Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More →
Socket
Sign inDemoInstall
Socket
Sign inDemoInstall

hls-fetcher

Package Overview
Dependencies
Maintainers
16
Versions
10
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

hls-fetcher - npm Package Compare versions

Comparing version 2.0.1 to 2.0.2

15

package.json
{
"name": "hls-fetcher",
"version": "2.0.1",
"version": "2.0.2",
"description": "Fetch HLS segments from an m3u8 playlist",
"main": "src/index.js",
"scripts": {
"lint": "vjsstandard",
"pretest": "npm run lint",
"test": "NODE_ENV=test mocha --opts test/opts/unit.opts test/unit"

@@ -26,14 +28,15 @@ },

"dependencies": {
"aes-decrypter": "^1.0.2",
"aes-decrypter": "^3.0.0",
"bluebird": "^3.4.0",
"m3u8-parser": "^1.0.2",
"m3u8-parser": "^4.2.0",
"mkdirp": "^0.5.1",
"pessimist": "^0.3.5",
"promise-streams": "^1.0.1",
"requestretry": "^1.8.0"
"request": "^2.87.0",
"requestretry": "^2.0.0"
},
"devDependencies": {
"mocha": "^5.2.0",
"nock": "^9.3.3"
"nock": "^9.3.3",
"videojs-standard": "^6.0.2"
}
}

12

README.md
# HLS-FETCHER
[![Build Status](https://travis-ci.org/videojs/hls-fetcher.svg?branch=master)](https://travis-ci.org/videojs/hls-fetcher)
[![Greenkeeper badge](https://badges.greenkeeper.io/videojs/hls-fetcher.svg)](https://greenkeeper.io/)
[![Slack Status](http://slack.videojs.com/badge.svg)](http://slack.videojs.com)
[![NPM](https://nodei.co/npm/hls-fetcher.png?downloads=true&downloadRank=true)](https://nodei.co/npm/hls-fetcher/)
A plugin that displays user-friendly messages when Video.js encounters an error.
Lead Maintainer: Brandon Casey [@brandonocasey](https://github.com/brandonocasey)
Maintenance Status: Stable
A simple CLI tool to fetch an entire hls manifest and it's segments and save it all locally.

@@ -4,0 +16,0 @@

47

src/cli.js
#!/usr/bin/env node
var path = require('path');
var start = require('./index');
var pessimist = require('pessimist')
.usage('Fetch and save the contents of an HLS playlist locally.\nUsage: $0 ')
.alias('i', 'input')
.demand('i')
.describe('i', 'uri to m3u8 (required)')
.alias('o', 'output')
.default('o', './hls-fetcher')
.describe('o', "output path (default:'./hls-fetcher')")
.alias('c', 'concurrency')
.default('c', Infinity)
.describe('c', 'number of simultaneous fetches (default: Infinity)')
.alias('d', 'decrypt')
.default('d', false)
.describe('d', 'decrypt and remove enryption from manifest (default: false)')
.argv;
/* eslint-disable no-console */
const path = require('path');
const start = require('./index');
const pessimist = require('pessimist')
.usage('Fetch and save the contents of an HLS playlist locally.\nUsage: $0 ')
.alias('i', 'input')
.demand('i')
.describe('i', 'uri to m3u8 (required)')
.alias('o', 'output')
.default('o', './hls-fetcher')
.describe('o', "output path (default:'./hls-fetcher')")
.alias('c', 'concurrency')
.default('c', Infinity)
.describe('c', 'number of simultaneous fetches (default: Infinity)')
.alias('d', 'decrypt')
.default('d', false)
.describe('d', 'decrypt and remove enryption from manifest (default: false)')
.argv;
// Make output path
var output = path.resolve(pessimist.o);
var startTime = Date.now();
var options = {
const output = path.resolve(pessimist.o);
const startTime = Date.now();
const options = {
input: pessimist.i,
output: output,
output,
concurrency: pessimist.c,

@@ -31,3 +33,4 @@ decrypt: pessimist.d

start(options).then(function() {
var timeTaken = ((Date.now() - startTime) / 1000).toFixed(2);
const timeTaken = ((Date.now() - startTime) / 1000).toFixed(2);
console.log('Operation completed successfully in', timeTaken, 'seconds.');

@@ -34,0 +37,0 @@ process.exit(0);

18

src/index.js

@@ -1,12 +0,14 @@

var WalkManifest = require('./walk-manifest');
var WriteData = require('./write-data');
/* eslint-disable no-console */
const WalkManifest = require('./walk-manifest');
const WriteData = require('./write-data');
var main = function(options) {
console.log("Gathering Manifest data...");
var options = {decrypt: options.decrypt, basedir: options.output, uri: options.input};
return WalkManifest(options)
const main = function(options) {
console.log('Gathering Manifest data...');
const settings = {decrypt: options.decrypt, basedir: options.output, uri: options.input};
return WalkManifest(settings)
.then(function(resources) {
console.log("Downloading additional data...");
console.log('Downloading additional data...');
return WriteData(options.decrypt, options.concurrency, resources);
})
});
};

@@ -13,0 +15,0 @@

29

src/utils.js

@@ -1,18 +0,21 @@

var url = require('url');
var path = require('path');
var fs = require('fs');
const url = require('url');
const path = require('path');
const fs = require('fs');
var Utils = {
joinUri: function(absolute, relative) {
var parse = url.parse(absolute);
const Utils = {
joinUri(absolute, relative) {
const parse = url.parse(absolute);
parse.pathname = path.join(parse.pathname, relative);
return url.format(parse);
},
getUriPath: function(uri, base) {
getUriPath(uri, base) {
base = base || '';
var parse = url.parse(uri);
const parse = url.parse(uri);
return path.relative('.', path.join(base, parse.pathname));
},
isAbsolute: function(uri) {
var parsed = url.parse(uri);
isAbsolute(uri) {
const parsed = url.parse(uri);
if (parsed.protocol) {

@@ -23,10 +26,10 @@ return true;

},
fileExists: function(file) {
fileExists(file) {
try {
return fs.statSync(file).isFile();
} catch(e) {
} catch (e) {
return false;
}
},
localize: function(parentUri, childUri) {
localize(parentUri, childUri) {
return path.join(path.basename(parentUri, path.extname(parentUri)), path.basename(childUri));

@@ -33,0 +36,0 @@ }

83

src/walk-manifest.js

@@ -1,9 +0,9 @@

var m3u8 = require('m3u8-parser');
var request = require('requestretry');
var url = require('url');
var path = require('path');
var fs = require('fs');
/* eslint-disable no-console */
const m3u8 = require('m3u8-parser');
const request = require('requestretry');
const url = require('url');
const path = require('path');
// replace invalid http/fs characters with valid representations
var fsSanitize = function(filepath) {
const fsSanitize = function(filepath) {
return filepath

@@ -13,4 +13,5 @@ .replace(/\?/g, '-questionmark-');

var joinURI = function(absolute, relative) {
var parse = url.parse(absolute);
const joinURI = function(absolute, relative) {
const parse = url.parse(absolute);
parse.pathname = path.join(parse.pathname, relative);

@@ -20,4 +21,5 @@ return url.format(parse);

var isAbsolute = function(uri) {
var parsed = url.parse(uri);
const isAbsolute = function(uri) {
const parsed = url.parse(uri);
if (parsed.protocol) {

@@ -29,6 +31,8 @@ return true;

var mediaGroupPlaylists = function(mediaGroups) {
var playlists = [];
const mediaGroupPlaylists = function(mediaGroups) {
const playlists = [];
['AUDIO', 'VIDEO', 'CLOSED-CAPTIONS', 'SUBTITLES'].forEach(function(type) {
var mediaGroupType = mediaGroups[type];
const mediaGroupType = mediaGroups[type];
if (mediaGroupType && !Object.keys(mediaGroupType).length) {

@@ -38,5 +42,6 @@ return;

for (var group in mediaGroupType) {
for (var item in mediaGroupType[group]) {
var props = mediaGroupType[group][item];
for (const group in mediaGroupType) {
for (const item in mediaGroupType[group]) {
const props = mediaGroupType[group][item];
playlists.push(props);

@@ -49,4 +54,5 @@ }

var parseManifest = function(content) {
var parser = new m3u8.Parser();
const parseManifest = function(content) {
const parser = new m3u8.Parser();
parser.push(content);

@@ -57,4 +63,4 @@ parser.end();

var parseKey = function(requestOptions, basedir, decrypt, resources, manifest, parent) {
return new Promise(function(resolve) {
const parseKey = function(requestOptions, basedir, decrypt, resources, manifest, parent) {
return new Promise(function(resolve, reject) {

@@ -64,5 +70,6 @@ if (!manifest.parsed.segments[0] || !manifest.parsed.segments[0].key) {

}
var key = manifest.parsed.segments[0].key;
const key = manifest.parsed.segments[0].key;
var keyUri = key.uri;
let keyUri = key.uri;
if (!isAbsolute(keyUri)) {

@@ -98,7 +105,8 @@ keyUri = joinURI(path.dirname(manifest.uri), keyUri);

const keyError = new Error(response.statusCode + '|' + keyUri);
console.error(keyError);
reject(keyError);
return reject(keyError);
}
keyContent = response.body;
const keyContent = response.body;

@@ -123,12 +131,13 @@ key.bytes = new Uint32Array([

const keyError = new Error(err.message + '|' + keyUri);
console.error(keyError, err);
reject(keyError);
})
});
});
};
var walkPlaylist = function(options) {
const walkPlaylist = function(options) {
return new Promise(function(resolve, reject) {
var {
const {
decrypt,

@@ -139,8 +148,8 @@ basedir,

manifestIndex = 0,
onError = function(err, uri, resources, resolve, reject) {
onError = function(err, errUri, resources, res, rej) {
// Avoid adding the top level uri to nested errors
if (err.message.includes('|')) {
reject(err);
rej(err);
} else {
reject(new Error(err.message + '|' + uri));
rej(new Error(err.message + '|' + errUri));
}

@@ -154,4 +163,5 @@ },

var resources = [];
var manifest = {};
let resources = [];
const manifest = {};
manifest.uri = uri;

@@ -188,3 +198,4 @@ manifest.file = path.join(basedir, fsSanitize(path.basename(uri)));

if (response.statusCode !== 200) {
var manifestError = new Error(response.statusCode + '|' + manifest.uri);
const manifestError = new Error(response.statusCode + '|' + manifest.uri);
manifestError.reponse = response;

@@ -204,3 +215,4 @@ return onError(manifestError, manifest.uri, resources, resolve, reject);

var playlists = manifest.parsed.playlists.concat(mediaGroupPlaylists(manifest.parsed.mediaGroups));
const playlists = manifest.parsed.playlists.concat(mediaGroupPlaylists(manifest.parsed.mediaGroups));
parseKey({

@@ -253,6 +265,7 @@ time: requestTimeout,

const flatten = [].concat.apply([], r);
resources = resources.concat(flatten);
resolve(resources);
}).catch(function(err) {
onError(err, manifest.uri, resources, resolve, reject)
onError(err, manifest.uri, resources, resolve, reject);
});

@@ -259,0 +272,0 @@ });

72

src/write-data.js

@@ -1,9 +0,10 @@

var Promise = require('bluebird');
var mkdirp = Promise.promisify(require('mkdirp'));
var request = require('requestretry');
var fs = Promise.promisifyAll(require('fs'));
var aesDecrypter = require('aes-decrypter').Decrypter;
var path = require('path');
/* eslint-disable no-console */
const Promise = require('bluebird');
const mkdirp = Promise.promisify(require('mkdirp'));
const request = require('requestretry');
const fs = Promise.promisifyAll(require('fs'));
const AesDecrypter = require('aes-decrypter').Decrypter;
const path = require('path');
var writeFile = function(file, content) {
const writeFile = function(file, content) {
return mkdirp(path.dirname(file)).then(function() {

@@ -16,9 +17,13 @@ return fs.writeFileAsync(file, content);

var requestFile = function(uri) {
var options = {
uri: uri,
timeout: 60000, // 60 seconds timeout
encoding: null, // treat all responses as a buffer
retryDelay: 1000 // retry 1s after on failure
const requestFile = function(uri) {
const options = {
uri,
// 60 seconds timeout
timeout: 60000,
// treat all responses as a buffer
encoding: null,
// retry 1s after on failure
retryDelay: 1000
};
return new Promise(function(resolve, reject) {

@@ -34,26 +39,35 @@ request(options, function(err, response, body) {

var toArrayBuffer = function(buffer) {
var ab = new ArrayBuffer(buffer.length);
var view = new Uint8Array(ab);
for (var i = 0; i < buffer.length; ++i) {
view[i] = buffer[i];
}
return ab;
const toArrayBuffer = function(buffer) {
const ab = new ArrayBuffer(buffer.length);
const view = new Uint8Array(ab);
for (let i = 0; i < buffer.length; ++i) {
view[i] = buffer[i];
}
return ab;
};
var decryptFile = function(content, encryption) {
const decryptFile = function(content, encryption) {
return new Promise(function(resolve, reject) {
var d = new aesDecrypter(toArrayBuffer(content), encryption.bytes, encryption.iv, function(err, bytes) {
/* eslint-disable no-new */
// this is how you use it, its kind of bad but :shrug:
new AesDecrypter(toArrayBuffer(content), encryption.bytes, encryption.iv, function(err, bytes) {
if (err) {
return reject(err);
}
return resolve(new Buffer(bytes));
})
});
/* eslint-enable no-new */
});
};
var WriteData = function(decrypt, concurrency, resources) {
var inProgress = [];
var operations = [];
const WriteData = function(decrypt, concurrency, resources) {
const inProgress = [];
const operations = [];
resources.forEach(function(r) {
if (r.content) {
operations.push(function() { return writeFile(r.file, r.content); });
operations.push(function() {
return writeFile(r.file, r.content);
});
} else if (r.key && decrypt) {

@@ -64,3 +78,3 @@ operations.push(function() {

}).then(function(content) {
return writeFile(r.file, content)
return writeFile(r.file, content);
});

@@ -80,3 +94,3 @@ });

return Promise.join(o());
}, {concurrency: concurrency}).all(function(o) {
}, {concurrency}).all(function(o) {
console.log('DONE!');

@@ -83,0 +97,0 @@ return Promise.resolve();

138

test/unit/walk-manifest.spec.js

@@ -1,19 +0,23 @@

var assert = require('assert');
var nock = require('nock');
/* eslint-env mocha */
/* eslint-disable max-nested-callbacks */
const assert = require('assert');
const nock = require('nock');
nock.disableNetConnect();
nock.enableNetConnect(/localhost/);
var walker = require('../../src/walk-manifest');
const walker = require('../../src/walk-manifest');
var TEST_URL = 'http://manifest-list-test.com';
const TEST_URL = 'http://manifest-list-test.com';
var customError = function(errors) {
const customError = function(errors) {
return function(err, uri, resources, resolve) {
// Avoid adding the top level uri to nested errors
if (err.message.includes('|'))
if (err.message.includes('|')) {
errors.push(err);
else
} else {
errors.push(new Error(err.message + '|' + uri));
}
resolve(resources);
}
};
};

@@ -35,3 +39,4 @@

var options = {decrypt: false, basedir: '.', uri: TEST_URL + '/test.m3u8', requestRetryMaxAttempts: 0};
const options = {decrypt: false, basedir: '.', uri: TEST_URL + '/test.m3u8', requestRetryMaxAttempts: 0};
walker(options)

@@ -42,3 +47,3 @@ .catch(function(err) {

done();
})
});
});

@@ -51,6 +56,8 @@

var options = {decrypt: false, basedir: '.', uri: TEST_URL + '/test.m3u8', requestRetryMaxAttempts: 0};
const options = {decrypt: false, basedir: '.', uri: TEST_URL + '/test.m3u8', requestRetryMaxAttempts: 0};
walker(options)
.then(function(resources) {
var setResources = new Set(resources);
const setResources = new Set(resources);
assert.equal(setResources.size, 1);

@@ -69,7 +76,9 @@ setResources.forEach(function(item) {

var options = {decrypt: false, basedir: '.', uri: TEST_URL + '/test.m3u8', requestRetryMaxAttempts: 0};
const options = {decrypt: false, basedir: '.', uri: TEST_URL + '/test.m3u8', requestRetryMaxAttempts: 0};
walker(options)
.then(function(resources) {
// m3u8 and 11 segments
var setResources = new Set(resources);
const setResources = new Set(resources);
assert.equal(setResources.size, 12);

@@ -90,7 +99,9 @@ setResources.forEach(function(item) {

var options = {decrypt: false, basedir: '.', uri: TEST_URL + '/test.m3u8', requestRetryMaxAttempts: 0};
const options = {decrypt: false, basedir: '.', uri: TEST_URL + '/test.m3u8', requestRetryMaxAttempts: 0};
walker(options)
.then(function(resources) {
// m3u8 and 11 segments
var setResources = new Set(resources);
const setResources = new Set(resources);
assert.equal(setResources.size, 12);

@@ -111,4 +122,4 @@ setResources.forEach(function(item) {

var errors = [];
var options = {
const errors = [];
const options = {
decrypt: false,

@@ -119,7 +130,11 @@ basedir: '.',

};
walker(options)
.then(function(resources) {
var setResources = new Set(resources);
assert.equal(setResources.size, 2); // 2 m3u8
assert.equal(errors.length, 0); // no errors on cycle
const setResources = new Set(resources);
// 2 m3u8
assert.equal(setResources.size, 2);
// no errors on cycle
assert.equal(errors.length, 0);
resources.forEach(function(item) {

@@ -129,3 +144,3 @@ assert(item.uri.includes('.m3u8'));

done();
})
});
});

@@ -139,3 +154,3 @@

var options = {
const options = {
decrypt: false,

@@ -148,2 +163,3 @@ basedir: '.',

};
walker(options)

@@ -162,4 +178,4 @@ .catch(function(err) {

var errors = [];
var options = {
const errors = [];
const options = {
decrypt: false,

@@ -173,2 +189,3 @@ basedir: '.',

};
walker(options)

@@ -187,3 +204,4 @@ .then(function(resources) {

var options = {decrypt: false, basedir: '.', uri: TEST_URL + '/test.m3u8', requestRetryMaxAttempts: 0};
const options = {decrypt: false, basedir: '.', uri: TEST_URL + '/test.m3u8', requestRetryMaxAttempts: 0};
walker(options)

@@ -210,7 +228,9 @@ .then(function(resources) {

var options = {decrypt: false, basedir: '.', uri: TEST_URL + '/test.m3u8', requestRetryMaxAttempts: 0};
const options = {decrypt: false, basedir: '.', uri: TEST_URL + '/test.m3u8', requestRetryMaxAttempts: 0};
walker(options)
.then(function(resources) {
// 4 m3u8 and 8 * 3 segments
var setResources = new Set(resources);
const setResources = new Set(resources);
assert.equal(setResources.size, 28);

@@ -237,7 +257,9 @@ setResources.forEach(function(item) {

var options = {decrypt: false, basedir: '.', uri: TEST_URL + '/test.m3u8', requestRetryMaxAttempts: 0};
const options = {decrypt: false, basedir: '.', uri: TEST_URL + '/test.m3u8', requestRetryMaxAttempts: 0};
walker(options)
.then(function(resources) {
// 4 m3u8 and 8 * 3 segments
var setResources = new Set(resources);
const setResources = new Set(resources);
assert.equal(setResources.size, 28);

@@ -262,3 +284,4 @@ setResources.forEach(function(item) {

var options = {decrypt: false, basedir: '.', uri: TEST_URL + '/test.m3u8', requestRetryMaxAttempts: 0};
const options = {decrypt: false, basedir: '.', uri: TEST_URL + '/test.m3u8', requestRetryMaxAttempts: 0};
walker(options)

@@ -282,4 +305,4 @@ .catch(function(err) {

var errors = [];
var options = {
const errors = [];
const options = {
decrypt: false,

@@ -291,6 +314,8 @@ basedir: '.',

};
walker(options)
.then(function(resources) {
// 3 m3u8 and 8 * 2 segments
var setResources = new Set(resources);
const setResources = new Set(resources);
assert.equal(setResources.size, 19);

@@ -319,8 +344,9 @@

var options = {
const options = {
decrypt: false,
basedir: '.',
uri: TEST_URL + '/test.m3u8',
requestRetryMaxAttempts: 0,
requestRetryMaxAttempts: 0
};
walker(options)

@@ -345,3 +371,3 @@ .catch(function(err) {

var options = {
const options = {
decrypt: false,

@@ -353,2 +379,3 @@ basedir: '.',

};
walker(options)

@@ -372,4 +399,4 @@ .catch(function(err) {

var errors = [];
var options = {
const errors = [];
const options = {
decrypt: false,

@@ -381,6 +408,8 @@ basedir: '.',

};
walker(options)
.then(function(resources) {
// 3 m3u8 and 8 * 2 segments
var setResources = new Set(resources);
const setResources = new Set(resources);
assert.equal(setResources.size, 19);

@@ -409,3 +438,4 @@

var options = {decrypt: false, basedir: '.', uri: TEST_URL + '/test.m3u8', requestRetryMaxAttempts: 0};
const options = {decrypt: false, basedir: '.', uri: TEST_URL + '/test.m3u8', requestRetryMaxAttempts: 0};
walker(options)

@@ -429,5 +459,4 @@ .catch(function(err) {

var errors = [];
var options = {
const errors = [];
const options = {
decrypt: false,

@@ -439,6 +468,8 @@ basedir: '.',

};
walker(options)
.then(function(resources) {
// 3 m3u8 and 8 * 2 segments
var setResources = new Set(resources);
const setResources = new Set(resources);
assert.equal(setResources.size, 19);

@@ -466,7 +497,9 @@

var options = {decrypt: false, basedir: '.', uri: TEST_URL + '/test.m3u8', requestRetryMaxAttempts: 0};
const options = {decrypt: false, basedir: '.', uri: TEST_URL + '/test.m3u8', requestRetryMaxAttempts: 0};
walker(options)
.then(function(resources) {
// 4 m3u8 and 8 * 2 segments
var setResources = new Set(resources);
const setResources = new Set(resources);
assert.equal(setResources.size, 20);

@@ -494,3 +527,3 @@ setResources.forEach(function(item) {

var options = {
const options = {
decrypt: false,

@@ -503,2 +536,3 @@ basedir: '.',

};
walker(options)

@@ -523,4 +557,4 @@ .catch(function(err) {

var errors = [];
var options = {
const errors = [];
const options = {
decrypt: false,

@@ -534,6 +568,8 @@ basedir: '.',

};
walker(options)
.then(function(resources) {
// 3 m3u8 and 8 * 2 segments
var setResources = new Set(resources);
const setResources = new Set(resources);
assert.equal(setResources.size, 19);

@@ -540,0 +576,0 @@ assert(errors.find(o => o.message === 'ESOCKETTIMEDOUT|' + TEST_URL + '/var500000/playlist.m3u8'));

.travis.yml

Sorry, the diff of this file is not supported yet

SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

About

Packages

npm

Go

Maven

PyPI

Rubygems

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc