hls-fetcher - npm Package Compare versions

Comparing version 1.1.2 to 2.0.0

package.json

 {
   "name": "hls-fetcher",
-  "version": "1.1.2",
+  "version": "2.0.0",
   "description": "Fetch HLS segments from an m3u8 playlist",
   "main": "src/index.js",
   "scripts": {
-    "test": "echo 'No tests to run'"
+    "test": "NODE_ENV=test mocha --opts test/opts/unit.opts test/unit"
   },
@@ -32,5 +32,8 @@ "bin": {
     "promise-streams": "^1.0.1",
-    "requestretry": "^1.8.0",
-    "sync-request": "^3.0.1"
+    "requestretry": "^1.8.0"
+  },
+  "devDependencies": {
+    "mocha": "^5.2.0",
+    "nock": "^9.3.3"
   }
 }

src/index.js

@@ -6,6 +6,11 @@ var WalkManifest = require('./walk-manifest');
   console.log("Gathering Manifest data...");
-  var resources = WalkManifest(options.decrypt, options.output, options.input);
-
-  console.log("Downloading additional data...");
-  return WriteData(options.decrypt, options.concurrency, resources);
+  var options = {decrypt: options.decrypt, basedir: options.output, uri: options.input};
+  WalkManifest(options)
+    .then(function(resources) {
+      console.log("Downloading additional data...");
+      return WriteData(options.decrypt, options.concurrency, resources);
+    })
+    .catch(function(err) {
+      throw err;
+    });
 };
@@ -12,0 +17,0 @@

src/walk-manifest.js

 var m3u8 = require('m3u8-parser');
-var syncRequest = require('sync-request');
+var request = require('requestretry');
 var url = require('url');
@@ -9,18 +9,18 @@ var path = require('path');
 var fsSanitize = function(filepath) {
-	return filepath
-		.replace(/\?/g, '-questionmark-');
+  return filepath
+    .replace(/\?/g, '-questionmark-');
 };
 
 var joinURI = function(absolute, relative) {
-	var parse = url.parse(absolute);
-	parse.pathname = path.join(parse.pathname, relative);
-	return url.format(parse);
+  var parse = url.parse(absolute);
+  parse.pathname = path.join(parse.pathname, relative);
+  return url.format(parse);
 };
 
 var isAbsolute = function(uri) {
-	var parsed = url.parse(uri);
-	if (parsed.protocol) {
-		return true;
-	}
-	return false;
+  var parsed = url.parse(uri);
+  if (parsed.protocol) {
+    return true;
+  }
+  return false;
 };
@@ -53,114 +53,205 @@
 
-var parseKey = function(basedir, decrypt, resources, manifest, parent) {
-	if (!manifest.parsed.segments[0] || !manifest.parsed.segments[0].key) {
-		return {};
-	}
-	var key = manifest.parsed.segments[0].key;
+var parseKey = function(requestOptions, basedir, decrypt, resources, manifest, parent) {
+  return new Promise(function(resolve) {
 
-	var keyUri = key.uri;
-	if (!isAbsolute(keyUri)) {
-		keyUri = joinURI(path.dirname(manifest.uri), keyUri);
-	}
+    if (!manifest.parsed.segments[0] || !manifest.parsed.segments[0].key) {
+      resolve({});
+    }
+    var key = manifest.parsed.segments[0].key;
 
-	// if we are not decrypting then we just download the key
-	if (!decrypt) {
-		// put keys in parent-dir/key-name.key
-		key.file = basedir;
-		if (parent) {
-			key.file = path.dirname(parent.file);
-		}
-		key.file = path.join(key.file, fsSanitize(path.basename(key.uri)));
+    var keyUri = key.uri;
+    if (!isAbsolute(keyUri)) {
+      keyUri = joinURI(path.dirname(manifest.uri), keyUri);
+    }
 
-		manifest.content = new Buffer(manifest.content.toString().replace(
-			key.uri,
-			path.relative(path.dirname(manifest.file), key.file)
-		));
-		key.uri = keyUri;
-		resources.push(key);
-		return key;
-	}
+    // if we are not decrypting then we just download the key
+    if (!decrypt) {
+      // put keys in parent-dir/key-name.key
+      key.file = basedir;
+      if (parent) {
+        key.file = path.dirname(parent.file);
+      }
+      key.file = path.join(key.file, fsSanitize(path.basename(key.uri)));
 
-	// get the aes key
-	var keyContent = syncRequest('GET', keyUri).getBody();
-	key.bytes = new Uint32Array([
-		keyContent.readUInt32BE(0),
-		keyContent.readUInt32BE(4),
-		keyContent.readUInt32BE(8),
-		keyContent.readUInt32BE(12)
-	]);
+      manifest.content = new Buffer(manifest.content.toString().replace(
+        key.uri,
+        path.relative(path.dirname(manifest.file), key.file)
+      ));
+      key.uri = keyUri;
+      resources.push(key);
+      resolve(key);
+    }
 
-	// remove the key from the manifest
-	manifest.content = new Buffer(manifest.content.toString().replace(
-		new RegExp('.*' + key.uri + '.*'),
-		''
-	));
+    requestOptions.url = keyUri;
+    requestOptions.encoding = null;
 
+    // get the aes key
+    request(requestOptions)
+      .then(function(response) {
+        if (response.statusCode !== 200) {
+          const keyError = new Error(response.statusCode + '|' + keyUri);
+          console.error(keyError);
+          reject(keyError);
+        }
 
-	return key;
+        keyContent = response.body;
+
+        key.bytes = new Uint32Array([
+          keyContent.readUInt32BE(0),
+          keyContent.readUInt32BE(4),
+          keyContent.readUInt32BE(8),
+          keyContent.readUInt32BE(12)
+        ]);
+
+        // remove the key from the manifest
+        manifest.content = new Buffer(manifest.content.toString().replace(
+          new RegExp('.*' + key.uri + '.*'),
+          ''
+        ));
+
+        resolve(key);
+      })
+      .catch(function(err) {
+        // TODO: do we even care about key errors; currently we just keep going and ignore them.
+        const keyError = new Error(err.message + '|' + keyUri);
+        console.error(keyError, err);
+        reject(keyError);
+      })
+  });
 };
 
-var walkPlaylist = function(decrypt, basedir, uri, parent, manifestIndex) {
-	var resources = [];
-	var manifest  = {};
-	manifest.uri  = uri;
-	manifest.file = path.join(basedir, fsSanitize(path.basename(uri)));
-	resources.push(manifest);
+var walkPlaylist = function(options) {
+  return new Promise(function(resolve, reject) {
 
-	// if we are not the master playlist
-	if (parent) {
-		manifest.file = path.join(
-			path.dirname(parent.file),
-			'manifest' + manifestIndex,
-			fsSanitize(path.basename(manifest.file))
-		);
-		// get the real uri of this playlist
-		if (!isAbsolute(manifest.uri)) {
-			manifest.uri = joinURI(path.dirname(parent.uri), manifest.uri);
-		}
-		// replace original uri in file with new file path
-		parent.content = new Buffer(parent.content.toString().replace(uri, path.relative(path.dirname(parent.file), manifest.file)));
-	}
+    var {
+      decrypt,
+      basedir,
+      uri,
+      parent = false,
+      manifestIndex = 0,
+      onError = function(err, uri, resources, resolve, reject) {
+        // Avoid adding the top level uri to nested errors
+        if (err.message.includes('|')) {
+          reject(err);
+        } else {
+          reject(new Error(err.message + '|' + uri));
+        }
+      },
+      visitedUrls = [],
+      requestTimeout = 1500,
+      requestRetryMaxAttempts = 5,
+      requestRetryDelay = 5000
+    } = options;
 
-  manifest.content = syncRequest('GET', manifest.uri).getBody();
-  manifest.parsed  = parseManifest(manifest.content);
-	manifest.parsed.segments = manifest.parsed.segments   || [];
-	manifest.parsed.playlists = manifest.parsed.playlists || [];
-	manifest.parsed.mediaGroups = manifest.parsed.mediaGroups || {};
+    var resources = [];
+    var manifest = {};
+    manifest.uri = uri;
+    manifest.file = path.join(basedir, fsSanitize(path.basename(uri)));
 
-  var playlists = manifest.parsed.playlists.concat(mediaGroupPlaylists(manifest.parsed.mediaGroups));
-	var key = parseKey(basedir, decrypt, resources, manifest, parent);
+    // if we are not the master playlist
+    if (parent) {
+      manifest.file = path.join(
+        path.dirname(parent.file),
+        'manifest' + manifestIndex,
+        fsSanitize(path.basename(manifest.file))
+      );
+      // get the real uri of this playlist
+      if (!isAbsolute(manifest.uri)) {
+        manifest.uri = joinURI(path.dirname(parent.uri), manifest.uri);
+      }
+      // replace original uri in file with new file path
+      parent.content = new Buffer(parent.content.toString().replace(uri, path.relative(path.dirname(parent.file), manifest.file)));
+    }
 
-	// SEGMENTS
-	manifest.parsed.segments.forEach(function(s, i) {
-		if (!s.uri) {
-			return;
-		}
-		// put segments in manifest-name/segment-name.ts
-		s.file = path.join(path.dirname(manifest.file), fsSanitize(path.basename(s.uri)));
-		if (!isAbsolute(s.uri)) {
-			s.uri = joinURI(path.dirname(manifest.uri), s.uri);
-		}
-		if (key) {
-			s.key = key;
-			s.key.iv = s.key.iv || new Uint32Array([0, 0, 0, manifest.parsed.mediaSequence, i]);
-		}
+    if (visitedUrls.includes(manifest.uri)) {
+      var manifestError = new Error('[WARN] Trying to visit the same uri again; skipping to avoid getting stuck in a cycle');
+      manifestError.uri = manifest.uri;
+      console.error(manifestError);
+      return resolve(resources);
+    }
 
-		manifest.content = new Buffer(manifest.content.toString().replace(
-			s.uri,
-			path.relative(path.dirname(manifest.file), s.file)
-		));
-		resources.push(s);
-	});
+    request({
+      url: manifest.uri,
+      timeout: requestTimeout,
+      maxAttempts: requestRetryMaxAttempts,
+      retryDelay: requestRetryDelay
+    })
+      .then(function(response) {
+        if (response.statusCode !== 200) {
+          var manifestError = new Error(response.statusCode + '|' + manifest.uri);
+          manifestError.reponse = response;
+          return onError(manifestError, manifest.uri, resources, resolve, reject);
+        }
+        // Only push manifest uris that get a non 200 and don't timeout
+        resources.push(manifest);
+        visitedUrls.push(manifest.uri);
 
-	// SUB Playlists
-	playlists.forEach(function(p, z) {
-		if (!p.uri) {
-			return;
-		}
-		resources = resources.concat(walkPlaylist(decrypt, basedir, p.uri, manifest, z));
-	});
+        manifest.content = response.body;
 
-	return resources;
+        manifest.parsed = parseManifest(manifest.content);
+        manifest.parsed.segments = manifest.parsed.segments || [];
+        manifest.parsed.playlists = manifest.parsed.playlists || [];
+        manifest.parsed.mediaGroups = manifest.parsed.mediaGroups || {};
+
+        var playlists = manifest.parsed.playlists.concat(mediaGroupPlaylists(manifest.parsed.mediaGroups));
+        parseKey({
+          time: requestTimeout,
+          maxAttempts: requestRetryMaxAttempts,
+          retryDelay: requestRetryDelay
+        }, basedir, decrypt, resources, manifest, parent).then(function(key) {
+          // SEGMENTS
+          manifest.parsed.segments.forEach(function(s, i) {
+            if (!s.uri) {
+              return;
+            }
+            // put segments in manifest-name/segment-name.ts
+            s.file = path.join(path.dirname(manifest.file), fsSanitize(path.basename(s.uri)));
+            if (!isAbsolute(s.uri)) {
+              s.uri = joinURI(path.dirname(manifest.uri), s.uri);
+            }
+            if (key) {
+              s.key = key;
+              s.key.iv = s.key.iv || new Uint32Array([0, 0, 0, manifest.parsed.mediaSequence, i]);
+            }
+            manifest.content = new Buffer(manifest.content.toString().replace(
+              s.uri,
+              path.relative(path.dirname(manifest.file), s.file)
+            ));
+            resources.push(s);
+          });
+
+          // SUB Playlists
+          const subs = playlists.map(function(p, z) {
+            if (!p.uri) {
+              return Promise().resolve(resources);
+            }
+            return walkPlaylist({
+              decrypt,
+              basedir,
+              uri: p.uri,
+              parent: manifest,
+              manifestIndex: z,
+              onError,
+              visitedUrls,
+              requestTimeout,
+              requestRetryMaxAttempts,
+              requestRetryDelay
+            });
+          });
+
+          Promise.all(subs).then(function(r) {
+            const flatten = [].concat.apply([], r);
+            resources = resources.concat(flatten);
+            resolve(resources);
+          }).catch(function(err) {
+            onError(err, manifest.uri, resources, resolve, reject)
+          });
+        });
+      })
+      .catch(function(err) {
+        onError(err, manifest.uri, resources, resolve, reject);
+      });
+  });
 };
 
 module.exports = walkPlaylist;

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

New author

Supply chain risk

A new npm collaborator published a version of the package for the first time. New collaborators are usually benign additions to a project, but do indicate a change to the security surface area of a package.

Found 1 instance in 1 package

No tests

Quality

Package does not have any tests. This is a strong signal of a poorly maintained or low quality package.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

38043

increased by221.39%

Dependency count

7

decreased by-12.5%

Number of package files

17

increased by112.5%

Lines of code

856

increased by191.16%

Number of low quality alerts

0

decreased by-100%

Number of medium supply chain risk alerts

1

decreased by-50%

Worsened metrics

Dev dependency count

2

increased byInfinity%

Dependency changes

• - Removedsync-request@^3.0.1

• - Removedasap@2.0.6(transitive)

• - Removedbuffer-from@1.1.2(transitive)

• - Removedcaseless@0.11.0(transitive)

• - Removedconcat-stream@1.6.2(transitive)

• - Removedcore-util-is@1.0.3(transitive)

• - Removedhttp-basic@2.5.1(transitive)

• - Removedhttp-response-object@1.1.0(transitive)

• - Removedinherits@2.0.4(transitive)

• - Removedisarray@1.0.0(transitive)

• - Removedprocess-nextick-args@2.0.1(transitive)

• - Removedpromise@7.3.1(transitive)

• - Removedreadable-stream@2.3.8(transitive)

• - Removedsafe-buffer@5.1.2(transitive)

• - Removedstring_decoder@1.1.1(transitive)

• - Removedsync-request@3.0.1(transitive)

• - Removedthen-request@2.2.0(transitive)

• - Removedtypedarray@0.0.6(transitive)

• - Removedutil-deprecate@1.0.2(transitive)