ytsr - npm Package Compare versions

Comparing version 0.1.7 to 0.1.8

lib/main.js

@@ -1,170 +0,123 @@
-"use strict";
+const URL = require('url');
+const UTIL = require('./util.js');
+const QS = require('querystring');
 
-var https = require('https');
-var util = require('./util.js');
+const main = module.exports = (searchString, options, callback) => { // eslint-disable-line consistent-return
+  // Check wether options wether no options were provided
+  if (typeof options === 'function') {
+    callback = options;
+    options = { limit: 100 };
+  }
+  // Return a promise when no callback is provided
+  if (!callback) {
+    return new Promise((resolve, reject) => {
+      main(searchString, options, (err, info) => { // eslint-disable-line consistent-return
+        if (err) return reject(err);
+        resolve(info);
+      });
+    });
+  }
+  if (!options) options = { limit: 100 };
+  if (!searchString && !options.nextpageRef) return callback(new Error('search string or nextpageRef is mandatory'));
+  if (isNaN(options.limit)) options.limit = 100;
 
-exports.search = function(search_string, options, callback) {
-	// check wether options wether no options were provided
-	if(typeof(options) == 'function') {
-		callback = options;
-		options = {limit: 100};
-	}
-	// return a promise when no callback is provided
-	if(!callback) {
-		return new Promise(function(resolve, reject) {
-			exports.search(search_string, options, function(err, info) {
-				if(err) return reject(err);
-				resolve(info);
-			});
-		});
-	}
-	if(!options) options = {limit: 100};
-	var afterfunc = function(resp) {
-		if(resp.statusCode != 200) {
-			callback(new Error('Status Code ' + resp.statusCode));
-		}
-		var resp_string = '';
-		resp.on('data', function(d) {
-			resp_string += d.toString();
-		})
-		resp.on('end', function() {
-			var parsed;
-			try {
-				parsed = JSON.parse(resp_string);
-			} catch(e) {
-				return callback(e);
-			}
-			var content = parsed[parsed.length - 1].body.content;
+  // Save provided nextpageRef and do the request
+  const currentRef = options.nextpageRef;
+  UTIL.getPage(currentRef ? UTIL.buildFromNextpage(currentRef) : UTIL.buildLink(searchString), (err, body) => { // eslint-disable-line consistent-return, max-len
+    if (err) return callback(err);
+    let content;
+    try {
+      const parsed = JSON.parse(body);
+      content = parsed[parsed.length - 1].body.content;
+    } catch (e) {
+      return callback(e);
+    }
 
-			// get the table of items and parse it(remove null items where the parsing failed)
-			var table = util.between(content, '<ol id="item-section-', '\n</ol>').split('</li>\n\n<li>');
-			table = table.filter(function(t) {
-				var condition_1 = !t.includes('<div class="pyv-afc-ads-container" style="visibility:visible">');
-				var condition_2 = !t.includes('<span class="spell-correction-corrected">');
-				var condition_3 = !t.includes('<div class="search-message">');
-				var condition_4 = !t.includes('<li class="search-exploratory-line">');
-				return condition_1 && condition_2 && condition_3 && condition_4;
-			});
-			table = table.map(function(t) { return util.parse_item(t, resp_string) }).filter(function(a) { return a });
+    // Get the table of items and parse it(remove null items where the parsing failed)
+    const items = UTIL
+      .between(content, '<ol id="item-section-', '\n</ol>')
+      .split('</li>\n\n<li>')
+      .filter(t => {
+        let condition1 = !t.includes('<div class="pyv-afc-ads-container" style="visibility:visible">');
+        let condition2 = !t.includes('<span class="spell-correction-corrected">');
+        let condition3 = !t.includes('<div class="search-message">');
+        let condition4 = !t.includes('<li class="search-exploratory-line">');
+        return condition1 && condition2 && condition3 && condition4;
+      })
+      .map(t => UTIL.parseItem(t, body, searchString))
+      .filter(a => a)
+      .filter((item, index) => !isNaN(options.limit) ? index < options.limit : true);
+    if (!isNaN(options.limit)) options.limit -= items.length;
 
-			// get amount of results
-			var results = util.between(util.between(content, '<p class="num-results', '</p>'), '>');
+    // Get amount of results
+    const results = UTIL.between(UTIL.between(content, '<p class="num-results', '</p>'), '>') || 0;
 
-			// get informations about set filters
-			var set_filters_holder = util.between(content, '<ul class="filter-crumb-list">', '</ul>').split('<li')
-			var set_filters = set_filters_holder.splice(1).map(function(f) { return util.between(f, '<span class="filter-text filter-ghost">', '<') });
+    // Get informations about set filters
+    const filters = UTIL.parseFilters(content);
+    const activeFilters = Array.from(filters).map(a => a[1].active).filter(a => a);
 
-			// were already on the last page so we cant parse more
-			var pages_container = util.between(content, '<div class="branded-page-box search-pager  spf-link ">', '</div>').split('<a');
-			var last_page_ref = pages_container[pages_container.length - 1];
-			if(last_page_ref.includes('data-redirect-url="/results?')) {
-				return callback(null, {
-					query: search_string,
-					results: results ? results : 0,
-					filters: set_filters,
-					current_ref: current_ref ? current_ref : undefined,
-					items: table
-				});
-			}
-			var nextpage_ref = util.remove_html(util.between(last_page_ref, 'href="', '"'));
+    const pagesContainer = UTIL
+      .between(content, '<div class="branded-page-box search-pager  spf-link ">', '</div>')
+      .split('<a');
+    const lastPageRef = pagesContainer[pagesContainer.length - 1];
+    const nextpageRef = UTIL.removeHtml(UTIL.between(lastPageRef, 'href="', '"')) || null;
 
-			// check wether we hit the set limit
-			if(options.limit && options.limit <= table.length) {
-				table = table.filter(function(item, index) { return index < options.limit });
-				return callback(null, {
-					query: search_string,
-					items: table,
-					nextpage_ref: nextpage_ref,
-					results: results ? results : 0,
-					filters: set_filters,
-					current_ref: current_ref ? current_ref : undefined,
-				});
-			}
-			if(nextpage_ref) {
-				options.nextpage_ref = nextpage_ref;
-				options.limit = options.limit ? options.limit - table.length : undefined;
-				return exports.search(search_string, options, function(err, data) {
-					if(err) {
-						return callback(err);
-					}
-					data.items = table.concat(data.items);
-					return callback(null, data);
-				});
-			}
+    // Were already on last page or hit the limit
+    if (lastPageRef.includes('data-redirect-url="/results?') ||
+      (!isNaN(options.limit) && options.limit < 1) ||
+      !nextpageRef) {
+      return callback(null, {
+        query: searchString || QS.unescape(URL.parse(currentRef, true).query.search_query),
+        items,
+        nextpageRef,
+        results,
+        filters: activeFilters,
+        currentRef: currentRef || null,
+      });
+    }
 
-			return callback(null, {
-				query: search_string,
-				items: table,
-				results: results ? results : 0,
-				filters: set_filters,
-				current_ref: current_ref ? current_ref : undefined,
-			});
-		});
-	}
-	// save provided nextpage_ref and do the request
-	var current_ref = options.nextpage_ref;
-	var request;
-	if(options.nextpage_ref) {
-		request = https.get('https://www.youtube.com' + options.nextpage_ref + '&spf=navigate', afterfunc);
-	} else {
-		request = https.get(util.build_link(search_string), afterfunc);
-	}
-	request.on('error', callback);
-}
+    options.nextpageRef = nextpageRef;
+    main(searchString, options, (e, data) => { // eslint-disable-line consistent-return, max-len
+      if (e) return callback(e);
+      items.push(...data.items);
+      callback(null, {
+        query: searchString || QS.unescape(URL.parse(currentRef, true).query.search_query),
+        items,
+        nextpageRef: data.nextpageRef,
+        results,
+        filters: activeFilters,
+        currentRef: data.currentRef,
+      });
+    });
+  });
+};
 
-exports.get_filters = function(search_string, callback) {
-	// return a promise when no callback is provided
-	if(!callback) {
-		return new Promise(function(resolve, reject) {
-			exports.get_filters(search_string, function(err, info) {
-				if(err) return reject(err);
-				resolve(info);
-			});
-		});
-	}
-	var afterfunc = function(resp) {
-		if(resp.statusCode != 200) {
-			callback(new Error('Status Code ' + resp.statusCode));
-		}
-		var resp_string = '';
-		resp.on('data', function(d) {
-			resp_string += d.toString();
-		})
-		resp.on('end', function() {
-			var parsed;
-			try {
-				parsed = JSON.parse(resp_string);
-			} catch(e) {
-				return callback(e);
-			}
-			var content = parsed[parsed.length - 1].body.content;
+const getFilters = main.getFilters = (searchString, callback) => { // eslint-disable-line consistent-return
+  // Return a promise when no callback is provided
+  if (!callback) {
+    return new Promise((resolve, reject) => {
+      getFilters(searchString, (err, info) => { // eslint-disable-line consistent-return
+        if (err) return reject(err);
+        resolve(info);
+      });
+    });
+  }
+  if (!searchString) return callback(new Error('search string is mandatory'));
 
-			// get informations about set filters
-			var set_filters_holder = util.between(content, '<ul class="filter-crumb-list">', '</ul>').split('<li');
-			var set_filters = set_filters_holder.splice(1).map(function(f) { return util.between(f, '<span class="filter-text filter-ghost">', '<') });
+  let queryString;
+  let parsedQuery = URL.parse(searchString, true);
+  if (parsedQuery.query.sp && parsedQuery.query.search_query) queryString = UTIL.buildFromNextpage(searchString);
+  else queryString = UTIL.buildLink(searchString);
 
-			// get avabile filters, parse and return them
-			var filters = util.between(content, '<div id="filter-dropdown"', '<ol id="item-section');
-			var coloms = filters.split('<h4 class="filter-col-title">');
-			coloms.splice(0, 1);
-			var results = {};
-			coloms.map(function(c) {
-				var parts = c.split('<a');
-				return parts.map(function(p, i) {
-					if(i == 0) {
-						return util.between(p, '', '<').toLowerCase();
-					} else {
-						return {
-							ref: util.remove_html(util.between(p, 'href="', '"')),
-							name: util.between(util.between(p, '>', '</span>'), '>'),
-						}
-					}
-				});
-			}).map(function(i) { results[i[0]] = i.splice(1) });
-			results['already_set'] = set_filters;
-			callback(null, results);
-		})
-	}
-	var request = https.get(util.build_link(search_string), afterfunc);
-	request.on('error', callback);
-}
+  UTIL.getPage(queryString, (err, body) => { // eslint-disable-line consistent-return
+    if (err) return callback(err);
+    let content;
+    try {
+      const parsed = JSON.parse(body);
+      content = parsed[parsed.length - 1].body.content;
+      callback(null, UTIL.parseFilters(content)); // eslint-disable-line callback-return
+    } catch (e) {
+      return callback(e);
+    }
+  });
+};

lib/util.js

@@ -1,213 +0,276 @@
-"use strict";
+const ENTITIES = require('html-entities').AllHtmlEntities;
+const PATH = require('path');
+const URL = require('url');
+const HTTPS = require('https');
+const FS = require('fs');
+const QUERYSTRING = require('querystring');
+const BASE_URL = 'https://www.youtube.com/results?';
 
-var Entities = require('html-entities').AllHtmlEntities;
-var url = require('url');
-var fs = require('fs');
-var querystring = require('querystring');
-var base_url = 'https://www.youtube.com/results?';
+// Builds the search query url
+exports.buildLink = query => BASE_URL + QUERYSTRING
+  .encode({
+    search_query: query,
+    spf: 'navigate',
+    gl: 'US',
+    hl: 'en',
+  });
 
-// builds the search query url
-exports.build_link = function(query) {
-	return base_url + querystring.encode({
-		search_query: query,
-		spf: 'navigate',
-		gl: 'US',
-		hl: 'en'
-	});
-}
+exports.buildFromNextpage = nextpageRef => {
+  let parsed = URL.parse(nextpageRef, true);
+  let overwrites = QUERYSTRING.decode(parsed.search.substr(1));
+  return BASE_URL + QUERYSTRING.encode(Object.assign({}, overwrites, {
+    spf: 'navigate',
+    gl: 'US',
+    hl: 'en',
+  }));
+};
 
-// start of parsing an item
-exports.parse_item = function(string, resp_string) {
-	var titles = exports.between(string, '<div class="', '"');
-	var type = exports.between(titles, 'yt-lockup yt-lockup-tile yt-lockup-', ' ');
-	if(type === 'playlist') {
-		return exports.parse_playlist(string);
-	} else if(type === 'channel') {
-		return exports.parse_channel(string);
-	} else if(type === 'video') {
-		return exports.parse_video(string);
-	} else if(type === 'movie-vertical-poster') {
-		return exports.parse_movie(string);
-	} else if(titles === 'search-refinements') {
-			return exports.parse_related_searches(string);
-	} else if(titles.includes('shelf') && string.includes('<div class="compact-shelf')) {
-			return exports.parse_shelf_compact(string);
-	} else if(titles.includes('shelf') && string.includes('<div class="vertical-shelf">')) {
-			return exports.parse_shelf_vertical(string);
-	} else {
-		console.error('\n/*****************************************************************************************************************************************************************************');
-		console.error('found an unknwon type |'+type+'|'+titles+'|');
-		console.error('pls post the content of to the files in ' + __dirname + require('path').sep + 'dumbs to https://github.com/TimeForANinja/node-ytsr/issues');
-		console.error('*****************************************************************************************************************************************************************************/\n');
-		fs.exists(__dirname + '/dumbs', function(exists) {
-			if(!exists) {
-				fs.mkdir(__dirname + '/dumbs/', function(err) {
-					fs.writeFile(__dirname + '/dumbs/' + Math.random().toString(36).substr(3) + '-' + Date.now() + '.dumb', type + '\n\n-----------------\n\n' + string + '\n\n-----------------\n\n' + resp_string, function(err) {});
-				});
-			}
-			else fs.writeFile(__dirname + '/dumbs/' + Math.random().toString(36).substr(3) + '-' +  Date.now() + '.dumb', type + '\n\n-----------------\n\n' + string + '\n\n-----------------\n\n' + resp_string, function(err) {});
-		});
-		return null;
-	}
-}
+// Start of parsing an item
+exports.parseItem = (string, respString, searchString) => {
+  const titles = exports.between(string, '<div class="', '"');
+  const type = exports.between(titles, 'yt-lockup yt-lockup-tile yt-lockup-', ' ');
+  if (type === 'playlist') {
+    if (string.includes('yt-pl-icon-mix')) return exports.parseMix(string);
+    return exports.parsePlaylist(string);
+  } else if (type === 'channel') {
+    return exports.parseChannel(string);
+  } else if (type === 'video') {
+    return exports.parseVideo(string);
+  } else if (type === 'movie-vertical-poster') {
+    return exports.parseMovie(string);
+  } else if (titles === 'search-refinements') {
+    return exports.parseRelatedSearches(string);
+  } else if (titles.includes('shelf') && string.includes('<div class="compact-shelf')) {
+    return exports.parseShelfCompact(string);
+  } else if (titles.includes('shelf') && string.includes('<div class="vertical-shelf">')) {
+    return exports.parseShelfVertical(string);
+  } else if (string.includes('<div class="display-message">No more results</div>')) {
+    return null;
+  } else {
+    const dir = PATH.resolve(__dirname, '../dumps/');
+    const file = PATH.resolve(dir, `${Math.random().toString(36).substr(3)}-${Date.now()}.dumb`);
+    const cfg = PATH.resolve(__dirname, '../package.json');
+    const bugsRef = require(cfg).bugs.url;
+    if (!FS.existsSync(dir)) FS.mkdirSync(dir);
+    FS.writeFileSync(file, JSON.stringify({ type, searchString, itemString: string, htmlBody: respString }));
+    /* eslint-disable no-console */
+    console.error(`\n/${'*'.repeat(200)}`);
+    console.error(`found an unknwon type |${type}|${titles}|`);
+    console.error(`pls post the the files in ${dir} to ${bugsRef}`);
+    console.error(`${'*'.repeat(200)}\\`);
+    /* eslint-enable no-console */
+    return null;
+  }
+};
 
-// parse an item of type playlist
-exports.parse_playlist = function(string) {
-	var owner_box = exports.between(string, '<div class="yt-lockup-byline ">', '</div>');
-	var thumbnail = exports.between(string, 'data-thumb="', '"');
-	thumbnail = thumbnail ? thumbnail : exports.between(string, 'src="', '"');
-	return {
-		type: 'playlist',
-		title: exports.remove_html(exports.between(exports.between(string, '<h3 class="yt-lockup-title ">', '</a>'), '>')),
-		link: 'https://www.youtube.com/playlist?list=' + exports.remove_html(exports.between(string, 'data-list-id="', '"')),
-		thumbnail: url.resolve(base_url, exports.remove_html(thumbnail)),
+exports.parseMix = string => {
+  const thumbnailRaw = exports.between(string, 'data-thumb="', '"');
+  const thumbnail = thumbnailRaw ? thumbnailRaw : exports.between(string, 'src="', '"');
+  const plistID = exports.removeHtml(exports.between(string, 'data-list-id="', '"'));
+  const videoID = exports.removeHtml(exports.between(string, 'data-video-ids="', '"'));
+  return {
+    type: 'mix',
+    title: exports.removeHtml(exports.between(exports.between(string, '<h3 class="yt-lockup-title ">', '</a>'), '>')),
+    firstItem: `https://www.youtube.com/watch?v=${videoID}&list=${plistID}`,
+    thumbnail: URL.resolve(BASE_URL, exports.removeHtml(thumbnail)),
+    length: exports.removeHtml(exports.between(string, '<span class="formatted-video-count-label">', '</span>')),
+  };
+};
 
-		author: {
-			name: exports.remove_html(exports.between(owner_box, '>', '</a>')),
-			id: exports.between(owner_box, 'data-ytid="', '"'),
-			ref: url.resolve(base_url, exports.remove_html(exports.between(owner_box, '<a href="', '"'))),
-			verified: string.includes('title="Verified"')
-		},
+// Parse an item of type playlist
+exports.parsePlaylist = string => {
+  const ownerBox = exports.between(string, '<div class="yt-lockup-byline ">', '</div>');
+  const thumbnailRaw = exports.between(string, 'data-thumb="', '"');
+  const thumbnail = thumbnailRaw ? thumbnailRaw : exports.between(string, 'src="', '"');
+  const cleanID = exports.removeHtml(exports.between(string, 'data-list-id="', '"'));
+  return {
+    type: 'playlist',
+    title: exports.removeHtml(exports.between(exports.between(string, '<h3 class="yt-lockup-title ">', '</a>'), '>')),
+    link: `https://www.youtube.com/playlist?list=${cleanID}`,
+    thumbnail: URL.resolve(BASE_URL, exports.removeHtml(thumbnail)),
 
-		length: exports.remove_html(exports.between(string, '<span class="formatted-video-count-label">', '</span>'))
-	}
-}
+    author: {
+      name: exports.removeHtml(exports.between(ownerBox, '>', '</a>')),
+      ref: URL.resolve(BASE_URL, exports.removeHtml(exports.between(ownerBox, '<a href="', '"'))),
+      verified: string.includes('title="Verified"'),
+    },
 
-// parse an item of type channel
-exports.parse_channel = function(string) {
-	var avatar = exports.between(string, 'data-thumb="', '"');
-	avatar = avatar ? avatar : exports.between(string, 'src="', '"');
-	return {
-		type: 'channel',
-		name: exports.remove_html(exports.between(exports.between(string, '<a href="', '</a>'), '>')),
-		channel_id: exports.between(string, 'data-ytid="', '"'),
-		link: url.resolve(base_url, exports.remove_html(exports.between(string, 'href="', '"'))),
-		avatar: url.resolve(base_url, exports.remove_html(avatar)),
-		verified: string.includes('title="Verified"') || string.includes('yt-channel-title-autogenerated'),
+    length: exports.removeHtml(exports.between(string, '<span class="formatted-video-count-label">', '</span>')),
+  };
+};
 
-		followers: Number(exports.between(exports.between(string, 'yt-subscriber-count"', '</span>'), '>').replace(/\.|,/g, '')),
-		description_short: exports.remove_html(exports.between(exports.between(string, '<div class="yt-lockup-description', '</div>'), '>')),
-		videos: Number(exports.between(string, '<ul class="yt-lockup-meta-info"><li>', '</li>').split(' ').splice(0,1)[0].replace(/\.|,/g, ''))
-	}
-}
+// Parse an item of type channel
+exports.parseChannel = string => {
+  const avatarRaw = exports.between(string, 'data-thumb="', '"');
+  const avatar = avatarRaw ? avatarRaw : exports.between(string, 'src="', '"');
+  const rawDesc = exports.between(exports.between(string, '<div class="yt-lockup-description', '</div>'), '>');
+  const rawFollows = exports.between(exports.between(string, 'yt-subscriber-count"', '</span>'), '>');
+  return {
+    type: 'channel',
+    name: exports.removeHtml(exports.between(exports.between(string, '<a href="', '</a>'), '>')),
+    channel_id: exports.between(string, 'data-channel-external-id="', '"'),
+    link: URL.resolve(BASE_URL, exports.removeHtml(exports.between(string, 'href="', '"'))),
+    avatar: URL.resolve(BASE_URL, exports.removeHtml(avatar)),
+    verified: string.includes('title="Verified"') || string.includes('yt-channel-title-autogenerated'),
 
-// parse an item of type video
-exports.parse_video = function(string) {
-	var owner_box = exports.between(string, '<div class="yt-lockup-byline ">', '</div>');
-	var meta_info = exports.between(string, '<div class="yt-lockup-meta ">', '</ul>').replace(/<\/li>/g, '').split('<li>').splice(1);
-	var thumbnail = exports.between(string, 'data-thumb="', '"');
-	thumbnail = thumbnail ? thumbnail : exports.between(string, 'src="', '"');
-	return {
-		type: 'video',
-		title: exports.remove_html(exports.between(exports.between(string, '<a href="', '</a>'), '>')),
-		link: url.resolve(base_url, exports.remove_html(exports.between(string, 'href="', '"'))),
-		thumbnail: url.resolve(base_url, exports.remove_html(thumbnail)),
+    followers: Number(rawFollows.replace(/\.|,/g, '')),
+    description_short: exports.removeHtml(rawDesc) || null,
+    videos: Number(exports.between(string, '<ul class="yt-lockup-meta-info"><li>', '</li>')
+      .split(' ')
+      .splice(0, 1)[0]
+      .replace(/\.|,/g, '')
+    ),
+  };
+};
 
-		author: {
-			name: exports.remove_html(exports.between(owner_box, '>', '</a>')),
-			id: exports.between(owner_box, 'data-ytid="', '"'),
-			ref: url.resolve(base_url, exports.remove_html(exports.between(owner_box, '<a href="', '"'))),
-			verified: owner_box.includes('title="Verified"')
-		},
+// Parse an item of type video
+exports.parseVideo = string => {
+  const ownerBox = exports.between(string, '<div class="yt-lockup-byline ">', '</div>');
+  const metaInfo = exports.between(string, '<div class="yt-lockup-meta ">', '</ul>')
+    .replace(/<\/li>/g, '')
+    .split('<li>')
+    .splice(1);
+  const thumbnailRaw = exports.between(string, 'data-thumb="', '"');
+  const thumbnail = thumbnailRaw ? thumbnailRaw : exports.between(string, 'src="', '"');
+  const rawDesc = exports.between(exports.between(string, '<div class="yt-lockup-description', '</div>'), '>');
+  return {
+    type: 'video',
+    title: exports.removeHtml(exports.between(exports.between(string, '<a href="', '</a>'), '>')),
+    link: URL.resolve(BASE_URL, exports.removeHtml(exports.between(string, 'href="', '"'))),
+    thumbnail: URL.resolve(BASE_URL, exports.removeHtml(thumbnail)),
 
-		description: exports.remove_html(exports.between(exports.between(string, '<div class="yt-lockup-description', '</div>'), '>')) || null,
-		views: meta_info[1] ? Number(meta_info[1].split(' ')[0].replace(/\.|,/g, '')) : null,
-		duration: exports.between(string, '<span class="video-time" aria-hidden="true">', '</span>'),
-		uploaded_at: meta_info[0] || null
-	}
-}
+    author: {
+      name: exports.removeHtml(exports.between(ownerBox, '>', '</a>')),
+      ref: URL.resolve(BASE_URL, exports.removeHtml(exports.between(ownerBox, '<a href="', '"'))),
+      verified: ownerBox.includes('title="Verified"'),
+    },
 
-// parse am item of type movie
-exports.parse_movie = function(string) {
-	var haystack = string.substr(string.lastIndexOf('<div class="yt-lockup-meta"><ul>') + 32);
-	var film_meta = haystack.substr(0, haystack.indexOf('</ul></div>'));
-	var author_info = string.substr(string.lastIndexOf('<a'), string.lastIndexOf('</a>')) + '</a>';
-	return {
-		type: 'movie',
-		title: exports.remove_html(exports.between(string, 'dir="ltr">', '</a>')),
-		link: url.resolve(base_url, exports.remove_html(exports.between(string, 'href="', '"'))),
-		thumbnail: url.resolve(base_url, exports.remove_html(exports.between(string, 'src="', '"'))),
+    description: exports.removeHtml(rawDesc) || null,
+    views: metaInfo[1] ? Number(metaInfo[1].split(' ')[0].replace(/\.|,/g, '')) : null,
+    duration: exports.between(string, '<span class="video-time" aria-hidden="true">', '</span>'),
+    uploaded_at: metaInfo[0] || null,
+  };
+};
 
-		author: {
-			name: exports.remove_html(exports.between(author_info, '>', '<')),
-			id: exports.between(author_info, 'data-ytid="', '"'),
-			ref: url.resolve(base_url, exports.remove_html(exports.between(author_info, '<a href="', '"'))),
-			verified: string.includes('title="Verified"')
-		},
+// Parse am item of type movie
+exports.parseMovie = string => {
+  const haystack = string.substr(string.lastIndexOf('<div class="yt-lockup-meta"><ul>') + 32);
+  const filmMeta = haystack.substr(0, haystack.indexOf('</ul></div>'));
+  const authorInfo = `${string.substr(string.lastIndexOf('<a'), string.lastIndexOf('</a>'))}</a>`;
+  const rawDesc = exports.between(string, 'yt-lockup-description', '</div>').replace(/[^>]+>/, '');
+  const rawMeta = exports.between(string, '<div class="yt-lockup-meta"><ul><li>', '</li></ul>');
+  return {
+    type: 'movie',
+    title: exports.removeHtml(exports.between(string, 'dir="ltr">', '</a>')),
+    link: URL.resolve(BASE_URL, exports.removeHtml(exports.between(string, 'href="', '"'))),
+    thumbnail: URL.resolve(BASE_URL, exports.removeHtml(exports.between(string, 'src="', '"'))),
 
-		description: exports.remove_html(exports.between(string, 'yt-lockup-description', '</div>').replace(/[^>]+>/, '')) || null,
-		meta: exports.remove_html(exports.between(string, '<div class="yt-lockup-meta"><ul><li>', '</li></ul>')).split(' · '),
-		actors: film_meta.split('<li>')[1].replace(/<[^>]+>|^[^:]+: /g, '').split(', ').map(function(a) { return exports.remove_html(a) }),
-		director: exports.remove_html(film_meta.split('<li>')[2].replace(/<[^>]+>|^[^:]+: /g, '')),
-		duration: exports.between(string, '<span class="video-time" aria-hidden="true">', '</span>')
-	}
-}
+    author: {
+      name: exports.removeHtml(exports.between(authorInfo, '>', '<')),
+      ref: URL.resolve(BASE_URL, exports.removeHtml(exports.between(authorInfo, '<a href="', '"'))),
+      verified: string.includes('title="Verified"'),
+    },
 
+    description: exports.removeHtml(rawDesc) || null,
+    meta: exports.removeHtml(rawMeta).split(' · '),
+    actors: filmMeta.split('<li>')[1].replace(/<[^>]+>|^[^:]+: /g, '').split(', ').map(a => exports.removeHtml(a)),
+    director: exports.removeHtml(filmMeta.split('<li>')[2].replace(/<[^>]+>|^[^:]+: /g, '')),
+    duration: exports.between(string, '<span class="video-time" aria-hidden="true">', '</span>'),
+  };
+};
 
-// parse an item of type related searches
-exports.parse_related_searches = function(string) {
-	let related = string.split('search-refinement').splice(1);
-	return related.map(function(item) {
-		return {
-			link: url.resolve(base_url, exports.remove_html(exports.between(item, 'href="', '"'))),
-			q: querystring.parse(exports.remove_html(exports.between(item, '/results?', '"')))['q']
-		}
-	});
-}
+// Parse an item of type related searches
+exports.parseRelatedSearches = string => {
+  const related = string.split('search-refinement').splice(2);
+  return {
+    type: 'search-refinements',
+    entrys: related.map(item => ({
+      link: URL.resolve(BASE_URL, exports.removeHtml(exports.between(item, 'href="', '"'))),
+      q: QUERYSTRING.parse(exports.removeHtml(exports.between(item, '/results?', '"'))).search_query || null,
+    })),
+  };
+};
 
-// horizontal shelf of youtube movie proposals
-exports.parse_shelf_compact = function(string) {
-	const items_raw = string.split('<li class="yt-uix-shelfslider-item').splice(1);
-	let items =  items_raw.map(function(item) {
-		const item_meta = exports.between(item, 'grid-movie-renderer-metadata"><li>', '</li>').split('·');
-		const views = exports.between(item, '<ul class="yt-lockup-meta-info">', '</li>').replace(/<[^>]+>| .*/g, '');
-		return {
-			type: exports.between(item, ' ', '-')+'-short',
-			ref: url.resolve(base_url, exports.remove_html(exports.between(item, 'href="', '"'))),
-			thumbnail: url.resolve(base_url, exports.remove_html(exports.between(item, 'src="', '"'))),
-			duration: exports.between(item, '"video-time"', '<').replace(/^[^>]+>/, ''),
-			published: item_meta[0].trim(),
-			genre: exports.remove_html(item_meta[1].trim()),
-			views: views ? Number(views.replace(/\.|,/g, '')) : null,
-			price: exports.between(item, '<span class="button-label">', '</span>').replace(/^[^ ]+ /, '') || null
-		}
-	})
-	return {
-		type: 'shelf-compact',
-		title: exports.remove_html(exports.between(string, '<span class="branded-page-module-title-text">', '</span>')),
-		items: items
-	};
-}
+// Horizontal shelf of youtube movie proposals
+exports.parseShelfCompact = string => {
+  const itemsRaw = string.split('<li class="yt-uix-shelfslider-item').splice(1);
+  const items = itemsRaw.map(item => ({
+    type: `${exports.between(item, ' ', '-')}-short`,
+    name: exports.removeHtml(exports.between(exports.between(item, '><a href="', '</a>'), '>', '')),
+    ref: URL.resolve(BASE_URL, exports.removeHtml(exports.between(item, 'href="', '"'))),
+    thumbnail: URL.resolve(BASE_URL, exports.removeHtml(exports.between(item, 'src="', '"'))),
+    duration: exports.between(item, '"video-time"', '<').replace(/^[^>]+>/, ''),
+    price: exports.between(item, '<span class="button-label">', '</span>').replace(/^[^ ]+ /, '') || null,
+  }));
+  return {
+    type: 'shelf-compact',
+    title: exports.removeHtml(exports.between(string, '<span class="branded-page-module-title-text">', '</span>')),
+    items,
+  };
+};
 
-// vertical shelf of youtube video proposals
-exports.parse_shelf_vertical = function(string) {
-	const items_raw = string.split('<a aria-hidden="').splice(1);
-	return {
-		type: 'shelf-vertical',
-		title: exports.remove_html(exports.between(string, '<span class="branded-page-module-title-text">', '</span>')),
-		items: items_raw.map(function(item) { return exports.parse_video(item) })
-	};
-}
+// Vertical shelf of youtube video proposals
+exports.parseShelfVertical = string => {
+  const itemsRaw = string.split('<a aria-hidden="').splice(1);
+  return {
+    type: 'shelf-vertical',
+    title: exports.removeHtml(exports.between(string, '<span class="branded-page-module-title-text">', '</span>')),
+    items: itemsRaw.map(item => exports.parseVideo(item)),
+  };
+};
 
-// taken from https://github.com/fent/node-ytdl-core/
-exports.between = function(haystack, left, right) {
-	var pos;
-	pos = haystack.indexOf(left);
-	if(pos === -1) { return ''; }
-	haystack = haystack.slice(pos + left.length);
-	if(!right) { return haystack; }
-	pos = haystack.indexOf(right);
-	if(pos === -1) { return ''; }
-	haystack = haystack.slice(0, pos);
-	return haystack;
+// Taken from https://github.com/fent/node-ytdl-core/
+const between = exports.between = (haystack, left, right) => {
+  let pos;
+  pos = haystack.indexOf(left);
+  if (pos === -1) { return ''; }
+  haystack = haystack.slice(pos + left.length);
+  if (!right) { return haystack; }
+  pos = haystack.indexOf(right);
+  if (pos === -1) { return ''; }
+  haystack = haystack.slice(0, pos);
+  return haystack;
 };
 
-// cleans up html text
-exports.remove_html = function(string) {
-	return new Entities().decode(
-		string.replace(/\n/g, ' ')
-		.replace(/\s*<\s*br\s*\/?\s*>\s*/gi, '\n')
-		.replace(/<\s*\/\s*p\s*>\s*<\s*p[^>]*>/gi, '\n')
-		.replace(/<.*?>/gi, '')
-	).trim();
+// Cleans up html text
+const removeHtml = exports.removeHtml = string => new ENTITIES().decode(
+  string.replace(/\n/g, ' ')
+    .replace(/\s*<\s*br\s*\/?\s*>\s*/gi, '\n')
+    .replace(/<\s*\/\s*p\s*>\s*<\s*p[^>]*>/gi, '\n')
+    .replace(/<.*?>/gi, '')
+).trim();
+
+exports.getPage = (ref, cb) => {
+  const request = HTTPS.get(ref, resp => { // eslint-disable-line consistent-return
+    if (resp.statusCode !== 200) return cb(new Error(`Status Code ${resp.statusCode}`));
+    const respBuffer = [];
+    resp.on('data', d => respBuffer.push(d));
+    resp.on('end', () => {
+      cb(null, Buffer.concat(respBuffer).toString());
+    });
+  });
+  request.on('error', cb);
 };
+
+exports.parseFilters = body => {
+  const filterContainer = between(body, '<div id="filter-dropdown"', '<ol id="item-section');
+  const coloms = filterContainer.split('<h4 class="filter-col-title">').splice(1);
+  const results = new Map();
+  coloms.forEach(c => {
+    const items = c.trim().split('<li>').filter(a => a);
+    const title = between(items.splice(0, 1)[0], '', '</h4>');
+    const array = results.set(title, []).get(title);
+    array.active = null;
+    items.forEach(i => {
+      let isActive = between(i, 'class="', '"').includes('filter-selected');
+      let parsedItem = {
+        ref: isActive ? null : URL.resolve(BASE_URL, removeHtml(between(i, 'href="', '"'))),
+        name: removeHtml(between(between(i, '>', '</span>'), '>')),
+        active: isActive,
+      };
+      if (isActive) array.active = parsedItem;
+      array.push(parsedItem);
+    });
+  });
+  return results;
+};

package.json

@@ -8,3 +8,3 @@ {
   ],
-  "version": "0.1.7",
+  "version": "0.1.8",
   "repository": {
@@ -20,3 +20,6 @@ "type": "git",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "npm run lint && npm run istanbul",
+    "istanbul": "istanbul cover node_modules/mocha/bin/_mocha -- -t 16000 test/*-test.js",
+    "lint": "eslint ./",
+    "lint:fix": "eslint --fix ./"
   },
@@ -26,6 +29,16 @@ "dependencies": {
   },
+  "devDependencies": {
+    "eslint": "^4.10.0",
+    "assert-diff": "^1.2.4",
+    "istanbul": "^0.4.5",
+    "mocha": "^5.0.0",
+    "nock": "^9.1.5"
+  },
   "engines": {
-    "node": ">=0.12"
+    "node": ">=6"
   },
+  "bugs": {
+    "url": "https://github.com/TimeForANinja/node-ytsr/issues"
+  },
   "license": "MIT"
 }

README.md

@@ -5,3 +5,4 @@ <div align="center">
     <a href="https://www.npmjs.com/package/ytsr"><img src="https://img.shields.io/npm/dt/ytsr.svg?maxAge=3600" alt="NPM downloads" /></a>
-    <a href="https://david-dm.org/timeforaninja/ytsr.svg"><img src="https://img.shields.io/david/timeforaninja/ytsr.svg?maxAge=3600" alt="Dependencies" /></a>
+    <a href="https://david-dm.org/"><img src="https://img.shields.io/david/timeforaninja/node-ytsr.svg?maxAge=3600" alt="Dependencies" /></a>
+    <a href="https://greenkeeper.io/"><img src="https://badges.greenkeeper.io/TimeForANinja/node-ytsr.svg" alt="Dependencies" /></a>
   </p>
@@ -21,15 +22,21 @@ <p>
 ```js
-var ytsr = require('ytsr');
+const ytsr = require('ytsr');
+let filter;
 
-ytsr.get_filters('github', function(err, filters) {
-	var filter = filters['type'].find((o) => {return o.name == 'Video'})
-	var options = {
-		limit: 5,
-		nextpage_ref: filter.ref,
-	}
-	ytsr.search(null, options, function(err, search_results) {
-		if(err) throw err;
-		dosth(search_results);
+ytsr.getFilters('github', function(err, filters) {
+  if(err) throw err;
+	filter = filters.get('Type').find(o => o.name === 'Video');
+  ytsr.getFilters(filter.ref, function(err, filters) {
+    if(err) throw err;
+  	filter = filters.get('Duration').find(o => o.name.startsWith('Short'));
+  	var options = {
+  		limit: 5,
+  		nextpageRef: filter.ref,
+  	}
+  	ytsr(null, options, function(err, searchResults) {
+  		if(err) throw err;
+  		dosth(searchResults);
+  	});
 	});
-})
+});
 ```
@@ -39,7 +46,7 @@
 # API
-### ytsr.search(search_string, [options, callback])
+### ytsr(searchString, [options, callback])
 
 Searches for the given string
 
-* `search_string`
+* `searchString`
     * string to search for
@@ -50,3 +57,3 @@ * `options`
     * limit[integer] -> limits the pulled items
-	* nextpage_ref[String] -> if u wanna continue a previous search
+	* nextpageRef[String] -> if u wanna continue a previous search or use filters
 * `callback(err, result)`
@@ -60,8 +67,9 @@ * function
 
-### ytsr.get_filters(search_string, [callback])
+### ytsr.getFilters(searchString, [callback])
 
-Pulls avaible filters for the given string
+Pulls avaible filters for the given string/ref
 
-* `search_string`
+* `searchString`
     * string to search for
+    * or previously optained filter ref
 * `callback(err, result)`
@@ -68,0 +76,0 @@ * function

New alerts

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Dynamic require

Supply chain risk

Dynamic require can indicate the package is performing dangerous or unsafe dynamic code execution.

Found 1 instance in 1 package

Fixed alerts

No bug tracker

Maintenance

Package does not have a linked bug tracker in package.json.

Found 1 instance in 1 package

No tests

Quality

Package does not have any tests. This is a strong signal of a poorly maintained or low quality package.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

21642

increased by5.97%

Lines of code

366

increased by3.98%

Number of low maintenance alerts

0

decreased by-100%

Number of low quality alerts

2

decreased by-33.33%

Number of lines in readme file

93

increased by9.41%

Worsened metrics

Dev dependency count

5

increased byInfinity%

Number of low supply chain risk alerts

2

increased by100%

Number of medium supply chain risk alerts

2

increased by100%

No dependency changes