ytsr - npm Package Compare versions

Comparing version 0.1.5 to 0.1.6

lib/main.js

@@ -39,3 +39,3 @@ "use strict";
 
-			// get the table of items and parse it
+			// get the table of items and parse it(remove null items where the parsing failed)
 			var table = util.between(content, '<ol id="item-section-', '\n</ol>').split('</li>\n\n<li>');
@@ -48,3 +48,3 @@ table = table.filter(function(t) {
 			});
-			table = table.map(function(t) { return util.parse_item(t) });
+			table = table.map(function(t) { return util.parse_item(t) }).filter(function(a) { return a });
 
@@ -64,9 +64,9 @@ // get amount of results
 					query: search_string,
-					items: table,
 					results: results ? results : 0,
 					filters: set_filters,
 					current_ref: current_ref ? current_ref : undefined,
+					items: table
 				});
 			}
-			var nextpage_ref = util.between(last_page_ref, 'href="', '"');
+			var nextpage_ref = util.remove_html(util.between(last_page_ref, 'href="', '"'));
 
@@ -160,3 +160,3 @@ // check wether we hit the set limit
 						return {
-							ref: util.between(p, 'href="', '"'),
+							ref: util.remove_html(util.between(p, 'href="', '"')),
 							name: util.between(util.between(p, '>', '</span>'), '>'),
@@ -163,0 +163,0 @@ }

lib/util.js

@@ -5,2 +5,3 @@ "use strict";
 var url = require('url');
+var fs = require('fs');
 var querystring = require('querystring');
@@ -21,14 +22,32 @@ var base_url = 'https://www.youtube.com/results?';
 exports.parse_item = function(string) {
-	var type = exports.between(string, '<div class="yt-lockup yt-lockup-tile yt-lockup-', ' ');
-	if(type == 'playlist') {
+	var titles = exports.between(string, '<div class="', '"');
+	var type = exports.between(titles, 'yt-lockup yt-lockup-tile yt-lockup-', ' ');
+	if(type === 'playlist') {
 		return exports.parse_playlist(string);
-	} else if(type == 'channel') {
+	} else if(type === 'channel') {
 		return exports.parse_channel(string);
-	} else if(type == 'video') {
+	} else if(type === 'video') {
 		return exports.parse_video(string);
-	} else if(exports.between(string, '<div class="', '"') === 'search-refinements') {
+	} else if(type === 'movie-vertical-poster') {
+		return exports.parse_movie(string);
+	} else if(titles === 'search-refinements') {
 			return exports.parse_related_searches(string);
+	} else if(titles.includes('shelf') && string.includes('<div class="compact-shelf')) {
+			return exports.parse_shelf_compact(string);
+	} else if(titles.includes('shelf') && string.includes('<div class="vertical-shelf">')) {
+			return exports.parse_shelf_vertical(string);
 	} else {
-		console.dir(string);
-		throw new Error('unknown type |' + type + '|')
+		console.error('\n/*****************************************************************************************************************************************************************************');
+		console.error('found an unknwon type |'+type+'|'+titles+'|');
+		console.error('pls post the content of to the files in ' + __dirname + require('path').sep + 'dumbs to https://github.com/TimeForANinja/node-ytsr/issues');
+		console.error('*****************************************************************************************************************************************************************************/\n');
+		fs.exists(__dirname + '/dumbs', function(exists) {
+			if(!exists) {
+				fs.mkdir(__dirname + '/dumbs/', function(err) {
+					fs.writeFile(__dirname + '/dumbs/' + Math.random().toString(36).substr(3) + '-' + Date.now() + '.dumb', type + '\n\n\n' + string, function(err) {});
+				});
+			}
+			else fs.writeFile(__dirname + '/dumbs/' + Math.random().toString(36).substr(3) + '-' +  Date.now() + '.dumb', type + '\n\n\n' + string, function(err) {});
+		});
+		return null;
 	}
@@ -45,4 +64,4 @@ }
 		title: exports.remove_html(exports.between(exports.between(string, '<h3 class="yt-lockup-title ">', '</a>'), '>')),
-		link: 'https://www.youtube.com/playlist?list=' + exports.between(string, 'data-list-id="', '"'),
-		thumbnail: url.resolve(base_url, thumbnail),
+		link: 'https://www.youtube.com/playlist?list=' + exports.remove_html(exports.between(string, 'data-list-id="', '"')),
+		thumbnail: url.resolve(base_url, exports.remove_html(thumbnail)),
 
@@ -52,6 +71,7 @@ author: {
 			id: exports.between(owner_box, 'data-ytid="', '"'),
-			ref: url.resolve(base_url, exports.between(owner_box, '<a href="', '"')),
+			ref: url.resolve(base_url, exports.remove_html(exports.between(owner_box, '<a href="', '"'))),
+			verified: string.includes('title="Verified"')
 		},
 
-		length: exports.remove_html(exports.between(string, '<span class="formatted-video-count-label">', '</span>')),
+		length: exports.remove_html(exports.between(string, '<span class="formatted-video-count-label">', '</span>'))
 	}
@@ -68,8 +88,9 @@ }
 		channel_id: exports.between(string, 'data-ytid="', '"'),
-		link: url.resolve(base_url, exports.between(string, 'href="', '"')),
-		avatar: url.resolve(base_url, avatar),
+		link: url.resolve(base_url, exports.remove_html(exports.between(string, 'href="', '"'))),
+		avatar: url.resolve(base_url, exports.remove_html(avatar)),
+		verified: string.includes('title="Verified"') || string.includes('yt-channel-title-autogenerated'),
 
-		followers: Number(exports.between(exports.between(string, 'yt-subscriber-count"', '</span>'), '>').replace(/\./g, '')),
+		followers: Number(exports.between(exports.between(string, 'yt-subscriber-count"', '</span>'), '>').replace(/\.|,/g, '')),
 		description_short: exports.remove_html(exports.between(exports.between(string, '<div class="yt-lockup-description', '</div>'), '>')),
-		videos: exports.between(string, '<ul class="yt-lockup-meta-info"><li>', '</li>'),
+		videos: Number(exports.between(string, '<ul class="yt-lockup-meta-info"><li>', '</li>').split(' ').splice(0,1)[0].replace(/\.|,/g, ''))
 	}
@@ -81,5 +102,4 @@ }
 	var owner_box = exports.between(string, '<div class="yt-lockup-byline ">', '</div>');
-	var meta_info = exports.between(string, '<ul class="yt-lockup-meta-info">', '</ul>').replace(/<\/li>/g, '').split('<li>');
+	var meta_info = exports.between(string, '<div class="yt-lockup-meta ">', '</ul>').replace(/<\/li>/g, '').split('<li>').splice(1);
 	var thumbnail = exports.between(string, 'data-thumb="', '"');
-	var description = exports.remove_html(exports.between(exports.between(string, '<div class="yt-lockup-description', '</div>'), '>'));
 	thumbnail = thumbnail ? thumbnail : exports.between(string, 'src="', '"');
@@ -89,4 +109,4 @@ return {
 		title: exports.remove_html(exports.between(exports.between(string, '<a href="', '</a>'), '>')),
-		link: url.resolve(base_url, exports.between(string, 'href="', '"')),
-		thumbnail: url.resolve(base_url, thumbnail),
+		link: url.resolve(base_url, exports.remove_html(exports.between(string, 'href="', '"'))),
+		thumbnail: url.resolve(base_url, exports.remove_html(thumbnail)),
 
@@ -96,18 +116,47 @@ author: {
 			id: exports.between(owner_box, 'data-ytid="', '"'),
-			ref: url.resolve(base_url, exports.between(owner_box, '<a href="', '"')),
+			ref: url.resolve(base_url, exports.remove_html(exports.between(owner_box, '<a href="', '"'))),
+			verified: owner_box.includes('title="Verified"')
 		},
 
-		description: description ? description : null,
-		meta: meta_info,
+		description: exports.remove_html(exports.between(exports.between(string, '<div class="yt-lockup-description', '</div>'), '>')) || null,
+		views: meta_info[1] ? Number(meta_info[1].split(' ')[0].replace(/\.|,/g, '')) : null,
 		duration: exports.between(string, '<span class="video-time" aria-hidden="true">', '</span>'),
-		uploaded_at: meta_info[1],
+		uploaded_at: meta_info[0] || null
 	}
 }
 
+// parse am item of type movie
+exports.parse_movie = function(string) {
+	var haystack = string.substr(string.lastIndexOf('<div class="yt-lockup-meta"><ul>') + 32);
+	var film_meta = haystack.substr(0, haystack.indexOf('</ul></div>'));
+	var author_info = string.substr(string.lastIndexOf('<a'), string.lastIndexOf('</a>')) + '</a>';
+	return {
+		type: 'movie',
+		title: exports.remove_html(exports.between(string, 'dir="ltr">', '</a>')),
+		link: url.resolve(base_url, exports.remove_html(exports.between(string, 'href="', '"'))),
+		thumbnail: url.resolve(base_url, exports.remove_html(exports.between(string, 'src="', '"'))),
+
+		author: {
+			name: exports.remove_html(exports.between(author_info, '>', '<')),
+			id: exports.between(author_info, 'data-ytid="', '"'),
+			ref: url.resolve(base_url, exports.remove_html(exports.between(author_info, '<a href="', '"'))),
+			verified: string.includes('title="Verified"')
+		},
+
+		description: exports.remove_html(exports.between(string, 'yt-lockup-description', '</div>').replace(/[^>]+>/, '')) || null,
+		meta: exports.remove_html(exports.between(string, '<div class="yt-lockup-meta"><ul><li>', '</li></ul>')).split(' · '),
+		actors: film_meta.split('<li>')[1].replace(/<[^>]+>|^[^:]+: /g, '').split(', ').map(function(a) { return exports.remove_html(a) }),
+		director: exports.remove_html(film_meta.split('<li>')[2].replace(/<[^>]+>|^[^:]+: /g, '')),
+		duration: exports.between(string, '<span class="video-time" aria-hidden="true">', '</span>')
+	}
+}
+
+
+// parse an item of type related searches
 exports.parse_related_searches = function(string) {
 	let related = string.split('search-refinement').splice(1);
-	return related.map(item => {
+	return related.map(function(item) {
 		return {
-			link: url.resolve(base_url, exports.between(item, 'href="', '"')),
-			q: querystring.parse(exports.between(item, '/results?', '"'))['q']
+			link: url.resolve(base_url, exports.remove_html(exports.between(item, 'href="', '"'))),
+			q: querystring.parse(exports.remove_html(exports.between(item, '/results?', '"')))['q']
 		}
@@ -117,2 +166,36 @@ });
 
+// horizontal shelf of youtube movie proposals
+exports.parse_shelf_compact = function(string) {
+	const items_raw = string.split('<li class="yt-uix-shelfslider-item').splice(1);
+	let items =  items_raw.map(function(item) {
+		const item_meta = exports.between(item, 'grid-movie-renderer-metadata"><li>', '</li>').split('·');
+		const views = exports.between(item, '<ul class="yt-lockup-meta-info">', '</li>').replace(/<[^>]+>| .*/g, '');
+		return {
+			type: exports.between(item, ' ', '-')+'-short',
+			ref: url.resolve(base_url, exports.remove_html(exports.between(item, 'href="', '"'))),
+			thumbnail: url.resolve(base_url, exports.remove_html(exports.between(item, 'src="', '"'))),
+			duration: exports.between(item, '"video-time"', '<').replace(/^[^>]+>/, ''),
+			published: item_meta[0].trim(),
+			genre: exports.remove_html(item_meta[1].trim()),
+			views: views ? Number(views.replace(/\.|,/g, '')) : null,
+			price: exports.between(item, '<span class="button-label">', '</span>').replace(/^[^ ]+ /, '') || null
+		}
+	})
+	return {
+		type: 'shelf-compact',
+		title: exports.remove_html(exports.between(string, '<span class="branded-page-module-title-text">', '</span>')),
+		items: items
+	};
+}
+
+// vertical shelf of youtube video proposals
+exports.parse_shelf_vertical = function(string) {
+	const items_raw = string.split('<a aria-hidden="').splice(1);
+	return {
+		type: 'shelf-vertical',
+		title: exports.remove_html(exports.between(string, '<span class="branded-page-module-title-text">', '</span>')),
+		items: items_raw.map(function(item) { return exports.parse_video(item) })
+	};
+}
+
 // taken from https://github.com/fent/node-ytdl-core/
@@ -119,0 +202,0 @@ exports.between = function(haystack, left, right) {

package.json

@@ -8,3 +8,3 @@ {
   ],
-  "version": "0.1.5",
+  "version": "0.1.6",
   "repository": {
@@ -11,0 +11,0 @@ "type": "git",

New alerts

Filesystem access

Supply chain risk

Accesses the file system, and could potentially read sensitive data.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

25664

increased by25.58%

Lines of code

351

increased by28.1%

Worsened metrics

Number of low supply chain risk alerts

1

increased byInfinity%

No dependency changes