@freetube/yt-trending-scraper - npm Package Compare versions

Comparing version 3.1.0 to 3.1.1

package.json

 {
   "name": "@freetube/yt-trending-scraper",
-  "version": "3.1.0",
+  "version": "3.1.1",
   "description": "Identifies the currently trending videos on YouTube and returns all trending site information about every video without accessing the YouTube API.",
@@ -8,3 +8,4 @@ "main": "index.js",
     "index.js",
-    "src/"
+    "src/",
+    "index.d.ts"
   ],
@@ -54,3 +55,5 @@ "scripts": {
     "jest": "^28.1.3",
-    "prettier": "^2.7.1"  }
+    "prettier": "^2.7.1"
+  },
+  "types": "index.d.ts"
 }

README.md

 # YouTube Trending Videos Scraper NodeJS Documentation
-This NodeJS library can scrape all available trending pages of YouTube without any API usage. It is developed for and tailored towards easy usage in the [FreeTube](https://github.com/FreeTubeApp/FreeTube) rewrite but can be used with any other project as well.
+This NodeJS library can scrape all available trending pages of YouTube without any API usage. It is developed for and tailored towards easy usage in [FreeTube](https://github.com/FreeTubeApp/FreeTube) but can be used with any other project as well.
 
@@ -10,6 +10,6 @@ Therefore, this library does not require any API keys, with the attached maximum quotas, but instead might take longer to receive the required data.
 ## Installation
-`npm install yt-trending-scraper`
+`npm install @freetubeapp/yt-trending-scraper`
 
 ## Usage
-`const ytrend = require("yt-trending-scraper")`
+`const ytrend = require("@freetubeapp/yt-trending-scraper")`
 
@@ -54,3 +54,3 @@ ## API
 }).catch((error)=>{
-    console.log(error);
+    console.error(error);
 });
@@ -79,3 +79,4 @@
     isVerified:         Boolean,
-    isVerifiedArist:    Boolean
+    isVerifiedArist:    Boolean,
+    isShort:            Boolean
 }
@@ -82,0 +83,0 @@

src/HtmlParser.js

 class HtmlParser {
   static parseNewHtml(htmlData, parseCreatorOnRise) {
     // matches the special setup of the video elements
-    let jsonContent = '{' + htmlData.match(/"sectionListRenderer".+?(},"tab)/)[0]
-    // remove the last chars in order to make it valid JSON
-    jsonContent = jsonContent.substring(0, jsonContent.length - 5)
-    const contentArrayJSON = JSON.parse(jsonContent).sectionListRenderer.contents
+    const jsonDataRegex = /ytInitialData = (.+)?(;<\/script>)/
+    const jsonObject = JSON.parse(htmlData.match(jsonDataRegex)[1])
+    const jsonContent = jsonObject.contents.twoColumnBrowseResultsRenderer.tabs
+      .find(e => e.tabRenderer.selected)
+      .tabRenderer
+      .content
+    let contentArrayJSON
+    if ('sectionListRenderer' in jsonContent) {
+      contentArrayJSON = jsonContent.sectionListRenderer.contents
+    } else if ('richGridRenderer' in jsonContent) {
+      contentArrayJSON = jsonContent.richGridRenderer.contents
+    }
     let videos = []
     const currentTime = Date.now()
     contentArrayJSON.forEach((data) => {
-      const videoList = this.buildApiOutput(data.itemSectionRenderer.contents[0].shelfRenderer.content, currentTime, parseCreatorOnRise)
-      videos = [...videos, ...videoList]
+      if ('itemSectionRenderer' in data) {
+        const videoList = this.buildApiOutput(data.itemSectionRenderer.contents[0].shelfRenderer.content, currentTime, parseCreatorOnRise)
+        videos = [...videos, ...videoList]
+      } else if ('richItemRenderer' in data) {
+        videos.push(
+          this.parseRichItemRenderer(data, currentTime)
+        )
+      } else if ('richSectionRenderer' in data) {
+        const videoList = this.parseRichSectionRenderer(data, currentTime)
+        videos = [...videos, ...videoList]
+      }
     })
-    return videos
+    return this.deduplicateVideoList(videos)
   }
 
+  static deduplicateVideoList(videos) {
+    const uniqueIds = new Set()
+    return videos.filter((video) => {
+      if (!uniqueIds.has(video.videoId)) {
+        uniqueIds.add(video.videoId)
+        return true
+      }
+      return false
+    })
+  }
+
   // access the one video container and build and object with all the data required
@@ -50,2 +78,12 @@ static buildApiOutput(videoList, currentTime, parseCreatorOnRise) {
 
+  static parseRichItemRenderer(data, currentTime) {
+    return this.parseVideo(data.richItemRenderer.content.videoRenderer, currentTime)
+  }
+
+  static parseRichSectionRenderer(data, currentTime) {
+    return data.richSectionRenderer.content.richShelfRenderer.contents.map(rsr => {
+      return this.parseRichItemRenderer(rsr, currentTime)
+    })
+  }
+
   static parseVideo(videoRenderer, currentTime) {
@@ -52,0 +90,0 @@ const videoEntry = {

Fixed alerts

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

New author

Supply chain risk

A new npm collaborator published a version of the package for the first time. New collaborators are usually benign additions to a project, but do indicate a change to the security surface area of a package.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

51888

increased by4.62%

Number of package files

8

increased by14.29%

Lines of code

298

increased by33.63%

Number of lines in readme file

91

increased by1.11%

Number of medium supply chain risk alerts

0

decreased by-100%

No dependency changes