@freetube/yt-comment-scraper - npm Package Compare versions

Comparing version 6.1.0 to 6.2.0

index.js

@@ -1,1 +0,1 @@
-module.exports = require("./src/Youtube-Scraper")
+module.exports = require('./src/Youtube-Scraper')

package.json

 {
   "name": "@freetube/yt-comment-scraper",
-  "version": "6.1.0",
+  "version": "6.2.0",
   "description": "Scrapes the comments of any YouTube video without YouTube API access. Uses the default YouTube Ajax calls to get the comment data.",
   "main": "index.js",
+  "files": [
+    "index.js",
+    "src/",
+    "index.d.ts"
+  ],
   "scripts": {
     "test": "jest --watchAll --verbose",
-    "test-ci": "jest --verbose --ci"
+    "test-ci": "jest --verbose --ci",
+    "lint-fix": "eslint --fix --ext .js ./",
+    "lint": "eslint --ext .js ./"
   },
@@ -45,7 +52,17 @@ "repository": {
   "dependencies": {
-    "axios": "^0.21.1"
+    "axios": "^0.27.2"
   },
   "devDependencies": {
-    "jest": "^27.0.4"
-  }
+    "eslint": "^8.22.0",
+    "eslint-config-prettier": "^8.5.0",
+    "eslint-config-standard": "^17.0.0",
+    "eslint-plugin-import": "^2.26.0",
+    "eslint-plugin-n": "15.2.5",
+    "eslint-plugin-node": "^11.1.0",
+    "eslint-plugin-prettier": "^4.2.1",
+    "eslint-plugin-promise": "^6.0.0",
+    "jest": "^28.1.3",
+    "prettier": "^2.7.1"
+  },
+  "types": "index.d.ts"
 }

README.md

@@ -57,2 +57,3 @@ # YouTube Comment Scraper NodeJS Documentation
 {
+  total: Number | null, // Total number of comments on the video
   comments: [
@@ -59,0 +60,0 @@ {

src/htmlParser.js

@@ -24,11 +24,7 @@ class HtmlParser {
       const icon = comment.authorCommentBadge?.authorCommentBadgeRenderer?.icon
-      const isVerified = icon?.iconType === "CHECK_CIRCLE_THICK" || icon?.iconType === "CHECK"
-      const isOfficialArtist = icon?.iconType === "OFFICIAL_ARTIST_BADGE"
+      const isVerified = icon?.iconType === 'CHECK_CIRCLE_THICK' || icon?.iconType === 'CHECK'
+      const isOfficialArtist = icon?.iconType === 'OFFICIAL_ARTIST_BADGE'
 
-      let isHearted = comment.actionButtons.commentActionButtonsRenderer.creatorHeart?.creatorHeartRenderer?.isHearted ?? false
+      const isHearted = comment.actionButtons.commentActionButtonsRenderer.creatorHeart?.creatorHeartRenderer?.isHearted ?? false
 
-      if (typeof heartBadge !== 'undefined') {
-        isHearted = heartBadge.creatorHeartRenderer.isHearted
-      }
-
       let isMember = false
@@ -45,4 +41,18 @@ let memberIconUrl = null
       contentText.forEach((content) => {
-        if (content.text.trim() === '') {
+        if (content.text === '\n') {
           text = text + '<br>'
+        } else if (content.bold || content.strikethrough || content.italics) {
+          let formattedText = content.text
+
+          if (content.bold) {
+            formattedText = '<b>' + formattedText + '</b>'
+          }
+          if (content.strikethrough) {
+            formattedText = '<s>' + formattedText + '</s>'
+          }
+          if (content.italics) {
+            formattedText = '<i>' + formattedText + '</i>'
+          }
+
+          text = text + formattedText
         } else {
@@ -85,8 +95,3 @@ text = text + content.text
         commentData.replyToken = continuation
-
-        const replyArrayLength = replyNode.viewReplies.buttonRenderer.text.runs.length
-        // lengths of: 1 = reply (not from owner), 2 = reply (from owner), 3 = replies (not from owner), 5 = replies (from owmer)
-        if (replyArrayLength === 2 || replyArrayLength === 5) {
-          commentData.hasOwnerReplied = true
-        }
+        commentData.hasOwnerReplied = replyNode.viewRepliesCreatorThumbnail !== undefined
       }
@@ -99,2 +104,3 @@
   }
+
   static parseShortedNumberString(string) {
@@ -101,0 +107,0 @@ const numberMultiplier = string.charAt(string.length - 1).toLowerCase()

src/HttpRequester.js

@@ -1,3 +0,3 @@
-const axios = require("axios")
-const baseURL = "https://www.youtube.com"
+const axios = require('axios')
+const baseURL = 'https://www.youtube.com'
 
@@ -20,3 +20,3 @@ class HttpRequester {
     if (this.mustSetCookie) {
-      this.session.defaults.headers.cookie = [`CONSENT=YES+`]
+      this.session.defaults.headers.cookie = ['CONSENT=YES+']
     }
@@ -37,14 +37,14 @@ }
 
-    const html_data = initialResponse.data
+    const htmlData = initialResponse.data
 
     // Cache data in the requester for future use
-    requester.cachedInitialData = html_data
+    requester.cachedInitialData = htmlData
 
     const ytConfigSelectors = [
-      /"INNERTUBE_CONTEXT_CLIENT_NAME":(\d*)/,     // X-YouTube-Client-Name
-      /"INNERTUBE_CONTEXT_CLIENT_VERSION":"([^"]*)"/,  // X-YouTube-Client-Version
+      /"INNERTUBE_CONTEXT_CLIENT_NAME":(\d*)/, // X-YouTube-Client-Name
+      /"INNERTUBE_CONTEXT_CLIENT_VERSION":"([^"]*)"/, // X-YouTube-Client-Version
     ]
 
-    const ytConfigData = html_data.match(
-      new RegExp(ytConfigSelectors.map(r => r.source).join(".*"))
+    const ytConfigData = htmlData.match(
+      new RegExp(ytConfigSelectors.map(r => r.source).join('.*'))
     )
@@ -71,6 +71,6 @@
 
-    let continuation = result[1]
+    const continuation = result[1]
     // Gets top comments by default, and new comments if true
-    return sortByNewest 
-      ? (continuation.slice(0, 47) + "B" + continuation.substring(48))
+    return sortByNewest
+      ? (continuation.slice(0, 47) + 'B' + continuation.substring(48))
       : continuation
@@ -91,3 +91,3 @@ }
     return await this.session.post(
-      `/youtubei/v1/next?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8`,
+      '/youtubei/v1/next?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
       payload
@@ -94,0 +94,0 @@ )

src/Youtube-Scraper.js

@@ -1,2 +0,2 @@
-const HttpRequester = require("./HttpRequester")
+const HttpRequester = require('./HttpRequester')
 const htmlParser = require('./htmlParser')
@@ -22,3 +22,3 @@
       if (typeof payload.continuation !== 'string') payload.continuation = null
-      return payload;
+      return payload
 
@@ -29,3 +29,3 @@ case ValidationId.getCommentReplies:
       }
-      return payload;
+      return payload
   }
@@ -45,3 +45,3 @@ }
     if (!token) {
-      return { comments: [], continuation: token}
+      return { comments: [], continuation: token }
     }
@@ -65,7 +65,19 @@
       if ('continuationItemRenderer' in continuationElem) {
-        token = continuationElem.continuationItemRenderer.continuationEndpoint.continuationCommand.token
+        if (typeof continuationElem.continuationItemRenderer.continuationEndpoint === 'undefined') {
+          token = continuationElem.continuationItemRenderer.button.buttonRenderer.command.continuationCommand.token
+        } else {
+          token = continuationElem.continuationItemRenderer.continuationEndpoint.continuationCommand.token
+        }
       }
     }
 
-    return { comments: commentData, continuation: token }
+    let total = null
+    if (!continuation) {
+      const headerElem = commentPageResponse.data.onResponseReceivedEndpoints[0].reloadContinuationItemsCommand.continuationItems[0]
+      if ('commentsHeaderRenderer' in headerElem) {
+        total = Number(headerElem?.commentsHeaderRenderer?.countText?.runs?.[0]?.text?.replace(',', '')) ?? null
+      }
+    }
+
+    return { total, comments: commentData, continuation: token }
   }
@@ -72,0 +84,0 @@

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

New author

Supply chain risk

A new npm collaborator published a version of the package for the first time. New collaborators are usually benign additions to a project, but do indicate a change to the security surface area of a package.

Found 1 instance in 1 package

Improved metrics

Number of lines in readme file

167

increased by0.6%

Number of medium supply chain risk alerts

0

decreased by-100%

Worsened metrics

Total package byte prevSize

56774

decreased by-7.36%

Dev dependency count

10

increased by900%

Number of package files

8

decreased by-27.27%

Lines of code

326

decreased by-10.93%

Dependency changes

• + Addedasynckit@0.4.0(transitive)

• + Addedaxios@0.27.2(transitive)

• + Addedcombined-stream@1.0.8(transitive)

• + Addeddelayed-stream@1.0.0(transitive)

• + Addedform-data@4.0.1(transitive)

• + Addedmime-db@1.52.0(transitive)

• + Addedmime-types@2.1.35(transitive)

• - Removedaxios@0.21.4(transitive)

• Updatedaxios@^0.27.2