tiktok-scraper-ts - npm Package Compare versions

Comparing version 1.3.1 to 1.3.2

dist/main.d.ts

@@ -531,3 +531,2 @@ /**
      * Replaces the window Object with a export string and writes the new JS file to work with the result as a JS Object
-     * @param type type of the data you are handling.
      * @param content the HTML content of the Page
@@ -576,2 +575,7 @@ * @deprecated No need for this function anymore since Tiktok now adds the json directly to the html
     noWaterMark(link: string): Promise<string | undefined | void>;
+    /**
+     * Scrapes a hashtag posts
+     * @param tag tiktok hashtag
+     * @returns Promise<IVideo[]>
+     */
     hashTag(tag: string): Promise<IVideo[]>;
@@ -610,3 +614,9 @@ }
 declare function fetchVideoNoWaterMark(url: string): Promise<string | void | undefined>;
+/**
+ * Scrapes a hashtag posts
+ * @param tag tiktok hashtag
+ * @returns Promise<IVideo[]>
+ */
+declare function hashtag(tag: string): Promise<IVideo[]>;
 
-export { IAuthor, IMusic, ITikTokResult, IUser, IVideo, Music, TTScraper, TikTokResult, User, Video, fetchAllVideosFromUser, fetchMusic, fetchUser, fetchVideo, fetchVideoNoWaterMark };
+export { IAuthor, IMusic, ITikTokResult, IUser, IVideo, Music, TTScraper, TikTokResult, User, Video, fetchAllVideosFromUser, fetchMusic, fetchUser, fetchVideo, fetchVideoNoWaterMark, hashtag };

dist/main.js

@@ -1,2 +0,2 @@
-var W=Object.create;var f=Object.defineProperty;var O=Object.getOwnPropertyDescriptor;var _=Object.getOwnPropertyNames;var q=Object.getPrototypeOf,D=Object.prototype.hasOwnProperty;var U=i=>f(i,"__esModule",{value:!0}),a=(i,r)=>f(i,"name",{value:r,configurable:!0});var j=(i,r)=>{for(var n in r)f(i,n,{get:r[n],enumerable:!0})},$=(i,r,n,s)=>{if(r&&typeof r=="object"||typeof r=="function")for(let e of _(r))!D.call(i,e)&&(n||e!=="default")&&f(i,e,{get:()=>r[e],enumerable:!(s=O(r,e))||s.enumerable});return i},v=(i,r)=>$(U(f(i!=null?W(q(i)):{},"default",!r&&i&&i.__esModule?{get:()=>i.default,enumerable:!0}:{value:i,enumerable:!0})),i),J=(i=>(r,n)=>i&&i.get(r)||(n=$(U({}),r,1),i&&i.set(r,n),n))(typeof WeakMap!="undefined"?new WeakMap:0);var K={};j(K,{Music:()=>p,TTScraper:()=>l,TikTokResult:()=>b,User:()=>I,Video:()=>g,fetchAllVideosFromUser:()=>P,fetchMusic:()=>F,fetchUser:()=>Y,fetchVideo:()=>G,fetchVideoNoWaterMark:()=>H});var L=v(require("cheerio")),x=v(require("miniget")),T=v(require("node-fetch")),m=require("fs"),S=v(require("http")),E=v(require("https")),R=require("process");var I=class{constructor(r,n,s,e,t,u,o,h,c,w,M,k,A,C,y){this.id=r,this.uniqueId=n,this.nickname=s,this.avatar=e,this.signature=t,this.createdAt=u,this.verified=o,this.secretUID=h,this.bioLink=c,this.privateAccount=w,this.isUnderAge18=M,this.followers=k,this.following=A,this.hearts=C,this.videos=y}};a(I,"User");var b=class{constructor(r,n,s,e,t,u,o,h,c,w){this.author=r,this.video=n,this.audio=s,this.shareCount=e,this.likesCount=t,this.commentCount=u,this.playCount=o,this.createdAt=h,this.tiktokLink=c,this.thumbnail=w}};a(b,"TikTokResult");var g=class{constructor(r,n,s,e,t,u,o,h,c,w,M,k,A,C,y,V,N){this.id=r,this.description=n,this.createdAt=s,this.height=e,this.width=t,this.duration=u,this.resolution=o,this.shareCount=h,this.likesCount=c,this.commentCount=w,this.playCount=M,this.downloadURL=k,this.cover=A,this.dynamicCover=C,this.playURL=y,this.format=V,this.author=N}};a(g,"Video");var p=class{constructor(r,n,s,e,t,u,o,h,c){this.id=r,this.title=n,this.playURL=s,this.coverLarge=e,this.coverThumb=t,this.author=u,this.duration=o,this.original=h,this.album=c}};a(p,"Music");var l=class{async requestWebsite(r,n){let s=new S.default.Agent({keepAlive:!0,maxSockets:20}),e=new E.default.Agent({keepAlive:!0,maxSockets:20}),t={agent:c=>c.protocol=="http:"?s:e,headers:{"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36 Edg/97.0.1072.62"}},o=await(await(0,T.default)(`${r}`,n||t)).text();return L.load(o,{xmlMode:!0})}handleHTMLContent(r){let n=r,s=n.split("window['SIGI_STATE']=")[1].indexOf(";window['SIGI_RETRY']=");return JSON.parse(n.split("window['SIGI_STATE']=")[1].slice(0,s))}async video(r){if(!r)throw new Error("A video URL must be provided");let s=(await this.requestWebsite(r))("#SIGI_STATE").text(),e=JSON.parse(s),t=e.ItemList.video.list[0];return new g(e.ItemModule[t].video.id,e.ItemModule[t].desc,new Date(Number(e.ItemModule[t].createTime)*1e3).toLocaleDateString(),Number(e.ItemModule[t].video.height),Number(e.ItemModule[t].video.width),Number(e.ItemModule[t].video.duration),e.ItemModule[t].video.ratio,e.ItemModule[t].stats.shareCount,e.ItemModule[t].stats.diggCount,e.ItemModule[t].stats.commentCount,e.ItemModule[t].stats.playCount,e.ItemModule[t].video.downloadAddr.trim(),e.ItemModule[t].video.cover,e.ItemModule[t].video.dynamicCover,e.ItemModule[t].video.playAddr.trim(),e.ItemModule[t].video.format,e.ItemModule[t].nickname)}async user(r){if(!r)throw new Error("Please enter a username");let s=(await this.requestWebsite(`https://www.tiktok.com/@${r}`))("#SIGI_STATE").text(),e=JSON.parse(s),t=e.UserModule.users[r];return new I(t.id,t.uniqueId,t.nickname,t.avatarLarger,t.signature.trim(),new Date(t.createTime*1e3).toLocaleDateString(),t.verified,t.secUid,t?.bioLink?.link,t.privateAccount,t.isUnderAge18,e.UserModule.stats[r].followerCount,e.UserModule.stats[r].followingCount,e.UserModule.stats[r].heart,e.UserModule.stats[r].videoCount)}async getAllVideosFromUser(r){if(!r)throw new Error("You must provide a username!");let s=(await this.requestWebsite(`https://www.tiktok.com/@${r}`))("#SIGI_STATE").text(),e=JSON.parse(s),t=[],{ItemList:u}=e;return u["user-post"].list.forEach(o=>{t.push(new g(e.ItemModule[o].video.id,e.ItemModule[o].desc,new Date(Number(e.ItemModule[o].createTime)*1e3).toLocaleDateString(),Number(e.ItemModule[o].video.height),Number(e.ItemModule[o].video.width),Number(e.ItemModule[o].video.duration),e.ItemModule[o].video.ratio,e.ItemModule[o].stats.shareCount,e.ItemModule[o].stats.diggCount,e.ItemModule[o].stats.commentCount,e.ItemModule[o].stats.playCount,e.ItemModule[o].video.downloadAddr.trim(),e.ItemModule[o].video.cover,e.ItemModule[o].video.dynamicCover,e.ItemModule[o].video.playAddr.trim(),e.ItemModule[o].video.format))}),t}async getMusic(r){if(!r)throw new Error("You must provide a link!");let s=(await this.requestWebsite(r))("#SIGI_STATE").text(),e=JSON.parse(s),t=e.ItemList.video.list[0];return new p(e.ItemModule[t].music.id,e.ItemModule[t].music.title,e.ItemModule[t].music.playUrl,e.ItemModule[t].music.coverLarge,e.ItemModule[t].music.coverThumb,e.ItemModule[t].music.authorName,Number(e.ItemModule[t].music.duration),e.ItemModule[t].music.original,e.ItemModule[t].music.album)}async downloadAllVideosFromUser(r,n){if(!r)throw new Error("Please enter a username!");let s=await this.getAllVideosFromUser(r);if(!s)throw new Error("No Videos were found for this username. Either the videos are private or the user has not videos");if(!n.path){if(n.path=`${__dirname}/../${r}`,(0,m.existsSync)(n.path)){console.log("A folder with this username exists, that is unusual!");try{(0,m.unlinkSync)(n.path)}catch(e){console.log(`[ERROR] Could not remove ${n.path}
- Error Message: ${e.message}`),(0,R.exit)(1)}}(0,m.existsSync)(n.path)||(0,m.mkdirSync)(n.path)}if(!n.watermark){for(let[e,t]of s.entries()){console.log(`Downloading Video: ${t.description?t.description:t.id}, [${e+1}/${s.length}]`);let u=await this.noWaterMark(t.id);if(!u){console.log(`Could not fetch ${t.description?t.description:t.id} with no watermark`);continue}(0,x.default)(u).pipe((0,m.createWriteStream)(`${n.path}/${t.id}_${t.resolution}.${t.format}`))}return}for(let[e,t]of s.entries())console.log(`Downloading Video: ${t.description?t.description:t.id}, [${e+1}/${s.length}]`),(0,x.default)(t.downloadURL).pipe((0,m.createWriteStream)(`${n.path}/${t.id}_${t.resolution}.${t.format}`))}async noWaterMark(r){let n="";r.startsWith("https")?n=(await this.video(r)).id:n=r;let e=await(await(0,T.default)("https://api2.musical.ly/aweme/v1/aweme/detail/?aweme_id="+n)).json();if(!e)throw new Error("There was an Error retrieveing this video without watermark!");let t=e.aweme_detail.video.download_addr.uri;if(!t)throw new Error("There was an Error retrieveing this video without watermark!");return`https://api-h2.tiktokv.com/aweme/v1/play/?video_id=${t}`}async hashTag(r){if(!r)throw new Error("You must provide a tag name to complete the search!");let s=(await this.requestWebsite(`https://www.tiktok.com/tag/${r}`))("#SIGI_STATE").text(),e=JSON.parse(s),{ItemList:t}=e,u=[];for(let o of t.challenge.list)u.push(new g(e.ItemModule[o].video.id,e.ItemModule[o].desc,new Date(Number(e.ItemModule[o].createTime)*1e3).toLocaleDateString(),Number(e.ItemModule[o].video.height),Number(e.ItemModule[o].video.width),Number(e.ItemModule[o].video.duration),e.ItemModule[o].video.ratio,e.ItemModule[o].stats.shareCount,e.ItemModule[o].stats.diggCount,e.ItemModule[o].stats.commentCount,e.ItemModule[o].stats.playCount,e.ItemModule[o].video.downloadAddr.trim(),e.ItemModule[o].video.cover,e.ItemModule[o].video.dynamicCover,e.ItemModule[o].video.playAddr.trim(),e.ItemModule[o].video.format,e.ItemModule[o].author));return u}};a(l,"TTScraper");async function G(i){if(!i)throw new Error("You must provide a Tiktok video url!");return await new l().video(i)}a(G,"fetchVideo");async function Y(i){if(!i)throw new Error("You must provide a username!");return await new l().user(i)}a(Y,"fetchUser");async function P(i){if(!i)throw new Error("You must provide a username!");return await new l().getAllVideosFromUser(i)}a(P,"fetchAllVideosFromUser");async function F(i){if(!i)throw new Error("You must provide a Tiktok video url!");return await new l().getMusic(i)}a(F,"fetchMusic");async function H(i){if(!i)throw new Error("You must provide a Tiktok video url!");return await new l().noWaterMark(i)}a(H,"fetchVideoNoWaterMark");module.exports=J(K);0&&(module.exports={Music,TTScraper,TikTokResult,User,Video,fetchAllVideosFromUser,fetchMusic,fetchUser,fetchVideo,fetchVideoNoWaterMark});
+var W=Object.create;var f=Object.defineProperty;var O=Object.getOwnPropertyDescriptor;var _=Object.getOwnPropertyNames;var q=Object.getPrototypeOf,D=Object.prototype.hasOwnProperty;var U=o=>f(o,"__esModule",{value:!0}),a=(o,r)=>f(o,"name",{value:r,configurable:!0});var j=(o,r)=>{for(var n in r)f(o,n,{get:r[n],enumerable:!0})},$=(o,r,n,s)=>{if(r&&typeof r=="object"||typeof r=="function")for(let e of _(r))!D.call(o,e)&&(n||e!=="default")&&f(o,e,{get:()=>r[e],enumerable:!(s=O(r,e))||s.enumerable});return o},v=(o,r)=>$(U(f(o!=null?W(q(o)):{},"default",!r&&o&&o.__esModule?{get:()=>o.default,enumerable:!0}:{value:o,enumerable:!0})),o),J=(o=>(r,n)=>o&&o.get(r)||(n=$(U({}),r,1),o&&o.set(r,n),n))(typeof WeakMap!="undefined"?new WeakMap:0);var z={};j(z,{Music:()=>p,TTScraper:()=>m,TikTokResult:()=>b,User:()=>I,Video:()=>g,fetchAllVideosFromUser:()=>P,fetchMusic:()=>F,fetchUser:()=>G,fetchVideo:()=>Y,fetchVideoNoWaterMark:()=>H,hashtag:()=>K});var L=v(require("cheerio")),x=v(require("miniget")),T=v(require("node-fetch")),l=require("fs"),S=v(require("http")),E=v(require("https")),R=require("process");var I=class{constructor(r,n,s,e,t,u,i,h,c,w,M,k,A,C,y){this.id=r,this.uniqueId=n,this.nickname=s,this.avatar=e,this.signature=t,this.createdAt=u,this.verified=i,this.secretUID=h,this.bioLink=c,this.privateAccount=w,this.isUnderAge18=M,this.followers=k,this.following=A,this.hearts=C,this.videos=y}};a(I,"User");var b=class{constructor(r,n,s,e,t,u,i,h,c,w){this.author=r,this.video=n,this.audio=s,this.shareCount=e,this.likesCount=t,this.commentCount=u,this.playCount=i,this.createdAt=h,this.tiktokLink=c,this.thumbnail=w}};a(b,"TikTokResult");var g=class{constructor(r,n,s,e,t,u,i,h,c,w,M,k,A,C,y,V,N){this.id=r,this.description=n,this.createdAt=s,this.height=e,this.width=t,this.duration=u,this.resolution=i,this.shareCount=h,this.likesCount=c,this.commentCount=w,this.playCount=M,this.downloadURL=k,this.cover=A,this.dynamicCover=C,this.playURL=y,this.format=V,this.author=N}};a(g,"Video");var p=class{constructor(r,n,s,e,t,u,i,h,c){this.id=r,this.title=n,this.playURL=s,this.coverLarge=e,this.coverThumb=t,this.author=u,this.duration=i,this.original=h,this.album=c}};a(p,"Music");var m=class{async requestWebsite(r,n){let s=new S.default.Agent({keepAlive:!0,maxSockets:20}),e=new E.default.Agent({keepAlive:!0,maxSockets:20}),t={agent:c=>c.protocol=="http:"?s:e,headers:{"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36 Edg/97.0.1072.62"}},i=await(await(0,T.default)(`${r}`,n||t)).text();return L.load(i,{xmlMode:!0})}handleHTMLContent(r){let n=r,s=n.split("window['SIGI_STATE']=")[1].indexOf(";window['SIGI_RETRY']=");return JSON.parse(n.split("window['SIGI_STATE']=")[1].slice(0,s))}async video(r){if(!r)throw new Error("A video URL must be provided");let s=(await this.requestWebsite(r))("#SIGI_STATE").text(),e=JSON.parse(s),t=e.ItemList.video.list[0];return new g(e.ItemModule[t].video.id,e.ItemModule[t].desc,new Date(Number(e.ItemModule[t].createTime)*1e3).toLocaleDateString(),Number(e.ItemModule[t].video.height),Number(e.ItemModule[t].video.width),Number(e.ItemModule[t].video.duration),e.ItemModule[t].video.ratio,e.ItemModule[t].stats.shareCount,e.ItemModule[t].stats.diggCount,e.ItemModule[t].stats.commentCount,e.ItemModule[t].stats.playCount,e.ItemModule[t].video.downloadAddr.trim(),e.ItemModule[t].video.cover,e.ItemModule[t].video.dynamicCover,e.ItemModule[t].video.playAddr.trim(),e.ItemModule[t].video.format,e.ItemModule[t].nickname)}async user(r){if(!r)throw new Error("Please enter a username");let s=(await this.requestWebsite(`https://www.tiktok.com/@${r}`))("#SIGI_STATE").text(),e=JSON.parse(s),t=e.UserModule.users[r];return new I(t.id,t.uniqueId,t.nickname,t.avatarLarger,t.signature.trim(),new Date(t.createTime*1e3).toLocaleDateString(),t.verified,t.secUid,t?.bioLink?.link,t.privateAccount,t.isUnderAge18,e.UserModule.stats[r].followerCount,e.UserModule.stats[r].followingCount,e.UserModule.stats[r].heart,e.UserModule.stats[r].videoCount)}async getAllVideosFromUser(r){if(!r)throw new Error("You must provide a username!");let s=(await this.requestWebsite(`https://www.tiktok.com/@${r}`))("#SIGI_STATE").text(),e=JSON.parse(s),t=[],{ItemList:u}=e;return u["user-post"].list.forEach(i=>{t.push(new g(e.ItemModule[i].video.id,e.ItemModule[i].desc,new Date(Number(e.ItemModule[i].createTime)*1e3).toLocaleDateString(),Number(e.ItemModule[i].video.height),Number(e.ItemModule[i].video.width),Number(e.ItemModule[i].video.duration),e.ItemModule[i].video.ratio,e.ItemModule[i].stats.shareCount,e.ItemModule[i].stats.diggCount,e.ItemModule[i].stats.commentCount,e.ItemModule[i].stats.playCount,e.ItemModule[i].video.downloadAddr.trim(),e.ItemModule[i].video.cover,e.ItemModule[i].video.dynamicCover,e.ItemModule[i].video.playAddr.trim(),e.ItemModule[i].video.format))}),t}async getMusic(r){if(!r)throw new Error("You must provide a link!");let s=(await this.requestWebsite(r))("#SIGI_STATE").text(),e=JSON.parse(s),t=e.ItemList.video.list[0];return new p(e.ItemModule[t].music.id,e.ItemModule[t].music.title,e.ItemModule[t].music.playUrl,e.ItemModule[t].music.coverLarge,e.ItemModule[t].music.coverThumb,e.ItemModule[t].music.authorName,Number(e.ItemModule[t].music.duration),e.ItemModule[t].music.original,e.ItemModule[t].music.album)}async downloadAllVideosFromUser(r,n){if(!r)throw new Error("Please enter a username!");let s=await this.getAllVideosFromUser(r);if(!s)throw new Error("No Videos were found for this username. Either the videos are private or the user has not videos");if(!n.path){if(n.path=`${__dirname}/../${r}`,(0,l.existsSync)(n.path)){console.log("A folder with this username exists, that is unusual!");try{(0,l.unlinkSync)(n.path)}catch(e){console.log(`[ERROR] Could not remove ${n.path}
+ Error Message: ${e.message}`),(0,R.exit)(1)}}(0,l.existsSync)(n.path)||(0,l.mkdirSync)(n.path)}if(!n.watermark){for(let[e,t]of s.entries()){console.log(`Downloading Video: ${t.description?t.description:t.id}, [${e+1}/${s.length}]`);let u=await this.noWaterMark(t.id);if(!u){console.log(`Could not fetch ${t.description?t.description:t.id} with no watermark`);continue}(0,x.default)(u).pipe((0,l.createWriteStream)(`${n.path}/${t.id}_${t.resolution}.${t.format}`))}return}for(let[e,t]of s.entries())console.log(`Downloading Video: ${t.description?t.description:t.id}, [${e+1}/${s.length}]`),(0,x.default)(t.downloadURL).pipe((0,l.createWriteStream)(`${n.path}/${t.id}_${t.resolution}.${t.format}`))}async noWaterMark(r){let n="";r.startsWith("https")?n=(await this.video(r)).id:n=r;let e=await(await(0,T.default)("https://api2.musical.ly/aweme/v1/aweme/detail/?aweme_id="+n)).json();if(!e)throw new Error("There was an Error retrieveing this video without watermark!");let t=e.aweme_detail.video.download_addr.uri;if(!t)throw new Error("There was an Error retrieveing this video without watermark!");return`https://api-h2.tiktokv.com/aweme/v1/play/?video_id=${t}`}async hashTag(r){if(!r)throw new Error("You must provide a tag name to complete the search!");let s=(await this.requestWebsite(`https://www.tiktok.com/tag/${r}`))("#SIGI_STATE").text(),e=JSON.parse(s),{ItemList:t}=e,u=[];for(let i of t.challenge.list)u.push(new g(e.ItemModule[i].video.id,e.ItemModule[i].desc,new Date(Number(e.ItemModule[i].createTime)*1e3).toLocaleDateString(),Number(e.ItemModule[i].video.height),Number(e.ItemModule[i].video.width),Number(e.ItemModule[i].video.duration),e.ItemModule[i].video.ratio,e.ItemModule[i].stats.shareCount,e.ItemModule[i].stats.diggCount,e.ItemModule[i].stats.commentCount,e.ItemModule[i].stats.playCount,e.ItemModule[i].video.downloadAddr.trim(),e.ItemModule[i].video.cover,e.ItemModule[i].video.dynamicCover,e.ItemModule[i].video.playAddr.trim(),e.ItemModule[i].video.format,e.ItemModule[i].author));return u}};a(m,"TTScraper");async function Y(o){if(!o)throw new Error("You must provide a Tiktok video url!");return await new m().video(o)}a(Y,"fetchVideo");async function G(o){if(!o)throw new Error("You must provide a username!");return await new m().user(o)}a(G,"fetchUser");async function P(o){if(!o)throw new Error("You must provide a username!");return await new m().getAllVideosFromUser(o)}a(P,"fetchAllVideosFromUser");async function F(o){if(!o)throw new Error("You must provide a Tiktok video url!");return await new m().getMusic(o)}a(F,"fetchMusic");async function H(o){if(!o)throw new Error("You must provide a Tiktok video url!");return await new m().noWaterMark(o)}a(H,"fetchVideoNoWaterMark");async function K(o){if(!o)throw new Error("You must provide a tiktok hashtag");return await new m().hashTag(o)}a(K,"hashtag");module.exports=J(z);0&&(module.exports={Music,TTScraper,TikTokResult,User,Video,fetchAllVideosFromUser,fetchMusic,fetchUser,fetchVideo,fetchVideoNoWaterMark,hashtag});

package.json

 {
   "name": "tiktok-scraper-ts",
-  "version": "1.3.1",
+  "version": "1.3.2",
   "description": "A scraper for tiktok videos, users and more",
@@ -5,0 +5,0 @@ "main": "dist/main.js",

README.md

@@ -38,2 +38,3 @@ # tiktok-scraper
 .noWaterMark(link) Returns a direct download link for the video without TikTok Watermark.
+.hashtag(tag) Scrapes a hashtag posts
 ```
@@ -53,2 +54,3 @@
 fetchVideoNoWaterMark(url); // Same as TTScraper.noWaterMark(url)
+hashtag(tag); // Same as TTScraper.hashtag(tag)
 ```
@@ -55,0 +57,0 @@

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

42260

increased by1.43%

Lines of code

689

increased by1.62%

Number of lines in readme file

342

increased by0.59%

No dependency changes