@the-convocation/twitter-scraper - npm Package Compare versions

Comparing version 0.5.0 to 0.6.0

dist/api.js

@@ -57,3 +57,3 @@ "use strict";
             success: false,
-            err: new errors_1.ApiError(res, `Response status: ${res.status}`),
+            err: await errors_1.ApiError.fromResponse(res),
         };
@@ -60,0 +60,0 @@ }

dist/errors.d.ts

 export declare class ApiError extends Error {
     readonly response: Response;
-    constructor(response: Response, message: string);
+    readonly data: any;
+    private constructor();
+    static fromResponse(response: Response): Promise<ApiError>;
 }
+interface Position {
+    line: number;
+    column: number;
+}
+interface TraceInfo {
+    trace_id: string;
+}
+interface TwitterApiErrorExtensions {
+    code?: number;
+    kind?: string;
+    name?: string;
+    source?: string;
+    tracing?: TraceInfo;
+}
+export interface TwitterApiErrorRaw extends TwitterApiErrorExtensions {
+    message?: string;
+    locations?: Position[];
+    path?: string[];
+    extensions?: TwitterApiErrorExtensions;
+}
+export {};
 //# sourceMappingURL=errors.d.ts.map

dist/errors.js

@@ -5,8 +5,23 @@ "use strict";
 class ApiError extends Error {
-    constructor(response, message) {
+    constructor(response, data, message) {
         super(message);
         this.response = response;
+        this.data = data;
     }
+    static async fromResponse(response) {
+        // Try our best to parse the result, but don't bother if we can't
+        let data = undefined;
+        try {
+            data = await response.json();
+        }
+        catch {
+            try {
+                data = await response.text();
+            }
+            catch { }
+        }
+        return new ApiError(response, data, `Response status: ${response.status}`);
+    }
 }
 exports.ApiError = ApiError;
 //# sourceMappingURL=errors.js.map

dist/profile.d.ts

 import { RequestApiResult } from './api';
 import { TwitterAuth } from './auth';
+import { TwitterApiErrorRaw } from './errors';
 export interface LegacyUserRaw {
@@ -64,6 +65,6 @@ created_at?: string;
     data: {
-        user_result: {
+        user: {
             result: {
                 rest_id?: string;
-                isBlueVerified: boolean;
+                is_blue_verified?: boolean;
                 legacy: LegacyUserRaw;
@@ -73,5 +74,3 @@ };
     };
-    errors?: {
-        message: string;
-    }[];
+    errors?: TwitterApiErrorRaw[];
 }
@@ -78,0 +77,0 @@ export declare function parseProfile(user: LegacyUserRaw, isBlueVerified?: boolean): Profile;

dist/profile.js

@@ -9,5 +9,8 @@ "use strict";
 const api_1 = require("./api");
+function getAvatarOriginalSizeUrl(avatarUrl) {
+    return avatarUrl ? avatarUrl.replace('_normal', '') : undefined;
+}
 function parseProfile(user, isBlueVerified) {
     const profile = {
-        avatar: user.profile_image_url_https,
+        avatar: getAvatarOriginalSizeUrl(user.profile_image_url_https),
         banner: user.profile_banner_url,
@@ -19,3 +22,3 @@ biography: user.description,
         mediaCount: user.media_count,
-        isPrivate: user.protected,
+        isPrivate: user.protected ?? false,
         isVerified: user.verified,
@@ -47,13 +50,18 @@ likesCount: user.favourites_count,
         screen_name: username,
-        withHighlightedLabel: true,
+        withSafetyModeUserFields: true,
     }));
-    const features = (0, api_1.addApiFeatures)({
-        interactive_text_enabled: true,
-        longform_notetweets_inline_media_enabled: false,
-        responsive_web_text_conversations_enabled: false,
-        tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: false,
-        vibe_api_enabled: false,
-    });
-    params.set('features', (0, json_stable_stringify_1.default)(features));
-    const res = await (0, api_1.requestApi)(`https://api.twitter.com/graphql/u7wQyGi6oExe8_TRWGMq4Q/UserResultByScreenNameQuery?${params.toString()}`, auth);
+    params.set('features', (0, json_stable_stringify_1.default)({
+        hidden_profile_likes_enabled: false,
+        hidden_profile_subscriptions_enabled: false,
+        responsive_web_graphql_exclude_directive_enabled: true,
+        verified_phone_label_enabled: false,
+        subscriptions_verification_info_is_identity_verified_enabled: false,
+        subscriptions_verification_info_verified_since_enabled: true,
+        highlights_tweets_tab_ui_enabled: true,
+        creator_subscriptions_tweet_preview_api_enabled: true,
+        responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
+        responsive_web_graphql_timeline_navigation_enabled: true,
+    }));
+    params.set('fieldToggles', (0, json_stable_stringify_1.default)({ withAuxiliaryUserLabels: false }));
+    const res = await (0, api_1.requestApi)(`https://twitter.com/i/api/graphql/G3KGOASz96M-Qu0nwmGXNg/UserByScreenName?${params.toString()}`, auth);
     if (!res.success) {
@@ -70,3 +78,3 @@ return res;
     }
-    const { result: user } = value.data.user_result;
+    const { result: user } = value.data.user;
     const { legacy } = user;
@@ -88,3 +96,3 @@ if (user.rest_id == null || user.rest_id.length === 0) {
         success: true,
-        value: parseProfile(user.legacy, user.isBlueVerified),
+        value: parseProfile(user.legacy, user.is_blue_verified),
     };
@@ -91,0 +99,0 @@ }

dist/timeline-tweet-util.js

@@ -19,2 +19,3 @@ "use strict";
                 url: m.media_url_https,
+                alt_text: m.ext_alt_text,
             });
@@ -21,0 +22,0 @@ }

dist/timeline-v1.d.ts

@@ -39,2 +39,3 @@ import { LegacyUserRaw, Profile } from './profile';
     video_info?: VideoInfo;
+    ext_alt_text: string | undefined;
 }
@@ -71,3 +72,3 @@ export interface SearchResultRaw {
     core?: {
-        user_result?: {
+        user_results?: {
             result?: {
@@ -74,0 +75,0 @@ is_blue_verified?: boolean;

dist/timeline-v2.d.ts

@@ -65,3 +65,3 @@ import { LegacyUserRaw } from './profile';
     data?: {
-        user_result?: {
+        user?: {
             result?: {
@@ -86,3 +86,4 @@ timeline_response?: {
 export declare function parseTimelineTweetsV2(timeline: TimelineV2): QueryTweetsResponse;
+export declare function parseTimelineEntryItemContentRaw(content: TimelineEntryItemContentRaw, entryId: string, isConversation?: boolean): Tweet | null;
 export declare function parseThreadedConversation(conversation: ThreadedConversation): Tweet[];
 //# sourceMappingURL=timeline-v2.d.ts.map

dist/timeline-v2.js

 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.parseThreadedConversation = exports.parseTimelineTweetsV2 = exports.parseLegacyTweet = void 0;
+exports.parseThreadedConversation = exports.parseTimelineEntryItemContentRaw = exports.parseTimelineTweetsV2 = exports.parseLegacyTweet = void 0;
 const timeline_tweet_util_1 = require("./timeline-tweet-util");
@@ -19,3 +19,3 @@ const type_util_1 = require("./type-util");
     }
-    if (tweet.id_str == null) {
+    if (!tweet.id_str) {
         if (!tweet.conversation_id_str) {
@@ -89,3 +89,3 @@ return {
         if (retweetedStatusResult) {
-            const parsedResult = parseLegacyTweet(retweetedStatusResult?.core?.user_result?.result?.legacy, retweetedStatusResult?.legacy);
+            const parsedResult = parseLegacyTweet(retweetedStatusResult?.core?.user_results?.result?.legacy, retweetedStatusResult?.legacy);
             if (parsedResult.success) {
@@ -117,3 +117,3 @@ tw.retweetedStatus = parsedResult.tweet;
     }
-    const tweetResult = parseLegacyTweet(result?.core?.user_result?.result?.legacy, result?.legacy);
+    const tweetResult = parseLegacyTweet(result?.core?.user_results?.result?.legacy, result?.legacy);
     if (!tweetResult.success) {
@@ -143,4 +143,4 @@ return tweetResult;
     const tweets = [];
-    const instructions = timeline.data?.user_result?.result?.timeline_response?.timeline
-        ?.instructions ?? [];
+    const instructions = timeline.data?.user?.result?.timeline_response?.timeline?.instructions ??
+        [];
     for (const instruction of instructions) {
@@ -168,8 +168,9 @@ const entries = instruction.entries ?? [];
 exports.parseTimelineTweetsV2 = parseTimelineTweetsV2;
-function parseAndPush(tweets, content, entryId, isConversation = false) {
+function parseTimelineEntryItemContentRaw(content, entryId, isConversation = false) {
     const result = content.tweetResult?.result;
     if (result?.__typename === 'Tweet') {
         if (result.legacy) {
-            const toReplace = isConversation ? 'tweet-' : 'conversation-';
-            result.legacy.id_str = entryId.replace(toReplace, '');
+            result.legacy.id_str = entryId
+                .replace('conversation-', '')
+                .replace('tweet-', '');
         }
@@ -183,6 +184,14 @@ const tweetResult = parseResult(result);
             }
-            tweets.push(tweetResult.tweet);
+            return tweetResult.tweet;
         }
     }
+    return null;
 }
+exports.parseTimelineEntryItemContentRaw = parseTimelineEntryItemContentRaw;
+function parseAndPush(tweets, content, entryId, isConversation = false) {
+    const tweet = parseTimelineEntryItemContentRaw(content, entryId, isConversation);
+    if (tweet) {
+        tweets.push(tweet);
+    }
+}
 function parseThreadedConversation(conversation) {
@@ -189,0 +198,0 @@ const tweets = [];

dist/tweets.d.ts

 import { TwitterAuth } from './auth';
 import { QueryTweetsResponse } from './timeline-v1';
+import { TimelineEntryItemContentRaw } from './timeline-v2';
 export interface Mention {
@@ -11,2 +12,3 @@ id: string;
     url: string;
+    alt_text: string | undefined;
 }
@@ -104,3 +106,6 @@ export interface Video {
 export declare function getLatestTweet(user: string, includeRetweets: boolean, max: number, auth: TwitterAuth): Promise<Tweet | null | void>;
+export interface TweetResultByRestId {
+    data?: TimelineEntryItemContentRaw;
+}
 export declare function getTweet(id: string, auth: TwitterAuth): Promise<Tweet | null>;
 //# sourceMappingURL=tweets.d.ts.map

dist/tweets.js

@@ -24,5 +24,8 @@ "use strict";
     const variables = {
-        includeHasBirdwatchNotes: false,
-        rest_id: userId,
+        userId: userId,
         count: maxTweets,
+        includePromotedContent: false,
+        withQuickPromoteEligibilityTweetFields: true,
+        withVoice: true,
+        withV2Timeline: true,
     };
@@ -34,4 +37,25 @@ if (cursor != null && cursor != '') {
     params.set('variables', (0, json_stable_stringify_1.default)(variables));
-    params.set('features', (0, json_stable_stringify_1.default)(exports.features));
-    const res = await (0, api_1.requestApi)(`https://api.twitter.com/graphql/8IS8MaO-2EN6GZZZb8jF0g/UserWithProfileTweetsAndRepliesQueryV2?${params.toString()}`, auth);
+    params.set('features', (0, json_stable_stringify_1.default)({
+        rweb_lists_timeline_redesign_enabled: true,
+        responsive_web_graphql_exclude_directive_enabled: true,
+        verified_phone_label_enabled: false,
+        creator_subscriptions_tweet_preview_api_enabled: true,
+        responsive_web_graphql_timeline_navigation_enabled: true,
+        responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
+        tweetypie_unmention_optimization_enabled: true,
+        responsive_web_edit_tweet_api_enabled: true,
+        graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
+        view_counts_everywhere_api_enabled: true,
+        longform_notetweets_consumption_enabled: true,
+        responsive_web_twitter_article_tweet_consumption_enabled: false,
+        tweet_awards_web_tipping_enabled: false,
+        freedom_of_speech_not_reach_fetch_enabled: true,
+        standardized_nudges_misinfo: true,
+        tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
+        longform_notetweets_rich_text_read_enabled: true,
+        longform_notetweets_inline_media_enabled: true,
+        responsive_web_media_download_video_enabled: false,
+        responsive_web_enhance_cards_enabled: false,
+    }));
+    const res = await (0, api_1.requestApi)(`https://twitter.com/i/api/graphql/XicnWRbyQ3WgVY__VataBQ/UserTweets?${params.toString()}`, auth);
     if (!res.success) {
@@ -101,16 +125,40 @@ throw res.err;
     const variables = {
-        focalTweetId: id,
-        includeHasBirdwatchNotes: false,
+        tweetId: id,
+        withCommunity: false,
+        includePromotedContent: false,
+        withVoice: false,
     };
     const params = new URLSearchParams();
-    params.set('features', (0, json_stable_stringify_1.default)(exports.features));
+    params.set('features', (0, json_stable_stringify_1.default)({
+        creator_subscriptions_tweet_preview_api_enabled: true,
+        tweetypie_unmention_optimization_enabled: true,
+        responsive_web_edit_tweet_api_enabled: true,
+        graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
+        view_counts_everywhere_api_enabled: true,
+        longform_notetweets_consumption_enabled: true,
+        responsive_web_twitter_article_tweet_consumption_enabled: false,
+        tweet_awards_web_tipping_enabled: false,
+        freedom_of_speech_not_reach_fetch_enabled: true,
+        standardized_nudges_misinfo: true,
+        tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
+        longform_notetweets_rich_text_read_enabled: true,
+        longform_notetweets_inline_media_enabled: true,
+        responsive_web_graphql_exclude_directive_enabled: true,
+        verified_phone_label_enabled: false,
+        responsive_web_media_download_video_enabled: false,
+        responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
+        responsive_web_graphql_timeline_navigation_enabled: true,
+        responsive_web_enhance_cards_enabled: false,
+    }));
     params.set('variables', (0, json_stable_stringify_1.default)(variables));
-    const res = await (0, api_1.requestApi)(`https://api.twitter.com/graphql/83h5UyHZ9wEKBVzALX8R_g/ConversationTimelineV2?${params.toString()}`, auth);
+    const res = await (0, api_1.requestApi)(`https://twitter.com/i/api/graphql/0hWvDhmW8YQ-S_ib3azIrw/TweetResultByRestId?${params.toString()}`, auth);
     if (!res.success) {
         throw res.err;
     }
-    const tweets = (0, timeline_v2_1.parseThreadedConversation)(res.value);
-    return tweets.find((t) => t.id === id) ?? null;
+    if (!res.value.data) {
+        return null;
+    }
+    return (0, timeline_v2_1.parseTimelineEntryItemContentRaw)(res.value.data, id);
 }
 exports.getTweet = getTweet;
 //# sourceMappingURL=tweets.js.map

package.json

 {
   "name": "@the-convocation/twitter-scraper",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "main": "dist/_module.js",
@@ -5,0 +5,0 @@ "repository": "https://github.com/the-convocation/twitter-scraper.git",

No alert changes

Improved metrics

Total package byte prevSize

194041

increased by3.93%

Lines of code

2846

increased by4.02%

No dependency changes