@the-convocation/twitter-scraper - npm Package Compare versions

Comparing version 0.4.0 to 0.5.0

dist/api.d.ts

@@ -60,2 +60,13 @@ import { TwitterAuth } from './auth';
     responsive_web_enhance_cards_enabled: boolean;
+    subscriptions_verification_info_enabled: boolean;
+    subscriptions_verification_info_reason_enabled: boolean;
+    subscriptions_verification_info_verified_since_enabled: boolean;
+    super_follow_badge_privacy_enabled: boolean;
+    super_follow_exclusive_tweet_notifications_enabled: boolean;
+    super_follow_tweet_api_enabled: boolean;
+    super_follow_user_api_enabled: boolean;
+    android_graphql_skip_api_media_color_palette: boolean;
+    creator_subscriptions_subscription_count_enabled: boolean;
+    blue_business_profile_image_shape_enabled: boolean;
+    unified_cards_ad_metadata_container_dynamic_card_content_query_enabled: boolean;
 };
@@ -62,0 +73,0 @@ export declare function addApiParams(params: URLSearchParams, includeTweetReplies: boolean): URLSearchParams;

dist/api.js

@@ -92,2 +92,13 @@ "use strict";
         responsive_web_enhance_cards_enabled: false,
+        subscriptions_verification_info_enabled: true,
+        subscriptions_verification_info_reason_enabled: true,
+        subscriptions_verification_info_verified_since_enabled: true,
+        super_follow_badge_privacy_enabled: false,
+        super_follow_exclusive_tweet_notifications_enabled: false,
+        super_follow_tweet_api_enabled: false,
+        super_follow_user_api_enabled: false,
+        android_graphql_skip_api_media_color_palette: false,
+        creator_subscriptions_subscription_count_enabled: false,
+        blue_business_profile_image_shape_enabled: false,
+        unified_cards_ad_metadata_container_dynamic_card_content_query_enabled: false,
     };
@@ -94,0 +105,0 @@ }

dist/profile.d.ts

@@ -16,2 +16,4 @@ import { RequestApiResult } from './api';
     friends_count?: number;
+    media_count?: number;
+    statuses_count?: number;
     id_str?: string;
@@ -21,3 +23,5 @@ listed_count?: number;
     location: string;
+    geo_enabled?: boolean;
     pinned_tweet_ids_str?: string[];
+    profile_background_color?: string;
     profile_banner_url?: string;
@@ -27,4 +31,6 @@ profile_image_url_https?: string;
     screen_name?: string;
-    statuses_count?: number;
     verified?: boolean;
+    has_custom_timelines?: boolean;
+    has_extended_profile?: boolean;
+    url?: string;
 }
@@ -42,4 +48,7 @@ /**
     friendsCount?: number;
+    mediaCount?: number;
+    statusesCount?: number;
     isPrivate?: boolean;
     isVerified?: boolean;
+    isBlueVerified?: boolean;
     joined?: Date;
@@ -59,5 +68,8 @@ likesCount?: number;
     data: {
-        user: {
-            rest_id?: string;
-            legacy: LegacyUserRaw;
+        user_result: {
+            result: {
+                rest_id?: string;
+                isBlueVerified: boolean;
+                legacy: LegacyUserRaw;
+            };
         };
@@ -69,5 +81,5 @@ };
 }
-export declare function parseProfile(user: LegacyUserRaw): Profile;
+export declare function parseProfile(user: LegacyUserRaw, isBlueVerified?: boolean): Profile;
 export declare function getProfile(username: string, auth: TwitterAuth): Promise<RequestApiResult<Profile>>;
 export declare function getUserIdByScreenName(screenName: string, auth: TwitterAuth): Promise<RequestApiResult<string>>;
 //# sourceMappingURL=profile.d.ts.map

dist/profile.js

 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getUserIdByScreenName = exports.getProfile = exports.parseProfile = void 0;
+const json_stable_stringify_1 = __importDefault(require("json-stable-stringify"));
 const api_1 = require("./api");
-function parseProfile(user) {
+function parseProfile(user, isBlueVerified) {
     const profile = {
@@ -13,2 +17,3 @@ avatar: user.profile_image_url_https,
         friendsCount: user.friends_count,
+        mediaCount: user.media_count,
         isPrivate: user.protected,
@@ -25,2 +30,3 @@ isVerified: user.verified,
         username: user.screen_name,
+        isBlueVerified: isBlueVerified ?? false,
     };
@@ -39,7 +45,15 @@ if (user.created_at != null) {
     const params = new URLSearchParams();
-    params.set('variables', JSON.stringify({
+    params.set('variables', (0, json_stable_stringify_1.default)({
         screen_name: username,
         withHighlightedLabel: true,
     }));
-    const res = await (0, api_1.requestApi)(`https://api.twitter.com/graphql/4S2ihIKfF3xhp-ENxvUAfQ/UserByScreenName?${params}`, auth);
+    const features = (0, api_1.addApiFeatures)({
+        interactive_text_enabled: true,
+        longform_notetweets_inline_media_enabled: false,
+        responsive_web_text_conversations_enabled: false,
+        tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: false,
+        vibe_api_enabled: false,
+    });
+    params.set('features', (0, json_stable_stringify_1.default)(features));
+    const res = await (0, api_1.requestApi)(`https://api.twitter.com/graphql/u7wQyGi6oExe8_TRWGMq4Q/UserResultByScreenNameQuery?${params.toString()}`, auth);
     if (!res.success) {
@@ -56,3 +70,3 @@ return res;
     }
-    const { user } = value.data;
+    const { result: user } = value.data.user_result;
     const { legacy } = user;
@@ -74,3 +88,3 @@ if (user.rest_id == null || user.rest_id.length === 0) {
         success: true,
-        value: parseProfile(user.legacy),
+        value: parseProfile(user.legacy, user.isBlueVerified),
     };
@@ -77,0 +91,0 @@ }

dist/scraper.d.ts

@@ -6,3 +6,3 @@ import { Cookie } from 'tough-cookie';
 import { QueryProfilesResponse, QueryTweetsResponse } from './timeline-v1';
-import { Tweet } from './tweets';
+import { Tweet, TweetQuery } from './tweets';
 import fetch from 'cross-fetch';
@@ -108,2 +108,36 @@ export interface ScraperOptions {
     /**
+     * Fetches the first tweet matching the given query.
+     *
+     * Example:
+     * ```js
+     * const timeline = scraper.getTweets('user', 200);
+     * const retweet = await scraper.getTweetWhere(timeline, { isRetweet: true });
+     * ```
+     * @param tweets The {@link AsyncIterable} of tweets to search through.
+     * @param query A query to test **all** tweets against. This may be either an
+     * object of key/value pairs or a predicate. If this query is an object, all
+     * key/value pairs must match a {@link Tweet} for it to be returned. If this query
+     * is a predicate, it must resolve to `true` for a {@link Tweet} to be returned.
+     * - All keys are optional.
+     * - If specified, the key must be implemented by that of {@link Tweet}.
+     */
+    getTweetWhere(tweets: AsyncIterable<Tweet>, query: TweetQuery): Promise<Tweet | null>;
+    /**
+     * Fetches all tweets matching the given query.
+     *
+     * Example:
+     * ```js
+     * const timeline = scraper.getTweets('user', 200);
+     * const retweets = await scraper.getTweetsWhere(timeline, { isRetweet: true });
+     * ```
+     * @param tweets The {@link AsyncIterable} of tweets to search through.
+     * @param query A query to test **all** tweets against. This may be either an
+     * object of key/value pairs or a predicate. If this query is an object, all
+     * key/value pairs must match a {@link Tweet} for it to be returned. If this query
+     * is a predicate, it must resolve to `true` for a {@link Tweet} to be returned.
+     * - All keys are optional.
+     * - If specified, the key must be implemented by that of {@link Tweet}.
+     */
+    getTweetsWhere(tweets: AsyncIterable<Tweet>, query: TweetQuery): Promise<Tweet[]>;
+    /**
      * Fetches the most recent tweet from a Twitter user.
@@ -114,3 +148,3 @@ * @param user The user whose latest tweet should be returned.
      */
-    getLatestTweet(user: string, includeRetweets?: boolean): Promise<Tweet | null | void>;
+    getLatestTweet(user: string, includeRetweets?: boolean, max?: number): Promise<Tweet | null | void>;
     /**
@@ -117,0 +151,0 @@ * Fetches a single tweet.

dist/scraper.js

@@ -122,2 +122,40 @@ "use strict";
     /**
+     * Fetches the first tweet matching the given query.
+     *
+     * Example:
+     * ```js
+     * const timeline = scraper.getTweets('user', 200);
+     * const retweet = await scraper.getTweetWhere(timeline, { isRetweet: true });
+     * ```
+     * @param tweets The {@link AsyncIterable} of tweets to search through.
+     * @param query A query to test **all** tweets against. This may be either an
+     * object of key/value pairs or a predicate. If this query is an object, all
+     * key/value pairs must match a {@link Tweet} for it to be returned. If this query
+     * is a predicate, it must resolve to `true` for a {@link Tweet} to be returned.
+     * - All keys are optional.
+     * - If specified, the key must be implemented by that of {@link Tweet}.
+     */
+    getTweetWhere(tweets, query) {
+        return (0, tweets_1.getTweetWhere)(tweets, query);
+    }
+    /**
+     * Fetches all tweets matching the given query.
+     *
+     * Example:
+     * ```js
+     * const timeline = scraper.getTweets('user', 200);
+     * const retweets = await scraper.getTweetsWhere(timeline, { isRetweet: true });
+     * ```
+     * @param tweets The {@link AsyncIterable} of tweets to search through.
+     * @param query A query to test **all** tweets against. This may be either an
+     * object of key/value pairs or a predicate. If this query is an object, all
+     * key/value pairs must match a {@link Tweet} for it to be returned. If this query
+     * is a predicate, it must resolve to `true` for a {@link Tweet} to be returned.
+     * - All keys are optional.
+     * - If specified, the key must be implemented by that of {@link Tweet}.
+     */
+    getTweetsWhere(tweets, query) {
+        return (0, tweets_1.getTweetsWhere)(tweets, query);
+    }
+    /**
      * Fetches the most recent tweet from a Twitter user.
@@ -128,4 +166,4 @@ * @param user The user whose latest tweet should be returned.
      */
-    getLatestTweet(user, includeRetweets = false) {
-        return (0, tweets_1.getLatestTweet)(user, includeRetweets, this.auth);
+    getLatestTweet(user, includeRetweets = false, max = 200) {
+        return (0, tweets_1.getLatestTweet)(user, includeRetweets, max, this.auth);
     }
@@ -132,0 +170,0 @@ /**

dist/search.js

@@ -63,2 +63,5 @@ "use strict";
         tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
+        interactive_text_enabled: false,
+        responsive_web_text_conversations_enabled: false,
+        vibe_api_enabled: false,
     });
@@ -91,3 +94,3 @@ const fieldToggles = {
     params.set('variables', (0, json_stable_stringify_1.default)(variables));
-    const res = await (0, api_1.requestApi)(`https://twitter.com/i/api/graphql/nK1dw4oV3k4w5TdtcAdSww/SearchTimeline?${params.toString()}`, auth);
+    const res = await (0, api_1.requestApi)(`https://api.twitter.com/graphql/gkjsKepM6gl_HmFWoWKfgg/SearchTimeline?${params.toString()}`, auth);
     if (!res.success) {
@@ -94,0 +97,0 @@ throw res.err;

dist/timeline-search.d.ts

 import { QueryProfilesResponse, QueryTweetsResponse } from './timeline-v1';
-import { TimelineEntryRaw } from './timeline-v2';
+import { SearchEntryRaw } from './timeline-v2';
 export interface SearchTimeline {
@@ -9,4 +9,4 @@ data?: {
                     instructions?: {
-                        entries?: TimelineEntryRaw[];
-                        entry?: TimelineEntryRaw;
+                        entries?: SearchEntryRaw[];
+                        entry?: SearchEntryRaw;
                         type?: string;
@@ -13,0 +13,0 @@ }[];

dist/timeline-search.js

@@ -18,3 +18,4 @@ "use strict";
             }
-            for (const entry of instruction.entries ?? []) {
+            const entries = instruction.entries ?? [];
+            for (const entry of entries) {
                 const itemContent = entry.content?.itemContent;
@@ -55,3 +56,4 @@ if (itemContent?.tweetDisplayType === 'Tweet') {
             }
-            for (const entry of instruction.entries ?? []) {
+            const entries = instruction.entries ?? [];
+            for (const entry of entries) {
                 const itemContent = entry.content?.itemContent;
@@ -61,5 +63,5 @@ if (itemContent?.userDisplayType === 'User') {
                     if (userResultRaw?.legacy) {
-                        const profile = (0, profile_1.parseProfile)(userResultRaw.legacy);
+                        const profile = (0, profile_1.parseProfile)(userResultRaw.legacy, userResultRaw.is_blue_verified);
                         if (!profile.userId) {
-                            profile.userId = itemContent.user_results?.result?.rest_id;
+                            profile.userId = userResultRaw.rest_id;
                         }
@@ -66,0 +68,0 @@ profiles.push(profile);

dist/timeline-v1.d.ts

@@ -40,3 +40,4 @@ import { LegacyUserRaw, Profile } from './profile';
 }
-export interface TimelineResultRaw {
+export interface SearchResultRaw {
+    rest_id?: string;
     __typename?: string;
@@ -62,2 +63,28 @@ core?: {
     quoted_status_result?: {
+        result?: SearchResultRaw;
+    };
+    legacy?: LegacyTweetRaw;
+}
+export interface TimelineResultRaw {
+    rest_id?: string;
+    __typename?: string;
+    core?: {
+        user_result?: {
+            result?: {
+                is_blue_verified?: boolean;
+                legacy?: LegacyUserRaw;
+            };
+        };
+    };
+    views?: {
+        count?: string;
+    };
+    note_tweet?: {
+        note_tweet_results?: {
+            result?: {
+                text?: string;
+            };
+        };
+    };
+    quoted_status_result?: {
         result?: TimelineResultRaw;
@@ -64,0 +91,0 @@ };

dist/timeline-v2.d.ts

 import { LegacyUserRaw } from './profile';
-import { LegacyTweetRaw, ParseTweetResult, QueryTweetsResponse, TimelineResultRaw } from './timeline-v1';
+import { LegacyTweetRaw, ParseTweetResult, QueryTweetsResponse, SearchResultRaw, TimelineResultRaw } from './timeline-v1';
 import { Tweet } from './tweets';
@@ -7,6 +7,7 @@ export interface TimelineUserResultRaw {
     legacy?: LegacyUserRaw;
+    is_blue_verified?: boolean;
 }
 export interface TimelineEntryItemContentRaw {
     tweetDisplayType?: string;
-    tweet_results?: {
+    tweetResult?: {
         result?: TimelineResultRaw;
@@ -20,2 +21,3 @@ };
 export interface TimelineEntryRaw {
+    entryId: string;
     content?: {
@@ -26,19 +28,46 @@ cursorType?: string;
             item?: {
-                itemContent?: TimelineEntryItemContentRaw;
+                content?: TimelineEntryItemContentRaw;
             };
         }[];
-        itemContent?: TimelineEntryItemContentRaw;
+        content?: TimelineEntryItemContentRaw;
     };
 }
+export interface SearchEntryItemContentRaw {
+    tweetDisplayType?: string;
+    tweet_results?: {
+        result?: SearchResultRaw;
+    };
+    userDisplayType?: string;
+    user_results?: {
+        result?: TimelineUserResultRaw;
+    };
+}
+export interface SearchEntryRaw {
+    entryId: string;
+    sortIndex: string;
+    content?: {
+        cursorType?: string;
+        entryType?: string;
+        __typename?: string;
+        value?: string;
+        items?: {
+            item?: {
+                content?: SearchEntryItemContentRaw;
+            };
+        }[];
+        itemContent?: SearchEntryItemContentRaw;
+    };
+}
+export interface TimelineInstruction {
+    entries: TimelineEntryRaw[];
+    entry?: TimelineEntryRaw;
+    type?: string;
+}
 export interface TimelineV2 {
     data?: {
-        user?: {
+        user_result?: {
             result?: {
-                timeline_v2?: {
+                timeline_response?: {
                     timeline?: {
-                        instructions?: {
-                            entries?: TimelineEntryRaw[];
-                            entry?: TimelineEntryRaw;
-                            type?: string;
-                        }[];
+                        instructions: TimelineInstruction[];
                     };
@@ -52,8 +81,4 @@ };
     data?: {
-        threaded_conversation_with_injections_v2?: {
-            instructions?: {
-                entries?: TimelineEntryRaw[];
-                entry?: TimelineEntryRaw;
-                type?: string;
-            }[];
+        timeline_response?: {
+            instructions?: TimelineInstruction[];
         };
@@ -60,0 +85,0 @@ };

dist/timeline-v2.js

@@ -19,2 +19,11 @@ "use strict";
     }
+    if (tweet.id_str == null) {
+        if (!tweet.conversation_id_str) {
+            return {
+                success: false,
+                err: new Error('Tweet ID was not found in object.'),
+            };
+        }
+        tweet.id_str = tweet.conversation_id_str;
+    }
     const hashtags = tweet.entities?.hashtags ?? [];
@@ -26,8 +35,2 @@ const mentions = tweet.entities?.user_mentions ?? [];
     const { photos, videos, sensitiveContent } = (0, timeline_tweet_util_1.parseMediaGroups)(media);
-    if (tweet.id_str == null) {
-        return {
-            success: false,
-            err: new Error('Tweet ID was not found in object.'),
-        };
-    }
     const tw = {
@@ -58,2 +61,7 @@ conversationId: tweet.conversation_id_str,
         videos,
+        isQuoted: false,
+        isReply: false,
+        isRetweet: false,
+        isPin: false,
+        sensitiveContent: false,
     };
@@ -67,17 +75,21 @@ if (tweet.created_at) {
     }
-    if (tweet.quoted_status_id_str) {
+    const quotedStatusIdStr = tweet.quoted_status_id_str;
+    const inReplyToStatusIdStr = tweet.in_reply_to_status_id_str;
+    const retweetedStatusIdStr = tweet.retweeted_status_id_str;
+    const retweetedStatusResult = tweet.retweeted_status_result?.result;
+    if (quotedStatusIdStr) {
         tw.isQuoted = true;
-        tw.quotedStatusId = tweet.quoted_status_id_str;
+        tw.quotedStatusId = quotedStatusIdStr;
     }
-    if (tweet.in_reply_to_status_id_str) {
+    if (inReplyToStatusIdStr) {
         tw.isReply = true;
-        tw.inReplyToStatusId = tweet.in_reply_to_status_id_str;
+        tw.inReplyToStatusId = inReplyToStatusIdStr;
     }
-    if (tweet.retweeted_status_id_str || tweet.retweeted_status_result?.result) {
+    if (retweetedStatusIdStr || retweetedStatusResult) {
         tw.isRetweet = true;
-        tw.retweetedStatusId = tweet.retweeted_status_id_str;
-        if (tweet.retweeted_status_result?.result) {
-            const retweetedStatusResult = parseLegacyTweet(tweet.retweeted_status_result.result.core?.user_results?.result?.legacy, tweet.retweeted_status_result.result.legacy);
-            if (retweetedStatusResult.success) {
-                tw.retweetedStatus = retweetedStatusResult.tweet;
+        tw.retweetedStatusId = retweetedStatusIdStr;
+        if (retweetedStatusResult) {
+            const parsedResult = parseLegacyTweet(retweetedStatusResult?.core?.user_result?.result?.legacy, retweetedStatusResult?.legacy);
+            if (parsedResult.success) {
+                tw.retweetedStatus = parsedResult.tweet;
             }
@@ -103,6 +115,7 @@ }
 function parseResult(result) {
-    if (result?.legacy && result.note_tweet?.note_tweet_results?.result?.text) {
-        result.legacy.full_text = result.note_tweet.note_tweet_results.result.text;
+    const noteTweetResultText = result?.note_tweet?.note_tweet_results?.result?.text;
+    if (result?.legacy && noteTweetResultText) {
+        result.legacy.full_text = noteTweetResultText;
     }
-    const tweetResult = parseLegacyTweet(result?.core?.user_results?.result?.legacy, result?.legacy);
+    const tweetResult = parseLegacyTweet(result?.core?.user_result?.result?.legacy, result?.legacy);
     if (!tweetResult.success) {
@@ -117,4 +130,8 @@ return tweetResult;
     }
-    if (result?.quoted_status_result?.result) {
-        const quotedTweetResult = parseResult(result.quoted_status_result.result);
+    const quotedResult = result?.quoted_status_result?.result;
+    if (quotedResult) {
+        if (quotedResult.legacy && quotedResult.rest_id) {
+            quotedResult.legacy.id_str = quotedResult.rest_id;
+        }
+        const quotedTweetResult = parseResult(quotedResult);
         if (quotedTweetResult.success) {
@@ -129,16 +146,21 @@ tweetResult.tweet.quotedStatus = quotedTweetResult.tweet;
     const tweets = [];
-    const instructions = timeline.data?.user?.result?.timeline_v2?.timeline?.instructions ?? [];
+    const instructions = timeline.data?.user_result?.result?.timeline_response?.timeline
+        ?.instructions ?? [];
     for (const instruction of instructions) {
-        for (const entry of instruction.entries ?? []) {
-            if (entry.content?.cursorType === 'Bottom') {
-                cursor = entry.content.value;
+        const entries = instruction.entries ?? [];
+        for (const entry of entries) {
+            const entryContent = entry.content;
+            if (!entryContent)
                 continue;
+            if (entryContent.cursorType === 'Bottom') {
+                cursor = entryContent.value;
+                continue;
             }
-            if (entry.content?.itemContent?.tweet_results?.result?.__typename ===
-                'Tweet') {
-                const tweetResult = parseResult(entry.content.itemContent.tweet_results.result);
-                if (tweetResult.success) {
-                    tweets.push(tweetResult.tweet);
-                }
+            const idStr = entry.entryId;
+            if (!idStr.startsWith('tweet')) {
+                continue;
             }
+            if (entryContent.content) {
+                parseAndPush(tweets, entryContent.content, idStr);
+            }
         }
@@ -149,27 +171,34 @@ }
 exports.parseTimelineTweetsV2 = parseTimelineTweetsV2;
+function parseAndPush(tweets, content, entryId, isConversation = false) {
+    const result = content.tweetResult?.result;
+    if (result?.__typename === 'Tweet') {
+        if (result.legacy) {
+            const toReplace = isConversation ? 'tweet-' : 'conversation-';
+            result.legacy.id_str = entryId.replace(toReplace, '');
+        }
+        const tweetResult = parseResult(result);
+        if (tweetResult.success) {
+            if (isConversation) {
+                if (content?.tweetDisplayType === 'SelfThread') {
+                    tweetResult.tweet.isSelfThread = true;
+                }
+            }
+            tweets.push(tweetResult.tweet);
+        }
+    }
+}
 function parseThreadedConversation(conversation) {
     const tweets = [];
-    const instructions = conversation.data?.threaded_conversation_with_injections_v2?.instructions ??
-        [];
+    const instructions = conversation.data?.timeline_response?.instructions ?? [];
     for (const instruction of instructions) {
-        for (const entry of instruction.entries ?? []) {
-            if (entry.content?.itemContent?.tweet_results?.result?.__typename ===
-                'Tweet') {
-                const tweetResult = parseResult(entry.content.itemContent.tweet_results.result);
-                if (tweetResult.success) {
-                    if (entry.content.itemContent.tweetDisplayType === 'SelfThread') {
-                        tweetResult.tweet.isSelfThread = true;
-                    }
-                    tweets.push(tweetResult.tweet);
-                }
+        const entries = instruction.entries ?? [];
+        for (const entry of entries) {
+            const entryContent = entry.content?.content;
+            if (entryContent) {
+                parseAndPush(tweets, entryContent, entry.entryId, true);
             }
             for (const item of entry.content?.items ?? []) {
-                if (item.item?.itemContent?.tweet_results?.result?.__typename === 'Tweet') {
-                    const tweetResult = parseResult(item.item.itemContent.tweet_results.result);
-                    if (tweetResult.success) {
-                        if (item.item.itemContent.tweetDisplayType === 'SelfThread') {
-                            tweetResult.tweet.isSelfThread = true;
-                        }
-                        tweets.push(tweetResult.tweet);
-                    }
+                const itemContent = item.item?.content;
+                if (itemContent) {
+                    parseAndPush(tweets, itemContent, entry.entryId, true);
                 }
@@ -176,0 +205,0 @@ }

dist/tweets.d.ts

@@ -67,7 +67,39 @@ import { TwitterAuth } from './auth';
 }
+export type TweetQuery = Partial<Tweet> | ((tweet: Tweet) => boolean | Promise<boolean>);
+export declare const features: {
+    rweb_lists_timeline_redesign_enabled: boolean;
+    responsive_web_graphql_exclude_directive_enabled: boolean;
+    verified_phone_label_enabled: boolean;
+    creator_subscriptions_tweet_preview_api_enabled: boolean;
+    responsive_web_graphql_timeline_navigation_enabled: boolean;
+    responsive_web_graphql_skip_user_profile_image_extensions_enabled: boolean;
+    tweetypie_unmention_optimization_enabled: boolean;
+    responsive_web_edit_tweet_api_enabled: boolean;
+    graphql_is_translatable_rweb_tweet_is_translatable_enabled: boolean;
+    view_counts_everywhere_api_enabled: boolean;
+    longform_notetweets_consumption_enabled: boolean;
+    tweet_awards_web_tipping_enabled: boolean;
+    freedom_of_speech_not_reach_fetch_enabled: boolean;
+    standardized_nudges_misinfo: boolean;
+    longform_notetweets_rich_text_read_enabled: boolean;
+    responsive_web_enhance_cards_enabled: boolean;
+    subscriptions_verification_info_enabled: boolean;
+    subscriptions_verification_info_reason_enabled: boolean;
+    subscriptions_verification_info_verified_since_enabled: boolean;
+    super_follow_badge_privacy_enabled: boolean;
+    super_follow_exclusive_tweet_notifications_enabled: boolean;
+    super_follow_tweet_api_enabled: boolean;
+    super_follow_user_api_enabled: boolean;
+    android_graphql_skip_api_media_color_palette: boolean;
+    creator_subscriptions_subscription_count_enabled: boolean;
+    blue_business_profile_image_shape_enabled: boolean;
+    unified_cards_ad_metadata_container_dynamic_card_content_query_enabled: boolean;
+};
 export declare function fetchTweets(userId: string, maxTweets: number, cursor: string | undefined, auth: TwitterAuth): Promise<QueryTweetsResponse>;
 export declare function getTweets(user: string, maxTweets: number, auth: TwitterAuth): AsyncGenerator<Tweet, void>;
 export declare function getTweetsByUserId(userId: string, maxTweets: number, auth: TwitterAuth): AsyncGenerator<Tweet, void>;
-export declare function getLatestTweet(user: string, includeRetweets: boolean, auth: TwitterAuth): Promise<Tweet | null | void>;
+export declare function getTweetWhere(tweets: AsyncIterable<Tweet>, query: TweetQuery): Promise<Tweet | null>;
+export declare function getTweetsWhere(tweets: AsyncIterable<Tweet>, query: TweetQuery): Promise<Tweet[]>;
+export declare function getLatestTweet(user: string, includeRetweets: boolean, max: number, auth: TwitterAuth): Promise<Tweet | null | void>;
 export declare function getTweet(id: string, auth: TwitterAuth): Promise<Tweet | null>;
 //# sourceMappingURL=tweets.d.ts.map

dist/tweets.js

@@ -6,3 +6,3 @@ "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getTweet = exports.getLatestTweet = exports.getTweetsByUserId = exports.getTweets = exports.fetchTweets = void 0;
+exports.getTweet = exports.getLatestTweet = exports.getTweetsWhere = exports.getTweetWhere = exports.getTweetsByUserId = exports.getTweets = exports.fetchTweets = exports.features = void 0;
 const api_1 = require("./api");
@@ -13,2 +13,9 @@ const profile_1 = require("./profile");
 const json_stable_stringify_1 = __importDefault(require("json-stable-stringify"));
+exports.features = (0, api_1.addApiFeatures)({
+    interactive_text_enabled: true,
+    longform_notetweets_inline_media_enabled: false,
+    responsive_web_text_conversations_enabled: false,
+    tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: false,
+    vibe_api_enabled: false,
+});
 async function fetchTweets(userId, maxTweets, cursor, auth) {
@@ -19,16 +26,6 @@ if (maxTweets > 200) {
     const variables = {
-        userId,
+        includeHasBirdwatchNotes: false,
+        rest_id: userId,
         count: maxTweets,
-        includePromotedContent: false,
-        withQuickPromoteEligibilityTweetFields: false,
-        withVoice: true,
-        withV2Timeline: true,
     };
-    const features = (0, api_1.addApiFeatures)({
-        interactive_text_enabled: true,
-        longform_notetweets_inline_media_enabled: false,
-        responsive_web_text_conversations_enabled: false,
-        tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: false,
-        vibe_api_enabled: true,
-    });
     if (cursor != null && cursor != '') {
@@ -39,4 +36,4 @@ variables['cursor'] = cursor;
     params.set('variables', (0, json_stable_stringify_1.default)(variables));
-    params.set('features', (0, json_stable_stringify_1.default)(features));
-    const res = await (0, api_1.requestApi)(`https://twitter.com/i/api/graphql/UGi7tjRPr-d_U3bCPIko5Q/UserTweets?${params.toString()}`, auth);
+    params.set('features', (0, json_stable_stringify_1.default)(exports.features));
+    const res = await (0, api_1.requestApi)(`https://api.twitter.com/graphql/8IS8MaO-2EN6GZZZb8jF0g/UserWithProfileTweetsAndRepliesQueryV2?${params.toString()}`, auth);
     if (!res.success) {
@@ -65,10 +62,9 @@ throw res.err;
 exports.getTweetsByUserId = getTweetsByUserId;
-async function getLatestTweet(user, includeRetweets, auth) {
-    const max = includeRetweets ? 1 : 200;
-    const timeline = getTweets(user, max, auth);
-    if (max == 1) {
-        return (await timeline.next()).value;
-    }
-    for await (const tweet of timeline) {
-        if (!tweet.isRetweet) {
+async function getTweetWhere(tweets, query) {
+    const isCallback = typeof query === 'function';
+    for await (const tweet of tweets) {
+        const matches = isCallback
+            ? await query(tweet)
+            : checkTweetMatches(tweet, query);
+        if (matches) {
             return tweet;
@@ -79,2 +75,28 @@ }
 }
+exports.getTweetWhere = getTweetWhere;
+async function getTweetsWhere(tweets, query) {
+    const isCallback = typeof query === 'function';
+    const filtered = [];
+    for await (const tweet of tweets) {
+        const matches = isCallback ? query(tweet) : checkTweetMatches(tweet, query);
+        if (!matches)
+            continue;
+        filtered.push(tweet);
+    }
+    return filtered;
+}
+exports.getTweetsWhere = getTweetsWhere;
+function checkTweetMatches(tweet, options) {
+    return Object.keys(options).every((k) => {
+        const key = k;
+        return tweet[key] === options[key];
+    });
+}
+async function getLatestTweet(user, includeRetweets, max, auth) {
+    const timeline = getTweets(user, max, auth);
+    // No point looping if max is 1, just use first entry.
+    return max === 1
+        ? (await timeline.next()).value
+        : await getTweetWhere(timeline, { isRetweet: includeRetweets });
+}
 exports.getLatestTweet = getLatestTweet;
@@ -84,18 +106,8 @@ async function getTweet(id, auth) {
         focalTweetId: id,
-        with_rux_injections: false,
-        includePromotedContent: true,
-        withCommunity: true,
-        withQuickPromoteEligibilityTweetFields: true,
-        withBirdwatchNotes: true,
-        withVoice: true,
-        withV2Timeline: true,
+        includeHasBirdwatchNotes: false,
     };
-    const features = (0, api_1.addApiFeatures)({
-        longform_notetweets_inline_media_enabled: true,
-        tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: false,
-    });
     const params = new URLSearchParams();
-    params.set('features', (0, json_stable_stringify_1.default)(features));
+    params.set('features', (0, json_stable_stringify_1.default)(exports.features));
     params.set('variables', (0, json_stable_stringify_1.default)(variables));
-    const res = await (0, api_1.requestApi)(`https://twitter.com/i/api/graphql/VWFGPVAGkZMGRKGe3GFFnA/TweetDetail?${params.toString()}`, auth);
+    const res = await (0, api_1.requestApi)(`https://api.twitter.com/graphql/83h5UyHZ9wEKBVzALX8R_g/ConversationTimelineV2?${params.toString()}`, auth);
     if (!res.success) {
@@ -105,10 +117,5 @@ throw res.err;
     const tweets = (0, timeline_v2_1.parseThreadedConversation)(res.value);
-    for (const tweet of tweets) {
-        if (tweet.id === id) {
-            return tweet;
-        }
-    }
-    return null;
+    return tweets.find((t) => t.id === id) ?? null;
 }
 exports.getTweet = getTweet;
 //# sourceMappingURL=tweets.js.map

package.json

 {
   "name": "@the-convocation/twitter-scraper",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "main": "dist/_module.js",
@@ -16,3 +16,3 @@ "repository": "https://github.com/the-convocation/twitter-scraper.git",
     "docs:deploy": "yarn docs:generate && gh-pages -d docs",
-    "format": "prettier --write \\\"src/**/*.ts\\\" \\\"test/**/*.ts\\\"",
+    "format": "prettier --write src/**/*.ts",
     "prepare": "husky install",
@@ -19,0 +19,0 @@ "test": "jest"

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

186695

increased by8.51%

Lines of code

2736

increased by9.84%

No dependency changes