google-sr - npm Package Compare versions

Comparing version 3.3.2 to 4.0.0

dist/index.d.ts

 import { AxiosRequestConfig } from 'axios';
-import * as selectors from 'google-sr-selectors';
-import { OrganicSearchSelector, TranslateSearchSelector, DictionarySearchSelector, TimeSearchSelector, CurrencyConvertSelector } from 'google-sr-selectors';
 import { CheerioAPI } from 'cheerio';
 
-declare enum ResultTypes {
-    SearchResult = "SEARCH",
-    TranslateResult = "TRANSLATE",
-    DictionaryResult = "DICTIONARY",
-    TimeResult = "TIME",
-    CurrencyResult = "CURRENCY"
+/**
+ * internal type to create a node with its type and other properties set as strings
+ * Example:
+ * Creates interface { type: "ORGANIC_RESULT", title: string } for ResultNodeTyper<typeof ResultTypes.OrganicSearchResult, "title">
+ * @private
+ */
+type ResultNodeTyper<T, K extends string> = {
+    type: T;
+} & Record<K, string | null>;
+/**
+ * Internal utility type to convert a array type (T[]) to a single type (T)
+ * if T is not an array, it will return T
+ * @private
+ */
+type AsArrayElement<T> = T extends Array<infer U> ? U : T;
+/**
+ * Internal utility type to get a record without null and undefined values
+ * @private
+ */
+type NonNullableRecord<T> = {
+    [K in keyof T]: NonNullable<T[K]>;
+};
+/**
+ * Generic type for the search results, derives the resultTypes from selector array.
+ */
+type SearchResultTypeFromSelector<R extends ResultSelector, S extends boolean = false> = S extends true ? NonNullableRecord<NonNullable<AsArrayElement<ReturnType<R>>>> : NonNullable<AsArrayElement<ReturnType<R>>>;
+
+declare const ResultTypes: {
+    readonly OrganicResult: "ORGANIC";
+    readonly TranslateResult: "TRANSLATE";
+    readonly DictionaryResult: "DICTIONARY";
+    readonly TimeResult: "TIME";
+    readonly CurrencyResult: "CURRENCY";
+};
+type OrganicResultNode = ResultNodeTyper<typeof ResultTypes.OrganicResult, "title" | "description" | "link">;
+type TranslateResultNode = ResultNodeTyper<typeof ResultTypes.TranslateResult, "sourceLanguage" | "translationLanguage" | "sourceText" | "translationText" | "translationPronunciation">;
+interface DictionaryDefinition {
+    partOfSpeech: string;
+    definition: string;
+    example: string;
+    synonyms: string[];
 }
-interface SearchResultNode {
-    /** Type of this result node */
-    type: ResultTypes.SearchResult;
-    /**
-     * Link or url of this search result
-     */
-    link: string;
-    description: string;
-    title: string;
+type DictionaryResultNode = ResultNodeTyper<typeof ResultTypes.DictionaryResult, "audio" | "phonetic" | "word"> & {
+    definitions: DictionaryDefinition[];
+};
+type TimeResultNode = ResultNodeTyper<typeof ResultTypes.TimeResult, "location" | "time" | "timeInWords">;
+type CurrencyResultNode = ResultNodeTyper<typeof ResultTypes.CurrencyResult, "from" | "to">;
+type SearchResultNode = OrganicResultNode | TranslateResultNode | DictionaryResultNode | TimeResultNode | CurrencyResultNode;
+interface SearchResultNodeLike {
+    type: string;
 }
-interface TranslateResultNode {
-    /** Type of this result node */
-    type: ResultTypes.TranslateResult;
+type ResultSelector<R extends SearchResultNodeLike = SearchResultNodeLike> = (cheerio: CheerioAPI, strictSelector: boolean) => R[] | R | null;
+/**
+ * Search options for single page search
+ */
+interface SearchOptions<R extends ResultSelector = ResultSelector> {
     /**
-     * Source for translation
+     * Search query
      */
-    source: {
-        /**
-         * Language of the source text
-         */
-        language: string;
-        /**
-         * Source text
-         */
-        text: string;
-    };
+    query: string;
     /**
-     * Translated content
+     * Control the type of results returned (can have a significant performance impact)
      */
-    translation: {
-        /**
-         * Language of the translated text
-         */
-        language: string;
-        /**
-         * translated text
-         */
-        text: string;
-        /**
-         * Pronunciation of the translation in english
-         * Only available in certain cases
-         */
-        pronunciation?: string;
-    };
-}
-interface DictionaryResultNode {
-    /** Type of this result node */
-    type: ResultTypes.DictionaryResult;
-    word: string;
-    phonetic: string;
+    resultTypes?: R[];
     /**
-     * Audio pronunciation of this word
+     * when true, will only return resultNodes that do not contain any undefined/empty properties
      */
-    audio?: string;
+    strictSelector?: boolean;
     /**
-     * Array of array containing definitions and their respective examples
-     * @example
-     *
-     * ```ts
-     * [
-     *  [
-     *    'causing great surprise or wonder; astonishing.',
-     *    'an amazing number of people registered'
-     *  ]
-     * ]
-     *
-     * ```
+     * Custom request configuration to be sent with the request
      */
-    definitions: [string, string][];
+    requestConfig?: AxiosRequestConfig;
 }
-interface TimeResultNode {
-    /** Type of this result node */
-    type: ResultTypes.TimeResult;
-    location: string;
-    time: string;
-    timeInWords: string;
-}
-interface CurrencyResultNode {
-    /** Type of this result node */
-    type: ResultTypes.CurrencyResult;
-    from: string;
-    to: string;
-    formula: string;
-}
-type ResultNode = SearchResultNode | TranslateResultNode | DictionaryResultNode | TimeResultNode | CurrencyResultNode;
 /**
- * Search options supported by the parser
+ * Search options for multiple pages search
  */
-interface SearchOptions {
+interface SearchOptionsWithPages<R extends ResultSelector = ResultSelector> extends SearchOptions<R> {
     /**
-     * raw config for axios
+     * Total number of pages to search or an array of specific pages to search
+     *
+     * google search uses cursor-based pagination.
+     *
+     * Specific page numbers are incremented by 10 starting from 0 (page 1)
+     *
+     * If total number of pages is provided, cursor will be created according to: start = page * 10
      */
-    requestConfig: AxiosRequestConfig;
+    pages: number | number[];
     /**
-     * Toggle to enable google safe mode
+     * Delay between each request in milliseconds. helps to avoid rate limiting issues.
+     *
+     * Default is 1000 ms (1 second). set to 0 to disable delay.
      */
-    safeMode: boolean;
-    /**
-     * Page number to fetch. Google page numbers are different that what you might expect
-     * we suggest you to use searchWithPages instead
-     */
-    page: number;
-    /**
-     * Base url of the service by default google.com/search
-     */
-    baseUrl: string;
-    /**
-     * Search query
-     */
-    query: string;
-    /**
-     * Filter the types of results returned (may have performance impact)
-     */
-    filterResults: ResultTypes[];
-    /**
-     * jquery selectors (cheerio) to extract data from scraped data
-     */
-    selectors: typeof selectors;
+    delay?: number;
 }
 
 /**
- * Convert a normal page to google query page
+ * Parses regular non-ads search results.
+ * @returns Array of OrganicSearchResultNodes
  */
-declare function pageToGoogleQueryPage(page: number): number;
-declare const generateArrayOfNumbers: (maxNumber: number) => number[];
-
+declare const OrganicResult: ResultSelector<OrganicResultNode>;
 /**
- * Search for a individual page
- * @param options Options for this search
- * @param options.query Search query
- * @returns Array of Results
- *
- * @example
- * ```ts
- * search({ query: 'nodejs' }).then(console.log);
- * // or if using await/async
- * const searchResults = await search({ query: 'nodejs' });
- * console.log(searchResults);
- * ```
+ * Parses translation search results.
+ * @returns Array of TranslateSearchResultNodes
  */
-declare function search(searchOptions: Partial<SearchOptions>): Promise<ResultNode[]>;
+declare const TranslateResult: ResultSelector<TranslateResultNode>;
 /**
- * Search multiple pages
- * @param options
- * @param options.pages no of pages / array of pages numbers to retrieve
- * @param options.searchDelay amount of milliseconds (ms) the package should wait between retrieving results (default is disabled) useful with ratelimits
- * @returns Array of arrays representing pages containing search results
- *
- * @example
- * Specify amount of pages to fetch
- *
- *```ts
- *
- * searchWithPages({ query: 'nodejs', pages: 5 }).then(console.log);
- * // or if using await/async
- * const searchResults = await searchWithPages({ query: 'nodejs', pages: 5 });
- * console.log(searchResults);
- * ```
- *
- * @example
- * Specifying specific pages to fetch
- *
- * ```ts
- * searchWithPages({ query: 'nodejs', pages: [1, 2, 5, 10] }).then(console.log);
- * // or if using await/async
- * const searchResults = await searchWithPages({ query: 'nodejs', pages: [1, 2, 5, 10] });
- * console.log(searchResults);
- * ```
+ * Parses dictionary search results.
+ * @returns Array of DictionaryResultNode
  */
-declare function searchWithPages({ pages, searchDelay, ...options }: Partial<Omit<SearchOptions, "page">> & {
-    pages: number | number[];
-    searchDelay?: number;
-}): Promise<ResultNode[][]>;
-
+declare const DictionaryResult: ResultSelector<DictionaryResultNode>;
 /**
- * Loader for Regular Results
- * @param $
- * @param selectors
- * @returns
- * @private
+ * Parses time search results.
+ * @returns Array of TimeResultNode
  */
-declare function loadSearchNodes($: CheerioAPI, selectors: typeof OrganicSearchSelector): SearchResultNode[];
+declare const TimeResult: ResultSelector<TimeResultNode>;
 /**
- * Loader for translation blocks
- * @param $
- * @param selectors
- * @returns
- * @private
+ * Parses currency convert search results.
+ * @returns Array of CurrencyResultNode
  */
-declare function loadTranslateNodes($: CheerioAPI, selectors: typeof TranslateSearchSelector): TranslateResultNode | null;
+declare const CurrencyResult: ResultSelector<CurrencyResultNode>;
+
 /**
- * Loader for dictionary blocks
- * @private
+ * Search google with the given query, only 1 page is returned
+ * @param options Search options
+ * @returns Search results as an array of SearchResultNodes
  */
-declare function loadDictionaryNodes($: CheerioAPI, selectors: typeof DictionarySearchSelector): DictionaryResultNode | null;
+declare function search<R extends ResultSelector = typeof OrganicResult>(options: SearchOptions<R> & {
+    strictSelector?: false;
+}): Promise<SearchResultTypeFromSelector<R>[]>;
+declare function search<R extends ResultSelector = typeof OrganicResult>(options: SearchOptions<R> & {
+    strictSelector: true;
+}): Promise<SearchResultTypeFromSelector<R, true>[]>;
 /**
- * @param $
- * @param selectors
- * @returns
- * @private
- */
-declare function loadTimeNode($: CheerioAPI, selectors: typeof TimeSearchSelector): TimeResultNode | null;
-/**
+ * Searches google with the given query, returns results for multiple pages.
+ * google uses cursor-based pagination (using param start=number).
  *
- * @param $
- * @param selectors
- * @returns
- * @private
+ * Therefore, when providing the specific page numbers, make sure to provide it in 10 increments.
+ *
+ * @example
+ * ```ts
+ * // search the first 5 pages
+ * const results = await searchWithPages({
+ *   query: "hello world",
+ *   pages: 5,
+ * });
+ *
+ * // or provide the specific page numbers
+ * const results = await searchWithPages({
+ *   query: "hello world",
+ *   pages: [0, 10, 20, 30, 40],
+ * });
+ *
+ * // pages can be skipped or be out of order
+ * const results = await searchWithPages({
+ *   query: "hello world",
+ *   pages: [10, 0, 20],
+ * });
+ * ```
+ * @returns Search results as an array of SearchResultNodes or an array of arrays of SearchResultNodes
  */
-declare function loadCurrencyNode($: CheerioAPI, selectors: typeof CurrencyConvertSelector): CurrencyResultNode | null;
+declare function searchWithPages<R extends ResultSelector = typeof OrganicResult>(options: SearchOptionsWithPages<R> & {
+    strictSelector?: false;
+}): Promise<SearchResultTypeFromSelector<R>[][]>;
+declare function searchWithPages<R extends ResultSelector = typeof OrganicResult>(options: SearchOptionsWithPages<R> & {
+    strictSelector: true;
+}): Promise<SearchResultTypeFromSelector<R, true>[][]>;
 
-export { CurrencyResultNode, DictionaryResultNode, ResultNode, ResultTypes, SearchOptions, SearchResultNode, TimeResultNode, TranslateResultNode, generateArrayOfNumbers, loadCurrencyNode, loadDictionaryNodes, loadSearchNodes, loadTimeNode, loadTranslateNodes, pageToGoogleQueryPage, search, searchWithPages };
+export { CurrencyResult, CurrencyResultNode, DictionaryDefinition, DictionaryResult, DictionaryResultNode, OrganicResult, OrganicResultNode, ResultNodeTyper, ResultSelector, ResultTypes, SearchOptions, SearchOptionsWithPages, SearchResultNode, SearchResultNodeLike, SearchResultTypeFromSelector, TimeResult, TimeResultNode, TranslateResult, TranslateResultNode, search, searchWithPages };

dist/index.js

@@ -34,10 +34,8 @@ "use strict";
 __export(src_exports, {
+  CurrencyResult: () => CurrencyResult,
+  DictionaryResult: () => DictionaryResult,
+  OrganicResult: () => OrganicResult,
   ResultTypes: () => ResultTypes,
-  generateArrayOfNumbers: () => generateArrayOfNumbers,
-  loadCurrencyNode: () => loadCurrencyNode,
-  loadDictionaryNodes: () => loadDictionaryNodes,
-  loadSearchNodes: () => loadSearchNodes,
-  loadTimeNode: () => loadTimeNode,
-  loadTranslateNodes: () => loadTranslateNodes,
-  pageToGoogleQueryPage: () => pageToGoogleQueryPage,
+  TimeResult: () => TimeResult,
+  TranslateResult: () => TranslateResult,
   search: () => search,
@@ -48,28 +46,28 @@ searchWithPages: () => searchWithPages
 
+// src/search.ts
+var import_axios = __toESM(require("axios"));
+var import_cheerio = require("cheerio");
+
+// src/results.ts
+var import_google_sr_selectors = require("google-sr-selectors");
+
 // src/constants.ts
-var selectors = __toESM(require("google-sr-selectors"));
-var ResultTypes = /* @__PURE__ */ ((ResultTypes2) => {
-  ResultTypes2["SearchResult"] = "SEARCH";
-  ResultTypes2["TranslateResult"] = "TRANSLATE";
-  ResultTypes2["DictionaryResult"] = "DICTIONARY";
-  ResultTypes2["TimeResult"] = "TIME";
-  ResultTypes2["CurrencyResult"] = "CURRENCY";
-  return ResultTypes2;
-})(ResultTypes || {});
-var defaultOptions = {
-  requestConfig: {},
-  safeMode: true,
-  // by default only the first page is resolved
-  page: 0,
-  query: "",
-  baseUrl: "https://www.google.com/search",
-  // do not add anything to this as deep merge will merge a new one with this
-  filterResults: [],
-  // these selectors must be updated when necessary
-  // last selector update was on 8/15/2023
-  selectors
+var ResultTypes = {
+  OrganicResult: "ORGANIC",
+  TranslateResult: "TRANSLATE",
+  DictionaryResult: "DICTIONARY",
+  TimeResult: "TIME",
+  CurrencyResult: "CURRENCY"
 };
 
-// src/helpers.ts
-var import_deepmerge_ts = require("deepmerge-ts");
+// src/utils.ts
+var baseHeaders = {
+  Accept: "text/html",
+  "Accept-Encoding": "gzip, deflate",
+  "Accept-Language": "en-US,en",
+  Referer: "https://www.google.com/",
+  "upgrade-insecure-requests": 1,
+  // the tested user agent is for Chrome 103 on Windows 10
+  "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36"
+};
 function extractUrlFromGoogleLink(googleLink) {
@@ -84,115 +82,115 @@ const match = googleLink.match(/\/url\?q=([^&]+)/);
 __name(extractUrlFromGoogleLink, "extractUrlFromGoogleLink");
-function constructSearchConfig(options) {
-  const AxiosRequestOptions = options.requestConfig || { headers: {} };
-  AxiosRequestOptions.headers = (0, import_deepmerge_ts.deepmerge)(
-    {
-      Accept: "text/html",
-      "Accept-Encoding": "gzip, deflate",
-      "Accept-Language": "en-US,en",
-      Referer: "https://www.google.com/",
-      "upgrade-insecure-requests": 1,
-      "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36"
-    },
-    AxiosRequestOptions.headers || {}
-  );
-  const params = new URLSearchParams(AxiosRequestOptions.params ?? void 0);
-  params.append("gbv", "1");
-  params.append("q", options.query);
-  if (options.safeMode) {
-    params.append("safe", "active");
+function prepareRequestConfig(opts) {
+  const requestConfig = opts.requestConfig ?? {};
+  if (typeof opts.query !== "string")
+    throw new TypeError(
+      `Search query must be a string, received ${typeof opts.query} instead.`
+    );
+  if (typeof requestConfig !== "object")
+    throw new TypeError(
+      `Request config must be an object if specified, received ${typeof requestConfig}.`
+    );
+  requestConfig.headers = requestConfig.headers ? Object.assign({}, baseHeaders, requestConfig.headers) : baseHeaders;
+  requestConfig.url = requestConfig.url ?? "https://www.google.com/search";
+  if (!(requestConfig.params instanceof URLSearchParams)) {
+    requestConfig.params = new URLSearchParams(requestConfig.params);
   }
-  if (options.page) {
-    params.append("start", options.page.toString());
-  }
-  const newConfig = {
-    params,
-    // require response data
-    responseType: "text",
-    responseEncoding: "utf-8"
-  };
-  return (0, import_deepmerge_ts.deepmerge)(AxiosRequestOptions, newConfig);
+  requestConfig.params.set("q", opts.query);
+  requestConfig.params.set("gbv", "1");
+  requestConfig.responseType = "text";
+  return requestConfig;
 }
-__name(constructSearchConfig, "constructSearchConfig");
-function pageToGoogleQueryPage(page) {
-  return Math.max(page * 10 - 10, 0);
+__name(prepareRequestConfig, "prepareRequestConfig");
+function throwNoCheerioError(resultParserName) {
+  throw new TypeError(
+    `CheerioAPI instance is missing, if using as a selector make sure to pass the raw function and not the result of calling it. (ex: [${resultParserName}] instead of [${resultParserName}()])`
+  );
 }
-__name(pageToGoogleQueryPage, "pageToGoogleQueryPage");
-var generateArrayOfNumbers = /* @__PURE__ */ __name((maxNumber) => new Array(maxNumber).fill(0).map((_, index) => index + 1), "generateArrayOfNumbers");
-var sleep = /* @__PURE__ */ __name((ms) => new Promise((resolve) => setTimeout(resolve, ms)), "sleep");
+__name(throwNoCheerioError, "throwNoCheerioError");
+function isEmpty(strictSelector, ...values) {
+  if (strictSelector)
+    return values.some(
+      (value) => value === "" || value === void 0 || value === null
+    );
+  return values.every(
+    (value) => value === "" || value === void 0 || value === null
+  );
+}
+__name(isEmpty, "isEmpty");
 
-// src/search.ts
-var import_axios = __toESM(require("axios"));
-var import_cheerio = require("cheerio");
-var import_deepmerge_ts2 = require("deepmerge-ts");
-
-// src/loaders.ts
-function loadSearchNodes($, selectors2) {
+// src/results.ts
+var OrganicResult = /* @__PURE__ */ __name(($, strictSelector) => {
+  if (!$)
+    throwNoCheerioError("OrganicResult");
   const parsedResults = [];
-  $(selectors2.block).each((_index, element) => {
-    const result = { type: "SEARCH" /* SearchResult */ };
-    const link = $(element).find(selectors2.link).attr("href");
-    const description = $(element).find(selectors2.description).text();
-    const title = $(element).find(selectors2.title).text();
-    if (typeof title === "string")
-      result.title = title;
+  const organicSearchBlocks = $(import_google_sr_selectors.OrganicSearchSelector.block).toArray();
+  for (const element of organicSearchBlocks) {
+    let link = $(element).find(import_google_sr_selectors.OrganicSearchSelector.link).attr("href");
+    const description = $(element).find(import_google_sr_selectors.OrganicSearchSelector.description).text();
+    const title = $(element).find(import_google_sr_selectors.OrganicSearchSelector.title).text();
     if (typeof link === "string")
-      result.link = extractUrlFromGoogleLink(link);
-    if (typeof description === "string")
-      result.description = description;
-    parsedResults.push(result);
-  });
+      link = extractUrlFromGoogleLink(link);
+    if (isEmpty(strictSelector, link, description, title))
+      continue;
+    parsedResults.push({
+      type: ResultTypes.OrganicResult,
+      link,
+      description,
+      title
+    });
+  }
   return parsedResults;
-}
-__name(loadSearchNodes, "loadSearchNodes");
-function loadTranslateNodes($, selectors2) {
-  const sourceLanguage = $(selectors2.sourceLanguage).text().trim();
-  const targetLanguage = $(selectors2.targetLanguage).text().trim();
-  const translation = $(selectors2.translationText).text().trim();
-  const source = $(selectors2.sourceText).val();
-  const pronunciation = $(selectors2.pronunciation).text().trim();
-  if ([translation, source, sourceLanguage, targetLanguage].some(
-    (value) => value === ""
+}, "OrganicResult");
+var TranslateResult = /* @__PURE__ */ __name(($, strictSelector) => {
+  if (!$)
+    throwNoCheerioError("TranslateResult");
+  const sourceLanguage = $(import_google_sr_selectors.TranslateSearchSelector.sourceLanguage).text().trim();
+  const sourceText = $(import_google_sr_selectors.TranslateSearchSelector.sourceText).val();
+  const translationText = $(import_google_sr_selectors.TranslateSearchSelector.translationText).text().trim();
+  const translationLanguage = $(import_google_sr_selectors.TranslateSearchSelector.targetLanguage).text().trim();
+  const translationPronunciation = $(import_google_sr_selectors.TranslateSearchSelector.pronunciation).text().trim();
+  if (isEmpty(
+    strictSelector,
+    sourceLanguage,
+    translationLanguage,
+    sourceText,
+    translationText,
+    translationPronunciation
   ))
     return null;
-  const result = {
-    type: "TRANSLATE" /* TranslateResult */,
-    source: {
-      language: sourceLanguage,
-      // expect this to be always a string
-      text: source
-    },
-    translation: {
-      language: targetLanguage,
-      text: translation,
-      pronunciation
-    }
+  return {
+    type: ResultTypes.TranslateResult,
+    sourceLanguage,
+    sourceText,
+    translationLanguage,
+    translationText,
+    translationPronunciation
   };
-  return result;
-}
-__name(loadTranslateNodes, "loadTranslateNodes");
-function loadDictionaryNodes($, selectors2) {
-  const audio = $(selectors2.audio).attr("src");
-  const phonetic = $(selectors2.phonetic).text().trim();
-  const word = $(selectors2.word).text().trim();
-  if ([audio, phonetic, word].some((val) => val === ""))
-    return null;
+}, "TranslateResult");
+var DictionaryResult = /* @__PURE__ */ __name(($, strictSelector) => {
+  if (!$)
+    throwNoCheerioError("DictionaryResult");
+  const audio = $(import_google_sr_selectors.DictionarySearchSelector.audio).attr("src") || "";
+  const phonetic = $(import_google_sr_selectors.DictionarySearchSelector.phonetic).first().text().trim();
+  const word = $(import_google_sr_selectors.DictionarySearchSelector.word).text().trim();
   const definitions = [];
-  $(selectors2.definitions).each(
-    (index, el) => {
-      definitions[index] = ["", ""];
-      definitions[index][0] = $(el).text().trim();
+  const definitionElements = $(import_google_sr_selectors.DictionarySearchSelector.definitionPartOfSpeech);
+  for (let i = 0; i < definitionElements.length; i++) {
+    const partOfSpeech = $(definitionElements[i]).text().trim();
+    if (partOfSpeech) {
+      const definition = $(import_google_sr_selectors.DictionarySearchSelector.definition).eq(i).text().trim();
+      const example = $(import_google_sr_selectors.DictionarySearchSelector.definitionExample).eq(i).text().trim();
+      const synonyms = $(import_google_sr_selectors.DictionarySearchSelector.definitionSynonyms).eq(i).text().replace("synonyms: ", "").split(",").map((synonym) => synonym.trim());
+      definitions.push({
+        partOfSpeech,
+        definition,
+        example,
+        synonyms
+      });
     }
-  );
-  $(selectors2.examples).each((index, el) => {
-    if (definitions[index]) {
-      let example = $(el).text().trim();
-      if (example.startsWith(`"`))
-        example = example.slice(1);
-      if (example.endsWith(`"`))
-        example = example.slice(0, example.length - 1);
-      definitions[index][1] = example;
-    }
-  });
-  const result = {
-    type: "DICTIONARY" /* DictionaryResult */,
+  }
+  if (isEmpty(strictSelector, audio, phonetic, word))
+    return null;
+  return {
+    type: ResultTypes.DictionaryResult,
     audio,
@@ -203,13 +201,13 @@ phonetic,
   };
-  return result;
-}
-__name(loadDictionaryNodes, "loadDictionaryNodes");
-function loadTimeNode($, selectors2) {
-  const location = $(selectors2.location).text().trim();
-  const time = $(selectors2.time).text().trim();
-  const timeInWords = $(selectors2.timeInWords).text().trim();
-  if ([location, time, timeInWords].some((val) => val === ""))
+}, "DictionaryResult");
+var TimeResult = /* @__PURE__ */ __name(($, strictSelector) => {
+  if (!$)
+    throwNoCheerioError("TimeResult");
+  const location = $(import_google_sr_selectors.TimeSearchSelector.location).text().trim();
+  const time = $(import_google_sr_selectors.TimeSearchSelector.time).text().trim();
+  const timeInWords = $(import_google_sr_selectors.TimeSearchSelector.timeInWords).text().trim();
+  if (isEmpty(strictSelector, location, time, timeInWords))
     return null;
-  const result = {
-    type: "TIME" /* TimeResult */,
+  return {
+    type: ResultTypes.TimeResult,
     location,
@@ -219,77 +217,68 @@ time,
   };
-  return result;
-}
-__name(loadTimeNode, "loadTimeNode");
-function loadCurrencyNode($, selectors2) {
-  const from = $(selectors2.from).text().trim();
-  const to = $(selectors2.to).text().trim();
-  const formula = `${from} ${to}`;
-  if ([to, from].some((val) => val === ""))
+}, "TimeResult");
+var CurrencyResult = /* @__PURE__ */ __name(($, strictSelector) => {
+  if (!$)
+    throwNoCheerioError("CurrencyResult");
+  const from = $(import_google_sr_selectors.CurrencyConvertSelector.from).text().replace("=", "").trim();
+  const to = $(import_google_sr_selectors.CurrencyConvertSelector.to).text().trim();
+  if (isEmpty(strictSelector, from, to))
     return null;
-  const result = {
-    type: "CURRENCY" /* CurrencyResult */,
-    formula,
-    from: from.replace("=", ""),
+  return {
+    type: ResultTypes.CurrencyResult,
+    from,
     to
   };
-  return result;
-}
-__name(loadCurrencyNode, "loadCurrencyNode");
+}, "CurrencyResult");
 
 // src/search.ts
-async function search(searchOptions) {
-  if (!searchOptions.query || typeof searchOptions.query !== "string")
+async function search(options) {
+  if (!options)
     throw new TypeError(
-      `Search query must be a string, received ${typeof searchOptions.query}`
+      `Search options must be provided. Received ${typeof options}`
     );
-  const options = (0, import_deepmerge_ts2.deepmerge)(defaultOptions, searchOptions);
-  if (options.filterResults.length === 0)
-    options.filterResults = ["SEARCH" /* SearchResult */];
-  const searchQuery = constructSearchConfig(options);
-  const searchRequest = await import_axios.default.get(options.baseUrl, searchQuery);
-  const html = searchRequest.data;
-  const $ = (0, import_cheerio.load)(html);
-  const result = [];
-  const selectors2 = options.selectors;
-  if (options.filterResults.includes("TRANSLATE" /* TranslateResult */)) {
-    const translateResult = loadTranslateNodes($, selectors2.TranslateSearchSelector);
-    if (translateResult)
-      result.push(translateResult);
+  const requestConfig = prepareRequestConfig(options);
+  const { data } = await (0, import_axios.default)(requestConfig);
+  const cheerioApi = (0, import_cheerio.load)(data);
+  const selectors = options.resultTypes || [OrganicResult];
+  let searchResults = [];
+  for (const selector of selectors) {
+    const result = selector(
+      cheerioApi,
+      Boolean(options.strictSelector)
+    );
+    if (result)
+      searchResults = searchResults.concat(result);
   }
-  if (options.filterResults.includes("DICTIONARY" /* DictionaryResult */)) {
-    const dictionaryResult = loadDictionaryNodes($, selectors2.DictionarySearchSelector);
-    if (dictionaryResult)
-      result.push(dictionaryResult);
-  }
-  if (options.filterResults.includes("TIME" /* TimeResult */)) {
-    const timeResult = loadTimeNode($, selectors2.TimeSearchSelector);
-    if (timeResult)
-      result.push(timeResult);
-  }
-  if (options.filterResults.includes("CURRENCY" /* CurrencyResult */)) {
-    const CurrencyResult = loadCurrencyNode($, selectors2.CurrencyConvertSelector);
-    if (CurrencyResult)
-      result.push(CurrencyResult);
-  }
-  if (options.filterResults.includes("SEARCH" /* SearchResult */)) {
-    const searchResults = loadSearchNodes($, selectors2.OrganicSearchSelector);
-    result.push(...searchResults);
-  }
-  return result;
+  return searchResults;
 }
 __name(search, "search");
-async function searchWithPages({
-  pages,
-  searchDelay = 0,
-  ...options
-}) {
-  const queryPages = Array.isArray(pages) ? pages : generateArrayOfNumbers(pages);
-  const pagesResults = [];
-  for (const page of queryPages) {
-    options.page = pageToGoogleQueryPage(page);
-    const result = await search(options);
-    pagesResults.push(result);
-    searchDelay && await sleep(searchDelay);
+async function searchWithPages(options) {
+  if (!options)
+    throw new TypeError(
+      `Search options must be provided. Received ${typeof options}`
+    );
+  if (typeof options.pages !== "number" && !Array.isArray(options.pages))
+    throw new TypeError(
+      `Page must be a number or an array of numbers. Received ${typeof options.pages}`
+    );
+  const searchResults = [];
+  const pages = Array.isArray(options.pages) ? options.pages : Array.from({ length: options.pages }, (_, i) => i * 10);
+  const baseRequestConfig = prepareRequestConfig(options);
+  const selectors = options.resultTypes || [OrganicResult];
+  for (const page of pages) {
+    baseRequestConfig.params.set("start", String(page));
+    const { data } = await (0, import_axios.default)(baseRequestConfig);
+    const cheerioApi = (0, import_cheerio.load)(data);
+    let pageResults = [];
+    for (const selector of selectors) {
+      const result = selector(
+        cheerioApi,
+        Boolean(options.strictSelector)
+      );
+      if (result)
+        pageResults = pageResults.concat(result);
+    }
+    searchResults.push(pageResults);
   }
-  return pagesResults;
+  return searchResults;
 }
@@ -299,12 +288,10 @@ __name(searchWithPages, "searchWithPages");
 0 && (module.exports = {
+  CurrencyResult,
+  DictionaryResult,
+  OrganicResult,
   ResultTypes,
-  generateArrayOfNumbers,
-  loadCurrencyNode,
-  loadDictionaryNodes,
-  loadSearchNodes,
-  loadTimeNode,
-  loadTranslateNodes,
-  pageToGoogleQueryPage,
+  TimeResult,
+  TranslateResult,
   search,
   searchWithPages
 });

package.json

 {
   "name": "google-sr",
-  "version": "3.3.2",
+  "version": "4.0.0",
   "description": "Fast and efficient Package for scraping Google search results without the need for an API key",
   "repository": "https://github.com/typicalninja/google-sr",
-  "homepage": "https://g-sr.vercel.app/google/sr",
+  "homepage": "https://github.com/typicalninja/google-sr",
   "main": "dist/index.js",
@@ -31,19 +31,14 @@ "types": "dist/index.d.ts",
     "cheerio": "1.0.0-rc.12",
-    "deepmerge-ts": "^5.1.0",
-    "tslib": "^2.6.1",
-    "google-sr-selectors": "^0.0.3"
+    "tslib": "^2.6.3",
+    "google-sr-selectors": "^1.0.0"
   },
   "devDependencies": {
-    "@types/chai": "^4.3.5",
-    "@types/mocha": "^10.0.1",
-    "chai": "^4.3.7",
-    "mocha": "^10.2.0",
-    "ts-node": "^10.9.1",
     "tsup": "^7.2.0",
-    "typescript": "^5.1.6"
+    "typescript": "^5.1.6",
+    "vitest": "^2.0.3"
   },
   "scripts": {
-    "test": "mocha --require ts-node/register tests/*.ts",
+    "test": "vitest",
     "build": "tsup"
   }
 }

README.md

@@ -11,90 +11,80 @@ # google-sr 🔍
 
-
 Simple & Fast Package for scraping Google search results without the need for an API key. 🚀
 
-* View documentation [here](https://g-sr.vercel.app)
+# Features
 
-* Come chat with us on [our discord](https://discord.gg/ynwckXS9T2)
-
-## Features ✨
-
-* Simple & Fast ⚡️ *
+* No API key is needed 🔑
+* 1st party typescript support
+* [Customizable selectors](https://github.com/typicalninja/google-sr/blob/master/apps/examples/src/custom-selector.ts) 🔍
 * [Well tested 🔄](#tests)
-* [Well documented 📚](https://g-sr.vercel.app)
-* TypeScript compatible 🧑‍💻
-* No API key is needed 🔑
-* [Wide variety of search result types supported 🌴](https://g-sr.vercel.app/google/sr/types)
 
-## Install 📦
+# Install 📦
 
-To get started, you can install **google-sr** using your preferred package manager:
+> google-sr is not supported on browser environments.
 
 ```bash
-
-# npm
-
 npm install google-sr
-
-# pnpm 
-
-pnpm add google-sr
-
-# yarn
-
-yarn add google-sr
-
+# or other supported runtimes/package managers
+[pnpm/yarn/bun] add google-sr
 ```
 
-## Usage
+# Usage
 
-You can easily perform a single-page search like this:
-
 ```ts
-import { search, ResultTypes } from 'google-sr';
+import { 
+    search, 
+    // import the result types you want
+    OrganicResult, 
+    DictionaryResult,
+    // helpful to import ResultTypes to filter results
+    ResultTypes 
+} from 'google-sr';
 
-// using await/async
-const searchResults = await search({ 
-    query: 'nodejs', 
-    safeMode: false, 
-    filterResults: [ResultTypes.SearchResult] 
+const queryResult = await search({
+    query: "nodejs",
+    // OrganicResult is the default, however it is recommended to ALWAYS specify the result type
+    resultTypes: [OrganicResult, DictionaryResult],
+    // to add additional configuration to the request, use the requestConfig option
+    // which accepts a AxiosRequestConfig object
+    // OPTIONAL
+    requestConfig: {
+		params: {
+            // enable "safe mode"
+			safe: 'active'
+		},
+	},
 });
 
-// will return a []
-console.log(searchResults);
-// should log: true
-console.log(searchResults[0].type === ResultTypes.SearchResult)
+// will return a SearchResult[]
+console.log(queryResult);
+console.log(queryResult[0].type === ResultTypes.OrganicResult); // true
 ```
 
-* **Read about the returned types [here](https://g-sr.vercel.app/google/sr/types)**
+> By default only `ResultTypes.OrganicResult` result type are returned, use the [resultTypes](#searchoptionsr--resultselector) option to configure it
 
-* **More detailed examples & usage can be found [here](https://g-sr.vercel.app/google/sr/usage)**
+* Additional examples can be found in [apps/examples](https://github.com/typicalninja/google-sr/tree/master/apps/examples) directory
 
-> By default only [`ResultTypes.SearchResult`](https://g-sr.vercel.app/google/sr/types#regular-search-results) are returned, use the [filterResults](https://g-sr.vercel.app/google/sr/usage#filtering-search-results) option to configure it
+# google-sr programatic API
 
-* Additional examples can be found in [tests](#tests) and apps directory
+Please refer to the google-sr API in [packages/google-sr](https://github.com/typicalninja/google-sr/blob/master/packages/google-sr/API.md)
 
 
-## Disclaimer
+# Related projects 🥂
 
-This is not sponsored, supported, or affiliated with Google Inc.
+* [google-that](https://g-sr.vercel.app/google/that) - CLI wrapper around google-sr
+* [google-sr-selectors](https://g-sr.vercel.app/google/selectors) - Selectors for google search results used by google-sr
 
-Unlike the conventional recommendation of using the Google API, this module scrapes the Google search result page (which might potentially infringe upon Google's terms of service).
+# ⚠️ Disclaimer
 
-The source code within this repository is intended solely for educational purposes.
+This is not sponsored, supported, or affiliated with Google Inc.
 
-The author (typicalninja) & contributors takes **no** responsibility for any issues that arise from misuse, such as IP blocking by Google. Your discretion in usage is advised.
+The source code within this repository is intended solely for educational & research purposes.
+The author (typicalninja) & contributors takes **NO** responsibility for any issues that arise from misuse, such as IP blocking by Google. Your discretion in usage is advised.
 
-## Related projects 🥂
-
-* [google-that](https://g-sr.vercel.app/google/that) - CLI wrapper around google-sr
-* [google-sr-selectors](https://g-sr.vercel.app/google/selectors) - Selectors for google search results used by google-sr
-
 # Tests
 
-Tests are written using [mocha](https://mochajs.org/) and can be run by using the `test` script.
+Tests are written using [vitest](https://vitest.dev/) and can be run by using the `test` script.
 
-> Weekly tests a executed using a github action to ensure compatibility
+> Weekly tests are executed using a github action to ensure compatibility and catch breakage due to google changes
 
-Project uses pnpm as its package manager
-
 ```bash
@@ -104,10 +94,4 @@ pnpm run test
 
-## Support & Bug Reporting 🛠️🐞
+# License
 
-> Make sure you are on the latest version before creating bug reports
-
-Support and bug reporting both can be done on  [github issues](https://github.com/typicalninja/google-sr/issues)
-
-## License
-
 This repository and the code inside it is licensed under the Apache-2.0 License. Read [LICENSE](./LICENSE) for more information.

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

51256

increased by5.42%

Dependency count

4

decreased by-20%

Dev dependency count

3

decreased by-57.14%

Number of package files

8

increased by14.29%

Number of medium supply chain risk alerts

0

decreased by-100%

Worsened metrics

Lines of code

700

decreased by-9.44%

Number of lines in readme file

95

decreased by-14.41%

Dependency changes

• + Addedgoogle-sr-selectors@1.0.0(transitive)

• - Removeddeepmerge-ts@^5.1.0

• - Removeddeepmerge-ts@5.1.0(transitive)

• - Removedgoogle-sr-selectors@0.0.3(transitive)

• Updatedgoogle-sr-selectors@^1.0.0

• Updatedtslib@^2.6.3