scrappey - npm Package Compare versions

Comparing version 1.1.0 to 1.2.0

index.js

@@ -5,3 +5,4 @@ /**
 
-const axios = require('axios');
+const axios = require('axios').default;
+const countries = require("./countries.json")
 
@@ -12,8 +13,36 @@ /**
 class Scrappey {
+
     /**
+    * Represents a proxy country.
+    *
+    * @typedef {('UnitedStates' | 'Canada' | 'Afghanistan' | 'Albania' | 'Algeria' | 'Argentina' | 'Armenia' | 'Aruba' | 'Australia' | 'Austria' | 'Azerbaijan' | 'Bahamas' | 'Bahrain' | 'Bangladesh' | 'Belarus' | 'Belgium' | 'BosniaandHerzegovina' | 'Brazil' | 'BritishVirginIslands' | 'Brunei' | 'Bulgaria' | 'Cambodia' | 'Cameroon' | 'Canada' | 'Chile' | 'China' | 'Colombia' | 'CostaRica' | 'Croatia' | 'Cuba' | 'Cyprus' | 'Czechia' | 'Denmark' | 'DominicanRepublic' | 'Ecuador' | 'Egypt' | 'ElSalvador' | 'Estonia' | 'Ethiopia' | 'Finland' | 'France' | 'Georgia' | 'Germany' | 'Ghana' | 'Greece' | 'Guatemala' | 'Guyana' | 'HashemiteKingdomofJordan' | 'HongKong' | 'Hungary' | 'India' | 'Indonesia' | 'Iran' | 'Iraq' | 'Ireland' | 'Israel' | 'Italy' | 'Jamaica' | 'Japan' | 'Kazakhstan' | 'Kenya' | 'Kosovo' | 'Kuwait' | 'Latvia' | 'Liechtenstein' | 'Luxembourg' | 'Macedonia' | 'Madagascar' | 'Malaysia' | 'Mauritius' | 'Mexico' | 'Mongolia' | 'Montenegro' | 'Morocco' | 'Mozambique' | 'Myanmar' | 'Nepal' | 'Netherlands' | 'NewZealand' | 'Nigeria' | 'Norway' | 'Oman' | 'Pakistan' | 'Palestine' | 'Panama' | 'PapuaNewGuinea' | 'Paraguay' | 'Peru' | 'Philippines' | 'Poland' | 'Portugal' | 'PuertoRico' | 'Qatar' | 'RepublicofLithuania' | 'RepublicofMoldova' | 'Romania' | 'Russia' | 'SaudiArabia' | 'Senegal' | 'Serbia' | 'Seychelles' | 'Singapore' | 'Slovakia' | 'Slovenia' | 'Somalia' | 'SouthAfrica' | 'SouthKorea' | 'Spain' | 'SriLanka' | 'Sudan' | 'Suriname' | 'Sweden' | 'Switzerland' | 'Syria' | 'Taiwan' | 'Tajikistan' | 'Thailand' | 'TrinidadandTobago' | 'Tunisia' | 'Turkey' | 'Uganda' | 'Ukraine' | 'UnitedArabEmirates' | 'UnitedKingdom' | 'UnitedStates' | 'Uzbekistan' | 'Venezuela' | 'Vietnam' | 'Zambia')} ProxyCountry
+    *
+    * @description
+    * The `proxyCountry` variable is a string that represents a country value from the provided array.
+    */
+
+    /**
+    * @typedef {Object} Options
+    * @property {string} url - The URL of the website to scrape.
+    * @property {string|null} [session] - (optional) The ID of the session to use for the request.
+    * @property {Array|null} [cookiejar] - (optional) Cookies to be sent with the request.
+    * @property {string|null} [proxy] - (optional) Proxy configuration for the request.
+    * @property {ProxyCountry|null} [proxyCountry] - (optional) Select the proxyCountry which will be used if no custom proxy is defined.
+    * @property {string|null} [autoparse] - (optional) Enables the autoParse feature from scrappey.com.
+    * @property {string|null} [properties] - (optional) Required properties when autoParse is enabled.
+    * @property {Object|null} [customHeaders] - (optional) Custom headers.
+    */
+
+    /**
+    * @typedef {Object} createOptions
+    * @property {string|null} [proxy] - (optional) Proxy configuration for the request.
+    * @property {ProxyCountry|null} [proxyCountry] - (optional) Select the proxyCountry which will be used if no custom proxy is defined.
+    */
+
+    /**
      * Create a Scrappey instance.
      * @param {string} apiKey - The API key for authentication.
+     * @param {boolean} disableVerboseErrors - Whether Verbose Errors should be disabled.
      */
-    constructor(apiKey) {
-
+    constructor(apiKey, disableVerboseErrors = false) {
         if (typeof apiKey === 'undefined' || apiKey.length === 0) {
@@ -24,11 +53,124 @@ throw new Error('apiKey parameter is required.');
         this.apiKey = apiKey;
+        this.disableVerboseErrors = disableVerboseErrors;
         this.baseUrl = 'https://publisher.scrappey.com/api/v1';
     }
+    /**
+     * 
+     * @param {string} endpoint 
+     * @param {Options} data 
+     * @returns 
+     */
+    isValidRequest(endpoint = "", data = {}) {
 
+        let sessionEndpoints = ["sessions.create", "sessions.destroy"];
+        let requestEndpoints = ["request.get", "request.post"];
+        let validEndpoints = [...sessionEndpoints, ...requestEndpoints];
+
+        // Validation is seperated into different parts.
+
+        // [1] Validate values which need to be checked on all Endpoints
+        if (!validEndpoints.includes(endpoint)) {
+            throw new Error(`Invalid endpoint. Valid endpoints are: ${validEndpoints.join(", ")}`);
+        }
+
+        // Validate the proxy
+        if (typeof data?.proxy !== "undefined") {
+            if (typeof data?.proxy !== "string") {
+                throw new Error("proxy parameter must be a string.");
+            }
+
+            let validProxyStarts = ["socks4://", "socks5://", "http://", "https://"];
+            if (!validProxyStarts.some(start => data.proxy.toLowerCase().startsWith(start))) {
+                throw new Error(`Proxy must start with: ${validProxyStarts.join(", ")}`);
+            }
+
+
+            if (typeof data.proxyCountry !== "undefined") {
+                if (!this.disableVerboseErrors)
+                    console.warn("The 'data.proxy' property is defined. As a result, the 'data.proxyCountry' property will be ignored by scrappey.com.");
+            }
+        }
+
+        if (typeof data?.proxyCountry !== "undefined") {
+            if (!countries.includes(data?.proxyCountry)) {
+                throw new Error(`Invalid proxyCountry. Valid proxyCountries are: ${countries.join(", ")}`);
+            }
+        }
+
+        // data.cmd should not be defined since the parameter endpoint already exists.
+        if (typeof data?.cmd !== "undefined") {
+            if (!this.disableVerboseErrors)
+                console.warn("The 'data.cmd' property is defined. The selected endpoint will be overwritten, which invalidates the validation performed by the wrapper. Please do not define the endpoint in the data.");
+        }
+
+        // End of [1]
+
+        // [2] Validation which only needs to be done on requests
+        if (requestEndpoints.includes(endpoint)) {
+            if (typeof data === "undefined") {
+                throw new Error("data parameter is required.");
+            }
+
+            if (typeof data?.url !== "string") {
+                throw new Error("url parameter is required.");
+            }
+
+            if (typeof data?.autoparse != "undefined") {
+                if (typeof data?.autoparse !== "boolean") {
+                    throw new Error("autoparse parameter must be a boolean.");
+                }
+
+                if (data.autoparse === true && typeof data?.properties === "undefined") {
+                    throw new Error("properties parameter is required when autoparse is enabled.");
+                }
+
+                if (typeof data?.properties !== "string") {
+                    throw new Error("properties parameter must be a string.");
+                }
+            }
+
+            if (typeof data?.customHeaders !== "undefined") {
+                if (typeof data?.customHeaders !== "object") {
+                    throw new Error("customHeaders parameter must be an object.");
+                }
+            }
+
+            if (typeof data?.session !== "undefined") {
+                if (typeof data?.session !== "string") {
+                    throw new Error("session parameter must be a string.");
+                }
+            }
+
+            if (typeof data?.cookiejar !== "undefined") {
+                // This throws error if non valid.
+                this.validCookies(data?.cookiejar);
+            }
+        }
+
+        // End of [2]
+
+        // [3] Validation which only needs to be done on sessions
+        if (sessionEndpoints.includes(endpoint)) {
+            if (typeof data !== "undefined") {
+                if (typeof data?.session !== "undefined") {
+                    if (typeof data?.session !== "string") {
+                        throw new Error("session parameter must be a string.");
+                    }
+                }
+            }
+        }
+
+        // End of [3]
+
+        return true;
+    }
+
     /**
-     * Sends a request to Scrappey.com.
+     * Sends a request to Scrappey.com API.
+     * Not all validation is performed here. 
+     * Additional endpoint-specific validation is conducted in the corresponding function. 
+     * Therefore, it is recommended to utilize the corresponding function instead of directly invoking 'sendRequest'.
      *
      * @param {string} endpoint - The endpoint for the request.
-     * @param {string} method - The HTTP method for the request.
-     * @param {Object} [data] - The request payload.
+     * @param {Options} [data] - The request payload.
      * @returns {Promise<Object>} The response from the request.
@@ -38,15 +180,17 @@ * @throws {Error} If the request fails.
      */
-    async sendRequest(endpoint, method, data = {}) {
-        const url = `${this.baseUrl}?key=${this.apiKey}`;
+    async sendRequest(endpoint, data = {}) {
 
-        if (typeof data?.proxy === "string") {
-            let validProxyStarts = ["socks4://", "socks5://", "http://", "https://"];
-            if (!validProxyStarts.some(start => data.proxy.toLowerCase().startsWith(start))) {
-                throw new Error(`Proxy must start with: ${validProxyStarts.join(", ")}`);
-            }
+        // This modification is necessary because the wrapper previously required a "sessionId" String instead of the "session" key as specified in the scrappey.com documentation, will be removed in feature.
+        if (typeof data?.sessionId === "string") {
+            data.session = data.sessionId;
+            data.sessionId = undefined;
         }
 
+        this.isValidRequest(endpoint, data);
+
+        const url = `${this.baseUrl}?key=${this.apiKey}`;
+
         const options = {
             url,
-            method: method,
+            method: "POST",
             headers: {
@@ -62,3 +206,3 @@ 'Content-Type': 'application/json'
         try {
-            const response = await axios(options);
+            const response = await axios(options, { timeout: 5 * 60 * 1000 });
             return response.data;
@@ -71,12 +215,54 @@ } catch (error) {
     /**
+    * Validates an array of cookies.
+    * 
+    * @param {Array<Object>} cookies - The cookies to validate.
+    * @returns {boolean} - A Promise that resolves to true if the cookies are valid.
+    * @throws {Error} - If the cookies are invalid.
+    */
+    validCookies(cookies) {
+        /**
+         * Cookies must look like this:
+         * [{"name": "cookie1", "value": "value1", "domain": "domain.com", "path": "/"},
+         *  {"name": "cookie2", "value": "value2", "domain": "domain.com", "path": "/"}]
+         */
+        if (!Array.isArray(cookies)) {
+            throw new Error("Cookies must be an array, check https://wiki.scrappey.com/ for examples how to use cookies.");
+        }
+
+        if (cookies.length === 0) {
+            throw new Error("Cookies should not be empty if defined.");
+        }
+
+        if (cookies.some(cookie => typeof cookie !== "object")) {
+            throw new Error("Cookies must be an array of objects.");
+        }
+
+        if (cookies.some(cookie => typeof cookie.name !== "string")) {
+            throw new Error("All cookies must have a name property.");
+        }
+
+        if (cookies.some(cookie => typeof cookie.value !== "string")) {
+            throw new Error("All cookies must have a value property which is a string.");
+        }
+
+        if (cookies.some(cookie => typeof cookie.domain !== "string")) {
+            throw new Error("All cookies must have a domain property.");
+        }
+
+        if (cookies.some(cookie => typeof cookie.path !== "string")) {
+            throw new Error("All cookies must have a path property.");
+        }
+
+        return true;
+    }
+
+
+    /**
     * Creates a new browser session.
     *
-    * @param {Object} options - The options for creating a session.
-    * @param {string|null} [options.sessionId] - The ID to assign to the session.
-    * @param {Object|null} [options.proxy] - Proxy configuration for the session.
-    * @returns {Promise<string>} The ID of the created session.
+    * @param {createOptions} options - The options for creating a session.
+    * @returns {Promise<Object>} The response from the request.
     */
     async createSession(options = {}) {
-        const { sessionId = null, proxy = null } = options;
-        return this.sendRequest('sessions.create', 'POST', { session: sessionId, proxy });
+        return this.sendRequest('sessions.create', { options });
     }
@@ -88,83 +274,123 @@
      *
-     * @param {string} sessionId - The ID of the session to destroy.
-     * @returns {Promise<void>}
+     * @param {string} session - The ID of the session to destroy.
+     * @returns {Promise<Object>} The response from the request.
      */
-    async destroySession(sessionId) {
-        if (typeof sessionId === 'undefined') {
-            throw new Error('sessionId parameter is required.');
+    async destroySession(session) {
+        if (typeof session === 'undefined') {
+            throw new Error('session parameter is required.');
         }
 
-        return this.sendRequest('sessions.destroy', 'POST', { session: sessionId });
+        return this.sendRequest('sessions.destroy', { session });
     }
 
     /**
+     * Checks whether the given data is a valid JSON string or object.
+     * @param {string|object} data - The data to be checked.
+     * @returns {boolean} - A boolean indicating whether the data is valid JSON.
+     */
+    isJSON(data) {
+        if (typeof data === "object") {
+            return true;
+        } else {
+            try {
+                JSON.parse(data);
+                return true;
+            } catch (error) {
+                return false;
+            }
+        }
+    }
+    /**
+    * Checks whether the given data is in valid form data format.
+    * @param {string} data - The data to be checked.
+    * @returns {boolean} - A boolean indicating whether the data is valid form data.
+    */
+    isFormData(data) {
+        try {
+            if (typeof data !== 'string') {
+                return false;
+            }
+
+            if (!data.includes('=')) {
+                return false;
+            }
+
+            const pairs = data.split('&');
+
+            for (let i = 0; i < pairs.length; i++) {
+                const pair = pairs[i];
+                const keyValue = pair.split('=');
+
+                if (keyValue.length !== 2) {
+                    return false;
+                }
+
+                const key = decodeURIComponent(keyValue[0]);
+                const value = decodeURIComponent(keyValue[1]);
+
+                if (key === '' || value === '') {
+                    return false;
+                }
+            }
+
+            return true;
+        } catch (error) {
+            return false;
+        }
+    }
+
+    /**
      * Sends a GET request through Scrappey.com.
      *
-     * @param {Object} options - The options for the GET request.
-     * @param {string} options.url - The URL of the website to scrape.
-     * @param {string|null} [options.sessionId] - The ID of the session to use for the request.
-     * @param {Object|null} [options.cookiejar] - Cookies to be sent with the request.
-     * @param {Object|null} [options.proxy] - Proxy configuration for the request.
+     * @param {Options} options - The options for the GET request.
      * @returns {Promise<Object>} The response from the request.
      */
     async getRequest(options) {
-        const { url, sessionId = null, cookiejar = null, proxy = null } = options;
-
-        if (typeof url === 'undefined') {
-            throw new Error('url parameter is required.');
-        }
-
-        return this.sendRequest('request.get', 'POST', {
-            url,
-            session: sessionId,
-            cookiejar,
-            proxy
+        return this.sendRequest('request.get', {
+            ...options
         });
     }
 
+
+
     /**
      * Sends a POST request through Scrappey.com.
      *
-     * @param {Object} options - The options for the POST request.
-     * @param {string} options.url - The URL of the website to scrape.
-     * @param {string|Object} options.postData - The data to be sent in the request body. It should be in `application/x-www-form-urlencoded` format.
-     * @param {string|null} [options.sessionId] - The ID of the session to use for the request.
-     * @param {Object|null} [options.cookiejar] - Cookies to be sent with the request.
-     * @param {Object|null} [options.proxy] - Proxy configuration for the request.
+     * @param {Options} options - The options for the POST request.
      * @returns {Promise<Object>} The response from the POST request.
-     * @throws {Error} If the postData is not in `application/x-www-form-urlencoded` format and cannot be converted.
      */
     async postRequest(options) {
-        const {
-            url,
-            postData,
-            sessionId = null,
-            cookiejar = null,
-            proxy = null
-        } = options;
+        if (typeof options?.postData !== "undefined") {
+            if (typeof options?.postData === "object") {
+                options.postData = JSON.stringify(options.postData);
+            }
 
-        // Check if postData is already in application/x-www-form-urlencoded format
-        const isFormData = typeof postData === 'string' && postData.includes('=');
+            // We should only check the postData if its length is not zero and it is of type string. This way, users can also send a value of "no postData" if desired.
+            if (typeof options?.postData === "string" && options?.postData?.length != 0) {
+                if (!this.isJSON(options?.postData) && !this.isFormData(options?.postData)) {
+                    throw new Error("postData must be in JSON or FormData (application/x-www-form-urlencoded) format.");
+                }
+            }
 
-        // If postData is not in the correct format, try to convert it
-        let requestData;
-        if (!isFormData) {
-            try {
-                requestData = new URLSearchParams(postData).toString();
-            } catch (error) {
-                throw new Error(
-                    'Invalid postData format. It must be in application/x-www-form-urlencoded format.'
-                );
+            if (typeof options?.postData != "string") {
+                throw new Error("postData must be a string.");
             }
         } else {
-            requestData = postData;
+            // We could also make postData default to empty String here.
+            // However, if users make a mistake, like misspelling postData or naming it differently (e.g., PostData), the request will still be sent without any postData.
+            // This behavior is not something the wrapper should do as it may lead to unexpected outcomes.
+            throw new Error("postData is required. Send empty String if you want to send no postData.");
         }
 
-        // Continue with the rest of the code
-        return this.sendRequest('request.post', 'POST', {
-            url,
-            postData: requestData,
-            session: sessionId,
-            cookiejar,
-            proxy
+        // If Users forget to give that content-type but data is json, it will be added automatically.
+        if (this.isJSON(options.postData)) {
+            let jsonpostData = JSON.parse(options.postData)
+
+            if (!Object.keys(jsonpostData).map(key => key.toLowerCase()).includes("content-type")) {
+                options.postData = JSON.stringify({ ...jsonpostData, "content-type": "application/json" })
+            }
+        }
+
+        return this.sendRequest('request.post', {
+            ...options
         });
@@ -171,0 +397,0 @@ }

package.json

 {
   "name": "scrappey",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "description": "",
@@ -5,0 +5,0 @@ "main": "index.js",

readme.md

@@ -7,3 +7,4 @@ # 🤖 Scrappey Wrapper - Data Extraction Made Easy
 
-Website: https://scrappey.com/
+Website: [https://scrappey.com/](https://scrappey.com/?via=martin)
+GitHub: [https://github.com/](https://github.com/DemonMartin/scrappey-wrapper)
 
@@ -36,3 +37,4 @@ ## Topics
 ```javascript
-const scrappey = new Scrappey('YOUR_API_KEY');
+const apiKey = 'YOUR_API_KEY';
+const scrappey = new Scrappey(apiKey);
 ```
@@ -48,3 +50,3 @@
 // Replace 'YOUR_API_KEY' with your Scrappey API key
-const apiKey = '';
+const apiKey = 'YOUR_API_KEY';
 
@@ -54,29 +56,45 @@ // Create an instance of Scrappey
 
-async function runTest() {
+async function run() {
     try {
         // Create a session
-        console.log("Creating sessionId.");
-        const session = await scrappey.createSession();
-        console.log(session);
+        const sessionRequest = await scrappey.createSession();
+        const { session } = sessionRequest;
 
+        console.log('Created Session:', session);
+
         // Make a GET request
-        const getRequestOptions = {
+        const getRequestResult = await scrappey.getRequest({
             url: 'https://reqres.in/api/users',
-            sessionId: session.session,
-        };
-        const getRequestResult = await scrappey.getRequest(getRequestOptions);
+            session,
+        });
         console.log('GET Request Result:', getRequestResult);
 
-        // Make a POST request
-        const postData = { username: 'user123', password: 'pass456' };
-        const postRequestOptions = {
+        // Make a POST request using FormData
+        const postFormData = { username: 'user123', password: 'pass456' };
+        const postRequestResultForm = await scrappey.postRequest({
             url: 'https://reqres.in/api/users',
-            postData,
-            sessionId: session.session,
-        };
-        const postRequestResult = await scrappey.postRequest(postRequestOptions);
-        console.log('POST Request Result:', postRequestResult);
+            postData: getQueryString(postFormData),
+            session
+        });
+        console.log('POST Request Result (FormData):', postRequestResultForm);
 
-        // Destroy the session
-        await scrappey.destroySession(session.session);
+        // Make a POST request using JSON data
+        const postJsonData = { email: '[e-mail]', password: 'pass123' };
+        const postRequestResultJson = await scrappey.postRequest({
+            url: 'https://reqres.in/api/users',
+            postData: JSON.stringify(postJsonData),
+            headers: {
+                'Content-Type': 'application/json', // Optional. To avoid issues please still add if you send JSON Data.
+            },
+            session,
+            // customHeaders: {
+            //     "auth": "token"
+            // },
+            // proxyCountry: "UnitedStates"
+            // & more!
+        });
+        console.log('POST Request Result (JSON):', postRequestResultJson);
+
+        // Manually destroy the session (automatically destroys after 4 minutes)
+        await scrappey.destroySession(session);
         console.log('Session destroyed.');
@@ -88,6 +106,24 @@ } catch (error) {
 
-runTest();
+run();
 
+function getQueryString(object) {
+    const queryString
+
+ = Object.keys(object)
+        .map(key => encodeURIComponent(key) + '=' + encodeURIComponent(object[key]))
+        .join('&');
+    return queryString;
+}
 ```
 
+**Scrappey Wrapper Features:**
+
+- Client-side error correction and handling
+- Easy session management with session creation and destruction
+- Support for GET and POST requests
+- Support for both FormData and JSON data formats
+- Customizable headers for requests
+- Robust and user-friendly
+- JSDocs supported
+
 For more information, please visit the [official Scrappey documentation](https://wiki.scrappey.com/getting-started). 📚
@@ -97,6 +133,2 @@
 
-This project is licensed under the MIT License.
-
-## Additional Tags
-
-cloudflare anti bot bypass, cloudflare solver, scraper, scraping, cloudflare scraper, cloudflare turnstile solver, turnstile solver, data extraction, web scraping, website scraping, data scraping, scraping tool, API scraping, scraping solution, web data extraction, website data extraction, web scraping library, website scraping library, cloudflare bypass, scraping API, web scraping API, cloudflare protection, data scraping tool, scraping service, cloudflare challenge solver, web scraping solution, web scraping service, cloudflare scraping, cloudflare bot protection, scraping framework, scraping library, cloudflare bypass tool, cloudflare anti-bot, cloudflare protection bypass, cloudflare solver tool, web scraping tool, data extraction library, website scraping tool, cloudflare turnstile bypass, cloudflare anti-bot solver, turnstile solver tool, cloudflare scraping solution, website data scraper, cloudflare challenge bypass, web scraping framework, cloudflare challenge solver tool, web data scraping, data scraper, scraping data from websites, SEO, data mining, data harvesting, data crawling, web scraping software, website scraping tool, web scraping framework, data extraction tool, web data scraper, data scraping service, scraping automation, scraping tutorial, scraping code, scraping techniques, scraping best practices, scraping scripts, scraping tutorial, scraping examples, scraping challenges, scraping tricks, scraping tips, scraping tricks, scraping strategies, scraping methods, cloudflare protection bypass, cloudflare security bypass, web scraping Python, web scraping JavaScript, web scraping PHP, web scraping Ruby, web scraping Java, web scraping C#, web scraping Node.js, web scraping BeautifulSoup, web scraping Selenium, web scraping Scrapy, web scraping Puppeteer, web scraping requests, web scraping headless browser, web scraping dynamic content, web scraping AJAX, web scraping pagination, web scraping authentication, web scraping cookies, web scraping session management, web scraping data parsing, web scraping data cleaning, web scraping data analysis, web scraping data visualization, web scraping legal issues, web scraping ethics, web scraping compliance, web scraping regulations, web scraping IP blocking, web scraping anti-scraping measures, web scraping proxy, web scraping CAPTCHA solving, web scraping IP rotation, web scraping rate limiting, web scraping data privacy, web scraping consent, web scraping terms of service, web scraping robots.txt, web scraping data storage, web scraping database integration, web scraping data integration, web scraping API integration, web scraping data export, web scraping data processing, web scraping data transformation, web scraping data enrichment, web scraping data validation, web scraping error handling, web scraping scalability, web scraping performance optimization, web scraping distributed scraping, web scraping cloud-based scraping, web scraping serverless scraping, akamai, datadome, perimetex, shape, kasada, queue-it, incapsula.
+This project is licensed under the MIT License.

test.js

 const Scrappey = require('.');
-
+const crypto = require("node:crypto");
+const fs = require("node:fs");
+const path = require("node:path")
 // Replace 'YOUR_API_KEY' with your Scrappey API key
 const apiKey = '';
+const showData = false;
 
-// Create an instance of Scrappey
-const scrappey = new Scrappey(apiKey);
+// Most error types look like this:
+// testMethod-region-errorIdentifier
+// Examples: 
+// test-client-get_request
+// advancedtest-server-get_request
 
+function getQueryString(object) {
+    //console.log("getQueryString Method was called.")
+    const queryString = Object.keys(object)
+        .map(key => encodeURIComponent(key) + '=' + encodeURIComponent(object[key]))
+        .join('&');
+
+
+    //console.log(`getQueryString Method returned: ${queryString}`)
+    return queryString;
+}
+
+function saveError(error, type = "undefined") {
+    if (!fs.existsSync(path.join(__dirname, "errors"))) {
+        fs.mkdirSync(path.join(__dirname, "errors"));
+    }
+
+    const errorId = crypto.randomUUID().slice(-5);
+    const errorDate = Date.now();
+
+    let errorObject;
+
+    if (typeof error === "string") {
+        errorObject = {
+            type,
+            errorId,
+            errorDate,
+            message: error
+        };
+    } else {
+        errorObject = {
+            type,
+            errorId,
+            errorDate,
+            name: error.name,
+            message: error.message,
+            stack: error.stack
+        };
+    }
+
+    const fileName = `${errorDate}-${errorId}.json`;
+    fs.writeFileSync(path.join(__dirname, 'errors', fileName), JSON.stringify(errorObject, null, 2));
+    console.log(`[~] Saved Error to: ./errors/${fileName}`);
+    return errorObject;
+}
+
+
 async function runTest() {
+
+    // Create an instance of Scrappey
+    const scrappey = new Scrappey(apiKey, false);
+    console.log("[test] [✅] Successfully created Scrappey Class.")
+
     try {
-        // Create a session
-        console.log("Creating sessionId.")
-        const session = await scrappey.createSession();
-        console.log(session);
+        let sessionId = `testSession-${crypto.randomUUID().slice(-5)}`;
+        try {
+            console.log("[test] [~] Create Session")
+            const sessionRequest = await scrappey.createSession();
+            console.log(`[test] [✅-client] Successfully created Session.`)
+            if (typeof sessionRequest?.error !== "undefined") {
+                console.log(`[test] [❌] Failed Creating Session.`)
+                saveError(
+                    JSON.stringify(sessionRequest, null, 2),
+                    "test-server-create_session"
+                )
+            } else {
+                const { session } = sessionRequest;
+                console.log("[test] [✅-server] Created Session: " + session)
+                sessionId = session;
+            }
 
-        // Make a GET request
-        const getRequestOptions = {
-            url: 'https://reqres.in/api/users',
-            sessionId: session.session,
-        };
-        const getRequestResult = await scrappey.getRequest(getRequestOptions);
-        console.log('GET Request Result:', getRequestResult);
+            if (showData) console.log(sessionRequest);
+        } catch (error) {
+            console.log("[test] [❌-client] Failed Creating Session")
+            saveError(error, "test-client-create_session");
+        }
 
-        // Make a POST request
-        const postData = { username: 'user123', password: 'pass456' };
-        const postRequestOptions = {
-            url: 'https://reqres.in/api/users',
-            postData,
-            sessionId: session.session,
-        };
-        const postRequestResult = await scrappey.postRequest(postRequestOptions);
-        console.log('POST Request Result:', postRequestResult);
+        try {
+            // Make a GET request
+            console.log("[test] [~] Get Request")
+            const getRequestResult = await scrappey.getRequest({
+                url: 'https://reqres.in/api/users',
+                session: sessionId
+            });
+            console.log(`[test] [✅-client] Successfully got Request.`)
+            if (showData) console.log(getRequestResult);
 
-        // Destroy the session
-        await scrappey.destroySession(session.session);
-        console.log('Session destroyed.');
+            if (typeof getRequestResult?.error !== "undefined") {
+                console.log(`[test] [❌-server] Failed Getting Request.`)
+                saveError(
+                    JSON.stringify(getRequestResult, null, 2),
+                    "test-server-get_request"
+                )
+            } else {
+                if (!scrappey.isJSON(getRequestResult.solution.innerText)) {
+                    console.log(`[test] [❌-server] Failed Getting Request. Response is not JSON.`)
+                    saveError(
+                        JSON.stringify(getRequestResult, null, 2),
+                        "test-server-get_request"
+                    )
+                } else if (getRequestResult.solution.innerText.includes("[e-mail]")) {
+                    console.log(`[test] [✅-server] Successfully got Request.`)
+                } else {
+                    console.log(`[test] [❌-server] Failed Getting Request. Response does not contain the correct data.`)
+                    saveError(
+                        JSON.stringify(getRequestResult, null, 2),
+                        "test-server-get_request"
+                    )
+                }
+            }
+        } catch (error) {
+            console.log("[test] [❌-client] Failed Getting Request")
+            saveError(error, "test-client-get_request");
+        }
+
+        try {
+            // Make a POST request using FormData
+            console.log("[test] [~] Post Request (FormData)")
+            const postFormData = { username: 'user123', password: 'pass456' };
+            const postRequestResult = await scrappey.postRequest({
+                url: 'https://reqres.in/api/users',
+                postData: getQueryString(postFormData), // The getQueryString Method will convert the Object to FormData. This will automatically be sent with the application/x-www-form-urlencoded header.
+                session: sessionId
+            });
+            console.log(`[test] [✅-client] Successfully posted Request (FormData).`)
+            if (showData) console.log(postRequestResult);
+
+            if (typeof postRequestResult?.error !== "undefined") {
+                console.log(`[test] [❌-server] Failed Posting Request (FormData).`)
+                saveError(
+                    JSON.stringify(postRequestResult, null, 2),
+                    "test-server-post_request_formdata"
+                )
+            } else {
+                if (!scrappey.isJSON(postRequestResult.solution.innerText)) {
+                    console.log(`[test] [❌-server] Failed Posting Request (FormData). Response is not JSON.`)
+                    saveError(
+                        JSON.stringify(postRequestResult, null, 2),
+                        "test-server-post_request_formdata"
+                    )
+                } else if (postRequestResult.solution.innerText.includes("user123")) {
+                    console.log(`[test] [✅-server] Successfully posted Request (FormData).`)
+                } else {
+                    console.log(`[test] [❌-server] Failed Posting Request (FormData). Response does not contain the correct data.`)
+                    saveError(
+                        JSON.stringify(postRequestResult, null, 2),
+                        "test-server-post_request_formdata"
+                    )
+                }
+            }
+
+
+        } catch (error) {
+            console.log("[test] [❌-client] Failed Posting Request (FormData).")
+            saveError(error, "test-post_request_formdata");
+        }
+
+        try {
+            // If you dont want to use formData, scrappey also supports pure JSON requests.
+            console.log("[test] [~] Post Request (JSON)")
+            const postDataJson = JSON.stringify({ email: "[e-mail]", password: "password" })
+            const postRequestResultJson = await scrappey.postRequest({
+                url: "https://reqres.in/api/users",
+                postData: postDataJson,
+                customHeaders: {
+                    'Content-Type': 'application/json', // This is optional, the wrapper should automatically add this if it detects JSON data.
+                },
+                session: sessionId
+            });
+            console.log(`[test] [✅-client] Successfully posted Request (JSON).`)
+            if (showData) console.log(postRequestResultJson);
+
+            if (typeof postRequestResultJson?.error !== "undefined") {
+                console.log(`[test] [❌-server] Failed Posting Request (JSON).`)
+                saveError(
+                    JSON.stringify(postRequestResultJson, null, 2),
+                    "test-server-post_request_json"
+                )
+            } else {
+                if (!scrappey.isJSON(postRequestResultJson.solution.innerText)) {
+                    console.log(`[test] [❌-server] Failed Posting Request (JSON). Response is not JSON.`)
+                    saveError(
+                        JSON.stringify(postRequestResultJson, null, 2),
+                        "test-server-post_request_json"
+                    )
+                } else if (postRequestResultJson.solution.innerText.includes("[e-mail]")) {
+                    console.log(`[test] [✅-server] Successfully posted Request (JSON).`)
+                } else {
+                    console.log(`[test] [❌-server] Failed Posting Request (JSON). Response does not contain the correct data.`)
+                    saveError(
+                        JSON.stringify(postRequestResultJson, null, 2),
+                        "test-server-post_request_json"
+                    )
+                }
+            }
+        } catch (error) {
+            console.log("[test] [❌-client] Failed Posting Request (JSON).")
+            saveError(error, "test-post_request_json");
+        }
+
+        // ! Important ! Either formData or JSON needs to be used.
+
+        try {
+            // Destroy the session
+            console.log('[test] [~] Session Destruction')
+            const destroySession = await scrappey.destroySession(sessionId);
+            console.log(`[test] [✅-client] Successfully destroyed Session.`)
+            if (showData) console.log(destroySession);
+
+            if (typeof destroySession?.error !== "undefined") {
+                console.log(`[test] [❌-server] Failed Destroying Session.`)
+                saveError(
+                    JSON.stringify(destroySession, null, 2),
+                    "test-server-destroy_session"
+                )
+            }
+        } catch (error) {
+            console.log("[test] [❌-client] Failed Destroying Session.")
+            saveError(error, "test-destroy_session");
+        }
+
+        console.log("[test] All tests have been now executed.")
     } catch (error) {
-        console.error(error);
+        console.error(
+            "[test] [❌] An error has occured while running the tests."
+        );
+        saveError(error, "test");
     }
 }
 
+async function advancedTests() {
+    // soon
+}
+
 runTest();
+advancedTests();

New alerts

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

37416

increased by150.12%

Number of package files

8

increased by33.33%

Lines of code

737

increased by304.95%

Number of lines in readme file

128

increased by33.33%

Worsened metrics

Number of medium supply chain risk alerts

1

increased byInfinity%

No dependency changes