puppeteer-page-proxy - npm Package Compare versions

Comparing version 1.2.5 to 1.2.6

changelog.md

 # Change log
+### [1.2.6] - 2020-06-18
+#### Changes
+- Updated for Puppeteer's v4.0.0 [breaking changes](https://github.com/puppeteer/puppeteer/releases/tag/v4.0.0) ([#22](https://github.com/Cuadrix/puppeteer-page-proxy/issues/22), [#23](https://github.com/Cuadrix/puppeteer-page-proxy/issues/23))
+- Modified cookie handling to fix ([#20](https://github.com/Cuadrix/puppeteer-page-proxy/issues/20)) among other cookie related errors
 ### [1.2.5] - 2020-05-21
@@ -3,0 +7,0 @@ #### Changes

package.json

 {
   "name": "puppeteer-page-proxy",
   "description": "Additional Node.js module to use with 'puppeteer' for setting proxies per page basis.",
-  "version": "1.2.5",
+  "version": "1.2.6",
   "author": "Cuadrix <[e-mail]> (https://github.com/Cuadrix)",
@@ -25,3 +25,3 @@ "homepage": "https://github.com/Cuadrix/puppeteer-page-proxy",
   "dependencies": {
-    "got": "^11.1.4",
+    "got": "^11.3.0",
     "http-proxy-agent": "^4.0.1",
@@ -28,0 +28,0 @@ "https-proxy-agent": "^5.0.0",

README.md

@@ -167,3 +167,3 @@ # puppeteer-page-proxy <img src="https://i.ibb.co/kQrN9QJ/puppeteer-page-proxy-logo.png" align="right" width="150" height="150">
 
-```json
+```
 connection: TLSSocket {
@@ -170,0 +170,0 @@ _tlsOptions: {

src/core/lookup.js

 const lookup = async (page, lookupService = "https://api.ipify.org?format=json", isJSON = true, timeout = 30000) => {
-    const XMLHttpRequest = async () => {
+    const doLookup = async () => {
         return await page.evaluate((lookupService, timeout, isJSON) => {
-            return new Promise(resolve => {
-                var req = new XMLHttpRequest();
-                req.open("GET", lookupService, true);
+            return new Promise((resolve) => {
+                const req = new XMLHttpRequest();
                 req.timeout = timeout;
@@ -12,8 +11,9 @@ req.onload = () => {
                     } else {
-                        resolve(xhrFailed(`Request from [${window.location.href.slice(0, -1)}] to [${lookupService}] failed with status code ${req.status}`));
+                        resolve(onLookupFailed(`Request from ${window.location.href} to ${lookupService} failed with status code ${req.status}`));
                     }
                 };
-                req.ontimeout = e => {
-                    resolve(xhrFailed(`Request from [${window.location.href.slice(0, -1)}] to [${lookupService}] timed out -> ${req.timeout} ms`));
+                req.ontimeout = (error) => {
+                    resolve(onLookupFailed(`Request from ${window.location.href} to ${lookupService} timed out -> ${req.timeout} ms`));
                 };
+                req.open("GET", lookupService, true);
                 req.send();
@@ -25,5 +25,5 @@ });
         await page.setBypassCSP(true);
-        const errName = "xhrFailed";
-        if (!page._pageBindings.has(errName)) {
-            await page.exposeFunction(errName, reason => {
+        const functionName = "onLookupFailed";
+        if (!page._pageBindings.has(functionName)) {
+            await page.exposeFunction(functionName, (reason) => {
                 console.error(reason);
@@ -33,8 +33,10 @@ return;
         }
-        return await XMLHttpRequest();
+        return await doLookup();
     } catch(error) {
-        if (error.message === "Execution context was destroyed, most likely because of a navigation." || error.message === "Execution context was destroyed.")
-            return await XMLHttpRequest();
+        if (error.message.startsWith("Execution context was destroyed")) {
+            return await doLookup();
+        }
     }
 };
+
 module.exports = lookup;

src/core/proxy.js

 const request = require("got");
-const {type} = require("../lib/types");
-const {getCookies, cookieStore} = require("../lib/cookies");
+const type = require("../lib/types");
+const cookieJar = require("../lib/cookies");
 const {setOverrides, setHeaders, setAgent} = require("../lib/options");
 
-const useProxy = async (target, proxy) => {
-    // Listener responsible for applying proxy
-    const $puppeteerPageProxyHandler = async req => {       
-        endpoint = req._client._connection._url;
-        targetId = req._frame._id;
-        const cookieJar = cookieStore(
-            await getCookies(endpoint, targetId)
-        );
-        const options = {
-            cookieJar,
-            method: req.method(),
-            body: req.postData(),
-            headers: setHeaders(req),
-            agent: setAgent(proxy),
-            responseType: "buffer",
-            throwHttpErrors: false
-        };
-        try {
-            const res = await request(req.url(), options);
-            await req.respond({
-                status: res.statusCode,
-                headers: res.headers,
-                body: res.body
-            });
-        } catch(error) {
-            await req.abort();
-        }
+// Responsible for applying proxy
+const proxyHandler = async (req, proxy) => {
+    const options = {
+        cookieJar,
+        method: req.method(),
+        body: req.postData(),
+        headers: setHeaders(req),
+        agent: setAgent(proxy),
+        responseType: "buffer",
+        maxRedirects: 15,
+        throwHttpErrors: false
     };
-    // Remove existing listener for reassigning proxy of current page
-    const removeRequestListener = (page, listenerName) => {
-        const listeners = page.listeners("request");
-        for (let i = 0; i < listeners.length; i++) {
-            if (listeners[i].name === listenerName) {
-                page.removeListener("request", listeners[i]);
-            }
+    try {
+        const res = await request(req.url(), options);
+        await req.respond({
+            status: res.statusCode,
+            headers: res.headers,
+            body: res.body
+        });
+    } catch(error) {await req.abort()}
+};
+
+// For reassigning proxy of page
+const removeRequestListener = (page, listenerName) => {
+    const eventName = "request";
+    const listeners = page.eventsMap.get(eventName);
+    if (listeners) {
+        const i = listeners.findIndex((listener) => {
+            return listener.name === listenerName
+        });
+        listeners.splice(i, 1);
+        if (!listeners.length) {
+            page.eventsMap.delete(eventName);
         }
-    };
-    // Proxy per request
-    if (target.constructor.name === "Request") {
-        if (type(proxy) == "object") {
-            target = setOverrides(target, proxy);
-            proxy = proxy.proxy;
+    }
+};
+
+// Calls this if request object passed
+const proxyPerRequest = async (req, data) => {
+    let proxy, overrides;
+    // Separate proxy and overrides
+    if (type(data) === "object") {
+        if (Object.keys(data).length !== 0) {
+            proxy = data.proxy;
+            delete data.proxy;
+            overrides = data;
         }
-        await $puppeteerPageProxyHandler(target);
-    // Page-wide proxy
-    } else if (target.constructor.name === "Page") {
-        if (type(proxy) == "object") {
-            proxy = proxy.proxy;
-        }
-        await target.setRequestInterception(true);
-        removeRequestListener(target, "$puppeteerPageProxyHandler");
-        if (proxy) {
-            target.on("request", $puppeteerPageProxyHandler);
-        } else {
-            await target.setRequestInterception(false);
-        }
+    } else {proxy = data}
+    req = setOverrides(req, overrides);
+    // Skip request if proxy omitted
+    if (proxy) {await proxyHandler(req, proxy)}
+    else {req.continue(overrides)}
+};
+
+// Calls this if page object passed
+const proxyPerPage = async (page, proxy) => {
+    await page.setRequestInterception(true);
+    removeRequestListener(page, "$ppp");
+    if (proxy) {
+        page.on("request", $ppp = async (req) => {
+            await proxyHandler(req, proxy);
+        });
+    } else {await page.setRequestInterception(false)}
+};
+
+// Main function
+const useProxy = async (target, data) => {
+    const targetType = target.constructor.name;
+    if (targetType === "HTTPRequest") {
+        await proxyPerRequest(target, data);
+    } else if (targetType === "Page") {
+        await proxyPerPage(target, data)
     }
 };
+
 module.exports = useProxy;

src/lib/cookies.js

@@ -1,40 +0,3 @@
-const WebSocket = require("ws");
 const {CookieJar} = require("tough-cookie");
-const {Target, Network} = require("./cdp");
 
-const cookies = {
-    async getCookies(endpoint, targetId) {
-        const ws = new WebSocket(endpoint, {
-            perMessageDeflate: false,
-            maxPayload: 180 * 4096 // 0.73728Mb
-        });
-        await new Promise(resolve => ws.once("open", resolve));
-        // Attach to target then get cookies
-        const sessionId = await Target.attachToTarget(ws, targetId);
-        return await Network.getCookies(ws, sessionId);
-    },
-    cookieStore(cookies) {
-        if (!cookies)
-            return;
-        return CookieJar.deserializeSync({
-            version: 'tough-cookie@4.0.0',
-            storeType: 'MemoryCookieStore',
-            rejectPublicSuffixes: true,
-            cookies: cookies.map((cookie) => {
-                return {
-                    key: cookie.name,
-                    value: cookie.value,
-                    expires: cookie.expires === -1 ? Infinity : new Date(cookie.expires * 1000).toISOString(),
-                    domain: cookie.domain.replace(/^\./, ""),
-                    path: cookie.path,
-                    secure: cookie.secure,
-                    httpOnly: cookie.httpOnly,
-                    sameSite: cookie.sameSite,
-                    creation: new Date().toISOString(),
-                    hostOnly: !(/^\./).test(cookie.domain)
-                };
-            })
-        });
-    }
-};
-module.exports = cookies;
+module.exports = new CookieJar();

src/lib/options.js

@@ -5,47 +5,59 @@ const HttpProxyAgent = require("http-proxy-agent");
 
-const options = {
-    setOverrides(req, overrides) {
-        const map = {url: true, method: true, postData: true, headers: true};
-        for (const key in overrides) {
-            if (key == "headers")
-                req.$headers = true
-            if (map[key])
+// For overriding request objects
+const setOverrides = (req, overrides) => {
+    const map = {
+        url: true,
+        method: true,
+        postData: true,
+        headers: true
+    };
+    for (const key in overrides) {
+        if (map[key]) {
+            if (key === "headers") {
+                req["$" + key] = () => overrides[key];
+            } else {
                 req[key] = () => overrides[key];
+            }
         }
-        return req;
-    },
-    setHeaders(req) {
-        // If headers have been overriden
-        if (req.$headers)
-            return req.headers();
-        // Extended default headers
-        const headers = {
-            ...req.headers(),
-            "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
-            "accept-encoding": "gzip, deflate, br",
-            "host": new URL(req.url()).hostname
-        }
-        if (req.isNavigationRequest()) {
-            headers["sec-fetch-mode"] = "navigate";
-            headers["sec-fetch-site"] = "none";
-            headers["sec-fetch-user"] = "?1";
-        } else {
-            headers["sec-fetch-mode"] = "no-cors";
-            headers["sec-fetch-site"] = "same-origin";
-        }
-        return headers;
-    },
-    // For applying proxy
-    setAgent(proxy) {
-        if (proxy.startsWith("socks"))
-            return {
-                http: new SocksProxyAgent(proxy),
-                https: new SocksProxyAgent(proxy)
-            };
+    }
+    return req;
+};
+
+// Some extra headers
+const setHeaders = (req) => {
+    // If headers have been overriden
+    if (req.$headers)
+        return req.$headers();
+    // Extended default headers
+    const headers = {
+        ...req.headers(),
+        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
+        "accept-encoding": "gzip, deflate, br",
+        "host": new URL(req.url()).hostname
+    }
+    if (req.isNavigationRequest()) {
+        headers["sec-fetch-mode"] = "navigate";
+        headers["sec-fetch-site"] = "none";
+        headers["sec-fetch-user"] = "?1";
+    } else {
+        headers["sec-fetch-mode"] = "no-cors";
+        headers["sec-fetch-site"] = "same-origin";
+    }
+    return headers;
+};
+
+// For applying proxy
+const setAgent = (proxy) => {
+    if (proxy.startsWith("socks")) {
         return {
-            http: new HttpProxyAgent(proxy),
-            https: new HttpsProxyAgent(proxy)
+            http: new SocksProxyAgent(proxy),
+            https: new SocksProxyAgent(proxy)
         };
     }
+    return {
+        http: new HttpProxyAgent(proxy),
+        https: new HttpsProxyAgent(proxy)
+    };
 };
-module.exports = options;
+
+module.exports = {setOverrides, setHeaders, setAgent};

src/lib/types.js

@@ -25,8 +25,8 @@ const map = {
 };
-const util = {
-    type(value) {
-        const type = Object.prototype.toString.call(value);
-        return (map[type] === undefined) ? "object" : map[type];
-    }
-};
-module.exports = util;
+
+const type = (value) => {
+    const type = Object.prototype.toString.call(value);
+    return (map[type] === undefined) ? "object" : map[type];
+}
+
+module.exports = type;

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Worsened metrics

Total package byte prevSize

19427

decreased by-9.68%

Number of package files

10

decreased by-9.09%

Lines of code

228

decreased by-19.15%

Dependency changes

• Updatedgot@^11.3.0