yahoo-finance2 - npm Package Compare versions

Comparing version 2.4.0 to 2.4.1

dist/cjs/package.json

 {
     "name": "yahoo-finance2",
-    "version": "2.4.0",
+    "version": "2.4.1",
     "description": "JS API for Yahoo Finance",
@@ -5,0 +5,0 @@ "type:": "commonjs",

dist/cjs/package.json.js

 module.exports = {
     "name": "yahoo-finance2",
-    "version": "2.4.0",
+    "version": "2.4.1",
     "description": "JS API for Yahoo Finance",
@@ -5,0 +5,0 @@ "type:": "commonjs",

dist/cjs/src/lib/cookieJar.js

@@ -8,2 +8,3 @@ "use strict";
         let cookies;
+        // console.log("setFromSetCookieHeaders", setCookieHeader);
         if (typeof setCookieHeader === "undefined") {
@@ -20,4 +21,6 @@ // no-op
             for (const cookie of cookies)
-                if (cookie instanceof tough_cookie_1.Cookie)
+                if (cookie instanceof tough_cookie_1.Cookie) {
+                    // console.log("setCookieSync", cookie, url);
                     this.setCookieSync(cookie, url);
+                }
     }
@@ -24,0 +27,0 @@ }

dist/cjs/src/lib/fetchDevel.js

@@ -41,3 +41,2 @@ "use strict";
 const fs_1 = __importDefault(require("fs"));
-const path_1 = __importDefault(require("path"));
 const crypto_1 = __importDefault(require("crypto"));
@@ -49,3 +48,6 @@ //const FILE_BASE = path.join(__dirname, "..", "..", "tests", "http");
         Object.keys(props).forEach((key) => (this[key] = props[key]));
-        this.headers = new node_fetch_1.Headers(this.headers);
+        const rawHeaders = this.headers;
+        this.headers = new node_fetch_1.Headers(rawHeaders);
+        // node-fetch extension, needed to handle multiple set-cookie headers
+        this.headers.raw = () => rawHeaders;
     }
@@ -109,4 +111,5 @@ json() {
                 };
-                const contentType = contentObj.response.headers["content-type"][0].split(";");
-                if (contentType[0] === "application/json") {
+                const contentTypeHeader = contentObj.response.headers["content-type"];
+                const contentType = contentTypeHeader && contentTypeHeader[0].split(";");
+                if (contentType === "application/json") {
                     contentObj.response.bodyJson = yield res.json();
@@ -113,0 +116,0 @@ }

dist/cjs/src/lib/getCrumb.js

@@ -20,6 +20,7 @@ "use strict";
 // const MAX_CRUMB_CACHE_TIME = 60_000 * 60 * 24;
+const parseHtmlEntities = (str) => str.replace(/&#x([0-9A-Fa-f]{1,3});/gi, (_, numStr) => String.fromCharCode(parseInt(numStr, 16)));
 function _getCrumb(fetch, fetchOptionsBase, url = "https://finance.yahoo.com/quote/AAPL", develOverride = "getCrumb-quote-AAPL.json", noCache = false, cookieJar = cookieJar_js_1.default) {
     return __awaiter(this, void 0, void 0, function* () {
         // if (crumb && crumbFetchTime + MAX_CRUMB_CACHE_TIME > Date.now()) return crumb;
-        if (!noCache) {
+        if (crumb && !noCache) {
             // If we still have a valid (non-expired) cookie, return the existing crumb.
@@ -30,15 +31,76 @@ const existingCookies = cookieJar.getCookiesSync(url, { expire: true });
         }
+        function processSetCookieHeader(header, url) {
+            if (header) {
+                cookieJar.setFromSetCookieHeaders(header, url);
+                return true;
+            }
+            return false;
+        }
         console.log("Fetching crumb and cookies from " + url + "...");
         const fetchOptions = Object.assign(Object.assign({}, fetchOptionsBase), { headers: Object.assign(Object.assign({}, fetchOptionsBase.headers), { 
                 // NB, we won't get a set-cookie header back without this:
-                accept: "text/html,application/xhtml+xml,application/xml" }), devel: 
+                accept: "text/html,application/xhtml+xml,application/xml" }), redirect: "manual", devel: 
             // @ts-expect-error: fetchDevel still has no types (yet)
             fetchOptionsBase.devel && develOverride });
         const response = yield fetch(url, fetchOptions);
-        const setCookieHeader = response.headers.get("set-cookie");
-        if (setCookieHeader)
-            cookieJar.setFromSetCookieHeaders(setCookieHeader, url);
-        // console.log(response.headers);
-        // console.log(setCookieHeader);
+        processSetCookieHeader(response.headers.raw()["set-cookie"], url);
+        // console.log(response.headers.raw());
         // console.log(cookieJar);
+        const location = response.headers.get("location");
+        if (location) {
+            if (location.match(/guce.yahoo/)) {
+                const consentFetchOptions = Object.assign(Object.assign({}, fetchOptions), { headers: Object.assign(Object.assign({}, fetchOptions.headers), { 
+                        // GUCS=XXXXXXXX; Max-Age=1800; Domain=.yahoo.com; Path=/; Secure
+                        cookie: cookieJar.getCookieStringSync(location) }), devel: "getCrumb-quote-AAPL-consent.html" });
+                // Returns 302 to collectConsent?sessionId=XXX
+                console.log("fetch", location /*, consentFetchOptions */);
+                const consentResponse = yield fetch(location, consentFetchOptions);
+                const consentLocation = consentResponse.headers.get("location");
+                if (consentLocation) {
+                    if (!consentLocation.match(/collectConsent/))
+                        throw new Error("Unexpected redirect to " + consentLocation);
+                    const collectConsentFetchOptions = Object.assign(Object.assign({}, consentFetchOptions), { headers: Object.assign(Object.assign({}, fetchOptions.headers), { cookie: cookieJar.getCookieStringSync(consentLocation) }), devel: "getCrumb-quote-AAPL-collectConsent.html" });
+                    console.log("fetch", consentLocation /*, collectConsentFetchOptions */);
+                    const collectConsentResponse = yield fetch(consentLocation, collectConsentFetchOptions);
+                    const collectConsentBody = yield collectConsentResponse.text();
+                    const collectConsentResponseParams = [
+                        ...collectConsentBody.matchAll(/<input type="hidden" name="([^"]+)" value="([^"]+)">/g),
+                    ]
+                        .map(([, name, value]) => `${name}=${encodeURIComponent(parseHtmlEntities(value))}&`)
+                        .join("") + "agree=agree&agree=agree";
+                    const collectConsentSubmitFetchOptions = Object.assign(Object.assign({}, consentFetchOptions), { headers: Object.assign(Object.assign({}, fetchOptions.headers), { cookie: cookieJar.getCookieStringSync(consentLocation), "content-type": "application/x-www-form-urlencoded" }), method: "POST", 
+                        // body: "csrfToken=XjJfOYU&sessionId=3_cc-session_bd9a3b0c-c1b4-4aa8-8c18-7a82ec68a5d5&originalDoneUrl=https%3A%2F%2Ffinance.yahoo.com%2Fquote%2FAAPL%3Fguccounter%3D1&namespace=yahoo&agree=agree&agree=agree",
+                        body: collectConsentResponseParams, devel: "getCrumb-quote-AAPL-collectConsentSubmit" });
+                    console.log("fetch", consentLocation /*, collectConsentSubmitFetchOptions */);
+                    const collectConsentSubmitResponse = yield fetch(consentLocation, collectConsentSubmitFetchOptions);
+                    // Set-Cookie: CFC=AQABCAFkWkdkjEMdLwQ9&s=AQAAAClxdtC-&g=ZFj24w; Expires=Wed, 8 May 2024 01:18:54 GMT; Domain=consent.yahoo.com; Path=/; Secure
+                    if (!processSetCookieHeader(collectConsentSubmitResponse.headers.raw()["set-cookie"], consentLocation))
+                        throw new Error("No set-cookie header on collectConsentSubmitResponse, please report.");
+                    // https://guce.yahoo.com/copyConsent?sessionId=3_cc-session_04da10ea-1025-4676-8175-60d2508bfc6c&lang=en-GB
+                    const collectConsentSubmitResponseLocation = collectConsentSubmitResponse.headers.get("location");
+                    if (!collectConsentSubmitResponseLocation)
+                        throw new Error("collectConsentSubmitResponse unexpectedly did not return a Location header, please report.");
+                    const copyConsentFetchOptions = Object.assign(Object.assign({}, consentFetchOptions), { headers: Object.assign(Object.assign({}, fetchOptions.headers), { cookie: cookieJar.getCookieStringSync(collectConsentSubmitResponseLocation) }), devel: "getCrumb-quote-AAPL-copyConsent" });
+                    console.log("fetch", collectConsentSubmitResponseLocation /*, copyConsentFetchOptions */);
+                    const copyConsentResponse = yield fetch(collectConsentSubmitResponseLocation, copyConsentFetchOptions);
+                    if (!processSetCookieHeader(copyConsentResponse.headers.raw()["set-cookie"], collectConsentSubmitResponseLocation))
+                        throw new Error("No set-cookie header on copyConsentResponse, please report.");
+                    const copyConsentResponseLocation = copyConsentResponse.headers.get("location");
+                    if (!copyConsentResponseLocation)
+                        throw new Error("collectConsentSubmitResponse unexpectedly did not return a Location header, please report.");
+                    const finalResponseFetchOptions = Object.assign(Object.assign({}, fetchOptions), { headers: Object.assign(Object.assign({}, fetchOptions.headers), { cookie: cookieJar.getCookieStringSync(collectConsentSubmitResponseLocation) }), devel: "getCrumb-quote-AAPL-consent-final-redirect.html" });
+                    /*
+                    console.log(
+                      "fetch",
+                      copyConsentResponseLocation,
+                      finalResponseFetchOptions
+                    );
+                    */
+                    return yield _getCrumb(fetch, finalResponseFetchOptions, copyConsentResponseLocation, "getCrumb-quote-AAPL-consent-final-redirect.html", noCache, cookieJar);
+                }
+            }
+            else {
+                throw new Error("Unsupported redirect to " + location + ", please report.");
+            }
+        }
         const cookie = cookieJar.getCookiesSync(url, { expire: true })[0];
@@ -45,0 +107,0 @@ if (cookie) {

dist/cjs/src/lib/yahooFinanceFetch.js

@@ -67,4 +67,5 @@ "use strict";
         // console.log(url);
-        const fetchOptions = Object.assign(Object.assign({}, fetchOptionsBase), { headers: Object.assign(Object.assign({}, fetchOptionsBase.headers), { cookie: cookieJar_js_1.default.getCookieStringSync(url) }) });
-        // console.log(fetchOptions);
+        // console.log(cookieJar.serializeSync());
+        const fetchOptions = Object.assign(Object.assign({}, fetchOptionsBase), { headers: Object.assign(Object.assign({}, fetchOptionsBase.headers), { cookie: cookieJar_js_1.default.getCookieStringSync(url, { allPaths: true }) }) });
+        // console.log("fetch", url, fetchOptions);
         // used in moduleExec.ts
@@ -74,5 +75,5 @@ if (func === "csv")
         const response = (yield queue.add(() => fetchFunc(url, fetchOptions)));
-        const setCookieHeader = response.headers.get("set-cookie");
-        if (setCookieHeader)
-            cookieJar_js_1.default.setFromSetCookieHeaders(setCookieHeader, url);
+        const setCookieHeaders = response.headers.raw()["set-cookie"];
+        if (setCookieHeaders)
+            cookieJar_js_1.default.setFromSetCookieHeaders(setCookieHeaders, url);
         const result = yield response[func]();
@@ -79,0 +80,0 @@ /*

dist/esm/package.json

 {
     "name": "yahoo-finance2",
-    "version": "2.4.0",
+    "version": "2.4.1",
     "description": "JS API for Yahoo Finance",
@@ -5,0 +5,0 @@ "type": "module",

dist/esm/package.json.js

 export default {
     "name": "yahoo-finance2",
-    "version": "2.4.0",
+    "version": "2.4.1",
     "description": "JS API for Yahoo Finance",
@@ -5,0 +5,0 @@ "type": "module",

dist/esm/src/lib/cookieJar.js

@@ -5,2 +5,3 @@ import { Cookie, CookieJar } from "tough-cookie";
         let cookies;
+        // console.log("setFromSetCookieHeaders", setCookieHeader);
         if (typeof setCookieHeader === "undefined") {
@@ -17,4 +18,6 @@ // no-op
             for (const cookie of cookies)
-                if (cookie instanceof Cookie)
+                if (cookie instanceof Cookie) {
+                    // console.log("setCookieSync", cookie, url);
                     this.setCookieSync(cookie, url);
+                }
     }
@@ -21,0 +24,0 @@ }

dist/esm/src/lib/fetchDevel.js

 /* istanbul ignore file */
 import nodeFetch, { Headers } from "node-fetch";
 import fs from "fs";
-import path from "path";
 import crypto from "crypto";
@@ -11,3 +10,6 @@ //const FILE_BASE = path.join(__dirname, "..", "..", "tests", "http");
         Object.keys(props).forEach((key) => (this[key] = props[key]));
-        this.headers = new Headers(this.headers);
+        const rawHeaders = this.headers;
+        this.headers = new Headers(rawHeaders);
+        // node-fetch extension, needed to handle multiple set-cookie headers
+        this.headers.raw = () => rawHeaders;
     }
@@ -66,4 +68,5 @@ async json() {
             };
-            const contentType = contentObj.response.headers["content-type"][0].split(";");
-            if (contentType[0] === "application/json") {
+            const contentTypeHeader = contentObj.response.headers["content-type"];
+            const contentType = contentTypeHeader && contentTypeHeader[0].split(";");
+            if (contentType === "application/json") {
                 contentObj.response.bodyJson = await res.json();
@@ -70,0 +73,0 @@ }

dist/esm/src/lib/getCrumb.js

@@ -5,5 +5,6 @@ import defaultCookieJar from "./cookieJar.js";
 // const MAX_CRUMB_CACHE_TIME = 60_000 * 60 * 24;
+const parseHtmlEntities = (str) => str.replace(/&#x([0-9A-Fa-f]{1,3});/gi, (_, numStr) => String.fromCharCode(parseInt(numStr, 16)));
 export async function _getCrumb(fetch, fetchOptionsBase, url = "https://finance.yahoo.com/quote/AAPL", develOverride = "getCrumb-quote-AAPL.json", noCache = false, cookieJar = defaultCookieJar) {
     // if (crumb && crumbFetchTime + MAX_CRUMB_CACHE_TIME > Date.now()) return crumb;
-    if (!noCache) {
+    if (crumb && !noCache) {
         // If we still have a valid (non-expired) cookie, return the existing crumb.
@@ -14,2 +15,9 @@ const existingCookies = cookieJar.getCookiesSync(url, { expire: true });
     }
+    function processSetCookieHeader(header, url) {
+        if (header) {
+            cookieJar.setFromSetCookieHeaders(header, url);
+            return true;
+        }
+        return false;
+    }
     console.log("Fetching crumb and cookies from " + url + "...");
@@ -25,2 +33,3 @@ const fetchOptions = {
         },
+        redirect: "manual",
         devel: 
@@ -31,8 +40,98 @@ // @ts-expect-error: fetchDevel still has no types (yet)
     const response = await fetch(url, fetchOptions);
-    const setCookieHeader = response.headers.get("set-cookie");
-    if (setCookieHeader)
-        cookieJar.setFromSetCookieHeaders(setCookieHeader, url);
-    // console.log(response.headers);
-    // console.log(setCookieHeader);
+    processSetCookieHeader(response.headers.raw()["set-cookie"], url);
+    // console.log(response.headers.raw());
     // console.log(cookieJar);
+    const location = response.headers.get("location");
+    if (location) {
+        if (location.match(/guce.yahoo/)) {
+            const consentFetchOptions = {
+                ...fetchOptions,
+                headers: {
+                    ...fetchOptions.headers,
+                    // GUCS=XXXXXXXX; Max-Age=1800; Domain=.yahoo.com; Path=/; Secure
+                    cookie: cookieJar.getCookieStringSync(location),
+                },
+                devel: "getCrumb-quote-AAPL-consent.html",
+            };
+            // Returns 302 to collectConsent?sessionId=XXX
+            console.log("fetch", location /*, consentFetchOptions */);
+            const consentResponse = await fetch(location, consentFetchOptions);
+            const consentLocation = consentResponse.headers.get("location");
+            if (consentLocation) {
+                if (!consentLocation.match(/collectConsent/))
+                    throw new Error("Unexpected redirect to " + consentLocation);
+                const collectConsentFetchOptions = {
+                    ...consentFetchOptions,
+                    headers: {
+                        ...fetchOptions.headers,
+                        cookie: cookieJar.getCookieStringSync(consentLocation),
+                    },
+                    devel: "getCrumb-quote-AAPL-collectConsent.html",
+                };
+                console.log("fetch", consentLocation /*, collectConsentFetchOptions */);
+                const collectConsentResponse = await fetch(consentLocation, collectConsentFetchOptions);
+                const collectConsentBody = await collectConsentResponse.text();
+                const collectConsentResponseParams = [
+                    ...collectConsentBody.matchAll(/<input type="hidden" name="([^"]+)" value="([^"]+)">/g),
+                ]
+                    .map(([, name, value]) => `${name}=${encodeURIComponent(parseHtmlEntities(value))}&`)
+                    .join("") + "agree=agree&agree=agree";
+                const collectConsentSubmitFetchOptions = {
+                    ...consentFetchOptions,
+                    headers: {
+                        ...fetchOptions.headers,
+                        cookie: cookieJar.getCookieStringSync(consentLocation),
+                        "content-type": "application/x-www-form-urlencoded",
+                    },
+                    method: "POST",
+                    // body: "csrfToken=XjJfOYU&sessionId=3_cc-session_bd9a3b0c-c1b4-4aa8-8c18-7a82ec68a5d5&originalDoneUrl=https%3A%2F%2Ffinance.yahoo.com%2Fquote%2FAAPL%3Fguccounter%3D1&namespace=yahoo&agree=agree&agree=agree",
+                    body: collectConsentResponseParams,
+                    devel: "getCrumb-quote-AAPL-collectConsentSubmit",
+                };
+                console.log("fetch", consentLocation /*, collectConsentSubmitFetchOptions */);
+                const collectConsentSubmitResponse = await fetch(consentLocation, collectConsentSubmitFetchOptions);
+                // Set-Cookie: CFC=AQABCAFkWkdkjEMdLwQ9&s=AQAAAClxdtC-&g=ZFj24w; Expires=Wed, 8 May 2024 01:18:54 GMT; Domain=consent.yahoo.com; Path=/; Secure
+                if (!processSetCookieHeader(collectConsentSubmitResponse.headers.raw()["set-cookie"], consentLocation))
+                    throw new Error("No set-cookie header on collectConsentSubmitResponse, please report.");
+                // https://guce.yahoo.com/copyConsent?sessionId=3_cc-session_04da10ea-1025-4676-8175-60d2508bfc6c&lang=en-GB
+                const collectConsentSubmitResponseLocation = collectConsentSubmitResponse.headers.get("location");
+                if (!collectConsentSubmitResponseLocation)
+                    throw new Error("collectConsentSubmitResponse unexpectedly did not return a Location header, please report.");
+                const copyConsentFetchOptions = {
+                    ...consentFetchOptions,
+                    headers: {
+                        ...fetchOptions.headers,
+                        cookie: cookieJar.getCookieStringSync(collectConsentSubmitResponseLocation),
+                    },
+                    devel: "getCrumb-quote-AAPL-copyConsent",
+                };
+                console.log("fetch", collectConsentSubmitResponseLocation /*, copyConsentFetchOptions */);
+                const copyConsentResponse = await fetch(collectConsentSubmitResponseLocation, copyConsentFetchOptions);
+                if (!processSetCookieHeader(copyConsentResponse.headers.raw()["set-cookie"], collectConsentSubmitResponseLocation))
+                    throw new Error("No set-cookie header on copyConsentResponse, please report.");
+                const copyConsentResponseLocation = copyConsentResponse.headers.get("location");
+                if (!copyConsentResponseLocation)
+                    throw new Error("collectConsentSubmitResponse unexpectedly did not return a Location header, please report.");
+                const finalResponseFetchOptions = {
+                    ...fetchOptions,
+                    headers: {
+                        ...fetchOptions.headers,
+                        cookie: cookieJar.getCookieStringSync(collectConsentSubmitResponseLocation),
+                    },
+                    devel: "getCrumb-quote-AAPL-consent-final-redirect.html",
+                };
+                /*
+                console.log(
+                  "fetch",
+                  copyConsentResponseLocation,
+                  finalResponseFetchOptions
+                );
+                */
+                return await _getCrumb(fetch, finalResponseFetchOptions, copyConsentResponseLocation, "getCrumb-quote-AAPL-consent-final-redirect.html", noCache, cookieJar);
+            }
+        }
+        else {
+            throw new Error("Unsupported redirect to " + location + ", please report.");
+        }
+    }
     const cookie = cookieJar.getCookiesSync(url, { expire: true })[0];
@@ -39,0 +138,0 @@ if (cookie) {

dist/esm/src/lib/yahooFinanceFetch.js

@@ -57,2 +57,3 @@ import Queue from "./queue.js";
     // console.log(url);
+    // console.log(cookieJar.serializeSync());
     const fetchOptions = {
@@ -62,6 +63,6 @@ ...fetchOptionsBase,
             ...fetchOptionsBase.headers,
-            cookie: cookieJar.getCookieStringSync(url),
+            cookie: cookieJar.getCookieStringSync(url, { allPaths: true }),
         },
     };
-    // console.log(fetchOptions);
+    // console.log("fetch", url, fetchOptions);
     // used in moduleExec.ts
@@ -71,5 +72,5 @@ if (func === "csv")
     const response = (await queue.add(() => fetchFunc(url, fetchOptions)));
-    const setCookieHeader = response.headers.get("set-cookie");
-    if (setCookieHeader)
-        cookieJar.setFromSetCookieHeaders(setCookieHeader, url);
+    const setCookieHeaders = response.headers.raw()["set-cookie"];
+    if (setCookieHeaders)
+        cookieJar.setFromSetCookieHeaders(setCookieHeaders, url);
     const result = await response[func]();
@@ -76,0 +77,0 @@ /*

package.json

 {
   "name": "yahoo-finance2",
-  "version": "2.4.0",
+  "version": "2.4.1",
   "description": "JS API for Yahoo Finance",
@@ -5,0 +5,0 @@ "type": "module",

No alert changes

Improved metrics

Total package byte prevSize

1640749

increased by0.79%

Lines of code

51673

increased by0.35%

Worsened metrics

Number of medium supply chain risk alerts

19

increased by111.11%

No dependency changes