quickscraper-sdk - npm Package Compare versions

Comparing version 1.1.9 to 2.0.0

error.js

@@ -10,4 +10,2 @@ "use strict";
     return function (d, b) {
-        if (typeof b !== "function" && b !== null)
-            throw new TypeError("Class extends value " + String(b) + " is not a constructor or null");
         extendStatics(d, b);
@@ -14,0 +12,0 @@ function __() { this.constructor = d; }

index.d.ts

@@ -8,2 +8,3 @@ /// <reference types="node" />
     data: string;
+    metadata: IQuickScraperMetadata;
 }
@@ -23,4 +24,12 @@ export interface IParseOptions {
     webhookRequestId?: string;
+    customSelectors?: IActions[];
+    formData?: IFormData;
+    keepSelection?: boolean;
+    isScroll?: boolean;
+    scrollTimeout?: number;
+    keepFormdataSelection?: boolean;
+    isPabbly?: boolean;
+    isZapier?: boolean;
 }
-type ParsedUrlQueryInput = NodeJS.Dict<string | number | boolean | ReadonlyArray<string> | ReadonlyArray<number> | ReadonlyArray<boolean> | null>;
+declare type ParsedUrlQueryInput = NodeJS.Dict<string | number | boolean | Array<any> | any | ReadonlyArray<string> | ReadonlyArray<number> | ReadonlyArray<boolean> | null>;
 export interface IUrlOptions extends ParsedUrlQueryInput {
@@ -35,2 +44,10 @@ URL?: string;
     device_type?: string;
+    customSelectors?: IActions[];
+    formData?: IFormData;
+    keepSelection?: boolean;
+    isScroll?: boolean;
+    scrollTimeout?: number;
+    keepFormdataSelection?: boolean;
+    isPabbly?: boolean;
+    isZapier?: boolean;
 }
@@ -47,2 +64,20 @@ export interface IAccountInfo {
 }
+export interface IActions {
+    name: string;
+    selectorScript: string;
+    isScript: boolean;
+    options?: string;
+}
+export interface IFormField {
+    isScript?: boolean;
+    selectorScript?: string;
+    value?: any;
+}
+export interface IFormData {
+    formSelector?: string;
+    submitButtonSelector?: string;
+    formSelectorScript?: string;
+    submitButtonSelectorScript?: string;
+    formFields: IFormField[];
+}
 export declare class QuickScraper {
@@ -56,2 +91,4 @@ private parseUrl;
     setAccessToken(accessToken: string): void;
+    private getContent;
+    private postContent;
     getHtml(url: string, parseOptions?: IParseOptions): Promise<IQuickScraperResponse>;
@@ -69,2 +106,6 @@ writeHtmlToFile(url: string, filePath: string, parseOptions?: IParseOptions): Promise<IQuickScraperResponse>;
     private prepareRequestUrl;
+    prepareRequestBody(url: string, parseOptions?: IParseOptions): {
+        requestBody: IUrlOptions;
+        requestUrl: string;
+    };
     private prepareHeaders;
@@ -71,0 +112,0 @@ executeJsonParser(url: string, parseOptions?: IParseOptions): Promise<IQuickScraperResponse>;

index.js

@@ -28,3 +28,3 @@ "use strict";
         if (f) throw new TypeError("Generator is already executing.");
-        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+        while (_) try {
             if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
@@ -60,3 +60,3 @@ if (y = 0, t) op = [op[0] & 2, t.value];
 var error_1 = require("./error");
-var debug = (0, debug_1.default)('QS:Main:index.ts');
+var debug = debug_1.default('QS:Main:index.ts');
 var QuickScraper = /** @class */ (function () {
@@ -86,8 +86,9 @@ function QuickScraper(accessToken) {
     };
-    QuickScraper.prototype.getHtml = function (url, parseOptions) {
+    QuickScraper.prototype.getContent = function (url, parseOptions) {
+        var _a, _b;
         if (parseOptions === void 0) { parseOptions = {}; }
         return __awaiter(this, void 0, void 0, function () {
-            var requestUrl, customHeaders, mergedHeaders, gotOptions, response, responseBody, gotError_1, errorBody, error, message_1, type_1, statusCode_1, message, type, statusCode;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
+            var requestUrl, customHeaders, mergedHeaders, gotOptions, response, metadata, responseBody, gotError_1, errorBody, error, message_1, type_1, statusCode_1, message, type, statusCode;
+            return __generator(this, function (_c) {
+                switch (_c.label) {
                     case 0:
@@ -102,11 +103,15 @@ debug('parseOptions ', parseOptions);
                         };
-                        debug('gotOptions ', JSON.stringify(gotOptions, null, 2));
-                        _a.label = 1;
+                        console.log('gotOptions ', JSON.stringify(gotOptions, null, 2));
+                        _c.label = 1;
                     case 1:
-                        _a.trys.push([1, 3, , 4]);
+                        _c.trys.push([1, 3, , 4]);
                         return [4 /*yield*/, got_1.default.get(requestUrl, gotOptions)];
                     case 2:
-                        response = _a.sent();
+                        response = _c.sent();
+                        metadata = {
+                            'quotaMax': (_a = response.headers['x-qs-quota-max']) === null || _a === void 0 ? void 0 : _a.toString(),
+                            'quotaRemaining': (_b = response.headers['x-qs-quota-remaining']) === null || _b === void 0 ? void 0 : _b.toString()
+                        };
                         responseBody = response.body;
-                        if (!(0, lodash_1.isEmpty)(parseOptions.parserSubscriptionId)
+                        if (!lodash_1.isEmpty(parseOptions.parserSubscriptionId)
                             && response.headers['content-type'] &&
@@ -118,6 +123,7 @@ response.headers['content-type'].includes('application/json')) {
                         return [2 /*return*/, {
-                                data: responseBody
+                                data: responseBody,
+                                metadata: metadata
                             }];
                     case 3:
-                        gotError_1 = _a.sent();
+                        gotError_1 = _c.sent();
                         if (gotError_1 &&
@@ -147,2 +153,108 @@ gotError_1.response &&
     };
+    QuickScraper.prototype.postContent = function (url, parseOptions) {
+        var _a, _b;
+        if (parseOptions === void 0) { parseOptions = {}; }
+        return __awaiter(this, void 0, void 0, function () {
+            var requestData, customHeaders, mergedHeaders, gotOptions, response, metadata, responseBody, gotError_2, errorBody, error, message_2, type_2, statusCode_2, message, type, statusCode;
+            return __generator(this, function (_c) {
+                switch (_c.label) {
+                    case 0:
+                        debug('parseOptions ', parseOptions);
+                        requestData = this.prepareRequestBody(url, parseOptions);
+                        debug('requestBody ', requestData);
+                        customHeaders = parseOptions.headers;
+                        mergedHeaders = this.prepareHeaders(customHeaders, parseOptions);
+                        gotOptions = {
+                            headers: mergedHeaders,
+                            json: requestData.requestBody
+                        };
+                        console.log('gotOptions ', JSON.stringify(gotOptions, null, 2));
+                        _c.label = 1;
+                    case 1:
+                        _c.trys.push([1, 3, , 4]);
+                        return [4 /*yield*/, got_1.default.post(requestData.requestUrl, gotOptions)];
+                    case 2:
+                        response = _c.sent();
+                        metadata = {
+                            'quotaMax': (_a = response.headers['x-qs-quota-max']) === null || _a === void 0 ? void 0 : _a.toString(),
+                            'quotaRemaining': (_b = response.headers['x-qs-quota-remaining']) === null || _b === void 0 ? void 0 : _b.toString()
+                        };
+                        responseBody = response.body;
+                        if (!lodash_1.isEmpty(parseOptions.parserSubscriptionId)
+                            && response.headers['content-type'] &&
+                            response.headers['content-type'].includes('application/json')) {
+                            responseBody = JSON.parse(responseBody);
+                        }
+                        debug('response type', typeof (responseBody));
+                        return [2 /*return*/, {
+                                data: responseBody,
+                                metadata: metadata
+                            }];
+                    case 3:
+                        gotError_2 = _c.sent();
+                        console.log(gotError_2, 'gotError');
+                        if (gotError_2 &&
+                            gotError_2.response &&
+                            gotError_2.response.body) {
+                            errorBody = gotError_2.response.body;
+                            debug('errorBody ', errorBody);
+                            error = JSON.parse(errorBody);
+                            message_2 = error.message || 'Failed to process request';
+                            type_2 = error.error || 'UNKNOWN';
+                            statusCode_2 = error.statusCode || 530;
+                            debug('error message ', message_2);
+                            debug('error statusCode ', statusCode_2);
+                            debug('error type ', type_2);
+                            throw new error_1.QsError(message_2, type_2, statusCode_2);
+                        }
+                        debug('gotError ', gotError_2);
+                        message = 'Failed to process request';
+                        type = 'UNKNOWN';
+                        statusCode = 530;
+                        throw new error_1.QsError(message, type, statusCode);
+                    case 4: return [2 /*return*/];
+                }
+            });
+        });
+    };
+    QuickScraper.prototype.getHtml = function (url, parseOptions) {
+        var _a;
+        if (parseOptions === void 0) { parseOptions = {}; }
+        return __awaiter(this, void 0, void 0, function () {
+            var gotError_3, errorBody, error, message_3, type_3, statusCode_3, message, type, statusCode;
+            return __generator(this, function (_b) {
+                switch (_b.label) {
+                    case 0:
+                        _b.trys.push([0, 4, , 5]);
+                        if (!((parseOptions.customSelectors && ((_a = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.customSelectors) === null || _a === void 0 ? void 0 : _a.length) > 0) || parseOptions.formData)) return [3 /*break*/, 2];
+                        return [4 /*yield*/, this.postContent(url, parseOptions)];
+                    case 1: return [2 /*return*/, _b.sent()];
+                    case 2: return [4 /*yield*/, this.getContent(url, parseOptions)];
+                    case 3: return [2 /*return*/, _b.sent()];
+                    case 4:
+                        gotError_3 = _b.sent();
+                        if (gotError_3 &&
+                            gotError_3.response &&
+                            gotError_3.response.body) {
+                            errorBody = gotError_3.response.body;
+                            debug('errorBody ', errorBody);
+                            error = JSON.parse(errorBody);
+                            message_3 = error.message || 'Failed to process request';
+                            type_3 = error.error || 'UNKNOWN';
+                            statusCode_3 = error.statusCode || 530;
+                            debug('error message ', message_3);
+                            debug('error statusCode ', statusCode_3);
+                            debug('error type ', type_3);
+                            throw new error_1.QsError(message_3, type_3, statusCode_3);
+                        }
+                        debug('gotError ', gotError_3);
+                        message = 'Failed to process request';
+                        type = 'UNKNOWN';
+                        statusCode = 530;
+                        throw new error_1.QsError(message, type, statusCode);
+                    case 5: return [2 /*return*/];
+                }
+            });
+        });
+    };
     QuickScraper.prototype.writeHtmlToFile = function (url, filePath, parseOptions) {
@@ -296,3 +408,3 @@ return __awaiter(this, void 0, void 0, function () {
         return __awaiter(this, void 0, void 0, function () {
-            var response, gotError_2, errorBody, error, message_2, type_2, statusCode_2, message, type, statusCode;
+            var response, gotError_4, errorBody, error, message_4, type_4, statusCode_4, message, type, statusCode;
             return __generator(this, function (_a) {
@@ -309,18 +421,18 @@ switch (_a.label) {
                     case 2:
-                        gotError_2 = _a.sent();
-                        if (gotError_2 &&
-                            gotError_2.response &&
-                            gotError_2.response.body) {
-                            errorBody = gotError_2.response.body;
+                        gotError_4 = _a.sent();
+                        if (gotError_4 &&
+                            gotError_4.response &&
+                            gotError_4.response.body) {
+                            errorBody = gotError_4.response.body;
                             debug('errorBody ', errorBody);
                             error = JSON.parse(errorBody);
-                            message_2 = error.message || 'Failed to process request';
-                            type_2 = error.error || 'UNKNOWN';
-                            statusCode_2 = error.statusCode || 530;
-                            debug('error message ', message_2);
-                            debug('error statusCode ', statusCode_2);
-                            debug('error type ', type_2);
-                            throw new error_1.QsError(message_2, type_2, statusCode_2);
+                            message_4 = error.message || 'Failed to process request';
+                            type_4 = error.error || 'UNKNOWN';
+                            statusCode_4 = error.statusCode || 530;
+                            debug('error message ', message_4);
+                            debug('error statusCode ', statusCode_4);
+                            debug('error type ', type_4);
+                            throw new error_1.QsError(message_4, type_4, statusCode_4);
                         }
-                        debug('gotError ', gotError_2);
+                        debug('gotError ', gotError_4);
                         message = 'Failed to process request';
@@ -367,2 +479,14 @@ type = 'UNKNOWN';
         }
+        if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.isScroll) === false) {
+            urlOptions.isScroll = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.isScroll;
+        }
+        if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.scrollTimeout) === false) {
+            urlOptions.scrollTimeout = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.scrollTimeout;
+        }
+        if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.isPabbly) === false) {
+            urlOptions.isPabbly = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.isPabbly;
+        }
+        if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.isZapier) === false) {
+            urlOptions.isZapier = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.isZapier;
+        }
         debug('urlOptions ', urlOptions);
@@ -375,2 +499,65 @@ var optionString = querystring.stringify(urlOptions);
     };
+    QuickScraper.prototype.prepareRequestBody = function (url, parseOptions) {
+        var urlOptions = {
+            access_token: this.accessToken,
+            URL: url
+        };
+        if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.render) === false) {
+            urlOptions.render = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.render;
+        }
+        if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.session_number) === false) {
+            urlOptions.session_number = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.session_number;
+        }
+        if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.country_code) === false) {
+            urlOptions.country_code = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.country_code;
+        }
+        if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.premium) === false) {
+            urlOptions.premium = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.premium;
+        }
+        if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.keep_headers) === false) {
+            urlOptions.keep_headers = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.keep_headers;
+        }
+        if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.format) === false) {
+            urlOptions.format = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.format;
+        }
+        if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.device_type) === false) {
+            urlOptions.device_type = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.device_type;
+        }
+        if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.parserSubscriptionId) === false) {
+            urlOptions.parserSubscriptionRequestId = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.parserSubscriptionId;
+        }
+        if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.webhookRequestId) === false) {
+            urlOptions.webhookRequestId = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.webhookRequestId;
+        }
+        if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.isScroll) === false) {
+            urlOptions.isScroll = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.isScroll;
+        }
+        if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.scrollTimeout) === false) {
+            urlOptions.scrollTimeout = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.scrollTimeout;
+        }
+        if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.isPabbly) === false) {
+            urlOptions.isPabbly = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.isPabbly;
+        }
+        if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.isZapier) === false) {
+            urlOptions.isZapier = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.isZapier;
+        }
+        if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.customSelectors) === false) {
+            urlOptions.customSelectors = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.customSelectors;
+        }
+        if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.keepSelection) === false) {
+            urlOptions.keepSelection = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.keepSelection;
+        }
+        if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.formData) === false) {
+            urlOptions.formData = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.formData;
+        }
+        if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.keepFormdataSelection) === false) {
+            urlOptions.keepFormdataSelection = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.keepFormdataSelection;
+        }
+        var requestData = {
+            requestBody: urlOptions, requestUrl: this.parseUrl
+        };
+        // debug('optionString ', optionString);
+        // console.log('requestUrl ', requestUrl);
+        return requestData;
+    };
     QuickScraper.prototype.prepareHeaders = function (customHeaders, parseOptions) {
@@ -377,0 +564,0 @@ if (parseOptions === void 0) { parseOptions = {}; }

package.json

 {
 	"name": "quickscraper-sdk",
-	"version": "1.1.9",
+	"version": "2.0.0",
 	"description": "Quick Scraper SDK NodeJS APIs",
@@ -5,0 +5,0 @@ "main": "index.js",

No alert changes

Improved metrics

Total package byte prevSize

47116

increased by40.59%

Lines of code

760

increased by42.32%

No dependency changes