New Case Study:See how Anthropic automated 95% of dependency reviews with Socket.Learn More →
quickscraper-sdk
Advanced tools
quickscraper-sdk - npm Package Compare versions
Comparing version 2.0.0 to 2.0.1
@@ -23,12 +23,4 @@ /// <reference types="node" /> | ||
| webhookRequestId?: string; | ||
| customSelectors?: IActions[]; | ||
| formData?: IFormData; | ||
| keepSelection?: boolean; | ||
| isScroll?: boolean; | ||
| scrollTimeout?: number; | ||
| keepFormdataSelection?: boolean; | ||
| isPabbly?: boolean; | ||
| isZapier?: boolean; | ||
| } | ||
| declare type ParsedUrlQueryInput = NodeJS.Dict<string | number | boolean | Array<any> | any | ReadonlyArray<string> | ReadonlyArray<number> | ReadonlyArray<boolean> | null>; | ||
| declare type ParsedUrlQueryInput = NodeJS.Dict<string | number | boolean | ReadonlyArray<string> | ReadonlyArray<number> | ReadonlyArray<boolean> | null>; | ||
| export interface IUrlOptions extends ParsedUrlQueryInput { | ||
@@ -43,10 +35,2 @@ URL?: string; | ||
| device_type?: string; | ||
| customSelectors?: IActions[]; | ||
| formData?: IFormData; | ||
| keepSelection?: boolean; | ||
| isScroll?: boolean; | ||
| scrollTimeout?: number; | ||
| keepFormdataSelection?: boolean; | ||
| isPabbly?: boolean; | ||
| isZapier?: boolean; | ||
| } | ||
@@ -63,20 +47,2 @@ export interface IAccountInfo { | ||
| } | ||
| export interface IActions { | ||
| name: string; | ||
| selectorScript: string; | ||
| isScript: boolean; | ||
| options?: string; | ||
| } | ||
| export interface IFormField { | ||
| isScript?: boolean; | ||
| selectorScript?: string; | ||
| value?: any; | ||
| } | ||
| export interface IFormData { | ||
| formSelector?: string; | ||
| submitButtonSelector?: string; | ||
| formSelectorScript?: string; | ||
| submitButtonSelectorScript?: string; | ||
| formFields: IFormField[]; | ||
| } | ||
| export declare class QuickScraper { | ||
@@ -90,4 +56,2 @@ private parseUrl; | ||
| setAccessToken(accessToken: string): void; | ||
| private getContent; | ||
| private postContent; | ||
| getHtml(url: string, parseOptions?: IParseOptions): Promise<IQuickScraperResponse>; | ||
@@ -105,6 +69,2 @@ writeHtmlToFile(url: string, filePath: string, parseOptions?: IParseOptions): Promise<IQuickScraperResponse>; | ||
| private prepareRequestUrl; | ||
| prepareRequestBody(url: string, parseOptions?: IParseOptions): { | ||
| requestBody: IUrlOptions; | ||
| requestUrl: string; | ||
| }; | ||
| private prepareHeaders; | ||
@@ -111,0 +71,0 @@ executeJsonParser(url: string, parseOptions?: IParseOptions): Promise<IQuickScraperResponse>; |
213
index.js
@@ -84,3 +84,3 @@ "use strict"; | ||
| }; | ||
| QuickScraper.prototype.getContent = function (url, parseOptions) { | ||
| QuickScraper.prototype.getHtml = function (url, parseOptions) { | ||
| var _a, _b; | ||
@@ -101,3 +101,3 @@ if (parseOptions === void 0) { parseOptions = {}; } | ||
| }; | ||
| console.log('gotOptions ', JSON.stringify(gotOptions, null, 2)); | ||
| debug('gotOptions ', JSON.stringify(gotOptions, null, 2)); | ||
| _c.label = 1; | ||
@@ -150,108 +150,2 @@ case 1: | ||
| }; | ||
| QuickScraper.prototype.postContent = function (url, parseOptions) { | ||
| var _a, _b; | ||
| if (parseOptions === void 0) { parseOptions = {}; } | ||
| return __awaiter(this, void 0, void 0, function () { | ||
| var requestData, customHeaders, mergedHeaders, gotOptions, response, metadata, responseBody, gotError_2, errorBody, error, message_2, type_2, statusCode_2, message, type, statusCode; | ||
| return __generator(this, function (_c) { | ||
| switch (_c.label) { | ||
| case 0: | ||
| debug('parseOptions ', parseOptions); | ||
| requestData = this.prepareRequestBody(url, parseOptions); | ||
| debug('requestBody ', requestData); | ||
| customHeaders = parseOptions.headers; | ||
| mergedHeaders = this.prepareHeaders(customHeaders, parseOptions); | ||
| gotOptions = { | ||
| headers: mergedHeaders, | ||
| json: requestData.requestBody | ||
| }; | ||
| console.log('gotOptions ', JSON.stringify(gotOptions, null, 2)); | ||
| _c.label = 1; | ||
| case 1: | ||
| _c.trys.push([1, 3, , 4]); | ||
| return [4 /*yield*/, got_1.default.post(requestData.requestUrl, gotOptions)]; | ||
| case 2: | ||
| response = _c.sent(); | ||
| metadata = { | ||
| 'quotaMax': (_a = response.headers['x-qs-quota-max']) === null || _a === void 0 ? void 0 : _a.toString(), | ||
| 'quotaRemaining': (_b = response.headers['x-qs-quota-remaining']) === null || _b === void 0 ? void 0 : _b.toString() | ||
| }; | ||
| responseBody = response.body; | ||
| if (!lodash_1.isEmpty(parseOptions.parserSubscriptionId) | ||
| && response.headers['content-type'] && | ||
| response.headers['content-type'].includes('application/json')) { | ||
| responseBody = JSON.parse(responseBody); | ||
| } | ||
| debug('response type', typeof (responseBody)); | ||
| return [2 /*return*/, { | ||
| data: responseBody, | ||
| metadata: metadata | ||
| }]; | ||
| case 3: | ||
| gotError_2 = _c.sent(); | ||
| console.log(gotError_2, 'gotError'); | ||
| if (gotError_2 && | ||
| gotError_2.response && | ||
| gotError_2.response.body) { | ||
| errorBody = gotError_2.response.body; | ||
| debug('errorBody ', errorBody); | ||
| error = JSON.parse(errorBody); | ||
| message_2 = error.message || 'Failed to process request'; | ||
| type_2 = error.error || 'UNKNOWN'; | ||
| statusCode_2 = error.statusCode || 530; | ||
| debug('error message ', message_2); | ||
| debug('error statusCode ', statusCode_2); | ||
| debug('error type ', type_2); | ||
| throw new error_1.QsError(message_2, type_2, statusCode_2); | ||
| } | ||
| debug('gotError ', gotError_2); | ||
| message = 'Failed to process request'; | ||
| type = 'UNKNOWN'; | ||
| statusCode = 530; | ||
| throw new error_1.QsError(message, type, statusCode); | ||
| case 4: return [2 /*return*/]; | ||
| } | ||
| }); | ||
| }); | ||
| }; | ||
| QuickScraper.prototype.getHtml = function (url, parseOptions) { | ||
| var _a; | ||
| if (parseOptions === void 0) { parseOptions = {}; } | ||
| return __awaiter(this, void 0, void 0, function () { | ||
| var gotError_3, errorBody, error, message_3, type_3, statusCode_3, message, type, statusCode; | ||
| return __generator(this, function (_b) { | ||
| switch (_b.label) { | ||
| case 0: | ||
| _b.trys.push([0, 4, , 5]); | ||
| if (!((parseOptions.customSelectors && ((_a = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.customSelectors) === null || _a === void 0 ? void 0 : _a.length) > 0) || parseOptions.formData)) return [3 /*break*/, 2]; | ||
| return [4 /*yield*/, this.postContent(url, parseOptions)]; | ||
| case 1: return [2 /*return*/, _b.sent()]; | ||
| case 2: return [4 /*yield*/, this.getContent(url, parseOptions)]; | ||
| case 3: return [2 /*return*/, _b.sent()]; | ||
| case 4: | ||
| gotError_3 = _b.sent(); | ||
| if (gotError_3 && | ||
| gotError_3.response && | ||
| gotError_3.response.body) { | ||
| errorBody = gotError_3.response.body; | ||
| debug('errorBody ', errorBody); | ||
| error = JSON.parse(errorBody); | ||
| message_3 = error.message || 'Failed to process request'; | ||
| type_3 = error.error || 'UNKNOWN'; | ||
| statusCode_3 = error.statusCode || 530; | ||
| debug('error message ', message_3); | ||
| debug('error statusCode ', statusCode_3); | ||
| debug('error type ', type_3); | ||
| throw new error_1.QsError(message_3, type_3, statusCode_3); | ||
| } | ||
| debug('gotError ', gotError_3); | ||
| message = 'Failed to process request'; | ||
| type = 'UNKNOWN'; | ||
| statusCode = 530; | ||
| throw new error_1.QsError(message, type, statusCode); | ||
| case 5: return [2 /*return*/]; | ||
| } | ||
| }); | ||
| }); | ||
| }; | ||
| QuickScraper.prototype.writeHtmlToFile = function (url, filePath, parseOptions) { | ||
@@ -405,3 +299,3 @@ return __awaiter(this, void 0, void 0, function () { | ||
| return __awaiter(this, void 0, void 0, function () { | ||
| var response, gotError_4, errorBody, error, message_4, type_4, statusCode_4, message, type, statusCode; | ||
| var response, gotError_2, errorBody, error, message_2, type_2, statusCode_2, message, type, statusCode; | ||
| return __generator(this, function (_a) { | ||
@@ -418,18 +312,18 @@ switch (_a.label) { | ||
| case 2: | ||
| gotError_4 = _a.sent(); | ||
| if (gotError_4 && | ||
| gotError_4.response && | ||
| gotError_4.response.body) { | ||
| errorBody = gotError_4.response.body; | ||
| gotError_2 = _a.sent(); | ||
| if (gotError_2 && | ||
| gotError_2.response && | ||
| gotError_2.response.body) { | ||
| errorBody = gotError_2.response.body; | ||
| debug('errorBody ', errorBody); | ||
| error = JSON.parse(errorBody); | ||
| message_4 = error.message || 'Failed to process request'; | ||
| type_4 = error.error || 'UNKNOWN'; | ||
| statusCode_4 = error.statusCode || 530; | ||
| debug('error message ', message_4); | ||
| debug('error statusCode ', statusCode_4); | ||
| debug('error type ', type_4); | ||
| throw new error_1.QsError(message_4, type_4, statusCode_4); | ||
| message_2 = error.message || 'Failed to process request'; | ||
| type_2 = error.error || 'UNKNOWN'; | ||
| statusCode_2 = error.statusCode || 530; | ||
| debug('error message ', message_2); | ||
| debug('error statusCode ', statusCode_2); | ||
| debug('error type ', type_2); | ||
| throw new error_1.QsError(message_2, type_2, statusCode_2); | ||
| } | ||
| debug('gotError ', gotError_4); | ||
| debug('gotError ', gotError_2); | ||
| message = 'Failed to process request'; | ||
@@ -476,14 +370,2 @@ type = 'UNKNOWN'; | ||
| } | ||
| if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.isScroll) === false) { | ||
| urlOptions.isScroll = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.isScroll; | ||
| } | ||
| if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.scrollTimeout) === false) { | ||
| urlOptions.scrollTimeout = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.scrollTimeout; | ||
| } | ||
| if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.isPabbly) === false) { | ||
| urlOptions.isPabbly = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.isPabbly; | ||
| } | ||
| if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.isZapier) === false) { | ||
| urlOptions.isZapier = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.isZapier; | ||
| } | ||
| debug('urlOptions ', urlOptions); | ||
@@ -496,65 +378,2 @@ var optionString = querystring.stringify(urlOptions); | ||
| }; | ||
| QuickScraper.prototype.prepareRequestBody = function (url, parseOptions) { | ||
| var urlOptions = { | ||
| access_token: this.accessToken, | ||
| URL: url | ||
| }; | ||
| if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.render) === false) { | ||
| urlOptions.render = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.render; | ||
| } | ||
| if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.session_number) === false) { | ||
| urlOptions.session_number = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.session_number; | ||
| } | ||
| if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.country_code) === false) { | ||
| urlOptions.country_code = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.country_code; | ||
| } | ||
| if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.premium) === false) { | ||
| urlOptions.premium = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.premium; | ||
| } | ||
| if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.keep_headers) === false) { | ||
| urlOptions.keep_headers = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.keep_headers; | ||
| } | ||
| if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.format) === false) { | ||
| urlOptions.format = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.format; | ||
| } | ||
| if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.device_type) === false) { | ||
| urlOptions.device_type = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.device_type; | ||
| } | ||
| if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.parserSubscriptionId) === false) { | ||
| urlOptions.parserSubscriptionRequestId = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.parserSubscriptionId; | ||
| } | ||
| if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.webhookRequestId) === false) { | ||
| urlOptions.webhookRequestId = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.webhookRequestId; | ||
| } | ||
| if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.isScroll) === false) { | ||
| urlOptions.isScroll = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.isScroll; | ||
| } | ||
| if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.scrollTimeout) === false) { | ||
| urlOptions.scrollTimeout = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.scrollTimeout; | ||
| } | ||
| if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.isPabbly) === false) { | ||
| urlOptions.isPabbly = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.isPabbly; | ||
| } | ||
| if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.isZapier) === false) { | ||
| urlOptions.isZapier = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.isZapier; | ||
| } | ||
| if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.customSelectors) === false) { | ||
| urlOptions.customSelectors = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.customSelectors; | ||
| } | ||
| if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.keepSelection) === false) { | ||
| urlOptions.keepSelection = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.keepSelection; | ||
| } | ||
| if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.formData) === false) { | ||
| urlOptions.formData = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.formData; | ||
| } | ||
| if (_.isUndefined(parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.keepFormdataSelection) === false) { | ||
| urlOptions.keepFormdataSelection = parseOptions === null || parseOptions === void 0 ? void 0 : parseOptions.keepFormdataSelection; | ||
| } | ||
| var requestData = { | ||
| requestBody: urlOptions, requestUrl: this.parseUrl | ||
| }; | ||
| // debug('optionString ', optionString); | ||
| // console.log('requestUrl ', requestUrl); | ||
| return requestData; | ||
| }; | ||
| QuickScraper.prototype.prepareHeaders = function (customHeaders, parseOptions) { | ||
@@ -561,0 +380,0 @@ if (parseOptions === void 0) { parseOptions = {}; } |
| { | ||
| "name": "quickscraper-sdk", | ||
| "version": "2.0.0", | ||
| "version": "2.0.1", | ||
| "description": "Quick Scraper SDK NodeJS APIs", | ||
@@ -5,0 +5,0 @@ "main": "index.js", |
No alert changes
Worsened metrics
- Total package byte prevSize
- decreased by-28.28%
33792
- Lines of code
- decreased by-29.08%
539
No dependency changes