got-scraping - npm Package Compare versions

Comparing version 1.0.0 to 1.0.1-beta.0

CHANGELOG.md

@@ -0,3 +1,8 @@
+1.0.1 / 2021/04/15
+====================
+- Improved ciphers
+- Fixed request payload sending
+
 1.0.0 / 2021/04/07
 ====================
 - Initial release

package.json

 {
-	"name": "got-scraping",
-	"version": "1.0.0",
-	"description": "HTTP client made for scraping based on got.",
-	"main": "src/index.js",
-	"files": [
-		"src"
-	],
-	"dependencies": {
-		"header-generator": "^0.0.1-beta.1",
-		"got": "^11.8.1",
-		"http-proxy-agent": "^4.0.1",
-		"http2-wrapper": "^2.0.0",
-		"https-proxy-agent": "^5.0.0",
-		"ow": "^0.23.0"
-	},
-	"devDependencies": {
-		"@apify/eslint-config": "^0.1.3",
-		"body-parser": "^1.19.0",
-		"eslint": "^7.0.0",
-		"express": "^4.17.1",
-		"fs-extra": "^9.1.0",
-		"jest": "^26.6.3",
-		"jest-extended": "^0.11.5",
-		"jsdoc-to-markdown": "^7.0.0",
-		"markdown-toc": "^1.2.0"
-	},
-	"scripts": {
-		"build-docs": "npm run build-toc",
-		"build-toc": "markdown-toc ./README.md -i",
-		"lint": "./node_modules/.bin/eslint ./src --ext .js,.jsx",
-		"lint:fix": "./node_modules/.bin/eslint ./src --ext .js,.jsx --fix",
-		"test": "jest --maxWorkers=3 --collect-coverage"
-	},
-	"author": {
-		"name": "Apify",
-		"email": "[e-mail]",
-		"url": "https://apify.com"
-	},
-	"license": "Apache-2.0",
-	"repository": {
-		"type": "git",
-		"url": "git+https://github.com/apify/got-scrapings"
-	},
-	"bugs": {
-		"url": "https://github.com/apify/got-scraping/issues"
-	}
+  "name": "got-scraping",
+  "version": "1.0.1-beta.0",
+  "description": "HTTP client made for scraping based on got.",
+  "main": "src/index.js",
+  "files": [
+    "src"
+  ],
+  "dependencies": {
+    "header-generator": "^0.0.1-beta.1",
+    "got": "^11.8.1",
+    "http-proxy-agent": "^4.0.1",
+    "http2-wrapper": "^2.0.0",
+    "https-proxy-agent": "^5.0.0",
+    "ow": "^0.23.0"
+  },
+  "devDependencies": {
+    "@apify/eslint-config": "^0.1.3",
+    "body-parser": "^1.19.0",
+    "eslint": "^7.0.0",
+    "express": "^4.17.1",
+    "fs-extra": "^9.1.0",
+    "jest": "^26.6.3",
+    "jest-extended": "^0.11.5",
+    "jsdoc-to-markdown": "^7.0.0",
+    "markdown-toc": "^1.2.0"
+  },
+  "scripts": {
+    "build-docs": "npm run build-toc",
+    "build-toc": "markdown-toc ./README.md -i",
+    "lint": "./node_modules/.bin/eslint ./src --ext .js,.jsx",
+    "lint:fix": "./node_modules/.bin/eslint ./src --ext .js,.jsx --fix",
+    "test": "jest --maxWorkers=3 --collect-coverage"
+  },
+  "author": {
+    "name": "Apify",
+    "email": "[e-mail]",
+    "url": "https://apify.com"
+  },
+  "license": "Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/apify/got-scrapings"
+  },
+  "bugs": {
+    "url": "https://github.com/apify/got-scraping/issues"
+  }
 }

README.md

@@ -112,3 +112,3 @@ # Got Scraping
         devices: ['desktop'],
-        languages: ['de-DE', 'en-US'],
+        locales: ['de-DE', 'en-US'],
         operatingSystems: ['windows', 'linux'] 
@@ -149,3 +149,4 @@ }
     // Node js uses different TLS ciphers by default.
-    ciphers: getCiphersBasedOnNode(),
+    // This is very useful at fighting protection, but can on some websites cause TLS errors.
+    ciphers: "TLS_AES_256_GCM_SHA384:TLS_AES_128_GCM_SHA256:TLS_CHACHA20_POLY1305_SHA256",
     // We need to have browser-like headers to blend in.
@@ -170,2 +171,2 @@ useHeaderGenerator: true,
 
-`RequestError: Client network socket disconnected before secure TLS connection was established` - Try changing the ciphers parameter to either `undefined` or a custom value.
+`RequestError: Client network socket disconnected before secure TLS connection was established` - Try changing the ciphers parameter to either `undefined` or a custom value.

src/handlers/custom-options.js

@@ -0,1 +1,3 @@
+const got = require('got');
+
 /**
@@ -12,6 +14,6 @@ * @param {object} options
         context,
-        ...gotOptions } = options;
+    } = options;
 
     // Got expects custom properties inside the context option.
-    gotOptions.context = {
+    const newContext = {
         ...context,
@@ -23,3 +25,9 @@ proxyUrl,
 
-    return next(gotOptions);
+    delete options.proxyUrl;
+    delete options.headerGeneratorOptions;
+    delete options.useHeaderGenerator;
+
+    const finalOptions = got.mergeOptions(options, { context: newContext });
+
+    return next(finalOptions);
 }
@@ -26,0 +34,0 @@

src/scraping-defaults.js

@@ -31,3 +31,3 @@ const SCRAPING_DEFAULT_OPTIONS = {
     }
-    return 'TLS_AES_256_GCM_SHA384';
+    return 'TLS_AES_256_GCM_SHA384:TLS_AES_128_GCM_SHA256:TLS_CHACHA20_POLY1305_SHA256';
 }
@@ -34,0 +34,0 @@

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

No v1

Quality

Package is not semver >=1. This means it is not stable and does not support ^ ranges.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

22412

increased by2.61%

Lines of code

356

increased by1.42%

Number of lines in readme file

170

increased by1.19%

Worsened metrics

Number of low quality alerts

2

increased by100%

No dependency changes