aimer - npm Package Compare versions
Comparing version
26
index.js
| 'use strict' | ||
| const cheerio = require('cheerio') | ||
| const got = require('got') | ||
| const HttpsProxyAgent = require('https-proxy-agent') | ||
| const fetch = require('node-fetch') | ||
@@ -12,6 +11,2 @@ module.exports = function (url, opts) { | ||
| opts = opts || {} | ||
| const proxy = opts.proxy === undefined ? | ||
| process.env.http_proxy : | ||
| opts.proxy | ||
| const cheerioOpts = Object.assign({ | ||
@@ -21,13 +16,10 @@ // keep the original unicode chars | ||
| }, opts.cheerio) | ||
| const gotOptions = Object.assign({ | ||
| agent: proxy && (new HttpsProxyAgent(proxy)) | ||
| }, opts.got) | ||
| const ret = got.get(url, gotOptions).then(data => { | ||
| if (data.statusCode !== 200) { | ||
| return Promise.reject(new Error(data.statusText)) | ||
| } | ||
| return data.body | ||
| }) | ||
| // normal mode, faster | ||
| const ret = fetch(url) | ||
| .then(data => { | ||
| if (data.status !== 200) { | ||
| return Promise.reject(new Error(data.statusText)) | ||
| } | ||
| return data.text() | ||
| }) | ||
| if (opts.htmlOnly) { | ||
@@ -34,0 +26,0 @@ return ret |
| { | ||
| "name": "aimer", | ||
| "version": "1.2.0", | ||
| "version": "1.3.0", | ||
| "description": "Remote web content crawler done right.", | ||
@@ -27,8 +27,6 @@ "license": "MIT", | ||
| "cheerio": "^0.20.0", | ||
| "got": "^6.3.0", | ||
| "https-proxy-agent": "^1.0.0" | ||
| "node-fetch": "^1.5.1" | ||
| }, | ||
| "devDependencies": { | ||
| "ava": "^0.14.0", | ||
| "is-ci": "^1.0.9", | ||
| "xo": "^0.15.0" | ||
@@ -35,0 +33,0 @@ }, |
@@ -41,12 +41,4 @@ # aimer [](https://npmjs.com/package/aimer) [](https://npmjs.com/package/aimer) [](https://circleci.com/gh/egoist/aimer/tree/master) | ||
| ##### got | ||
| [got](https://github.com/sindresorhus/got) options, this module is used for retriving web contents. | ||
| ##### proxy | ||
| An http or https proxy address, eg: `http://localhost:8787` for Lantern. If `process.env.http_proxy` is set and `opts.proxy` is not set we use the former. Set `opts.proxy` to `false` to disable proxy anyway. | ||
| ## License | ||
| MIT © [EGOIST](https://github.com/egoist) |
New alerts
Network access
Supply chain riskThis module accesses the network.
Found 1 instance in 1 package
Fixed alerts
Network access
Supply chain riskThis module accesses the network.
Found 1 instance in 1 package
Environment variable access
Supply chain riskPackage accesses environment variables, which may be a sign of credential stuffing or data theft.
Found 1 instance in 1 package
Improved metrics
- Dependency count
2
-33.33%- Dev dependency count
2
-33.33%- Number of low supply chain risk alerts
1
-50%Worsened metrics
- Total package byte prevSize
3594
-14.22%- Lines of code
25
-16.67%- Number of lines in readme file
44
-15.38%Dependency changes
+ Added
node-fetch@^1.5.1+ Added
encoding@0.1.13(transitive)+ Added
iconv-lite@0.6.3(transitive)+ Added
node-fetch@1.7.3(transitive)- Removed
got@^6.3.0- Removed
https-proxy-agent@^1.0.0- Removed
agent-base@2.1.1(transitive)- Removed
capture-stack-trace@1.0.2(transitive)- Removed
create-error-class@3.0.2(transitive)- Removed
debug@2.6.9(transitive)- Removed
duplexer3@0.1.5(transitive)- Removed
get-stream@3.0.0(transitive)- Removed
got@6.7.1(transitive)- Removed
https-proxy-agent@1.0.0(transitive)- Removed
is-redirect@1.0.0(transitive)- Removed
is-retry-allowed@1.2.0(transitive)- Removed
lowercase-keys@1.0.1(transitive)- Removed
ms@2.0.0(transitive)- Removed
prepend-http@1.0.4(transitive)- Removed
semver@5.0.3(transitive)- Removed
timed-out@4.0.1(transitive)- Removed
unzip-response@2.0.1(transitive)- Removed
url-parse-lax@1.0.0(transitive)