lastcall-nightcrawler
Advanced tools
Comparing version 1.2.0 to 2.0.0-beta1
@@ -7,2 +7,8 @@ # Changelog | ||
## Unreleased | ||
### Breaking | ||
- HTTP authentication has changed due to the default request driver changing. Now, follow the specifications of the Node http.request method when you construct the Driver. See https://nodejs.org/api/http.html#http_http_request_url_options_callback | ||
- Some of the return values for the crawler methods have changed. See the example configuration for more information on how to write valid 2.x configuration. | ||
## [1.2.0] - 2020-02-18 | ||
@@ -9,0 +15,0 @@ ### Fixed |
@@ -1,145 +0,59 @@ | ||
'use strict'; | ||
"use strict"; | ||
var _ora = require('ora'); | ||
Object.defineProperty(exports, "__esModule", { | ||
value: true | ||
}); | ||
exports.default = _default; | ||
var _ora2 = _interopRequireDefault(_ora); | ||
var _errors = require("../errors"); | ||
var _os = require('os'); | ||
var _ConsoleReporter = _interopRequireDefault(require("../formatters/ConsoleReporter")); | ||
var _fs = require('fs'); | ||
var _JUnitReporter = _interopRequireDefault(require("../formatters/JUnitReporter")); | ||
var _fs2 = _interopRequireDefault(_fs); | ||
var _JSONReporter = _interopRequireDefault(require("../formatters/JSONReporter")); | ||
var _errors = require('../errors'); | ||
var _util = require("../util"); | ||
var _console = require('../formatters/console'); | ||
var _console2 = _interopRequireDefault(_console); | ||
var _junit = require('../formatters/junit'); | ||
var _junit2 = _interopRequireDefault(_junit); | ||
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } | ||
exports.command = 'crawl [crawlerfile]'; | ||
exports.describe = 'execute the crawl defined in the active config file'; | ||
exports.builder = yargs => { | ||
yargs.option('concurrency', { | ||
alias: 'c', | ||
describe: 'number of requests allowed in-flight at once', | ||
type: 'number', | ||
required: true, | ||
default: 3 | ||
}); | ||
yargs.option('silent', { | ||
alias: 'n', | ||
describe: 'silence all output', | ||
type: 'boolean', | ||
default: false | ||
}); | ||
yargs.option('json', { | ||
alias: 'j', | ||
describe: 'filename to write JSON report to', | ||
normalize: true, | ||
type: 'string', | ||
default: '' | ||
}); | ||
yargs.option('junit', { | ||
alias: 'u', | ||
describe: 'filename to write JUnit report to', | ||
normalize: true, | ||
type: 'string', | ||
default: '' | ||
}); | ||
}; | ||
exports.handler = async function (argv, crawler) { | ||
const { | ||
json = '', | ||
junit = '', | ||
concurrency = 3, | ||
stdout = process.stdout | ||
} = argv; | ||
const spunCrawler = new CrawlerSpinnerDecorator(crawler, stdout); | ||
function getReporters(argv, stdout) { | ||
var _argv$junit, _argv$json; | ||
await spunCrawler.setup(); | ||
const reporters = []; | ||
const data = await spunCrawler.work(concurrency); | ||
const analysis = await spunCrawler.analyze(data); | ||
stdout.write((0, _console2.default)(analysis, { color: true, minLevel: 1 }) + _os.EOL); | ||
if (json.length) { | ||
_fs2.default.writeFileSync(json, JSON.stringify(data), 'utf8'); | ||
if (!argv.silent) { | ||
reporters.push(new _ConsoleReporter.default(stdout)); | ||
} | ||
if (junit.length) { | ||
(0, _junit2.default)(analysis, { filename: junit }); | ||
if ((_argv$junit = argv.junit) === null || _argv$junit === void 0 ? void 0 : _argv$junit.length) { | ||
reporters.push(new _JUnitReporter.default(argv.junit)); | ||
} | ||
if (analysis.hasFailures()) { | ||
throw new _errors.FailedAnalysisError('Analysis reported an error'); | ||
if ((_argv$json = argv.json) === null || _argv$json === void 0 ? void 0 : _argv$json.length) { | ||
reporters.push(new _JSONReporter.default(argv.json)); | ||
} | ||
}; | ||
class CrawlerSpinnerDecorator { | ||
constructor(inner, stream) { | ||
this.inner = inner; | ||
this.stream = stream; | ||
return reporters; | ||
} | ||
async function _default(argv, stdout) { | ||
const { | ||
context, | ||
concurrency = 5 | ||
} = argv; | ||
const reporters = getReporters(argv, stdout); | ||
let hasAnyFailure = false; | ||
await Promise.all(reporters.map(reporter => reporter.start())); | ||
for await (const [url, result] of context.crawl(concurrency)) { | ||
reporters.forEach(r => r.report(url, result)); | ||
hasAnyFailure = hasAnyFailure || (0, _util.hasFailure)(result); | ||
} | ||
setup() { | ||
let spinner = (0, _ora2.default)({ | ||
stream: this.stream | ||
}).start('Setup'); | ||
// Handle non-TTY by giving some output | ||
if (!spinner.enabled) { | ||
this.stream.write(`Setup${_os.EOL}`); | ||
} | ||
return this.inner.setup().then(res => { | ||
spinner.succeed('Setup'); | ||
return res; | ||
}).catch(res => { | ||
spinner.fail('Setup'); | ||
return Promise.reject(res); | ||
}); | ||
} | ||
work(concurrency) { | ||
let spinner = (0, _ora2.default)({ | ||
stream: this.stream | ||
}).start('Crawl'); | ||
// Handle non-TTY by giving some output | ||
if (!spinner.enabled) { | ||
this.stream.write(`Crawl${_os.EOL}`); | ||
} | ||
let done = 0; | ||
let tick = () => { | ||
spinner.text = `Crawled ${++done} of ${this.inner.queue.length}`; | ||
}; | ||
this.inner.on('response.success', tick).on('response.error', tick); | ||
return this.inner.work(concurrency).then(res => { | ||
spinner.succeed('Crawl'); | ||
return res; | ||
}).catch(res => { | ||
spinner.fail('Crawl'); | ||
return Promise.reject(res); | ||
}); | ||
await Promise.all(reporters.map(r => r.stop())); | ||
if (hasAnyFailure) { | ||
throw new _errors.FailedAnalysisError('Testing reported an error.'); | ||
} | ||
analyze(data) { | ||
let spinner = (0, _ora2.default)({ | ||
stream: this.stream | ||
}).start('Analyze'); | ||
// Handle non-TTY by giving some output | ||
if (!spinner.enabled) { | ||
this.stream.write(`Analyze${_os.EOL}`); | ||
} | ||
return this.inner.analyze(data).then(res => { | ||
spinner.succeed('Analyze'); | ||
return res; | ||
}).catch(res => { | ||
spinner.fail('Analyze'); | ||
return Promise.reject(res); | ||
}); | ||
} | ||
} |
@@ -6,3 +6,6 @@ "use strict"; | ||
}); | ||
exports.FailedAnalysisError = void 0; | ||
class FailedAnalysisError extends Error {} | ||
exports.FailedAnalysisError = FailedAnalysisError; |
@@ -1,36 +0,51 @@ | ||
'use strict'; | ||
"use strict"; | ||
var _yargs = require('yargs'); | ||
Object.defineProperty(exports, "__esModule", { | ||
value: true | ||
}); | ||
exports.default = _default; | ||
var _yargs2 = _interopRequireDefault(_yargs); | ||
var _minimist = _interopRequireDefault(require("minimist")); | ||
var _util = require('./util'); | ||
var _crawl = _interopRequireDefault(require("./commands/crawl")); | ||
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } | ||
var _version = _interopRequireDefault(require("./commands/version")); | ||
_yargs2.default.option('config').default('config', './nightcrawler.js'); | ||
var _help = _interopRequireDefault(require("./commands/help")); | ||
let crawler; | ||
var _util = require("./util"); | ||
try { | ||
crawler = (0, _util.requireCrawler)(_yargs2.default.argv.config); | ||
} catch (e) { | ||
throw new Error(`Unable to load crawler from ${_yargs2.default.argv.config}`); | ||
} | ||
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } | ||
function visitCommand(command) { | ||
const oldHandler = command.handler; | ||
function massageConcurrency(concurrency, defaultValue) { | ||
if (concurrency === undefined || concurrency === '') { | ||
return defaultValue; | ||
} | ||
// Inject the crawler into the command. | ||
command.handler = function (argv) { | ||
return oldHandler(argv, crawler); | ||
}; | ||
return command; | ||
return typeof concurrency === 'string' ? parseInt(concurrency) : concurrency; | ||
} | ||
// I need environment, sample size, auth. | ||
async function _default(input, stdout, cwd) { | ||
const argv = (0, _minimist.default)(input, { | ||
string: ['config', 'json', 'concurrency', 'junit'], | ||
boolean: ['silent', 'help', 'version'], | ||
default: { | ||
config: './nightcrawler.js', | ||
concurrency: 5, | ||
silent: false, | ||
help: false, | ||
version: false | ||
} | ||
}); | ||
_yargs2.default.commandDir('commands', { | ||
visit: visitCommand | ||
}).demandCommand(1, '').help().argv; | ||
if (argv.help) { | ||
return (0, _help.default)(stdout); | ||
} else if (argv.version) { | ||
return (0, _version.default)(stdout); | ||
} else { | ||
return (0, _crawl.default)({ ...argv, | ||
concurrency: massageConcurrency(argv.concurrency, 5), | ||
context: (0, _util.loadContext)(argv.config, cwd) | ||
}, stdout); | ||
} | ||
} |
@@ -1,2 +0,2 @@ | ||
'use strict'; | ||
"use strict"; | ||
@@ -6,50 +6,37 @@ Object.defineProperty(exports, "__esModule", { | ||
}); | ||
exports.requireCrawler = requireCrawler; | ||
exports.consoleDisplayValue = consoleDisplayValue; | ||
exports.stringLength = stringLength; | ||
exports.makeResult = makeResult; | ||
exports.hasFailure = hasFailure; | ||
exports.loadContext = loadContext; | ||
var _path = require('path'); | ||
var _TestContext = _interopRequireDefault(require("../testing/TestContext")); | ||
var _path2 = _interopRequireDefault(_path); | ||
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } | ||
var _fs = require('fs'); | ||
function makeResult(obj) { | ||
return new Map(Object.entries(obj)); | ||
} | ||
var _fs2 = _interopRequireDefault(_fs); | ||
function hasFailure(result) { | ||
return Array.from(result.values()).some(r => !r.pass); | ||
} | ||
var _chalk = require('chalk'); | ||
function loadContext(configFile, cwd) { | ||
let context; | ||
var _chalk2 = _interopRequireDefault(_chalk); | ||
try { | ||
const resolved = require.resolve(configFile, { | ||
paths: [cwd] | ||
}); // eslint-disable-next-line @typescript-eslint/no-var-requires | ||
var _crawler = require('../crawler'); | ||
var _crawler2 = _interopRequireDefault(_crawler); | ||
context = require(resolved); | ||
} catch (e) { | ||
throw new Error(`Unable to find configuration file at ${configFile}.`); | ||
} | ||
var _stripAnsi = require('strip-ansi'); | ||
var _stripAnsi2 = _interopRequireDefault(_stripAnsi); | ||
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } | ||
function requireCrawler(file) { | ||
if (typeof file === 'string') { | ||
var resolved = _path2.default.resolve(process.cwd(), file); | ||
// $FlowFixMe | ||
return require(resolved); | ||
if (context instanceof _TestContext.default) { | ||
return context; | ||
} | ||
// Allow full crawler instances to be passed in during testing. | ||
return file; | ||
} | ||
function consoleDisplayValue(level, value) { | ||
switch (level) { | ||
case 2: | ||
return _chalk2.default.red(value); | ||
case 1: | ||
return _chalk2.default.yellow(value); | ||
default: | ||
return value; | ||
} | ||
} | ||
function stringLength(data) { | ||
return (0, _stripAnsi2.default)(data).length; | ||
throw new Error(`The configuration file at ${configFile} does not export a valid test context.`); | ||
} |
@@ -1,25 +0,31 @@ | ||
'use strict'; | ||
"use strict"; | ||
var _crawler = require('./crawler'); | ||
Object.defineProperty(exports, "__esModule", { | ||
value: true | ||
}); | ||
var _exportNames = { | ||
Crawler: true | ||
}; | ||
Object.defineProperty(exports, "Crawler", { | ||
enumerable: true, | ||
get: function () { | ||
return _Crawler.default; | ||
} | ||
}); | ||
var _crawler2 = _interopRequireDefault(_crawler); | ||
var _Crawler = _interopRequireDefault(require("./Crawler")); | ||
var _request = require('./driver/request'); | ||
var _functions = require("./testing/functions"); | ||
var _request2 = _interopRequireDefault(_request); | ||
Object.keys(_functions).forEach(function (key) { | ||
if (key === "default" || key === "__esModule") return; | ||
if (Object.prototype.hasOwnProperty.call(_exportNames, key)) return; | ||
Object.defineProperty(exports, key, { | ||
enumerable: true, | ||
get: function () { | ||
return _functions[key]; | ||
} | ||
}); | ||
}); | ||
var _metrics = require('./metrics'); | ||
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } | ||
_crawler2.default.drivers = { | ||
request: _request2.default | ||
}; | ||
_crawler2.default.metrics = { | ||
Number: _metrics.Number, | ||
Milliseconds: _metrics.Milliseconds, | ||
Percent: _metrics.Percent | ||
}; | ||
module.exports = _crawler2.default; | ||
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } |
{ | ||
"name": "lastcall-nightcrawler", | ||
"version": "1.2.0", | ||
"version": "2.0.0-beta1", | ||
"main": "dist/index.js", | ||
"license": "MIT", | ||
"dependencies": { | ||
"bluebird": "^3.5.1", | ||
"chalk": "^2.3.0", | ||
"debug": "^3.1.0", | ||
"events-async": "^1.2.1", | ||
"forever-agent": "^0.6.1", | ||
"junit-report-builder": "^1.2.0", | ||
"markdown-table": "^1.1.1", | ||
"ora": "^2.0.0", | ||
"request": "^2.83.0", | ||
"request-promise": "^4.2.2", | ||
"yargs": "^12.0.1" | ||
"chalk": "^2.0.0", | ||
"debug": "^3.0.0", | ||
"indent-string": "^4.0.0", | ||
"minimist": "^1.2.5", | ||
"strip-ansi": "^6.0.0", | ||
"wrap-ansi": "^6.0.0", | ||
"xml": "^1.0.0" | ||
}, | ||
@@ -23,22 +19,33 @@ "bin": { | ||
"devDependencies": { | ||
"babel-cli": "^6.26.0", | ||
"babel-eslint": "^8.2.5", | ||
"babel-preset-env": "^1.6.1", | ||
"babel-preset-flow": "^6.23.0", | ||
"@babel/cli": "^7.8.4", | ||
"@babel/core": "^7.8.4", | ||
"@babel/plugin-proposal-class-properties": "^7.8.3", | ||
"@babel/plugin-proposal-object-rest-spread": "^7.8.3", | ||
"@babel/preset-env": "^7.8.4", | ||
"@babel/preset-typescript": "^7.8.3", | ||
"@types/debug": "^4.1.5", | ||
"@types/jest": "^25.1.2", | ||
"@types/minimist": "^1.2.0", | ||
"@types/node": "^13.7.2", | ||
"@types/tmp": "^0.1.0", | ||
"@types/wrap-ansi": "^3.0.0", | ||
"@types/xml": "^1.0.4", | ||
"@typescript-eslint/eslint-plugin": "^2.21.0", | ||
"@typescript-eslint/parser": "^2.21.0", | ||
"chai": "^4.1.2", | ||
"eslint": "^4.17.0", | ||
"eslint-config-prettier": "^2.9.0", | ||
"eslint-plugin-flowtype": "^2.50.0", | ||
"flow": "^0.2.3", | ||
"flow-bin": "^0.76.0", | ||
"jest": "^23.4.0", | ||
"jest-junit": "^5.1.0", | ||
"nock": "^9.1.6", | ||
"prettier": "^1.10.2" | ||
"eslint": "^6.8.0", | ||
"jest": "^25.1.0", | ||
"jest-junit": "^10.0.0", | ||
"nock": "^12.0.1", | ||
"prettier": "^1.19.1", | ||
"tmp-promise": "^2.0.2", | ||
"ts-jest": "^25.2.1", | ||
"typescript": "^3.8.3" | ||
}, | ||
"scripts": { | ||
"prettier": "prettier --single-quote --write '{src,test}/**/*.{js,jsx}' '*.js'", | ||
"prettier": "prettier --single-quote --write './src/**/*.ts'", | ||
"test": "jest", | ||
"flow": "flow", | ||
"build": "babel src/ --out-dir=dist" | ||
"check-types": "tsc --noEmit", | ||
"build": "babel src/ --out-dir=dist --extensions '.ts' --ignore 'src/**/__tests__/**' --ignore 'src/**/__mocks__/**' --ignore 'src/**/__stubs__/**' && tsc --emitDeclarationOnly", | ||
"lint": "eslint ./src/ --ext .js,.jsx,.ts,.tsx" | ||
}, | ||
@@ -49,28 +56,2 @@ "files": [ | ||
], | ||
"babel": { | ||
"presets": [ | ||
[ | ||
"env", | ||
{ | ||
"targets": { | ||
"node": "8" | ||
} | ||
} | ||
], | ||
"flow" | ||
] | ||
}, | ||
"eslintConfig": { | ||
"extends": [ | ||
"prettier", | ||
"plugin:flowtype/recommended" | ||
], | ||
"plugins": [ | ||
"flowtype" | ||
], | ||
"parserOptions": { | ||
"ecmaVersion": 2017, | ||
"sourceType": "module" | ||
} | ||
}, | ||
"jest": { | ||
@@ -77,0 +58,0 @@ "collectCoverage": true, |
177
README.md
@@ -15,20 +15,17 @@ ![Nightcrawler](docs/logo.png) | ||
# nightcrawler.js | ||
const Crawler = require('lastcall-nightcrawler'); | ||
const Number = Crawler.metrics.Number; | ||
const {crawl, test} = require('./dist'); | ||
const expect = require('expect'); | ||
const myCrawler = new Crawler('My Crawler'); | ||
module.exports = crawl('Response code validation', function() { | ||
test('Should return 2xx', function(unit) { | ||
expect(unit.response.statusCode).toBeGreaterThanOrEqual(200); | ||
expect(unit.response.statusCode).toBeLessThan(300); | ||
}); | ||
myCrawler.on('setup', function(crawler) { | ||
// On setup, give the crawler a list of URLs to crawl. | ||
crawler.enqueue('http://localhost/'); | ||
crawler.enqueue('http://localhost/foo'); | ||
return [ | ||
{url: 'https://example.com'}, | ||
{url: 'https://example.com?q=1'}, | ||
{url: 'https://example.com?q=2'} | ||
]; | ||
}); | ||
myCrawler.on('analyze', function(crawlReport, analysis) { | ||
// On analysis, derive the metrics you need from the | ||
// array of collected data. | ||
analysis.addMetric('count', new Number('Total Requests', 0, crawlReport.data.length)); | ||
}); | ||
module.exports = myCrawler; | ||
``` | ||
@@ -41,112 +38,88 @@ Run your crawler: | ||
Queueing Requests | ||
----------------- | ||
Requests can be queued during the `setup` event. You can queue a new request by calling the `enqueue()` method, using either a string (representing the URL) or an object containing a `url` property. If you pass an object, you will have access to that object's properties later on during analysis. | ||
Specifying what URLs to crawl | ||
----------------------------- | ||
The `crawl` function expects a return value of an iterable (or async iterable) containing "requests". The simplest version of this is just an array of objects that have a `url` property. Eg: | ||
```js | ||
myCrawler.on('setup', function(crawler) { | ||
// This works | ||
crawler.enqueue('http://localhost/'); | ||
// So does this: | ||
crawler.enqueue({ | ||
url: 'http://localhost/foo', | ||
group: 'awesome' | ||
}); | ||
module.exports = crawl('Crawl a static list of URLs', function() { | ||
return [ | ||
{url: 'https://example.com'} | ||
] | ||
}); | ||
``` | ||
myCrawler.on('analyze', function(crawlReport, analysis) { | ||
var awesomeRequests = crawlReport.data.filter(function(point) { | ||
// *group property is only available if you added it during queuing. | ||
return point.group === 'awesome'; | ||
}); | ||
// Do additional analysis only on pages in the awesome group. | ||
analysis.addMetric('awesome.count', new Number('Awesome Requests', 0, awesomeRequests.length)); | ||
For more advanced use cases, you may want to use async generators to fetch a list of URLs from somewhere else (eg: a database). Eg: | ||
```js | ||
async function* getURLs() { | ||
const result = await queryDB(); | ||
for(const url of result) { | ||
yield {url: url}; | ||
} | ||
} | ||
module.exports = crawl('Crawl a dynamic list of URLs', function() { | ||
return getURLs(); | ||
}) | ||
``` | ||
Collecting data | ||
--------------- | ||
By default, only the following information is collected for each response: | ||
* `url` (string) : The URL that was crawled. | ||
* `error` (bool) : Whether the response was determined to be an error response. | ||
* `status` (int): The HTTP status code received. | ||
* `backendResponseTime` (int): The duration of HTTP server response (see the [request module's documentation](https://github.com/request/request) on `timingPhases.firstByte`). | ||
Performing assertions on responses | ||
---------------------------------- | ||
If there is other data you're interested in knowing, you can collect it like this: | ||
One of the primary goals of Nightcrawler is to detect URLs that don't meet your expectations. To achieve this, you can use the `test` function within a `crawl` to make assertions about the response received. | ||
```js | ||
// Collect the `Expires` header for each request. | ||
myCrawler.on('response', function(response, data) { | ||
data.expires = response.headers['expires']; | ||
const {crawl, test} = require('./dist'); | ||
// Use the expect module from NPM for assertions. | ||
// You can use any assertion library, including the built-in assert module. | ||
const expect = require('expect'); | ||
module.exports = crawl('Check that the homepage is cacheable', function() { | ||
test('Should have cache-control header', function(unit) { | ||
expect(unit.response.headers).toHaveProperty('cache-control'); | ||
expect(unit.response.headers['cache-control']).toBe('public; max-age: 1800'); | ||
}) | ||
return [{url: 'https://example.com/'}] | ||
}); | ||
``` | ||
The response event is triggered on request success or error, as long as the server sends a response. Anything put into the `data` object will end up in the final JSON report. | ||
The `test` function will receive a `unit` of crawler work, which includes the following properties: | ||
Dynamic Crawling | ||
---------------- | ||
You may wish to be able to crawl a list of URLs that isn't static (it's determined at runtime). For example, you may want to query a remote API or a database and enqueue a list of URLs based on that data. To support this, the `setup` event allows you to return a promise. | ||
* `request`: The request, as you passed it into the Crawler. This will include any additional properties you passed in, and you can use those properties to do conditional checking of units of work. | ||
* `response`: The response object, as returned by the Driver. The default `NativeDriver` will produce a response in the shape of a Node [`http.IncomingMessage`](https://nodejs.org/api/http.html#http_class_http_incomingmessage) object. All `response` objects are guaranteed to have both a `statusCode` and a `time` property. | ||
Performing assertions about the overall status of the crawl | ||
----------------------------------------------------------- | ||
For some use cases, you will want make assertions about many requests. For example, checking the average response type of all requests. To do this, you may use the `after` function to perform assertions after all the URLs have been requested. Just use the `test` function to collect the data you need from each request, then perform the final assertion in `after`: | ||
```js | ||
// Fetch a list of URLs from a remote API, then enqueue them all. | ||
myCrawler.on('setup', function(crawler) { | ||
return fetchData().then(function(myData) { | ||
myData.forEach(function(url) { | ||
crawler.enqueue(url); | ||
}) | ||
}) | ||
}) | ||
``` | ||
const {crawl, test, after} = require('./dist'); | ||
const expect = require('expect'); | ||
Analysis | ||
-------- | ||
Once the crawl has been completed, you will probably want to analyze the data in some way. Data analysis in Nightcrawler is intentionally loose - the crawler fires an `analyze` event with an array of collected data, and you are responsible for analyzing your own data. Here are some examples of things you might do during analysis: | ||
```js | ||
const Crawler = require('lastcall-nightcrawler'); | ||
const Number = Crawler.metrics.Number; | ||
const Milliseconds = Crawler.metrics.Milliseconds; | ||
const Percent = Crawler.metrics.Percent; | ||
module.exports = crawl('Check that pages load quickly', function() { | ||
const times = []; | ||
myCrawler.on('analyze', function(crawlReport, analysis) { | ||
var data = crawlReport.data; | ||
test('Collect response time', function(unit) { | ||
times.push(unit.response.time); | ||
}) | ||
// Calculate the number of requests that were made: | ||
analysis.addMetric('count', new Number('Total Requests', 0, data.length)); | ||
// Calculate the average response time: | ||
var avgTime = data.reduce(function(sum, dataPoint) { | ||
return sum + dataPoint.backendTime | ||
}, 0) / data.length; | ||
analysis.addMetric('time', new Milliseconds('Avg Response Time', 0, avgTime)); | ||
// Calculate the percent of requests that were marked failed: | ||
var failRatio = data.filter(function(dataPoint) { | ||
return dataPoint.fail === true; | ||
}).length / data.length; | ||
var level = failRatio > 0 ? 2 : 0; | ||
analysis.addMetric('fail', new Percent('% Failed', level, failRatio)); | ||
// Calculate the percent of requests that resulted in a 500 response. | ||
var serverErrorRatio = data.filter(function(dataPoint) { | ||
return dataPoint.statusCode >= 500; | ||
}).length / data.length; | ||
var level = serverErrorRatio > 0 ? 2 : 0; | ||
analysis.add('500', new Percent('% 500', level, serverErrorRatio)); | ||
after('Response time should be less than 500ms', function() { | ||
const sum = times.reduce((total, value) => total + value, 0); | ||
expect(sum / times.length).toBeLessThan(500); | ||
}) | ||
return [{url: 'https://example.com/'}] | ||
}); | ||
``` | ||
The [`analysis`](./src/analysis.js) object can consist of many metrics, added through the `add` method. See [`src/metrics.js`](./src/metrics.js) for more information about metrics. | ||
Analysis can also be performed on individual requests to mark them passed or failed. | ||
Drivers | ||
------- | ||
```js | ||
myCrawler.on('analyze', function(crawlReport, analysis) { | ||
var data = crawlReport.data; | ||
Right now, there is only one "Driver" available for making requests. It uses Node's built-in `http` and `https` modules to issue HTTP requests to the target URL. In the future, we may have additional drivers available. | ||
data.forEach(function(request) { | ||
var level = request.statusCode > 499 ? 2 : 0 | ||
analysis.addResult(request.url, level) | ||
}); | ||
}) | ||
``` | ||
CI Setup | ||
@@ -153,0 +126,0 @@ -------- |
Sorry, the diff of this file is not supported yet
Major refactor
Supply chain riskPackage has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.
Found 1 instance in 1 package
Network access
Supply chain riskThis module accesses the network.
Found 2 instances in 1 package
Debug access
Supply chain riskUses debug, reflection and dynamic code execution features.
Found 1 instance in 1 package
Dynamic require
Supply chain riskDynamic require can indicate the package is performing dangerous or unsafe dynamic code execution.
Found 1 instance in 1 package
Filesystem access
Supply chain riskAccesses the file system, and could potentially read sensitive data.
Found 1 instance in 1 package
No v1
QualityPackage is not semver >=1. This means it is not stable and does not support ^ ranges.
Found 1 instance in 1 package
Filesystem access
Supply chain riskAccesses the file system, and could potentially read sensitive data.
Found 1 instance in 1 package
7
42
37319
24
854
2
136
3
+ Addedindent-string@^4.0.0
+ Addedminimist@^1.2.5
+ Addedstrip-ansi@^6.0.0
+ Addedwrap-ansi@^6.0.0
+ Addedxml@^1.0.0
+ Addedansi-regex@5.0.1(transitive)
+ Addedansi-styles@4.3.0(transitive)
+ Addedcolor-convert@2.0.1(transitive)
+ Addedcolor-name@1.1.4(transitive)
+ Addedemoji-regex@8.0.0(transitive)
+ Addedindent-string@4.0.0(transitive)
+ Addedis-fullwidth-code-point@3.0.0(transitive)
+ Addedstring-width@4.2.3(transitive)
+ Addedstrip-ansi@6.0.1(transitive)
+ Addedwrap-ansi@6.2.0(transitive)
+ Addedxml@1.0.1(transitive)
- Removedbluebird@^3.5.1
- Removedevents-async@^1.2.1
- Removedforever-agent@^0.6.1
- Removedjunit-report-builder@^1.2.0
- Removedmarkdown-table@^1.1.1
- Removedora@^2.0.0
- Removedrequest@^2.83.0
- Removedrequest-promise@^4.2.2
- Removedyargs@^12.0.1
- Removedajv@6.12.6(transitive)
- Removedansi-regex@2.1.13.0.1(transitive)
- Removedasn1@0.2.6(transitive)
- Removedassert-plus@1.0.0(transitive)
- Removedasynckit@0.4.0(transitive)
- Removedaws-sign2@0.7.0(transitive)
- Removedaws4@1.13.2(transitive)
- Removedbcrypt-pbkdf@1.0.2(transitive)
- Removedbluebird@3.7.2(transitive)
- Removedcamelcase@5.3.1(transitive)
- Removedcaseless@0.12.0(transitive)
- Removedcli-cursor@2.1.0(transitive)
- Removedcli-spinners@1.3.1(transitive)
- Removedcliui@4.1.0(transitive)
- Removedclone@1.0.4(transitive)
- Removedcode-point-at@1.1.0(transitive)
- Removedcombined-stream@1.0.8(transitive)
- Removedcore-util-is@1.0.2(transitive)
- Removedcross-spawn@6.0.6(transitive)
- Removeddashdash@1.14.1(transitive)
- Removeddate-format@0.0.2(transitive)
- Removeddecamelize@1.2.0(transitive)
- Removeddefaults@1.0.4(transitive)
- Removeddelayed-stream@1.0.0(transitive)
- Removedecc-jsbn@0.1.2(transitive)
- Removedend-of-stream@1.4.4(transitive)
- Removedevents-async@1.2.1(transitive)
- Removedexeca@1.0.0(transitive)
- Removedextend@3.0.2(transitive)
- Removedextsprintf@1.3.0(transitive)
- Removedfast-deep-equal@3.1.3(transitive)
- Removedfast-json-stable-stringify@2.1.0(transitive)
- Removedfind-up@3.0.0(transitive)
- Removedforever-agent@0.6.1(transitive)
- Removedform-data@2.3.3(transitive)
- Removedget-caller-file@1.0.3(transitive)
- Removedget-stream@4.1.0(transitive)
- Removedgetpass@0.1.7(transitive)
- Removedhar-schema@2.0.0(transitive)
- Removedhar-validator@5.1.5(transitive)
- Removedhttp-signature@1.2.0(transitive)
- Removedinvert-kv@2.0.0(transitive)
- Removedis-fullwidth-code-point@1.0.02.0.0(transitive)
- Removedis-stream@1.1.0(transitive)
- Removedis-typedarray@1.0.0(transitive)
- Removedisexe@2.0.0(transitive)
- Removedisstream@0.1.2(transitive)
- Removedjsbn@0.1.1(transitive)
- Removedjson-schema@0.4.0(transitive)
- Removedjson-schema-traverse@0.4.1(transitive)
- Removedjson-stringify-safe@5.0.1(transitive)
- Removedjsprim@1.4.2(transitive)
- Removedjunit-report-builder@1.3.3(transitive)
- Removedlcid@2.0.0(transitive)
- Removedlocate-path@3.0.0(transitive)
- Removedlodash@4.17.21(transitive)
- Removedlog-symbols@2.2.0(transitive)
- Removedmap-age-cleaner@0.1.3(transitive)
- Removedmarkdown-table@1.1.3(transitive)
- Removedmem@4.3.0(transitive)
- Removedmime-db@1.52.0(transitive)
- Removedmime-types@2.1.35(transitive)
- Removedmimic-fn@1.2.02.1.0(transitive)
- Removedmkdirp@0.5.6(transitive)
- Removednice-try@1.0.5(transitive)
- Removednpm-run-path@2.0.2(transitive)
- Removednumber-is-nan@1.0.1(transitive)
- Removedoauth-sign@0.9.0(transitive)
- Removedonce@1.4.0(transitive)
- Removedonetime@2.0.1(transitive)
- Removedora@2.1.0(transitive)
- Removedos-locale@3.1.0(transitive)
- Removedp-defer@1.0.0(transitive)
- Removedp-finally@1.0.0(transitive)
- Removedp-is-promise@2.1.0(transitive)
- Removedp-limit@2.3.0(transitive)
- Removedp-locate@3.0.0(transitive)
- Removedp-try@2.2.0(transitive)
- Removedpath-exists@3.0.0(transitive)
- Removedpath-key@2.0.1(transitive)
- Removedperformance-now@2.1.0(transitive)
- Removedpsl@1.15.0(transitive)
- Removedpump@3.0.2(transitive)
- Removedpunycode@2.3.1(transitive)
- Removedqs@6.5.3(transitive)
- Removedrequest@2.88.2(transitive)
- Removedrequest-promise@4.2.6(transitive)
- Removedrequest-promise-core@1.1.4(transitive)
- Removedrequire-directory@2.1.1(transitive)
- Removedrequire-main-filename@1.0.1(transitive)
- Removedrestore-cursor@2.0.0(transitive)
- Removedsafe-buffer@5.2.1(transitive)
- Removedsafer-buffer@2.1.2(transitive)
- Removedsemver@5.7.2(transitive)
- Removedset-blocking@2.0.0(transitive)
- Removedshebang-command@1.2.0(transitive)
- Removedshebang-regex@1.0.0(transitive)
- Removedsignal-exit@3.0.7(transitive)
- Removedsshpk@1.18.0(transitive)
- Removedstealthy-require@1.1.1(transitive)
- Removedstring-width@1.0.22.1.1(transitive)
- Removedstrip-ansi@3.0.14.0.0(transitive)
- Removedstrip-eof@1.0.0(transitive)
- Removedtough-cookie@2.5.0(transitive)
- Removedtunnel-agent@0.6.0(transitive)
- Removedtweetnacl@0.14.5(transitive)
- Removeduri-js@4.4.1(transitive)
- Removeduuid@3.4.0(transitive)
- Removedverror@1.10.0(transitive)
- Removedwcwidth@1.0.1(transitive)
- Removedwhich@1.3.1(transitive)
- Removedwhich-module@2.0.1(transitive)
- Removedwrap-ansi@2.1.0(transitive)
- Removedwrappy@1.0.2(transitive)
- Removedxmlbuilder@10.1.1(transitive)
- Removedy18n@4.0.3(transitive)
- Removedyargs@12.0.5(transitive)
- Removedyargs-parser@11.1.1(transitive)
Updatedchalk@^2.0.0
Updateddebug@^3.0.0