pa11y - npm Package Compare versions

Comparing version 5.0.0-beta.3 to 5.0.0-beta.4

bin/pa11y.js

@@ -28,14 +28,11 @@ #!/usr/bin/env node
 			'-r, --reporter <reporter>',
-			'the reporter to use: cli (default), csv, json',
-			'cli'
+			'the reporter to use: cli (default), csv, json'
 		)
 		.option(
 			'-l, --level <level>',
-			'the level of issue to fail on (exit with code 2): error, warning, notice',
-			'error'
+			'the level of issue to fail on (exit with code 2): error, warning, notice'
 		)
 		.option(
 			'-T, --threshold <number>',
-			'permit this number of errors, warnings, or notices, otherwise fail with exit code 2',
-			'0'
+			'permit this number of errors, warnings, or notices, otherwise fail with exit code 2'
 		)
@@ -104,8 +101,12 @@ .option(
 	}
-	const report = loadReporter(program.reporter);
-	const options = processOptions(report.log);
+	const options = processOptions();
+	const report = loadReporter(options.reporter);
+	options.log = report.log;
+	if (!program.debug) {
+		options.log.debug = () => {};
+	}
 	await report.begin(program.url);
 	try {
 		const results = await pa11y(program.url, options);
-		if (reportShouldFail(program.level, results.issues, program.threshold)) {
+		if (reportShouldFail(options.level, results.issues, options.threshold)) {
 			process.once('exit', () => {
@@ -122,4 +123,10 @@ process.exit(2);
 
-function processOptions(log) {
-	const options = extend({}, loadConfig(program.config), {
+function processOptions() {
+	// CLI options take precedence over config options (which take precedence over defaults)
+	// 'level', 'reporter', and 'threshold' defaults are given here as they are not relevant when using lib/pa11y via JavaScript
+	const options = extend({
+		level: 'error',
+		reporter: 'cli',
+		threshold: 0
+	}, loadConfig(program.config), {
 		hideElements: program.hideElements,
@@ -129,2 +136,4 @@ ignore: (program.ignore.length ? program.ignore : undefined),
 		includeWarnings: program.includeWarnings,
+		level: program.level,
+		reporter: program.reporter,
 		rootElement: program.rootElement,
@@ -134,10 +143,6 @@ rules: (program.addRule.length ? program.addRule : undefined),
 		standard: program.standard,
+		threshold: program.threshold,
 		timeout: program.timeout,
-		wait: program.wait,
-		log
+		wait: program.wait
 	});
-
-	if (!program.debug) {
-		options.log.debug = () => {};
-	}
 	return options;
@@ -144,0 +149,0 @@ }

CHANGELOG.md

 
 # Changelog
 
+## 5.0.0-beta.4 pre-release (2017-12-06)
+
+  * Add reporter, threshold and level as configuration options
+  * Clarify some documentation
+
 ## 5.0.0-beta.3 pre-release (2017-11-26)
@@ -5,0 +10,0 @@

lib/pa11y.js

@@ -261,3 +261,4 @@ 'use strict';
 /**
- * Default options.
+ * Default options (excluding 'level', 'reporter', and 'threshold' which are only
+ * relevant when calling bin/pa11y from the CLI)
  * @public
@@ -264,0 +265,0 @@ */

package.json

 {
   "name": "pa11y",
-  "version": "5.0.0-beta.3",
+  "version": "5.0.0-beta.4",
   "description": "Pa11y is your automated accessibility testing pal",
@@ -5,0 +5,0 @@ "keywords": [

README.md

@@ -173,2 +173,4 @@
 
+If any configuration is set both in a configuration file and also as a command-line option, the value set in the latter will take priority.
+
 For more information on configuring Pa11y, see the [configuration documentation](#configuration).
@@ -345,3 +347,3 @@
 
-Pa11y has lots of options you can use to change the way Headless Chrome runs, or the way your page is loaded. Options can be set either as a parameter on the `pa11y` function or in a config file used by the command-line interface.
+Pa11y has lots of options you can use to change the way Headless Chrome runs, or the way your page is loaded. Options can be set either as a parameter on the `pa11y` function or in a [JSON configuration file](#command-line-configuration). Some are also available directly as [command-line options](#command-line-interface).
 
@@ -452,2 +454,14 @@ Below is a reference of all the options that are available:
 
+### `level` (string)
+
+The level of issue which can fail the test (and cause it to exit with code 2) when running via the CLI. This should be one of `error` (the default), `warning`, or `notice`.
+
+```json
+{
+    "level": "warning"
+}
+```
+
+Defaults to `error`. Note this configuration is only available when using Pa11y on the command line, not via the JavaScript Interface.
+
 ### `log` (object)
@@ -509,2 +523,14 @@
 
+### `reporter` (string)
+
+The reporter to use while running the test via the CLI. [More about reporters](#reporters).
+
+```json
+{
+    "reporter": "json"
+}
+```
+
+Defaults to `cli`. Note this configuration is only available when using Pa11y on the command line, not via the JavaScript Interface.
+
 ### `rootElement` (element)
@@ -557,2 +583,14 @@
 
+### `threshold` (number)
+
+The number of errors, warnings, or notices to permit before the test is considered to have failed (with exit code 2) when running via the CLI.
+
+```json
+{
+    "threshold": 9
+}
+```
+
+Defaults to `0`. Note this configuration is only available when using Pa11y on the command line, not via the JavaScript Interface.
+
 ### `timeout` (number)
@@ -562,2 +600,4 @@
 
+Please note that this is the timeout for the _entire_ test run (including time to initialise Chrome, load the page, and run the tests).
+
 ```js
@@ -564,0 +604,0 @@ pa11y('http://example.com/', {

test/integration/cli/exit-codes.test.js

@@ -88,2 +88,87 @@ 'use strict';
 
+	describe('when the `level` config is set to "warning"', () => {
+
+		describe('and Pa11y is run on a page with no warnings or errors', () => {
+
+			before(async () => {
+				pa11yResponse = await runPa11yCli(`${global.mockWebsiteAddress}/notices`, {
+					arguments: [
+						'--config', './mock/config/level-warning.json',
+						'--include-notices',
+						'--include-warnings'
+					]
+				});
+			});
+
+			it('exits with a code of `0`', () => {
+				assert.strictEqual(pa11yResponse.exitCode, 0);
+			});
+
+		});
+
+		describe('and Pa11y is run on a page with warnings', () => {
+
+			before(async () => {
+				pa11yResponse = await runPa11yCli(`${global.mockWebsiteAddress}/warnings`, {
+					arguments: [
+						'--config', './mock/config/level-warning.json',
+						'--include-notices',
+						'--include-warnings'
+					]
+				});
+			});
+
+			it('exits with a code of `2`', () => {
+				assert.strictEqual(pa11yResponse.exitCode, 2);
+			});
+
+		});
+
+	});
+
+	describe('when the `level` config is set to "warning" but the `--level` flag is set to "notice"', () => {
+
+		describe('and Pa11y is run on a page with no notices, warnings, or errors', () => {
+
+			before(async () => {
+				pa11yResponse = await runPa11yCli(`${global.mockWebsiteAddress}/notices`, {
+					arguments: [
+						'--config', './mock/config/level-warning.json',
+						'--include-notices',
+						'--include-warnings',
+						'--level', 'notice',
+						// We can't build a page that doesn't include notices, so we have
+						// to fake it by ignoring the only one there is
+						'--ignore', 'WCAG2AA.Principle2.Guideline2_4.2_4_2.H25.2'
+					]
+				});
+			});
+
+			it('exits with a code of `0`', () => {
+				assert.strictEqual(pa11yResponse.exitCode, 0);
+			});
+
+		});
+
+		describe('and Pa11y is run on a page with notices', () => {
+
+			before(async () => {
+				pa11yResponse = await runPa11yCli(`${global.mockWebsiteAddress}/notices`, {
+					arguments: [
+						'--config', './mock/config/level-warning.json',
+						'--include-notices',
+						'--include-warnings',
+						'--level', 'notice'
+					]
+				});
+			});
+
+			it('exits with a code of `2`', () => {
+				assert.strictEqual(pa11yResponse.exitCode, 2);
+			});
+
+		});
+
+	});
+
 	describe('when the `--level` flag is set to "notice"', () => {
@@ -132,2 +217,39 @@
 
+	describe('when the `threshold` config is set to more than the number of errors present', () => {
+
+		before(async () => {
+			pa11yResponse = await runPa11yCli(`${global.mockWebsiteAddress}/many-errors`, {
+				arguments: [
+					'--config', './mock/config/threshold-large.json',
+					'--include-notices',
+					'--include-warnings'
+				]
+			});
+		});
+
+		it('exits with a code of `0`', () => {
+			assert.strictEqual(pa11yResponse.exitCode, 0);
+		});
+
+	});
+
+	describe('when the `threshold` config is set to less than the number of errors present but the `--threshold` flag is set to more', () => {
+
+		before(async () => {
+			pa11yResponse = await runPa11yCli(`${global.mockWebsiteAddress}/many-errors`, {
+				arguments: [
+					'--config', './mock/config/threshold-small.json',
+					'--include-notices',
+					'--include-warnings',
+					'--threshold', '5'
+				]
+			});
+		});
+
+		it('exits with a code of `0`', () => {
+			assert.strictEqual(pa11yResponse.exitCode, 0);
+		});
+
+	});
+
 	describe('when the `--threshold` flag is set to more than the number of errors present', () => {
@@ -134,0 +256,0 @@

test/integration/cli/reporter-json.test.js

@@ -27,2 +27,20 @@ 'use strict';
 
+	describe('when the `reporter` config is set to "json"', () => {
+
+		before(async () => {
+			pa11yResponse = await runPa11yCli(`${global.mockWebsiteAddress}/errors`, {
+				arguments: [
+					'--config', './mock/config/reporter-json.json'
+				]
+			});
+		});
+
+		it('outputs issues in JSON format', () => {
+			const json = JSON.parse(pa11yResponse.output);
+			assert.isArray(json);
+			assert.lengthEquals(json, 1);
+		});
+
+	});
+
 });

test/integration/cli/standard.test.js

@@ -81,2 +81,26 @@ 'use strict';
 
+	describe('when the `standard` config is set to "WCAG2AAA" but the `--standard` flag is set to "Section508"', () => {
+
+		before(async () => {
+			pa11yResponse = await runPa11yCli(`${global.mockWebsiteAddress}/errors`, {
+				arguments: [
+					'--config', './mock/config/standard.json',
+					'--include-notices',
+					'--include-warnings',
+					'--standard', 'Section508',
+					'--reporter', 'json'
+				]
+			});
+		});
+
+		it('outputs the expected issues', () => {
+			assert.isArray(pa11yResponse.json);
+			assert.lengthEquals(pa11yResponse.json, 1);
+			pa11yResponse.json.forEach(issue => {
+				assert.match(issue.code, /^Section508\./);
+			});
+		});
+
+	});
+
 });

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

352243

increased by1.82%

Number of package files

101

increased by5.21%

Lines of code

5644

increased by2.73%

Number of lines in readme file

894

increased by4.68%

Number of medium supply chain risk alerts

3

decreased by-25%

No dependency changes