@indutny/bencher - npm Package Compare versions

Comparing version 1.1.5 to 1.2.0

dist/bin/bencher.js

@@ -35,4 +35,7 @@ #!/usr/bin/env node
 // ANSI colors
-const BOLD = 1;
-const ITALIC = 3;
+const BOLD = '\x1b[1m';
+const ITALIC = '\x1b[3m';
+const RESET = '\x1b[m';
+const GREY = '\x1b[90m';
+const RED = '\x1b[31m';
 // Go back to previous line, clear the line
@@ -65,9 +68,14 @@ const PREV_LINE = '\x1b[F\x1b[K';
         type: 'number',
-        default: 10,
+        default: 5,
         describe: 'width of iteration sweep',
     })
-        .option('warm-up-iterations', {
+        .option('warm-up-duration', {
         type: 'number',
-        default: 100,
-        describe: 'number of warm up iterations',
+        default: 0.5,
+        describe: 'duration of warm up',
+    })
+        .option('ignore-outliers', {
+        alias: 'q',
+        type: 'boolean',
+        describe: "don't report severe outliers",
     }).argv;
@@ -78,3 +86,3 @@ const modules = await Promise.all(argv._.map(async (file) => {
         const m = await (_a = path, Promise.resolve().then(() => __importStar(require(_a))));
-        const { duration = argv['duration'], sweepWidth = argv['sweepWidth'], samples = argv['samples'], warmUpIterations = argv['warmUpIterations'], } = m.options ?? {};
+        const { duration = argv['duration'], sweepWidth = argv['sweepWidth'], samples = argv['samples'], warmUpDuration = argv['warmUpDuration'], ignoreOutliers = argv['ignoreOutliers'], } = m.options ?? {};
         if (duration <= 0) {
@@ -92,4 +100,4 @@ throw new Error(`${file}: options.duration must be positive`);
         }
-        if (warmUpIterations <= 0) {
-            throw new Error(`${file}: options.warmUpIterations must be positive`);
+        if (warmUpDuration <= 0) {
+            throw new Error(`${file}: options.warmUpDuration must be positive`);
         }
@@ -102,3 +110,4 @@ return {
                 samples,
-                warmUpIterations,
+                warmUpDuration,
+                ignoreOutliers,
             },
@@ -110,3 +119,3 @@ default: m.default,
     for (const m of modules) {
-        const paddedName = style(m.name, BOLD) + ':' + ' '.repeat(maxNameLength - m.name.length);
+        const paddedName = BOLD + m.name + RESET + ':' + ' '.repeat(maxNameLength - m.name.length);
         // Just to reserve the line
@@ -120,8 +129,16 @@ (0, fs_1.writeSync)(process.stdout.fd, '\n');
         onTick();
-        const { ops, maxError, usedSamples } = run(m, {
+        const { ops, maxError, outliers, severeOutliers } = run(m, {
             onTick,
         });
-        const stats = '\x1b[90m' +
-            style(`(±${nice(maxError)}, p=${P_VALUE}, n=${usedSamples})`, ITALIC);
-        (0, fs_1.writeSync)(process.stdout.fd, `${PREV_LINE}${paddedName} ${nice(ops)} ops/sec ${stats}\n`);
+        const stats = [
+            `±${nice(maxError)}`,
+            `p=${P_VALUE}`,
+            `o=${outliers + severeOutliers}/${m.options.samples}`,
+        ];
+        let warning = '';
+        if (!m.options.ignoreOutliers && severeOutliers !== 0) {
+            warning = `${RESET}${RED} severe outliers=${severeOutliers}`;
+        }
+        (0, fs_1.writeSync)(process.stdout.fd, `${PREV_LINE}${paddedName} ${nice(ops)} ops/sec ` +
+            `${GREY + ITALIC}(${stats.join(', ')})${warning}${RESET}\n`);
     }
@@ -145,3 +162,3 @@ }
     }
-    const { beta, confidence, outliers } = regress(m, samples);
+    const { beta, confidence, outliers, severeOutliers } = regress(m, samples);
     const ops = 1 / beta;
@@ -151,7 +168,7 @@ const lowOps = 1 / (beta + confidence);
     const maxError = Math.max(highOps - ops, ops - lowOps);
-    const usedSamples = samples.length - outliers;
     return {
         ops,
         maxError,
-        usedSamples,
+        outliers,
+        severeOutliers,
     };
@@ -161,3 +178,5 @@ }
     // Initial warm-up
-    for (let i = 0; i < m.options.warmUpIterations; i++) {
+    const coldDuration = measure(m, 1);
+    const warmUpIterations = m.options.warmUpDuration / coldDuration;
+    for (let i = 0; i < warmUpIterations; i++) {
         m.default();
@@ -205,5 +224,6 @@ }
     }
-    // Within each iteration bin get rid of the outliers.
-    const withoutOutliers = new Array();
-    for (const [iterations, durations] of bins) {
+    let outliers = 0;
+    let severeOutliers = 0;
+    // Within each iteration bin identify the outliers for reporting purposes.
+    for (const [, durations] of bins) {
         durations.sort();
@@ -215,9 +235,15 @@ const p25 = durations[Math.floor(durations.length * 0.25)] ?? -Infinity;
         const outlierHigh = p75 + iqr * 1.5;
+        const badOutlierLow = p25 - iqr * 3;
+        const badOutlierHigh = p75 + iqr * 3;
         // Tukey's method
-        const filtered = durations.filter((d) => d >= outlierLow && d <= outlierHigh);
-        for (const duration of filtered) {
-            withoutOutliers.push({ iterations, duration });
+        for (const d of durations) {
+            if (d < badOutlierLow || d > badOutlierHigh) {
+                severeOutliers++;
+            }
+            else if (d < outlierLow || d > outlierHigh) {
+                outliers++;
+            }
         }
     }
-    if (withoutOutliers.length < 2) {
+    if (samples.length < 2) {
         throw new Error(`${m.name}: low sample count`);
@@ -227,11 +253,11 @@ }
     let meanIterations = 0;
-    for (const { duration, iterations } of withoutOutliers) {
+    for (const { duration, iterations } of samples) {
         meanDuration += duration;
         meanIterations += iterations;
     }
-    meanDuration /= withoutOutliers.length;
-    meanIterations /= withoutOutliers.length;
+    meanDuration /= samples.length;
+    meanIterations /= samples.length;
     let betaNum = 0;
     let betaDenom = 0;
-    for (const { duration, iterations } of withoutOutliers) {
+    for (const { duration, iterations } of samples) {
         betaNum += (duration - meanDuration) * (iterations - meanIterations);
@@ -245,6 +271,6 @@ betaDenom += (iterations - meanIterations) ** 2;
     let stdError = 0;
-    for (const { duration, iterations } of withoutOutliers) {
+    for (const { duration, iterations } of samples) {
         stdError += (duration - alpha - beta * iterations) ** 2;
     }
-    stdError /= withoutOutliers.length - 2;
+    stdError /= samples.length - 2;
     stdError /= betaDenom;
@@ -256,8 +282,6 @@ stdError = Math.sqrt(stdError);
         confidence: STUDENT_T * stdError,
-        outliers: samples.length - withoutOutliers.length,
+        outliers,
+        severeOutliers,
     };
 }
-function style(text, code) {
-    return `\x1b[${code}m${text}\x1b[m`;
-}
 function nice(n) {
@@ -264,0 +288,0 @@ let result = n.toFixed(1);

package.json

 {
   "name": "@indutny/bencher",
-  "version": "1.1.5",
+  "version": "1.2.0",
   "description": "Simple benchmarking tool",
@@ -5,0 +5,0 @@ "bin": {

README.md

@@ -46,3 +46,3 @@ # @indutny/bencher
 $ bencher benchmark.js
-runner: 1058.6 ops/s (±4.5, p=0.05, n=98)
+runner: 1’037.8 ops/sec (±18.8, p=0.001, n=98)
 ```
@@ -49,0 +49,0 @@

No alert changes

Improved metrics

Total package byte prevSize

13614

increased by5.36%

Lines of code

282

increased by9.3%

No dependency changes