@vscode/vscode-perf - npm Package Compare versions

Comparing version 0.0.5 to 0.0.6

index.d.ts

@@ -6,9 +6,31 @@ /*---------------------------------------------------------------------------------------------
 
+export enum Quality {
+	/**
+	 * Stable quality
+	 */
+	Stable = 'stable',
+
+	/**
+	 * Insider quality
+	 */
+	Insider = 'insider',
+
+	/**
+	 * Exploration quality
+	 */
+	Exploration = 'exploration'
+}
+
 export interface Options {
 	/**
-	 * executable location of the build to measure the performance of
+	 * quality or the location of the build to measure the performance of. Location can be a path to a build or a URL to a build
 	 */
-	readonly build: string;
+	readonly build: string | Quality;
 
 	/**
+	 * Include unreleased builds in the search for the build to measure the performance of. Defaults to false.
+	 */
+	readonly unreleased?: boolean;
+
+	/**
 	 * pair of markers separated by `-` between which the duration has to be measured. Eg: `code/didLoadWorkbenchMain-code/didLoadExtensions
@@ -47,4 +69,14 @@ */
 	readonly profAppendTimers?: string;
+
+	/**
+	 * whether to measure the performance of desktop or web runtime. Defaults to desktop.
+	 */
+	readonly runtime?: 'desktop' | 'web';
+
+	/**
+	 * a GitHub token of scopes 'repo', 'workflow', 'user:email', 'read:user' to enable additional performance tests targetting web
+	 */
+	readonly token?: string;
 }
 
 export function run(options?: Options): Promise<void>;

out/constants.js

@@ -7,10 +7,14 @@ "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.platform = exports.Platform = exports.PERFORMANCE_RUNS = exports.PERFORMANCE_FILE = exports.EXTENSIONS_FOLDER = exports.USER_DATA_FOLDER = exports.ROOT = void 0;
+exports.platform = exports.Quality = exports.Runtime = exports.Platform = exports.INSIDERS_VSCODE_DEV_HOST_NAME = exports.VSCODE_DEV_HOST_NAME = exports.PERFORMANCE_RUNS = exports.PERFORMANCE_FILE = exports.EXTENSIONS_FOLDER = exports.USER_DATA_FOLDER = exports.DATA_FOLDER = exports.BUILDS_FOLDER = exports.ROOT = void 0;
 const os_1 = require("os");
 const path_1 = require("path");
 exports.ROOT = (0, path_1.join)((0, os_1.tmpdir)(), 'vscode-perf');
-exports.USER_DATA_FOLDER = (0, path_1.join)(exports.ROOT, 'user-data-dir');
-exports.EXTENSIONS_FOLDER = (0, path_1.join)(exports.ROOT, 'extensions-dir');
+exports.BUILDS_FOLDER = (0, path_1.join)(exports.ROOT, '.builds');
+exports.DATA_FOLDER = (0, path_1.join)(exports.ROOT, '.data');
+exports.USER_DATA_FOLDER = (0, path_1.join)(exports.DATA_FOLDER, 'data');
+exports.EXTENSIONS_FOLDER = (0, path_1.join)(exports.DATA_FOLDER, 'extensions');
 exports.PERFORMANCE_FILE = (0, path_1.join)(exports.ROOT, 'startup-perf.txt');
 exports.PERFORMANCE_RUNS = 10;
+exports.VSCODE_DEV_HOST_NAME = 'vscode.dev';
+exports.INSIDERS_VSCODE_DEV_HOST_NAME = 'insiders.vscode.dev';
 var Platform;
@@ -25,2 +29,13 @@ (function (Platform) {
 })(Platform = exports.Platform || (exports.Platform = {}));
+var Runtime;
+(function (Runtime) {
+    Runtime[Runtime["Web"] = 1] = "Web";
+    Runtime[Runtime["Desktop"] = 2] = "Desktop";
+})(Runtime = exports.Runtime || (exports.Runtime = {}));
+var Quality;
+(function (Quality) {
+    Quality["Exploration"] = "exploration";
+    Quality["Insider"] = "insider";
+    Quality["Stable"] = "stable";
+})(Quality = exports.Quality || (exports.Quality = {}));
 exports.platform = (() => {
@@ -27,0 +42,0 @@ if (process.platform === 'win32') {

out/index.js

@@ -13,2 +13,5 @@ "use strict";
 const commander_1 = require("commander");
+const fs_1 = require("fs");
+const builds_1 = require("./builds");
+const constants_1 = require("./constants");
 const perf_1 = require("./perf");
@@ -18,3 +21,4 @@ async function run(options) {
         commander_1.program
-            .requiredOption('-b, --build <build>', 'executable location of the build to measure the performance of')
+            .requiredOption('-b, --build <build>', 'quality or the location of the build to measure the performance of. Location can be a path to a build or a URL to a build. Quality options: `stable`, `insider`, `exploration`.')
+            .option('--unreleased', 'Include unreleased builds in the search for the build to measure the performance of.')
             .option('-m, --duration-markers <duration-markers>', 'pair of markers separated by `-` between which the duration has to be measured. Eg: `code/didLoadWorkbenchMain-code/didLoadExtensions')
@@ -26,2 +30,4 @@ .option('--duration-markers-file <duration-markers-file>', 'file in which the performance measurements shall be recorded')
             .option('-v, --verbose', 'logs verbose output to the console when errors occur')
+            .option('-t, --token <token>', `a GitHub token of scopes 'repo', 'workflow', 'user:email', 'read:user' to enable additional performance tests targetting web`)
+            .addOption(new commander_1.Option('-r, --runtime <runtime>', 'whether to measure the performance of desktop or web runtime').choices(['desktop', 'web']))
             .addOption(new commander_1.Option('--prof-append-timers <prof-append-timers>').hideHelp(true));
@@ -31,4 +37,19 @@ options = commander_1.program.parse(process.argv).opts();
     try {
+        try {
+            (0, fs_1.rmSync)(constants_1.ROOT, { recursive: true });
+        }
+        catch (error) { }
+        (0, fs_1.mkdirSync)(constants_1.ROOT, { recursive: true });
+        const runtime = options.runtime === 'web' ? constants_1.Runtime.Web : constants_1.Runtime.Desktop;
+        let build = options.build;
+        switch (build) {
+            case 'stable':
+            case 'insider':
+            case 'exploration':
+                build = await (0, builds_1.installBuild)(runtime, build, options.unreleased);
+                break;
+        }
         await (0, perf_1.launch)({
-            build: options.build,
+            build,
+            runtime,
             durationMarkers: options.durationMarkers ? Array.isArray(options.durationMarkers) ? options.durationMarkers : [options.durationMarkers] : undefined,
@@ -39,3 +60,4 @@ durationMarkersFile: options.durationMarkersFile,
             fileToOpen: options.file,
-            profAppendTimers: options.profAppendTimers
+            profAppendTimers: options.profAppendTimers,
+            token: options.token,
         });
@@ -42,0 +64,0 @@ }

out/perf.js

@@ -37,10 +37,60 @@ "use strict";
 const cp = __importStar(require("child_process"));
+const playwright_1 = __importDefault(require("playwright"));
 const chalk_1 = __importDefault(require("chalk"));
+const auth_1 = require("./auth");
+const PERFORMANCE_RUN_TIMEOUT = 60000;
 async function launch(options) {
     try {
-        fs.rmSync(constants_1.ROOT, { recursive: true });
+        fs.rmSync(constants_1.DATA_FOLDER, { recursive: true });
     }
     catch (error) { }
-    fs.mkdirSync(constants_1.ROOT, { recursive: true });
+    fs.mkdirSync(constants_1.DATA_FOLDER, { recursive: true });
+    const runs = options.runs ?? constants_1.PERFORMANCE_RUNS;
+    const durations = new Map();
     const perfFile = options.durationMarkersFile ?? constants_1.PERFORMANCE_FILE;
+    const markers = options.durationMarkers?.length ? [...options.durationMarkers] : ['ellapsed'];
+    const playwrightStorageState = options.runtime === constants_1.Runtime.Web ? await preparePlaywright(options) : undefined;
+    for (let i = 0; i < runs; i++) {
+        console.log(`${chalk_1.default.gray('[perf]')} running session ${chalk_1.default.green(`${i + 1}`)} of ${chalk_1.default.green(`${runs}`)}...`);
+        let timedOut = false;
+        let promise;
+        const abortController = new AbortController();
+        switch (options.runtime) {
+            case constants_1.Runtime.Desktop:
+                promise = launchDesktop(options, perfFile, markers, abortController.signal);
+                break;
+            case constants_1.Runtime.Web:
+                promise = launchWeb(options, perfFile, markers[0], playwrightStorageState, abortController.signal);
+                break;
+        }
+        const content = await Promise.race([
+            new Promise(resolve => setTimeout(() => {
+                timedOut = true;
+                resolve();
+            }, PERFORMANCE_RUN_TIMEOUT)),
+            promise
+        ]);
+        if (timedOut) {
+            console.log(`${chalk_1.default.red('[perf]')} timeout after ${chalk_1.default.green(`${PERFORMANCE_RUN_TIMEOUT}ms`)}`);
+            abortController.abort();
+        }
+        else {
+            if (content) {
+                for (const marker of markers) {
+                    logMarker(content, marker, durations);
+                }
+            }
+            else {
+                console.log(`${chalk_1.default.red('[perf]')} no perf data found.`);
+            }
+        }
+    }
+    console.log(`${chalk_1.default.gray('[perf]')} ${chalk_1.default.blueBright('Summary')}:`);
+    for (const marker of markers) {
+        const markerDurations = durations.get(marker) ?? [];
+        console.log(`${chalk_1.default.gray('[perf]')} ${marker}: ${chalk_1.default.green(`${markerDurations[0]}ms`)} (fastest), ${chalk_1.default.green(`${markerDurations[markerDurations.length - 1]}ms`)} (slowest), ${chalk_1.default.green(`${markerDurations[Math.floor(markerDurations.length / 2)]}ms`)} (median)`);
+    }
+}
+exports.launch = launch;
+async function launchDesktop(options, perfFile, markers, signal) {
     const codeArgs = [
@@ -66,3 +116,2 @@ '--accept-server-license-terms',
     }
-    const markers = options.durationMarkers?.length ? [...options.durationMarkers] : ['ellapsed'];
     for (const marker of markers) {
@@ -78,48 +127,87 @@ codeArgs.push('--prof-duration-markers');
     }
-    const runs = options.runs ?? constants_1.PERFORMANCE_RUNS;
-    const durations = new Map();
     let childProcess;
-    process.on('exit', () => {
-        if (childProcess) {
-            childProcess.kill();
+    signal.addEventListener('abort', () => childProcess?.kill());
+    childProcess = cp.spawn(options.build, codeArgs);
+    childProcess.stdout.on('data', data => {
+        if (options.verbose) {
+            console.log(`${chalk_1.default.gray('[electron]')}: ${data.toString()}`);
         }
     });
-    for (let i = 0; i < runs; i++) {
-        console.log(`${chalk_1.default.gray('[perf]')} running session ${chalk_1.default.green(`${i + 1}`)} of ${chalk_1.default.green(`${runs}`)}...`);
-        childProcess = cp.spawn(options.build, codeArgs);
-        childProcess.stdout.on('data', data => {
-            if (options.verbose) {
-                console.log(`${chalk_1.default.gray('[electron]')}: ${data.toString()}`);
+    childProcess.stderr.on('data', data => {
+        if (options.verbose) {
+            console.log(`${chalk_1.default.red('[electron]')}: ${data.toString()}`);
+        }
+    });
+    await (new Promise(resolve => childProcess?.on('exit', () => resolve())));
+    childProcess = undefined;
+    if (fs.existsSync(perfFile)) {
+        return readLastLineSync(perfFile);
+    }
+    if (options.profAppendTimers) {
+        const content = readLastLineSync(options.profAppendTimers);
+        return `ellapsed	${content}`;
+    }
+    return undefined;
+}
+async function preparePlaywright(options) {
+    const url = new URL(options.build);
+    if (options.token && (url.hostname === constants_1.VSCODE_DEV_HOST_NAME || url.hostname === constants_1.INSIDERS_VSCODE_DEV_HOST_NAME)) {
+        return (0, auth_1.generateVscodeDevAuthState)(options.token);
+    }
+    return undefined;
+}
+async function launchWeb(options, perfFile, durationMarker, playwrightStorageState, signal) {
+    const url = new URL(options.build);
+    if (options.folderToOpen) {
+        url.searchParams.set('folder', options.folderToOpen);
+    }
+    const payload = [];
+    // profDurationMarkers
+    payload.push(['profDurationMarkers', durationMarker === 'ellapsed' ? 'code/timeOrigin,code/didStartWorkbench' : durationMarker.split('-').join(',')]);
+    if (options.fileToOpen) {
+        payload.push(['openFile', options.fileToOpen]);
+    }
+    // disable annoyers
+    payload.push(['skipWelcome', 'true']);
+    payload.push(['skipReleaseNotes', 'true']);
+    url.searchParams.set('payload', JSON.stringify(payload));
+    // Use playwright to open the page
+    // and watch out for the desired performance measurement to
+    // be printed to the console.
+    const browser = await playwright_1.default.chromium.launch({ headless: false });
+    signal.addEventListener('abort', () => browser.close());
+    if (signal.aborted) {
+        browser.close();
+    }
+    const page = await browser.newPage({
+        storageState: playwrightStorageState,
+        viewport: { width: 1200, height: 800 }
+    });
+    if (options.verbose) {
+        page.on('pageerror', error => console.error(`Playwright ERROR: page error: ${error}`));
+        page.on('crash', () => console.error('Playwright ERROR: page crash'));
+        page.on('requestfailed', e => console.error('Playwright ERROR: Request Failed', e.url(), e.failure()?.errorText));
+        page.on('response', response => {
+            if (response.status() >= 400) {
+                console.error(`Playwright ERROR: HTTP status ${response.status()} for ${response.url()}`);
             }
         });
-        childProcess.stderr.on('data', data => {
+    }
+    return new Promise(resolve => {
+        page.on('console', async (msg) => {
+            const text = msg.text();
             if (options.verbose) {
-                console.log(`${chalk_1.default.red('[electron]')}: ${data.toString()}`);
+                console.error(`Playwright Console: ${text}`);
             }
+            // Write full message to perf file if we got a path
+            const matches = /\[prof-timers\] (.+)/.exec(text);
+            if (matches?.[1]) {
+                browser.close();
+                fs.appendFileSync(perfFile, `${matches[1]}\n`);
+                resolve(`${durationMarker}	${matches[1]}`);
+            }
         });
-        await (new Promise(resolve => childProcess?.on('exit', () => resolve())));
-        childProcess = undefined;
-        if (fs.existsSync(perfFile)) {
-            const content = readLastLineSync(perfFile);
-            for (const marker of markers) {
-                logMarker(content, marker, durations);
-            }
-        }
-        else if (options.profAppendTimers) {
-            const content = readLastLineSync(options.profAppendTimers);
-            const marker = 'ellapsed';
-            logMarker(`${marker}	${content}`, marker, durations);
-        }
-        else {
-            console.error('No perf file found');
-            process.exit(1);
-        }
-    }
-    console.log(`${chalk_1.default.gray('[perf]')} ${chalk_1.default.blueBright('Summary')}:`);
-    for (const marker of markers) {
-        const markerDurations = durations.get(marker) ?? [];
-        console.log(`${chalk_1.default.gray('[perf]')} ${marker}: ${chalk_1.default.green(`${markerDurations[0]}ms`)} (fastest), ${chalk_1.default.green(`${markerDurations[markerDurations.length - 1]}ms`)} (slowest), ${chalk_1.default.green(`${markerDurations[Math.floor(markerDurations.length / 2)]}ms`)} (median)`);
-    }
+        page.goto(url.href);
+    });
 }
-exports.launch = launch;
 function logMarker(content, marker, durations) {
@@ -126,0 +214,0 @@ const regex = new RegExp(`${escapeRegExpCharacters(marker)}\\s+(\\d+)`);

package.json

 {
     "name": "@vscode/vscode-perf",
-    "version": "0.0.5",
+    "version": "0.0.6",
     "description": "Tooling for evaluating performance of VS Code",
@@ -31,8 +31,14 @@ "repository": {
         "chalk": "^4.x",
-        "commander": "^9.4.0"
+        "commander": "^9.4.0",
+        "cookie": "^0.5.0",
+        "js-base64": "^3.7.4",
+        "node-fetch": "2.6.8",
+        "playwright": "^1.29.2"
     },
     "devDependencies": {
-        "@types/node": "16.x",
-        "typescript": "4.8.x"
+        "@types/node": "18.x",
+        "typescript": "4.8.x",
+        "@types/node-fetch": "^2.6.2",
+        "@types/cookie": "^0.5.1"
     }
 }

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Network access

Supply chain risk

This module accesses the network.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

46037

increased by135.62%

Number of package files

14

increased by27.27%

Lines of code

803

increased by204.17%

Worsened metrics

Dependency count

6

increased by200%

Dev dependency count

4

increased by100%

Number of low supply chain risk alerts

3

increased by200%

Number of medium supply chain risk alerts

4

increased by300%

Dependency changes

• + Addedcookie@^0.5.0

• + Addedjs-base64@^3.7.4

• + Addednode-fetch@2.6.8

• + Addedplaywright@^1.29.2

• + Addedcookie@0.5.0(transitive)

• + Addedfsevents@2.3.2(transitive)

• + Addedjs-base64@3.7.7(transitive)

• + Addednode-fetch@2.6.8(transitive)

• + Addedplaywright@1.49.1(transitive)

• + Addedplaywright-core@1.49.1(transitive)

• + Addedtr46@0.0.3(transitive)

• + Addedwebidl-conversions@3.0.1(transitive)

• + Addedwhatwg-url@5.0.0(transitive)