Big News: Socket raises $60M Series C at a $1B valuation to secure software supply chains for AI-driven development.Announcement
Sign In

ctrf

Package Overview
Dependencies
Maintainers
1
Versions
34
Alerts
File Explorer

Advanced tools

Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

ctrf - npm Package Compare versions

Comparing version
0.1.0
to
0.2.0-next-0
+195
dist/cli/cli.cjs
#!/usr/bin/env node
"use strict";
var __create = Object.create;
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __getProtoOf = Object.getPrototypeOf;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
// If the importer is in node compatibility mode or this is not an ESM
// file that has been converted to a CommonJS file using a Babel-
// compatible transform (i.e. "__esModule" has not been set), then set
// "default" to the CommonJS "module.exports" for node compatibility.
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
mod
));
// src/cli/cli.ts
var import_yargs = __toESM(require("yargs/yargs"), 1);
var import_helpers = require("yargs/helpers");
// src/cli/merge.ts
var import_fs = __toESM(require("fs"), 1);
var import_path = __toESM(require("path"), 1);
async function mergeReports(directory, output, outputDir, keepReports) {
try {
const directoryPath = import_path.default.resolve(directory);
const outputFileName = output;
const resolvedOutputDir = outputDir ? import_path.default.resolve(outputDir) : directoryPath;
const outputPath = import_path.default.join(resolvedOutputDir, outputFileName);
console.log("Merging CTRF reports...");
const files = import_fs.default.readdirSync(directoryPath);
files.forEach((file) => {
console.log("Found file:", file);
});
const ctrfReportFiles = files.filter((file) => {
try {
if (import_path.default.extname(file) !== ".json") {
console.log(`Skipping non-CTRF file: ${file}`);
return false;
}
const filePath = import_path.default.join(directoryPath, file);
const fileContent = import_fs.default.readFileSync(filePath, "utf8");
const jsonData = JSON.parse(fileContent);
if (!("results" in jsonData)) {
console.log(`Skipping non-CTRF file: ${file}`);
return false;
}
return true;
} catch (error) {
console.error(`Error reading JSON file '${file}':`, error);
return false;
}
});
if (ctrfReportFiles.length === 0) {
console.log("No CTRF reports found in the specified directory.");
return;
}
if (!import_fs.default.existsSync(resolvedOutputDir)) {
import_fs.default.mkdirSync(resolvedOutputDir, { recursive: true });
console.log(`Created output directory: ${resolvedOutputDir}`);
}
const mergedReport = ctrfReportFiles.map((file) => {
console.log("Merging report:", file);
const filePath = import_path.default.join(directoryPath, file);
const fileContent = import_fs.default.readFileSync(filePath, "utf8");
return JSON.parse(fileContent);
}).reduce(
(acc, curr) => {
if (!acc.results) {
return curr;
}
acc.results.summary.tests += curr.results.summary.tests;
acc.results.summary.passed += curr.results.summary.passed;
acc.results.summary.failed += curr.results.summary.failed;
acc.results.summary.skipped += curr.results.summary.skipped;
acc.results.summary.pending += curr.results.summary.pending;
acc.results.summary.other += curr.results.summary.other;
acc.results.tests.push(...curr.results.tests);
acc.results.summary.start = Math.min(
acc.results.summary.start,
curr.results.summary.start
);
acc.results.summary.stop = Math.max(
acc.results.summary.stop,
curr.results.summary.stop
);
return acc;
},
{ results: null }
);
import_fs.default.writeFileSync(outputPath, JSON.stringify(mergedReport, null, 2));
if (!keepReports) {
ctrfReportFiles.forEach((file) => {
const filePath = import_path.default.join(directoryPath, file);
if (file !== outputFileName) {
import_fs.default.unlinkSync(filePath);
}
});
}
console.log("CTRF reports merged successfully.");
console.log(`Merged report saved to: ${outputPath}`);
} catch (error) {
console.error("Error merging CTRF reports:", error);
}
}
// src/cli/flaky.ts
var import_fs2 = __toESM(require("fs"), 1);
var import_path2 = __toESM(require("path"), 1);
async function identifyFlakyTests(filePath) {
try {
const resolvedFilePath = import_path2.default.resolve(filePath);
if (!import_fs2.default.existsSync(resolvedFilePath)) {
console.error(`The file ${resolvedFilePath} does not exist.`);
return;
}
const fileContent = import_fs2.default.readFileSync(resolvedFilePath, "utf8");
const report = JSON.parse(fileContent);
const flakyTests = report.results.tests.filter(
(test) => test.flaky === true
);
if (flakyTests.length > 0) {
console.log(`Found ${flakyTests.length} flaky test(s):`);
flakyTests.forEach((test) => {
console.log(`- Test Name: ${test.name}, Retries: ${test.retries}`);
});
} else {
console.log(`No flaky tests found in ${resolvedFilePath}.`);
}
} catch (error) {
console.error("Error identifying flaky tests:", error);
}
}
// src/cli/cli.ts
console.warn(
"\u26A0\uFE0F DEPRECATION NOTICE: This CLI is deprecated and will be removed in v1."
);
console.warn("\u{1F4E6} Please use the standalone ctrf-cli package instead:");
console.warn(" https://github.com/ctrf-io/ctrf-cli\n");
void (0, import_yargs.default)((0, import_helpers.hideBin)(process.argv)).command(
"merge <directory>",
"Merge CTRF reports into a single report",
(yargs2) => {
return yargs2.positional("directory", {
describe: "Directory of the CTRF reports",
type: "string",
demandOption: true
}).option("output", {
alias: "o",
describe: "Output file name for merged report",
type: "string",
default: "ctrf-report.json"
}).option("output-dir", {
alias: "d",
describe: "Output directory for merged report",
type: "string"
}).option("keep-reports", {
alias: "k",
describe: "Keep existing reports after merging",
type: "boolean",
default: false
});
},
async (argv) => {
await mergeReports(
argv.directory,
argv.output,
argv["output-dir"],
argv["keep-reports"]
);
}
).command(
"flaky <file>",
"Identify flaky tests from a CTRF report file",
(yargs2) => {
return yargs2.positional("file", {
describe: "CTRF report file",
type: "string",
demandOption: true
});
},
async (argv) => {
await identifyFlakyTests(argv.file);
}
).help().demandCommand(1, "You need at least one command before moving on").argv;
"use strict";
var __create = Object.create;
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __getProtoOf = Object.getPrototypeOf;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __export = (target, all) => {
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
// If the importer is in node compatibility mode or this is not an ESM
// file that has been converted to a CommonJS file using a Babel-
// compatible transform (i.e. "__esModule" has not been set), then set
// "default" to the CommonJS "module.exports" for node compatibility.
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
mod
));
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
// src/index.ts
var src_exports = {};
__export(src_exports, {
BuilderError: () => BuilderError,
CTRFError: () => CTRFError,
CTRF_NAMESPACE: () => CTRF_NAMESPACE,
CURRENT_SPEC_VERSION: () => CURRENT_SPEC_VERSION,
FileError: () => FileError,
ParseError: () => ParseError,
REPORT_FORMAT: () => REPORT_FORMAT,
ReportBuilder: () => ReportBuilder,
SUPPORTED_SPEC_VERSIONS: () => SUPPORTED_SPEC_VERSIONS,
SchemaVersionError: () => SchemaVersionError,
TEST_STATUSES: () => TEST_STATUSES,
TestBuilder: () => TestBuilder,
ValidationError: () => ValidationError,
addInsights: () => addInsights,
calculateSummary: () => calculateSummary,
filterTests: () => filterTests,
findTest: () => findTest,
generateReportId: () => generateReportId,
generateTestId: () => generateTestId,
getCurrentSpecVersion: () => getCurrentSpecVersion,
getSchema: () => getSchema,
getSupportedSpecVersions: () => getSupportedSpecVersions,
hasInsights: () => hasInsights,
isCTRFReport: () => isCTRFReport,
isRetryAttempt: () => isRetryAttempt,
isTest: () => isTest,
isTestFlaky: () => isTestFlaky,
isTestStatus: () => isTestStatus,
isValid: () => isValid,
merge: () => merge,
parse: () => parse,
schema: () => schema,
stringify: () => stringify,
validate: () => validate,
validateStrict: () => validateStrict
});
module.exports = __toCommonJS(src_exports);
// node_modules/tsup/assets/cjs_shims.js
var getImportMetaUrl = () => typeof document === "undefined" ? new URL(`file:${__filename}`).href : document.currentScript && document.currentScript.tagName.toUpperCase() === "SCRIPT" ? document.currentScript.src : new URL("main.js", document.baseURI).href;
var importMetaUrl = /* @__PURE__ */ getImportMetaUrl();
// src/constants.ts
var REPORT_FORMAT = "CTRF";
var CURRENT_SPEC_VERSION = "0.0.0";
var TEST_STATUSES = [
"passed",
"failed",
"skipped",
"pending",
"other"
];
var SUPPORTED_SPEC_VERSIONS = ["0.0.0"];
var CTRF_NAMESPACE = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
// src/validate.ts
var import_ajv = __toESM(require("ajv"), 1);
var import_ajv_formats = __toESM(require("ajv-formats"), 1);
// src/schema.ts
var import_fs = __toESM(require("fs"), 1);
var import_path = __toESM(require("path"), 1);
var import_url = require("url");
// src/errors.ts
var CTRFError = class _CTRFError extends Error {
constructor(message) {
super(message);
this.name = "CTRFError";
Object.setPrototypeOf(this, _CTRFError.prototype);
}
};
var ValidationError = class _ValidationError extends CTRFError {
/** Detailed validation errors */
errors;
constructor(message, errors = []) {
super(message);
this.name = "ValidationError";
this.errors = errors;
Object.setPrototypeOf(this, _ValidationError.prototype);
}
};
var ParseError = class _ParseError extends CTRFError {
/** The original error that caused the parse failure */
cause;
constructor(message, cause) {
super(message);
this.name = "ParseError";
this.cause = cause;
Object.setPrototypeOf(this, _ParseError.prototype);
}
};
var SchemaVersionError = class _SchemaVersionError extends CTRFError {
/** The unsupported version */
version;
/** Supported versions */
supportedVersions;
constructor(version, supportedVersions) {
super(
`Unsupported schema version: ${version}. Supported versions: ${supportedVersions.join(", ")}`
);
this.name = "SchemaVersionError";
this.version = version;
this.supportedVersions = supportedVersions;
Object.setPrototypeOf(this, _SchemaVersionError.prototype);
}
};
var FileError = class _FileError extends CTRFError {
/** The file path that caused the error */
filePath;
/** The original error */
cause;
constructor(message, filePath, cause) {
super(message);
this.name = "FileError";
this.filePath = filePath;
this.cause = cause;
Object.setPrototypeOf(this, _FileError.prototype);
}
};
var BuilderError = class _BuilderError extends CTRFError {
constructor(message) {
super(message);
this.name = "BuilderError";
Object.setPrototypeOf(this, _BuilderError.prototype);
}
};
// src/schema.ts
var __filename2 = (0, import_url.fileURLToPath)(importMetaUrl);
var __dirname = import_path.default.dirname(__filename2);
var _schemas = /* @__PURE__ */ new Map();
function loadSchemaForVersion(version) {
const versionParts = version.split(".");
if (versionParts.length < 2) {
throw new SchemaVersionError(version, [...SUPPORTED_SPEC_VERSIONS]);
}
const majorMinor = `${versionParts[0]}.${versionParts[1]}`;
if (_schemas.has(majorMinor)) {
return _schemas.get(majorMinor);
}
const schemaPath = import_path.default.resolve(__dirname, `ctrf-schema-${majorMinor}.json`);
if (!import_fs.default.existsSync(schemaPath)) {
throw new SchemaVersionError(version, [...SUPPORTED_SPEC_VERSIONS]);
}
const schema2 = JSON.parse(import_fs.default.readFileSync(schemaPath, "utf8"));
_schemas.set(majorMinor, schema2);
return schema2;
}
var schema = loadSchemaForVersion(CURRENT_SPEC_VERSION);
function getSchema(version) {
if (!SUPPORTED_SPEC_VERSIONS.includes(
version
)) {
throw new SchemaVersionError(version, [...SUPPORTED_SPEC_VERSIONS]);
}
return loadSchemaForVersion(version);
}
function getCurrentSpecVersion() {
return CURRENT_SPEC_VERSION;
}
function getSupportedSpecVersions() {
return SUPPORTED_SPEC_VERSIONS;
}
// src/validate.ts
function validate(report, options = {}) {
const ajv = new import_ajv.default({ allErrors: true });
(0, import_ajv_formats.default)(ajv);
const schemaToUse = options.specVersion ? getSchema(options.specVersion) : schema;
const validateFn = ajv.compile(schemaToUse);
const valid = validateFn(report);
if (valid) {
return { valid: true, errors: [] };
}
const errors = validateFn.errors?.map((error) => ({
message: error.message || "Unknown validation error",
path: error.instancePath || "/",
keyword: error.keyword
})) || [];
return { valid: false, errors };
}
function isValid(report) {
const result = validate(report);
return result.valid;
}
function validateStrict(report) {
const result = validate(report);
if (!result.valid) {
const errorMessages = result.errors.map((e) => `${e.path}: ${e.message}`).join("\n");
throw new ValidationError(
`CTRF validation failed:
${errorMessages}`,
result.errors
);
}
}
function isCTRFReport(report) {
return typeof report === "object" && report !== null && "reportFormat" in report && report.reportFormat === REPORT_FORMAT;
}
function isTest(obj) {
return typeof obj === "object" && obj !== null && "name" in obj && typeof obj.name === "string" && "status" in obj && typeof obj.status === "string" && "duration" in obj && typeof obj.duration === "number";
}
function isTestStatus(value) {
return typeof value === "string" && TEST_STATUSES.includes(value);
}
function isRetryAttempt(obj) {
return typeof obj === "object" && obj !== null && "attempt" in obj && typeof obj.attempt === "number" && "status" in obj && typeof obj.status === "string";
}
function hasInsights(report) {
return report.insights !== void 0 && Object.keys(report.insights).length > 0;
}
// src/summary.ts
function calculateSummary(tests, options = {}) {
const statusCounts = {
passed: 0,
failed: 0,
skipped: 0,
pending: 0,
other: 0
};
let flakyCount = 0;
const suites = /* @__PURE__ */ new Set();
let totalDuration = 0;
let minStart = options.start ?? Number.MAX_SAFE_INTEGER;
let maxStop = options.stop ?? 0;
for (const test of tests) {
statusCounts[test.status]++;
if (test.flaky) {
flakyCount++;
}
if (test.suite) {
for (let i = 1; i <= test.suite.length; i++) {
suites.add(test.suite.slice(0, i).join("/"));
}
}
totalDuration += test.duration;
if (test.start !== void 0) {
minStart = Math.min(minStart, test.start);
}
if (test.stop !== void 0) {
maxStop = Math.max(maxStop, test.stop);
}
}
const start = options.start ?? (minStart === Number.MAX_SAFE_INTEGER ? 0 : minStart);
const stop = options.stop ?? maxStop;
const summary = {
tests: tests.length,
passed: statusCounts.passed,
failed: statusCounts.failed,
skipped: statusCounts.skipped,
pending: statusCounts.pending,
other: statusCounts.other,
start,
stop
};
if (flakyCount > 0) {
summary.flaky = flakyCount;
}
if (suites.size > 0) {
summary.suites = suites.size;
}
if (totalDuration > 0 || tests.length > 0) {
summary.duration = totalDuration;
}
return summary;
}
// src/id.ts
var import_crypto = require("crypto");
function generateTestId(properties) {
const { name, suite, filePath } = properties;
const suiteString = suite ? suite.join("/") : "";
const identifier = `${name}|${suiteString}|${filePath || ""}`;
const namespaceBytes = CTRF_NAMESPACE.replace(/-/g, "").match(/.{2}/g).map((byte) => parseInt(byte, 16));
const input = Buffer.concat([
Buffer.from(namespaceBytes),
Buffer.from(identifier, "utf8")
]);
const hash = (0, import_crypto.createHash)("sha1").update(input).digest("hex");
const uuid = [
hash.substring(0, 8),
hash.substring(8, 12),
"5" + hash.substring(13, 16),
// Version 5
(parseInt(hash.substring(16, 17), 16) & 3 | 8).toString(16) + hash.substring(17, 20),
// Variant bits
hash.substring(20, 32)
].join("-");
return uuid;
}
function generateReportId() {
return (0, import_crypto.randomUUID)();
}
// src/builder.ts
var ReportBuilder = class {
constructor(options = {}) {
this.options = options;
if (options.autoGenerateId) {
this._reportId = generateReportId();
}
if (options.autoTimestamp) {
this._timestamp = (/* @__PURE__ */ new Date()).toISOString();
}
}
_specVersion = CURRENT_SPEC_VERSION;
_reportId;
_timestamp;
_generatedBy;
_tool;
_environment;
_tests = [];
_insights;
_baseline;
_extra;
_summaryOverrides;
/**
* Set the spec version.
*/
specVersion(version) {
this._specVersion = version;
return this;
}
/**
* Set or generate the report ID.
* @param uuid - UUID to use, or undefined to auto-generate
*/
reportId(uuid) {
this._reportId = uuid ?? generateReportId();
return this;
}
/**
* Set the timestamp.
* @param date - Date to use, or undefined for current time
*/
timestamp(date) {
if (date instanceof Date) {
this._timestamp = date.toISOString();
} else if (typeof date === "string") {
this._timestamp = date;
} else {
this._timestamp = (/* @__PURE__ */ new Date()).toISOString();
}
return this;
}
/**
* Set the generator name.
*/
generatedBy(name) {
this._generatedBy = name;
return this;
}
/**
* Set the tool information.
*/
tool(tool) {
this._tool = tool;
return this;
}
/**
* Set the environment information.
*/
environment(env) {
this._environment = env;
return this;
}
/**
* Add a single test.
*/
addTest(test) {
this._tests.push(test);
return this;
}
/**
* Add multiple tests.
*/
addTests(tests) {
this._tests.push(...tests);
return this;
}
/**
* Set run-level insights.
*/
insights(insights) {
this._insights = insights;
return this;
}
/**
* Set the baseline reference.
*/
baseline(baseline) {
this._baseline = baseline;
return this;
}
/**
* Set extra metadata.
*/
extra(data) {
this._extra = data;
return this;
}
/**
* Override specific summary fields.
* Useful when you want to set specific timing or counts.
*/
summaryOverrides(overrides) {
this._summaryOverrides = overrides;
return this;
}
/**
* Build and return the CTRF report.
* @throws BuilderError if required fields are missing
*/
build() {
if (!this._tool) {
throw new BuilderError("Tool is required. Call .tool() before .build()");
}
const calculatedSummary = calculateSummary(this._tests);
const summary = {
...calculatedSummary,
...this._summaryOverrides
};
const results = {
tool: this._tool,
summary,
tests: this._tests
};
if (this._environment) {
results.environment = this._environment;
}
const report = {
reportFormat: REPORT_FORMAT,
specVersion: this._specVersion,
results
};
if (this._reportId) {
report.reportId = this._reportId;
}
if (this._timestamp) {
report.timestamp = this._timestamp;
}
if (this._generatedBy) {
report.generatedBy = this._generatedBy;
}
if (this._insights) {
report.insights = this._insights;
}
if (this._baseline) {
report.baseline = this._baseline;
}
if (this._extra) {
report.extra = this._extra;
}
return report;
}
};
var TestBuilder = class {
constructor(options = {}) {
this.options = options;
}
_id;
_name;
_status;
_duration;
_start;
_stop;
_suite;
_message;
_trace;
_snippet;
_ai;
_line;
_rawStatus;
_tags;
_type;
_filePath;
_retries;
_retryAttempts;
_flaky;
_stdout;
_stderr;
_threadId;
_browser;
_device;
_screenshot;
_attachments;
_parameters;
_steps;
_insights;
_extra;
/**
* Set or generate the test ID.
* @param uuid - UUID to use, or undefined to auto-generate based on properties
*/
id(uuid) {
this._id = uuid;
return this;
}
/**
* Set the test name.
*/
name(name) {
this._name = name;
return this;
}
/**
* Set the test status.
*/
status(status) {
this._status = status;
return this;
}
/**
* Set the duration in milliseconds.
*/
duration(ms) {
this._duration = ms;
return this;
}
/**
* Set the start timestamp.
*/
start(timestamp) {
this._start = timestamp;
return this;
}
/**
* Set the stop timestamp.
*/
stop(timestamp) {
this._stop = timestamp;
return this;
}
/**
* Set the suite hierarchy.
*/
suite(suites) {
this._suite = suites;
return this;
}
/**
* Set the error message.
*/
message(msg) {
this._message = msg;
return this;
}
/**
* Set the stack trace.
*/
trace(trace) {
this._trace = trace;
return this;
}
/**
* Set the code snippet.
*/
snippet(code) {
this._snippet = code;
return this;
}
/**
* Set AI-generated analysis.
*/
ai(analysis) {
this._ai = analysis;
return this;
}
/**
* Set the line number.
*/
line(num) {
this._line = num;
return this;
}
/**
* Set the raw status from the test framework.
*/
rawStatus(status) {
this._rawStatus = status;
return this;
}
/**
* Set tags.
*/
tags(tags) {
this._tags = tags;
return this;
}
/**
* Set test type.
*/
type(type) {
this._type = type;
return this;
}
/**
* Set file path.
*/
filePath(path2) {
this._filePath = path2;
return this;
}
/**
* Set retry count.
*/
retries(count) {
this._retries = count;
return this;
}
/**
* Add a retry attempt.
*/
addRetryAttempt(attempt) {
if (!this._retryAttempts) {
this._retryAttempts = [];
}
this._retryAttempts.push(attempt);
return this;
}
/**
* Mark as flaky.
*/
flaky(isFlaky = true) {
this._flaky = isFlaky;
return this;
}
/**
* Set stdout.
*/
stdout(lines) {
this._stdout = lines;
return this;
}
/**
* Set stderr.
*/
stderr(lines) {
this._stderr = lines;
return this;
}
/**
* Set thread ID.
*/
threadId(id) {
this._threadId = id;
return this;
}
/**
* Set browser name.
*/
browser(name) {
this._browser = name;
return this;
}
/**
* Set device name.
*/
device(name) {
this._device = name;
return this;
}
/**
* Set screenshot (base64).
*/
screenshot(base64) {
this._screenshot = base64;
return this;
}
/**
* Add an attachment.
*/
addAttachment(attachment) {
if (!this._attachments) {
this._attachments = [];
}
this._attachments.push(attachment);
return this;
}
/**
* Set parameters.
*/
parameters(params) {
this._parameters = params;
return this;
}
/**
* Add a step.
*/
addStep(step) {
if (!this._steps) {
this._steps = [];
}
this._steps.push(step);
return this;
}
/**
* Set test-level insights.
*/
insights(insights) {
this._insights = insights;
return this;
}
/**
* Set extra metadata.
*/
extra(data) {
this._extra = data;
return this;
}
/**
* Build and return the Test object.
* @throws BuilderError if required fields are missing
*/
build() {
if (!this._name) {
throw new BuilderError(
"Test name is required. Call .name() before .build()"
);
}
if (!this._status) {
throw new BuilderError(
"Test status is required. Call .status() before .build()"
);
}
if (this._duration === void 0) {
throw new BuilderError(
"Test duration is required. Call .duration() before .build()"
);
}
const test = {
name: this._name,
status: this._status,
duration: this._duration
};
if (this.options.autoGenerateId && !this._id) {
test.id = generateTestId({
name: this._name,
suite: this._suite,
filePath: this._filePath
});
} else if (this._id) {
test.id = this._id;
}
if (this._start !== void 0) test.start = this._start;
if (this._stop !== void 0) test.stop = this._stop;
if (this._suite) test.suite = this._suite;
if (this._message) test.message = this._message;
if (this._trace) test.trace = this._trace;
if (this._snippet) test.snippet = this._snippet;
if (this._ai) test.ai = this._ai;
if (this._line !== void 0) test.line = this._line;
if (this._rawStatus) test.rawStatus = this._rawStatus;
if (this._tags) test.tags = this._tags;
if (this._type) test.type = this._type;
if (this._filePath) test.filePath = this._filePath;
if (this._retries !== void 0) test.retries = this._retries;
if (this._retryAttempts) test.retryAttempts = this._retryAttempts;
if (this._flaky !== void 0) test.flaky = this._flaky;
if (this._stdout) test.stdout = this._stdout;
if (this._stderr) test.stderr = this._stderr;
if (this._threadId) test.threadId = this._threadId;
if (this._browser) test.browser = this._browser;
if (this._device) test.device = this._device;
if (this._screenshot) test.screenshot = this._screenshot;
if (this._attachments) test.attachments = this._attachments;
if (this._parameters) test.parameters = this._parameters;
if (this._steps) test.steps = this._steps;
if (this._insights) test.insights = this._insights;
if (this._extra) test.extra = this._extra;
return test;
}
};
// src/parse.ts
var import_fs2 = __toESM(require("fs"), 1);
var import_util = require("util");
var readFileAsync = (0, import_util.promisify)(import_fs2.default.readFile);
var writeFileAsync = (0, import_util.promisify)(import_fs2.default.writeFile);
function parse(json, options = {}) {
let parsed;
try {
parsed = JSON.parse(json);
} catch (error) {
throw new ParseError(
`Failed to parse JSON: ${error instanceof Error ? error.message : "Unknown error"}`,
error instanceof Error ? error : void 0
);
}
if (options.validate) {
validateStrict(parsed);
}
return parsed;
}
function stringify(report, options = {}) {
const { pretty = false, indent = 2 } = options;
if (pretty) {
return JSON.stringify(report, null, indent);
}
return JSON.stringify(report);
}
// src/merge.ts
function merge(reports, options = {}) {
if (!reports || reports.length === 0) {
throw new Error("No reports provided for merging");
}
if (reports.length === 1) {
return { ...reports[0] };
}
const {
deduplicateTests = false,
mergeSummary = true,
preserveEnvironment = "merge"
} = options;
let allTests = [];
for (const report of reports) {
allTests.push(...report.results.tests);
}
if (deduplicateTests) {
const seen = /* @__PURE__ */ new Map();
for (const test of allTests) {
if (test.id) {
seen.set(test.id, test);
} else {
seen.set(`no-id-${seen.size}`, test);
}
}
allTests = Array.from(seen.values());
}
let summary;
if (mergeSummary) {
summary = calculateSummary(allTests);
let minStart = Number.MAX_SAFE_INTEGER;
let maxStop = 0;
for (const report of reports) {
minStart = Math.min(minStart, report.results.summary.start);
maxStop = Math.max(maxStop, report.results.summary.stop);
}
summary.start = minStart === Number.MAX_SAFE_INTEGER ? 0 : minStart;
summary.stop = maxStop;
summary.duration = summary.stop - summary.start;
} else {
summary = sumSummaries(reports.map((r) => r.results.summary));
}
let environment;
switch (preserveEnvironment) {
case "first":
environment = reports[0].results.environment;
break;
case "last":
environment = reports[reports.length - 1].results.environment;
break;
case "merge":
default:
environment = mergeEnvironments(
reports.map((r) => r.results.environment).filter(Boolean)
);
break;
}
const tool = reports[0].results.tool;
const merged = {
reportFormat: REPORT_FORMAT,
specVersion: CURRENT_SPEC_VERSION,
reportId: generateReportId(),
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
results: {
tool,
summary,
tests: allTests
}
};
if (environment && Object.keys(environment).length > 0) {
merged.results.environment = environment;
}
const mergedResultsExtra = mergeExtras(
reports.map((r) => r.results.extra).filter(Boolean)
);
if (mergedResultsExtra && Object.keys(mergedResultsExtra).length > 0) {
merged.results.extra = mergedResultsExtra;
}
const mergedReportExtra = mergeExtras(
reports.map((r) => r.extra).filter(Boolean)
);
if (mergedReportExtra && Object.keys(mergedReportExtra).length > 0) {
merged.extra = mergedReportExtra;
}
return merged;
}
function sumSummaries(summaries) {
const result = {
tests: 0,
passed: 0,
failed: 0,
skipped: 0,
pending: 0,
other: 0,
start: Number.MAX_SAFE_INTEGER,
stop: 0
};
let totalFlaky = 0;
let totalSuites = 0;
let totalDuration = 0;
let hasFlaky = false;
let hasSuites = false;
let hasDuration = false;
for (const summary of summaries) {
result.tests += summary.tests;
result.passed += summary.passed;
result.failed += summary.failed;
result.skipped += summary.skipped;
result.pending += summary.pending;
result.other += summary.other;
result.start = Math.min(result.start, summary.start);
result.stop = Math.max(result.stop, summary.stop);
if (summary.flaky !== void 0) {
hasFlaky = true;
totalFlaky += summary.flaky;
}
if (summary.suites !== void 0) {
hasSuites = true;
totalSuites += summary.suites;
}
if (summary.duration !== void 0) {
hasDuration = true;
totalDuration += summary.duration;
}
}
if (result.start === Number.MAX_SAFE_INTEGER) {
result.start = 0;
}
if (hasFlaky) result.flaky = totalFlaky;
if (hasSuites) result.suites = totalSuites;
if (hasDuration) result.duration = totalDuration;
return result;
}
function mergeEnvironments(environments) {
const merged = {};
for (const env of environments) {
for (const [key, value] of Object.entries(env)) {
if (value !== void 0 && merged[key] === void 0) {
;
merged[key] = value;
}
}
}
return merged;
}
function mergeExtras(extras) {
if (extras.length === 0) {
return void 0;
}
const merged = {};
for (const extra of extras) {
for (const [key, value] of Object.entries(extra)) {
if (merged[key] === void 0) {
merged[key] = value;
}
}
}
return Object.keys(merged).length > 0 ? merged : void 0;
}
// src/filter.ts
function filterTests(report, criteria) {
return report.results.tests.filter((test) => matchesCriteria(test, criteria));
}
function findTest(report, criteria) {
const { id, name, ...filterCriteria } = criteria;
return report.results.tests.find((test) => {
if (id !== void 0 && test.id !== id) {
return false;
}
if (name !== void 0 && test.name !== name) {
return false;
}
return matchesCriteria(test, filterCriteria);
});
}
function matchesCriteria(test, criteria) {
if (criteria.status !== void 0) {
const statuses = Array.isArray(criteria.status) ? criteria.status : [criteria.status];
if (!statuses.includes(test.status)) {
return false;
}
}
if (criteria.tags !== void 0) {
const requiredTags = Array.isArray(criteria.tags) ? criteria.tags : [criteria.tags];
if (!test.tags || !requiredTags.some((tag) => test.tags.includes(tag))) {
return false;
}
}
if (criteria.suite !== void 0) {
const requiredSuites = Array.isArray(criteria.suite) ? criteria.suite : [criteria.suite];
if (!test.suite || !requiredSuites.some((suite) => test.suite.includes(suite))) {
return false;
}
}
if (criteria.flaky !== void 0 && test.flaky !== criteria.flaky) {
return false;
}
if (criteria.browser !== void 0 && test.browser !== criteria.browser) {
return false;
}
if (criteria.device !== void 0 && test.device !== criteria.device) {
return false;
}
return true;
}
// src/insights.ts
function isTestFlaky(test) {
return test.flaky === true || test.retries !== void 0 && test.retries > 0 && test.status === "passed";
}
function calculateP95(values) {
if (values.length === 0) return 0;
const sorted = [...values].sort((a, b) => a - b);
const index = Math.ceil(sorted.length * 0.95) - 1;
return Number(sorted[Math.max(0, index)].toFixed(2));
}
function validateReportForInsights(report) {
return !!(report?.results?.tests && Array.isArray(report.results.tests));
}
function sortReportsByTimestamp(reports) {
return [...reports].sort((a, b) => {
const aStart = a.results?.summary?.start ?? 0;
const bStart = b.results?.summary?.start ?? 0;
return bStart - aStart;
});
}
function aggregateTestMetricsAcrossReports(reports) {
const metricsMap = /* @__PURE__ */ new Map();
for (const report of reports) {
if (!validateReportForInsights(report)) continue;
for (const test of report.results.tests) {
const isPassed = test.status === "passed";
const isFailed = test.status === "failed";
const isSkipped = test.status === "skipped";
const isPending = test.status === "pending";
const isOther = test.status === "other";
const testName = test.name;
if (!metricsMap.has(testName)) {
metricsMap.set(testName, {
totalAttempts: 0,
totalAttemptsFailed: 0,
totalResults: 0,
totalResultsFailed: 0,
totalResultsPassed: 0,
totalResultsSkipped: 0,
totalResultsFlaky: 0,
totalAttemptsFlaky: 0,
totalResultsDuration: 0,
appearsInRuns: 0,
reportsAnalyzed: 0,
durations: []
});
}
const metrics = metricsMap.get(testName);
metrics.totalResults += 1;
metrics.totalAttempts += 1 + (test.retries || 0);
metrics.totalAttemptsFailed += test.retries || 0;
if (isFailed) {
metrics.totalResultsFailed += 1;
metrics.totalAttemptsFailed += 1 + (test.retries || 0);
} else if (isPassed) {
metrics.totalResultsPassed += 1;
} else if (isSkipped || isPending || isOther) {
metrics.totalResultsSkipped += 1;
}
if (isTestFlaky(test)) {
metrics.totalResultsFlaky += 1;
metrics.totalAttemptsFlaky += test.retries || 0;
}
metrics.totalResultsDuration += test.duration || 0;
metrics.durations.push(test.duration || 0);
}
const testsInThisReport = /* @__PURE__ */ new Set();
for (const test of report.results.tests) {
testsInThisReport.add(test.name);
}
for (const testName of testsInThisReport) {
const metrics = metricsMap.get(testName);
metrics.appearsInRuns += 1;
}
}
return metricsMap;
}
function consolidateTestMetricsToRunMetrics(metricsMap) {
let totalAttempts = 0;
let totalAttemptsFailed = 0;
let totalResults = 0;
let totalResultsFailed = 0;
let totalResultsPassed = 0;
let totalResultsSkipped = 0;
let totalResultsFlaky = 0;
let totalAttemptsFlaky = 0;
let totalResultsDuration = 0;
for (const metrics of metricsMap.values()) {
totalAttempts += metrics.totalAttempts;
totalAttemptsFailed += metrics.totalAttemptsFailed;
totalResults += metrics.totalResults;
totalResultsFailed += metrics.totalResultsFailed;
totalResultsPassed += metrics.totalResultsPassed;
totalResultsSkipped += metrics.totalResultsSkipped;
totalResultsFlaky += metrics.totalResultsFlaky;
totalAttemptsFlaky += metrics.totalAttemptsFlaky;
totalResultsDuration += metrics.totalResultsDuration;
}
return {
totalAttempts,
totalAttemptsFailed,
totalResults,
totalResultsFailed,
totalResultsPassed,
totalResultsSkipped,
totalResultsFlaky,
totalAttemptsFlaky,
totalResultsDuration,
reportsAnalyzed: metricsMap.size
};
}
function calculateFlakyRateFromMetrics(runMetrics) {
if (runMetrics.totalAttempts === 0) {
return 0;
}
return Number(
(runMetrics.totalAttemptsFlaky / (runMetrics.totalResults + runMetrics.totalAttemptsFlaky)).toFixed(4)
);
}
function calculateFailRateFromMetrics(runMetrics) {
if (runMetrics.totalResults === 0) {
return 0;
}
return Number(
(runMetrics.totalResultsFailed / runMetrics.totalResults).toFixed(4)
);
}
function calculatePassRateFromMetrics(runMetrics) {
if (runMetrics.totalResults === 0) {
return 0;
}
return Number(
(runMetrics.totalResultsPassed / runMetrics.totalResults).toFixed(4)
);
}
function calculateAverageTestDurationFromMetrics(runMetrics) {
if (runMetrics.totalResults === 0) {
return 0;
}
return Number(
(runMetrics.totalResultsDuration / runMetrics.totalResults).toFixed(2)
);
}
function calculateAverageRunDurationFromMetrics(runMetrics) {
if (runMetrics.reportsAnalyzed === 0) {
return 0;
}
return Number(
(runMetrics.totalResultsDuration / runMetrics.reportsAnalyzed).toFixed(2)
);
}
function calculateP95RunDurationFromReports(reports) {
const runDurations = [];
for (const report of reports) {
if (validateReportForInsights(report) && report.results?.summary) {
const { start, stop } = report.results.summary;
if (start && stop && stop > start) {
const runDuration = stop - start;
runDurations.push(runDuration);
}
}
}
return calculateP95(runDurations);
}
function calculateRunInsights(reports, index = 0) {
if (index >= reports.length) {
return reports;
}
const currentReport = reports[index];
const previousReports = reports.slice(index + 1);
const allReportsUpToThisPoint = [currentReport, ...previousReports];
const testMetrics = aggregateTestMetricsAcrossReports(allReportsUpToThisPoint);
const runMetrics = consolidateTestMetricsToRunMetrics(testMetrics);
const { ...relevantMetrics } = runMetrics;
currentReport.insights = {
passRate: {
current: calculatePassRateFromMetrics(runMetrics),
baseline: 0,
change: 0
},
flakyRate: {
current: calculateFlakyRateFromMetrics(runMetrics),
baseline: 0,
change: 0
},
failRate: {
current: calculateFailRateFromMetrics(runMetrics),
baseline: 0,
change: 0
},
averageTestDuration: {
current: calculateAverageTestDurationFromMetrics(runMetrics),
baseline: 0,
change: 0
},
averageRunDuration: {
current: calculateAverageRunDurationFromMetrics(runMetrics),
baseline: 0,
change: 0
},
p95RunDuration: {
current: calculateP95RunDurationFromReports(allReportsUpToThisPoint),
baseline: 0,
change: 0
},
runsAnalyzed: allReportsUpToThisPoint.length,
extra: relevantMetrics
};
return calculateRunInsights(reports, index + 1);
}
function calculateTestInsightsWithBaseline(currentTestMetrics, baselineTestMetrics) {
const currentPassRate = currentTestMetrics.totalResults === 0 ? 0 : Number(
(currentTestMetrics.totalResultsPassed / currentTestMetrics.totalResults).toFixed(4)
);
const currentFlakyRate = currentTestMetrics.totalAttempts === 0 ? 0 : Number(
(currentTestMetrics.totalAttemptsFlaky / (currentTestMetrics.totalResults + currentTestMetrics.totalAttemptsFlaky)).toFixed(4)
);
const currentFailRate = currentTestMetrics.totalResults === 0 ? 0 : Number(
(currentTestMetrics.totalResultsFailed / currentTestMetrics.totalResults).toFixed(4)
);
const currentAverageDuration = currentTestMetrics.totalResults === 0 ? 0 : Number(
(currentTestMetrics.totalResultsDuration / currentTestMetrics.totalResults).toFixed(2)
);
const currentP95Duration = calculateP95(currentTestMetrics.durations);
let baselinePassRate = 0;
let baselineFlakyRate = 0;
let baselineFailRate = 0;
let baselineAverageDuration = 0;
let baselineP95Duration = 0;
if (baselineTestMetrics) {
baselinePassRate = baselineTestMetrics.totalResults === 0 ? 0 : Number(
(baselineTestMetrics.totalResultsPassed / baselineTestMetrics.totalResults).toFixed(4)
);
baselineFlakyRate = baselineTestMetrics.totalAttempts === 0 ? 0 : Number(
(baselineTestMetrics.totalAttemptsFlaky / baselineTestMetrics.totalAttempts).toFixed(4)
);
baselineFailRate = baselineTestMetrics.totalResults === 0 ? 0 : Number(
(baselineTestMetrics.totalResultsFailed / baselineTestMetrics.totalResults).toFixed(4)
);
baselineAverageDuration = baselineTestMetrics.totalResults === 0 ? 0 : Number(
(baselineTestMetrics.totalResultsDuration / baselineTestMetrics.totalResults).toFixed(2)
);
baselineP95Duration = calculateP95(baselineTestMetrics.durations);
}
const { durations: _durations, ...relevantMetrics } = currentTestMetrics;
return {
passRate: {
current: currentPassRate,
baseline: baselinePassRate,
change: Number((currentPassRate - baselinePassRate).toFixed(4))
},
flakyRate: {
current: currentFlakyRate,
baseline: baselineFlakyRate,
change: Number((currentFlakyRate - baselineFlakyRate).toFixed(4))
},
failRate: {
current: currentFailRate,
baseline: baselineFailRate,
change: Number((currentFailRate - baselineFailRate).toFixed(4))
},
averageTestDuration: {
current: currentAverageDuration,
baseline: baselineAverageDuration,
change: currentAverageDuration - baselineAverageDuration
},
p95TestDuration: {
current: currentP95Duration,
baseline: baselineP95Duration,
change: currentP95Duration - baselineP95Duration
},
executedInRuns: currentTestMetrics.appearsInRuns,
extra: relevantMetrics
};
}
function addTestInsightsWithBaselineToCurrentReport(currentReport, previousReports, baselineReport) {
if (!validateReportForInsights(currentReport)) {
return currentReport;
}
const allReports = [currentReport, ...previousReports];
const currentTestMetrics = aggregateTestMetricsAcrossReports(allReports);
let baselineTestMetrics;
if (baselineReport && validateReportForInsights(baselineReport)) {
const baselineIndex = previousReports.findIndex(
(report) => report.results?.summary?.start === baselineReport.results?.summary?.start
);
if (baselineIndex >= 0) {
const reportsUpToBaseline = previousReports.slice(baselineIndex);
baselineTestMetrics = aggregateTestMetricsAcrossReports(reportsUpToBaseline);
}
}
const reportWithInsights = {
...currentReport,
results: {
...currentReport.results,
tests: currentReport.results.tests.map((test) => {
const testName = test.name;
const currentMetrics = currentTestMetrics.get(testName);
if (currentMetrics) {
const baselineMetrics = baselineTestMetrics?.get(testName);
const testInsights = calculateTestInsightsWithBaseline(
currentMetrics,
baselineMetrics
);
return {
...test,
insights: testInsights
};
}
return test;
})
}
};
return reportWithInsights;
}
function calculateReportInsightsBaseline(currentReport, baselineReport) {
const currentInsights = currentReport.insights;
const previousInsights = baselineReport.insights;
if (!currentInsights || !previousInsights) {
console.log("Both reports must have insights populated");
return currentReport.insights;
}
return {
passRate: {
current: currentInsights?.passRate?.current ?? 0,
baseline: previousInsights?.passRate?.current ?? 0,
change: Number(
((currentInsights?.passRate?.current ?? 0) - (previousInsights?.passRate?.current ?? 0)).toFixed(4)
)
},
flakyRate: {
current: currentInsights?.flakyRate?.current ?? 0,
baseline: previousInsights?.flakyRate?.current ?? 0,
change: Number(
((currentInsights?.flakyRate?.current ?? 0) - (previousInsights?.flakyRate?.current ?? 0)).toFixed(4)
)
},
failRate: {
current: currentInsights?.failRate?.current ?? 0,
baseline: previousInsights?.failRate?.current ?? 0,
change: Number(
((currentInsights?.failRate?.current ?? 0) - (previousInsights?.failRate?.current ?? 0)).toFixed(4)
)
},
averageTestDuration: {
current: currentInsights?.averageTestDuration?.current ?? 0,
baseline: previousInsights?.averageTestDuration?.current ?? 0,
change: (currentInsights?.averageTestDuration?.current ?? 0) - (previousInsights?.averageTestDuration?.current ?? 0)
},
averageRunDuration: {
current: currentInsights?.averageRunDuration?.current ?? 0,
baseline: previousInsights?.averageRunDuration?.current ?? 0,
change: (currentInsights?.averageRunDuration?.current ?? 0) - (previousInsights?.averageRunDuration?.current ?? 0)
},
p95RunDuration: {
current: currentInsights?.p95RunDuration?.current ?? 0,
baseline: previousInsights?.p95RunDuration?.current ?? 0,
change: (currentInsights?.p95RunDuration?.current ?? 0) - (previousInsights?.p95RunDuration?.current ?? 0)
},
runsAnalyzed: currentInsights?.runsAnalyzed ?? 0,
extra: currentInsights.extra
};
}
function getTestsRemovedSinceBaseline(currentReport, baselineReport) {
if (!validateReportForInsights(currentReport) || !validateReportForInsights(baselineReport)) {
return [];
}
const currentTestNames = new Set(
currentReport.results.tests.map((test) => test.name)
);
const removedTests = baselineReport.results.tests.filter(
(test) => !currentTestNames.has(test.name)
);
return removedTests.map((test) => ({
name: test.name,
suite: test.suite,
filePath: test.filePath
}));
}
function getTestsAddedSinceBaseline(currentReport, baselineReport) {
if (!validateReportForInsights(currentReport) || !validateReportForInsights(baselineReport)) {
return [];
}
const baselineTestNames = new Set(
baselineReport.results.tests.map((test) => test.name)
);
const addedTests = currentReport.results.tests.filter(
(test) => !baselineTestNames.has(test.name)
);
return addedTests.map((test) => ({
name: test.name,
suite: test.suite,
filePath: test.filePath
}));
}
function setTestsRemovedToInsights(insights, currentReport, baselineReport) {
const removedTests = getTestsRemovedSinceBaseline(
currentReport,
baselineReport
);
return {
...insights,
extra: {
...insights.extra,
testsRemoved: removedTests
}
};
}
function setTestsAddedToInsights(insights, currentReport, baselineReport) {
const addedTests = getTestsAddedSinceBaseline(currentReport, baselineReport);
return {
...insights,
extra: {
...insights.extra,
testsAdded: addedTests
}
};
}
function addInsights(report, historicalReports = [], options = {}) {
if (!validateReportForInsights(report)) {
console.warn("Current report is not valid for insights calculation");
return report;
}
const baseline = options.baseline;
const sortedPreviousReports = sortReportsByTimestamp(historicalReports);
const allReports = [report, ...sortedPreviousReports];
const reportsWithRunInsights = calculateRunInsights([...allReports]);
const currentReportWithRunInsights = reportsWithRunInsights[0];
const currentReportWithTestInsights = addTestInsightsWithBaselineToCurrentReport(
currentReportWithRunInsights,
sortedPreviousReports,
baseline
);
if (!baseline) {
return currentReportWithTestInsights;
}
let baselineInsights = calculateReportInsightsBaseline(
currentReportWithTestInsights,
baseline
);
baselineInsights = setTestsAddedToInsights(
baselineInsights,
currentReportWithTestInsights,
baseline
);
baselineInsights = setTestsRemovedToInsights(
baselineInsights,
currentReportWithTestInsights,
baseline
);
if (baselineInsights.extra?.testsAdded) {
delete baselineInsights.extra.testsAdded;
}
if (baselineInsights.extra?.testsRemoved) {
delete baselineInsights.extra.testsRemoved;
}
return {
...currentReportWithTestInsights,
insights: baselineInsights
};
}
// Annotate the CommonJS export names for ESM import in node:
0 && (module.exports = {
BuilderError,
CTRFError,
CTRF_NAMESPACE,
CURRENT_SPEC_VERSION,
FileError,
ParseError,
REPORT_FORMAT,
ReportBuilder,
SUPPORTED_SPEC_VERSIONS,
SchemaVersionError,
TEST_STATUSES,
TestBuilder,
ValidationError,
addInsights,
calculateSummary,
filterTests,
findTest,
generateReportId,
generateTestId,
getCurrentSpecVersion,
getSchema,
getSupportedSpecVersions,
hasInsights,
isCTRFReport,
isRetryAttempt,
isTest,
isTestFlaky,
isTestStatus,
isValid,
merge,
parse,
schema,
stringify,
validate,
validateStrict
});
/**
* CTRF TypeScript Types
* Generated from the CTRF JSON Schema specification
*/
/**
* The root CTRF report object
*
* @group Core Types
*/
interface CTRFReport {
/** Must be 'CTRF' */
reportFormat: 'CTRF';
/** Semantic version of the CTRF specification */
specVersion: string;
/** Unique identifier for this report (UUID v4) */
reportId?: string;
/** ISO 8601 timestamp when the report was generated */
timestamp?: string;
/** Name of the tool/library that generated this report */
generatedBy?: string;
/** The test results */
results: Results;
/** Run-level insights computed from historical data */
insights?: Insights;
/** Reference to a baseline report for comparison */
baseline?: Baseline;
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* Container for test results
*
* @group Core Types
*/
interface Results {
/** Information about the test tool */
tool: Tool;
/** Aggregated test statistics */
summary: Summary;
/** Array of individual test results */
tests: Test[];
/** Environment information */
environment?: Environment;
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* Test tool information
*
* @group Core Types
*/
interface Tool {
/** Name of the test tool (e.g., 'jest', 'playwright') */
name: string;
/** Version of the test tool */
version?: string;
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* Aggregated test statistics
*
* @group Core Types
*/
interface Summary {
/** Total number of tests */
tests: number;
/** Number of passed tests */
passed: number;
/** Number of failed tests */
failed: number;
/** Number of skipped tests */
skipped: number;
/** Number of pending tests */
pending: number;
/** Number of tests with other status */
other: number;
/** Number of flaky tests */
flaky?: number;
/** Number of test suites */
suites?: number;
/** Start timestamp (Unix epoch milliseconds) */
start: number;
/** Stop timestamp (Unix epoch milliseconds) */
stop: number;
/** Total duration in milliseconds */
duration?: number;
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* Individual test result
*
* @group Core Types
*/
interface Test {
/** Unique test identifier (UUID) */
id?: string;
/** Test name */
name: string;
/** Test execution status */
status: TestStatus;
/** Test duration in milliseconds */
duration: number;
/** Start timestamp (Unix epoch milliseconds) */
start?: number;
/** Stop timestamp (Unix epoch milliseconds) */
stop?: number;
/** Test suite hierarchy */
suite?: string[];
/** Error message (for failed tests) */
message?: string;
/** Stack trace (for failed tests) */
trace?: string;
/** Code snippet where failure occurred */
snippet?: string;
/** AI-generated analysis or suggestion */
ai?: string;
/** Line number where test is defined or failed */
line?: number;
/** Original status from the test framework */
rawStatus?: string;
/** Tags for categorization */
tags?: string[];
/** Test type (e.g., 'unit', 'integration', 'e2e') */
type?: string;
/** Path to the test file */
filePath?: string;
/** Number of retry attempts */
retries?: number;
/** Details of each retry attempt */
retryAttempts?: RetryAttempt[];
/** Whether the test is flaky */
flaky?: boolean;
/** Standard output captured during test */
stdout?: string[];
/** Standard error captured during test */
stderr?: string[];
/** Thread/worker ID that ran this test */
threadId?: string;
/** Browser name (for browser tests) */
browser?: string;
/** Device name (for device tests) */
device?: string;
/** Base64 encoded screenshot */
screenshot?: string;
/** File attachments */
attachments?: Attachment[];
/** Test parameters (for parameterized tests) */
parameters?: Record<string, unknown>;
/** Test steps */
steps?: Step[];
/** Test-level insights */
insights?: TestInsights;
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* Test status enum
*
* @group Core Types
*/
type TestStatus = 'passed' | 'failed' | 'skipped' | 'pending' | 'other';
/**
* Details of a test retry attempt
*
* @group Core Types
*/
interface RetryAttempt {
/** Attempt number (1-indexed) */
attempt: number;
/** Status of this attempt */
status: TestStatus;
/** Duration of this attempt in milliseconds */
duration?: number;
/** Error message */
message?: string;
/** Stack trace */
trace?: string;
/** Line number */
line?: number;
/** Code snippet */
snippet?: string;
/** Standard output */
stdout?: string[];
/** Standard error */
stderr?: string[];
/** Start timestamp */
start?: number;
/** Stop timestamp */
stop?: number;
/** Attachments for this attempt */
attachments?: Attachment[];
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* File attachment
*
* @group Core Types
*/
interface Attachment {
/** Attachment name */
name: string;
/** MIME content type */
contentType: string;
/** Path to the attachment file */
path: string;
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* Test step
*
* @group Core Types
*/
interface Step {
/** Step name */
name: string;
/** Step status */
status: TestStatus;
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* Environment information
*
* @group Core Types
*/
interface Environment {
/** Custom report name */
reportName?: string;
/** Application name */
appName?: string;
/** Application version */
appVersion?: string;
/** Build identifier */
buildId?: string;
/** Build name */
buildName?: string;
/** Build number */
buildNumber?: number;
/** Build URL */
buildUrl?: string;
/** Repository name */
repositoryName?: string;
/** Repository URL */
repositoryUrl?: string;
/** Git commit SHA */
commit?: string;
/** Git branch name */
branchName?: string;
/** Operating system platform */
osPlatform?: string;
/** Operating system release */
osRelease?: string;
/** Operating system version */
osVersion?: string;
/** Test environment name */
testEnvironment?: string;
/** Whether the environment is healthy */
healthy?: boolean;
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* Run-level insights computed from historical data
*
* @group Insights
*/
interface Insights {
/** Pass rate metric */
passRate?: MetricDelta;
/** Fail rate metric */
failRate?: MetricDelta;
/** Flaky rate metric */
flakyRate?: MetricDelta;
/** Average run duration metric */
averageRunDuration?: MetricDelta;
/** 95th percentile run duration metric */
p95RunDuration?: MetricDelta;
/** Average test duration metric */
averageTestDuration?: MetricDelta;
/** Number of historical runs analyzed */
runsAnalyzed?: number;
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* Test-level insights computed from historical data
*
* @group Insights
*/
interface TestInsights {
/** Pass rate metric */
passRate?: MetricDelta;
/** Fail rate metric */
failRate?: MetricDelta;
/** Flaky rate metric */
flakyRate?: MetricDelta;
/** Average test duration metric */
averageTestDuration?: MetricDelta;
/** 95th percentile test duration metric */
p95TestDuration?: MetricDelta;
/** Number of runs this test was executed in */
executedInRuns?: number;
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* Metric with current value, baseline, and change
*
* @group Insights
*/
interface MetricDelta {
/** Current value */
current?: number;
/** Baseline value for comparison */
baseline?: number;
/** Change from baseline (current - baseline) */
change?: number;
}
/**
* Reference to a baseline report
*
* @group Core Types
*/
interface Baseline {
/** Report ID of the baseline report */
reportId: string;
/** Timestamp of the baseline report */
timestamp?: string;
/** Source description (e.g., 'main-branch', 'previous-run') */
source?: string;
/** Build number of the baseline */
buildNumber?: number;
/** Build name of the baseline */
buildName?: string;
/** Build URL of the baseline */
buildUrl?: string;
/** Git commit of the baseline */
commit?: string;
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* Result of schema validation
*
* @group Validation Options
*/
interface ValidationResult {
/** Whether the report is valid */
valid: boolean;
/** Array of validation errors */
errors: ValidationErrorDetail[];
}
/**
* Details of a validation error
*
* @group Validation Options
*/
interface ValidationErrorDetail {
/** Human-readable error message */
message: string;
/** JSON path to the error location */
path: string;
/** JSON Schema keyword that failed */
keyword: string;
}
/**
* Options for merging reports
*
* @group Merge Options
*/
interface MergeOptions {
/** Remove duplicate tests by ID */
deduplicateTests?: boolean;
/** Recalculate summary from merged tests */
mergeSummary?: boolean;
/** Strategy for handling environments */
preserveEnvironment?: 'first' | 'last' | 'merge';
}
/**
* Criteria for filtering and finding tests.
*
* @group Query & Filter Options
*/
interface FilterCriteria {
/** Filter by test ID (UUID) */
id?: string;
/** Filter by test name */
name?: string;
/** Filter by status */
status?: TestStatus | TestStatus[];
/** Filter by tags */
tags?: string | string[];
/** Filter by suite */
suite?: string | string[];
/** Filter by flaky flag */
flaky?: boolean;
/** Filter by browser */
browser?: string;
/** Filter by device */
device?: string;
}
/**
* Options for insights calculation
*
* @group Insights Options
*/
interface InsightsOptions {
/** Baseline report for comparison */
baseline?: CTRFReport;
/** Number of historical reports to analyze */
window?: number;
}
/**
* Options for ReportBuilder
*
* @group Builder Options
*/
interface ReportBuilderOptions {
/** Automatically generate report ID */
autoGenerateId?: boolean;
/** Automatically set timestamp */
autoTimestamp?: boolean;
}
/**
* Options for TestBuilder
*
* @group Builder Options
*/
interface TestBuilderOptions {
/** Automatically generate test ID */
autoGenerateId?: boolean;
}
/**
* Options for calculating summary
*
* @group Core Options
*/
interface SummaryOptions {
/** Start timestamp */
start?: number;
/** Stop timestamp */
stop?: number;
}
/**
* Options for parsing JSON
*
* @group Core Options
*/
interface ParseOptions {
/** Validate after parsing */
validate?: boolean;
}
/**
* Options for stringifying to JSON
*
* @group Core Options
*/
interface StringifyOptions {
/** Pretty print with indentation */
pretty?: boolean;
/** Number of spaces for indentation (default: 2) */
indent?: number;
}
/**
* Options for validation
*
* @group Validation Options
*/
interface ValidateOptions {
/** Specific spec version to validate against */
specVersion?: string;
}
/**
* CTRF Constants
*/
/** The CTRF report format identifier */
declare const REPORT_FORMAT: "CTRF";
/** Current spec version */
declare const CURRENT_SPEC_VERSION = "0.0.0";
/** All valid test statuses */
declare const TEST_STATUSES: readonly ["passed", "failed", "skipped", "pending", "other"];
/** Supported specification versions */
declare const SUPPORTED_SPEC_VERSIONS: readonly ["0.0.0"];
/** CTRF namespace UUID for deterministic ID generation */
declare const CTRF_NAMESPACE = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
/**
* CTRF Validation
*/
/**
* Validate a CTRF report against the JSON schema.
*
* @group Core Operations
* @param report - The object to validate
* @param options - Validation options (e.g., specific spec version)
* @returns Validation result containing `valid` boolean and `errors` array
*
* @example
* ```typescript
* const result = validate(report);
* if (!result.valid) {
* console.log(result.errors);
* }
*
* // Validate against specific version
* const result = validate(report, { specVersion: '1.0.0' });
* ```
*/
declare function validate(report: unknown, options?: ValidateOptions): ValidationResult;
/**
*
* @group Core Operations
* Check if a report is valid (type guard).
*
* @param report - The object to validate
* @returns true if the report is a valid CTRFReport
*
* @example
* ```typescript
* if (isValid(report)) {
* // TypeScript now knows report is CTRFReport
* console.log(report.results.summary.passed);
* }
* ```
*/
declare function isValid(report: unknown): report is CTRFReport;
/**
*
* @group Core Operations
* Validate a report and throw if invalid (assertion).
*
* @param report - The object to validate
* @throws ValidationError if the report is invalid
*
* @example
* ```typescript
* try {
* validateStrict(report);
* // TypeScript now knows report is CTRFReport
* } catch (e) {
* if (e instanceof ValidationError) {
* console.log(e.errors);
* }
* }
* ```
*/
declare function validateStrict(report: unknown): asserts report is CTRFReport;
/**
*
* @group Type Guards
* Checks if an object has the basic structure of a CTRF report.
* This is a quick, lightweight check that doesn't validate against the full schema.
*
* @param report - The object to check
* @returns true if the object appears to be a CTRF report
*
* @example
* ```typescript
* if (isCTRFReport(data)) {
* // data has reportFormat: 'CTRF'
* }
* ```
*/
declare function isCTRFReport(report: unknown): report is {
reportFormat: 'CTRF';
};
/**
*
* @group Type Guards
* Type guard for Test objects.
*
* @param obj - Object to check
* @returns true if the object is a Test
*/
declare function isTest(obj: unknown): obj is {
name: string;
status: string;
duration: number;
};
/**
*
* @group Type Guards
* Type guard for TestStatus values.
*
* @param value - Value to check
* @returns true if the value is a valid TestStatus
*/
declare function isTestStatus(value: unknown): value is (typeof TEST_STATUSES)[number];
/**
*
* @group Type Guards
* Type guard for RetryAttempt objects.
*
* @param obj - Object to check
* @returns true if the object is a RetryAttempt
*/
declare function isRetryAttempt(obj: unknown): obj is {
attempt: number;
status: string;
};
/**
*
* @group Type Guards
* Check if a report has insights.
*
* @param report - The report to check
* @returns true if the report has insights
*/
declare function hasInsights(report: CTRFReport): boolean;
/**
* CTRF Summary Calculation
*/
/**
*
* @group Core Operations
* Calculate summary statistics from an array of tests.
*
* @param tests - Array of test results
* @param options - Optional timing information
* @returns Calculated summary object
*
* @example
* ```typescript
* const summary = calculateSummary(tests);
*
* // With timing
* const summary = calculateSummary(tests, {
* start: 1704067200000,
* stop: 1704067260000
* });
* ```
*/
declare function calculateSummary(tests: Test[], options?: SummaryOptions): Summary;
/**
* CTRF Report and Test Builders
* Fluent API for constructing CTRF reports
*/
/**
*
* @group Builders
* Fluent builder for constructing CTRF reports.
*
* @example
* ```typescript
* const report = new ReportBuilder()
* .specVersion('1.0.0')
* .tool({ name: 'jest', version: '29.0.0' })
* .environment({ branchName: 'main' })
* .addTest(
* new TestBuilder()
* .name('should add numbers')
* .status('passed')
* .duration(150)
* .build()
* )
* .build();
* ```
*/
declare class ReportBuilder {
private options;
private _specVersion;
private _reportId?;
private _timestamp?;
private _generatedBy?;
private _tool?;
private _environment?;
private _tests;
private _insights?;
private _baseline?;
private _extra?;
private _summaryOverrides?;
constructor(options?: ReportBuilderOptions);
/**
* Set the spec version.
*/
specVersion(version: string): this;
/**
* Set or generate the report ID.
* @param uuid - UUID to use, or undefined to auto-generate
*/
reportId(uuid?: string): this;
/**
* Set the timestamp.
* @param date - Date to use, or undefined for current time
*/
timestamp(date?: Date | string): this;
/**
* Set the generator name.
*/
generatedBy(name: string): this;
/**
* Set the tool information.
*/
tool(tool: Tool): this;
/**
* Set the environment information.
*/
environment(env: Environment): this;
/**
* Add a single test.
*/
addTest(test: Test): this;
/**
* Add multiple tests.
*/
addTests(tests: Test[]): this;
/**
* Set run-level insights.
*/
insights(insights: Insights): this;
/**
* Set the baseline reference.
*/
baseline(baseline: Baseline): this;
/**
* Set extra metadata.
*/
extra(data: Record<string, unknown>): this;
/**
* Override specific summary fields.
* Useful when you want to set specific timing or counts.
*/
summaryOverrides(overrides: Partial<Summary>): this;
/**
* Build and return the CTRF report.
* @throws BuilderError if required fields are missing
*/
build(): CTRFReport;
}
/**
*
* @group Builders
* Fluent builder for constructing Test objects.
*
* @example
* ```typescript
* const test = new TestBuilder()
* .name('should add numbers')
* .status('passed')
* .duration(150)
* .suite(['math', 'addition'])
* .build();
* ```
*/
declare class TestBuilder {
private options;
private _id?;
private _name?;
private _status?;
private _duration?;
private _start?;
private _stop?;
private _suite?;
private _message?;
private _trace?;
private _snippet?;
private _ai?;
private _line?;
private _rawStatus?;
private _tags?;
private _type?;
private _filePath?;
private _retries?;
private _retryAttempts?;
private _flaky?;
private _stdout?;
private _stderr?;
private _threadId?;
private _browser?;
private _device?;
private _screenshot?;
private _attachments?;
private _parameters?;
private _steps?;
private _insights?;
private _extra?;
constructor(options?: TestBuilderOptions);
/**
* Set or generate the test ID.
* @param uuid - UUID to use, or undefined to auto-generate based on properties
*/
id(uuid?: string): this;
/**
* Set the test name.
*/
name(name: string): this;
/**
* Set the test status.
*/
status(status: TestStatus): this;
/**
* Set the duration in milliseconds.
*/
duration(ms: number): this;
/**
* Set the start timestamp.
*/
start(timestamp: number): this;
/**
* Set the stop timestamp.
*/
stop(timestamp: number): this;
/**
* Set the suite hierarchy.
*/
suite(suites: string[]): this;
/**
* Set the error message.
*/
message(msg: string): this;
/**
* Set the stack trace.
*/
trace(trace: string): this;
/**
* Set the code snippet.
*/
snippet(code: string): this;
/**
* Set AI-generated analysis.
*/
ai(analysis: string): this;
/**
* Set the line number.
*/
line(num: number): this;
/**
* Set the raw status from the test framework.
*/
rawStatus(status: string): this;
/**
* Set tags.
*/
tags(tags: string[]): this;
/**
* Set test type.
*/
type(type: string): this;
/**
* Set file path.
*/
filePath(path: string): this;
/**
* Set retry count.
*/
retries(count: number): this;
/**
* Add a retry attempt.
*/
addRetryAttempt(attempt: RetryAttempt): this;
/**
* Mark as flaky.
*/
flaky(isFlaky?: boolean): this;
/**
* Set stdout.
*/
stdout(lines: string[]): this;
/**
* Set stderr.
*/
stderr(lines: string[]): this;
/**
* Set thread ID.
*/
threadId(id: string): this;
/**
* Set browser name.
*/
browser(name: string): this;
/**
* Set device name.
*/
device(name: string): this;
/**
* Set screenshot (base64).
*/
screenshot(base64: string): this;
/**
* Add an attachment.
*/
addAttachment(attachment: Attachment): this;
/**
* Set parameters.
*/
parameters(params: Record<string, unknown>): this;
/**
* Add a step.
*/
addStep(step: Step): this;
/**
* Set test-level insights.
*/
insights(insights: TestInsights): this;
/**
* Set extra metadata.
*/
extra(data: Record<string, unknown>): this;
/**
* Build and return the Test object.
* @throws BuilderError if required fields are missing
*/
build(): Test;
}
/**
* CTRF Test ID Generation
*/
/**
*
* @group ID Generation
* Generate a deterministic UUID v5 for a test based on its properties.
* The same inputs will always produce the same UUID, enabling
* cross-run analysis and test identification.
*
* @param properties - Test properties to generate ID from
* @param properties.name - Test name (required)
* @param properties.suite - Suite hierarchy (optional)
* @param properties.filePath - File path (optional)
* @returns A deterministic UUID v5 string
*
* @example
* ```typescript
* const id = generateTestId({
* name: 'should add numbers',
* suite: ['math', 'addition'],
* filePath: 'tests/math.test.ts'
* });
* // Always returns the same UUID for these inputs
* ```
*/
declare function generateTestId(properties: {
name: string;
suite?: string[];
filePath?: string;
}): string;
/**
*
* @group ID Generation
* Generate a random UUID v4 for report identification.
*
* @returns A random UUID v4 string
*
* @example
* ```typescript
* const reportId = generateReportId();
* // => 'f47ac10b-58cc-4372-a567-0e02b2c3d479'
* ```
*/
declare function generateReportId(): string;
/**
* CTRF Parsing and Serialization
*/
/**
*
* @group Core Operations
* Parse a JSON string into a CTRFReport.
*
* @param json - JSON string to parse
* @param options - Parse options (e.g., enable validation)
* @returns Parsed CTRFReport object
* @throws ParseError if JSON is invalid
* @throws ValidationError if validation is enabled and fails
*
* @example
* ```typescript
* const report = parse(jsonString);
*
* // With validation
* const report = parse(jsonString, { validate: true });
* ```
*/
declare function parse(json: string, options?: ParseOptions): CTRFReport;
/**
*
* @group Core Operations
* Serialize a CTRFReport to a JSON string.
*
* @param report - The CTRF report to serialize
* @param options - Stringify options (pretty print, indent)
* @returns JSON string representation
*
* @example
* ```typescript
* const json = stringify(report);
*
* // Pretty print
* const json = stringify(report, { pretty: true });
*
* // Custom indent
* const json = stringify(report, { pretty: true, indent: 4 });
* ```
*/
declare function stringify(report: CTRFReport, options?: StringifyOptions): string;
/**
* CTRF Report Merging
*/
/**
*
* @group Merge
* Merge multiple CTRF reports into a single report.
* Useful for combining results from parallel or sharded test runs.
*
* @param reports - Array of CTRF reports to merge
* @param options - Merge options (deduplication, environment handling)
* @returns A new merged CTRFReport
* @throws Error if no reports are provided
*
* @example
* ```typescript
* const merged = merge([report1, report2, report3]);
*
* // With deduplication by test ID
* const merged = merge(reports, { deduplicateTests: true });
*
* // Keep first environment only
* const merged = merge(reports, { preserveEnvironment: 'first' });
* ```
*/
declare function merge(reports: CTRFReport[], options?: MergeOptions): CTRFReport;
/**
* CTRF Filtering and Querying
*/
/**
*
* @group Query & Filter
* Filter tests in a report by criteria.
*
* @param report - The CTRF report containing tests to filter
* @param criteria - Filter criteria (status, tags, suite, flaky, browser, device)
* @returns Array of tests matching all specified criteria
*
* @example
* ```typescript
* // Filter by status
* const failed = filterTests(report, { status: 'failed' });
*
* // Filter by multiple criteria
* const filtered = filterTests(report, {
* status: ['failed', 'skipped'],
* tags: ['smoke'],
* flaky: true
* });
* ```
*/
declare function filterTests(report: CTRFReport, criteria: FilterCriteria): Test[];
/**
*
* @group Query & Filter
* Find a single test in a report.
*
* @param report - The CTRF report to search
* @param criteria - Filter criteria including id, name, status, tags, etc.
* @returns The first matching test, or undefined if not found
*
* @example
* ```typescript
* // Find by ID
* const test = findTest(report, { id: 'uuid' });
*
* // Find by name
* const test = findTest(report, { name: 'should login' });
*
* // Find by multiple criteria
* const test = findTest(report, { status: 'failed', flaky: true });
* ```
*/
declare function findTest(report: CTRFReport, criteria: FilterCriteria): Test | undefined;
/**
* CTRF Insights Calculation
*
* This module replicates the functionality of the existing run-insights.ts
* while using the reference implementation's type system and API signatures.
*/
/**
*
* @group Insights
* Determines if a test is flaky based on the CTRF specification.
*
* A test is considered flaky if:
* - The `flaky` field is explicitly set to `true`, OR
* - The test has retries > 0 AND final status is 'passed'
*
* @param test - The test to check
* @returns true if the test is flaky
*
* @example
* ```typescript
* if (isTestFlaky(test)) {
* console.log('Test is flaky:', test.name);
* }
* ```
*/
declare function isTestFlaky(test: Test): boolean;
/**
*
* @group Insights
* Add insights to a CTRF report using historical data.
*
* Computes run-level and test-level insights according to the CTRF specification,
* including pass rate, fail rate, flaky rate, and duration metrics.
*
* @param report - The current report to enrich with insights
* @param historicalReports - Array of previous reports for trend analysis
* @param options - Options including baseline for comparison
* @returns A new report with insights populated
*
* @example
* ```typescript
* // Basic usage
* const reportWithInsights = addInsights(currentReport, previousReports);
*
* // With baseline comparison
* const reportWithInsights = addInsights(currentReport, previousReports, {
* baseline: baselineReport
* });
* ```
*/
declare function addInsights(report: CTRFReport, historicalReports?: CTRFReport[], options?: InsightsOptions): CTRFReport;
/**
* CTRF Schema Access
*/
/**
*
* @group Schema & Versioning
* The current version CTRF JSON Schema object.
*
* @example
* ```typescript
* import { schema } from 'ctrf';
* console.log(schema.$schema);
* ```
*/
declare const schema: object;
/**
*
* @group Schema & Versioning
* Get the JSON Schema for a specific CTRF spec version.
*
* @param version - The spec version (MAJOR.MINOR.PATCH) to get the schema for
* @returns The JSON Schema object for that version
* @throws SchemaVersionError if the version is not supported
*
* @example
* ```typescript
* const v0_0Schema = getSchema('0.0.0');
* const v1_0Schema = getSchema('1.0.0');
* ```
*/
declare function getSchema(version: string): object;
/**
*
* @group Schema & Versioning
* Get the current spec version.
*
* @returns The current spec version string
*/
declare function getCurrentSpecVersion(): string;
/**
*
* @group Schema & Versioning
* Get all supported spec versions.
*
* @returns Array of supported version strings
*/
declare function getSupportedSpecVersions(): readonly string[];
/**
* CTRF Error Classes
*/
/**
*
* @group Errors
* Base error class for all CTRF errors.
* All CTRF-specific errors extend this class.
*/
declare class CTRFError extends Error {
constructor(message: string);
}
/**
*
* @group Errors
* Error thrown when schema validation fails.
* Contains detailed error information for each validation issue.
*/
declare class ValidationError extends CTRFError {
/** Detailed validation errors */
readonly errors: ValidationErrorDetail[];
constructor(message: string, errors?: ValidationErrorDetail[]);
}
/**
*
* @group Errors
* Error thrown when JSON parsing fails.
*/
declare class ParseError extends CTRFError {
/** The original error that caused the parse failure */
readonly cause?: Error;
constructor(message: string, cause?: Error);
}
/**
*
* @group Errors
* Error thrown when an unsupported CTRF specification version is encountered.
*/
declare class SchemaVersionError extends CTRFError {
/** The unsupported version */
readonly version: string;
/** Supported versions */
readonly supportedVersions: string[];
constructor(version: string, supportedVersions: string[]);
}
/**
*
* @group Errors
* Error thrown when a file read or write operation fails.
*/
declare class FileError extends CTRFError {
/** The file path that caused the error */
readonly filePath: string;
/** The original error */
readonly cause?: Error;
constructor(message: string, filePath: string, cause?: Error);
}
/**
*
* @group Errors
* Error thrown when building a report or test fails due to missing required fields.
*/
declare class BuilderError extends CTRFError {
constructor(message: string);
}
export { type Attachment, type Baseline, BuilderError, CTRFError, type CTRFReport, CTRF_NAMESPACE, CURRENT_SPEC_VERSION, type Environment, FileError, type FilterCriteria, type Insights, type InsightsOptions, type MergeOptions, type MetricDelta, ParseError, type ParseOptions, REPORT_FORMAT, ReportBuilder, type ReportBuilderOptions, type Results, type RetryAttempt, SUPPORTED_SPEC_VERSIONS, SchemaVersionError, type Step, type StringifyOptions, type Summary, type SummaryOptions, TEST_STATUSES, type Test, TestBuilder, type TestBuilderOptions, type TestInsights, type TestStatus, type Tool, type ValidateOptions, ValidationError, type ValidationErrorDetail, type ValidationResult, addInsights, calculateSummary, filterTests, findTest, generateReportId, generateTestId, getCurrentSpecVersion, getSchema, getSupportedSpecVersions, hasInsights, isCTRFReport, isRetryAttempt, isTest, isTestFlaky, isTestStatus, isValid, merge, parse, schema, stringify, validate, validateStrict };
+168
-43
#!/usr/bin/env node
import yargs from 'yargs/yargs';
import { hideBin } from 'yargs/helpers';
import { mergeReports } from './merge.js';
import { identifyFlakyTests } from './flaky.js';
console.warn('⚠️ DEPRECATION NOTICE: This CLI is deprecated and will be removed in v1.');
console.warn('📦 Please use the standalone ctrf-cli package instead:');
console.warn(' https://github.com/ctrf-io/ctrf-cli\n');
void yargs(hideBin(process.argv))
.command('merge <directory>', 'Merge CTRF reports into a single report', yargs => {
return yargs
.positional('directory', {
describe: 'Directory of the CTRF reports',
type: 'string',
demandOption: true,
})
.option('output', {
alias: 'o',
describe: 'Output file name for merged report',
type: 'string',
default: 'ctrf-report.json',
})
.option('output-dir', {
alias: 'd',
describe: 'Output directory for merged report',
type: 'string',
})
.option('keep-reports', {
alias: 'k',
describe: 'Keep existing reports after merging',
type: 'boolean',
default: false,
// src/cli/cli.ts
import yargs from "yargs/yargs";
import { hideBin } from "yargs/helpers";
// src/cli/merge.ts
import fs from "fs";
import path from "path";
async function mergeReports(directory, output, outputDir, keepReports) {
try {
const directoryPath = path.resolve(directory);
const outputFileName = output;
const resolvedOutputDir = outputDir ? path.resolve(outputDir) : directoryPath;
const outputPath = path.join(resolvedOutputDir, outputFileName);
console.log("Merging CTRF reports...");
const files = fs.readdirSync(directoryPath);
files.forEach((file) => {
console.log("Found file:", file);
});
}, async (argv) => {
await mergeReports(argv.directory, argv.output, argv['output-dir'], argv['keep-reports']);
})
.command('flaky <file>', 'Identify flaky tests from a CTRF report file', yargs => {
return yargs.positional('file', {
describe: 'CTRF report file',
type: 'string',
demandOption: true,
const ctrfReportFiles = files.filter((file) => {
try {
if (path.extname(file) !== ".json") {
console.log(`Skipping non-CTRF file: ${file}`);
return false;
}
const filePath = path.join(directoryPath, file);
const fileContent = fs.readFileSync(filePath, "utf8");
const jsonData = JSON.parse(fileContent);
if (!("results" in jsonData)) {
console.log(`Skipping non-CTRF file: ${file}`);
return false;
}
return true;
} catch (error) {
console.error(`Error reading JSON file '${file}':`, error);
return false;
}
});
}, async (argv) => {
if (ctrfReportFiles.length === 0) {
console.log("No CTRF reports found in the specified directory.");
return;
}
if (!fs.existsSync(resolvedOutputDir)) {
fs.mkdirSync(resolvedOutputDir, { recursive: true });
console.log(`Created output directory: ${resolvedOutputDir}`);
}
const mergedReport = ctrfReportFiles.map((file) => {
console.log("Merging report:", file);
const filePath = path.join(directoryPath, file);
const fileContent = fs.readFileSync(filePath, "utf8");
return JSON.parse(fileContent);
}).reduce(
(acc, curr) => {
if (!acc.results) {
return curr;
}
acc.results.summary.tests += curr.results.summary.tests;
acc.results.summary.passed += curr.results.summary.passed;
acc.results.summary.failed += curr.results.summary.failed;
acc.results.summary.skipped += curr.results.summary.skipped;
acc.results.summary.pending += curr.results.summary.pending;
acc.results.summary.other += curr.results.summary.other;
acc.results.tests.push(...curr.results.tests);
acc.results.summary.start = Math.min(
acc.results.summary.start,
curr.results.summary.start
);
acc.results.summary.stop = Math.max(
acc.results.summary.stop,
curr.results.summary.stop
);
return acc;
},
{ results: null }
);
fs.writeFileSync(outputPath, JSON.stringify(mergedReport, null, 2));
if (!keepReports) {
ctrfReportFiles.forEach((file) => {
const filePath = path.join(directoryPath, file);
if (file !== outputFileName) {
fs.unlinkSync(filePath);
}
});
}
console.log("CTRF reports merged successfully.");
console.log(`Merged report saved to: ${outputPath}`);
} catch (error) {
console.error("Error merging CTRF reports:", error);
}
}
// src/cli/flaky.ts
import fs2 from "fs";
import path2 from "path";
async function identifyFlakyTests(filePath) {
try {
const resolvedFilePath = path2.resolve(filePath);
if (!fs2.existsSync(resolvedFilePath)) {
console.error(`The file ${resolvedFilePath} does not exist.`);
return;
}
const fileContent = fs2.readFileSync(resolvedFilePath, "utf8");
const report = JSON.parse(fileContent);
const flakyTests = report.results.tests.filter(
(test) => test.flaky === true
);
if (flakyTests.length > 0) {
console.log(`Found ${flakyTests.length} flaky test(s):`);
flakyTests.forEach((test) => {
console.log(`- Test Name: ${test.name}, Retries: ${test.retries}`);
});
} else {
console.log(`No flaky tests found in ${resolvedFilePath}.`);
}
} catch (error) {
console.error("Error identifying flaky tests:", error);
}
}
// src/cli/cli.ts
console.warn(
"\u26A0\uFE0F DEPRECATION NOTICE: This CLI is deprecated and will be removed in v1."
);
console.warn("\u{1F4E6} Please use the standalone ctrf-cli package instead:");
console.warn(" https://github.com/ctrf-io/ctrf-cli\n");
void yargs(hideBin(process.argv)).command(
"merge <directory>",
"Merge CTRF reports into a single report",
(yargs2) => {
return yargs2.positional("directory", {
describe: "Directory of the CTRF reports",
type: "string",
demandOption: true
}).option("output", {
alias: "o",
describe: "Output file name for merged report",
type: "string",
default: "ctrf-report.json"
}).option("output-dir", {
alias: "d",
describe: "Output directory for merged report",
type: "string"
}).option("keep-reports", {
alias: "k",
describe: "Keep existing reports after merging",
type: "boolean",
default: false
});
},
async (argv) => {
await mergeReports(
argv.directory,
argv.output,
argv["output-dir"],
argv["keep-reports"]
);
}
).command(
"flaky <file>",
"Identify flaky tests from a CTRF report file",
(yargs2) => {
return yargs2.positional("file", {
describe: "CTRF report file",
type: "string",
demandOption: true
});
},
async (argv) => {
await identifyFlakyTests(argv.file);
})
.help()
.demandCommand(1, 'You need at least one command before moving on').argv;
}
).help().demandCommand(1, "You need at least one command before moving on").argv;
/**
* CTRF TypeScript SDK - Reference Implementation
* CTRF TypeScript Types
* Generated from the CTRF JSON Schema specification
*/
/**
* The root CTRF report object
*
* A complete TypeScript implementation for working with CTRF (Common Test Report Format) reports.
* @group Core Types
*/
interface CTRFReport {
/** Must be 'CTRF' */
reportFormat: 'CTRF';
/** Semantic version of the CTRF specification */
specVersion: string;
/** Unique identifier for this report (UUID v4) */
reportId?: string;
/** ISO 8601 timestamp when the report was generated */
timestamp?: string;
/** Name of the tool/library that generated this report */
generatedBy?: string;
/** The test results */
results: Results;
/** Run-level insights computed from historical data */
insights?: Insights;
/** Reference to a baseline report for comparison */
baseline?: Baseline;
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* Container for test results
*
* @packageDocumentation
* @group Core Types
*/
export type { CTRFReport, Results, Tool, Summary, Test, Environment, RetryAttempt, Attachment, Step, Insights, TestInsights, MetricDelta, Baseline, TestStatus, ValidationResult, ValidationErrorDetail, MergeOptions, FilterCriteria, InsightsOptions, ReportBuilderOptions, TestBuilderOptions, SummaryOptions, ParseOptions, StringifyOptions, ValidateOptions, } from './types.js';
export { REPORT_FORMAT, CURRENT_SPEC_VERSION, TEST_STATUSES, SUPPORTED_SPEC_VERSIONS, CTRF_NAMESPACE, } from './constants.js';
export { validate, isValid, validateStrict, isCTRFReport, isTest, isTestStatus, isRetryAttempt, hasInsights, } from './validate.js';
export { calculateSummary } from './summary.js';
export { ReportBuilder, TestBuilder } from './builder.js';
export { generateTestId, generateReportId } from './id.js';
export { parse, stringify } from './parse.js';
export { merge } from './merge.js';
export { filterTests, findTest } from './filter.js';
export { addInsights, isTestFlaky } from './insights.js';
export { schema, getSchema, getCurrentSpecVersion, getSupportedSpecVersions, } from './schema.js';
export { CTRFError, ValidationError, ParseError, SchemaVersionError, FileError, BuilderError, } from './errors.js';
interface Results {
/** Information about the test tool */
tool: Tool;
/** Aggregated test statistics */
summary: Summary;
/** Array of individual test results */
tests: Test[];
/** Environment information */
environment?: Environment;
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* Test tool information
*
* @group Core Types
*/
interface Tool {
/** Name of the test tool (e.g., 'jest', 'playwright') */
name: string;
/** Version of the test tool */
version?: string;
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* Aggregated test statistics
*
* @group Core Types
*/
interface Summary {
/** Total number of tests */
tests: number;
/** Number of passed tests */
passed: number;
/** Number of failed tests */
failed: number;
/** Number of skipped tests */
skipped: number;
/** Number of pending tests */
pending: number;
/** Number of tests with other status */
other: number;
/** Number of flaky tests */
flaky?: number;
/** Number of test suites */
suites?: number;
/** Start timestamp (Unix epoch milliseconds) */
start: number;
/** Stop timestamp (Unix epoch milliseconds) */
stop: number;
/** Total duration in milliseconds */
duration?: number;
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* Individual test result
*
* @group Core Types
*/
interface Test {
/** Unique test identifier (UUID) */
id?: string;
/** Test name */
name: string;
/** Test execution status */
status: TestStatus;
/** Test duration in milliseconds */
duration: number;
/** Start timestamp (Unix epoch milliseconds) */
start?: number;
/** Stop timestamp (Unix epoch milliseconds) */
stop?: number;
/** Test suite hierarchy */
suite?: string[];
/** Error message (for failed tests) */
message?: string;
/** Stack trace (for failed tests) */
trace?: string;
/** Code snippet where failure occurred */
snippet?: string;
/** AI-generated analysis or suggestion */
ai?: string;
/** Line number where test is defined or failed */
line?: number;
/** Original status from the test framework */
rawStatus?: string;
/** Tags for categorization */
tags?: string[];
/** Test type (e.g., 'unit', 'integration', 'e2e') */
type?: string;
/** Path to the test file */
filePath?: string;
/** Number of retry attempts */
retries?: number;
/** Details of each retry attempt */
retryAttempts?: RetryAttempt[];
/** Whether the test is flaky */
flaky?: boolean;
/** Standard output captured during test */
stdout?: string[];
/** Standard error captured during test */
stderr?: string[];
/** Thread/worker ID that ran this test */
threadId?: string;
/** Browser name (for browser tests) */
browser?: string;
/** Device name (for device tests) */
device?: string;
/** Base64 encoded screenshot */
screenshot?: string;
/** File attachments */
attachments?: Attachment[];
/** Test parameters (for parameterized tests) */
parameters?: Record<string, unknown>;
/** Test steps */
steps?: Step[];
/** Test-level insights */
insights?: TestInsights;
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* Test status enum
*
* @group Core Types
*/
type TestStatus = 'passed' | 'failed' | 'skipped' | 'pending' | 'other';
/**
* Details of a test retry attempt
*
* @group Core Types
*/
interface RetryAttempt {
/** Attempt number (1-indexed) */
attempt: number;
/** Status of this attempt */
status: TestStatus;
/** Duration of this attempt in milliseconds */
duration?: number;
/** Error message */
message?: string;
/** Stack trace */
trace?: string;
/** Line number */
line?: number;
/** Code snippet */
snippet?: string;
/** Standard output */
stdout?: string[];
/** Standard error */
stderr?: string[];
/** Start timestamp */
start?: number;
/** Stop timestamp */
stop?: number;
/** Attachments for this attempt */
attachments?: Attachment[];
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* File attachment
*
* @group Core Types
*/
interface Attachment {
/** Attachment name */
name: string;
/** MIME content type */
contentType: string;
/** Path to the attachment file */
path: string;
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* Test step
*
* @group Core Types
*/
interface Step {
/** Step name */
name: string;
/** Step status */
status: TestStatus;
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* Environment information
*
* @group Core Types
*/
interface Environment {
/** Custom report name */
reportName?: string;
/** Application name */
appName?: string;
/** Application version */
appVersion?: string;
/** Build identifier */
buildId?: string;
/** Build name */
buildName?: string;
/** Build number */
buildNumber?: number;
/** Build URL */
buildUrl?: string;
/** Repository name */
repositoryName?: string;
/** Repository URL */
repositoryUrl?: string;
/** Git commit SHA */
commit?: string;
/** Git branch name */
branchName?: string;
/** Operating system platform */
osPlatform?: string;
/** Operating system release */
osRelease?: string;
/** Operating system version */
osVersion?: string;
/** Test environment name */
testEnvironment?: string;
/** Whether the environment is healthy */
healthy?: boolean;
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* Run-level insights computed from historical data
*
* @group Insights
*/
interface Insights {
/** Pass rate metric */
passRate?: MetricDelta;
/** Fail rate metric */
failRate?: MetricDelta;
/** Flaky rate metric */
flakyRate?: MetricDelta;
/** Average run duration metric */
averageRunDuration?: MetricDelta;
/** 95th percentile run duration metric */
p95RunDuration?: MetricDelta;
/** Average test duration metric */
averageTestDuration?: MetricDelta;
/** Number of historical runs analyzed */
runsAnalyzed?: number;
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* Test-level insights computed from historical data
*
* @group Insights
*/
interface TestInsights {
/** Pass rate metric */
passRate?: MetricDelta;
/** Fail rate metric */
failRate?: MetricDelta;
/** Flaky rate metric */
flakyRate?: MetricDelta;
/** Average test duration metric */
averageTestDuration?: MetricDelta;
/** 95th percentile test duration metric */
p95TestDuration?: MetricDelta;
/** Number of runs this test was executed in */
executedInRuns?: number;
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* Metric with current value, baseline, and change
*
* @group Insights
*/
interface MetricDelta {
/** Current value */
current?: number;
/** Baseline value for comparison */
baseline?: number;
/** Change from baseline (current - baseline) */
change?: number;
}
/**
* Reference to a baseline report
*
* @group Core Types
*/
interface Baseline {
/** Report ID of the baseline report */
reportId: string;
/** Timestamp of the baseline report */
timestamp?: string;
/** Source description (e.g., 'main-branch', 'previous-run') */
source?: string;
/** Build number of the baseline */
buildNumber?: number;
/** Build name of the baseline */
buildName?: string;
/** Build URL of the baseline */
buildUrl?: string;
/** Git commit of the baseline */
commit?: string;
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* Result of schema validation
*
* @group Validation Options
*/
interface ValidationResult {
/** Whether the report is valid */
valid: boolean;
/** Array of validation errors */
errors: ValidationErrorDetail[];
}
/**
* Details of a validation error
*
* @group Validation Options
*/
interface ValidationErrorDetail {
/** Human-readable error message */
message: string;
/** JSON path to the error location */
path: string;
/** JSON Schema keyword that failed */
keyword: string;
}
/**
* Options for merging reports
*
* @group Merge Options
*/
interface MergeOptions {
/** Remove duplicate tests by ID */
deduplicateTests?: boolean;
/** Recalculate summary from merged tests */
mergeSummary?: boolean;
/** Strategy for handling environments */
preserveEnvironment?: 'first' | 'last' | 'merge';
}
/**
* Criteria for filtering and finding tests.
*
* @group Query & Filter Options
*/
interface FilterCriteria {
/** Filter by test ID (UUID) */
id?: string;
/** Filter by test name */
name?: string;
/** Filter by status */
status?: TestStatus | TestStatus[];
/** Filter by tags */
tags?: string | string[];
/** Filter by suite */
suite?: string | string[];
/** Filter by flaky flag */
flaky?: boolean;
/** Filter by browser */
browser?: string;
/** Filter by device */
device?: string;
}
/**
* Options for insights calculation
*
* @group Insights Options
*/
interface InsightsOptions {
/** Baseline report for comparison */
baseline?: CTRFReport;
/** Number of historical reports to analyze */
window?: number;
}
/**
* Options for ReportBuilder
*
* @group Builder Options
*/
interface ReportBuilderOptions {
/** Automatically generate report ID */
autoGenerateId?: boolean;
/** Automatically set timestamp */
autoTimestamp?: boolean;
}
/**
* Options for TestBuilder
*
* @group Builder Options
*/
interface TestBuilderOptions {
/** Automatically generate test ID */
autoGenerateId?: boolean;
}
/**
* Options for calculating summary
*
* @group Core Options
*/
interface SummaryOptions {
/** Start timestamp */
start?: number;
/** Stop timestamp */
stop?: number;
}
/**
* Options for parsing JSON
*
* @group Core Options
*/
interface ParseOptions {
/** Validate after parsing */
validate?: boolean;
}
/**
* Options for stringifying to JSON
*
* @group Core Options
*/
interface StringifyOptions {
/** Pretty print with indentation */
pretty?: boolean;
/** Number of spaces for indentation (default: 2) */
indent?: number;
}
/**
* Options for validation
*
* @group Validation Options
*/
interface ValidateOptions {
/** Specific spec version to validate against */
specVersion?: string;
}
/**
* CTRF Constants
*/
/** The CTRF report format identifier */
declare const REPORT_FORMAT: "CTRF";
/** Current spec version */
declare const CURRENT_SPEC_VERSION = "0.0.0";
/** All valid test statuses */
declare const TEST_STATUSES: readonly ["passed", "failed", "skipped", "pending", "other"];
/** Supported specification versions */
declare const SUPPORTED_SPEC_VERSIONS: readonly ["0.0.0"];
/** CTRF namespace UUID for deterministic ID generation */
declare const CTRF_NAMESPACE = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
/**
* CTRF Validation
*/
/**
* Validate a CTRF report against the JSON schema.
*
* @group Core Operations
* @param report - The object to validate
* @param options - Validation options (e.g., specific spec version)
* @returns Validation result containing `valid` boolean and `errors` array
*
* @example
* ```typescript
* const result = validate(report);
* if (!result.valid) {
* console.log(result.errors);
* }
*
* // Validate against specific version
* const result = validate(report, { specVersion: '1.0.0' });
* ```
*/
declare function validate(report: unknown, options?: ValidateOptions): ValidationResult;
/**
*
* @group Core Operations
* Check if a report is valid (type guard).
*
* @param report - The object to validate
* @returns true if the report is a valid CTRFReport
*
* @example
* ```typescript
* if (isValid(report)) {
* // TypeScript now knows report is CTRFReport
* console.log(report.results.summary.passed);
* }
* ```
*/
declare function isValid(report: unknown): report is CTRFReport;
/**
*
* @group Core Operations
* Validate a report and throw if invalid (assertion).
*
* @param report - The object to validate
* @throws ValidationError if the report is invalid
*
* @example
* ```typescript
* try {
* validateStrict(report);
* // TypeScript now knows report is CTRFReport
* } catch (e) {
* if (e instanceof ValidationError) {
* console.log(e.errors);
* }
* }
* ```
*/
declare function validateStrict(report: unknown): asserts report is CTRFReport;
/**
*
* @group Type Guards
* Checks if an object has the basic structure of a CTRF report.
* This is a quick, lightweight check that doesn't validate against the full schema.
*
* @param report - The object to check
* @returns true if the object appears to be a CTRF report
*
* @example
* ```typescript
* if (isCTRFReport(data)) {
* // data has reportFormat: 'CTRF'
* }
* ```
*/
declare function isCTRFReport(report: unknown): report is {
reportFormat: 'CTRF';
};
/**
*
* @group Type Guards
* Type guard for Test objects.
*
* @param obj - Object to check
* @returns true if the object is a Test
*/
declare function isTest(obj: unknown): obj is {
name: string;
status: string;
duration: number;
};
/**
*
* @group Type Guards
* Type guard for TestStatus values.
*
* @param value - Value to check
* @returns true if the value is a valid TestStatus
*/
declare function isTestStatus(value: unknown): value is (typeof TEST_STATUSES)[number];
/**
*
* @group Type Guards
* Type guard for RetryAttempt objects.
*
* @param obj - Object to check
* @returns true if the object is a RetryAttempt
*/
declare function isRetryAttempt(obj: unknown): obj is {
attempt: number;
status: string;
};
/**
*
* @group Type Guards
* Check if a report has insights.
*
* @param report - The report to check
* @returns true if the report has insights
*/
declare function hasInsights(report: CTRFReport): boolean;
/**
* CTRF Summary Calculation
*/
/**
*
* @group Core Operations
* Calculate summary statistics from an array of tests.
*
* @param tests - Array of test results
* @param options - Optional timing information
* @returns Calculated summary object
*
* @example
* ```typescript
* const summary = calculateSummary(tests);
*
* // With timing
* const summary = calculateSummary(tests, {
* start: 1704067200000,
* stop: 1704067260000
* });
* ```
*/
declare function calculateSummary(tests: Test[], options?: SummaryOptions): Summary;
/**
* CTRF Report and Test Builders
* Fluent API for constructing CTRF reports
*/
/**
*
* @group Builders
* Fluent builder for constructing CTRF reports.
*
* @example
* ```typescript
* const report = new ReportBuilder()
* .specVersion('1.0.0')
* .tool({ name: 'jest', version: '29.0.0' })
* .environment({ branchName: 'main' })
* .addTest(
* new TestBuilder()
* .name('should add numbers')
* .status('passed')
* .duration(150)
* .build()
* )
* .build();
* ```
*/
declare class ReportBuilder {
private options;
private _specVersion;
private _reportId?;
private _timestamp?;
private _generatedBy?;
private _tool?;
private _environment?;
private _tests;
private _insights?;
private _baseline?;
private _extra?;
private _summaryOverrides?;
constructor(options?: ReportBuilderOptions);
/**
* Set the spec version.
*/
specVersion(version: string): this;
/**
* Set or generate the report ID.
* @param uuid - UUID to use, or undefined to auto-generate
*/
reportId(uuid?: string): this;
/**
* Set the timestamp.
* @param date - Date to use, or undefined for current time
*/
timestamp(date?: Date | string): this;
/**
* Set the generator name.
*/
generatedBy(name: string): this;
/**
* Set the tool information.
*/
tool(tool: Tool): this;
/**
* Set the environment information.
*/
environment(env: Environment): this;
/**
* Add a single test.
*/
addTest(test: Test): this;
/**
* Add multiple tests.
*/
addTests(tests: Test[]): this;
/**
* Set run-level insights.
*/
insights(insights: Insights): this;
/**
* Set the baseline reference.
*/
baseline(baseline: Baseline): this;
/**
* Set extra metadata.
*/
extra(data: Record<string, unknown>): this;
/**
* Override specific summary fields.
* Useful when you want to set specific timing or counts.
*/
summaryOverrides(overrides: Partial<Summary>): this;
/**
* Build and return the CTRF report.
* @throws BuilderError if required fields are missing
*/
build(): CTRFReport;
}
/**
*
* @group Builders
* Fluent builder for constructing Test objects.
*
* @example
* ```typescript
* const test = new TestBuilder()
* .name('should add numbers')
* .status('passed')
* .duration(150)
* .suite(['math', 'addition'])
* .build();
* ```
*/
declare class TestBuilder {
private options;
private _id?;
private _name?;
private _status?;
private _duration?;
private _start?;
private _stop?;
private _suite?;
private _message?;
private _trace?;
private _snippet?;
private _ai?;
private _line?;
private _rawStatus?;
private _tags?;
private _type?;
private _filePath?;
private _retries?;
private _retryAttempts?;
private _flaky?;
private _stdout?;
private _stderr?;
private _threadId?;
private _browser?;
private _device?;
private _screenshot?;
private _attachments?;
private _parameters?;
private _steps?;
private _insights?;
private _extra?;
constructor(options?: TestBuilderOptions);
/**
* Set or generate the test ID.
* @param uuid - UUID to use, or undefined to auto-generate based on properties
*/
id(uuid?: string): this;
/**
* Set the test name.
*/
name(name: string): this;
/**
* Set the test status.
*/
status(status: TestStatus): this;
/**
* Set the duration in milliseconds.
*/
duration(ms: number): this;
/**
* Set the start timestamp.
*/
start(timestamp: number): this;
/**
* Set the stop timestamp.
*/
stop(timestamp: number): this;
/**
* Set the suite hierarchy.
*/
suite(suites: string[]): this;
/**
* Set the error message.
*/
message(msg: string): this;
/**
* Set the stack trace.
*/
trace(trace: string): this;
/**
* Set the code snippet.
*/
snippet(code: string): this;
/**
* Set AI-generated analysis.
*/
ai(analysis: string): this;
/**
* Set the line number.
*/
line(num: number): this;
/**
* Set the raw status from the test framework.
*/
rawStatus(status: string): this;
/**
* Set tags.
*/
tags(tags: string[]): this;
/**
* Set test type.
*/
type(type: string): this;
/**
* Set file path.
*/
filePath(path: string): this;
/**
* Set retry count.
*/
retries(count: number): this;
/**
* Add a retry attempt.
*/
addRetryAttempt(attempt: RetryAttempt): this;
/**
* Mark as flaky.
*/
flaky(isFlaky?: boolean): this;
/**
* Set stdout.
*/
stdout(lines: string[]): this;
/**
* Set stderr.
*/
stderr(lines: string[]): this;
/**
* Set thread ID.
*/
threadId(id: string): this;
/**
* Set browser name.
*/
browser(name: string): this;
/**
* Set device name.
*/
device(name: string): this;
/**
* Set screenshot (base64).
*/
screenshot(base64: string): this;
/**
* Add an attachment.
*/
addAttachment(attachment: Attachment): this;
/**
* Set parameters.
*/
parameters(params: Record<string, unknown>): this;
/**
* Add a step.
*/
addStep(step: Step): this;
/**
* Set test-level insights.
*/
insights(insights: TestInsights): this;
/**
* Set extra metadata.
*/
extra(data: Record<string, unknown>): this;
/**
* Build and return the Test object.
* @throws BuilderError if required fields are missing
*/
build(): Test;
}
/**
* CTRF Test ID Generation
*/
/**
*
* @group ID Generation
* Generate a deterministic UUID v5 for a test based on its properties.
* The same inputs will always produce the same UUID, enabling
* cross-run analysis and test identification.
*
* @param properties - Test properties to generate ID from
* @param properties.name - Test name (required)
* @param properties.suite - Suite hierarchy (optional)
* @param properties.filePath - File path (optional)
* @returns A deterministic UUID v5 string
*
* @example
* ```typescript
* const id = generateTestId({
* name: 'should add numbers',
* suite: ['math', 'addition'],
* filePath: 'tests/math.test.ts'
* });
* // Always returns the same UUID for these inputs
* ```
*/
declare function generateTestId(properties: {
name: string;
suite?: string[];
filePath?: string;
}): string;
/**
*
* @group ID Generation
* Generate a random UUID v4 for report identification.
*
* @returns A random UUID v4 string
*
* @example
* ```typescript
* const reportId = generateReportId();
* // => 'f47ac10b-58cc-4372-a567-0e02b2c3d479'
* ```
*/
declare function generateReportId(): string;
/**
* CTRF Parsing and Serialization
*/
/**
*
* @group Core Operations
* Parse a JSON string into a CTRFReport.
*
* @param json - JSON string to parse
* @param options - Parse options (e.g., enable validation)
* @returns Parsed CTRFReport object
* @throws ParseError if JSON is invalid
* @throws ValidationError if validation is enabled and fails
*
* @example
* ```typescript
* const report = parse(jsonString);
*
* // With validation
* const report = parse(jsonString, { validate: true });
* ```
*/
declare function parse(json: string, options?: ParseOptions): CTRFReport;
/**
*
* @group Core Operations
* Serialize a CTRFReport to a JSON string.
*
* @param report - The CTRF report to serialize
* @param options - Stringify options (pretty print, indent)
* @returns JSON string representation
*
* @example
* ```typescript
* const json = stringify(report);
*
* // Pretty print
* const json = stringify(report, { pretty: true });
*
* // Custom indent
* const json = stringify(report, { pretty: true, indent: 4 });
* ```
*/
declare function stringify(report: CTRFReport, options?: StringifyOptions): string;
/**
* CTRF Report Merging
*/
/**
*
* @group Merge
* Merge multiple CTRF reports into a single report.
* Useful for combining results from parallel or sharded test runs.
*
* @param reports - Array of CTRF reports to merge
* @param options - Merge options (deduplication, environment handling)
* @returns A new merged CTRFReport
* @throws Error if no reports are provided
*
* @example
* ```typescript
* const merged = merge([report1, report2, report3]);
*
* // With deduplication by test ID
* const merged = merge(reports, { deduplicateTests: true });
*
* // Keep first environment only
* const merged = merge(reports, { preserveEnvironment: 'first' });
* ```
*/
declare function merge(reports: CTRFReport[], options?: MergeOptions): CTRFReport;
/**
* CTRF Filtering and Querying
*/
/**
*
* @group Query & Filter
* Filter tests in a report by criteria.
*
* @param report - The CTRF report containing tests to filter
* @param criteria - Filter criteria (status, tags, suite, flaky, browser, device)
* @returns Array of tests matching all specified criteria
*
* @example
* ```typescript
* // Filter by status
* const failed = filterTests(report, { status: 'failed' });
*
* // Filter by multiple criteria
* const filtered = filterTests(report, {
* status: ['failed', 'skipped'],
* tags: ['smoke'],
* flaky: true
* });
* ```
*/
declare function filterTests(report: CTRFReport, criteria: FilterCriteria): Test[];
/**
*
* @group Query & Filter
* Find a single test in a report.
*
* @param report - The CTRF report to search
* @param criteria - Filter criteria including id, name, status, tags, etc.
* @returns The first matching test, or undefined if not found
*
* @example
* ```typescript
* // Find by ID
* const test = findTest(report, { id: 'uuid' });
*
* // Find by name
* const test = findTest(report, { name: 'should login' });
*
* // Find by multiple criteria
* const test = findTest(report, { status: 'failed', flaky: true });
* ```
*/
declare function findTest(report: CTRFReport, criteria: FilterCriteria): Test | undefined;
/**
* CTRF Insights Calculation
*
* This module replicates the functionality of the existing run-insights.ts
* while using the reference implementation's type system and API signatures.
*/
/**
*
* @group Insights
* Determines if a test is flaky based on the CTRF specification.
*
* A test is considered flaky if:
* - The `flaky` field is explicitly set to `true`, OR
* - The test has retries > 0 AND final status is 'passed'
*
* @param test - The test to check
* @returns true if the test is flaky
*
* @example
* ```typescript
* if (isTestFlaky(test)) {
* console.log('Test is flaky:', test.name);
* }
* ```
*/
declare function isTestFlaky(test: Test): boolean;
/**
*
* @group Insights
* Add insights to a CTRF report using historical data.
*
* Computes run-level and test-level insights according to the CTRF specification,
* including pass rate, fail rate, flaky rate, and duration metrics.
*
* @param report - The current report to enrich with insights
* @param historicalReports - Array of previous reports for trend analysis
* @param options - Options including baseline for comparison
* @returns A new report with insights populated
*
* @example
* ```typescript
* // Basic usage
* const reportWithInsights = addInsights(currentReport, previousReports);
*
* // With baseline comparison
* const reportWithInsights = addInsights(currentReport, previousReports, {
* baseline: baselineReport
* });
* ```
*/
declare function addInsights(report: CTRFReport, historicalReports?: CTRFReport[], options?: InsightsOptions): CTRFReport;
/**
* CTRF Schema Access
*/
/**
*
* @group Schema & Versioning
* The current version CTRF JSON Schema object.
*
* @example
* ```typescript
* import { schema } from 'ctrf';
* console.log(schema.$schema);
* ```
*/
declare const schema: object;
/**
*
* @group Schema & Versioning
* Get the JSON Schema for a specific CTRF spec version.
*
* @param version - The spec version (MAJOR.MINOR.PATCH) to get the schema for
* @returns The JSON Schema object for that version
* @throws SchemaVersionError if the version is not supported
*
* @example
* ```typescript
* const v0_0Schema = getSchema('0.0.0');
* const v1_0Schema = getSchema('1.0.0');
* ```
*/
declare function getSchema(version: string): object;
/**
*
* @group Schema & Versioning
* Get the current spec version.
*
* @returns The current spec version string
*/
declare function getCurrentSpecVersion(): string;
/**
*
* @group Schema & Versioning
* Get all supported spec versions.
*
* @returns Array of supported version strings
*/
declare function getSupportedSpecVersions(): readonly string[];
/**
* CTRF Error Classes
*/
/**
*
* @group Errors
* Base error class for all CTRF errors.
* All CTRF-specific errors extend this class.
*/
declare class CTRFError extends Error {
constructor(message: string);
}
/**
*
* @group Errors
* Error thrown when schema validation fails.
* Contains detailed error information for each validation issue.
*/
declare class ValidationError extends CTRFError {
/** Detailed validation errors */
readonly errors: ValidationErrorDetail[];
constructor(message: string, errors?: ValidationErrorDetail[]);
}
/**
*
* @group Errors
* Error thrown when JSON parsing fails.
*/
declare class ParseError extends CTRFError {
/** The original error that caused the parse failure */
readonly cause?: Error;
constructor(message: string, cause?: Error);
}
/**
*
* @group Errors
* Error thrown when an unsupported CTRF specification version is encountered.
*/
declare class SchemaVersionError extends CTRFError {
/** The unsupported version */
readonly version: string;
/** Supported versions */
readonly supportedVersions: string[];
constructor(version: string, supportedVersions: string[]);
}
/**
*
* @group Errors
* Error thrown when a file read or write operation fails.
*/
declare class FileError extends CTRFError {
/** The file path that caused the error */
readonly filePath: string;
/** The original error */
readonly cause?: Error;
constructor(message: string, filePath: string, cause?: Error);
}
/**
*
* @group Errors
* Error thrown when building a report or test fails due to missing required fields.
*/
declare class BuilderError extends CTRFError {
constructor(message: string);
}
export { type Attachment, type Baseline, BuilderError, CTRFError, type CTRFReport, CTRF_NAMESPACE, CURRENT_SPEC_VERSION, type Environment, FileError, type FilterCriteria, type Insights, type InsightsOptions, type MergeOptions, type MetricDelta, ParseError, type ParseOptions, REPORT_FORMAT, ReportBuilder, type ReportBuilderOptions, type Results, type RetryAttempt, SUPPORTED_SPEC_VERSIONS, SchemaVersionError, type Step, type StringifyOptions, type Summary, type SummaryOptions, TEST_STATUSES, type Test, TestBuilder, type TestBuilderOptions, type TestInsights, type TestStatus, type Tool, type ValidateOptions, ValidationError, type ValidationErrorDetail, type ValidationResult, addInsights, calculateSummary, filterTests, findTest, generateReportId, generateTestId, getCurrentSpecVersion, getSchema, getSupportedSpecVersions, hasInsights, isCTRFReport, isRetryAttempt, isTest, isTestFlaky, isTestStatus, isValid, merge, parse, schema, stringify, validate, validateStrict };

@@ -1,48 +0,1468 @@

/**
* CTRF TypeScript SDK - Reference Implementation
*
* A complete TypeScript implementation for working with CTRF (Common Test Report Format) reports.
*
* @packageDocumentation
*/
export { REPORT_FORMAT, CURRENT_SPEC_VERSION, TEST_STATUSES, SUPPORTED_SPEC_VERSIONS, CTRF_NAMESPACE, } from './constants.js';
// ============================================================================
// Validation
// ============================================================================
export { validate, isValid, validateStrict, isCTRFReport, isTest, isTestStatus, isRetryAttempt, hasInsights, } from './validate.js';
// ============================================================================
// Summary Calculation
// ============================================================================
export { calculateSummary } from './summary.js';
// ============================================================================
// Builders
// ============================================================================
export { ReportBuilder, TestBuilder } from './builder.js';
// ============================================================================
// ID Generation
// ============================================================================
export { generateTestId, generateReportId } from './id.js';
// ============================================================================
// Parsing & Serialization
// ============================================================================
export { parse, stringify } from './parse.js';
// ============================================================================
// Merging
// ============================================================================
export { merge } from './merge.js';
// ============================================================================
// Filtering & Querying
// ============================================================================
export { filterTests, findTest } from './filter.js';
// ============================================================================
// Insights
// ============================================================================
export { addInsights, isTestFlaky } from './insights.js';
// ============================================================================
// Schema
// ============================================================================
export { schema, getSchema, getCurrentSpecVersion, getSupportedSpecVersions, } from './schema.js';
// ============================================================================
// Errors
// ============================================================================
export { CTRFError, ValidationError, ParseError, SchemaVersionError, FileError, BuilderError, } from './errors.js';
// src/constants.ts
var REPORT_FORMAT = "CTRF";
var CURRENT_SPEC_VERSION = "0.0.0";
var TEST_STATUSES = [
"passed",
"failed",
"skipped",
"pending",
"other"
];
var SUPPORTED_SPEC_VERSIONS = ["0.0.0"];
var CTRF_NAMESPACE = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
// src/validate.ts
import Ajv from "ajv";
import addFormats from "ajv-formats";
// src/schema.ts
import fs from "fs";
import path from "path";
import { fileURLToPath } from "url";
// src/errors.ts
var CTRFError = class _CTRFError extends Error {
constructor(message) {
super(message);
this.name = "CTRFError";
Object.setPrototypeOf(this, _CTRFError.prototype);
}
};
var ValidationError = class _ValidationError extends CTRFError {
/** Detailed validation errors */
errors;
constructor(message, errors = []) {
super(message);
this.name = "ValidationError";
this.errors = errors;
Object.setPrototypeOf(this, _ValidationError.prototype);
}
};
var ParseError = class _ParseError extends CTRFError {
/** The original error that caused the parse failure */
cause;
constructor(message, cause) {
super(message);
this.name = "ParseError";
this.cause = cause;
Object.setPrototypeOf(this, _ParseError.prototype);
}
};
var SchemaVersionError = class _SchemaVersionError extends CTRFError {
/** The unsupported version */
version;
/** Supported versions */
supportedVersions;
constructor(version, supportedVersions) {
super(
`Unsupported schema version: ${version}. Supported versions: ${supportedVersions.join(", ")}`
);
this.name = "SchemaVersionError";
this.version = version;
this.supportedVersions = supportedVersions;
Object.setPrototypeOf(this, _SchemaVersionError.prototype);
}
};
var FileError = class _FileError extends CTRFError {
/** The file path that caused the error */
filePath;
/** The original error */
cause;
constructor(message, filePath, cause) {
super(message);
this.name = "FileError";
this.filePath = filePath;
this.cause = cause;
Object.setPrototypeOf(this, _FileError.prototype);
}
};
var BuilderError = class _BuilderError extends CTRFError {
constructor(message) {
super(message);
this.name = "BuilderError";
Object.setPrototypeOf(this, _BuilderError.prototype);
}
};
// src/schema.ts
var __filename2 = fileURLToPath(import.meta.url);
var __dirname2 = path.dirname(__filename2);
var _schemas = /* @__PURE__ */ new Map();
function loadSchemaForVersion(version) {
const versionParts = version.split(".");
if (versionParts.length < 2) {
throw new SchemaVersionError(version, [...SUPPORTED_SPEC_VERSIONS]);
}
const majorMinor = `${versionParts[0]}.${versionParts[1]}`;
if (_schemas.has(majorMinor)) {
return _schemas.get(majorMinor);
}
const schemaPath = path.resolve(__dirname2, `ctrf-schema-${majorMinor}.json`);
if (!fs.existsSync(schemaPath)) {
throw new SchemaVersionError(version, [...SUPPORTED_SPEC_VERSIONS]);
}
const schema2 = JSON.parse(fs.readFileSync(schemaPath, "utf8"));
_schemas.set(majorMinor, schema2);
return schema2;
}
var schema = loadSchemaForVersion(CURRENT_SPEC_VERSION);
function getSchema(version) {
if (!SUPPORTED_SPEC_VERSIONS.includes(
version
)) {
throw new SchemaVersionError(version, [...SUPPORTED_SPEC_VERSIONS]);
}
return loadSchemaForVersion(version);
}
function getCurrentSpecVersion() {
return CURRENT_SPEC_VERSION;
}
function getSupportedSpecVersions() {
return SUPPORTED_SPEC_VERSIONS;
}
// src/validate.ts
function validate(report, options = {}) {
const ajv = new Ajv({ allErrors: true });
addFormats(ajv);
const schemaToUse = options.specVersion ? getSchema(options.specVersion) : schema;
const validateFn = ajv.compile(schemaToUse);
const valid = validateFn(report);
if (valid) {
return { valid: true, errors: [] };
}
const errors = validateFn.errors?.map((error) => ({
message: error.message || "Unknown validation error",
path: error.instancePath || "/",
keyword: error.keyword
})) || [];
return { valid: false, errors };
}
function isValid(report) {
const result = validate(report);
return result.valid;
}
function validateStrict(report) {
const result = validate(report);
if (!result.valid) {
const errorMessages = result.errors.map((e) => `${e.path}: ${e.message}`).join("\n");
throw new ValidationError(
`CTRF validation failed:
${errorMessages}`,
result.errors
);
}
}
function isCTRFReport(report) {
return typeof report === "object" && report !== null && "reportFormat" in report && report.reportFormat === REPORT_FORMAT;
}
function isTest(obj) {
return typeof obj === "object" && obj !== null && "name" in obj && typeof obj.name === "string" && "status" in obj && typeof obj.status === "string" && "duration" in obj && typeof obj.duration === "number";
}
function isTestStatus(value) {
return typeof value === "string" && TEST_STATUSES.includes(value);
}
function isRetryAttempt(obj) {
return typeof obj === "object" && obj !== null && "attempt" in obj && typeof obj.attempt === "number" && "status" in obj && typeof obj.status === "string";
}
function hasInsights(report) {
return report.insights !== void 0 && Object.keys(report.insights).length > 0;
}
// src/summary.ts
function calculateSummary(tests, options = {}) {
const statusCounts = {
passed: 0,
failed: 0,
skipped: 0,
pending: 0,
other: 0
};
let flakyCount = 0;
const suites = /* @__PURE__ */ new Set();
let totalDuration = 0;
let minStart = options.start ?? Number.MAX_SAFE_INTEGER;
let maxStop = options.stop ?? 0;
for (const test of tests) {
statusCounts[test.status]++;
if (test.flaky) {
flakyCount++;
}
if (test.suite) {
for (let i = 1; i <= test.suite.length; i++) {
suites.add(test.suite.slice(0, i).join("/"));
}
}
totalDuration += test.duration;
if (test.start !== void 0) {
minStart = Math.min(minStart, test.start);
}
if (test.stop !== void 0) {
maxStop = Math.max(maxStop, test.stop);
}
}
const start = options.start ?? (minStart === Number.MAX_SAFE_INTEGER ? 0 : minStart);
const stop = options.stop ?? maxStop;
const summary = {
tests: tests.length,
passed: statusCounts.passed,
failed: statusCounts.failed,
skipped: statusCounts.skipped,
pending: statusCounts.pending,
other: statusCounts.other,
start,
stop
};
if (flakyCount > 0) {
summary.flaky = flakyCount;
}
if (suites.size > 0) {
summary.suites = suites.size;
}
if (totalDuration > 0 || tests.length > 0) {
summary.duration = totalDuration;
}
return summary;
}
// src/id.ts
import { createHash, randomUUID } from "crypto";
function generateTestId(properties) {
const { name, suite, filePath } = properties;
const suiteString = suite ? suite.join("/") : "";
const identifier = `${name}|${suiteString}|${filePath || ""}`;
const namespaceBytes = CTRF_NAMESPACE.replace(/-/g, "").match(/.{2}/g).map((byte) => parseInt(byte, 16));
const input = Buffer.concat([
Buffer.from(namespaceBytes),
Buffer.from(identifier, "utf8")
]);
const hash = createHash("sha1").update(input).digest("hex");
const uuid = [
hash.substring(0, 8),
hash.substring(8, 12),
"5" + hash.substring(13, 16),
// Version 5
(parseInt(hash.substring(16, 17), 16) & 3 | 8).toString(16) + hash.substring(17, 20),
// Variant bits
hash.substring(20, 32)
].join("-");
return uuid;
}
function generateReportId() {
return randomUUID();
}
// src/builder.ts
var ReportBuilder = class {
constructor(options = {}) {
this.options = options;
if (options.autoGenerateId) {
this._reportId = generateReportId();
}
if (options.autoTimestamp) {
this._timestamp = (/* @__PURE__ */ new Date()).toISOString();
}
}
_specVersion = CURRENT_SPEC_VERSION;
_reportId;
_timestamp;
_generatedBy;
_tool;
_environment;
_tests = [];
_insights;
_baseline;
_extra;
_summaryOverrides;
/**
* Set the spec version.
*/
specVersion(version) {
this._specVersion = version;
return this;
}
/**
* Set or generate the report ID.
* @param uuid - UUID to use, or undefined to auto-generate
*/
reportId(uuid) {
this._reportId = uuid ?? generateReportId();
return this;
}
/**
* Set the timestamp.
* @param date - Date to use, or undefined for current time
*/
timestamp(date) {
if (date instanceof Date) {
this._timestamp = date.toISOString();
} else if (typeof date === "string") {
this._timestamp = date;
} else {
this._timestamp = (/* @__PURE__ */ new Date()).toISOString();
}
return this;
}
/**
* Set the generator name.
*/
generatedBy(name) {
this._generatedBy = name;
return this;
}
/**
* Set the tool information.
*/
tool(tool) {
this._tool = tool;
return this;
}
/**
* Set the environment information.
*/
environment(env) {
this._environment = env;
return this;
}
/**
* Add a single test.
*/
addTest(test) {
this._tests.push(test);
return this;
}
/**
* Add multiple tests.
*/
addTests(tests) {
this._tests.push(...tests);
return this;
}
/**
* Set run-level insights.
*/
insights(insights) {
this._insights = insights;
return this;
}
/**
* Set the baseline reference.
*/
baseline(baseline) {
this._baseline = baseline;
return this;
}
/**
* Set extra metadata.
*/
extra(data) {
this._extra = data;
return this;
}
/**
* Override specific summary fields.
* Useful when you want to set specific timing or counts.
*/
summaryOverrides(overrides) {
this._summaryOverrides = overrides;
return this;
}
/**
* Build and return the CTRF report.
* @throws BuilderError if required fields are missing
*/
build() {
if (!this._tool) {
throw new BuilderError("Tool is required. Call .tool() before .build()");
}
const calculatedSummary = calculateSummary(this._tests);
const summary = {
...calculatedSummary,
...this._summaryOverrides
};
const results = {
tool: this._tool,
summary,
tests: this._tests
};
if (this._environment) {
results.environment = this._environment;
}
const report = {
reportFormat: REPORT_FORMAT,
specVersion: this._specVersion,
results
};
if (this._reportId) {
report.reportId = this._reportId;
}
if (this._timestamp) {
report.timestamp = this._timestamp;
}
if (this._generatedBy) {
report.generatedBy = this._generatedBy;
}
if (this._insights) {
report.insights = this._insights;
}
if (this._baseline) {
report.baseline = this._baseline;
}
if (this._extra) {
report.extra = this._extra;
}
return report;
}
};
var TestBuilder = class {
constructor(options = {}) {
this.options = options;
}
_id;
_name;
_status;
_duration;
_start;
_stop;
_suite;
_message;
_trace;
_snippet;
_ai;
_line;
_rawStatus;
_tags;
_type;
_filePath;
_retries;
_retryAttempts;
_flaky;
_stdout;
_stderr;
_threadId;
_browser;
_device;
_screenshot;
_attachments;
_parameters;
_steps;
_insights;
_extra;
/**
* Set or generate the test ID.
* @param uuid - UUID to use, or undefined to auto-generate based on properties
*/
id(uuid) {
this._id = uuid;
return this;
}
/**
* Set the test name.
*/
name(name) {
this._name = name;
return this;
}
/**
* Set the test status.
*/
status(status) {
this._status = status;
return this;
}
/**
* Set the duration in milliseconds.
*/
duration(ms) {
this._duration = ms;
return this;
}
/**
* Set the start timestamp.
*/
start(timestamp) {
this._start = timestamp;
return this;
}
/**
* Set the stop timestamp.
*/
stop(timestamp) {
this._stop = timestamp;
return this;
}
/**
* Set the suite hierarchy.
*/
suite(suites) {
this._suite = suites;
return this;
}
/**
* Set the error message.
*/
message(msg) {
this._message = msg;
return this;
}
/**
* Set the stack trace.
*/
trace(trace) {
this._trace = trace;
return this;
}
/**
* Set the code snippet.
*/
snippet(code) {
this._snippet = code;
return this;
}
/**
* Set AI-generated analysis.
*/
ai(analysis) {
this._ai = analysis;
return this;
}
/**
* Set the line number.
*/
line(num) {
this._line = num;
return this;
}
/**
* Set the raw status from the test framework.
*/
rawStatus(status) {
this._rawStatus = status;
return this;
}
/**
* Set tags.
*/
tags(tags) {
this._tags = tags;
return this;
}
/**
* Set test type.
*/
type(type) {
this._type = type;
return this;
}
/**
* Set file path.
*/
filePath(path2) {
this._filePath = path2;
return this;
}
/**
* Set retry count.
*/
retries(count) {
this._retries = count;
return this;
}
/**
* Add a retry attempt.
*/
addRetryAttempt(attempt) {
if (!this._retryAttempts) {
this._retryAttempts = [];
}
this._retryAttempts.push(attempt);
return this;
}
/**
* Mark as flaky.
*/
flaky(isFlaky = true) {
this._flaky = isFlaky;
return this;
}
/**
* Set stdout.
*/
stdout(lines) {
this._stdout = lines;
return this;
}
/**
* Set stderr.
*/
stderr(lines) {
this._stderr = lines;
return this;
}
/**
* Set thread ID.
*/
threadId(id) {
this._threadId = id;
return this;
}
/**
* Set browser name.
*/
browser(name) {
this._browser = name;
return this;
}
/**
* Set device name.
*/
device(name) {
this._device = name;
return this;
}
/**
* Set screenshot (base64).
*/
screenshot(base64) {
this._screenshot = base64;
return this;
}
/**
* Add an attachment.
*/
addAttachment(attachment) {
if (!this._attachments) {
this._attachments = [];
}
this._attachments.push(attachment);
return this;
}
/**
* Set parameters.
*/
parameters(params) {
this._parameters = params;
return this;
}
/**
* Add a step.
*/
addStep(step) {
if (!this._steps) {
this._steps = [];
}
this._steps.push(step);
return this;
}
/**
* Set test-level insights.
*/
insights(insights) {
this._insights = insights;
return this;
}
/**
* Set extra metadata.
*/
extra(data) {
this._extra = data;
return this;
}
/**
* Build and return the Test object.
* @throws BuilderError if required fields are missing
*/
build() {
if (!this._name) {
throw new BuilderError(
"Test name is required. Call .name() before .build()"
);
}
if (!this._status) {
throw new BuilderError(
"Test status is required. Call .status() before .build()"
);
}
if (this._duration === void 0) {
throw new BuilderError(
"Test duration is required. Call .duration() before .build()"
);
}
const test = {
name: this._name,
status: this._status,
duration: this._duration
};
if (this.options.autoGenerateId && !this._id) {
test.id = generateTestId({
name: this._name,
suite: this._suite,
filePath: this._filePath
});
} else if (this._id) {
test.id = this._id;
}
if (this._start !== void 0) test.start = this._start;
if (this._stop !== void 0) test.stop = this._stop;
if (this._suite) test.suite = this._suite;
if (this._message) test.message = this._message;
if (this._trace) test.trace = this._trace;
if (this._snippet) test.snippet = this._snippet;
if (this._ai) test.ai = this._ai;
if (this._line !== void 0) test.line = this._line;
if (this._rawStatus) test.rawStatus = this._rawStatus;
if (this._tags) test.tags = this._tags;
if (this._type) test.type = this._type;
if (this._filePath) test.filePath = this._filePath;
if (this._retries !== void 0) test.retries = this._retries;
if (this._retryAttempts) test.retryAttempts = this._retryAttempts;
if (this._flaky !== void 0) test.flaky = this._flaky;
if (this._stdout) test.stdout = this._stdout;
if (this._stderr) test.stderr = this._stderr;
if (this._threadId) test.threadId = this._threadId;
if (this._browser) test.browser = this._browser;
if (this._device) test.device = this._device;
if (this._screenshot) test.screenshot = this._screenshot;
if (this._attachments) test.attachments = this._attachments;
if (this._parameters) test.parameters = this._parameters;
if (this._steps) test.steps = this._steps;
if (this._insights) test.insights = this._insights;
if (this._extra) test.extra = this._extra;
return test;
}
};
// src/parse.ts
import fs2 from "fs";
import { promisify } from "util";
var readFileAsync = promisify(fs2.readFile);
var writeFileAsync = promisify(fs2.writeFile);
function parse(json, options = {}) {
let parsed;
try {
parsed = JSON.parse(json);
} catch (error) {
throw new ParseError(
`Failed to parse JSON: ${error instanceof Error ? error.message : "Unknown error"}`,
error instanceof Error ? error : void 0
);
}
if (options.validate) {
validateStrict(parsed);
}
return parsed;
}
function stringify(report, options = {}) {
const { pretty = false, indent = 2 } = options;
if (pretty) {
return JSON.stringify(report, null, indent);
}
return JSON.stringify(report);
}
// src/merge.ts
function merge(reports, options = {}) {
if (!reports || reports.length === 0) {
throw new Error("No reports provided for merging");
}
if (reports.length === 1) {
return { ...reports[0] };
}
const {
deduplicateTests = false,
mergeSummary = true,
preserveEnvironment = "merge"
} = options;
let allTests = [];
for (const report of reports) {
allTests.push(...report.results.tests);
}
if (deduplicateTests) {
const seen = /* @__PURE__ */ new Map();
for (const test of allTests) {
if (test.id) {
seen.set(test.id, test);
} else {
seen.set(`no-id-${seen.size}`, test);
}
}
allTests = Array.from(seen.values());
}
let summary;
if (mergeSummary) {
summary = calculateSummary(allTests);
let minStart = Number.MAX_SAFE_INTEGER;
let maxStop = 0;
for (const report of reports) {
minStart = Math.min(minStart, report.results.summary.start);
maxStop = Math.max(maxStop, report.results.summary.stop);
}
summary.start = minStart === Number.MAX_SAFE_INTEGER ? 0 : minStart;
summary.stop = maxStop;
summary.duration = summary.stop - summary.start;
} else {
summary = sumSummaries(reports.map((r) => r.results.summary));
}
let environment;
switch (preserveEnvironment) {
case "first":
environment = reports[0].results.environment;
break;
case "last":
environment = reports[reports.length - 1].results.environment;
break;
case "merge":
default:
environment = mergeEnvironments(
reports.map((r) => r.results.environment).filter(Boolean)
);
break;
}
const tool = reports[0].results.tool;
const merged = {
reportFormat: REPORT_FORMAT,
specVersion: CURRENT_SPEC_VERSION,
reportId: generateReportId(),
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
results: {
tool,
summary,
tests: allTests
}
};
if (environment && Object.keys(environment).length > 0) {
merged.results.environment = environment;
}
const mergedResultsExtra = mergeExtras(
reports.map((r) => r.results.extra).filter(Boolean)
);
if (mergedResultsExtra && Object.keys(mergedResultsExtra).length > 0) {
merged.results.extra = mergedResultsExtra;
}
const mergedReportExtra = mergeExtras(
reports.map((r) => r.extra).filter(Boolean)
);
if (mergedReportExtra && Object.keys(mergedReportExtra).length > 0) {
merged.extra = mergedReportExtra;
}
return merged;
}
function sumSummaries(summaries) {
const result = {
tests: 0,
passed: 0,
failed: 0,
skipped: 0,
pending: 0,
other: 0,
start: Number.MAX_SAFE_INTEGER,
stop: 0
};
let totalFlaky = 0;
let totalSuites = 0;
let totalDuration = 0;
let hasFlaky = false;
let hasSuites = false;
let hasDuration = false;
for (const summary of summaries) {
result.tests += summary.tests;
result.passed += summary.passed;
result.failed += summary.failed;
result.skipped += summary.skipped;
result.pending += summary.pending;
result.other += summary.other;
result.start = Math.min(result.start, summary.start);
result.stop = Math.max(result.stop, summary.stop);
if (summary.flaky !== void 0) {
hasFlaky = true;
totalFlaky += summary.flaky;
}
if (summary.suites !== void 0) {
hasSuites = true;
totalSuites += summary.suites;
}
if (summary.duration !== void 0) {
hasDuration = true;
totalDuration += summary.duration;
}
}
if (result.start === Number.MAX_SAFE_INTEGER) {
result.start = 0;
}
if (hasFlaky) result.flaky = totalFlaky;
if (hasSuites) result.suites = totalSuites;
if (hasDuration) result.duration = totalDuration;
return result;
}
function mergeEnvironments(environments) {
const merged = {};
for (const env of environments) {
for (const [key, value] of Object.entries(env)) {
if (value !== void 0 && merged[key] === void 0) {
;
merged[key] = value;
}
}
}
return merged;
}
function mergeExtras(extras) {
if (extras.length === 0) {
return void 0;
}
const merged = {};
for (const extra of extras) {
for (const [key, value] of Object.entries(extra)) {
if (merged[key] === void 0) {
merged[key] = value;
}
}
}
return Object.keys(merged).length > 0 ? merged : void 0;
}
// src/filter.ts
function filterTests(report, criteria) {
return report.results.tests.filter((test) => matchesCriteria(test, criteria));
}
function findTest(report, criteria) {
const { id, name, ...filterCriteria } = criteria;
return report.results.tests.find((test) => {
if (id !== void 0 && test.id !== id) {
return false;
}
if (name !== void 0 && test.name !== name) {
return false;
}
return matchesCriteria(test, filterCriteria);
});
}
function matchesCriteria(test, criteria) {
if (criteria.status !== void 0) {
const statuses = Array.isArray(criteria.status) ? criteria.status : [criteria.status];
if (!statuses.includes(test.status)) {
return false;
}
}
if (criteria.tags !== void 0) {
const requiredTags = Array.isArray(criteria.tags) ? criteria.tags : [criteria.tags];
if (!test.tags || !requiredTags.some((tag) => test.tags.includes(tag))) {
return false;
}
}
if (criteria.suite !== void 0) {
const requiredSuites = Array.isArray(criteria.suite) ? criteria.suite : [criteria.suite];
if (!test.suite || !requiredSuites.some((suite) => test.suite.includes(suite))) {
return false;
}
}
if (criteria.flaky !== void 0 && test.flaky !== criteria.flaky) {
return false;
}
if (criteria.browser !== void 0 && test.browser !== criteria.browser) {
return false;
}
if (criteria.device !== void 0 && test.device !== criteria.device) {
return false;
}
return true;
}
// src/insights.ts
function isTestFlaky(test) {
return test.flaky === true || test.retries !== void 0 && test.retries > 0 && test.status === "passed";
}
function calculateP95(values) {
if (values.length === 0) return 0;
const sorted = [...values].sort((a, b) => a - b);
const index = Math.ceil(sorted.length * 0.95) - 1;
return Number(sorted[Math.max(0, index)].toFixed(2));
}
function validateReportForInsights(report) {
return !!(report?.results?.tests && Array.isArray(report.results.tests));
}
function sortReportsByTimestamp(reports) {
return [...reports].sort((a, b) => {
const aStart = a.results?.summary?.start ?? 0;
const bStart = b.results?.summary?.start ?? 0;
return bStart - aStart;
});
}
function aggregateTestMetricsAcrossReports(reports) {
const metricsMap = /* @__PURE__ */ new Map();
for (const report of reports) {
if (!validateReportForInsights(report)) continue;
for (const test of report.results.tests) {
const isPassed = test.status === "passed";
const isFailed = test.status === "failed";
const isSkipped = test.status === "skipped";
const isPending = test.status === "pending";
const isOther = test.status === "other";
const testName = test.name;
if (!metricsMap.has(testName)) {
metricsMap.set(testName, {
totalAttempts: 0,
totalAttemptsFailed: 0,
totalResults: 0,
totalResultsFailed: 0,
totalResultsPassed: 0,
totalResultsSkipped: 0,
totalResultsFlaky: 0,
totalAttemptsFlaky: 0,
totalResultsDuration: 0,
appearsInRuns: 0,
reportsAnalyzed: 0,
durations: []
});
}
const metrics = metricsMap.get(testName);
metrics.totalResults += 1;
metrics.totalAttempts += 1 + (test.retries || 0);
metrics.totalAttemptsFailed += test.retries || 0;
if (isFailed) {
metrics.totalResultsFailed += 1;
metrics.totalAttemptsFailed += 1 + (test.retries || 0);
} else if (isPassed) {
metrics.totalResultsPassed += 1;
} else if (isSkipped || isPending || isOther) {
metrics.totalResultsSkipped += 1;
}
if (isTestFlaky(test)) {
metrics.totalResultsFlaky += 1;
metrics.totalAttemptsFlaky += test.retries || 0;
}
metrics.totalResultsDuration += test.duration || 0;
metrics.durations.push(test.duration || 0);
}
const testsInThisReport = /* @__PURE__ */ new Set();
for (const test of report.results.tests) {
testsInThisReport.add(test.name);
}
for (const testName of testsInThisReport) {
const metrics = metricsMap.get(testName);
metrics.appearsInRuns += 1;
}
}
return metricsMap;
}
function consolidateTestMetricsToRunMetrics(metricsMap) {
let totalAttempts = 0;
let totalAttemptsFailed = 0;
let totalResults = 0;
let totalResultsFailed = 0;
let totalResultsPassed = 0;
let totalResultsSkipped = 0;
let totalResultsFlaky = 0;
let totalAttemptsFlaky = 0;
let totalResultsDuration = 0;
for (const metrics of metricsMap.values()) {
totalAttempts += metrics.totalAttempts;
totalAttemptsFailed += metrics.totalAttemptsFailed;
totalResults += metrics.totalResults;
totalResultsFailed += metrics.totalResultsFailed;
totalResultsPassed += metrics.totalResultsPassed;
totalResultsSkipped += metrics.totalResultsSkipped;
totalResultsFlaky += metrics.totalResultsFlaky;
totalAttemptsFlaky += metrics.totalAttemptsFlaky;
totalResultsDuration += metrics.totalResultsDuration;
}
return {
totalAttempts,
totalAttemptsFailed,
totalResults,
totalResultsFailed,
totalResultsPassed,
totalResultsSkipped,
totalResultsFlaky,
totalAttemptsFlaky,
totalResultsDuration,
reportsAnalyzed: metricsMap.size
};
}
function calculateFlakyRateFromMetrics(runMetrics) {
if (runMetrics.totalAttempts === 0) {
return 0;
}
return Number(
(runMetrics.totalAttemptsFlaky / (runMetrics.totalResults + runMetrics.totalAttemptsFlaky)).toFixed(4)
);
}
function calculateFailRateFromMetrics(runMetrics) {
if (runMetrics.totalResults === 0) {
return 0;
}
return Number(
(runMetrics.totalResultsFailed / runMetrics.totalResults).toFixed(4)
);
}
function calculatePassRateFromMetrics(runMetrics) {
if (runMetrics.totalResults === 0) {
return 0;
}
return Number(
(runMetrics.totalResultsPassed / runMetrics.totalResults).toFixed(4)
);
}
function calculateAverageTestDurationFromMetrics(runMetrics) {
if (runMetrics.totalResults === 0) {
return 0;
}
return Number(
(runMetrics.totalResultsDuration / runMetrics.totalResults).toFixed(2)
);
}
function calculateAverageRunDurationFromMetrics(runMetrics) {
if (runMetrics.reportsAnalyzed === 0) {
return 0;
}
return Number(
(runMetrics.totalResultsDuration / runMetrics.reportsAnalyzed).toFixed(2)
);
}
function calculateP95RunDurationFromReports(reports) {
const runDurations = [];
for (const report of reports) {
if (validateReportForInsights(report) && report.results?.summary) {
const { start, stop } = report.results.summary;
if (start && stop && stop > start) {
const runDuration = stop - start;
runDurations.push(runDuration);
}
}
}
return calculateP95(runDurations);
}
function calculateRunInsights(reports, index = 0) {
if (index >= reports.length) {
return reports;
}
const currentReport = reports[index];
const previousReports = reports.slice(index + 1);
const allReportsUpToThisPoint = [currentReport, ...previousReports];
const testMetrics = aggregateTestMetricsAcrossReports(allReportsUpToThisPoint);
const runMetrics = consolidateTestMetricsToRunMetrics(testMetrics);
const { ...relevantMetrics } = runMetrics;
currentReport.insights = {
passRate: {
current: calculatePassRateFromMetrics(runMetrics),
baseline: 0,
change: 0
},
flakyRate: {
current: calculateFlakyRateFromMetrics(runMetrics),
baseline: 0,
change: 0
},
failRate: {
current: calculateFailRateFromMetrics(runMetrics),
baseline: 0,
change: 0
},
averageTestDuration: {
current: calculateAverageTestDurationFromMetrics(runMetrics),
baseline: 0,
change: 0
},
averageRunDuration: {
current: calculateAverageRunDurationFromMetrics(runMetrics),
baseline: 0,
change: 0
},
p95RunDuration: {
current: calculateP95RunDurationFromReports(allReportsUpToThisPoint),
baseline: 0,
change: 0
},
runsAnalyzed: allReportsUpToThisPoint.length,
extra: relevantMetrics
};
return calculateRunInsights(reports, index + 1);
}
function calculateTestInsightsWithBaseline(currentTestMetrics, baselineTestMetrics) {
const currentPassRate = currentTestMetrics.totalResults === 0 ? 0 : Number(
(currentTestMetrics.totalResultsPassed / currentTestMetrics.totalResults).toFixed(4)
);
const currentFlakyRate = currentTestMetrics.totalAttempts === 0 ? 0 : Number(
(currentTestMetrics.totalAttemptsFlaky / (currentTestMetrics.totalResults + currentTestMetrics.totalAttemptsFlaky)).toFixed(4)
);
const currentFailRate = currentTestMetrics.totalResults === 0 ? 0 : Number(
(currentTestMetrics.totalResultsFailed / currentTestMetrics.totalResults).toFixed(4)
);
const currentAverageDuration = currentTestMetrics.totalResults === 0 ? 0 : Number(
(currentTestMetrics.totalResultsDuration / currentTestMetrics.totalResults).toFixed(2)
);
const currentP95Duration = calculateP95(currentTestMetrics.durations);
let baselinePassRate = 0;
let baselineFlakyRate = 0;
let baselineFailRate = 0;
let baselineAverageDuration = 0;
let baselineP95Duration = 0;
if (baselineTestMetrics) {
baselinePassRate = baselineTestMetrics.totalResults === 0 ? 0 : Number(
(baselineTestMetrics.totalResultsPassed / baselineTestMetrics.totalResults).toFixed(4)
);
baselineFlakyRate = baselineTestMetrics.totalAttempts === 0 ? 0 : Number(
(baselineTestMetrics.totalAttemptsFlaky / baselineTestMetrics.totalAttempts).toFixed(4)
);
baselineFailRate = baselineTestMetrics.totalResults === 0 ? 0 : Number(
(baselineTestMetrics.totalResultsFailed / baselineTestMetrics.totalResults).toFixed(4)
);
baselineAverageDuration = baselineTestMetrics.totalResults === 0 ? 0 : Number(
(baselineTestMetrics.totalResultsDuration / baselineTestMetrics.totalResults).toFixed(2)
);
baselineP95Duration = calculateP95(baselineTestMetrics.durations);
}
const { durations: _durations, ...relevantMetrics } = currentTestMetrics;
return {
passRate: {
current: currentPassRate,
baseline: baselinePassRate,
change: Number((currentPassRate - baselinePassRate).toFixed(4))
},
flakyRate: {
current: currentFlakyRate,
baseline: baselineFlakyRate,
change: Number((currentFlakyRate - baselineFlakyRate).toFixed(4))
},
failRate: {
current: currentFailRate,
baseline: baselineFailRate,
change: Number((currentFailRate - baselineFailRate).toFixed(4))
},
averageTestDuration: {
current: currentAverageDuration,
baseline: baselineAverageDuration,
change: currentAverageDuration - baselineAverageDuration
},
p95TestDuration: {
current: currentP95Duration,
baseline: baselineP95Duration,
change: currentP95Duration - baselineP95Duration
},
executedInRuns: currentTestMetrics.appearsInRuns,
extra: relevantMetrics
};
}
function addTestInsightsWithBaselineToCurrentReport(currentReport, previousReports, baselineReport) {
if (!validateReportForInsights(currentReport)) {
return currentReport;
}
const allReports = [currentReport, ...previousReports];
const currentTestMetrics = aggregateTestMetricsAcrossReports(allReports);
let baselineTestMetrics;
if (baselineReport && validateReportForInsights(baselineReport)) {
const baselineIndex = previousReports.findIndex(
(report) => report.results?.summary?.start === baselineReport.results?.summary?.start
);
if (baselineIndex >= 0) {
const reportsUpToBaseline = previousReports.slice(baselineIndex);
baselineTestMetrics = aggregateTestMetricsAcrossReports(reportsUpToBaseline);
}
}
const reportWithInsights = {
...currentReport,
results: {
...currentReport.results,
tests: currentReport.results.tests.map((test) => {
const testName = test.name;
const currentMetrics = currentTestMetrics.get(testName);
if (currentMetrics) {
const baselineMetrics = baselineTestMetrics?.get(testName);
const testInsights = calculateTestInsightsWithBaseline(
currentMetrics,
baselineMetrics
);
return {
...test,
insights: testInsights
};
}
return test;
})
}
};
return reportWithInsights;
}
function calculateReportInsightsBaseline(currentReport, baselineReport) {
const currentInsights = currentReport.insights;
const previousInsights = baselineReport.insights;
if (!currentInsights || !previousInsights) {
console.log("Both reports must have insights populated");
return currentReport.insights;
}
return {
passRate: {
current: currentInsights?.passRate?.current ?? 0,
baseline: previousInsights?.passRate?.current ?? 0,
change: Number(
((currentInsights?.passRate?.current ?? 0) - (previousInsights?.passRate?.current ?? 0)).toFixed(4)
)
},
flakyRate: {
current: currentInsights?.flakyRate?.current ?? 0,
baseline: previousInsights?.flakyRate?.current ?? 0,
change: Number(
((currentInsights?.flakyRate?.current ?? 0) - (previousInsights?.flakyRate?.current ?? 0)).toFixed(4)
)
},
failRate: {
current: currentInsights?.failRate?.current ?? 0,
baseline: previousInsights?.failRate?.current ?? 0,
change: Number(
((currentInsights?.failRate?.current ?? 0) - (previousInsights?.failRate?.current ?? 0)).toFixed(4)
)
},
averageTestDuration: {
current: currentInsights?.averageTestDuration?.current ?? 0,
baseline: previousInsights?.averageTestDuration?.current ?? 0,
change: (currentInsights?.averageTestDuration?.current ?? 0) - (previousInsights?.averageTestDuration?.current ?? 0)
},
averageRunDuration: {
current: currentInsights?.averageRunDuration?.current ?? 0,
baseline: previousInsights?.averageRunDuration?.current ?? 0,
change: (currentInsights?.averageRunDuration?.current ?? 0) - (previousInsights?.averageRunDuration?.current ?? 0)
},
p95RunDuration: {
current: currentInsights?.p95RunDuration?.current ?? 0,
baseline: previousInsights?.p95RunDuration?.current ?? 0,
change: (currentInsights?.p95RunDuration?.current ?? 0) - (previousInsights?.p95RunDuration?.current ?? 0)
},
runsAnalyzed: currentInsights?.runsAnalyzed ?? 0,
extra: currentInsights.extra
};
}
function getTestsRemovedSinceBaseline(currentReport, baselineReport) {
if (!validateReportForInsights(currentReport) || !validateReportForInsights(baselineReport)) {
return [];
}
const currentTestNames = new Set(
currentReport.results.tests.map((test) => test.name)
);
const removedTests = baselineReport.results.tests.filter(
(test) => !currentTestNames.has(test.name)
);
return removedTests.map((test) => ({
name: test.name,
suite: test.suite,
filePath: test.filePath
}));
}
function getTestsAddedSinceBaseline(currentReport, baselineReport) {
if (!validateReportForInsights(currentReport) || !validateReportForInsights(baselineReport)) {
return [];
}
const baselineTestNames = new Set(
baselineReport.results.tests.map((test) => test.name)
);
const addedTests = currentReport.results.tests.filter(
(test) => !baselineTestNames.has(test.name)
);
return addedTests.map((test) => ({
name: test.name,
suite: test.suite,
filePath: test.filePath
}));
}
function setTestsRemovedToInsights(insights, currentReport, baselineReport) {
const removedTests = getTestsRemovedSinceBaseline(
currentReport,
baselineReport
);
return {
...insights,
extra: {
...insights.extra,
testsRemoved: removedTests
}
};
}
function setTestsAddedToInsights(insights, currentReport, baselineReport) {
const addedTests = getTestsAddedSinceBaseline(currentReport, baselineReport);
return {
...insights,
extra: {
...insights.extra,
testsAdded: addedTests
}
};
}
function addInsights(report, historicalReports = [], options = {}) {
if (!validateReportForInsights(report)) {
console.warn("Current report is not valid for insights calculation");
return report;
}
const baseline = options.baseline;
const sortedPreviousReports = sortReportsByTimestamp(historicalReports);
const allReports = [report, ...sortedPreviousReports];
const reportsWithRunInsights = calculateRunInsights([...allReports]);
const currentReportWithRunInsights = reportsWithRunInsights[0];
const currentReportWithTestInsights = addTestInsightsWithBaselineToCurrentReport(
currentReportWithRunInsights,
sortedPreviousReports,
baseline
);
if (!baseline) {
return currentReportWithTestInsights;
}
let baselineInsights = calculateReportInsightsBaseline(
currentReportWithTestInsights,
baseline
);
baselineInsights = setTestsAddedToInsights(
baselineInsights,
currentReportWithTestInsights,
baseline
);
baselineInsights = setTestsRemovedToInsights(
baselineInsights,
currentReportWithTestInsights,
baseline
);
if (baselineInsights.extra?.testsAdded) {
delete baselineInsights.extra.testsAdded;
}
if (baselineInsights.extra?.testsRemoved) {
delete baselineInsights.extra.testsRemoved;
}
return {
...currentReportWithTestInsights,
insights: baselineInsights
};
}
export {
BuilderError,
CTRFError,
CTRF_NAMESPACE,
CURRENT_SPEC_VERSION,
FileError,
ParseError,
REPORT_FORMAT,
ReportBuilder,
SUPPORTED_SPEC_VERSIONS,
SchemaVersionError,
TEST_STATUSES,
TestBuilder,
ValidationError,
addInsights,
calculateSummary,
filterTests,
findTest,
generateReportId,
generateTestId,
getCurrentSpecVersion,
getSchema,
getSupportedSpecVersions,
hasInsights,
isCTRFReport,
isRetryAttempt,
isTest,
isTestFlaky,
isTestStatus,
isValid,
merge,
parse,
schema,
stringify,
validate,
validateStrict
};
{
"name": "ctrf",
"version": "0.1.0",
"version": "0.2.0-next-0",
"description": "CTRF reference implementation in TypeScript for creating and validating CTRF documents.",
"type": "module",
"main": "dist/index.js",
"main": "dist/index.cjs",
"module": "dist/index.js",
"types": "dist/index.d.ts",
"exports": {
".": {
"import": {
"types": "./dist/index.d.ts",
"default": "./dist/index.js"
},
"require": {
"types": "./dist/index.d.cts",
"default": "./dist/index.cjs"
}
}
},
"bin": {

@@ -15,5 +28,5 @@ "ctrf": "dist/cli/cli.js"

"scripts": {
"build": "tsc -p . && cp src/ctrf-schema-*.json dist/",
"build:check": "tsc -p . -noEmit",
"build:watch": "tsc -p . --watch",
"build": "tsup && cp src/ctrf-schema-*.json dist/",
"build:check": "tsc -p . --noEmit",
"build:watch": "tsup --watch",
"clean": "rm -rf dist && rm -rf coverage && rm -rf ctrf",

@@ -29,2 +42,3 @@ "test": "vitest run",

"update-readme": "npx tsx update-readme.ts",
"attw": "npm pack && attw ctrf-*.tgz && rm ctrf-*.tgz",
"all": "npm run build:check && npm run test:coverage && npm run lint && npm run format && npm run docs && npm run build"

@@ -75,15 +89,17 @@ },

"devDependencies": {
"@arethetypeswrong/cli": "^0.18.2",
"@d2t/vitest-ctrf-json-reporter": "^1.2.0",
"@eslint/js": "^9.32.0",
"@eslint/js": "^10.0.1",
"@types/node": "^20.12.7",
"@types/yargs": "^17.0.32",
"@vitest/coverage-v8": "^3.2.4",
"eslint": "^9.32.0",
"@vitest/coverage-v8": "^4.0.18",
"eslint": "^10.0.2",
"prettier": "^3.5.3",
"tsup": "^8.5.1",
"typedoc": "^0.28.9",
"typedoc-plugin-markdown": "^4.8.0",
"typescript": "^5.8.3",
"typescript-eslint": "^8.38.0",
"vitest": "^3.2.4"
"typescript-eslint": "^8.57.0",
"vitest": "^4.0.18"
}
}
/**
* CTRF Report and Test Builders
* Fluent API for constructing CTRF reports
*/
import type { CTRFReport, Tool, Summary, Test, Environment, Insights, TestInsights, Baseline, Attachment, RetryAttempt, Step, TestStatus, ReportBuilderOptions, TestBuilderOptions } from './types.js';
/**
*
* @group Builders
* Fluent builder for constructing CTRF reports.
*
* @example
* ```typescript
* const report = new ReportBuilder()
* .specVersion('1.0.0')
* .tool({ name: 'jest', version: '29.0.0' })
* .environment({ branchName: 'main' })
* .addTest(
* new TestBuilder()
* .name('should add numbers')
* .status('passed')
* .duration(150)
* .build()
* )
* .build();
* ```
*/
export declare class ReportBuilder {
private options;
private _specVersion;
private _reportId?;
private _timestamp?;
private _generatedBy?;
private _tool?;
private _environment?;
private _tests;
private _insights?;
private _baseline?;
private _extra?;
private _summaryOverrides?;
constructor(options?: ReportBuilderOptions);
/**
* Set the spec version.
*/
specVersion(version: string): this;
/**
* Set or generate the report ID.
* @param uuid - UUID to use, or undefined to auto-generate
*/
reportId(uuid?: string): this;
/**
* Set the timestamp.
* @param date - Date to use, or undefined for current time
*/
timestamp(date?: Date | string): this;
/**
* Set the generator name.
*/
generatedBy(name: string): this;
/**
* Set the tool information.
*/
tool(tool: Tool): this;
/**
* Set the environment information.
*/
environment(env: Environment): this;
/**
* Add a single test.
*/
addTest(test: Test): this;
/**
* Add multiple tests.
*/
addTests(tests: Test[]): this;
/**
* Set run-level insights.
*/
insights(insights: Insights): this;
/**
* Set the baseline reference.
*/
baseline(baseline: Baseline): this;
/**
* Set extra metadata.
*/
extra(data: Record<string, unknown>): this;
/**
* Override specific summary fields.
* Useful when you want to set specific timing or counts.
*/
summaryOverrides(overrides: Partial<Summary>): this;
/**
* Build and return the CTRF report.
* @throws BuilderError if required fields are missing
*/
build(): CTRFReport;
}
/**
*
* @group Builders
* Fluent builder for constructing Test objects.
*
* @example
* ```typescript
* const test = new TestBuilder()
* .name('should add numbers')
* .status('passed')
* .duration(150)
* .suite(['math', 'addition'])
* .build();
* ```
*/
export declare class TestBuilder {
private options;
private _id?;
private _name?;
private _status?;
private _duration?;
private _start?;
private _stop?;
private _suite?;
private _message?;
private _trace?;
private _snippet?;
private _ai?;
private _line?;
private _rawStatus?;
private _tags?;
private _type?;
private _filePath?;
private _retries?;
private _retryAttempts?;
private _flaky?;
private _stdout?;
private _stderr?;
private _threadId?;
private _browser?;
private _device?;
private _screenshot?;
private _attachments?;
private _parameters?;
private _steps?;
private _insights?;
private _extra?;
constructor(options?: TestBuilderOptions);
/**
* Set or generate the test ID.
* @param uuid - UUID to use, or undefined to auto-generate based on properties
*/
id(uuid?: string): this;
/**
* Set the test name.
*/
name(name: string): this;
/**
* Set the test status.
*/
status(status: TestStatus): this;
/**
* Set the duration in milliseconds.
*/
duration(ms: number): this;
/**
* Set the start timestamp.
*/
start(timestamp: number): this;
/**
* Set the stop timestamp.
*/
stop(timestamp: number): this;
/**
* Set the suite hierarchy.
*/
suite(suites: string[]): this;
/**
* Set the error message.
*/
message(msg: string): this;
/**
* Set the stack trace.
*/
trace(trace: string): this;
/**
* Set the code snippet.
*/
snippet(code: string): this;
/**
* Set AI-generated analysis.
*/
ai(analysis: string): this;
/**
* Set the line number.
*/
line(num: number): this;
/**
* Set the raw status from the test framework.
*/
rawStatus(status: string): this;
/**
* Set tags.
*/
tags(tags: string[]): this;
/**
* Set test type.
*/
type(type: string): this;
/**
* Set file path.
*/
filePath(path: string): this;
/**
* Set retry count.
*/
retries(count: number): this;
/**
* Add a retry attempt.
*/
addRetryAttempt(attempt: RetryAttempt): this;
/**
* Mark as flaky.
*/
flaky(isFlaky?: boolean): this;
/**
* Set stdout.
*/
stdout(lines: string[]): this;
/**
* Set stderr.
*/
stderr(lines: string[]): this;
/**
* Set thread ID.
*/
threadId(id: string): this;
/**
* Set browser name.
*/
browser(name: string): this;
/**
* Set device name.
*/
device(name: string): this;
/**
* Set screenshot (base64).
*/
screenshot(base64: string): this;
/**
* Add an attachment.
*/
addAttachment(attachment: Attachment): this;
/**
* Set parameters.
*/
parameters(params: Record<string, unknown>): this;
/**
* Add a step.
*/
addStep(step: Step): this;
/**
* Set test-level insights.
*/
insights(insights: TestInsights): this;
/**
* Set extra metadata.
*/
extra(data: Record<string, unknown>): this;
/**
* Build and return the Test object.
* @throws BuilderError if required fields are missing
*/
build(): Test;
}
/**
* CTRF Report and Test Builders
* Fluent API for constructing CTRF reports
*/
import { REPORT_FORMAT, CURRENT_SPEC_VERSION } from './constants.js';
import { generateReportId, generateTestId } from './id.js';
import { calculateSummary } from './summary.js';
import { BuilderError } from './errors.js';
/**
*
* @group Builders
* Fluent builder for constructing CTRF reports.
*
* @example
* ```typescript
* const report = new ReportBuilder()
* .specVersion('1.0.0')
* .tool({ name: 'jest', version: '29.0.0' })
* .environment({ branchName: 'main' })
* .addTest(
* new TestBuilder()
* .name('should add numbers')
* .status('passed')
* .duration(150)
* .build()
* )
* .build();
* ```
*/
export class ReportBuilder {
options;
_specVersion = CURRENT_SPEC_VERSION;
_reportId;
_timestamp;
_generatedBy;
_tool;
_environment;
_tests = [];
_insights;
_baseline;
_extra;
_summaryOverrides;
constructor(options = {}) {
this.options = options;
if (options.autoGenerateId) {
this._reportId = generateReportId();
}
if (options.autoTimestamp) {
this._timestamp = new Date().toISOString();
}
}
/**
* Set the spec version.
*/
specVersion(version) {
this._specVersion = version;
return this;
}
/**
* Set or generate the report ID.
* @param uuid - UUID to use, or undefined to auto-generate
*/
reportId(uuid) {
this._reportId = uuid ?? generateReportId();
return this;
}
/**
* Set the timestamp.
* @param date - Date to use, or undefined for current time
*/
timestamp(date) {
if (date instanceof Date) {
this._timestamp = date.toISOString();
}
else if (typeof date === 'string') {
this._timestamp = date;
}
else {
this._timestamp = new Date().toISOString();
}
return this;
}
/**
* Set the generator name.
*/
generatedBy(name) {
this._generatedBy = name;
return this;
}
/**
* Set the tool information.
*/
tool(tool) {
this._tool = tool;
return this;
}
/**
* Set the environment information.
*/
environment(env) {
this._environment = env;
return this;
}
/**
* Add a single test.
*/
addTest(test) {
this._tests.push(test);
return this;
}
/**
* Add multiple tests.
*/
addTests(tests) {
this._tests.push(...tests);
return this;
}
/**
* Set run-level insights.
*/
insights(insights) {
this._insights = insights;
return this;
}
/**
* Set the baseline reference.
*/
baseline(baseline) {
this._baseline = baseline;
return this;
}
/**
* Set extra metadata.
*/
extra(data) {
this._extra = data;
return this;
}
/**
* Override specific summary fields.
* Useful when you want to set specific timing or counts.
*/
summaryOverrides(overrides) {
this._summaryOverrides = overrides;
return this;
}
/**
* Build and return the CTRF report.
* @throws BuilderError if required fields are missing
*/
build() {
if (!this._tool) {
throw new BuilderError('Tool is required. Call .tool() before .build()');
}
// Calculate summary from tests
const calculatedSummary = calculateSummary(this._tests);
const summary = {
...calculatedSummary,
...this._summaryOverrides,
};
const results = {
tool: this._tool,
summary,
tests: this._tests,
};
if (this._environment) {
results.environment = this._environment;
}
const report = {
reportFormat: REPORT_FORMAT,
specVersion: this._specVersion,
results,
};
if (this._reportId) {
report.reportId = this._reportId;
}
if (this._timestamp) {
report.timestamp = this._timestamp;
}
if (this._generatedBy) {
report.generatedBy = this._generatedBy;
}
if (this._insights) {
report.insights = this._insights;
}
if (this._baseline) {
report.baseline = this._baseline;
}
if (this._extra) {
report.extra = this._extra;
}
return report;
}
}
/**
*
* @group Builders
* Fluent builder for constructing Test objects.
*
* @example
* ```typescript
* const test = new TestBuilder()
* .name('should add numbers')
* .status('passed')
* .duration(150)
* .suite(['math', 'addition'])
* .build();
* ```
*/
export class TestBuilder {
options;
_id;
_name;
_status;
_duration;
_start;
_stop;
_suite;
_message;
_trace;
_snippet;
_ai;
_line;
_rawStatus;
_tags;
_type;
_filePath;
_retries;
_retryAttempts;
_flaky;
_stdout;
_stderr;
_threadId;
_browser;
_device;
_screenshot;
_attachments;
_parameters;
_steps;
_insights;
_extra;
constructor(options = {}) {
this.options = options;
}
/**
* Set or generate the test ID.
* @param uuid - UUID to use, or undefined to auto-generate based on properties
*/
id(uuid) {
this._id = uuid;
return this;
}
/**
* Set the test name.
*/
name(name) {
this._name = name;
return this;
}
/**
* Set the test status.
*/
status(status) {
this._status = status;
return this;
}
/**
* Set the duration in milliseconds.
*/
duration(ms) {
this._duration = ms;
return this;
}
/**
* Set the start timestamp.
*/
start(timestamp) {
this._start = timestamp;
return this;
}
/**
* Set the stop timestamp.
*/
stop(timestamp) {
this._stop = timestamp;
return this;
}
/**
* Set the suite hierarchy.
*/
suite(suites) {
this._suite = suites;
return this;
}
/**
* Set the error message.
*/
message(msg) {
this._message = msg;
return this;
}
/**
* Set the stack trace.
*/
trace(trace) {
this._trace = trace;
return this;
}
/**
* Set the code snippet.
*/
snippet(code) {
this._snippet = code;
return this;
}
/**
* Set AI-generated analysis.
*/
ai(analysis) {
this._ai = analysis;
return this;
}
/**
* Set the line number.
*/
line(num) {
this._line = num;
return this;
}
/**
* Set the raw status from the test framework.
*/
rawStatus(status) {
this._rawStatus = status;
return this;
}
/**
* Set tags.
*/
tags(tags) {
this._tags = tags;
return this;
}
/**
* Set test type.
*/
type(type) {
this._type = type;
return this;
}
/**
* Set file path.
*/
filePath(path) {
this._filePath = path;
return this;
}
/**
* Set retry count.
*/
retries(count) {
this._retries = count;
return this;
}
/**
* Add a retry attempt.
*/
addRetryAttempt(attempt) {
if (!this._retryAttempts) {
this._retryAttempts = [];
}
this._retryAttempts.push(attempt);
return this;
}
/**
* Mark as flaky.
*/
flaky(isFlaky = true) {
this._flaky = isFlaky;
return this;
}
/**
* Set stdout.
*/
stdout(lines) {
this._stdout = lines;
return this;
}
/**
* Set stderr.
*/
stderr(lines) {
this._stderr = lines;
return this;
}
/**
* Set thread ID.
*/
threadId(id) {
this._threadId = id;
return this;
}
/**
* Set browser name.
*/
browser(name) {
this._browser = name;
return this;
}
/**
* Set device name.
*/
device(name) {
this._device = name;
return this;
}
/**
* Set screenshot (base64).
*/
screenshot(base64) {
this._screenshot = base64;
return this;
}
/**
* Add an attachment.
*/
addAttachment(attachment) {
if (!this._attachments) {
this._attachments = [];
}
this._attachments.push(attachment);
return this;
}
/**
* Set parameters.
*/
parameters(params) {
this._parameters = params;
return this;
}
/**
* Add a step.
*/
addStep(step) {
if (!this._steps) {
this._steps = [];
}
this._steps.push(step);
return this;
}
/**
* Set test-level insights.
*/
insights(insights) {
this._insights = insights;
return this;
}
/**
* Set extra metadata.
*/
extra(data) {
this._extra = data;
return this;
}
/**
* Build and return the Test object.
* @throws BuilderError if required fields are missing
*/
build() {
if (!this._name) {
throw new BuilderError('Test name is required. Call .name() before .build()');
}
if (!this._status) {
throw new BuilderError('Test status is required. Call .status() before .build()');
}
if (this._duration === undefined) {
throw new BuilderError('Test duration is required. Call .duration() before .build()');
}
const test = {
name: this._name,
status: this._status,
duration: this._duration,
};
// Generate ID if auto-generate is enabled or if id() was called without a value
if (this.options.autoGenerateId && !this._id) {
test.id = generateTestId({
name: this._name,
suite: this._suite,
filePath: this._filePath,
});
}
else if (this._id) {
test.id = this._id;
}
// Add optional fields only if set
if (this._start !== undefined)
test.start = this._start;
if (this._stop !== undefined)
test.stop = this._stop;
if (this._suite)
test.suite = this._suite;
if (this._message)
test.message = this._message;
if (this._trace)
test.trace = this._trace;
if (this._snippet)
test.snippet = this._snippet;
if (this._ai)
test.ai = this._ai;
if (this._line !== undefined)
test.line = this._line;
if (this._rawStatus)
test.rawStatus = this._rawStatus;
if (this._tags)
test.tags = this._tags;
if (this._type)
test.type = this._type;
if (this._filePath)
test.filePath = this._filePath;
if (this._retries !== undefined)
test.retries = this._retries;
if (this._retryAttempts)
test.retryAttempts = this._retryAttempts;
if (this._flaky !== undefined)
test.flaky = this._flaky;
if (this._stdout)
test.stdout = this._stdout;
if (this._stderr)
test.stderr = this._stderr;
if (this._threadId)
test.threadId = this._threadId;
if (this._browser)
test.browser = this._browser;
if (this._device)
test.device = this._device;
if (this._screenshot)
test.screenshot = this._screenshot;
if (this._attachments)
test.attachments = this._attachments;
if (this._parameters)
test.parameters = this._parameters;
if (this._steps)
test.steps = this._steps;
if (this._insights)
test.insights = this._insights;
if (this._extra)
test.extra = this._extra;
return test;
}
}
import { describe, it, expect } from 'vitest';
import { ReportBuilder, TestBuilder } from './builder.js';
import { BuilderError } from './errors.js';
import { REPORT_FORMAT, CURRENT_SPEC_VERSION } from './constants.js';
describe('builder', () => {
describe('ReportBuilder', () => {
it('should build a minimal valid report', () => {
const report = new ReportBuilder().tool({ name: 'jest' }).build();
expect(report.reportFormat).toBe(REPORT_FORMAT);
expect(report.specVersion).toBe(CURRENT_SPEC_VERSION);
expect(report.results.tool.name).toBe('jest');
expect(report.results.tests).toEqual([]);
});
it('should set spec version', () => {
const report = new ReportBuilder()
.specVersion('0.1.0')
.tool({ name: 'jest' })
.build();
expect(report.specVersion).toBe('0.1.0');
});
it('should set report ID', () => {
const report = new ReportBuilder()
.reportId('custom-id')
.tool({ name: 'jest' })
.build();
expect(report.reportId).toBe('custom-id');
});
it('should auto-generate report ID', () => {
const report = new ReportBuilder()
.reportId()
.tool({ name: 'jest' })
.build();
expect(report.reportId).toBeDefined();
expect(report.reportId).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i);
});
it('should set timestamp', () => {
const date = new Date('2024-01-01T00:00:00Z');
const report = new ReportBuilder()
.timestamp(date)
.tool({ name: 'jest' })
.build();
expect(report.timestamp).toBe('2024-01-01T00:00:00.000Z');
});
it('should set timestamp from string', () => {
const report = new ReportBuilder()
.timestamp('2024-01-01T00:00:00Z')
.tool({ name: 'jest' })
.build();
expect(report.timestamp).toBe('2024-01-01T00:00:00Z');
});
it('should auto-set timestamp to now', () => {
const before = new Date().toISOString();
const report = new ReportBuilder()
.timestamp()
.tool({ name: 'jest' })
.build();
const after = new Date().toISOString();
expect(report.timestamp).toBeDefined();
expect(report.timestamp >= before).toBe(true);
expect(report.timestamp <= after).toBe(true);
});
it('should set generatedBy', () => {
const report = new ReportBuilder()
.generatedBy('my-reporter')
.tool({ name: 'jest' })
.build();
expect(report.generatedBy).toBe('my-reporter');
});
it('should set environment', () => {
const report = new ReportBuilder()
.tool({ name: 'jest' })
.environment({ branchName: 'main', commit: 'abc123' })
.build();
expect(report.results.environment?.branchName).toBe('main');
expect(report.results.environment?.commit).toBe('abc123');
});
it('should add tests', () => {
const test1 = new TestBuilder()
.name('test 1')
.status('passed')
.duration(100)
.build();
const test2 = new TestBuilder()
.name('test 2')
.status('failed')
.duration(200)
.build();
const report = new ReportBuilder()
.tool({ name: 'jest' })
.addTest(test1)
.addTest(test2)
.build();
expect(report.results.tests).toHaveLength(2);
expect(report.results.tests[0].name).toBe('test 1');
expect(report.results.tests[1].name).toBe('test 2');
});
it('should add multiple tests at once', () => {
const tests = [
new TestBuilder().name('test 1').status('passed').duration(100).build(),
new TestBuilder().name('test 2').status('failed').duration(200).build(),
];
const report = new ReportBuilder()
.tool({ name: 'jest' })
.addTests(tests)
.build();
expect(report.results.tests).toHaveLength(2);
});
it('should calculate summary automatically', () => {
const report = new ReportBuilder()
.tool({ name: 'jest' })
.addTest(new TestBuilder().name('t1').status('passed').duration(100).build())
.addTest(new TestBuilder().name('t2').status('passed').duration(100).build())
.addTest(new TestBuilder().name('t3').status('failed').duration(100).build())
.addTest(new TestBuilder().name('t4').status('skipped').duration(0).build())
.build();
expect(report.results.summary.tests).toBe(4);
expect(report.results.summary.passed).toBe(2);
expect(report.results.summary.failed).toBe(1);
expect(report.results.summary.skipped).toBe(1);
});
it('should allow summary overrides', () => {
const report = new ReportBuilder()
.tool({ name: 'jest' })
.summaryOverrides({ start: 1000, stop: 2000 })
.build();
expect(report.results.summary.start).toBe(1000);
expect(report.results.summary.stop).toBe(2000);
});
it('should set insights', () => {
const report = new ReportBuilder()
.tool({ name: 'jest' })
.insights({ runsAnalyzed: 10 })
.build();
expect(report.insights?.runsAnalyzed).toBe(10);
});
it('should set baseline', () => {
const report = new ReportBuilder()
.tool({ name: 'jest' })
.baseline({ reportId: 'baseline-id', source: 'main' })
.build();
expect(report.baseline?.reportId).toBe('baseline-id');
expect(report.baseline?.source).toBe('main');
});
it('should set extra metadata', () => {
const report = new ReportBuilder()
.tool({ name: 'jest' })
.extra({ custom: 'data' })
.build();
expect(report.extra?.custom).toBe('data');
});
it('should throw if tool is not set', () => {
expect(() => new ReportBuilder().build()).toThrow(BuilderError);
});
it('should support auto-generate options', () => {
const report = new ReportBuilder({
autoGenerateId: true,
autoTimestamp: true,
})
.tool({ name: 'jest' })
.build();
expect(report.reportId).toBeDefined();
expect(report.timestamp).toBeDefined();
});
});
describe('TestBuilder', () => {
it('should build a minimal valid test', () => {
const test = new TestBuilder()
.name('should work')
.status('passed')
.duration(100)
.build();
expect(test.name).toBe('should work');
expect(test.status).toBe('passed');
expect(test.duration).toBe(100);
});
it('should throw if name is missing', () => {
expect(() => new TestBuilder().status('passed').duration(100).build()).toThrow(BuilderError);
});
it('should throw if status is missing', () => {
expect(() => new TestBuilder().name('test').duration(100).build()).toThrow(BuilderError);
});
it('should throw if duration is missing', () => {
expect(() => new TestBuilder().name('test').status('passed').build()).toThrow(BuilderError);
});
it('should set all optional fields', () => {
const test = new TestBuilder()
.id('test-id')
.name('test')
.status('failed')
.duration(100)
.start(1000)
.stop(1100)
.suite(['unit', 'auth'])
.message('Error message')
.trace('Stack trace')
.snippet('expect(true).toBe(false)')
.ai('AI analysis')
.line(42)
.rawStatus('FAILED')
.tags(['smoke', 'critical'])
.type('e2e')
.filePath('test/auth.test.ts')
.retries(2)
.flaky(true)
.stdout(['output'])
.stderr(['error'])
.threadId('worker-1')
.browser('chrome')
.device('desktop')
.screenshot('base64data')
.parameters({ user: 'test' })
.extra({ custom: 'data' })
.build();
expect(test.id).toBe('test-id');
expect(test.start).toBe(1000);
expect(test.stop).toBe(1100);
expect(test.suite).toEqual(['unit', 'auth']);
expect(test.message).toBe('Error message');
expect(test.trace).toBe('Stack trace');
expect(test.snippet).toBe('expect(true).toBe(false)');
expect(test.ai).toBe('AI analysis');
expect(test.line).toBe(42);
expect(test.rawStatus).toBe('FAILED');
expect(test.tags).toEqual(['smoke', 'critical']);
expect(test.type).toBe('e2e');
expect(test.filePath).toBe('test/auth.test.ts');
expect(test.retries).toBe(2);
expect(test.flaky).toBe(true);
expect(test.stdout).toEqual(['output']);
expect(test.stderr).toEqual(['error']);
expect(test.threadId).toBe('worker-1');
expect(test.browser).toBe('chrome');
expect(test.device).toBe('desktop');
expect(test.screenshot).toBe('base64data');
expect(test.parameters).toEqual({ user: 'test' });
expect(test.extra).toEqual({ custom: 'data' });
});
it('should add retry attempts', () => {
const test = new TestBuilder()
.name('test')
.status('passed')
.duration(100)
.addRetryAttempt({ attempt: 1, status: 'failed', duration: 50 })
.addRetryAttempt({ attempt: 2, status: 'passed', duration: 100 })
.build();
expect(test.retryAttempts).toHaveLength(2);
expect(test.retryAttempts[0].attempt).toBe(1);
expect(test.retryAttempts[1].attempt).toBe(2);
});
it('should add attachments', () => {
const test = new TestBuilder()
.name('test')
.status('failed')
.duration(100)
.addAttachment({
name: 'screenshot.png',
contentType: 'image/png',
path: '/tmp/screenshot.png',
})
.build();
expect(test.attachments).toHaveLength(1);
expect(test.attachments[0].name).toBe('screenshot.png');
});
it('should add steps', () => {
const test = new TestBuilder()
.name('test')
.status('failed')
.duration(100)
.addStep({ name: 'Step 1', status: 'passed' })
.addStep({ name: 'Step 2', status: 'failed' })
.build();
expect(test.steps).toHaveLength(2);
expect(test.steps[0].name).toBe('Step 1');
expect(test.steps[1].status).toBe('failed');
});
it('should set insights', () => {
const test = new TestBuilder()
.name('test')
.status('passed')
.duration(100)
.insights({ executedInRuns: 10 })
.build();
expect(test.insights?.executedInRuns).toBe(10);
});
it('should auto-generate ID when option enabled', () => {
const test = new TestBuilder({ autoGenerateId: true })
.name('test')
.status('passed')
.duration(100)
.suite(['unit'])
.filePath('test.ts')
.build();
expect(test.id).toBeDefined();
expect(test.id).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-5[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i);
});
});
});
#!/usr/bin/env node
export {};
export declare function identifyFlakyTests(filePath: string): Promise<void>;
import fs from 'fs';
import path from 'path';
export async function identifyFlakyTests(filePath) {
try {
const resolvedFilePath = path.resolve(filePath);
if (!fs.existsSync(resolvedFilePath)) {
console.error(`The file ${resolvedFilePath} does not exist.`);
return;
}
const fileContent = fs.readFileSync(resolvedFilePath, 'utf8');
const report = JSON.parse(fileContent);
const flakyTests = report.results.tests.filter((test) => test.flaky === true);
if (flakyTests.length > 0) {
console.log(`Found ${flakyTests.length} flaky test(s):`);
flakyTests.forEach((test) => {
console.log(`- Test Name: ${test.name}, Retries: ${test.retries}`);
});
}
else {
console.log(`No flaky tests found in ${resolvedFilePath}.`);
}
}
catch (error) {
console.error('Error identifying flaky tests:', error);
}
}
export declare function mergeReports(directory: string, output: string, outputDir: string, keepReports: boolean): Promise<void>;
import fs from 'fs';
import path from 'path';
export async function mergeReports(directory, output, outputDir, keepReports) {
try {
const directoryPath = path.resolve(directory);
const outputFileName = output;
const resolvedOutputDir = outputDir
? path.resolve(outputDir)
: directoryPath;
const outputPath = path.join(resolvedOutputDir, outputFileName);
console.log('Merging CTRF reports...');
const files = fs.readdirSync(directoryPath);
files.forEach(file => {
console.log('Found file:', file);
});
const ctrfReportFiles = files.filter(file => {
try {
if (path.extname(file) !== '.json') {
console.log(`Skipping non-CTRF file: ${file}`);
return false;
}
const filePath = path.join(directoryPath, file);
const fileContent = fs.readFileSync(filePath, 'utf8');
const jsonData = JSON.parse(fileContent);
if (!('results' in jsonData)) {
console.log(`Skipping non-CTRF file: ${file}`);
return false;
}
return true;
}
catch (error) {
console.error(`Error reading JSON file '${file}':`, error);
return false;
}
});
if (ctrfReportFiles.length === 0) {
console.log('No CTRF reports found in the specified directory.');
return;
}
if (!fs.existsSync(resolvedOutputDir)) {
fs.mkdirSync(resolvedOutputDir, { recursive: true });
console.log(`Created output directory: ${resolvedOutputDir}`);
}
const mergedReport = ctrfReportFiles
.map(file => {
console.log('Merging report:', file);
const filePath = path.join(directoryPath, file);
const fileContent = fs.readFileSync(filePath, 'utf8');
return JSON.parse(fileContent);
})
.reduce((acc, curr) => {
if (!acc.results) {
return curr;
}
acc.results.summary.tests += curr.results.summary.tests;
acc.results.summary.passed += curr.results.summary.passed;
acc.results.summary.failed += curr.results.summary.failed;
acc.results.summary.skipped += curr.results.summary.skipped;
acc.results.summary.pending += curr.results.summary.pending;
acc.results.summary.other += curr.results.summary.other;
acc.results.tests.push(...curr.results.tests);
acc.results.summary.start = Math.min(acc.results.summary.start, curr.results.summary.start);
acc.results.summary.stop = Math.max(acc.results.summary.stop, curr.results.summary.stop);
return acc;
}, { results: null });
fs.writeFileSync(outputPath, JSON.stringify(mergedReport, null, 2));
if (!keepReports) {
ctrfReportFiles.forEach(file => {
const filePath = path.join(directoryPath, file);
if (file !== outputFileName) {
fs.unlinkSync(filePath);
}
});
}
console.log('CTRF reports merged successfully.');
console.log(`Merged report saved to: ${outputPath}`);
}
catch (error) {
console.error('Error merging CTRF reports:', error);
}
}
export interface Report {
reportFormat: 'CTRF';
specVersion: `${number}.${number}.${number}`;
reportId?: string;
timestamp?: string;
generatedBy?: string;
results: Results;
insights?: RootInsights;
baseline?: Baseline;
extra?: Record<string, unknown>;
}
export interface Results {
tool: Tool;
summary: Summary;
tests: Test[];
environment?: Environment;
extra?: Record<string, unknown>;
}
export interface Summary {
tests: number;
passed: number;
failed: number;
skipped: number;
pending: number;
other: number;
flaky?: number;
suites?: number;
start: number;
stop: number;
duration?: number;
extra?: Record<string, unknown>;
}
export interface Test {
id?: string;
name: string;
status: TestStatus;
duration: number;
start?: number;
stop?: number;
suite?: string[];
message?: string;
trace?: string;
snippet?: string;
line?: number;
ai?: string;
rawStatus?: string;
tags?: string[];
type?: string;
filePath?: string;
retries?: number;
retryAttempts?: RetryAttempt[];
flaky?: boolean;
stdout?: string[];
stderr?: string[];
threadId?: string;
attachments?: Attachment[];
browser?: string;
device?: string;
screenshot?: string;
parameters?: Record<string, unknown>;
steps?: Step[];
insights?: TestInsights;
extra?: Record<string, unknown>;
}
export interface Environment {
reportName?: string;
appName?: string;
appVersion?: string;
buildId?: string;
buildName?: string;
buildNumber?: number;
buildUrl?: string;
repositoryName?: string;
repositoryUrl?: string;
commit?: string;
branchName?: string;
osPlatform?: string;
osRelease?: string;
osVersion?: string;
testEnvironment?: string;
extra?: Record<string, unknown>;
}
export interface Tool {
name: string;
version?: string;
extra?: Record<string, unknown>;
}
export interface Step {
name: string;
status: TestStatus;
extra?: Record<string, unknown>;
}
export interface Attachment {
name: string;
contentType: string;
path: string;
extra?: Record<string, unknown>;
}
export interface RetryAttempt {
attempt: number;
status: TestStatus;
duration?: number;
message?: string;
trace?: string;
line?: number;
snippet?: string;
stdout?: string[];
stderr?: string[];
start?: number;
stop?: number;
attachments?: Attachment[];
extra?: Record<string, unknown>;
}
export interface RootInsights {
runsAnalyzed?: number;
passRate?: InsightsMetric;
failRate?: InsightsMetric;
flakyRate?: InsightsMetric;
averageRunDuration?: InsightsMetric;
p95RunDuration?: InsightsMetric;
averageTestDuration?: InsightsMetric;
extra?: Record<string, unknown>;
}
export interface TestInsights {
passRate?: InsightsMetric;
failRate?: InsightsMetric;
flakyRate?: InsightsMetric;
averageTestDuration?: InsightsMetric;
p95TestDuration?: InsightsMetric;
executedInRuns?: number;
extra?: Record<string, unknown>;
}
export interface InsightsMetric {
current: number;
baseline: number;
change: number;
}
export interface Baseline {
reportId: string;
source?: string;
timestamp?: string;
commit?: string;
buildName?: string;
buildNumber?: number;
buildUrl?: string;
extra?: Record<string, unknown>;
}
export type TestStatus = 'passed' | 'failed' | 'skipped' | 'pending' | 'other';
export {};
/**
* CTRF Constants
*/
/** The CTRF report format identifier */
export declare const REPORT_FORMAT: "CTRF";
/** Current spec version */
export declare const CURRENT_SPEC_VERSION = "0.0.0";
/** All valid test statuses */
export declare const TEST_STATUSES: readonly ["passed", "failed", "skipped", "pending", "other"];
/** Supported specification versions */
export declare const SUPPORTED_SPEC_VERSIONS: readonly ["0.0.0"];
/** CTRF namespace UUID for deterministic ID generation */
export declare const CTRF_NAMESPACE = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
/**
* CTRF Constants
*/
/** The CTRF report format identifier */
export const REPORT_FORMAT = 'CTRF';
/** Current spec version */
export const CURRENT_SPEC_VERSION = '0.0.0';
/** All valid test statuses */
export const TEST_STATUSES = [
'passed',
'failed',
'skipped',
'pending',
'other',
];
/** Supported specification versions */
export const SUPPORTED_SPEC_VERSIONS = ['0.0.0'];
/** CTRF namespace UUID for deterministic ID generation */
export const CTRF_NAMESPACE = '6ba7b810-9dad-11d1-80b4-00c04fd430c8';
import { describe, it, expect } from 'vitest';
import { REPORT_FORMAT, CURRENT_SPEC_VERSION, TEST_STATUSES, SUPPORTED_SPEC_VERSIONS, CTRF_NAMESPACE, } from './constants.js';
describe('constants', () => {
describe('REPORT_FORMAT', () => {
it('should be CTRF', () => {
expect(REPORT_FORMAT).toBe('CTRF');
});
});
describe('CURRENT_SPEC_VERSION', () => {
it('should be a valid semver string', () => {
expect(CURRENT_SPEC_VERSION).toMatch(/^\d+\.\d+\.\d+$/);
});
});
describe('TEST_STATUSES', () => {
it('should contain all valid statuses', () => {
expect(TEST_STATUSES).toContain('passed');
expect(TEST_STATUSES).toContain('failed');
expect(TEST_STATUSES).toContain('skipped');
expect(TEST_STATUSES).toContain('pending');
expect(TEST_STATUSES).toContain('other');
});
it('should have exactly 5 statuses', () => {
expect(TEST_STATUSES).toHaveLength(5);
});
});
describe('SUPPORTED_SPEC_VERSIONS', () => {
it('should include the current version', () => {
expect(SUPPORTED_SPEC_VERSIONS).toContain(CURRENT_SPEC_VERSION);
});
});
describe('CTRF_NAMESPACE', () => {
it('should be a valid UUID', () => {
expect(CTRF_NAMESPACE).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i);
});
});
});
/**
* CTRF Error Classes
*/
import type { ValidationErrorDetail } from './types.js';
/**
*
* @group Errors
* Base error class for all CTRF errors.
* All CTRF-specific errors extend this class.
*/
export declare class CTRFError extends Error {
constructor(message: string);
}
/**
*
* @group Errors
* Error thrown when schema validation fails.
* Contains detailed error information for each validation issue.
*/
export declare class ValidationError extends CTRFError {
/** Detailed validation errors */
readonly errors: ValidationErrorDetail[];
constructor(message: string, errors?: ValidationErrorDetail[]);
}
/**
*
* @group Errors
* Error thrown when JSON parsing fails.
*/
export declare class ParseError extends CTRFError {
/** The original error that caused the parse failure */
readonly cause?: Error;
constructor(message: string, cause?: Error);
}
/**
*
* @group Errors
* Error thrown when an unsupported CTRF specification version is encountered.
*/
export declare class SchemaVersionError extends CTRFError {
/** The unsupported version */
readonly version: string;
/** Supported versions */
readonly supportedVersions: string[];
constructor(version: string, supportedVersions: string[]);
}
/**
*
* @group Errors
* Error thrown when a file read or write operation fails.
*/
export declare class FileError extends CTRFError {
/** The file path that caused the error */
readonly filePath: string;
/** The original error */
readonly cause?: Error;
constructor(message: string, filePath: string, cause?: Error);
}
/**
*
* @group Errors
* Error thrown when building a report or test fails due to missing required fields.
*/
export declare class BuilderError extends CTRFError {
constructor(message: string);
}
/**
* CTRF Error Classes
*/
/**
*
* @group Errors
* Base error class for all CTRF errors.
* All CTRF-specific errors extend this class.
*/
export class CTRFError extends Error {
constructor(message) {
super(message);
this.name = 'CTRFError';
Object.setPrototypeOf(this, CTRFError.prototype);
}
}
/**
*
* @group Errors
* Error thrown when schema validation fails.
* Contains detailed error information for each validation issue.
*/
export class ValidationError extends CTRFError {
/** Detailed validation errors */
errors;
constructor(message, errors = []) {
super(message);
this.name = 'ValidationError';
this.errors = errors;
Object.setPrototypeOf(this, ValidationError.prototype);
}
}
/**
*
* @group Errors
* Error thrown when JSON parsing fails.
*/
export class ParseError extends CTRFError {
/** The original error that caused the parse failure */
cause;
constructor(message, cause) {
super(message);
this.name = 'ParseError';
this.cause = cause;
Object.setPrototypeOf(this, ParseError.prototype);
}
}
/**
*
* @group Errors
* Error thrown when an unsupported CTRF specification version is encountered.
*/
export class SchemaVersionError extends CTRFError {
/** The unsupported version */
version;
/** Supported versions */
supportedVersions;
constructor(version, supportedVersions) {
super(`Unsupported schema version: ${version}. Supported versions: ${supportedVersions.join(', ')}`);
this.name = 'SchemaVersionError';
this.version = version;
this.supportedVersions = supportedVersions;
Object.setPrototypeOf(this, SchemaVersionError.prototype);
}
}
/**
*
* @group Errors
* Error thrown when a file read or write operation fails.
*/
export class FileError extends CTRFError {
/** The file path that caused the error */
filePath;
/** The original error */
cause;
constructor(message, filePath, cause) {
super(message);
this.name = 'FileError';
this.filePath = filePath;
this.cause = cause;
Object.setPrototypeOf(this, FileError.prototype);
}
}
/**
*
* @group Errors
* Error thrown when building a report or test fails due to missing required fields.
*/
export class BuilderError extends CTRFError {
constructor(message) {
super(message);
this.name = 'BuilderError';
Object.setPrototypeOf(this, BuilderError.prototype);
}
}
import { describe, it, expect } from 'vitest';
import { CTRFError, ValidationError, ParseError, SchemaVersionError, FileError, BuilderError, } from './errors.js';
describe('errors', () => {
describe('CTRFError', () => {
it('should create error with message', () => {
const error = new CTRFError('test error');
expect(error.message).toBe('test error');
expect(error.name).toBe('CTRFError');
expect(error).toBeInstanceOf(Error);
expect(error).toBeInstanceOf(CTRFError);
});
});
describe('ValidationError', () => {
it('should create error with message and errors array', () => {
const details = [
{
message: 'Invalid status',
path: '/results/tests/0/status',
keyword: 'enum',
},
];
const error = new ValidationError('Validation failed', details);
expect(error.message).toBe('Validation failed');
expect(error.name).toBe('ValidationError');
expect(error.errors).toEqual(details);
expect(error).toBeInstanceOf(CTRFError);
});
it('should default to empty errors array', () => {
const error = new ValidationError('Validation failed');
expect(error.errors).toEqual([]);
});
});
describe('ParseError', () => {
it('should create error with message and cause', () => {
const cause = new Error('JSON syntax error');
const error = new ParseError('Failed to parse', cause);
expect(error.message).toBe('Failed to parse');
expect(error.name).toBe('ParseError');
expect(error.cause).toBe(cause);
expect(error).toBeInstanceOf(CTRFError);
});
});
describe('SchemaVersionError', () => {
it('should create error with version info', () => {
const error = new SchemaVersionError('2.0.0', ['0.0.0', '1.0.0']);
expect(error.message).toContain('2.0.0');
expect(error.message).toContain('0.0.0');
expect(error.message).toContain('1.0.0');
expect(error.name).toBe('SchemaVersionError');
expect(error.version).toBe('2.0.0');
expect(error.supportedVersions).toEqual(['0.0.0', '1.0.0']);
expect(error).toBeInstanceOf(CTRFError);
});
});
describe('FileError', () => {
it('should create error with file path and cause', () => {
const cause = new Error('ENOENT');
const error = new FileError('File not found', '/path/to/file.json', cause);
expect(error.message).toBe('File not found');
expect(error.name).toBe('FileError');
expect(error.filePath).toBe('/path/to/file.json');
expect(error.cause).toBe(cause);
expect(error).toBeInstanceOf(CTRFError);
});
});
describe('BuilderError', () => {
it('should create error with message', () => {
const error = new BuilderError('Missing required field');
expect(error.message).toBe('Missing required field');
expect(error.name).toBe('BuilderError');
expect(error).toBeInstanceOf(CTRFError);
});
});
});
/**
* CTRF Filtering and Querying
*/
import type { CTRFReport, Test, FilterCriteria, TestStatus } from './types.js';
/**
*
* @group Query & Filter
* Filter tests in a report by criteria.
*
* @param report - The CTRF report containing tests to filter
* @param criteria - Filter criteria (status, tags, suite, flaky, browser, device)
* @returns Array of tests matching all specified criteria
*
* @example
* ```typescript
* // Filter by status
* const failed = filterTests(report, { status: 'failed' });
*
* // Filter by multiple criteria
* const filtered = filterTests(report, {
* status: ['failed', 'skipped'],
* tags: ['smoke'],
* flaky: true
* });
* ```
*/
export declare function filterTests(report: CTRFReport, criteria: FilterCriteria): Test[];
/**
*
* @group Query & Filter
* Find a single test in a report.
*
* @param report - The CTRF report to search
* @param criteria - Filter criteria including id, name, status, tags, etc.
* @returns The first matching test, or undefined if not found
*
* @example
* ```typescript
* // Find by ID
* const test = findTest(report, { id: 'uuid' });
*
* // Find by name
* const test = findTest(report, { name: 'should login' });
*
* // Find by multiple criteria
* const test = findTest(report, { status: 'failed', flaky: true });
* ```
*/
export declare function findTest(report: CTRFReport, criteria: FilterCriteria): Test | undefined;
/**
* Group tests by a field.
*
* @param tests - Array of tests to group
* @param field - Field to group by
* @returns Object with field values as keys and arrays of tests as values
*
* @example
* ```typescript
* const byStatus = groupBy(report.results.tests, 'status');
* // => { passed: [...], failed: [...], ... }
*
* const bySuite = groupBy(report.results.tests, 'suite');
* // Groups by first suite level
* ```
*/
export declare function groupBy<K extends keyof Test>(tests: Test[], field: K): Record<string, Test[]>;
/**
* Get tests by status.
*
* @param report - The report to query
* @param status - The status to filter by
* @returns Array of tests with the given status
*/
export declare function getTestsByStatus(report: CTRFReport, status: TestStatus): Test[];
/**
* Get all failed tests.
*
* @param report - The report to query
* @returns Array of failed tests
*/
export declare function getFailedTests(report: CTRFReport): Test[];
/**
* Get all passed tests.
*
* @param report - The report to query
* @returns Array of passed tests
*/
export declare function getPassedTests(report: CTRFReport): Test[];
/**
* Get all skipped tests.
*
* @param report - The report to query
* @returns Array of skipped tests
*/
export declare function getSkippedTests(report: CTRFReport): Test[];
/**
* Get all flaky tests.
*
* @param report - The report to query
* @returns Array of flaky tests
*/
export declare function getFlakyTests(report: CTRFReport): Test[];
/**
* Get tests by tag.
*
* @param report - The report to query
* @param tag - The tag to filter by
* @returns Array of tests with the given tag
*/
export declare function getTestsByTag(report: CTRFReport, tag: string): Test[];
/**
* Get tests by suite.
*
* @param report - The report to query
* @param suiteName - The suite name to filter by (can be any level in the hierarchy)
* @returns Array of tests in the given suite
*/
export declare function getTestsBySuite(report: CTRFReport, suiteName: string): Test[];
/**
* Get unique suite names from a report.
*
* @param report - The report to query
* @returns Array of unique suite names
*/
export declare function getUniqueSuites(report: CTRFReport): string[];
/**
* Get unique tags from a report.
*
* @param report - The report to query
* @returns Array of unique tags
*/
export declare function getUniqueTags(report: CTRFReport): string[];
/**
* CTRF Filtering and Querying
*/
/**
*
* @group Query & Filter
* Filter tests in a report by criteria.
*
* @param report - The CTRF report containing tests to filter
* @param criteria - Filter criteria (status, tags, suite, flaky, browser, device)
* @returns Array of tests matching all specified criteria
*
* @example
* ```typescript
* // Filter by status
* const failed = filterTests(report, { status: 'failed' });
*
* // Filter by multiple criteria
* const filtered = filterTests(report, {
* status: ['failed', 'skipped'],
* tags: ['smoke'],
* flaky: true
* });
* ```
*/
export function filterTests(report, criteria) {
return report.results.tests.filter(test => matchesCriteria(test, criteria));
}
/**
*
* @group Query & Filter
* Find a single test in a report.
*
* @param report - The CTRF report to search
* @param criteria - Filter criteria including id, name, status, tags, etc.
* @returns The first matching test, or undefined if not found
*
* @example
* ```typescript
* // Find by ID
* const test = findTest(report, { id: 'uuid' });
*
* // Find by name
* const test = findTest(report, { name: 'should login' });
*
* // Find by multiple criteria
* const test = findTest(report, { status: 'failed', flaky: true });
* ```
*/
export function findTest(report, criteria) {
const { id, name, ...filterCriteria } = criteria;
return report.results.tests.find(test => {
if (id !== undefined && test.id !== id) {
return false;
}
if (name !== undefined && test.name !== name) {
return false;
}
return matchesCriteria(test, filterCriteria);
});
}
/**
* Group tests by a field.
*
* @param tests - Array of tests to group
* @param field - Field to group by
* @returns Object with field values as keys and arrays of tests as values
*
* @example
* ```typescript
* const byStatus = groupBy(report.results.tests, 'status');
* // => { passed: [...], failed: [...], ... }
*
* const bySuite = groupBy(report.results.tests, 'suite');
* // Groups by first suite level
* ```
*/
export function groupBy(tests, field) {
const groups = {};
for (const test of tests) {
let key;
if (field === 'suite') {
key = test.suite?.[0] ?? 'root';
}
else if (field === 'tags') {
const tags = test.tags ?? ['untagged'];
for (const tag of tags) {
if (!groups[tag]) {
groups[tag] = [];
}
groups[tag].push(test);
}
continue;
}
else {
const value = test[field];
key = value !== undefined ? String(value) : 'undefined';
}
if (!groups[key]) {
groups[key] = [];
}
groups[key].push(test);
}
return groups;
}
/**
* Get tests by status.
*
* @param report - The report to query
* @param status - The status to filter by
* @returns Array of tests with the given status
*/
export function getTestsByStatus(report, status) {
return report.results.tests.filter(test => test.status === status);
}
/**
* Get all failed tests.
*
* @param report - The report to query
* @returns Array of failed tests
*/
export function getFailedTests(report) {
return getTestsByStatus(report, 'failed');
}
/**
* Get all passed tests.
*
* @param report - The report to query
* @returns Array of passed tests
*/
export function getPassedTests(report) {
return getTestsByStatus(report, 'passed');
}
/**
* Get all skipped tests.
*
* @param report - The report to query
* @returns Array of skipped tests
*/
export function getSkippedTests(report) {
return getTestsByStatus(report, 'skipped');
}
/**
* Get all flaky tests.
*
* @param report - The report to query
* @returns Array of flaky tests
*/
export function getFlakyTests(report) {
return report.results.tests.filter(test => test.flaky === true);
}
/**
* Get tests by tag.
*
* @param report - The report to query
* @param tag - The tag to filter by
* @returns Array of tests with the given tag
*/
export function getTestsByTag(report, tag) {
return report.results.tests.filter(test => test.tags?.includes(tag));
}
/**
* Get tests by suite.
*
* @param report - The report to query
* @param suiteName - The suite name to filter by (can be any level in the hierarchy)
* @returns Array of tests in the given suite
*/
export function getTestsBySuite(report, suiteName) {
return report.results.tests.filter(test => test.suite?.includes(suiteName));
}
/**
* Get unique suite names from a report.
*
* @param report - The report to query
* @returns Array of unique suite names
*/
export function getUniqueSuites(report) {
const suites = new Set();
for (const test of report.results.tests) {
if (test.suite) {
for (const suite of test.suite) {
suites.add(suite);
}
}
}
return Array.from(suites);
}
/**
* Get unique tags from a report.
*
* @param report - The report to query
* @returns Array of unique tags
*/
export function getUniqueTags(report) {
const tags = new Set();
for (const test of report.results.tests) {
if (test.tags) {
for (const tag of test.tags) {
tags.add(tag);
}
}
}
return Array.from(tags);
}
/**
* Check if a test matches the given criteria.
*/
function matchesCriteria(test, criteria) {
if (criteria.status !== undefined) {
const statuses = Array.isArray(criteria.status)
? criteria.status
: [criteria.status];
if (!statuses.includes(test.status)) {
return false;
}
}
if (criteria.tags !== undefined) {
const requiredTags = Array.isArray(criteria.tags)
? criteria.tags
: [criteria.tags];
if (!test.tags || !requiredTags.some(tag => test.tags.includes(tag))) {
return false;
}
}
if (criteria.suite !== undefined) {
const requiredSuites = Array.isArray(criteria.suite)
? criteria.suite
: [criteria.suite];
if (!test.suite ||
!requiredSuites.some(suite => test.suite.includes(suite))) {
return false;
}
}
if (criteria.flaky !== undefined && test.flaky !== criteria.flaky) {
return false;
}
if (criteria.browser !== undefined && test.browser !== criteria.browser) {
return false;
}
if (criteria.device !== undefined && test.device !== criteria.device) {
return false;
}
return true;
}
import { describe, it, expect } from 'vitest';
import { filterTests, findTest, groupBy, getTestsByStatus, getFailedTests, getPassedTests, getSkippedTests, getFlakyTests, getTestsByTag, getTestsBySuite, getUniqueSuites, getUniqueTags, } from './filter.js';
import { ReportBuilder, TestBuilder } from './builder.js';
describe('filter', () => {
const createReport = (tests) => {
const builder = new ReportBuilder().tool({ name: 'jest' });
for (const t of tests) {
const testBuilder = new TestBuilder()
.name(t.name || 'test')
.status(t.status || 'passed')
.duration(t.duration || 100);
if (t.id)
testBuilder.id(t.id);
if (t.tags)
testBuilder.tags(t.tags);
if (t.suite)
testBuilder.suite(t.suite);
if (t.flaky !== undefined)
testBuilder.flaky(t.flaky);
if (t.browser)
testBuilder.browser(t.browser);
if (t.device)
testBuilder.device(t.device);
builder.addTest(testBuilder.build());
}
return builder.build();
};
describe('filterTests', () => {
it('should filter by single status', () => {
const report = createReport([
{ name: 'test1', status: 'passed' },
{ name: 'test2', status: 'failed' },
{ name: 'test3', status: 'passed' },
]);
const result = filterTests(report, { status: 'passed' });
expect(result).toHaveLength(2);
expect(result.every(t => t.status === 'passed')).toBe(true);
});
it('should filter by multiple statuses', () => {
const report = createReport([
{ name: 'test1', status: 'passed' },
{ name: 'test2', status: 'failed' },
{ name: 'test3', status: 'skipped' },
]);
const result = filterTests(report, { status: ['failed', 'skipped'] });
expect(result).toHaveLength(2);
});
it('should filter by single tag', () => {
const report = createReport([
{ name: 'test1', tags: ['smoke', 'fast'] },
{ name: 'test2', tags: ['slow'] },
{ name: 'test3', tags: ['smoke'] },
]);
const result = filterTests(report, { tags: 'smoke' });
expect(result).toHaveLength(2);
});
it('should filter by multiple tags (any match)', () => {
const report = createReport([
{ name: 'test1', tags: ['smoke'] },
{ name: 'test2', tags: ['critical'] },
{ name: 'test3', tags: ['other'] },
]);
const result = filterTests(report, { tags: ['smoke', 'critical'] });
expect(result).toHaveLength(2);
});
it('should filter by suite', () => {
const report = createReport([
{ name: 'test1', suite: ['unit', 'auth'] },
{ name: 'test2', suite: ['unit', 'api'] },
{ name: 'test3', suite: ['integration'] },
]);
const result = filterTests(report, { suite: 'auth' });
expect(result).toHaveLength(1);
expect(result[0].name).toBe('test1');
});
it('should filter by flaky', () => {
const report = createReport([
{ name: 'test1', flaky: true },
{ name: 'test2', flaky: false },
{ name: 'test3' },
]);
const result = filterTests(report, { flaky: true });
expect(result).toHaveLength(1);
expect(result[0].name).toBe('test1');
});
it('should filter by browser', () => {
const report = createReport([
{ name: 'test1', browser: 'chrome' },
{ name: 'test2', browser: 'firefox' },
]);
const result = filterTests(report, { browser: 'chrome' });
expect(result).toHaveLength(1);
});
it('should filter by device', () => {
const report = createReport([
{ name: 'test1', device: 'mobile' },
{ name: 'test2', device: 'desktop' },
]);
const result = filterTests(report, { device: 'mobile' });
expect(result).toHaveLength(1);
});
it('should combine multiple criteria', () => {
const report = createReport([
{ name: 'test1', status: 'failed', tags: ['smoke'], flaky: true },
{ name: 'test2', status: 'failed', tags: ['smoke'], flaky: false },
{ name: 'test3', status: 'passed', tags: ['smoke'], flaky: true },
]);
const result = filterTests(report, {
status: 'failed',
tags: 'smoke',
flaky: true,
});
expect(result).toHaveLength(1);
expect(result[0].name).toBe('test1');
});
});
describe('findTest', () => {
it('should find test by id', () => {
const report = createReport([
{ name: 'test1', id: 'id-1' },
{ name: 'test2', id: 'id-2' },
]);
const result = findTest(report, { id: 'id-2' });
expect(result?.name).toBe('test2');
});
it('should find test by name', () => {
const report = createReport([{ name: 'test1' }, { name: 'test2' }]);
const result = findTest(report, { name: 'test1' });
expect(result?.name).toBe('test1');
});
it('should return undefined for non-existent test', () => {
const report = createReport([{ name: 'test1' }]);
const result = findTest(report, { id: 'non-existent' });
expect(result).toBeUndefined();
});
it('should combine id/name with filter criteria', () => {
const report = createReport([
{ name: 'test', status: 'passed', tags: ['smoke'] },
{ name: 'test', status: 'failed', tags: ['smoke'] },
]);
const result = findTest(report, { name: 'test', status: 'failed' });
expect(result?.status).toBe('failed');
});
});
describe('groupBy', () => {
it('should group by status', () => {
const report = createReport([
{ name: 'test1', status: 'passed' },
{ name: 'test2', status: 'passed' },
{ name: 'test3', status: 'failed' },
]);
const groups = groupBy(report.results.tests, 'status');
expect(groups['passed']).toHaveLength(2);
expect(groups['failed']).toHaveLength(1);
});
it('should group by first suite level', () => {
const report = createReport([
{ name: 'test1', suite: ['unit', 'auth'] },
{ name: 'test2', suite: ['unit', 'api'] },
{ name: 'test3', suite: ['integration'] },
{ name: 'test4' },
]);
const groups = groupBy(report.results.tests, 'suite');
expect(groups['unit']).toHaveLength(2);
expect(groups['integration']).toHaveLength(1);
expect(groups['root']).toHaveLength(1);
});
it('should group by tags (tests may appear in multiple groups)', () => {
const report = createReport([
{ name: 'test1', tags: ['smoke', 'fast'] },
{ name: 'test2', tags: ['smoke'] },
{ name: 'test3', tags: ['slow'] },
]);
const groups = groupBy(report.results.tests, 'tags');
expect(groups['smoke']).toHaveLength(2);
expect(groups['fast']).toHaveLength(1);
expect(groups['slow']).toHaveLength(1);
});
});
describe('getTestsByStatus', () => {
it('should return tests with given status', () => {
const report = createReport([
{ name: 'test1', status: 'passed' },
{ name: 'test2', status: 'failed' },
]);
const result = getTestsByStatus(report, 'failed');
expect(result).toHaveLength(1);
expect(result[0].name).toBe('test2');
});
});
describe('getFailedTests', () => {
it('should return only failed tests', () => {
const report = createReport([
{ name: 'test1', status: 'passed' },
{ name: 'test2', status: 'failed' },
]);
expect(getFailedTests(report)).toHaveLength(1);
});
});
describe('getPassedTests', () => {
it('should return only passed tests', () => {
const report = createReport([
{ name: 'test1', status: 'passed' },
{ name: 'test2', status: 'failed' },
]);
expect(getPassedTests(report)).toHaveLength(1);
});
});
describe('getSkippedTests', () => {
it('should return only skipped tests', () => {
const report = createReport([
{ name: 'test1', status: 'passed' },
{ name: 'test2', status: 'skipped' },
]);
expect(getSkippedTests(report)).toHaveLength(1);
});
});
describe('getFlakyTests', () => {
it('should return only flaky tests', () => {
const report = createReport([
{ name: 'test1', flaky: true },
{ name: 'test2', flaky: false },
{ name: 'test3' },
]);
expect(getFlakyTests(report)).toHaveLength(1);
});
});
describe('getTestsByTag', () => {
it('should return tests with given tag', () => {
const report = createReport([
{ name: 'test1', tags: ['smoke', 'fast'] },
{ name: 'test2', tags: ['slow'] },
]);
const result = getTestsByTag(report, 'smoke');
expect(result).toHaveLength(1);
expect(result[0].name).toBe('test1');
});
});
describe('getTestsBySuite', () => {
it('should return tests in given suite', () => {
const report = createReport([
{ name: 'test1', suite: ['unit', 'auth'] },
{ name: 'test2', suite: ['integration'] },
]);
const result = getTestsBySuite(report, 'unit');
expect(result).toHaveLength(1);
expect(result[0].name).toBe('test1');
});
});
describe('getUniqueSuites', () => {
it('should return all unique suite names', () => {
const report = createReport([
{ name: 'test1', suite: ['unit', 'auth'] },
{ name: 'test2', suite: ['unit', 'api'] },
{ name: 'test3', suite: ['integration'] },
]);
const suites = getUniqueSuites(report);
expect(suites).toContain('unit');
expect(suites).toContain('auth');
expect(suites).toContain('api');
expect(suites).toContain('integration');
});
});
describe('getUniqueTags', () => {
it('should return all unique tags', () => {
const report = createReport([
{ name: 'test1', tags: ['smoke', 'fast'] },
{ name: 'test2', tags: ['smoke', 'slow'] },
]);
const tags = getUniqueTags(report);
expect(tags).toContain('smoke');
expect(tags).toContain('fast');
expect(tags).toContain('slow');
expect(tags).toHaveLength(3);
});
});
});
export declare function identifyFlakyTests(filePath: string): Promise<void>;
import fs from 'fs';
import path from 'path';
export async function identifyFlakyTests(filePath) {
try {
const resolvedFilePath = path.resolve(filePath);
if (!fs.existsSync(resolvedFilePath)) {
console.error(`The file ${resolvedFilePath} does not exist.`);
return;
}
const fileContent = fs.readFileSync(resolvedFilePath, 'utf8');
const report = JSON.parse(fileContent);
const flakyTests = report.results.tests.filter((test) => test.flaky === true);
if (flakyTests.length > 0) {
console.log(`Found ${flakyTests.length} flaky test(s):`);
flakyTests.forEach((test) => {
console.log(`- Test Name: ${test.name}, Retries: ${test.retries}`);
});
}
else {
console.log(`No flaky tests found in ${resolvedFilePath}.`);
}
}
catch (error) {
console.error('Error identifying flaky tests:', error);
}
}
/**
* CTRF Test ID Generation
*/
/**
*
* @group ID Generation
* Generate a deterministic UUID v5 for a test based on its properties.
* The same inputs will always produce the same UUID, enabling
* cross-run analysis and test identification.
*
* @param properties - Test properties to generate ID from
* @param properties.name - Test name (required)
* @param properties.suite - Suite hierarchy (optional)
* @param properties.filePath - File path (optional)
* @returns A deterministic UUID v5 string
*
* @example
* ```typescript
* const id = generateTestId({
* name: 'should add numbers',
* suite: ['math', 'addition'],
* filePath: 'tests/math.test.ts'
* });
* // Always returns the same UUID for these inputs
* ```
*/
export declare function generateTestId(properties: {
name: string;
suite?: string[];
filePath?: string;
}): string;
/**
*
* @group ID Generation
* Generate a random UUID v4 for report identification.
*
* @returns A random UUID v4 string
*
* @example
* ```typescript
* const reportId = generateReportId();
* // => 'f47ac10b-58cc-4372-a567-0e02b2c3d479'
* ```
*/
export declare function generateReportId(): string;
/**
* CTRF Test ID Generation
*/
import { createHash, randomUUID } from 'crypto';
import { CTRF_NAMESPACE } from './constants.js';
/**
*
* @group ID Generation
* Generate a deterministic UUID v5 for a test based on its properties.
* The same inputs will always produce the same UUID, enabling
* cross-run analysis and test identification.
*
* @param properties - Test properties to generate ID from
* @param properties.name - Test name (required)
* @param properties.suite - Suite hierarchy (optional)
* @param properties.filePath - File path (optional)
* @returns A deterministic UUID v5 string
*
* @example
* ```typescript
* const id = generateTestId({
* name: 'should add numbers',
* suite: ['math', 'addition'],
* filePath: 'tests/math.test.ts'
* });
* // Always returns the same UUID for these inputs
* ```
*/
export function generateTestId(properties) {
const { name, suite, filePath } = properties;
const suiteString = suite ? suite.join('/') : '';
const identifier = `${name}|${suiteString}|${filePath || ''}`;
const namespaceBytes = CTRF_NAMESPACE.replace(/-/g, '')
.match(/.{2}/g)
.map(byte => parseInt(byte, 16));
const input = Buffer.concat([
Buffer.from(namespaceBytes),
Buffer.from(identifier, 'utf8'),
]);
const hash = createHash('sha1').update(input).digest('hex');
const uuid = [
hash.substring(0, 8),
hash.substring(8, 12),
'5' + hash.substring(13, 16), // Version 5
((parseInt(hash.substring(16, 17), 16) & 0x3) | 0x8).toString(16) +
hash.substring(17, 20), // Variant bits
hash.substring(20, 32),
].join('-');
return uuid;
}
/**
*
* @group ID Generation
* Generate a random UUID v4 for report identification.
*
* @returns A random UUID v4 string
*
* @example
* ```typescript
* const reportId = generateReportId();
* // => 'f47ac10b-58cc-4372-a567-0e02b2c3d479'
* ```
*/
export function generateReportId() {
return randomUUID();
}
export {};
import { describe, it, expect } from 'vitest';
import { generateTestId, generateReportId } from './id.js';
describe('id', () => {
describe('generateTestId', () => {
it('should generate a deterministic UUID v5', () => {
const id1 = generateTestId({
name: 'should add numbers',
suite: ['math', 'addition'],
filePath: 'tests/math.test.ts',
});
const id2 = generateTestId({
name: 'should add numbers',
suite: ['math', 'addition'],
filePath: 'tests/math.test.ts',
});
expect(id1).toBe(id2);
});
it('should generate different IDs for different inputs', () => {
const id1 = generateTestId({ name: 'test 1' });
const id2 = generateTestId({ name: 'test 2' });
expect(id1).not.toBe(id2);
});
it('should generate valid UUID v5 format', () => {
const id = generateTestId({ name: 'test' });
// UUID v5 format: xxxxxxxx-xxxx-5xxx-yxxx-xxxxxxxxxxxx
// where y is 8, 9, a, or b
expect(id).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-5[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i);
});
it('should handle missing suite', () => {
const id = generateTestId({
name: 'test',
filePath: 'test.ts',
});
expect(id).toBeDefined();
});
it('should handle missing filePath', () => {
const id = generateTestId({
name: 'test',
suite: ['unit'],
});
expect(id).toBeDefined();
});
it('should handle only name', () => {
const id = generateTestId({ name: 'test' });
expect(id).toBeDefined();
});
it('should include suite in hash', () => {
const withoutSuite = generateTestId({ name: 'test' });
const withSuite = generateTestId({ name: 'test', suite: ['unit'] });
expect(withoutSuite).not.toBe(withSuite);
});
it('should include filePath in hash', () => {
const withoutPath = generateTestId({ name: 'test' });
const withPath = generateTestId({ name: 'test', filePath: 'test.ts' });
expect(withoutPath).not.toBe(withPath);
});
});
describe('generateReportId', () => {
it('should generate a valid UUID v4', () => {
const id = generateReportId();
// UUID v4 format: xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx
expect(id).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i);
});
it('should generate unique IDs', () => {
const ids = new Set();
for (let i = 0; i < 100; i++) {
ids.add(generateReportId());
}
expect(ids.size).toBe(100);
});
});
});
/**
* CTRF Insights Calculation
*
* This module replicates the functionality of the existing run-insights.ts
* while using the reference implementation's type system and API signatures.
*/
import type { CTRFReport, Test, Insights, TestInsights, MetricDelta, InsightsOptions } from './types.js';
/**
*
* @group Insights
* Determines if a test is flaky based on the CTRF specification.
*
* A test is considered flaky if:
* - The `flaky` field is explicitly set to `true`, OR
* - The test has retries > 0 AND final status is 'passed'
*
* @param test - The test to check
* @returns true if the test is flaky
*
* @example
* ```typescript
* if (isTestFlaky(test)) {
* console.log('Test is flaky:', test.name);
* }
* ```
*/
export declare function isTestFlaky(test: Test): boolean;
/**
* Utility function that formats a ratio (0-1) as a percentage string for display.
*/
export declare function formatAsPercentage(ratio: number, decimals?: number): string;
/**
* Utility function that formats a MetricDelta as percentage strings for display.
*/
export declare function formatMetricDeltaAsPercentage(metric: MetricDelta, decimals?: number): {
current: string;
baseline: string;
change: string;
};
/**
* Utility function that calculates percent fractional change between current and baseline values.
*/
export declare function calculatePercentChange(current: number, baseline: number, decimals?: number): number;
/**
* Calculate run-level insights from multiple historical reports.
*
* This is a simplified API for getting insights from an array of reports.
* For the full enrichment workflow, use `addInsights`.
*
* @param reports - Array of historical reports (most recent last)
* @param options - Insights options
* @returns Calculated insights
*
* @example
* ```typescript
* const insights = calculateInsights(historicalReports);
*
* // With baseline
* const insights = calculateInsights(reports, { baseline: baselineReport });
*
* // With limited window
* const insights = calculateInsights(reports, { window: 10 });
* ```
*/
export declare function calculateInsights(reports: CTRFReport[], options?: InsightsOptions): Insights;
/**
* Calculate insights for a specific test across multiple reports.
*
* @param reports - Array of historical reports
* @param testId - The test ID to calculate insights for
* @returns Calculated test insights
*
* @example
* ```typescript
* const insights = calculateTestInsights(reports, 'test-uuid');
* ```
*/
export declare function calculateTestInsights(reports: CTRFReport[], testId: string): TestInsights;
/**
* Calculate the metric delta between current and baseline values.
*
* @param current - Current value
* @param baseline - Baseline value
* @returns MetricDelta object
*/
export declare function calculateMetricDelta(current: number, baseline: number): MetricDelta;
/**
*
* @group Insights
* Add insights to a CTRF report using historical data.
*
* Computes run-level and test-level insights according to the CTRF specification,
* including pass rate, fail rate, flaky rate, and duration metrics.
*
* @param report - The current report to enrich with insights
* @param historicalReports - Array of previous reports for trend analysis
* @param options - Options including baseline for comparison
* @returns A new report with insights populated
*
* @example
* ```typescript
* // Basic usage
* const reportWithInsights = addInsights(currentReport, previousReports);
*
* // With baseline comparison
* const reportWithInsights = addInsights(currentReport, previousReports, {
* baseline: baselineReport
* });
* ```
*/
export declare function addInsights(report: CTRFReport, historicalReports?: CTRFReport[], options?: InsightsOptions): CTRFReport;
/**
* CTRF Insights Calculation
*
* This module replicates the functionality of the existing run-insights.ts
* while using the reference implementation's type system and API signatures.
*/
// ============================================================================
// Utility Functions
// ============================================================================
/**
*
* @group Insights
* Determines if a test is flaky based on the CTRF specification.
*
* A test is considered flaky if:
* - The `flaky` field is explicitly set to `true`, OR
* - The test has retries > 0 AND final status is 'passed'
*
* @param test - The test to check
* @returns true if the test is flaky
*
* @example
* ```typescript
* if (isTestFlaky(test)) {
* console.log('Test is flaky:', test.name);
* }
* ```
*/
export function isTestFlaky(test) {
return (test.flaky === true ||
(test.retries !== undefined && test.retries > 0 && test.status === 'passed'));
}
/**
* Utility function that formats a ratio (0-1) as a percentage string for display.
*/
export function formatAsPercentage(ratio, decimals = 2) {
return `${(ratio * 100).toFixed(decimals)}%`;
}
/**
* Utility function that formats a MetricDelta as percentage strings for display.
*/
export function formatMetricDeltaAsPercentage(metric, decimals = 2) {
return {
current: formatAsPercentage(metric.current ?? 0, decimals),
baseline: formatAsPercentage(metric.baseline ?? 0, decimals),
change: `${(metric.change ?? 0) >= 0 ? '+' : ''}${formatAsPercentage(metric.change ?? 0, decimals)}`,
};
}
/**
* Utility function that calculates percent fractional change between current and baseline values.
*/
export function calculatePercentChange(current, baseline, decimals = 4) {
if (baseline === 0) {
return 0;
}
return Number(((current - baseline) / baseline).toFixed(decimals));
}
/**
* Calculates the 95th percentile from an array of numbers.
*/
function calculateP95(values) {
if (values.length === 0)
return 0;
const sorted = [...values].sort((a, b) => a - b);
const index = Math.ceil(sorted.length * 0.95) - 1;
return Number(sorted[Math.max(0, index)].toFixed(2));
}
/**
* Helper function to validate that reports have the necessary data for insights calculation.
*/
function validateReportForInsights(report) {
return !!(report?.results?.tests && Array.isArray(report.results.tests));
}
/**
* Sort reports by timestamp (newest first).
*/
function sortReportsByTimestamp(reports) {
return [...reports].sort((a, b) => {
const aStart = a.results?.summary?.start ?? 0;
const bStart = b.results?.summary?.start ?? 0;
return bStart - aStart;
});
}
// ============================================================================
// Metrics Aggregation Functions
// ============================================================================
/**
* Aggregates test metrics across multiple reports.
*/
function aggregateTestMetricsAcrossReports(reports) {
const metricsMap = new Map();
for (const report of reports) {
if (!validateReportForInsights(report))
continue;
for (const test of report.results.tests) {
const isPassed = test.status === 'passed';
const isFailed = test.status === 'failed';
const isSkipped = test.status === 'skipped';
const isPending = test.status === 'pending';
const isOther = test.status === 'other';
const testName = test.name;
if (!metricsMap.has(testName)) {
metricsMap.set(testName, {
totalAttempts: 0,
totalAttemptsFailed: 0,
totalResults: 0,
totalResultsFailed: 0,
totalResultsPassed: 0,
totalResultsSkipped: 0,
totalResultsFlaky: 0,
totalAttemptsFlaky: 0,
totalResultsDuration: 0,
appearsInRuns: 0,
reportsAnalyzed: 0,
durations: [],
});
}
const metrics = metricsMap.get(testName);
metrics.totalResults += 1;
metrics.totalAttempts += 1 + (test.retries || 0);
metrics.totalAttemptsFailed += test.retries || 0;
if (isFailed) {
metrics.totalResultsFailed += 1;
metrics.totalAttemptsFailed += 1 + (test.retries || 0);
}
else if (isPassed) {
metrics.totalResultsPassed += 1;
}
else if (isSkipped || isPending || isOther) {
metrics.totalResultsSkipped += 1;
}
if (isTestFlaky(test)) {
metrics.totalResultsFlaky += 1;
metrics.totalAttemptsFlaky += test.retries || 0;
}
metrics.totalResultsDuration += test.duration || 0;
metrics.durations.push(test.duration || 0);
}
const testsInThisReport = new Set();
for (const test of report.results.tests) {
testsInThisReport.add(test.name);
}
for (const testName of testsInThisReport) {
const metrics = metricsMap.get(testName);
metrics.appearsInRuns += 1;
}
}
return metricsMap;
}
/**
* Consolidates all test-level metrics into overall run-level metrics.
*/
function consolidateTestMetricsToRunMetrics(metricsMap) {
let totalAttempts = 0;
let totalAttemptsFailed = 0;
let totalResults = 0;
let totalResultsFailed = 0;
let totalResultsPassed = 0;
let totalResultsSkipped = 0;
let totalResultsFlaky = 0;
let totalAttemptsFlaky = 0;
let totalResultsDuration = 0;
for (const metrics of metricsMap.values()) {
totalAttempts += metrics.totalAttempts;
totalAttemptsFailed += metrics.totalAttemptsFailed;
totalResults += metrics.totalResults;
totalResultsFailed += metrics.totalResultsFailed;
totalResultsPassed += metrics.totalResultsPassed;
totalResultsSkipped += metrics.totalResultsSkipped;
totalResultsFlaky += metrics.totalResultsFlaky;
totalAttemptsFlaky += metrics.totalAttemptsFlaky;
totalResultsDuration += metrics.totalResultsDuration;
}
return {
totalAttempts,
totalAttemptsFailed,
totalResults,
totalResultsFailed,
totalResultsPassed,
totalResultsSkipped,
totalResultsFlaky,
totalAttemptsFlaky,
totalResultsDuration,
reportsAnalyzed: metricsMap.size,
};
}
// ============================================================================
// Rate Calculation Functions
// ============================================================================
function calculateFlakyRateFromMetrics(runMetrics) {
if (runMetrics.totalAttempts === 0) {
return 0;
}
return Number((runMetrics.totalAttemptsFlaky /
(runMetrics.totalResults + runMetrics.totalAttemptsFlaky)).toFixed(4));
}
function calculateFailRateFromMetrics(runMetrics) {
if (runMetrics.totalResults === 0) {
return 0;
}
return Number((runMetrics.totalResultsFailed / runMetrics.totalResults).toFixed(4));
}
function calculatePassRateFromMetrics(runMetrics) {
if (runMetrics.totalResults === 0) {
return 0;
}
return Number((runMetrics.totalResultsPassed / runMetrics.totalResults).toFixed(4));
}
function calculateAverageTestDurationFromMetrics(runMetrics) {
if (runMetrics.totalResults === 0) {
return 0;
}
return Number((runMetrics.totalResultsDuration / runMetrics.totalResults).toFixed(2));
}
function calculateAverageRunDurationFromMetrics(runMetrics) {
if (runMetrics.reportsAnalyzed === 0) {
return 0;
}
return Number((runMetrics.totalResultsDuration / runMetrics.reportsAnalyzed).toFixed(2));
}
function calculateP95RunDurationFromReports(reports) {
const runDurations = [];
for (const report of reports) {
if (validateReportForInsights(report) && report.results?.summary) {
const { start, stop } = report.results.summary;
if (start && stop && stop > start) {
const runDuration = stop - start;
runDurations.push(runDuration);
}
}
}
return calculateP95(runDurations);
}
// ============================================================================
// Insights Calculation Functions
// ============================================================================
/**
* Internal helper function that recursively calculates insights for each report.
*/
function calculateRunInsights(reports, index = 0) {
if (index >= reports.length) {
return reports;
}
const currentReport = reports[index];
const previousReports = reports.slice(index + 1);
const allReportsUpToThisPoint = [currentReport, ...previousReports];
const testMetrics = aggregateTestMetricsAcrossReports(allReportsUpToThisPoint);
const runMetrics = consolidateTestMetricsToRunMetrics(testMetrics);
const { ...relevantMetrics } = runMetrics;
currentReport.insights = {
passRate: {
current: calculatePassRateFromMetrics(runMetrics),
baseline: 0,
change: 0,
},
flakyRate: {
current: calculateFlakyRateFromMetrics(runMetrics),
baseline: 0,
change: 0,
},
failRate: {
current: calculateFailRateFromMetrics(runMetrics),
baseline: 0,
change: 0,
},
averageTestDuration: {
current: calculateAverageTestDurationFromMetrics(runMetrics),
baseline: 0,
change: 0,
},
averageRunDuration: {
current: calculateAverageRunDurationFromMetrics(runMetrics),
baseline: 0,
change: 0,
},
p95RunDuration: {
current: calculateP95RunDurationFromReports(allReportsUpToThisPoint),
baseline: 0,
change: 0,
},
runsAnalyzed: allReportsUpToThisPoint.length,
extra: relevantMetrics,
};
return calculateRunInsights(reports, index + 1);
}
/**
* Calculates test-level insights with baseline comparison for a specific test.
*/
function calculateTestInsightsWithBaseline(currentTestMetrics, baselineTestMetrics) {
const currentPassRate = currentTestMetrics.totalResults === 0
? 0
: Number((currentTestMetrics.totalResultsPassed /
currentTestMetrics.totalResults).toFixed(4));
const currentFlakyRate = currentTestMetrics.totalAttempts === 0
? 0
: Number((currentTestMetrics.totalAttemptsFlaky /
(currentTestMetrics.totalResults +
currentTestMetrics.totalAttemptsFlaky)).toFixed(4));
const currentFailRate = currentTestMetrics.totalResults === 0
? 0
: Number((currentTestMetrics.totalResultsFailed /
currentTestMetrics.totalResults).toFixed(4));
const currentAverageDuration = currentTestMetrics.totalResults === 0
? 0
: Number((currentTestMetrics.totalResultsDuration /
currentTestMetrics.totalResults).toFixed(2));
const currentP95Duration = calculateP95(currentTestMetrics.durations);
let baselinePassRate = 0;
let baselineFlakyRate = 0;
let baselineFailRate = 0;
let baselineAverageDuration = 0;
let baselineP95Duration = 0;
if (baselineTestMetrics) {
baselinePassRate =
baselineTestMetrics.totalResults === 0
? 0
: Number((baselineTestMetrics.totalResultsPassed /
baselineTestMetrics.totalResults).toFixed(4));
baselineFlakyRate =
baselineTestMetrics.totalAttempts === 0
? 0
: Number((baselineTestMetrics.totalAttemptsFlaky /
baselineTestMetrics.totalAttempts).toFixed(4));
baselineFailRate =
baselineTestMetrics.totalResults === 0
? 0
: Number((baselineTestMetrics.totalResultsFailed /
baselineTestMetrics.totalResults).toFixed(4));
baselineAverageDuration =
baselineTestMetrics.totalResults === 0
? 0
: Number((baselineTestMetrics.totalResultsDuration /
baselineTestMetrics.totalResults).toFixed(2));
baselineP95Duration = calculateP95(baselineTestMetrics.durations);
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const { durations: _durations, ...relevantMetrics } = currentTestMetrics;
return {
passRate: {
current: currentPassRate,
baseline: baselinePassRate,
change: Number((currentPassRate - baselinePassRate).toFixed(4)),
},
flakyRate: {
current: currentFlakyRate,
baseline: baselineFlakyRate,
change: Number((currentFlakyRate - baselineFlakyRate).toFixed(4)),
},
failRate: {
current: currentFailRate,
baseline: baselineFailRate,
change: Number((currentFailRate - baselineFailRate).toFixed(4)),
},
averageTestDuration: {
current: currentAverageDuration,
baseline: baselineAverageDuration,
change: currentAverageDuration - baselineAverageDuration,
},
p95TestDuration: {
current: currentP95Duration,
baseline: baselineP95Duration,
change: currentP95Duration - baselineP95Duration,
},
executedInRuns: currentTestMetrics.appearsInRuns,
extra: relevantMetrics,
};
}
/**
* Internal helper function that adds test-level insights with baseline comparison to all tests.
*/
function addTestInsightsWithBaselineToCurrentReport(currentReport, previousReports, baselineReport) {
if (!validateReportForInsights(currentReport)) {
return currentReport;
}
const allReports = [currentReport, ...previousReports];
const currentTestMetrics = aggregateTestMetricsAcrossReports(allReports);
let baselineTestMetrics;
if (baselineReport && validateReportForInsights(baselineReport)) {
const baselineIndex = previousReports.findIndex(report => report.results?.summary?.start ===
baselineReport.results?.summary?.start);
if (baselineIndex >= 0) {
const reportsUpToBaseline = previousReports.slice(baselineIndex);
baselineTestMetrics =
aggregateTestMetricsAcrossReports(reportsUpToBaseline);
}
}
const reportWithInsights = {
...currentReport,
results: {
...currentReport.results,
tests: currentReport.results.tests.map(test => {
const testName = test.name;
const currentMetrics = currentTestMetrics.get(testName);
if (currentMetrics) {
const baselineMetrics = baselineTestMetrics?.get(testName);
const testInsights = calculateTestInsightsWithBaseline(currentMetrics, baselineMetrics);
return {
...test,
insights: testInsights,
};
}
return test;
}),
},
};
return reportWithInsights;
}
/**
* Internal helper function that calculates baseline report-level insights.
*/
function calculateReportInsightsBaseline(currentReport, baselineReport) {
const currentInsights = currentReport.insights;
const previousInsights = baselineReport.insights;
if (!currentInsights || !previousInsights) {
console.log('Both reports must have insights populated');
return currentReport.insights;
}
return {
passRate: {
current: currentInsights?.passRate?.current ?? 0,
baseline: previousInsights?.passRate?.current ?? 0,
change: Number(((currentInsights?.passRate?.current ?? 0) -
(previousInsights?.passRate?.current ?? 0)).toFixed(4)),
},
flakyRate: {
current: currentInsights?.flakyRate?.current ?? 0,
baseline: previousInsights?.flakyRate?.current ?? 0,
change: Number(((currentInsights?.flakyRate?.current ?? 0) -
(previousInsights?.flakyRate?.current ?? 0)).toFixed(4)),
},
failRate: {
current: currentInsights?.failRate?.current ?? 0,
baseline: previousInsights?.failRate?.current ?? 0,
change: Number(((currentInsights?.failRate?.current ?? 0) -
(previousInsights?.failRate?.current ?? 0)).toFixed(4)),
},
averageTestDuration: {
current: currentInsights?.averageTestDuration?.current ?? 0,
baseline: previousInsights?.averageTestDuration?.current ?? 0,
change: (currentInsights?.averageTestDuration?.current ?? 0) -
(previousInsights?.averageTestDuration?.current ?? 0),
},
averageRunDuration: {
current: currentInsights?.averageRunDuration?.current ?? 0,
baseline: previousInsights?.averageRunDuration?.current ?? 0,
change: (currentInsights?.averageRunDuration?.current ?? 0) -
(previousInsights?.averageRunDuration?.current ?? 0),
},
p95RunDuration: {
current: currentInsights?.p95RunDuration?.current ?? 0,
baseline: previousInsights?.p95RunDuration?.current ?? 0,
change: (currentInsights?.p95RunDuration?.current ?? 0) -
(previousInsights?.p95RunDuration?.current ?? 0),
},
runsAnalyzed: currentInsights?.runsAnalyzed ?? 0,
extra: currentInsights.extra,
};
}
/**
* Gets test details for tests that have been removed since the baseline report.
*/
function getTestsRemovedSinceBaseline(currentReport, baselineReport) {
if (!validateReportForInsights(currentReport) ||
!validateReportForInsights(baselineReport)) {
return [];
}
const currentTestNames = new Set(currentReport.results.tests.map(test => test.name));
const removedTests = baselineReport.results.tests.filter(test => !currentTestNames.has(test.name));
return removedTests.map(test => ({
name: test.name,
suite: test.suite,
filePath: test.filePath,
}));
}
/**
* Gets test details for tests that have been added since the baseline report.
*/
function getTestsAddedSinceBaseline(currentReport, baselineReport) {
if (!validateReportForInsights(currentReport) ||
!validateReportForInsights(baselineReport)) {
return [];
}
const baselineTestNames = new Set(baselineReport.results.tests.map(test => test.name));
const addedTests = currentReport.results.tests.filter(test => !baselineTestNames.has(test.name));
return addedTests.map(test => ({
name: test.name,
suite: test.suite,
filePath: test.filePath,
}));
}
/**
* Sets the removed tests array to insights.extra.testsRemoved.
*/
function setTestsRemovedToInsights(insights, currentReport, baselineReport) {
const removedTests = getTestsRemovedSinceBaseline(currentReport, baselineReport);
return {
...insights,
extra: {
...insights.extra,
testsRemoved: removedTests,
},
};
}
/**
* Sets the added tests array to insights.extra.testsAdded.
*/
function setTestsAddedToInsights(insights, currentReport, baselineReport) {
const addedTests = getTestsAddedSinceBaseline(currentReport, baselineReport);
return {
...insights,
extra: {
...insights.extra,
testsAdded: addedTests,
},
};
}
// ============================================================================
// Main Public API
// ============================================================================
/**
* Calculate run-level insights from multiple historical reports.
*
* This is a simplified API for getting insights from an array of reports.
* For the full enrichment workflow, use `addInsights`.
*
* @param reports - Array of historical reports (most recent last)
* @param options - Insights options
* @returns Calculated insights
*
* @example
* ```typescript
* const insights = calculateInsights(historicalReports);
*
* // With baseline
* const insights = calculateInsights(reports, { baseline: baselineReport });
*
* // With limited window
* const insights = calculateInsights(reports, { window: 10 });
* ```
*/
export function calculateInsights(reports, options = {}) {
const { baseline, window } = options;
// Apply window limit if specified
const reportsToAnalyze = window && window < reports.length ? reports.slice(-window) : reports;
if (reportsToAnalyze.length === 0) {
return { runsAnalyzed: 0 };
}
const testMetrics = aggregateTestMetricsAcrossReports(reportsToAnalyze);
const runMetrics = consolidateTestMetricsToRunMetrics(testMetrics);
const currentInsights = {
passRate: {
current: calculatePassRateFromMetrics(runMetrics),
baseline: 0,
change: 0,
},
failRate: {
current: calculateFailRateFromMetrics(runMetrics),
baseline: 0,
change: 0,
},
flakyRate: {
current: calculateFlakyRateFromMetrics(runMetrics),
baseline: 0,
change: 0,
},
averageTestDuration: {
current: calculateAverageTestDurationFromMetrics(runMetrics),
baseline: 0,
change: 0,
},
averageRunDuration: {
current: calculateAverageRunDurationFromMetrics(runMetrics),
baseline: 0,
change: 0,
},
p95RunDuration: {
current: calculateP95RunDurationFromReports(reportsToAnalyze),
baseline: 0,
change: 0,
},
runsAnalyzed: reportsToAnalyze.length,
extra: runMetrics,
};
// If baseline provided, calculate baseline metrics and changes
if (baseline && validateReportForInsights(baseline)) {
const baselineTestMetrics = aggregateTestMetricsAcrossReports([baseline]);
const baselineRunMetrics = consolidateTestMetricsToRunMetrics(baselineTestMetrics);
const baselinePassRate = calculatePassRateFromMetrics(baselineRunMetrics);
const baselineFailRate = calculateFailRateFromMetrics(baselineRunMetrics);
const baselineFlakyRate = calculateFlakyRateFromMetrics(baselineRunMetrics);
const baselineAvgTestDuration = calculateAverageTestDurationFromMetrics(baselineRunMetrics);
const baselineAvgRunDuration = calculateAverageRunDurationFromMetrics(baselineRunMetrics);
const baselineP95RunDuration = calculateP95RunDurationFromReports([
baseline,
]);
currentInsights.passRate.baseline = baselinePassRate;
currentInsights.passRate.change = Number(((currentInsights.passRate.current ?? 0) - baselinePassRate).toFixed(4));
currentInsights.failRate.baseline = baselineFailRate;
currentInsights.failRate.change = Number(((currentInsights.failRate.current ?? 0) - baselineFailRate).toFixed(4));
currentInsights.flakyRate.baseline = baselineFlakyRate;
currentInsights.flakyRate.change = Number(((currentInsights.flakyRate.current ?? 0) - baselineFlakyRate).toFixed(4));
currentInsights.averageTestDuration.baseline = baselineAvgTestDuration;
currentInsights.averageTestDuration.change =
(currentInsights.averageTestDuration.current ?? 0) -
baselineAvgTestDuration;
currentInsights.averageRunDuration.baseline = baselineAvgRunDuration;
currentInsights.averageRunDuration.change =
(currentInsights.averageRunDuration.current ?? 0) -
baselineAvgRunDuration;
currentInsights.p95RunDuration.baseline = baselineP95RunDuration;
currentInsights.p95RunDuration.change =
(currentInsights.p95RunDuration.current ?? 0) - baselineP95RunDuration;
}
return currentInsights;
}
/**
* Calculate insights for a specific test across multiple reports.
*
* @param reports - Array of historical reports
* @param testId - The test ID to calculate insights for
* @returns Calculated test insights
*
* @example
* ```typescript
* const insights = calculateTestInsights(reports, 'test-uuid');
* ```
*/
export function calculateTestInsights(reports, testId) {
// Collect all instances of this test across reports by ID
const testInstances = [];
let executedCount = 0;
for (const report of reports) {
const test = report.results.tests.find(t => t.id === testId);
if (test) {
testInstances.push(test);
executedCount++;
}
}
if (testInstances.length === 0) {
return { executedInRuns: 0 };
}
// Build aggregated metrics for this single test
const metrics = {
totalAttempts: 0,
totalAttemptsFailed: 0,
totalResults: 0,
totalResultsFailed: 0,
totalResultsPassed: 0,
totalResultsSkipped: 0,
totalResultsFlaky: 0,
totalAttemptsFlaky: 0,
totalResultsDuration: 0,
appearsInRuns: executedCount,
reportsAnalyzed: executedCount,
durations: [],
};
for (const test of testInstances) {
metrics.totalResults += 1;
metrics.totalAttempts += 1 + (test.retries || 0);
metrics.totalAttemptsFailed += test.retries || 0;
if (test.status === 'failed') {
metrics.totalResultsFailed += 1;
metrics.totalAttemptsFailed += 1 + (test.retries || 0);
}
else if (test.status === 'passed') {
metrics.totalResultsPassed += 1;
}
else {
metrics.totalResultsSkipped += 1;
}
if (isTestFlaky(test)) {
metrics.totalResultsFlaky += 1;
metrics.totalAttemptsFlaky += test.retries || 0;
}
metrics.totalResultsDuration += test.duration || 0;
metrics.durations.push(test.duration || 0);
}
return calculateTestInsightsWithBaseline(metrics, undefined);
}
/**
* Calculate the metric delta between current and baseline values.
*
* @param current - Current value
* @param baseline - Baseline value
* @returns MetricDelta object
*/
export function calculateMetricDelta(current, baseline) {
return {
current,
baseline,
change: current - baseline,
};
}
/**
*
* @group Insights
* Add insights to a CTRF report using historical data.
*
* Computes run-level and test-level insights according to the CTRF specification,
* including pass rate, fail rate, flaky rate, and duration metrics.
*
* @param report - The current report to enrich with insights
* @param historicalReports - Array of previous reports for trend analysis
* @param options - Options including baseline for comparison
* @returns A new report with insights populated
*
* @example
* ```typescript
* // Basic usage
* const reportWithInsights = addInsights(currentReport, previousReports);
*
* // With baseline comparison
* const reportWithInsights = addInsights(currentReport, previousReports, {
* baseline: baselineReport
* });
* ```
*/
export function addInsights(report, historicalReports = [], options = {}) {
if (!validateReportForInsights(report)) {
console.warn('Current report is not valid for insights calculation');
return report;
}
const baseline = options.baseline;
const sortedPreviousReports = sortReportsByTimestamp(historicalReports);
const allReports = [report, ...sortedPreviousReports];
const reportsWithRunInsights = calculateRunInsights([...allReports]);
const currentReportWithRunInsights = reportsWithRunInsights[0];
const currentReportWithTestInsights = addTestInsightsWithBaselineToCurrentReport(currentReportWithRunInsights, sortedPreviousReports, baseline);
if (!baseline) {
return currentReportWithTestInsights;
}
let baselineInsights = calculateReportInsightsBaseline(currentReportWithTestInsights, baseline);
baselineInsights = setTestsAddedToInsights(baselineInsights, currentReportWithTestInsights, baseline);
baselineInsights = setTestsRemovedToInsights(baselineInsights, currentReportWithTestInsights, baseline);
// Remove testsAdded and testsRemoved as they're not part of the official schema yet
if (baselineInsights.extra?.testsAdded) {
delete baselineInsights.extra.testsAdded;
}
if (baselineInsights.extra?.testsRemoved) {
delete baselineInsights.extra.testsRemoved;
}
return {
...currentReportWithTestInsights,
insights: baselineInsights,
};
}
import { describe, it, expect } from 'vitest';
import { calculateInsights, calculateTestInsights, calculateMetricDelta, addInsights, isTestFlaky, formatAsPercentage, formatMetricDeltaAsPercentage, calculatePercentChange, } from './insights.js';
import { ReportBuilder, TestBuilder } from './builder.js';
describe('insights', () => {
const createReport = (tests, timing) => {
const builder = new ReportBuilder().tool({ name: 'jest' });
if (timing) {
builder.summaryOverrides(timing);
}
for (const t of tests) {
const testBuilder = new TestBuilder()
.name(t.name)
.status(t.status)
.duration(t.duration || 100);
if (t.id)
testBuilder.id(t.id);
if (t.flaky !== undefined)
testBuilder.flaky(t.flaky);
if (t.retries !== undefined)
testBuilder.retries(t.retries);
builder.addTest(testBuilder.build());
}
return builder.build();
};
describe('isTestFlaky', () => {
it('should return true when flaky flag is true', () => {
const test = new TestBuilder()
.name('test')
.status('passed')
.duration(100)
.flaky(true)
.build();
expect(isTestFlaky(test)).toBe(true);
});
it('should return true when test passed with retries', () => {
const test = new TestBuilder()
.name('test')
.status('passed')
.duration(100)
.retries(2)
.build();
expect(isTestFlaky(test)).toBe(true);
});
it('should return false when test failed with retries', () => {
const test = new TestBuilder()
.name('test')
.status('failed')
.duration(100)
.retries(2)
.build();
expect(isTestFlaky(test)).toBe(false);
});
it('should return false for normal passing test', () => {
const test = new TestBuilder()
.name('test')
.status('passed')
.duration(100)
.build();
expect(isTestFlaky(test)).toBe(false);
});
});
describe('formatAsPercentage', () => {
it('should format ratio as percentage', () => {
expect(formatAsPercentage(0.5)).toBe('50.00%');
expect(formatAsPercentage(1)).toBe('100.00%');
expect(formatAsPercentage(0)).toBe('0.00%');
});
it('should respect decimal places', () => {
expect(formatAsPercentage(0.5555, 1)).toBe('55.5%');
expect(formatAsPercentage(0.5555, 0)).toBe('56%');
});
});
describe('formatMetricDeltaAsPercentage', () => {
it('should format metric delta as percentages', () => {
const result = formatMetricDeltaAsPercentage({
current: 0.8,
baseline: 0.6,
change: 0.2,
});
expect(result.current).toBe('80.00%');
expect(result.baseline).toBe('60.00%');
expect(result.change).toBe('+20.00%');
});
it('should handle negative change', () => {
const result = formatMetricDeltaAsPercentage({
current: 0.5,
baseline: 0.8,
change: -0.3,
});
expect(result.change).toBe('-30.00%');
});
});
describe('calculatePercentChange', () => {
it('should calculate percent change', () => {
expect(calculatePercentChange(120, 100)).toBe(0.2);
expect(calculatePercentChange(80, 100)).toBe(-0.2);
});
it('should return 0 when baseline is 0', () => {
expect(calculatePercentChange(100, 0)).toBe(0);
});
});
describe('calculateInsights', () => {
it('should return empty insights for empty reports', () => {
const insights = calculateInsights([]);
expect(insights.runsAnalyzed).toBe(0);
});
it('should calculate pass rate', () => {
const reports = [
createReport([
{ name: 'test1', status: 'passed' },
{ name: 'test2', status: 'failed' },
]),
];
const insights = calculateInsights(reports);
expect(insights.passRate?.current).toBe(0.5);
});
it('should calculate fail rate', () => {
const reports = [
createReport([
{ name: 'test1', status: 'passed' },
{ name: 'test2', status: 'failed' },
]),
];
const insights = calculateInsights(reports);
expect(insights.failRate?.current).toBe(0.5);
});
it('should calculate flaky rate from retries', () => {
// Flaky rate is calculated as: totalAttemptsFlaky / (totalResults + totalAttemptsFlaky)
// A test with flaky: true and retries: 2 contributes 2 to totalAttemptsFlaky
const reports = [
createReport([
{ name: 'test1', status: 'passed', flaky: true, retries: 2 },
{ name: 'test2', status: 'passed' },
]),
];
const insights = calculateInsights(reports);
// totalAttemptsFlaky = 2 (from retries), totalResults = 2
// flakyRate = 2 / (2 + 2) = 0.5
expect(insights.flakyRate?.current).toBe(0.5);
});
it('should track runs analyzed', () => {
const reports = [
createReport([{ name: 'test1', status: 'passed' }]),
createReport([{ name: 'test1', status: 'passed' }]),
createReport([{ name: 'test1', status: 'passed' }]),
];
const insights = calculateInsights(reports);
expect(insights.runsAnalyzed).toBe(3);
});
it('should use explicit baseline when provided', () => {
const explicitBaseline = createReport([
{ name: 'test1', status: 'failed' },
]);
const reports = [
createReport([{ name: 'test1', status: 'passed' }]),
createReport([{ name: 'test1', status: 'passed' }]),
];
const insights = calculateInsights(reports, {
baseline: explicitBaseline,
});
expect(insights.passRate?.baseline).toBe(0); // Explicit baseline had 0% pass
expect(insights.passRate?.current).toBe(1);
});
it('should respect window option', () => {
const reports = [
createReport([{ name: 'test1', status: 'failed' }]),
createReport([{ name: 'test1', status: 'passed' }]),
createReport([{ name: 'test1', status: 'passed' }]),
createReport([{ name: 'test1', status: 'passed' }]),
createReport([{ name: 'test1', status: 'passed' }]),
];
const insights = calculateInsights(reports, { window: 2 });
expect(insights.runsAnalyzed).toBe(2);
});
it('should calculate average test duration', () => {
const reports = [
createReport([
{ name: 'test1', status: 'passed', duration: 100 },
{ name: 'test2', status: 'passed', duration: 200 },
]),
];
const insights = calculateInsights(reports);
expect(insights.averageTestDuration?.current).toBe(150);
});
});
describe('calculateTestInsights', () => {
it('should return empty insights for test not found', () => {
const reports = [
createReport([{ name: 'test1', status: 'passed', id: 'id-1' }]),
];
const insights = calculateTestInsights(reports, 'non-existent');
expect(insights.executedInRuns).toBe(0);
});
it('should calculate pass rate for specific test', () => {
const reports = [
createReport([{ name: 'test1', status: 'passed', id: 'id-1' }]),
createReport([{ name: 'test1', status: 'passed', id: 'id-1' }]),
createReport([{ name: 'test1', status: 'failed', id: 'id-1' }]),
];
const insights = calculateTestInsights(reports, 'id-1');
expect(insights.passRate?.current).toBeCloseTo(0.67, 1);
expect(insights.executedInRuns).toBe(3);
});
it('should track flaky rate for specific test with retries', () => {
// Flaky rate requires retries to count
const reports = [
createReport([
{
name: 'test1',
status: 'passed',
id: 'id-1',
flaky: true,
retries: 1,
},
]),
createReport([
{ name: 'test1', status: 'passed', id: 'id-1', flaky: false },
]),
];
const insights = calculateTestInsights(reports, 'id-1');
// totalAttemptsFlaky = 1, totalResults = 2
// flakyRate = 1 / (2 + 1) = 0.3333
expect(insights.flakyRate?.current).toBeCloseTo(0.33, 1);
});
it('should calculate average duration for specific test', () => {
const reports = [
createReport([
{ name: 'test1', status: 'passed', id: 'id-1', duration: 100 },
]),
createReport([
{ name: 'test1', status: 'passed', id: 'id-1', duration: 200 },
]),
createReport([
{ name: 'test1', status: 'passed', id: 'id-1', duration: 300 },
]),
];
const insights = calculateTestInsights(reports, 'id-1');
expect(insights.averageTestDuration?.current).toBe(200);
});
});
describe('calculateMetricDelta', () => {
it('should calculate delta correctly', () => {
const delta = calculateMetricDelta(0.8, 0.6);
expect(delta.current).toBe(0.8);
expect(delta.baseline).toBe(0.6);
expect(delta.change).toBeCloseTo(0.2);
});
it('should handle negative change', () => {
const delta = calculateMetricDelta(0.5, 0.8);
expect(delta.change).toBeCloseTo(-0.3);
});
});
describe('addInsights', () => {
it('should add run-level insights to report', () => {
const historical = [createReport([{ name: 'test1', status: 'passed' }])];
const current = createReport([{ name: 'test1', status: 'passed' }]);
const enriched = addInsights(current, historical);
expect(enriched.insights).toBeDefined();
expect(enriched.insights?.runsAnalyzed).toBe(2);
});
it('should add test-level insights to tests', () => {
const historical = [
createReport([{ name: 'test1', status: 'passed', id: 'id-1' }]),
createReport([{ name: 'test1', status: 'failed', id: 'id-1' }]),
];
const current = createReport([
{ name: 'test1', status: 'passed', id: 'id-1' },
]);
const enriched = addInsights(current, historical);
// Tests get insights based on test name aggregation
expect(enriched.results.tests[0].insights).toBeDefined();
});
it('should handle empty historical reports', () => {
const current = createReport([{ name: 'test1', status: 'passed' }]);
const enriched = addInsights(current, []);
expect(enriched.insights).toBeDefined();
expect(enriched.insights?.runsAnalyzed).toBe(1);
});
it('should use baseline for comparison when provided', () => {
const baseline = createReport([{ name: 'test1', status: 'failed' }]);
const historical = [createReport([{ name: 'test1', status: 'passed' }])];
const current = createReport([{ name: 'test1', status: 'passed' }]);
const enriched = addInsights(current, historical, {
baseline,
});
expect(enriched.insights).toBeDefined();
});
});
});
/**
* CTRF Report Merging
*/
import type { CTRFReport, MergeOptions } from './types.js';
/**
*
* @group Merge
* Merge multiple CTRF reports into a single report.
* Useful for combining results from parallel or sharded test runs.
*
* @param reports - Array of CTRF reports to merge
* @param options - Merge options (deduplication, environment handling)
* @returns A new merged CTRFReport
* @throws Error if no reports are provided
*
* @example
* ```typescript
* const merged = merge([report1, report2, report3]);
*
* // With deduplication by test ID
* const merged = merge(reports, { deduplicateTests: true });
*
* // Keep first environment only
* const merged = merge(reports, { preserveEnvironment: 'first' });
* ```
*/
export declare function merge(reports: CTRFReport[], options?: MergeOptions): CTRFReport;
/**
* CTRF Report Merging
*/
import { REPORT_FORMAT, CURRENT_SPEC_VERSION } from './constants.js';
import { calculateSummary } from './summary.js';
import { generateReportId } from './id.js';
/**
*
* @group Merge
* Merge multiple CTRF reports into a single report.
* Useful for combining results from parallel or sharded test runs.
*
* @param reports - Array of CTRF reports to merge
* @param options - Merge options (deduplication, environment handling)
* @returns A new merged CTRFReport
* @throws Error if no reports are provided
*
* @example
* ```typescript
* const merged = merge([report1, report2, report3]);
*
* // With deduplication by test ID
* const merged = merge(reports, { deduplicateTests: true });
*
* // Keep first environment only
* const merged = merge(reports, { preserveEnvironment: 'first' });
* ```
*/
export function merge(reports, options = {}) {
if (!reports || reports.length === 0) {
throw new Error('No reports provided for merging');
}
if (reports.length === 1) {
return { ...reports[0] };
}
const { deduplicateTests = false, mergeSummary = true, preserveEnvironment = 'merge', } = options;
let allTests = [];
for (const report of reports) {
allTests.push(...report.results.tests);
}
if (deduplicateTests) {
const seen = new Map();
for (const test of allTests) {
if (test.id) {
seen.set(test.id, test);
}
else {
seen.set(`no-id-${seen.size}`, test);
}
}
allTests = Array.from(seen.values());
}
let summary;
if (mergeSummary) {
summary = calculateSummary(allTests);
let minStart = Number.MAX_SAFE_INTEGER;
let maxStop = 0;
for (const report of reports) {
minStart = Math.min(minStart, report.results.summary.start);
maxStop = Math.max(maxStop, report.results.summary.stop);
}
summary.start = minStart === Number.MAX_SAFE_INTEGER ? 0 : minStart;
summary.stop = maxStop;
summary.duration = summary.stop - summary.start;
}
else {
summary = sumSummaries(reports.map(r => r.results.summary));
}
let environment;
switch (preserveEnvironment) {
case 'first':
environment = reports[0].results.environment;
break;
case 'last':
environment = reports[reports.length - 1].results.environment;
break;
case 'merge':
default:
environment = mergeEnvironments(reports.map(r => r.results.environment).filter(Boolean));
break;
}
const tool = reports[0].results.tool;
const merged = {
reportFormat: REPORT_FORMAT,
specVersion: CURRENT_SPEC_VERSION,
reportId: generateReportId(),
timestamp: new Date().toISOString(),
results: {
tool,
summary,
tests: allTests,
},
};
if (environment && Object.keys(environment).length > 0) {
merged.results.environment = environment;
}
// Merge extra metadata from results
const mergedResultsExtra = mergeExtras(reports.map(r => r.results.extra).filter(Boolean));
if (mergedResultsExtra && Object.keys(mergedResultsExtra).length > 0) {
merged.results.extra = mergedResultsExtra;
}
// Merge report-level extra
const mergedReportExtra = mergeExtras(reports.map(r => r.extra).filter(Boolean));
if (mergedReportExtra && Object.keys(mergedReportExtra).length > 0) {
merged.extra = mergedReportExtra;
}
return merged;
}
/**
* Sum multiple summaries together.
*/
function sumSummaries(summaries) {
const result = {
tests: 0,
passed: 0,
failed: 0,
skipped: 0,
pending: 0,
other: 0,
start: Number.MAX_SAFE_INTEGER,
stop: 0,
};
let totalFlaky = 0;
let totalSuites = 0;
let totalDuration = 0;
let hasFlaky = false;
let hasSuites = false;
let hasDuration = false;
for (const summary of summaries) {
result.tests += summary.tests;
result.passed += summary.passed;
result.failed += summary.failed;
result.skipped += summary.skipped;
result.pending += summary.pending;
result.other += summary.other;
result.start = Math.min(result.start, summary.start);
result.stop = Math.max(result.stop, summary.stop);
if (summary.flaky !== undefined) {
hasFlaky = true;
totalFlaky += summary.flaky;
}
if (summary.suites !== undefined) {
hasSuites = true;
totalSuites += summary.suites;
}
if (summary.duration !== undefined) {
hasDuration = true;
totalDuration += summary.duration;
}
}
if (result.start === Number.MAX_SAFE_INTEGER) {
result.start = 0;
}
if (hasFlaky)
result.flaky = totalFlaky;
if (hasSuites)
result.suites = totalSuites;
if (hasDuration)
result.duration = totalDuration;
return result;
}
/**
* Merge multiple environments into one.
*/
function mergeEnvironments(environments) {
const merged = {};
for (const env of environments) {
for (const [key, value] of Object.entries(env)) {
if (value !== undefined &&
merged[key] === undefined) {
;
merged[key] = value;
}
}
}
return merged;
}
/**
* Merge multiple extra objects.
*/
function mergeExtras(extras) {
if (extras.length === 0) {
return undefined;
}
const merged = {};
for (const extra of extras) {
for (const [key, value] of Object.entries(extra)) {
if (merged[key] === undefined) {
merged[key] = value;
}
}
}
return Object.keys(merged).length > 0 ? merged : undefined;
}
import { describe, it, expect } from 'vitest';
import { merge } from './merge.js';
import { ReportBuilder, TestBuilder } from './builder.js';
describe('merge', () => {
const createReport = (tests, overrides = {}) => {
const builder = new ReportBuilder()
.tool({ name: 'jest' })
.summaryOverrides({ start: 1000, stop: 2000 });
for (const t of tests) {
builder.addTest(new TestBuilder().name(t.name).status(t.status).duration(100).build());
}
const report = builder.build();
return { ...report, ...overrides };
};
describe('merge', () => {
it('should throw for empty reports array', () => {
expect(() => merge([])).toThrow('No reports provided');
});
it('should return copy for single report', () => {
const report = createReport([{ name: 'test', status: 'passed' }]);
const merged = merge([report]);
expect(merged.results.tests).toHaveLength(1);
expect(merged).not.toBe(report);
});
it('should merge tests from multiple reports', () => {
const report1 = createReport([{ name: 'test1', status: 'passed' }]);
const report2 = createReport([{ name: 'test2', status: 'failed' }]);
const merged = merge([report1, report2]);
expect(merged.results.tests).toHaveLength(2);
expect(merged.results.tests[0].name).toBe('test1');
expect(merged.results.tests[1].name).toBe('test2');
});
it('should recalculate summary', () => {
const report1 = createReport([
{ name: 'test1', status: 'passed' },
{ name: 'test2', status: 'passed' },
]);
const report2 = createReport([{ name: 'test3', status: 'failed' }]);
const merged = merge([report1, report2]);
expect(merged.results.summary.tests).toBe(3);
expect(merged.results.summary.passed).toBe(2);
expect(merged.results.summary.failed).toBe(1);
});
it('should merge timing bounds', () => {
const report1 = createReport([{ name: 'test1', status: 'passed' }]);
report1.results.summary.start = 1000;
report1.results.summary.stop = 2000;
const report2 = createReport([{ name: 'test2', status: 'passed' }]);
report2.results.summary.start = 1500;
report2.results.summary.stop = 3000;
const merged = merge([report1, report2]);
expect(merged.results.summary.start).toBe(1000);
expect(merged.results.summary.stop).toBe(3000);
});
it('should use first tool info', () => {
const report1 = createReport([{ name: 'test1', status: 'passed' }]);
report1.results.tool = { name: 'jest', version: '29.0.0' };
const report2 = createReport([{ name: 'test2', status: 'passed' }]);
report2.results.tool = { name: 'vitest', version: '1.0.0' };
const merged = merge([report1, report2]);
expect(merged.results.tool.name).toBe('jest');
expect(merged.results.tool.version).toBe('29.0.0');
});
it('should deduplicate tests by ID when enabled', () => {
const report1 = createReport([{ name: 'test1', status: 'failed' }]);
report1.results.tests[0].id = 'same-id';
const report2 = createReport([{ name: 'test1', status: 'passed' }]);
report2.results.tests[0].id = 'same-id';
const merged = merge([report1, report2], {
deduplicateTests: true,
});
expect(merged.results.tests).toHaveLength(1);
// Keeps the last one
expect(merged.results.tests[0].status).toBe('passed');
});
it('should not deduplicate by default', () => {
const report1 = createReport([{ name: 'test1', status: 'failed' }]);
report1.results.tests[0].id = 'same-id';
const report2 = createReport([{ name: 'test1', status: 'passed' }]);
report2.results.tests[0].id = 'same-id';
const merged = merge([report1, report2]);
expect(merged.results.tests).toHaveLength(2);
});
it('should preserve first environment', () => {
const report1 = createReport([{ name: 'test1', status: 'passed' }]);
report1.results.environment = { branchName: 'main', commit: 'abc' };
const report2 = createReport([{ name: 'test2', status: 'passed' }]);
report2.results.environment = { branchName: 'feature', commit: 'def' };
const merged = merge([report1, report2], {
preserveEnvironment: 'first',
});
expect(merged.results.environment?.branchName).toBe('main');
});
it('should preserve last environment', () => {
const report1 = createReport([{ name: 'test1', status: 'passed' }]);
report1.results.environment = { branchName: 'main' };
const report2 = createReport([{ name: 'test2', status: 'passed' }]);
report2.results.environment = { branchName: 'feature' };
const merged = merge([report1, report2], {
preserveEnvironment: 'last',
});
expect(merged.results.environment?.branchName).toBe('feature');
});
it('should merge environments by default', () => {
const report1 = createReport([{ name: 'test1', status: 'passed' }]);
report1.results.environment = { branchName: 'main', buildId: 'build-1' };
const report2 = createReport([{ name: 'test2', status: 'passed' }]);
report2.results.environment = { commit: 'abc123' };
const merged = merge([report1, report2]);
expect(merged.results.environment?.branchName).toBe('main');
expect(merged.results.environment?.buildId).toBe('build-1');
expect(merged.results.environment?.commit).toBe('abc123');
});
it('should generate new report ID and timestamp', () => {
const report1 = createReport([{ name: 'test1', status: 'passed' }]);
report1.reportId = 'old-id-1';
const report2 = createReport([{ name: 'test2', status: 'passed' }]);
report2.reportId = 'old-id-2';
const merged = merge([report1, report2]);
expect(merged.reportId).toBeDefined();
expect(merged.reportId).not.toBe('old-id-1');
expect(merged.reportId).not.toBe('old-id-2');
expect(merged.timestamp).toBeDefined();
});
it('should merge extra metadata', () => {
const report1 = createReport([{ name: 'test1', status: 'passed' }]);
report1.results.extra = { key1: 'value1' };
const report2 = createReport([{ name: 'test2', status: 'passed' }]);
report2.results.extra = { key2: 'value2' };
const merged = merge([report1, report2]);
expect(merged.results.extra?.key1).toBe('value1');
expect(merged.results.extra?.key2).toBe('value2');
});
});
});
/**
* CTRF Parsing and Serialization
*/
import type { CTRFReport, ParseOptions, StringifyOptions } from './types.js';
/**
*
* @group Core Operations
* Parse a JSON string into a CTRFReport.
*
* @param json - JSON string to parse
* @param options - Parse options (e.g., enable validation)
* @returns Parsed CTRFReport object
* @throws ParseError if JSON is invalid
* @throws ValidationError if validation is enabled and fails
*
* @example
* ```typescript
* const report = parse(jsonString);
*
* // With validation
* const report = parse(jsonString, { validate: true });
* ```
*/
export declare function parse(json: string, options?: ParseOptions): CTRFReport;
/**
*
* @group Core Operations
* Serialize a CTRFReport to a JSON string.
*
* @param report - The CTRF report to serialize
* @param options - Stringify options (pretty print, indent)
* @returns JSON string representation
*
* @example
* ```typescript
* const json = stringify(report);
*
* // Pretty print
* const json = stringify(report, { pretty: true });
*
* // Custom indent
* const json = stringify(report, { pretty: true, indent: 4 });
* ```
*/
export declare function stringify(report: CTRFReport, options?: StringifyOptions): string;
/**
* Read a CTRF report from a file (async).
*
* @param path - Path to the report file
* @param options - Parse options
* @returns Promise resolving to the parsed report
* @throws FileError if file cannot be read
* @throws ParseError if JSON is invalid
*
* @example
* ```typescript
* const report = await readReport('ctrf-report.json');
* ```
*/
export declare function readReport(path: string, options?: ParseOptions): Promise<CTRFReport>;
/**
* Read a CTRF report from a file (sync).
*
* @param path - Path to the report file
* @param options - Parse options
* @returns The parsed report
* @throws FileError if file cannot be read
* @throws ParseError if JSON is invalid
*
* @example
* ```typescript
* const report = readReportSync('ctrf-report.json');
* ```
*/
export declare function readReportSync(path: string, options?: ParseOptions): CTRFReport;
/**
* Write a CTRF report to a file (async).
*
* @param path - Path to write the report
* @param report - Report to write
* @param options - Stringify options
* @throws FileError if file cannot be written
*
* @example
* ```typescript
* await writeReport('ctrf-report.json', report);
*
* // Pretty print
* await writeReport('ctrf-report.json', report, { pretty: true });
* ```
*/
export declare function writeReport(path: string, report: CTRFReport, options?: StringifyOptions): Promise<void>;
/**
* Write a CTRF report to a file (sync).
*
* @param path - Path to write the report
* @param report - Report to write
* @param options - Stringify options
* @throws FileError if file cannot be written
*
* @example
* ```typescript
* writeReportSync('ctrf-report.json', report);
* ```
*/
export declare function writeReportSync(path: string, report: CTRFReport, options?: StringifyOptions): void;
/**
* CTRF Parsing and Serialization
*/
import fs from 'fs';
import { promisify } from 'util';
import { ParseError, FileError } from './errors.js';
import { validateStrict } from './validate.js';
const readFileAsync = promisify(fs.readFile);
const writeFileAsync = promisify(fs.writeFile);
/**
*
* @group Core Operations
* Parse a JSON string into a CTRFReport.
*
* @param json - JSON string to parse
* @param options - Parse options (e.g., enable validation)
* @returns Parsed CTRFReport object
* @throws ParseError if JSON is invalid
* @throws ValidationError if validation is enabled and fails
*
* @example
* ```typescript
* const report = parse(jsonString);
*
* // With validation
* const report = parse(jsonString, { validate: true });
* ```
*/
export function parse(json, options = {}) {
let parsed;
try {
parsed = JSON.parse(json);
}
catch (error) {
throw new ParseError(`Failed to parse JSON: ${error instanceof Error ? error.message : 'Unknown error'}`, error instanceof Error ? error : undefined);
}
if (options.validate) {
validateStrict(parsed);
}
return parsed;
}
/**
*
* @group Core Operations
* Serialize a CTRFReport to a JSON string.
*
* @param report - The CTRF report to serialize
* @param options - Stringify options (pretty print, indent)
* @returns JSON string representation
*
* @example
* ```typescript
* const json = stringify(report);
*
* // Pretty print
* const json = stringify(report, { pretty: true });
*
* // Custom indent
* const json = stringify(report, { pretty: true, indent: 4 });
* ```
*/
export function stringify(report, options = {}) {
const { pretty = false, indent = 2 } = options;
if (pretty) {
return JSON.stringify(report, null, indent);
}
return JSON.stringify(report);
}
/**
* Read a CTRF report from a file (async).
*
* @param path - Path to the report file
* @param options - Parse options
* @returns Promise resolving to the parsed report
* @throws FileError if file cannot be read
* @throws ParseError if JSON is invalid
*
* @example
* ```typescript
* const report = await readReport('ctrf-report.json');
* ```
*/
export async function readReport(path, options = {}) {
let content;
try {
content = await readFileAsync(path, 'utf8');
}
catch (error) {
throw new FileError(`Failed to read file: ${error instanceof Error ? error.message : 'Unknown error'}`, path, error instanceof Error ? error : undefined);
}
return parse(content, options);
}
/**
* Read a CTRF report from a file (sync).
*
* @param path - Path to the report file
* @param options - Parse options
* @returns The parsed report
* @throws FileError if file cannot be read
* @throws ParseError if JSON is invalid
*
* @example
* ```typescript
* const report = readReportSync('ctrf-report.json');
* ```
*/
export function readReportSync(path, options = {}) {
let content;
try {
content = fs.readFileSync(path, 'utf8');
}
catch (error) {
throw new FileError(`Failed to read file: ${error instanceof Error ? error.message : 'Unknown error'}`, path, error instanceof Error ? error : undefined);
}
return parse(content, options);
}
/**
* Write a CTRF report to a file (async).
*
* @param path - Path to write the report
* @param report - Report to write
* @param options - Stringify options
* @throws FileError if file cannot be written
*
* @example
* ```typescript
* await writeReport('ctrf-report.json', report);
*
* // Pretty print
* await writeReport('ctrf-report.json', report, { pretty: true });
* ```
*/
export async function writeReport(path, report, options = { pretty: true }) {
const json = stringify(report, options);
try {
await writeFileAsync(path, json, 'utf8');
}
catch (error) {
throw new FileError(`Failed to write file: ${error instanceof Error ? error.message : 'Unknown error'}`, path, error instanceof Error ? error : undefined);
}
}
/**
* Write a CTRF report to a file (sync).
*
* @param path - Path to write the report
* @param report - Report to write
* @param options - Stringify options
* @throws FileError if file cannot be written
*
* @example
* ```typescript
* writeReportSync('ctrf-report.json', report);
* ```
*/
export function writeReportSync(path, report, options = { pretty: true }) {
const json = stringify(report, options);
try {
fs.writeFileSync(path, json, 'utf8');
}
catch (error) {
throw new FileError(`Failed to write file: ${error instanceof Error ? error.message : 'Unknown error'}`, path, error instanceof Error ? error : undefined);
}
}
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import { writeFileSync, mkdirSync, rmSync } from 'fs';
import { join } from 'path';
import { tmpdir } from 'os';
import { parse, stringify, readReport, readReportSync, writeReport, writeReportSync, } from './parse.js';
import { ParseError, FileError, ValidationError } from './errors.js';
describe('parse', () => {
const validReport = {
reportFormat: 'CTRF',
specVersion: '1.0.0',
results: {
tool: { name: 'jest' },
summary: {
tests: 1,
passed: 1,
failed: 0,
skipped: 0,
pending: 0,
other: 0,
start: 1000,
stop: 2000,
},
tests: [
{
name: 'test1',
status: 'passed',
duration: 100,
},
],
},
};
let tempDir;
beforeEach(() => {
tempDir = join(tmpdir(), `ctrf-test-${Date.now()}`);
mkdirSync(tempDir, { recursive: true });
});
afterEach(() => {
rmSync(tempDir, { recursive: true, force: true });
});
describe('parse', () => {
it('should parse valid JSON', () => {
const json = JSON.stringify(validReport);
const result = parse(json);
expect(result.reportFormat).toBe('CTRF');
expect(result.results.tests).toHaveLength(1);
});
it('should throw ParseError for invalid JSON', () => {
expect(() => parse('not valid json')).toThrow(ParseError);
});
it('should validate when option is true', () => {
const json = JSON.stringify(validReport);
const result = parse(json, { validate: true });
expect(result.reportFormat).toBe('CTRF');
});
it('should throw ValidationError when validation fails', () => {
const invalid = JSON.stringify({ invalid: true });
expect(() => parse(invalid, { validate: true })).toThrow(ValidationError);
});
});
describe('stringify', () => {
it('should stringify report', () => {
const json = stringify(validReport);
expect(json).toBe(JSON.stringify(validReport));
});
it('should pretty print when option is true', () => {
const json = stringify(validReport, { pretty: true });
expect(json).toBe(JSON.stringify(validReport, null, 2));
});
it('should use custom indent', () => {
const json = stringify(validReport, { pretty: true, indent: 4 });
expect(json).toBe(JSON.stringify(validReport, null, 4));
});
});
describe('readReport', () => {
it('should read report from file', async () => {
const filePath = join(tempDir, 'report.json');
writeFileSync(filePath, JSON.stringify(validReport));
const result = await readReport(filePath);
expect(result.reportFormat).toBe('CTRF');
});
it('should throw FileError for non-existent file', async () => {
await expect(readReport('/non/existent/file.json')).rejects.toThrow(FileError);
});
it('should validate when option is true', async () => {
const filePath = join(tempDir, 'report.json');
writeFileSync(filePath, JSON.stringify(validReport));
const result = await readReport(filePath, { validate: true });
expect(result.reportFormat).toBe('CTRF');
});
});
describe('readReportSync', () => {
it('should read report from file synchronously', () => {
const filePath = join(tempDir, 'report.json');
writeFileSync(filePath, JSON.stringify(validReport));
const result = readReportSync(filePath);
expect(result.reportFormat).toBe('CTRF');
});
it('should throw FileError for non-existent file', () => {
expect(() => readReportSync('/non/existent/file.json')).toThrow(FileError);
});
});
describe('writeReport', () => {
it('should write report to file', async () => {
const filePath = join(tempDir, 'output.json');
await writeReport(filePath, validReport);
const result = readReportSync(filePath);
expect(result.reportFormat).toBe('CTRF');
});
it('should pretty print by default', async () => {
const filePath = join(tempDir, 'output.json');
await writeReport(filePath, validReport);
const content = require('fs').readFileSync(filePath, 'utf8');
expect(content).toBe(JSON.stringify(validReport, null, 2));
});
it('should throw FileError for invalid path', async () => {
await expect(writeReport('/invalid/path/report.json', validReport)).rejects.toThrow(FileError);
});
});
describe('writeReportSync', () => {
it('should write report to file synchronously', () => {
const filePath = join(tempDir, 'output.json');
writeReportSync(filePath, validReport);
const result = readReportSync(filePath);
expect(result.reportFormat).toBe('CTRF');
});
it('should throw FileError for invalid path', () => {
expect(() => writeReportSync('/invalid/path/report.json', validReport)).toThrow(FileError);
});
});
});
/**
* CTRF Schema Access
*/
/**
*
* @group Schema & Versioning
* The current version CTRF JSON Schema object.
*
* @example
* ```typescript
* import { schema } from 'ctrf';
* console.log(schema.$schema);
* ```
*/
export declare const schema: object;
/**
*
* @group Schema & Versioning
* Get the JSON Schema for a specific CTRF spec version.
*
* @param version - The spec version (MAJOR.MINOR.PATCH) to get the schema for
* @returns The JSON Schema object for that version
* @throws SchemaVersionError if the version is not supported
*
* @example
* ```typescript
* const v0_0Schema = getSchema('0.0.0');
* const v1_0Schema = getSchema('1.0.0');
* ```
*/
export declare function getSchema(version: string): object;
/**
*
* @group Schema & Versioning
* Get the current spec version.
*
* @returns The current spec version string
*/
export declare function getCurrentSpecVersion(): string;
/**
*
* @group Schema & Versioning
* Get all supported spec versions.
*
* @returns Array of supported version strings
*/
export declare function getSupportedSpecVersions(): readonly string[];
/**
* CTRF Schema Access
*/
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
import { CURRENT_SPEC_VERSION, SUPPORTED_SPEC_VERSIONS } from './constants.js';
import { SchemaVersionError } from './errors.js';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const _schemas = new Map();
/**
* Load a schema file for a specific version.
* Files should be named: ctrf-schema-{major}.{minor}.json
*/
function loadSchemaForVersion(version) {
const versionParts = version.split('.');
if (versionParts.length < 2) {
throw new SchemaVersionError(version, [...SUPPORTED_SPEC_VERSIONS]);
}
const majorMinor = `${versionParts[0]}.${versionParts[1]}`;
if (_schemas.has(majorMinor)) {
return _schemas.get(majorMinor);
}
const schemaPath = path.resolve(__dirname, `ctrf-schema-${majorMinor}.json`);
if (!fs.existsSync(schemaPath)) {
throw new SchemaVersionError(version, [...SUPPORTED_SPEC_VERSIONS]);
}
const schema = JSON.parse(fs.readFileSync(schemaPath, 'utf8'));
_schemas.set(majorMinor, schema);
return schema;
}
/**
*
* @group Schema & Versioning
* The current version CTRF JSON Schema object.
*
* @example
* ```typescript
* import { schema } from 'ctrf';
* console.log(schema.$schema);
* ```
*/
export const schema = loadSchemaForVersion(CURRENT_SPEC_VERSION);
/**
*
* @group Schema & Versioning
* Get the JSON Schema for a specific CTRF spec version.
*
* @param version - The spec version (MAJOR.MINOR.PATCH) to get the schema for
* @returns The JSON Schema object for that version
* @throws SchemaVersionError if the version is not supported
*
* @example
* ```typescript
* const v0_0Schema = getSchema('0.0.0');
* const v1_0Schema = getSchema('1.0.0');
* ```
*/
export function getSchema(version) {
if (!SUPPORTED_SPEC_VERSIONS.includes(version)) {
throw new SchemaVersionError(version, [...SUPPORTED_SPEC_VERSIONS]);
}
return loadSchemaForVersion(version);
}
/**
*
* @group Schema & Versioning
* Get the current spec version.
*
* @returns The current spec version string
*/
export function getCurrentSpecVersion() {
return CURRENT_SPEC_VERSION;
}
/**
*
* @group Schema & Versioning
* Get all supported spec versions.
*
* @returns Array of supported version strings
*/
export function getSupportedSpecVersions() {
return SUPPORTED_SPEC_VERSIONS;
}
import { describe, it, expect } from 'vitest';
import { schema, getSchema, getCurrentSpecVersion, getSupportedSpecVersions, } from './schema.js';
import { SchemaVersionError } from './errors.js';
import { CURRENT_SPEC_VERSION, SUPPORTED_SPEC_VERSIONS } from './constants.js';
describe('schema', () => {
describe('schema', () => {
it('should be a valid JSON Schema object', () => {
expect(schema).toBeDefined();
expect(typeof schema).toBe('object');
expect(schema.$schema).toBe('http://json-schema.org/draft-07/schema#');
});
it('should define required CTRF properties', () => {
const s = schema;
const required = s.required;
expect(required).toContain('reportFormat');
expect(required).toContain('specVersion');
expect(required).toContain('results');
});
});
describe('getSchema', () => {
it('should return schema for supported version', () => {
const result = getSchema(CURRENT_SPEC_VERSION);
expect(result).toBeDefined();
expect(typeof result).toBe('object');
});
it('should throw SchemaVersionError for unsupported version', () => {
expect(() => getSchema('99.0.0')).toThrow(SchemaVersionError);
});
it('should include supported versions in error', () => {
try {
getSchema('99.0.0');
expect.fail('Should have thrown');
}
catch (error) {
expect(error).toBeInstanceOf(SchemaVersionError);
expect(error.version).toBe('99.0.0');
expect(error.supportedVersions).toEqual([
...SUPPORTED_SPEC_VERSIONS,
]);
}
});
});
describe('getCurrentSpecVersion', () => {
it('should return current spec version', () => {
expect(getCurrentSpecVersion()).toBe(CURRENT_SPEC_VERSION);
});
});
describe('getSupportedSpecVersions', () => {
it('should return array of supported versions', () => {
const versions = getSupportedSpecVersions();
expect(Array.isArray(versions)).toBe(true);
expect(versions.length).toBeGreaterThan(0);
});
it('should include current version', () => {
const versions = getSupportedSpecVersions();
expect(versions).toContain(CURRENT_SPEC_VERSION);
});
});
});
/**
* CTRF Summary Calculation
*/
import type { Test, Summary, SummaryOptions } from './types.js';
/**
*
* @group Core Operations
* Calculate summary statistics from an array of tests.
*
* @param tests - Array of test results
* @param options - Optional timing information
* @returns Calculated summary object
*
* @example
* ```typescript
* const summary = calculateSummary(tests);
*
* // With timing
* const summary = calculateSummary(tests, {
* start: 1704067200000,
* stop: 1704067260000
* });
* ```
*/
export declare function calculateSummary(tests: Test[], options?: SummaryOptions): Summary;
/**
* CTRF Summary Calculation
*/
/**
*
* @group Core Operations
* Calculate summary statistics from an array of tests.
*
* @param tests - Array of test results
* @param options - Optional timing information
* @returns Calculated summary object
*
* @example
* ```typescript
* const summary = calculateSummary(tests);
*
* // With timing
* const summary = calculateSummary(tests, {
* start: 1704067200000,
* stop: 1704067260000
* });
* ```
*/
export function calculateSummary(tests, options = {}) {
const statusCounts = {
passed: 0,
failed: 0,
skipped: 0,
pending: 0,
other: 0,
};
let flakyCount = 0;
const suites = new Set();
let totalDuration = 0;
let minStart = options.start ?? Number.MAX_SAFE_INTEGER;
let maxStop = options.stop ?? 0;
for (const test of tests) {
// Count by status
statusCounts[test.status]++;
// Count flaky tests
if (test.flaky) {
flakyCount++;
}
// Collect unique suites
if (test.suite) {
// Add all suite levels as individual suites
for (let i = 1; i <= test.suite.length; i++) {
suites.add(test.suite.slice(0, i).join('/'));
}
}
// Accumulate duration
totalDuration += test.duration;
// Track timing bounds
if (test.start !== undefined) {
minStart = Math.min(minStart, test.start);
}
if (test.stop !== undefined) {
maxStop = Math.max(maxStop, test.stop);
}
}
// Use provided times or calculated bounds
const start = options.start ?? (minStart === Number.MAX_SAFE_INTEGER ? 0 : minStart);
const stop = options.stop ?? maxStop;
const summary = {
tests: tests.length,
passed: statusCounts.passed,
failed: statusCounts.failed,
skipped: statusCounts.skipped,
pending: statusCounts.pending,
other: statusCounts.other,
start,
stop,
};
// Only add optional fields if they have meaningful values
if (flakyCount > 0) {
summary.flaky = flakyCount;
}
if (suites.size > 0) {
summary.suites = suites.size;
}
if (totalDuration > 0 || tests.length > 0) {
summary.duration = totalDuration;
}
return summary;
}
import { describe, it, expect } from 'vitest';
import { calculateSummary } from './summary.js';
describe('summary', () => {
const createTest = (overrides = {}) => ({
name: 'test',
status: 'passed',
duration: 100,
...overrides,
});
describe('calculateSummary', () => {
it('should calculate counts by status', () => {
const tests = [
createTest({ status: 'passed' }),
createTest({ status: 'passed' }),
createTest({ status: 'failed' }),
createTest({ status: 'skipped' }),
createTest({ status: 'pending' }),
createTest({ status: 'other' }),
];
const summary = calculateSummary(tests);
expect(summary.tests).toBe(6);
expect(summary.passed).toBe(2);
expect(summary.failed).toBe(1);
expect(summary.skipped).toBe(1);
expect(summary.pending).toBe(1);
expect(summary.other).toBe(1);
});
it('should count flaky tests', () => {
const tests = [
createTest({ flaky: true }),
createTest({ flaky: true }),
createTest({ flaky: false }),
createTest(),
];
const summary = calculateSummary(tests);
expect(summary.flaky).toBe(2);
});
it('should not include flaky if no tests are flaky', () => {
const tests = [createTest(), createTest()];
const summary = calculateSummary(tests);
expect(summary.flaky).toBeUndefined();
});
it('should count unique suites', () => {
const tests = [
createTest({ suite: ['unit', 'auth'] }),
createTest({ suite: ['unit', 'auth'] }),
createTest({ suite: ['unit', 'api'] }),
createTest({ suite: ['integration'] }),
];
const summary = calculateSummary(tests);
// Counts: unit, unit/auth, unit/api, integration = 4 unique suite paths
expect(summary.suites).toBe(4);
});
it('should calculate total duration', () => {
const tests = [
createTest({ duration: 100 }),
createTest({ duration: 200 }),
createTest({ duration: 300 }),
];
const summary = calculateSummary(tests);
expect(summary.duration).toBe(600);
});
it('should use provided start/stop times', () => {
const tests = [createTest()];
const summary = calculateSummary(tests, {
start: 1000,
stop: 2000,
});
expect(summary.start).toBe(1000);
expect(summary.stop).toBe(2000);
});
it('should calculate start/stop from test times', () => {
const tests = [
createTest({ start: 1000, stop: 1100 }),
createTest({ start: 1050, stop: 1200 }),
createTest({ start: 1100, stop: 1300 }),
];
const summary = calculateSummary(tests);
expect(summary.start).toBe(1000);
expect(summary.stop).toBe(1300);
});
it('should handle empty tests array', () => {
const summary = calculateSummary([]);
expect(summary.tests).toBe(0);
expect(summary.passed).toBe(0);
expect(summary.failed).toBe(0);
expect(summary.skipped).toBe(0);
expect(summary.pending).toBe(0);
expect(summary.other).toBe(0);
expect(summary.start).toBe(0);
expect(summary.stop).toBe(0);
});
});
});
/**
* CTRF TypeScript Types
* Generated from the CTRF JSON Schema specification
*/
/**
* The root CTRF report object
*
* @group Core Types
*/
export interface CTRFReport {
/** Must be 'CTRF' */
reportFormat: 'CTRF';
/** Semantic version of the CTRF specification */
specVersion: string;
/** Unique identifier for this report (UUID v4) */
reportId?: string;
/** ISO 8601 timestamp when the report was generated */
timestamp?: string;
/** Name of the tool/library that generated this report */
generatedBy?: string;
/** The test results */
results: Results;
/** Run-level insights computed from historical data */
insights?: Insights;
/** Reference to a baseline report for comparison */
baseline?: Baseline;
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* Container for test results
*
* @group Core Types
*/
export interface Results {
/** Information about the test tool */
tool: Tool;
/** Aggregated test statistics */
summary: Summary;
/** Array of individual test results */
tests: Test[];
/** Environment information */
environment?: Environment;
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* Test tool information
*
* @group Core Types
*/
export interface Tool {
/** Name of the test tool (e.g., 'jest', 'playwright') */
name: string;
/** Version of the test tool */
version?: string;
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* Aggregated test statistics
*
* @group Core Types
*/
export interface Summary {
/** Total number of tests */
tests: number;
/** Number of passed tests */
passed: number;
/** Number of failed tests */
failed: number;
/** Number of skipped tests */
skipped: number;
/** Number of pending tests */
pending: number;
/** Number of tests with other status */
other: number;
/** Number of flaky tests */
flaky?: number;
/** Number of test suites */
suites?: number;
/** Start timestamp (Unix epoch milliseconds) */
start: number;
/** Stop timestamp (Unix epoch milliseconds) */
stop: number;
/** Total duration in milliseconds */
duration?: number;
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* Individual test result
*
* @group Core Types
*/
export interface Test {
/** Unique test identifier (UUID) */
id?: string;
/** Test name */
name: string;
/** Test execution status */
status: TestStatus;
/** Test duration in milliseconds */
duration: number;
/** Start timestamp (Unix epoch milliseconds) */
start?: number;
/** Stop timestamp (Unix epoch milliseconds) */
stop?: number;
/** Test suite hierarchy */
suite?: string[];
/** Error message (for failed tests) */
message?: string;
/** Stack trace (for failed tests) */
trace?: string;
/** Code snippet where failure occurred */
snippet?: string;
/** AI-generated analysis or suggestion */
ai?: string;
/** Line number where test is defined or failed */
line?: number;
/** Original status from the test framework */
rawStatus?: string;
/** Tags for categorization */
tags?: string[];
/** Test type (e.g., 'unit', 'integration', 'e2e') */
type?: string;
/** Path to the test file */
filePath?: string;
/** Number of retry attempts */
retries?: number;
/** Details of each retry attempt */
retryAttempts?: RetryAttempt[];
/** Whether the test is flaky */
flaky?: boolean;
/** Standard output captured during test */
stdout?: string[];
/** Standard error captured during test */
stderr?: string[];
/** Thread/worker ID that ran this test */
threadId?: string;
/** Browser name (for browser tests) */
browser?: string;
/** Device name (for device tests) */
device?: string;
/** Base64 encoded screenshot */
screenshot?: string;
/** File attachments */
attachments?: Attachment[];
/** Test parameters (for parameterized tests) */
parameters?: Record<string, unknown>;
/** Test steps */
steps?: Step[];
/** Test-level insights */
insights?: TestInsights;
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* Test status enum
*
* @group Core Types
*/
export type TestStatus = 'passed' | 'failed' | 'skipped' | 'pending' | 'other';
/**
* Details of a test retry attempt
*
* @group Core Types
*/
export interface RetryAttempt {
/** Attempt number (1-indexed) */
attempt: number;
/** Status of this attempt */
status: TestStatus;
/** Duration of this attempt in milliseconds */
duration?: number;
/** Error message */
message?: string;
/** Stack trace */
trace?: string;
/** Line number */
line?: number;
/** Code snippet */
snippet?: string;
/** Standard output */
stdout?: string[];
/** Standard error */
stderr?: string[];
/** Start timestamp */
start?: number;
/** Stop timestamp */
stop?: number;
/** Attachments for this attempt */
attachments?: Attachment[];
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* File attachment
*
* @group Core Types
*/
export interface Attachment {
/** Attachment name */
name: string;
/** MIME content type */
contentType: string;
/** Path to the attachment file */
path: string;
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* Test step
*
* @group Core Types
*/
export interface Step {
/** Step name */
name: string;
/** Step status */
status: TestStatus;
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* Environment information
*
* @group Core Types
*/
export interface Environment {
/** Custom report name */
reportName?: string;
/** Application name */
appName?: string;
/** Application version */
appVersion?: string;
/** Build identifier */
buildId?: string;
/** Build name */
buildName?: string;
/** Build number */
buildNumber?: number;
/** Build URL */
buildUrl?: string;
/** Repository name */
repositoryName?: string;
/** Repository URL */
repositoryUrl?: string;
/** Git commit SHA */
commit?: string;
/** Git branch name */
branchName?: string;
/** Operating system platform */
osPlatform?: string;
/** Operating system release */
osRelease?: string;
/** Operating system version */
osVersion?: string;
/** Test environment name */
testEnvironment?: string;
/** Whether the environment is healthy */
healthy?: boolean;
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* Run-level insights computed from historical data
*
* @group Insights
*/
export interface Insights {
/** Pass rate metric */
passRate?: MetricDelta;
/** Fail rate metric */
failRate?: MetricDelta;
/** Flaky rate metric */
flakyRate?: MetricDelta;
/** Average run duration metric */
averageRunDuration?: MetricDelta;
/** 95th percentile run duration metric */
p95RunDuration?: MetricDelta;
/** Average test duration metric */
averageTestDuration?: MetricDelta;
/** Number of historical runs analyzed */
runsAnalyzed?: number;
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* Test-level insights computed from historical data
*
* @group Insights
*/
export interface TestInsights {
/** Pass rate metric */
passRate?: MetricDelta;
/** Fail rate metric */
failRate?: MetricDelta;
/** Flaky rate metric */
flakyRate?: MetricDelta;
/** Average test duration metric */
averageTestDuration?: MetricDelta;
/** 95th percentile test duration metric */
p95TestDuration?: MetricDelta;
/** Number of runs this test was executed in */
executedInRuns?: number;
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* Metric with current value, baseline, and change
*
* @group Insights
*/
export interface MetricDelta {
/** Current value */
current?: number;
/** Baseline value for comparison */
baseline?: number;
/** Change from baseline (current - baseline) */
change?: number;
}
/**
* Reference to a baseline report
*
* @group Core Types
*/
export interface Baseline {
/** Report ID of the baseline report */
reportId: string;
/** Timestamp of the baseline report */
timestamp?: string;
/** Source description (e.g., 'main-branch', 'previous-run') */
source?: string;
/** Build number of the baseline */
buildNumber?: number;
/** Build name of the baseline */
buildName?: string;
/** Build URL of the baseline */
buildUrl?: string;
/** Git commit of the baseline */
commit?: string;
/** Custom metadata */
extra?: Record<string, unknown>;
}
/**
* Result of schema validation
*
* @group Validation Options
*/
export interface ValidationResult {
/** Whether the report is valid */
valid: boolean;
/** Array of validation errors */
errors: ValidationErrorDetail[];
}
/**
* Details of a validation error
*
* @group Validation Options
*/
export interface ValidationErrorDetail {
/** Human-readable error message */
message: string;
/** JSON path to the error location */
path: string;
/** JSON Schema keyword that failed */
keyword: string;
}
/**
* Options for merging reports
*
* @group Merge Options
*/
export interface MergeOptions {
/** Remove duplicate tests by ID */
deduplicateTests?: boolean;
/** Recalculate summary from merged tests */
mergeSummary?: boolean;
/** Strategy for handling environments */
preserveEnvironment?: 'first' | 'last' | 'merge';
}
/**
* Criteria for filtering and finding tests.
*
* @group Query & Filter Options
*/
export interface FilterCriteria {
/** Filter by test ID (UUID) */
id?: string;
/** Filter by test name */
name?: string;
/** Filter by status */
status?: TestStatus | TestStatus[];
/** Filter by tags */
tags?: string | string[];
/** Filter by suite */
suite?: string | string[];
/** Filter by flaky flag */
flaky?: boolean;
/** Filter by browser */
browser?: string;
/** Filter by device */
device?: string;
}
/**
* Options for insights calculation
*
* @group Insights Options
*/
export interface InsightsOptions {
/** Baseline report for comparison */
baseline?: CTRFReport;
/** Number of historical reports to analyze */
window?: number;
}
/**
* Options for ReportBuilder
*
* @group Builder Options
*/
export interface ReportBuilderOptions {
/** Automatically generate report ID */
autoGenerateId?: boolean;
/** Automatically set timestamp */
autoTimestamp?: boolean;
}
/**
* Options for TestBuilder
*
* @group Builder Options
*/
export interface TestBuilderOptions {
/** Automatically generate test ID */
autoGenerateId?: boolean;
}
/**
* Options for calculating summary
*
* @group Core Options
*/
export interface SummaryOptions {
/** Start timestamp */
start?: number;
/** Stop timestamp */
stop?: number;
}
/**
* Options for parsing JSON
*
* @group Core Options
*/
export interface ParseOptions {
/** Validate after parsing */
validate?: boolean;
}
/**
* Options for stringifying to JSON
*
* @group Core Options
*/
export interface StringifyOptions {
/** Pretty print with indentation */
pretty?: boolean;
/** Number of spaces for indentation (default: 2) */
indent?: number;
}
/**
* Options for validation
*
* @group Validation Options
*/
export interface ValidateOptions {
/** Specific spec version to validate against */
specVersion?: string;
}
/**
* CTRF TypeScript Types
* Generated from the CTRF JSON Schema specification
*/
export {};
/**
* CTRF Validation
*/
import type { CTRFReport, ValidationResult, ValidateOptions } from './types.js';
import { TEST_STATUSES } from './constants.js';
/**
* Validate a CTRF report against the JSON schema.
*
* @group Core Operations
* @param report - The object to validate
* @param options - Validation options (e.g., specific spec version)
* @returns Validation result containing `valid` boolean and `errors` array
*
* @example
* ```typescript
* const result = validate(report);
* if (!result.valid) {
* console.log(result.errors);
* }
*
* // Validate against specific version
* const result = validate(report, { specVersion: '1.0.0' });
* ```
*/
export declare function validate(report: unknown, options?: ValidateOptions): ValidationResult;
/**
*
* @group Core Operations
* Check if a report is valid (type guard).
*
* @param report - The object to validate
* @returns true if the report is a valid CTRFReport
*
* @example
* ```typescript
* if (isValid(report)) {
* // TypeScript now knows report is CTRFReport
* console.log(report.results.summary.passed);
* }
* ```
*/
export declare function isValid(report: unknown): report is CTRFReport;
/**
*
* @group Core Operations
* Validate a report and throw if invalid (assertion).
*
* @param report - The object to validate
* @throws ValidationError if the report is invalid
*
* @example
* ```typescript
* try {
* validateStrict(report);
* // TypeScript now knows report is CTRFReport
* } catch (e) {
* if (e instanceof ValidationError) {
* console.log(e.errors);
* }
* }
* ```
*/
export declare function validateStrict(report: unknown): asserts report is CTRFReport;
/**
*
* @group Type Guards
* Checks if an object has the basic structure of a CTRF report.
* This is a quick, lightweight check that doesn't validate against the full schema.
*
* @param report - The object to check
* @returns true if the object appears to be a CTRF report
*
* @example
* ```typescript
* if (isCTRFReport(data)) {
* // data has reportFormat: 'CTRF'
* }
* ```
*/
export declare function isCTRFReport(report: unknown): report is {
reportFormat: 'CTRF';
};
/**
*
* @group Type Guards
* Type guard for Test objects.
*
* @param obj - Object to check
* @returns true if the object is a Test
*/
export declare function isTest(obj: unknown): obj is {
name: string;
status: string;
duration: number;
};
/**
*
* @group Type Guards
* Type guard for TestStatus values.
*
* @param value - Value to check
* @returns true if the value is a valid TestStatus
*/
export declare function isTestStatus(value: unknown): value is (typeof TEST_STATUSES)[number];
/**
*
* @group Type Guards
* Type guard for RetryAttempt objects.
*
* @param obj - Object to check
* @returns true if the object is a RetryAttempt
*/
export declare function isRetryAttempt(obj: unknown): obj is {
attempt: number;
status: string;
};
/**
*
* @group Type Guards
* Check if a report has insights.
*
* @param report - The report to check
* @returns true if the report has insights
*/
export declare function hasInsights(report: CTRFReport): boolean;
/**
* CTRF Validation
*/
import Ajv from 'ajv';
import addFormats from 'ajv-formats';
import { schema, getSchema } from './schema.js';
import { ValidationError } from './errors.js';
import { REPORT_FORMAT, TEST_STATUSES } from './constants.js';
/**
* Validate a CTRF report against the JSON schema.
*
* @group Core Operations
* @param report - The object to validate
* @param options - Validation options (e.g., specific spec version)
* @returns Validation result containing `valid` boolean and `errors` array
*
* @example
* ```typescript
* const result = validate(report);
* if (!result.valid) {
* console.log(result.errors);
* }
*
* // Validate against specific version
* const result = validate(report, { specVersion: '1.0.0' });
* ```
*/
export function validate(report, options = {}) {
const ajv = new Ajv({ allErrors: true });
addFormats(ajv);
const schemaToUse = options.specVersion
? getSchema(options.specVersion)
: schema;
const validateFn = ajv.compile(schemaToUse);
const valid = validateFn(report);
if (valid) {
return { valid: true, errors: [] };
}
const errors = validateFn.errors?.map(error => ({
message: error.message || 'Unknown validation error',
path: error.instancePath || '/',
keyword: error.keyword,
})) || [];
return { valid: false, errors };
}
/**
*
* @group Core Operations
* Check if a report is valid (type guard).
*
* @param report - The object to validate
* @returns true if the report is a valid CTRFReport
*
* @example
* ```typescript
* if (isValid(report)) {
* // TypeScript now knows report is CTRFReport
* console.log(report.results.summary.passed);
* }
* ```
*/
export function isValid(report) {
const result = validate(report);
return result.valid;
}
/**
*
* @group Core Operations
* Validate a report and throw if invalid (assertion).
*
* @param report - The object to validate
* @throws ValidationError if the report is invalid
*
* @example
* ```typescript
* try {
* validateStrict(report);
* // TypeScript now knows report is CTRFReport
* } catch (e) {
* if (e instanceof ValidationError) {
* console.log(e.errors);
* }
* }
* ```
*/
export function validateStrict(report) {
const result = validate(report);
if (!result.valid) {
const errorMessages = result.errors
.map(e => `${e.path}: ${e.message}`)
.join('\n');
throw new ValidationError(`CTRF validation failed:\n${errorMessages}`, result.errors);
}
}
/**
*
* @group Type Guards
* Checks if an object has the basic structure of a CTRF report.
* This is a quick, lightweight check that doesn't validate against the full schema.
*
* @param report - The object to check
* @returns true if the object appears to be a CTRF report
*
* @example
* ```typescript
* if (isCTRFReport(data)) {
* // data has reportFormat: 'CTRF'
* }
* ```
*/
export function isCTRFReport(report) {
return (typeof report === 'object' &&
report !== null &&
'reportFormat' in report &&
report.reportFormat === REPORT_FORMAT);
}
/**
*
* @group Type Guards
* Type guard for Test objects.
*
* @param obj - Object to check
* @returns true if the object is a Test
*/
export function isTest(obj) {
return (typeof obj === 'object' &&
obj !== null &&
'name' in obj &&
typeof obj.name === 'string' &&
'status' in obj &&
typeof obj.status === 'string' &&
'duration' in obj &&
typeof obj.duration === 'number');
}
/**
*
* @group Type Guards
* Type guard for TestStatus values.
*
* @param value - Value to check
* @returns true if the value is a valid TestStatus
*/
export function isTestStatus(value) {
return (typeof value === 'string' &&
TEST_STATUSES.includes(value));
}
/**
*
* @group Type Guards
* Type guard for RetryAttempt objects.
*
* @param obj - Object to check
* @returns true if the object is a RetryAttempt
*/
export function isRetryAttempt(obj) {
return (typeof obj === 'object' &&
obj !== null &&
'attempt' in obj &&
typeof obj.attempt === 'number' &&
'status' in obj &&
typeof obj.status === 'string');
}
/**
*
* @group Type Guards
* Check if a report has insights.
*
* @param report - The report to check
* @returns true if the report has insights
*/
export function hasInsights(report) {
return (report.insights !== undefined && Object.keys(report.insights).length > 0);
}
import { describe, it, expect } from 'vitest';
import { validate, isValid, validateStrict, isCTRFReport, isTest, isTestStatus, isRetryAttempt, hasInsights, } from './validate.js';
import { ValidationError } from './errors.js';
describe('validate', () => {
const validReport = {
reportFormat: 'CTRF',
specVersion: '1.0.0',
results: {
tool: { name: 'jest' },
summary: {
tests: 1,
passed: 1,
failed: 0,
skipped: 0,
pending: 0,
other: 0,
start: Date.now(),
stop: Date.now() + 1000,
},
tests: [
{
name: 'test1',
status: 'passed',
duration: 100,
},
],
},
};
describe('validate', () => {
it('should return valid for a valid report', () => {
const result = validate(validReport);
expect(result.valid).toBe(true);
expect(result.errors).toHaveLength(0);
});
it('should return errors for invalid report', () => {
const invalid = {
reportFormat: 'INVALID',
specVersion: '1.0.0',
results: {},
};
const result = validate(invalid);
expect(result.valid).toBe(false);
expect(result.errors.length).toBeGreaterThan(0);
});
it('should validate missing required fields', () => {
const missing = {
reportFormat: 'CTRF',
specVersion: '1.0.0',
};
const result = validate(missing);
expect(result.valid).toBe(false);
expect(result.errors.some(e => e.message.includes('results'))).toBe(true);
});
it('should validate invalid status', () => {
const invalidStatus = {
...validReport,
results: {
...validReport.results,
tests: [{ name: 'test', status: 'invalid', duration: 100 }],
},
};
const result = validate(invalidStatus);
expect(result.valid).toBe(false);
});
});
describe('isValid', () => {
it('should return true for valid report', () => {
expect(isValid(validReport)).toBe(true);
});
it('should return false for invalid report', () => {
expect(isValid({ invalid: true })).toBe(false);
});
it('should return false for null', () => {
expect(isValid(null)).toBe(false);
});
it('should return false for undefined', () => {
expect(isValid(undefined)).toBe(false);
});
});
describe('validateStrict', () => {
it('should not throw for valid report', () => {
expect(() => validateStrict(validReport)).not.toThrow();
});
it('should throw ValidationError for invalid report', () => {
expect(() => validateStrict({ invalid: true })).toThrow(ValidationError);
});
it('should include error details in exception', () => {
try {
validateStrict({ invalid: true });
expect.fail('Should have thrown');
}
catch (error) {
expect(error).toBeInstanceOf(ValidationError);
expect(error.errors.length).toBeGreaterThan(0);
}
});
});
describe('isCTRFReport', () => {
it('should return true for object with CTRF reportFormat', () => {
expect(isCTRFReport({ reportFormat: 'CTRF' })).toBe(true);
});
it('should return false for wrong reportFormat', () => {
expect(isCTRFReport({ reportFormat: 'OTHER' })).toBe(false);
});
it('should return false for missing reportFormat', () => {
expect(isCTRFReport({ other: 'field' })).toBe(false);
});
it('should return false for null', () => {
expect(isCTRFReport(null)).toBe(false);
});
it('should return false for primitives', () => {
expect(isCTRFReport('string')).toBe(false);
expect(isCTRFReport(123)).toBe(false);
});
});
describe('isTest', () => {
it('should return true for valid test object', () => {
expect(isTest({ name: 'test', status: 'passed', duration: 100 })).toBe(true);
});
it('should return false for missing name', () => {
expect(isTest({ status: 'passed', duration: 100 })).toBe(false);
});
it('should return false for missing status', () => {
expect(isTest({ name: 'test', duration: 100 })).toBe(false);
});
it('should return false for missing duration', () => {
expect(isTest({ name: 'test', status: 'passed' })).toBe(false);
});
it('should return false for null', () => {
expect(isTest(null)).toBe(false);
});
});
describe('isTestStatus', () => {
it('should return true for valid statuses', () => {
expect(isTestStatus('passed')).toBe(true);
expect(isTestStatus('failed')).toBe(true);
expect(isTestStatus('skipped')).toBe(true);
expect(isTestStatus('pending')).toBe(true);
expect(isTestStatus('other')).toBe(true);
});
it('should return false for invalid status', () => {
expect(isTestStatus('invalid')).toBe(false);
expect(isTestStatus('PASSED')).toBe(false);
});
it('should return false for non-strings', () => {
expect(isTestStatus(123)).toBe(false);
expect(isTestStatus(null)).toBe(false);
});
});
describe('isRetryAttempt', () => {
it('should return true for valid retry attempt', () => {
expect(isRetryAttempt({ attempt: 1, status: 'failed' })).toBe(true);
});
it('should return false for missing attempt', () => {
expect(isRetryAttempt({ status: 'failed' })).toBe(false);
});
it('should return false for missing status', () => {
expect(isRetryAttempt({ attempt: 1 })).toBe(false);
});
});
describe('hasInsights', () => {
it('should return true if report has insights', () => {
const withInsights = {
...validReport,
insights: { runsAnalyzed: 10 },
};
expect(hasInsights(withInsights)).toBe(true);
});
it('should return false if report has no insights', () => {
expect(hasInsights(validReport)).toBe(false);
});
it('should return false for empty insights object', () => {
const emptyInsights = {
...validReport,
insights: {},
};
expect(hasInsights(emptyInsights)).toBe(false);
});
});
});