Big News: Socket raises $60M Series C at a $1B valuation to secure software supply chains for AI-driven development.Announcement
Sign In

ctrf-cli

Package Overview
Dependencies
Maintainers
1
Versions
6
Alerts
File Explorer

Advanced tools

Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

ctrf-cli - npm Package Compare versions

Comparing version
0.0.4
to
0.0.5-next-1
+4
dist/add-insights.d.ts
export interface AddInsightsOptions {
output?: string;
}
export declare function addInsightsCommand(currentReportPath: string, historicalReportsDirectory: string, options?: AddInsightsOptions): Promise<void>;
import fs from 'fs';
import path from 'path';
import { parse, addInsights, stringify } from 'ctrf';
const EXIT_SUCCESS = 0;
const EXIT_GENERAL_ERROR = 1;
const EXIT_FILE_NOT_FOUND = 3;
export async function addInsightsCommand(currentReportPath, historicalReportsDirectory, options = {}) {
try {
const resolvedCurrentPath = path.resolve(currentReportPath);
const resolvedHistoricalDir = path.resolve(historicalReportsDirectory);
if (!fs.existsSync(resolvedCurrentPath)) {
console.error(`Error: Report file not found: ${resolvedCurrentPath}`);
process.exit(EXIT_FILE_NOT_FOUND);
}
if (!fs.statSync(resolvedCurrentPath).isFile()) {
console.error(`Error: Path is not a file: ${resolvedCurrentPath}`);
process.exit(EXIT_GENERAL_ERROR);
}
if (!fs.existsSync(resolvedHistoricalDir)) {
console.error(`Error: Directory not found: ${resolvedHistoricalDir}`);
process.exit(EXIT_FILE_NOT_FOUND);
}
if (!fs.statSync(resolvedHistoricalDir).isDirectory()) {
console.error(`Error: Path is not a directory: ${resolvedHistoricalDir}`);
process.exit(EXIT_GENERAL_ERROR);
}
let currentReport;
try {
const currentContent = fs.readFileSync(resolvedCurrentPath, 'utf-8');
currentReport = parse(currentContent);
if (!currentReport ||
!currentReport.results ||
!currentReport.results.tests) {
console.error(`Error: Invalid CTRF report: ${resolvedCurrentPath}`);
process.exit(EXIT_GENERAL_ERROR);
}
}
catch {
console.error(`Error: Failed to parse current report: ${resolvedCurrentPath}`);
process.exit(EXIT_GENERAL_ERROR);
}
const files = fs.readdirSync(resolvedHistoricalDir);
const historicalReports = [];
let totalHistoricalTests = 0;
for (const file of files) {
if (path.extname(file) !== '.json') {
continue;
}
const filePath = path.join(resolvedHistoricalDir, file);
const resolvedFilePath = path.resolve(filePath);
if (resolvedFilePath.toLowerCase() === resolvedCurrentPath.toLowerCase()) {
console.warn('Note: Current report found in historical reports directory and excluded from analysis');
continue;
}
try {
const fileContent = fs.readFileSync(filePath, 'utf-8');
const report = parse(fileContent);
if (report && report.results && report.results.tests) {
historicalReports.push(report);
totalHistoricalTests += report.results.tests.length;
}
else {
console.warn(`Skipping non-CTRF file: ${file}`);
}
}
catch {
console.warn(`Skipping invalid file: ${file}`);
}
}
if (historicalReports.length === 0) {
console.warn('No valid CTRF historical reports found in the specified directory.');
}
const reportWithInsights = addInsights(currentReport, historicalReports);
if (options.output) {
const outputPath = path.resolve(options.output);
const outputDir = path.dirname(outputPath);
if (!fs.existsSync(outputDir)) {
fs.mkdirSync(outputDir, { recursive: true });
}
fs.writeFileSync(outputPath, stringify(reportWithInsights), 'utf-8');
console.error(`✓ Analyzed current report with ${historicalReports.length} historical report(s) (${totalHistoricalTests} total tests)`);
console.error(`✓ Added insights including trends, patterns, and behavioral analysis`);
console.error(`✓ Saved to ${options.output}`);
process.exit(EXIT_SUCCESS);
}
else {
const output = stringify(reportWithInsights);
console.log(output);
process.exit(EXIT_SUCCESS);
}
}
catch (error) {
console.error(`Error: ${error.message}`);
process.exit(EXIT_GENERAL_ERROR);
}
}
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import fs from 'fs';
import path from 'path';
import os from 'os';
import { ReportBuilder, TestBuilder } from 'ctrf';
import { addInsightsCommand } from './add-insights.js';
describe('addInsightsCommand', () => {
let tmpDir;
let currentReportPath;
let historicalReportsDir;
let exitSpy;
let consoleLogSpy;
let consoleErrorSpy;
let consoleWarnSpy;
const createReport = (index, passedCount, failedCount) => {
const builder = new ReportBuilder().tool({ name: 'test-tool' });
for (let i = 0; i < passedCount; i++) {
builder.addTest(new TestBuilder()
.name(`test ${i + 1}`)
.status('passed')
.duration(100 + i * 10)
.build());
}
for (let i = 0; i < failedCount; i++) {
builder.addTest(new TestBuilder()
.name(`test ${passedCount + i + 1}`)
.status('failed')
.duration(200 + i * 10)
.build());
}
return builder.build();
};
beforeEach(() => {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ctrf-insights-test-'));
historicalReportsDir = path.join(tmpDir, 'historical');
fs.mkdirSync(historicalReportsDir, { recursive: true });
fs.writeFileSync(path.join(historicalReportsDir, 'report1.json'), JSON.stringify(createReport(1, 8, 2), null, 2));
fs.writeFileSync(path.join(historicalReportsDir, 'report2.json'), JSON.stringify(createReport(2, 7, 3), null, 2));
currentReportPath = path.join(tmpDir, 'current-report.json');
fs.writeFileSync(currentReportPath, JSON.stringify(createReport(3, 9, 1), null, 2));
exitSpy = vi
.spyOn(process, 'exit')
.mockImplementation((() => undefined));
consoleLogSpy = vi.spyOn(console, 'log').mockImplementation(() => { });
consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => { });
consoleWarnSpy = vi.spyOn(console, 'warn').mockImplementation(() => { });
});
afterEach(() => {
fs.rmSync(tmpDir, { recursive: true, force: true });
exitSpy.mockRestore();
consoleLogSpy.mockRestore();
consoleErrorSpy.mockRestore();
consoleWarnSpy.mockRestore();
});
describe('insights generation', () => {
it('should analyze historical reports and add insights to current report', async () => {
await addInsightsCommand(currentReportPath, historicalReportsDir);
expect(exitSpy).toHaveBeenCalledWith(0);
const output = consoleLogSpy.mock.calls[0][0];
const result = JSON.parse(output);
expect(result.reportFormat).toBe('CTRF');
expect(result.results).toBeDefined();
});
it('should produce valid CTRF output', async () => {
await addInsightsCommand(currentReportPath, historicalReportsDir);
expect(exitSpy).toHaveBeenCalledWith(0);
const output = consoleLogSpy.mock.calls[0][0];
const result = JSON.parse(output);
expect(result.reportFormat).toBe('CTRF');
expect(result.specVersion).toBeDefined();
expect(result.results).toBeDefined();
expect(result.results.tool).toBeDefined();
expect(result.results.summary).toBeDefined();
expect(result.results.tests).toBeDefined();
});
});
describe('output option', () => {
it('should write to file when --output is specified', async () => {
const outputPath = path.join(tmpDir, 'with-insights.json');
await addInsightsCommand(currentReportPath, historicalReportsDir, {
output: outputPath,
});
expect(exitSpy).toHaveBeenCalledWith(0);
expect(fs.existsSync(outputPath)).toBe(true);
const savedContent = JSON.parse(fs.readFileSync(outputPath, 'utf-8'));
expect(savedContent.reportFormat).toBe('CTRF');
expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('Analyzed'));
expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('Saved to'));
});
it('should print to stdout when no --output specified', async () => {
await addInsightsCommand(currentReportPath, historicalReportsDir);
expect(exitSpy).toHaveBeenCalledWith(0);
expect(consoleLogSpy).toHaveBeenCalled();
const output = consoleLogSpy.mock.calls[0][0];
expect(() => JSON.parse(output)).not.toThrow();
});
});
describe('error handling', () => {
it('should exit with code 3 for current report not found', async () => {
const nonExistentReportPath = path.join(tmpDir, 'nonexistent.json');
await addInsightsCommand(nonExistentReportPath, historicalReportsDir);
expect(exitSpy).toHaveBeenCalledWith(3);
expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('Report file not found'));
});
it('should exit with code 3 for historical directory not found', async () => {
const nonExistentDir = path.join(tmpDir, 'nonexistent');
await addInsightsCommand(currentReportPath, nonExistentDir);
expect(exitSpy).toHaveBeenCalledWith(3);
expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('Directory not found'));
});
it('should warn but succeed when no valid historical reports found', async () => {
const emptyDir = path.join(tmpDir, 'empty');
fs.mkdirSync(emptyDir, { recursive: true });
await addInsightsCommand(currentReportPath, emptyDir);
expect(exitSpy).toHaveBeenCalledWith(0);
expect(consoleWarnSpy).toHaveBeenCalledWith(expect.stringContaining('No valid CTRF historical reports found'));
});
it('should skip non-CTRF files with warning', async () => {
fs.writeFileSync(path.join(historicalReportsDir, 'not-ctrf.json'), JSON.stringify({ foo: 'bar' }));
await addInsightsCommand(currentReportPath, historicalReportsDir);
expect(exitSpy).toHaveBeenCalledWith(0);
expect(consoleLogSpy).toHaveBeenCalled();
});
it('should skip invalid JSON files', async () => {
fs.writeFileSync(path.join(historicalReportsDir, 'invalid.json'), 'not valid json');
await addInsightsCommand(currentReportPath, historicalReportsDir);
expect(exitSpy).toHaveBeenCalledWith(0);
expect(consoleLogSpy).toHaveBeenCalled();
});
});
describe('single historical report', () => {
it('should work with a single historical report', async () => {
await addInsightsCommand(currentReportPath, historicalReportsDir);
expect(exitSpy).toHaveBeenCalledWith(0);
const output = consoleLogSpy.mock.calls[0][0];
const result = JSON.parse(output);
expect(result.reportFormat).toBe('CTRF');
});
});
describe('current report in historical directory', () => {
it('should exclude current report if it has the same absolute path', async () => {
const parentDir = tmpDir;
const reportsDirAsParent = path.join(parentDir, 'reports');
fs.mkdirSync(reportsDirAsParent, { recursive: true });
const reportInParent = path.join(parentDir, 'report.json');
fs.writeFileSync(reportInParent, JSON.stringify(createReport(1, 8, 2), null, 2));
fs.writeFileSync(path.join(reportsDirAsParent, 'historical.json'), JSON.stringify(createReport(2, 7, 3), null, 2));
await addInsightsCommand(reportInParent, reportsDirAsParent);
expect(exitSpy).toHaveBeenCalledWith(0);
expect(consoleLogSpy).toHaveBeenCalled();
});
});
});
export declare const EXIT_SUCCESS = 0;
export declare const EXIT_GENERAL_ERROR = 1;
export declare const EXIT_VALIDATION_FAILED = 2;
export declare const EXIT_FILE_NOT_FOUND = 3;
export declare const EXIT_INVALID_CTRF = 4;
export const EXIT_SUCCESS = 0;
export const EXIT_GENERAL_ERROR = 1;
export const EXIT_VALIDATION_FAILED = 2;
export const EXIT_FILE_NOT_FOUND = 3;
export const EXIT_INVALID_CTRF = 4;
export interface FilterOptions {
id?: string;
name?: string;
status?: string;
tags?: string;
suite?: string;
type?: string;
browser?: string;
device?: string;
flaky?: boolean;
output?: string;
}
export declare function filterReport(filePath: string, options: FilterOptions): Promise<void>;
import fs from 'fs';
import path from 'path';
import { parse, filterTests, stringify, calculateSummary, } from 'ctrf';
import { EXIT_SUCCESS, EXIT_GENERAL_ERROR, EXIT_FILE_NOT_FOUND, EXIT_INVALID_CTRF, } from './exit-codes.js';
export async function filterReport(filePath, options) {
try {
let fileContent;
let displayPath;
if (filePath === '-') {
fileContent = await readStdin();
displayPath = 'stdin';
}
else {
const resolvedPath = path.resolve(filePath);
if (!fs.existsSync(resolvedPath)) {
console.error(`Error: File not found: ${resolvedPath}`);
process.exit(EXIT_FILE_NOT_FOUND);
}
fileContent = fs.readFileSync(resolvedPath, 'utf-8');
displayPath = path.basename(filePath);
}
let report;
try {
report = parse(fileContent);
}
catch (parseError) {
console.error(`Error: Invalid CTRF report - ${parseError.message}`);
process.exit(EXIT_INVALID_CTRF);
}
const criteria = {};
if (options.id) {
criteria.id = options.id;
}
if (options.name) {
criteria.name = options.name;
}
if (options.status) {
const statuses = options.status
.split(',')
.map(s => s.trim());
criteria.status = statuses;
}
if (options.tags) {
const tags = options.tags.split(',').map(t => t.trim());
criteria.tags = tags;
}
if (options.suite) {
criteria.suite = options.suite;
}
if (options.browser) {
criteria.browser = options.browser;
}
if (options.device) {
criteria.device = options.device;
}
if (options.flaky !== undefined) {
criteria.flaky = options.flaky;
}
let filteredTests = filterTests(report, criteria);
if (options.type) {
filteredTests = filteredTests.filter(test => test.type === options.type);
}
const filteredReport = {
...report,
results: {
...report.results,
tests: filteredTests,
summary: calculateSummary(filteredTests),
},
};
const output = stringify(filteredReport);
if (options.output) {
const outputPath = path.resolve(options.output);
const outputDir = path.dirname(outputPath);
if (!fs.existsSync(outputDir)) {
fs.mkdirSync(outputDir, { recursive: true });
}
fs.writeFileSync(outputPath, output, 'utf-8');
console.error(`✓ Filtered ${filteredTests.length} tests from ${displayPath}`);
console.error(`✓ Saved to ${options.output}`);
process.exit(EXIT_SUCCESS);
}
else {
console.log(output);
process.exit(EXIT_SUCCESS);
}
}
catch (error) {
console.error(`Error: ${error.message}`);
process.exit(EXIT_GENERAL_ERROR);
}
}
function readStdin() {
return new Promise((resolve, reject) => {
let data = '';
process.stdin.setEncoding('utf-8');
process.stdin.on('data', chunk => {
data += chunk;
});
process.stdin.on('end', () => {
resolve(data);
});
process.stdin.on('error', reject);
});
}
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import fs from 'fs';
import path from 'path';
import os from 'os';
import { ReportBuilder, TestBuilder, isCTRFReport, stringify, } from 'ctrf';
import { filterReport } from './filter.js';
describe('filterReport', () => {
let tmpDir;
let reportPath;
let exitSpy;
let consoleLogSpy;
let consoleErrorSpy;
const sampleReport = new ReportBuilder()
.tool({ name: 'test-tool' })
.addTest(new TestBuilder()
.name('test 1')
.status('passed')
.duration(100)
.tags(['smoke'])
.suite(['Unit'])
.browser('chrome')
.build())
.addTest(new TestBuilder()
.name('test 2')
.status('failed')
.duration(200)
.tags(['regression'])
.suite(['Integration'])
.build())
.addTest(new TestBuilder()
.name('test 3')
.status('passed')
.duration(150)
.tags(['smoke', 'regression'])
.suite(['Unit'])
.build())
.addTest(new TestBuilder()
.name('test 4')
.status('failed')
.duration(180)
.tags(['regression'])
.suite(['E2E'])
.device('mobile')
.build())
.addTest(new TestBuilder()
.name('test 5')
.status('skipped')
.duration(0)
.tags(['flaky'])
.suite(['Unit'])
.flaky(true)
.build())
.build();
beforeEach(() => {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ctrf-filter-test-'));
reportPath = path.join(tmpDir, 'report.json');
fs.writeFileSync(reportPath, JSON.stringify(sampleReport, null, 2));
exitSpy = vi
.spyOn(process, 'exit')
.mockImplementation((() => undefined));
consoleLogSpy = vi.spyOn(console, 'log').mockImplementation(() => { });
consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => { });
});
afterEach(() => {
fs.rmSync(tmpDir, { recursive: true, force: true });
exitSpy.mockRestore();
consoleLogSpy.mockRestore();
consoleErrorSpy.mockRestore();
});
describe('status filtering', () => {
it('should filter by single status', async () => {
await filterReport(reportPath, { status: 'failed' });
expect(exitSpy).toHaveBeenCalledWith(0);
const output = consoleLogSpy.mock.calls[0][0];
const result = JSON.parse(output);
expect(result.results.tests).toHaveLength(2);
expect(result.results.tests.every((t) => t.status === 'failed')).toBe(true);
expect(result.results.summary.failed).toBe(2);
});
it('should filter by multiple statuses (OR logic)', async () => {
await filterReport(reportPath, { status: 'passed,failed' });
expect(exitSpy).toHaveBeenCalledWith(0);
const output = consoleLogSpy.mock.calls[0][0];
const result = JSON.parse(output);
expect(result.results.tests).toHaveLength(4);
expect(result.results.tests.every((t) => ['passed', 'failed'].includes(t.status))).toBe(true);
});
});
describe('tags filtering', () => {
it('should filter by tag', async () => {
await filterReport(reportPath, { tags: 'smoke' });
expect(exitSpy).toHaveBeenCalledWith(0);
const output = consoleLogSpy.mock.calls[0][0];
const result = JSON.parse(output);
expect(result.results.tests).toHaveLength(2);
expect(result.results.tests.every((t) => t.tags?.includes('smoke'))).toBe(true);
expect(isCTRFReport(result)).toBe(true);
});
});
describe('suite filtering', () => {
it('should filter by suite', async () => {
await filterReport(reportPath, { suite: 'Unit' });
expect(exitSpy).toHaveBeenCalledWith(0);
const output = consoleLogSpy.mock.calls[0][0];
const result = JSON.parse(output);
expect(result.results.tests).toHaveLength(3);
});
});
describe('flaky filtering', () => {
it('should filter to flaky tests only', async () => {
await filterReport(reportPath, { flaky: true });
expect(exitSpy).toHaveBeenCalledWith(0);
const output = consoleLogSpy.mock.calls[0][0];
const result = JSON.parse(output);
expect(result.results.tests).toHaveLength(1);
expect(result.results.tests[0].name).toBe('test 5');
});
});
describe('browser filtering', () => {
it('should filter by browser', async () => {
await filterReport(reportPath, { browser: 'chrome' });
expect(exitSpy).toHaveBeenCalledWith(0);
const output = consoleLogSpy.mock.calls[0][0];
const result = JSON.parse(output);
expect(result.results.tests).toHaveLength(1);
expect(result.results.tests[0].name).toBe('test 1');
});
});
describe('device filtering', () => {
it('should filter by device', async () => {
await filterReport(reportPath, { device: 'mobile' });
expect(exitSpy).toHaveBeenCalledWith(0);
const output = consoleLogSpy.mock.calls[0][0];
const result = JSON.parse(output);
expect(result.results.tests).toHaveLength(1);
expect(result.results.tests[0].name).toBe('test 4');
});
});
describe('combined filtering (AND logic)', () => {
it('should apply multiple criteria with AND logic', async () => {
await filterReport(reportPath, { status: 'passed', suite: 'Unit' });
expect(exitSpy).toHaveBeenCalledWith(0);
const output = consoleLogSpy.mock.calls[0][0];
const result = JSON.parse(output);
expect(result.results.tests).toHaveLength(2);
expect(result.results.tests.every((t) => t.status === 'passed')).toBe(true);
});
});
describe('output option', () => {
it('should write to file when --output is specified', async () => {
const outputPath = path.join(tmpDir, 'filtered.json');
await filterReport(reportPath, { status: 'failed', output: outputPath });
expect(exitSpy).toHaveBeenCalledWith(0);
expect(fs.existsSync(outputPath)).toBe(true);
const savedContent = JSON.parse(fs.readFileSync(outputPath, 'utf-8'));
expect(savedContent.results.tests).toHaveLength(2);
expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('Filtered 2 tests'));
expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('Saved to'));
});
});
describe('error handling', () => {
it('should exit with code 3 for file not found', async () => {
const nonExistentPath = path.join(tmpDir, 'nonexistent.json');
await filterReport(nonExistentPath, {});
expect(exitSpy).toHaveBeenCalledWith(3);
expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('File not found'));
});
it('should produce valid CTRF output', async () => {
await filterReport(reportPath, { status: 'failed' });
expect(exitSpy).toHaveBeenCalledWith(0);
const output = consoleLogSpy.mock.calls[0][0];
const result = JSON.parse(output);
expect(result.reportFormat).toBe('CTRF');
expect(result.specVersion).toBeDefined();
expect(result.results).toBeDefined();
expect(result.results.tool).toBeDefined();
expect(result.results.summary).toBeDefined();
expect(result.results.tests).toBeDefined();
});
});
describe('stdin input', () => {
it('should read from stdin when filePath is "-"', async () => {
const report = new ReportBuilder()
.tool({ name: 'test-tool' })
.addTest(new TestBuilder()
.name('test 1')
.status('passed')
.duration(100)
.build())
.build();
const reportJson = stringify(report);
const stdinMock = {
setEncoding: vi.fn(),
on: vi.fn((event, callback) => {
if (event === 'data') {
callback(reportJson);
}
else if (event === 'end') {
callback();
}
}),
};
vi.spyOn(process, 'stdin', 'get').mockReturnValue(stdinMock);
await filterReport('-', { name: 'test 1' });
expect(exitSpy).toHaveBeenCalledWith(0);
expect(consoleLogSpy).toHaveBeenCalled();
});
});
describe('type filtering', () => {
it('should filter by type when manually specified', async () => {
const report = new ReportBuilder()
.tool({ name: 'test-tool' })
.addTest(new TestBuilder()
.name('unit test')
.status('passed')
.duration(100)
.type('unit')
.build())
.addTest(new TestBuilder()
.name('e2e test')
.status('passed')
.duration(100)
.type('e2e')
.build())
.build();
fs.writeFileSync(reportPath, stringify(report));
await filterReport(reportPath, { type: 'unit' });
expect(exitSpy).toHaveBeenCalledWith(0);
const output = consoleLogSpy.mock.calls[0][0];
const result = JSON.parse(output);
expect(result.results.tests).toHaveLength(1);
expect(result.results.tests[0].name).toBe('unit test');
expect(result.results.tests[0].type).toBe('unit');
});
it('should correctly update summary when type filtering', async () => {
const report = new ReportBuilder()
.tool({ name: 'test-tool' })
.addTest(new TestBuilder()
.name('unit 1')
.status('passed')
.duration(100)
.type('unit')
.build())
.addTest(new TestBuilder()
.name('unit 2')
.status('failed')
.duration(100)
.type('unit')
.build())
.addTest(new TestBuilder()
.name('e2e test')
.status('passed')
.duration(100)
.type('e2e')
.build())
.build();
fs.writeFileSync(reportPath, stringify(report));
await filterReport(reportPath, { type: 'unit' });
const output = consoleLogSpy.mock.calls[0][0];
const result = JSON.parse(output);
expect(result.results.summary.tests).toBe(2);
expect(result.results.summary.passed).toBe(1);
expect(result.results.summary.failed).toBe(1);
});
});
});
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import fs from 'fs';
import path from 'path';
import os from 'os';
import { ReportBuilder, TestBuilder } from 'ctrf';
import { identifyFlakyTests } from './flaky.js';
describe('identifyFlakyTests', () => {
let tmpDir;
let reportPath;
let consoleLogSpy;
let consoleErrorSpy;
beforeEach(() => {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ctrf-flaky-test-'));
reportPath = path.join(tmpDir, 'report.json');
consoleLogSpy = vi.spyOn(console, 'log').mockImplementation(() => { });
consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => { });
});
afterEach(() => {
vi.restoreAllMocks();
if (fs.existsSync(tmpDir)) {
fs.rmSync(tmpDir, { recursive: true });
}
});
it('should identify flaky tests', async () => {
const report = new ReportBuilder()
.tool({ name: 'test-tool' })
.addTest(new TestBuilder()
.name('flaky test 1')
.status('passed')
.duration(100)
.flaky(true)
.retries(2)
.build())
.addTest(new TestBuilder()
.name('flaky test 2')
.status('passed')
.duration(100)
.flaky(true)
.retries(1)
.build())
.addTest(new TestBuilder()
.name('stable test')
.status('passed')
.duration(100)
.build())
.build();
fs.writeFileSync(reportPath, JSON.stringify(report, null, 2));
await identifyFlakyTests(reportPath);
expect(consoleLogSpy).toHaveBeenCalledWith('Found 2 flaky test(s):');
expect(consoleLogSpy).toHaveBeenCalledWith('- Test Name: flaky test 1, Retries: 2');
expect(consoleLogSpy).toHaveBeenCalledWith('- Test Name: flaky test 2, Retries: 1');
});
it('should report when no flaky tests found', async () => {
const report = new ReportBuilder()
.tool({ name: 'test-tool' })
.addTest(new TestBuilder()
.name('stable test')
.status('passed')
.duration(100)
.build())
.build();
fs.writeFileSync(reportPath, JSON.stringify(report, null, 2));
await identifyFlakyTests(reportPath);
expect(consoleLogSpy).toHaveBeenCalledWith(expect.stringContaining('No flaky tests found'));
});
it('should handle file not found', async () => {
const nonExistentPath = path.join(tmpDir, 'non-existent.json');
await identifyFlakyTests(nonExistentPath);
expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('does not exist'));
});
it('should handle invalid JSON', async () => {
fs.writeFileSync(reportPath, 'invalid json {');
await identifyFlakyTests(reportPath);
expect(consoleErrorSpy).toHaveBeenCalledWith('Error identifying flaky tests:', expect.any(Error));
});
it('should handle missing results property', async () => {
fs.writeFileSync(reportPath, JSON.stringify({ invalid: 'structure' }, null, 2));
await identifyFlakyTests(reportPath);
expect(consoleErrorSpy).toHaveBeenCalledWith('Error identifying flaky tests:', expect.any(Error));
});
it('should display test retries correctly', async () => {
const report = new ReportBuilder()
.tool({ name: 'test-tool' })
.addTest(new TestBuilder()
.name('flaky with retries')
.status('passed')
.duration(100)
.flaky(true)
.retries(5)
.build())
.build();
fs.writeFileSync(reportPath, JSON.stringify(report, null, 2));
await identifyFlakyTests(reportPath);
expect(consoleLogSpy).toHaveBeenCalledWith('- Test Name: flaky with retries, Retries: 5');
});
});
export interface GenerateReportIdOptions {
output?: string;
}
export declare function generateReportIdCommand(filePath: string, options?: GenerateReportIdOptions): Promise<void>;
import fs from 'fs';
import path from 'path';
import { parse, generateReportId, stringify } from 'ctrf';
const EXIT_SUCCESS = 0;
const EXIT_GENERAL_ERROR = 1;
const EXIT_FILE_NOT_FOUND = 3;
const EXIT_INVALID_CTRF = 4;
export async function generateReportIdCommand(filePath, options = {}) {
try {
const resolvedPath = path.resolve(filePath);
if (!fs.existsSync(resolvedPath)) {
console.error(`Error: File not found: ${resolvedPath}`);
process.exit(EXIT_FILE_NOT_FOUND);
}
const fileContent = fs.readFileSync(resolvedPath, 'utf-8');
let report;
try {
report = parse(fileContent);
}
catch (parseError) {
console.error(`Error: Invalid CTRF report - ${parseError.message}`);
process.exit(EXIT_INVALID_CTRF);
}
const reportId = generateReportId();
const updatedReport = {
...report,
reportId,
};
const output = stringify(updatedReport);
if (options.output) {
const outputPath = path.resolve(options.output);
const outputDir = path.dirname(outputPath);
if (!fs.existsSync(outputDir)) {
fs.mkdirSync(outputDir, { recursive: true });
}
fs.writeFileSync(outputPath, output, 'utf-8');
console.error(`✓ Generated report ID: ${reportId}`);
console.error(`✓ Saved to ${options.output}`);
process.exit(EXIT_SUCCESS);
}
else {
console.log(output);
process.exit(EXIT_SUCCESS);
}
}
catch (error) {
console.error(`Error: ${error.message}`);
process.exit(EXIT_GENERAL_ERROR);
}
}
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import fs from 'fs';
import path from 'path';
import os from 'os';
import { ReportBuilder, TestBuilder, } from 'ctrf';
import { generateReportIdCommand } from './generate-report-id.js';
describe('generateReportIdCommand', () => {
let tmpDir;
let reportPath;
let exitSpy;
let consoleLogSpy;
let consoleErrorSpy;
const sampleReport = new ReportBuilder()
.tool({ name: 'test-tool' })
.addTest(new TestBuilder().name('test 1').status('passed').duration(100).build())
.addTest(new TestBuilder().name('test 2').status('failed').duration(200).build())
.build();
beforeEach(() => {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ctrf-report-id-test-'));
reportPath = path.join(tmpDir, 'report.json');
fs.writeFileSync(reportPath, JSON.stringify(sampleReport, null, 2));
exitSpy = vi
.spyOn(process, 'exit')
.mockImplementation((() => undefined));
consoleLogSpy = vi.spyOn(console, 'log').mockImplementation(() => { });
consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => { });
});
afterEach(() => {
fs.rmSync(tmpDir, { recursive: true, force: true });
exitSpy.mockRestore();
consoleLogSpy.mockRestore();
consoleErrorSpy.mockRestore();
});
describe('report ID generation', () => {
it('should add reportId to the report', async () => {
await generateReportIdCommand(reportPath);
expect(exitSpy).toHaveBeenCalledWith(0);
const output = consoleLogSpy.mock.calls[0][0];
const result = JSON.parse(output);
expect(result.reportId).toBeDefined();
});
it('should generate a UUID', async () => {
await generateReportIdCommand(reportPath);
expect(exitSpy).toHaveBeenCalledWith(0);
const output = consoleLogSpy.mock.calls[0][0];
const result = JSON.parse(output);
const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
expect(uuidRegex.test(result.reportId)).toBe(true);
});
it('should generate unique IDs on each call', async () => {
await generateReportIdCommand(reportPath);
const output1 = JSON.parse(consoleLogSpy.mock.calls[0][0]);
consoleLogSpy.mockClear();
await generateReportIdCommand(reportPath);
const output2 = JSON.parse(consoleLogSpy.mock.calls[0][0]);
expect(output1.reportId).not.toBe(output2.reportId);
});
it('should replace existing reportId', async () => {
const reportWithId = { ...sampleReport, reportId: 'existing-id' };
const pathWithId = path.join(tmpDir, 'report-with-id.json');
fs.writeFileSync(pathWithId, JSON.stringify(reportWithId, null, 2));
await generateReportIdCommand(pathWithId);
expect(exitSpy).toHaveBeenCalledWith(0);
const output = consoleLogSpy.mock.calls[0][0];
const result = JSON.parse(output);
expect(result.reportId).not.toBe('existing-id');
});
});
describe('output option', () => {
it('should write to file when --output is specified', async () => {
const outputPath = path.join(tmpDir, 'with-id.json');
await generateReportIdCommand(reportPath, { output: outputPath });
expect(exitSpy).toHaveBeenCalledWith(0);
expect(fs.existsSync(outputPath)).toBe(true);
const savedContent = JSON.parse(fs.readFileSync(outputPath, 'utf-8'));
expect(savedContent.reportId).toBeDefined();
expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('Generated report ID'));
expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('Saved to'));
});
it('should print to stdout when no --output specified', async () => {
await generateReportIdCommand(reportPath);
expect(exitSpy).toHaveBeenCalledWith(0);
expect(consoleLogSpy).toHaveBeenCalled();
const output = consoleLogSpy.mock.calls[0][0];
expect(() => JSON.parse(output)).not.toThrow();
});
});
describe('error handling', () => {
it('should exit with code 3 for file not found', async () => {
const nonExistentPath = path.join(tmpDir, 'nonexistent.json');
await generateReportIdCommand(nonExistentPath);
expect(exitSpy).toHaveBeenCalledWith(3);
expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('File not found'));
});
it('should exit with code 4 for invalid JSON', async () => {
const invalidJsonPath = path.join(tmpDir, 'invalid.json');
fs.writeFileSync(invalidJsonPath, 'not valid json');
await generateReportIdCommand(invalidJsonPath);
expect(exitSpy).toHaveBeenCalledWith(4);
});
});
describe('output validity', () => {
it('should produce valid CTRF output', async () => {
await generateReportIdCommand(reportPath);
expect(exitSpy).toHaveBeenCalledWith(0);
const output = consoleLogSpy.mock.calls[0][0];
const result = JSON.parse(output);
expect(result.reportFormat).toBe('CTRF');
expect(result.specVersion).toBeDefined();
expect(result.results).toBeDefined();
expect(result.results.tool).toBeDefined();
expect(result.results.summary).toBeDefined();
expect(result.results.tests).toBeDefined();
});
});
});
export interface GenerateTestIdsOptions {
output?: string;
}
export declare function generateTestIds(filePath: string, options?: GenerateTestIdsOptions): Promise<void>;
import fs from 'fs';
import path from 'path';
import { parse, generateTestId, stringify } from 'ctrf';
const EXIT_SUCCESS = 0;
const EXIT_GENERAL_ERROR = 1;
const EXIT_FILE_NOT_FOUND = 3;
const EXIT_INVALID_CTRF = 4;
export async function generateTestIds(filePath, options = {}) {
try {
const resolvedPath = path.resolve(filePath);
if (!fs.existsSync(resolvedPath)) {
console.error(`Error: File not found: ${resolvedPath}`);
process.exit(EXIT_FILE_NOT_FOUND);
}
const fileContent = fs.readFileSync(resolvedPath, 'utf-8');
let report;
try {
report = parse(fileContent);
}
catch (parseError) {
console.error(`Error: Invalid CTRF report - ${parseError.message}`);
process.exit(EXIT_INVALID_CTRF);
}
const testsWithIds = report.results.tests.map(test => ({
...test,
id: generateTestId(test),
}));
const updatedReport = {
...report,
results: {
...report.results,
tests: testsWithIds,
},
};
const output = stringify(updatedReport);
if (options.output) {
const outputPath = path.resolve(options.output);
const outputDir = path.dirname(outputPath);
if (!fs.existsSync(outputDir)) {
fs.mkdirSync(outputDir, { recursive: true });
}
fs.writeFileSync(outputPath, output, 'utf-8');
console.error(`✓ Generated IDs for ${testsWithIds.length} tests`);
console.error(`✓ Saved to ${options.output}`);
process.exit(EXIT_SUCCESS);
}
else {
console.log(output);
process.exit(EXIT_SUCCESS);
}
}
catch (error) {
console.error(`Error: ${error.message}`);
process.exit(EXIT_GENERAL_ERROR);
}
}
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import fs from 'fs';
import path from 'path';
import os from 'os';
import { ReportBuilder, TestBuilder, } from 'ctrf';
import { generateTestIds } from './generate-test-ids.js';
describe('generateTestIds', () => {
let tmpDir;
let reportPath;
let exitSpy;
let consoleLogSpy;
let consoleErrorSpy;
const sampleReport = new ReportBuilder()
.tool({ name: 'test-tool' })
.addTest(new TestBuilder().name('test 1').status('passed').duration(100).build())
.addTest(new TestBuilder().name('test 2').status('failed').duration(200).build())
.addTest(new TestBuilder().name('test 3').status('passed').duration(150).build())
.build();
beforeEach(() => {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ctrf-generate-ids-test-'));
reportPath = path.join(tmpDir, 'report.json');
fs.writeFileSync(reportPath, JSON.stringify(sampleReport, null, 2));
exitSpy = vi
.spyOn(process, 'exit')
.mockImplementation((() => undefined));
consoleLogSpy = vi.spyOn(console, 'log').mockImplementation(() => { });
consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => { });
});
afterEach(() => {
fs.rmSync(tmpDir, { recursive: true, force: true });
exitSpy.mockRestore();
consoleLogSpy.mockRestore();
consoleErrorSpy.mockRestore();
});
describe('ID generation', () => {
it('should generate IDs for all tests', async () => {
await generateTestIds(reportPath);
expect(exitSpy).toHaveBeenCalledWith(0);
const output = consoleLogSpy.mock.calls[0][0];
const result = JSON.parse(output);
expect(result.results.tests).toHaveLength(3);
expect(result.results.tests.every((t) => t.id)).toBe(true);
});
it('should generate deterministic IDs (same input = same ID)', async () => {
await generateTestIds(reportPath);
const output1 = JSON.parse(consoleLogSpy.mock.calls[0][0]);
consoleLogSpy.mockClear();
await generateTestIds(reportPath);
const output2 = JSON.parse(consoleLogSpy.mock.calls[0][0]);
expect(output1.results.tests[0].id).toBe(output2.results.tests[0].id);
expect(output1.results.tests[1].id).toBe(output2.results.tests[1].id);
expect(output1.results.tests[2].id).toBe(output2.results.tests[2].id);
});
it('should generate UUIDs', async () => {
await generateTestIds(reportPath);
expect(exitSpy).toHaveBeenCalledWith(0);
const output = consoleLogSpy.mock.calls[0][0];
const result = JSON.parse(output);
const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
result.results.tests.forEach((test) => {
expect(test.id).toMatch(uuidRegex);
});
});
});
describe('output option', () => {
it('should write to file when --output is specified', async () => {
const outputPath = path.join(tmpDir, 'with-ids.json');
await generateTestIds(reportPath, { output: outputPath });
expect(exitSpy).toHaveBeenCalledWith(0);
expect(fs.existsSync(outputPath)).toBe(true);
const savedContent = JSON.parse(fs.readFileSync(outputPath, 'utf-8'));
expect(savedContent.results.tests.every((t) => t.id)).toBe(true);
expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('Generated IDs for 3 tests'));
expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('Saved to'));
});
it('should print to stdout when no --output specified', async () => {
await generateTestIds(reportPath);
expect(exitSpy).toHaveBeenCalledWith(0);
expect(consoleLogSpy).toHaveBeenCalled();
const output = consoleLogSpy.mock.calls[0][0];
expect(() => JSON.parse(output)).not.toThrow();
});
});
describe('error handling', () => {
it('should exit with code 3 for file not found', async () => {
const nonExistentPath = path.join(tmpDir, 'nonexistent.json');
await generateTestIds(nonExistentPath);
expect(exitSpy).toHaveBeenCalledWith(3);
expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('File not found'));
});
it('should exit with code 4 for invalid JSON', async () => {
const invalidJsonPath = path.join(tmpDir, 'invalid.json');
fs.writeFileSync(invalidJsonPath, 'not valid json');
await generateTestIds(invalidJsonPath);
expect(exitSpy).toHaveBeenCalledWith(4);
});
});
describe('output validity', () => {
it('should produce valid CTRF output', async () => {
await generateTestIds(reportPath);
expect(exitSpy).toHaveBeenCalledWith(0);
const output = consoleLogSpy.mock.calls[0][0];
const result = JSON.parse(output);
expect(result.reportFormat).toBe('CTRF');
expect(result.specVersion).toBeDefined();
expect(result.results).toBeDefined();
expect(result.results.tool).toBeDefined();
expect(result.results.summary).toBeDefined();
expect(result.results.tests).toBeDefined();
});
});
});
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import fs from 'fs';
import path from 'path';
import os from 'os';
import { mergeReports } from './merge.js';
describe('mergeReports', () => {
let tmpDir;
let testReportDir;
let testReport1;
let testReport2;
beforeEach(() => {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ctrf-merge-test-'));
testReportDir = path.join(tmpDir, 'reports');
fs.mkdirSync(testReportDir, { recursive: true });
const report1 = {
results: {
tool: { name: 'test-tool' },
summary: {
tests: 2,
passed: 1,
failed: 1,
skipped: 0,
pending: 0,
other: 0,
start: 1708979371669,
stop: 1708979388927,
},
tests: [
{
name: 'test 1',
status: 'passed',
duration: 100,
},
{
name: 'test 2',
status: 'failed',
duration: 200,
},
],
},
};
const report2 = {
results: {
tool: { name: 'test-tool' },
summary: {
tests: 1,
passed: 1,
failed: 0,
skipped: 0,
pending: 0,
other: 0,
start: 1708979400000,
stop: 1708979410000,
},
tests: [
{
name: 'test 3',
status: 'passed',
duration: 150,
},
],
},
};
testReport1 = path.join(testReportDir, 'report1.json');
testReport2 = path.join(testReportDir, 'report2.json');
fs.writeFileSync(testReport1, JSON.stringify(report1, null, 2));
fs.writeFileSync(testReport2, JSON.stringify(report2, null, 2));
});
afterEach(() => {
fs.rmSync(tmpDir, { recursive: true, force: true });
});
describe('output path handling', () => {
it('should save with custom filename in input directory', async () => {
const outputFilename = 'my-merged-report.json';
await mergeReports(testReportDir, outputFilename, true);
const outputPath = path.join(testReportDir, outputFilename);
expect(fs.existsSync(outputPath)).toBe(true);
const merged = JSON.parse(fs.readFileSync(outputPath, 'utf8'));
expect(merged.results.summary.tests).toBe(3);
expect(merged.results.summary.passed).toBe(2);
expect(merged.results.summary.failed).toBe(1);
});
it('should save with relative path from current directory', async () => {
const outputDir = path.join(tmpDir, 'output');
const outputPath = path.join(outputDir, 'merged.json');
const relativeOutputPath = path.join(outputDir, 'merged.json');
const originalCwd = process.cwd();
try {
process.chdir(tmpDir);
await mergeReports(testReportDir, relativeOutputPath, true);
expect(fs.existsSync(outputPath)).toBe(true);
const merged = JSON.parse(fs.readFileSync(outputPath, 'utf8'));
expect(merged.results.summary.tests).toBe(3);
expect(merged.results.summary.passed).toBe(2);
expect(merged.results.summary.failed).toBe(1);
}
finally {
process.chdir(originalCwd);
}
});
it('should save to directory with default filename', async () => {
const outputDirPath = path.join(tmpDir, 'output');
await mergeReports(testReportDir, outputDirPath + '/', true);
const expectedOutputPath = path.join(outputDirPath, 'ctrf-report.json');
expect(fs.existsSync(expectedOutputPath)).toBe(true);
const merged = JSON.parse(fs.readFileSync(expectedOutputPath, 'utf8'));
expect(merged.results.summary.tests).toBe(3);
expect(merged.results.summary.passed).toBe(2);
expect(merged.results.summary.failed).toBe(1);
});
it('should save to absolute path', async () => {
const outputAbsPath = path.join(tmpDir, 'absolute-output', 'merged.json');
await mergeReports(testReportDir, outputAbsPath, true);
expect(fs.existsSync(outputAbsPath)).toBe(true);
const merged = JSON.parse(fs.readFileSync(outputAbsPath, 'utf8'));
expect(merged.results.summary.tests).toBe(3);
expect(merged.results.summary.passed).toBe(2);
expect(merged.results.summary.failed).toBe(1);
});
it('should detect directory without trailing slash and use default filename', async () => {
const outputDirPath = path.join(tmpDir, 'output-no-slash');
fs.mkdirSync(outputDirPath, { recursive: true });
await mergeReports(testReportDir, outputDirPath, true);
const expectedOutputPath = path.join(outputDirPath, 'ctrf-report.json');
expect(fs.existsSync(expectedOutputPath)).toBe(true);
const merged = JSON.parse(fs.readFileSync(expectedOutputPath, 'utf8'));
expect(merged.results.summary.tests).toBe(3);
});
it('should create nested output directories if they do not exist', async () => {
const outputPath = path.join(tmpDir, 'deep', 'nested', 'output', 'merged.json');
await mergeReports(testReportDir, outputPath, true);
expect(fs.existsSync(outputPath)).toBe(true);
const merged = JSON.parse(fs.readFileSync(outputPath, 'utf8'));
expect(merged.results.summary.tests).toBe(3);
});
});
describe('report merging', () => {
it('should correctly merge test summaries', async () => {
const outputPath = path.join(tmpDir, 'merged.json');
await mergeReports(testReportDir, outputPath, true);
const merged = JSON.parse(fs.readFileSync(outputPath, 'utf8'));
expect(merged.results.summary.tests).toBe(3);
expect(merged.results.summary.passed).toBe(2);
expect(merged.results.summary.failed).toBe(1);
expect(merged.results.summary.skipped).toBe(0);
expect(merged.results.summary.pending).toBe(0);
expect(merged.results.summary.other).toBe(0);
});
it('should combine all test cases from multiple reports', async () => {
const outputPath = path.join(tmpDir, 'merged.json');
await mergeReports(testReportDir, outputPath, true);
const merged = JSON.parse(fs.readFileSync(outputPath, 'utf8'));
expect(merged.results.tests.length).toBe(3);
expect(merged.results.tests[0].name).toBe('test 1');
expect(merged.results.tests[1].name).toBe('test 2');
expect(merged.results.tests[2].name).toBe('test 3');
});
it('should use min start time and max stop time from all reports', async () => {
const outputPath = path.join(tmpDir, 'merged.json');
await mergeReports(testReportDir, outputPath, true);
const merged = JSON.parse(fs.readFileSync(outputPath, 'utf8'));
expect(merged.results.summary.start).toBe(1708979371669);
expect(merged.results.summary.stop).toBe(1708979410000);
});
});
describe('file retention', () => {
it('should keep original reports when keepReports is true', async () => {
const outputPath = path.join(tmpDir, 'merged.json');
await mergeReports(testReportDir, outputPath, true);
expect(fs.existsSync(testReport1)).toBe(true);
expect(fs.existsSync(testReport2)).toBe(true);
});
it('should delete original reports when keepReports is false', async () => {
const outputPath = path.join(tmpDir, 'merged.json');
await mergeReports(testReportDir, outputPath, false);
expect(fs.existsSync(testReport1)).toBe(false);
expect(fs.existsSync(testReport2)).toBe(false);
expect(fs.existsSync(outputPath)).toBe(true);
});
it('should not delete the output report itself', async () => {
const outputFilename = 'ctrf-report.json';
const outputPath = path.join(testReportDir, outputFilename);
await mergeReports(testReportDir, outputFilename, false);
expect(fs.existsSync(outputPath)).toBe(true);
});
});
});
export declare function validateReport(filePath: string, strict?: boolean): Promise<void>;
import fs from 'fs';
import path from 'path';
import { parse, validate, validateStrict, ValidationError, } from 'ctrf';
import { EXIT_SUCCESS, EXIT_GENERAL_ERROR, EXIT_VALIDATION_FAILED, EXIT_FILE_NOT_FOUND, EXIT_INVALID_CTRF, } from './exit-codes.js';
export async function validateReport(filePath, strict = false) {
try {
const resolvedPath = path.resolve(filePath);
if (!fs.existsSync(resolvedPath)) {
console.error(`Error: File not found: ${resolvedPath}`);
process.exit(EXIT_FILE_NOT_FOUND);
}
const fileContent = fs.readFileSync(resolvedPath, 'utf-8');
let report;
try {
report = parse(fileContent);
}
catch (parseError) {
console.error(`Error: Invalid CTRF report - ${parseError.message}`);
process.exit(EXIT_INVALID_CTRF);
}
if (strict) {
try {
validateStrict(report);
console.log(`✓ ${path.basename(filePath)} is valid CTRF (strict)`);
process.exit(EXIT_SUCCESS);
}
catch (error) {
console.error(`✗ ${path.basename(filePath)} failed strict validation:`);
if (error instanceof ValidationError && error.errors) {
error.errors.forEach(err => {
const errPath = err.path || '';
const errMessage = err.message || String(err);
console.error(` - ${errPath ? errPath + ': ' : ''}${errMessage}`);
});
}
else {
console.error(` - ${error.message}`);
}
process.exit(EXIT_VALIDATION_FAILED);
}
}
else {
const validationResult = validate(report);
if (validationResult.valid) {
console.log(`✓ ${path.basename(filePath)} is valid CTRF`);
process.exit(EXIT_SUCCESS);
}
else {
console.error(`✗ ${path.basename(filePath)} failed validation:`);
if (validationResult.errors && validationResult.errors.length > 0) {
validationResult.errors.forEach(error => {
const errPath = error.path || '';
const errMessage = error.message || String(error);
console.error(` - ${errPath ? errPath + ': ' : ''}${errMessage}`);
});
}
process.exit(EXIT_VALIDATION_FAILED);
}
}
}
catch (error) {
console.error(`Error: ${error.message}`);
process.exit(EXIT_GENERAL_ERROR);
}
}
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import fs from 'fs';
import path from 'path';
import os from 'os';
import { ReportBuilder, TestBuilder, validate, parse, } from 'ctrf';
import { validateReport } from './validate.js';
describe('validateReport', () => {
let tmpDir;
let validReportPath;
let invalidReportPath;
let exitSpy;
let consoleLogSpy;
let consoleErrorSpy;
const validReport = new ReportBuilder()
.tool({ name: 'test-tool' })
.addTest(new TestBuilder().name('test 1').status('passed').duration(100).build())
.addTest(new TestBuilder().name('test 2').status('failed').duration(200).build())
.build();
const invalidReport = {
results: {
tests: [],
},
};
beforeEach(() => {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ctrf-validate-test-'));
validReportPath = path.join(tmpDir, 'valid-report.json');
invalidReportPath = path.join(tmpDir, 'invalid-report.json');
fs.writeFileSync(validReportPath, JSON.stringify(validReport, null, 2));
fs.writeFileSync(invalidReportPath, JSON.stringify(invalidReport, null, 2));
exitSpy = vi
.spyOn(process, 'exit')
.mockImplementation((() => undefined));
consoleLogSpy = vi.spyOn(console, 'log').mockImplementation(() => { });
consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => { });
});
afterEach(() => {
fs.rmSync(tmpDir, { recursive: true, force: true });
exitSpy.mockRestore();
consoleLogSpy.mockRestore();
consoleErrorSpy.mockRestore();
});
describe('validate (non-strict)', () => {
it('should validate a valid CTRF report', async () => {
await validateReport(validReportPath, false);
expect(exitSpy).toHaveBeenCalledWith(0);
expect(consoleLogSpy).toHaveBeenCalledWith(expect.stringContaining('is valid CTRF'));
const reportContent = fs.readFileSync(validReportPath, 'utf-8');
const parsedReport = parse(reportContent);
const result = validate(parsedReport);
expect(result.valid).toBe(true);
});
it('should reject an invalid CTRF report', async () => {
await validateReport(invalidReportPath, false);
expect(exitSpy).toHaveBeenCalledWith(2);
expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('failed validation'));
});
it('should exit with code 3 for file not found', async () => {
const nonExistentPath = path.join(tmpDir, 'nonexistent.json');
await validateReport(nonExistentPath, false);
expect(exitSpy).toHaveBeenCalledWith(3);
expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('File not found'));
});
it('should exit with code 4 for invalid JSON', async () => {
const invalidJsonPath = path.join(tmpDir, 'invalid.json');
fs.writeFileSync(invalidJsonPath, 'not valid json');
await validateReport(invalidJsonPath, false);
expect(exitSpy).toHaveBeenCalledWith(4);
expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('Invalid CTRF report'));
});
});
describe('validate-strict', () => {
it('should validate a valid CTRF report in strict mode', async () => {
await validateReport(validReportPath, true);
expect(exitSpy).toHaveBeenCalledWith(0);
expect(consoleLogSpy).toHaveBeenCalledWith(expect.stringContaining('is valid CTRF (strict)'));
});
it('should reject an invalid CTRF report in strict mode', async () => {
await validateReport(invalidReportPath, true);
expect(exitSpy).toHaveBeenCalledWith(2);
expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('failed strict validation'));
});
});
describe('file not found', () => {
it('should exit with code 3 when file not found', async () => {
const nonExistentPath = path.join(tmpDir, 'nonexistent.json');
await validateReport(nonExistentPath);
expect(exitSpy).toHaveBeenCalledWith(3);
expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('File not found'));
});
});
describe('invalid JSON', () => {
it('should exit with code 4 for invalid CTRF JSON', async () => {
const invalidJsonPath = path.join(tmpDir, 'invalid.json');
fs.writeFileSync(invalidJsonPath, 'not valid json {');
await validateReport(invalidJsonPath);
expect(exitSpy).toHaveBeenCalledWith(4);
expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('Invalid CTRF report'));
});
});
describe('strict mode error details', () => {
it('should display validation error paths in strict mode', async () => {
const reportWithPath = {
reportFormat: 'CTRF',
specVersion: '1.0.0',
results: {
tool: { name: 'test' },
summary: {},
tests: [],
},
};
fs.writeFileSync(invalidReportPath, JSON.stringify(reportWithPath, null, 2));
await validateReport(invalidReportPath, true);
expect(exitSpy).toHaveBeenCalledWith(2);
expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('failed strict validation'));
});
});
describe('standard mode error handling', () => {
it('should display validation errors without error.errors array', async () => {
const malformedReport = {
reportFormat: 'WRONG',
specVersion: '1.0.0',
results: {
tool: { name: 'test' },
summary: {},
tests: [],
},
};
fs.writeFileSync(invalidReportPath, JSON.stringify(malformedReport, null, 2));
await validateReport(invalidReportPath, false);
expect(exitSpy).toHaveBeenCalledWith(2);
expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('failed validation'));
});
});
});
+152
-25
#!/usr/bin/env node
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const yargs_1 = __importDefault(require("yargs/yargs"));
const helpers_1 = require("yargs/helpers");
const merge_1 = require("./merge");
const flaky_1 = require("./flaky");
const argv = (0, yargs_1.default)((0, helpers_1.hideBin)(process.argv))
import yargs from 'yargs/yargs';
import { hideBin } from 'yargs/helpers';
import { mergeReports } from './merge.js';
import { identifyFlakyTests } from './flaky.js';
import { validateReport } from './validate.js';
import { filterReport } from './filter.js';
import { generateTestIds } from './generate-test-ids.js';
import { generateReportIdCommand } from './generate-report-id.js';
import { addInsightsCommand } from './add-insights.js';
const argv = yargs(hideBin(process.argv))
.command('merge <directory>', 'Merge CTRF reports into a single report', yargs => {

@@ -47,5 +38,5 @@ return yargs

});
}, (argv) => __awaiter(void 0, void 0, void 0, function* () {
yield (0, merge_1.mergeReports)(argv.directory, argv.output, argv['keep-reports'], argv['output-dir']);
}))
}, async (argv) => {
await mergeReports(argv.directory, argv.output, argv['keep-reports'], argv['output-dir']);
})
.command('flaky <file>', 'Identify flaky tests from a CTRF report file', yargs => {

@@ -57,6 +48,142 @@ return yargs.positional('file', {

});
}, (argv) => __awaiter(void 0, void 0, void 0, function* () {
yield (0, flaky_1.identifyFlakyTests)(argv.file);
}))
}, async (argv) => {
await identifyFlakyTests(argv.file);
})
.command('validate <file>', 'Validate a CTRF report against the JSON schema', yargs => {
return yargs.positional('file', {
describe: 'Path to the CTRF report file to validate',
type: 'string',
demandOption: true,
});
}, async (argv) => {
await validateReport(argv.file, false);
})
.command('validate-strict <file>', 'Strict validation with additionalProperties enforcement', yargs => {
return yargs.positional('file', {
describe: 'Path to the CTRF report file to validate',
type: 'string',
demandOption: true,
});
}, async (argv) => {
await validateReport(argv.file, true);
})
.command('filter <file>', 'Filter tests from a CTRF report based on criteria', yargs => {
return yargs
.positional('file', {
describe: 'Path to the CTRF report file (use - for stdin)',
type: 'string',
demandOption: true,
})
.option('id', {
describe: 'Filter by test ID (UUID)',
type: 'string',
})
.option('name', {
describe: 'Filter by test name',
type: 'string',
})
.option('status', {
describe: 'Filter by test status (comma-separated: passed,failed,skipped,pending,other)',
type: 'string',
})
.option('tags', {
describe: 'Filter by tags (comma-separated)',
type: 'string',
})
.option('suite', {
describe: 'Filter by suite name',
type: 'string',
})
.option('type', {
describe: 'Filter by test type',
type: 'string',
})
.option('browser', {
describe: 'Filter by browser (e.g., chrome, firefox)',
type: 'string',
})
.option('device', {
describe: 'Filter by device',
type: 'string',
})
.option('flaky', {
describe: 'Filter to flaky tests only',
type: 'boolean',
})
.option('output', {
alias: 'o',
describe: 'Output file path (optional; defaults to stdout)',
type: 'string',
});
}, async (argv) => {
await filterReport(argv.file, {
id: argv.id,
name: argv.name,
status: argv.status,
tags: argv.tags,
suite: argv.suite,
type: argv.type,
browser: argv.browser,
device: argv.device,
flaky: argv.flaky,
output: argv.output,
});
})
.command('generate-test-ids <file>', 'Generate deterministic UUIDs for all tests in a report', yargs => {
return yargs
.positional('file', {
describe: 'Path to the CTRF report file',
type: 'string',
demandOption: true,
})
.option('output', {
alias: 'o',
describe: 'Output file path (optional; defaults to stdout)',
type: 'string',
});
}, async (argv) => {
await generateTestIds(argv.file, {
output: argv.output,
});
})
.command('generate-report-id <file>', 'Generate a unique identifier for the CTRF report', yargs => {
return yargs
.positional('file', {
describe: 'Path to the CTRF report file',
type: 'string',
demandOption: true,
})
.option('output', {
alias: 'o',
describe: 'Output file path (optional; defaults to stdout)',
type: 'string',
});
}, async (argv) => {
await generateReportIdCommand(argv.file, {
output: argv.output,
});
})
.command('add-insights <current-report> <historical-reports>', 'Analyze historical reports and add insights to current report', yargs => {
return yargs
.positional('current-report', {
describe: 'Path to the current CTRF report file to enhance',
type: 'string',
demandOption: true,
})
.positional('historical-reports', {
describe: 'Path to directory containing historical CTRF reports',
type: 'string',
demandOption: true,
})
.option('output', {
alias: 'o',
describe: 'Output file path for report with insights (optional; defaults to stdout)',
type: 'string',
});
}, async (argv) => {
await addInsightsCommand(argv['current-report'], argv['historical-reports'], {
output: argv.output,
});
})
.help()
.demandCommand(1, 'You need at least one command before moving on').argv;
+23
-41

@@ -1,44 +0,26 @@

"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.identifyFlakyTests = void 0;
const fs_1 = __importDefault(require("fs"));
const path_1 = __importDefault(require("path"));
function identifyFlakyTests(filePath) {
return __awaiter(this, void 0, void 0, function* () {
try {
const resolvedFilePath = path_1.default.resolve(filePath);
if (!fs_1.default.existsSync(resolvedFilePath)) {
console.error(`The file ${resolvedFilePath} does not exist.`);
return;
}
const fileContent = fs_1.default.readFileSync(resolvedFilePath, 'utf8');
const report = JSON.parse(fileContent);
const flakyTests = report.results.tests.filter((test) => test.flaky === true);
if (flakyTests.length > 0) {
console.log(`Found ${flakyTests.length} flaky test(s):`);
flakyTests.forEach((test) => {
console.log(`- Test Name: ${test.name}, Retries: ${test.retries}`);
});
}
else {
console.log(`No flaky tests found in ${resolvedFilePath}.`);
}
import fs from 'fs';
import path from 'path';
export async function identifyFlakyTests(filePath) {
try {
const resolvedFilePath = path.resolve(filePath);
if (!fs.existsSync(resolvedFilePath)) {
console.error(`The file ${resolvedFilePath} does not exist.`);
return;
}
catch (error) {
console.error('Error identifying flaky tests:', error);
const fileContent = fs.readFileSync(resolvedFilePath, 'utf8');
const report = JSON.parse(fileContent);
const flakyTests = report.results.tests.filter((test) => test.flaky === true);
if (flakyTests.length > 0) {
console.log(`Found ${flakyTests.length} flaky test(s):`);
flakyTests.forEach((test) => {
console.log(`- Test Name: ${test.name}, Retries: ${test.retries}`);
});
}
});
else {
console.log(`No flaky tests found in ${resolvedFilePath}.`);
}
}
catch (error) {
console.error('Error identifying flaky tests:', error);
}
}
exports.identifyFlakyTests = identifyFlakyTests;
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.readReportsFromGlobPattern = exports.readReportsFromDirectory = exports.mergeReports = void 0;
var merge_reports_1 = require("./methods/merge-reports");
Object.defineProperty(exports, "mergeReports", { enumerable: true, get: function () { return merge_reports_1.mergeReports; } });
var read_reports_1 = require("./methods/read-reports");
Object.defineProperty(exports, "readReportsFromDirectory", { enumerable: true, get: function () { return read_reports_1.readReportsFromDirectory; } });
var read_reports_2 = require("./methods/read-reports");
Object.defineProperty(exports, "readReportsFromGlobPattern", { enumerable: true, get: function () { return read_reports_2.readReportsFromGlobPattern; } });

@@ -1,118 +0,100 @@

"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.mergeReports = void 0;
const fs_1 = __importDefault(require("fs"));
const path_1 = __importDefault(require("path"));
function mergeReports(directory, output, keepReports, outputDir) {
return __awaiter(this, void 0, void 0, function* () {
try {
const directoryPath = path_1.default.resolve(directory);
let outputPath;
if (outputDir) {
console.warn('Warning: --output-dir is deprecated. Use --output with a path instead.');
const outputFileName = output;
const resolvedOutputDir = path_1.default.resolve(outputDir);
outputPath = path_1.default.join(resolvedOutputDir, outputFileName);
import fs from 'fs';
import path from 'path';
export async function mergeReports(directory, output, keepReports, outputDir) {
try {
const directoryPath = path.resolve(directory);
let outputPath;
if (outputDir) {
console.warn('Warning: --output-dir is deprecated. Use --output with a path instead.');
const outputFileName = output;
const resolvedOutputDir = path.resolve(outputDir);
outputPath = path.join(resolvedOutputDir, outputFileName);
}
else if (output.includes('/') || output.includes('\\')) {
const resolvedPath = path.resolve(output);
const endsWithSeparator = output.endsWith('/') || output.endsWith('\\');
const isExistingDirectory = fs.existsSync(resolvedPath) && fs.statSync(resolvedPath).isDirectory();
const hasNoExtension = path.extname(output) === '';
if (endsWithSeparator || isExistingDirectory || hasNoExtension) {
outputPath = path.join(resolvedPath, 'ctrf-report.json');
}
else if (output.includes('/') || output.includes('\\')) {
const resolvedPath = path_1.default.resolve(output);
const endsWithSeparator = output.endsWith('/') || output.endsWith('\\');
const isExistingDirectory = fs_1.default.existsSync(resolvedPath) && fs_1.default.statSync(resolvedPath).isDirectory();
const hasNoExtension = path_1.default.extname(output) === '';
if (endsWithSeparator || isExistingDirectory || hasNoExtension) {
outputPath = path_1.default.join(resolvedPath, 'ctrf-report.json');
}
else {
outputPath = resolvedPath;
}
}
else {
outputPath = path_1.default.join(directoryPath, output);
outputPath = resolvedPath;
}
console.log('Merging CTRF reports...');
const files = fs_1.default.readdirSync(directoryPath);
files.forEach(file => {
console.log('Found file:', file);
});
const ctrfReportFiles = files.filter(file => {
try {
if (path_1.default.extname(file) !== '.json') {
console.log(`Skipping non-CTRF file: ${file}`);
return false;
}
const filePath = path_1.default.join(directoryPath, file);
const fileContent = fs_1.default.readFileSync(filePath, 'utf8');
const jsonData = JSON.parse(fileContent);
if (!Object.prototype.hasOwnProperty.call(jsonData, 'results')) {
console.log(`Skipping non-CTRF file: ${file}`);
return false;
}
return true;
}
else {
outputPath = path.join(directoryPath, output);
}
console.log('Merging CTRF reports...');
const files = fs.readdirSync(directoryPath);
files.forEach(file => {
console.log('Found file:', file);
});
const ctrfReportFiles = files.filter(file => {
try {
if (path.extname(file) !== '.json') {
console.log(`Skipping non-CTRF file: ${file}`);
return false;
}
catch (error) {
console.error(`Error reading JSON file '${file}':`, error);
const filePath = path.join(directoryPath, file);
const fileContent = fs.readFileSync(filePath, 'utf8');
const jsonData = JSON.parse(fileContent);
if (!Object.prototype.hasOwnProperty.call(jsonData, 'results')) {
console.log(`Skipping non-CTRF file: ${file}`);
return false;
}
});
if (ctrfReportFiles.length === 0) {
console.log('No CTRF reports found in the specified directory.');
return;
return true;
}
const outputDirPath = path_1.default.dirname(outputPath);
if (!fs_1.default.existsSync(outputDirPath)) {
fs_1.default.mkdirSync(outputDirPath, { recursive: true });
console.log(`Created output directory: ${outputDirPath}`);
catch (error) {
console.error(`Error reading JSON file '${file}':`, error);
return false;
}
const mergedReport = ctrfReportFiles
.map(file => {
console.log('Merging report:', file);
const filePath = path_1.default.join(directoryPath, file);
const fileContent = fs_1.default.readFileSync(filePath, 'utf8');
return JSON.parse(fileContent);
})
.reduce((acc, curr) => {
if (!acc.results) {
return curr;
});
if (ctrfReportFiles.length === 0) {
console.log('No CTRF reports found in the specified directory.');
return;
}
const outputDirPath = path.dirname(outputPath);
if (!fs.existsSync(outputDirPath)) {
fs.mkdirSync(outputDirPath, { recursive: true });
console.log(`Created output directory: ${outputDirPath}`);
}
const mergedReport = ctrfReportFiles
.map(file => {
console.log('Merging report:', file);
const filePath = path.join(directoryPath, file);
const fileContent = fs.readFileSync(filePath, 'utf8');
return JSON.parse(fileContent);
})
.reduce((acc, curr) => {
if (!acc.results) {
return curr;
}
acc.results.summary.tests += curr.results.summary.tests;
acc.results.summary.passed += curr.results.summary.passed;
acc.results.summary.failed += curr.results.summary.failed;
acc.results.summary.skipped += curr.results.summary.skipped;
acc.results.summary.pending += curr.results.summary.pending;
acc.results.summary.other += curr.results.summary.other;
acc.results.tests.push(...curr.results.tests);
acc.results.summary.start = Math.min(acc.results.summary.start, curr.results.summary.start);
acc.results.summary.stop = Math.max(acc.results.summary.stop, curr.results.summary.stop);
return acc;
}, { results: null });
fs.writeFileSync(outputPath, JSON.stringify(mergedReport, null, 2));
if (!keepReports) {
const outputFileName = path.basename(outputPath);
ctrfReportFiles.forEach(file => {
const filePath = path.join(directoryPath, file);
if (file !== outputFileName) {
fs.unlinkSync(filePath);
}
acc.results.summary.tests += curr.results.summary.tests;
acc.results.summary.passed += curr.results.summary.passed;
acc.results.summary.failed += curr.results.summary.failed;
acc.results.summary.skipped += curr.results.summary.skipped;
acc.results.summary.pending += curr.results.summary.pending;
acc.results.summary.other += curr.results.summary.other;
acc.results.tests.push(...curr.results.tests);
acc.results.summary.start = Math.min(acc.results.summary.start, curr.results.summary.start);
acc.results.summary.stop = Math.max(acc.results.summary.stop, curr.results.summary.stop);
return acc;
}, { results: null });
fs_1.default.writeFileSync(outputPath, JSON.stringify(mergedReport, null, 2));
if (!keepReports) {
const outputFileName = path_1.default.basename(outputPath);
ctrfReportFiles.forEach(file => {
const filePath = path_1.default.join(directoryPath, file);
if (file !== outputFileName) {
fs_1.default.unlinkSync(filePath);
}
});
}
console.log('CTRF reports merged successfully.');
console.log(`Merged report saved to: ${outputPath}`);
});
}
catch (error) {
console.error('Error merging CTRF reports:', error);
}
});
console.log('CTRF reports merged successfully.');
console.log(`Merged report saved to: ${outputPath}`);
}
catch (error) {
console.error('Error merging CTRF reports:', error);
}
}
exports.mergeReports = mergeReports;
MIT License
Copyright (c) 2024 Matthew Thomas
Copyright (c) 2024 Matthew Poulton-White

@@ -5,0 +5,0 @@ Permission is hereby granted, free of charge, to any person obtaining a copy

{
"type": "module",
"name": "ctrf-cli",
"version": "0.0.4",
"version": "0.0.5-next-1",
"description": "Various CTRF utilities available from the command line",
"main": "dist/index.js",
"bin": {
"ctrf": "dist/cli.js",
"ctrf-cli": "dist/cli.js"

@@ -17,3 +19,3 @@ },

"format:check": "prettier --check .",
"all": "npm run lint && npm run format:check && npm run test && npm run build"
"all": "npm run lint && npm run format:check && npm run test:coverage && npm run build"
},

@@ -36,2 +38,3 @@ "files": [

"dependencies": {
"ctrf": "^0.0.18-next-1",
"glob": "^11.0.1",

@@ -43,8 +46,8 @@ "typescript": "^5.4.5",

"@d2t/vitest-ctrf-json-reporter": "^1.2.0",
"@types/node": "^20.12.7",
"@eslint/js": "^9.32.0",
"@types/node": "^25.0.10",
"@types/yargs": "^17.0.32",
"@vitest/coverage-v8": "^3.2.4",
"@eslint/js": "^9.32.0",
"eslint": "^9.32.0",
"prettier": "^3.5.3",
"@types/yargs": "^17.0.32",
"typescript": "^5.4.5",

@@ -51,0 +54,0 @@ "typescript-eslint": "^8.38.0",

+212
-67

@@ -1,112 +0,257 @@

# CTRF CLI
# CTRF CLI Reference Implementation
Various CTRF utilities available from the command line
Command line tooling for the [Common Test Report Format (CTRF)](https://github.com/ctrf-io/ctrf) specification.
<div align="center">
<div style="padding: 1.5rem; border-radius: 8px; margin: 1rem 0; border: 1px solid #30363d;">
<span style="font-size: 23px;">💚</span>
<h3 style="margin: 1rem 0;">CTRF tooling is open source and free to use</h3>
<p style="font-size: 16px;">You can support the project with a follow and a star</p>
## Open Standard
<div style="margin-top: 1.5rem;">
<a href="https://github.com/ctrf-io/ctrf-cli">
<img src="https://img.shields.io/github/stars/ctrf-io/ctrf-cli?style=for-the-badge&color=2ea043" alt="GitHub stars">
</a>
<a href="https://github.com/ctrf-io">
<img src="https://img.shields.io/github/followers/ctrf-io?style=for-the-badge&color=2ea043" alt="GitHub followers">
</a>
</div>
</div>
[CTRF](https://github.com/ctrf-io/ctrf) is an open standard built and shaped by community contributions.
<p style="font-size: 14px; margin: 1rem 0;">
Your feedback and contributions are essential to the project's success:
Contributions are very welcome! <br/>
Explore more <a href="https://www.ctrf.io/integrations">integrations</a> <br/>
<a href="https://app.formbricks.com/s/cmefs524mhlh1tl01gkpvefrb">Let us know your thoughts</a>
- [Contribute](CONTRIBUTING.md)
- [Discuss](https://github.com/orgs/ctrf-io/discussions)
</p>
</div>
## Support
## Command Line Utilities
You can support the project by giving this repository a star ⭐
| Name |Details |
| ------------ | ----------------------------------------------------------------------------------- |
| `merge` | Merge multiple CTRF reports into a single report. |
| `flaky` | Output flaky test name and retries. |
## Usage
## Merge
### Without Installation
This might be useful if you need a single report, but your chosen reporter generates multiple reports through design, parallelisation or otherwise.
Use `npx` to run the CLI without installing:
To merge CTRF reports in a specified directory, use the following command:
```bash
npx ctrf-cli@0.0.5-next-1 validate report.json
```
### Global Installation
Or install globally for repeated use:
```bash [npm]
npm install -g ctrf-cli@0.0.5-next-1
```
After global installation, use the `ctrf` command:
```bash
ctrf validate report.json
```
## Commands
| Command | Purpose |
|---------|---------|
| `merge` | Merge multiple CTRF reports into a single report |
| `validate` | Validate a CTRF report against the JSON schema |
| `validate-strict` | Strict validation with additionalProperties enforcement |
| `filter` | Filter tests from a CTRF report based on criteria |
| `generate-test-ids` | Generate deterministic UUIDs for all tests |
| `generate-report-id` | Generate a unique UUID v4 identifier for report |
| `add-insights` | Analyze trends and add insights across multiple reports |
| `flaky` | Identify and output flaky tests |
## Exit Codes
- `0`: Command completed successfully
- `1`: General error
- `2`: Validation failed
- `3`: File or directory not found
- `4`: Invalid CTRF report
- `5`: No CTRF reports found in directory
## merge
Combines multiple CTRF reports into a single unified report. **Writes to file.**
**Syntax:**
```sh
npx ctrf-cli@0.0.3 merge <directory>
ctrf merge <directory> [--output <path>] [--keep-reports]
```
Replace `directory` with the path to the directory containing the CTRF reports you want to merge. Your merged report will be saved as `ctrf-report.json` in the same directory by default.
**Parameters:**
### Options
- `directory`: Path to directory containing CTRF reports (required)
- `--output, -o`: Output file path (default: ctrf-report.json)
- `--keep-reports, -k`: Preserve original reports after merging
-o, --output `path`: Output file path for the merged report. Can be a filename (saved in input directory), relative path from current working directory, or absolute path. Default is `ctrf-report.json`.
**Example:**
```sh
# Save with custom filename in input directory
npx ctrf-cli@0.0.3 merge ./reports --output my-merged-report.json
# Merged report saved to: ./reports/my-merged-report.json
ctrf merge ./reports --output ./merged.json
```
# Save with relative path from current directory
npx ctrf-cli@0.0.3 merge ./reports --output ./output/merged.json
# Merged report saved to: ./output/merged.json
## validate
# Save to directory with default filename
npx ctrf-cli@0.0.3 merge ./reports --output ./output/
# Merged report saved to: ./output/ctrf-report.json
Validates CTRF report conformance to the JSON Schema specification. **Outputs to stdout.**
# Save to absolute path
npx ctrf-cli@0.0.3 merge ./reports --output /tmp/merged.json
# Merged report saved to: /tmp/merged.json
**Syntax:**
```sh
ctrf validate <file-path>
ctrf validate-strict <file-path>
```
-k, --keep-reports: Keep existing reports after merging. By default, the original reports will be deleted after merging.
**Parameters:**
- `file-path`: Path to CTRF report file (required)
**Modes:**
- `validate`: Standard validation allowing additional properties
- `validate-strict`: Strict validation enforcing additionalProperties: false
**Example:**
```sh
npx ctrf-cli@0.0.3 merge <directory> --keep-reports
ctrf validate report.json
ctrf validate-strict report.json
```
## Flaky
## filter
The flaky command is useful for identifying tests marked as flaky in your CTRF report. Flaky tests are tests that pass or fail inconsistently and may require special attention or retries to determine their reliability.
Extracts a subset of tests from a CTRF report based on specified criteria. **Outputs to stdout or file.**
Usage
To output flaky tests, use the following command:
**Syntax:**
```sh
npx ctrf-cli@0.0.3 flaky <file-path>
ctrf filter <file-path> [options]
```
Replace <file-path> with the path to the CTRF report file you want to analyze.
**Parameters:**
### Output
- `file-path`: Path to CTRF report (use `-` for stdin) (required)
- `--id <uuid>`: Filter by test ID
- `--name <string>`: Filter by test name (exact match)
- `--status <statuses>`: Filter by status (comma-separated: passed,failed,skipped,pending,other)
- `--tags <tags>`: Filter by tags (comma-separated)
- `--suite <string>`: Filter by suite name (exact match)
- `--type <string>`: Filter by test type
- `--browser <string>`: Filter by browser
- `--device <string>`: Filter by device
- `--flaky`: Filter to flaky tests only
- `--output, -o`: Output file path (optional; defaults to stdout)
The command will output the names of the flaky tests and the number of retries each test has undergone. For example:
**Examples:**
```zsh
Processing report: reports/sample-report.json
Found 1 flaky test(s) in reports/sample-report.json:
- Test Name: Test 1, Retries: 2
```sh
# Filter failed tests to stdout
ctrf filter report.json --status failed
# Filter by multiple criteria and save to file
ctrf filter report.json --status failed,skipped --tags critical --output filtered.json
# Filter flaky tests and save
ctrf filter report.json --flaky --output flaky-report.json
# Read from stdin
cat report.json | ctrf filter - --status failed
```
## What is CTRF?
## generate-test-ids
CTRF is a universal JSON test report schema that addresses the lack of a standardized format for JSON test reports.
Generates deterministic UUID v5 identifiers for all tests in a report. **Outputs to stdout or file.**
**Consistency Across Tools:** Different testing tools and frameworks often produce reports in varied formats. CTRF ensures a uniform structure, making it easier to understand and compare reports, regardless of the testing tool used.
**Syntax:**
**Language and Framework Agnostic:** It provides a universal reporting schema that works seamlessly with any programming language and testing framework.
```sh
ctrf generate-test-ids <file-path> [--output <path>]
```
**Facilitates Better Analysis:** With a standardized format, programatically analyzing test outcomes across multiple platforms becomes more straightforward.
**Parameters:**
## Support Us
- `file-path`: Path to CTRF report (use `-` for stdin) (required)
- `--output, -o`: Output file path (optional; defaults to stdout)
If you find this project useful, consider giving it a GitHub star ⭐ It means a lot to us.
**Example:**
```sh
# Output to stdout
ctrf generate-test-ids report.json
# Save to file
ctrf generate-test-ids report.json --output report-with-ids.json
```
## generate-report-id
Generates a unique UUID v4 identifier for a CTRF report. **Outputs to stdout or file.**
**Syntax:**
```sh
ctrf generate-report-id <file-path> [--output <path>]
```
**Parameters:**
- `file-path`: Path to CTRF report (required)
- `--output, -o`: Output file path (optional; defaults to stdout)
**Example:**
```sh
# Output to stdout
ctrf generate-report-id report.json
# Save to file
ctrf generate-report-id report.json --output report-with-id.json
```
## add-insights
Analyzes historical test reports and adds insights metrics to the current report. **Writes to file (or stdout for piping).**
Reads all CTRF reports in the historical reports directory and calculates trends, patterns, and metrics. Writes the enhanced current report with insights appended.
**Syntax:**
```sh
ctrf add-insights <current-report> <historical-reports> [--output <path>]
```
**Parameters:**
- `current-report`: Path to the CTRF report file to enhance (required)
- `historical-reports`: Path to directory containing historical CTRF reports for analysis (required)
- `--output, -o`: Output file path for the report with insights (optional; defaults to stdout)
**Example:**
```sh
# Analyze historical reports and enhance current report
ctrf add-insights ./report.json ./historical-reports --output ./report-with-insights.json
# Output enhanced report to stdout for piping
ctrf add-insights ./report.json ./historical-reports | jq .
```
## flaky
Identifies and reports tests marked as flaky in a CTRF report. **Outputs to stdout.**
**Syntax:**
```sh
ctrf flaky <file-path>
```
**Parameters:**
- `file-path`: Path to CTRF report file (required)
**Example:**
```sh
ctrf flaky reports/sample-report.json
```
**Output:**
```bash
Processing report: reports/sample-report.json
Found 1 flaky test(s) in reports/sample-report.json:
- Test Name: Test 1, Retries: 2
```
import { CtrfReport } from '../../types/ctrf';
/**
* Merges multiple CTRF reports into a single report.
*
* @param reports Array of CTRF report objects to be merged.
* @returns The merged CTRF report object.
*/
export declare function mergeReports(reports: CtrfReport[]): CtrfReport;
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.mergeReports = void 0;
/**
* Merges multiple CTRF reports into a single report.
*
* @param reports Array of CTRF report objects to be merged.
* @returns The merged CTRF report object.
*/
function mergeReports(reports) {
if (!reports || reports.length === 0) {
throw new Error('No reports provided for merging.');
}
const mergedReport = {
results: {
tool: reports[0].results.tool,
summary: initializeEmptySummary(),
tests: [],
},
};
reports.forEach(report => {
const { summary, tests, environment, extra } = report.results;
mergedReport.results.summary.tests += summary.tests;
mergedReport.results.summary.passed += summary.passed;
mergedReport.results.summary.failed += summary.failed;
mergedReport.results.summary.skipped += summary.skipped;
mergedReport.results.summary.pending += summary.pending;
mergedReport.results.summary.other += summary.other;
if (summary.suites !== undefined) {
mergedReport.results.summary.suites =
(mergedReport.results.summary.suites || 0) + summary.suites;
}
mergedReport.results.summary.start = Math.min(mergedReport.results.summary.start, summary.start);
mergedReport.results.summary.stop = Math.max(mergedReport.results.summary.stop, summary.stop);
mergedReport.results.tests.push(...tests);
if (environment) {
mergedReport.results.environment = Object.assign(Object.assign({}, mergedReport.results.environment), environment);
}
if (extra) {
mergedReport.results.extra = Object.assign(Object.assign({}, mergedReport.results.extra), extra);
}
});
return mergedReport;
}
exports.mergeReports = mergeReports;
/**
* Initializes an empty summary object.
*
* @returns An empty Summary object.
*/
function initializeEmptySummary() {
return {
tests: 0,
passed: 0,
failed: 0,
skipped: 0,
pending: 0,
other: 0,
start: Number.MAX_SAFE_INTEGER,
stop: 0,
};
}
import { CtrfReport } from '../../types/ctrf';
/**
* Reads a single CTRF report file from a specified path.
*
* @param filePath Path to the JSON file containing the CTRF report.
* @returns The parsed `CtrfReport` object.
* @throws If the file does not exist, is not a valid JSON, or does not conform to the `CtrfReport` structure.
*/
export declare function readSingleReport(filePath: string): CtrfReport;
/**
* Reads all CTRF report files from a given directory.
*
* @param directory Path to the directory containing JSON files.
* @returns An array of parsed `CtrfReport` objects.
* @throws If the directory does not exist or no valid CTRF reports are found.
*/
export declare function readReportsFromDirectory(directoryPath: string): CtrfReport[];
/**
* Reads all CTRF report files matching a glob pattern.
*
* @param pattern The glob pattern to match files (e.g., ctrf/*.json).
* @returns An array of parsed `CtrfReport` objects.
* @throws If no valid CTRF reports are found.
*/
export declare function readReportsFromGlobPattern(pattern: string): CtrfReport[];
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.readReportsFromGlobPattern = exports.readReportsFromDirectory = exports.readSingleReport = void 0;
// TO BE REMOVED, WILL USE THE CTRF LIBRARY INSTEAD
const fs_1 = __importDefault(require("fs"));
const path_1 = __importDefault(require("path"));
const glob_1 = require("glob");
/**
* Reads a single CTRF report file from a specified path.
*
* @param filePath Path to the JSON file containing the CTRF report.
* @returns The parsed `CtrfReport` object.
* @throws If the file does not exist, is not a valid JSON, or does not conform to the `CtrfReport` structure.
*/
function readSingleReport(filePath) {
if (!fs_1.default.existsSync(filePath)) {
throw new Error(`JSON file not found: ${filePath}`);
}
const resolvedPath = path_1.default.resolve(filePath);
if (!fs_1.default.existsSync(resolvedPath)) {
throw new Error(`The file '${resolvedPath}' does not exist.`);
}
try {
const content = fs_1.default.readFileSync(resolvedPath, 'utf8');
const parsed = JSON.parse(content);
if (!isCtrfReport(parsed)) {
throw new Error(`The file '${resolvedPath}' is not a valid CTRF report.`);
}
return parsed;
}
catch (error) {
const errorMessage = error.message || 'Unknown error';
throw new Error(`Failed to read or parse the file '${resolvedPath}': ${errorMessage}`);
}
}
exports.readSingleReport = readSingleReport;
/**
* Reads all CTRF report files from a given directory.
*
* @param directory Path to the directory containing JSON files.
* @returns An array of parsed `CtrfReport` objects.
* @throws If the directory does not exist or no valid CTRF reports are found.
*/
function readReportsFromDirectory(directoryPath) {
directoryPath = path_1.default.resolve(directoryPath);
if (!fs_1.default.existsSync(directoryPath)) {
throw new Error(`The directory '${directoryPath}' does not exist.`);
}
const files = fs_1.default.readdirSync(directoryPath);
const reports = files
.filter(file => path_1.default.extname(file) === '.json')
.map(file => {
const filePath = path_1.default.join(directoryPath, file);
try {
const content = fs_1.default.readFileSync(filePath, 'utf8');
const parsed = JSON.parse(content);
if (!isCtrfReport(parsed)) {
console.warn(`Skipping invalid CTRF report file: ${file}`);
return null;
}
return parsed;
}
catch (error) {
console.warn(`Failed to read or parse file '${file}':`, error);
return null;
}
})
.filter((report) => report !== null);
if (reports.length === 0) {
throw new Error(`No valid CTRF reports found in the directory '${directoryPath}'.`);
}
return reports;
}
exports.readReportsFromDirectory = readReportsFromDirectory;
/**
* Reads all CTRF report files matching a glob pattern.
*
* @param pattern The glob pattern to match files (e.g., ctrf/*.json).
* @returns An array of parsed `CtrfReport` objects.
* @throws If no valid CTRF reports are found.
*/
function readReportsFromGlobPattern(pattern) {
const files = glob_1.glob.sync(pattern);
if (files.length === 0) {
throw new Error(`No files found matching the pattern '${pattern}'.`);
}
const reports = files
.map(file => {
try {
const content = fs_1.default.readFileSync(file, 'utf8');
const parsed = JSON.parse(content);
if (!isCtrfReport(parsed)) {
console.warn(`Skipping invalid CTRF report file: ${file}`);
return null;
}
return parsed;
}
catch (error) {
console.warn(`Failed to read or parse file '${file}':`, error);
return null;
}
})
.filter((report) => report !== null);
if (reports.length === 0) {
throw new Error(`No valid CTRF reports found matching the pattern '${pattern}'.`);
}
return reports;
}
exports.readReportsFromGlobPattern = readReportsFromGlobPattern;
/**
* Checks if an object conforms to the `CtrfReport` structure.
*
* @param obj The object to validate.
* @returns `true` if the object matches the `CtrfReport` type; otherwise, `false`.
*/
function isCtrfReport(obj) {
return (obj &&
typeof obj === 'object' &&
obj.results &&
Array.isArray(obj.results.tests) &&
typeof obj.results.summary === 'object' &&
typeof obj.results.tool === 'object');
}