inferbench-cli - npm Package Compare versions

Comparing version 0.1.0 to 0.1.1

dist/benchmark.d.ts

+import type { EngineAdapter, EngineBenchmarkResult } from "./types.js";
+export interface RunBenchmarkOptions {
+    model: string;
+    maxTokens?: number;
+    serverStartTimeoutMs?: number;
+    requestTimeoutMs?: number;
+    prompts?: string[];
+    verbose?: boolean;
+    onProgress?: (line: string) => void;
+}
+/**
+ * Benchmarks a single engine. Never throws for an engine-level failure
+ * (not installed, server start timeout, request failure) -- engines are
+ * isolated from each other by design, so one bad engine never blocks
+ * results from the others. Callers get a result object with installed:false
+ * or per-run errors instead.
+ */
+export declare function benchmarkEngine(adapter: EngineAdapter, opts: RunBenchmarkOptions): Promise<EngineBenchmarkResult>;

dist/benchmark.js

+import { timedCompletion } from "./harness/measure.js";
+import { DEFAULT_PROMPTS, WARMUP_PROMPT } from "./prompts.js";
+const DEFAULT_MAX_TOKENS = 200;
+const DEFAULT_SERVER_START_TIMEOUT_MS = 5 * 60 * 1000;
+const DEFAULT_REQUEST_TIMEOUT_MS = 2 * 60 * 1000;
+let nextPort = 41000;
+function claimPort() {
+    return nextPort++;
+}
+/**
+ * Benchmarks a single engine. Never throws for an engine-level failure
+ * (not installed, server start timeout, request failure) -- engines are
+ * isolated from each other by design, so one bad engine never blocks
+ * results from the others. Callers get a result object with installed:false
+ * or per-run errors instead.
+ */
+export async function benchmarkEngine(adapter, opts) {
+    const progress = opts.onProgress ?? (() => { });
+    const prompts = opts.prompts ?? DEFAULT_PROMPTS;
+    const maxTokens = opts.maxTokens ?? DEFAULT_MAX_TOKENS;
+    if (!adapter.isInstalled()) {
+        progress(`${adapter.name}: not installed, skipped`);
+        return {
+            engine: adapter.name,
+            installed: false,
+            error: `binary "${adapter.binary}" not found`,
+            runs: [],
+            avgTokensPerSecond: null,
+            minTokensPerSecond: null,
+            maxTokensPerSecond: null,
+        };
+    }
+    progress(`${adapter.name}: starting server...`);
+    let server;
+    try {
+        server = await adapter.startServer({
+            model: opts.model,
+            port: claimPort(),
+            timeoutMs: opts.serverStartTimeoutMs ?? DEFAULT_SERVER_START_TIMEOUT_MS,
+            verbose: opts.verbose,
+        });
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        progress(`${adapter.name}: failed to start (${message})`);
+        return {
+            engine: adapter.name,
+            installed: true,
+            error: message,
+            runs: [],
+            avgTokensPerSecond: null,
+            minTokensPerSecond: null,
+            maxTokensPerSecond: null,
+        };
+    }
+    try {
+        progress(`${adapter.name}: warming up...`);
+        try {
+            await timedCompletion({
+                engine: adapter.name,
+                baseUrl: server.baseUrl,
+                modelId: server.modelId,
+                prompt: WARMUP_PROMPT,
+                maxTokens: 16,
+                timeoutMs: opts.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS,
+            });
+        }
+        catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            progress(`${adapter.name}: warm-up failed (${message})`);
+            return {
+                engine: adapter.name,
+                installed: true,
+                error: `warm-up failed: ${message}`,
+                runs: [],
+                avgTokensPerSecond: null,
+                minTokensPerSecond: null,
+                maxTokensPerSecond: null,
+            };
+        }
+        const runs = [];
+        for (let i = 0; i < prompts.length; i++) {
+            const prompt = prompts[i];
+            progress(`${adapter.name}: [${i + 1}/${prompts.length}] benchmarking...`);
+            try {
+                const result = await timedCompletion({
+                    engine: adapter.name,
+                    baseUrl: server.baseUrl,
+                    modelId: server.modelId,
+                    prompt,
+                    maxTokens,
+                    timeoutMs: opts.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS,
+                });
+                runs.push({ prompt, ok: true, result });
+            }
+            catch (err) {
+                const message = err instanceof Error ? err.message : String(err);
+                runs.push({ prompt, ok: false, error: message });
+            }
+        }
+        const valid = runs
+            .filter((r) => r.ok && r.result?.tokensPerSecond)
+            .map((r) => r.result.tokensPerSecond);
+        return {
+            engine: adapter.name,
+            installed: true,
+            runs,
+            avgTokensPerSecond: valid.length > 0
+                ? Math.round((valid.reduce((a, b) => a + b, 0) / valid.length) * 100) / 100
+                : null,
+            minTokensPerSecond: valid.length > 0 ? Math.min(...valid) : null,
+            maxTokensPerSecond: valid.length > 0 ? Math.max(...valid) : null,
+        };
+    }
+    finally {
+        server.stop();
+    }
+}

dist/cli.d.ts

+#!/usr/bin/env node
+export {};

dist/cli.js

+#!/usr/bin/env node
+import { Command } from "commander";
+import { detectHardware } from "./hardware/detect.js";
+import { resolveEngines, allEngines, SUPPORTED_ENGINES } from "./engines/registry.js";
+import { benchmarkEngine } from "./benchmark.js";
+import { recommend } from "./recommend/config.js";
+import { writeJsonReport } from "./report/json.js";
+import { NoEnginesFoundError, UsageError } from "./errors.js";
+const program = new Command();
+program
+    .name("inferbench")
+    .description("Benchmarks local-LLM-inference engines (omlx, llama.cpp) on your own hardware, live.")
+    .version("0.1.0");
+program
+    .command("run")
+    .description("Benchmark installed engines against a model")
+    .requiredOption("--model <spec>", "Model spec (engine-specific, see README)")
+    .option("--engines <list>", `Comma-separated engines to test (default: all supported -- ${SUPPORTED_ENGINES.join(", ")})`)
+    .option("--max-tokens <n>", "Max completion tokens per prompt", "200")
+    .option("--json", "Output machine-readable JSON instead of a human table")
+    .option("--out <file>", "Also write the full JSON report to this file")
+    .option("--verbose", "Show raw engine server stdout/stderr")
+    .action(async (options) => {
+    try {
+        await runCommand(options);
+    }
+    catch (err) {
+        if (err instanceof UsageError || err instanceof NoEnginesFoundError) {
+            console.error(err.message);
+            process.exit(1);
+        }
+        throw err;
+    }
+});
+function parseMaxTokens(value) {
+    const parsed = Number(value);
+    if (!Number.isInteger(parsed)) {
+        throw new UsageError(`Invalid --max-tokens value "${value}": must be a whole number, e.g. --max-tokens 200`);
+    }
+    if (parsed <= 0) {
+        throw new UsageError(`Invalid --max-tokens value "${value}": must be a positive number`);
+    }
+    return parsed;
+}
+async function runCommand(options) {
+    const adapters = options.engines
+        ? resolveEngines(options.engines.split(","))
+        : allEngines();
+    const maxTokens = parseMaxTokens(options.maxTokens);
+    const hardware = detectHardware();
+    if (!options.json) {
+        console.log(`Hardware: ${hardware.cpuModel} (${hardware.platform}/${hardware.arch}), ${hardware.totalMemoryGb}GB\n`);
+    }
+    const results = [];
+    for (const adapter of adapters) {
+        const result = await benchmarkEngine(adapter, {
+            model: options.model,
+            maxTokens,
+            verbose: options.verbose,
+            onProgress: options.json ? undefined : (line) => console.log(line),
+        });
+        results.push(result);
+    }
+    const testedAny = results.some((r) => r.installed);
+    if (!testedAny) {
+        throw new NoEnginesFoundError();
+    }
+    const report = {
+        timestamp: new Date().toISOString(),
+        hardware,
+        model: options.model,
+        engines: results,
+        recommendation: recommend(results),
+    };
+    if (options.out) {
+        await writeJsonReport(report, options.out);
+    }
+    if (options.json) {
+        console.log(JSON.stringify(report, null, 2));
+        return;
+    }
+    console.log("\nResults:");
+    for (const r of results) {
+        if (!r.installed) {
+            console.log(`  ${r.engine}: not installed, skipped`);
+            continue;
+        }
+        if (r.avgTokensPerSecond === null) {
+            console.log(`  ${r.engine}: FAILED (${r.error ?? "no successful runs"})`);
+            continue;
+        }
+        console.log(`  ${r.engine}: avg ${r.avgTokensPerSecond} tok/s (range ${r.minTokensPerSecond}-${r.maxTokensPerSecond}, n=${r.runs.filter((x) => x.ok).length})`);
+    }
+    if (report.recommendation) {
+        console.log(`\nRecommendation: ${report.recommendation.engine} -- ${report.recommendation.reason}`);
+    }
+    if (options.out) {
+        console.log(`\nFull report: ${options.out}`);
+    }
+}
+program.parseAsync(process.argv);

dist/cost/cloud_comparison.d.ts

+/**
+ * Static, versioned pricing snapshot -- not a live API call. Prices drift;
+ * this table is a directional reference, not a real-time quote. Update the
+ * date below whenever prices are refreshed.
+ */
+export declare const PRICING_SNAPSHOT_DATE = "2026-07-15";
+export declare const CLOUD_PRICING_PER_1K_OUTPUT_TOKENS: Record<string, number>;
+export interface CostComparison {
+    cloudModel: string;
+    cloudCostPer1kTokensUsd: number;
+    pricingSnapshotDate: string;
+    note: string;
+}
+export declare function compareToCloud(cloudModel: string): CostComparison | null;

dist/cost/cloud_comparison.js

+/**
+ * Static, versioned pricing snapshot -- not a live API call. Prices drift;
+ * this table is a directional reference, not a real-time quote. Update the
+ * date below whenever prices are refreshed.
+ */
+export const PRICING_SNAPSHOT_DATE = "2026-07-15";
+export const CLOUD_PRICING_PER_1K_OUTPUT_TOKENS = {
+    "claude-5-haiku": 0.0008,
+    "claude-5-sonnet": 0.006,
+};
+export function compareToCloud(cloudModel) {
+    const price = CLOUD_PRICING_PER_1K_OUTPUT_TOKENS[cloudModel];
+    if (price === undefined)
+        return null;
+    return {
+        cloudModel,
+        cloudCostPer1kTokensUsd: price,
+        pricingSnapshotDate: PRICING_SNAPSHOT_DATE,
+        note: "Local hardware's own amortized cost is not included -- this compares raw " +
+            "generation cost only. Local inference has $0 marginal per-token cost once " +
+            "hardware is already owned.",
+    };
+}

dist/engines/llamacpp.d.ts

+import type { EngineAdapter, StartedServer } from "../types.js";
+export declare class LlamaCppAdapter implements EngineAdapter {
+    readonly name = "llama.cpp";
+    readonly binary = "llama-server";
+    isInstalled(): boolean;
+    /**
+     * `model` is treated as a Hugging Face repo spec (e.g.
+     * "bartowski/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M") -- llama.cpp's own
+     * -hf flag downloads and caches it automatically, no manual step needed.
+     */
+    startServer(opts: {
+        model: string;
+        port: number;
+        timeoutMs: number;
+        verbose?: boolean;
+    }): Promise<StartedServer>;
+}

dist/engines/llamacpp.js

+import { execFileSync } from "node:child_process";
+import { spawnServerAndWaitReady } from "../harness/spawn-server.js";
+import { EngineNotFoundError, UsageError } from "../errors.js";
+const BINARY = "llama-server";
+export class LlamaCppAdapter {
+    name = "llama.cpp";
+    binary = BINARY;
+    isInstalled() {
+        try {
+            execFileSync(BINARY, ["--version"], { stdio: "ignore" });
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * `model` is treated as a Hugging Face repo spec (e.g.
+     * "bartowski/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M") -- llama.cpp's own
+     * -hf flag downloads and caches it automatically, no manual step needed.
+     */
+    async startServer(opts) {
+        if (!this.isInstalled()) {
+            throw new EngineNotFoundError(this.name, BINARY);
+        }
+        if (opts.model.startsWith("-")) {
+            throw new UsageError(`Invalid --model value "${opts.model}": cannot start with "-" (would be parsed as a flag by llama-server, not a model spec)`);
+        }
+        const spawned = await spawnServerAndWaitReady({
+            engine: this.name,
+            command: BINARY,
+            args: [
+                "-hf",
+                opts.model,
+                "--port",
+                String(opts.port),
+                "--host",
+                "127.0.0.1",
+            ],
+            readyCheckUrl: `http://127.0.0.1:${opts.port}/v1/models`,
+            timeoutMs: opts.timeoutMs,
+            verbose: opts.verbose,
+        });
+        return {
+            process: spawned.process,
+            baseUrl: `http://127.0.0.1:${opts.port}`,
+            modelId: "default",
+            stop: spawned.stop,
+        };
+    }
+}

dist/engines/omlx.d.ts

+import type { EngineAdapter, StartedServer } from "../types.js";
+/**
+ * omlx has no CLI benchmark tool of its own (verified against its real
+ * README -- its "Performance Benchmark" feature is a GUI-only admin-panel
+ * one-click action). This adapter only starts the server; all measurement
+ * goes through the shared HTTP harness in src/harness/measure.ts, same as
+ * every other engine.
+ */
+export declare class OmlxAdapter implements EngineAdapter {
+    readonly name = "omlx";
+    readonly binary = "omlx";
+    isInstalled(): boolean;
+    /**
+     * `model` is treated as a model-dir subdirectory name under
+     * ~/.omlx/models/<model> -- omlx's `serve` command has no positional
+     * model argument and discovers models from --model-dir subdirectories
+     * (or the standard HF cache). Unlike llama.cpp, omlx does not auto-download
+     * an arbitrary HF repo from a CLI flag, so the model must already be
+     * present locally -- an honest v0.1 limitation, not hidden from the user.
+     */
+    startServer(opts: {
+        model: string;
+        port: number;
+        timeoutMs: number;
+        verbose?: boolean;
+    }): Promise<StartedServer>;
+    private resolveModelId;
+}

dist/engines/omlx.js

+import { execFileSync } from "node:child_process";
+import os from "node:os";
+import path from "node:path";
+import { spawnServerAndWaitReady } from "../harness/spawn-server.js";
+import { EngineNotFoundError, EngineStartTimeoutError } from "../errors.js";
+const BINARY = "omlx";
+/**
+ * omlx has no CLI benchmark tool of its own (verified against its real
+ * README -- its "Performance Benchmark" feature is a GUI-only admin-panel
+ * one-click action). This adapter only starts the server; all measurement
+ * goes through the shared HTTP harness in src/harness/measure.ts, same as
+ * every other engine.
+ */
+export class OmlxAdapter {
+    name = "omlx";
+    binary = BINARY;
+    isInstalled() {
+        try {
+            execFileSync(BINARY, ["--version"], { stdio: "ignore" });
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * `model` is treated as a model-dir subdirectory name under
+     * ~/.omlx/models/<model> -- omlx's `serve` command has no positional
+     * model argument and discovers models from --model-dir subdirectories
+     * (or the standard HF cache). Unlike llama.cpp, omlx does not auto-download
+     * an arbitrary HF repo from a CLI flag, so the model must already be
+     * present locally -- an honest v0.1 limitation, not hidden from the user.
+     */
+    async startServer(opts) {
+        if (!this.isInstalled()) {
+            throw new EngineNotFoundError(this.name, BINARY);
+        }
+        const modelDir = path.join(os.homedir(), ".omlx", "models");
+        const spawned = await spawnServerAndWaitReady({
+            engine: this.name,
+            command: BINARY,
+            args: ["serve", "--model-dir", modelDir, "--port", String(opts.port)],
+            readyCheckUrl: `http://127.0.0.1:${opts.port}/v1/models`,
+            timeoutMs: opts.timeoutMs,
+            verbose: opts.verbose,
+        });
+        const modelId = await this.resolveModelId(opts.port, opts.model);
+        if (!modelId) {
+            spawned.stop();
+            throw new EngineStartTimeoutError(this.name, opts.timeoutMs);
+        }
+        return {
+            process: spawned.process,
+            baseUrl: `http://127.0.0.1:${opts.port}`,
+            modelId,
+            stop: spawned.stop,
+        };
+    }
+    async resolveModelId(port, requestedModel) {
+        const res = await fetch(`http://127.0.0.1:${port}/v1/models`);
+        if (!res.ok)
+            return null;
+        const payload = (await res.json());
+        const models = payload.data ?? [];
+        if (models.length === 0)
+            return null;
+        const exact = models.find((m) => m.id === requestedModel);
+        return exact ? exact.id : models[0].id;
+    }
+}

dist/engines/registry.d.ts

+import type { EngineAdapter } from "../types.js";
+export declare const SUPPORTED_ENGINES: string[];
+/** Dedupes and validates a user-supplied --engines list. */
+export declare function resolveEngines(names: string[]): EngineAdapter[];
+export declare function allEngines(): EngineAdapter[];

dist/engines/registry.js

+import { OmlxAdapter } from "./omlx.js";
+import { LlamaCppAdapter } from "./llamacpp.js";
+import { UsageError } from "../errors.js";
+const ADAPTERS = {
+    omlx: () => new OmlxAdapter(),
+    "llama.cpp": () => new LlamaCppAdapter(),
+};
+export const SUPPORTED_ENGINES = Object.keys(ADAPTERS);
+/** Dedupes and validates a user-supplied --engines list. */
+export function resolveEngines(names) {
+    const deduped = [...new Set(names)];
+    const unknown = deduped.filter((n) => !(n in ADAPTERS));
+    if (unknown.length > 0) {
+        throw new UsageError(`Unknown engine(s): ${unknown.join(", ")}. Supported: ${SUPPORTED_ENGINES.join(", ")}`);
+    }
+    return deduped.map((n) => ADAPTERS[n]());
+}
+export function allEngines() {
+    return SUPPORTED_ENGINES.map((n) => ADAPTERS[n]());
+}

dist/errors.d.ts

+export declare class EngineNotFoundError extends Error {
+    readonly engine: string;
+    constructor(engine: string, binary: string);
+}
+export declare class EngineStartTimeoutError extends Error {
+    readonly engine: string;
+    constructor(engine: string, timeoutMs: number);
+}
+export declare class EngineRequestTimeoutError extends Error {
+    readonly engine: string;
+    constructor(engine: string, timeoutMs: number);
+}
+export declare class BenchmarkParseError extends Error {
+    readonly engine: string;
+    constructor(engine: string, rawOutput: string);
+    rawOutput: string;
+}
+export declare class NoEnginesFoundError extends Error {
+    constructor();
+}
+export declare class UsageError extends Error {
+    constructor(message: string);
+}

dist/errors.js

+export class EngineNotFoundError extends Error {
+    engine;
+    constructor(engine, binary) {
+        super(`${engine}: binary "${binary}" not found on PATH -- skipped`);
+        this.engine = engine;
+        this.name = "EngineNotFoundError";
+    }
+}
+export class EngineStartTimeoutError extends Error {
+    engine;
+    constructor(engine, timeoutMs) {
+        super(`${engine}: server did not become ready within ${timeoutMs}ms`);
+        this.engine = engine;
+        this.name = "EngineStartTimeoutError";
+    }
+}
+export class EngineRequestTimeoutError extends Error {
+    engine;
+    constructor(engine, timeoutMs) {
+        super(`${engine}: request timed out after ${timeoutMs}ms`);
+        this.engine = engine;
+        this.name = "EngineRequestTimeoutError";
+    }
+}
+export class BenchmarkParseError extends Error {
+    engine;
+    constructor(engine, rawOutput) {
+        super(`${engine}: could not parse benchmark response`);
+        this.engine = engine;
+        this.name = "BenchmarkParseError";
+        this.rawOutput = rawOutput;
+    }
+    rawOutput;
+}
+export class NoEnginesFoundError extends Error {
+    constructor() {
+        super("No supported engines found on this machine. Install omlx (brew install omlx) " +
+            "or llama.cpp (brew install llama.cpp) and try again.");
+        this.name = "NoEnginesFoundError";
+    }
+}
+export class UsageError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "UsageError";
+    }
+}

dist/hardware/detect.d.ts

+import type { HardwareProfile } from "../types.js";
+export declare function detectHardware(): HardwareProfile;

dist/hardware/detect.js

+import os from "node:os";
+export function detectHardware() {
+    const platform = os.platform();
+    const arch = os.arch();
+    const cpus = os.cpus();
+    return {
+        platform,
+        arch,
+        totalMemoryGb: Math.round((os.totalmem() / 1024 ** 3) * 10) / 10,
+        cpuModel: cpus.length > 0 ? cpus[0].model : "unknown",
+        isAppleSilicon: platform === "darwin" && arch === "arm64",
+    };
+}

dist/harness/measure.d.ts

+import type { CompletionResult } from "../types.js";
+export interface TimedCompletionOptions {
+    engine: string;
+    baseUrl: string;
+    modelId: string;
+    prompt: string;
+    maxTokens: number;
+    timeoutMs: number;
+}
+/**
+ * Sends one timed chat-completion request to an engine's OpenAI-compatible
+ * server and measures wall-clock time + reported token counts. This is the
+ * ONE shared measurement code path for every engine -- the architecture
+ * decision that replaced per-engine benchmark-CLI parsing (omlx has no CLI
+ * benchmark tool at all; this makes the comparison genuinely apples-to-apples
+ * since the same code measures every engine).
+ */
+export declare function timedCompletion(opts: TimedCompletionOptions): Promise<CompletionResult>;

dist/harness/measure.js

+import { EngineRequestTimeoutError, BenchmarkParseError } from "../errors.js";
+/**
+ * Sends one timed chat-completion request to an engine's OpenAI-compatible
+ * server and measures wall-clock time + reported token counts. This is the
+ * ONE shared measurement code path for every engine -- the architecture
+ * decision that replaced per-engine benchmark-CLI parsing (omlx has no CLI
+ * benchmark tool at all; this makes the comparison genuinely apples-to-apples
+ * since the same code measures every engine).
+ */
+export async function timedCompletion(opts) {
+    const body = JSON.stringify({
+        model: opts.modelId,
+        messages: [{ role: "user", content: opts.prompt }],
+        max_tokens: opts.maxTokens,
+        stream: false,
+    });
+    // Elapsed time must be measured across the FULL request, including
+    // reading the response body -- fetch() itself resolves as soon as
+    // headers arrive, before generation is done streaming back. Measuring
+    // right after `await fetch(...)` (a real bug caught during a live
+    // end-to-end run: it produced a physically impossible 64,646 tok/s)
+    // only captures time-to-first-byte, not actual generation time.
+    const start = performance.now();
+    let response;
+    try {
+        response = await fetch(`${opts.baseUrl}/v1/chat/completions`, {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body,
+            signal: AbortSignal.timeout(opts.timeoutMs),
+        });
+    }
+    catch (err) {
+        if (err instanceof Error && err.name === "TimeoutError") {
+            throw new EngineRequestTimeoutError(opts.engine, opts.timeoutMs);
+        }
+        throw err;
+    }
+    if (!response.ok) {
+        const errorBody = await response.text().catch(() => "");
+        throw new BenchmarkParseError(opts.engine, `HTTP ${response.status}: ${errorBody}`);
+    }
+    let payload;
+    const rawText = await response.text();
+    const elapsedMs = performance.now() - start;
+    try {
+        payload = JSON.parse(rawText);
+    }
+    catch {
+        throw new BenchmarkParseError(opts.engine, rawText);
+    }
+    const usage = payload?.usage;
+    const completionTokens = typeof usage?.completion_tokens === "number" ? usage.completion_tokens : null;
+    const promptTokensDetails = usage?.prompt_tokens_details;
+    const cachedPromptTokens = typeof promptTokensDetails?.cached_tokens === "number"
+        ? promptTokensDetails.cached_tokens
+        : 0;
+    const tokensPerSecond = completionTokens && elapsedMs > 0
+        ? completionTokens / (elapsedMs / 1000)
+        : null;
+    return {
+        elapsedMs: Math.round(elapsedMs),
+        completionTokens,
+        cachedPromptTokens,
+        tokensPerSecond: tokensPerSecond ? Math.round(tokensPerSecond * 100) / 100 : null,
+    };
+}

dist/harness/spawn-server.d.ts

+import { spawn } from "node:child_process";
+export interface SpawnServerOptions {
+    /** Engine name, used only for error messages. */
+    engine: string;
+    /** Executable name or path -- never a shell string. */
+    command: string;
+    /** Argv array -- never interpolated into a shell string (command injection safety). */
+    args: string[];
+    /** URL to poll until it responds successfully, indicating the server is ready. */
+    readyCheckUrl: string;
+    /** Max time to wait for readyCheckUrl to respond before giving up. */
+    timeoutMs: number;
+    /** Poll interval while waiting for the server to become ready. */
+    pollIntervalMs?: number;
+    verbose?: boolean;
+}
+export interface SpawnedServer {
+    process: ReturnType<typeof spawn>;
+    stop(): void;
+}
+/**
+ * Spawns a long-running server process (never through a shell -- argv array
+ * only, to avoid command-injection risk) and polls a URL until it responds,
+ * so callers get a server that is actually ready to accept requests rather
+ * than racing against startup.
+ */
+export declare function spawnServerAndWaitReady(opts: SpawnServerOptions): Promise<SpawnedServer>;

dist/harness/spawn-server.js

+import { spawn } from "node:child_process";
+import { EngineStartTimeoutError } from "../errors.js";
+/**
+ * Spawns a long-running server process (never through a shell -- argv array
+ * only, to avoid command-injection risk) and polls a URL until it responds,
+ * so callers get a server that is actually ready to accept requests rather
+ * than racing against startup.
+ */
+export async function spawnServerAndWaitReady(opts) {
+    const pollIntervalMs = opts.pollIntervalMs ?? 500;
+    const child = spawn(opts.command, opts.args, {
+        stdio: opts.verbose ? "inherit" : "ignore",
+    });
+    const stop = () => {
+        if (!child.killed) {
+            child.kill();
+        }
+    };
+    const deadline = Date.now() + opts.timeoutMs;
+    let lastError;
+    while (Date.now() < deadline) {
+        if (child.exitCode !== null) {
+            throw new Error(`${opts.engine}: process exited early (code ${child.exitCode}) before becoming ready`);
+        }
+        try {
+            const res = await fetch(opts.readyCheckUrl, {
+                signal: AbortSignal.timeout(2000),
+            });
+            if (res.ok) {
+                return { process: child, stop };
+            }
+        }
+        catch (err) {
+            lastError = err;
+        }
+        await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
+    }
+    stop();
+    const timeoutErr = new EngineStartTimeoutError(opts.engine, opts.timeoutMs);
+    if (lastError instanceof Error) {
+        timeoutErr.cause = lastError;
+    }
+    throw timeoutErr;
+}

dist/prompts.d.ts

+/**
+ * Default varied prompt set: 8 distinct prompts across different topics and
+ * lengths, deliberately never repeating the exact same prompt, to avoid the
+ * prefix-cache skew observed during real validation (a repeated identical
+ * prompt let llama.cpp reuse 39/40 cached prompt tokens on the second call,
+ * which is not a fair steady-state comparison).
+ */
+export declare const DEFAULT_PROMPTS: string[];
+export declare const WARMUP_PROMPT = "Say hello.";

dist/prompts.js

+/**
+ * Default varied prompt set: 8 distinct prompts across different topics and
+ * lengths, deliberately never repeating the exact same prompt, to avoid the
+ * prefix-cache skew observed during real validation (a repeated identical
+ * prompt let llama.cpp reuse 39/40 cached prompt tokens on the second call,
+ * which is not a fair steady-state comparison).
+ */
+export const DEFAULT_PROMPTS = [
+    "Explain in one paragraph why the sky is blue.",
+    "Write a short haiku about autumn leaves falling in a quiet forest.",
+    "What are three practical tips for someone learning to cook rice perfectly every time?",
+    "Summarize the plot of a story about a lighthouse keeper who discovers a message in a bottle.",
+    "List five differences between a cat and a dog as household pets.",
+    "Describe how a bicycle chain transfers power from the pedals to the rear wheel.",
+    "Give a brief explanation of why leaves change color in autumn, covering chlorophyll and other pigments.",
+    "What is the difference between weather and climate? Explain with a simple example.",
+];
+export const WARMUP_PROMPT = "Say hello.";

dist/recommend/config.d.ts

+import type { EngineBenchmarkResult, Recommendation } from "../types.js";
+/**
+ * v0.1 recommendation rule: highest measured average tok/s among engines
+ * that were actually installed and tested. Deliberately simple -- richer
+ * multi-factor scoring (memory, cost) is intentionally deferred until real
+ * usage shows this simple rule picks wrong recommendations.
+ */
+export declare function recommend(results: EngineBenchmarkResult[]): Recommendation | null;

dist/recommend/config.js

+/**
+ * v0.1 recommendation rule: highest measured average tok/s among engines
+ * that were actually installed and tested. Deliberately simple -- richer
+ * multi-factor scoring (memory, cost) is intentionally deferred until real
+ * usage shows this simple rule picks wrong recommendations.
+ */
+export function recommend(results) {
+    const candidates = results.filter((r) => r.installed && r.avgTokensPerSecond !== null);
+    if (candidates.length === 0)
+        return null;
+    const best = candidates.reduce((a, b) => (b.avgTokensPerSecond ?? 0) > (a.avgTokensPerSecond ?? 0) ? b : a);
+    return {
+        engine: best.engine,
+        reason: `highest measured throughput on this run (${best.avgTokensPerSecond} tok/s avg) -- specific to this hardware and model, not a universal ranking`,
+    };
+}

dist/report/json.d.ts

+import type { BenchmarkReport } from "../types.js";
+export declare class UnsafeOutputPathError extends Error {
+}
+export declare function writeJsonReport(report: BenchmarkReport, filePath: string): Promise<void>;

dist/report/json.js

+import { writeFile } from "node:fs/promises";
+import * as path from "node:path";
+export class UnsafeOutputPathError extends Error {
+}
+// --out is a plain CLI flag today, but this CLI is also meant to be invoked
+// programmatically by agents that may derive the value from less-trusted
+// input (a fetched benchmark config, an LLM-generated argument list, etc).
+// A relative path containing `..` segments can escape the intended output
+// location entirely (`--out ../../../etc/cron.d/x`) -- reject any --out
+// value that resolves outside the current working directory. An explicit
+// absolute path is still allowed: that's a value the caller typed/passed
+// directly, not one that silently escaped via traversal.
+function assertSafeOutputPath(filePath) {
+    if (path.isAbsolute(filePath))
+        return;
+    const cwd = process.cwd();
+    const resolved = path.resolve(cwd, filePath);
+    if (resolved !== cwd && !resolved.startsWith(cwd + path.sep)) {
+        throw new UnsafeOutputPathError(`--out "${filePath}" resolves outside the current working directory (${resolved}). ` +
+            "Pass an absolute path if you intend to write outside the working directory.");
+    }
+}
+export async function writeJsonReport(report, filePath) {
+    assertSafeOutputPath(filePath);
+    await writeFile(filePath, JSON.stringify(report, null, 2), "utf-8");
+}

dist/types.d.ts

+export interface HardwareProfile {
+    platform: NodeJS.Platform;
+    arch: string;
+    totalMemoryGb: number;
+    cpuModel: string;
+    isAppleSilicon: boolean;
+}
+export interface EngineAdapter {
+    readonly name: string;
+    readonly binary: string;
+    /** Check whether the engine's binary is installed on this machine. */
+    isInstalled(): boolean;
+    /**
+     * Start the engine's OpenAI-compatible server for the given model spec.
+     * Returns the running process and the base URL to send requests to.
+     */
+    startServer(opts: {
+        model: string;
+        port: number;
+        timeoutMs: number;
+        verbose?: boolean;
+    }): Promise<StartedServer>;
+}
+export interface StartedServer {
+    process: import("node:child_process").ChildProcess;
+    baseUrl: string;
+    modelId: string;
+    stop(): void;
+}
+export interface CompletionResult {
+    elapsedMs: number;
+    completionTokens: number | null;
+    cachedPromptTokens: number;
+    tokensPerSecond: number | null;
+}
+export interface PromptRunResult {
+    prompt: string;
+    ok: boolean;
+    error?: string;
+    result?: CompletionResult;
+}
+export interface EngineBenchmarkResult {
+    engine: string;
+    installed: boolean;
+    error?: string;
+    runs: PromptRunResult[];
+    avgTokensPerSecond: number | null;
+    minTokensPerSecond: number | null;
+    maxTokensPerSecond: number | null;
+}
+export interface BenchmarkReport {
+    timestamp: string;
+    hardware: HardwareProfile;
+    model: string;
+    engines: EngineBenchmarkResult[];
+    recommendation: Recommendation | null;
+}
+export interface Recommendation {
+    engine: string;
+    reason: string;
+}

dist/types.js

+export {};

package.json

 {
   "name": "inferbench-cli",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "description": "Benchmarks local-LLM-inference engines (omlx, llama.cpp) on your own hardware, live, and recommends the fastest engine/config combination for your model.",
@@ -5,0 +5,0 @@ "keywords": [

README.md

@@ -138,4 +138,29 @@ # InferBench
 
+## Documentation
+
+- [docs/getting-started.md](./docs/getting-started.md) -- install, first run, and using the library instead of the CLI, for both distributions.
+- [docs/concepts.md](./docs/concepts.md) -- the measurement architecture, the hardware detector, the recommendation rule, and the exit-code contract.
+- [docs/integrations/ci.md](./docs/integrations/ci.md) -- why InferBench is deliberately not a per-PR CI gate, and what patterns work instead.
+
+## Demo
+
+Install, first run, and a real omlx benchmark against a cached model:
+
+![InferBench install and first run: pip install inferbench-cli, then a live omlx benchmark reporting real tokens/second and a recommendation](./docs/demo.gif)
+
+Machine-readable output written to a file with `--json --out`, useful for CI or for an agent parsing the result:
+
+![InferBench --json --out usage: a live omlx benchmark run whose full JSON report (per-prompt tokens/second, recommendation) is printed to stdout and also saved to report.json](./docs/usage.gif)
+
 ## FAQ
 
+**What is InferBench, exactly?**
+A benchmarking tool for local-LLM-inference engines already installed on your machine -- currently `omlx` and `llama.cpp`. It runs a fixed, varied prompt set against whichever of those are present, measures real tokens/second for each, and recommends whichever one was fastest on that specific run. It ships as two packages under the same name, `inferbench-cli`: one on npm (JavaScript/TypeScript) and one on PyPI (Python).
+
+**How is InferBench different from llama.cpp's own `llama-bench`?**
+`llama-bench` (bundled with llama.cpp) only benchmarks llama.cpp itself, with fine-grained tuning knobs (batch size, cache type, thread count, and more). InferBench benchmarks *across* engines -- currently `omlx` and `llama.cpp` -- using the same prompt set and the same measurement code for both, so the resulting tokens/second numbers are directly comparable to each other on your hardware, not just tunable in isolation for one engine.
+
+**Does InferBench work on Linux and Windows, or only macOS?**
+The `llama.cpp` engine works on any platform llama.cpp itself supports (Linux, macOS, Windows), since InferBench just starts `llama-server` and measures its OpenAI-compatible endpoint. The `omlx` engine is Apple Silicon-only, matching omlx's own scope -- on Linux or Windows, `--engines omlx` reports that engine as not installed and InferBench benchmarks whatever supported engine actually is present. Node.js >=18 is required for the npm package, Python >=3.9 for the PyPI package.
+
 **Does InferBench download models for me?**
@@ -153,8 +178,11 @@ For llama.cpp, yes -- pass a Hugging Face repo spec and `llama-server`'s own `-hf` flag downloads and caches it. For omlx, no -- omlx's `serve` command only discovers models already present in a local directory, so you need to have the model downloaded there first.
 
-## Documentation
+**Is `--out` safe to point at a path that comes from an agent or other less-trusted input?**
+Yes, with one documented restriction: `--out` rejects a relative path that resolves outside the current working directory (for example `--out ../../etc/cron.d/x`), specifically so a benchmark invoked with an agent-supplied path can't be tricked into writing outside the intended directory. An absolute path is still accepted, since that's a value the caller passed directly rather than one that escaped via `..` traversal.
 
-- [docs/getting-started.md](./docs/getting-started.md) -- install, first run, and using the library instead of the CLI, for both distributions.
-- [docs/concepts.md](./docs/concepts.md) -- the measurement architecture, the hardware detector, the recommendation rule, and the exit-code contract.
-- [docs/integrations/ci.md](./docs/integrations/ci.md) -- why InferBench is deliberately not a per-PR CI gate, and what patterns work instead.
+**What happens if no supported engine is installed, or a run fails partway through?**
+If neither `omlx` nor `llama.cpp` is found, InferBench exits with code `1` and a message naming both install commands rather than returning a silent empty result. If an engine is installed but a specific run fails, that engine's line in the report reads `FAILED` with the underlying error instead of a number -- any other engine that did complete still gets a real result and remains eligible for the recommendation.
 
+**Can I use InferBench commercially, and is it free?**
+Yes. InferBench is Apache License 2.0, which permits commercial use, modification, and redistribution with no licensing fee. It has no paid API dependency -- every benchmark request goes to a server it starts locally on your own machine.
+
 ## Contributing
@@ -161,0 +189,0 @@

New alerts

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Network access

Supply chain risk

This module accesses the network.

Shell access

Supply chain risk

This module accesses the system shell. Accessing the system shell increases the risk of executing arbitrary code.

Filesystem access

Supply chain risk

Accesses the file system, and could potentially read sensitive data.

URL strings

Supply chain risk

Package contains fragments of external URLs or IP addresses, which the package may be accessing at runtime.

Fixed alerts

Empty package

Supply chain risk

Package does not contain any code. It may be removed, is name squatting, or the result of a faulty package publish.

Improved metrics

Total package byte prevSize

59174

158.88%

Number of package files

31

933.33%

Lines of code

851

Infinity%

Number of lines in readme file

197

16.57%

Worsened metrics

Number of low supply chain risk alerts

2

Infinity%

Number of medium supply chain risk alerts

8

700%

No dependency changes