skillgrade - npm Package Compare versions

Comparing version 0.1.3 to 0.1.4

dist/agents/acp.d.ts

+import { BaseAgent, CommandResult } from '../types';
+/**
+ * Configuration for ACP agent
+ */
+export interface AcpAgentConfig {
+    /** Command to start the ACP agent (e.g., "gemini --acp") */
+    command: string;
+    /** Optional environment variables */
+    env?: Record<string, string>;
+    /** Optional API key for authentication */
+    apiKey?: string;
+    /** Timeout in milliseconds for agent operations (default: 300000 = 5 min) */
+    timeout?: number;
+}
+/**
+ * ACP Agent implementation that communicates with ACP-compatible agents.
+ *
+ * The Agent Client Protocol (ACP) is a standardized protocol for communication
+ * between code editors/clients and AI coding agents. It uses JSON-RPC 2.0
+ * over stdio for transport.
+ *
+ * Example usage:
+ * - Gemini CLI: `gemini --acp`
+ * - Any ACP-compatible agent
+ */
+export declare class AcpAgent extends BaseAgent {
+    private config;
+    private process;
+    private connection;
+    private sessionId;
+    private sessionOutputs;
+    constructor(config: AcpAgentConfig);
+    /**
+     * Run an instruction using the ACP agent.
+     */
+    run(instruction: string, workspacePath: string, runCommand: (cmd: string) => Promise<CommandResult>): Promise<string>;
+    /**
+     * Initialize the ACP connection and create a session.
+     */
+    private initialize;
+    /**
+     * Create the client handler for handling agent requests.
+     */
+    private createClientHandler;
+    /**
+     * Handle session updates from the agent.
+     */
+    private handleSessionUpdate;
+    /**
+     * Wrap a promise with a timeout.
+     */
+    private withTimeout;
+    /**
+     * Cleanup the ACP connection and process.
+     */
+    cleanup(): Promise<void>;
+}
+//# sourceMappingURL=acp.d.ts.map

dist/agents/acp.js

+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AcpAgent = void 0;
+/**
+ * ACP (Agent Client Protocol) Agent Implementation
+ *
+ * This agent communicates with ACP-compatible agents (like Gemini CLI with --acp flag)
+ * using JSON-RPC 2.0 over stdio. This allows using any ACP-compatible agent without
+ * requiring direct API key configuration.
+ *
+ * @see https://agentclientprotocol.com/
+ */
+const child_process_1 = require("child_process");
+const types_1 = require("../types");
+const acp = __importStar(require("@agentclientprotocol/sdk"));
+/**
+ * ACP Agent implementation that communicates with ACP-compatible agents.
+ *
+ * The Agent Client Protocol (ACP) is a standardized protocol for communication
+ * between code editors/clients and AI coding agents. It uses JSON-RPC 2.0
+ * over stdio for transport.
+ *
+ * Example usage:
+ * - Gemini CLI: `gemini --acp`
+ * - Any ACP-compatible agent
+ */
+class AcpAgent extends types_1.BaseAgent {
+    config;
+    process = null;
+    connection = null;
+    sessionId = null;
+    sessionOutputs = [];
+    constructor(config) {
+        super();
+        this.config = {
+            timeout: 300000,
+            ...config,
+        };
+    }
+    /**
+     * Run an instruction using the ACP agent.
+     */
+    async run(instruction, workspacePath, runCommand) {
+        try {
+            // Initialize connection if not already established
+            if (!this.connection) {
+                await this.initialize(workspacePath);
+            }
+            if (!this.sessionId) {
+                throw new Error('No active session. Failed to create session.');
+            }
+            // Clear session outputs
+            this.sessionOutputs = [];
+            // Send the prompt - prompt is an array of ContentBlock
+            const promptContent = [
+                { type: 'text', text: instruction },
+            ];
+            const response = await this.withTimeout(this.connection.prompt({
+                sessionId: this.sessionId,
+                prompt: promptContent,
+            }), this.config.timeout, 'Prompt execution');
+            // Format response
+            const outputs = [...this.sessionOutputs];
+            if (response.stopReason) {
+                outputs.push(`\n[Agent finished: ${response.stopReason}]`);
+            }
+            return outputs.join('\n').trim() || 'Agent completed without output.';
+        }
+        catch (error) {
+            const errorMsg = error instanceof Error ? error.message : String(error);
+            return `ACP Agent error: ${errorMsg}`;
+        }
+    }
+    /**
+     * Initialize the ACP connection and create a session.
+     */
+    async initialize(cwd) {
+        // Parse command into parts
+        const parts = this.config.command.split(' ');
+        const command = parts[0];
+        const args = parts.slice(1);
+        // Spawn the ACP agent process
+        this.process = (0, child_process_1.spawn)(command, args, {
+            cwd,
+            env: { ...process.env, ...this.config.env },
+            stdio: ['pipe', 'pipe', 'pipe'],
+        });
+        if (!this.process.stdin || !this.process.stdout) {
+            throw new Error('Failed to create stdio streams for ACP process');
+        }
+        // Create the ACP stream from stdio
+        // Convert Node.js streams to Web streams
+        const stdoutWeb = this.process.stdout;
+        const stdinWeb = this.process.stdin;
+        const stream = acp.ndJsonStream(stdoutWeb, stdinWeb);
+        // Create client handler
+        const clientHandler = this.createClientHandler();
+        // Create client-side connection
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        this.connection = new acp.ClientSideConnection(clientHandler, stream);
+        // Initialize the connection
+        const initResponse = await this.withTimeout(this.connection.initialize({
+            protocolVersion: acp.PROTOCOL_VERSION,
+            clientCapabilities: {
+                fs: {
+                    readTextFile: true,
+                    writeTextFile: true,
+                },
+            },
+        }), 30000, 'ACP initialization');
+        // Authenticate if we have an API key
+        if (this.config.apiKey) {
+            // Find the API key auth method
+            const authMethods = initResponse.authMethods || [];
+            const apiKeyMethod = authMethods.find((m) => m.id === 'use_gemini' || m.id === 'api_key');
+            if (apiKeyMethod) {
+                const authRequest = {
+                    methodId: apiKeyMethod.id,
+                    _meta: {
+                        'api-key': this.config.apiKey,
+                    },
+                };
+                await this.withTimeout(this.connection.authenticate(authRequest), 60000, 'ACP authentication');
+            }
+        }
+        // Create a new session
+        const sessionResponse = await this.withTimeout(this.connection.newSession({
+            cwd,
+            mcpServers: [],
+        }), 30000, 'ACP session creation');
+        this.sessionId = sessionResponse.sessionId;
+    }
+    /**
+     * Create the client handler for handling agent requests.
+     */
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    createClientHandler() {
+        const self = this;
+        return {
+            // Handle permission requests from agent (auto-approve for evaluation)
+            async requestPermission(params) {
+                // Auto-approve for evaluation context by selecting the first option
+                const options = params.options || [];
+                if (options.length > 0) {
+                    // Select the first option (usually "allow_once" or similar)
+                    const response = {
+                        outcome: 'selected',
+                        optionId: options[0].optionId,
+                    };
+                    return response;
+                }
+                // If no options, cancel
+                return { outcome: 'cancelled' };
+            },
+            // Handle session updates from agent
+            async sessionUpdate(params) {
+                // Collect text content from update
+                self.handleSessionUpdate(params.update);
+            },
+            // File system operations (not supported in evaluation mode)
+            async readTextFile(params) {
+                throw new Error('File system access not supported in ACP evaluation mode');
+            },
+            async writeTextFile(params) {
+                throw new Error('File system access not supported in ACP evaluation mode');
+            },
+        };
+    }
+    /**
+     * Handle session updates from the agent.
+     */
+    handleSessionUpdate(update) {
+        // Check for text content
+        const updateRecord = update;
+        if ('content' in updateRecord && updateRecord.content) {
+            const content = updateRecord.content;
+            if ('text' in content && content.text) {
+                this.sessionOutputs.push(String(content.text));
+            }
+        }
+        // Check for tool call info
+        if ('name' in updateRecord && updateRecord.name) {
+            const status = 'status' in updateRecord ? updateRecord.status : 'unknown';
+            this.sessionOutputs.push(`[Tool: ${updateRecord.name} - ${status}]`);
+        }
+    }
+    /**
+     * Wrap a promise with a timeout.
+     */
+    withTimeout(promise, timeoutMs, label) {
+        return new Promise((resolve, reject) => {
+            const timer = setTimeout(() => {
+                reject(new Error(`${label} timed out after ${timeoutMs / 1000}s`));
+            }, timeoutMs);
+            promise.then((result) => {
+                clearTimeout(timer);
+                resolve(result);
+            }, (error) => {
+                clearTimeout(timer);
+                reject(error);
+            });
+        });
+    }
+    /**
+     * Cleanup the ACP connection and process.
+     */
+    async cleanup() {
+        if (this.process) {
+            this.process.kill();
+            this.process = null;
+        }
+        this.connection = null;
+        this.sessionId = null;
+    }
+}
+exports.AcpAgent = AcpAgent;
+//# sourceMappingURL=acp.js.map

dist/agents/codex.d.ts

@@ -1,5 +0,10 @@
-import { BaseAgent, CommandResult } from '../types';
+import { BaseAgent, CommandResult, AgentResult } from '../types';
 export declare class CodexAgent extends BaseAgent {
-    run(instruction: string, _workspacePath: string, runCommand: (cmd: string) => Promise<CommandResult>): Promise<string>;
+    /**
+     * Read API keys from ~/.codex/auth.json
+     * Returns environment variables to inject for codex CLI
+     */
+    private getCodexEnvVars;
+    run(instruction: string, _workspacePath: string, runCommand: (cmd: string, env?: Record<string, string>) => Promise<CommandResult>): Promise<AgentResult>;
 }
 //# sourceMappingURL=codex.d.ts.map

dist/agents/codex.js

 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.CodexAgent = void 0;
 const types_1 = require("../types");
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const os = __importStar(require("os"));
+/**
+ * Parse --json JSONL output from Codex CLI to extract structured info.
+ *
+ * JSONL events of interest:
+ *   - type=item.completed, item.type="command_execution" → tool/command usage
+ *   - type=item.completed, item.type="agent_message"     → agent text output
+ *   - type=turn.completed, usage                         → token counts
+ */
+function parseCodexJsonOutput(rawOutput) {
+    const lines = rawOutput.split('\n').filter(l => l.trim());
+    const toolsUsed = new Set();
+    const skillsTriggered = [];
+    const seenSkills = new Set();
+    const messageParts = [];
+    let inputTokens = 0;
+    let outputTokens = 0;
+    let numTurns = 0;
+    for (const line of lines) {
+        let event;
+        try {
+            event = JSON.parse(line);
+        }
+        catch {
+            continue;
+        }
+        if (event.type === 'item.completed' && event.item) {
+            const item = event.item;
+            // Agent text message
+            if (item.type === 'agent_message' && item.text) {
+                messageParts.push(item.text);
+            }
+            // Command execution → record as tool usage
+            if (item.type === 'command_execution') {
+                toolsUsed.add('command_execution');
+                // Detect skill file reads from commands
+                const cmd = item.command || '';
+                const skillMatch = cmd.match(/(?:\.claude\/skills|\.agents\/skills|\.codefuse\/fuse\/skills)\/([^/\s]+)/);
+                if (skillMatch) {
+                    const skillName = skillMatch[1];
+                    if (!seenSkills.has(`cmd:${skillName}`)) {
+                        seenSkills.add(`cmd:${skillName}`);
+                        skillsTriggered.push({
+                            name: skillName,
+                            source: 'file_read',
+                            timestamp: new Date().toISOString(),
+                            details: `Command referenced skill: ${cmd}`,
+                        });
+                    }
+                }
+            }
+            // Function call (if Codex supports tool_use style items in future)
+            if (item.type === 'tool_use' || item.type === 'function_call') {
+                const toolName = item.name || item.function?.name || 'unknown';
+                toolsUsed.add(toolName);
+            }
+        }
+        // Token usage from turn completion
+        if (event.type === 'turn.completed' && event.usage) {
+            numTurns++;
+            inputTokens += event.usage.input_tokens || 0;
+            outputTokens += event.usage.output_tokens || 0;
+        }
+    }
+    const finalOutput = messageParts.join('\n');
+    return {
+        output: finalOutput,
+        skills_triggered: skillsTriggered,
+        tools_used: Array.from(toolsUsed),
+        num_turns: numTurns || undefined,
+    };
+}
 class CodexAgent extends types_1.BaseAgent {
+    /**
+     * Read API keys from ~/.codex/auth.json
+     * Returns environment variables to inject for codex CLI
+     */
+    getCodexEnvVars() {
+        const authPath = path.join(os.homedir(), '.codex', 'auth.json');
+        const envVars = {};
+        try {
+            if (fs.existsSync(authPath)) {
+                const authContent = fs.readFileSync(authPath, 'utf-8');
+                const auth = JSON.parse(authContent);
+                // Map auth.json keys to environment variables
+                if (auth.OPENAI_API_KEY) {
+                    envVars.OPENAI_API_KEY = auth.OPENAI_API_KEY;
+                }
+                if (auth.ZENMUX_API_KEY) {
+                    envVars.ZENMUX_API_KEY = auth.ZENMUX_API_KEY;
+                }
+                // Also check for other common API key formats
+                for (const [key, value] of Object.entries(auth)) {
+                    if (key.endsWith('_API_KEY') && typeof value === 'string') {
+                        envVars[key] = value;
+                    }
+                }
+            }
+        }
+        catch (e) {
+            // Ignore errors - auth.json may not exist or be malformed
+        }
+        return envVars;
+    }
     async run(instruction, _workspacePath, runCommand) {
@@ -10,8 +148,22 @@ // Write instruction to a temp file to avoid shell escaping issues with long prompts
         await runCommand(`echo '${b64}' | base64 -d > /tmp/.prompt.md`);
-        const command = `codex --approval-mode full-auto "$(cat /tmp/.prompt.md)"`;
-        const result = await runCommand(command);
+        // Get API keys from auth.json
+        const envVars = this.getCodexEnvVars();
+        // Use --json for structured JSONL output, --ephemeral to avoid writing session files
+        // codex exec runs non-interactively; --full-auto enables sandboxed auto-execution
+        // --skip-git-repo-check allows running in non-git temp directories
+        const command = `cat /tmp/.prompt.md | codex exec --full-auto --skip-git-repo-check --json --ephemeral`;
+        const result = await runCommand(command, Object.keys(envVars).length > 0 ? envVars : undefined);
         if (result.exitCode !== 0) {
             console.error('CodexAgent: Codex CLI failed to execute correctly.');
         }
-        return result.stdout + '\n' + result.stderr;
+        // Parse JSONL events to extract structured info
+        const agentResult = parseCodexJsonOutput(result.stdout);
+        agentResult.raw_output = result.stdout.length > 256 * 1024
+            ? result.stdout.slice(0, 256 * 1024) + '\n... [truncated]'
+            : result.stdout;
+        // Fallback: if parsing didn't extract any output, use raw stdout+stderr
+        if (!agentResult.output) {
+            agentResult.output = result.stdout + '\n' + result.stderr;
+        }
+        return agentResult;
     }
@@ -18,0 +170,0 @@ }

dist/agents/registry.d.ts

@@ -8,8 +8,15 @@ /**
  *   - codex: OpenAI Codex CLI
+ *   - acp: Agent Client Protocol compatible agents
  */
 import { BaseAgent } from '../types';
+import { AcpAgentConfig } from './acp';
+/** Configuration for agent creation */
+export interface AgentConfig {
+    /** ACP-specific configuration */
+    acp?: AcpAgentConfig;
+}
 /** Get the list of supported agent names */
 export declare function getAgentNames(): string[];
 /** Create an agent instance by name. Throws if the name is unknown. */
-export declare function createAgent(name: string): BaseAgent;
+export declare function createAgent(name: string, config?: AgentConfig): BaseAgent;
 //# sourceMappingURL=registry.d.ts.map

dist/agents/registry.js

@@ -8,2 +8,3 @@ "use strict";
 const codex_1 = require("./codex");
+const acp_1 = require("./acp");
 /** Registry of available agent implementations */
@@ -14,2 +15,4 @@ const AGENT_REGISTRY = {
     codex: () => new codex_1.CodexAgent(),
+    // ACP agent requires config, registered as placeholder
+    acp: () => new acp_1.AcpAgent({ command: 'gemini --acp' }),
 };
@@ -21,3 +24,7 @@ /** Get the list of supported agent names */
 /** Create an agent instance by name. Throws if the name is unknown. */
-function createAgent(name) {
+function createAgent(name, config) {
+    // Handle ACP agent with custom configuration
+    if (name === 'acp' && config?.acp) {
+        return new acp_1.AcpAgent(config.acp);
+    }
     const factory = AGENT_REGISTRY[name];
@@ -24,0 +31,0 @@ if (!factory) {

dist/commands/run.d.ts

@@ -13,2 +13,3 @@ interface RunOptions {
     grader?: string;
+    acpCommand?: string;
 }
@@ -15,0 +16,0 @@ export declare function runEvals(dir: string, opts: RunOptions): Promise<void>;

dist/commands/run.js

@@ -155,2 +155,16 @@ "use strict";
         const providerName = opts.provider || resolved.provider;
+        // Build agent config (for ACP agent)
+        const agentConfig = {};
+        if (agentName === 'acp') {
+            // Use CLI flag > eval.yaml config > default
+            const acpCommand = opts.acpCommand || resolved.acp?.command;
+            if (!acpCommand) {
+                throw new Error('ACP agent requires a command. Specify via --acp-command or acp.command in eval.yaml');
+            }
+            agentConfig.acp = {
+                command: acpCommand,
+                env: resolved.acp?.env,
+                apiKey: env.GEMINI_API_KEY || env.ANTHROPIC_API_KEY || env.OPENAI_API_KEY,
+            };
+        }
         // Pick provider
@@ -186,3 +200,3 @@ const provider = providerName === 'docker'
             // Normal eval mode
-            const agent = (0, registry_1.createAgent)(agentName);
+            const agent = (0, registry_1.createAgent)(agentName, agentConfig);
             (0, cli_1.header)(resolved.name);
@@ -279,2 +293,9 @@ console.log(`    ${cli_1.fmt.dim('agent')} ${agentName}  ${cli_1.fmt.dim('provider')} ${providerName}  ${cli_1.fmt.dim('trials')} ${trials}${parallel > 1 ? `  ${cli_1.fmt.dim('parallel')} ${parallel}` : ''}`);
     }
+    else if (resolved.agent === 'acp') {
+        // For ACP agent, install gemini-cli as the default ACP-compatible agent
+        // Users can customize the command via acp.command in eval.yaml
+        // Note: ACP agent works best with --provider=local since it requires
+        // the ACP command to be available in the host environment
+        dockerfileContent += `RUN npm install -g @google/gemini-cli\n\n`;
+    }
     // Docker setup commands
@@ -281,0 +302,0 @@ if (resolved.docker.setup) {

dist/core/config.js

@@ -87,2 +87,13 @@ "use strict";
     const version = raw.version || '1';
+    // Handle ACP config
+    let acp;
+    if (raw.defaults?.acp) {
+        if (!raw.defaults.acp.command) {
+            throw new Error('eval.yaml: acp.command is required when using ACP agent');
+        }
+        acp = {
+            command: raw.defaults.acp.command,
+            env: raw.defaults.acp.env,
+        };
+    }
     const defaults = {
@@ -100,2 +111,6 @@ ...DEFAULT_CONFIG,
     };
+    // Add ACP config if present
+    if (acp) {
+        defaults.acp = acp;
+    }
     if (!raw.tasks || !Array.isArray(raw.tasks) || raw.tasks.length === 0) {
@@ -162,2 +177,3 @@ throw new Error('eval.yaml must have at least one task in the "tasks" array');
     const grader_model = task.grader_model || defaults.grader_model;
+    const acp = defaults.acp; // ACP config is only at defaults level
     // Resolve instruction — could be inline text or file path
@@ -196,2 +212,3 @@ const instruction = await resolveFileOrInline(task.instruction, baseDir);
         grader_model,
+        acp,
         docker,
@@ -198,0 +215,0 @@ environment,

dist/core/config.types.d.ts

@@ -32,2 +32,9 @@ /**
 }
+/** ACP (Agent Client Protocol) configuration */
+export interface AcpConfig {
+    /** Command to start the ACP agent (e.g., "gemini --acp") */
+    command: string;
+    /** Optional environment variables for the ACP process */
+    env?: Record<string, string>;
+}
 /** Single eval task */
@@ -56,2 +63,3 @@ export interface EvalTaskConfig {
     grader_model?: string;
+    acp?: AcpConfig;
     docker: DockerConfig;
@@ -79,2 +87,3 @@ environment: EnvironmentConfig;
     grader_model?: string;
+    acp?: AcpConfig;
     docker: DockerConfig;
@@ -81,0 +90,0 @@ environment: EnvironmentConfig;

dist/evalRunner.js

@@ -73,2 +73,13 @@ "use strict";
 }
+/** Normalize agent output to AgentResult format */
+function normalizeAgentOutput(raw) {
+    if (typeof raw === 'string') {
+        return {
+            output: raw,
+            skills_triggered: [],
+            tools_used: [],
+        };
+    }
+    return raw;
+}
 class EvalRunner {
@@ -172,7 +183,8 @@ provider;
             const agentTimeoutMs = opts.timeoutSec * 1000;
-            const agentLogs = await withTimeout(agent.run(instruction, workspace, loggedRunCommand), agentTimeoutMs, `Agent (limit: ${opts.timeoutSec}s)`);
+            const agentRaw = await withTimeout(agent.run(instruction, workspace, loggedRunCommand), agentTimeoutMs, `Agent (limit: ${opts.timeoutSec}s)`);
+            const agentResult = normalizeAgentOutput(agentRaw);
             sessionLog.push({
                 type: 'agent_result',
                 timestamp: this.timestamp(),
-                output: agentLogs
+                output: agentResult.output
             });
@@ -179,0 +191,0 @@ // Run all graders

dist/skillgrade.js

@@ -130,2 +130,3 @@ #!/usr/bin/env node
         output: outputDir,
+        acpCommand: getFlag('acp-command'),
     });
@@ -156,4 +157,5 @@ if (openPreview) {
     --parallel=N       Run trials concurrently
-    --agent=gemini|claude|codex   Override agent (default: auto-detect from API key)
+    --agent=gemini|claude|codex|acp   Override agent (default: auto-detect from API key)
     --provider=docker|local Override provider (default: docker)
+    --acp-command=CMD  ACP agent command (e.g., "gemini --acp")
     --output=DIR       Output directory for reports and temp files
@@ -174,2 +176,3 @@ Default: $TMPDIR/skillgrade
     skillgrade --regression --ci   # CI regression with 30 trials
+    skillgrade --agent=acp --acp-command="gemini --acp"  # use ACP-compatible agent
     skillgrade preview browser     # open web UI
@@ -176,0 +179,0 @@ `);

dist/types.d.ts

@@ -40,3 +40,22 @@ export interface CommandResult {
     session_log: LogEntry[];
+    skills_triggered?: SkillTriggerInfo[];
+    tools_used?: string[];
 }
+/** Skill trigger information during agent execution */
+export interface SkillTriggerInfo {
+    name: string;
+    source: 'tool_use' | 'file_read' | 'init_list';
+    timestamp?: string;
+    details?: string;
+}
+/** Structured agent execution result */
+export interface AgentResult {
+    output: string;
+    raw_output?: string;
+    skills_triggered: SkillTriggerInfo[];
+    tools_used: string[];
+    num_turns?: number;
+    duration_api_ms?: number;
+    cost_usd?: number;
+}
 export interface EvalReport {
@@ -51,3 +70,3 @@ task: string;
 export declare abstract class BaseAgent {
-    abstract run(instruction: string, workspacePath: string, runCommand: (cmd: string) => Promise<CommandResult>): Promise<string>;
+    abstract run(instruction: string, workspacePath: string, runCommand: (cmd: string, env?: Record<string, string>) => Promise<CommandResult>): Promise<string | AgentResult>;
 }
@@ -54,0 +73,0 @@ /** Options passed to environment providers for setup */

package.json

 {
   "name": "skillgrade",
-  "version": "0.1.3",
+  "version": "0.1.4",
   "description": "The easiest way to evaluate your Agent Skills — test that AI agents correctly discover and use your skills",
@@ -64,2 +64,3 @@ "main": "dist/skillgrade.js",
   "dependencies": {
+    "@agentclientprotocol/sdk": "^0.19.0",
     "dockerode": "^4.0.9",
@@ -70,2 +71,2 @@ "fs-extra": "^11.3.3",
   }
-}
+}

README.md

@@ -62,4 +62,5 @@ # Skillgrade
 | `--parallel=N` | Run trials concurrently |
-| `--agent=gemini\|claude\|codex` | Override agent (default: auto-detect from API key) |
+| `--agent=gemini\|claude\|codex\|acp` | Override agent (default: auto-detect from API key) |
 | `--provider=docker\|local` | Override provider |
+| `--acp-command=CMD` | ACP agent command (e.g., `gemini --acp`) |
 | `--output=DIR` | Output directory (default: `$TMPDIR/skillgrade`) |
@@ -80,3 +81,3 @@ | `--validate` | Verify graders using reference solutions |
 defaults:
-  agent: gemini          # gemini | claude | codex
+  agent: gemini          # gemini | claude | codex | acp
   provider: docker       # docker | local
@@ -87,2 +88,6 @@ trials: 5
   grader_model: gemini-3-flash-preview  # default LLM grader model
+  acp:                   # ACP agent configuration (optional)
+    command: gemini --acp  # command to start ACP-compatible agent
+    env:                  # optional environment variables
+      DEBUG: "1"
   docker:
@@ -238,2 +243,47 @@ base: node:20-slim
 
+## ACP Agent
+
+[Agent Client Protocol (ACP)](https://agentclientprotocol.com/) is an open protocol that standardizes communication between AI coding agents and clients. Using an ACP-compatible agent allows you to evaluate skills without managing API keys directly.
+
+### Quick Start
+
+```bash
+# Use Gemini CLI in ACP mode (requires gemini CLI installed)
+skillgrade --agent=acp --acp-command="gemini --acp"
+
+# Or configure in eval.yaml
+```
+
+```yaml
+defaults:
+  agent: acp
+  acp:
+    command: gemini --acp
+```
+
+### ACP-Compatible Agents
+
+Any agent that supports the ACP protocol can be used:
+
+| Agent | Command |
+|-------|---------|
+| Gemini CLI | `gemini --acp` |
+| Other ACP agents | Check agent documentation |
+
+### How It Works
+
+1. skillgrade starts the ACP agent as a subprocess
+2. Communication happens via JSON-RPC 2.0 over stdio
+3. No API key required — authentication is handled by the ACP agent
+4. Works best with `--provider=local` since the ACP agent needs to be available in your environment
+
+### CLI Options
+
+| Flag | Description |
+|------|-------------|
+| `--agent=acp` | Use ACP-compatible agent |
+| `--acp-command=CMD` | Command to start the ACP agent |
+
+The `--acp-command` can also be set in `eval.yaml` under `defaults.acp.command`.
+
 ## Best Practices
@@ -240,0 +290,0 @@

New alerts

Filesystem access

Supply chain risk

Accesses the file system, and could potentially read sensitive data.

Long strings

Supply chain risk

Contains long string literals, which may be a sign of obfuscated or packed code.

URL strings

Supply chain risk

Package contains fragments of external URLs or IP addresses, which the package may be accessing at runtime.

Fixed alerts

AI-detected potential code anomaly

Supply chain risk

AI has identified unusual behaviors that may pose a security risk.

Long strings

Supply chain risk

Contains long string literals, which may be a sign of obfuscated or packed code.

URL strings

Supply chain risk

Package contains fragments of external URLs or IP addresses, which the package may be accessing at runtime.

Improved metrics

Total package byte prevSize

183372

14.34%

Number of package files

50

4.17%

Lines of code

3610

18.32%

Number of lines in readme file

304

19.69%

Worsened metrics

Dependency count

5

25%

Number of low supply chain risk alerts

21

5%

Number of medium supply chain risk alerts

8

14.29%

Dependency changes

• + Added

  @agentclientprotocol/sdk@^0.19.0

• + Added

  @agentclientprotocol/sdk@0.19.2
  (transitive)

• + Added

  zod@4.4.3
  (transitive)