skillgrade
Advanced tools
| import { BaseAgent, CommandResult } from '../types'; | ||
| export interface OpenCodeAgentConfig { | ||
| agent?: string; | ||
| model?: string; | ||
| } | ||
| export declare class OpenCodeAgent extends BaseAgent { | ||
| private config; | ||
| constructor(config?: OpenCodeAgentConfig); | ||
| run(instruction: string, _workspacePath: string, runCommand: (cmd: string) => Promise<CommandResult>): Promise<string>; | ||
| } | ||
| //# sourceMappingURL=opencode.d.ts.map |
| "use strict"; | ||
| Object.defineProperty(exports, "__esModule", { value: true }); | ||
| exports.OpenCodeAgent = void 0; | ||
| const types_1 = require("../types"); | ||
| class OpenCodeAgent extends types_1.BaseAgent { | ||
| config; | ||
| constructor(config = {}) { | ||
| super(); | ||
| this.config = config; | ||
| } | ||
| async run(instruction, _workspacePath, runCommand) { | ||
| const fullCommand = `cd '${_workspacePath.replace(/'/g, "'\\''")}' && echo '${instruction.replace(/'/g, "'\\''")}' | opencode run ${this.config.model ? `--model ${this.config.model}` : ''}`; | ||
| const result = await runCommand(fullCommand); | ||
| return result.stdout + '\n' + result.stderr; | ||
| } | ||
| } | ||
| exports.OpenCodeAgent = OpenCodeAgent; | ||
| //# sourceMappingURL=opencode.js.map |
@@ -9,5 +9,7 @@ /** | ||
| * - acp: Agent Client Protocol compatible agents | ||
| * - opencode: OpenCode AI coding agent | ||
| */ | ||
| import { BaseAgent } from '../types'; | ||
| import { AcpAgentConfig } from './acp'; | ||
| import { OpenCodeAgentConfig } from './opencode'; | ||
| /** Configuration for agent creation */ | ||
@@ -17,2 +19,4 @@ export interface AgentConfig { | ||
| acp?: AcpAgentConfig; | ||
| /** OpenCode-specific configuration */ | ||
| opencode?: OpenCodeAgentConfig; | ||
| } | ||
@@ -19,0 +23,0 @@ /** Get the list of supported agent names */ |
@@ -9,2 +9,3 @@ "use strict"; | ||
| const acp_1 = require("./acp"); | ||
| const opencode_1 = require("./opencode"); | ||
| /** Registry of available agent implementations */ | ||
@@ -16,3 +17,4 @@ const AGENT_REGISTRY = { | ||
| // ACP agent requires config, registered as placeholder | ||
| acp: () => new acp_1.AcpAgent({ command: 'gemini --acp' }), | ||
| acp: (config) => new acp_1.AcpAgent(config?.acp || { command: 'gemini --acp' }), | ||
| opencode: (config) => new opencode_1.OpenCodeAgent(config?.opencode || {}), | ||
| }; | ||
@@ -25,6 +27,2 @@ /** Get the list of supported agent names */ | ||
| function createAgent(name, config) { | ||
| // Handle ACP agent with custom configuration | ||
| if (name === 'acp' && config?.acp) { | ||
| return new acp_1.AcpAgent(config.acp); | ||
| } | ||
| const factory = AGENT_REGISTRY[name]; | ||
@@ -35,4 +33,4 @@ if (!factory) { | ||
| } | ||
| return factory(); | ||
| return factory(config); | ||
| } | ||
| //# sourceMappingURL=registry.js.map |
@@ -14,2 +14,4 @@ interface RunOptions { | ||
| acpCommand?: string; | ||
| openCodeAgent?: string; | ||
| openCodeModel?: string; | ||
| } | ||
@@ -16,0 +18,0 @@ export declare function runEvals(dir: string, opts: RunOptions): Promise<void>; |
+28
-11
@@ -135,2 +135,3 @@ "use strict"; | ||
| graderModel: resolved.grader_model, | ||
| graderProvider: resolved.grader_provider, | ||
| environment: resolved.environment, | ||
@@ -156,6 +157,5 @@ }; | ||
| const providerName = opts.provider || resolved.provider; | ||
| // Build agent config (for ACP agent) | ||
| // Build agent config | ||
| const agentConfig = {}; | ||
| if (agentName === 'acp') { | ||
| // Use CLI flag > eval.yaml config > default | ||
| const acpCommand = opts.acpCommand || resolved.acp?.command; | ||
@@ -171,2 +171,11 @@ if (!acpCommand) { | ||
| } | ||
| else if (agentName === 'opencode') { | ||
| agentConfig.opencode = {}; | ||
| if (opts.openCodeAgent) { | ||
| agentConfig.opencode.agent = opts.openCodeAgent; | ||
| } | ||
| if (opts.openCodeModel) { | ||
| agentConfig.opencode.model = opts.openCodeModel; | ||
| } | ||
| } | ||
| // Pick provider | ||
@@ -295,8 +304,7 @@ const provider = providerName === 'docker' | ||
| else if (resolved.agent === 'acp') { | ||
| // For ACP agent, install gemini-cli as the default ACP-compatible agent | ||
| // Users can customize the command via acp.command in eval.yaml | ||
| // Note: ACP agent works best with --provider=local since it requires | ||
| // the ACP command to be available in the host environment | ||
| dockerfileContent += `RUN npm install -g @google/gemini-cli\n\n`; | ||
| } | ||
| else if (resolved.agent === 'opencode') { | ||
| dockerfileContent += `RUN npm install -g opencode\n\n`; | ||
| } | ||
| // Docker setup commands | ||
@@ -315,9 +323,18 @@ if (resolved.docker.setup) { | ||
| const srcPath = path.resolve(baseDir, w.src); | ||
| const destInTmp = path.join(tmpDir, path.basename(w.src)); | ||
| if (await fs.pathExists(srcPath)) { | ||
| await fs.copy(srcPath, destInTmp); | ||
| dockerfileContent += `COPY ${path.basename(w.src)} ${w.dest}\n`; | ||
| if (w.chmod) { | ||
| dockerfileContent += `RUN chmod ${w.chmod} ${w.dest}\n`; | ||
| if (resolved.provider === 'local') { | ||
| // For local provider: copy directly to destination path in tmpDir | ||
| const destPath = path.join(tmpDir, w.dest); | ||
| await fs.ensureDir(path.dirname(destPath)); | ||
| await fs.copy(srcPath, destPath); | ||
| } | ||
| else { | ||
| // For Docker: copy to tmpDir root, Dockerfile will place it | ||
| const destInTmp = path.join(tmpDir, path.basename(w.src)); | ||
| await fs.copy(srcPath, destInTmp); | ||
| dockerfileContent += `COPY ${path.basename(w.src)} ${w.dest}\n`; | ||
| if (w.chmod) { | ||
| dockerfileContent += `RUN chmod ${w.chmod} ${w.dest}\n`; | ||
| } | ||
| } | ||
| } | ||
@@ -324,0 +341,0 @@ } |
+28
-8
@@ -114,2 +114,7 @@ "use strict"; | ||
| } | ||
| // Validate grader_provider | ||
| const validGraderProviders = ['gemini', 'anthropic', 'openai']; | ||
| if (defaults.grader_provider && !validGraderProviders.includes(defaults.grader_provider)) { | ||
| throw new Error(`eval.yaml: grader_provider must be one of ${validGraderProviders.join(', ')}, got "${defaults.grader_provider}"`); | ||
| } | ||
| if (!raw.tasks || !Array.isArray(raw.tasks) || raw.tasks.length === 0) { | ||
@@ -126,2 +131,5 @@ throw new Error('eval.yaml must have at least one task in the "tasks" array'); | ||
| } | ||
| if (t.grader_provider && !validGraderProviders.includes(t.grader_provider)) { | ||
| throw new Error(`Task "${t.name}" has invalid grader_provider "${t.grader_provider}", must be one of ${validGraderProviders.join(', ')}`); | ||
| } | ||
| const workspace = (t.workspace || []).map((w) => { | ||
@@ -141,10 +149,16 @@ if (typeof w === 'string') { | ||
| workspace, | ||
| graders: t.graders.map((g) => ({ | ||
| type: g.type, | ||
| setup: g.setup, | ||
| run: g.run, | ||
| rubric: g.rubric, | ||
| model: g.model, | ||
| weight: g.weight ?? 1.0, | ||
| })), | ||
| graders: t.graders.map((g) => { | ||
| if (g.provider && !validGraderProviders.includes(g.provider)) { | ||
| throw new Error(`Task "${t.name}" grader has invalid provider "${g.provider}", must be one of ${validGraderProviders.join(', ')}`); | ||
| } | ||
| return { | ||
| type: g.type, | ||
| setup: g.setup, | ||
| run: g.run, | ||
| rubric: g.rubric, | ||
| model: g.model, | ||
| provider: g.provider, | ||
| weight: g.weight ?? 1.0, | ||
| }; | ||
| }), | ||
| solution: t.solution, | ||
@@ -155,3 +169,6 @@ agent: t.agent, | ||
| timeout: t.timeout, | ||
| grader_model: t.grader_model, | ||
| grader_provider: t.grader_provider, | ||
| docker: t.docker, | ||
| environment: t.environment, | ||
| }; | ||
@@ -179,2 +196,3 @@ }); | ||
| const grader_model = task.grader_model || defaults.grader_model; | ||
| const grader_provider = task.grader_provider || defaults.grader_provider; | ||
| const acp = defaults.acp; // ACP config is only at defaults level | ||
@@ -189,2 +207,3 @@ // Resolve instruction — could be inline text or file path | ||
| model: g.model, | ||
| provider: g.provider, | ||
| weight: g.weight, | ||
@@ -215,2 +234,3 @@ }; | ||
| grader_model, | ||
| grader_provider, | ||
| acp, | ||
@@ -217,0 +237,0 @@ docker, |
@@ -20,2 +20,3 @@ /** | ||
| model?: string; | ||
| provider?: 'gemini' | 'anthropic' | 'openai'; | ||
| weight: number; | ||
@@ -52,2 +53,3 @@ } | ||
| grader_model?: string; | ||
| grader_provider?: 'gemini' | 'anthropic' | 'openai'; | ||
| docker?: DockerConfig; | ||
@@ -64,2 +66,3 @@ environment?: Partial<EnvironmentConfig>; | ||
| grader_model?: string; | ||
| grader_provider?: 'gemini' | 'anthropic' | 'openai'; | ||
| acp?: AcpConfig; | ||
@@ -88,2 +91,3 @@ docker: DockerConfig; | ||
| grader_model?: string; | ||
| grader_provider?: 'gemini' | 'anthropic' | 'openai'; | ||
| acp?: AcpConfig; | ||
@@ -99,4 +103,5 @@ docker: DockerConfig; | ||
| model?: string; | ||
| provider?: 'gemini' | 'anthropic' | 'openai'; | ||
| weight: number; | ||
| } | ||
| //# sourceMappingURL=config.types.d.ts.map |
@@ -9,2 +9,3 @@ import { BaseAgent, EnvironmentProvider, EvalReport } from './types'; | ||
| graderModel?: string; | ||
| graderProvider?: 'gemini' | 'anthropic' | 'openai'; | ||
| graderTimeoutSec?: number; | ||
@@ -11,0 +12,0 @@ environment: { |
@@ -207,2 +207,3 @@ "use strict"; | ||
| model: graderDef.model || opts.graderModel, | ||
| provider: graderDef.provider || opts.graderProvider, | ||
| weight: graderDef.weight, | ||
@@ -209,0 +210,0 @@ }; |
@@ -20,8 +20,18 @@ import { GraderConfig, GraderResult, EnvironmentProvider } from '../types'; | ||
| * Uses an LLM to evaluate the agent's session transcript against a rubric. | ||
| * Requires GEMINI_API_KEY or ANTHROPIC_API_KEY in the environment. | ||
| * | ||
| * Supported providers (selected via `config.provider`, defaults to "gemini"): | ||
| * - gemini → Google Gemini (GEMINI_API_KEY) | ||
| * - anthropic → Anthropic Claude or compatible (ANTHROPIC_API_KEY; optional ANTHROPIC_BASE_URL) | ||
| * - openai → OpenAI or compatible (OPENAI_API_KEY; optional OPENAI_BASE_URL for Ollama, vLLM, etc.) | ||
| * | ||
| * Each provider method resolves its own API key, makes the HTTP call, and | ||
| * handles errors — returning a zero-score GraderResult on any failure. | ||
| */ | ||
| export declare class LLMGrader implements Grader { | ||
| /** Default models when no model override is configured. */ | ||
| private static readonly DEFAULT_MODELS; | ||
| grade(_workspace: string, _provider: EnvironmentProvider, config: GraderConfig, taskPath: string, sessionLog: any[], env?: Record<string, string>): Promise<GraderResult>; | ||
| private callGemini; | ||
| private callAnthropic; | ||
| private callOpenAI; | ||
| private parseResponse; | ||
@@ -28,0 +38,0 @@ } |
+121
-38
@@ -94,5 +94,18 @@ "use strict"; | ||
| * Uses an LLM to evaluate the agent's session transcript against a rubric. | ||
| * Requires GEMINI_API_KEY or ANTHROPIC_API_KEY in the environment. | ||
| * | ||
| * Supported providers (selected via `config.provider`, defaults to "gemini"): | ||
| * - gemini → Google Gemini (GEMINI_API_KEY) | ||
| * - anthropic → Anthropic Claude or compatible (ANTHROPIC_API_KEY; optional ANTHROPIC_BASE_URL) | ||
| * - openai → OpenAI or compatible (OPENAI_API_KEY; optional OPENAI_BASE_URL for Ollama, vLLM, etc.) | ||
| * | ||
| * Each provider method resolves its own API key, makes the HTTP call, and | ||
| * handles errors — returning a zero-score GraderResult on any failure. | ||
| */ | ||
| class LLMGrader { | ||
| /** Default models when no model override is configured. */ | ||
| static DEFAULT_MODELS = { | ||
| gemini: 'gemini-3-flash-preview', | ||
| anthropic: 'claude-sonnet-4-20250514', | ||
| openai: 'gpt-4o', | ||
| }; | ||
| async grade(_workspace, _provider, config, taskPath, sessionLog, env) { | ||
@@ -147,20 +160,30 @@ const rubricPath = path.join(taskPath, config.rubric || 'prompts/quality.md'); | ||
| Respond with ONLY a JSON object: {"score": <number>, "reasoning": "<brief explanation>"}`; | ||
| // Try Gemini API first, fall back to Anthropic | ||
| const providerName = config.provider || 'gemini'; | ||
| const model = config.model || LLMGrader.DEFAULT_MODELS[providerName] || 'gemini-3-flash-preview'; | ||
| switch (providerName) { | ||
| case "gemini": | ||
| return this.callGemini(prompt, model, config, env); | ||
| case "anthropic": | ||
| return this.callAnthropic(prompt, model, config, env); | ||
| case "openai": | ||
| return this.callOpenAI(prompt, model, config, env); | ||
| default: | ||
| return { | ||
| grader_type: 'llm_rubric', | ||
| score: 0, | ||
| weight: config.weight, | ||
| details: `Unknown grader provider: "${providerName}". Supported: gemini, anthropic, openai`, | ||
| }; | ||
| } | ||
| } | ||
| async callGemini(prompt, model, config, env) { | ||
| const apiKey = env?.GEMINI_API_KEY || process.env.GEMINI_API_KEY; | ||
| const anthropicKey = env?.ANTHROPIC_API_KEY || process.env.ANTHROPIC_API_KEY; | ||
| if (apiKey) { | ||
| return this.callGemini(prompt, apiKey, config); | ||
| if (!apiKey) { | ||
| return { | ||
| grader_type: 'llm_rubric', | ||
| score: 0, | ||
| weight: config.weight, | ||
| details: 'Missing GEMINI_API_KEY. Set the GEMINI_API_KEY environment variable to use the "gemini" grader provider.' | ||
| }; | ||
| } | ||
| else if (anthropicKey) { | ||
| return this.callAnthropic(prompt, anthropicKey, config); | ||
| } | ||
| return { | ||
| grader_type: 'llm_rubric', | ||
| score: 0, | ||
| weight: config.weight, | ||
| details: 'No API key available for LLM grading (set GEMINI_API_KEY or ANTHROPIC_API_KEY)' | ||
| }; | ||
| } | ||
| async callGemini(prompt, apiKey, config) { | ||
| const model = config.model || 'gemini-3-flash-preview'; | ||
| const url = `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent?key=${apiKey}`; | ||
@@ -184,6 +207,16 @@ try { | ||
| } | ||
| async callAnthropic(prompt, apiKey, config) { | ||
| const model = config.model || 'claude-sonnet-4-20250514'; | ||
| async callAnthropic(prompt, model, config, env) { | ||
| const apiKey = env?.ANTHROPIC_API_KEY || process.env.ANTHROPIC_API_KEY; | ||
| if (!apiKey) { | ||
| return { | ||
| grader_type: 'llm_rubric', | ||
| score: 0, | ||
| weight: config.weight, | ||
| details: 'Missing ANTHROPIC_API_KEY. Set the ANTHROPIC_API_KEY environment variable to use the "anthropic" grader provider.' | ||
| }; | ||
| } | ||
| const baseUrl = (env?.ANTHROPIC_BASE_URL || process.env.ANTHROPIC_BASE_URL || 'https://api.anthropic.com/v1').replace(/\/+$/, ''); | ||
| const url = `${baseUrl}/messages`; | ||
| try { | ||
| const response = await fetch('https://api.anthropic.com/v1/messages', { | ||
| const response = await fetch(url, { | ||
| method: 'POST', | ||
@@ -209,9 +242,43 @@ headers: { | ||
| } | ||
| async callOpenAI(prompt, model, config, env) { | ||
| const apiKey = env?.OPENAI_API_KEY || process.env.OPENAI_API_KEY; | ||
| if (!apiKey) { | ||
| return { | ||
| grader_type: 'llm_rubric', | ||
| score: 0, | ||
| weight: config.weight, | ||
| details: 'Missing OPENAI_API_KEY. Set the OPENAI_API_KEY environment variable to use the "openai" grader provider.' | ||
| }; | ||
| } | ||
| const baseUrl = (env?.OPENAI_BASE_URL || process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1').replace(/\/+$/, ''); | ||
| const url = `${baseUrl}/chat/completions`; | ||
| try { | ||
| const response = await fetch(url, { | ||
| method: 'POST', | ||
| headers: { | ||
| 'Content-Type': 'application/json', | ||
| 'Authorization': `Bearer ${apiKey}`, | ||
| }, | ||
| body: JSON.stringify({ | ||
| model, | ||
| temperature: 0, | ||
| max_tokens: 4096, | ||
| messages: [{ role: 'user', content: prompt }], | ||
| }), | ||
| }); | ||
| const data = await response.json(); | ||
| const text = data?.choices?.[0]?.message?.content || ''; | ||
| return this.parseResponse(text, config); | ||
| } | ||
| catch (e) { | ||
| return { grader_type: 'llm_rubric', score: 0, weight: config.weight, details: `OpenAI API error: ${e}` }; | ||
| } | ||
| } | ||
| parseResponse(text, config) { | ||
| try { | ||
| // Strip markdown code fences if present | ||
| let cleaned = text.replace(/```(?:json)?\s*/g, '').replace(/```/g, '').trim(); | ||
| // Extract JSON from response | ||
| const jsonMatch = cleaned.match(/\{[\s\S]*\}/); | ||
| if (jsonMatch) { | ||
| // Strip markdown code fences if present | ||
| let cleaned = text.replace(/```(?:json)?\s*/g, '').replace(/```/g, '').trim(); | ||
| // Try to extract and parse JSON | ||
| const jsonMatch = cleaned.match(/\{[\s\S]*\}/); | ||
| if (jsonMatch) { | ||
| try { | ||
| const parsed = JSON.parse(jsonMatch[0]); | ||
@@ -226,16 +293,32 @@ const score = Math.max(0, Math.min(1, parseFloat(parsed.score) || 0)); | ||
| } | ||
| } | ||
| catch (e) { | ||
| // JSON parse failed — try to extract score from truncated response | ||
| const scoreMatch = text.match(/"score"\s*:\s*([\d.]+)/); | ||
| if (scoreMatch) { | ||
| const score = Math.max(0, Math.min(1, parseFloat(scoreMatch[1]) || 0)); | ||
| return { | ||
| grader_type: 'llm_rubric', | ||
| score, | ||
| weight: config.weight, | ||
| details: 'Parsed score from truncated LLM response' | ||
| }; | ||
| catch (e) { | ||
| // JSON parse failed, likely truncated or malformed - try to extract score anyway | ||
| const scoreMatch = jsonMatch[0].match(/"score"\s*:\s*([\d.]+)/); | ||
| if (scoreMatch) { | ||
| const score = Math.max(0, Math.min(1, parseFloat(scoreMatch[1]) || 0)); | ||
| // Try to extract partial reasoning if available | ||
| const reasoningMatch = jsonMatch[0].match(/"reasoning"\s*:\s*"([^"]*)/); | ||
| const reasoning = reasoningMatch | ||
| ? reasoningMatch[1] + '... (response truncated)' | ||
| : 'Score extracted from incomplete LLM response'; | ||
| return { | ||
| grader_type: 'llm_rubric', | ||
| score, | ||
| weight: config.weight, | ||
| details: reasoning | ||
| }; | ||
| } | ||
| } | ||
| } | ||
| // No JSON found at all - try to extract score from plain text | ||
| const scoreMatch = text.match(/"score"\s*:\s*([\d.]+)|score[:\s]+(\d+\.?\d*)/i); | ||
| if (scoreMatch) { | ||
| const score = Math.max(0, Math.min(1, parseFloat(scoreMatch[1] || scoreMatch[2]) || 0)); | ||
| return { | ||
| grader_type: 'llm_rubric', | ||
| score, | ||
| weight: config.weight, | ||
| details: 'Score extracted from malformed LLM response' | ||
| }; | ||
| } | ||
| return { grader_type: 'llm_rubric', score: 0, weight: config.weight, details: `Failed to parse LLM response: ${text.substring(0, 200)}` }; | ||
@@ -242,0 +325,0 @@ } |
@@ -131,2 +131,4 @@ #!/usr/bin/env node | ||
| acpCommand: getFlag('acp-command'), | ||
| openCodeAgent: getFlag('opencode-agent'), | ||
| openCodeModel: getFlag('opencode-model'), | ||
| }); | ||
@@ -157,5 +159,7 @@ if (openPreview) { | ||
| --parallel=N Run trials concurrently | ||
| --agent=gemini|claude|codex|acp Override agent (default: auto-detect from API key) | ||
| --agent=gemini|claude|codex|acp|opencode Override agent (default: auto-detect from API key) | ||
| --provider=docker|local Override provider (default: docker) | ||
| --acp-command=CMD ACP agent command (e.g., "gemini --acp") | ||
| --opencode-agent=NAME OpenCode agent (build|plan|explore) | ||
| --opencode-model=MODEL OpenCode model (provider/model format) | ||
| --output=DIR Output directory for reports and temp files | ||
@@ -162,0 +166,0 @@ Default: $TMPDIR/skillgrade |
+1
-0
@@ -11,2 +11,3 @@ export interface CommandResult { | ||
| model?: string; | ||
| provider?: 'gemini' | 'anthropic' | 'openai'; | ||
| weight: number; | ||
@@ -13,0 +14,0 @@ } |
+1
-1
| { | ||
| "name": "skillgrade", | ||
| "version": "0.1.4", | ||
| "version": "0.1.5", | ||
| "description": "The easiest way to evaluate your Agent Skills — test that AI agents correctly discover and use your skills", | ||
@@ -5,0 +5,0 @@ "main": "dist/skillgrade.js", |
+70
-5
@@ -62,5 +62,7 @@ # Skillgrade | ||
| | `--parallel=N` | Run trials concurrently | | ||
| | `--agent=gemini\|claude\|codex\|acp` | Override agent (default: auto-detect from API key) | | ||
| | `--agent=gemini\|claude\|codex\|acp\|opencode` | Override agent (default: auto-detect from API key) | | ||
| | `--provider=docker\|local` | Override provider | | ||
| | `--acp-command=CMD` | ACP agent command (e.g., `gemini --acp`) | | ||
| | `--opencode-agent=NAME` | OpenCode agent (build\|plan\|explore) | | ||
| | `--opencode-model=MODEL` | OpenCode model (provider/model format) | | ||
| | `--output=DIR` | Output directory (default: `$TMPDIR/skillgrade`) | | ||
@@ -87,2 +89,3 @@ | `--validate` | Verify graders using reference solutions | | ||
| grader_model: gemini-3-flash-preview # default LLM grader model | ||
| grader_provider: gemini # default LLM grader provider: gemini | anthropic | openai | ||
| acp: # ACP agent configuration (optional) | ||
@@ -120,2 +123,3 @@ command: gemini --acp # command to start ACP-compatible agent | ||
| Did the agent follow the check → fix → verify workflow? | ||
| provider: gemini # optional: gemini (default) | anthropic | openai | ||
| model: gemini-2.0-flash # optional model override | ||
@@ -126,2 +130,3 @@ weight: 0.3 | ||
| agent: claude | ||
| grader_provider: anthropic # override default LLM grader provider | ||
| trials: 10 | ||
@@ -199,7 +204,16 @@ timeout: 600 | ||
| weight: 0.3 | ||
| provider: gemini # gemini (default) | anthropic | openai | ||
| model: gemini-2.0-flash # optional, auto-detected from API key | ||
| ``` | ||
| Uses Gemini or Anthropic based on available API key. Override with the `model` field. | ||
| The `provider` field selects which LLM API to call: | ||
| | Provider | API Key Env Var | Base URL Env Var (optional) | Default Model | | ||
| |------------|---------------------|-----------------------------|----------------------------| | ||
| | `gemini` | `GEMINI_API_KEY` | - | `gemini-3-flash-preview` | | ||
| | `anthropic`| `ANTHROPIC_API_KEY` | `ANTHROPIC_BASE_URL` | `claude-sonnet-4-20250514` | | ||
| | `openai` | `OPENAI_API_KEY` | `OPENAI_BASE_URL` | `gpt-4o` | | ||
| `ANTHROPIC_BASE_URL` and `OPENAI_BASE_URL` enable custom/self-hosted endpoints (Ollama, vLLM, etc.). | ||
| ### Combining Graders | ||
@@ -239,8 +253,59 @@ | ||
| |----------|---------| | ||
| | `GEMINI_API_KEY` | Agent execution, LLM grading, `skillgrade init` | | ||
| | `ANTHROPIC_API_KEY` | Agent execution, LLM grading, `skillgrade init` | | ||
| | `OPENAI_API_KEY` | Agent execution (Codex), `skillgrade init` | | ||
| | `GEMINI_API_KEY` | Agent execution, LLM grading (`provider: gemini`), `skillgrade init` | | ||
| | `ANTHROPIC_API_KEY` | Agent execution, LLM grading (`provider: anthropic`), `skillgrade init` | | ||
| | `OPENAI_API_KEY` | Agent execution (Codex), LLM grading (`provider: openai`), `skillgrade init` | | ||
| | `ANTHROPIC_BASE_URL` | LLM grading (`provider: anthropic`) — custom Anthropic-compatible endpoint | | ||
| | `OPENAI_BASE_URL` | LLM grading (`provider: openai`) — custom OpenAI-compatible endpoint (Ollama, vLLM, etc.) | | ||
| Variables are also loaded from `.env` in the skill directory. Shell values override `.env`. All values are **redacted** from persisted session logs. | ||
| ## OpenCode Agent | ||
| [OpenCode](https://opencode.ai/) is an AI coding agent that supports multiple AI models and specialized subagents. | ||
| ### Quick Start | ||
| ```bash | ||
| # Use OpenCode with default agent and model | ||
| skillgrade --agent=opencode | ||
| # Specify OpenCode agent (build|plan|explore) | ||
| skillgrade --agent=opencode --opencode-agent=build | ||
| # Specify both agent and model (provider/model format) | ||
| skillgrade --agent=opencode --opencode-agent=build --opencode-model=anthropic/claude-sonnet-4-20250514 | ||
| ``` | ||
| ### OpenCode Agents | ||
| | Agent | Description | | ||
| |-------|-------------| | ||
| | `build` | Default primary agent with full tool access | | ||
| | `plan` | Read-only planning/analysis agent | | ||
| | `explore` | Fast codebase exploration agent | | ||
| ### OpenCode Models | ||
| Models are specified in `provider/model` format: | ||
| | Model | Format | | ||
| |-------|--------| | ||
| | Claude Sonnet 4 | `anthropic/claude-sonnet-4-20250514` | | ||
| | GPT 5.1 Codex | `opencode/gpt-5.1-codex` | | ||
| ### CLI Options | ||
| | Flag | Description | | ||
| |------|-------------| | ||
| | `--agent=opencode` | Use OpenCode agent | | ||
| | `--opencode-agent=NAME` | OpenCode agent (build\|plan\|explore) | | ||
| | `--opencode-model=MODEL` | OpenCode model (provider/model format) | | ||
| ### How It Works | ||
| 1. skillgrade invokes OpenCode CLI with `opencode run` | ||
| 2. Passes instruction via temp file to avoid shell escaping issues | ||
| 3. Supports both agent and model specification | ||
| 4. Works with `--provider=docker` or `--provider=local` | ||
| ## ACP Agent | ||
@@ -247,0 +312,0 @@ |
Environment variable access
Supply chain riskPackage accesses environment variables, which may be a sign of credential stuffing or data theft.
Found 2 instances
Long strings
Supply chain riskContains long string literals, which may be a sign of obfuscated or packed code.
URL strings
Supply chain riskPackage contains fragments of external URLs or IP addresses, which the package may be accessing at runtime.
Long strings
Supply chain riskContains long string literals, which may be a sign of obfuscated or packed code.
URL strings
Supply chain riskPackage contains fragments of external URLs or IP addresses, which the package may be accessing at runtime.
194502
6.07%52
4%3783
4.79%369
21.38%24
14.29%9
12.5%