🎩 You're Invited:Meet the Socket team at Black Hat in Las Vegas, August 3-6.RSVP
Sign In

skillgrade

Package Overview
Dependencies
Maintainers
1
Versions
10
Alerts
File Explorer

Advanced tools

Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

skillgrade - npm Package Compare versions

Comparing version
0.1.4
to
0.1.5
+11
dist/agents/opencode.d.ts
import { BaseAgent, CommandResult } from '../types';
export interface OpenCodeAgentConfig {
agent?: string;
model?: string;
}
export declare class OpenCodeAgent extends BaseAgent {
private config;
constructor(config?: OpenCodeAgentConfig);
run(instruction: string, _workspacePath: string, runCommand: (cmd: string) => Promise<CommandResult>): Promise<string>;
}
//# sourceMappingURL=opencode.d.ts.map
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.OpenCodeAgent = void 0;
const types_1 = require("../types");
class OpenCodeAgent extends types_1.BaseAgent {
config;
constructor(config = {}) {
super();
this.config = config;
}
async run(instruction, _workspacePath, runCommand) {
const fullCommand = `cd '${_workspacePath.replace(/'/g, "'\\''")}' && echo '${instruction.replace(/'/g, "'\\''")}' | opencode run ${this.config.model ? `--model ${this.config.model}` : ''}`;
const result = await runCommand(fullCommand);
return result.stdout + '\n' + result.stderr;
}
}
exports.OpenCodeAgent = OpenCodeAgent;
//# sourceMappingURL=opencode.js.map
+4
-0

@@ -9,5 +9,7 @@ /**

* - acp: Agent Client Protocol compatible agents
* - opencode: OpenCode AI coding agent
*/
import { BaseAgent } from '../types';
import { AcpAgentConfig } from './acp';
import { OpenCodeAgentConfig } from './opencode';
/** Configuration for agent creation */

@@ -17,2 +19,4 @@ export interface AgentConfig {

acp?: AcpAgentConfig;
/** OpenCode-specific configuration */
opencode?: OpenCodeAgentConfig;
}

@@ -19,0 +23,0 @@ /** Get the list of supported agent names */

+4
-6

@@ -9,2 +9,3 @@ "use strict";

const acp_1 = require("./acp");
const opencode_1 = require("./opencode");
/** Registry of available agent implementations */

@@ -16,3 +17,4 @@ const AGENT_REGISTRY = {

// ACP agent requires config, registered as placeholder
acp: () => new acp_1.AcpAgent({ command: 'gemini --acp' }),
acp: (config) => new acp_1.AcpAgent(config?.acp || { command: 'gemini --acp' }),
opencode: (config) => new opencode_1.OpenCodeAgent(config?.opencode || {}),
};

@@ -25,6 +27,2 @@ /** Get the list of supported agent names */

function createAgent(name, config) {
// Handle ACP agent with custom configuration
if (name === 'acp' && config?.acp) {
return new acp_1.AcpAgent(config.acp);
}
const factory = AGENT_REGISTRY[name];

@@ -35,4 +33,4 @@ if (!factory) {

}
return factory();
return factory(config);
}
//# sourceMappingURL=registry.js.map

@@ -14,2 +14,4 @@ interface RunOptions {

acpCommand?: string;
openCodeAgent?: string;
openCodeModel?: string;
}

@@ -16,0 +18,0 @@ export declare function runEvals(dir: string, opts: RunOptions): Promise<void>;

@@ -135,2 +135,3 @@ "use strict";

graderModel: resolved.grader_model,
graderProvider: resolved.grader_provider,
environment: resolved.environment,

@@ -156,6 +157,5 @@ };

const providerName = opts.provider || resolved.provider;
// Build agent config (for ACP agent)
// Build agent config
const agentConfig = {};
if (agentName === 'acp') {
// Use CLI flag > eval.yaml config > default
const acpCommand = opts.acpCommand || resolved.acp?.command;

@@ -171,2 +171,11 @@ if (!acpCommand) {

}
else if (agentName === 'opencode') {
agentConfig.opencode = {};
if (opts.openCodeAgent) {
agentConfig.opencode.agent = opts.openCodeAgent;
}
if (opts.openCodeModel) {
agentConfig.opencode.model = opts.openCodeModel;
}
}
// Pick provider

@@ -295,8 +304,7 @@ const provider = providerName === 'docker'

else if (resolved.agent === 'acp') {
// For ACP agent, install gemini-cli as the default ACP-compatible agent
// Users can customize the command via acp.command in eval.yaml
// Note: ACP agent works best with --provider=local since it requires
// the ACP command to be available in the host environment
dockerfileContent += `RUN npm install -g @google/gemini-cli\n\n`;
}
else if (resolved.agent === 'opencode') {
dockerfileContent += `RUN npm install -g opencode\n\n`;
}
// Docker setup commands

@@ -315,9 +323,18 @@ if (resolved.docker.setup) {

const srcPath = path.resolve(baseDir, w.src);
const destInTmp = path.join(tmpDir, path.basename(w.src));
if (await fs.pathExists(srcPath)) {
await fs.copy(srcPath, destInTmp);
dockerfileContent += `COPY ${path.basename(w.src)} ${w.dest}\n`;
if (w.chmod) {
dockerfileContent += `RUN chmod ${w.chmod} ${w.dest}\n`;
if (resolved.provider === 'local') {
// For local provider: copy directly to destination path in tmpDir
const destPath = path.join(tmpDir, w.dest);
await fs.ensureDir(path.dirname(destPath));
await fs.copy(srcPath, destPath);
}
else {
// For Docker: copy to tmpDir root, Dockerfile will place it
const destInTmp = path.join(tmpDir, path.basename(w.src));
await fs.copy(srcPath, destInTmp);
dockerfileContent += `COPY ${path.basename(w.src)} ${w.dest}\n`;
if (w.chmod) {
dockerfileContent += `RUN chmod ${w.chmod} ${w.dest}\n`;
}
}
}

@@ -324,0 +341,0 @@ }

@@ -114,2 +114,7 @@ "use strict";

}
// Validate grader_provider
const validGraderProviders = ['gemini', 'anthropic', 'openai'];
if (defaults.grader_provider && !validGraderProviders.includes(defaults.grader_provider)) {
throw new Error(`eval.yaml: grader_provider must be one of ${validGraderProviders.join(', ')}, got "${defaults.grader_provider}"`);
}
if (!raw.tasks || !Array.isArray(raw.tasks) || raw.tasks.length === 0) {

@@ -126,2 +131,5 @@ throw new Error('eval.yaml must have at least one task in the "tasks" array');

}
if (t.grader_provider && !validGraderProviders.includes(t.grader_provider)) {
throw new Error(`Task "${t.name}" has invalid grader_provider "${t.grader_provider}", must be one of ${validGraderProviders.join(', ')}`);
}
const workspace = (t.workspace || []).map((w) => {

@@ -141,10 +149,16 @@ if (typeof w === 'string') {

workspace,
graders: t.graders.map((g) => ({
type: g.type,
setup: g.setup,
run: g.run,
rubric: g.rubric,
model: g.model,
weight: g.weight ?? 1.0,
})),
graders: t.graders.map((g) => {
if (g.provider && !validGraderProviders.includes(g.provider)) {
throw new Error(`Task "${t.name}" grader has invalid provider "${g.provider}", must be one of ${validGraderProviders.join(', ')}`);
}
return {
type: g.type,
setup: g.setup,
run: g.run,
rubric: g.rubric,
model: g.model,
provider: g.provider,
weight: g.weight ?? 1.0,
};
}),
solution: t.solution,

@@ -155,3 +169,6 @@ agent: t.agent,

timeout: t.timeout,
grader_model: t.grader_model,
grader_provider: t.grader_provider,
docker: t.docker,
environment: t.environment,
};

@@ -179,2 +196,3 @@ });

const grader_model = task.grader_model || defaults.grader_model;
const grader_provider = task.grader_provider || defaults.grader_provider;
const acp = defaults.acp; // ACP config is only at defaults level

@@ -189,2 +207,3 @@ // Resolve instruction — could be inline text or file path

model: g.model,
provider: g.provider,
weight: g.weight,

@@ -215,2 +234,3 @@ };

grader_model,
grader_provider,
acp,

@@ -217,0 +237,0 @@ docker,

@@ -20,2 +20,3 @@ /**

model?: string;
provider?: 'gemini' | 'anthropic' | 'openai';
weight: number;

@@ -52,2 +53,3 @@ }

grader_model?: string;
grader_provider?: 'gemini' | 'anthropic' | 'openai';
docker?: DockerConfig;

@@ -64,2 +66,3 @@ environment?: Partial<EnvironmentConfig>;

grader_model?: string;
grader_provider?: 'gemini' | 'anthropic' | 'openai';
acp?: AcpConfig;

@@ -88,2 +91,3 @@ docker: DockerConfig;

grader_model?: string;
grader_provider?: 'gemini' | 'anthropic' | 'openai';
acp?: AcpConfig;

@@ -99,4 +103,5 @@ docker: DockerConfig;

model?: string;
provider?: 'gemini' | 'anthropic' | 'openai';
weight: number;
}
//# sourceMappingURL=config.types.d.ts.map

@@ -9,2 +9,3 @@ import { BaseAgent, EnvironmentProvider, EvalReport } from './types';

graderModel?: string;
graderProvider?: 'gemini' | 'anthropic' | 'openai';
graderTimeoutSec?: number;

@@ -11,0 +12,0 @@ environment: {

@@ -207,2 +207,3 @@ "use strict";

model: graderDef.model || opts.graderModel,
provider: graderDef.provider || opts.graderProvider,
weight: graderDef.weight,

@@ -209,0 +210,0 @@ };

@@ -20,8 +20,18 @@ import { GraderConfig, GraderResult, EnvironmentProvider } from '../types';

* Uses an LLM to evaluate the agent's session transcript against a rubric.
* Requires GEMINI_API_KEY or ANTHROPIC_API_KEY in the environment.
*
* Supported providers (selected via `config.provider`, defaults to "gemini"):
* - gemini → Google Gemini (GEMINI_API_KEY)
* - anthropic → Anthropic Claude or compatible (ANTHROPIC_API_KEY; optional ANTHROPIC_BASE_URL)
* - openai → OpenAI or compatible (OPENAI_API_KEY; optional OPENAI_BASE_URL for Ollama, vLLM, etc.)
*
* Each provider method resolves its own API key, makes the HTTP call, and
* handles errors — returning a zero-score GraderResult on any failure.
*/
export declare class LLMGrader implements Grader {
/** Default models when no model override is configured. */
private static readonly DEFAULT_MODELS;
grade(_workspace: string, _provider: EnvironmentProvider, config: GraderConfig, taskPath: string, sessionLog: any[], env?: Record<string, string>): Promise<GraderResult>;
private callGemini;
private callAnthropic;
private callOpenAI;
private parseResponse;

@@ -28,0 +38,0 @@ }

@@ -94,5 +94,18 @@ "use strict";

* Uses an LLM to evaluate the agent's session transcript against a rubric.
* Requires GEMINI_API_KEY or ANTHROPIC_API_KEY in the environment.
*
* Supported providers (selected via `config.provider`, defaults to "gemini"):
* - gemini → Google Gemini (GEMINI_API_KEY)
* - anthropic → Anthropic Claude or compatible (ANTHROPIC_API_KEY; optional ANTHROPIC_BASE_URL)
* - openai → OpenAI or compatible (OPENAI_API_KEY; optional OPENAI_BASE_URL for Ollama, vLLM, etc.)
*
* Each provider method resolves its own API key, makes the HTTP call, and
* handles errors — returning a zero-score GraderResult on any failure.
*/
class LLMGrader {
/** Default models when no model override is configured. */
static DEFAULT_MODELS = {
gemini: 'gemini-3-flash-preview',
anthropic: 'claude-sonnet-4-20250514',
openai: 'gpt-4o',
};
async grade(_workspace, _provider, config, taskPath, sessionLog, env) {

@@ -147,20 +160,30 @@ const rubricPath = path.join(taskPath, config.rubric || 'prompts/quality.md');

Respond with ONLY a JSON object: {"score": <number>, "reasoning": "<brief explanation>"}`;
// Try Gemini API first, fall back to Anthropic
const providerName = config.provider || 'gemini';
const model = config.model || LLMGrader.DEFAULT_MODELS[providerName] || 'gemini-3-flash-preview';
switch (providerName) {
case "gemini":
return this.callGemini(prompt, model, config, env);
case "anthropic":
return this.callAnthropic(prompt, model, config, env);
case "openai":
return this.callOpenAI(prompt, model, config, env);
default:
return {
grader_type: 'llm_rubric',
score: 0,
weight: config.weight,
details: `Unknown grader provider: "${providerName}". Supported: gemini, anthropic, openai`,
};
}
}
async callGemini(prompt, model, config, env) {
const apiKey = env?.GEMINI_API_KEY || process.env.GEMINI_API_KEY;
const anthropicKey = env?.ANTHROPIC_API_KEY || process.env.ANTHROPIC_API_KEY;
if (apiKey) {
return this.callGemini(prompt, apiKey, config);
if (!apiKey) {
return {
grader_type: 'llm_rubric',
score: 0,
weight: config.weight,
details: 'Missing GEMINI_API_KEY. Set the GEMINI_API_KEY environment variable to use the "gemini" grader provider.'
};
}
else if (anthropicKey) {
return this.callAnthropic(prompt, anthropicKey, config);
}
return {
grader_type: 'llm_rubric',
score: 0,
weight: config.weight,
details: 'No API key available for LLM grading (set GEMINI_API_KEY or ANTHROPIC_API_KEY)'
};
}
async callGemini(prompt, apiKey, config) {
const model = config.model || 'gemini-3-flash-preview';
const url = `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent?key=${apiKey}`;

@@ -184,6 +207,16 @@ try {

}
async callAnthropic(prompt, apiKey, config) {
const model = config.model || 'claude-sonnet-4-20250514';
async callAnthropic(prompt, model, config, env) {
const apiKey = env?.ANTHROPIC_API_KEY || process.env.ANTHROPIC_API_KEY;
if (!apiKey) {
return {
grader_type: 'llm_rubric',
score: 0,
weight: config.weight,
details: 'Missing ANTHROPIC_API_KEY. Set the ANTHROPIC_API_KEY environment variable to use the "anthropic" grader provider.'
};
}
const baseUrl = (env?.ANTHROPIC_BASE_URL || process.env.ANTHROPIC_BASE_URL || 'https://api.anthropic.com/v1').replace(/\/+$/, '');
const url = `${baseUrl}/messages`;
try {
const response = await fetch('https://api.anthropic.com/v1/messages', {
const response = await fetch(url, {
method: 'POST',

@@ -209,9 +242,43 @@ headers: {

}
async callOpenAI(prompt, model, config, env) {
const apiKey = env?.OPENAI_API_KEY || process.env.OPENAI_API_KEY;
if (!apiKey) {
return {
grader_type: 'llm_rubric',
score: 0,
weight: config.weight,
details: 'Missing OPENAI_API_KEY. Set the OPENAI_API_KEY environment variable to use the "openai" grader provider.'
};
}
const baseUrl = (env?.OPENAI_BASE_URL || process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1').replace(/\/+$/, '');
const url = `${baseUrl}/chat/completions`;
try {
const response = await fetch(url, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${apiKey}`,
},
body: JSON.stringify({
model,
temperature: 0,
max_tokens: 4096,
messages: [{ role: 'user', content: prompt }],
}),
});
const data = await response.json();
const text = data?.choices?.[0]?.message?.content || '';
return this.parseResponse(text, config);
}
catch (e) {
return { grader_type: 'llm_rubric', score: 0, weight: config.weight, details: `OpenAI API error: ${e}` };
}
}
parseResponse(text, config) {
try {
// Strip markdown code fences if present
let cleaned = text.replace(/```(?:json)?\s*/g, '').replace(/```/g, '').trim();
// Extract JSON from response
const jsonMatch = cleaned.match(/\{[\s\S]*\}/);
if (jsonMatch) {
// Strip markdown code fences if present
let cleaned = text.replace(/```(?:json)?\s*/g, '').replace(/```/g, '').trim();
// Try to extract and parse JSON
const jsonMatch = cleaned.match(/\{[\s\S]*\}/);
if (jsonMatch) {
try {
const parsed = JSON.parse(jsonMatch[0]);

@@ -226,16 +293,32 @@ const score = Math.max(0, Math.min(1, parseFloat(parsed.score) || 0));

}
}
catch (e) {
// JSON parse failed — try to extract score from truncated response
const scoreMatch = text.match(/"score"\s*:\s*([\d.]+)/);
if (scoreMatch) {
const score = Math.max(0, Math.min(1, parseFloat(scoreMatch[1]) || 0));
return {
grader_type: 'llm_rubric',
score,
weight: config.weight,
details: 'Parsed score from truncated LLM response'
};
catch (e) {
// JSON parse failed, likely truncated or malformed - try to extract score anyway
const scoreMatch = jsonMatch[0].match(/"score"\s*:\s*([\d.]+)/);
if (scoreMatch) {
const score = Math.max(0, Math.min(1, parseFloat(scoreMatch[1]) || 0));
// Try to extract partial reasoning if available
const reasoningMatch = jsonMatch[0].match(/"reasoning"\s*:\s*"([^"]*)/);
const reasoning = reasoningMatch
? reasoningMatch[1] + '... (response truncated)'
: 'Score extracted from incomplete LLM response';
return {
grader_type: 'llm_rubric',
score,
weight: config.weight,
details: reasoning
};
}
}
}
// No JSON found at all - try to extract score from plain text
const scoreMatch = text.match(/"score"\s*:\s*([\d.]+)|score[:\s]+(\d+\.?\d*)/i);
if (scoreMatch) {
const score = Math.max(0, Math.min(1, parseFloat(scoreMatch[1] || scoreMatch[2]) || 0));
return {
grader_type: 'llm_rubric',
score,
weight: config.weight,
details: 'Score extracted from malformed LLM response'
};
}
return { grader_type: 'llm_rubric', score: 0, weight: config.weight, details: `Failed to parse LLM response: ${text.substring(0, 200)}` };

@@ -242,0 +325,0 @@ }

@@ -131,2 +131,4 @@ #!/usr/bin/env node

acpCommand: getFlag('acp-command'),
openCodeAgent: getFlag('opencode-agent'),
openCodeModel: getFlag('opencode-model'),
});

@@ -157,5 +159,7 @@ if (openPreview) {

--parallel=N Run trials concurrently
--agent=gemini|claude|codex|acp Override agent (default: auto-detect from API key)
--agent=gemini|claude|codex|acp|opencode Override agent (default: auto-detect from API key)
--provider=docker|local Override provider (default: docker)
--acp-command=CMD ACP agent command (e.g., "gemini --acp")
--opencode-agent=NAME OpenCode agent (build|plan|explore)
--opencode-model=MODEL OpenCode model (provider/model format)
--output=DIR Output directory for reports and temp files

@@ -162,0 +166,0 @@ Default: $TMPDIR/skillgrade

@@ -11,2 +11,3 @@ export interface CommandResult {

model?: string;
provider?: 'gemini' | 'anthropic' | 'openai';
weight: number;

@@ -13,0 +14,0 @@ }

{
"name": "skillgrade",
"version": "0.1.4",
"version": "0.1.5",
"description": "The easiest way to evaluate your Agent Skills — test that AI agents correctly discover and use your skills",

@@ -5,0 +5,0 @@ "main": "dist/skillgrade.js",

@@ -62,5 +62,7 @@ # Skillgrade

| `--parallel=N` | Run trials concurrently |
| `--agent=gemini\|claude\|codex\|acp` | Override agent (default: auto-detect from API key) |
| `--agent=gemini\|claude\|codex\|acp\|opencode` | Override agent (default: auto-detect from API key) |
| `--provider=docker\|local` | Override provider |
| `--acp-command=CMD` | ACP agent command (e.g., `gemini --acp`) |
| `--opencode-agent=NAME` | OpenCode agent (build\|plan\|explore) |
| `--opencode-model=MODEL` | OpenCode model (provider/model format) |
| `--output=DIR` | Output directory (default: `$TMPDIR/skillgrade`) |

@@ -87,2 +89,3 @@ | `--validate` | Verify graders using reference solutions |

grader_model: gemini-3-flash-preview # default LLM grader model
grader_provider: gemini # default LLM grader provider: gemini | anthropic | openai
acp: # ACP agent configuration (optional)

@@ -120,2 +123,3 @@ command: gemini --acp # command to start ACP-compatible agent

Did the agent follow the check → fix → verify workflow?
provider: gemini # optional: gemini (default) | anthropic | openai
model: gemini-2.0-flash # optional model override

@@ -126,2 +130,3 @@ weight: 0.3

agent: claude
grader_provider: anthropic # override default LLM grader provider
trials: 10

@@ -199,7 +204,16 @@ timeout: 600

weight: 0.3
provider: gemini # gemini (default) | anthropic | openai
model: gemini-2.0-flash # optional, auto-detected from API key
```
Uses Gemini or Anthropic based on available API key. Override with the `model` field.
The `provider` field selects which LLM API to call:
| Provider | API Key Env Var | Base URL Env Var (optional) | Default Model |
|------------|---------------------|-----------------------------|----------------------------|
| `gemini` | `GEMINI_API_KEY` | - | `gemini-3-flash-preview` |
| `anthropic`| `ANTHROPIC_API_KEY` | `ANTHROPIC_BASE_URL` | `claude-sonnet-4-20250514` |
| `openai` | `OPENAI_API_KEY` | `OPENAI_BASE_URL` | `gpt-4o` |
`ANTHROPIC_BASE_URL` and `OPENAI_BASE_URL` enable custom/self-hosted endpoints (Ollama, vLLM, etc.).
### Combining Graders

@@ -239,8 +253,59 @@

|----------|---------|
| `GEMINI_API_KEY` | Agent execution, LLM grading, `skillgrade init` |
| `ANTHROPIC_API_KEY` | Agent execution, LLM grading, `skillgrade init` |
| `OPENAI_API_KEY` | Agent execution (Codex), `skillgrade init` |
| `GEMINI_API_KEY` | Agent execution, LLM grading (`provider: gemini`), `skillgrade init` |
| `ANTHROPIC_API_KEY` | Agent execution, LLM grading (`provider: anthropic`), `skillgrade init` |
| `OPENAI_API_KEY` | Agent execution (Codex), LLM grading (`provider: openai`), `skillgrade init` |
| `ANTHROPIC_BASE_URL` | LLM grading (`provider: anthropic`) — custom Anthropic-compatible endpoint |
| `OPENAI_BASE_URL` | LLM grading (`provider: openai`) — custom OpenAI-compatible endpoint (Ollama, vLLM, etc.) |
Variables are also loaded from `.env` in the skill directory. Shell values override `.env`. All values are **redacted** from persisted session logs.
## OpenCode Agent
[OpenCode](https://opencode.ai/) is an AI coding agent that supports multiple AI models and specialized subagents.
### Quick Start
```bash
# Use OpenCode with default agent and model
skillgrade --agent=opencode
# Specify OpenCode agent (build|plan|explore)
skillgrade --agent=opencode --opencode-agent=build
# Specify both agent and model (provider/model format)
skillgrade --agent=opencode --opencode-agent=build --opencode-model=anthropic/claude-sonnet-4-20250514
```
### OpenCode Agents
| Agent | Description |
|-------|-------------|
| `build` | Default primary agent with full tool access |
| `plan` | Read-only planning/analysis agent |
| `explore` | Fast codebase exploration agent |
### OpenCode Models
Models are specified in `provider/model` format:
| Model | Format |
|-------|--------|
| Claude Sonnet 4 | `anthropic/claude-sonnet-4-20250514` |
| GPT 5.1 Codex | `opencode/gpt-5.1-codex` |
### CLI Options
| Flag | Description |
|------|-------------|
| `--agent=opencode` | Use OpenCode agent |
| `--opencode-agent=NAME` | OpenCode agent (build\|plan\|explore) |
| `--opencode-model=MODEL` | OpenCode model (provider/model format) |
### How It Works
1. skillgrade invokes OpenCode CLI with `opencode run`
2. Passes instruction via temp file to avoid shell escaping issues
3. Supports both agent and model specification
4. Works with `--provider=docker` or `--provider=local`
## ACP Agent

@@ -247,0 +312,0 @@