skillgrade - npm Package Compare versions

Comparing version 0.1.1 to 0.1.2

dist/commands/init.js

@@ -235,2 +235,3 @@ "use strict";
     let text;
+    const fetchOpts = { signal: AbortSignal.timeout(120_000) };
     if (provider === 'anthropic') {
@@ -250,2 +251,3 @@ const response = await fetch('https://api.anthropic.com/v1/messages', {
             }),
+            ...fetchOpts,
         });
@@ -273,2 +275,3 @@ if (!response.ok) {
             }),
+            ...fetchOpts,
         });
@@ -291,2 +294,3 @@ if (!response.ok) {
             }),
+            ...fetchOpts,
         });
@@ -293,0 +297,0 @@ if (!response.ok) {

dist/commands/run.d.ts

 interface RunOptions {
-    task?: string;
+    eval?: string;
     trials?: number;
@@ -12,2 +12,3 @@ parallel?: number;
     output?: string;
+    grader?: string;
 }
@@ -14,0 +15,0 @@ export declare function runEvals(dir: string, opts: RunOptions): Promise<void>;

dist/commands/run.js

@@ -98,10 +98,11 @@ "use strict";
     }
-    // Filter tasks
+    // Filter evals
     let tasksToRun = config.tasks;
-    if (opts.task) {
-        tasksToRun = config.tasks.filter(t => t.name === opts.task);
+    if (opts.eval) {
+        const evalNames = opts.eval.split(',').map(s => s.trim());
+        tasksToRun = config.tasks.filter(t => evalNames.includes(t.name));
         if (tasksToRun.length === 0) {
-            console.error(`  ${cli_1.fmt.red('error')}  task "${opts.task}" not found`);
+            console.error(`  ${cli_1.fmt.red('error')}  eval "${opts.eval}" not found`);
             console.log(`  ${cli_1.fmt.dim('available:')} ${config.tasks.map(t => t.name).join(', ')}`);
-            throw new Error(`Task "${opts.task}" not found`);
+            throw new Error(`Eval "${opts.eval}" not found`);
         }
@@ -130,3 +131,5 @@ }
             instruction: resolved.instruction,
-            graders: resolved.graders,
+            graders: opts.grader
+                ? resolved.graders.filter(g => g.type === opts.grader)
+                : resolved.graders,
             timeoutSec: resolved.timeout,
@@ -133,0 +136,0 @@ graderModel: resolved.grader_model,

dist/skillgrade.d.ts

@@ -9,3 +9,3 @@ #!/usr/bin/env node
  *   skillgrade preview [browser]   View results (CLI default, or browser)
- *   skillgrade <task-name>         Run a specific task
+ *   skillgrade <task-name>         Run a specific eval
  *
@@ -12,0 +12,0 @@ * Options:

dist/skillgrade.js

@@ -10,3 +10,3 @@ #!/usr/bin/env node
  *   skillgrade preview [browser]   View results (CLI default, or browser)
- *   skillgrade <task-name>         Run a specific task
+ *   skillgrade <task-name>         Run a specific eval
  *
@@ -58,2 +58,3 @@ * Options:
 const preview_1 = require("./commands/preview");
+const cli_1 = require("./utils/cli");
 const os = __importStar(require("os"));
@@ -106,5 +107,17 @@ const path = __importStar(require("path"));
     const explicitTrials = getFlag('trials') ? parseInt(getFlag('trials')) : undefined;
+    // Resolve eval filter: --eval flag, deprecated --task flag, or positional arg
+    let evalFilter;
+    if (getFlag('eval')) {
+        evalFilter = getFlag('eval');
+    }
+    else if (getFlag('task')) {
+        console.log(`  ${cli_1.fmt.dim('note:')} --task is deprecated, use --eval instead\n`);
+        evalFilter = getFlag('task');
+    }
+    else if (taskName) {
+        evalFilter = taskName;
+    }
     const outputDir = getFlag('output') || path.join(os.tmpdir(), 'skillgrade');
     await (0, run_1.runEvals)(cwd, {
-        task: taskName,
+        eval: evalFilter,
         trials: explicitTrials ?? presetTrials,
@@ -118,2 +131,3 @@ parallel: getFlag('parallel') ? parseInt(getFlag('parallel')) : undefined,
         provider: getFlag('provider'),
+        grader: getFlag('grader'),
         output: outputDir,
@@ -127,9 +141,9 @@ });
     console.log(`
-  📊 skillgrade — Evaluation framework for Agent Skills
+  skillgrade - The easiest way to evaluate your Agent Skills
 
   Usage:
-    skillgrade                     Run all tasks from eval.yaml
+    skillgrade                     Run all evals from eval.yaml
     skillgrade init [--force]      Generate eval.yaml (--force to overwrite)
     skillgrade preview [browser]   View results (CLI default, or browser)
-    skillgrade <task-name>         Run a specific task
+    skillgrade <eval-name>         Run a specific eval
 
@@ -142,2 +156,4 @@ Presets:
   Options:
+    --eval=NAME[,NAME] Run specific evals by name (comma-separated)
+    --grader=TYPE      Run only graders of this type (deterministic|llm_rubric)
     --trials=N         Override trial count (overrides preset)
@@ -159,2 +175,4 @@ --parallel=N       Run trials concurrently
     skillgrade --smoke             # quick 5-trial smoke test
+    skillgrade --eval=fix-linting  # run a specific eval
+    skillgrade --eval=foo,bar      # run multiple evals
     skillgrade --regression --ci   # CI regression with 30 trials
@@ -161,0 +179,0 @@ skillgrade preview browser     # open web UI

package.json

 {
   "name": "skillgrade",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "description": "The easiest way to evaluate your Agent Skills — test that AI agents correctly discover and use your skills",
@@ -5,0 +5,0 @@ "main": "dist/skillgrade.js",

README.md

@@ -58,2 +58,4 @@ # Skillgrade
 |------|-------------|
+| `--eval=NAME[,NAME]` | Run specific evals by name (comma-separated) |
+| `--grader=TYPE` | Run only graders of a type (`deterministic` or `llm_rubric`) |
 | `--trials=N` | Override trial count |
@@ -60,0 +62,0 @@ | `--parallel=N` | Run trials concurrently |

New alerts

Long strings

Supply chain risk

Contains long string literals, which may be a sign of obfuscated or packed code.

URL strings

Supply chain risk

Package contains fragments of external URLs or IP addresses, which the package may be accessing at runtime.

Fixed alerts

Long strings

Supply chain risk

Contains long string literals, which may be a sign of obfuscated or packed code.

URL strings

Supply chain risk

Package contains fragments of external URLs or IP addresses, which the package may be accessing at runtime.

Improved metrics

Total package byte prevSize

160362

0.78%

Lines of code

3051

0.86%

Number of lines in readme file

254

0.79%

No dependency changes