skillgrade - npm Package Compare versions

Comparing version 0.2.0 to 0.2.1

dist/utils/models.d.ts

+/**
+ * Fetch latest models from Gemini API.
+ */
+export declare function fetchLatestGeminiModel(apiKey: string): Promise<string>;
+/**
+ * Fetch latest models from OpenAI API.
+ */
+export declare function fetchLatestOpenAIModel(apiKey: string): Promise<string>;
+/**
+ * Fetch latest models from Anthropic API.
+ */
+export declare function fetchLatestAnthropicModel(apiKey: string): Promise<string>;
+/**
+ * Resolve Gemini Model based on environment variables or dynamic lookup.
+ */
+export declare function resolveGeminiModel(apiKey: string | undefined, env?: Record<string, string | undefined>, context?: 'init' | 'grader'): Promise<string>;
+/**
+ * Resolve Anthropic Model based on environment variables or dynamic lookup.
+ */
+export declare function resolveAnthropicModel(apiKey: string | undefined, env?: Record<string, string | undefined>, context?: 'init' | 'grader'): Promise<string>;
+/**
+ * Resolve OpenAI Model based on environment variables or dynamic lookup.
+ */
+export declare function resolveOpenAIModel(apiKey: string | undefined, env?: Record<string, string | undefined>, context?: 'init' | 'grader'): Promise<string>;

dist/utils/models.js

+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.fetchLatestGeminiModel = fetchLatestGeminiModel;
+exports.fetchLatestOpenAIModel = fetchLatestOpenAIModel;
+exports.fetchLatestAnthropicModel = fetchLatestAnthropicModel;
+exports.resolveGeminiModel = resolveGeminiModel;
+exports.resolveAnthropicModel = resolveAnthropicModel;
+exports.resolveOpenAIModel = resolveOpenAIModel;
+/**
+ * Fetch latest models from Gemini API.
+ */
+async function fetchLatestGeminiModel(apiKey) {
+    const res = await fetch(`https://generativelanguage.googleapis.com/v1beta/models?key=${apiKey}`, {
+        signal: AbortSignal.timeout(10000),
+    });
+    if (!res.ok) {
+        throw new Error(`Gemini API returned status ${res.status}`);
+    }
+    const data = await res.json();
+    if (!data.models || !Array.isArray(data.models)) {
+        throw new Error('Invalid response from Gemini API models endpoint');
+    }
+    let filtered = data.models.filter((m) => m.name &&
+        m.supportedGenerationMethods &&
+        m.supportedGenerationMethods.includes('generateContent') &&
+        m.name.includes('gemini-') &&
+        m.name.includes('-flash'));
+    if (filtered.length === 0) {
+        filtered = data.models.filter((m) => m.name &&
+            m.supportedGenerationMethods &&
+            m.supportedGenerationMethods.includes('generateContent') &&
+            m.name.includes('gemini-'));
+    }
+    const models = filtered
+        .map((m) => {
+        const cleanName = m.name.replace(/^models\//, '');
+        const match = cleanName.match(/gemini-([\d.]+)-/);
+        const version = match ? parseFloat(match[1]) : 0;
+        return { name: cleanName, version };
+    })
+        .filter((m) => m.version > 0)
+        .sort((a, b) => b.version - a.version);
+    if (models.length > 0) {
+        return models[0].name;
+    }
+    throw new Error('No suitable Gemini models found from the API');
+}
+/**
+ * Fetch latest models from OpenAI API.
+ */
+async function fetchLatestOpenAIModel(apiKey) {
+    const res = await fetch('https://api.openai.com/v1/models', {
+        headers: {
+            'Authorization': `Bearer ${apiKey}`,
+        },
+        signal: AbortSignal.timeout(10000),
+    });
+    if (!res.ok) {
+        throw new Error(`OpenAI API returned status ${res.status}`);
+    }
+    const data = await res.json();
+    if (!data.data || !Array.isArray(data.data)) {
+        throw new Error('Invalid response from OpenAI API models endpoint');
+    }
+    let filtered = data.data.filter((m) => {
+        const id = m.id;
+        const isSuitable = id.startsWith('gpt-') || id.startsWith('o1') || id.startsWith('o2') || id.startsWith('o3') || id.startsWith('o4') || id.startsWith('o5');
+        const isFlashEquivalent = id.includes('mini') || id.includes('nano') || id.includes('flash');
+        const isAuxiliary = id.includes('instruct') || id.includes('realtime') || id.includes('search') || id.includes('transcribe') || id.includes('tts') || id.includes('embedding') || id.includes('moderation') || id.includes('audio') || id.includes('vision');
+        return isSuitable && isFlashEquivalent && !isAuxiliary;
+    });
+    if (filtered.length === 0) {
+        filtered = data.data.filter((m) => {
+            const id = m.id;
+            const isSuitable = id.startsWith('gpt-') || id.startsWith('o1') || id.startsWith('o2') || id.startsWith('o3') || id.startsWith('o4') || id.startsWith('o5');
+            const isAuxiliary = id.includes('instruct') || id.includes('realtime') || id.includes('search') || id.includes('transcribe') || id.includes('tts') || id.includes('embedding') || id.includes('moderation') || id.includes('audio') || id.includes('vision');
+            return isSuitable && !isAuxiliary;
+        });
+    }
+    const sorted = filtered.sort((a, b) => b.created - a.created);
+    if (sorted.length > 0) {
+        return sorted[0].id;
+    }
+    throw new Error('No suitable OpenAI models found from the API');
+}
+/**
+ * Fetch latest models from Anthropic API.
+ */
+async function fetchLatestAnthropicModel(apiKey) {
+    const res = await fetch('https://api.anthropic.com/v1/models', {
+        headers: {
+            'x-api-key': apiKey,
+            'anthropic-version': '2023-06-01',
+        },
+        signal: AbortSignal.timeout(10000),
+    });
+    if (!res.ok) {
+        throw new Error(`Anthropic API returned status ${res.status}`);
+    }
+    const data = await res.json();
+    if (!data.data || !Array.isArray(data.data)) {
+        throw new Error('Invalid response from Anthropic API models endpoint');
+    }
+    let filtered = data.data.filter((m) => {
+        const id = m.id;
+        const isSuitable = id.startsWith('claude-');
+        const isFlashEquivalent = id.includes('haiku') || id.includes('flash');
+        const isAuxiliary = id.includes('search') || id.includes('embedding') || id.includes('moderation');
+        return isSuitable && isFlashEquivalent && !isAuxiliary;
+    });
+    if (filtered.length === 0) {
+        filtered = data.data.filter((m) => {
+            const id = m.id;
+            const isSuitable = id.startsWith('claude-');
+            const isAuxiliary = id.includes('search') || id.includes('embedding') || id.includes('moderation');
+            return isSuitable && !isAuxiliary;
+        });
+    }
+    const sorted = filtered.sort((a, b) => {
+        const timeA = a.created_at ? new Date(a.created_at).getTime() : 0;
+        const timeB = b.created_at ? new Date(b.created_at).getTime() : 0;
+        return timeB - timeA;
+    });
+    if (sorted.length > 0) {
+        return sorted[0].id;
+    }
+    throw new Error('No suitable Anthropic models found from the API');
+}
+/**
+ * Resolve Gemini Model based on environment variables or dynamic lookup.
+ */
+async function resolveGeminiModel(apiKey, env = process.env, context = 'grader') {
+    if (context === 'init') {
+        if (env.INIT_GEMINI_MODEL)
+            return env.INIT_GEMINI_MODEL;
+    }
+    if (env.GEMINI_MODEL)
+        return env.GEMINI_MODEL;
+    if (!apiKey) {
+        throw new Error('Missing GEMINI_API_KEY. Cannot dynamically resolve the latest Gemini model without an API key.');
+    }
+    return await fetchLatestGeminiModel(apiKey);
+}
+/**
+ * Resolve Anthropic Model based on environment variables or dynamic lookup.
+ */
+async function resolveAnthropicModel(apiKey, env = process.env, context = 'grader') {
+    if (context === 'init') {
+        if (env.INIT_ANTHROPIC_MODEL)
+            return env.INIT_ANTHROPIC_MODEL;
+    }
+    if (env.ANTHROPIC_MODEL)
+        return env.ANTHROPIC_MODEL;
+    if (!apiKey) {
+        throw new Error('Missing ANTHROPIC_API_KEY. Cannot dynamically resolve the latest Anthropic model without an API key.');
+    }
+    return await fetchLatestAnthropicModel(apiKey);
+}
+/**
+ * Resolve OpenAI Model based on environment variables or dynamic lookup.
+ */
+async function resolveOpenAIModel(apiKey, env = process.env, context = 'grader') {
+    if (context === 'init') {
+        if (env.INIT_OPENAI_MODEL)
+            return env.INIT_OPENAI_MODEL;
+    }
+    if (env.OPENAI_MODEL)
+        return env.OPENAI_MODEL;
+    if (!apiKey) {
+        throw new Error('Missing OPENAI_API_KEY. Cannot dynamically resolve the latest OpenAI model without an API key.');
+    }
+    return await fetchLatestOpenAIModel(apiKey);
+}

dist/commands/init.js

@@ -48,2 +48,3 @@ "use strict";
 const env_1 = require("../utils/env");
+const models_1 = require("../utils/models");
 async function runInit(dir, opts = {}) {
@@ -238,2 +239,3 @@ const evalPath = path.join(dir, 'eval.yaml');
     if (provider === 'anthropic') {
+        const model = await (0, models_1.resolveAnthropicModel)(apiKey, process.env, 'init');
         const response = await fetch('https://api.anthropic.com/v1/messages', {
@@ -247,3 +249,3 @@ method: 'POST',
             body: JSON.stringify({
-                model: 'claude-sonnet-4-20250514',
+                model,
                 max_tokens: 4096,
@@ -264,2 +266,3 @@ messages: [{ role: 'user', content: prompt }],
     else if (provider === 'openai') {
+        const model = await (0, models_1.resolveOpenAIModel)(apiKey, process.env, 'init');
         const response = await fetch('https://api.openai.com/v1/chat/completions', {
@@ -272,3 +275,3 @@ method: 'POST',
             body: JSON.stringify({
-                model: 'gpt-4o',
+                model,
                 max_tokens: 4096,
@@ -289,3 +292,4 @@ messages: [{ role: 'user', content: prompt }],
     else {
-        const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/gemini-3-flash-preview:generateContent?key=${apiKey}`, {
+        const model = await (0, models_1.resolveGeminiModel)(apiKey, process.env, 'init');
+        const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent?key=${apiKey}`, {
             method: 'POST',
@@ -292,0 +296,0 @@ headers: { 'Content-Type': 'application/json' },

dist/graders/index.d.ts

@@ -30,4 +30,2 @@ import { GraderConfig, GraderResult, EnvironmentProvider } from '../types';
 export declare class LLMGrader implements Grader {
-    /** Default models when no model override is configured. */
-    private static readonly DEFAULT_MODELS;
     grade(_workspace: string, _provider: EnvironmentProvider, config: GraderConfig, taskPath: string, sessionLog: any[], env?: Record<string, string>): Promise<GraderResult>;
@@ -34,0 +32,0 @@ private callGemini;

dist/graders/index.js

@@ -40,2 +40,3 @@ "use strict";
 const path = __importStar(require("path"));
+const models_1 = require("../utils/models");
 /**
@@ -105,8 +106,2 @@ * Runs a command and parses structured JSON from stdout.
 class LLMGrader {
-    /** Default models when no model override is configured. */
-    static DEFAULT_MODELS = {
-        gemini: 'gemini-3-flash-preview',
-        anthropic: 'claude-sonnet-4-20250514',
-        openai: 'gpt-4o',
-    };
     async grade(_workspace, _provider, config, taskPath, sessionLog, env) {
@@ -162,3 +157,27 @@ const rubricPath = path.join(taskPath, config.rubric || 'prompts/quality.md');
         const providerName = config.provider || 'gemini';
-        const model = config.model || LLMGrader.DEFAULT_MODELS[providerName] || 'gemini-3-flash-preview';
+        let model = config.model;
+        if (!model) {
+            try {
+                if (providerName === 'gemini') {
+                    model = await (0, models_1.resolveGeminiModel)(env?.GEMINI_API_KEY || process.env.GEMINI_API_KEY, env, 'grader');
+                }
+                else if (providerName === 'anthropic') {
+                    model = await (0, models_1.resolveAnthropicModel)(env?.ANTHROPIC_API_KEY || process.env.ANTHROPIC_API_KEY, env, 'grader');
+                }
+                else if (providerName === 'openai') {
+                    model = await (0, models_1.resolveOpenAIModel)(env?.OPENAI_API_KEY || process.env.OPENAI_API_KEY, env, 'grader');
+                }
+                else {
+                    throw new Error(`Unknown grader provider: "${providerName}". Supported: gemini, anthropic, openai`);
+                }
+            }
+            catch (err) {
+                return {
+                    grader_type: 'llm_rubric',
+                    score: 0,
+                    weight: config.weight,
+                    details: err.message || String(err),
+                };
+            }
+        }
         switch (providerName) {
@@ -165,0 +184,0 @@ case "gemini":

package.json

 {
   "name": "skillgrade",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "description": "The easiest way to evaluate your Agent Skills — test that AI agents correctly discover and use your skills",
@@ -21,2 +21,9 @@ "main": "dist/skillgrade.js",
   ],
+  "scripts": {
+    "test": "vitest run",
+    "test:coverage": "vitest run --coverage",
+    "dev": "ts-node src/skillgrade.ts",
+    "build": "tsc -p tsconfig.build.json && cp src/viewer.html dist/viewer.html",
+    "prepublishOnly": "npm run build"
+  },
   "keywords": [
@@ -63,9 +70,3 @@ "ai",
     "tar-stream": "^3.1.7"
-  },
-  "scripts": {
-    "test": "vitest run",
-    "test:coverage": "vitest run --coverage",
-    "dev": "ts-node src/skillgrade.ts",
-    "build": "tsc -p tsconfig.build.json && cp src/viewer.html dist/viewer.html"
   }
-}
+}

README.md

@@ -124,3 +124,3 @@ # Skillgrade
         provider: gemini                 # optional: gemini (default) | anthropic | openai
-        model: gemini-2.0-flash          # optional model override
+        model: gemini-3.5-flash          # optional model override
         weight: 0.3
@@ -211,5 +211,5 @@
 |------------|---------------------|-----------------------------|----------------------------|
-| `gemini`   | `GEMINI_API_KEY`    | -                           | `gemini-3-flash-preview`   |
-| `anthropic`| `ANTHROPIC_API_KEY` | `ANTHROPIC_BASE_URL`        | `claude-sonnet-4-20250514` |
-| `openai`   | `OPENAI_API_KEY`    | `OPENAI_BASE_URL`           | `gpt-4o`                   |
+| `gemini`   | `GEMINI_API_KEY`    | -                           | Dynamically resolved latest Flash model (via API) |
+| `anthropic`| `ANTHROPIC_API_KEY` | `ANTHROPIC_BASE_URL`        | Dynamically resolved latest Haiku model (via API) |
+| `openai`   | `OPENAI_API_KEY`    | `OPENAI_BASE_URL`           | Dynamically resolved latest Mini/Flash model (via API) |
 
@@ -257,2 +257,8 @@ `ANTHROPIC_BASE_URL` and `OPENAI_BASE_URL` enable custom/self-hosted endpoints (Ollama, vLLM, etc.).
 | `OPENAI_BASE_URL` | LLM grading (`provider: openai`) — custom OpenAI-compatible endpoint (Ollama, vLLM, etc.) |
+| `GEMINI_MODEL` | Override the default model used for Gemini LLM grading (defaults to dynamic API lookup; throws if resolution fails) |
+| `INIT_GEMINI_MODEL` | Override the model used for Gemini in `skillgrade init` (defaults to `GEMINI_MODEL` or dynamic API lookup; throws if resolution fails) |
+| `ANTHROPIC_MODEL` | Override the default model used for Anthropic LLM grading (defaults to dynamic API lookup; throws if resolution fails) |
+| `INIT_ANTHROPIC_MODEL` | Override the model used for Anthropic in `skillgrade init` (defaults to `ANTHROPIC_MODEL` or dynamic API lookup; throws if resolution fails) |
+| `OPENAI_MODEL` | Override the default model used for OpenAI LLM grading (defaults to dynamic API lookup; throws if resolution fails) |
+| `INIT_OPENAI_MODEL` | Override the model used for OpenAI in `skillgrade init` (defaults to `OPENAI_MODEL` or dynamic API lookup; throws if resolution fails) |
 
@@ -259,0 +265,0 @@ Variables are also loaded from `.env` in the skill directory. Shell values override `.env`. All values are **redacted** from persisted session logs.

New alerts

URL strings

Supply chain risk

Package contains fragments of external URLs or IP addresses, which the package may be accessing at runtime.

Fixed alerts

URL strings

Supply chain risk

Package contains fragments of external URLs or IP addresses, which the package may be accessing at runtime.

Improved metrics

Total package byte prevSize

210525

5.23%

Number of package files

56

3.7%

Lines of code

4106

5.61%

Number of lines in readme file

418

1.46%

Worsened metrics

Number of low supply chain risk alerts

33

37.5%

Number of medium supply chain risk alerts

12

33.33%

No dependency changes