agent-obs - npm Package Compare versions

Comparing version 0.1.0 to 0.2.0

cli.js

@@ -6,2 +6,3 @@ #!/usr/bin/env node
 const { startMcpServer } = require('./mcp-server');
+const { getSessions, getToolCalls, getDashboardStats } = require('./database');
 
@@ -22,2 +23,4 @@ const command = process.argv[2];
     server                  Run as MCP server (agents self-report via MCP tools)
+    check [--last <n>]      Show latest session summary (or last n sessions)
+    stats                   Show aggregate stats across all sessions
     dashboard [--port <n>]  Start the web dashboard
@@ -29,2 +32,5 @@ inspect <session-id>    Show session details in terminal
     agent-obs server
+    agent-obs check
+    agent-obs check --last 3
+    agent-obs stats
     agent-obs dashboard
@@ -37,2 +43,13 @@ agent-obs inspect abc12345
 
+function timeAgo(sqliteUtc) {
+  if (!sqliteUtc) return 'unknown';
+  const then = new Date(sqliteUtc.replace(' ', 'T') + 'Z').getTime();
+  if (isNaN(then)) return sqliteUtc;
+  const diffSec = Math.max(0, Math.floor((Date.now() - then) / 1000));
+  if (diffSec < 60) return `${diffSec}s ago`;
+  if (diffSec < 3600) return `${Math.floor(diffSec / 60)}m ago`;
+  if (diffSec < 86400) return `${Math.floor(diffSec / 3600)}h ago`;
+  return `${Math.floor(diffSec / 86400)}d ago`;
+}
+
 async function main() {
@@ -56,2 +73,59 @@ if (!command || command === 'help' || command === '--help' || command === '-h') {
 
+  if (command === 'check') {
+    const lastN = args.indexOf('--last') >= 0 ? parseInt(args[args.indexOf('--last') + 1]) || 1 : 1;
+    const sessions = getSessions({ limit: lastN });
+    if (!sessions.length) {
+      console.log('No agent sessions recorded yet.');
+      console.log('Start the dashboard with: agent-obs dashboard');
+      process.exit(0);
+    }
+    if (lastN === 1) {
+      const s = sessions[0];
+      const calls = getToolCalls(s.id);
+      const errors = calls.filter(c => c.status === 'error').length;
+      const totalMs = calls.reduce((sum, c) => sum + (c.duration_ms || 0), 0);
+      console.log(`Session: ${s.id.slice(0, 8)}`);
+      console.log(`Agent:   ${s.agent_type}`);
+      console.log(`Task:    ${s.task_description || '(none)'}`);
+      console.log(`Status:  ${s.status} | Grade: ${s.grade || 'N/A'}`);
+      console.log(`Tools:   ${calls.length} calls | ${errors} ${errors === 1 ? 'error' : 'errors'} | ${(totalMs / 1000).toFixed(1)}s total`);
+      console.log(`Tokens:  ${(s.total_tokens || 0).toLocaleString()} | Cost: $${s.estimated_cost_usd}`);
+      console.log(`Started: ${timeAgo(s.started_at)}`);
+    } else {
+      for (const s of sessions) {
+        const calls = getToolCalls(s.id);
+        const errors = calls.filter(c => c.status === 'error').length;
+        const statusIcon = s.status === 'complete' ? '✓' : s.status === 'error' ? '✗' : '⋯';
+        console.log(`${statusIcon} ${s.id.slice(0, 8)} | ${s.grade || '?'} | ${(s.task_description || '(none)').slice(0, 50)} | ${calls.length} calls | ${errors} errors | $${s.estimated_cost_usd}`);
+      }
+    }
+    process.exit(0);
+  }
+
+  if (command === 'stats') {
+    const stats = getDashboardStats();
+    console.log('── Agent Observability ──');
+    console.log(`Total sessions:    ${stats.totalSessions}`);
+    console.log(`Completed:         ${stats.completedSessions}`);
+    console.log(`Failed:            ${stats.failedSessions}`);
+    console.log(`Total tool calls:  ${stats.totalToolCalls}`);
+    console.log(`Total tokens:      ${(stats.totalTokens || 0).toLocaleString()}`);
+    console.log(`Total cost:        $${stats.totalCost.toFixed(2)}`);
+    console.log(`Avg duration:      ${Math.round(stats.avgDurationSeconds)}s`);
+
+    const gradeValues = { A: 4, B: 3, C: 2, D: 1, F: 0 };
+    const distribution = { A: 0, B: 0, C: 0, D: 0, F: 0 };
+    const graded = getSessions({ limit: 10000 }).filter(s => s.grade in gradeValues);
+    for (const s of graded) distribution[s.grade]++;
+    if (graded.length) {
+      const avg = graded.reduce((sum, s) => sum + gradeValues[s.grade], 0) / graded.length;
+      const avgGrade = ['F', 'D', 'C', 'B', 'A'][Math.round(avg)];
+      console.log(`Avg grade:         ${avgGrade}`);
+    }
+    console.log('');
+    console.log('Grade distribution:');
+    console.log(`  A: ${distribution.A}  B: ${distribution.B}  C: ${distribution.C}  D: ${distribution.D}  F: ${distribution.F}`);
+    process.exit(0);
+  }
+
   if (command === 'proxy') {
@@ -58,0 +132,0 @@ const separatorIdx = args.indexOf('--');

mcp-server.js

@@ -70,2 +70,20 @@ const readline = require('readline');
   },
+  {
+    name: 'check_session',
+    description: 'Check the current session status — tool call count, error count, grade estimate, and session duration',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        sessionId: { type: 'string', description: 'The session ID to check (optional — uses most recent active session if omitted)' },
+      },
+    },
+  },
+  {
+    name: 'get_session_stats',
+    description: 'Get aggregate stats across all sessions — total sessions, total tool calls, total cost, grade distribution',
+    inputSchema: {
+      type: 'object',
+      properties: {},
+    },
+  },
 ];
@@ -219,2 +237,3 @@
           estimatedCostUsd: session.estimated_cost_usd,
+          toolCallCount: toolCalls.length,
           toolCount: toolCalls.length,
@@ -229,2 +248,47 @@ errorCount,
 
+      case 'check_session': {
+        const sessions = database.getSessions({ limit: 1, status: undefined });
+        const session = args.sessionId
+          ? database.getSession(args.sessionId)
+          : (sessions.length > 0 ? sessions[0] : null);
+
+        if (!session) {
+          sendToolResult(id, { error: 'No session found. Start one with start_session.' });
+          break;
+        }
+
+        const toolCalls = database.getToolCalls(session.id);
+        const errors = toolCalls.filter(tc => tc.status === 'error').length;
+        const grade = database.computeGrade({ errorCount: errors, totalCalls: toolCalls.length, durationMs: 0 });
+        const cost = database.estimateCost(session.total_tokens || 0);
+
+        sendToolResult(id, {
+          sessionId: session.id,
+          status: session.status,
+          grade: session.grade || grade.grade,
+          score: grade.score,
+          toolCalls: toolCalls.length,
+          errors,
+          totalTokens: session.total_tokens || 0,
+          estimatedCostUsd: session.estimated_cost_usd || cost,
+          startedAt: session.started_at,
+          taskDescription: session.task_description,
+        });
+        break;
+      }
+
+      case 'get_session_stats': {
+        const stats = database.getDashboardStats();
+        const sessions = database.getSessions({ limit: 1000 });
+        const grades = { A: 0, B: 0, C: 0, D: 0, F: 0 };
+        sessions.forEach(s => { if (s.grade && grades.hasOwnProperty(s.grade)) grades[s.grade]++; });
+
+        sendToolResult(id, {
+          ...stats,
+          gradeDistribution: grades,
+          latestSession: sessions.length > 0 ? sessions[0].id : null,
+        });
+        break;
+      }
+
       default:
@@ -231,0 +295,0 @@ sendError(id, -32601, `Unknown tool: ${name}`);

package.json

 {
   "name": "agent-obs",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "Open source agent observability — see what your agents did, why they failed, and what it cost. Runs locally.",
@@ -5,0 +5,0 @@ "main": "cli.js",

README.md

@@ -7,2 +7,4 @@ # agent-observability
 
+Here's how to connect your agent in 2 minutes: install the package, add agent-obs as an MCP server to your agent platform, tell the agent to self-report, and watch sessions appear in the dashboard.
+
 ### Install
@@ -14,28 +16,56 @@
 
-### Three ways to use it
+### 1. opencode
 
-#### 1. MCP Proxy (transparent capture)
+Add the MCP server to your `opencode.json`:
 
-Intercepts all MCP tool calls automatically. Wrap any MCP server command with `agent-obs proxy` and every tool invocation gets traced without modifying the agent or server.
+```json
+{
+  "mcp": {
+    "agent-obs": {
+      "type": "local",
+      "command": ["node", "/Users/williamweishuhn/Documents/New OpenCode Project/agent-observability/cli.js", "server"],
+      "enabled": true
+    }
+  }
+}
+```
 
-```bash
-agent-obs proxy --desc "fix login bug" -- npx @modelcontextprotocol/server-filesystem /tmp
+Then add this to your system prompt (e.g. in `AGENTS.md`):
+
+> After every tool call, call agent_obs_log_tool_call with the tool name, duration, status, input summary, and output summary. Start a session at the beginning of each task with agent_obs_start_session.
+
+### 2. Claude Code
+
+Add to your project's `.mcp.json`:
+
+```json
+{
+  "mcpServers": {
+    "agent-obs": {
+      "command": "agent-obs",
+      "args": ["server"]
+    }
+  }
+}
 ```
 
-The proxy sits between an MCP client and its target server. It captures:
+Then add a project-level skill so Claude self-reports. Create `.claude/skills/agent-obs/SKILL.md`:
 
-- Tool call name, arguments, and results
-- Server name and version
-- Duration of each call
-- Success/failure status
-- Token counts if the tool communicates with an LLM
+```markdown
+---
+name: agent-obs
+description: Report every tool call to agent-obs for observability tracking.
+---
 
-All data is written to a local SQLite database in `~/.agent-observability/`. No data leaves your machine.
+At the start of each task, call agent_obs_start_session with a description
+of the task. After every tool call, call agent_obs_log_tool_call with the
+tool name, duration, status, input summary, and output summary. When the
+task is done, call agent_obs_end_session with the final status and token
+counts.
+```
 
-#### 2. MCP Server (full capture, recommended)
+### 3. Cursor
 
-Connect agent-observability as an MCP server. Your agent self-reports every action it takes, along with reasoning context that the proxy can't observe.
+Add to Cursor's MCP settings (`~/.cursor/mcp.json`, or Settings → MCP → Add new global MCP server):
 
-Add to your `.mcp.json` (Claude Code, Cursor, or any MCP-compatible agent):
-
 ```json
@@ -52,22 +82,29 @@ {
 
-Available MCP tools:
+Then add the same self-reporting instruction to your Cursor rules (`.cursorrules` or Settings → Rules):
 
-| Tool | Description |
-|------|-------------|
-| `obs_record_tool_call` | Record a tool invocation with name, server, duration, status, input/output |
-| `obs_record_token_usage` | Record token consumption (input/output/total) for the current session |
-| `obs_record_decision` | Capture a decision point — what the agent chose and why |
-| `obs_record_grade` | Assign a session grade (A-F) with reasoning |
-| `obs_get_session_report` | Retrieve a full session summary with all calls, costs, and grades |
-| `obs_list_sessions` | List recent sessions with grades and timestamps |
+> After every tool call, call agent_obs_log_tool_call with the tool name, duration, status, input summary, and output summary. Start a session at the beginning of each task with agent_obs_start_session.
 
-#### 3. Dashboard
+### 4. Self-reporting mode (recommended)
 
-Launch the web dashboard to explore sessions, filter by grade, search tool calls, and export data.
+The MCP server mode above is the primary way to use agent-obs. The agent self-reports every action it takes — including built-in tools (Read, Write, Edit, Bash, etc.) that never travel over MCP and therefore can't be captured by a proxy. Self-reporting also captures reasoning context the wire protocol never sees: why a tool was chosen, what the agent was trying to accomplish, and how the session should be graded.
 
+For MCP-only workloads, there's also a transparent proxy mode that requires no agent cooperation. Wrap any MCP server command and every tool invocation through it gets traced automatically:
+
 ```bash
+agent-obs proxy --desc "fix login bug" -- npx @modelcontextprotocol/server-filesystem /tmp
+```
+
+The proxy captures tool names, arguments, results, durations, and success/failure status — but only for calls to the wrapped server. Use it as a supplement, not a replacement, for self-reporting.
+
+All data is written to a local SQLite database in `~/.agent-observability/`. No data leaves your machine.
+
+### 5. How to verify it's working
+
+```bash
 agent-obs dashboard
+# Open http://localhost:9400
+# You should see a session appear after your agent runs a task
 ```
 
-Then open **http://localhost:9400** in your browser. The dashboard shows:
+The dashboard shows:
 
@@ -74,0 +111,0 @@ - Session list with grade badges (A-F), timestamps, and token totals

New alerts

Long strings

Supply chain risk

Contains long string literals, which may be a sign of obfuscated or packed code.

Improved metrics

Total package byte prevSize

91762

8.78%

Lines of code

1571

8.87%

Number of lines in readme file

312

13.45%

Worsened metrics

Number of low supply chain risk alerts

5

25%

No dependency changes