harness-evolver - npm Package Compare versions

Comparing version 6.1.0 to 6.2.0

.claude-plugin/marketplace.json

+{
+  "name": "harness-evolver-marketplace",
+  "owner": {
+    "name": "Raphael Valdetaro"
+  },
+  "metadata": {
+    "description": "LangSmith-native autonomous agent optimization plugin"
+  },
+  "plugins": [
+    {
+      "name": "harness-evolver",
+      "source": "./",
+      "description": "Evolves LLM agent code using multi-agent proposers, LangSmith experiments, and git worktrees",
+      "version": "6.1.0",
+      "author": {
+        "name": "Raphael Valdetaro"
+      },
+      "homepage": "https://github.com/raphaelchristi/harness-evolver",
+      "repository": "https://github.com/raphaelchristi/harness-evolver",
+      "license": "MIT",
+      "keywords": ["langsmith", "optimization", "evolution", "llm", "agent"],
+      "category": "development"
+    }
+  ]
+}

tools/log_iteration.py

+#!/usr/bin/env python3
+"""Log an evolution iteration as a LangSmith run.
+
+Creates a run in the evolution tracing project with iteration metadata.
+Returns the run ID and dotted_order for nesting proposer traces as children.
+
+Usage:
+    python3 log_iteration.py --config .evolver.json --action start --version v001
+    python3 log_iteration.py --config .evolver.json --action end --run-id <id> --score 0.85 --merged true
+
+Requires: pip install langsmith
+"""
+
+import argparse
+import json
+import os
+import sys
+from datetime import datetime, timezone
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from _common import ensure_langsmith_api_key, load_config
+
+
+def start_iteration(client, project_name, config, version):
+    """Create a new LangSmith run for an iteration."""
+    from langsmith import RunTree
+
+    run = RunTree(
+        name=f"iteration-{version}",
+        run_type="chain",
+        project_name=project_name,
+        inputs={
+            "version": version,
+            "best_score": config.get("best_score", 0),
+            "iterations": config.get("iterations", 0),
+            "mode": config.get("mode", "balanced"),
+            "evaluators": config.get("evaluators", []),
+        },
+        extra={
+            "metadata": {
+                "evolver_version": version,
+                "agent_project": config.get("project", "unknown"),
+                "mode": config.get("mode", "balanced"),
+            }
+        },
+    )
+    run.post()
+
+    return {
+        "run_id": str(run.id),
+        "dotted_order": run.dotted_order,
+        "trace_id": str(run.trace_id),
+    }
+
+
+def end_iteration(client, run_id, score, merged, approach, lens, candidates, duration):
+    """Update an existing iteration run with results."""
+    outputs = {
+        "score": score,
+        "merged": merged,
+        "approach": approach,
+        "lens": lens,
+        "candidates_evaluated": candidates,
+    }
+
+    client.update_run(
+        run_id=run_id,
+        outputs=outputs,
+        end_time=datetime.now(timezone.utc),
+        extra={
+            "metadata": {
+                "score": score,
+                "merged": merged,
+                "approach": approach,
+                "lens": lens,
+                "duration_seconds": duration,
+            }
+        },
+    )
+
+    try:
+        client.create_feedback(
+            run_id=run_id,
+            key="score",
+            score=score,
+            comment=f"{'Merged' if merged else 'Not merged'}: {approach}",
+        )
+    except Exception:
+        pass
+
+    return {"run_id": run_id, "score": score, "merged": merged}
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Log evolution iteration to LangSmith")
+    parser.add_argument("--config", default=".evolver.json")
+    parser.add_argument("--action", required=True, choices=["start", "end"])
+    parser.add_argument("--version", default=None)
+    parser.add_argument("--project", default=None)
+    parser.add_argument("--run-id", default=None)
+    parser.add_argument("--score", type=float, default=0.0)
+    parser.add_argument("--merged", type=lambda x: x.lower() == "true", default=False)
+    parser.add_argument("--approach", default="")
+    parser.add_argument("--lens", default="")
+    parser.add_argument("--candidates", type=int, default=0)
+    parser.add_argument("--duration", type=int, default=0)
+    parser.add_argument("--output", default=None)
+    args = parser.parse_args()
+
+    ensure_langsmith_api_key()
+
+    config = load_config(args.config)
+    if not config:
+        print('{"error": "config not found"}')
+        sys.exit(1)
+
+    from langsmith import Client
+    client = Client()
+
+    project_name = args.project or f"harness-evolution-{config.get('project', 'unknown')}"
+
+    if args.action == "start":
+        version = args.version or f"v{config.get('iterations', 0) + 1:03d}"
+        result = start_iteration(client, project_name, config, version)
+        output = json.dumps(result, indent=2)
+        if args.output:
+            with open(args.output, "w") as f:
+                f.write(output)
+        print(output)
+
+    elif args.action == "end":
+        if not args.run_id:
+            print('{"error": "--run-id required for --action end"}', file=sys.stderr)
+            sys.exit(1)
+        result = end_iteration(
+            client, args.run_id, args.score, args.merged,
+            args.approach, args.lens, args.candidates, args.duration,
+        )
+        print(json.dumps(result, indent=2))
+
+
+if __name__ == "__main__":
+    main()

.claude-plugin/plugin.json

 {
   "name": "harness-evolver",
   "description": "LangSmith-native autonomous agent optimization — evolves LLM agent code using multi-agent proposers, LangSmith experiments, and git worktrees",
-  "version": "6.1.0",
+  "version": "6.2.0",
   "author": {
@@ -6,0 +6,0 @@ "name": "Raphael Valdetaro"

package.json

 {
   "name": "harness-evolver",
-  "version": "6.1.0",
+  "version": "6.2.0",
   "description": "LangSmith-native autonomous agent optimization for Claude Code",
@@ -5,0 +5,0 @@ "author": "Raphael Valdetaro",

README.md

@@ -53,2 +53,29 @@ <p align="center">
 
+## What It Looks Like
+
+```mermaid
+xychart-beta
+    title "Best Score Over Evolution Iterations"
+    x-axis ["base", "v001", "v002", "v003", "v004", "v005", "v006", "v007", "v008", "v009"]
+    y-axis "Correctness" 0 --> 1
+    line [0.31, 0.48, 0.52, 0.52, 0.67, 0.71, 0.71, 0.71, 0.79, 0.84]
+```
+
+| Iter | Score | Merged? | What happened |
+|---|---|---|---|
+| baseline | 0.31 | — | Broken tool calls, hallucinations, no error handling |
+| v001 | 0.48 | Yes | Fixed input parsing, added retry logic (+0.17) |
+| v002 | 0.52 | Yes | Prompt rewrite to reduce hallucinations (+0.04) |
+| v003 | 0.49 | **No** | Attempted retrieval change — regressed, rejected by gate |
+| v004 | 0.67 | Yes | Architect triggered: chain → ReAct restructure (+0.15) |
+| v005 | 0.71 | Yes | Output validation + citation grounding (+0.04) |
+| v006 | 0.68 | **No** | Tried fewer tool calls — broke edge cases, rejected |
+| v007 | 0.70 | **No** | Prompt tweak — within noise margin, not merged |
+| v008 | 0.79 | Yes | Evolution memory insight: combined v003's retrieval with v005's validation (+0.08) |
+| v009 | 0.84 | Yes | Fine-tuned rubric alignment from judge feedback (+0.05) |
+
+Real pattern: initial jump → plateau → architectural breakthrough → small gains → stagnation → memory-driven recovery. Regressions rejected automatically. Not every iteration improves — that's the point of gate checks.
+
+---
+
 ## How It Works
@@ -110,2 +137,15 @@
 
+## Companion: LangSmith Tracing
+
+For full observability into what each proposer does during evolution (every file read, edit, and commit), install the [LangSmith tracing plugin](https://github.com/langchain-ai/langsmith-claude-code-plugins):
+
+```
+/plugin marketplace add langchain-ai/langsmith-claude-code-plugins
+/plugin install langsmith-tracing@langsmith-claude-code-plugins
+```
+
+With both plugins installed, the evolution loop traces to LangSmith as a hierarchy: iteration → proposers → tool calls.
+
+---
+
 ## References
@@ -112,0 +152,0 @@

skills/evolve/SKILL.md

@@ -82,3 +82,3 @@ ---
 
-### 0. Read State
+### 0. Read State + Start Iteration Trace
 
@@ -88,4 +88,14 @@ ```bash
 PROJECT_DIR=$(python3 -c "import json; print(json.load(open('.evolver.json')).get('project_dir', ''))")
+ITER_START=$(date +%s)
 ```
 
+**Start iteration trace** (logs to LangSmith for observability):
+```bash
+ITER_TRACE=$($EVOLVER_PY $TOOLS/log_iteration.py --config .evolver.json --action start --version v{NNN} 2>/dev/null)
+ITER_RUN_ID=$(echo "$ITER_TRACE" | python3 -c "import sys,json; print(json.load(sys.stdin).get('run_id',''))" 2>/dev/null)
+ITER_DOTTED_ORDER=$(echo "$ITER_TRACE" | python3 -c "import sys,json; print(json.load(sys.stdin).get('dotted_order',''))" 2>/dev/null)
+```
+
+If log_iteration.py fails (no LangSmith, no key), the loop continues — tracing is optional.
+
 If `$BEST` is empty (no baseline ran), skip data gathering — proposers work from code analysis only.
@@ -124,5 +134,7 @@
 
-**IMPORTANT**: After each proposer worktree is created, copy untracked files BEFORE the agent starts reading. Always use **absolute paths** (relative paths fail when Bash CWD differs from project root):
+**IMPORTANT**: After each proposer worktree is created, copy untracked files and set trace nesting. Always use **absolute paths**:
 ```bash
 SRC="$(pwd)"
+# If langsmith-tracing companion is installed, proposer traces nest under iteration:
+[ -n "$ITER_DOTTED_ORDER" ] && export CC_LANGSMITH_PARENT_DOTTED_ORDER="$ITER_DOTTED_ORDER"
 # For each worktree (after Agent creates it, before agent reads files):
@@ -270,2 +282,11 @@ cp "$SRC/.evolver.json" "$WT_PROJECT/.evolver.json"
 
+**End iteration trace**:
+```bash
+ITER_DURATION=$(( $(date +%s) - ITER_START ))
+$EVOLVER_PY $TOOLS/log_iteration.py --config .evolver.json --action end \
+    --run-id "$ITER_RUN_ID" --score {winner_score} --merged {true|false} \
+    --approach "{approach}" --lens "{lens}" --candidates {num_evaluated} \
+    --duration $ITER_DURATION 2>/dev/null
+```
+
 **Consolidate** (background):
@@ -272,0 +293,0 @@ ```

skills/setup/SKILL.md

@@ -260,1 +260,2 @@ ---
 - **Eval concurrency** defaults to 3 (runs 3 examples in parallel). If the agent can't handle parallel execution (writes to shared files, uses a fixed port, holds a DB lock), set `eval_concurrency: 1` in `.evolver.json` after setup.
+- **Companion plugin**: For full proposer observability, recommend installing `langsmith-tracing` from `langchain-ai/langsmith-claude-code-plugins`. Each proposer's file reads, edits, and commits become visible in LangSmith.

New alerts

AI-detected potential code anomaly

Supply chain risk

AI has identified unusual behaviors that may pose a security risk.

Fixed alerts

AI-detected potential code anomaly

Supply chain risk

AI has identified unusual behaviors that may pose a security risk.

Improved metrics

Total package byte prevSize

336597

2.58%

Number of package files

41

5.13%

Lines of code

6112

2.41%

Number of lines in readme file

169

31.01%

No dependency changes