🎩 You're Invited:Meet the Socket team at Black Hat in Las Vegas, August 3-6.RSVP →
Sign In

harness-evolver

Package Overview
Dependencies
Maintainers
1
Versions
104
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

harness-evolver - npm Package Compare versions

Comparing version
6.2.0
 to
6.3.0
+179
tools/cleanup_worktrees.py
#!/usr/bin/env python3
"""Remove orphan git worktrees from .claude/worktrees/ after evaluation.
Prevents accumulation of 6+ worktrees per session. Cleans up directories
and runs `git worktree prune` afterward.
Stdlib-only — no langsmith dependency.
Usage:
# Dry run — show what would be removed
python3 cleanup_worktrees.py --dry-run
# Remove all worktrees
python3 cleanup_worktrees.py
# Keep specific worktrees by name
python3 cleanup_worktrees.py --keep winner-v003 candidate-v004a
# Specify project directory
python3 cleanup_worktrees.py --dir /path/to/project --dry-run
"""
import argparse
import os
import shutil
import subprocess
import sys
WORKTREE_SUBDIR = os.path.join(".claude", "worktrees")
def find_worktrees(base_dir):
"""Find all directories under .claude/worktrees/ in the given project.
Returns a list of absolute paths to worktree directories.
"""
worktrees_root = os.path.join(base_dir, WORKTREE_SUBDIR)
if not os.path.isdir(worktrees_root):
return []
entries = []
for name in sorted(os.listdir(worktrees_root)):
full = os.path.join(worktrees_root, name)
if os.path.isdir(full):
entries.append(full)
return entries
def remove_worktree(path, dry_run=False):
"""Remove a single worktree directory.
Tries `git worktree remove --force` first. If that fails (e.g. the
worktree wasn't registered with git), falls back to shutil.rmtree.
Returns a dict with keys: path, method, success, error.
"""
result = {"path": path, "method": None, "success": False, "error": None}
if dry_run:
result["method"] = "dry-run"
result["success"] = True
return result
# Try git worktree remove --force
try:
proc = subprocess.run(
["git", "worktree", "remove", "--force", path],
capture_output=True,
text=True,
timeout=30,
)
if proc.returncode == 0:
result["method"] = "git worktree remove"
result["success"] = True
return result
except (subprocess.TimeoutExpired, FileNotFoundError, OSError):
pass
# Fallback: shutil.rmtree
try:
shutil.rmtree(path)
result["method"] = "shutil.rmtree"
result["success"] = True
except OSError as exc:
result["method"] = "shutil.rmtree"
result["error"] = str(exc)
return result
def prune_worktrees(project_dir):
"""Run `git worktree prune` to clean up stale worktree bookkeeping."""
try:
subprocess.run(
["git", "worktree", "prune"],
cwd=project_dir,
capture_output=True,
text=True,
timeout=30,
)
except (subprocess.TimeoutExpired, FileNotFoundError, OSError):
pass
def main():
parser = argparse.ArgumentParser(
description="Remove orphan git worktrees from .claude/worktrees/"
)
parser.add_argument(
"--dir",
default=".",
help="Project directory (default: current directory)",
)
parser.add_argument(
"--keep",
nargs="*",
default=[],
metavar="NAME",
help="Worktree directory names to keep (basenames, not full paths)",
)
parser.add_argument(
"--dry-run",
action="store_true",
help="Show what would be removed without actually removing",
)
args = parser.parse_args()
project_dir = os.path.abspath(args.dir)
keep_set = set(args.keep)
worktrees = find_worktrees(project_dir)
if not worktrees:
print(f"No worktrees found under {os.path.join(project_dir, WORKTREE_SUBDIR)}")
return
# Partition into keep / remove
to_remove = []
to_keep = []
for wt in worktrees:
name = os.path.basename(wt)
if name in keep_set:
to_keep.append(wt)
else:
to_remove.append(wt)
if to_keep:
print(f"Keeping {len(to_keep)} worktree(s): {', '.join(os.path.basename(w) for w in to_keep)}")
if not to_remove:
print("Nothing to remove.")
return
action = "Would remove" if args.dry_run else "Removing"
print(f"{action} {len(to_remove)} worktree(s):\n")
results = []
for wt in to_remove:
res = remove_worktree(wt, dry_run=args.dry_run)
results.append(res)
name = os.path.basename(wt)
if res["success"]:
method = f" ({res['method']})" if res["method"] != "dry-run" else ""
print(f" [ok] {name}{method}")
else:
print(f" [FAIL] {name} — {res['error']}")
# Prune stale worktree references
if not args.dry_run:
prune_worktrees(project_dir)
print("\nRan `git worktree prune`.")
failed = [r for r in results if not r["success"]]
if failed:
print(f"\n{len(failed)} removal(s) failed.", file=sys.stderr)
sys.exit(1)
if __name__ == "__main__":
main()
+164
tools/update_config.py
#!/usr/bin/env python3
"""Atomic .evolver.json updates after merge.
Replaces inline Python for config backup/restore/update during the evolve loop.
Three actions:
backup — save .evolver.json to .evolver.json.bak before merge
restore — restore from .bak after merge overwrites config, delete .bak
update — update best_experiment, best_score, increment iterations,
append enriched history entry
Stdlib-only. No external dependencies.
Usage:
# Before merge — save config
python3 update_config.py --config .evolver.json --action backup
# After merge — restore config (merge brought worktree's stale copy)
python3 update_config.py --config .evolver.json --action restore
# Update config with winner data
python3 update_config.py --config .evolver.json --action update \
--winner-experiment v003-abc --winner-score 0.87 \
--approach "fixed JSON parsing" --lens "failure_cluster" \
--tokens 15000 --latency-ms 4500 --error-count 1 \
--passing 18 --total 20 --per-evaluator '{"accuracy":0.9,"format":0.85}' \
--code-loc 120
"""
import argparse
import json
import os
import shutil
import sys
from datetime import datetime, timezone
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from _common import write_config_atomic, load_config
def action_backup(config_path):
"""Copy .evolver.json to .evolver.json.bak."""
bak_path = config_path + ".bak"
if not os.path.exists(config_path):
print(f"Error: config not found: {config_path}", file=sys.stderr)
return False
shutil.copy2(config_path, bak_path)
print(json.dumps({"action": "backup", "path": bak_path}))
return True
def action_restore(config_path):
"""Restore .evolver.json from .bak, delete .bak."""
bak_path = config_path + ".bak"
if not os.path.exists(bak_path):
print(f"Error: backup not found: {bak_path}", file=sys.stderr)
return False
shutil.copy2(bak_path, config_path)
os.remove(bak_path)
print(json.dumps({"action": "restore", "path": config_path}))
return True
def action_update(config_path, args):
"""Update best_experiment, best_score, iterations, and append history."""
config = load_config(config_path)
# Update top-level fields
config["best_experiment"] = args.winner_experiment
config["best_score"] = args.winner_score
config["iterations"] = config.get("iterations", 0) + 1
# Build enriched history entry
version = f"v{config['iterations']:03d}"
entry = {
"version": version,
"experiment": args.winner_experiment,
"score": args.winner_score,
"timestamp": datetime.now(timezone.utc).isoformat(),
}
# Optional enrichment fields
if args.approach:
entry["approach"] = args.approach
if args.lens:
entry["lens"] = args.lens
if args.tokens is not None:
entry["tokens"] = args.tokens
if args.latency_ms is not None:
entry["latency_ms"] = args.latency_ms
if args.error_count is not None:
entry["error_count"] = args.error_count
if args.passing is not None:
entry["passing"] = args.passing
if args.total is not None:
entry["total"] = args.total
if args.per_evaluator:
try:
entry["per_evaluator"] = json.loads(args.per_evaluator)
except json.JSONDecodeError:
print(f"Warning: --per-evaluator is not valid JSON, skipping", file=sys.stderr)
if args.code_loc is not None:
entry["code_loc"] = args.code_loc
# Append to history
if "history" not in config:
config["history"] = []
config["history"].append(entry)
write_config_atomic(config_path, config)
print(json.dumps({
"action": "update",
"version": version,
"best_score": args.winner_score,
"iterations": config["iterations"],
}))
return True
def main():
parser = argparse.ArgumentParser(
description="Atomic .evolver.json updates after merge."
)
parser.add_argument(
"--config", required=True, help="Path to .evolver.json"
)
parser.add_argument(
"--action", required=True, choices=["backup", "restore", "update"],
help="Action: backup, restore, or update"
)
# Update-specific flags
parser.add_argument("--winner-experiment", help="Winning experiment name")
parser.add_argument("--winner-score", type=float, help="Winning score")
parser.add_argument("--approach", default="", help="Brief description of winning approach")
parser.add_argument("--lens", default="", help="Investigation lens used")
parser.add_argument("--tokens", type=int, default=None, help="Token usage")
parser.add_argument("--latency-ms", type=int, default=None, help="Latency in milliseconds")
parser.add_argument("--error-count", type=int, default=None, help="Number of errors")
parser.add_argument("--passing", type=int, default=None, help="Number of passing examples")
parser.add_argument("--total", type=int, default=None, help="Total number of examples")
parser.add_argument("--per-evaluator", default=None, help="Per-evaluator scores as JSON string")
parser.add_argument("--code-loc", type=int, default=None, help="Lines of code changed")
args = parser.parse_args()
if args.action == "update":
if not args.winner_experiment or args.winner_score is None:
parser.error("--winner-experiment and --winner-score are required for --action update")
if args.action == "backup":
ok = action_backup(args.config)
elif args.action == "restore":
ok = action_restore(args.config)
elif args.action == "update":
ok = action_update(args.config, args)
else:
parser.error(f"Unknown action: {args.action}")
ok = False
sys.exit(0 if ok else 1)
if __name__ == "__main__":
main()
+1
-1
.claude-plugin/plugin.json
{
"name": "harness-evolver",
"description": "LangSmith-native autonomous agent optimization — evolves LLM agent code using multi-agent proposers, LangSmith experiments, and git worktrees",
"version": "6.2.0",
"version": "6.3.0",
"author": {

@@ -6,0 +6,0 @@ "name": "Raphael Valdetaro"

+5
-2
agents/harness-evaluator.md

@@ -141,3 +141,4 @@ ---

Example for one run:
**Rubric pinning**: Include the rubric text (if available) in the comment. This makes scores reproducible and diagnosable across iterations:
```bash

@@ -147,6 +148,8 @@ langsmith-cli --json feedback create "run-uuid-here" \

--score 1.0 \
--comment "Response correctly identifies the applicable regulation and provides accurate guidance." \
--comment "RUBRIC: Should mention null safety and Android. JUDGMENT: Lists all features correctly." \
--source model
```
If no rubric exists, use standard format without the RUBRIC prefix. The `RUBRIC:` prefix lets downstream tools compare rubric interpretations across iterations.
### Phase 4: Summary

@@ -153,0 +156,0 @@ 

+1
-1
package.json
{
"name": "harness-evolver",
"version": "6.2.0",
"version": "6.3.0",
"description": "LangSmith-native autonomous agent optimization for Claude Code",

@@ -5,0 +5,0 @@ "author": "Raphael Valdetaro",

+18
-11
skills/evolve/SKILL.md

@@ -238,19 +238,21 @@ ---

1. **Save config before merge** (merge will overwrite with worktree's stale copy):
```bash
cp .evolver.json .evolver.json.bak
```
# 1. Backup config (merge will overwrite with worktree's stale copy)
$EVOLVER_PY $TOOLS/update_config.py --config .evolver.json --action backup
2. **Merge the winner**:
```bash
# 2. Merge
git merge {winner_branch} --no-edit -m "evolve: merge v{NNN} (score: {score})"
```
3. **Restore config and update** (the merge brought the worktree's old .evolver.json — restore ours):
```bash
cp .evolver.json.bak .evolver.json
# 3. Restore config (merge brought stale copy)
$EVOLVER_PY $TOOLS/update_config.py --config .evolver.json --action restore
# 4. Update config with enriched history (one command, no inline Python)
$EVOLVER_PY $TOOLS/update_config.py --config .evolver.json --action update \
--winner-experiment "{winner}" --winner-score {score} \
--approach "{approach}" --lens "{lens}" \
--tokens {tokens} --latency-ms {latency} --error-count {errors} \
--passing {passing} --total {total} \
--per-evaluator '{json_dict}' --code-loc {loc}
```
4. **Update config** with enriched history (score, tokens, latency, errors, passing, total, per_evaluator, approach, lens, code_loc).
5. **Git-tag** for rollback:

@@ -304,2 +306,7 @@

**Cleanup worktrees** (free disk space after eval):
```bash
$EVOLVER_PY $TOOLS/cleanup_worktrees.py --dir "$(pwd)"
```
### 7. Gate Check

@@ -306,0 +313,0 @@ 

+8
-0
tools/run_eval.py

@@ -153,2 +153,4 @@ #!/usr/bin/env python3

parser.add_argument("--preflight-only", action="store_true", help="Run preflight checks only (API key, config, canary) then exit")
parser.add_argument("--retry-on-rate-limit", action="store_true",
help="If rate-limited, wait 60s and suggest re-run")
parser.add_argument("--sample", type=int, default=None, help="Evaluate a random sample of N examples instead of all")

@@ -321,2 +323,8 @@ args = parser.parse_args()

if aborted_early and args.retry_on_rate_limit:
import time
print(f"\n Rate-limited. Waiting 60s before suggesting re-run...", file=sys.stderr)
time.sleep(60)
print(f" Wait complete. Re-run this command to retry remaining examples.", file=sys.stderr)
mean_score = sum(scores) / len(scores) if scores else 0.0

@@ -323,0 +331,0 @@ num_examples = len(per_example)