From 95981d41e2a08470fe92933d336562f73faad154 Mon Sep 17 00:00:00 2001 From: Kevin Date: Fri, 3 Apr 2026 00:16:54 -0700 Subject: [PATCH 01/12] clean up README structure and add thirdlayer branding Co-Authored-By: Claude Opus 4.6 (1M context) --- README.md | 65 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 39 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 60a54620..b9efe502 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,28 @@ # autoagent +> Like [autoresearch](https://github.com/karpathy/autoresearch) but for agent engineering. Give an AI agent a task, let it build and iterate on an agent harness autonomously overnight. It modifies the system prompt, tools, agent configuration, and orchestration, runs the benchmark, checks the score, keeps or discards the change, and repeats. + ![teaser](progress.png) -Like [autoresearch](https://github.com/karpathy/autoresearch) but for agent engineering. Give an AI agent a task, let it build and iterate on an agent harness autonomously overnight. It modifies the system prompt, tools, agent configuration, and orchestration, runs the benchmark, checks the score, keeps or discards the change, and repeats. +The core idea is the same: you're not touching the harness Python files like you normally would as an engineer. Instead, you program `program.md`, the Markdown file that provides context to the meta-agent and defines the agent-engineering loop. -The core idea is the same: you're not touching the harness Python files like you normally would as an engineer. Instead, you program `program.md` — the Markdown file that provides context to the meta-agent and defines the agent-engineering loop. +If you're interested in auto-configuring agents for your product, [get in touch](https://form.typeform.com/to/ZQbnbO09). ## How it works The repo has a few files and directories that matter: -- **`agent.py`** — the entire harness under test in a single file. It contains +- **`agent.py`** -- the entire harness under test in a single file. It contains config, tool definitions, agent registry, routing/orchestration, and the Harbor adapter boundary. The adapter section is explicitly marked as fixed; the rest is the primary edit surface for the meta-agent. -- **`program.md`** — instructions for the meta-agent + the directive (what +- **`program.md`** -- instructions for the meta-agent + the directive (what kind of agent to build). **This file is edited by the human**. -- **`tasks/`** — evaluation tasks in +- **`tasks/`** -- evaluation tasks in [harbor](https://github.com/laude-institute/harbor) format. In a clean baseline branch, benchmark payloads may be omitted and added in benchmark-specific branches. -- **`.agent/`** — optional workspace artifacts for reusable instructions, +- **`.agent/`** -- optional workspace artifacts for reusable instructions, notes, prompts, or skills. The metric is total **score** produced by the benchmark's task test suites. The @@ -70,16 +72,16 @@ benchmark, diagnose failures, modify `agent.py`, and iterate. ## Project structure ```text -agent.py — single-file harness under test - editable harness section — prompt, registries, tools, routing - fixed adapter section — Harbor integration + trajectory serialization -program.md — meta-agent instructions + directive -Dockerfile.base — base image -.agent/ — optional agent workspace artifacts -tasks/ — benchmark tasks, typically added in benchmark-specific branches -jobs/ — Harbor job outputs -results.tsv — experiment log (created by meta-agent, gitignored) -run.log — latest run output +agent.py -- single-file harness under test + editable harness section -- prompt, registries, tools, routing + fixed adapter section -- Harbor integration + trajectory serialization +program.md -- meta-agent instructions + directive +Dockerfile.base -- base image +.agent/ -- optional agent workspace artifacts +tasks/ -- benchmark tasks, typically added in benchmark-specific branches +jobs/ -- Harbor job outputs +results.tsv -- experiment log (created by meta-agent, gitignored) +run.log -- latest run output ``` ## Task format @@ -88,17 +90,17 @@ When present, tasks follow [harbor](https://github.com/laude-institute/harbor)'s ```text tasks/my-task/ - task.toml — config (timeouts, metadata) - instruction.md — prompt sent to the agent + task.toml -- config (timeouts, metadata) + instruction.md -- prompt sent to the agent tests/ - test.sh — entry point, writes /logs/reward.txt - test.py — verification (deterministic or LLM-as-judge) + test.sh -- entry point, writes /logs/reward.txt + test.py -- verification (deterministic or LLM-as-judge) environment/ - Dockerfile — task container (FROM autoagent-base) - files/ — reference files mounted into container + Dockerfile -- task container (FROM autoagent-base) + files/ -- reference files mounted into container ``` -Tests write a score (0.0–1.0) to the verifier logs. The meta-agent hill-climbs +Tests write a score (0.0-1.0) to the verifier logs. The meta-agent hill-climbs on this. ## Design choices @@ -108,8 +110,7 @@ on this. - **Single-file, registry-driven harness.** The implementation lives in one file for simplicity, but agent and tool registration stay structured so the harness can still evolve cleanly. -- **Docker isolation.** The agent-under-test runs in a container. It can't - damage the host. +- **Docker isolation.** The agent runs in a container. It can't damage the host. - **Score-driven.** Every experiment produces a numeric score. Keep if better, discard if not. Same loop as autoresearch. - **Harbor-compatible tasks.** Tasks use the same format as harbor benchmarks, @@ -130,7 +131,7 @@ docker system prune -a -f docker container prune -f ``` -If Docker becomes unresponsive (e.g. after many concurrent runs), restart +If Docker becomes unresponsive (for example after many concurrent runs), restart Docker Desktop: ```bash @@ -144,3 +145,15 @@ You can equip the agent with [Agent Skills for Context Engineering](https://gith ## License MIT + +--- + +

maintained by

+

+ + thirdlayer + +

+

+ thirdlayer.inc +

From 74c0bf361174f9e47b7fb6efb098e643615e9547 Mon Sep 17 00:00:00 2001 From: Kevin Date: Fri, 3 Apr 2026 00:17:25 -0700 Subject: [PATCH 02/12] move thirdlayer branding to top of README Co-Authored-By: Claude Opus 4.6 (1M context) --- README.md | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index b9efe502..2ca89f9f 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,12 @@ +

+ + thirdlayer + +

+

+ thirdlayer.inc +

+ # autoagent > Like [autoresearch](https://github.com/karpathy/autoresearch) but for agent engineering. Give an AI agent a task, let it build and iterate on an agent harness autonomously overnight. It modifies the system prompt, tools, agent configuration, and orchestration, runs the benchmark, checks the score, keeps or discards the change, and repeats. @@ -146,14 +155,3 @@ You can equip the agent with [Agent Skills for Context Engineering](https://gith MIT ---- - -

maintained by

-

- - thirdlayer - -

-

- thirdlayer.inc -

From 467540d047cd4fa9030be74871e62b201b2f55e2 Mon Sep 17 00:00:00 2001 From: Kevin Date: Fri, 3 Apr 2026 00:18:59 -0700 Subject: [PATCH 03/12] resize progress graph to 600px width Co-Authored-By: Claude Opus 4.6 (1M context) --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2ca89f9f..e2323d8c 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,9 @@ > Like [autoresearch](https://github.com/karpathy/autoresearch) but for agent engineering. Give an AI agent a task, let it build and iterate on an agent harness autonomously overnight. It modifies the system prompt, tools, agent configuration, and orchestration, runs the benchmark, checks the score, keeps or discards the change, and repeats. -![teaser](progress.png) +

+ teaser +

The core idea is the same: you're not touching the harness Python files like you normally would as an engineer. Instead, you program `program.md`, the Markdown file that provides context to the meta-agent and defines the agent-engineering loop. From 2c9fcd643d6a43975fde2a79ea769ebb8b3cd1f4 Mon Sep 17 00:00:00 2001 From: Kevin Date: Fri, 3 Apr 2026 00:24:01 -0700 Subject: [PATCH 04/12] add product signup quote under thirdlayer branding Co-Authored-By: Claude Opus 4.6 (1M context) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e2323d8c..e9bb9fef 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,8 @@ thirdlayer.inc

+> We're launching a product around self-configuring agents soon. [Sign up here](https://form.typeform.com/to/ZQbnbO09). + # autoagent > Like [autoresearch](https://github.com/karpathy/autoresearch) but for agent engineering. Give an AI agent a task, let it build and iterate on an agent harness autonomously overnight. It modifies the system prompt, tools, agent configuration, and orchestration, runs the benchmark, checks the score, keeps or discards the change, and repeats. @@ -17,8 +19,6 @@ The core idea is the same: you're not touching the harness Python files like you normally would as an engineer. Instead, you program `program.md`, the Markdown file that provides context to the meta-agent and defines the agent-engineering loop. -If you're interested in auto-configuring agents for your product, [get in touch](https://form.typeform.com/to/ZQbnbO09). - ## How it works The repo has a few files and directories that matter: From 0602cbb79e90a714237e2ed66524dc0f4b3d11e7 Mon Sep 17 00:00:00 2001 From: Kevin Date: Fri, 3 Apr 2026 00:25:28 -0700 Subject: [PATCH 05/12] Built by thirdlayer.inc, capitalize AutoAgent, full-width graph Co-Authored-By: Claude Opus 4.6 (1M context) --- README.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index e9bb9fef..3e6bac5c 100644 --- a/README.md +++ b/README.md @@ -4,18 +4,16 @@

- thirdlayer.inc + Built by thirdlayer.inc

> We're launching a product around self-configuring agents soon. [Sign up here](https://form.typeform.com/to/ZQbnbO09). -# autoagent +# AutoAgent > Like [autoresearch](https://github.com/karpathy/autoresearch) but for agent engineering. Give an AI agent a task, let it build and iterate on an agent harness autonomously overnight. It modifies the system prompt, tools, agent configuration, and orchestration, runs the benchmark, checks the score, keeps or discards the change, and repeats. -

- teaser -

+![teaser](progress.png) The core idea is the same: you're not touching the harness Python files like you normally would as an engineer. Instead, you program `program.md`, the Markdown file that provides context to the meta-agent and defines the agent-engineering loop. From b0b0a854e5b20824ba443b64601cbada32090e28 Mon Sep 17 00:00:00 2001 From: Kevin Date: Fri, 3 Apr 2026 00:26:05 -0700 Subject: [PATCH 06/12] remove autoresearch link Co-Authored-By: Claude Opus 4.6 (1M context) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3e6bac5c..6888ac5e 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ # AutoAgent -> Like [autoresearch](https://github.com/karpathy/autoresearch) but for agent engineering. Give an AI agent a task, let it build and iterate on an agent harness autonomously overnight. It modifies the system prompt, tools, agent configuration, and orchestration, runs the benchmark, checks the score, keeps or discards the change, and repeats. +> Like autoresearch but for agent engineering. Give an AI agent a task, let it build and iterate on an agent harness autonomously overnight. It modifies the system prompt, tools, agent configuration, and orchestration, runs the benchmark, checks the score, keeps or discards the change, and repeats. ![teaser](progress.png) From 2618a8b074da5b0e6f5a9f5fb855fd7a73d2236f Mon Sep 17 00:00:00 2001 From: Szymon Paluch Date: Fri, 3 Apr 2026 11:42:28 +0200 Subject: [PATCH 07/12] feat: add Claude Code CLI adapter for Harbor Add a new agent variant that uses the Claude Code CLI (`claude --print`) as the execution backend instead of direct SDK calls. This enables running AutoAgent benchmarks with Claude Code's built-in tool suite. Files added: - agent-claude-code.py: Harbor adapter with editable/fixed boundary - Dockerfile.claude-code: base image with Node.js + Claude Code CLI - program-claude-code.md: meta-agent directive for this variant --- Dockerfile.claude-code | 25 ++++ agent-claude-code.py | 329 +++++++++++++++++++++++++++++++++++++++++ program-claude-code.md | 89 +++++++++++ 3 files changed, 443 insertions(+) create mode 100644 Dockerfile.claude-code create mode 100644 agent-claude-code.py create mode 100644 program-claude-code.md diff --git a/Dockerfile.claude-code b/Dockerfile.claude-code new file mode 100644 index 00000000..50671a29 --- /dev/null +++ b/Dockerfile.claude-code @@ -0,0 +1,25 @@ +FROM ghcr.io/astral-sh/uv:python3.12-bookworm-slim + +RUN apt-get update && \ + apt-get install -y --no-install-recommends ca-certificates git curl && \ + rm -rf /var/lib/apt/lists/* + +# Install Node.js (required for Claude Code CLI) +RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - && \ + apt-get install -y --no-install-recommends nodejs && \ + rm -rf /var/lib/apt/lists/* + +# Install Claude Code CLI globally +RUN npm install -g @anthropic-ai/claude-code + +WORKDIR /app + +# Python deps +COPY pyproject.toml ./ +RUN uv pip install --system . + +# Agent code +COPY agent-claude-code.py ./ + +RUN ln -sf $(which python3) /usr/local/bin/python +RUN mkdir -p /logs /app/output diff --git a/agent-claude-code.py b/agent-claude-code.py new file mode 100644 index 00000000..6093491d --- /dev/null +++ b/agent-claude-code.py @@ -0,0 +1,329 @@ +""" +Claude Code CLI adapter for Harbor. + +Spawns `claude` as a subprocess inside the container. The meta-agent iterates +on the editable section (system prompt, CLI flags, context strategy). + +Run all tasks: + docker build -f Dockerfile.claude-code -t autoagent-base . + set -a && source .env && set +a + uv run harbor run -p tasks/ --agent-import-path agent-claude-code:AutoAgent -o jobs --job-name latest > run.log 2>&1 +""" + +from __future__ import annotations + +import asyncio +import json +import os +import time +from datetime import datetime, timezone +from pathlib import Path + + +# =========================================================================== +# AGENT CONFIG — meta-agent modifies this section +# =========================================================================== + +SYSTEM_PROMPT = """You are a highly capable task-completion agent. You solve tasks by reading instructions, analyzing the problem, writing and executing code, and producing the required output files. + +## Approach +1. Read /task/instruction.md to understand what's required. +2. Explore the working environment — check what files, tools, and libraries are available. +3. Plan your approach, then execute step by step. +4. Write output files to the exact paths specified in the instructions. +5. Verify your output before finishing. + +## Key rules +- Use python3 (not python) for running scripts. +- Use Bash to run shell commands, install packages, inspect files. +- For data analysis: pandas, numpy, openpyxl are available. +- For file manipulation: use standard Python or shell tools. +- Always verify output files exist and contain valid content before finishing. +- If a task involves git repos, use git commands directly. +- If a task involves databases, use sqlite3 CLI or Python sqlite3 module. +- If a task involves images, use PIL/Pillow. +- Read error messages carefully and fix issues iteratively. +- Never give up — try multiple approaches if one fails. +""" + +MODEL = "sonnet" +MAX_TURNS = 30 +PERMISSION_MODE = "bypassPermissions" +ALLOWED_TOOLS = [] +CLI_EXTRA_FLAGS: list[str] = [] + + +def build_cli_args(instruction_path: str) -> list[str]: + """Build the claude CLI argument list. Modify to change execution strategy.""" + args = [ + "claude", + "--print", + "--output-format", "json", + "--model", MODEL, + "--max-turns", str(MAX_TURNS), + "--permission-mode", PERMISSION_MODE, + ] + if SYSTEM_PROMPT: + args.extend(["--system-prompt", SYSTEM_PROMPT]) + for tool in ALLOWED_TOOLS: + args.extend(["--allowedTools", tool]) + args.extend(CLI_EXTRA_FLAGS) + args.extend(["--prompt", f"Read the task at {instruction_path} and complete it."]) + return args + + +# =========================================================================== +# HARBOR ADAPTER — fixed harness, do not modify +# =========================================================================== + +from dotenv import dotenv_values # noqa: E402 +from harbor.agents.base import BaseAgent # noqa: E402 +from harbor.environments.base import BaseEnvironment # noqa: E402 +from harbor.models.agent.context import AgentContext # noqa: E402 + + +def _parse_claude_json_output(raw: str) -> list[dict]: + """Parse Claude Code JSON output (newline-delimited JSON objects).""" + messages = [] + for line in raw.strip().splitlines(): + line = line.strip() + if not line: + continue + try: + messages.append(json.loads(line)) + except json.JSONDecodeError: + continue + return messages + + +def _to_atif(messages: list[dict], duration_ms: int) -> dict: + """Convert Claude Code JSON output to an ATIF trajectory dict.""" + steps: list[dict] = [] + step_id = 0 + now = datetime.now(timezone.utc).isoformat() + + total_input_tokens = 0 + total_output_tokens = 0 + total_cache_tokens = 0 + cost_usd = None + session_id = "unknown" + num_turns = 0 + model_name = MODEL + + def _step(source: str, message: str, **extra: object) -> dict: + nonlocal step_id + step_id += 1 + step = { + "step_id": step_id, + "timestamp": now, + "source": source, + "message": message, + } + step.update({k: v for k, v in extra.items() if v is not None}) + return step + + for msg in messages: + msg_type = msg.get("type", "") + + if msg_type == "assistant": + content_blocks = msg.get("message", {}).get("content", []) + texts = [] + reasoning = None + for block in content_blocks: + block_type = block.get("type", "") + if block_type == "text": + texts.append(block.get("text", "")) + elif block_type == "thinking": + reasoning = block.get("thinking", "") + elif block_type == "tool_use": + pass + if texts or reasoning: + steps.append(_step( + "agent", + "\n".join(texts) or "(thinking)", + reasoning_content=reasoning, + model_name=model_name, + )) + usage = msg.get("message", {}).get("usage", {}) + total_input_tokens += usage.get("input_tokens", 0) + total_output_tokens += usage.get("output_tokens", 0) + total_cache_tokens += usage.get("cache_read_input_tokens", 0) + num_turns += 1 + if msg.get("message", {}).get("model"): + model_name = msg["message"]["model"] + if msg.get("session_id"): + session_id = msg["session_id"] + + elif msg_type == "tool_use": + tool_name = msg.get("tool_name", msg.get("name", "unknown")) + tool_input = msg.get("tool_input", msg.get("input", {})) + tool_id = msg.get("tool_use_id", msg.get("id", "")) + steps.append(_step( + "agent", + f"Tool: {tool_name}", + tool_calls=[{ + "tool_call_id": tool_id, + "function_name": tool_name, + "arguments": tool_input, + }], + )) + + elif msg_type == "tool_result": + tool_id = msg.get("tool_use_id", "") + content = msg.get("content", "") + if isinstance(content, list): + content = "\n".join( + b.get("text", str(b)) for b in content + ) + if steps and steps[-1].get("tool_calls"): + last_call = steps[-1]["tool_calls"][0] + steps[-1]["observation"] = { + "results": [{ + "source_call_id": last_call.get("tool_call_id", tool_id), + "content": str(content), + }] + } + else: + steps.append(_step("tool", str(content))) + + elif msg_type == "result": + cost_usd = msg.get("cost_usd", msg.get("total_cost_usd")) + if msg.get("session_id"): + session_id = msg["session_id"] + result_usage = msg.get("usage", {}) + if result_usage: + total_input_tokens = result_usage.get("input_tokens", total_input_tokens) + total_output_tokens = result_usage.get("output_tokens", total_output_tokens) + total_cache_tokens = result_usage.get("cache_read_input_tokens", total_cache_tokens) + num_turns = msg.get("num_turns", num_turns) + + if not steps: + steps.append(_step("user", "(empty)")) + + return { + "schema_version": "ATIF-v1.6", + "session_id": session_id, + "agent": {"name": "autoagent-claude-code", "version": "0.1.0", "model_name": model_name}, + "steps": steps, + "final_metrics": { + "total_prompt_tokens": total_input_tokens, + "total_completion_tokens": total_output_tokens, + "total_cached_tokens": total_cache_tokens, + "total_cost_usd": cost_usd, + "total_steps": len(steps), + "extra": { + "duration_ms": duration_ms, + "num_turns": num_turns, + }, + }, + } + + +class AutoAgent(BaseAgent): + """Harbor agent adapter. Execs Claude Code CLI inside the container.""" + + SUPPORTS_ATIF = True + + def __init__(self, *args, extra_env: dict[str, str] | None = None, **kwargs): + super().__init__(*args, **kwargs) + self._extra_env = dict(extra_env) if extra_env else {} + + @staticmethod + def name() -> str: + return "autoagent" + + def version(self) -> str | None: + return "0.1.0" + + async def setup(self, environment: BaseEnvironment) -> None: + pass + + async def run( + self, instruction: str, environment: BaseEnvironment, context: AgentContext + ) -> None: + await environment.exec(command="mkdir -p /task") + instr_file = self.logs_dir / "instruction.md" + instr_file.write_text(instruction) + await environment.upload_file( + source_path=instr_file, target_path="/task/instruction.md" + ) + + env = {"IS_SANDBOX": "1", **dotenv_values()} + env = {k: v for k, v in env.items() if v} + env.update(self._extra_env) + + result = await environment.exec( + command="cd /app && python agent-claude-code.py", + env=env, + timeout_sec=600, + ) + if result.stdout: + (self.logs_dir / "agent_stdout.txt").write_text(result.stdout) + if result.stderr: + (self.logs_dir / "agent_stderr.txt").write_text(result.stderr) + + traj_path = self.logs_dir / "trajectory.json" + if traj_path.exists(): + try: + fm = json.loads(traj_path.read_text()).get("final_metrics", {}) + context.cost_usd = fm.get("total_cost_usd") + context.n_input_tokens = fm.get("total_prompt_tokens", 0) + context.n_output_tokens = fm.get("total_completion_tokens", 0) + context.n_cache_tokens = fm.get("total_cached_tokens", 0) + except Exception: + pass + + +# =========================================================================== +# CONTAINER ENTRYPOINT — fixed harness, do not modify +# =========================================================================== + + +def _run_in_container(): + """Container entrypoint — spawns claude CLI, captures output, writes ATIF.""" + import subprocess + + instruction_path = "/task/instruction.md" + cli_args = build_cli_args(instruction_path) + + t0 = time.time() + try: + result = subprocess.run( + cli_args, + capture_output=True, + text=True, + timeout=540, + cwd="/task", + ) + duration_ms = int((time.time() - t0) * 1000) + raw_output = result.stdout or "" + stderr_output = result.stderr or "" + except subprocess.TimeoutExpired: + duration_ms = int((time.time() - t0) * 1000) + raw_output = "" + stderr_output = "ERROR: claude CLI timed out after 540s" + except FileNotFoundError: + duration_ms = int((time.time() - t0) * 1000) + raw_output = "" + stderr_output = "ERROR: claude CLI not found. Ensure @anthropic-ai/claude-code is installed." + + messages = _parse_claude_json_output(raw_output) + atif = _to_atif(messages, duration_ms) + + traj_dir = Path("/logs/agent") + traj_dir.mkdir(parents=True, exist_ok=True) + (traj_dir / "trajectory.json").write_text(json.dumps(atif, indent=2)) + + if raw_output: + (traj_dir / "claude_raw_output.txt").write_text(raw_output) + if stderr_output: + (traj_dir / "claude_stderr.txt").write_text(stderr_output) + + fm = atif.get("final_metrics", {}) + cost = fm.get("total_cost_usd") or 0 + turns = fm.get("extra", {}).get("num_turns", 0) + print(f"cost_usd={cost:.4f} turns={turns} duration_ms={duration_ms}") + + +if __name__ == "__main__": + _run_in_container() diff --git a/program-claude-code.md b/program-claude-code.md new file mode 100644 index 00000000..d243e395 --- /dev/null +++ b/program-claude-code.md @@ -0,0 +1,89 @@ +# autoagent — Claude Code CLI variant + +Autonomous agent engineering. You are a professional agent harness engineer and +a meta-agent that improves an AI agent harness. + +Your job is not to solve benchmark tasks directly. Your job is to improve the +harness in `agent-claude-code.py` so the agent gets better at solving tasks on +its own. + +## Directive + +Build a generally capable autonomous coding and terminal agent using Claude Code +CLI as the execution backend. + +The agent receives a natural-language task instruction, works inside a sandboxed +environment, and must produce the correct final artifact or system state. +Evaluation is done by task-specific verifiers. + +## Architecture + +This variant uses Claude Code CLI (`claude`) as a subprocess rather than a +direct SDK call. The CLI handles tool execution, file operations, and shell +commands internally. The harness controls: + +- **System prompt**: what instructions the agent receives +- **Model selection**: which Claude model to use +- **CLI flags**: permission mode, max turns, allowed tools +- **Prompt framing**: how the task instruction is presented to the CLI + +## Setup + +Before starting a new experiment: + +1. Read `README.md`, this file, and `agent-claude-code.py`. +2. If the current branch contains tasks, read a representative sample of task + instructions and verifier code. +3. Build the base image and verify the agent imports cleanly: + ```bash + docker build -f Dockerfile.claude-code -t autoagent-base . + ``` +4. Initialize `results.tsv` if it does not exist. + +The first run must always be the unmodified baseline. + +## What You Can Modify + +Everything above the `HARBOR ADAPTER` comment in `agent-claude-code.py`: + +- `SYSTEM_PROMPT` — agent instructions +- `MODEL` — Claude model to use (sonnet, haiku, opus) +- `MAX_TURNS` — maximum conversation turns +- `PERMISSION_MODE` — CLI permission mode +- `ALLOWED_TOOLS` — restrict which tools the CLI can use +- `CLI_EXTRA_FLAGS` — additional CLI flags +- `build_cli_args()` — how the CLI is invoked + +## Improvement Axes + +Since Claude Code CLI handles its own tool execution internally, the main +levers are: + +1. **System prompt engineering** — task decomposition strategies, verification + steps, error recovery patterns +2. **Model selection** — balancing capability vs cost +3. **Prompt framing** — how the task instruction is presented +4. **Tool restrictions** — limiting tools to reduce distraction +5. **Turn budget** — balancing thoroughness vs cost + +## What You Must Not Modify + +Inside `agent-claude-code.py`, there is a fixed adapter boundary marked by +comments. Do not modify that fixed section unless the human explicitly asks. + +## How to Run + +```bash +docker build -f Dockerfile.claude-code -t autoagent-base . +rm -rf jobs; mkdir -p jobs && uv run harbor run -p tasks/ -n 100 --agent-import-path agent-claude-code:AutoAgent -o jobs --job-name latest > run.log 2>&1 +``` + +## Goal, Logging, Experiment Loop, Keep/Discard Rules + +Same as `program.md`. Maximize passed tasks. Log to `results.tsv`. Keep if +passed improves or harness is simpler at same performance. Discard otherwise. + +## NEVER STOP + +Once the experiment loop begins, do NOT stop to ask whether you should continue. +Continue iterating until the human explicitly interrupts you. From 1c5f7bc88c6a6804d54788dd8eb7ea240ade64c9 Mon Sep 17 00:00:00 2001 From: Szymon Paluch Date: Fri, 3 Apr 2026 11:45:35 +0200 Subject: [PATCH 08/12] fix: use local Claude CLI auth instead of API key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The adapter now copies the host's ~/.claude credentials into the container at runtime. No ANTHROPIC_API_KEY or .env file needed — Claude Code CLI uses its own OAuth session from the host machine. --- Dockerfile.claude-code | 5 +++++ agent-claude-code.py | 27 +++++++++++++++++++-------- program-claude-code.md | 19 +++++++++++++++---- 3 files changed, 39 insertions(+), 12 deletions(-) diff --git a/Dockerfile.claude-code b/Dockerfile.claude-code index 50671a29..cdc28b4d 100644 --- a/Dockerfile.claude-code +++ b/Dockerfile.claude-code @@ -10,6 +10,8 @@ RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - && \ rm -rf /var/lib/apt/lists/* # Install Claude Code CLI globally +# No API key needed — auth is provided by mounting the host's ~/.claude +# directory (handled by the Harbor adapter at runtime). RUN npm install -g @anthropic-ai/claude-code WORKDIR /app @@ -21,5 +23,8 @@ RUN uv pip install --system . # Agent code COPY agent-claude-code.py ./ +# Prepare auth mount point — the adapter uploads credentials at runtime +RUN mkdir -p /root/.claude + RUN ln -sf $(which python3) /usr/local/bin/python RUN mkdir -p /logs /app/output diff --git a/agent-claude-code.py b/agent-claude-code.py index 6093491d..2cd86e6a 100644 --- a/agent-claude-code.py +++ b/agent-claude-code.py @@ -1,20 +1,21 @@ """ Claude Code CLI adapter for Harbor. -Spawns `claude` as a subprocess inside the container. The meta-agent iterates -on the editable section (system prompt, CLI flags, context strategy). +Spawns the locally-installed `claude` CLI as a subprocess. No API key needed — +Claude Code uses its own OAuth session from the host machine. + +Prerequisites: + - Claude Code CLI installed: npm install -g @anthropic-ai/claude-code + - Authenticated locally: claude login (or already logged in) Run all tasks: docker build -f Dockerfile.claude-code -t autoagent-base . - set -a && source .env && set +a uv run harbor run -p tasks/ --agent-import-path agent-claude-code:AutoAgent -o jobs --job-name latest > run.log 2>&1 """ from __future__ import annotations -import asyncio import json -import os import time from datetime import datetime, timezone from pathlib import Path @@ -76,7 +77,6 @@ def build_cli_args(instruction_path: str) -> list[str]: # HARBOR ADAPTER — fixed harness, do not modify # =========================================================================== -from dotenv import dotenv_values # noqa: E402 from harbor.agents.base import BaseAgent # noqa: E402 from harbor.environments.base import BaseEnvironment # noqa: E402 from harbor.models.agent.context import AgentContext # noqa: E402 @@ -248,8 +248,19 @@ async def run( source_path=instr_file, target_path="/task/instruction.md" ) - env = {"IS_SANDBOX": "1", **dotenv_values()} - env = {k: v for k, v in env.items() if v} + # Upload host's Claude auth config into the container so the CLI + # can authenticate without an API key. + host_claude_dir = Path.home() / ".claude" + if host_claude_dir.exists(): + for auth_file in ("credentials.json", "config.json", ".credentials.json"): + src = host_claude_dir / auth_file + if src.exists(): + await environment.upload_file( + source_path=src, + target_path=f"/root/.claude/{auth_file}", + ) + + env = {"IS_SANDBOX": "1"} env.update(self._extra_env) result = await environment.exec( diff --git a/program-claude-code.md b/program-claude-code.md index d243e395..69ec0127 100644 --- a/program-claude-code.md +++ b/program-claude-code.md @@ -27,18 +27,26 @@ commands internally. The harness controls: - **CLI flags**: permission mode, max turns, allowed tools - **Prompt framing**: how the task instruction is presented to the CLI +## Prerequisites + +- Claude Code CLI installed on the host: `npm install -g @anthropic-ai/claude-code` +- Authenticated: `claude login` (or already logged in — no API key needed) +- The adapter automatically copies your `~/.claude` auth credentials into the + container at runtime. + ## Setup Before starting a new experiment: 1. Read `README.md`, this file, and `agent-claude-code.py`. -2. If the current branch contains tasks, read a representative sample of task +2. Verify `claude --version` works on your host machine. +3. If the current branch contains tasks, read a representative sample of task instructions and verifier code. -3. Build the base image and verify the agent imports cleanly: +4. Build the base image and verify the agent imports cleanly: ```bash docker build -f Dockerfile.claude-code -t autoagent-base . ``` -4. Initialize `results.tsv` if it does not exist. +5. Initialize `results.tsv` if it does not exist. The first run must always be the unmodified baseline. @@ -75,9 +83,12 @@ comments. Do not modify that fixed section unless the human explicitly asks. ```bash docker build -f Dockerfile.claude-code -t autoagent-base . -rm -rf jobs; mkdir -p jobs && uv run harbor run -p tasks/ -n 100 --agent-import-path agent-claude-code:AutoAgent -o jobs --job-name latest > run.log 2>&1 +rm -rf jobs && mkdir -p jobs && uv run harbor run -p tasks/ -n 100 --agent-import-path agent-claude-code:AutoAgent -o jobs --job-name latest > run.log 2>&1 ``` +No `.env` file or API key needed — the adapter copies your local Claude auth +into the container automatically. + ## Goal, Logging, Experiment Loop, Keep/Discard Rules Same as `program.md`. Maximize passed tasks. Log to `results.tsv`. Keep if From 8ede701fccee1106ee776f07e9431f5a974ef2ca Mon Sep 17 00:00:00 2001 From: Szymon Paluch Date: Fri, 3 Apr 2026 11:55:24 +0200 Subject: [PATCH 09/12] fix: update ATIF parser for stream-json format - Fixed CLI args: added --verbose (required for stream-json), prompt is positional not --prompt flag - Rewrote ATIF parser to handle actual stream-json message structure: assistant messages with tool_use content blocks, user messages with tool_result blocks, pending tool pairing - Tested locally: all 4 tasks produce valid ATIF-v1.6 trajectories --- agent-claude-code.py | 112 ++++++++++++++++++++++++------------------- 1 file changed, 64 insertions(+), 48 deletions(-) diff --git a/agent-claude-code.py b/agent-claude-code.py index 2cd86e6a..2b910a5d 100644 --- a/agent-claude-code.py +++ b/agent-claude-code.py @@ -59,7 +59,8 @@ def build_cli_args(instruction_path: str) -> list[str]: args = [ "claude", "--print", - "--output-format", "json", + "--output-format", "stream-json", + "--verbose", "--model", MODEL, "--max-turns", str(MAX_TURNS), "--permission-mode", PERMISSION_MODE, @@ -69,7 +70,8 @@ def build_cli_args(instruction_path: str) -> list[str]: for tool in ALLOWED_TOOLS: args.extend(["--allowedTools", tool]) args.extend(CLI_EXTRA_FLAGS) - args.extend(["--prompt", f"Read the task at {instruction_path} and complete it."]) + # Prompt is a positional argument in claude CLI + args.append(f"Read the task at {instruction_path} and complete it.") return args @@ -97,18 +99,30 @@ def _parse_claude_json_output(raw: str) -> list[dict]: def _to_atif(messages: list[dict], duration_ms: int) -> dict: - """Convert Claude Code JSON output to an ATIF trajectory dict.""" + """Convert Claude Code stream-json output to an ATIF trajectory dict. + + Stream-json format (with --verbose) emits NDJSON with these types: + - system: init metadata (tools, model, session_id) + - assistant: content blocks (thinking, text, tool_use) — may arrive + as multiple messages for the same API response + - user: tool_result content blocks + - result: final summary with cost, usage, duration + - rate_limit_event: rate limit info (ignored) + """ steps: list[dict] = [] step_id = 0 now = datetime.now(timezone.utc).isoformat() - total_input_tokens = 0 - total_output_tokens = 0 - total_cache_tokens = 0 cost_usd = None session_id = "unknown" num_turns = 0 model_name = MODEL + total_input_tokens = 0 + total_output_tokens = 0 + total_cache_tokens = 0 + + # Track pending tool_use blocks to pair with their results + pending_tools: dict[str, dict] = {} def _step(source: str, message: str, **extra: object) -> dict: nonlocal step_id @@ -136,66 +150,68 @@ def _step(source: str, message: str, **extra: object) -> dict: elif block_type == "thinking": reasoning = block.get("thinking", "") elif block_type == "tool_use": - pass + tool_id = block.get("id", "") + tool_name = block.get("name", "unknown") + tool_input = block.get("input", {}) + pending_tools[tool_id] = { + "tool_call_id": tool_id, + "function_name": tool_name, + "arguments": tool_input, + } if texts or reasoning: steps.append(_step( "agent", "\n".join(texts) or "(thinking)", reasoning_content=reasoning, - model_name=model_name, + model_name=msg.get("message", {}).get("model", model_name), )) - usage = msg.get("message", {}).get("usage", {}) - total_input_tokens += usage.get("input_tokens", 0) - total_output_tokens += usage.get("output_tokens", 0) - total_cache_tokens += usage.get("cache_read_input_tokens", 0) - num_turns += 1 if msg.get("message", {}).get("model"): model_name = msg["message"]["model"] if msg.get("session_id"): session_id = msg["session_id"] - elif msg_type == "tool_use": - tool_name = msg.get("tool_name", msg.get("name", "unknown")) - tool_input = msg.get("tool_input", msg.get("input", {})) - tool_id = msg.get("tool_use_id", msg.get("id", "")) - steps.append(_step( - "agent", - f"Tool: {tool_name}", - tool_calls=[{ - "tool_call_id": tool_id, - "function_name": tool_name, - "arguments": tool_input, - }], - )) - - elif msg_type == "tool_result": - tool_id = msg.get("tool_use_id", "") - content = msg.get("content", "") + elif msg_type == "user": + # User messages contain tool_result blocks + content = msg.get("message", {}).get("content", []) if isinstance(content, list): - content = "\n".join( - b.get("text", str(b)) for b in content - ) - if steps and steps[-1].get("tool_calls"): - last_call = steps[-1]["tool_calls"][0] - steps[-1]["observation"] = { - "results": [{ - "source_call_id": last_call.get("tool_call_id", tool_id), - "content": str(content), - }] - } - else: - steps.append(_step("tool", str(content))) + for block in content: + if block.get("type") == "tool_result": + tool_id = block.get("tool_use_id", "") + result_content = block.get("content", "") + if isinstance(result_content, list): + result_content = "\n".join( + b.get("text", str(b)) for b in result_content + ) + tc = pending_tools.pop(tool_id, None) + if tc: + steps.append(_step( + "agent", + f"Tool: {tc['function_name']}", + tool_calls=[tc], + observation={"results": [{ + "source_call_id": tool_id, + "content": str(result_content), + }]}, + )) elif msg_type == "result": - cost_usd = msg.get("cost_usd", msg.get("total_cost_usd")) + cost_usd = msg.get("total_cost_usd") if msg.get("session_id"): session_id = msg["session_id"] + num_turns = msg.get("num_turns", num_turns) result_usage = msg.get("usage", {}) if result_usage: - total_input_tokens = result_usage.get("input_tokens", total_input_tokens) - total_output_tokens = result_usage.get("output_tokens", total_output_tokens) - total_cache_tokens = result_usage.get("cache_read_input_tokens", total_cache_tokens) - num_turns = msg.get("num_turns", num_turns) + total_input_tokens = result_usage.get("input_tokens", 0) + total_output_tokens = result_usage.get("output_tokens", 0) + total_cache_tokens = result_usage.get("cache_read_input_tokens", 0) + + # Flush any pending tool calls that never got a result + for tc in pending_tools.values(): + steps.append(_step( + "agent", + f"Tool: {tc['function_name']}", + tool_calls=[tc], + )) if not steps: steps.append(_step("user", "(empty)")) From 6f2b8d7ae18b625e5fe63a401f425ad2a7dad85e Mon Sep 17 00:00:00 2001 From: Szymon Paluch Date: Fri, 3 Apr 2026 12:32:47 +0200 Subject: [PATCH 10/12] fix: run claude CLI host-side with OAuth auth Major restructure: claude CLI now runs on the HOST machine (not inside Docker) using the user's existing OAuth session. No API key needed. - Agent syncs files from container to host temp dir before running - Claude executes with full OAuth auth from host keychain - Results synced back to container for verifier - Simplified Dockerfile (no Node.js/Claude Code needed in image) - Rewrites /task/ paths to temp dir paths for correct file placement - Fixed download_file arg order for container-to-host sync Tested: 4/4 tasks pass (hello-world, fibonacci, csv-analysis, git-log) with Harbor e2e Docker pipeline. Mean score: 1.000. --- Dockerfile.claude-code | 19 +-- agent-claude-code.py | 256 +++++++++++++++++++++-------------------- 2 files changed, 135 insertions(+), 140 deletions(-) diff --git a/Dockerfile.claude-code b/Dockerfile.claude-code index cdc28b4d..6e0d0a2f 100644 --- a/Dockerfile.claude-code +++ b/Dockerfile.claude-code @@ -1,30 +1,17 @@ FROM ghcr.io/astral-sh/uv:python3.12-bookworm-slim RUN apt-get update && \ - apt-get install -y --no-install-recommends ca-certificates git curl && \ + apt-get install -y --no-install-recommends ca-certificates git && \ rm -rf /var/lib/apt/lists/* -# Install Node.js (required for Claude Code CLI) -RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - && \ - apt-get install -y --no-install-recommends nodejs && \ - rm -rf /var/lib/apt/lists/* - -# Install Claude Code CLI globally -# No API key needed — auth is provided by mounting the host's ~/.claude -# directory (handled by the Harbor adapter at runtime). -RUN npm install -g @anthropic-ai/claude-code - WORKDIR /app -# Python deps +# Python deps — only what the agent needs (harbor excluded via .dockerignore) COPY pyproject.toml ./ RUN uv pip install --system . # Agent code COPY agent-claude-code.py ./ -# Prepare auth mount point — the adapter uploads credentials at runtime -RUN mkdir -p /root/.claude - RUN ln -sf $(which python3) /usr/local/bin/python -RUN mkdir -p /logs /app/output +RUN mkdir -p /logs /app/output /task/output diff --git a/agent-claude-code.py b/agent-claude-code.py index 2b910a5d..1da1d4a5 100644 --- a/agent-claude-code.py +++ b/agent-claude-code.py @@ -1,12 +1,12 @@ """ Claude Code CLI adapter for Harbor. -Spawns the locally-installed `claude` CLI as a subprocess. No API key needed — -Claude Code uses its own OAuth session from the host machine. +Runs the locally-installed `claude` CLI on the HOST machine (not inside Docker). +No API key needed — uses your existing Claude Code OAuth session. Prerequisites: - Claude Code CLI installed: npm install -g @anthropic-ai/claude-code - - Authenticated locally: claude login (or already logged in) + - Authenticated locally: claude login (or already logged in via claude.ai) Run all tasks: docker build -f Dockerfile.claude-code -t autoagent-base . @@ -15,7 +15,10 @@ from __future__ import annotations +import asyncio import json +import subprocess +import tempfile import time from datetime import datetime, timezone from pathlib import Path @@ -25,36 +28,34 @@ # AGENT CONFIG — meta-agent modifies this section # =========================================================================== -SYSTEM_PROMPT = """You are a highly capable task-completion agent. You solve tasks by reading instructions, analyzing the problem, writing and executing code, and producing the required output files. +SYSTEM_PROMPT = """You are a highly capable task-completion agent working inside a sandboxed environment. ## Approach -1. Read /task/instruction.md to understand what's required. +1. Read the task instruction to understand what's required. 2. Explore the working environment — check what files, tools, and libraries are available. 3. Plan your approach, then execute step by step. 4. Write output files to the exact paths specified in the instructions. 5. Verify your output before finishing. ## Key rules +- All paths referenced in the task are relative to your current working directory. - Use python3 (not python) for running scripts. -- Use Bash to run shell commands, install packages, inspect files. - For data analysis: pandas, numpy, openpyxl are available. - For file manipulation: use standard Python or shell tools. - Always verify output files exist and contain valid content before finishing. - If a task involves git repos, use git commands directly. - If a task involves databases, use sqlite3 CLI or Python sqlite3 module. -- If a task involves images, use PIL/Pillow. - Read error messages carefully and fix issues iteratively. - Never give up — try multiple approaches if one fails. """ MODEL = "sonnet" MAX_TURNS = 30 -PERMISSION_MODE = "bypassPermissions" -ALLOWED_TOOLS = [] +ALLOWED_TOOLS: list[str] = [] CLI_EXTRA_FLAGS: list[str] = [] -def build_cli_args(instruction_path: str) -> list[str]: +def build_cli_args(workdir: str, instruction_text: str) -> list[str]: """Build the claude CLI argument list. Modify to change execution strategy.""" args = [ "claude", @@ -63,15 +64,14 @@ def build_cli_args(instruction_path: str) -> list[str]: "--verbose", "--model", MODEL, "--max-turns", str(MAX_TURNS), - "--permission-mode", PERMISSION_MODE, + "--permission-mode", "bypassPermissions", ] if SYSTEM_PROMPT: args.extend(["--system-prompt", SYSTEM_PROMPT]) for tool in ALLOWED_TOOLS: args.extend(["--allowedTools", tool]) args.extend(CLI_EXTRA_FLAGS) - # Prompt is a positional argument in claude CLI - args.append(f"Read the task at {instruction_path} and complete it.") + args.append(instruction_text) return args @@ -99,16 +99,7 @@ def _parse_claude_json_output(raw: str) -> list[dict]: def _to_atif(messages: list[dict], duration_ms: int) -> dict: - """Convert Claude Code stream-json output to an ATIF trajectory dict. - - Stream-json format (with --verbose) emits NDJSON with these types: - - system: init metadata (tools, model, session_id) - - assistant: content blocks (thinking, text, tool_use) — may arrive - as multiple messages for the same API response - - user: tool_result content blocks - - result: final summary with cost, usage, duration - - rate_limit_event: rate limit info (ignored) - """ + """Convert Claude Code stream-json output to an ATIF trajectory dict.""" steps: list[dict] = [] step_id = 0 now = datetime.now(timezone.utc).isoformat() @@ -121,7 +112,6 @@ def _to_atif(messages: list[dict], duration_ms: int) -> dict: total_output_tokens = 0 total_cache_tokens = 0 - # Track pending tool_use blocks to pair with their results pending_tools: dict[str, dict] = {} def _step(source: str, message: str, **extra: object) -> dict: @@ -150,13 +140,10 @@ def _step(source: str, message: str, **extra: object) -> dict: elif block_type == "thinking": reasoning = block.get("thinking", "") elif block_type == "tool_use": - tool_id = block.get("id", "") - tool_name = block.get("name", "unknown") - tool_input = block.get("input", {}) - pending_tools[tool_id] = { - "tool_call_id": tool_id, - "function_name": tool_name, - "arguments": tool_input, + pending_tools[block.get("id", "")] = { + "tool_call_id": block.get("id", ""), + "function_name": block.get("name", "unknown"), + "arguments": block.get("input", {}), } if texts or reasoning: steps.append(_step( @@ -171,7 +158,6 @@ def _step(source: str, message: str, **extra: object) -> dict: session_id = msg["session_id"] elif msg_type == "user": - # User messages contain tool_result blocks content = msg.get("message", {}).get("content", []) if isinstance(content, list): for block in content: @@ -205,12 +191,9 @@ def _step(source: str, message: str, **extra: object) -> dict: total_output_tokens = result_usage.get("output_tokens", 0) total_cache_tokens = result_usage.get("cache_read_input_tokens", 0) - # Flush any pending tool calls that never got a result for tc in pending_tools.values(): steps.append(_step( - "agent", - f"Tool: {tc['function_name']}", - tool_calls=[tc], + "agent", f"Tool: {tc['function_name']}", tool_calls=[tc], )) if not steps: @@ -235,8 +218,46 @@ def _step(source: str, message: str, **extra: object) -> dict: } +async def _sync_dir_to_container( + local_dir: Path, container_path: str, environment: BaseEnvironment +) -> None: + """Upload all files from a local directory into the container.""" + for fpath in local_dir.rglob("*"): + if fpath.is_file(): + rel = fpath.relative_to(local_dir) + target = f"{container_path}/{rel}" + parent = str(Path(target).parent) + await environment.exec(command=f"mkdir -p {parent}") + await environment.upload_file(source_path=fpath, target_path=target) + + +async def _sync_container_to_dir( + container_path: str, local_dir: Path, environment: BaseEnvironment +) -> None: + """Download files from the container into a local directory.""" + result = await environment.exec( + command=f"find {container_path} -type f 2>/dev/null || true" + ) + if not result.stdout or not result.stdout.strip(): + return + for line in result.stdout.strip().splitlines(): + line = line.strip() + if not line: + continue + rel = line.removeprefix(container_path).lstrip("/") + local_file = local_dir / rel + local_file.parent.mkdir(parents=True, exist_ok=True) + try: + await environment.download_file( + source_path=line, target_path=local_file + ) + except Exception: + pass + + class AutoAgent(BaseAgent): - """Harbor agent adapter. Execs Claude Code CLI inside the container.""" + """Harbor agent adapter. Runs Claude Code CLI on the HOST, + syncs files to/from the container for verification.""" SUPPORTS_ATIF = True @@ -257,42 +278,75 @@ async def setup(self, environment: BaseEnvironment) -> None: async def run( self, instruction: str, environment: BaseEnvironment, context: AgentContext ) -> None: - await environment.exec(command="mkdir -p /task") - instr_file = self.logs_dir / "instruction.md" - instr_file.write_text(instruction) - await environment.upload_file( - source_path=instr_file, target_path="/task/instruction.md" - ) - - # Upload host's Claude auth config into the container so the CLI - # can authenticate without an API key. - host_claude_dir = Path.home() / ".claude" - if host_claude_dir.exists(): - for auth_file in ("credentials.json", "config.json", ".credentials.json"): - src = host_claude_dir / auth_file - if src.exists(): - await environment.upload_file( - source_path=src, - target_path=f"/root/.claude/{auth_file}", - ) - - env = {"IS_SANDBOX": "1"} - env.update(self._extra_env) - - result = await environment.exec( - command="cd /app && python agent-claude-code.py", - env=env, - timeout_sec=600, - ) - if result.stdout: - (self.logs_dir / "agent_stdout.txt").write_text(result.stdout) - if result.stderr: - (self.logs_dir / "agent_stderr.txt").write_text(result.stderr) - - traj_path = self.logs_dir / "trajectory.json" - if traj_path.exists(): + # 1. Create a temp workdir on the host that mirrors /task in container + with tempfile.TemporaryDirectory(prefix="autoagent_") as tmpdir: + workdir = Path(tmpdir) + task_dir = workdir / "task" + task_dir.mkdir() + output_dir = task_dir / "output" + output_dir.mkdir() + + # Write instruction + instr_file = task_dir / "instruction.md" + instr_file.write_text(instruction) + + # Download any pre-existing files from the container's /task/ + await _sync_container_to_dir("/task", task_dir, environment) + + # 2. Rewrite absolute /task/ paths to the temp workdir path + # so claude writes files to the correct location + task_prefix = str(task_dir) + local_instruction = instruction.replace("/task/", f"{task_prefix}/") + + # Run claude CLI on the HOST, pointed at the temp workdir + cli_args = build_cli_args( + str(workdir), + local_instruction, + ) + + t0 = time.time() + try: + result = subprocess.run( + cli_args, + capture_output=True, + text=True, + timeout=540, + cwd=str(task_dir), + ) + duration_ms = int((time.time() - t0) * 1000) + raw_output = result.stdout or "" + stderr_output = result.stderr or "" + except subprocess.TimeoutExpired: + duration_ms = int((time.time() - t0) * 1000) + raw_output = "" + stderr_output = "ERROR: claude CLI timed out after 540s" + except FileNotFoundError: + duration_ms = int((time.time() - t0) * 1000) + raw_output = "" + stderr_output = ( + "ERROR: claude CLI not found on host. " + "Install: npm install -g @anthropic-ai/claude-code" + ) + + # Save raw output for debugging + if raw_output: + (self.logs_dir / "claude_raw_output.txt").write_text(raw_output) + if stderr_output: + (self.logs_dir / "claude_stderr.txt").write_text(stderr_output) + + # 3. Sync files created by claude back to the container + await _sync_dir_to_container(task_dir, "/task", environment) + + # 4. Build ATIF trajectory + messages = _parse_claude_json_output(raw_output) + atif = _to_atif(messages, duration_ms) + + traj_path = self.logs_dir / "trajectory.json" + traj_path.write_text(json.dumps(atif, indent=2)) + + # 5. Populate Harbor metrics try: - fm = json.loads(traj_path.read_text()).get("final_metrics", {}) + fm = atif.get("final_metrics", {}) context.cost_usd = fm.get("total_cost_usd") context.n_input_tokens = fm.get("total_prompt_tokens", 0) context.n_output_tokens = fm.get("total_completion_tokens", 0) @@ -300,57 +354,11 @@ async def run( except Exception: pass - -# =========================================================================== -# CONTAINER ENTRYPOINT — fixed harness, do not modify -# =========================================================================== + cost = atif.get("final_metrics", {}).get("total_cost_usd") or 0 + turns = atif.get("final_metrics", {}).get("extra", {}).get("num_turns", 0) + print( + f"cost_usd={cost:.4f} turns={turns} duration_ms={duration_ms}" + ) -def _run_in_container(): - """Container entrypoint — spawns claude CLI, captures output, writes ATIF.""" - import subprocess - - instruction_path = "/task/instruction.md" - cli_args = build_cli_args(instruction_path) - - t0 = time.time() - try: - result = subprocess.run( - cli_args, - capture_output=True, - text=True, - timeout=540, - cwd="/task", - ) - duration_ms = int((time.time() - t0) * 1000) - raw_output = result.stdout or "" - stderr_output = result.stderr or "" - except subprocess.TimeoutExpired: - duration_ms = int((time.time() - t0) * 1000) - raw_output = "" - stderr_output = "ERROR: claude CLI timed out after 540s" - except FileNotFoundError: - duration_ms = int((time.time() - t0) * 1000) - raw_output = "" - stderr_output = "ERROR: claude CLI not found. Ensure @anthropic-ai/claude-code is installed." - - messages = _parse_claude_json_output(raw_output) - atif = _to_atif(messages, duration_ms) - - traj_dir = Path("/logs/agent") - traj_dir.mkdir(parents=True, exist_ok=True) - (traj_dir / "trajectory.json").write_text(json.dumps(atif, indent=2)) - - if raw_output: - (traj_dir / "claude_raw_output.txt").write_text(raw_output) - if stderr_output: - (traj_dir / "claude_stderr.txt").write_text(stderr_output) - - fm = atif.get("final_metrics", {}) - cost = fm.get("total_cost_usd") or 0 - turns = fm.get("extra", {}).get("num_turns", 0) - print(f"cost_usd={cost:.4f} turns={turns} duration_ms={duration_ms}") - - -if __name__ == "__main__": - _run_in_container() +__all__ = ["AutoAgent"] From ad5c2fe2e50030e2a1c8be7ea8e50f0aa61b8f2f Mon Sep 17 00:00:00 2001 From: Szymon Paluch Date: Fri, 3 Apr 2026 12:40:12 +0200 Subject: [PATCH 11/12] fix: address all PR review findings Critical fixes: - Replace manual file sync with Harbor's upload_dir/download_dir (fixes shell injection, silent file loss, and fragile find+loop) - Check subprocess exit code and log errors - Preserve partial stdout on TimeoutExpired Important fixes: - Use asyncio.to_thread() for subprocess.run to avoid blocking event loop - Remove bare except Exception: pass on metrics (no longer needed) - Log non-JSON lines from CLI output instead of silently dropping - Add logging throughout via logger instead of print() Doc fixes: - Remove false claim about copying ~/.claude auth into container - Add PERMISSION_MODE as configurable constant (was hardcoded) - Add security warning about host-side execution - Fix ATIF acronym expansion in docstring - Remove unused workdir param from build_cli_args - Remove unused output_dir creation Tested: 4/4 tasks pass (Mean: 1.000) after all changes. --- agent-claude-code.py | 136 ++++++++++++++++++----------------------- program-claude-code.md | 21 ++++--- 2 files changed, 73 insertions(+), 84 deletions(-) diff --git a/agent-claude-code.py b/agent-claude-code.py index 1da1d4a5..7a9f3906 100644 --- a/agent-claude-code.py +++ b/agent-claude-code.py @@ -4,9 +4,13 @@ Runs the locally-installed `claude` CLI on the HOST machine (not inside Docker). No API key needed — uses your existing Claude Code OAuth session. +SECURITY NOTE: Unlike the SDK-based adapters that run inside containers, this +adapter executes `claude` on the host with bypassPermissions. The CLI can access +any host file/network resource. Only run on trusted task sets. + Prerequisites: - Claude Code CLI installed: npm install -g @anthropic-ai/claude-code - - Authenticated locally: claude login (or already logged in via claude.ai) + - Authenticated locally: claude login Run all tasks: docker build -f Dockerfile.claude-code -t autoagent-base . @@ -17,12 +21,15 @@ import asyncio import json +import logging import subprocess import tempfile import time from datetime import datetime, timezone from pathlib import Path +logger = logging.getLogger(__name__) + # =========================================================================== # AGENT CONFIG — meta-agent modifies this section @@ -51,11 +58,12 @@ MODEL = "sonnet" MAX_TURNS = 30 +PERMISSION_MODE = "bypassPermissions" ALLOWED_TOOLS: list[str] = [] CLI_EXTRA_FLAGS: list[str] = [] -def build_cli_args(workdir: str, instruction_text: str) -> list[str]: +def build_cli_args(instruction_text: str) -> list[str]: """Build the claude CLI argument list. Modify to change execution strategy.""" args = [ "claude", @@ -64,7 +72,7 @@ def build_cli_args(workdir: str, instruction_text: str) -> list[str]: "--verbose", "--model", MODEL, "--max-turns", str(MAX_TURNS), - "--permission-mode", "bypassPermissions", + "--permission-mode", PERMISSION_MODE, ] if SYSTEM_PROMPT: args.extend(["--system-prompt", SYSTEM_PROMPT]) @@ -85,7 +93,10 @@ def build_cli_args(workdir: str, instruction_text: str) -> list[str]: def _parse_claude_json_output(raw: str) -> list[dict]: - """Parse Claude Code JSON output (newline-delimited JSON objects).""" + """Parse Claude Code NDJSON output (newline-delimited JSON objects). + + Non-JSON lines (e.g. CLI error messages) are logged as warnings. + """ messages = [] for line in raw.strip().splitlines(): line = line.strip() @@ -94,12 +105,13 @@ def _parse_claude_json_output(raw: str) -> list[dict]: try: messages.append(json.loads(line)) except json.JSONDecodeError: - continue + logger.warning("Non-JSON line in claude output: %s", line[:200]) return messages def _to_atif(messages: list[dict], duration_ms: int) -> dict: - """Convert Claude Code stream-json output to an ATIF trajectory dict.""" + """Convert Claude Code stream-json output to an ATIF (Agent Trajectory + Interchange Format) v1.6 trajectory dict.""" steps: list[dict] = [] step_id = 0 now = datetime.now(timezone.utc).isoformat() @@ -218,46 +230,10 @@ def _step(source: str, message: str, **extra: object) -> dict: } -async def _sync_dir_to_container( - local_dir: Path, container_path: str, environment: BaseEnvironment -) -> None: - """Upload all files from a local directory into the container.""" - for fpath in local_dir.rglob("*"): - if fpath.is_file(): - rel = fpath.relative_to(local_dir) - target = f"{container_path}/{rel}" - parent = str(Path(target).parent) - await environment.exec(command=f"mkdir -p {parent}") - await environment.upload_file(source_path=fpath, target_path=target) - - -async def _sync_container_to_dir( - container_path: str, local_dir: Path, environment: BaseEnvironment -) -> None: - """Download files from the container into a local directory.""" - result = await environment.exec( - command=f"find {container_path} -type f 2>/dev/null || true" - ) - if not result.stdout or not result.stdout.strip(): - return - for line in result.stdout.strip().splitlines(): - line = line.strip() - if not line: - continue - rel = line.removeprefix(container_path).lstrip("/") - local_file = local_dir / rel - local_file.parent.mkdir(parents=True, exist_ok=True) - try: - await environment.download_file( - source_path=line, target_path=local_file - ) - except Exception: - pass - - class AutoAgent(BaseAgent): - """Harbor agent adapter. Runs Claude Code CLI on the HOST, - syncs files to/from the container for verification.""" + """Harbor agent adapter. Runs Claude Code CLI on the HOST in a temp + directory, syncing task files from the container beforehand and results + back afterward.""" SUPPORTS_ATIF = True @@ -283,30 +259,27 @@ async def run( workdir = Path(tmpdir) task_dir = workdir / "task" task_dir.mkdir() - output_dir = task_dir / "output" - output_dir.mkdir() - # Write instruction instr_file = task_dir / "instruction.md" instr_file.write_text(instruction) - # Download any pre-existing files from the container's /task/ - await _sync_container_to_dir("/task", task_dir, environment) + # Download pre-existing files from the container's /task/ + await environment.download_dir( + source_dir="/task", target_dir=task_dir + ) # 2. Rewrite absolute /task/ paths to the temp workdir path # so claude writes files to the correct location task_prefix = str(task_dir) local_instruction = instruction.replace("/task/", f"{task_prefix}/") - # Run claude CLI on the HOST, pointed at the temp workdir - cli_args = build_cli_args( - str(workdir), - local_instruction, - ) + cli_args = build_cli_args(local_instruction) + # 3. Run claude CLI on the HOST (async to avoid blocking event loop) t0 = time.time() try: - result = subprocess.run( + result = await asyncio.to_thread( + subprocess.run, cli_args, capture_output=True, text=True, @@ -316,10 +289,22 @@ async def run( duration_ms = int((time.time() - t0) * 1000) raw_output = result.stdout or "" stderr_output = result.stderr or "" - except subprocess.TimeoutExpired: + + if result.returncode != 0: + logger.error( + "claude CLI exited with code %d. stderr: %s", + result.returncode, + stderr_output[:500], + ) + except subprocess.TimeoutExpired as exc: duration_ms = int((time.time() - t0) * 1000) - raw_output = "" + raw_output = ( + exc.stdout.decode("utf-8", errors="replace") + if isinstance(exc.stdout, bytes) + else (exc.stdout or "") + ) stderr_output = "ERROR: claude CLI timed out after 540s" + logger.error("claude CLI timed out after 540s (partial output preserved)") except FileNotFoundError: duration_ms = int((time.time() - t0) * 1000) raw_output = "" @@ -327,37 +312,36 @@ async def run( "ERROR: claude CLI not found on host. " "Install: npm install -g @anthropic-ai/claude-code" ) + logger.error("claude CLI not found on host") - # Save raw output for debugging if raw_output: (self.logs_dir / "claude_raw_output.txt").write_text(raw_output) if stderr_output: (self.logs_dir / "claude_stderr.txt").write_text(stderr_output) - # 3. Sync files created by claude back to the container - await _sync_dir_to_container(task_dir, "/task", environment) + # 4. Sync files created by claude back to the container + await environment.upload_dir( + source_dir=task_dir, target_dir="/task" + ) - # 4. Build ATIF trajectory + # 5. Build ATIF trajectory messages = _parse_claude_json_output(raw_output) atif = _to_atif(messages, duration_ms) traj_path = self.logs_dir / "trajectory.json" traj_path.write_text(json.dumps(atif, indent=2)) - # 5. Populate Harbor metrics - try: - fm = atif.get("final_metrics", {}) - context.cost_usd = fm.get("total_cost_usd") - context.n_input_tokens = fm.get("total_prompt_tokens", 0) - context.n_output_tokens = fm.get("total_completion_tokens", 0) - context.n_cache_tokens = fm.get("total_cached_tokens", 0) - except Exception: - pass - - cost = atif.get("final_metrics", {}).get("total_cost_usd") or 0 - turns = atif.get("final_metrics", {}).get("extra", {}).get("num_turns", 0) - print( - f"cost_usd={cost:.4f} turns={turns} duration_ms={duration_ms}" + # 6. Populate Harbor metrics + fm = atif.get("final_metrics", {}) + context.cost_usd = fm.get("total_cost_usd") + context.n_input_tokens = fm.get("total_prompt_tokens", 0) + context.n_output_tokens = fm.get("total_completion_tokens", 0) + context.n_cache_tokens = fm.get("total_cached_tokens", 0) + + cost = fm.get("total_cost_usd") or 0 + turns = fm.get("extra", {}).get("num_turns", 0) + logger.info( + "cost_usd=%.4f turns=%d duration_ms=%d", cost, turns, duration_ms ) diff --git a/program-claude-code.md b/program-claude-code.md index 69ec0127..ebbc7e6a 100644 --- a/program-claude-code.md +++ b/program-claude-code.md @@ -18,9 +18,16 @@ Evaluation is done by task-specific verifiers. ## Architecture -This variant uses Claude Code CLI (`claude`) as a subprocess rather than a -direct SDK call. The CLI handles tool execution, file operations, and shell -commands internally. The harness controls: +This variant runs the Claude Code CLI on the **host machine** (not inside the +container) using the user's existing OAuth session. The CLI handles tool +execution, file operations, and shell commands internally. The adapter syncs +task files from the container to a host temp directory, runs `claude`, then +syncs results back for verification. + +**Security note:** Because the CLI runs on the host with `bypassPermissions`, +it can access any host file or network resource. Only run on trusted task sets. + +The harness controls: - **System prompt**: what instructions the agent receives - **Model selection**: which Claude model to use @@ -30,9 +37,7 @@ commands internally. The harness controls: ## Prerequisites - Claude Code CLI installed on the host: `npm install -g @anthropic-ai/claude-code` -- Authenticated: `claude login` (or already logged in — no API key needed) -- The adapter automatically copies your `~/.claude` auth credentials into the - container at runtime. +- Authenticated: `claude login` (no API key needed — uses your subscription) ## Setup @@ -86,8 +91,8 @@ docker build -f Dockerfile.claude-code -t autoagent-base . rm -rf jobs && mkdir -p jobs && uv run harbor run -p tasks/ -n 100 --agent-import-path agent-claude-code:AutoAgent -o jobs --job-name latest > run.log 2>&1 ``` -No `.env` file or API key needed — the adapter copies your local Claude auth -into the container automatically. +No `.env` file or API key needed — the CLI runs on the host and uses your +existing Claude Code session directly. ## Goal, Logging, Experiment Loop, Keep/Discard Rules From 7a021f558f41998dc1f98bc046290a57240bd759 Mon Sep 17 00:00:00 2001 From: Szymon Paluch Date: Fri, 3 Apr 2026 13:37:27 +0200 Subject: [PATCH 12/12] feat: add timeout and budget controls - Add MAX_BUDGET_USD (default $1.00) via --max-budget-usd flag - Add TIMEOUT_SEC (default 540s) as configurable constant - Prevents meta-agent from getting stuck on hung tasks --- agent-claude-code.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/agent-claude-code.py b/agent-claude-code.py index 7a9f3906..946e4cd9 100644 --- a/agent-claude-code.py +++ b/agent-claude-code.py @@ -58,6 +58,8 @@ MODEL = "sonnet" MAX_TURNS = 30 +MAX_BUDGET_USD = 1.0 +TIMEOUT_SEC = 540 PERMISSION_MODE = "bypassPermissions" ALLOWED_TOOLS: list[str] = [] CLI_EXTRA_FLAGS: list[str] = [] @@ -72,6 +74,7 @@ def build_cli_args(instruction_text: str) -> list[str]: "--verbose", "--model", MODEL, "--max-turns", str(MAX_TURNS), + "--max-budget-usd", str(MAX_BUDGET_USD), "--permission-mode", PERMISSION_MODE, ] if SYSTEM_PROMPT: @@ -283,7 +286,7 @@ async def run( cli_args, capture_output=True, text=True, - timeout=540, + timeout=TIMEOUT_SEC, cwd=str(task_dir), ) duration_ms = int((time.time() - t0) * 1000) @@ -303,8 +306,8 @@ async def run( if isinstance(exc.stdout, bytes) else (exc.stdout or "") ) - stderr_output = "ERROR: claude CLI timed out after 540s" - logger.error("claude CLI timed out after 540s (partial output preserved)") + stderr_output = f"ERROR: claude CLI timed out after {TIMEOUT_SEC}s" + logger.error("claude CLI timed out after %ds (partial output preserved)", TIMEOUT_SEC) except FileNotFoundError: duration_ms = int((time.time() - t0) * 1000) raw_output = ""