From acc558c9c28b18e30e232e514c397fbe4ab02ce5 Mon Sep 17 00:00:00 2001 From: Fengzdadi <453788063@qq.com> Date: Fri, 2 Oct 2026 20:55:54 -0400 Subject: [PATCH] feat(e2e): run paired continuation workloads with Pi `paired --host pi` runs both arms with Harbor's Pi agent as a fourth PluginHost. Harbor installs Pi from its former npm name, which ends before the versions the PowerContext Pi package supports, so the agent installs `@earendil-works/pi-coding-agent` 0.82.1, the version the package's lockfile and CI use, with Harbor's own steps. The ON arm runs `pi install` on the package, as `powercontext setup pi` does, and its POWERCONTEXT_PI_* environment carries the Server URL, Scope, plain-HTTP consent, and the harness Client's token. As on the other hosts, the OFF arm installs no package and mounts nothing. The ON container sees only the package's manifest, extension, sources, and Skill. The reasoning effort becomes `--thinking`. Harbor runs Pi with `--no-session`, so Pi saves no session. Pi's bash tool keeps the full output of a command over 2,000 lines or 50 KB as `pi-bash-*.log` in the temporary directory, and nothing clears it between the steps of a trial, so the agent removes those files before each session in both arms. Co-Authored-By: Claude Fable 5.1 --- e2e/bub/README.md | 37 +++++++-- e2e/bub/src/powercontext_e2e/__main__.py | 2 +- e2e/bub/src/powercontext_e2e/harbor_pi.py | 94 +++++++++++++++++++++++ e2e/bub/src/powercontext_e2e/hosts.py | 16 +++- e2e/bub/tests/test_evidence_redaction.py | 1 + e2e/bub/tests/test_harbor_job_config.py | 73 +++++++++++++++++- 6 files changed, 213 insertions(+), 10 deletions(-) create mode 100644 e2e/bub/src/powercontext_e2e/harbor_pi.py diff --git a/e2e/bub/README.md b/e2e/bub/README.md index eecdb96cd..61811a1a6 100644 --- a/e2e/bub/README.md +++ b/e2e/bub/README.md @@ -6,8 +6,8 @@ on Memory collection, grounding, and recall rather than the native task reward. The common architecture separates workload selection, execution, evidence, Memory evaluation, and reporting. Bub is the execution adapter for acceptance workloads because its model, tools, context injection, capture, and checkpoints -are observable. The OFF/ON comparison can also run on Codex, Claude Code, and OpenCode, without changing the workload -or evaluation contracts. +are observable. The OFF/ON comparison can also run on Codex, Claude Code, OpenCode, and Pi, without changing the +workload or evaluation contracts. Every workload follows one execution path: @@ -154,8 +154,8 @@ step, because Harbor would then skip the recall step when that step's unrelated The `paired` command runs each selected workload with PowerContext off and on, in separate containers, and repeats this for `--trials` trials. The arm that runs first alternates between trials. `--host` selects the agent host for -both arms: `bub` by default, `codex`, `claude-code`, or `opencode`. Each host uses its own PowerContext integration, so -ON means what that integration does for its users. +both arms: `bub` by default, `codex`, `claude-code`, `opencode`, or `pi`. Each host uses its own PowerContext +integration, so ON means what that integration does for its users. - OFF is the host as a user without PowerContext has it. The agent is not told that PowerContext is off, and it cannot find PowerContext: no integration is installed, no PowerContext sources are mounted in its container, and it @@ -165,9 +165,10 @@ ON means what that integration does for its users. - ON installs the integration, binds it to a new Scope, and gives it the harness Client's Server token. For Bub this means the plugin with `capture_events` enabled, so that, like the other host integrations, it captures what the user says without relying on the model to call a memory tool. This is not the plugin's default setting. Codex runs - with `--enable plugins`, Claude Code has the plugin enabled, and OpenCode loads it from its plugins directory. In all - three, the plugin captures each user prompt and asks for context before each turn, and the plugin's tools (MCP for - Codex and Claude Code, native tools for OpenCode) and Skill are available to the model. + with `--enable plugins`, Claude Code has the plugin enabled, OpenCode loads it from its plugins directory, and Pi + loads the installed package. In all four, the plugin captures each user prompt and asks for context before each + turn, and the plugin's tools (MCP for Codex and Claude Code, native tools for OpenCode and Pi) and Skill are + available to the model. - Everything else is the same in both arms: image, host version, model, reasoning settings, and budget. Both arms' containers can reach the Server, and the Server keeps the ON arms' Memory across trials. The command @@ -283,6 +284,28 @@ export OPENROUTER_API_KEY=replace-me make harness-paired ARGS='--host opencode --trials 2' ``` +Pi 0.82.1, the version the PowerContext Pi package tests against, runs through Harbor's Pi agent. Harbor installs Pi +from its former npm name, which ends before that version, so the harness installs `@earendil-works/pi-coding-agent` +with the same steps. The ON arm then runs `pi install` on the package, as `powercontext setup pi` does; the agent +container sees only its `package.json`, `extensions`, `src`, and `skills`. The package reads +`POWERCONTEXT_PI_SERVER_URL`, the Scope, and the plain-HTTP consent `POWERCONTEXT_PI_ALLOW_INSECURE_HTTP` from its +environment. Pi runs every session with `--no-session`, so it saves no session. Its bash tool keeps the full output of +a command over 2,000 lines or 50 KB as `pi-bash-*.log` in the temporary directory, so before each session in both arms +the harness removes those files. The harness selects the model with `POWERCONTEXT_E2E_PI_MODEL` in Pi's +`provider/model` form and passes `POWERCONTEXT_E2E_PI_REASONING_EFFORT`, default `medium`, as `--thinking`. Harbor +passes the provider's key, such as `OPENROUTER_API_KEY`. + +```bash +export POWERCONTEXT_CLIENT_SERVER_URL=http://127.0.0.1:8000 +export POWERCONTEXT_CLIENT_API_TOKEN=replace-me +export POWERCONTEXT_CLIENT_TIMEOUT=150 +export POWERCONTEXT_PI_SERVER_URL=http://host-gateway:8000 +export POWERCONTEXT_PI_ALLOW_INSECURE_HTTP=true +export POWERCONTEXT_E2E_PI_MODEL=openrouter/z-ai/glm-5.3 +export OPENROUTER_API_KEY=replace-me +make harness-paired ARGS='--host pi --trials 2' +``` + Each arm writes `observation.json`, which includes the per-session Server snapshots for ON, and its Harbor jobs: ```text diff --git a/e2e/bub/src/powercontext_e2e/__main__.py b/e2e/bub/src/powercontext_e2e/__main__.py index d88b1cbf9..7ce3d6fbe 100644 --- a/e2e/bub/src/powercontext_e2e/__main__.py +++ b/e2e/bub/src/powercontext_e2e/__main__.py @@ -102,7 +102,7 @@ def main() -> None: paired_parser.add_argument("--trials", type=int, default=2, help="Trials per arm; the arm order alternates.") paired_parser.add_argument( "--host", - choices=("bub", "codex", "claude-code", "opencode"), + choices=("bub", "codex", "claude-code", "opencode", "pi"), default="bub", help="Agent host that runs both arms with its own PowerContext integration.", ) diff --git a/e2e/bub/src/powercontext_e2e/harbor_pi.py b/e2e/bub/src/powercontext_e2e/harbor_pi.py new file mode 100644 index 000000000..9b804683e --- /dev/null +++ b/e2e/bub/src/powercontext_e2e/harbor_pi.py @@ -0,0 +1,94 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Harbor's Pi agent with the local PowerContext Pi package installed and switched per arm.""" + +from __future__ import annotations + +from typing import Any, override + +from harbor.agents.installed.pi import Pi +from harbor.environments.base import BaseEnvironment +from harbor.models.agent.context import AgentContext + +from .harbor_agent import REMOTE_SOURCE + +# The version the PowerContext Pi package's lockfile and CI test against. +PI_VERSION = "0.82.1" +PI_PACKAGE = "@earendil-works/pi-coding-agent" +REMOTE_PLUGIN = f"{REMOTE_SOURCE}/integrations/pi/plugins/powercontext" +# Pi's bash tool keeps the full output of a command it truncated, over 2,000 lines or 50 KB, in its temporary +# directory, and nothing else clears it between the steps of a trial. Node's os.tmpdir() reads TMPDIR, then TMP, then +# TEMP. +PI_TOOL_OUTPUT = '"${TMPDIR:-${TMP:-${TEMP:-/tmp}}}"/pi-bash-*.log' + + +class PowerContextPiAgent(Pi): + """Install the PowerContext Pi package only in the ON arm, and start every session without Pi's earlier output. + + As on the other hosts, the OFF arm runs without the package. The package reads its Server URL, Scope, consent, and + Server token from its own environment, which only the ON arm receives. Harbor runs Pi without a saved session, and + each session starts without the tool output an earlier session saved, so neither arm can read an earlier session + from Pi's own files. + """ + + def __init__( + self, + *, + server_url: str, + powercontext: bool = True, + reasoning_effort: str | None = None, + **kwargs: Any, + ) -> None: + del server_url # The ON arm passes it to the package through the package's environment. + self._powercontext = powercontext + super().__init__(version=PI_VERSION, thinking=reasoning_effort, **kwargs) + + @override + async def install(self, environment: BaseEnvironment) -> None: + # Harbor installs Pi from its deprecated npm name, which stopped before the versions this package supports, + # so Harbor 0.16's steps install it from the current name. Compare them when upgrading Harbor. + await self.exec_as_root( + environment, + command="apt-get update && apt-get install -y curl", + env={"DEBIAN_FRONTEND": "noninteractive"}, + ) + await self.exec_as_agent( + environment, + command=( + "set -euo pipefail; " + "curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.2/install.sh | bash && " + 'export NVM_DIR="$HOME/.nvm" && ' + '\\. "$NVM_DIR/nvm.sh" || true && ' + "command -v nvm &>/dev/null || { echo 'Error: NVM failed to load' >&2; exit 1; } && " + "nvm install 22 && " + f"npm install -g {PI_PACKAGE}@{self._version} && " + "pi --version" + ), + ) + if self._powercontext: + # As `powercontext setup pi` does; Pi reads the package in place from the read-only mount. + await self.exec_as_agent(environment, command=install_plugin_command()) + + @override + async def run(self, instruction: str, environment: BaseEnvironment, context: AgentContext) -> None: + await self.exec_as_agent(environment, command=f"rm -f {PI_TOOL_OUTPUT}") + await super().run(instruction, environment, context) + + +def install_plugin_command() -> str: + """Install the mounted PowerContext Pi package into Pi.""" + + # Harbor's Pi agent loads Node the same way before each run. + return f". ~/.nvm/nvm.sh; pi install {REMOTE_PLUGIN}" diff --git a/e2e/bub/src/powercontext_e2e/hosts.py b/e2e/bub/src/powercontext_e2e/hosts.py index a6d4a2c7e..f5dfc5108 100644 --- a/e2e/bub/src/powercontext_e2e/hosts.py +++ b/e2e/bub/src/powercontext_e2e/hosts.py @@ -29,6 +29,7 @@ from .harbor_claude_code import CLAUDE_CODE_VERSION from .harbor_codex import CODEX_VERSION from .harbor_opencode import OPENCODE_VERSION +from .harbor_pi import PI_VERSION from .settings import ( bub_environment, codex_auth_path, @@ -259,6 +260,19 @@ def _plugin_environment(self) -> dict[str, str]: ), ) +# Harbor passes the key of the model's provider, such as OPENROUTER_API_KEY. The package's extension is TypeScript that +# Pi loads directly with the Skill, and reads its Server URL from its environment. +PI = PluginHost( + name="pi", + version=PI_VERSION, + agent_import_path="powercontext_e2e.harbor_pi:PowerContextPiAgent", + plugin_prefix="POWERCONTEXT_PI_", + setting_prefix="POWERCONTEXT_E2E_PI_", + plugin_paths=tuple( + f"integrations/pi/plugins/powercontext/{name}" for name in ("package.json", "extensions", "src", "skills") + ), +) + def _capture_settings(task: E2ETask) -> tuple[bool, int, int]: evaluation = task.evaluation @@ -289,7 +303,7 @@ def read_only_bind(source: Path, target: str) -> ServiceVolumeConfig: } -_HOSTS: dict[str, HostAdapter] = {host.name: host for host in (BubHost(), CODEX, CLAUDE_CODE, OPENCODE)} +_HOSTS: dict[str, HostAdapter] = {host.name: host for host in (BubHost(), CODEX, CLAUDE_CODE, OPENCODE, PI)} def host_adapter(name: str) -> HostAdapter: diff --git a/e2e/bub/tests/test_evidence_redaction.py b/e2e/bub/tests/test_evidence_redaction.py index 2d5d22f94..09bde2106 100644 --- a/e2e/bub/tests/test_evidence_redaction.py +++ b/e2e/bub/tests/test_evidence_redaction.py @@ -72,6 +72,7 @@ def test_resolved_instruction_evidence_matches_harbor_acp_summaries( "POWERCONTEXT_OPENCODE_AUTHORIZATION", "HF_TOKEN", "AWS_SECRET_ACCESS_KEY", + "POWERCONTEXT_PI_AUTHORIZATION", ) ), # The Client and the Server accept a token of any length. diff --git a/e2e/bub/tests/test_harbor_job_config.py b/e2e/bub/tests/test_harbor_job_config.py index c71eee9da..b9fe81248 100644 --- a/e2e/bub/tests/test_harbor_job_config.py +++ b/e2e/bub/tests/test_harbor_job_config.py @@ -23,6 +23,7 @@ import pytest from harbor.agents.installed.opencode import OpenCode +from harbor.agents.installed.pi import Pi from harbor.environments.base import ExecResult from harbor.models.agent.context import AgentContext from harbor.models.job.config import JobConfig @@ -31,6 +32,7 @@ from powercontext_e2e.harbor_claude_code import PowerContextClaudeCodeAgent from powercontext_e2e.harbor_codex import PowerContextCodexAgent from powercontext_e2e.harbor_opencode import PowerContextOpenCodeAgent +from powercontext_e2e.harbor_pi import PowerContextPiAgent from powercontext_e2e.hosts import host_adapter from powercontext_e2e.runner import _job_config, prepare_runtime_task, require_runtime_models, run_tasks from powercontext_e2e.settings import HarnessSettings, ModelNotConfiguredError @@ -240,6 +242,13 @@ class _PluginHost(NamedTuple): "POWERCONTEXT_OPENCODE_ALLOW_INSECURE_HTTP", "POWERCONTEXT_OPENCODE_SCOPE_ID", ), + _PluginHost( + "pi", + "POWERCONTEXT_E2E_PI_MODEL", + "POWERCONTEXT_PI_SERVER_URL", + "POWERCONTEXT_PI_ALLOW_INSECURE_HTTP", + "POWERCONTEXT_PI_SCOPE_ID", + ), ] @@ -426,6 +435,68 @@ def test_opencode_reasoning_effort_selects_the_model_variant(tmp_path: Path) -> assert _opencode_agent(tmp_path, powercontext=True).build_cli_flags() == "--variant medium" +def _pi_agent(tmp_path: Path, *, powercontext: bool) -> PowerContextPiAgent: + return PowerContextPiAgent( + logs_dir=tmp_path, + model_name="openrouter/model-test", + server_url="http://host-gateway:8000", + powercontext=powercontext, + ) + + +@pytest.mark.parametrize("powercontext", [True, False]) +def test_pi_installs_the_package_only_for_on(tmp_path: Path, powercontext: bool) -> None: + environment = _RecordingEnvironment() + + asyncio.run(_pi_agent(tmp_path, powercontext=powercontext).install(environment)) + + installed = [command for command in environment.commands if "pi install" in command] + assert bool(installed) is powercontext + + +@pytest.mark.parametrize("powercontext", [True, False]) +def test_pi_sessions_start_without_tool_output_an_earlier_session_left( + monkeypatch, tmp_path: Path, powercontext: bool +) -> None: + # Harbor leaves the container's temporary directory in place between the steps of a trial. Pi's bash tool keeps + # the full output of a truncated command there, which a later session could otherwise read. + started: list[set[str]] = [] + environment = _ShellEnvironment(tmp_path) + (environment.tmp / "pi-bash-0123456789abcdef.log").write_text("The team chose OceanBase with 12 shards.") + (environment.tmp / "notes.txt").write_text("Not Pi's file.") + + async def run_pi(self, instruction, environment, context) -> None: + started.append({path.name for path in environment.tmp.iterdir()}) + + monkeypatch.setattr(Pi, "run", run_pi) + + asyncio.run(_pi_agent(tmp_path, powercontext=powercontext).run("task", environment, AgentContext())) + + assert started == [{"notes.txt"}] + + +def test_pi_sessions_start_when_no_earlier_tool_output_exists(monkeypatch, tmp_path: Path) -> None: + started: list[str] = [] + + async def run_pi(self, instruction, environment, context) -> None: + started.append(instruction) + + monkeypatch.setattr(Pi, "run", run_pi) + + asyncio.run(_pi_agent(tmp_path, powercontext=False).run("task", _ShellEnvironment(tmp_path), AgentContext())) + + assert started == ["task"] + + +def test_pi_runs_without_a_saved_session(tmp_path: Path) -> None: + # Pi saves every session unless told not to, so the harness relies on Harbor passing `--no-session`. + environment = _RecordingEnvironment() + + asyncio.run(_pi_agent(tmp_path, powercontext=False).run("task", environment, AgentContext())) + + assert any("pi --print" in command and "--no-session" in command for command in environment.commands) + + @pytest.mark.parametrize("host", _PLUGIN_HOSTS, ids=lambda host: host.name) def test_plugin_host_requires_a_model_and_the_server_url_before_any_run(monkeypatch, host: _PluginHost) -> None: adapter = host_adapter(host.name) @@ -438,7 +509,7 @@ def test_plugin_host_requires_a_model_and_the_server_url_before_any_run(monkeypa require_runtime_models((_PAIRED_TASK,), adapter) -@pytest.mark.parametrize("host", ["bub", "codex", "claude-code", "opencode"]) +@pytest.mark.parametrize("host", ["bub", "codex", "claude-code", "opencode", "pi"]) @pytest.mark.parametrize( "manifest", ["paired-tasks/project-decision-continuation.yaml", "tasks/acceptance-01-project-database-decision.yaml"],