run all scenarios

Makefile

@@ -1,4 +1,4 @@
-.PHONY: sync up down test logs orchestrate smoke dump-logs server server-test webui-build serve
+.PHONY: sync up down test logs orchestrate smoke dump-logs server server-test webui-build serve mint-rate-limit-fallback-test
 
 E2E_TIMEOUT ?= 60
 
@@ -36,6 +36,16 @@ down:
 	docker compose down -v
 	@echo "Done."
 
+# Hermetic PR #597 scenario: primary mint returns 429, secondary mint settles.
+# Override ROUTSTR_CORE_REF to test an unmerged branch/PR, for example:
+#   ROUTSTR_CORE_REF=refs/pull/597/head make mint-rate-limit-fallback-test
+mint-rate-limit-fallback-test: sync
+	CASHU_MINTS=http://fault-proxy:3340,http://primary-mint:3338,http://fee-mint:3338 \
+		docker compose up -d --build relay mock-openai fault-mint fault-proxy primary-mint fee-mint node-a
+	@bash scripts/wait_for.sh node-a http://localhost:8001/v1/info $(E2E_TIMEOUT)
+	@bash scripts/wait_for.sh fault-proxy http://localhost:3340/__proxy__/stats $(E2E_TIMEOUT)
+	.venv/bin/pytest -q -s tests/integration/test_mint_rate_limit_fallback.py
+
 dump-logs:
 	@bash scripts/dump_logs.sh
 

runner/orchestrate.py

@@ -595,6 +595,15 @@ def orchestrate(
             )
         for idx, admin_token in enumerate(remote_admin_tokens or []):
             scenario_env[f"REMOTE_NODE_ADMIN_TOKEN_{idx}"] = admin_token
+        if token:
+            # UI-submitted Cashu is primarily used to top up local routstrd,
+            # but remote/direct node tests also accept raw Cashu as auth.
+            # Export it under their conventional env names without clobbering
+            # operator-provided values already present in the server env.
+            if not os.environ.get("NODE_A_API_KEY"):
+                scenario_env["NODE_A_API_KEY"] = token
+            if not os.environ.get("X_CASHU_TOKENS"):
+                scenario_env["X_CASHU_TOKENS"] = token
         if not token_ok:
             scenario_env["TOPUP_FAILED"] = "1"
         # Paid tests append their precise spend (millisats) here; summed below

scenarios/remote_all.yaml

@@ -0,0 +1,20 @@
+id: remote_all
+name: Remote node full harness
+description: >
+  Runs the full routstr-testing node-facing harness from the Web UI against a
+  deployed Routstr node. Choose target_profile=remote in the Run modal, paste
+  one or more node URLs, and paste a funded Cashu token. The orchestrator passes
+  that token to tests as NODE_A_API_KEY and X_CASHU_TOKENS so paid request tests
+  can authenticate directly against the remote node. Tests that need extra admin
+  access or multiple one-shot X-Cashu tokens will skip when those credentials are
+  not supplied.
+services_required: false
+target_profile: remote
+upstream_profile: mock
+selection:
+  paths:
+    - tests/integration
+parameters: {}
+expected_cost_sats: 5
+estimated_upstream_cost_usd: 0.0
+timeout_seconds: 600

server/scenarios.py

@@ -78,6 +78,7 @@ def _parse(scenarios_dir: Path, path: Path) -> ScenarioDetail:
         name=str(data.get("name", scenario_id)),
         description=str(data.get("description", "")),
         expected_cost_sats=expected_cost_sats,
+        target_profile=str(data.get("target_profile", "local")).lower(),
         upstream_profile=str(data.get("upstream_profile", "mock")).lower(),
         estimated_upstream_cost_usd=estimated_upstream_cost_usd,
         yaml=raw,
@@ -147,6 +148,7 @@ def list_scenarios(
             name=d.name,
             description=d.description,
             expected_cost_sats=d.expected_cost_sats,
+            target_profile=d.target_profile,
             upstream_profile=d.upstream_profile,
             estimated_upstream_cost_usd=d.estimated_upstream_cost_usd,
             stats=stats.get(d.id, ScenarioStats()),

server/schemas.py

@@ -24,6 +24,7 @@ class ScenarioSummary(BaseModel):
     name: str
     description: str = ""
     expected_cost_sats: int = 0
+    target_profile: str = "local"
     # ROU-153 — surfaced so the Run modal can show the USD cost preview and
     # the Scenarios list can flag real-upstream scenarios.
     upstream_profile: str = "mock"

tests/integration/fault_proxy.py

@@ -12,8 +12,9 @@
 -> retry. The melt-quote path (`POST /v1/melt/quote/bolt11`) is never faulted.
 
 Control plane (not forwarded):
-  POST /__proxy__/reset?faults=1  -> arm N faults, clear stats
-  GET  /__proxy__/stats           -> {melt_attempts, faulted, fault_remaining}
+  POST /__proxy__/reset?faults=1&kind=melt_insufficient
+  POST /__proxy__/reset?faults=1&kind=mint_quote_429&retry_after=0
+  GET  /__proxy__/stats
 
 Runs on the nutshell image (already ships fastapi/uvicorn/httpx) — no new build.
 """
@@ -31,6 +32,7 @@
 # NUT-05 melt EXECUTE paths (path param has no leading slash). The melt QUOTE
 # path (v1/melt/quote/bolt11) is deliberately excluded — only the execute fails.
 MELT_EXECUTE_PATHS = {"v1/melt/bolt11", "v1/melt"}
+MINT_QUOTE_PATHS = {"v1/mint/quote/bolt11", "v1/mint/quote"}
 
 # Raw NUT-00 error body a mint returns when inputs don't cover amount + fee.
 # The node's classifier keys on the code (11000 = nutshell TransactionError) and
@@ -40,23 +42,39 @@
 )
 
 app = FastAPI()
-_state = {"fault_remaining": 0}
+_state = {
+    "fault_remaining": 0,
+    "fault_kind": "melt_insufficient",
+    "retry_after": "0",
+}
 _stats: Counter = Counter()
 
 
 @app.post("/__proxy__/reset")
-async def reset(faults: int = 1) -> dict:
-    _state["fault_remaining"] = faults
+async def reset(
+    faults: int = 1,
+    kind: str = "melt_insufficient",
+    retry_after: str = "0",
+) -> dict:
+    if kind not in {"melt_insufficient", "mint_quote_429"}:
+        return {"error": f"unknown fault kind: {kind}"}
+    _state.update(
+        fault_remaining=faults,
+        fault_kind=kind,
+        retry_after=retry_after,
+    )
     _stats.clear()
-    return {"fault_remaining": faults}
+    return dict(_state)
 
 
 @app.get("/__proxy__/stats")
 async def stats() -> dict:
     return {
         "melt_attempts": _stats["melt_attempts"],
+        "mint_quote_attempts": _stats["mint_quote_attempts"],
+        "forwarded": _stats["forwarded"],
         "faulted": _stats["faulted"],
-        "fault_remaining": _state["fault_remaining"],
+        **_state,
     }
 
 
@@ -70,13 +88,32 @@ async def proxy(path: str, request: Request) -> Response:
         _stats["melt_attempts"] += 1
         print(f"[fault-proxy] melt execute #{_stats['melt_attempts']} "
               f"(fault_remaining={_state['fault_remaining']})", flush=True)
-        if _state["fault_remaining"] > 0:
+        if (
+            _state["fault_kind"] == "melt_insufficient"
+            and _state["fault_remaining"] > 0
+        ):
             _state["fault_remaining"] -= 1
             _stats["faulted"] += 1
             return Response(
                 content=_FAULT_BODY, status_code=400, media_type="application/json"
             )
 
+    if request.method == "POST" and path in MINT_QUOTE_PATHS:
+        _stats["mint_quote_attempts"] += 1
+        if (
+            _state["fault_kind"] == "mint_quote_429"
+            and _state["fault_remaining"] > 0
+        ):
+            _state["fault_remaining"] -= 1
+            _stats["faulted"] += 1
+            return Response(
+                content=json.dumps({"detail": "Rate limit exceeded."}),
+                status_code=429,
+                headers={"Retry-After": _state["retry_after"]},
+                media_type="application/json",
+            )
+
+    _stats["forwarded"] += 1
     async with httpx.AsyncClient(timeout=90) as client:
         upstream = await client.request(
             request.method,

tests/integration/test_mint_rate_limit_fallback.py

@@ -0,0 +1,141 @@
+"""End-to-end mint 429 fallback and invoice provenance coverage for PR #597.
+
+The primary mint is the fault proxy. It returns HTTP 429 for mint-quote requests,
+forcing routstr-core to create the invoice on the secondary trusted mint. The
+scenario then follows the invoice through settlement and verifies that a top-up
+for the resulting key stays on its backing mint instead of mixing collateral.
+"""
+from __future__ import annotations
+
+import os
+import subprocess
+import time
+
+import httpx
+import pytest
+
+from tests.integration.targets import require_node, unavailable
+
+pytestmark = pytest.mark.destructive
+
+NODE = os.environ.get("NODE_A_URL", "http://localhost:8001").rstrip("/")
+PROXY_CTL = os.environ.get("FAULT_PROXY_URL", "http://localhost:3340").rstrip("/")
+NODE_CONTAINER = os.environ.get("NODE_CONTAINER", "routstr-testing-node-a-1")
+PRIMARY_FAULT_MINT = "http://fault-proxy:3340"
+SECONDARY_MINT = "http://primary-mint:3338"
+
+
+@pytest.fixture(scope="module", autouse=True)
+def _require_stack() -> None:
+    require_node()
+    try:
+        response = httpx.get(f"{PROXY_CTL}/__proxy__/stats", timeout=5)
+        if response.status_code != 200:
+            unavailable(f"fault-proxy not reachable at {PROXY_CTL}; run `make up`")
+    except httpx.HTTPError:
+        unavailable(f"fault-proxy not reachable at {PROXY_CTL}; run `make up`")
+
+    try:
+        inspected = subprocess.run(
+            [
+                "docker",
+                "inspect",
+                NODE_CONTAINER,
+                "--format",
+                "{{range .Config.Env}}{{println .}}{{end}}",
+            ],
+            capture_output=True,
+            text=True,
+            timeout=10,
+            check=True,
+        )
+    except (FileNotFoundError, subprocess.SubprocessError) as exc:
+        unavailable(f"cannot verify node mint topology: {exc}")
+    expected = f"CASHU_MINTS={PRIMARY_FAULT_MINT},{SECONDARY_MINT}"
+    if expected not in inspected.stdout:
+        unavailable(
+            "rate-limit topology is not active; run "
+            "`make mint-rate-limit-fallback-test`"
+        )
+
+
+def _invoice_mint_url(invoice_id: str) -> str:
+    script = (
+        "import sqlite3; "
+        "db=sqlite3.connect('/data/node-a.db'); "
+        f"row=db.execute(\"select mint_url from lightning_invoices where id='{invoice_id}'\").fetchone(); "
+        "print(row[0] if row else '')"
+    )
+    try:
+        result = subprocess.run(
+            ["docker", "exec", NODE_CONTAINER, "python", "-c", script],
+            capture_output=True,
+            text=True,
+            timeout=10,
+            check=True,
+        )
+    except (FileNotFoundError, subprocess.SubprocessError) as exc:
+        unavailable(f"cannot inspect node invoice provenance: {exc}")
+    return result.stdout.strip()
+
+
+def _wait_paid(invoice_id: str, timeout: float = 30) -> dict:
+    deadline = time.monotonic() + timeout
+    last: dict = {}
+    while time.monotonic() < deadline:
+        response = httpx.get(
+            f"{NODE}/lightning/invoice/{invoice_id}/status", timeout=15
+        )
+        assert response.status_code == 200, response.text
+        last = response.json()
+        if last["status"] == "paid":
+            return last
+        time.sleep(1)
+    pytest.fail(f"invoice {invoice_id} did not settle: {last}")
+
+
+def _create_invoice(amount: int, *, purpose: str, api_key: str | None = None) -> dict:
+    headers = {"Authorization": f"Bearer {api_key}"} if api_key else None
+    response = httpx.post(
+        f"{NODE}/lightning/invoice",
+        json={"amount_sats": amount, "purpose": purpose},
+        headers=headers,
+        timeout=30,
+    )
+    assert response.status_code == 200, response.text
+    return response.json()
+
+
+def test_429_fallback_persists_mint_and_topup_stays_on_backing_mint() -> None:
+    reset = httpx.post(
+        f"{PROXY_CTL}/__proxy__/reset",
+        params={"faults": 10, "kind": "mint_quote_429", "retry_after": "0"},
+        timeout=10,
+    )
+    assert reset.status_code == 200, reset.text
+
+    created = _create_invoice(32, purpose="create")
+    create_stats = httpx.get(f"{PROXY_CTL}/__proxy__/stats", timeout=10).json()
+    assert create_stats["faulted"] >= 1, create_stats
+    assert create_stats["mint_quote_attempts"] >= 1, create_stats
+    assert _invoice_mint_url(created["invoice_id"]) == SECONDARY_MINT
+
+    paid = _wait_paid(created["invoice_id"])
+    api_key = paid.get("api_key")
+    assert api_key and api_key.startswith("sk-")
+
+    attempts_before_topup = httpx.get(
+        f"{PROXY_CTL}/__proxy__/stats", timeout=10
+    ).json()["mint_quote_attempts"]
+    topup = _create_invoice(16, purpose="topup", api_key=api_key)
+    attempts_after_topup = httpx.get(
+        f"{PROXY_CTL}/__proxy__/stats", timeout=10
+    ).json()["mint_quote_attempts"]
+
+    assert attempts_after_topup == attempts_before_topup, (
+        "top-up retried the rate-limited primary instead of staying on the "
+        "API key's backing mint"
+    )
+    assert _invoice_mint_url(topup["invoice_id"]) == SECONDARY_MINT
+    topup_paid = _wait_paid(topup["invoice_id"])
+    assert topup_paid["api_key"] == api_key

tests/test_orchestrate_balance.py

@@ -251,7 +251,7 @@ def fake_run_pytest(scenario, junit_path, env):
     db_path = tmp_path / "runs.db"
     run_id = orch_mod.orchestrate(
         scenario_id="remote_smoke",
-        token=None,
+        token="cashu-test-token",
         db_path=db_path,
         scenarios_dir=scenarios_dir,
         compose_file=tmp_path / "compose.yml",
@@ -278,6 +278,8 @@ def fake_run_pytest(scenario, junit_path, env):
     )
     assert seen_env["REMOTE_NODE_ADMIN_TOKEN_0"] == "secret-1"
     assert seen_env["REMOTE_NODE_ADMIN_TOKEN_1"] == "secret-2"
+    assert seen_env["NODE_A_API_KEY"] == "cashu-test-token"
+    assert seen_env["X_CASHU_TOKENS"] == "cashu-test-token"
 
 
 def test_orchestrate_remote_profile_requires_urls(tmp_path, monkeypatch):

webui/pnpm-workspace.yaml

@@ -0,0 +1,4 @@
+allowBuilds:
+  esbuild: true
+onlyBuiltDependencies:
+  - esbuild

webui/src/components/RunTokenModal.tsx

@@ -16,6 +16,8 @@ interface Props {
   estimatedCostSats?: number;
   /** Scenario's declared real-upstream cost (USD) — drives the cost preview. */
   estimatedUpstreamCostUsd?: number;
+  /** Scenario's declared target_profile, used as the dropdown default. */
+  scenarioTargetProfile?: TargetProfile;
   /** Scenario's declared upstream_profile, used as the dropdown default. */
   scenarioUpstreamProfile?: string;
   onClose: () => void;
@@ -31,6 +33,7 @@ export function RunTokenModal({
   scenarioName,
   estimatedCostSats,
   estimatedUpstreamCostUsd,
+  scenarioTargetProfile,
   scenarioUpstreamProfile,
   onClose,
   onSubmit,
@@ -51,6 +54,7 @@ export function RunTokenModal({
 
   useEffect(() => {
     if (!open) return;
+    setTargetProfile(scenarioTargetProfile || 'local');
     setUpstreamProfile(scenarioUpstreamProfile || MOCK_UPSTREAM);
     let active = true;
     api
@@ -60,7 +64,7 @@ export function RunTokenModal({
     return () => {
       active = false;
     };
-  }, [open, scenarioUpstreamProfile]);
+  }, [open, scenarioTargetProfile, scenarioUpstreamProfile]);
 
   const parsedUrls = useMemo(
     () =>

webui/src/lib/types.ts

@@ -7,6 +7,7 @@ export interface ScenarioSummary {
   name: string;
   description: string;
   expected_cost_sats: number;
+  target_profile: TargetProfile;
   upstream_profile: string;
   estimated_upstream_cost_usd: number;
   stats: Record<string, unknown>;