feat(conformance): ADPP conformance harness — foundation (#25)

.github/workflows/ci.yml

@@ -22,10 +22,85 @@ jobs:
           breaking: true
           breaking_against: "https://github.com/anolishq/anolis-protocol.git#branch=main"
 
+  conformance:
+    name: conformance (verifier self-tests)
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        # 3.10 exercises the tomli fallback (stdlib tomllib is 3.11+); 3.12 is
+        # the primary lane.
+        python-version: ["3.10", "3.12"]
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+        with:
+          fetch-depth: 0 # hatch-vcs needs tags for the version
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: bufbuild/buf-action@fd21066df7214747548607aaa45548ba2b9bc1ff # v1
+        with:
+          setup_only: true
+      - name: Generate Python bindings + package shims
+        run: |
+          buf generate
+          touch gen/python/anolis/__init__.py \
+                gen/python/anolis/deviceprovider/__init__.py \
+                gen/python/anolis/deviceprovider/v1/__init__.py
+          cat > gen/python/protocol_pb2.py <<'EOF'
+          """Compatibility shim: re-exports all ADPP v1 generated classes."""
+          from anolis.deviceprovider.v1.call_pb2 import *
+          from anolis.deviceprovider.v1.envelope_pb2 import *
+          from anolis.deviceprovider.v1.handshake_pb2 import *
+          from anolis.deviceprovider.v1.health_pb2 import *
+          from anolis.deviceprovider.v1.inventory_pb2 import *
+          from anolis.deviceprovider.v1.readiness_pb2 import *
+          from anolis.deviceprovider.v1.status_pb2 import *
+          from anolis.deviceprovider.v1.telemetry_pb2 import *
+          from anolis.deviceprovider.v1.types_pb2 import *
+          from anolis.deviceprovider.v1.value_pb2 import *
+          EOF
+      - name: Install the wheel (with conformance extra)
+        run: pip install ".[conformance]"
+      - name: Verifier self-tests (hermetic — no external binary)
+        run: anolis-adpp-conformance --self-test -p no:cacheprovider -ra
+      - name: Plugin isolation regression (unrelated pytest must still pass)
+        run: |
+          mkdir -p /tmp/unrelated
+          printf 'def test_unrelated():\n    assert True\n' > /tmp/unrelated/test_unrelated.py
+          python -m pytest /tmp/unrelated -q -p no:cacheprovider
+      - name: Misconfiguration must fail, never report green
+        run: |
+          set +e
+          fail() { echo "EXPECTED NONZERO: $1"; exit 1; }
+          anolis-adpp-conformance -p no:cacheprovider;            [ $? -ne 0 ] || fail "bare invocation (no mode)"
+          anolis-adpp-conformance --provider-bin /x -p no:cacheprovider; [ $? -ne 0 ] || fail "one provider arg"
+          anolis-adpp-conformance --provider-bin /x --provider-config /y -p no:cacheprovider; [ $? -ne 0 ] || fail "two provider args"
+          anolis-adpp-conformance --self-test --provider-bin /x -p no:cacheprovider; [ $? -ne 0 ] || fail "--self-test + provider arg"
+          # A waiver targeting a non-executable-profile test must be rejected at
+          # collection. Use REAL bin+config paths so the only possible failure
+          # reason is the waiver scope — and assert the specific diagnostic.
+          real_bin="$(mktemp)"; printf '#!/bin/sh\nexit 0\n' > "$real_bin"; chmod +x "$real_bin"
+          real_cfg="$(mktemp)"; echo '{}' > "$real_cfg"
+          printf 'provider_name = "p"\n[waivers]\ntest_read_signals_response_shape = "bogus"\n' > /tmp/bad-profile.toml
+          out="$(anolis-adpp-conformance --provider-bin "$real_bin" --provider-config "$real_cfg" --provider-profile /tmp/bad-profile.toml -p no:cacheprovider 2>&1)"
+          [ $? -ne 0 ] || fail "out-of-scope waiver (exit)"
+          echo "$out" | grep -q "waivers may only target executable_profile tests" || fail "out-of-scope waiver fired for the wrong reason:\n$out"
+          echo "OK: every misconfiguration returned nonzero (waiver scope for the right reason)"
+      # NOTE: cross-provider conformance (running the harness against a real
+      # provider binary) lives in each PROVIDER repo's CI — the provider pulls
+      # this pinned wheel + supplies its own --provider-profile. The contract
+      # repo must not depend on a concrete implementer's release, so the only
+      # thing gated here is that the verifier itself is sound (the hermetic
+      # self-tests above against in-repo fake providers).
+
   ok:
     name: ok
     if: always()
-    needs: [lint]
+    needs: [lint, conformance]
     runs-on: ubuntu-latest
     timeout-minutes: 5
     steps:

.gitignore

@@ -45,3 +45,13 @@ working/
 
 # buf generate output (CI artifacts, not committed)
 gen/
+
+# Python (conformance harness + wheel build)
+__pycache__/
+*.py[cod]
+.pytest_cache/
+*.egg-info/
+.eggs/
+dist/
+.venv/
+venv/

README.md

@@ -23,13 +23,24 @@ FetchContent_MakeAvailable(anolis_protocol)
 
 Replace the version and hash from the [latest release](https://github.com/anolishq/anolis-protocol/releases/latest) `SHA256SUMS` file.
 
-**Python (PyPI wheel):**
+**Python (wheel from the GitHub Release):**
+
+Each tagged release attaches a built wheel (generated protobuf bindings) to the [GitHub Release](https://github.com/anolishq/anolis-protocol/releases/latest) (there is no PyPI publish). Pin it by URL:
 
 ```sh
-pip install anolis-protocol
+pip install "anolis-protocol @ https://github.com/anolishq/anolis-protocol/releases/download/v1.1.4/anolis_protocol-1.1.4-py3-none-any.whl"
 ```
 
-The PyPI package is published on each tagged release and includes the generated protobuf bindings.
+The cross-provider **conformance harness** (the `[conformance]` extra + the
+`anolis-adpp-conformance` script) ships in the wheel from its **first release**
+onward — releases up to and including v1.2.0 predate it. Until that release is
+cut, install it from a checkout:
+
+```sh
+pip install ".[conformance]"
+# from a tagged release that contains the harness (replace once one exists):
+# pip install "anolis-protocol[conformance] @ https://github.com/anolishq/anolis-protocol/releases/download/<first-release-with-harness>/anolis_protocol-<ver>-py3-none-any.whl"
+```
 
 **buf BSR (code generation):**
 
@@ -58,8 +69,10 @@ buf generate
 ```
 
 - Owns ADPP protobuf schema used by `anolis` and providers.
-- Contains protocol-only artifacts (schema + semantics + compatibility docs).
-- Contains no runtime/provider implementation code.
+- Contains the protocol contract (schema + normative semantics/profile docs) plus
+  the generic **conformance harness** that verifies any provider against it.
+- Contains **no provider/runtime implementation code** — the harness is a generic
+  verifier, and it ships no knowledge of any specific provider.
 
 ## Layout
 
@@ -82,9 +95,15 @@ anolis-protocol/
 │               ├── readiness.proto   # WaitReady request/response
 │               ├── types.proto       # Device, CapabilitySet, FunctionSpec, ArgSpec
 │               └── value.proto       # Value, ValueType
+├── conformance/            # generic ADPP conformance harness (shipped in the wheel)
+│   ├── anolis_conformance/ # AdppClient, spec, checks, the three contract suites
+│   └── ADPP-CONFORMANCE.md # how to run it (non-normative)
 └── docs/
     ├── index.md
-    ├── semantics.md
+    ├── semantics.md        # core ADPP v1 (normative)
+    ├── profiles/
+    │   ├── framed-stdio-v1.md            # stdio transport binding (normative)
+    │   └── anolis-executable-profile-v1.md  # Anolis executable conventions
     └── versioning.md
 ```
 

conformance/ADPP-CONFORMANCE.md

@@ -0,0 +1,136 @@
+# ADPP provider conformance harness
+
+A binary-level acceptance harness that drives an Anolis provider **executable**
+through the ADPP wire lifecycle and checks its behavior. It is the verifier for
+cross-provider convergence work and the future provider-SDK's acceptance test
+(anolis-protocol#25).
+
+> **This document is not normative.** The normative sources are
+> `docs/semantics.md` (core ADPP), `docs/profiles/framed-stdio-v1.md` (the stdio
+> transport binding), and `docs/profiles/anolis-executable-profile-v1.md` (Anolis
+> executable conventions — an organizational profile, not ADPP). Each test suite
+> cites its source. Where a normative document permits a choice, the harness
+> accepts every permitted behavior; if a check here ever conflicts with its
+> source, the source wins and the check is the bug.
+
+**Status:** *foundation.* This PR delivers the generic harness + hermetic
+verifier self-tests. It ships **no implementer-specific data** — providers pull
+this pinned artifact and supply their own `--provider-profile`. Per-provider CI
+lanes (in each provider repo), the full assertion set, the version alignment, and
+an org-level cross-version compatibility matrix are tracked follow-ups under #25 —
+do not read this as "all of #25 is done".
+
+**Platform:** Linux/POSIX only for now (the client uses `select` on pipes).
+Windows support is a tracked follow-up; don't present it as cross-platform yet.
+
+## Running it
+
+```bash
+pip install anolis-protocol[conformance]
+# --provider-config is the mock-mode config for CI (no real i2c);
+# --provider-profile is the provider-owned manifest (identity + waivers).
+anolis-adpp-conformance \
+  --provider-bin ./build/.../anolis-provider-X \
+  --provider-config config/conformance.yaml \
+  --provider-profile conformance.toml
+```
+
+`--provider-profile` points at a manifest **owned by the provider repo** — the
+harness ships no knowledge of any specific provider. See *Waiver policy* below
+for its format.
+
+There are two explicit modes, and **a misconfigured run never reports green**:
+
+- **provider mode** — all three `--provider-*` args are required; a partial set
+  or a bare invocation is a usage error (nonzero exit), so a broken provider-CI
+  command can't masquerade as conformant.
+- **self-test mode** — `anolis-adpp-conformance --self-test` runs only the
+  hermetic verifier self-tests (no provider args allowed).
+
+The console script loads the plugin explicitly and defaults to the gating set
+(`-m "not experimental"`). The plugin is **not** a global `pytest11` entry point,
+so installing the wheel never affects unrelated pytest runs.
+
+## Three separate contracts
+
+The harness tests three distinct contracts (kept in separate modules so they are
+not conflated):
+
+1. **ADPP core protocol** (`test_adpp_core.py`) — messages, status codes,
+   `request_id` correlation, capabilities, and read/call **semantics** per
+   `semantics.md`. Examples of deferring to the spec:
+   - Unknown signal id → the provider must pick **one consistent** behavior:
+     fail `NOT_FOUND` **or** return partial results omitting it (§7.4). The
+     harness accepts either.
+   - Both `function_id` and `function_name` given → the provider **MUST prefer
+     `function_id`** (§6.2). The harness does **not** require rejecting a
+     conflict.
+   - Unsupported version → `FAILED_PRECONDITION` **or** `UNIMPLEMENTED` (§3).
+2. **ADPP framed-stdio profile** (`test_framed_stdio.py`) — `uint32_le` framing,
+   the 1 MiB cap, fragmentation/coalescing, and **controlled** handling of a
+   malformed stream: a well-formed framed error response, or a clean documented
+   exit (codes 0/2/3). A crash (process killed by a signal → negative return
+   code), a hang, or an over-cap/malformed response is a **failure**.
+3. **Anolis provider executable profile** (`test_executable_profile.py`) — CLI
+   surface (`--version`, `--check-config`), the WaitReady diagnostics the runtime
+   reads (`init_time_ms`), and process lifecycle. **These are Anolis conventions,
+   not ADPP requirements** — a binary can be ADPP-conformant and still diverge
+   here.
+
+Concurrency note: `semantics.md` allows concurrent processing and out-of-order
+responses (correlate by `request_id`). The runtime's one-in-flight serialization
+is a runtime profile, **not** an ADPP restriction — the harness does not require it.
+
+## Verifier self-tests
+
+`test_selftest.py` drives the harness against deliberately-faulty fake providers
+and asserts the harness **rejects** each. These are hermetic (no external binary)
+and are what make the verifier trustworthy. They cover:
+
+- **transport faults** — hang, signal-crash, over-cap response, byte-drip,
+  mid-frame close, wrong `request_id`, missing status;
+- **the real malformed-input validator** (`checks.assert_controlled_malformed`,
+  the same code the framed-stdio suite runs) against a provider that answers
+  garbage with `CODE_OK`, with `CODE_UNSPECIFIED`, or with a response-then-crash
+  — each must be rejected, while a framed *error* response is accepted;
+- **the profile loader** — valid manifests, defaults, and every rejection path
+  (missing/invalid fields, unknown keys, malformed TOML).
+
+The CI also asserts at the command level that a misconfigured invocation (no
+mode, partial provider args, or a waiver targeting a non-executable-profile test)
+exits **nonzero** — the verifier must never report green having tested nothing.
+
+## Waiver (`xfail`) policy
+
+A provider's identity and tracked gaps live in a **provider-owned manifest** —
+this repo ships only the schema + loader, never a provider's data. Pass it with
+`--provider-profile conformance.toml`:
+
+```toml
+provider_name = "anolis-provider-<name>"    # required; asserted against Hello
+has_mock_devices = true                     # optional, default true
+
+[waivers]                                   # test base-name -> reason (issue link)
+test_cli_version_flag = "no --version (<owner>/<repo>#<issue-number>)"
+```
+
+Waivers apply as **strict** `xfail`s and may target **only executable-profile**
+tests (those carry the `executable_profile` marker). A waiver key naming any
+other test — a core, framed-stdio, or verifier test — is rejected at collection,
+so a waiver can never mask a real protocol/transport failure. Strict means a
+fixed divergence fails as `XPASS` until its waiver is removed, rather than
+silently preserving stale conformance debt. Because the manifest lives in the
+provider repo, the protocol package never re-releases for a provider-specific
+exception. Each waiver reason should carry an issue link (and ideally an
+owner/expiry).
+
+## Not yet covered (follow-ups, #25)
+
+Positive calls (by id and name) with valid args; argument type/bound validation;
+deadline behavior; typed-value/quality/timestamp assertions; full readiness
+diagnostics; pre-Hello handling; and capability id/name-convention checks.
+Provider-side concerns — each provider's `provider.conformance` CI lane (pulling
+this pinned wheel), its `conformance.toml`/mock config, and version-pin alignment
+— are tracked in the respective provider repos, plus an org-level cross-version
+compatibility matrix. (ADPP is currently implemented by `anolis-provider-sim`,
+`-ezo`, and `-bread`.)

conformance/anolis_conformance/__init__.py

@@ -0,0 +1,21 @@
+"""Cross-provider ADPP conformance harness.
+
+Drives any ADPP provider binary through the ADPP v1 wire lifecycle and asserts
+compliance. See ``ADPP-CONFORMANCE.md`` for the executable spec.
+
+Run it against a provider binary:
+
+    anolis-adpp-conformance \\
+        --provider-bin ./build/.../anolis-provider-X \\
+        --provider-config config/conformance.yaml \\
+        --provider-profile conformance.toml
+
+or equivalently (the plugin is not globally registered, so load it explicitly)::
+
+    pytest -p anolis_conformance.plugin --pyargs anolis_conformance --provider-bin ...
+"""
+
+__all__ = ["AdppClient", "ProviderProfile", "load_profile"]
+
+from .client import AdppClient
+from .profiles import ProviderProfile, load_profile

conformance/anolis_conformance/_process.py

@@ -0,0 +1,70 @@
+"""Process lifecycle + output capture helpers."""
+
+from __future__ import annotations
+
+import subprocess
+import threading
+from typing import Any
+
+
+class LineCapture:
+    """Capture a text/bytes stream's lines in a background thread (diagnostics)."""
+
+    def __init__(self, stream: Any | None):
+        self._stream = stream
+        self._lines: list[str] = []
+        self._lock = threading.Lock()
+        self._stop = threading.Event()
+        self._thread: threading.Thread | None = None
+
+    def start(self) -> None:
+        if self._stream is None:
+            return
+        self._thread = threading.Thread(target=self._run, daemon=True)
+        self._thread.start()
+
+    def _run(self) -> None:
+        assert self._stream is not None
+        try:
+            while not self._stop.is_set():
+                line = self._stream.readline()
+                if isinstance(line, bytes):
+                    if line == b"":
+                        break
+                    cleaned = line.decode("utf-8", errors="replace").rstrip("\r\n")
+                else:
+                    if line == "":
+                        break
+                    cleaned = line.rstrip("\r\n")
+                with self._lock:
+                    self._lines.append(cleaned)
+        except Exception as exc:  # best-effort diagnostics only
+            with self._lock:
+                self._lines.append(f"[capture-error] {exc}")
+
+    def tail(self, lines: int = 80) -> str:
+        with self._lock:
+            chosen = self._lines[-lines:] if lines > 0 else self._lines
+            return "\n".join(chosen)
+
+    def stop(self, timeout: float = 1.0) -> None:
+        self._stop.set()
+        if self._thread is not None:
+            self._thread.join(timeout=timeout)
+
+
+def terminate_process(proc: subprocess.Popen, timeout: float = 5.0) -> None:
+    """Terminate a process, graceful (SIGTERM) then forced (SIGKILL)."""
+    if proc.poll() is not None:
+        return
+    proc.terminate()
+    try:
+        proc.wait(timeout=timeout)
+        return
+    except subprocess.TimeoutExpired:
+        pass
+    proc.kill()
+    try:
+        proc.wait(timeout=2.0)
+    except subprocess.TimeoutExpired:
+        pass