From 65385596a36322cb84ae73df3d7f16bc28160e72 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 6 Oct 2026 13:46:48 -0700 Subject: [PATCH 1/3] test(k3s): connect tmachine clients directly to ClusterIP Signed-off-by: Matthew Grossman --- .agents/skills/watch-github-actions/SKILL.md | 6 +- .github/workflows/integration-runner.yml | 9 + CI.md | 10 + skills/debug-openshell-cluster/SKILL.md | 8 +- tests/ansible/playbooks/conformance/cli.yaml | 5 + .../playbooks/openshell-k3s-ready.yaml | 9 + tests/ansible/playbooks/openshell-k3s.yaml | 307 +++++++++--------- .../files/collect.sh | 48 +++ .../openshell_k3s_diagnostics/tasks/main.yaml | 31 ++ .../roles/openshell_k3s_ready/tasks/main.yaml | 50 +++ tests/config.nix | 2 + tests/tmachine/src/config.rs | 28 ++ tests/tmachine/src/qemu/test.rs | 5 + 13 files changed, 359 insertions(+), 159 deletions(-) create mode 100644 tests/ansible/playbooks/openshell-k3s-ready.yaml create mode 100644 tests/ansible/roles/openshell_k3s_diagnostics/files/collect.sh create mode 100644 tests/ansible/roles/openshell_k3s_diagnostics/tasks/main.yaml create mode 100644 tests/ansible/roles/openshell_k3s_ready/tasks/main.yaml diff --git a/.agents/skills/watch-github-actions/SKILL.md b/.agents/skills/watch-github-actions/SKILL.md index fc42aaf947..1f5d5b91b5 100644 --- a/.agents/skills/watch-github-actions/SKILL.md +++ b/.agents/skills/watch-github-actions/SKILL.md @@ -245,7 +245,11 @@ gh run rerun ```bash gh run view --log-failed ``` -4. Rerun the failed jobs if needed: +4. For tmachine K3s failures, download `tmachine-diagnostics-*` artifacts and + preserve the first failure logs before rerunning. Inspect Service routing, + EndpointSlices and CLI status alongside the API, node and gateway logs. + Readiness polling within a job is different from rerunning a failed suite. +5. Rerun the failed jobs if needed: ```bash gh run rerun --failed ``` diff --git a/.github/workflows/integration-runner.yml b/.github/workflows/integration-runner.yml index 89cc0ca920..4be954eced 100644 --- a/.github/workflows/integration-runner.yml +++ b/.github/workflows/integration-runner.yml @@ -89,3 +89,12 @@ jobs: # Retry dependency preparation, then execute the test suite once. nix build --no-link .#tmachine || nix build --no-link .#tmachine nix run .#tmachine -- test "${ENVIRONMENT}" "${INSTALLER}" "${TESTSUITE}" + + - name: Upload tmachine diagnostics + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: tmachine-diagnostics-${{ inputs.category }}-${{ matrix.environment }}-${{ matrix.installer }}-${{ matrix.testsuite }} + path: artifacts/tmachine-diagnostics + if-no-files-found: ignore + retention-days: 7 diff --git a/CI.md b/CI.md index a433754151..b0dae21e15 100644 --- a/CI.md +++ b/CI.md @@ -105,6 +105,16 @@ compatibility baseline for the v1beta1 Sandbox API. It does not track the local K3s development default, currently v1.0.3. OpenShell also supports v0.4.6 through its v1alpha1 fallback, so v0.5.0 is not the overall minimum supported version. +The conformance CLI runs on the K3s node inside the tmachine guest and connects +directly to the gateway's default ClusterIP Service on port 8080. Installation +discovers the Service IP and registers it once; the installed disk preserves that +registration across test boots. Automated boots wait for the API, nodes, gateway +StatefulSet and a connected CLI before conformance. Interactive shell boots skip +this preparation so failed gateways remain debuggable. Recreating the Service +requires reprovisioning the fixture rather than repairing registrations at boot. +Installation, preparation and conformance failures preserve bounded diagnostics +under `artifacts/tmachine-diagnostics`, also uploaded by `Integration Tests`. + ### Run only the policy advisor conformance tests Manually dispatch `Integration Tests` on the candidate branch with an diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index bd30ceb8ef..343cd354f8 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -723,7 +723,13 @@ kubectl -n openshell get svc openshell -o wide kubectl -n openshell get endpoints openshell ``` -For local port-forward testing: +Clients running on a Kubernetes node can use the Service's ClusterIP and gRPC +port directly when node routing supports it. The IP stays stable across Pod +replacement and node reboots while the Service exists. It is not an external +endpoint; clients outside that network need another exposure method. Recreating +the Service can change its IP, so check the registered endpoint afterward. + +For local port-forward testing from outside the cluster network: ```bash kubectl -n openshell port-forward service/openshell 8080:8080 diff --git a/tests/ansible/playbooks/conformance/cli.yaml b/tests/ansible/playbooks/conformance/cli.yaml index dd8ce2b592..fee1eac41c 100644 --- a/tests/ansible/playbooks/conformance/cli.yaml +++ b/tests/ansible/playbooks/conformance/cli.yaml @@ -106,6 +106,11 @@ var: openshell_gateway_logs.stdout_lines when: conformance_result.rc != 0 + - name: Collect K3s conformance diagnostics + ansible.builtin.include_role: + name: openshell_k3s_diagnostics + when: conformance_result.rc != 0 + - name: Require OpenShell conformance success ansible.builtin.assert: that: diff --git a/tests/ansible/playbooks/openshell-k3s-ready.yaml b/tests/ansible/playbooks/openshell-k3s-ready.yaml new file mode 100644 index 0000000000..3a0c7cc61d --- /dev/null +++ b/tests/ansible/playbooks/openshell-k3s-ready.yaml @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +- name: Check OpenShell K3s readiness after boot + hosts: all + gather_facts: false + roles: + - openshell_k3s_ready diff --git a/tests/ansible/playbooks/openshell-k3s.yaml b/tests/ansible/playbooks/openshell-k3s.yaml index dd00aa4bda..b875644759 100644 --- a/tests/ansible/playbooks/openshell-k3s.yaml +++ b/tests/ansible/playbooks/openshell-k3s.yaml @@ -10,166 +10,159 @@ - name: Wait for SSH ansible.builtin.wait_for_connection: - - name: Install OpenShell CLI - ansible.builtin.copy: - src: "{{ openshell_cli_binary }}" - dest: /usr/local/bin/openshell - mode: "0755" - - - name: Create OpenShell artifact directory - ansible.builtin.file: - path: /var/lib/openshell/artifacts - state: directory - mode: "0700" - - - name: Copy OpenShell images - ansible.builtin.copy: - src: "{{ item.src }}" - dest: "/var/lib/openshell/artifacts/{{ item.name }}.tar" - mode: "0600" - loop: - - name: gateway - src: "{{ openshell_gateway_image }}" - - name: sandbox - src: "{{ openshell_sandbox_image }}" - - name: supervisor - src: "{{ openshell_supervisor_image }}" - - - name: Import OpenShell images into K3s - ansible.builtin.command: - argv: - - /usr/local/bin/k3s - - ctr - - --namespace - - k8s.io - - images - - import - - "/var/lib/openshell/artifacts/{{ item }}.tar" - loop: - - gateway - - sandbox - - supervisor - - - name: Copy OpenShell Helm chart - ansible.builtin.copy: - src: "{{ openshell_helm_chart }}" - dest: /var/lib/openshell/artifacts/helm-chart.tgz - mode: "0600" - - - name: Write OpenShell Helm values - ansible.builtin.copy: - dest: /var/lib/openshell/artifacts/values.yaml - mode: "0600" - content: | - global: - image: - registry: "" - gateway: - image: - repository: openshell/gateway - tag: tmachine - pullPolicy: Never - sandboxRuntime: - image: - repository: openshell/sandbox - tag: tmachine - pullPolicy: never - supervisor: - image: - repository: openshell/supervisor - tag: tmachine - pullPolicy: never - networkPolicy: - enabled: true - server: - auth: - allowUnauthenticatedUsers: true - disableTls: true - telemetryEnabled: false - - - name: Install Agent Sandbox - ansible.builtin.command: - argv: - - /usr/local/bin/k3s - - kubectl - - apply - - --filename - - "https://github.com/kubernetes-sigs/agent-sandbox/releases/download/v{{ agent_sandbox_version }}/manifest.yaml" - - - name: Wait for Agent Sandbox CRD - ansible.builtin.command: - argv: - - /usr/local/bin/k3s - - kubectl - - wait - - --for=condition=Established - - crd/sandboxes.agents.x-k8s.io - - --timeout=120s - changed_when: false - - - name: Wait for Agent Sandbox controller - ansible.builtin.command: - argv: - - /usr/local/bin/k3s - - kubectl - - --namespace - - agent-sandbox-system - - rollout - - status - - deployment/agent-sandbox-controller - - --timeout=300s - changed_when: false - - - name: Install OpenShell Helm chart - ansible.builtin.command: - argv: - - /usr/local/bin/helm - - install - - openshell - - /var/lib/openshell/artifacts/helm-chart.tgz - - --namespace - - openshell - - --create-namespace - - --values - - /var/lib/openshell/artifacts/values.yaml - - --wait - - --timeout=5m - environment: - KUBECONFIG: /etc/rancher/k3s/k3s.yaml - - - name: Install gateway port-forward service - ansible.builtin.copy: - dest: /etc/systemd/system/openshell-k3s-port-forward.service - mode: "0644" - content: | - [Unit] - Description=OpenShell K3s gateway port forward - After=k3s.service - Requires=k3s.service - - [Service] - ExecStart=/usr/local/bin/k3s kubectl --namespace openshell port-forward --address 127.0.0.1 service/openshell 17670:8080 - Restart=always - RestartSec=1 - - [Install] - WantedBy=multi-user.target - - - name: Start gateway port-forward service - ansible.builtin.systemd_service: - name: openshell-k3s-port-forward.service - daemon_reload: true - enabled: true - state: started - - - name: Wait for OpenShell gateway - ansible.builtin.wait_for: - host: 127.0.0.1 - port: 17670 - timeout: 60 + - name: Provision the K3s gateway + block: + - name: Install OpenShell CLI + ansible.builtin.copy: + src: "{{ openshell_cli_binary }}" + dest: /usr/local/bin/openshell + mode: "0755" + + - name: Create OpenShell artifact directory + ansible.builtin.file: + path: /var/lib/openshell/artifacts + state: directory + mode: "0700" + + - name: Copy OpenShell images + ansible.builtin.copy: + src: "{{ item.src }}" + dest: "/var/lib/openshell/artifacts/{{ item.name }}.tar" + mode: "0600" + loop: + - name: gateway + src: "{{ openshell_gateway_image }}" + - name: sandbox + src: "{{ openshell_sandbox_image }}" + - name: supervisor + src: "{{ openshell_supervisor_image }}" + + - name: Import OpenShell images into K3s + ansible.builtin.command: + argv: + - /usr/local/bin/k3s + - ctr + - --namespace + - k8s.io + - images + - import + - "/var/lib/openshell/artifacts/{{ item }}.tar" + loop: + - gateway + - sandbox + - supervisor + + - name: Copy OpenShell Helm chart + ansible.builtin.copy: + src: "{{ openshell_helm_chart }}" + dest: /var/lib/openshell/artifacts/helm-chart.tgz + mode: "0600" + + - name: Write OpenShell Helm values + ansible.builtin.copy: + dest: /var/lib/openshell/artifacts/values.yaml + mode: "0600" + content: | + global: + image: + registry: "" + gateway: + image: + repository: openshell/gateway + tag: tmachine + pullPolicy: Never + sandboxRuntime: + image: + repository: openshell/sandbox + tag: tmachine + pullPolicy: never + supervisor: + image: + repository: openshell/supervisor + tag: tmachine + pullPolicy: never + networkPolicy: + enabled: true + server: + auth: + allowUnauthenticatedUsers: true + disableTls: true + telemetryEnabled: false + + - name: Install Agent Sandbox + ansible.builtin.command: + argv: + - /usr/local/bin/k3s + - kubectl + - apply + - --filename + - "https://github.com/kubernetes-sigs/agent-sandbox/releases/download/v{{ agent_sandbox_version }}/manifest.yaml" + + - name: Wait for Agent Sandbox CRD + ansible.builtin.command: + argv: + - /usr/local/bin/k3s + - kubectl + - wait + - --for=condition=Established + - crd/sandboxes.agents.x-k8s.io + - --timeout=120s + changed_when: false + + - name: Wait for Agent Sandbox controller + ansible.builtin.command: + argv: + - /usr/local/bin/k3s + - kubectl + - --namespace + - agent-sandbox-system + - rollout + - status + - deployment/agent-sandbox-controller + - --timeout=300s + changed_when: false + + - name: Install OpenShell Helm chart + ansible.builtin.command: + argv: + - /usr/local/bin/helm + - install + - openshell + - /var/lib/openshell/artifacts/helm-chart.tgz + - --namespace + - openshell + - --create-namespace + - --values + - /var/lib/openshell/artifacts/values.yaml + - --wait + - --timeout=5m + environment: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + + - name: Discover the gateway ClusterIP + ansible.builtin.command: /usr/local/bin/k3s kubectl --request-timeout=10s --namespace openshell get service openshell --output=jsonpath={.spec.clusterIP} + register: k3s_gateway_ip + changed_when: false + + - name: Require a gateway Service address + ansible.builtin.assert: + that: + - k3s_gateway_ip.stdout | length > 0 + - k3s_gateway_ip.stdout != 'None' + rescue: + - name: Collect K3s installation diagnostics + ansible.builtin.include_role: + name: openshell_k3s_diagnostics + + - name: Fail K3s installation + ansible.builtin.fail: + msg: OpenShell K3s installation failed; see K3s diagnostics - name: Register OpenShell gateway for test client hosts: all gather_facts: false + vars: + openshell_client_gateway_endpoint: "http://{{ k3s_gateway_ip.stdout }}:8080" roles: - openshell_client + - openshell_k3s_ready diff --git a/tests/ansible/roles/openshell_k3s_diagnostics/files/collect.sh b/tests/ansible/roles/openshell_k3s_diagnostics/files/collect.sh new file mode 100644 index 0000000000..e6b2ce320f --- /dev/null +++ b/tests/ansible/roles/openshell_k3s_diagnostics/files/collect.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# No kubeconfig, Secrets, environment dumps, or complete Pod specifications. +set -u +umask 077 +mkdir -p /var/lib/openshell/diagnostics +output=/var/lib/openshell/diagnostics/k3s.txt + +collect() { + printf '\n### %s\n' "$1" + shift + # Bound both stalled commands and unusually large log streams. + timeout 10s "$@" 2>&1 | tail -c 262144 + printf '\ncommand status: %s\n' "${PIPESTATUS[0]}" +} + +{ + date --utc --iso-8601=seconds + collect 'K3s service state' systemctl show k3s.service \ + -p ActiveState -p SubState -p Result -p NRestarts -p ExecMainStatus \ + -p ExecMainPID -p ActiveEnterTimestamp -p ExecMainStartTimestamp \ + -p StartLimitIntervalUSec -p StartLimitBurst -p RestartUSec + collect 'K3s boot journal' journalctl -b --no-pager --lines=500 -u k3s.service + collect 'Gateway Service routing rules' bash -c \ + 'iptables-save | awk "/openshell\/openshell:grpc/ { print }"' + collect 'K3s version' /usr/local/bin/k3s --version + collect 'API readiness' /usr/local/bin/k3s kubectl --request-timeout=5s get --raw=/readyz + collect 'Nodes' /usr/local/bin/k3s kubectl --request-timeout=5s get nodes -o wide + collect 'Gateway Pod identity and state' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get pods \ + -o 'custom-columns=NAME:.metadata.name,UID:.metadata.uid,PHASE:.status.phase,CONDITIONS:.status.conditions,CONTAINERS:.status.containerStatuses' + collect 'Gateway Services' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get services \ + -o 'custom-columns=NAME:.metadata.name,TYPE:.spec.type,CLUSTERIP:.spec.clusterIP,SELECTOR:.spec.selector,PORTS:.spec.ports' + collect 'Test client gateway registrations' runuser -u tmachine -- openshell gateway list --output json + collect 'Test client gateway status' runuser -u tmachine -- openshell status --output json + collect 'Endpoint readiness' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get endpointslices \ + -o 'custom-columns=NAME:.metadata.name,PORTS:.ports,ADDRESSES:.endpoints[*].addresses,CONDITIONS:.endpoints[*].conditions' + collect 'Gateway events' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get events --sort-by=.lastTimestamp + collect 'Gateway current logs' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs openshell-0 -c openshell-gateway --tail=300 --timestamps + collect 'Gateway previous logs' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs openshell-0 -c openshell-gateway --previous --tail=300 --timestamps + collect 'Memory' free -m + collect 'Disk' df -h / /var/lib/rancher/k3s +} | sed -E \ + -e 's/(Bearer )[A-Za-z0-9._~+\/-]+/\1[REDACTED]/gI' \ + -e 's/((token|password|secret|authorization)[" ]*[=:][" ]*)[^ ,"]+/\1[REDACTED]/gI' \ + > "$output" +cat "$output" diff --git a/tests/ansible/roles/openshell_k3s_diagnostics/tasks/main.yaml b/tests/ansible/roles/openshell_k3s_diagnostics/tasks/main.yaml new file mode 100644 index 0000000000..e1ee0ba622 --- /dev/null +++ b/tests/ansible/roles/openshell_k3s_diagnostics/tasks/main.yaml @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +- name: Check for K3s + become: true + ansible.builtin.stat: + path: /usr/local/bin/k3s + register: k3s_diagnostics_binary + failed_when: false + +- name: Preserve K3s diagnostics before the VM exits + when: k3s_diagnostics_binary.stat.exists | default(false) + become: true + block: + - name: Collect bounded K3s diagnostics + ansible.builtin.script: collect.sh + register: k3s_diagnostics_collection + changed_when: false + failed_when: false + + - name: Show K3s diagnostics + ansible.builtin.debug: + var: k3s_diagnostics_collection.stdout_lines + + - name: Fetch K3s diagnostics + ansible.builtin.fetch: + src: /var/lib/openshell/diagnostics/k3s.txt + dest: "{{ lookup('env', 'TMACHINE_DIAGNOSTICS_DIR') | default(role_path + '/../../../../artifacts/tmachine-diagnostics', true) }}/{{ inventory_hostname }}/k3s.txt" + flat: true + failed_when: false diff --git a/tests/ansible/roles/openshell_k3s_ready/tasks/main.yaml b/tests/ansible/roles/openshell_k3s_ready/tasks/main.yaml new file mode 100644 index 0000000000..2145118bfe --- /dev/null +++ b/tests/ansible/roles/openshell_k3s_ready/tasks/main.yaml @@ -0,0 +1,50 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +- name: Wait for the installed K3s gateway + block: + - name: Wait for SSH + ansible.builtin.wait_for_connection: + timeout: 120 + + - name: Wait for the K3s API + become: true + ansible.builtin.command: /usr/local/bin/k3s kubectl --request-timeout=5s get --raw=/readyz + register: k3s_api_ready + changed_when: false + until: k3s_api_ready.rc == 0 + retries: 30 + delay: 2 + + - name: Wait for K3s nodes + become: true + ansible.builtin.command: /usr/local/bin/k3s kubectl --request-timeout=130s wait nodes --all --for=condition=Ready --timeout=120s + changed_when: false + + - name: Wait for the gateway StatefulSet + become: true + ansible.builtin.command: /usr/local/bin/k3s kubectl --request-timeout=190s --namespace openshell rollout status statefulset/openshell --timeout=180s + changed_when: false + + # Rollout readiness does not prove that Service routing has recovered. + - name: Wait for the registered gateway to answer the CLI + become: true + become_user: tmachine + ansible.builtin.command: timeout 10s openshell status --output json + register: k3s_gateway_ready + changed_when: false + until: >- + k3s_gateway_ready.rc == 0 and + (k3s_gateway_ready.stdout | from_json).status == 'connected' and + (k3s_gateway_ready.stdout | from_json).gateway == 'tmachine' + retries: 24 + delay: 5 + rescue: + - name: Collect K3s readiness diagnostics + ansible.builtin.include_role: + name: openshell_k3s_diagnostics + + - name: Fail K3s boot preparation + ansible.builtin.fail: + msg: OpenShell K3s gateway did not become ready before conformance; see K3s diagnostics diff --git a/tests/config.nix b/tests/config.nix index 26d27b712b..28bedc8f3f 100644 --- a/tests/config.nix +++ b/tests/config.nix @@ -85,6 +85,7 @@ let name = "k3s"; use_galaxy = false; playbooks = [ "ansible/playbooks/openshell-k3s.yaml" ]; + prepare_playbooks = [ "ansible/playbooks/openshell-k3s-ready.yaml" ]; inputs = { agent_sandbox_version = "0.5.0"; openshell_cli_binary = "../artifacts/binaries/${muslTarget}/openshell"; @@ -213,6 +214,7 @@ let root=$(git rev-parse --show-toplevel) cd "$root/tests" export ANSIBLE_CONFIG="$PWD/ansible/ansible.cfg" + export TMACHINE_DIAGNOSTICS_DIR="$root/artifacts/tmachine-diagnostics" exec ${tmachine}/bin/tmachine --config ${config} "$@" ''; }; diff --git a/tests/tmachine/src/config.rs b/tests/tmachine/src/config.rs index bc19cbbb1f..44a614538b 100644 --- a/tests/tmachine/src/config.rs +++ b/tests/tmachine/src/config.rs @@ -40,6 +40,34 @@ pub struct Installer { pub use_galaxy: bool, pub playbooks: Vec, pub inputs: BTreeMap, + /// Readiness checks run on non-interactive test boots, including cached installations. + #[serde(default)] + pub prepare_playbooks: Vec, +} + +#[cfg(test)] +mod tests { + use super::Installer; + + #[test] + fn existing_installers_need_no_boot_preparation() { + let installer: Installer = + serde_saphyr::from_str("name: none\nuse_galaxy: false\nplaybooks: []\ninputs: {}\n") + .unwrap(); + assert!(installer.prepare_playbooks.is_empty()); + } + + #[test] + fn installer_can_request_boot_preparation() { + let installer: Installer = serde_saphyr::from_str( + "name: k3s\nuse_galaxy: false\nplaybooks: []\ninputs: {}\nprepare_playbooks: [ready.yaml]\n", + ) + .unwrap(); + assert_eq!( + installer.prepare_playbooks, + [std::path::PathBuf::from("ready.yaml")] + ); + } } #[derive(Clone, Deserialize)] diff --git a/tests/tmachine/src/qemu/test.rs b/tests/tmachine/src/qemu/test.rs index ac8a8bdfc0..0dd8f9888c 100644 --- a/tests/tmachine/src/qemu/test.rs +++ b/tests/tmachine/src/qemu/test.rs @@ -26,6 +26,11 @@ pub async fn test( let image = QemuImage::create(&install_disk, test_disk).await; let vm = QemuVm::start(&image).await; + // Installation readiness does not survive the shutdown of the cached VM. + // Keep the interactive shell accessible when diagnosing boot failures. + if !testsuite.interactive { + run_playbooks(&installer.prepare_playbooks, &installer.inputs).await?; + } run_playbooks(&testsuite.playbooks, &testsuite.inputs).await?; if testsuite.interactive { From b14566a640b02e41f8979fd00f4f7375354e032d Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Fri, 9 Oct 2026 10:02:29 -0700 Subject: [PATCH 2/3] test(conformance): wait for a connected gateway on every installer Replace the K3s-specific boot readiness hook with one shared gateway wait in the conformance playbook. Skip conformance and collect the existing diagnostics when the gateway never connects. Discover the gateway Service port with its ClusterIP, and capture both iptables backends in K3s diagnostics. Signed-off-by: Matthew Grossman --- CI.md | 19 ++++--- tests/ansible/playbooks/conformance/cli.yaml | 29 +++++++++-- .../playbooks/openshell-k3s-ready.yaml | 9 ---- tests/ansible/playbooks/openshell-k3s.yaml | 24 ++++++--- .../files/collect.sh | 10 +++- .../openshell_k3s_diagnostics/tasks/main.yaml | 2 +- .../roles/openshell_k3s_ready/tasks/main.yaml | 50 ------------------- tests/config.nix | 2 - tests/tmachine/src/config.rs | 28 ----------- tests/tmachine/src/qemu/test.rs | 5 -- 10 files changed, 62 insertions(+), 116 deletions(-) delete mode 100644 tests/ansible/playbooks/openshell-k3s-ready.yaml delete mode 100644 tests/ansible/roles/openshell_k3s_ready/tasks/main.yaml diff --git a/CI.md b/CI.md index b0dae21e15..230db049ef 100644 --- a/CI.md +++ b/CI.md @@ -106,14 +106,17 @@ K3s development default, currently v1.0.3. OpenShell also supports v0.4.6 throug its v1alpha1 fallback, so v0.5.0 is not the overall minimum supported version. The conformance CLI runs on the K3s node inside the tmachine guest and connects -directly to the gateway's default ClusterIP Service on port 8080. Installation -discovers the Service IP and registers it once; the installed disk preserves that -registration across test boots. Automated boots wait for the API, nodes, gateway -StatefulSet and a connected CLI before conformance. Interactive shell boots skip -this preparation so failed gateways remain debuggable. Recreating the Service -requires reprovisioning the fixture rather than repairing registrations at boot. -Installation, preparation and conformance failures preserve bounded diagnostics -under `artifacts/tmachine-diagnostics`, also uploaded by `Integration Tests`. +directly to the gateway's default ClusterIP Service on its gRPC port. Installation +discovers the Service address and registers it once; the installed disk preserves that +registration across test boots. Recreating the Service requires reprovisioning +the fixture rather than repairing registrations at boot. K3s installation and +conformance failures preserve bounded diagnostics under +`artifacts/tmachine-diagnostics`, also uploaded by `Integration Tests`. + +On every installer, the conformance playbook waits up to five minutes for +`openshell status` to report a connected gateway before running tests, because +each test boot restarts the gateway. Interactive shell boots skip this wait so +failed gateways remain debuggable. ### Run only the policy advisor conformance tests diff --git a/tests/ansible/playbooks/conformance/cli.yaml b/tests/ansible/playbooks/conformance/cli.yaml index fee1eac41c..52d7a81430 100644 --- a/tests/ansible/playbooks/conformance/cli.yaml +++ b/tests/ansible/playbooks/conformance/cli.yaml @@ -55,6 +55,20 @@ register: openshell_cli changed_when: false + # Every test boot restarts the gateway, and some installers take minutes to + # recover. Wait once here; the runner's per-scenario preflight stays short. + - name: Wait for the registered gateway + ansible.builtin.command: + argv: [timeout, 10s, "{{ openshell_cli.stdout }}", status, --output, json] + register: gateway_status + changed_when: false + until: >- + gateway_status.rc == 0 and + (gateway_status.stdout | from_json).status == 'connected' + retries: 60 + delay: 5 + ignore_errors: true + - name: Run OpenShell conformance archive ansible.builtin.command: argv: @@ -77,13 +91,14 @@ register: conformance_result changed_when: false failed_when: false + when: gateway_status is succeeded # Preserve both streams for failures without adding passing-test output to # every tmachine run. - name: Show OpenShell conformance diagnostics ansible.builtin.debug: var: conformance_result - when: conformance_result.rc != 0 + when: conformance_result.rc | default(0) != 0 - name: Read OpenShell gateway logs become: true @@ -99,17 +114,23 @@ register: openshell_gateway_logs changed_when: false failed_when: false - when: conformance_result.rc != 0 + when: conformance_result.rc | default(1) != 0 - name: Show OpenShell gateway logs ansible.builtin.debug: var: openshell_gateway_logs.stdout_lines - when: conformance_result.rc != 0 + when: conformance_result.rc | default(1) != 0 - name: Collect K3s conformance diagnostics ansible.builtin.include_role: name: openshell_k3s_diagnostics - when: conformance_result.rc != 0 + when: conformance_result.rc | default(1) != 0 + + - name: Require a connected gateway + ansible.builtin.assert: + that: + - gateway_status is succeeded + fail_msg: OpenShell gateway did not connect before conformance; see gateway diagnostics - name: Require OpenShell conformance success ansible.builtin.assert: diff --git a/tests/ansible/playbooks/openshell-k3s-ready.yaml b/tests/ansible/playbooks/openshell-k3s-ready.yaml deleted file mode 100644 index 3a0c7cc61d..0000000000 --- a/tests/ansible/playbooks/openshell-k3s-ready.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - ---- -- name: Check OpenShell K3s readiness after boot - hosts: all - gather_facts: false - roles: - - openshell_k3s_ready diff --git a/tests/ansible/playbooks/openshell-k3s.yaml b/tests/ansible/playbooks/openshell-k3s.yaml index b875644759..0d02e6a745 100644 --- a/tests/ansible/playbooks/openshell-k3s.yaml +++ b/tests/ansible/playbooks/openshell-k3s.yaml @@ -139,16 +139,27 @@ environment: KUBECONFIG: /etc/rancher/k3s/k3s.yaml - - name: Discover the gateway ClusterIP - ansible.builtin.command: /usr/local/bin/k3s kubectl --request-timeout=10s --namespace openshell get service openshell --output=jsonpath={.spec.clusterIP} - register: k3s_gateway_ip + - name: Discover the gateway ClusterIP and gRPC port + ansible.builtin.command: + argv: + - /usr/local/bin/k3s + - kubectl + - --request-timeout=10s + - --namespace + - openshell + - get + - service + - openshell + - --output=jsonpath={.spec.clusterIP}:{.spec.ports[?(@.name=="grpc")].port} + register: k3s_gateway_address changed_when: false + # Rejects headless Services (ClusterIP "None") and a missing grpc port. - name: Require a gateway Service address ansible.builtin.assert: that: - - k3s_gateway_ip.stdout | length > 0 - - k3s_gateway_ip.stdout != 'None' + - k3s_gateway_address.stdout is match('^[0-9.]+:[0-9]+$') + fail_msg: "Unexpected gateway Service address: {{ k3s_gateway_address.stdout }}" rescue: - name: Collect K3s installation diagnostics ansible.builtin.include_role: @@ -162,7 +173,6 @@ hosts: all gather_facts: false vars: - openshell_client_gateway_endpoint: "http://{{ k3s_gateway_ip.stdout }}:8080" + openshell_client_gateway_endpoint: "http://{{ k3s_gateway_address.stdout }}" roles: - openshell_client - - openshell_k3s_ready diff --git a/tests/ansible/roles/openshell_k3s_diagnostics/files/collect.sh b/tests/ansible/roles/openshell_k3s_diagnostics/files/collect.sh index e6b2ce320f..c20334bb13 100644 --- a/tests/ansible/roles/openshell_k3s_diagnostics/files/collect.sh +++ b/tests/ansible/roles/openshell_k3s_diagnostics/files/collect.sh @@ -23,8 +23,14 @@ collect() { -p ExecMainPID -p ActiveEnterTimestamp -p ExecMainStartTimestamp \ -p StartLimitIntervalUSec -p StartLimitBurst -p RestartUSec collect 'K3s boot journal' journalctl -b --no-pager --lines=500 -u k3s.service - collect 'Gateway Service routing rules' bash -c \ - 'iptables-save | awk "/openshell\/openshell:grpc/ { print }"' + # kube-proxy may program either iptables backend; dump whichever exist. + # shellcheck disable=SC2016 # Expanded by the inner shell. + collect 'Gateway Service routing rules' bash -c ' + for save in iptables-nft-save iptables-legacy-save; do + command -v "$save" >/dev/null || continue + echo "# $save" + "$save" 2>/dev/null | awk "/openshell\/openshell:grpc/ { print }" + done' collect 'K3s version' /usr/local/bin/k3s --version collect 'API readiness' /usr/local/bin/k3s kubectl --request-timeout=5s get --raw=/readyz collect 'Nodes' /usr/local/bin/k3s kubectl --request-timeout=5s get nodes -o wide diff --git a/tests/ansible/roles/openshell_k3s_diagnostics/tasks/main.yaml b/tests/ansible/roles/openshell_k3s_diagnostics/tasks/main.yaml index e1ee0ba622..6a4891b32d 100644 --- a/tests/ansible/roles/openshell_k3s_diagnostics/tasks/main.yaml +++ b/tests/ansible/roles/openshell_k3s_diagnostics/tasks/main.yaml @@ -26,6 +26,6 @@ - name: Fetch K3s diagnostics ansible.builtin.fetch: src: /var/lib/openshell/diagnostics/k3s.txt - dest: "{{ lookup('env', 'TMACHINE_DIAGNOSTICS_DIR') | default(role_path + '/../../../../artifacts/tmachine-diagnostics', true) }}/{{ inventory_hostname }}/k3s.txt" + dest: "{{ role_path }}/../../../../artifacts/tmachine-diagnostics/{{ inventory_hostname }}/k3s.txt" flat: true failed_when: false diff --git a/tests/ansible/roles/openshell_k3s_ready/tasks/main.yaml b/tests/ansible/roles/openshell_k3s_ready/tasks/main.yaml deleted file mode 100644 index 2145118bfe..0000000000 --- a/tests/ansible/roles/openshell_k3s_ready/tasks/main.yaml +++ /dev/null @@ -1,50 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - ---- -- name: Wait for the installed K3s gateway - block: - - name: Wait for SSH - ansible.builtin.wait_for_connection: - timeout: 120 - - - name: Wait for the K3s API - become: true - ansible.builtin.command: /usr/local/bin/k3s kubectl --request-timeout=5s get --raw=/readyz - register: k3s_api_ready - changed_when: false - until: k3s_api_ready.rc == 0 - retries: 30 - delay: 2 - - - name: Wait for K3s nodes - become: true - ansible.builtin.command: /usr/local/bin/k3s kubectl --request-timeout=130s wait nodes --all --for=condition=Ready --timeout=120s - changed_when: false - - - name: Wait for the gateway StatefulSet - become: true - ansible.builtin.command: /usr/local/bin/k3s kubectl --request-timeout=190s --namespace openshell rollout status statefulset/openshell --timeout=180s - changed_when: false - - # Rollout readiness does not prove that Service routing has recovered. - - name: Wait for the registered gateway to answer the CLI - become: true - become_user: tmachine - ansible.builtin.command: timeout 10s openshell status --output json - register: k3s_gateway_ready - changed_when: false - until: >- - k3s_gateway_ready.rc == 0 and - (k3s_gateway_ready.stdout | from_json).status == 'connected' and - (k3s_gateway_ready.stdout | from_json).gateway == 'tmachine' - retries: 24 - delay: 5 - rescue: - - name: Collect K3s readiness diagnostics - ansible.builtin.include_role: - name: openshell_k3s_diagnostics - - - name: Fail K3s boot preparation - ansible.builtin.fail: - msg: OpenShell K3s gateway did not become ready before conformance; see K3s diagnostics diff --git a/tests/config.nix b/tests/config.nix index 28bedc8f3f..26d27b712b 100644 --- a/tests/config.nix +++ b/tests/config.nix @@ -85,7 +85,6 @@ let name = "k3s"; use_galaxy = false; playbooks = [ "ansible/playbooks/openshell-k3s.yaml" ]; - prepare_playbooks = [ "ansible/playbooks/openshell-k3s-ready.yaml" ]; inputs = { agent_sandbox_version = "0.5.0"; openshell_cli_binary = "../artifacts/binaries/${muslTarget}/openshell"; @@ -214,7 +213,6 @@ let root=$(git rev-parse --show-toplevel) cd "$root/tests" export ANSIBLE_CONFIG="$PWD/ansible/ansible.cfg" - export TMACHINE_DIAGNOSTICS_DIR="$root/artifacts/tmachine-diagnostics" exec ${tmachine}/bin/tmachine --config ${config} "$@" ''; }; diff --git a/tests/tmachine/src/config.rs b/tests/tmachine/src/config.rs index 44a614538b..bc19cbbb1f 100644 --- a/tests/tmachine/src/config.rs +++ b/tests/tmachine/src/config.rs @@ -40,34 +40,6 @@ pub struct Installer { pub use_galaxy: bool, pub playbooks: Vec, pub inputs: BTreeMap, - /// Readiness checks run on non-interactive test boots, including cached installations. - #[serde(default)] - pub prepare_playbooks: Vec, -} - -#[cfg(test)] -mod tests { - use super::Installer; - - #[test] - fn existing_installers_need_no_boot_preparation() { - let installer: Installer = - serde_saphyr::from_str("name: none\nuse_galaxy: false\nplaybooks: []\ninputs: {}\n") - .unwrap(); - assert!(installer.prepare_playbooks.is_empty()); - } - - #[test] - fn installer_can_request_boot_preparation() { - let installer: Installer = serde_saphyr::from_str( - "name: k3s\nuse_galaxy: false\nplaybooks: []\ninputs: {}\nprepare_playbooks: [ready.yaml]\n", - ) - .unwrap(); - assert_eq!( - installer.prepare_playbooks, - [std::path::PathBuf::from("ready.yaml")] - ); - } } #[derive(Clone, Deserialize)] diff --git a/tests/tmachine/src/qemu/test.rs b/tests/tmachine/src/qemu/test.rs index 0dd8f9888c..ac8a8bdfc0 100644 --- a/tests/tmachine/src/qemu/test.rs +++ b/tests/tmachine/src/qemu/test.rs @@ -26,11 +26,6 @@ pub async fn test( let image = QemuImage::create(&install_disk, test_disk).await; let vm = QemuVm::start(&image).await; - // Installation readiness does not survive the shutdown of the cached VM. - // Keep the interactive shell accessible when diagnosing boot failures. - if !testsuite.interactive { - run_playbooks(&installer.prepare_playbooks, &installer.inputs).await?; - } run_playbooks(&testsuite.playbooks, &testsuite.inputs).await?; if testsuite.interactive { From 1767ec0eeb2a6d47502fd28ca06b19895a84311a Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Fri, 9 Oct 2026 10:47:02 -0700 Subject: [PATCH 3/3] test(tmachine): centralize gateway diagnostics and trim K3s changes Move failure diagnostics into one openshell_diagnostics role that collects the systemd gateway journal and, when K3s is installed, the K3s diagnostics. Drop install-time diagnostics so the K3s installer change stays minimal, revert unrelated skill edits, and shorten the CI.md note. Signed-off-by: Matthew Grossman --- .agents/skills/watch-github-actions/SKILL.md | 6 +- CI.md | 17 +- skills/debug-openshell-cluster/SKILL.md | 8 +- tests/ansible/playbooks/conformance/cli.yaml | 32 +- tests/ansible/playbooks/openshell-k3s.yaml | 284 +++++++++--------- .../files/collect-k3s.sh} | 0 .../tasks/main.yaml | 24 +- 7 files changed, 173 insertions(+), 198 deletions(-) rename tests/ansible/roles/{openshell_k3s_diagnostics/files/collect.sh => openshell_diagnostics/files/collect-k3s.sh} (100%) rename tests/ansible/roles/{openshell_k3s_diagnostics => openshell_diagnostics}/tasks/main.yaml (59%) diff --git a/.agents/skills/watch-github-actions/SKILL.md b/.agents/skills/watch-github-actions/SKILL.md index 1f5d5b91b5..fc42aaf947 100644 --- a/.agents/skills/watch-github-actions/SKILL.md +++ b/.agents/skills/watch-github-actions/SKILL.md @@ -245,11 +245,7 @@ gh run rerun ```bash gh run view --log-failed ``` -4. For tmachine K3s failures, download `tmachine-diagnostics-*` artifacts and - preserve the first failure logs before rerunning. Inspect Service routing, - EndpointSlices and CLI status alongside the API, node and gateway logs. - Readiness polling within a job is different from rerunning a failed suite. -5. Rerun the failed jobs if needed: +4. Rerun the failed jobs if needed: ```bash gh run rerun --failed ``` diff --git a/CI.md b/CI.md index 230db049ef..62d72f78a6 100644 --- a/CI.md +++ b/CI.md @@ -105,18 +105,11 @@ compatibility baseline for the v1beta1 Sandbox API. It does not track the local K3s development default, currently v1.0.3. OpenShell also supports v0.4.6 through its v1alpha1 fallback, so v0.5.0 is not the overall minimum supported version. -The conformance CLI runs on the K3s node inside the tmachine guest and connects -directly to the gateway's default ClusterIP Service on its gRPC port. Installation -discovers the Service address and registers it once; the installed disk preserves that -registration across test boots. Recreating the Service requires reprovisioning -the fixture rather than repairing registrations at boot. K3s installation and -conformance failures preserve bounded diagnostics under -`artifacts/tmachine-diagnostics`, also uploaded by `Integration Tests`. - -On every installer, the conformance playbook waits up to five minutes for -`openshell status` to report a connected gateway before running tests, because -each test boot restarts the gateway. Interactive shell boots skip this wait so -failed gateways remain debuggable. +The `ubuntu-k3s` lane registers the gateway's ClusterIP Service directly, so +recreating the Service requires reprovisioning the fixture. Before conformance, +every installer waits up to five minutes for a connected gateway; interactive +shells skip the wait. K3s failures upload diagnostics as `tmachine-diagnostics-*` +artifacts. ### Run only the policy advisor conformance tests diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 343cd354f8..bd30ceb8ef 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -723,13 +723,7 @@ kubectl -n openshell get svc openshell -o wide kubectl -n openshell get endpoints openshell ``` -Clients running on a Kubernetes node can use the Service's ClusterIP and gRPC -port directly when node routing supports it. The IP stays stable across Pod -replacement and node reboots while the Service exists. It is not an external -endpoint; clients outside that network need another exposure method. Recreating -the Service can change its IP, so check the registered endpoint afterward. - -For local port-forward testing from outside the cluster network: +For local port-forward testing: ```bash kubectl -n openshell port-forward service/openshell 8080:8080 diff --git a/tests/ansible/playbooks/conformance/cli.yaml b/tests/ansible/playbooks/conformance/cli.yaml index 52d7a81430..11eb07e36d 100644 --- a/tests/ansible/playbooks/conformance/cli.yaml +++ b/tests/ansible/playbooks/conformance/cli.yaml @@ -5,6 +5,9 @@ - name: Run OpenShell conformance tests hosts: all gather_facts: false + vars: + # Conformance is skipped when the gateway never connects. + conformance_failed: "{{ gateway_status is failed or conformance_result.rc != 0 }}" tasks: - name: Wait for SSH ansible.builtin.wait_for_connection: @@ -98,33 +101,12 @@ - name: Show OpenShell conformance diagnostics ansible.builtin.debug: var: conformance_result - when: conformance_result.rc | default(0) != 0 + when: conformance_failed - - name: Read OpenShell gateway logs - become: true - ansible.builtin.command: - argv: - - journalctl - - --no-pager - - --lines - - "500" - - _SYSTEMD_UNIT=openshell-gateway.service - - "+" - - _SYSTEMD_USER_UNIT=openshell-gateway.service - register: openshell_gateway_logs - changed_when: false - failed_when: false - when: conformance_result.rc | default(1) != 0 - - - name: Show OpenShell gateway logs - ansible.builtin.debug: - var: openshell_gateway_logs.stdout_lines - when: conformance_result.rc | default(1) != 0 - - - name: Collect K3s conformance diagnostics + - name: Collect OpenShell gateway diagnostics ansible.builtin.include_role: - name: openshell_k3s_diagnostics - when: conformance_result.rc | default(1) != 0 + name: openshell_diagnostics + when: conformance_failed - name: Require a connected gateway ansible.builtin.assert: diff --git a/tests/ansible/playbooks/openshell-k3s.yaml b/tests/ansible/playbooks/openshell-k3s.yaml index 0d02e6a745..591262a933 100644 --- a/tests/ansible/playbooks/openshell-k3s.yaml +++ b/tests/ansible/playbooks/openshell-k3s.yaml @@ -10,164 +10,154 @@ - name: Wait for SSH ansible.builtin.wait_for_connection: - - name: Provision the K3s gateway - block: - - name: Install OpenShell CLI - ansible.builtin.copy: - src: "{{ openshell_cli_binary }}" - dest: /usr/local/bin/openshell - mode: "0755" + - name: Install OpenShell CLI + ansible.builtin.copy: + src: "{{ openshell_cli_binary }}" + dest: /usr/local/bin/openshell + mode: "0755" - - name: Create OpenShell artifact directory - ansible.builtin.file: - path: /var/lib/openshell/artifacts - state: directory - mode: "0700" + - name: Create OpenShell artifact directory + ansible.builtin.file: + path: /var/lib/openshell/artifacts + state: directory + mode: "0700" - - name: Copy OpenShell images - ansible.builtin.copy: - src: "{{ item.src }}" - dest: "/var/lib/openshell/artifacts/{{ item.name }}.tar" - mode: "0600" - loop: - - name: gateway - src: "{{ openshell_gateway_image }}" - - name: sandbox - src: "{{ openshell_sandbox_image }}" - - name: supervisor - src: "{{ openshell_supervisor_image }}" + - name: Copy OpenShell images + ansible.builtin.copy: + src: "{{ item.src }}" + dest: "/var/lib/openshell/artifacts/{{ item.name }}.tar" + mode: "0600" + loop: + - name: gateway + src: "{{ openshell_gateway_image }}" + - name: sandbox + src: "{{ openshell_sandbox_image }}" + - name: supervisor + src: "{{ openshell_supervisor_image }}" - - name: Import OpenShell images into K3s - ansible.builtin.command: - argv: - - /usr/local/bin/k3s - - ctr - - --namespace - - k8s.io - - images - - import - - "/var/lib/openshell/artifacts/{{ item }}.tar" - loop: - - gateway - - sandbox - - supervisor + - name: Import OpenShell images into K3s + ansible.builtin.command: + argv: + - /usr/local/bin/k3s + - ctr + - --namespace + - k8s.io + - images + - import + - "/var/lib/openshell/artifacts/{{ item }}.tar" + loop: + - gateway + - sandbox + - supervisor - - name: Copy OpenShell Helm chart - ansible.builtin.copy: - src: "{{ openshell_helm_chart }}" - dest: /var/lib/openshell/artifacts/helm-chart.tgz - mode: "0600" + - name: Copy OpenShell Helm chart + ansible.builtin.copy: + src: "{{ openshell_helm_chart }}" + dest: /var/lib/openshell/artifacts/helm-chart.tgz + mode: "0600" - - name: Write OpenShell Helm values - ansible.builtin.copy: - dest: /var/lib/openshell/artifacts/values.yaml - mode: "0600" - content: | - global: - image: - registry: "" - gateway: - image: - repository: openshell/gateway - tag: tmachine - pullPolicy: Never - sandboxRuntime: - image: - repository: openshell/sandbox - tag: tmachine - pullPolicy: never - supervisor: - image: - repository: openshell/supervisor - tag: tmachine - pullPolicy: never - networkPolicy: - enabled: true - server: - auth: - allowUnauthenticatedUsers: true - disableTls: true - telemetryEnabled: false + - name: Write OpenShell Helm values + ansible.builtin.copy: + dest: /var/lib/openshell/artifacts/values.yaml + mode: "0600" + content: | + global: + image: + registry: "" + gateway: + image: + repository: openshell/gateway + tag: tmachine + pullPolicy: Never + sandboxRuntime: + image: + repository: openshell/sandbox + tag: tmachine + pullPolicy: never + supervisor: + image: + repository: openshell/supervisor + tag: tmachine + pullPolicy: never + networkPolicy: + enabled: true + server: + auth: + allowUnauthenticatedUsers: true + disableTls: true + telemetryEnabled: false - - name: Install Agent Sandbox - ansible.builtin.command: - argv: - - /usr/local/bin/k3s - - kubectl - - apply - - --filename - - "https://github.com/kubernetes-sigs/agent-sandbox/releases/download/v{{ agent_sandbox_version }}/manifest.yaml" + - name: Install Agent Sandbox + ansible.builtin.command: + argv: + - /usr/local/bin/k3s + - kubectl + - apply + - --filename + - "https://github.com/kubernetes-sigs/agent-sandbox/releases/download/v{{ agent_sandbox_version }}/manifest.yaml" - - name: Wait for Agent Sandbox CRD - ansible.builtin.command: - argv: - - /usr/local/bin/k3s - - kubectl - - wait - - --for=condition=Established - - crd/sandboxes.agents.x-k8s.io - - --timeout=120s - changed_when: false + - name: Wait for Agent Sandbox CRD + ansible.builtin.command: + argv: + - /usr/local/bin/k3s + - kubectl + - wait + - --for=condition=Established + - crd/sandboxes.agents.x-k8s.io + - --timeout=120s + changed_when: false - - name: Wait for Agent Sandbox controller - ansible.builtin.command: - argv: - - /usr/local/bin/k3s - - kubectl - - --namespace - - agent-sandbox-system - - rollout - - status - - deployment/agent-sandbox-controller - - --timeout=300s - changed_when: false + - name: Wait for Agent Sandbox controller + ansible.builtin.command: + argv: + - /usr/local/bin/k3s + - kubectl + - --namespace + - agent-sandbox-system + - rollout + - status + - deployment/agent-sandbox-controller + - --timeout=300s + changed_when: false - - name: Install OpenShell Helm chart - ansible.builtin.command: - argv: - - /usr/local/bin/helm - - install - - openshell - - /var/lib/openshell/artifacts/helm-chart.tgz - - --namespace - - openshell - - --create-namespace - - --values - - /var/lib/openshell/artifacts/values.yaml - - --wait - - --timeout=5m - environment: - KUBECONFIG: /etc/rancher/k3s/k3s.yaml + - name: Install OpenShell Helm chart + ansible.builtin.command: + argv: + - /usr/local/bin/helm + - install + - openshell + - /var/lib/openshell/artifacts/helm-chart.tgz + - --namespace + - openshell + - --create-namespace + - --values + - /var/lib/openshell/artifacts/values.yaml + - --wait + - --timeout=5m + environment: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml - - name: Discover the gateway ClusterIP and gRPC port - ansible.builtin.command: - argv: - - /usr/local/bin/k3s - - kubectl - - --request-timeout=10s - - --namespace - - openshell - - get - - service - - openshell - - --output=jsonpath={.spec.clusterIP}:{.spec.ports[?(@.name=="grpc")].port} - register: k3s_gateway_address - changed_when: false + - name: Discover the gateway ClusterIP and gRPC port + ansible.builtin.command: + argv: + - /usr/local/bin/k3s + - kubectl + - --request-timeout=10s + - --namespace + - openshell + - get + - service + - openshell + - --output=jsonpath={.spec.clusterIP}:{.spec.ports[?(@.name=="grpc")].port} + register: k3s_gateway_address + changed_when: false - # Rejects headless Services (ClusterIP "None") and a missing grpc port. - - name: Require a gateway Service address - ansible.builtin.assert: - that: - - k3s_gateway_address.stdout is match('^[0-9.]+:[0-9]+$') - fail_msg: "Unexpected gateway Service address: {{ k3s_gateway_address.stdout }}" - rescue: - - name: Collect K3s installation diagnostics - ansible.builtin.include_role: - name: openshell_k3s_diagnostics - - - name: Fail K3s installation - ansible.builtin.fail: - msg: OpenShell K3s installation failed; see K3s diagnostics + # Rejects headless Services (ClusterIP "None") and a missing grpc port. + - name: Require a gateway Service address + ansible.builtin.assert: + that: + - k3s_gateway_address.stdout is match('^[0-9.]+:[0-9]+$') + fail_msg: "Unexpected gateway Service address: {{ k3s_gateway_address.stdout }}" - name: Register OpenShell gateway for test client hosts: all diff --git a/tests/ansible/roles/openshell_k3s_diagnostics/files/collect.sh b/tests/ansible/roles/openshell_diagnostics/files/collect-k3s.sh similarity index 100% rename from tests/ansible/roles/openshell_k3s_diagnostics/files/collect.sh rename to tests/ansible/roles/openshell_diagnostics/files/collect-k3s.sh diff --git a/tests/ansible/roles/openshell_k3s_diagnostics/tasks/main.yaml b/tests/ansible/roles/openshell_diagnostics/tasks/main.yaml similarity index 59% rename from tests/ansible/roles/openshell_k3s_diagnostics/tasks/main.yaml rename to tests/ansible/roles/openshell_diagnostics/tasks/main.yaml index 6a4891b32d..74a204ab96 100644 --- a/tests/ansible/roles/openshell_k3s_diagnostics/tasks/main.yaml +++ b/tests/ansible/roles/openshell_diagnostics/tasks/main.yaml @@ -2,6 +2,26 @@ # SPDX-License-Identifier: Apache-2.0 --- +# Collect whatever applies to the installed gateway before the VM exits. +- name: Read OpenShell gateway logs + become: true + ansible.builtin.command: + argv: + - journalctl + - --no-pager + - --lines + - "500" + - _SYSTEMD_UNIT=openshell-gateway.service + - "+" + - _SYSTEMD_USER_UNIT=openshell-gateway.service + register: openshell_gateway_logs + changed_when: false + failed_when: false + +- name: Show OpenShell gateway logs + ansible.builtin.debug: + var: openshell_gateway_logs.stdout_lines + - name: Check for K3s become: true ansible.builtin.stat: @@ -9,12 +29,12 @@ register: k3s_diagnostics_binary failed_when: false -- name: Preserve K3s diagnostics before the VM exits +- name: Preserve K3s diagnostics when: k3s_diagnostics_binary.stat.exists | default(false) become: true block: - name: Collect bounded K3s diagnostics - ansible.builtin.script: collect.sh + ansible.builtin.script: collect-k3s.sh register: k3s_diagnostics_collection changed_when: false failed_when: false