test(tmachine): use K3s ClusterIP, wait for gateway, add failure diagnostics

.github/workflows/integration-runner.yml

@@ -89,3 +89,12 @@ jobs:
           # Retry dependency preparation, then execute the test suite once.
           nix build --no-link .#tmachine || nix build --no-link .#tmachine
           nix run .#tmachine -- test "${ENVIRONMENT}" "${INSTALLER}" "${TESTSUITE}"
+
+      - name: Upload tmachine diagnostics
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: tmachine-diagnostics-${{ inputs.category }}-${{ matrix.environment }}-${{ matrix.installer }}-${{ matrix.testsuite }}
+          path: artifacts/tmachine-diagnostics
+          if-no-files-found: ignore
+          retention-days: 7

CI.md

@@ -105,6 +105,12 @@ compatibility baseline for the v1beta1 Sandbox API. It does not track the local
 K3s development default, currently v1.0.3. OpenShell also supports v0.4.6 through
 its v1alpha1 fallback, so v0.5.0 is not the overall minimum supported version.
 
+The `ubuntu-k3s` lane registers the gateway's ClusterIP Service directly, so
+recreating the Service requires reprovisioning the fixture. Before conformance,
+every installer waits up to five minutes for a connected gateway; interactive
+shells skip the wait. K3s failures upload diagnostics as `tmachine-diagnostics-*`
+artifacts.
+
 ### Run only the policy advisor conformance tests
 
 Manually dispatch `Integration Tests` on the candidate branch with an

tests/ansible/playbooks/conformance/cli.yaml

@@ -5,6 +5,9 @@
 - name: Run OpenShell conformance tests
   hosts: all
   gather_facts: false
+  vars:
+    # Conformance is skipped when the gateway never connects.
+    conformance_failed: "{{ gateway_status is failed or conformance_result.rc != 0 }}"
   tasks:
     - name: Wait for SSH
       ansible.builtin.wait_for_connection:
@@ -55,6 +58,20 @@
       register: openshell_cli
       changed_when: false
 
+    # Every test boot restarts the gateway, and some installers take minutes to
+    # recover. Wait once here; the runner's per-scenario preflight stays short.
+    - name: Wait for the registered gateway
+      ansible.builtin.command:
+        argv: [timeout, 10s, "{{ openshell_cli.stdout }}", status, --output, json]
+      register: gateway_status
+      changed_when: false
+      until: >-
+        gateway_status.rc == 0 and
+        (gateway_status.stdout | from_json).status == 'connected'
+      retries: 60
+      delay: 5
+      ignore_errors: true
+
     - name: Run OpenShell conformance archive
       ansible.builtin.command:
         argv:
@@ -77,34 +94,25 @@
       register: conformance_result
       changed_when: false
       failed_when: false
+      when: gateway_status is succeeded
 
     # Preserve both streams for failures without adding passing-test output to
     # every tmachine run.
     - name: Show OpenShell conformance diagnostics
       ansible.builtin.debug:
         var: conformance_result
-      when: conformance_result.rc != 0
+      when: conformance_failed
 
-    - name: Read OpenShell gateway logs
-      become: true
-      ansible.builtin.command:
-        argv:
-          - journalctl
-          - --no-pager
-          - --lines
-          - "500"
-          - _SYSTEMD_UNIT=openshell-gateway.service
-          - "+"
-          - _SYSTEMD_USER_UNIT=openshell-gateway.service
-      register: openshell_gateway_logs
-      changed_when: false
-      failed_when: false
-      when: conformance_result.rc != 0
+    - name: Collect OpenShell gateway diagnostics
+      ansible.builtin.include_role:
+        name: openshell_diagnostics
+      when: conformance_failed
 
-    - name: Show OpenShell gateway logs
-      ansible.builtin.debug:
-        var: openshell_gateway_logs.stdout_lines
-      when: conformance_result.rc != 0
+    - name: Require a connected gateway
+      ansible.builtin.assert:
+        that:
+          - gateway_status is succeeded
+        fail_msg: OpenShell gateway did not connect before conformance; see gateway diagnostics
 
     - name: Require OpenShell conformance success
       ansible.builtin.assert:

tests/ansible/playbooks/openshell-k3s.yaml

@@ -137,39 +137,32 @@
       environment:
         KUBECONFIG: /etc/rancher/k3s/k3s.yaml
 
-    - name: Install gateway port-forward service
-      ansible.builtin.copy:
-        dest: /etc/systemd/system/openshell-k3s-port-forward.service
-        mode: "0644"
-        content: |
-          [Unit]
-          Description=OpenShell K3s gateway port forward
-          After=k3s.service
-          Requires=k3s.service
-
-          [Service]
-          ExecStart=/usr/local/bin/k3s kubectl --namespace openshell port-forward --address 127.0.0.1 service/openshell 17670:8080
-          Restart=always
-          RestartSec=1
-
-          [Install]
-          WantedBy=multi-user.target
-
-    - name: Start gateway port-forward service
-      ansible.builtin.systemd_service:
-        name: openshell-k3s-port-forward.service
-        daemon_reload: true
-        enabled: true
-        state: started
+    - name: Discover the gateway ClusterIP and gRPC port
+      ansible.builtin.command:
+        argv:
+          - /usr/local/bin/k3s
+          - kubectl
+          - --request-timeout=10s
+          - --namespace
+          - openshell
+          - get
+          - service
+          - openshell
+          - --output=jsonpath={.spec.clusterIP}:{.spec.ports[?(@.name=="grpc")].port}
+      register: k3s_gateway_address
+      changed_when: false
 
-    - name: Wait for OpenShell gateway
-      ansible.builtin.wait_for:
-        host: 127.0.0.1
-        port: 17670
-        timeout: 60
+    # Rejects headless Services (ClusterIP "None") and a missing grpc port.
+    - name: Require a gateway Service address
+      ansible.builtin.assert:
+        that:
+          - k3s_gateway_address.stdout is match('^[0-9.]+:[0-9]+$')
+        fail_msg: "Unexpected gateway Service address: {{ k3s_gateway_address.stdout }}"
 
 - name: Register OpenShell gateway for test client
   hosts: all
   gather_facts: false
+  vars:
+    openshell_client_gateway_endpoint: "http://{{ k3s_gateway_address.stdout }}"
   roles:
     - openshell_client

tests/ansible/roles/openshell_diagnostics/files/collect-k3s.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# No kubeconfig, Secrets, environment dumps, or complete Pod specifications.
+set -u
+umask 077
+mkdir -p /var/lib/openshell/diagnostics
+output=/var/lib/openshell/diagnostics/k3s.txt
+
+collect() {
+  printf '\n### %s\n' "$1"
+  shift
+  # Bound both stalled commands and unusually large log streams.
+  timeout 10s "$@" 2>&1 | tail -c 262144
+  printf '\ncommand status: %s\n' "${PIPESTATUS[0]}"
+}
+
+{
+  date --utc --iso-8601=seconds
+  collect 'K3s service state' systemctl show k3s.service \
+    -p ActiveState -p SubState -p Result -p NRestarts -p ExecMainStatus \
+    -p ExecMainPID -p ActiveEnterTimestamp -p ExecMainStartTimestamp \
+    -p StartLimitIntervalUSec -p StartLimitBurst -p RestartUSec
+  collect 'K3s boot journal' journalctl -b --no-pager --lines=500 -u k3s.service
+  # kube-proxy may program either iptables backend; dump whichever exist.
+  # shellcheck disable=SC2016 # Expanded by the inner shell.
+  collect 'Gateway Service routing rules' bash -c '
+    for save in iptables-nft-save iptables-legacy-save; do
+      command -v "$save" >/dev/null || continue
+      echo "# $save"
+      "$save" 2>/dev/null | awk "/openshell\/openshell:grpc/ { print }"
+    done'
+  collect 'K3s version' /usr/local/bin/k3s --version
+  collect 'API readiness' /usr/local/bin/k3s kubectl --request-timeout=5s get --raw=/readyz
+  collect 'Nodes' /usr/local/bin/k3s kubectl --request-timeout=5s get nodes -o wide
+  collect 'Gateway Pod identity and state' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get pods \
+    -o 'custom-columns=NAME:.metadata.name,UID:.metadata.uid,PHASE:.status.phase,CONDITIONS:.status.conditions,CONTAINERS:.status.containerStatuses'
+  collect 'Gateway Services' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get services \
+    -o 'custom-columns=NAME:.metadata.name,TYPE:.spec.type,CLUSTERIP:.spec.clusterIP,SELECTOR:.spec.selector,PORTS:.spec.ports'
+  collect 'Test client gateway registrations' runuser -u tmachine -- openshell gateway list --output json
+  collect 'Test client gateway status' runuser -u tmachine -- openshell status --output json
+  collect 'Endpoint readiness' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get endpointslices \
+    -o 'custom-columns=NAME:.metadata.name,PORTS:.ports,ADDRESSES:.endpoints[*].addresses,CONDITIONS:.endpoints[*].conditions'
+  collect 'Gateway events' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get events --sort-by=.lastTimestamp
+  collect 'Gateway current logs' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs openshell-0 -c openshell-gateway --tail=300 --timestamps
+  collect 'Gateway previous logs' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs openshell-0 -c openshell-gateway --previous --tail=300 --timestamps
+  collect 'Memory' free -m
+  collect 'Disk' df -h / /var/lib/rancher/k3s
+} | sed -E \
+  -e 's/(Bearer )[A-Za-z0-9._~+\/-]+/\1[REDACTED]/gI' \
+  -e 's/((token|password|secret|authorization)[" ]*[=:][" ]*)[^ ,"]+/\1[REDACTED]/gI' \
+  > "$output"
+cat "$output"

tests/ansible/roles/openshell_diagnostics/tasks/main.yaml

@@ -0,0 +1,51 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+---
+# Collect whatever applies to the installed gateway before the VM exits.
+- name: Read OpenShell gateway logs
+  become: true
+  ansible.builtin.command:
+    argv:
+      - journalctl
+      - --no-pager
+      - --lines
+      - "500"
+      - _SYSTEMD_UNIT=openshell-gateway.service
+      - "+"
+      - _SYSTEMD_USER_UNIT=openshell-gateway.service
+  register: openshell_gateway_logs
+  changed_when: false
+  failed_when: false
+
+- name: Show OpenShell gateway logs
+  ansible.builtin.debug:
+    var: openshell_gateway_logs.stdout_lines
+
+- name: Check for K3s
+  become: true
+  ansible.builtin.stat:
+    path: /usr/local/bin/k3s
+  register: k3s_diagnostics_binary
+  failed_when: false
+
+- name: Preserve K3s diagnostics
+  when: k3s_diagnostics_binary.stat.exists | default(false)
+  become: true
+  block:
+    - name: Collect bounded K3s diagnostics
+      ansible.builtin.script: collect-k3s.sh
+      register: k3s_diagnostics_collection
+      changed_when: false
+      failed_when: false
+
+    - name: Show K3s diagnostics
+      ansible.builtin.debug:
+        var: k3s_diagnostics_collection.stdout_lines
+
+    - name: Fetch K3s diagnostics
+      ansible.builtin.fetch:
+        src: /var/lib/openshell/diagnostics/k3s.txt
+        dest: "{{ role_path }}/../../../../artifacts/tmachine-diagnostics/{{ inventory_hostname }}/k3s.txt"
+        flat: true
+      failed_when: false