Fix NFT lifecycle baseline test and apptainer timeout

src/orchestrator/roles/cleanup/components/openchami/tasks/cleanup.yml

@@ -73,8 +73,8 @@
   tags: openchami
 
 - name: Wait for services to stop
-  ansible.builtin.pause:
-    seconds: 5
+  ansible.builtin.wait_for:
+    timeout: 5
   when:
     - not (dry_run | default(false) | bool)
     - >-

test/orchestrator/conftest.py

@@ -248,6 +248,7 @@ def pytest_configure(config):
         "performance": "Lifecycle duration checks",
         "idempotency": "Repeated lifecycle execution checks",
         "security": "Credential, key, log, and vault protection checks",
+        "lifecycle": "Clean-baseline and fresh-install lifecycle checks",
     }
     for name, desc in markers.items():
         config.addinivalue_line("markers", f"{name}: {desc}")

test/orchestrator/library/functions/__init__.py

@@ -103,8 +103,10 @@
 )
 from .network_inventory_func import check_network_inventory
 from .nft_func import (
+    check_clean_baseline,
     check_cleanup_idempotency,
     check_credential_file_permissions,
+    check_lifecycle_fresh_install,
     check_lifecycle_performance,
     check_log_file_permissions,
     check_precheck_idempotency,
@@ -230,6 +232,7 @@ def run_playbook(tag: str | None = None, **kwargs):
     "check_apptainer_slurm_environment",
     "check_boot_configurations",
     "check_boot_nodes",
+    "check_clean_baseline",
     "check_cleanup_artifacts",
     "check_cleanup_credentials",
     "check_cleanup_idempotency",
@@ -256,6 +259,7 @@ def run_playbook(tag: str | None = None, **kwargs):
     "check_kubernetes_version_compatibility",
     "check_kubernetes_virtual_ip",
     "check_kubernetes_workload_scheduling",
+    "check_lifecycle_fresh_install",
     "check_lifecycle_performance",
     "check_log_file_permissions",
     "check_metadata_groups",

test/orchestrator/library/functions/nft_func.py

@@ -127,7 +127,14 @@ def persistent_changed_count(output: str) -> int:
 
 def _playbook_failure(tag: str, result: dict[str, Any]) -> str:
     """Return a concise playbook failure suitable for reports."""
-    return str(result.get("error") or f"{tag} exited with rc={result.get('rc')}")
+    raw = str(result.get("error") or "")
+    rc = result.get("rc", "unknown")
+    # Extract only the first meaningful line; drop generic HOW TO FIX blocks
+    # that are aimed at interactive runner output, not test reports.
+    if raw:
+        first_line = raw.split("\n")[0].strip()
+        return f"{tag} failed (rc={rc}): {first_line}"
+    return f"{tag} exited with rc={rc}"
 
 
 def _run_lifecycle(tag: str, **kwargs) -> dict[str, Any]:
@@ -546,3 +553,157 @@ def check_vault_encryption(host) -> dict[str, Any]:
         fields,
         "; ".join(failures),
     )
+
+
+# -----------------------------------------------------------------
+# Lifecycle — clean-baseline and fresh-install contracts
+# -----------------------------------------------------------------
+
+
+def _lifecycle_cleanup_postconditions(
+    host,
+) -> tuple[list[tuple[str, object]], list[str]]:
+    """Reuse cleanup FVT postconditions, excluding credentials.
+
+    The lifecycle baseline preserves credentials so the subsequent
+    fresh-install can proceed without interactive prompts.
+    """
+    checks: tuple[tuple[str, Callable], ...] = (
+        ("OpenCHAMI", check_cleanup_openchami),
+        ("OpenLDAP", check_cleanup_openldap),
+        ("Slurm", check_cleanup_slurm),
+        ("Kubernetes", check_cleanup_kubernetes),
+        ("Artifacts", check_cleanup_artifacts),
+    )
+    fields = []
+    failures = []
+    for label, checker in checks:
+        result = checker(host)
+        fields.append(
+            (f"{label} postcondition", "passed" if result["success"] else "FAILED")
+        )
+        if not result["success"]:
+            failures.append(f"{label}: {result['error']}")
+    fields.append(("Credentials postcondition", "skipped (preserved for lifecycle)"))
+    return fields, failures
+
+
+def check_clean_baseline(host) -> dict[str, Any]:
+    """Run cleanup and verify all postconditions to prove a clean OIM state.
+
+    Credentials are preserved (``cleanup_credentials=false``) so the
+    subsequent fresh-install lifecycle can run without interactive
+    password prompts.
+    """
+    try:
+        extra_vars = cleanup_extra_vars()
+    except (TypeError, ValueError) as exc:
+        return _result(False, "Cleanup configuration is invalid", [], str(exc))
+
+    # Preserve credentials so prepare does not prompt for manual input.
+    extra_vars["cleanup_credentials"] = "false"
+
+    result = _run_lifecycle("cleanup", extra_vars=extra_vars)
+    if not result.get("success"):
+        return _result(
+            False,
+            "Baseline cleanup execution failed",
+            [
+                ("Lifecycle", "cleanup"),
+                ("Return code", result.get("rc", "unknown")),
+                ("Duration seconds", f"{float(result.get('duration', 0)):.1f}"),
+            ],
+            _playbook_failure("cleanup", result),
+        )
+
+    postcondition_fields, postcondition_failures = (
+        _lifecycle_cleanup_postconditions(host)
+    )
+    fields = [
+        ("Cleanup duration seconds", f"{float(result.get('duration', 0)):.1f}"),
+        ("Credentials policy", "preserved for lifecycle"),
+        *postcondition_fields,
+    ]
+    return _result(
+        not postcondition_failures,
+        "OIM baseline is clean — all cleanup postconditions passed",
+        fields,
+        "; ".join(postcondition_failures),
+    )
+
+
+def check_lifecycle_fresh_install(host) -> dict[str, Any]:
+    """Run precheck, prepare, and provision from a proven-clean baseline."""
+    lifecycles = ("precheck", "prepare", "provision")
+    durations: dict[str, float] = {}
+    for lifecycle in lifecycles:
+        result = _run_lifecycle(lifecycle)
+        durations[lifecycle] = float(result.get("duration", 0))
+        if not result.get("success"):
+            completed = [
+                lc for lc in lifecycles if lc in durations and lc != lifecycle
+            ]
+            fields = [
+                ("Failed lifecycle", lifecycle),
+                ("Return code", result.get("rc", "unknown")),
+                ("Completed phases", ", ".join(completed) if completed else "none"),
+                *(
+                    (f"{lc} duration seconds", f"{durations[lc]:.1f}")
+                    for lc in lifecycles
+                    if lc in durations
+                ),
+            ]
+            return _result(
+                False,
+                f"Fresh install failed at {lifecycle} phase",
+                fields,
+                _playbook_failure(lifecycle, result),
+            )
+
+    # Verify prepare postconditions
+    readiness_fields, readiness_failures = _prepare_readiness(host)
+
+    # Verify provision postconditions by importing checkers directly
+    from .smd_provision_func import check_smd_identity, check_smd_groups
+    from .boot_service_provision_func import (
+        check_boot_configurations,
+        check_boot_nodes,
+    )
+    from .metadata_service_provision_func import (
+        check_metadata_groups,
+        check_metadata_instances,
+    )
+
+    provision_checks: tuple[tuple[str, Callable], ...] = (
+        ("SMD identity", check_smd_identity),
+        ("SMD functional groups", check_smd_groups),
+        ("Boot configurations", check_boot_configurations),
+        ("Boot node identity", check_boot_nodes),
+        ("Metadata groups", check_metadata_groups),
+        ("Metadata instances", check_metadata_instances),
+    )
+    provision_fields: list[tuple[str, object]] = []
+    provision_failures: list[str] = []
+    for label, checker in provision_checks:
+        check_result = checker(host)
+        provision_fields.append(
+            (label, "passed" if check_result["success"] else "FAILED")
+        )
+        if not check_result["success"]:
+            provision_failures.append(f"{label}: {check_result['error']}")
+
+    all_failures = [*readiness_failures, *provision_failures]
+    fields = [
+        *(
+            (f"{lc} duration seconds", f"{durations[lc]:.1f}")
+            for lc in lifecycles
+        ),
+        *readiness_fields,
+        *provision_fields,
+    ]
+    return _result(
+        not all_failures,
+        "Fresh-install lifecycle completed and all postconditions passed",
+        fields,
+        "; ".join(all_failures),
+    )

test/orchestrator/library/vars/domain_vars.py

@@ -58,6 +58,7 @@
     "performance",
     "idempotency",
     "security",
+    "lifecycle",
 ]
 
 # =====================================================================

test/orchestrator/library/vars/pxeboot_vars.py

@@ -43,7 +43,7 @@
 APPTAINER_SCRIPT_DIRECTORY = "/hpc_tools/scripts"
 APPTAINER_DOWNLOAD_SCRIPT = "/hpc_tools/scripts/download_container_image.sh"
 APPTAINER_IMAGE_LIST = "/hpc_tools/scripts/container_image.list"
-APPTAINER_DOWNLOAD_TIMEOUT_SECONDS = 1200
+APPTAINER_DOWNLOAD_TIMEOUT_SECONDS = 1800
 LONG_OPERATION_POLL_SECONDS = 20
 APPTAINER_DOWNLOAD_MAX_RSS_KIB = 1048576
 APPTAINER_JOB_TIMEOUT_SECONDS = 300

test/orchestrator/library/vars/test_case_vars.py

@@ -715,6 +715,14 @@
         "id": "ORCH_NFT_011",
         "title": "Verify Orchestrator vault encryption",
     },
+    "clean_baseline": {
+        "id": "ORCH_NFT_012",
+        "title": "Verify clean OIM baseline before fresh install",
+    },
+    "lifecycle_fresh_install": {
+        "id": "ORCH_NFT_013",
+        "title": "Verify complete fresh-install lifecycle from clean baseline",
+    },
 }
 
 TEST_CASES: dict[str, dict[str, str]] = {

test/orchestrator/nft/test_lifecycle.py

@@ -0,0 +1,46 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Orchestrator clean-baseline and fresh-install lifecycle contracts.
+
+Addresses OMN-DEF #850: proves that the complete fresh-install lifecycle
+succeeds from a verified clean OIM state.  The baseline test runs cleanup
+and asserts every cleanup postcondition, then the fresh-install test runs
+precheck -> prepare -> provision in sequence and verifies all prepare and
+provision postconditions are bound to that run.
+"""
+
+import pytest
+from library.functions import (
+    check_clean_baseline,
+    check_lifecycle_fresh_install,
+)
+
+from nft.result import verify_nft
+
+pytestmark = [pytest.mark.nft, pytest.mark.lifecycle]
+
+
+@pytest.mark.destructive
+@pytest.mark.order(100)
+def test_clean_baseline(host):
+    """Require a provably clean OIM state before the fresh-install lifecycle."""
+    verify_nft(host, "clean_baseline", check_clean_baseline)
+
+
+@pytest.mark.destructive
+@pytest.mark.order(101)
+def test_lifecycle_fresh_install(host):
+    """Require the complete fresh-install lifecycle to succeed from clean baseline."""
+    verify_nft(host, "lifecycle_fresh_install", check_lifecycle_fresh_install)