From 99b71971cf58d2a73467ec3da50df8a54e113ca1 Mon Sep 17 00:00:00 2001 From: SujalS27 Date: Sat, 26 Sep 2026 00:39:14 +0530 Subject: [PATCH] Fix NFT lifecycle baseline test and apptainer timeout - Add NFT lifecycle test (test_lifecycle.py) to prove complete fresh-install lifecycle from verified clean baseline, addressing OMN-DEF #850 - Increase apptainer image download timeout from 1200s to 1800s to handle larger image downloads - Fix OpenCHAMI cleanup pause task causing ENOTTY error on non-interactive execution by replacing ansible.builtin.pause with ansible.builtin.wait_for - Preserve credentials during lifecycle baseline cleanup to avoid interactive password prompts in subsequent prepare phase - Improve NFT error messages to show concise failure details instead of generic HOW TO FIX boilerplate - Add lifecycle marker to domain configuration and conftest Signed-off-by: SujalS27 --- .../components/openchami/tasks/cleanup.yml | 4 +- test/orchestrator/conftest.py | 1 + .../library/functions/__init__.py | 4 + .../library/functions/nft_func.py | 163 +++++++++++++++++- test/orchestrator/library/vars/domain_vars.py | 1 + .../orchestrator/library/vars/pxeboot_vars.py | 2 +- .../library/vars/test_case_vars.py | 8 + test/orchestrator/nft/test_lifecycle.py | 46 +++++ 8 files changed, 225 insertions(+), 4 deletions(-) create mode 100644 test/orchestrator/nft/test_lifecycle.py diff --git a/src/orchestrator/roles/cleanup/components/openchami/tasks/cleanup.yml b/src/orchestrator/roles/cleanup/components/openchami/tasks/cleanup.yml index dd7070dd6..9de5035dd 100644 --- a/src/orchestrator/roles/cleanup/components/openchami/tasks/cleanup.yml +++ b/src/orchestrator/roles/cleanup/components/openchami/tasks/cleanup.yml @@ -73,8 +73,8 @@ tags: openchami - name: Wait for services to stop - ansible.builtin.pause: - seconds: 5 + ansible.builtin.wait_for: + timeout: 5 when: - not (dry_run | default(false) | bool) - >- diff --git a/test/orchestrator/conftest.py b/test/orchestrator/conftest.py index 0d653b7ff..5908cdd14 100644 --- a/test/orchestrator/conftest.py +++ b/test/orchestrator/conftest.py @@ -248,6 +248,7 @@ def pytest_configure(config): "performance": "Lifecycle duration checks", "idempotency": "Repeated lifecycle execution checks", "security": "Credential, key, log, and vault protection checks", + "lifecycle": "Clean-baseline and fresh-install lifecycle checks", } for name, desc in markers.items(): config.addinivalue_line("markers", f"{name}: {desc}") diff --git a/test/orchestrator/library/functions/__init__.py b/test/orchestrator/library/functions/__init__.py index 634ab9093..65423120f 100644 --- a/test/orchestrator/library/functions/__init__.py +++ b/test/orchestrator/library/functions/__init__.py @@ -103,8 +103,10 @@ ) from .network_inventory_func import check_network_inventory from .nft_func import ( + check_clean_baseline, check_cleanup_idempotency, check_credential_file_permissions, + check_lifecycle_fresh_install, check_lifecycle_performance, check_log_file_permissions, check_precheck_idempotency, @@ -230,6 +232,7 @@ def run_playbook(tag: str | None = None, **kwargs): "check_apptainer_slurm_environment", "check_boot_configurations", "check_boot_nodes", + "check_clean_baseline", "check_cleanup_artifacts", "check_cleanup_credentials", "check_cleanup_idempotency", @@ -256,6 +259,7 @@ def run_playbook(tag: str | None = None, **kwargs): "check_kubernetes_version_compatibility", "check_kubernetes_virtual_ip", "check_kubernetes_workload_scheduling", + "check_lifecycle_fresh_install", "check_lifecycle_performance", "check_log_file_permissions", "check_metadata_groups", diff --git a/test/orchestrator/library/functions/nft_func.py b/test/orchestrator/library/functions/nft_func.py index e12b18e66..5289b3d76 100644 --- a/test/orchestrator/library/functions/nft_func.py +++ b/test/orchestrator/library/functions/nft_func.py @@ -127,7 +127,14 @@ def persistent_changed_count(output: str) -> int: def _playbook_failure(tag: str, result: dict[str, Any]) -> str: """Return a concise playbook failure suitable for reports.""" - return str(result.get("error") or f"{tag} exited with rc={result.get('rc')}") + raw = str(result.get("error") or "") + rc = result.get("rc", "unknown") + # Extract only the first meaningful line; drop generic HOW TO FIX blocks + # that are aimed at interactive runner output, not test reports. + if raw: + first_line = raw.split("\n")[0].strip() + return f"{tag} failed (rc={rc}): {first_line}" + return f"{tag} exited with rc={rc}" def _run_lifecycle(tag: str, **kwargs) -> dict[str, Any]: @@ -546,3 +553,157 @@ def check_vault_encryption(host) -> dict[str, Any]: fields, "; ".join(failures), ) + + +# ----------------------------------------------------------------- +# Lifecycle — clean-baseline and fresh-install contracts +# ----------------------------------------------------------------- + + +def _lifecycle_cleanup_postconditions( + host, +) -> tuple[list[tuple[str, object]], list[str]]: + """Reuse cleanup FVT postconditions, excluding credentials. + + The lifecycle baseline preserves credentials so the subsequent + fresh-install can proceed without interactive prompts. + """ + checks: tuple[tuple[str, Callable], ...] = ( + ("OpenCHAMI", check_cleanup_openchami), + ("OpenLDAP", check_cleanup_openldap), + ("Slurm", check_cleanup_slurm), + ("Kubernetes", check_cleanup_kubernetes), + ("Artifacts", check_cleanup_artifacts), + ) + fields = [] + failures = [] + for label, checker in checks: + result = checker(host) + fields.append( + (f"{label} postcondition", "passed" if result["success"] else "FAILED") + ) + if not result["success"]: + failures.append(f"{label}: {result['error']}") + fields.append(("Credentials postcondition", "skipped (preserved for lifecycle)")) + return fields, failures + + +def check_clean_baseline(host) -> dict[str, Any]: + """Run cleanup and verify all postconditions to prove a clean OIM state. + + Credentials are preserved (``cleanup_credentials=false``) so the + subsequent fresh-install lifecycle can run without interactive + password prompts. + """ + try: + extra_vars = cleanup_extra_vars() + except (TypeError, ValueError) as exc: + return _result(False, "Cleanup configuration is invalid", [], str(exc)) + + # Preserve credentials so prepare does not prompt for manual input. + extra_vars["cleanup_credentials"] = "false" + + result = _run_lifecycle("cleanup", extra_vars=extra_vars) + if not result.get("success"): + return _result( + False, + "Baseline cleanup execution failed", + [ + ("Lifecycle", "cleanup"), + ("Return code", result.get("rc", "unknown")), + ("Duration seconds", f"{float(result.get('duration', 0)):.1f}"), + ], + _playbook_failure("cleanup", result), + ) + + postcondition_fields, postcondition_failures = ( + _lifecycle_cleanup_postconditions(host) + ) + fields = [ + ("Cleanup duration seconds", f"{float(result.get('duration', 0)):.1f}"), + ("Credentials policy", "preserved for lifecycle"), + *postcondition_fields, + ] + return _result( + not postcondition_failures, + "OIM baseline is clean — all cleanup postconditions passed", + fields, + "; ".join(postcondition_failures), + ) + + +def check_lifecycle_fresh_install(host) -> dict[str, Any]: + """Run precheck, prepare, and provision from a proven-clean baseline.""" + lifecycles = ("precheck", "prepare", "provision") + durations: dict[str, float] = {} + for lifecycle in lifecycles: + result = _run_lifecycle(lifecycle) + durations[lifecycle] = float(result.get("duration", 0)) + if not result.get("success"): + completed = [ + lc for lc in lifecycles if lc in durations and lc != lifecycle + ] + fields = [ + ("Failed lifecycle", lifecycle), + ("Return code", result.get("rc", "unknown")), + ("Completed phases", ", ".join(completed) if completed else "none"), + *( + (f"{lc} duration seconds", f"{durations[lc]:.1f}") + for lc in lifecycles + if lc in durations + ), + ] + return _result( + False, + f"Fresh install failed at {lifecycle} phase", + fields, + _playbook_failure(lifecycle, result), + ) + + # Verify prepare postconditions + readiness_fields, readiness_failures = _prepare_readiness(host) + + # Verify provision postconditions by importing checkers directly + from .smd_provision_func import check_smd_identity, check_smd_groups + from .boot_service_provision_func import ( + check_boot_configurations, + check_boot_nodes, + ) + from .metadata_service_provision_func import ( + check_metadata_groups, + check_metadata_instances, + ) + + provision_checks: tuple[tuple[str, Callable], ...] = ( + ("SMD identity", check_smd_identity), + ("SMD functional groups", check_smd_groups), + ("Boot configurations", check_boot_configurations), + ("Boot node identity", check_boot_nodes), + ("Metadata groups", check_metadata_groups), + ("Metadata instances", check_metadata_instances), + ) + provision_fields: list[tuple[str, object]] = [] + provision_failures: list[str] = [] + for label, checker in provision_checks: + check_result = checker(host) + provision_fields.append( + (label, "passed" if check_result["success"] else "FAILED") + ) + if not check_result["success"]: + provision_failures.append(f"{label}: {check_result['error']}") + + all_failures = [*readiness_failures, *provision_failures] + fields = [ + *( + (f"{lc} duration seconds", f"{durations[lc]:.1f}") + for lc in lifecycles + ), + *readiness_fields, + *provision_fields, + ] + return _result( + not all_failures, + "Fresh-install lifecycle completed and all postconditions passed", + fields, + "; ".join(all_failures), + ) diff --git a/test/orchestrator/library/vars/domain_vars.py b/test/orchestrator/library/vars/domain_vars.py index 60f942dc1..540679fd0 100644 --- a/test/orchestrator/library/vars/domain_vars.py +++ b/test/orchestrator/library/vars/domain_vars.py @@ -58,6 +58,7 @@ "performance", "idempotency", "security", + "lifecycle", ] # ===================================================================== diff --git a/test/orchestrator/library/vars/pxeboot_vars.py b/test/orchestrator/library/vars/pxeboot_vars.py index 2153674e4..5b7c0f983 100644 --- a/test/orchestrator/library/vars/pxeboot_vars.py +++ b/test/orchestrator/library/vars/pxeboot_vars.py @@ -43,7 +43,7 @@ APPTAINER_SCRIPT_DIRECTORY = "/hpc_tools/scripts" APPTAINER_DOWNLOAD_SCRIPT = "/hpc_tools/scripts/download_container_image.sh" APPTAINER_IMAGE_LIST = "/hpc_tools/scripts/container_image.list" -APPTAINER_DOWNLOAD_TIMEOUT_SECONDS = 1200 +APPTAINER_DOWNLOAD_TIMEOUT_SECONDS = 1800 LONG_OPERATION_POLL_SECONDS = 20 APPTAINER_DOWNLOAD_MAX_RSS_KIB = 1048576 APPTAINER_JOB_TIMEOUT_SECONDS = 300 diff --git a/test/orchestrator/library/vars/test_case_vars.py b/test/orchestrator/library/vars/test_case_vars.py index 07ebf8ba0..a75925cec 100644 --- a/test/orchestrator/library/vars/test_case_vars.py +++ b/test/orchestrator/library/vars/test_case_vars.py @@ -715,6 +715,14 @@ "id": "ORCH_NFT_011", "title": "Verify Orchestrator vault encryption", }, + "clean_baseline": { + "id": "ORCH_NFT_012", + "title": "Verify clean OIM baseline before fresh install", + }, + "lifecycle_fresh_install": { + "id": "ORCH_NFT_013", + "title": "Verify complete fresh-install lifecycle from clean baseline", + }, } TEST_CASES: dict[str, dict[str, str]] = { diff --git a/test/orchestrator/nft/test_lifecycle.py b/test/orchestrator/nft/test_lifecycle.py new file mode 100644 index 000000000..fae6c770c --- /dev/null +++ b/test/orchestrator/nft/test_lifecycle.py @@ -0,0 +1,46 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Orchestrator clean-baseline and fresh-install lifecycle contracts. + +Addresses OMN-DEF #850: proves that the complete fresh-install lifecycle +succeeds from a verified clean OIM state. The baseline test runs cleanup +and asserts every cleanup postcondition, then the fresh-install test runs +precheck -> prepare -> provision in sequence and verifies all prepare and +provision postconditions are bound to that run. +""" + +import pytest +from library.functions import ( + check_clean_baseline, + check_lifecycle_fresh_install, +) + +from nft.result import verify_nft + +pytestmark = [pytest.mark.nft, pytest.mark.lifecycle] + + +@pytest.mark.destructive +@pytest.mark.order(100) +def test_clean_baseline(host): + """Require a provably clean OIM state before the fresh-install lifecycle.""" + verify_nft(host, "clean_baseline", check_clean_baseline) + + +@pytest.mark.destructive +@pytest.mark.order(101) +def test_lifecycle_fresh_install(host): + """Require the complete fresh-install lifecycle to succeed from clean baseline.""" + verify_nft(host, "lifecycle_fresh_install", check_lifecycle_fresh_install)