From 62cd50c48db78337597cb780cb73c2ad3afbeaa9 Mon Sep 17 00:00:00 2001 From: gabriela-reznik Date: Wed, 5 Aug 2026 11:28:33 +0200 Subject: [PATCH 1/4] nextaur json validations --- .../workflows/validate-nextflow-assets.yml | 27 ++++ .../dxpy/nextflow/validate_nextflow_assets.py | 141 ++++++++++++++++++ 2 files changed, 168 insertions(+) create mode 100644 .github/workflows/validate-nextflow-assets.yml create mode 100644 src/python/dxpy/nextflow/validate_nextflow_assets.py diff --git a/.github/workflows/validate-nextflow-assets.yml b/.github/workflows/validate-nextflow-assets.yml new file mode 100644 index 000000000..7dbd3db47 --- /dev/null +++ b/.github/workflows/validate-nextflow-assets.yml @@ -0,0 +1,27 @@ +name: Validate Nextflow asset JSON files + +on: + pull_request: + paths: + - "src/python/dxpy/nextflow/**" + push: + branches: + - master + paths: + - "src/python/dxpy/nextflow/**" + +jobs: + validate-assets: + name: Validate asset record/project IDs + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Run Nextflow asset validator + run: python src/python/dxpy/nextflow/validate_nextflow_assets.py + diff --git a/src/python/dxpy/nextflow/validate_nextflow_assets.py b/src/python/dxpy/nextflow/validate_nextflow_assets.py new file mode 100644 index 000000000..17f1a63fa --- /dev/null +++ b/src/python/dxpy/nextflow/validate_nextflow_assets.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +""" +Standalone validator for Nextflow asset JSON files. + +Checks that every asset file referenced in versions.json: + - exists on disk + - is valid JSON + - contains only well-formed DNAnexus record IDs (^record-[A-Za-z0-9]{24}$) + - has identical region key sets in the prod and staging variants + +Also validates app_asset_projects_ids_{prod,staging}.json for project IDs. + +No dxpy dependency — stdlib only. Safe to run without any DX credentials. + +Exit code: 0 on success, 1 on any validation failure. +""" + +import json +import os +import re +import sys + +NEXTFLOW_DIR = os.path.join(os.path.dirname(__file__)) +VERSIONS_JSON = os.path.join(NEXTFLOW_DIR, "versions.json") + +RECORD_ID_RE = re.compile(r"^record-[A-Za-z0-9]{24}$") +PROJECT_ID_RE = re.compile(r"^project-[A-Za-z0-9]{24}$") + +ASSET_KEYS = ("nextaur_assets", "nextflow_assets", "awscli_assets") + + +def _load_json(filepath): + """Load and return parsed JSON. Returns (data, error_message).""" + if not os.path.isfile(filepath): + return None, f"File not found: {filepath}" + try: + with open(filepath) as f: + return json.load(f), None + except json.JSONDecodeError as e: + return None, f"Invalid JSON in {filepath}: {e}" + + +def validate_asset_file(filepath, id_re, id_type, errors): + """Validate a single asset JSON file. + + Returns the set of region keys if the file is valid, else None. + Appends error strings to `errors`. + """ + data, err = _load_json(filepath) + if err: + errors.append(err) + return None + + if not isinstance(data, dict): + errors.append(f"{filepath}: expected a JSON object, got {type(data).__name__}") + return None + + for region, obj_id in data.items(): + if not id_re.match(obj_id): + errors.append( + f"{filepath}[{region!r}]: {obj_id!r} is not a valid {id_type} ID " + f"(expected pattern {id_re.pattern})" + ) + + return set(data.keys()) + + +def validate_assets(): + errors = [] + + # ------------------------------------------------------------------ + # 1. Load versions.json + # ------------------------------------------------------------------ + manifest, err = _load_json(VERSIONS_JSON) + if err: + print(f"FAIL {err}", file=sys.stderr) + return False + + if "default" not in manifest or "versions" not in manifest: + errors.append(f"{VERSIONS_JSON}: missing 'default' or 'versions' key") + else: + # ------------------------------------------------------------------ + # 2. Validate every version's prod and staging asset files + # ------------------------------------------------------------------ + for ver, config in manifest["versions"].items(): + for key in ASSET_KEYS: + prod_filename = config.get(key) + if not prod_filename: + errors.append(f"versions.json[{ver!r}]: missing key {key!r}") + continue + + prod_path = os.path.join(NEXTFLOW_DIR, prod_filename) + staging_filename = prod_filename.replace(".json", ".staging.json") + staging_path = os.path.join(NEXTFLOW_DIR, staging_filename) + + prod_regions = validate_asset_file( + prod_path, RECORD_ID_RE, "record", errors + ) + staging_regions = validate_asset_file( + staging_path, RECORD_ID_RE, "record", errors + ) + + # Region-set consistency between prod and staging + if prod_regions is not None and staging_regions is not None: + only_in_prod = prod_regions - staging_regions + only_in_staging = staging_regions - prod_regions + if only_in_prod: + errors.append( + f"Region mismatch for {ver} / {key}: " + f"regions in prod but not staging: {sorted(only_in_prod)}" + ) + if only_in_staging: + errors.append( + f"Region mismatch for {ver} / {key}: " + f"regions in staging but not prod: {sorted(only_in_staging)}" + ) + + # ------------------------------------------------------------------ + # 3. Validate app_asset_projects_ids_{prod,staging}.json + # ------------------------------------------------------------------ + for filename in ( + "app_asset_projects_ids_prod.json", + "app_asset_projects_ids_staging.json", + ): + filepath = os.path.join(NEXTFLOW_DIR, filename) + validate_asset_file(filepath, PROJECT_ID_RE, "project", errors) + + # ------------------------------------------------------------------ + # 4. Report + # ------------------------------------------------------------------ + if errors: + for err in errors: + print(f"FAIL {err}", file=sys.stderr) + return False + + print(f"OK All Nextflow asset JSON files passed validation.", file=sys.stderr) + return True + + +if __name__ == "__main__": + sys.exit(0 if validate_assets() else 1) From 422cb3b02826a14319a9c1eb4eca2d0e2f420ead Mon Sep 17 00:00:00 2001 From: gabriela-reznik <110951027+gabriela-reznik@users.noreply.github.com> Date: Tue, 18 Aug 2026 05:20:25 -0300 Subject: [PATCH 2/4] Apply suggestion from @simunic-cz MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fixing python hashes Co-authored-by: Patrik Šimunič --- .github/workflows/validate-nextflow-assets.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate-nextflow-assets.yml b/.github/workflows/validate-nextflow-assets.yml index 7dbd3db47..3394e3c2d 100644 --- a/.github/workflows/validate-nextflow-assets.yml +++ b/.github/workflows/validate-nextflow-assets.yml @@ -16,9 +16,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 with: python-version: "3.x" From 48e8b5cb3de92cc5ef66bcaa3b21734ddea89fc0 Mon Sep 17 00:00:00 2001 From: gabriela-reznik <110951027+gabriela-reznik@users.noreply.github.com> Date: Tue, 18 Aug 2026 05:56:46 -0300 Subject: [PATCH 3/4] Apply suggestion from @simunic-cz MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Patrik Šimunič --- src/python/dxpy/nextflow/validate_nextflow_assets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/dxpy/nextflow/validate_nextflow_assets.py b/src/python/dxpy/nextflow/validate_nextflow_assets.py index 17f1a63fa..b264ef453 100644 --- a/src/python/dxpy/nextflow/validate_nextflow_assets.py +++ b/src/python/dxpy/nextflow/validate_nextflow_assets.py @@ -133,7 +133,7 @@ def validate_assets(): print(f"FAIL {err}", file=sys.stderr) return False - print(f"OK All Nextflow asset JSON files passed validation.", file=sys.stderr) + print(f"OK All Nextflow asset JSON files passed validation.", file=sys.stdout) return True From 3ba4f64339816836b1da701793dae37c0145dc38 Mon Sep 17 00:00:00 2001 From: gabriela-reznik Date: Tue, 18 Aug 2026 10:58:27 +0200 Subject: [PATCH 4/4] extra parity checks --- .../dxpy/nextflow/validate_nextflow_assets.py | 87 ++++++++++++++----- 1 file changed, 65 insertions(+), 22 deletions(-) diff --git a/src/python/dxpy/nextflow/validate_nextflow_assets.py b/src/python/dxpy/nextflow/validate_nextflow_assets.py index 17f1a63fa..3f5fb354f 100644 --- a/src/python/dxpy/nextflow/validate_nextflow_assets.py +++ b/src/python/dxpy/nextflow/validate_nextflow_assets.py @@ -7,6 +7,7 @@ - is valid JSON - contains only well-formed DNAnexus record IDs (^record-[A-Za-z0-9]{24}$) - has identical region key sets in the prod and staging variants + - has identical region key sets across all asset types (nextaur, nextflow, awscli) Also validates app_asset_projects_ids_{prod,staging}.json for project IDs. @@ -83,6 +84,10 @@ def validate_assets(): # 2. Validate every version's prod and staging asset files # ------------------------------------------------------------------ for ver, config in manifest["versions"].items(): + # Phase 1: load all prod and staging region sets for this version + prod_regions_by_key = {} + staging_regions_by_key = {} + for key in ASSET_KEYS: prod_filename = config.get(key) if not prod_filename: @@ -90,40 +95,78 @@ def validate_assets(): continue prod_path = os.path.join(NEXTFLOW_DIR, prod_filename) - staging_filename = prod_filename.replace(".json", ".staging.json") - staging_path = os.path.join(NEXTFLOW_DIR, staging_filename) - - prod_regions = validate_asset_file( - prod_path, RECORD_ID_RE, "record", errors + staging_path = os.path.join( + NEXTFLOW_DIR, prod_filename.replace(".json", ".staging.json") ) - staging_regions = validate_asset_file( + + regions = validate_asset_file(prod_path, RECORD_ID_RE, "record", errors) + if regions is not None: + prod_regions_by_key[key] = regions + + regions = validate_asset_file( staging_path, RECORD_ID_RE, "record", errors ) - - # Region-set consistency between prod and staging - if prod_regions is not None and staging_regions is not None: - only_in_prod = prod_regions - staging_regions - only_in_staging = staging_regions - prod_regions - if only_in_prod: + if regions is not None: + staging_regions_by_key[key] = regions + + # Phase 2: union of all regions mentioned in any successfully loaded file + all_regions = set() + for regions in prod_regions_by_key.values(): + all_regions |= regions + for regions in staging_regions_by_key.values(): + all_regions |= regions + + # Phase 3: for each region, verify prod and staging coverage per asset + for region in sorted(all_regions): + for key in ASSET_KEYS: + if ( + key not in prod_regions_by_key + and key not in staging_regions_by_key + ): + continue # file-load failure already reported + if ( + key in prod_regions_by_key + and region not in prod_regions_by_key[key] + ): errors.append( f"Region mismatch for {ver} / {key}: " - f"regions in prod but not staging: {sorted(only_in_prod)}" + f"region '{region}' missing from prod" ) - if only_in_staging: + if ( + key in staging_regions_by_key + and region not in staging_regions_by_key[key] + ): errors.append( f"Region mismatch for {ver} / {key}: " - f"regions in staging but not prod: {sorted(only_in_staging)}" + f"region '{region}' missing from staging" ) # ------------------------------------------------------------------ - # 3. Validate app_asset_projects_ids_{prod,staging}.json + # 3. Validate app_asset_projects_ids_{prod,staging}.json and compare # ------------------------------------------------------------------ - for filename in ( - "app_asset_projects_ids_prod.json", - "app_asset_projects_ids_staging.json", - ): - filepath = os.path.join(NEXTFLOW_DIR, filename) - validate_asset_file(filepath, PROJECT_ID_RE, "project", errors) + prod_project_regions = validate_asset_file( + os.path.join(NEXTFLOW_DIR, "app_asset_projects_ids_prod.json"), + PROJECT_ID_RE, + "project", + errors, + ) + staging_project_regions = validate_asset_file( + os.path.join(NEXTFLOW_DIR, "app_asset_projects_ids_staging.json"), + PROJECT_ID_RE, + "project", + errors, + ) + if prod_project_regions is not None and staging_project_regions is not None: + only_in_prod = prod_project_regions - staging_project_regions + only_in_staging = staging_project_regions - prod_project_regions + if only_in_prod: + errors.append( + f"app_asset_projects_ids: regions in prod but not staging: {sorted(only_in_prod)}" + ) + if only_in_staging: + errors.append( + f"app_asset_projects_ids: regions in staging but not prod: {sorted(only_in_staging)}" + ) # ------------------------------------------------------------------ # 4. Report