From 4711c814ad805e84654dbe8da6152c77fa8d7644 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 27 Oct 2026 18:14:38 +0530 Subject: [PATCH 1/2] Cite real policy sections in the check registry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first version of the checker numbered its rules by their row in the Rules §9.1 table, and those row numbers ended up written as section numbers: §10 load pattern, §12 query count, §14 metric consistency, §15 accuracy, §1 for the directory checks, §§3–6 for region coverage. 26 of the 84 rules in messages.yaml cited one of them. Each now points to the section that defines what it checks. Seed rotation and publication cohorts are in the Submission Rules (§4.6, §4.2), but were cited bare, which reads as the Rules, where §4.6 does not exist. They now say "Submission Rules §4.6". A catalog test now fails on a spec outside the Rules' §1–§9 or the Submission Rules' §1–§11. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 66 ++++++++--------- docs/endpoints-submission-cli.md | 4 +- pyproject.toml | 2 +- .../commands/submissions/remove_run.py | 4 +- src/submission_checker/checker.py | 2 +- src/submission_checker/cli.py | 4 +- src/submission_checker/cohorts.py | 19 ++--- src/submission_checker/data/messages.yaml | 70 +++++++++---------- src/submission_checker/data/seed_sets.yaml | 4 +- .../models/aggregate/context.py | 10 +-- .../models/aggregate/point_result.py | 10 +-- .../models/aggregate/seed_binding.py | 6 +- .../models/file/point_config.py | 14 ++-- src/submission_checker/seed_sets.py | 8 +-- .../commands/test_check_submission_command.py | 2 +- .../commands/test_submissions_commands.py | 12 ++-- tests/endpoints_submission_cli/conftest.py | 2 +- .../submissions/test_builder.py | 2 +- .../test_checks_agentic_accuracy.py | 2 +- .../test_checks_seed_binding.py | 6 +- tests/submission_checker/test_cohorts.py | 8 +-- tests/submission_checker/test_messages.py | 22 +++++- tests/submission_checker/test_seed_sets.py | 6 +- tests/tools/regenerate_fixtures.py | 8 +-- 24 files changed, 159 insertions(+), 134 deletions(-) diff --git a/README.md b/README.md index 76a50371..b0c30637 100644 --- a/README.md +++ b/README.md @@ -218,7 +218,7 @@ directory below it; a submission root is the level holding `results/` and `docs/ | `--strict` | Treat warnings as errors (exit 1 on any warning) | | `--quiet` / `-q` | Suppress INFO-level passing checks | | `--output FILE` / `-o FILE` | Write full results as JSON to *FILE* | -| `--seed-sets FILE` | Published seed sets to check against (§4.6). Defaults to the bundled set; also settable via `$MLPERF_ENDPOINTS_SEED_SETS`. | +| `--seed-sets FILE` | Published seed sets to check against (Submission Rules §4.6). Defaults to the bundled set; also settable via `$MLPERF_ENDPOINTS_SEED_SETS`. | | `--approved-drafters FILE` | Published approved drafters (§2.9.4). Defaults to the bundled list; also settable via `$MLPERF_ENDPOINTS_APPROVED_DRAFTERS`. | **Exit codes:** `0` = all checks passed, `1` = one or more errors (or warnings with `--strict`). @@ -302,17 +302,17 @@ submission root (§9.1). | Rule | Spec | Description | |------|------|-------------| -| `path-exists` | §1 | Submission root directory exists | -| `required-dir` | §1 | `results/` and `docs/` present | -| `src-dir` | §2.2.1 | `src/` present with at least one implementation directory | -| `src-readme` | §2.2.1 | Each `src//` has a `README.md` | -| `system-results-dir` | §1 | At least one `results//` directory exists | -| `benchmark-model-dir` | §1 | At least one benchmark-model directory per system | -| `point-dirs` | §1 | At least one `r/` Pareto-point directory per model | -| `measurement-points-present` | §1 | Every `r/` carries a `point.yaml` | +| `path-exists` | §8.1 | Submission root directory exists | +| `required-dir` | §8.1 | `results/` and `docs/` present | +| `src-dir` | §2.2.1, §8.1 | `src/` present with at least one implementation directory | +| `src-readme` | §2.2.1, §8.1 | Each `src//` has a `README.md` | +| `system-results-dir` | §8.1 | At least one `results//` directory exists | +| `benchmark-model-dir` | §8.1 | At least one benchmark-model directory per system | +| `point-dirs` | §8.1 | At least one `r/` Pareto-point directory per model | +| `measurement-points-present` | §8.1 | Every `r/` carries a `point.yaml` | | `config-file-present` | §8.1 | Every `r/` carries a `config.yaml` that is a YAML mapping | -| `point-dirname-concurrency` | §1 | `r/` name matches the declared concurrency (warn) | -| `result-summary-present` | §1 | `result_summary.json` exists for each point | +| `point-dirname-concurrency` | §8.1 | `r/` name matches the declared concurrency (warn) | +| `result-summary-present` | §8.1 | `result_summary.json` exists for each point | | `shared-path-resolution` | §9.1 | `shared_src` / `shared_docs` resolve under the submission root | ### System description (§8.2) @@ -324,7 +324,7 @@ submission root (§9.1). | `system-description-consistency` | §8.5 | Every point of a frontier describes the same system (parallelism, `batch` and `config_summary` may vary by point) | | `model-name-valid` | §3.2 | `point.yaml`'s `model_name` is exactly one of the round's supported models, spelled canonically | | `model-name-consistency` | §8.1 | It is exactly the results directory name | -| `max-concurrency-declared` | §7 | `max_supported_concurrency` (C_max) present and > 32 | +| `max-concurrency-declared` | §5.4 | `max_supported_concurrency` (C_max) present and > 32 | | `tps-utilization` | §8.2 | Equals `system_tps / max(system_tps)` over the point's own frontier | | `power-descriptor` | §4.5.2, App. E | `system_power.json` present per system and valid under Appendix E.7; optional for RDI and Serviced | | `power-estimated` | §4.5.2, App. D | "MLC Estimated Power": an Appendix D value reaches the total (warn) | @@ -468,19 +468,19 @@ validation by the service or reviewers. | `region-basis` | §5.4 | Reports whether `C_min` was derived or explicitly overridden | | `c-min-override` | §5.4 | Override is a consistent integer in 1–32 matching a submitted point | | `region-computation` | §5.5 | `(C_max, C_min)` is a valid input to the reference algorithm | -| `concurrency-in-range` | §9.1 | Each concurrency falls in a valid region, margin included | +| `concurrency-in-range` | §5.5 | Each concurrency falls in a valid region, margin included | | `region-declared` | §8.3 | Declared `region` is one of the spec's values | | `region-placement` | §8.3 | Declared region matches the computed one (warn) | | `offline-declared` | §5.7 | `offline` is `dedicated`, `elected`, or `none` | | `offline-point-present` | §5.7 | Exactly one Offline point, `elected` sitting on the C_max point — or **none at all** for an agentic benchmark, which §5.7 says "neither requires nor may include" one | | `offline-ordering` | §5.7.2 | Offline beats C_max on throughput (2% tolerance) and concurrency (warn) | | `ultra-low-concurrency-coverage` | §5.4 | At least one point in 1–32 by default, or 1–C_min with an override | -| `low-concurrency-coverage` | §9.1 | At least one point in the Low Concurrency region | -| `med-concurrency-coverage` | §9.1 | At least one point in the Medium Concurrency region | -| `high-concurrency-coverage` | §9.1 | At least one point in the High Concurrency region | +| `low-concurrency-coverage` | §5.4 | At least one point in the Low Concurrency region | +| `med-concurrency-coverage` | §5.4 | At least one point in the Medium Concurrency region | +| `high-concurrency-coverage` | §5.4 | At least one point in the High Concurrency region | | `point-count` | §5.3 | 7–32 measurement points; 8 with a dedicated Offline run; 7 for an agentic benchmark | | `benchmark-type-consistency` | §6.1, §8.5 | Every point on a frontier declares the same `load_pattern`, since one frontier is one benchmark | -| `point-cap` | §2, §8 | Point count does not exceed 32 | +| `point-cap` | §5.6 | Point count does not exceed 32 | ### Measurement points (§8.3, §6) @@ -501,17 +501,17 @@ validation by the service or reviewers. | `config-consistency-dataset` | §9.1 | All points use the same dataset | | `config-consistency-model` | §9.1 | All points declare the same `model_name` | -### Seed binding (§4.6) +### Seed binding (Submission Rules §4.6) | Rule | Spec | Description | |------|------|-------------| | `seed-set-consistency` | §9.1 | Every point records the same seed set | | `seed-set-membership` | §9.1 | The bound set is one MLCommons published | | `seed-runtime-match` | §2.1.1 | The RNG seeds equal the bound set's values | -| `target-cohort` | §4.6 | `target_cohort` matches `YYYY-MM-C0` / `YYYY-MM-C1` | -| `seed-set-adoption` | §4.6 | `target_cohort` falls inside the set's four-cohort adoption window | -| `seed-config-legacy` | §4.6 | v0.7 fallback: seeds == 42 when no `seed_set` is declared | -| `seed-set-registry` | §4.6 | Warns when the seed-set file itself cannot be read | +| `target-cohort` | Submission Rules §4.6 | `target_cohort` matches `YYYY-MM-C0` / `YYYY-MM-C1` | +| `seed-set-adoption` | Submission Rules §4.6 | `target_cohort` falls inside the set's four-cohort adoption window | +| `seed-config-legacy` | Submission Rules §4.6 | v0.7 fallback: seeds == 42 when no `seed_set` is declared | +| `seed-set-registry` | Submission Rules §4.6 | Warns when the seed-set file itself cannot be read | ### Speculative decoding (§2.9.4) @@ -546,33 +546,33 @@ cohorts in the same registry. The published sets ship as data (`src/submission_checker/data/seed_sets.yaml`), mirrored from the policies repo's `seedset.yaml`. The file's `cohort-id` is the cohort its sets -were published for; §4.6's four-cohort adoption window is derived from it. Point +were published for; Submission Rules §4.6's four-cohort adoption window is derived from it. Point `--seed-sets FILE` or `$MLPERF_ENDPOINTS_SEED_SETS` at a newer file to check against a set published after this release. -### Metrics (§9.1, §14) +### Metrics (§4.1, §9.1) | Rule | Spec | Description | |------|------|-------------| | `result-file-valid` | §8.3 | `result_summary.json` parses against `PointSummary` | -| `metric-consistency-duration` | §14 | `duration_ns` > 0 | -| `metric-consistency-accounting` | §14 | `completed + failed == issued` | -| `metric-consistency-output-tokens` | §14 | `total_output_tokens` ≥ 0 | +| `metric-consistency-duration` | §9.1 | `duration_ns` > 0 | +| `metric-consistency-accounting` | §9.1 | `completed + failed == issued` | +| `metric-consistency-output-tokens` | §9.1 | `total_output_tokens` ≥ 0 | | `metric-consistency-system-tps` | §9.1 | Stored `system_tps` matches the derived value | | `metric-consistency-tpot-p90` | §9.1 | Reported TPOT P90 present, finite, strictly positive; a dedicated Offline run is exempt (§5.7.1, §6.5) | | `metric-consistency-tps-per-user` | §9.1 | Stored `tps_per_user` matches `1000 / tpot_p90_ms`; not applied to a dedicated Offline run | | `metric-consistency-tps-per-kw` | §4.5.3 | Stored `system_tps_per_kw` matches `system_tps / point_power_kw` — see [per-point normalisation](#per-point-normalisation-453) | | `agentic-metric-consistency` | §4.1 | `e2e_avg_interactivity` is derivable from its reported inputs: the per-turn sums, or the client's `output_sequence_lengths.total / (latency.total / 1e9)` when no sample failed (latency is in nanoseconds) | -### Accuracy (§15) +### Accuracy (§4.3, §6.6) | Rule | Spec | Description | |------|------|-------------| -| `accuracy-present` | §15 | At least one model in the submission carries accuracy results | +| `accuracy-present` | §6.6 | At least one model in the submission carries accuracy results | | `accuracy-coverage` | §4.3, §5.3 | Exactly one accuracy result in each of the four mandatory bands, plus the Offline point (N=5; N=4 for an agentic benchmark, which has none). A result at any other point is an error, and one result cannot fill two bands | -| `accuracy-valid` | §15 | `accuracy_results.json` parses correctly | -| `accuracy-sample-count` | §15 | Issued sample count meets the model's minimum | -| `accuracy-gate` | §15 | Score meets the benchmark quality target | +| `accuracy-valid` | §4.3 | `accuracy_results.json` parses correctly | +| `accuracy-sample-count` | §4.3, §6.5 | Issued sample count meets the model's minimum | +| `accuracy-gate` | §4.3 | Score meets the benchmark quality target | | `agentic-accuracy` | §3.2 | The agentic model is one the reference implementation publishes thresholds for, and they are not TBD | | `agentic-accuracy-inline` | §4.3 | Inline accuracy clears the model's floor at **every** point | | `agentic-accuracy-swebench` | §4.3 | The **mean** of the N SWE-bench results clears the model's floor; individual results need not | @@ -614,7 +614,7 @@ using the same assignment for coverage and the multi-turn mean. The exception is §5.7.2's elected `C_max` point, which counts once as its region point and once as the Offline result. The gates below then judge every result present. -Accuracy is gated differently too. §15's gate folds every dataset of a point into one +Accuracy is gated differently too. The single-turn gate folds every dataset of a point into one sample-weighted score per metric; the agentic benchmarks gate three quantities that do not reduce that way, so for a recognised agentic model it stands down in favour of the three rules above: diff --git a/docs/endpoints-submission-cli.md b/docs/endpoints-submission-cli.md index 56791bd0..751c5c2b 100644 --- a/docs/endpoints-submission-cli.md +++ b/docs/endpoints-submission-cli.md @@ -503,7 +503,7 @@ dataset_name: CNN/DailyMail dataset_type: Performance dataset_link: https://example.com/dataset -# §4.6 seed binding — the set is named here and its values used below +# Submission Rules §4.6 seed binding — the set is named here and its values used below seed_set: A target_cohort: 2026-10-C1 @@ -542,7 +542,7 @@ of every run's `src/` folder — does not exist until the builder assembles the no single run archive can name it. A value the run *does* declare is validated and never overwritten; one that does not resolve fails the build. -v1.0 also rotates the RNG seeds (§4.6). v0.7 fixed every seed at 42; a v1.0 point names +v1.0 also rotates the RNG seeds (Submission Rules §4.6). v0.7 fixed every seed at 42; a v1.0 point names a published seed set and seeds `scheduler_rng_seed`, `sample_index_rng_seed` and `model_seed` from it. diff --git a/pyproject.toml b/pyproject.toml index f309f50b..9a8a0596 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,7 +63,7 @@ packages = [ "" = "src" [tool.setuptools.package-data] -# The published seed sets (§4.6) ship as data so a newly published set can be +# The published seed sets (Submission Rules §4.6) ship as data so a newly published set can be # supplied without a checker release; see submission_checker.seed_sets. submission_checker = ["data/*.yaml"] endpoints_submission_cli = ["skills/*/SKILL.md"] diff --git a/src/endpoints_submission_cli/commands/submissions/remove_run.py b/src/endpoints_submission_cli/commands/submissions/remove_run.py index 16323b99..ee975224 100644 --- a/src/endpoints_submission_cli/commands/submissions/remove_run.py +++ b/src/endpoints_submission_cli/commands/submissions/remove_run.py @@ -28,8 +28,8 @@ def _rollback_remove_run(token: str, submission_id: str, run_id: str) -> None: """Restore a run this command just removed, after a later step failed. - This re-adds a run the submission already had, so it is not a §8 post-submission - addition — it is the undo half of an operation that did not complete. + This re-adds a run the submission already had, so it is not a Submission Rules §8 + post-submission addition — it is the undo half of an operation that did not complete. """ _console.print(f"[yellow]Rolling back: re-adding run {run_id} to record…[/yellow]") try: diff --git a/src/submission_checker/checker.py b/src/submission_checker/checker.py index a0fd94da..519544b2 100644 --- a/src/submission_checker/checker.py +++ b/src/submission_checker/checker.py @@ -414,7 +414,7 @@ def run(self) -> Report: # within each / Pareto frontier. report.results.extend(self._check_tps_utilization(results_dir)) - # §15: at least one model must carry accuracy results — either as accuracy_scores + # §6.6: at least one model must carry accuracy results — either as accuracy_scores # embedded in a results.json, or as a standalone accuracy/results.json. has_full_accuracy = any(True for _ in results_dir.rglob("accuracy_results.json")) or any( _results_has_accuracy_scores(p) for p in results_dir.rglob("results.json") diff --git a/src/submission_checker/cli.py b/src/submission_checker/cli.py index 0c171193..3a22a531 100644 --- a/src/submission_checker/cli.py +++ b/src/submission_checker/cli.py @@ -53,8 +53,8 @@ def main() -> None: default=None, envvar=SEED_SETS_ENV_VAR, help=( - "Published seed sets to check against (§4.6). Defaults to the set bundled with " - f"this checker; also settable via ${SEED_SETS_ENV_VAR}." + "Published seed sets to check against (Submission Rules §4.6). Defaults to the set " + f"bundled with this checker; also settable via ${SEED_SETS_ENV_VAR}." ), ) @click.option( diff --git a/src/submission_checker/cohorts.py b/src/submission_checker/cohorts.py index f2a2ef3f..2ea721f3 100644 --- a/src/submission_checker/cohorts.py +++ b/src/submission_checker/cohorts.py @@ -1,13 +1,14 @@ # SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons # SPDX-License-Identifier: Apache-2.0 -"""Publication cohorts (§4.2) — identifiers and the arithmetic §4.6 needs. +"""Publication cohorts — identifiers and arithmetic for Submission Rules §4.2 and §4.6. MLCommons publishes results on a bi-weekly cadence, two cohorts per calendar month, -identified as ``YYYY-MM-C0`` and ``YYYY-MM-C1`` (§4.2). §4.6 then counts in cohorts: -a seed set is adoptable for "its publication cohort and the following three", which -only means something if a cohort has a well-defined successor. That successor lives -here rather than in the seed-set loader, because it is a property of the publication -calendar and the next rule to count in cohorts will want it too. +identified as ``YYYY-MM-C0`` and ``YYYY-MM-C1`` (Submission Rules §4.2). Submission +Rules §4.6 then counts in cohorts: a seed set is adoptable for "its publication cohort +and the following three", which only means something if a cohort has a well-defined +successor. That successor lives here rather than in the seed-set loader, because it is +a property of the publication calendar and the next rule to count in cohorts will want +it too. """ from __future__ import annotations @@ -20,7 +21,7 @@ #: A cohort identifier: calendar year-month plus the index within that month. COHORT_RE = re.compile(r"(?P\d{4})-(?P\d{2})-C(?P[01])") -#: Cohorts published per calendar month (§4.2's 1st and 3rd Wednesday). +#: Cohorts published per calendar month (Submission Rules §4.2's 1st and 3rd Wednesday). _COHORTS_PER_MONTH = 2 @@ -71,12 +72,12 @@ def next(self) -> Cohort: def adoption_window(published: Cohort, length: int = 4) -> tuple[Cohort, ...]: """The cohorts during which a set published in *published* may be adopted. - §4.6: "Each published seed set is available for adoption by new submissions for + Submission Rules §4.6: "Each published seed set is available for adoption by new submissions for **four consecutive cohorts** — its publication cohort and the following three." Args: published: The cohort the set was published for. - length: Window length in cohorts. Defaults to §4.6's four; a parameter so a + length: Window length in cohorts. Defaults to Submission Rules §4.6's four; a parameter so a working-group change is a call-site edit rather than a rewrite. Returns: diff --git a/src/submission_checker/data/messages.yaml b/src/submission_checker/data/messages.yaml index ede426cd..6bf56464 100644 --- a/src/submission_checker/data/messages.yaml +++ b/src/submission_checker/data/messages.yaml @@ -85,7 +85,7 @@ accuracy-coverage: fix: Keep exactly one accuracy result per mandatory band and remove the rest. accuracy-gate: title: Accuracy gate - spec: §15 + spec: §4.3 messages: no-thresholds: text: No accuracy thresholds are defined for {model!r}, so its accuracy was not gated @@ -97,7 +97,7 @@ accuracy-gate: pass: '{matched_key} = {score:.4f} PASSED (target {bound})' accuracy-present: title: Accuracy results present - spec: §15 + spec: §6.6 messages: pass: At least one model has accuracy results fail: @@ -106,7 +106,7 @@ accuracy-present: fix: Include accuracy_results.json, or accuracy_scores in results.json, at the mandatory points. accuracy-sample-count: title: Accuracy sample count - spec: §15 + spec: §4.3, §6.5 messages: fail: '{total} samples{suffix} < required {min_queries}' pass: '{total} samples{suffix} ≥ required {min_queries}' @@ -186,7 +186,7 @@ approved-drafter: identifies an entry by weight checksum, or by target checksum plus configuration' benchmark-model-dir: title: Benchmark-model directory - spec: §1 + spec: §8.1 messages: fail: text: No benchmark-model directories in results/{system_id}/ @@ -202,7 +202,7 @@ benchmark-type-consistency: spec: §6.1, §8.5 concurrency-in-range: title: Concurrency in range - spec: §9 + spec: §5.5 messages: pass: text: 'Dedicated Offline run: concurrency {concurrency} is the dataset cardinality (§5.7.1), exempt @@ -237,7 +237,7 @@ drafter-list-registry: unavailable: '{detail}' high-concurrency-coverage: title: High Concurrency coverage - spec: §§3–6 + spec: §5.4 messages: pass: '{label} region covered: {covered} (range {bounds})' fail: @@ -245,7 +245,7 @@ high-concurrency-coverage: fix: Add a point with concurrency in {bounds}. load-pattern: title: Load pattern - spec: §10 + spec: §6.1 messages: pass: text: 'Offline point: load pattern {lp!r} exempt from fixed concurrency' @@ -255,7 +255,7 @@ load-pattern: pass-2: 'Point {concurrency}: load pattern OK ({lp})' low-concurrency-coverage: title: Low Concurrency coverage - spec: §§3–6 + spec: §5.4 messages: pass: '{label} region covered: {covered} (range {bounds})' fail: @@ -263,7 +263,7 @@ low-concurrency-coverage: fix: Add a point with concurrency in {bounds}. max-concurrency-declared: title: Maximum supported concurrency - spec: §7 + spec: §5.4 messages: pass: max_supported_concurrency = {c_max} maximal-engagement: @@ -289,7 +289,7 @@ maximal-engagement: declared but there is no shortfall' measurement-points-present: title: Point configuration present - spec: §1 + spec: §8.1 messages: fail: text: No usable r/point.yaml files in {relative_to} @@ -300,7 +300,7 @@ measurement-points-present: fix: Add the point's point.yaml (§8.3 disclosure) to this directory. med-concurrency-coverage: title: Medium Concurrency coverage - spec: §§3–6 + spec: §5.4 messages: pass: '{label} region covered: {covered} (range {bounds})' fail: @@ -308,19 +308,19 @@ med-concurrency-coverage: fix: Add a point with concurrency in {bounds}. metric-consistency-accounting: title: Metric consistency accounting - spec: §14 + spec: §9.1 messages: fail: completed ({n_samples_completed}) + failed ({n_samples_failed}) = {accounted} ≠ issued ({n_samples_issued}) pass: 'Sample accounting consistent: {n_samples_issued} issued' metric-consistency-duration: title: Metric consistency duration - spec: §14 + spec: §9.1 messages: fail: 'duration_ns is not positive: {duration_ns}' pass: duration_ns={duration_ns:.0f} ns metric-consistency-output-tokens: title: Metric consistency output tokens - spec: §14 + spec: §9.1 messages: fail: 'total_output_tokens is negative: {total_output_tokens}' pass: total_output_tokens={total_output_tokens} @@ -354,7 +354,7 @@ metric-consistency-tps-per-user: fail: stored tps_per_user {stored:.4f} ≠ derived 1000 / tpot_p90_ms {derived:.4f} (rel err {rel_err:.1%}) min-query-count: title: Min query count - spec: §12 + spec: §6.4 messages: fail: 'Dataset ''{dataset}'': completed {completed} < minimum {min_queries} (§6.4)' pass: 'Dataset ''{dataset}'': completed {completed} ≥ minimum {min_queries}' @@ -429,13 +429,13 @@ offline-point-present: fix: Elect the C_max point (r{c_max}), or run a dedicated Offline point instead. path-exists: title: Submission path - spec: §1 + spec: §8.1 messages: pass: Submission path exists fail: 'Submission path does not exist: {submission_path}' point-cap: title: Point cap - spec: §2, §8 + spec: §5.6 messages: fail: '{n} points exceed the {max_points}-point cap' point-config-valid: @@ -448,7 +448,7 @@ point-config-valid: nonzero point-count: title: Number of points - spec: §2, §8 + spec: §5.3 messages: fail: text: Only {n} measurement point(s) — minimum {minimum} required @@ -456,14 +456,14 @@ point-count: pass: 'Point count OK: {n}' point-dirname-concurrency: title: Point dirname concurrency - spec: §1 + spec: §8.1 messages: warn: text: '{point_dir_name}/: directory concurrency {dir_concurrency} ≠ declared {concurrency}' fix: Rename the directory to r{concurrency}, or correct concurrency in point.yaml. point-dirs: title: Pareto point directories - spec: §1 + spec: §8.1 messages: fail: 'Missing benchmark-model directory: {rel}/' pass: 'Found {dirs_count} Pareto point director{value} in {rel}/: {dirs}' @@ -484,7 +484,7 @@ point-duration: pass: 'Point {c}: {basis} duration {duration_ms:.0f} ms meets minimum for {region}' point-rules-skipped: title: Point rules skipped - spec: §1 + spec: §8.3 messages: warn: '{point_dir_name}/: point.yaml is missing or invalid, so the rules that read it (region placement, metric consistency, power) were not run for this point; fixing it may report more' @@ -594,7 +594,7 @@ region-placement: pass: Declared region '{declared}' matches concurrency {concurrency} required-dir: title: Required directories - spec: §1 + spec: §8.1 messages: pass: 'Found required directory: {name}/' fail: @@ -619,7 +619,7 @@ result-file-valid: percentile-conflict: two keys for percentile {percentile} give different values result-summary-present: title: Result summary present - spec: §1 + spec: §8.1 messages: fail: text: 'Missing result_summary.json for r{concurrency}: {relative_to}' @@ -629,15 +629,15 @@ result-summary-present: fix: Add the point's result_summary.json, as the benchmark writes it. seed-config-legacy: title: Seed config legacy - spec: §4.6 + spec: Submission Rules §4.6 messages: fail: 'Point {concurrency}: {field_name} = {val!r}; v0.7 required 42 and v1.0 requires a declared - seed_set (§4.6) — neither is satisfied' + seed_set (Submission Rules §4.6) — neither is satisfied' warn: 'Point {concurrency}: {field_name} = 42 with no seed_set declared; v1.0 binds seeds to a published - set (§4.6)' + set (Submission Rules §4.6)' seed-runtime-match: title: Seed runtime match - spec: §4.6 + spec: Submission Rules §4.6 messages: fail: 'runtime_settings.runtime does not match seed set {published_id!r}: {mismatched}' pass: RNG seeds match seed set {published_id!r} @@ -645,7 +645,7 @@ seed-runtime-match: v1.0 names them scheduler_rng_seed / sample_index_rng_seed and adds model_seed seed-set-adoption: title: Seed set adoption - spec: §4.6 + spec: Submission Rules §4.6 messages: pass: Seed set {published_id!r} was published for cohort {target} fail: 'Seed set {published_id!r} was not published for cohort {target} (published for: {value})' @@ -655,20 +655,20 @@ seed-set-adoption: adoption window cannot be evaluated (points target {cohorts})' seed-set-consistency: title: Seed set consistency - spec: §4.6 + spec: Submission Rules §4.6 messages: fail: 'Points declare {declared_count} different seed sets: {declared} — a submission binds exactly one' pass: All points bind seed set {next!r} seed-set-membership: title: Seed set membership - spec: §4.6 + spec: Submission Rules §4.6 messages: pass: seed_set {set_id!r} is a published set fail: 'seed_set {set_id!r} is not a published set (known: {value})' seed-set-registry: title: Seed-set registry - spec: §4.6 + spec: Submission Rules §4.6 messages: unavailable: '{detail}' shared-path-resolution: @@ -683,7 +683,7 @@ shared-path-resolution: pass: '{field_name} {value!r} resolves to {relative_to}/ ({where})' src-dir: title: Implementation sources (src/) - spec: §1 + spec: §2.2.1, §8.1 messages: pass: 'src/ present with {impl_dirs_count} implementation director{value}: {impl_dirs}' fail: @@ -695,7 +695,7 @@ src-dir: fix: Add one directory per implementation under src/ (vllm/, sglang/, …). src-readme: title: Implementation README - spec: §1 + spec: §2.2.1, §8.1 messages: pass: src/{impl_dir_name}/README.md present fail: @@ -791,14 +791,14 @@ system-description-valid: fix: Shorten shortened_system_name to {limit} characters or fewer. system-results-dir: title: System results directory - spec: §1 + spec: §8.1 messages: fail: text: No results// directories found fix: Put each system's results under results///r/. target-cohort: title: Target cohort - spec: §4.6 + spec: Submission Rules §4.6 messages: pass: target_cohort='{target_cohort}' fail: target_cohort '{target_cohort}' is not of the form YYYY-MM-C0/C1 diff --git a/src/submission_checker/data/seed_sets.yaml b/src/submission_checker/data/seed_sets.yaml index dc261165..2775eb29 100644 --- a/src/submission_checker/data/seed_sets.yaml +++ b/src/submission_checker/data/seed_sets.yaml @@ -1,9 +1,9 @@ -# MLPerf Endpoints published seed sets (§4.6 Seed Rotation). +# MLPerf Endpoints published seed sets (Submission Rules §4.6 Seed Rotation). # # Mirrored verbatim from `mlcommons/endpoints_policies` `seedset.yaml`, merged to # `v1.0_rules_dev` in PR #117 (`a36d158`, 2026-09-15). # -# `cohort-id` is the cohort the sets below were published for. §4.6 makes each set +# `cohort-id` is the cohort the sets below were published for. Submission Rules §4.6 makes each set # adoptable for that cohort and the three following it, so the adoption window is # derived rather than listed — see `submission_checker.seed_sets`. # diff --git a/src/submission_checker/models/aggregate/context.py b/src/submission_checker/models/aggregate/context.py index e0db145d..ec7cd5a7 100644 --- a/src/submission_checker/models/aggregate/context.py +++ b/src/submission_checker/models/aggregate/context.py @@ -48,7 +48,7 @@ "med_concurrency", "high_concurrency", ) -#: How each mandatory band is named to a submitter (§3–6's region names). +#: How each mandatory band is named to a submitter (§5.4's region names). _BAND_LABELS = { "ultra_low_concurrency": "Ultra Low Concurrency", "low_concurrency": "Low Concurrency", @@ -443,7 +443,7 @@ def _check_ultra_low_concurrency_coverage(self) -> ModelContext: @model_validator(mode="after") def _check_regional_coverage(self) -> ModelContext: - """§3–6: at least one valid point must fall in each of the three concurrency regions. + """§5.4: at least one valid point must fall in each of the three concurrency regions. Attribution goes through :func:`~submission_checker.models.regions.covered_region`, so a point in the @@ -720,7 +720,7 @@ def _dataset_score(self, concurrency: int, dataset: str, rule: str) -> float | N @model_validator(mode="after") def _check_agentic_accuracy(self) -> ModelContext: - """The agentic accuracy gates (§3.2, §4.3), which do not reduce to §15's. + """The agentic accuracy gates (§3.2, §4.3), which do not reduce to the single-turn gate. Three quantities, aggregated three different ways — see :mod:`submission_checker.agentic_targets` for why they cannot share the @@ -967,7 +967,7 @@ def _gate_agentic_osl(self, targets: AgenticTargets) -> None: @model_validator(mode="after") def _check_accuracy(self) -> ModelContext: - """§15: every accuracy metric must meet its quality threshold. + """§4.3: every accuracy metric must meet its quality threshold. Runs over every point that carries results — §5.3 requires N of them, and a failing gate at any one is a failing submission. @@ -988,7 +988,7 @@ def _check_accuracy(self) -> ModelContext: return self def _gate_accuracy(self, accuracy_result: AccuracyResult) -> None: - """Gate one point's accuracy results against the model's thresholds (§15).""" + """Gate one point's accuracy results against the model's thresholds (§4.3).""" json_path = ( (self.accuracy_dir / "results.json") if self.accuracy_dir diff --git a/src/submission_checker/models/aggregate/point_result.py b/src/submission_checker/models/aggregate/point_result.py index 407c21f2..1b63115f 100644 --- a/src/submission_checker/models/aggregate/point_result.py +++ b/src/submission_checker/models/aggregate/point_result.py @@ -139,7 +139,7 @@ def _check_steady_state_basis(self, info: ValidationInfo) -> PointResult: @model_validator(mode="after") def _check_min_query_count(self, info: ValidationInfo) -> PointResult: - """§12: n_samples_completed must meet the dataset's minimum query count (§6.4). + """§6.4: n_samples_completed must meet the dataset's minimum query count. Skipped when the dataset is not in MIN_QUERY_COUNT (unknown datasets are not yet mapped; add them as the spec is ratified). @@ -191,7 +191,7 @@ def _duration_basis(self) -> tuple[float, str]: # ------------------------------------------------------------------ def _check_duration_positive(self, s: PointSummary, path: Path | None) -> None: - """Emit ok/err for the duration_ns > 0 invariant (§14).""" + """Emit ok/err for the duration_ns > 0 invariant (§9.1).""" if s.duration_ns <= 0: self._check_results.append( err("metric-consistency-duration", "fail", path, duration_ns=s.duration_ns) @@ -202,7 +202,7 @@ def _check_duration_positive(self, s: PointSummary, path: Path | None) -> None: ) def _check_sample_accounting(self, s: PointSummary, path: Path | None) -> None: - """Emit ok/err for the completed + failed == issued invariant (§14). + """Emit ok/err for the completed + failed == issued invariant (§9.1). Skipped when n_samples_issued is 0 (point tool did not track issued count). """ @@ -231,7 +231,7 @@ def _check_sample_accounting(self, s: PointSummary, path: Path | None) -> None: ) def _check_output_tokens_nonnegative(self, s: PointSummary, path: Path | None) -> None: - """Emit ok/err for the total_output_tokens >= 0 invariant (§14).""" + """Emit ok/err for the total_output_tokens >= 0 invariant (§9.1).""" if s.total_output_tokens < 0: self._check_results.append( err( @@ -381,7 +381,7 @@ def _check_agentic_metrics(self, s: PointSummary, path: Path | None) -> None: @model_validator(mode="after") def _check_metric_consistency(self, info: ValidationInfo) -> PointResult: - """§14 + §9.1: validate point-log accounting invariants and tps derivability.""" + """§9.1: validate point-log accounting invariants and tps derivability.""" summary_path: Path | None = (info.context or {}).get("summary_path") s = self.summary self._check_duration_positive(s, summary_path) diff --git a/src/submission_checker/models/aggregate/seed_binding.py b/src/submission_checker/models/aggregate/seed_binding.py index decaafd6..e29b0d18 100644 --- a/src/submission_checker/models/aggregate/seed_binding.py +++ b/src/submission_checker/models/aggregate/seed_binding.py @@ -1,4 +1,4 @@ -"""Seed binding — §4.6 seed rotation, checked across one Pareto frontier. +"""Seed binding — Submission Rules §4.6 seed rotation, checked across one Pareto frontier. §9.1's "Seed-set validity" row makes this a property of the whole submission, not of one file: "every point must record the same seed set, and that set must have been @@ -119,7 +119,9 @@ def _check_seed_runtime_match(self) -> SeedBinding: @model_validator(mode="after") def _check_seed_set_adoption(self) -> SeedBinding: - """§4.6: the bound set must be published for ``target_cohort`` or the three before it. + """The bound set must be published for ``target_cohort`` or the three before it. + + Submission Rules §4.6. Reported as INFO while the published registry carries no cohort keys: with no cohort attached to any set, every submission would either pass vacuously or diff --git a/src/submission_checker/models/file/point_config.py b/src/submission_checker/models/file/point_config.py index 119a08b4..63491957 100644 --- a/src/submission_checker/models/file/point_config.py +++ b/src/submission_checker/models/file/point_config.py @@ -33,7 +33,7 @@ # algorithm and §9.1's check names both use *_concurrency. The algorithm wins. _VALID_REGIONS = frozenset({*REGION_NAMES, SUBMITTERS_CHOICE}) -#: Cohort identifier (§4.6): calendar year-month plus the cohort index within it. +#: Cohort identifier (Submission Rules §4.6): calendar year-month plus the cohort index within it. _COHORT_RE = re.compile(r"\d{4}-\d{2}-C[01]") #: §8.3's `offline` vocabulary. ``dedicated`` — this point is its own Offline run; @@ -153,7 +153,7 @@ class RuntimeSettings(BaseModel): stream_all_chunks: bool = True class Runtime(BaseModel): - """RNG seed configuration, bound to a published seed set (§4.6). + """RNG seed configuration, bound to a published seed set (Submission Rules §4.6). v1.0 renamed every seed and stopped fixing them at 42: `seedset.yaml` names them ``scheduler_rng_seed``, ``sample_index_rng_seed`` and ``model_seed``. The @@ -251,7 +251,7 @@ class PointConfig(BaseModel): dataset_link: Link to the data used for the submission. shared_src: Root-relative path to the implementation directory under ``src/``. shared_docs: Root-relative path to the shared ``docs/`` directory. - seed_set: Identifier of the bound seed set (§4.6). + seed_set: Identifier of the bound seed set (Submission Rules §4.6). target_cohort: Cohort this submission targets, ``YYYY-MM-C0`` or ``…-C1``. nodes_used: §4.5.3's per-point node declaration, by node type. Absent means the point is normalised by the system's full provisioned power. @@ -476,7 +476,7 @@ def _check_disclosure_complete(self, info: ValidationInfo) -> PointConfig: @model_validator(mode="after") def _check_target_cohort(self, info: ValidationInfo) -> PointConfig: - """§4.6: ``target_cohort`` names a cohort as ``YYYY-MM-C0`` or ``YYYY-MM-C1``.""" + """Submission Rules §4.6: ``target_cohort`` is ``YYYY-MM-C0`` or ``YYYY-MM-C1``.""" path: Path | None = (info.context or {}).get("yaml_path") if self.target_cohort is None: return self # absence is reported by point-disclosure-complete @@ -494,9 +494,9 @@ def _check_target_cohort(self, info: ValidationInfo) -> PointConfig: def _check_seeds_legacy(self, info: ValidationInfo) -> PointConfig: """v0.7 fallback: seeds must equal 42 when the point binds no seed set. - v0.7 fixed every seed at 42. v1.0 rotates them (§4.6), so the constant is now - wrong for any point that names a ``seed_set`` — those are checked against the - published registry by + v0.7 fixed every seed at 42. v1.0 rotates them (Submission Rules §4.6), so the + constant is now wrong for any point that names a ``seed_set`` — those are checked + against the published registry by :class:`~submission_checker.models.aggregate.SeedBinding` instead. This rule survives only to give a v0.7 bundle a comprehensible message. """ diff --git a/src/submission_checker/seed_sets.py b/src/submission_checker/seed_sets.py index e4be526e..7d336732 100644 --- a/src/submission_checker/seed_sets.py +++ b/src/submission_checker/seed_sets.py @@ -1,4 +1,4 @@ -"""Published seed sets (§4.6 Seed Rotation) and the registry that loads them. +"""Published seed sets (Submission Rules §4.6 Seed Rotation) and the registry that loads them. §2.1.1 requires the client's request-issue / sample-order RNG and the per-query salt to be seeded from a set MLCommons published for the submission's cohort, and §9.1's @@ -52,7 +52,7 @@ class SeedSet: sample_index_rng_seed: Seed for the sample-order RNG. model_seed: Seed passed through to the model / per-query salt. cohorts: The cohorts during which a *new* submission may adopt this set — - §4.6's four-cohort adoption window, derived from the registry's + Submission Rules §4.6's four-cohort adoption window, derived from the registry's publication cohort. Empty only when the file declares no ``cohort-id``, in which case the adoption test cannot run. """ @@ -100,7 +100,7 @@ def load_seed_sets(path: Path | None = None) -> dict[str, SeedSet]: entries, published = _unwrap(chosen, raw) - # §4.6 states the window in terms of the publication cohort, so the registry + # Submission Rules §4.6 states the window in terms of the publication cohort, so the registry # carries one `cohort-id` rather than listing the window per set. cohorts: tuple[str, ...] = () if published is not None: @@ -142,7 +142,7 @@ def _unwrap(path: Path, raw: object) -> tuple[list[object], str | None]: """Return ``(seed-set entries, publication cohort)`` from either registry shape. Upstream nests everything under a ``cohort:`` mapping that also carries the - ``cohort-id`` §4.6 counts from. The older flat ``seed_sets:`` form is still + ``cohort-id`` Submission Rules §4.6 counts from. The older flat ``seed_sets:`` form is still accepted so an operator can point ``--seed-sets`` at a file predating that change; such a file simply declares no cohort, and the adoption test stands down. """ diff --git a/tests/endpoints_submission_cli/commands/test_check_submission_command.py b/tests/endpoints_submission_cli/commands/test_check_submission_command.py index cb9ce7cf..f375630f 100644 --- a/tests/endpoints_submission_cli/commands/test_check_submission_command.py +++ b/tests/endpoints_submission_cli/commands/test_check_submission_command.py @@ -82,7 +82,7 @@ def test_annotate_emits_github_commands_on_stderr(self) -> None: assert result.exit_code == 1 assert "::error " in result.stderr # Titled with the rule's title and spec section, not its id twice. - assert "title=Low Concurrency coverage (§§3–6)::" in result.stderr + assert "title=Low Concurrency coverage (§5.4)::" in result.stderr assert "[low-concurrency-coverage]" not in result.stderr def test_no_annotate_by_default_outside_ci(self) -> None: diff --git a/tests/endpoints_submission_cli/commands/test_submissions_commands.py b/tests/endpoints_submission_cli/commands/test_submissions_commands.py index 36bfc0e1..dec418db 100644 --- a/tests/endpoints_submission_cli/commands/test_submissions_commands.py +++ b/tests/endpoints_submission_cli/commands/test_submissions_commands.py @@ -197,8 +197,9 @@ class TestSubmissionsUpdate: def test_update_run_ids(self, tmp_path: Path) -> None: """--run-ids triggers full rebuild pipeline; update_submission called with new run list.""" - # Removing _OLD_RUN_ID and keeping RUN_ID: §8 permits withdrawing a point, - # not adding one, so the desired list must be a subset of the current one. + # Removing _OLD_RUN_ID and keeping RUN_ID: Submission Rules §8.1 permits + # withdrawing a point, not adding one, so the desired list must be a subset of + # the current one. current_sub = { **SUBMISSION_OUT, "run_ids": [self._OLD_RUN_ID, RUN_ID], @@ -255,8 +256,9 @@ def test_update_run_ids(self, tmp_path: Path) -> None: def test_update_run_ids_merge_failure_rolls_back(self, tmp_path: Path) -> None: """A rebuild failure rolls back the DB PATCH with the original run IDs.""" - # Removing _OLD_RUN_ID and keeping RUN_ID: §8 permits withdrawing a point, - # not adding one, so the desired list must be a subset of the current one. + # Removing _OLD_RUN_ID and keeping RUN_ID: Submission Rules §8.1 permits + # withdrawing a point, not adding one, so the desired list must be a subset of + # the current one. current_sub = { **SUBMISSION_OUT, "run_ids": [self._OLD_RUN_ID, RUN_ID], @@ -344,7 +346,7 @@ def test_update_run_ids_download_failure_rolls_back(self, tmp_path: Path) -> Non mock_patch.assert_any_call(TOKEN, SUBMISSION_ID, {"run_ids": [self._OLD_RUN_ID, RUN_ID]}) def test_update_run_ids_rejects_additions(self) -> None: - """§8 no longer provides a post-submission window for adding points. + """Submission Rules §8 no longer provide a post-submission window for adding points. The guard fires before anything is fetched or rebuilt, so a mistaken --run-ids list cannot leave the submission half-updated. diff --git a/tests/endpoints_submission_cli/conftest.py b/tests/endpoints_submission_cli/conftest.py index 804580b3..4870561f 100644 --- a/tests/endpoints_submission_cli/conftest.py +++ b/tests/endpoints_submission_cli/conftest.py @@ -209,7 +209,7 @@ # a config carries runtime knobs, while point.yaml carries the disclosure the checker # validates (seeds, warmup counts, data source). Validated against the checker's # PointConfig, so the fixture represents a bundle that actually passes. -#: Seed set A (§4.6), mirrored from policies PR #117 into data/seed_sets.yaml. +#: Seed set A (Submission Rules §4.6), mirrored from policies PR #117 into data/seed_sets.yaml. _SEED_SET_ID = "A" _SEEDS = { "scheduler_rng_seed": 10487924139932647040, diff --git a/tests/endpoints_submission_cli/submissions/test_builder.py b/tests/endpoints_submission_cli/submissions/test_builder.py index cda2c140..5ea1127f 100644 --- a/tests/endpoints_submission_cli/submissions/test_builder.py +++ b/tests/endpoints_submission_cli/submissions/test_builder.py @@ -1269,7 +1269,7 @@ def test_built_points_parse_through_checker_pointconfig( data = yaml.safe_load(py.read_text()) # Must not raise — this is exactly what failed before the runtime fix. cfg = PointConfig.model_validate(data, context={"yaml_path": py}) - # §4.6 binds the seeds to a published set; the fixture uses set A. + # Submission Rules §4.6 binds the seeds to a published set; the fixture uses set A. assert cfg.seed_set == "A" assert cfg.runtime_settings.runtime.scheduler_rng_seed == 10487924139932647040 assert cfg.runtime_settings.runtime.sample_index_rng_seed == 586478644936801402 diff --git a/tests/submission_checker/test_checks_agentic_accuracy.py b/tests/submission_checker/test_checks_agentic_accuracy.py index 702dc274..ab6db5b8 100644 --- a/tests/submission_checker/test_checks_agentic_accuracy.py +++ b/tests/submission_checker/test_checks_agentic_accuracy.py @@ -148,7 +148,7 @@ def test_tbd_thresholds_warn(self, tmp_path: Path, monkeypatch) -> None: assert not _hits(ctx, "agentic-accuracy-inline") def test_single_turn_gate_stands_down(self, tmp_path: Path) -> None: - """§15's gate would fold inline and SWE-bench into one weighted mean.""" + """The single-turn gate would fold inline and SWE-bench into one weighted mean.""" ctx = _ctx(tmp_path, accuracy_by_point={64: _accuracy(inline=60.0, swebench=95.0)}) assert not _hits(ctx, "accuracy-gate") diff --git a/tests/submission_checker/test_checks_seed_binding.py b/tests/submission_checker/test_checks_seed_binding.py index 8d217992..edd3ae6d 100644 --- a/tests/submission_checker/test_checks_seed_binding.py +++ b/tests/submission_checker/test_checks_seed_binding.py @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons # SPDX-License-Identifier: Apache-2.0 -"""Tests for SeedBinding — §4.6 seed rotation, checked across a Pareto frontier.""" +"""Tests for SeedBinding — Submission Rules §4.6 seed rotation, checked across a Pareto frontier.""" from __future__ import annotations @@ -136,7 +136,7 @@ def test_cohort_outside_the_window_errors(self, tmp_path: Path) -> None: assert _errors(binding, "seed-set-adoption") def test_last_cohort_of_the_window_is_accepted(self, tmp_path: Path) -> None: - """The window is inclusive of its fourth cohort (§4.6).""" + """The window is inclusive of its fourth cohort (Submission Rules §4.6).""" points = [_point(tmp_path, target_cohort="2026-12-C0")] binding = SeedBinding(points=points, registry=load_seed_sets(), model_dir=tmp_path) assert not _errors(binding, "seed-set-adoption") @@ -147,7 +147,7 @@ def test_one_cohort_past_the_window_errors(self, tmp_path: Path) -> None: assert _errors(binding, "seed-set-adoption") def test_skipped_only_when_the_registry_declares_no_cohort(self, tmp_path: Path) -> None: - """A pre-cohort registry file leaves §4.6 unevaluable rather than failing.""" + """A pre-cohort registry file leaves Submission Rules §4.6 unevaluable, not failing.""" cohortless = {"A": SeedSet(id="A", **_SET_A.seeds, cohorts=())} binding = SeedBinding(points=[_point(tmp_path)], registry=cohortless, model_dir=tmp_path) results = [r for r in binding._check_results if r.rule == "seed-set-adoption"] diff --git a/tests/submission_checker/test_cohorts.py b/tests/submission_checker/test_cohorts.py index ed219000..8baed950 100644 --- a/tests/submission_checker/test_cohorts.py +++ b/tests/submission_checker/test_cohorts.py @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons # SPDX-License-Identifier: Apache-2.0 -"""Tests for publication-cohort identifiers and arithmetic (§4.2, §4.6).""" +"""Tests for publication-cohort identifiers and arithmetic (Submission Rules §4.2, §4.6).""" from __future__ import annotations @@ -60,7 +60,7 @@ def test_succession_is_strictly_increasing(self) -> None: c = c.next() def test_two_cohorts_per_calendar_month(self) -> None: - """§4.2's bi-weekly cadence: a year advances in exactly 24 steps.""" + """Submission Rules §4.2's bi-weekly cadence: a year advances in exactly 24 steps.""" c = Cohort(2026, 1, 0) for _ in range(24): c = c.next() @@ -69,7 +69,7 @@ def test_two_cohorts_per_calendar_month(self) -> None: class TestAdoptionWindow: def test_four_consecutive_cohorts_by_default(self) -> None: - """§4.6: the publication cohort and the following three.""" + """Submission Rules §4.6: the publication cohort and the following three.""" window = adoption_window(Cohort(2026, 10, 1)) assert [str(c) for c in window] == [ "2026-10-C1", @@ -102,7 +102,7 @@ def test_window_is_contiguous(self) -> None: assert earlier.next() == later def test_two_sets_published_two_cohorts_apart_overlap_by_two(self) -> None: - """§4.6: refresh every two cohorts, adoptable for four — so two sets overlap.""" + """Submission Rules §4.6: refresh every two cohorts, adoptable for four, so two overlap.""" first = set(adoption_window(Cohort(2026, 10, 1))) second = set(adoption_window(Cohort(2026, 11, 1))) assert len(first & second) == 2 diff --git a/tests/submission_checker/test_messages.py b/tests/submission_checker/test_messages.py index d3ac9f26..df6ef2c3 100644 --- a/tests/submission_checker/test_messages.py +++ b/tests/submission_checker/test_messages.py @@ -17,6 +17,7 @@ import ast import json +import re from dataclasses import dataclass from pathlib import Path @@ -176,13 +177,32 @@ def test_every_rule_has_a_title_and_spec(self) -> None: if rule in _SECTIONS: continue assert entry.title and entry.title != rule, rule - assert entry.spec.startswith("§"), f"{rule}: spec {entry.spec!r}" + assert entry.spec.startswith(("§", "Submission Rules §")), ( + f"{rule}: spec {entry.spec!r}" + ) def test_specs_use_one_notation(self) -> None: for rule, entry in _CATALOG.items(): for key, message in entry.messages.items(): assert "#" not in message.spec, f"{rule}.{key}: {message.spec!r}" + def test_specs_name_sections_that_exist(self) -> None: + """Each spec cites a section of the Rules (§1–§9) or Submission Rules (§1–§11). + + A bare § is the Rules; the Submission Rules are named. The early checker + numbered its rules by their row in the §9.1 table (§10 load pattern, §15 + accuracy, ...), which no reader could look up. + """ + top_level = {"": 9, "Submission Rules ": 11} + for rule, entry in _CATALOG.items(): + for key, message in entry.messages.items(): + for ref in filter(None, message.spec.split(", ")): + match = re.fullmatch(r"(Submission Rules )?§(\d+)(\.\d+)*", ref) + assert match, f"{rule}.{key}: {ref!r} is not a § citation" + assert 1 <= int(match[2]) <= top_level[match[1] or ""], ( + f"{rule}.{key}: {ref!r} names no section" + ) + @pytest.mark.parametrize( ("rule", "key"), [(rule, key) for rule, entry in _CATALOG.items() for key in entry.messages], diff --git a/tests/submission_checker/test_seed_sets.py b/tests/submission_checker/test_seed_sets.py index e66e8de7..49c2258a 100644 --- a/tests/submission_checker/test_seed_sets.py +++ b/tests/submission_checker/test_seed_sets.py @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons # SPDX-License-Identifier: Apache-2.0 -"""Tests for the published seed-set registry (§4.6 Seed Rotation).""" +"""Tests for the published seed-set registry (Submission Rules §4.6 Seed Rotation).""" from __future__ import annotations @@ -41,7 +41,7 @@ def test_seeds_are_not_the_v07_constant(self) -> None: assert all(value != 42 for value in load_seed_sets()["A"].seeds.values()) def test_adoption_window_is_derived_from_the_publication_cohort(self) -> None: - """§4.6: adoptable for the publication cohort and the three following it.""" + """Submission Rules §4.6: adoptable for the publication cohort and the next three.""" assert load_seed_sets()["A"].cohorts == ( "2026-10-C1", "2026-11-C0", @@ -160,7 +160,7 @@ def test_explicit_path_beats_the_environment( assert list(load_seed_sets(arg_path)) == ["A"] def test_cohorts_are_read_when_present(self, tmp_path: Path) -> None: - """Forward-compatibility: the loader already reads the key §4.6 needs.""" + """Forward-compatibility: the loader already reads the key Submission Rules §4.6 needs.""" path = _write_registry( tmp_path / "seeds.yaml", [ diff --git a/tests/tools/regenerate_fixtures.py b/tests/tools/regenerate_fixtures.py index ccf30301..d4e7e29d 100644 --- a/tests/tools/regenerate_fixtures.py +++ b/tests/tools/regenerate_fixtures.py @@ -21,8 +21,8 @@ with ``system_tps`` derived from ``result_summary.json``, so the stored value agrees with the measurement the checker recomputes it from. 4. **§8.3 disclosure added to every ``point.yaml``** — the ten §8.3 fields plus - ``shared_src``, ``shared_docs``, ``seed_set`` and ``target_cohort``, and the §4.6 RNG - seeds under their v1.0 names. + ``shared_src``, ``shared_docs``, ``seed_set`` and ``target_cohort``, and the + Submission Rules §4.6 RNG seeds under their v1.0 names. 5. **``result_summary.json`` gains ``tpot``** (migrated from ``run_metadata``'s ``measured_latency_tpot_*``, in nanoseconds), a TTFT P90, and ``system_tps`` / ``tps_per_user`` consistent with the v1.0 formulas. @@ -79,7 +79,7 @@ #: The seed set every fixture binds, and the cohort it targets. #: -#: §4.6 makes a set adoptable only for its publication cohort and the three +#: Submission Rules §4.6 makes a set adoptable only for its publication cohort and the three #: following, so this must sit inside the window derived from the registry's #: `cohort-id` — see submission_checker.cohorts.adoption_window. SEED_SET_ID = "A" @@ -638,7 +638,7 @@ def _write_point_yaml( *, dry_run: bool, ) -> list[str]: - """Add the §8.3 disclosure and §4.6 seeds to a point's ``point.yaml``.""" + """Add the §8.3 disclosure and Submission Rules §4.6 seeds to a point's ``point.yaml``.""" path = point_dir / layout.POINT_YAML current = yaml.safe_load(path.read_text()) if path.exists() else None point: dict[str, Any] = dict(current) if isinstance(current, dict) else {} From 1c98f1337d483808d46705663338a24ebafcbd7e Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 27 Oct 2026 18:25:26 +0530 Subject: [PATCH 2/2] Correct three remaps after checking them against the policy text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Region coverage: the requirement is in §5.3; §5.4 defines the regions. Cite both. - src/ checks: §2.2.1 does not mention src/ or its README; §8.1 does. - duration, output-token and sample-accounting checks: the §9.1 "Metric consistency" row now covers only TPOT P90 and tps_per_user. Duration and output tokens are §4.1's system_tps inputs; the issued and completed counts are §6.4's. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 16 ++++++++-------- src/submission_checker/data/messages.yaml | 16 ++++++++-------- .../models/aggregate/context.py | 2 +- .../models/aggregate/point_result.py | 8 ++++---- .../commands/test_check_submission_command.py | 2 +- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index b0c30637..6921eef5 100644 --- a/README.md +++ b/README.md @@ -304,8 +304,8 @@ submission root (§9.1). |------|------|-------------| | `path-exists` | §8.1 | Submission root directory exists | | `required-dir` | §8.1 | `results/` and `docs/` present | -| `src-dir` | §2.2.1, §8.1 | `src/` present with at least one implementation directory | -| `src-readme` | §2.2.1, §8.1 | Each `src//` has a `README.md` | +| `src-dir` | §8.1 | `src/` present with at least one implementation directory | +| `src-readme` | §8.1 | Each `src//` has a `README.md` | | `system-results-dir` | §8.1 | At least one `results//` directory exists | | `benchmark-model-dir` | §8.1 | At least one benchmark-model directory per system | | `point-dirs` | §8.1 | At least one `r/` Pareto-point directory per model | @@ -475,9 +475,9 @@ validation by the service or reviewers. | `offline-point-present` | §5.7 | Exactly one Offline point, `elected` sitting on the C_max point — or **none at all** for an agentic benchmark, which §5.7 says "neither requires nor may include" one | | `offline-ordering` | §5.7.2 | Offline beats C_max on throughput (2% tolerance) and concurrency (warn) | | `ultra-low-concurrency-coverage` | §5.4 | At least one point in 1–32 by default, or 1–C_min with an override | -| `low-concurrency-coverage` | §5.4 | At least one point in the Low Concurrency region | -| `med-concurrency-coverage` | §5.4 | At least one point in the Medium Concurrency region | -| `high-concurrency-coverage` | §5.4 | At least one point in the High Concurrency region | +| `low-concurrency-coverage` | §5.3, §5.4 | At least one point in the Low Concurrency region | +| `med-concurrency-coverage` | §5.3, §5.4 | At least one point in the Medium Concurrency region | +| `high-concurrency-coverage` | §5.3, §5.4 | At least one point in the High Concurrency region | | `point-count` | §5.3 | 7–32 measurement points; 8 with a dedicated Offline run; 7 for an agentic benchmark | | `benchmark-type-consistency` | §6.1, §8.5 | Every point on a frontier declares the same `load_pattern`, since one frontier is one benchmark | | `point-cap` | §5.6 | Point count does not exceed 32 | @@ -555,9 +555,9 @@ set published after this release. | Rule | Spec | Description | |------|------|-------------| | `result-file-valid` | §8.3 | `result_summary.json` parses against `PointSummary` | -| `metric-consistency-duration` | §9.1 | `duration_ns` > 0 | -| `metric-consistency-accounting` | §9.1 | `completed + failed == issued` | -| `metric-consistency-output-tokens` | §9.1 | `total_output_tokens` ≥ 0 | +| `metric-consistency-duration` | §4.1 | `duration_ns` > 0 | +| `metric-consistency-accounting` | §6.4 | `completed + failed == issued` | +| `metric-consistency-output-tokens` | §4.1 | `total_output_tokens` ≥ 0 | | `metric-consistency-system-tps` | §9.1 | Stored `system_tps` matches the derived value | | `metric-consistency-tpot-p90` | §9.1 | Reported TPOT P90 present, finite, strictly positive; a dedicated Offline run is exempt (§5.7.1, §6.5) | | `metric-consistency-tps-per-user` | §9.1 | Stored `tps_per_user` matches `1000 / tpot_p90_ms`; not applied to a dedicated Offline run | diff --git a/src/submission_checker/data/messages.yaml b/src/submission_checker/data/messages.yaml index 6bf56464..cc424d1d 100644 --- a/src/submission_checker/data/messages.yaml +++ b/src/submission_checker/data/messages.yaml @@ -237,7 +237,7 @@ drafter-list-registry: unavailable: '{detail}' high-concurrency-coverage: title: High Concurrency coverage - spec: §5.4 + spec: §5.3, §5.4 messages: pass: '{label} region covered: {covered} (range {bounds})' fail: @@ -255,7 +255,7 @@ load-pattern: pass-2: 'Point {concurrency}: load pattern OK ({lp})' low-concurrency-coverage: title: Low Concurrency coverage - spec: §5.4 + spec: §5.3, §5.4 messages: pass: '{label} region covered: {covered} (range {bounds})' fail: @@ -300,7 +300,7 @@ measurement-points-present: fix: Add the point's point.yaml (§8.3 disclosure) to this directory. med-concurrency-coverage: title: Medium Concurrency coverage - spec: §5.4 + spec: §5.3, §5.4 messages: pass: '{label} region covered: {covered} (range {bounds})' fail: @@ -308,19 +308,19 @@ med-concurrency-coverage: fix: Add a point with concurrency in {bounds}. metric-consistency-accounting: title: Metric consistency accounting - spec: §9.1 + spec: §6.4 messages: fail: completed ({n_samples_completed}) + failed ({n_samples_failed}) = {accounted} ≠ issued ({n_samples_issued}) pass: 'Sample accounting consistent: {n_samples_issued} issued' metric-consistency-duration: title: Metric consistency duration - spec: §9.1 + spec: §4.1 messages: fail: 'duration_ns is not positive: {duration_ns}' pass: duration_ns={duration_ns:.0f} ns metric-consistency-output-tokens: title: Metric consistency output tokens - spec: §9.1 + spec: §4.1 messages: fail: 'total_output_tokens is negative: {total_output_tokens}' pass: total_output_tokens={total_output_tokens} @@ -683,7 +683,7 @@ shared-path-resolution: pass: '{field_name} {value!r} resolves to {relative_to}/ ({where})' src-dir: title: Implementation sources (src/) - spec: §2.2.1, §8.1 + spec: §8.1 messages: pass: 'src/ present with {impl_dirs_count} implementation director{value}: {impl_dirs}' fail: @@ -695,7 +695,7 @@ src-dir: fix: Add one directory per implementation under src/ (vllm/, sglang/, …). src-readme: title: Implementation README - spec: §2.2.1, §8.1 + spec: §8.1 messages: pass: src/{impl_dir_name}/README.md present fail: diff --git a/src/submission_checker/models/aggregate/context.py b/src/submission_checker/models/aggregate/context.py index ec7cd5a7..9aeba638 100644 --- a/src/submission_checker/models/aggregate/context.py +++ b/src/submission_checker/models/aggregate/context.py @@ -443,7 +443,7 @@ def _check_ultra_low_concurrency_coverage(self) -> ModelContext: @model_validator(mode="after") def _check_regional_coverage(self) -> ModelContext: - """§5.4: at least one valid point must fall in each of the three concurrency regions. + """§5.3: at least one valid point must fall in each of the three concurrency regions. Attribution goes through :func:`~submission_checker.models.regions.covered_region`, so a point in the diff --git a/src/submission_checker/models/aggregate/point_result.py b/src/submission_checker/models/aggregate/point_result.py index 1b63115f..c8143f9a 100644 --- a/src/submission_checker/models/aggregate/point_result.py +++ b/src/submission_checker/models/aggregate/point_result.py @@ -191,7 +191,7 @@ def _duration_basis(self) -> tuple[float, str]: # ------------------------------------------------------------------ def _check_duration_positive(self, s: PointSummary, path: Path | None) -> None: - """Emit ok/err for the duration_ns > 0 invariant (§9.1).""" + """Emit ok/err for the duration_ns > 0 invariant (§4.1).""" if s.duration_ns <= 0: self._check_results.append( err("metric-consistency-duration", "fail", path, duration_ns=s.duration_ns) @@ -202,7 +202,7 @@ def _check_duration_positive(self, s: PointSummary, path: Path | None) -> None: ) def _check_sample_accounting(self, s: PointSummary, path: Path | None) -> None: - """Emit ok/err for the completed + failed == issued invariant (§9.1). + """Emit ok/err for the completed + failed == issued invariant (§6.4). Skipped when n_samples_issued is 0 (point tool did not track issued count). """ @@ -231,7 +231,7 @@ def _check_sample_accounting(self, s: PointSummary, path: Path | None) -> None: ) def _check_output_tokens_nonnegative(self, s: PointSummary, path: Path | None) -> None: - """Emit ok/err for the total_output_tokens >= 0 invariant (§9.1).""" + """Emit ok/err for the total_output_tokens >= 0 invariant (§4.1).""" if s.total_output_tokens < 0: self._check_results.append( err( @@ -381,7 +381,7 @@ def _check_agentic_metrics(self, s: PointSummary, path: Path | None) -> None: @model_validator(mode="after") def _check_metric_consistency(self, info: ValidationInfo) -> PointResult: - """§9.1: validate point-log accounting invariants and tps derivability.""" + """§4.1, §6.4, §9.1: validate point-log accounting invariants and tps derivability.""" summary_path: Path | None = (info.context or {}).get("summary_path") s = self.summary self._check_duration_positive(s, summary_path) diff --git a/tests/endpoints_submission_cli/commands/test_check_submission_command.py b/tests/endpoints_submission_cli/commands/test_check_submission_command.py index f375630f..ef5f46b3 100644 --- a/tests/endpoints_submission_cli/commands/test_check_submission_command.py +++ b/tests/endpoints_submission_cli/commands/test_check_submission_command.py @@ -82,7 +82,7 @@ def test_annotate_emits_github_commands_on_stderr(self) -> None: assert result.exit_code == 1 assert "::error " in result.stderr # Titled with the rule's title and spec section, not its id twice. - assert "title=Low Concurrency coverage (§5.4)::" in result.stderr + assert "title=Low Concurrency coverage (§5.3%2C §5.4)::" in result.stderr assert "[low-concurrency-coverage]" not in result.stderr def test_no_annotate_by_default_outside_ci(self) -> None: