diff --git a/every_eval_ever/converters/helm/adapter.py b/every_eval_ever/converters/helm/adapter.py index b667628bd..e01016f17 100644 --- a/every_eval_ever/converters/helm/adapter.py +++ b/every_eval_ever/converters/helm/adapter.py @@ -23,7 +23,6 @@ get_model_deployment, ) from helm.benchmark.run_spec import RunSpec - from helm.common.codec import from_json except ( Exception ) as ex: # pragma: no cover - exercised only when optional deps missing @@ -38,7 +37,6 @@ RunSpec = cast(Any, None) get_model_deployment = cast(Any, None) register_builtin_configs_from_helm_package = cast(Any, None) - from_json = cast(Any, None) ModelDeploymentNotFoundError = cast(Any, Exception) from every_eval_ever.converters import SCHEMA_VERSION @@ -224,7 +222,14 @@ def _load_evaluation_run_logfiles(self, dir_path) -> Dict: stats = self._load_file_if_exists(dir_path, self.STATS_FILE) with open(f'{dir_path}/{self.PER_INSTANCE_STATS_FILE}', 'r') as f: - per_instance_stats = from_json(f.read(), List[PerInstanceStats]) + per_instance_stats = [ + from_dict( + data_class=PerInstanceStats, + data=entry, + config=DaciteConfig(cast=[str]), + ) + for entry in json.load(f) + ] return { 'per_instance_stats': per_instance_stats, diff --git a/pyproject.toml b/pyproject.toml index e252f670f..6022c5d98 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,16 +31,14 @@ inspect = ["inspect-ai>=0.3.160,<0.4.0"] helm = [ "crfm-helm>=0.5.14", "typer>=0.12,<1.0", - # crfm-helm pulls nltk transitively. nltk 3.10.1 added an import guard + # crfm-helm pulls nltk transitively. nltk 3.10.1 alone ships an import guard # (nltk/inisec.py, a CWE-427 mitigation) that blocks nltk-initiated imports # of any module resolving *under the current working directory*. uv places # .venv/ inside the project, so site-packages is under the CWD and the guard - # false-positives on nltk's own `regex` dependency -> `import helm` fails at - # import time (breaks the HELM converter in CI's `loose` matrix, and for any - # default venv layout). Cap below the guarded release until nltk fixes the - # false positive (tracking: https://github.com/nltk/nltk/issues/3730); safe - # because crfm-helm is frozen. - "nltk<3.10.1", + # false-positives on nltk's own `regex` dependency -> `import nltk` fails at + # import time (tracking: https://github.com/nltk/nltk/issues/3730). 3.10.2 + # dropped it; its pathsec guard covers corpus paths, not imports. + "nltk!=3.10.1", ] all = [ "every-eval-ever[inspect]", diff --git a/tests/test_helm_generation_args.py b/tests/test_helm_generation_args.py index 666a51249..2c9fff341 100644 --- a/tests/test_helm_generation_args.py +++ b/tests/test_helm_generation_args.py @@ -4,18 +4,47 @@ not silently replaced by adapter defaults. """ +import importlib.util +from types import SimpleNamespace + import pytest -pytest.importorskip( - 'helm', reason='crfm-helm not installed; install with: uv sync --extra helm' +import every_eval_ever.converters.helm.adapter as helm_adapter_module +from every_eval_ever.converters.helm.adapter import HELMAdapter + +# `import helm` alone is not enough: on Python 3.14 the top-level package imports +# but `helm.common.codec` does not, so the converter's own import guard is the +# only reliable signal. Same condition as tests/test_helm_adapter.py. Scoped to +# the class rather than the whole module so the sentinel below stays live even +# when the guard has fired. +_requires_helm = pytest.mark.skipif( + helm_adapter_module._HELM_IMPORT_ERROR is not None, + reason=( + 'HELM converter dependencies are missing: ' + f'{helm_adapter_module._HELM_IMPORT_ERROR!r}. ' + 'Install with: uv sync --extra helm' + ), ) -from types import SimpleNamespace -from every_eval_ever.converters.helm.adapter import HELMAdapter +def test_helm_extra_is_importable_when_installed(): + """If HELM is installed at all, the converter's guarded imports must work. + + The class-scoped skip above hides a broken guarded import (e.g. a HELM + release whose `helm.common.codec` stops importing) whenever + ``_HELM_IMPORT_ERROR`` is set, so a full ``uv sync --all-extras`` CI row + would report green with HELM silently skipped. This sentinel skips only + when the top-level `helm` package is genuinely absent and otherwise fails, + turning an installed-but-broken extra into a hard failure. + """ + if importlib.util.find_spec('helm') is None: + pytest.skip('HELM is not installed; the extra is optional for core.') + helm_adapter_module._require_helm_dependencies() -def _make_request_state(temperature=None, max_tokens=None, top_p=None, top_k=None): +def _make_request_state( + temperature=None, max_tokens=None, top_p=None, top_k=None +): """Build a minimal mock RequestState with the given request-level values.""" request = SimpleNamespace( temperature=temperature, @@ -29,7 +58,9 @@ def _make_request_state(temperature=None, max_tokens=None, top_p=None, top_k=Non ) -def _make_adapter_spec(temperature=None, max_tokens=None, top_p=None, top_k=None): +def _make_adapter_spec( + temperature=None, max_tokens=None, top_p=None, top_k=None +): """Build a minimal mock AdapterSpec with the given fallback values.""" return SimpleNamespace( temperature=temperature, @@ -39,6 +70,7 @@ def _make_adapter_spec(temperature=None, max_tokens=None, top_p=None, top_k=None ) +@_requires_helm class TestExtractGenerationArgsFalsyValues: """Verify that 0 is treated as a real value, not as missing.""" diff --git a/uv.lock b/uv.lock index 4065c6532..e755c1ab8 100644 --- a/uv.lock +++ b/uv.lock @@ -895,7 +895,7 @@ requires-dist = [ { name = "inspect-ai", marker = "extra == 'inspect'", specifier = ">=0.3.160,<0.4.0" }, { name = "jsonschema", specifier = ">=4.26.0,<5.0.0" }, { name = "matplotlib", specifier = ">=3.10.8" }, - { name = "nltk", marker = "extra == 'helm'", specifier = "<3.10.1" }, + { name = "nltk", marker = "extra == 'helm'", specifier = "!=3.10.1" }, { name = "numpy", specifier = ">=2.4.1" }, { name = "pandas", specifier = ">=2.3.3" }, { name = "pydantic", specifier = ">=2.12.5,<3.0.0" },