helm: drop the codec import and relax the nltk pin, fixing the extra on Python 3.14

every_eval_ever/converters/helm/adapter.py

@@ -23,7 +23,6 @@
         get_model_deployment,
     )
     from helm.benchmark.run_spec import RunSpec
-    from helm.common.codec import from_json
 except (
     Exception
 ) as ex:  # pragma: no cover - exercised only when optional deps missing
@@ -38,7 +37,6 @@
     RunSpec = cast(Any, None)
     get_model_deployment = cast(Any, None)
     register_builtin_configs_from_helm_package = cast(Any, None)
-    from_json = cast(Any, None)
     ModelDeploymentNotFoundError = cast(Any, Exception)
 
 from every_eval_ever.converters import SCHEMA_VERSION
@@ -224,7 +222,14 @@ def _load_evaluation_run_logfiles(self, dir_path) -> Dict:
         stats = self._load_file_if_exists(dir_path, self.STATS_FILE)
 
         with open(f'{dir_path}/{self.PER_INSTANCE_STATS_FILE}', 'r') as f:
-            per_instance_stats = from_json(f.read(), List[PerInstanceStats])
+            per_instance_stats = [
+                from_dict(
+                    data_class=PerInstanceStats,
+                    data=entry,
+                    config=DaciteConfig(cast=[str]),
+                )
+                for entry in json.load(f)
+            ]
 
         return {
             'per_instance_stats': per_instance_stats,

pyproject.toml

@@ -31,16 +31,14 @@ inspect = ["inspect-ai>=0.3.160,<0.4.0"]
 helm = [
     "crfm-helm>=0.5.14",
     "typer>=0.12,<1.0",
-    # crfm-helm pulls nltk transitively. nltk 3.10.1 added an import guard
+    # crfm-helm pulls nltk transitively. nltk 3.10.1 alone ships an import guard
     # (nltk/inisec.py, a CWE-427 mitigation) that blocks nltk-initiated imports
     # of any module resolving *under the current working directory*. uv places
     # .venv/ inside the project, so site-packages is under the CWD and the guard
-    # false-positives on nltk's own `regex` dependency -> `import helm` fails at
-    # import time (breaks the HELM converter in CI's `loose` matrix, and for any
-    # default venv layout). Cap below the guarded release until nltk fixes the
-    # false positive (tracking: https://github.com/nltk/nltk/issues/3730); safe
-    # because crfm-helm is frozen.
-    "nltk<3.10.1",
+    # false-positives on nltk's own `regex` dependency -> `import nltk` fails at
+    # import time (tracking: https://github.com/nltk/nltk/issues/3730). 3.10.2
+    # dropped it; its pathsec guard covers corpus paths, not imports.
+    "nltk!=3.10.1",
 ]
 all = [
     "every-eval-ever[inspect]",

tests/test_helm_generation_args.py

@@ -4,18 +4,47 @@
 not silently replaced by adapter defaults.
 """
 
+import importlib.util
+from types import SimpleNamespace
+
 import pytest
 
-pytest.importorskip(
-    'helm', reason='crfm-helm not installed; install with: uv sync --extra helm'
+import every_eval_ever.converters.helm.adapter as helm_adapter_module
+from every_eval_ever.converters.helm.adapter import HELMAdapter
+
+# `import helm` alone is not enough: on Python 3.14 the top-level package imports
+# but `helm.common.codec` does not, so the converter's own import guard is the
+# only reliable signal. Same condition as tests/test_helm_adapter.py. Scoped to
+# the class rather than the whole module so the sentinel below stays live even
+# when the guard has fired.
+_requires_helm = pytest.mark.skipif(
+    helm_adapter_module._HELM_IMPORT_ERROR is not None,
+    reason=(
+        'HELM converter dependencies are missing: '
+        f'{helm_adapter_module._HELM_IMPORT_ERROR!r}. '
+        'Install with: uv sync --extra helm'
+    ),
 )
 
-from types import SimpleNamespace
 
-from every_eval_ever.converters.helm.adapter import HELMAdapter
+def test_helm_extra_is_importable_when_installed():
+    """If HELM is installed at all, the converter's guarded imports must work.
+
+    The class-scoped skip above hides a broken guarded import (e.g. a HELM
+    release whose `helm.common.codec` stops importing) whenever
+    ``_HELM_IMPORT_ERROR`` is set, so a full ``uv sync --all-extras`` CI row
+    would report green with HELM silently skipped. This sentinel skips only
+    when the top-level `helm` package is genuinely absent and otherwise fails,
+    turning an installed-but-broken extra into a hard failure.
+    """
+    if importlib.util.find_spec('helm') is None:
+        pytest.skip('HELM is not installed; the extra is optional for core.')
+    helm_adapter_module._require_helm_dependencies()
 
 
-def _make_request_state(temperature=None, max_tokens=None, top_p=None, top_k=None):
+def _make_request_state(
+    temperature=None, max_tokens=None, top_p=None, top_k=None
+):
     """Build a minimal mock RequestState with the given request-level values."""
     request = SimpleNamespace(
         temperature=temperature,
@@ -29,7 +58,9 @@ def _make_request_state(temperature=None, max_tokens=None, top_p=None, top_k=Non
     )
 
 
-def _make_adapter_spec(temperature=None, max_tokens=None, top_p=None, top_k=None):
+def _make_adapter_spec(
+    temperature=None, max_tokens=None, top_p=None, top_k=None
+):
     """Build a minimal mock AdapterSpec with the given fallback values."""
     return SimpleNamespace(
         temperature=temperature,
@@ -39,6 +70,7 @@ def _make_adapter_spec(temperature=None, max_tokens=None, top_p=None, top_k=None
     )
 
 
+@_requires_helm
 class TestExtractGenerationArgsFalsyValues:
     """Verify that 0 is treated as a real value, not as missing."""