(5/n) Package m-ArenaHard task families with language variants

judgearena/artifacts/metadata.py

@@ -232,7 +232,7 @@ def _task_definition_metadata(run: dict[str, Any]) -> dict[str, Any] | None:
     resolved = get_packaged_task(task_id)
     if resolved is None:
         return None
-    return {
+    metadata = {
         "schema_version": resolved.spec.schema_version,
         "task_version": resolved.spec.task_version,
         "resolved_sha256": resolved.provenance.resolved_sha256,
@@ -241,6 +241,13 @@ def _task_definition_metadata(run: dict[str, Any]) -> dict[str, Any] | None:
             for resource in resolved.provenance.resources
         ],
     }
+    if resolved.selection is not None:
+        metadata["selection"] = {
+            "selector": resolved.selection.selector,
+            "name": resolved.selection.name,
+            "values": list(resolved.selection.values),
+        }
+    return metadata
 
 
 def write_run_metadata(

judgearena/benchmarks/pairwise/baselines.py

@@ -4,16 +4,11 @@
 
 from collections.abc import Mapping
 
-from judgearena.datasets.m_arenahard import (
-    M_ARENA_HARD_BASELINES,
-    split_m_arena_hard_dataset,
-)
 from judgearena.datasets.mt_bench import MT_BENCH_BASELINES
 from judgearena.tasks.registry import get_packaged_task
 from judgearena.tasks.schema import CategoryDefaultsBaseline, TaskDefaultBaseline
 
 LEGACY_PAIRWISE_BASELINES: dict[str, str | Mapping[str, str]] = {
-    **M_ARENA_HARD_BASELINES,
     **MT_BENCH_BASELINES,
 }
 
@@ -31,8 +26,4 @@ def native_pairwise_baseline(task: str) -> str | Mapping[str, str] | None:
 
     if task in LEGACY_PAIRWISE_BASELINES:
         return LEGACY_PAIRWISE_BASELINES[task]
-    parsed_m_arena_hard = split_m_arena_hard_dataset(task)
-    if parsed_m_arena_hard is not None:
-        version_key, _lang_or_subset = parsed_m_arena_hard
-        return LEGACY_PAIRWISE_BASELINES[version_key]
     return None

judgearena/dataset_revisions.py

@@ -22,9 +22,6 @@
     "lmarena-ai/arena-human-preference-55k": "18c298340948c0e7f7727399fd459cca6ce0ca6f",
     # ComparIA (already pinned via the legacy comparia_revision argument).
     "ministere-culture/comparia-votes": "7a40bce496c1f2aa3be4001da85a49cb4743042b",
-    # m-ArenaHard (Cohere release)
-    "CohereLabs/m-ArenaHard": "ab393a96cd0b134a1acfa96e080af31e5e73a393",
-    "CohereLabs/m-ArenaHard-v2.0": "24c65eff42cec85e30dd5db99d1a702c7ebaa8ab",
     # MT-Bench questions (LMSYS Space).
     "lmsys/mt-bench": "a4b674ca573c24143824ac7f60d9173e7081e37d",
     # Multilingual fluency contexts (generated by scripts/fluency/generate_fluency.py).

judgearena/datasets/__init__.py

@@ -1,9 +1,5 @@
 import pandas as pd
 
-from judgearena.datasets.m_arenahard import (
-    load_m_arenahard,
-    split_m_arena_hard_dataset,
-)
 from judgearena.log import get_logger
 from judgearena.tasks.registry import get_packaged_task
 
@@ -26,31 +22,6 @@ def load_instructions(dataset: str, n_instructions: int | None = None) -> pd.Dat
 
         df_instructions = load_mt_bench()
 
-    elif (parsed := split_m_arena_hard_dataset(dataset)) is not None:
-        from judgearena import utils as judgearena_utils
-
-        version_key, lang_or_subset = parsed
-        logger.info(
-            "Loading %s with language specification set to %s",
-            version_key,
-            lang_or_subset,
-        )
-        df_instructions = load_m_arenahard(
-            local_path=judgearena_utils.data_root,
-            version=version_key,
-            language=lang_or_subset,
-        )
-        # sort by question_id, then language so that we get multiple languages if we truncate
-        df_instructions.sort_values(["question_id", "lang"], inplace=True)
-        df_instructions.rename(
-            {
-                "question_id": "instruction_index",
-                "prompt": "instruction",
-            },
-            axis=1,
-            inplace=True,
-        )
-
     else:
         raise ValueError(f"Unsupported instruction dataset {dataset!r}.")