diff --git a/every_eval_ever/converters/README.md b/every_eval_ever/converters/README.md index b9733fdd6..41b94a561 100644 --- a/every_eval_ever/converters/README.md +++ b/every_eval_ever/converters/README.md @@ -46,7 +46,18 @@ The exact command for converting an example evaluation log is: uv run --extra inspect every_eval_ever convert inspect --log_path tests/data/inspect/2026-02-07T11-26-57+00-00_gaia_4V8zHbbRKpU5Yv2BMoBcjE.json ``` -Optional: pass `--supplemental_eval_details path/to/supplemental_eval_details.json` to enrich converted output. `additional_details` maps are extend-only (existing keys are preserved), while synthetic `metric_config` defaults can be overridden for these fields: `evaluation_description`, `lower_is_better`, `score_type`, `level_names`, `level_metadata`, `has_unknown_level`, `min_score`, `max_score`. Use top-level fields (`model_info`, `source_data`, `generation_config`, `agentic_eval_config`) for shared details and `evaluation_results` for per-result metric/score details keyed by `evaluation_name`. +Optional: pass `--supplemental_eval_details path/to/supplemental_eval_details.json` to enrich converted output. `additional_details` maps are extend-only (existing keys are preserved), while synthetic `metric_config` defaults can be overridden for these fields: `evaluation_description`, `lower_is_better`, `score_type`, `level_names`, `level_metadata`, `has_unknown_level`, `min_score`, `max_score`. Use top-level fields (`model_info`, `source_data`, `generation_config`, `agentic_eval_config`) for shared details and `evaluation_results` for per-result metric/score details. + +An `evaluation_results` entry selects which results it applies to by either key: + +| Key | Selects | +|---|---| +| `evaluation_result_id` | one result — `":"`, e.g. `choice:accuracy` | +| `evaluation_name` | every result of that evaluation, e.g. `inspect_evals/pubmedqa` | + +`evaluation_result_id` wins where both match. An entry with neither key applies +positionally. Convert once without a supplement to see the available keys — a key +that selects nothing is logged as a warning rather than applied. Example `supplemental_eval_details.json`: @@ -74,7 +85,7 @@ Example `supplemental_eval_details.json`: }, "evaluation_results": [ { - "evaluation_name": "inspect_evals/pubmedqa - choice", + "evaluation_result_id": "choice:accuracy", "score_details": { "details": { "notes": [ @@ -100,7 +111,7 @@ Example `supplemental_eval_details.json`: Use it with: ```bash -uv run python -m eval_converters.inspect \ +uv run --extra inspect every_eval_ever convert inspect \ --log_path tests/data/inspect/data_pubmedqa_gpt4o_mini.json \ --supplemental_eval_details path/to/supplemental_eval_details.json ``` diff --git a/every_eval_ever/converters/helm/adapter.py b/every_eval_ever/converters/helm/adapter.py index b667628bd..fbd231241 100644 --- a/every_eval_ever/converters/helm/adapter.py +++ b/every_eval_ever/converters/helm/adapter.py @@ -52,6 +52,7 @@ HELMInstanceLevelDataAdapter, _evaluation_result_id, _score_from_stat, + _stat_name_part, ) from every_eval_ever.converters.helm.metrics import is_core_metric from every_eval_ever.converters.helm.utils import extract_reasoning @@ -513,6 +514,7 @@ def _transform_single( metric_config = MetricConfig( evaluation_description=metric_name, + metric_name=metric_name, lower_is_better=False, # TODO schema.json check score_type=ScoreType.continuous, min_score=0, @@ -521,19 +523,12 @@ def _transform_single( split = getattr(stat.name, 'split', None) perturbation = getattr(stat.name, 'perturbation', None) - name_parts = [metric_name] - if split: - name_parts.append(str(split)) - if perturbation: - name_parts.append(str(perturbation)) - evaluation_name = ( - f'{" ".join(name_parts)} on {source_data.dataset_name}' - ) + perturbation_label = _stat_name_part(perturbation) evaluation_results.append( EvaluationResult( evaluation_result_id=evaluation_result_id, - evaluation_name=evaluation_name, + evaluation_name=source_data.dataset_name, source_data=source_data, evaluation_timestamp=evaluation_timestamp, metric_config=metric_config, @@ -553,10 +548,8 @@ def _transform_single( ), details={ 'count': str(getattr(stat, 'count', '')), - 'split': str(split) if split else '', - 'perturbation': str(perturbation) - if perturbation - else '', + 'split': _stat_name_part(split) or '', + 'perturbation': perturbation_label or '', }, ), generation_config=GenerationConfig( diff --git a/every_eval_ever/converters/inspect/adapter.py b/every_eval_ever/converters/inspect/adapter.py index 6033b3110..b5cf171ce 100644 --- a/every_eval_ever/converters/inspect/adapter.py +++ b/every_eval_ever/converters/inspect/adapter.py @@ -56,6 +56,7 @@ def _require_inspect_dependencies() -> None: ) from every_eval_ever.converters.inspect.instance_level_adapter import ( InspectInstanceLevelDataAdapter, + evaluation_result_id, ) from every_eval_ever.converters.inspect.utils import ( apply_supplemental_eval_details, @@ -147,11 +148,15 @@ def _build_evaluation_result( num_samples: int = 0, ) -> EvaluationResult: return EvaluationResult( - evaluation_name=f'{metric_info.name} on {evaluation_task_name} for scorer {scorer_name}', + evaluation_result_id=evaluation_result_id( + scorer_name, metric_info.name + ), + evaluation_name=evaluation_task_name, source_data=source_data, evaluation_timestamp=evaluation_timestamp, metric_config=MetricConfig( - evaluation_description=metric_info.name, + evaluation_description=f'{metric_info.name} from scorer {scorer_name}', + metric_name=metric_info.name, lower_is_better=False, # no metadata available score_type=ScoreType.continuous, min_score=0, @@ -175,8 +180,15 @@ def _extract_evaluation_results( generation_config: GenerationConfig, num_samples: int, timestamp: str, - ) -> List[EvaluationResult]: + ) -> Tuple[List[EvaluationResult], Dict[str, List[str]]]: + """Convert Inspect's per-scorer metrics into aggregate results. + + Returns the results plus a scorer name -> `evaluation_result_id` map, + which the instance-level converter needs to emit one row per aggregate + result a sample contributed to. + """ results: List[EvaluationResult] = [] + result_ids_by_scorer: Dict[str, List[str]] = {} for scorer in scores: llm_grader = None @@ -220,22 +232,26 @@ def _extract_evaluation_results( scorer_name = scorer.name or scorer.scorer - results.append( - self._build_evaluation_result( - evaluation_task_name=evaluation_task_name, - scorer_name=scorer_name, - metric_info=metric_info, - llm_grader=llm_grader, - source_data=source_data, - evaluation_timestamp=timestamp, - generation_config=generation_config, - stderr_value=stderr_value, - stddev_value=stddev_value, - num_samples=num_samples, - ) + result = self._build_evaluation_result( + evaluation_task_name=evaluation_task_name, + scorer_name=scorer_name, + metric_info=metric_info, + llm_grader=llm_grader, + source_data=source_data, + evaluation_timestamp=timestamp, + generation_config=generation_config, + stderr_value=stderr_value, + stddev_value=stddev_value, + num_samples=num_samples, ) + results.append(result) + + if scorer_name and result.evaluation_result_id: + result_ids_by_scorer.setdefault(scorer_name, []).append( + result.evaluation_result_id + ) - return results + return results, result_ids_by_scorer # A HuggingFace repo identifier: exactly `namespace/name` with no # extra path segments, schemes, or path-unsafe prefixes. We use an @@ -634,7 +650,7 @@ def _transform_single( evaluation_task_name = eval_spec.task_display_name or eval_spec.task - evaluation_results = ( + evaluation_results, result_ids_by_scorer = ( self._extract_evaluation_results( evaluation_task_name, results.scores if results else [], @@ -644,7 +660,7 @@ def _transform_single( evaluation_unix_timestamp, ) if results and results.scores - else [] + else ([], {}) ) supplemental_eval_details = parse_supplemental_eval_details( @@ -690,6 +706,7 @@ def _transform_single( model_info.id, raw_eval_log.samples, getattr(raw_eval_log, 'reductions', None), + result_ids_by_scorer, ) ) diff --git a/every_eval_ever/converters/inspect/instance_level_adapter.py b/every_eval_ever/converters/inspect/instance_level_adapter.py index 6854817f9..a3f0a6505 100644 --- a/every_eval_ever/converters/inspect/instance_level_adapter.py +++ b/every_eval_ever/converters/inspect/instance_level_adapter.py @@ -45,6 +45,21 @@ def _require_inspect_dependencies() -> None: ) +def evaluation_result_id( + scorer_name: str | None, metric_name: str | None +) -> str | None: + """Build the join key shared by aggregate results and instance rows. + + Inspect reports one set of metrics per scorer, so a metric name alone is + not unique within a task: two scorers can each report ``accuracy``. + """ + if not metric_name: + return None + if not scorer_name: + return metric_name + return f'{scorer_name}:{metric_name}' + + class InspectInstanceLevelDataAdapter: def __init__( self, @@ -271,12 +286,35 @@ def _resolve_evaluation_score( Tuple[str, str], Tuple[float, bool] ], reductions_by_sample: Dict[str, List[Tuple[float, bool]]], + scorer_name: str | None = None, ) -> Tuple[float, bool]: + """Resolve one row's score, preferring the scorer that owns the row. + + `scorer_name` is the scorer whose aggregate result this row joins to; + with it, a sample scored by several scorers reports each scorer's own + value instead of whichever one matched first. + """ sample_id = self._normalize_sample_id(getattr(sample, 'id', None)) + if scorer_name is not None: + matched = reductions_by_sample_and_scorer.get( + (sample_id, self._normalize_sample_id(scorer_name)) + ) + if matched is not None: + score, _ = matched + return score, False + + own_score = (sample.scores or {}).get(scorer_name) + if own_score is not None: + parsed_score, _ = self._parse_score_value( + getattr(own_score, 'value', None) + ) + if parsed_score is not None: + return parsed_score, False + if sample.scores: - for scorer_name in sample.scores.keys(): - scorer_key = self._normalize_sample_id(scorer_name) + for candidate_name in sample.scores.keys(): + scorer_key = self._normalize_sample_id(candidate_name) matched = reductions_by_sample_and_scorer.get( (sample_id, scorer_key) ) @@ -301,17 +339,84 @@ def _resolve_evaluation_score( fallback_score = 1.0 if response_in_reference else 0.0 return fallback_score, True + def _scorer_emissions( + self, + sample: EvalSample, + result_ids_by_scorer: Dict[str, List[str]], + ) -> List[Tuple[str | None, str | None]]: + """The (evaluation_result_id, scorer_name) rows one sample owes. + + The instance schema asks for one record per aggregate result a sample + contributed to, so a sample graded by one scorer reporting three + metrics produces three rows. When no aggregate result can be + attributed, a single row carries no `evaluation_result_id`. + """ + emissions: List[Tuple[str | None, str | None]] = [] + for scorer_name in sample.scores or {}: + for result_id in result_ids_by_scorer.get(scorer_name, []): + emissions.append((result_id, scorer_name)) + + return emissions or [(None, None)] + + def _response_from_output( + self, sample: EvalSample + ) -> Tuple[str, str | None]: + """The model's own response text and reasoning trace for a sample.""" + if not sample.output.choices: + # Samples with no model output (e.g. sandbox failures in + # agentic evals) have `output.choices == []`. Treat this + # as an empty response rather than crashing at choices[0]. + return '', None + + content = sample.output.choices[0].message.content + if isinstance(content, list): + return self._parse_content_with_reasoning(content) + + return content, None + + def _response_for_scorer( + self, + sample: EvalSample, + scorer_name: str | None, + model_response: str, + ) -> str: + """The answer text a row's own scorer graded, else the model output. + + Inspect scorers may restate the answer they graded (`answer`) or + explain it (`explanation`), which is closer to what the score refers + to than the raw model output. + """ + if not sample.scores: + return model_response + + if scorer_name is None: + scores = list(sample.scores.values()) + else: + own_score = sample.scores.get(scorer_name) + scores = [own_score] if own_score is not None else [] + + response = model_response + for score in scores: + if score.answer: + response = score.answer + elif score.explanation: + response = score.explanation + + return response + def convert_instance_level_logs( self, evaluation_name: str, model_id: str, samples: List[EvalSample], reductions: List[EvalSampleReductions] | None = None, + result_ids_by_scorer: Dict[str, List[str]] | None = None, ) -> Tuple[str, int]: instance_level_logs: List[InstanceLevelEvaluationLog] = [] reductions_by_sample_and_scorer, reductions_by_sample = ( self._build_reduction_lookups(reductions) ) + result_ids_by_scorer = result_ids_by_scorer or {} for sample in samples: sample_input = Input( @@ -325,31 +430,7 @@ def convert_instance_level_logs( formatted=None, ) - reasoning_trace = None - if sample.output.choices: - message = sample.output.choices[0].message - content = message.content - - if isinstance(content, list): - ( - response, - reasoning_trace, - ) = self._parse_content_with_reasoning(content) - else: - response = content - else: - # Samples with no model output (e.g. sandbox failures in - # agentic evals) have `output.choices == []`. Treat this - # as an empty response rather than crashing at choices[0]. - response = '' - - if sample.scores: - # TODO Consider multiple scores - for scorer_name, score in sample.scores.items(): - if score.answer: - response = score.answer - elif score.explanation: - response = score.explanation + model_response, reasoning_trace = self._response_from_output(sample) processed_messages = [ self._handle_chat_messages(msg_idx, msg) @@ -371,49 +452,10 @@ def convert_instance_level_logs( interaction_type = InteractionType.single_turn if interaction_type == InteractionType.single_turn: - sample_output = Output( - raw=[response] - if isinstance(response, str) - else list(response), - reasoning_trace=[reasoning_trace] - if isinstance(reasoning_trace, str) - else reasoning_trace, - ) messages = None else: - sample_output = None messages = processed_messages - response_in_reference = response in sample_input.reference - ( - evaluation_score, - is_fallback_score, - ) = self._resolve_evaluation_score( - sample, - response_in_reference, - reductions_by_sample_and_scorer, - reductions_by_sample, - ) - is_correct = ( - response_in_reference - if is_fallback_score - else evaluation_score > 0 - ) - - evaluation = Evaluation( - score=evaluation_score, - is_correct=is_correct, - num_turns=len(messages) if messages else 1, - tool_calls_count=sum( - len(msg.tool_calls) if msg.tool_calls else 0 - for msg in messages - ) - if messages - else 0, - ) - - answer_attribution: List[AnswerAttributionItem] = [] - token_usage = self._get_token_usage(sample.output.usage) if sample.total_time and sample.working_time: @@ -426,38 +468,89 @@ def convert_instance_level_logs( else: performance = None - instance_level_log = InstanceLevelEvaluationLog( - schema_version=SCHEMA_VERSION, - evaluation_id=self.evaluation_id, - model_id=model_id, - evaluation_name=evaluation_name, - sample_id=str(sample.id), - sample_hash=sha256_string( - sample_input.raw + ''.join(sample_input.reference) - ), - interaction_type=interaction_type, - input=sample_input, - output=sample_output, - messages=messages, - answer_attribution=answer_attribution, - evaluation=evaluation, - token_usage=token_usage, - performance=performance, - error=f'{sample.error.message}\n{sample.error.traceback}' - if sample.error - else None, - metadata={ - # `stop_reason` is documented as reflecting the first - # choice; guard against empty `choices` so it is only - # surfaced when a choice actually exists. - 'stop_reason': str(sample.output.stop_reason) - if sample.output.choices and sample.output.stop_reason - else '', - 'epoch': str(sample.epoch), - }, - ) + for result_id, scorer_name in self._scorer_emissions( + sample, result_ids_by_scorer + ): + response = self._response_for_scorer( + sample, scorer_name, model_response + ) + + if messages is None: + sample_output = Output( + raw=[response] + if isinstance(response, str) + else list(response), + reasoning_trace=[reasoning_trace] + if isinstance(reasoning_trace, str) + else reasoning_trace, + ) + else: + sample_output = None + + response_in_reference = response in sample_input.reference + ( + evaluation_score, + is_fallback_score, + ) = self._resolve_evaluation_score( + sample, + response_in_reference, + reductions_by_sample_and_scorer, + reductions_by_sample, + scorer_name, + ) + is_correct = ( + response_in_reference + if is_fallback_score + else evaluation_score > 0 + ) + + evaluation = Evaluation( + score=evaluation_score, + is_correct=is_correct, + num_turns=len(messages) if messages else 1, + tool_calls_count=sum( + len(msg.tool_calls) if msg.tool_calls else 0 + for msg in messages + ) + if messages + else 0, + ) + + answer_attribution: List[AnswerAttributionItem] = [] + + instance_level_log = InstanceLevelEvaluationLog( + schema_version=SCHEMA_VERSION, + evaluation_id=self.evaluation_id, + model_id=model_id, + evaluation_name=evaluation_name, + evaluation_result_id=result_id, + sample_id=str(sample.id), + sample_hash=sha256_string( + sample_input.raw + ''.join(sample_input.reference) + ), + interaction_type=interaction_type, + input=sample_input, + output=sample_output, + messages=messages, + answer_attribution=answer_attribution, + evaluation=evaluation, + token_usage=token_usage, + performance=performance, + error=f'{sample.error.message}\n{sample.error.traceback}' + if sample.error + else None, + metadata={ + # `stop_reason` is documented as reflecting the first + # choice; guard against empty `choices` so it is only + # surfaced when a choice actually exists. + 'stop_reason': str(sample.output.stop_reason) + if sample.output.choices and sample.output.stop_reason + else '', + 'epoch': str(sample.epoch), + }, + ) - instance_level_logs.append(instance_level_log) + instance_level_logs.append(instance_level_log) self._save_json(instance_level_logs) diff --git a/every_eval_ever/converters/inspect/supplemental_eval_details.py b/every_eval_ever/converters/inspect/supplemental_eval_details.py index d777f8ef8..05883e3a4 100644 --- a/every_eval_ever/converters/inspect/supplemental_eval_details.py +++ b/every_eval_ever/converters/inspect/supplemental_eval_details.py @@ -37,6 +37,7 @@ class SupplementalMetricConfig(_StrictSupplementalModel): additional_details: dict[str, Any] | None = None class SupplementalForEvaluationResults(_StrictSupplementalModel): + evaluation_result_id: str | None = None evaluation_name: str | None = None metric_config: SupplementalMetricConfig | None = None score_details: SupplementalScoreDetails | None = None diff --git a/every_eval_ever/converters/inspect/utils.py b/every_eval_ever/converters/inspect/utils.py index 6b5b46fc6..3f28009a5 100644 --- a/every_eval_ever/converters/inspect/utils.py +++ b/every_eval_ever/converters/inspect/utils.py @@ -1,4 +1,5 @@ import json +import logging import re from typing import Any, Dict, List, Type @@ -19,6 +20,8 @@ ModelInfo, ) +logger = logging.getLogger(__name__) + class ModelPathHandler: """Base class for all model path parsing strategies.""" @@ -486,6 +489,68 @@ def apply_result_supplement( apply_metric_config_supplement(evaluation_result, supplement) +def _key_supplements( + result_supplements: list[SupplementalForEvaluationResults], + key_field: str, +) -> dict[str, SupplementalForEvaluationResults]: + """Index per-result supplements by one key field, rejecting duplicates.""" + keyed = { + getattr(supplement, key_field): supplement + for supplement in result_supplements + if getattr(supplement, key_field) is not None + } + provided = [ + supplement + for supplement in result_supplements + if getattr(supplement, key_field) is not None + ] + if len(keyed) != len(provided): + raise ValueError( + f'Duplicate {key_field} values in ' + 'supplemental_eval_details.evaluation_results.' + ) + return keyed + + +def _warn_unmatched_supplements( + by_result_id: dict[str, SupplementalForEvaluationResults], + by_evaluation_name: dict[str, SupplementalForEvaluationResults], + matched_keys: set[tuple[str, str]], + evaluation_results: list[EvaluationResult], +) -> None: + """Report supplement keys that selected no result. + + A supplemental file is hand-written, so a typo would otherwise be applied + to nothing and reported as a successful conversion. One file may cover a + directory of logs, so an unmatched key is a warning rather than an error. + """ + unmatched = [ + f'{key_field}={key!r}' + for key_field, keys in ( + ('evaluation_result_id', by_result_id), + ('evaluation_name', by_evaluation_name), + ) + for key in keys + if (key_field, key) not in matched_keys + ] + if not unmatched: + return + + logger.warning( + 'supplemental_eval_details entries matched no evaluation result: %s. ' + 'Available evaluation_result_id values: %s; evaluation_name values: %s.', + ', '.join(unmatched), + sorted( + { + result.evaluation_result_id + for result in evaluation_results + if result.evaluation_result_id + } + ), + sorted({result.evaluation_name for result in evaluation_results}), + ) + + def apply_supplemental_eval_details( model_info: ModelInfo, evaluation_results: list[EvaluationResult], @@ -508,26 +573,54 @@ def apply_supplemental_eval_details( ) result_supplements = supplemental_eval_details.evaluation_results or [] - named_supplements = { - supplement.evaluation_name: supplement + both_selectors = [ + supplement for supplement in result_supplements - if supplement.evaluation_name is not None - } - if len(named_supplements) != len( - [s for s in result_supplements if s.evaluation_name is not None] - ): + if supplement.evaluation_result_id is not None + and supplement.evaluation_name is not None + ] + if both_selectors: raise ValueError( - "Duplicate evaluation_name values in supplemental_eval_details.evaluation_results." + 'A supplemental_eval_details.evaluation_results entry sets both ' + 'evaluation_result_id and evaluation_name. An id selects one result ' + 'and a name selects every result of that evaluation, so an entry ' + 'carrying both would apply to that one result and, separately, to ' + 'every sibling sharing the name. Use one selector per entry.' ) - unnamed_supplements = [ - supplement for supplement in result_supplements if supplement.evaluation_name is None + by_result_id = _key_supplements(result_supplements, 'evaluation_result_id') + by_evaluation_name = _key_supplements(result_supplements, 'evaluation_name') + unkeyed_supplements = [ + supplement + for supplement in result_supplements + if supplement.evaluation_result_id is None + and supplement.evaluation_name is None ] - unnamed_idx = 0 + unkeyed_idx = 0 + matched_keys: set[tuple[str, str]] = set() for evaluation_result in evaluation_results: - supplement = named_supplements.get(evaluation_result.evaluation_name) - if supplement is None and unnamed_idx < len(unnamed_supplements): - supplement = unnamed_supplements[unnamed_idx] - unnamed_idx += 1 + # `evaluation_result_id` selects one result; `evaluation_name` selects + # every result of that evaluation, so the specific key wins. + supplement = by_result_id.get(evaluation_result.evaluation_result_id) + if supplement is not None: + matched_keys.add( + ('evaluation_result_id', evaluation_result.evaluation_result_id) + ) + else: + supplement = by_evaluation_name.get( + evaluation_result.evaluation_name + ) + if supplement is not None: + matched_keys.add( + ('evaluation_name', evaluation_result.evaluation_name) + ) + + if supplement is None and unkeyed_idx < len(unkeyed_supplements): + supplement = unkeyed_supplements[unkeyed_idx] + unkeyed_idx += 1 apply_result_supplement(evaluation_result, supplement) + + _warn_unmatched_supplements( + by_result_id, by_evaluation_name, matched_keys, evaluation_results + ) diff --git a/every_eval_ever/converters/lm_eval/adapter.py b/every_eval_ever/converters/lm_eval/adapter.py index af9bac1e8..610b3bd64 100644 --- a/every_eval_ever/converters/lm_eval/adapter.py +++ b/every_eval_ever/converters/lm_eval/adapter.py @@ -39,6 +39,7 @@ KNOWN_METRIC_BOUNDS, MODEL_TYPE_TO_INFERENCE_ENGINE, MODEL_TYPE_TO_INFERENCE_PLATFORM, + evaluation_result_id, parse_model_args, ) @@ -216,6 +217,9 @@ def _build_evaluation_results( task_name, {} ) n_samples = raw_data.get('n-samples', {}).get(task_name, {}) + bootstrap_iters = raw_data.get('config', {}).get('bootstrap_iters') + if not isinstance(bootstrap_iters, int): + bootstrap_iters = None source_data = self._build_source_data(task_config, task_name) gen_config = self._build_generation_config(task_config) @@ -265,12 +269,14 @@ def _build_evaluation_results( # without falsely declaring them continuous and unbounded. metric_config = MetricConfig( evaluation_description=description, + metric_name=metric_name, lower_is_better=not is_higher_better, additional_details={'bounds_status': 'unknown'}, ) else: metric_config = MetricConfig( evaluation_description=description, + metric_name=metric_name, lower_is_better=not is_higher_better, score_type=ScoreType.continuous, min_score=bounds[0], @@ -291,6 +297,9 @@ def _build_evaluation_results( else None ), num_samples=num_samples, + num_bootstrap_samples=( + bootstrap_iters if stderr_val is not None else None + ), ) eval_name = task_name @@ -299,6 +308,9 @@ def _build_evaluation_results( results.append( EvaluationResult( + evaluation_result_id=evaluation_result_id( + metric_name, filter_name + ), evaluation_name=eval_name, source_data=source_data, evaluation_timestamp=eval_timestamp, diff --git a/every_eval_ever/converters/lm_eval/instance_level_adapter.py b/every_eval_ever/converters/lm_eval/instance_level_adapter.py index e46b1dc72..18976f5e3 100644 --- a/every_eval_ever/converters/lm_eval/instance_level_adapter.py +++ b/every_eval_ever/converters/lm_eval/instance_level_adapter.py @@ -22,6 +22,8 @@ Output, ) +from .utils import evaluation_result_id + class LMEvalInstanceLevelAdapter: """Converts lm-eval per-sample JSONL to instance-level every_eval_ever format.""" @@ -42,10 +44,11 @@ def transform_samples( if not line.strip(): continue sample = json.loads(line) - log = self._transform_sample( - sample, evaluation_id, model_id, task_name + results.extend( + self._transform_sample( + sample, evaluation_id, model_id, task_name + ) ) - results.append(log) return results @@ -123,8 +126,13 @@ def _transform_sample( evaluation_id: str, model_id: str, task_name: str, - ) -> InstanceLevelEvaluationLog: - """Transform a single lm-eval sample into an instance-level log.""" + ) -> List[InstanceLevelEvaluationLog]: + """Transform one lm-eval sample into a row per metric it reports. + + The schema asks for one instance record per aggregate result the sample + contributed to. lm-eval stores each metric's own value on the sample, + so every row carries the score its own result aggregates. + """ # Extract prompt from arguments arguments = sample.get('arguments', {}) prompt = '' @@ -137,21 +145,13 @@ def _transform_sample( # Extract model output raw_output = self._extract_output(sample) - # Determine correctness from metric values metrics = sample.get('metrics', []) - score = None - is_correct = None - for metric_name in metrics: - if metric_name in sample: - val = sample[metric_name] - if isinstance(val, (int, float)): - score = float(val) - is_correct = score == 1.0 - break - - if score is None: - score = 0.0 - is_correct = False + scored = [ + (metric_name, float(sample[metric_name])) + for metric_name in metrics + if isinstance(sample.get(metric_name), (int, float)) + and not isinstance(sample.get(metric_name), bool) + ] # Build sample hash from input + reference for cross-model comparison hash_input = json.dumps( @@ -182,35 +182,47 @@ def _transform_sample( ) ] - return InstanceLevelEvaluationLog( - schema_version=SCHEMA_VERSION, - evaluation_id=evaluation_id, - model_id=model_id, - evaluation_name=eval_name, - sample_id=str(sample.get('doc_id', 0)), - sample_hash=sample_hash, - interaction_type=InteractionType.single_turn, - input=Input( - raw=prompt, - reference=[target], - choices=self._extract_choices(sample), - ), - output=Output(raw=[raw_output]), - answer_attribution=answer_attribution, - evaluation=Evaluation( - score=score, - is_correct=is_correct, - ), - metadata={ - 'doc_hash': str(sample.get('doc_hash', '')), - 'prompt_hash': str(sample.get('prompt_hash', '')), - 'target_hash': str(sample.get('target_hash', '')), - 'filter': str(filter_name), - 'lm_eval_metrics': json.dumps( - {m: sample.get(m) for m in metrics if m in sample} + # A sample whose metrics are all non-numeric still belongs in the + # sidecar, with no aggregate result to point at. + emissions = scored or [(None, 0.0)] + + return [ + InstanceLevelEvaluationLog( + schema_version=SCHEMA_VERSION, + evaluation_id=evaluation_id, + model_id=model_id, + evaluation_name=eval_name, + evaluation_result_id=( + evaluation_result_id(metric_name, filter_name) + if metric_name is not None + else None ), - }, - ) + sample_id=str(sample.get('doc_id', 0)), + sample_hash=sample_hash, + interaction_type=InteractionType.single_turn, + input=Input( + raw=prompt, + reference=[target], + choices=self._extract_choices(sample), + ), + output=Output(raw=[raw_output]), + answer_attribution=answer_attribution, + evaluation=Evaluation( + score=score, + is_correct=score == 1.0, + ), + metadata={ + 'doc_hash': str(sample.get('doc_hash', '')), + 'prompt_hash': str(sample.get('prompt_hash', '')), + 'target_hash': str(sample.get('target_hash', '')), + 'filter': str(filter_name), + 'lm_eval_metrics': json.dumps( + {m: sample.get(m) for m in metrics if m in sample} + ), + }, + ) + for metric_name, score in emissions + ] def _is_multiple_choice(self, sample: dict[str, Any]) -> bool: """Check if a sample is multiple-choice by inspecting the arguments structure.""" diff --git a/every_eval_ever/converters/lm_eval/utils.py b/every_eval_ever/converters/lm_eval/utils.py index 1bdc3dd3f..56d9c3ccb 100644 --- a/every_eval_ever/converters/lm_eval/utils.py +++ b/every_eval_ever/converters/lm_eval/utils.py @@ -4,6 +4,18 @@ from typing import Dict, Optional +def evaluation_result_id(metric_name: str, filter_name: str) -> str: + """Build the join key shared by aggregate results and instance rows. + + lm-eval reports a metric once per filter, so a metric name alone is not + unique within a task: `exact_match` under `flexible-extract` and under no + filter are separate results. + """ + if not filter_name or filter_name == 'none': + return metric_name + return f'{metric_name}:{filter_name}' + + def parse_model_args(model_args: str | None) -> Dict[str, str]: """Parse lm-eval model_args string (comma-separated key=value pairs). diff --git a/tests/converter_cases.py b/tests/converter_cases.py index 8919b42d1..b60f90d92 100644 --- a/tests/converter_cases.py +++ b/tests/converter_cases.py @@ -31,10 +31,19 @@ class ConverterCase: # Total `evaluation_results` across every aggregate. For a converter that emits more # results than are worth listing in `scores`, this is what catches one going missing. results: int | None = None + # Rows in the instance-level sidecars, which is one per aggregate result a sample + # contributed to, not one per sample. + sidecar_rows: int | None = None model_id: str | None = None - # Keyed by `/`, since one task can be scored by several - # metrics and each becomes its own result. + # Keyed by `/`, the pair that addresses one + # result, falling back to the metric description for a converter that sets no + # `evaluation_result_id`. A metric name alone would not do: HELM reports the same + # metric on the `valid` split and worst-case over each perturbation. scores: dict[str, float] | None = None + # Distinct `metric_config.metric_name` values across every result. The metric + # belongs in this field rather than in `evaluation_name` or the description, and a + # converter that leaves it unset shows up here as `None` in the set. + metric_names: frozenset[str] | None = None extra_argv: tuple[str, ...] = () # Upstream key paths the converter cannot work without, `*` matching any one key. required_source_paths: tuple[str, ...] = () @@ -82,7 +91,18 @@ def source_payload(self) -> Any: # One scorer reporting three metrics, which is what makes this fixture worth # using: a converter that collapses them to one result fails here. results=3, + # One sample, three aggregate results, so three rows. + sidecar_rows=3, model_id='mistral/mistral-large-latest', + scores={ + 'inspect_evals/cyse2_vulnerability_exploit/' + 'vul_exploit_scorer:accuracy': 0.38108974358974373, + 'inspect_evals/cyse2_vulnerability_exploit/' + 'vul_exploit_scorer:mean': 0.38108974358974357, + 'inspect_evals/cyse2_vulnerability_exploit/' + 'vul_exploit_scorer:std': 0.3115628730565127, + }, + metric_names=frozenset({'accuracy', 'mean', 'std'}), required_source_paths=( 'eval.model', 'eval.task', @@ -104,7 +124,25 @@ def source_payload(self) -> Any: # Eight metrics on the `valid` split, each also reported worst-case over the # robustness and fairness perturbations. results=24, + # 10 instances against the 8 `valid` results; the perturbation results report no + # per-instance stats, so no row joins to them. + sidecar_rows=80, model_id='eleutherai/pythia-1b-v0', + # The 24 results are these 8 metrics on `valid` plus each one's worst case over + # the robustness and fairness perturbations, so the names are listed rather than + # the 24 scores; `results` above is what counts them. + metric_names=frozenset( + { + 'exact_match', + 'exact_match@5', + 'quasi_exact_match', + 'quasi_exact_match@5', + 'prefix_exact_match', + 'prefix_exact_match@5', + 'quasi_prefix_exact_match', + 'quasi_prefix_exact_match@5', + } + ), # `--log_path` is a HELM run directory, not one file, so there is no single # payload for `missing_paths` to address. The gate and the counts above are # what cover this converter. diff --git a/tests/test_converter_conversion.py b/tests/test_converter_conversion.py index c7b87a7d8..602af4b13 100644 --- a/tests/test_converter_conversion.py +++ b/tests/test_converter_conversion.py @@ -95,6 +95,20 @@ def test_conversion_yields_the_expected_records(case, tmp_path): f'{sorted(key for key in set(keys) if keys.count(key) > 1)}' ) assert dict(scored) == case.scores + if case.metric_names is not None: + # `.get`, because an unset `metric_name` is absent from the record rather than + # null — which is the case this assertion exists to report. + converted = { + result['metric_config'].get('metric_name') + for log in logs + for result in log['evaluation_results'] + } + assert converted == case.metric_names, ( + f'{case.source} named its metrics {sorted(converted, key=str)}, ' + f'expected {sorted(case.metric_names)}. The metric belongs in ' + '`metric_config.metric_name`; `None` here means the converter left it ' + 'unset, and `evaluation_name` is for the evaluation.' + ) for log, path in zip(logs, aggregates, strict=True): detailed = log.get('detailed_evaluation_results') @@ -105,6 +119,32 @@ def test_conversion_yields_the_expected_records(case, tmp_path): assert detailed['file_path'].endswith(f'{path.stem}_samples.jsonl') assert detailed['total_rows'] > 0 + if case.sidecar_rows is not None: + declared = sum( + log['detailed_evaluation_results']['total_rows'] + for log in logs + if log.get('detailed_evaluation_results') + ) + written = sum( + len( + [ + line + for line in path.read_text(encoding='utf-8').splitlines() + if line.strip() + ] + ) + for path in sidecars + ) + assert declared == case.sidecar_rows, ( + f'{case.source} reported {declared} instance-level row(s), expected ' + f'{case.sidecar_rows}. A row is owed per aggregate result a sample ' + 'contributed to, so this changes when the results do.' + ) + # `total_rows` is what a reader trusts without opening the sidecar. + assert written == declared, ( + f'{case.source} wrote {written} row(s) but reported {declared}' + ) + def test_required_source_paths_are_present_in_the_fixture(case): """The keys the converter reads must still exist in the committed log. diff --git a/tests/test_helm_adapter.py b/tests/test_helm_adapter.py index b9f1316e9..ca3387a91 100644 --- a/tests/test_helm_adapter.py +++ b/tests/test_helm_adapter.py @@ -189,6 +189,77 @@ def test_narrativeqa_eval(): assert converted_eval.detailed_evaluation_results.total_rows >= 5 +HELLASWAG_RUN = ( + 'tests/data/helm/commonsense-dataset=hellaswag,' + 'method=multiple_choice_joint,model=eleutherai_pythia-1b-v0' +) + + +def test_evaluation_name_is_the_benchmark_and_the_metric_is_named(): + """Each field carries one thing: the eval, the metric, the split. + + HELM reports one stat per (metric, split, perturbation), and those three + already identify a result through `evaluation_result_id`. `evaluation_name` is + the benchmark, which is what the instance-level rows carry and what a registry + lookup can resolve. + """ + adapter = HELMAdapter() + converted_eval = _load_eval( + adapter, + HELLASWAG_RUN, + { + 'source_organization_name': 'TestOrg', + 'evaluator_relationship': EvaluatorRelationship.first_party, + }, + ) + results = converted_eval.evaluation_results + + assert {result.evaluation_name for result in results} == {'hellaswag'} + assert all( + result.metric_config.metric_name + and result.evaluation_result_id.startswith( + result.metric_config.metric_name + ) + for result in results + ) + assert { + result.score_details.details['perturbation'] for result in results + } == {'', 'robustness', 'fairness'} + + +def test_instance_rows_join_the_aggregate_results_they_belong_to(): + """A sample row the aggregate cannot be joined to is a row nobody can read.""" + import json + + adapter = HELMAdapter() + with tempfile.TemporaryDirectory() as tmpdir: + converted_eval = adapter.transform_from_directory( + Path(HELLASWAG_RUN), + metadata_args={ + 'source_organization_name': 'TestOrg', + 'evaluator_relationship': EvaluatorRelationship.first_party, + 'file_uuid': TEST_UUID, + 'parent_eval_output_dir': tmpdir, + }, + )[0] + sidecars = list(Path(tmpdir).rglob('*_samples.jsonl')) + assert len(sidecars) == 1 + rows = [ + json.loads(line) + for line in sidecars[0].read_text(encoding='utf-8').splitlines() + if line + ] + + assert rows + assert {row['evaluation_name'] for row in rows} == { + result.evaluation_name for result in converted_eval.evaluation_results + } + assert {row['evaluation_result_id'] for row in rows} <= { + result.evaluation_result_id + for result in converted_eval.evaluation_results + } + + def test_missing_model_deployment_falls_back_to_model(): """ Copies a helm data item and explicitly removes a field to test robustness diff --git a/tests/test_inspect_adapter.py b/tests/test_inspect_adapter.py index 4432b76fe..74ad6734b 100644 --- a/tests/test_inspect_adapter.py +++ b/tests/test_inspect_adapter.py @@ -6,6 +6,7 @@ ) import contextlib +import logging import tempfile from pathlib import Path from types import SimpleNamespace @@ -106,11 +107,9 @@ def test_pubmedqa_eval(): assert converted_eval.model_info.inference_engine is None results = converted_eval.evaluation_results - assert ( - results[0].evaluation_name - == 'accuracy on inspect_evals/pubmedqa for scorer choice' - ) - assert results[0].metric_config.evaluation_description == 'accuracy' + assert results[0].evaluation_name == 'inspect_evals/pubmedqa' + assert results[0].evaluation_result_id == 'choice:accuracy' + assert results[0].metric_config.metric_name == 'accuracy' assert results[0].score_details.score == 1.0 assert converted_eval.detailed_evaluation_results is not None @@ -239,10 +238,9 @@ def test_arc_sonnet_eval(): assert converted_eval.model_info.inference_engine is None results = converted_eval.evaluation_results - assert ( - results[0].evaluation_name == 'accuracy on arc_easy for scorer choice' - ) - assert results[0].metric_config.evaluation_description == 'accuracy' + assert results[0].evaluation_name == 'arc_easy' + assert results[0].evaluation_result_id == 'choice:accuracy' + assert results[0].metric_config.metric_name == 'accuracy' assert results[0].score_details.score == 1.0 assert converted_eval.detailed_evaluation_results is not None @@ -281,10 +279,9 @@ def test_arc_qwen_eval(): assert converted_eval.model_info.inference_engine.name == 'ollama' results = converted_eval.evaluation_results - assert ( - results[0].evaluation_name == 'accuracy on arc_easy for scorer choice' - ) - assert results[0].metric_config.evaluation_description == 'accuracy' + assert results[0].evaluation_name == 'arc_easy' + assert results[0].evaluation_result_id == 'choice:accuracy' + assert results[0].metric_config.metric_name == 'accuracy' assert results[0].score_details.score == 0.3333333333333333 assert converted_eval.detailed_evaluation_results is not None @@ -332,10 +329,13 @@ def test_gaia_eval(): results = converted_eval.evaluation_results assert len(results) > 0 + assert results[0].evaluation_name == 'gaia' + assert results[0].evaluation_result_id == 'gaia_scorer:accuracy' + assert results[0].metric_config.metric_name == 'accuracy' assert ( - results[0].evaluation_name == 'accuracy on gaia for scorer gaia_scorer' + results[0].metric_config.evaluation_description + == 'accuracy from scorer gaia_scorer' ) - assert results[0].metric_config.evaluation_description == 'accuracy' assert results[0].score_details.score >= 0.0 assert converted_eval.detailed_evaluation_results is not None @@ -343,6 +343,36 @@ def test_gaia_eval(): assert converted_eval.detailed_evaluation_results.total_rows > 0 +def test_evaluation_name_is_the_benchmark_and_the_metric_is_named(): + """One scorer reporting three metrics: same eval, three named metrics.""" + adapter = InspectAIAdapter() + metadata_args = { + 'source_organization_name': 'TestOrg', + 'evaluator_relationship': EvaluatorRelationship.first_party, + } + + converted_eval = _load_eval( + adapter, + 'tests/data/inspect/data_cyse2_vuln_exploit_challenges.json', + metadata_args, + ) + + results = converted_eval.evaluation_results + assert len(results) == 3 + assert {result.evaluation_name for result in results} == { + 'inspect_evals/cyse2_vulnerability_exploit' + } + assert {result.metric_config.metric_name for result in results} == { + 'accuracy', + 'mean', + 'std', + } + for result in results: + assert result.evaluation_result_id == ( + f'vul_exploit_scorer:{result.metric_config.metric_name}' + ) + + def test_humaneval_eval(): adapter = InspectAIAdapter() metadata_args = { @@ -375,7 +405,7 @@ def test_extract_evaluation_results_one_scorer_with_two_metrics(): ) ] - results = adapter._extract_evaluation_results( + results, result_ids_by_scorer = adapter._extract_evaluation_results( evaluation_task_name='synthetic/task', scores=scores, source_data=source_data, @@ -385,10 +415,17 @@ def test_extract_evaluation_results_one_scorer_with_two_metrics(): ) assert len(results) == 2 - assert {result.evaluation_name for result in results} == { - 'accuracy on synthetic/task for scorer choice', - 'f1 on synthetic/task for scorer choice', + # The eval is named once; the metric is a metric field, not part of the name. + assert {result.evaluation_name for result in results} == {'synthetic/task'} + assert {result.metric_config.metric_name for result in results} == { + 'accuracy', + 'f1', } + assert {result.evaluation_result_id for result in results} == { + 'choice:accuracy', + 'choice:f1', + } + assert result_ids_by_scorer == {'choice': ['choice:accuracy', 'choice:f1']} def test_extract_evaluation_results_two_scorers_two_metrics_each(): @@ -414,7 +451,7 @@ def test_extract_evaluation_results_two_scorers_two_metrics_each(): ), ] - results = adapter._extract_evaluation_results( + results, result_ids_by_scorer = adapter._extract_evaluation_results( evaluation_task_name='synthetic/task', scores=scores, source_data=source_data, @@ -424,11 +461,18 @@ def test_extract_evaluation_results_two_scorers_two_metrics_each(): ) assert len(results) == 4 - assert {result.evaluation_name for result in results} == { - 'accuracy on synthetic/task for scorer scorer_a', - 'f1 on synthetic/task for scorer scorer_a', - 'accuracy on synthetic/task for scorer scorer_b', - 'f1 on synthetic/task for scorer scorer_b', + assert {result.evaluation_name for result in results} == {'synthetic/task'} + # Two scorers reporting the same metric name must not collide in + # `evaluation_result_id`, or sample rows cannot say which one they join. + assert {result.evaluation_result_id for result in results} == { + 'scorer_a:accuracy', + 'scorer_a:f1', + 'scorer_b:accuracy', + 'scorer_b:f1', + } + assert result_ids_by_scorer == { + 'scorer_a': ['scorer_a:accuracy', 'scorer_a:f1'], + 'scorer_b': ['scorer_b:accuracy', 'scorer_b:f1'], } @@ -498,7 +542,7 @@ def test_supplemental_eval_details_fill_only_top_level_fields(): }, 'evaluation_results': [ { - 'evaluation_name': 'accuracy on inspect_evals/pubmedqa for scorer choice', + 'evaluation_result_id': 'choice:accuracy', 'score_details': { 'details': { 'notes': ['a', 'b'], @@ -562,7 +606,7 @@ def test_supplemental_eval_details_applies_top_level_score_details(): 'supplemental_eval_details': { 'evaluation_results': [ { - 'evaluation_name': 'accuracy on inspect_evals/pubmedqa for scorer choice', + 'evaluation_result_id': 'choice:accuracy', 'score_details': { 'details': { 'matched': 1, @@ -612,7 +656,9 @@ def test_supplemental_eval_details_does_not_overwrite_existing_generation_detail assert result.generation_config.additional_details['added_field'] == 'yes' -def test_supplemental_eval_details_does_not_apply_when_evaluation_name_does_not_match(): +def test_supplemental_eval_details_does_not_apply_when_evaluation_name_does_not_match( + caplog, +): adapter = InspectAIAdapter() metadata_args = { 'source_organization_name': 'TestOrg', @@ -627,13 +673,60 @@ def test_supplemental_eval_details_does_not_apply_when_evaluation_name_does_not_ }, } + with caplog.at_level(logging.WARNING): + converted_eval = _load_eval( + adapter, + 'tests/data/inspect/data_pubmedqa_gpt4o_mini.json', + metadata_args, + ) + result = converted_eval.evaluation_results[0] + assert result.score_details.details is None + # A supplemental file is hand-written, so a key that selects nothing is a + # typo the contributor needs to hear about, not a silent no-op. + assert 'matched no evaluation result' in caplog.text + assert "'some_other_eval - choice'" in caplog.text + + +def test_supplemental_eval_details_matches_all_results_of_an_evaluation(): + """`evaluation_name` now names the eval, so it selects every result.""" + adapter = InspectAIAdapter() + metadata_args = { + 'source_organization_name': 'TestOrg', + 'evaluator_relationship': EvaluatorRelationship.first_party, + 'supplemental_eval_details': { + 'evaluation_results': [ + { + 'evaluation_name': 'inspect_evals/cyse2_vulnerability_exploit', + 'score_details': {'details': {'reviewed': 'yes'}}, + }, + { + 'evaluation_result_id': 'vul_exploit_scorer:std', + 'score_details': {'details': {'reviewed': 'separately'}}, + }, + ], + }, + } + converted_eval = _load_eval( adapter, - 'tests/data/inspect/data_pubmedqa_gpt4o_mini.json', + 'tests/data/inspect/data_cyse2_vuln_exploit_challenges.json', metadata_args, ) - result = converted_eval.evaluation_results[0] - assert result.score_details.details is None + + details_by_result_id = { + result.evaluation_result_id: result.score_details.details + for result in converted_eval.evaluation_results + } + assert details_by_result_id['vul_exploit_scorer:accuracy'] == { + 'reviewed': 'yes' + } + assert details_by_result_id['vul_exploit_scorer:mean'] == { + 'reviewed': 'yes' + } + # The specific key wins over the evaluation-wide one. + assert details_by_result_id['vul_exploit_scorer:std'] == { + 'reviewed': 'separately' + } def test_supplemental_eval_details_fails_on_deprecated_per_result_schema(): @@ -645,7 +738,7 @@ def test_supplemental_eval_details_fails_on_deprecated_per_result_schema(): 'per_result': [ { 'match': { - 'evaluation_name': 'accuracy on inspect_evals/pubmedqa for scorer choice', + 'evaluation_result_id': 'choice:accuracy', }, 'score_details': {'details': {'matched': 1}}, }, @@ -664,7 +757,45 @@ def test_supplemental_eval_details_fails_on_deprecated_per_result_schema(): ) -def test_supplemental_eval_details_fails_on_duplicate_evaluation_name(): +@pytest.mark.parametrize( + 'key_field, key', + [ + ('evaluation_result_id', 'choice:accuracy'), + ('evaluation_name', 'inspect_evals/pubmedqa'), + ], +) +def test_supplemental_eval_details_fails_on_duplicate_key(key_field, key): + adapter = InspectAIAdapter() + metadata_args = { + 'source_organization_name': 'TestOrg', + 'evaluator_relationship': EvaluatorRelationship.first_party, + 'supplemental_eval_details': { + 'evaluation_results': [ + {key_field: key, 'score_details': {'details': {'a': 1}}}, + {key_field: key, 'score_details': {'details': {'b': 2}}}, + ] + }, + } + + with tempfile.TemporaryDirectory() as tmpdir: + metadata_args = dict(metadata_args) + metadata_args['file_uuid'] = TEST_UUID + metadata_args['parent_eval_output_dir'] = tmpdir + with pytest.raises(AdapterError): + adapter.transform_from_file( + Path('tests/data/inspect/data_pubmedqa_gpt4o_mini.json'), + metadata_args=metadata_args, + ) + + +def test_supplemental_eval_details_fails_when_one_entry_sets_both_selectors(): + """An id selects one result; a name selects every result of the evaluation. + + An entry that sets both applies to its own result by id and, through the + shared name, to that result's siblings as well -- never what one entry is + meant to do. The two behaviours are still available through two entries, so + the ambiguous single entry is rejected rather than silently fanned out. + """ adapter = InspectAIAdapter() metadata_args = { 'source_organization_name': 'TestOrg', @@ -672,13 +803,10 @@ def test_supplemental_eval_details_fails_on_duplicate_evaluation_name(): 'supplemental_eval_details': { 'evaluation_results': [ { - 'evaluation_name': 'accuracy on inspect_evals/pubmedqa for scorer choice', + 'evaluation_result_id': 'choice:accuracy', + 'evaluation_name': 'inspect_evals/pubmedqa', 'score_details': {'details': {'a': 1}}, }, - { - 'evaluation_name': 'accuracy on inspect_evals/pubmedqa for scorer choice', - 'score_details': {'details': {'b': 2}}, - }, ] }, } diff --git a/tests/test_inspect_instance_level_adapter.py b/tests/test_inspect_instance_level_adapter.py index 237d96e52..acb93c769 100644 --- a/tests/test_inspect_instance_level_adapter.py +++ b/tests/test_inspect_instance_level_adapter.py @@ -196,6 +196,112 @@ def test_gaia_instance_level(): assert log.token_usage.output_tokens >= 0 +def test_instance_rows_join_the_aggregate_results_they_belong_to(): + """One sample, three aggregate metrics: three rows, one per result. + + The instance schema asks for a record per aggregate result a sample + contributed to, and `evaluation_result_id` is the only field that says + which result a row belongs to. + """ + adapter = InspectAIAdapter() + + with tempfile.TemporaryDirectory() as tmpdir: + metadata_args = { + 'source_organization_name': 'TestOrg', + 'evaluator_relationship': EvaluatorRelationship.first_party, + 'parent_eval_output_dir': tmpdir, + 'file_uuid': TEST_UUID, + } + + converted_eval, instance_logs = _load_instance_level_data( + adapter, + 'tests/data/inspect/data_cyse2_vuln_exploit_challenges.json', + metadata_args, + ) + + aggregate_result_ids = { + result.evaluation_result_id + for result in converted_eval.evaluation_results + } + assert len(instance_logs) == 3 + assert {log.sample_id for log in instance_logs} == {'1'} + assert { + log.evaluation_result_id for log in instance_logs + } == aggregate_result_ids + assert {log.evaluation_name for log in instance_logs} == { + result.evaluation_name for result in converted_eval.evaluation_results + } + assert converted_eval.detailed_evaluation_results.total_rows == 3 + + +def test_multiple_scorers_report_their_own_score_on_their_own_row(): + sample = _make_synthetic_sample( + sample_id='sample_1', + target='target_answer', + response_content='generated_response', + score_value='C', + scorer_name='scorer_a', + ) + sample.scores['scorer_b'] = SimpleNamespace( + value='I', answer='b_answer', explanation=None + ) + + with tempfile.TemporaryDirectory() as tmpdir: + adapter = InspectInstanceLevelDataAdapter( + 'synthetic_test', 'synthetic_test', 'jsonl', 'sha256', tmpdir + ) + path, rows_count = adapter.convert_instance_level_logs( + 'synthetic_eval', + 'synthetic/model', + [sample], + None, + { + 'scorer_a': ['scorer_a:accuracy'], + 'scorer_b': ['scorer_b:accuracy', 'scorer_b:std'], + }, + ) + rows = [ + InstanceLevelEvaluationLog.model_validate(json.loads(line)) + for line in Path(path).read_text(encoding='utf-8').splitlines() + ] + + assert rows_count == 3 + by_result_id = {row.evaluation_result_id: row for row in rows} + assert set(by_result_id) == { + 'scorer_a:accuracy', + 'scorer_b:accuracy', + 'scorer_b:std', + } + assert {row.sample_id for row in rows} == {'sample_1'} + + assert by_result_id['scorer_a:accuracy'].evaluation.score == 1.0 + assert by_result_id['scorer_a:accuracy'].evaluation.is_correct is True + assert by_result_id['scorer_b:accuracy'].evaluation.score == 0.0 + assert by_result_id['scorer_b:accuracy'].evaluation.is_correct is False + assert by_result_id['scorer_b:std'].evaluation.score == 0.0 + + # Each row shows the answer its own scorer graded, not the last scorer's. + assert by_result_id['scorer_a:accuracy'].output.raw == [ + 'generated_response' + ] + assert by_result_id['scorer_b:accuracy'].output.raw == ['b_answer'] + + +def test_sample_row_omits_result_id_when_no_aggregate_result_matches(): + """Without an attributable aggregate result, emit one unjoined row.""" + sample = _make_synthetic_sample( + sample_id='sample_1', + target='target_answer', + response_content='target_answer', + score_value='C', + ) + + instance_log = _convert_single_synthetic_sample(sample) + + assert instance_log.evaluation_result_id is None + assert instance_log.evaluation.score == 1.0 + + def test_serialize_input_skips_non_user_messages(): adapter = InspectInstanceLevelDataAdapter( 'test_id', 'test_id', 'jsonl', 'sha256', '/tmp' diff --git a/tests/test_lm_eval_adapter.py b/tests/test_lm_eval_adapter.py index cbdcb5116..2456f6762 100644 --- a/tests/test_lm_eval_adapter.py +++ b/tests/test_lm_eval_adapter.py @@ -148,6 +148,9 @@ def test_transform_from_file_evaluation_results(): assert perturbed_results[0].metric_config.lower_is_better is False assert perturbed_results[0].metric_config.min_score == 0.0 assert perturbed_results[0].metric_config.max_score == 1.0 + # The metric belongs in metric_name, not only in the description. + assert perturbed_results[0].metric_config.metric_name == 'exact_match' + assert perturbed_results[0].evaluation_result_id == 'exact_match' # Second task: math_rephrased_full with exact_match = 0.0004 rephrased_results = logs[1].evaluation_results @@ -163,6 +166,9 @@ def test_transform_from_file_uncertainty(): assert uncertainty.standard_error.value == 0.0002828144211304471 assert uncertainty.standard_error.method == 'bootstrap' assert uncertainty.num_samples == 5000 + # The resamples the standard error came from, distinct from the 5000 + # documents the score came from. + assert uncertainty.num_bootstrap_samples == 100000 def test_transform_from_file_generation_config(): @@ -296,6 +302,101 @@ def test_instance_level_transform_and_save_no_output_dir(): assert result is None +def _write_samples(tmpdir: str, samples: list[dict]) -> Path: + path = Path(tmpdir) / 'samples_mytask_2026-01-01T00-00-00.jsonl' + path.write_text( + '\n'.join(json.dumps(sample) for sample in samples) + '\n', + encoding='utf-8', + ) + return path + + +def test_instance_level_one_row_per_metric(): + """A sample scored by several metrics becomes one row per metric. + + Each row carries its own metric's score and the evaluation_result_id of the + aggregate result that score feeds, so the sidecar joins back per metric + rather than attributing one metric's value to all of them. + """ + inst_adapter = LMEvalInstanceLevelAdapter() + with tempfile.TemporaryDirectory() as tmpdir: + samples_path = _write_samples( + tmpdir, + [ + { + 'doc_id': 0, + 'target': '3', + 'filter': 'flexible-extract', + 'metrics': ['acc', 'acc_norm', 'brier_score'], + 'acc': 1.0, + 'acc_norm': 0.0, + 'brier_score': 0.25, + 'arguments': {'gen_args_0': {'arg_0': 'What is 1 + 2?'}}, + 'filtered_resps': ['3'], + } + ], + ) + logs = inst_adapter.transform_samples( + samples_path, + evaluation_id='test/eval/123', + model_id='test-model', + task_name='mytask', + ) + + assert [ + (log.evaluation_result_id, log.evaluation.score) for log in logs + ] == [ + ('acc:flexible-extract', 1.0), + ('acc_norm:flexible-extract', 0.0), + ('brier_score:flexible-extract', 0.25), + ] + assert [log.evaluation.is_correct for log in logs] == [True, False, False] + # One underlying interaction, so the rows share a sample id and hash. + assert {log.sample_id for log in logs} == {'0'} + assert len({log.sample_hash for log in logs}) == 1 + assert {log.evaluation_name for log in logs} == {'mytask/flexible-extract'} + + +def test_instance_level_non_numeric_metrics_only(): + """A sample whose metrics are all non-numeric still reaches the sidecar. + + It has no aggregate result to point at, so it gets a single row with no + evaluation_result_id rather than being dropped. + """ + inst_adapter = LMEvalInstanceLevelAdapter() + with tempfile.TemporaryDirectory() as tmpdir: + samples_path = _write_samples( + tmpdir, + [ + { + 'doc_id': 7, + 'target': 'yes', + 'filter': 'none', + 'metrics': ['verdict', 'passed'], + 'verdict': 'correct', + 'passed': True, + 'arguments': {'gen_args_0': {'arg_0': 'Is this right?'}}, + 'filtered_resps': ['yes'], + } + ], + ) + logs = inst_adapter.transform_samples( + samples_path, + evaluation_id='test/eval/123', + model_id='test-model', + task_name='mytask', + ) + + assert len(logs) == 1 + assert logs[0].evaluation_result_id is None + assert logs[0].sample_id == '7' + # The unscored values are still recoverable from the row. + assert json.loads(logs[0].metadata['lm_eval_metrics']) == { + 'verdict': 'correct', + 'passed': True, + } + + def test_na_stderr_treated_as_absent(): """lm-eval reports stderr as the string 'N/A' for non-bootstrapped metrics (aggregated/grouped or custom metrics, e.g. ECLeKTic). Conversion must not