Merge pull request #90 from evaleval/fix_different_uuids_for_json_and_jsonl

Fix different uuids for json and jsonl files for the same log in the CLI

Author: nelaturuharsha

every_eval_ever/cli.py

@@ -4,6 +4,7 @@
 
 import argparse
 import json
+import os
 import sys
 import uuid
 from pathlib import Path
@@ -108,23 +109,37 @@ def _cmd_convert_lm_eval(args: argparse.Namespace) -> int:
 
 
 def _cmd_convert_inspect(args: argparse.Namespace) -> int:
-    from every_eval_ever.converters.inspect.adapter import InspectAIAdapter
+    from every_eval_ever.converters.inspect.adapter import (
+        InspectAIAdapter,
+        list_eval_logs,
+    )
 
     adapter = InspectAIAdapter()
     metadata = _common_metadata(args)
-    metadata['file_uuid'] = str(uuid.uuid4())
 
     log_path = Path(args.log_path)
+    eval_uuids: list[str]
     if log_path.is_file():
+        eval_uuids = [str(uuid.uuid4())]
+        metadata['file_uuid'] = eval_uuids[0]
         logs = [adapter.transform_from_file(log_path, metadata)]
     elif log_path.is_dir():
+        eval_paths = list_eval_logs(log_path.absolute().as_posix())
+        eval_uuids = [str(uuid.uuid4()) for _ in eval_paths]
+        metadata['file_uuids'] = eval_uuids
         logs = adapter.transform_from_directory(log_path, metadata)
     else:
         raise FileNotFoundError(f'Path is not a file or directory: {log_path}')
 
+    if len(logs) != len(eval_uuids):
+        raise RuntimeError(
+            'Inspect conversion produced a different number of logs than '
+            'the generated UUID list.'
+        )
+
     output_dir = Path(args.output_dir)
-    for log in logs:
-        print(_write_log(log, output_dir))
+    for log, eval_uuid in zip(logs, eval_uuids):
+        print(_write_log(log, output_dir, eval_uuid=eval_uuid))
 
     print(f'Converted {len(logs)} evaluation log(s).')
     return 0
@@ -135,16 +150,39 @@ def _cmd_convert_helm(args: argparse.Namespace) -> int:
 
     adapter = HELMAdapter()
     metadata = _common_metadata(args)
-    metadata['file_uuid'] = str(uuid.uuid4())
+    log_path = Path(args.log_path)
+
+    eval_uuids: list[str]
+    if adapter._directory_contains_required_files(log_path):
+        eval_uuids = [str(uuid.uuid4())]
+        metadata['file_uuid'] = eval_uuids[0]
+    elif log_path.is_dir():
+        run_dirs = [
+            entry.path
+            for entry in os.scandir(log_path)
+            if entry.is_dir()
+            and adapter._directory_contains_required_files(entry.path)
+        ]
+        eval_uuids = [str(uuid.uuid4()) for _ in run_dirs]
+        metadata['file_uuids'] = eval_uuids
+    else:
+        raise FileNotFoundError(f'Path is not a file or directory: {log_path}')
 
     logs = adapter.transform_from_directory(
-        Path(args.log_path),
+        log_path,
         output_path=str(Path(args.output_dir) / 'helm_output'),
         metadata_args=metadata,
     )
+
+    if len(logs) != len(eval_uuids):
+        raise RuntimeError(
+            'HELM conversion produced a different number of logs than '
+            'the generated UUID list.'
+        )
+
     output_dir = Path(args.output_dir)
-    for log in logs:
-        print(_write_log(log, output_dir))
+    for log, eval_uuid in zip(logs, eval_uuids):
+        print(_write_log(log, output_dir, eval_uuid=eval_uuid))
 
     print(f'Converted {len(logs)} evaluation log(s).')
     return 0

every_eval_ever/converters/common/utils.py

@@ -1,9 +1,16 @@
 import hashlib
+import re
 from datetime import datetime
-from typing import Dict
+from pathlib import Path
+from typing import Any, Dict
 
 from huggingface_hub import HfApi
 
+_UUID_FILE_RE = re.compile(
+    r'(?P<uuid>[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})(?:_samples)?(?:\.jsonl?)?$',
+    re.IGNORECASE,
+)
+
 
 def convert_timestamp_to_unix_format(timestamp: str) -> str:
     dt = datetime.fromisoformat(timestamp)
@@ -78,3 +85,20 @@ def sha256_file(path, chunk_size=8192):
 
 def sha256_string(text: str, chunk_size=8192):
     return hashlib.sha256(text.encode('utf-8')).hexdigest()
+
+
+def extract_file_uuid_from_detailed_results(log: Any) -> str | None:
+    detailed = getattr(log, 'detailed_evaluation_results', None)
+    if not detailed:
+        return None
+
+    file_path = getattr(detailed, 'file_path', None)
+    if not file_path:
+        return None
+
+    filename = Path(str(file_path)).name
+    uuid_match = _UUID_FILE_RE.search(filename)
+    if uuid_match:
+        return uuid_match.group('uuid')
+
+    return None

every_eval_ever/converters/helm/adapter.py

@@ -1,6 +1,7 @@
 import datetime
 import json
 import os
+import uuid
 from pathlib import Path
 from typing import Any, Dict, List, Tuple
 
@@ -68,7 +69,8 @@ def _require_helm_dependencies() -> None:
     if _HELM_IMPORT_ERROR is not None:
         raise ImportError(
             'HELM converter dependencies are missing. '
-            "Install with: pip install 'every_eval_ever[helm]'"
+            "Install with: uv sync --extra helm "
+            "(or pip install 'every_eval_ever[helm]')."
         ) from _HELM_IMPORT_ERROR
 
 
@@ -171,18 +173,46 @@ def transform_from_directory(
         # all_instance_logs: List[InstanceLevelEvaluationLog] = []
         aggregate_logs: List[EvaluationLog] = []
 
+        file_uuids = metadata_args.get('file_uuids')
+
         if self._directory_contains_required_files(dir_path):
             data = self._load_evaluation_run_logfiles(dir_path)
-            agg = self._transform_single(data, metadata_args)
+            per_log_metadata_args = dict(metadata_args)
+            if (
+                isinstance(file_uuids, list)
+                and file_uuids
+                and file_uuids[0]
+            ):
+                per_log_metadata_args['file_uuid'] = file_uuids[0]
+            else:
+                per_log_metadata_args['file_uuid'] = metadata_args.get(
+                    'file_uuid'
+                ) or str(uuid.uuid4())
+            agg = self._transform_single(data, per_log_metadata_args)
             aggregate_logs.append(agg)
         else:
+            converted_idx = 0
             for entry in os.scandir(dir_path):
                 if entry.is_dir() and self._directory_contains_required_files(
                     entry.path
                 ):
                     data = self._load_evaluation_run_logfiles(entry.path)
-                    agg = self._transform_single(data, metadata_args)
+                    per_log_metadata_args = dict(metadata_args)
+                    if (
+                        isinstance(file_uuids, list)
+                        and converted_idx < len(file_uuids)
+                        and file_uuids[converted_idx]
+                    ):
+                        per_log_metadata_args['file_uuid'] = file_uuids[
+                            converted_idx
+                        ]
+                    else:
+                        per_log_metadata_args['file_uuid'] = str(
+                            uuid.uuid4()
+                        )
+                    agg = self._transform_single(data, per_log_metadata_args)
                     aggregate_logs.append(agg)
+                    converted_idx += 1
 
         # # Write all consolidated instance logs to JSONL
         # with open(output_path, 'w', encoding='utf-8') as f:

every_eval_ever/converters/inspect/__main__.py

@@ -12,7 +12,9 @@
     from inspect_ai.log import list_eval_logs
 
     from every_eval_ever.converters.inspect.adapter import InspectAIAdapter
-    from every_eval_ever.converters.inspect.supplemental_eval_details import SupplementalEvalDetails
+    from every_eval_ever.converters.inspect.supplemental_eval_details import (
+        SupplementalEvalDetails,
+    )
 except ImportError as exc:
     raise SystemExit(
         "The 'inspect-ai' package is required to use the Inspect AI converter.\n"
@@ -165,16 +167,6 @@ def save_evaluation_log(
         return False
 
 
-def extract_file_uuid_from_output(unified_output: EvaluationLog) -> str | None:
-    detailed = unified_output.detailed_evaluation_results
-    if detailed and detailed.file_path:
-        stem = Path(detailed.file_path).stem
-        suffix = '_samples'
-        if stem.endswith(suffix):
-            return stem[: -len(suffix)]
-    return None
-
-
 if __name__ == '__main__':
     logging.basicConfig(level=logging.INFO)
     args = parse_args()
@@ -216,21 +208,14 @@ def extract_file_uuid_from_output(unified_output: EvaluationLog) -> str | None:
                 metadata_args
             )
             if unified_output and isinstance(unified_output, List):
-                for idx, single_unified_output in enumerate(unified_output):
-                    file_uuid = (
-                        file_uuids[idx] if idx < len(file_uuids) else None
+                if len(unified_output) != len(file_uuids):
+                    raise RuntimeError(
+                        'Inspect conversion produced a different number of '
+                        'logs than the generated UUID list.'
                     )
-                    if not file_uuid:
-                        file_uuid = extract_file_uuid_from_output(
-                            single_unified_output
-                        )
-                    if not file_uuid:
-                        file_uuid = str(uuid.uuid4())
-                        logger.warning(
-                            'Missing UUID for output %s; generated %s for aggregate save.',
-                            single_unified_output.evaluation_id,
-                            file_uuid,
-                        )
+                for single_unified_output, file_uuid in zip(
+                    unified_output, file_uuids
+                ):
                     save_evaluation_log(
                         single_unified_output,
                         inspect_converter,

every_eval_ever/converters/inspect/utils.py

@@ -1,19 +1,10 @@
 import json
 import re
 from pathlib import Path
+from typing import Any, Dict, List, Type
 
 from pydantic import BaseModel
-from typing import Any, Dict, List, Type
 
-from every_eval_ever.eval_types import (
-    AgenticEvalConfig,
-    EvaluationResult,
-    GenerationArgs,
-    GenerationConfig,
-    InferenceEngine,
-    MetricConfig,
-    ModelInfo
-)
 from every_eval_ever.converters.common.utils import get_model_organization_info
 from every_eval_ever.converters.inspect.supplemental_eval_details import (
     SupplementalAgenticEvalConfig,
@@ -22,6 +13,15 @@
     SupplementalGenerationConfig,
     SupplementalSourceData,
 )
+from every_eval_ever.eval_types import (
+    AgenticEvalConfig,
+    EvaluationResult,
+    GenerationArgs,
+    GenerationConfig,
+    InferenceEngine,
+    MetricConfig,
+    ModelInfo,
+)
 
 
 class ModelPathHandler:
@@ -549,4 +549,4 @@ def apply_supplemental_eval_details(
             supplement = unnamed_supplements[unnamed_idx]
             unnamed_idx += 1
 
-        apply_result_supplement(evaluation_result, supplement)
+        apply_result_supplement(evaluation_result, supplement)

pyproject.toml

@@ -22,7 +22,10 @@ dependencies = [
 
 [project.optional-dependencies]
 inspect = ["inspect-ai>=0.3.160,<0.4.0"]
-helm = ["crfm-helm>=0.5.12"]
+helm = [
+    "crfm-helm>=0.5.12",
+    "typer>=0.12,<1.0",
+]
 all = [
     "every-eval-ever[inspect]",
     "every-eval-ever[helm]",