Implementation of get_lr_system and get_reference_data (#23)

* First basic implementation

* update based on review comments

* Add functions to load LR system and reference data from disk

* Update .gitignore, remove test_validation_yaml, and add test_api for loading LR systems

* Update lir package version to 1.4.0

Author: PimMeulensteen

.gitignore

@@ -5,6 +5,7 @@ output/
 lrsystem_output/
 .testmondata
 tests/test_model_storage/
+tests/saved_models/
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

lrmodule/__init__.py

@@ -1,15 +1,60 @@
+import pickle
 from pathlib import Path
 
 import numpy as np
 from lir.config.lrsystem_architectures import specific_source
 from lir.data.models import FeatureData, LLRData
+from lir.datasets.feature_data_csv import FeatureDataCsvFileParser
 from lir.lrsystems.lrsystems import LRSystem
 
 from lrmodule import persistence
 from lrmodule.data_types import ModelSettings
 from lrmodule.lrsystem import get_trained_model
 
 
+def get_lr_system(lr_system_folder: Path, file_name: str = "model.pkl") -> LRSystem:
+    """
+    Load a trained LR system from disk from a given folder.
+
+    It is expected that the folder contains a file named "model.pkl" (or another name specified by file_name),
+    which is a pickled LRSystem object. The function loads this object and returns it.
+
+    The system is returned as an instance of the LRSystem class. This class provides an apply method, which can be used
+    to calculate LLRs for given features. These features should be contained in a FeatureData object from lir.
+
+    Example usage:
+    ```
+    from lir.data.models import FeatureData
+
+    lr_system = get_lr_system(Path("path/to/lr_system_folder/"))
+
+    # Create three instances of features, each with two feature values.
+    features = np.array([[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]])
+    feature_data = FeatureData(features=features)
+    llr_data = lr_system.apply(feature_data)
+    ```
+    """
+    with (lr_system_folder / file_name).open("rb") as f:
+        return pickle.load(f)  # noqa: S301
+
+
+def get_reference_data(lr_system_folder: Path, file_name: str = "reference_data.csv") -> FeatureData:
+    """Load reference data from disk.
+
+    It is expected that the folder contains a file named "reference_data.csv" (or another name specified by file_name),
+    which is a CSV file containing the reference data. The function loads this data and returns it as a FeatureData
+    object.
+
+    If the data has `n` features, the CSV file should have `n+1` columns. One of the columns should be named
+    "hypothesis", and contain the labels for the data. The other columns should contain the feature values.
+
+    The data is returned as an instance of the FeatureData class from lir. This class has a features and a labels
+    attribute, which can be used to access the feature values and the labels, respectively.
+    """
+    reference_data_file = lr_system_folder / file_name
+    return FeatureDataCsvFileParser(file=reference_data_file, label_column="hypothesis").get_instances()
+
+
 def get_model(settings: ModelSettings, training_data: FeatureData, model_storage_path: Path | None) -> LRSystem:
     """
     Obtain a model by loading it from disk, or by fitting it from training data.

pdm.lock

@@ -14,7 +14,7 @@ license = {text = "Apache Software License 2.0"}
 
 lir = {url = "https://github.com/NetherlandsForensicInstitute/lir", rev = "335f8e0dc458b3d969b1a126e492c04cbf8499ab" }
 dependencies = [
-    "lir==1.3.3",
+    "lir==1.4",
     "pandas>=2.3.3",
     "pytest-randomly>=4.0.1",
 ]

pyproject.toml

@@ -0,0 +1,45 @@
+import os
+
+import confidence
+import pytest
+from lir.data.models import FeatureData
+from lir.lrsystems import LRSystem
+from lir.main import initialize_experiments
+from lir.util import check_type
+
+from lrmodule import Path, get_lr_system, get_reference_data
+
+# TEST_FOLDER is a folder that contains packages (folders) with everything relevant for that specific model
+# (e.g. reference data, trained model, etc.).
+TEST_FOLDER = Path(__file__).parent / 'saved_models'
+os.makedirs(TEST_FOLDER, exist_ok=True)
+
+# Generate models at import time so that TEST_FOLDER is populated before @pytest.mark.parametrize
+# evaluates its argument list during collection.
+_yaml_file = Path(__file__).parent.parent / 'validation.yaml'
+_cfg = confidence.Configuration(confidence.loadf(_yaml_file), {'output_path': TEST_FOLDER})
+_exps, _ = initialize_experiments(_cfg)
+for _exp in _exps.values():
+    _exp.run()
+
+
+# Test for every folder in TEST_FOLDER that the `get_lr_system` function can load a trained LR system and that it
+# returns an LR system object.
+@pytest.mark.parametrize("model_folder", [f for f in TEST_FOLDER.iterdir() if f.is_dir()])
+def test_get_lr_system_loads_trained_model(model_folder: Path):
+    """Check that the `get_lr_system` function can load a trained LR system."""
+    lr_system = get_lr_system(model_folder)
+
+    # We should get an LR system object
+    assert lr_system is not None
+    check_type(LRSystem, lr_system)
+
+
+@pytest.mark.parametrize("model_folder", [f for f in TEST_FOLDER.iterdir() if f.is_dir()])
+def test_get_reference_data(model_folder: Path):
+    """Check that the `get_reference_data` function can load reference data."""
+    reference_data = get_reference_data(model_folder)
+
+    # We should get reference data
+    assert reference_data is not None
+    check_type(FeatureData, reference_data)

tests/test_api.py

@@ -152,6 +152,13 @@ experiments:
       - pav
       - ece
       - llr_interval
+      - save_model
+      - method: lrmodule.copy_csv.copy_csv
+        file: ${data_root}/aperture_shear.csv
+        new_file_name: reference_data.csv
+        columns:
+          - hypothesis
+          - ccf
       # - score_histogram (ongetransformeerd)
       # - score_histogram (normalized (area=1))
 
@@ -167,9 +174,16 @@ experiments:
         columns:
           - cllr
           - cllr_min
+      - method: lrmodule.copy_csv.copy_csv
+        file: ${data_root}/firing_pin_impression.csv
+        new_file_name: reference_data.csv
+        columns:
+          - hypothesis
+          - accf
       - pav
       - ece
       - llr_interval
+      - save_model
 
   - name: breech_face_impression_model_accf
     strategy: single_run
@@ -189,6 +203,12 @@ experiments:
       - method: lrmodule.copy_csv.copy_csv
         file: ${data_root}/breech_face_impression.csv
         new_file_name: source_data.csv
+      - method: lrmodule.copy_csv.copy_csv
+        file: ${data_root}/breech_face_impression.csv
+        new_file_name: reference_data.csv
+        columns:
+          - hypothesis
+          - accf
       - save_model
 
 
@@ -204,7 +224,15 @@ experiments:
         columns:
           - cllr
           - cllr_min
+      - method: lrmodule.copy_csv.copy_csv
+        file: ${data_root}/breech_face_impression.csv
+        new_file_name: reference_data.csv
+        columns:
+          - hypothesis
+          - cmc
+          - n
       - pav
       - ece
       - llr_interval
+      - save_model