diff --git a/.gitignore b/.gitignore index 03505ba..d216827 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ output/ lrsystem_output/ .testmondata tests/test_model_storage/ +tests/saved_models/ # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/lrmodule/__init__.py b/lrmodule/__init__.py index 533d8ab..53d06ff 100644 --- a/lrmodule/__init__.py +++ b/lrmodule/__init__.py @@ -1,8 +1,10 @@ +import pickle from pathlib import Path import numpy as np from lir.config.lrsystem_architectures import specific_source from lir.data.models import FeatureData, LLRData +from lir.datasets.feature_data_csv import FeatureDataCsvFileParser from lir.lrsystems.lrsystems import LRSystem from lrmodule import persistence @@ -10,6 +12,49 @@ from lrmodule.lrsystem import get_trained_model +def get_lr_system(lr_system_folder: Path, file_name: str = "model.pkl") -> LRSystem: + """ + Load a trained LR system from disk from a given folder. + + It is expected that the folder contains a file named "model.pkl" (or another name specified by file_name), + which is a pickled LRSystem object. The function loads this object and returns it. + + The system is returned as an instance of the LRSystem class. This class provides an apply method, which can be used + to calculate LLRs for given features. These features should be contained in a FeatureData object from lir. + + Example usage: + ``` + from lir.data.models import FeatureData + + lr_system = get_lr_system(Path("path/to/lr_system_folder/")) + + # Create three instances of features, each with two feature values. + features = np.array([[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]]) + feature_data = FeatureData(features=features) + llr_data = lr_system.apply(feature_data) + ``` + """ + with (lr_system_folder / file_name).open("rb") as f: + return pickle.load(f) # noqa: S301 + + +def get_reference_data(lr_system_folder: Path, file_name: str = "reference_data.csv") -> FeatureData: + """Load reference data from disk. + + It is expected that the folder contains a file named "reference_data.csv" (or another name specified by file_name), + which is a CSV file containing the reference data. The function loads this data and returns it as a FeatureData + object. + + If the data has `n` features, the CSV file should have `n+1` columns. One of the columns should be named + "hypothesis", and contain the labels for the data. The other columns should contain the feature values. + + The data is returned as an instance of the FeatureData class from lir. This class has a features and a labels + attribute, which can be used to access the feature values and the labels, respectively. + """ + reference_data_file = lr_system_folder / file_name + return FeatureDataCsvFileParser(file=reference_data_file, label_column="hypothesis").get_instances() + + def get_model(settings: ModelSettings, training_data: FeatureData, model_storage_path: Path | None) -> LRSystem: """ Obtain a model by loading it from disk, or by fitting it from training data. diff --git a/pdm.lock b/pdm.lock index 924281c..0a45604 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:d3c66b18110dbf634944d8965f9867a5418c14258fa7c4bd2a50d757c8232541" +content_hash = "sha256:fbd9387bbac757a746bc9c3282b18f4ca020292dbcb406e63e138c4fe0a72d5f" [[metadata.targets]] requires_python = ">=3.13,<3.14" @@ -594,7 +594,7 @@ files = [ [[package]] name = "lir" -version = "1.3.3" +version = "1.4.0" requires_python = "<3.15,>=3.12" summary = "Package for optimising and evaluating Likelihood Ratio (LR) systems." groups = ["default"] @@ -615,7 +615,7 @@ dependencies = [ "tqdm>=4.67.1", ] files = [ - {file = "lir-1.3.3-py3-none-any.whl", hash = "sha256:290b3886fa11093ed28a56c8a0e80e12840c3b5f2ad8f697a2589ebd10918b93"}, + {file = "lir-1.4.0-py3-none-any.whl", hash = "sha256:d7219a9c43a14b7760add4ecb8cba840953f59c72e7a0557de2435e54e9e0b89"}, ] [[package]] diff --git a/pyproject.toml b/pyproject.toml index 471ac10..7dfa93f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ license = {text = "Apache Software License 2.0"} lir = {url = "https://github.com/NetherlandsForensicInstitute/lir", rev = "335f8e0dc458b3d969b1a126e492c04cbf8499ab" } dependencies = [ - "lir==1.3.3", + "lir==1.4", "pandas>=2.3.3", "pytest-randomly>=4.0.1", ] diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 0000000..3528581 --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,45 @@ +import os + +import confidence +import pytest +from lir.data.models import FeatureData +from lir.lrsystems import LRSystem +from lir.main import initialize_experiments +from lir.util import check_type + +from lrmodule import Path, get_lr_system, get_reference_data + +# TEST_FOLDER is a folder that contains packages (folders) with everything relevant for that specific model +# (e.g. reference data, trained model, etc.). +TEST_FOLDER = Path(__file__).parent / 'saved_models' +os.makedirs(TEST_FOLDER, exist_ok=True) + +# Generate models at import time so that TEST_FOLDER is populated before @pytest.mark.parametrize +# evaluates its argument list during collection. +_yaml_file = Path(__file__).parent.parent / 'validation.yaml' +_cfg = confidence.Configuration(confidence.loadf(_yaml_file), {'output_path': TEST_FOLDER}) +_exps, _ = initialize_experiments(_cfg) +for _exp in _exps.values(): + _exp.run() + + +# Test for every folder in TEST_FOLDER that the `get_lr_system` function can load a trained LR system and that it +# returns an LR system object. +@pytest.mark.parametrize("model_folder", [f for f in TEST_FOLDER.iterdir() if f.is_dir()]) +def test_get_lr_system_loads_trained_model(model_folder: Path): + """Check that the `get_lr_system` function can load a trained LR system.""" + lr_system = get_lr_system(model_folder) + + # We should get an LR system object + assert lr_system is not None + check_type(LRSystem, lr_system) + + +@pytest.mark.parametrize("model_folder", [f for f in TEST_FOLDER.iterdir() if f.is_dir()]) +def test_get_reference_data(model_folder: Path): + """Check that the `get_reference_data` function can load reference data.""" + reference_data = get_reference_data(model_folder) + + # We should get reference data + assert reference_data is not None + check_type(FeatureData, reference_data) diff --git a/tests/test_validation_yaml.py b/tests/test_validation_yaml.py deleted file mode 100644 index fce71d2..0000000 --- a/tests/test_validation_yaml.py +++ /dev/null @@ -1,16 +0,0 @@ -from pathlib import Path - -import confidence -from lir.main import initialize_experiments - - -def test_validation_yaml(tmpdir: Path): - """Test if the validation.yaml file can be parsed correctly. - - Does not test correctness of the content, only that it can be parsed without errors. - Running the whole setup will take too long for a unit test. - """ - validation_file = Path(__file__).parent.parent / "validation.yaml" - cfg = confidence.Configuration(confidence.loadf(validation_file), {'output_path': tmpdir}) - setup, _ = initialize_experiments(cfg) - diff --git a/validation.yaml b/validation.yaml index 7f88546..e9c7343 100644 --- a/validation.yaml +++ b/validation.yaml @@ -152,6 +152,13 @@ experiments: - pav - ece - llr_interval + - save_model + - method: lrmodule.copy_csv.copy_csv + file: ${data_root}/aperture_shear.csv + new_file_name: reference_data.csv + columns: + - hypothesis + - ccf # - score_histogram (ongetransformeerd) # - score_histogram (normalized (area=1)) @@ -167,9 +174,16 @@ experiments: columns: - cllr - cllr_min + - method: lrmodule.copy_csv.copy_csv + file: ${data_root}/firing_pin_impression.csv + new_file_name: reference_data.csv + columns: + - hypothesis + - accf - pav - ece - llr_interval + - save_model - name: breech_face_impression_model_accf strategy: single_run @@ -189,6 +203,12 @@ experiments: - method: lrmodule.copy_csv.copy_csv file: ${data_root}/breech_face_impression.csv new_file_name: source_data.csv + - method: lrmodule.copy_csv.copy_csv + file: ${data_root}/breech_face_impression.csv + new_file_name: reference_data.csv + columns: + - hypothesis + - accf - save_model @@ -204,7 +224,15 @@ experiments: columns: - cllr - cllr_min + - method: lrmodule.copy_csv.copy_csv + file: ${data_root}/breech_face_impression.csv + new_file_name: reference_data.csv + columns: + - hypothesis + - cmc + - n - pav - ece - llr_interval + - save_model