Creating File bundles (#51)

* file_bundle

* absolute path for pytest

* tests for file bundle

* solving windows path

* Typo

* Correction file bundle

* new tests for file bundel

* typo

* typo

Author: Peyman-N

bids2openminds/main.py

@@ -333,12 +333,71 @@ def create_subjects(subject_id, layout_df, layout, collection):
     return subjects_dict, subject_state_dict, subjects_list
 
 
+def create_file_bundle(BIDS_path, path, collection, parent_file_bundle=None, is_file_repository=False):
+
+    if is_file_repository:
+        openminds_file_bundle = omcore.FileRepository(format=omcore.ContentType.by_name("application/vnd.bids"),
+                                                      iri=IRI(pathlib.Path(BIDS_path).absolute().as_uri()))
+    else:
+        relative_path = os.path.relpath(path, BIDS_path)
+        name = str(relative_path).replace("\\", "/")
+        if name[0] == "_":
+            name = name[1:]
+        openminds_file_bundle = omcore.FileBundle(content_description=f"File bundle created for {relative_path}",
+                                                  name=name,
+                                                  is_part_of=parent_file_bundle)
+
+    files = {}
+    files_size = 0
+    all = os.listdir(path)
+
+    for item in all:
+
+        item_path = str(pathlib.PurePath(path, item))
+
+        if os.path.isfile(item_path) and os.path.basename(item_path) != "openminds.jsonld":
+
+            if is_file_repository:
+                files[item_path] = None
+            else:
+                files[item_path] = [openminds_file_bundle]
+
+            files_size += os.stat(item_path).st_size
+
+        if os.path.isdir(item_path) and os.path.basename(item_path) != "openminds":
+
+            child_files, child_filesizes, _ = create_file_bundle(
+                BIDS_path, item_path, collection, parent_file_bundle=openminds_file_bundle, is_file_repository=False)
+
+            for child_file_path in child_files.keys():
+                if child_file_path not in files:
+                    files[child_file_path] = []
+
+                files[child_file_path].extend(child_files[child_file_path])
+
+            files_size += child_filesizes
+
+    openminds_file_bundle.storage_size = omcore.QuantitativeValue(value=files_size,
+                                                                  unit=controlled_terms.UnitOfMeasurement.by_name(
+                                                                      "byte")
+                                                                  )
+    collection.add(openminds_file_bundle)
+
+    if is_file_repository:
+        openminds_file_repository = openminds_file_bundle
+    else:
+        openminds_file_repository = None
+
+    return files, files_size, openminds_file_repository
+
+
 def create_file(layout_df, BIDS_path, collection):
 
-    file_repository = omcore.FileRepository(
-        format=omcore.ContentType.by_name("application/vnd.bids"),
-        iri=IRI(pathlib.Path(BIDS_path).absolute().as_uri()))
-    collection.add(file_repository)
+    BIDS_path_absolute = pathlib.Path(BIDS_path).absolute()
+
+    file2file_bundle_dic, _, file_repository = create_file_bundle(
+        BIDS_path_absolute, BIDS_path_absolute, collection, is_file_repository=True)
+
     files_list = []
     for index, file in layout_df.iterrows():
         file_format = None
@@ -380,14 +439,16 @@ def create_file(layout_df, BIDS_path, collection):
                         "event sequence")
                     file_format = omcore.ContentType.by_name(
                         "text/tab-separated-values")
+
         file = omcore.File(
             iri=iri,
             content_description=content_description,
             data_types=data_types,
             file_repository=file_repository,
             format=file_format,
             hashes=hashes,
-            # is_part_of=file_bundels
+            is_part_of=file2file_bundle_dic[str(
+                pathlib.Path(path))],
             name=name,
             # special_usage_role
             storage_size=storage_size_obj,

test/test_bids_examples.py

@@ -4,18 +4,18 @@
 import bids2openminds.converter
 
 
-# Dataset information in following order dataset_label, dataset_subject_number, dataset_subject_state_number, dataset_person_number, dataset_files_number, dataset_behavioral_protocol_number
-example_dataset = [("ds003", 13, 13, 2, 58, 1),
-                   ("ds000247", 6, 10, 5, 202, 2),
+# Dataset information in following order dataset_label, dataset_subject_number, dataset_subject_state_number, dataset_person_number, dataset_files_number, dataset_file_bundles_number, dataset_behavioral_protocol_number
+example_dataset = [("ds003", 13, 13, 2, 58, 39, 1),
+                   ("ds000247", 6, 10, 5, 202, 41, 2),
                    # The authors list in 'eeg_cbm' contains non person entities 2 is not correct name (issue raied #43)
-                   ("eeg_cbm", 20, 20, 2, 104, 1),
-                   ("asl001", 1, 1, 2, 8, 0),
+                   ("eeg_cbm", 20, 20, 2, 104, 40, 1),
+                   ("asl001", 1, 1, 2, 8, 3, 0),
                    # Number of files in 'eeg_rest_fmri' is not correct as it doesn't contain files in derivated (issue raied #42)
-                   ("eeg_rest_fmri", 3, 3, 6, 46, 1)]
+                   ("eeg_rest_fmri", 3, 3, 6, 46, 22, 1)]
 
 
-@pytest.mark.parametrize("dataset_label, dataset_subject_number, dataset_subject_state_number, dataset_person_number, dataset_files_number, dataset_behavioral_protocol_number", example_dataset)
-def test_example_datasets(dataset_label, dataset_subject_number, dataset_subject_state_number, dataset_person_number, dataset_files_number, dataset_behavioral_protocol_number):
+@pytest.mark.parametrize("dataset_label, dataset_subject_number, dataset_subject_state_number, dataset_person_number, dataset_files_number, dataset_file_bundles_number, dataset_behavioral_protocol_number", example_dataset)
+def test_example_datasets(dataset_label, dataset_subject_number, dataset_subject_state_number, dataset_person_number, dataset_files_number, dataset_file_bundles_number, dataset_behavioral_protocol_number):
     test_dir = os.path.join("bids-examples", dataset_label)
     bids2openminds.converter.convert(test_dir, save_output=True)
     c = Collection()
@@ -25,23 +25,26 @@ def test_example_datasets(dataset_label, dataset_subject_number, dataset_subject
     subject_state_number = 0
     person_number = 0
     files_number = 0
+    file_bundles_number = 0
     behavioral_protocol_number = 0
 
     for item in c:
-        match item.type_:
-            case "https://openminds.ebrains.eu/core/Subject":
-                subject_number += 1
-            case "https://openminds.ebrains.eu/core/SubjectState":
-                subject_state_number += 1
-            case "https://openminds.ebrains.eu/core/Person":
-                person_number += 1
-            case "https://openminds.ebrains.eu/core/File":
-                files_number += 1
-            case "https://openminds.ebrains.eu/core/BehavioralProtocol":
-                behavioral_protocol_number += 1
+        if item.type_ == "https://openminds.ebrains.eu/core/Subject":
+            subject_number += 1
+        if item.type_ == "https://openminds.ebrains.eu/core/SubjectState":
+            subject_state_number += 1
+        if item.type_ == "https://openminds.ebrains.eu/core/Person":
+            person_number += 1
+        if item.type_ == "https://openminds.ebrains.eu/core/File":
+            files_number += 1
+        if item.type_ == "https://openminds.ebrains.eu/core/FileBundle":
+            file_bundles_number += 1
+        if item.type_ == "https://openminds.ebrains.eu/core/BehavioralProtocol":
+            behavioral_protocol_number += 1
 
     assert dataset_subject_number == subject_number
     assert dataset_subject_state_number == subject_state_number
     assert dataset_person_number == person_number
     assert dataset_files_number == files_number
+    assert dataset_file_bundles_number == file_bundles_number
     assert dataset_behavioral_protocol_number == behavioral_protocol_number

test/test_example_datasets_click.py

@@ -1,9 +1,10 @@
 import os
+import shutil
 from openminds import Collection
 from bids2openminds.converter import convert_click
 from click.testing import CliRunner
 
-(test_data_set, number_of_openminds_files) = ("ds003", 102)
+(test_data_set, number_of_openminds_files) = ("ds003", 141)
 
 
 def test_example_datasets_click():
@@ -17,10 +18,12 @@ def test_example_datasets_click():
 
 def test_example_datasets_click_seperate_files():
     test_dir = os.path.join("bids-examples", test_data_set)
+    path_openminds = os.path.join(test_dir, "openminds")
+    if os.path.isdir(path_openminds):
+        shutil.rmtree(path_openminds)
     runner = CliRunner()
     result = runner.invoke(convert_click, ["--multiple-files", test_dir])
     assert result.exit_code == 0
-    path_openminds = os.path.join(test_dir, "openminds")
     numer_of_files = len(os.listdir(path_openminds))
     assert numer_of_files == number_of_openminds_files
 

test/test_file_bundle.py

@@ -0,0 +1,98 @@
+import pytest
+import os
+import pathlib
+from openminds import Collection
+from bids2openminds.main import create_file_bundle
+
+(test_data_set, number_of_openminds_bundle) = (
+    "ds007", 60)
+
+# example_file_filebundle = ([list of the path to this file],[expected file bundle])
+example_file_filebundle = ((["sub-04", "anat", "sub-04_inplaneT2.nii.gz"], ["sub-04/anat"]),
+                           (["sub-03", "func", "sub-03_task-stopsignalwithmanualresponse_run-02_events.tsv"], ["sub-03/func"]))
+# example_folder_filebundle= ([list of path to this folder],[expected folder name],[expected parent filebundle (None for highest level file bundles)])
+example_folder_filebundle = ((["sub-04", "anat"], ["sub-04/anat"], ["sub-04"]),
+                             (["sub-01"], ["sub-01"], [None]))
+
+
+@pytest.fixture(scope="session")
+def test_dir():
+    test_dir = os.path.join("bids-examples", test_data_set)
+    return test_dir
+
+
+def path_name(path):
+    name = str(path).replace("\\", "/")
+    return name
+
+
+def example_path(test_dir, path_list):
+    path = test_dir
+    for item in path_list:
+        path = os.path.join(path, item)
+    return path
+
+
+@pytest.fixture(scope="session")
+def generate_file_bundle_collection(test_dir):
+    collection = Collection()
+    file_bundles, _, file_repository = create_file_bundle(
+        test_dir, test_dir, collection, is_file_repository=True)
+    return file_bundles, collection, file_repository
+
+
+def test_file_bundles_type(generate_file_bundle_collection):
+    file_bundles, _, _ = generate_file_bundle_collection
+    assert type(file_bundles) is dict
+
+
+def test_number_file_bundle(generate_file_bundle_collection):
+    _, collection, _ = generate_file_bundle_collection
+    m = 0
+    for item in collection:
+        if item.type_ == "https://openminds.ebrains.eu/core/FileBundle":
+            m += 1
+    assert m == number_of_openminds_bundle
+
+
+@pytest.mark.parametrize("path_list, bundle", example_file_filebundle)
+def test_random_file(test_dir, path_list, bundle, generate_file_bundle_collection):
+
+    file_bundles, _, _ = generate_file_bundle_collection
+
+    example_file_path = example_path(test_dir, path_list)
+
+    test_bundles = file_bundles[example_file_path]
+
+    assert len(test_bundles) == len(bundle)
+
+    for test_bundle in test_bundles:
+        assert test_bundle.name in bundle
+
+
+@pytest.mark.parametrize("path_list, bundle, parent_bundle", example_folder_filebundle)
+def test_random_folder(test_dir, path_list, bundle, parent_bundle, generate_file_bundle_collection):
+
+    _, collection, _ = generate_file_bundle_collection
+
+    file_repository_iri = pathlib.Path(test_dir).absolute().as_uri()
+
+    folder_name = path_name(example_path("", path_list))
+
+    dataset_bundle = None
+
+    for item in collection:
+        if item.type_ == "https://openminds.ebrains.eu/core/FileBundle" and item.name == folder_name:
+            # detects only one file bundle have this name
+            assert dataset_bundle is None
+            dataset_bundle = item
+    # asserts at least one file bundle have this name
+    assert dataset_bundle is not None
+
+    assert dataset_bundle.name == bundle[0]
+
+    if parent_bundle is not None:
+        assert dataset_bundle.is_part_of.name == parent_bundle[0]
+    else:
+        assert dataset_bundle.is_part_of.type_ == "https://openminds.ebrains.eu/core/FileRepository"
+        assert dataset_bundle.is_part_of.iri.value == file_repository_iri