wip 2

Author: MaximeBICMTL

pyproject.toml

@@ -57,7 +57,7 @@ preview = true
 
 [tool.ruff.lint]
 ignore = ["E202", "E203", "E221", "E241", "E251", "E272"]
-select = ["E", "EXE", "F", "I", "N", "RUF", "UP", "W"]
+select = ["E", "EXE", "F", "FAST", "I", "N", "RUF", "UP", "W"]
 
 [tool.ruff.lint.pycodestyle]
 max-doc-length = 100

python/lib/config.py

@@ -34,22 +34,7 @@ def get_data_dir_path_config(env: Env) -> Path:
     """
 
     data_dir_path = Path(_get_config_value(env, 'dataDirBasepath'))
-
-    if not data_dir_path.is_dir():
-        log_error_exit(
-            env,
-            (
-                f"The LORIS base data directory path configuration value '{data_dir_path}' does not refer to an"
-                " existing directory."
-            )
-        )
-
-    if not os.access(data_dir_path, os.R_OK) or not os.access(data_dir_path, os.W_OK):
-        log_error_exit(
-            env,
-            f"Missing read or write permission on the LORIS base data directory '{data_dir_path}'.",
-        )
-
+    check_loris_directory(env, data_dir_path, "data")
     return data_dir_path
 
 
@@ -60,22 +45,7 @@ def get_dicom_archive_dir_path_config(env: Env) -> Path:
     """
 
     dicom_archive_dir_path = Path(_get_config_value(env, 'tarchiveLibraryDir'))
-
-    if not dicom_archive_dir_path.is_dir():
-        log_error_exit(
-            env,
-            (
-                f"The LORIS DICOM archive directory path configuration value '{dicom_archive_dir_path}' does not refer"
-                " to an existing directory."
-            ),
-        )
-
-    if not os.access(dicom_archive_dir_path, os.R_OK) or not os.access(dicom_archive_dir_path, os.W_OK):
-        log_error_exit(
-            env,
-            f"Missing read or write permission on the LORIS DICOM archive directory '{dicom_archive_dir_path}'.",
-        )
-
+    check_loris_directory(env, dicom_archive_dir_path, "DICOM archive")
     return dicom_archive_dir_path
 
 
@@ -87,74 +57,43 @@ def get_default_bids_visit_label_config(env: Env) -> str | None:
     return _try_get_config_value(env, 'default_bids_vl')
 
 
-def get_eeg_viz_enabled_config(env: Env) -> bool:
+def get_ephys_visualization_enabled_config(env: Env) -> bool:
     """
-    Get whether the EEG visualization is enabled from the in-database configuration.
+    Get whether the electrophysiology visualization is enabled from the in-database configuration.
     """
 
     eeg_viz_enabled = _try_get_config_value(env, 'useEEGBrowserVisualizationComponents')
     return eeg_viz_enabled == 'true' or eeg_viz_enabled == '1'
 
 
-def get_eeg_chunks_dir_path_config(env: Env) -> Path | None:
+def get_ephys_chunks_dir_path_config(env: Env) -> Path | None:
     """
-    Get the EEG chunks directory path configuration value from the in-database configuration.
+    Get the electrophysiology chunks directory path configuration value from the in-database
+    configuration.
     """
 
-    eeg_chunks_path = _try_get_config_value(env, 'EEGChunksPath')
-    if eeg_chunks_path is None:
+    ephys_chunks_path = _try_get_config_value(env, 'EEGChunksPath')
+    if ephys_chunks_path is None:
         return None
 
-    eeg_chunks_path = Path(eeg_chunks_path)
-
-    if not eeg_chunks_path.is_dir():
-        log_error_exit(
-            env,
-            (
-                f"The configuration value for the LORIS EEG chunks directory path '{eeg_chunks_path}' does not refer to"
-                " an existing directory."
-            ),
-        )
-
-    if not os.access(eeg_chunks_path, os.R_OK) or not os.access(eeg_chunks_path, os.W_OK):
-        log_error_exit(
-            env,
-            f"Missing read or write permission on the LORIS EEG chunks directory '{eeg_chunks_path}'.",
-        )
-
-    return eeg_chunks_path
+    ephys_chunks_path = Path(ephys_chunks_path)
+    check_loris_directory(env, ephys_chunks_path, "electrophysiology chunks")
+    return ephys_chunks_path
 
 
-def get_eeg_pre_package_download_dir_path_config(env: Env) -> Path | None:
+def get_ephys_archive_dir_path_config(env: Env) -> Path | None:
     """
-    Get the EEG pre-packaged download path configuration value from the in-database configuration.
+    Get the electrophysiology archive directory path configuration value from the in-database
+    configuration.
     """
 
-    eeg_pre_package_path = _try_get_config_value(env, 'prePackagedDownloadPath')
-    if eeg_pre_package_path is None:
+    ephys_archive_dir_path = _try_get_config_value(env, 'prePackagedDownloadPath')
+    if ephys_archive_dir_path is None:
         return None
 
-    eeg_pre_package_path = Path(eeg_pre_package_path)
-
-    if not eeg_pre_package_path.is_dir():
-        log_error_exit(
-            env,
-            (
-                "The configuration value for the LORIS EEG pre-packaged download directory path"
-                f" '{eeg_pre_package_path}' does not refer to an existing directory."
-            ),
-        )
-
-    if not os.access(eeg_pre_package_path, os.R_OK) or not os.access(eeg_pre_package_path, os.W_OK):
-        log_error_exit(
-            env,
-            (
-                "Missing read or write permission on the LORIS EEG pre-packaged download directory"
-                f" '{eeg_pre_package_path}'."
-            ),
-        )
-
-    return eeg_pre_package_path
+    ephys_archive_dir_path = Path(ephys_archive_dir_path)
+    check_loris_directory(env, ephys_archive_dir_path, "electrophysiology archive")
+    return ephys_archive_dir_path
 
 
 def _get_config_value(env: Env, setting_name: str) -> str:
@@ -179,6 +118,28 @@ def _get_config_value(env: Env, setting_name: str) -> str:
     return config.value
 
 
+def check_loris_directory(env: Env, dir_path: Path, display_name: str):
+    """
+    Check that a LORIS directory exists and is readable and writable, or exit the program with an
+    error otherwise.
+    """
+
+    if not dir_path.is_dir():
+        log_error_exit(
+            env,
+            (
+                f"The LORIS {display_name} directory path configuration value '{dir_path}' does not refer to an"
+                " existing directory."
+            ),
+        )
+
+    if not os.access(dir_path, os.R_OK) or not os.access(dir_path, os.W_OK):
+        log_error_exit(
+            env,
+            f"Missing read or write permission on the {display_name} directory '{dir_path}'.",
+        )
+
+
 def _try_get_config_value(env: Env, setting_name: str) -> str | None:
     """
     Get a configuration value from the database using a configuration setting name, or return

python/lib/eeg.py

@@ -14,7 +14,7 @@
 
 import lib.exitcode
 import lib.utilities as utilities
-from lib.config import get_eeg_viz_enabled_config
+from lib.config import get_ephys_visualization_enabled_config
 from lib.db.models.physio_file import DbPhysioFile
 from lib.db.models.session import DbSession
 from lib.db.queries.physio_file import try_get_physio_file_with_path
@@ -206,7 +206,7 @@ def register_data(self, derivatives=False, detect=True):
             import_physio_file_archive(self.env, eeg_file, files_to_archive)
 
             # create data chunks for React visualization
-            if get_eeg_viz_enabled_config(self.env):
+            if get_ephys_visualization_enabled_config(self.env):
                 create_physio_channels_chunks(self.env, eeg_file)
 
     def fetch_and_insert_eeg_files(self, derivatives=False, detect=True):

python/lib/import_bids_dataset/archive.py

@@ -4,7 +4,7 @@
 from loris_utils.crypto import compute_file_blake2b_hash
 from loris_utils.path import remove_path_extension
 
-from lib.config import get_data_dir_path_config, get_eeg_pre_package_download_dir_path_config
+from lib.config import get_data_dir_path_config, get_ephys_archive_dir_path_config
 from lib.db.models.physio_event_archive import DbPhysioEventArchive
 from lib.db.models.physio_file import DbPhysioFile
 from lib.db.models.physio_file_archive import DbPhysioFileArchive
@@ -70,9 +70,9 @@ def get_archive_path(env: Env, file_path: Path) -> Path:
     """
 
     archive_rel_path = remove_path_extension(file_path).with_suffix('.tgz')
-    archives_dir_path = get_eeg_pre_package_download_dir_path_config(env)
-    if archives_dir_path is not None:
+    archive_dir_path = get_ephys_archive_dir_path_config(env)
+    if archive_dir_path is not None:
         data_dir_path = get_data_dir_path_config(env)
-        return (archives_dir_path / 'raw' / archive_rel_path.name).relative_to(data_dir_path)
+        return (archive_dir_path / 'raw' / archive_rel_path.name).relative_to(data_dir_path)
     else:
         return archive_rel_path

python/lib/import_bids_dataset/main.py

@@ -23,7 +23,7 @@
 )
 from lib.import_bids_dataset.env import BidsImportEnv
 from lib.import_bids_dataset.events import get_root_events_metadata
-from lib.import_bids_dataset.meg import import_bids_meg_data_type
+from lib.import_bids_dataset.meg.ctf import import_bids_meg_data_type
 from lib.import_bids_dataset.mri import import_bids_mri_data_type
 from lib.import_bids_dataset.print import print_bids_import_summary
 from lib.logging import log, log_error_exit, log_warning
@@ -93,7 +93,7 @@ def import_bids_dataset(env: Env, args: Args, legacy_db: Database):
 
     import_env = BidsImportEnv(
         data_dir_path     = data_dir_path,
-        loris_bids_path   = loris_bids_path,
+        loris_bids_path   = loris_bids_path.relative_to(data_dir_path) if loris_bids_path is not None else None,
         total_files_count = acquisitions_count,
     )
 

python/lib/import_bids_dataset/meg.py python/lib/import_bids_dataset/meg/ctf.pypython/lib/import_bids_dataset/meg.py renamed to python/lib/import_bids_dataset/meg/ctf.py

@@ -3,9 +3,15 @@
 from loris_bids_reader.info import BidsAcquisitionInfo
 from loris_bids_reader.meg.acquisition import MegAcquisition
 from loris_bids_reader.meg.reader import BidsMegDataTypeReader
+from loris_utils.archive import create_archive_with_file
 from loris_utils.error import group_errors_tuple
+from loris_utils.path import add_path_extension
 
-from lib.config import get_eeg_viz_enabled_config
+from lib.config import (
+    get_data_dir_path_config,
+    get_ephys_archive_dir_path_config,
+    get_ephys_visualization_enabled_config,
+)
 from lib.db.models.meg_ctf_head_shape_file import DbMegCtfHeadShapeFile
 from lib.db.models.session import DbSession
 from lib.db.queries.physio_file import try_get_physio_file_with_path
@@ -18,9 +24,12 @@
 from lib.import_bids_dataset.events import insert_events_metadata_file
 from lib.import_bids_dataset.events_tsv import insert_bids_events_file
 from lib.import_bids_dataset.file_type import get_check_bids_imaging_file_type
-from lib.import_bids_dataset.head_shape import insert_head_shape_file
-from lib.import_bids_dataset.meg_channels import read_meg_channels
-from lib.import_bids_dataset.physio import get_check_bids_physio_modality, get_check_bids_physio_output_type
+from lib.import_bids_dataset.meg.ctf_head_shape import insert_head_shape_file
+from lib.import_bids_dataset.physio import (
+    get_check_bids_physio_file_hash,
+    get_check_bids_physio_modality,
+    get_check_bids_physio_output_type,
+)
 from lib.logging import log, log_warning
 from lib.physio.chunking import create_physio_channels_chunks
 from lib.physio.events import FileSource
@@ -73,13 +82,12 @@ def import_bids_meg_acquisition(
     bids_info: BidsAcquisitionInfo,
     head_shape_file: DbMegCtfHeadShapeFile | None,
 ):
-    # TODO: The file is actually a directory, it should be tared before proceeding to the hash.
-    modality, output_type, file_type = group_errors_tuple(
+    modality, output_type, file_type, file_hash = group_errors_tuple(
         f"Error while checking database information for MEG acquisition '{bids_info.name}'.",
         lambda: get_check_bids_physio_modality(env, bids_info.data_type),
         lambda: get_check_bids_physio_output_type(env, args.type or 'raw'),
         lambda: get_check_bids_imaging_file_type(env, 'ctf'),
-        # lambda: get_check_bids_physio_file_hash(env, acquisition),
+        lambda: get_check_bids_physio_file_hash(env, acquisition.ctf_path),
     )
 
     # The files to copy to LORIS, with the source path on the left and the LORIS path on the right.
@@ -96,6 +104,9 @@ def import_bids_meg_acquisition(
 
     check_bids_meg_metadata_files(env, acquisition, bids_info)
 
+    ctf_archive_path = get_ctf_archive_path(env, loris_file_path)
+    create_archive_with_file(import_env.data_dir_path / ctf_archive_path, acquisition.ctf_path)
+
     physio_file = insert_physio_file(
         env,
         session,
@@ -104,11 +115,11 @@ def import_bids_meg_acquisition(
         modality,
         output_type,
         bids_info.scan_row.get_acquisition_time() if bids_info.scan_row is not None else None,
-        None,  # TODO: Use archive.
+        ctf_archive_path,
         head_shape_file,
     )
 
-    # insert_physio_file_parameter(env, physio_file, 'physiological_json_file_blake2b_hash', file_hash)  # ruff:noqa
+    insert_physio_file_parameter(env, physio_file, 'physiological_file_blake2b_hash', file_hash)
     for name, value in acquisition.sidecar_file.data.items():
         insert_physio_file_parameter(env, physio_file, name, value)
 
@@ -139,17 +150,12 @@ def import_bids_meg_acquisition(
 
     log(env, f"MEG file succesfully imported with ID: {physio_file.id}.")
 
-    # TODO: Remove the false.
-    if get_eeg_viz_enabled_config(env):
+    if get_ephys_visualization_enabled_config(env):
         log(env, "Creating visualization chunks...")
         create_physio_channels_chunks(env, physio_file)
 
-    read_meg_channels(env, import_env, physio_file, acquisition, bids_info)
-
     env.db.commit()
 
-    import_env.imported_files_count += 1
-
 
 def check_bids_meg_metadata_files(env: Env, acquisition: MegAcquisition, bids_info: BidsAcquisitionInfo):
     """
@@ -165,3 +171,19 @@ def check_bids_meg_metadata_files(env: Env, acquisition: MegAcquisition, bids_in
 
     if acquisition.events_file is not None and acquisition.events_file.dictionary is not None:
         log_warning(env, f"No events dictionary file found for acquisition '{bids_info.name}'.")
+
+
+def get_ctf_archive_path(env: Env, loris_ctf_path: Path) -> Path:
+    """
+    Get the path of a CTF archive.
+    """
+
+    archive_rel_path = add_path_extension(loris_ctf_path, 'tgz')
+    archive_dir_path = get_ephys_archive_dir_path_config(env)
+    if archive_dir_path is not None:
+        data_dir_path = get_data_dir_path_config(env)
+        archive_path = archive_dir_path / 'ctf' / archive_rel_path.name
+        archive_path.parent.mkdir(exist_ok=True, parents=True)
+        return (archive_path).relative_to(data_dir_path)
+    else:
+        return archive_rel_path

…on/lib/import_bids_dataset/head_shape.py …mport_bids_dataset/meg/ctf_head_shape.pypython/lib/import_bids_dataset/head_shape.py renamed to python/lib/import_bids_dataset/meg/ctf_head_shape.py python/lib/import_bids_dataset/head_shape.py renamed to python/lib/import_bids_dataset/meg/ctf_head_shape.py

@@ -27,13 +27,13 @@ def insert_head_shape_file(
     env.db.add(db_head_shape_file)
     env.db.flush()
 
-    for name, coordinates in head_shape_file.points.items():
+    for name, point in head_shape_file.points.items():
         env.db.add(DbMegCtfHeadShapePoint(
             file_id = db_head_shape_file.id,
             name    = name,
-            x       = coordinates.x,
-            y       = coordinates.y,
-            z       = coordinates.z,
+            x       = point.x,
+            y       = point.y,
+            z       = point.z,
         ))
 
     env.db.flush()