From 7b38875814fbd1edc86268318c1e4d372759a474 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Fri, 27 Jun 2025 13:18:32 +0100 Subject: [PATCH] First commit aeon conveter. --- datashuttle/converters/convert_sumiya.py | 185 ++++++++++++++++++ .../behav/ExperimentEvents/20250516_M8_ci.bin | 0 .../ExperimentEvents/20250516_M8_ci_.bin | 0 .../behav/ExperimentEvents/config.yaml | 183 +++++++++++++++++ .../behav/ExperimentEvents/position_log .csv | 0 .../behav/SessionSettings/session.yaml | 49 +++++ .../VideoData/Camera1_2025-05-16T14_11_17.avi | 0 .../VideoData/Camera2_2025-05-16T14_11_17.avi | 0 .../behav/ExperimentEvents/20250516_M8_ci.bin | 0 .../ExperimentEvents/20250516_M8_ci_.bin | 0 .../behav/ExperimentEvents/config.yaml | 183 +++++++++++++++++ .../behav/ExperimentEvents/position_log .csv | 0 .../behav/SessionSettings/session.yaml | 49 +++++ .../VideoData/Camera1_2025-05-16T14_11_17.avi | 0 .../VideoData/Camera2_2025-05-16T14_11_17.avi | 0 .../behav/ExperimentEvents/20250516_M8_ci.bin | 0 .../ExperimentEvents/20250516_M8_ci_.bin | 0 .../behav/ExperimentEvents/config.yaml | 183 +++++++++++++++++ .../behav/ExperimentEvents/position_log .csv | 0 .../behav/SessionSettings/session.yaml | 49 +++++ .../VideoData/Camera1_2025-05-16T14_11_17.avi | 0 .../VideoData/Camera2_2025-05-16T14_11_17.avi | 0 .../behav/ExperimentEvents/20250516_M8_ci.bin | 0 .../ExperimentEvents/20250516_M8_ci_.bin | 0 .../behav/ExperimentEvents/config.yaml | 183 +++++++++++++++++ .../behav/ExperimentEvents/position_log .csv | 0 .../behav/SessionSettings/session.yaml | 49 +++++ .../VideoData/Camera1_2025-05-16T14_11_17.avi | 0 .../VideoData/Camera2_2025-05-16T14_11_17.avi | 0 .../behav/SessionSettings/session.yaml | 49 +++++ .../behav/ExperimentEvents/20250516_M8_ci.bin | 0 .../ExperimentEvents/20250516_M8_ci_.bin | 0 .../behav/ExperimentEvents/config.yaml | 183 +++++++++++++++++ .../behav/ExperimentEvents/position_log .csv | 0 .../behav/SessionSettings/session.yaml | 49 +++++ .../VideoData/Camera1_2025-05-16T14_11_17.avi | 0 .../VideoData/Camera2_2025-05-16T14_11_17.avi | 0 datashuttle/converters/run_convert_sumiya.py | 14 ++ datashuttle/datashuttle_class.py | 62 +++++- 39 files changed, 1468 insertions(+), 2 deletions(-) create mode 100644 datashuttle/converters/convert_sumiya.py create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-001_datetime-20240607T125530/behav/ExperimentEvents/20250516_M8_ci.bin create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-001_datetime-20240607T125530/behav/ExperimentEvents/20250516_M8_ci_.bin create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-001_datetime-20240607T125530/behav/ExperimentEvents/config.yaml create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-001_datetime-20240607T125530/behav/ExperimentEvents/position_log .csv create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-001_datetime-20240607T125530/behav/SessionSettings/session.yaml create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-001_datetime-20240607T125530/behav/VideoData/Camera1_2025-05-16T14_11_17.avi create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-001_datetime-20240607T125530/behav/VideoData/Camera2_2025-05-16T14_11_17.avi create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-002_datetime-20240607T125530/behav/ExperimentEvents/20250516_M8_ci.bin create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-002_datetime-20240607T125530/behav/ExperimentEvents/20250516_M8_ci_.bin create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-002_datetime-20240607T125530/behav/ExperimentEvents/config.yaml create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-002_datetime-20240607T125530/behav/ExperimentEvents/position_log .csv create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-002_datetime-20240607T125530/behav/SessionSettings/session.yaml create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-002_datetime-20240607T125530/behav/VideoData/Camera1_2025-05-16T14_11_17.avi create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-002_datetime-20240607T125530/behav/VideoData/Camera2_2025-05-16T14_11_17.avi create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-003_datetime-20240607T125530/behav/ExperimentEvents/20250516_M8_ci.bin create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-003_datetime-20240607T125530/behav/ExperimentEvents/20250516_M8_ci_.bin create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-003_datetime-20240607T125530/behav/ExperimentEvents/config.yaml create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-003_datetime-20240607T125530/behav/ExperimentEvents/position_log .csv create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-004_datetime-20240607T125530/behav/SessionSettings/session.yaml create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-004_datetime-20240607T125530/behav/VideoData/Camera1_2025-05-16T14_11_17.avi create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-004_datetime-20240607T125530/behav/VideoData/Camera2_2025-05-16T14_11_17.avi create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-005_datetime-20240607T125530/behav/ExperimentEvents/20250516_M8_ci.bin create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-005_datetime-20240607T125530/behav/ExperimentEvents/20250516_M8_ci_.bin create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-005_datetime-20240607T125530/behav/ExperimentEvents/config.yaml create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-005_datetime-20240607T125530/behav/ExperimentEvents/position_log .csv create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-005_datetime-20240607T125530/behav/SessionSettings/session.yaml create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-005_datetime-20240607T125530/behav/VideoData/Camera1_2025-05-16T14_11_17.avi create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-005_datetime-20240607T125530/behav/VideoData/Camera2_2025-05-16T14_11_17.avi create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-006_datetime-20240607T125530/behav/SessionSettings/session.yaml create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-007_datetime-20240607T125530/behav/ExperimentEvents/20250516_M8_ci.bin create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-007_datetime-20240607T125530/behav/ExperimentEvents/20250516_M8_ci_.bin create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-007_datetime-20240607T125530/behav/ExperimentEvents/config.yaml create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-007_datetime-20240607T125530/behav/ExperimentEvents/position_log .csv create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-007_datetime-20240607T125530/behav/SessionSettings/session.yaml create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-007_datetime-20240607T125530/behav/VideoData/Camera1_2025-05-16T14_11_17.avi create mode 100644 datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-007_datetime-20240607T125530/behav/VideoData/Camera2_2025-05-16T14_11_17.avi create mode 100644 datashuttle/converters/run_convert_sumiya.py diff --git a/datashuttle/converters/convert_sumiya.py b/datashuttle/converters/convert_sumiya.py new file mode 100644 index 000000000..8781d22a8 --- /dev/null +++ b/datashuttle/converters/convert_sumiya.py @@ -0,0 +1,185 @@ +"""Custom converter to NeuroBlueprint. + +In datashuttle, `project.convert(path_to_module)` can take a custom converter to allow +conversion of non-NeuroBlueprint formats to NeuroBlueprint. + +To write a custom converter, create a new module that includes the function +`converter_func(project)` that takes the current datashuttle `project` as an input. +When `project.convert(path_to_module) is called, the project will be passed +to this custom convter function. You then have access to the current project +and can detect and convert your non-NeuroBlueprint folders. + +Within the converter function, you can use `local_path = project.get_local_path() / "rawdata"` +to get the current local path. You can glob for folders formatted to your style, and +iterate through them to convert. + +Often, you may collect new sessions in a non-NeuroBlueprint format for subjects +that are already in NeuroBlueprint format. `project` has a number of convenience +functions to return the state of the project and infer the NeuroBlueprint subject names. + +For example, `project.get_sub_names_from_key_value_pair(key, value)` will return +subject names that include any given key-value pair. You can use this to match +identifiers from your format to an existing NeuroBlueprint format name with +that identifier as a key-value pair. It is therefore recommended to include unique +subject identifiers from your format as a field in the subject folder name +e.g. `sub-001_myID-1324`. + +If the subject does not exist, you can use `project.get_next_sub()` to return +the next NeuroBlueprint subject id e.g. sub-002. You can then append your format +id to the subject name. + +To find the next NeuroBlueprint session ID, you can use `project.get_next_ses()`. +Then, you can place output folders into NeuroBlueprint datatype folders. + +Once you have your NeuroBlueprint subject, session names and paths to +the datatype folders, you can create a new tree, move the data, and +delete the old (now empty) folders. + +`project.convert()` starts a logger than can be +accessed with `project.log_and_message()` or `project.log_and_raise_error()`. +The corresponding log file is saved with the rest of the datashuttle logs. +""" + +import shutil +from datetime import datetime + +import yaml + +# TODO: test some more complex cases, add proper tests. +# TODO: could do a check here that all dates are older than this one +# TODO: add to general validation, that all date / datetime keys are in order e.g. ses-001 must be later than ses-002 + + +def get_mapping(): + return { + "ExperimentEvents": "behav", + "SessionSettings": "behav", # TODO: convert this to a metadata file, we will add metadata spec soon + "VideoData": "behav", + } + + +def converter_func(project): + """Convert Aeon-formatted Bonsai output to NeuroBlueprint. + + `project.convert()` starts a logger than can be + accessed with `project.log_and_message()` or `project.log_and_raise_error()`. + The corresponding log file is saved with the rest of the datashuttle logs. + + First, search for all folders in the local "rawdata" path. + Folders that are a valid ISO8601 datetime are assumed to be + Aeon-formatted sessions. Iterate through these sessions + and convert them to NeuroBlueprint. + + For each session, find the Aeon subject ID from the .yaml file. + Use this in the subject name e.g. `sub-001_id-plimbo`. + Check if thus subject already exists in the NeuroBlueprint project, + if so then use that, otherwise create a new subject based on + `project.get_next_sub()`. + + Create a session name as ses-..._datetime-...where datetime is the Aeon + session folder name. Use `project.get_next_ses()` to find the NeuroBlueprint + session ID if the subject already exists, otherwise just use `ses-001_datetime-...`. + + Next, use the `get_mapping()` dictionary to map Aeon device folders to their + NeuroBlueprint datatypes. Now we have the subject, session and datatype name + for a given decide folder, so we can move it to a new NeuroBlueprint folder. + + Once all device folders are moved, delete the old Aeon folder tree. It assumes + all files have been moved, if any remain an error is raised. + """ + rawdata_path = project.get_local_path() / "rawdata" + + aeon_session_folders = [] + + # Find all folders in the project that are aeon format (datetime) + for item in rawdata_path.glob("*"): + if item.is_dir() and item.name[:4] != "sub-": + try: + datetime.fromisoformat(item.name) + except ValueError: + continue + + aeon_session_folders.append(item) + + # Confidence check the folders are sorted by datetime, otherwise + # the session ids for per-session conversion will go out of order. + assert aeon_session_folders == sorted( + aeon_session_folders, key=lambda p: p.stat().st_ctime + ) + + # Convert each session + for session_folder in aeon_session_folders: + convert_aeon_session_folder(project, session_folder) + + +def convert_aeon_session_folder(project, input_folder): + """Move a single Aeon session to NeuroBlueprint folder.""" + # Process the aeon session folder name to daetime + aeon_ses_name = input_folder.name + + datetime_ = datetime.fromisoformat(aeon_ses_name) + formatted_datetime = datetime_.strftime("%Y%m%dT%H%M%S") + + # Find the subject in the Aeon session + session_yaml_path = input_folder / "SessionSettings" / "session.yaml" + + if not session_yaml_path.is_file(): + project.log_and_raise_error( + f"Cannot find session.yaml for input session {input_folder}. Cannot convert.", + FileNotFoundError, + ) + + with open(input_folder / "SessionSettings" / "session.yaml") as f: + aeon_subject_id = yaml.safe_load(f)["animalId"] + + # If the subject already exists, then get the full NeuroBlueprint + # subject name and next available session id. Otherwise, create a new + # name and use session id 001. + if sub_name := project.get_sub_names_from_key_value_pair( + "id", aeon_subject_id + ): + assert len(sub_name) == 1, ( + f"Multiple folders detected with the same unique aeon subject id: {aeon_subject_id}." + ) + sub_name = sub_name[0] + + ses_id = project.get_next_ses("rawdata", sub=sub_name) + ses_name = f"{ses_id}_datetime-{formatted_datetime}" + + else: + sub_id = project.get_next_sub("rawdata") + sub_name = f"{sub_id}_id-{aeon_subject_id}" + ses_name = f"ses-001_datetime-{formatted_datetime}" + + # Create the paths for the new NeuroBlueprint folder + nb_sub_path = project.get_local_path() / "rawdata" / sub_name + nb_ses_path = nb_sub_path / ses_name + + # For each device, get its datatype and move it + # to the new NeuroBlueprint folder + aeon_device_folders = list(input_folder.glob("*")) + + datatype_mapping = get_mapping() + + for device_folder in aeon_device_folders: + if device_folder.name in datatype_mapping: # use get + datatype = datatype_mapping[device_folder.name] + + nb_datatype_path = nb_ses_path / datatype + nb_datatype_path.mkdir(parents=True, exist_ok=True) + + project.log_and_message( + f"Moving folders:\nsource: {device_folder.as_posix()}\ntarget: {nb_datatype_path.as_posix()}" + ) + + shutil.move(device_folder, nb_datatype_path) + + # Delete the original folder. We can make this crash + # if there are files remaining instead of using rmtree + if any([file for file in input_folder.rglob("*") if f.is_file()]): + project.log_and_raise_error( + f"Session {aeon_ses_name} is not empty after conversion, cannot delete.", + RuntimeError, + ) + else: + shutil.rmtree(input_folder) # TODO: input folder path? diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-001_datetime-20240607T125530/behav/ExperimentEvents/20250516_M8_ci.bin b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-001_datetime-20240607T125530/behav/ExperimentEvents/20250516_M8_ci.bin new file mode 100644 index 000000000..e69de29bb diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-001_datetime-20240607T125530/behav/ExperimentEvents/20250516_M8_ci_.bin b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-001_datetime-20240607T125530/behav/ExperimentEvents/20250516_M8_ci_.bin new file mode 100644 index 000000000..e69de29bb diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-001_datetime-20240607T125530/behav/ExperimentEvents/config.yaml b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-001_datetime-20240607T125530/behav/ExperimentEvents/config.yaml new file mode 100644 index 000000000..b30771dda --- /dev/null +++ b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-001_datetime-20240607T125530/behav/ExperimentEvents/config.yaml @@ -0,0 +1,183 @@ +base_tunnel: + eye_fov: + fov: 100 + fov_shift: 57 + speed_gain: 1.5 + wall_length: 4.5 + wall_model: walls.egg + wall_spacing: 9 +card: + position: + - 0.9 + - 0.9 + size: + - 0.2 + - 0.2 +daqChannel: + finalV1: Dev1/port1/line0 + finalV2: Dev1/port1/line3 + fiveV: Dev1/port2/line3 + mo1: Dev1/port0/line0 + mo2: Dev1/port2/line6 + odour0: Dev1/port2/line4 + odour1: Dev1/port0/line4 + odour2: Dev1/port1/line7 + odour3: Dev1/port0/line7 + odour4: Dev1/port1/line6 + odour5: Dev1/port0/line2 + odour6: Dev1/port2/line5 + odour7: Dev1/port0/line3 + odour8: Dev1/port1/line4 + odour9: Dev1/port0/line5 + spout1: + chan: Dev1/ai10 + max_value: 10 + min_value: 0 + threshold: 1 + threeV: Dev1/port1/line1 + threeV2: Dev1/port1/line2 + valve1: Dev2/port0/line1 +flip_tunnel: + continuous_corridor: true + goals: + - - 20 + - 36 + - - 74 + - 90 + - - 155 + - 171 + - - 263 + - 279 + io_module: nidaq + landmarks: + - - 20 + - 36 + - - 47 + - 63 + - - 74 + - 90 + - - 101 + - 117 + - - 128 + - 144 + - - 155 + - 171 + - - 182 + - 198 + - - 209 + - 225 + - - 236 + - 252 + - - 263 + - 279 + landmarks_sequence: + - 0 + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + length: 288 + manual_reward_with_space: true + margin_start: 9 + modd1_odours: + - 1 + - 3 + - 5 + - 7 + - 9 + neutral_texture: grey.png + odour_diffs: + flush: 1 + flush_other: 0.75 + flush_same: 1.5 + odour_overlap: 0.05 + reward_distance: 9 + reward_length: + assist: 0.3 + correct: 0.3 + manual: 0.1 + wrong: 0.3 + reward_prob: 1 + reward_tone_length: 1 + sleep_time: 0 + sound_at_A: true + sound_dir: examples/sounds/ + sounds: + 0: 6kHz_tone.ogg + speed_limit: 15 + stimulus_onset: 12 +goal_ids: +- 1 +- 0 +- 2 +- 0 +- 0 +- 3 +- 0 +- 0 +- 0 +- 4 +inputs: + speed: + chan: Dev1/ctr0 + diameter: 0.197 + error_value: 4000000000 + pulses_per_rev: 1000 + threshold: 0.001 +logger: + date_time: '2025-05-16 14:11:49.094411' + experiment_start_time: '14:11' + foldername: Z:\public\projects\AtApSuKuSaRe_20250129_HFScohort2\TAA0000066\TrainingData\250516 +monitor: + dual_monitor: true + monitor1: + width: 1920 + monitor2: + height: 1080 + width: 1920 + monitor3: + height: 1080 + width: 1920 +outputs: {} +sequence_task: + protocol: olfactory_support_l1 + rulename: olfactory_support +walls_sequence: +- random_dots.png +- random_dots.png +- random_dots.png +- logs.png +- logs.png +- random_dots.png +- grating1.jpg +- grating1.jpg +- random_dots.png +- tiles.png +- tiles.png +- random_dots.png +- big_light_rectangles.png +- big_light_rectangles.png +- random_dots.png +- grass.png +- grass.png +- random_dots.png +- big_dark_rectangles.png +- big_dark_rectangles.png +- random_dots.png +- leaves.png +- leaves.png +- random_dots.png +- waves.png +- waves.png +- random_dots.png +- bark.png +- bark.png +- random_dots.png +- big_light_circles.png +- big_light_circles.png +- random_dots.png diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-001_datetime-20240607T125530/behav/ExperimentEvents/position_log .csv b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-001_datetime-20240607T125530/behav/ExperimentEvents/position_log .csv new file mode 100644 index 000000000..e69de29bb diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-001_datetime-20240607T125530/behav/SessionSettings/session.yaml b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-001_datetime-20240607T125530/behav/SessionSettings/session.yaml new file mode 100644 index 000000000..f298dc806 --- /dev/null +++ b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-001_datetime-20240607T125530/behav/SessionSettings/session.yaml @@ -0,0 +1,49 @@ +# yaml-language-server: $schema=schema.json + +animalId: plimbo +drawUntilPositionOffset: 5 +drawSetsWhenDistanceFromEndReached: 20 +availableRewardPositions: 4 +corridorWidth: 2 + +trial: { + randomiseSequence: true, + offsets: [0, 1, 5], + landmarks: [ + [{size: 3, texture: grating1, odour: odour1, rewardSequencePosition: 0}], + [{size: 3, texture: logs, odour: odour0, rewardSequencePosition: -1}, {size: 3, texture: bark, odour: odour0, rewardSequencePosition: -1}], + [{size: 3, texture: grating1, odour: odour2, rewardSequencePosition: 1}], + [{size: 3, texture: logs, odour: odour0, rewardSequencePosition: -1}, {size: 3, texture: bark, odour: odour0, rewardSequencePosition: -1}], + [{size: 3, texture: grating1, odour: odour3, rewardSequencePosition: 2}], + [{size: 3, texture: logs, odour: odour0, rewardSequencePosition: -1}, {size: 3, texture: bark, odour: odour0, rewardSequencePosition: -1}], + [{size: 3, texture: grating1, odour: odour4, rewardSequencePosition: 3}], + [{size: 3, texture: logs, odour: odour0, rewardSequencePosition: -1}, {size: 3, texture: bark, odour: odour0, rewardSequencePosition: -1}], + ] +} + +olfactometerCommands: [ + { + alias: odour1, + prepareCommand: [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + releaseCommand: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0], + flushCommand: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] + }, + { + alias: odour2, + prepareCommand: [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + releaseCommand: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0], + flushCommand: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] + }, + { + alias: odour3, + prepareCommand: [0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0], + releaseCommand: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1], + flushCommand: [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] + }, + { + alias: odour4, + prepareCommand: [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0], + releaseCommand: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + flushCommand: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + } +] diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-001_datetime-20240607T125530/behav/VideoData/Camera1_2025-05-16T14_11_17.avi b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-001_datetime-20240607T125530/behav/VideoData/Camera1_2025-05-16T14_11_17.avi new file mode 100644 index 000000000..e69de29bb diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-001_datetime-20240607T125530/behav/VideoData/Camera2_2025-05-16T14_11_17.avi b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-001_datetime-20240607T125530/behav/VideoData/Camera2_2025-05-16T14_11_17.avi new file mode 100644 index 000000000..e69de29bb diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-002_datetime-20240607T125530/behav/ExperimentEvents/20250516_M8_ci.bin b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-002_datetime-20240607T125530/behav/ExperimentEvents/20250516_M8_ci.bin new file mode 100644 index 000000000..e69de29bb diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-002_datetime-20240607T125530/behav/ExperimentEvents/20250516_M8_ci_.bin b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-002_datetime-20240607T125530/behav/ExperimentEvents/20250516_M8_ci_.bin new file mode 100644 index 000000000..e69de29bb diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-002_datetime-20240607T125530/behav/ExperimentEvents/config.yaml b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-002_datetime-20240607T125530/behav/ExperimentEvents/config.yaml new file mode 100644 index 000000000..b30771dda --- /dev/null +++ b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-002_datetime-20240607T125530/behav/ExperimentEvents/config.yaml @@ -0,0 +1,183 @@ +base_tunnel: + eye_fov: + fov: 100 + fov_shift: 57 + speed_gain: 1.5 + wall_length: 4.5 + wall_model: walls.egg + wall_spacing: 9 +card: + position: + - 0.9 + - 0.9 + size: + - 0.2 + - 0.2 +daqChannel: + finalV1: Dev1/port1/line0 + finalV2: Dev1/port1/line3 + fiveV: Dev1/port2/line3 + mo1: Dev1/port0/line0 + mo2: Dev1/port2/line6 + odour0: Dev1/port2/line4 + odour1: Dev1/port0/line4 + odour2: Dev1/port1/line7 + odour3: Dev1/port0/line7 + odour4: Dev1/port1/line6 + odour5: Dev1/port0/line2 + odour6: Dev1/port2/line5 + odour7: Dev1/port0/line3 + odour8: Dev1/port1/line4 + odour9: Dev1/port0/line5 + spout1: + chan: Dev1/ai10 + max_value: 10 + min_value: 0 + threshold: 1 + threeV: Dev1/port1/line1 + threeV2: Dev1/port1/line2 + valve1: Dev2/port0/line1 +flip_tunnel: + continuous_corridor: true + goals: + - - 20 + - 36 + - - 74 + - 90 + - - 155 + - 171 + - - 263 + - 279 + io_module: nidaq + landmarks: + - - 20 + - 36 + - - 47 + - 63 + - - 74 + - 90 + - - 101 + - 117 + - - 128 + - 144 + - - 155 + - 171 + - - 182 + - 198 + - - 209 + - 225 + - - 236 + - 252 + - - 263 + - 279 + landmarks_sequence: + - 0 + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + length: 288 + manual_reward_with_space: true + margin_start: 9 + modd1_odours: + - 1 + - 3 + - 5 + - 7 + - 9 + neutral_texture: grey.png + odour_diffs: + flush: 1 + flush_other: 0.75 + flush_same: 1.5 + odour_overlap: 0.05 + reward_distance: 9 + reward_length: + assist: 0.3 + correct: 0.3 + manual: 0.1 + wrong: 0.3 + reward_prob: 1 + reward_tone_length: 1 + sleep_time: 0 + sound_at_A: true + sound_dir: examples/sounds/ + sounds: + 0: 6kHz_tone.ogg + speed_limit: 15 + stimulus_onset: 12 +goal_ids: +- 1 +- 0 +- 2 +- 0 +- 0 +- 3 +- 0 +- 0 +- 0 +- 4 +inputs: + speed: + chan: Dev1/ctr0 + diameter: 0.197 + error_value: 4000000000 + pulses_per_rev: 1000 + threshold: 0.001 +logger: + date_time: '2025-05-16 14:11:49.094411' + experiment_start_time: '14:11' + foldername: Z:\public\projects\AtApSuKuSaRe_20250129_HFScohort2\TAA0000066\TrainingData\250516 +monitor: + dual_monitor: true + monitor1: + width: 1920 + monitor2: + height: 1080 + width: 1920 + monitor3: + height: 1080 + width: 1920 +outputs: {} +sequence_task: + protocol: olfactory_support_l1 + rulename: olfactory_support +walls_sequence: +- random_dots.png +- random_dots.png +- random_dots.png +- logs.png +- logs.png +- random_dots.png +- grating1.jpg +- grating1.jpg +- random_dots.png +- tiles.png +- tiles.png +- random_dots.png +- big_light_rectangles.png +- big_light_rectangles.png +- random_dots.png +- grass.png +- grass.png +- random_dots.png +- big_dark_rectangles.png +- big_dark_rectangles.png +- random_dots.png +- leaves.png +- leaves.png +- random_dots.png +- waves.png +- waves.png +- random_dots.png +- bark.png +- bark.png +- random_dots.png +- big_light_circles.png +- big_light_circles.png +- random_dots.png diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-002_datetime-20240607T125530/behav/ExperimentEvents/position_log .csv b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-002_datetime-20240607T125530/behav/ExperimentEvents/position_log .csv new file mode 100644 index 000000000..e69de29bb diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-002_datetime-20240607T125530/behav/SessionSettings/session.yaml b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-002_datetime-20240607T125530/behav/SessionSettings/session.yaml new file mode 100644 index 000000000..f298dc806 --- /dev/null +++ b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-002_datetime-20240607T125530/behav/SessionSettings/session.yaml @@ -0,0 +1,49 @@ +# yaml-language-server: $schema=schema.json + +animalId: plimbo +drawUntilPositionOffset: 5 +drawSetsWhenDistanceFromEndReached: 20 +availableRewardPositions: 4 +corridorWidth: 2 + +trial: { + randomiseSequence: true, + offsets: [0, 1, 5], + landmarks: [ + [{size: 3, texture: grating1, odour: odour1, rewardSequencePosition: 0}], + [{size: 3, texture: logs, odour: odour0, rewardSequencePosition: -1}, {size: 3, texture: bark, odour: odour0, rewardSequencePosition: -1}], + [{size: 3, texture: grating1, odour: odour2, rewardSequencePosition: 1}], + [{size: 3, texture: logs, odour: odour0, rewardSequencePosition: -1}, {size: 3, texture: bark, odour: odour0, rewardSequencePosition: -1}], + [{size: 3, texture: grating1, odour: odour3, rewardSequencePosition: 2}], + [{size: 3, texture: logs, odour: odour0, rewardSequencePosition: -1}, {size: 3, texture: bark, odour: odour0, rewardSequencePosition: -1}], + [{size: 3, texture: grating1, odour: odour4, rewardSequencePosition: 3}], + [{size: 3, texture: logs, odour: odour0, rewardSequencePosition: -1}, {size: 3, texture: bark, odour: odour0, rewardSequencePosition: -1}], + ] +} + +olfactometerCommands: [ + { + alias: odour1, + prepareCommand: [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + releaseCommand: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0], + flushCommand: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] + }, + { + alias: odour2, + prepareCommand: [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + releaseCommand: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0], + flushCommand: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] + }, + { + alias: odour3, + prepareCommand: [0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0], + releaseCommand: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1], + flushCommand: [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] + }, + { + alias: odour4, + prepareCommand: [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0], + releaseCommand: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + flushCommand: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + } +] diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-002_datetime-20240607T125530/behav/VideoData/Camera1_2025-05-16T14_11_17.avi b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-002_datetime-20240607T125530/behav/VideoData/Camera1_2025-05-16T14_11_17.avi new file mode 100644 index 000000000..e69de29bb diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-002_datetime-20240607T125530/behav/VideoData/Camera2_2025-05-16T14_11_17.avi b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-002_datetime-20240607T125530/behav/VideoData/Camera2_2025-05-16T14_11_17.avi new file mode 100644 index 000000000..e69de29bb diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-003_datetime-20240607T125530/behav/ExperimentEvents/20250516_M8_ci.bin b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-003_datetime-20240607T125530/behav/ExperimentEvents/20250516_M8_ci.bin new file mode 100644 index 000000000..e69de29bb diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-003_datetime-20240607T125530/behav/ExperimentEvents/20250516_M8_ci_.bin b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-003_datetime-20240607T125530/behav/ExperimentEvents/20250516_M8_ci_.bin new file mode 100644 index 000000000..e69de29bb diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-003_datetime-20240607T125530/behav/ExperimentEvents/config.yaml b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-003_datetime-20240607T125530/behav/ExperimentEvents/config.yaml new file mode 100644 index 000000000..b30771dda --- /dev/null +++ b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-003_datetime-20240607T125530/behav/ExperimentEvents/config.yaml @@ -0,0 +1,183 @@ +base_tunnel: + eye_fov: + fov: 100 + fov_shift: 57 + speed_gain: 1.5 + wall_length: 4.5 + wall_model: walls.egg + wall_spacing: 9 +card: + position: + - 0.9 + - 0.9 + size: + - 0.2 + - 0.2 +daqChannel: + finalV1: Dev1/port1/line0 + finalV2: Dev1/port1/line3 + fiveV: Dev1/port2/line3 + mo1: Dev1/port0/line0 + mo2: Dev1/port2/line6 + odour0: Dev1/port2/line4 + odour1: Dev1/port0/line4 + odour2: Dev1/port1/line7 + odour3: Dev1/port0/line7 + odour4: Dev1/port1/line6 + odour5: Dev1/port0/line2 + odour6: Dev1/port2/line5 + odour7: Dev1/port0/line3 + odour8: Dev1/port1/line4 + odour9: Dev1/port0/line5 + spout1: + chan: Dev1/ai10 + max_value: 10 + min_value: 0 + threshold: 1 + threeV: Dev1/port1/line1 + threeV2: Dev1/port1/line2 + valve1: Dev2/port0/line1 +flip_tunnel: + continuous_corridor: true + goals: + - - 20 + - 36 + - - 74 + - 90 + - - 155 + - 171 + - - 263 + - 279 + io_module: nidaq + landmarks: + - - 20 + - 36 + - - 47 + - 63 + - - 74 + - 90 + - - 101 + - 117 + - - 128 + - 144 + - - 155 + - 171 + - - 182 + - 198 + - - 209 + - 225 + - - 236 + - 252 + - - 263 + - 279 + landmarks_sequence: + - 0 + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + length: 288 + manual_reward_with_space: true + margin_start: 9 + modd1_odours: + - 1 + - 3 + - 5 + - 7 + - 9 + neutral_texture: grey.png + odour_diffs: + flush: 1 + flush_other: 0.75 + flush_same: 1.5 + odour_overlap: 0.05 + reward_distance: 9 + reward_length: + assist: 0.3 + correct: 0.3 + manual: 0.1 + wrong: 0.3 + reward_prob: 1 + reward_tone_length: 1 + sleep_time: 0 + sound_at_A: true + sound_dir: examples/sounds/ + sounds: + 0: 6kHz_tone.ogg + speed_limit: 15 + stimulus_onset: 12 +goal_ids: +- 1 +- 0 +- 2 +- 0 +- 0 +- 3 +- 0 +- 0 +- 0 +- 4 +inputs: + speed: + chan: Dev1/ctr0 + diameter: 0.197 + error_value: 4000000000 + pulses_per_rev: 1000 + threshold: 0.001 +logger: + date_time: '2025-05-16 14:11:49.094411' + experiment_start_time: '14:11' + foldername: Z:\public\projects\AtApSuKuSaRe_20250129_HFScohort2\TAA0000066\TrainingData\250516 +monitor: + dual_monitor: true + monitor1: + width: 1920 + monitor2: + height: 1080 + width: 1920 + monitor3: + height: 1080 + width: 1920 +outputs: {} +sequence_task: + protocol: olfactory_support_l1 + rulename: olfactory_support +walls_sequence: +- random_dots.png +- random_dots.png +- random_dots.png +- logs.png +- logs.png +- random_dots.png +- grating1.jpg +- grating1.jpg +- random_dots.png +- tiles.png +- tiles.png +- random_dots.png +- big_light_rectangles.png +- big_light_rectangles.png +- random_dots.png +- grass.png +- grass.png +- random_dots.png +- big_dark_rectangles.png +- big_dark_rectangles.png +- random_dots.png +- leaves.png +- leaves.png +- random_dots.png +- waves.png +- waves.png +- random_dots.png +- bark.png +- bark.png +- random_dots.png +- big_light_circles.png +- big_light_circles.png +- random_dots.png diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-003_datetime-20240607T125530/behav/ExperimentEvents/position_log .csv b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-003_datetime-20240607T125530/behav/ExperimentEvents/position_log .csv new file mode 100644 index 000000000..e69de29bb diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-004_datetime-20240607T125530/behav/SessionSettings/session.yaml b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-004_datetime-20240607T125530/behav/SessionSettings/session.yaml new file mode 100644 index 000000000..f298dc806 --- /dev/null +++ b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-004_datetime-20240607T125530/behav/SessionSettings/session.yaml @@ -0,0 +1,49 @@ +# yaml-language-server: $schema=schema.json + +animalId: plimbo +drawUntilPositionOffset: 5 +drawSetsWhenDistanceFromEndReached: 20 +availableRewardPositions: 4 +corridorWidth: 2 + +trial: { + randomiseSequence: true, + offsets: [0, 1, 5], + landmarks: [ + [{size: 3, texture: grating1, odour: odour1, rewardSequencePosition: 0}], + [{size: 3, texture: logs, odour: odour0, rewardSequencePosition: -1}, {size: 3, texture: bark, odour: odour0, rewardSequencePosition: -1}], + [{size: 3, texture: grating1, odour: odour2, rewardSequencePosition: 1}], + [{size: 3, texture: logs, odour: odour0, rewardSequencePosition: -1}, {size: 3, texture: bark, odour: odour0, rewardSequencePosition: -1}], + [{size: 3, texture: grating1, odour: odour3, rewardSequencePosition: 2}], + [{size: 3, texture: logs, odour: odour0, rewardSequencePosition: -1}, {size: 3, texture: bark, odour: odour0, rewardSequencePosition: -1}], + [{size: 3, texture: grating1, odour: odour4, rewardSequencePosition: 3}], + [{size: 3, texture: logs, odour: odour0, rewardSequencePosition: -1}, {size: 3, texture: bark, odour: odour0, rewardSequencePosition: -1}], + ] +} + +olfactometerCommands: [ + { + alias: odour1, + prepareCommand: [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + releaseCommand: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0], + flushCommand: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] + }, + { + alias: odour2, + prepareCommand: [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + releaseCommand: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0], + flushCommand: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] + }, + { + alias: odour3, + prepareCommand: [0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0], + releaseCommand: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1], + flushCommand: [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] + }, + { + alias: odour4, + prepareCommand: [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0], + releaseCommand: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + flushCommand: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + } +] diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-004_datetime-20240607T125530/behav/VideoData/Camera1_2025-05-16T14_11_17.avi b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-004_datetime-20240607T125530/behav/VideoData/Camera1_2025-05-16T14_11_17.avi new file mode 100644 index 000000000..e69de29bb diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-004_datetime-20240607T125530/behav/VideoData/Camera2_2025-05-16T14_11_17.avi b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-004_datetime-20240607T125530/behav/VideoData/Camera2_2025-05-16T14_11_17.avi new file mode 100644 index 000000000..e69de29bb diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-005_datetime-20240607T125530/behav/ExperimentEvents/20250516_M8_ci.bin b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-005_datetime-20240607T125530/behav/ExperimentEvents/20250516_M8_ci.bin new file mode 100644 index 000000000..e69de29bb diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-005_datetime-20240607T125530/behav/ExperimentEvents/20250516_M8_ci_.bin b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-005_datetime-20240607T125530/behav/ExperimentEvents/20250516_M8_ci_.bin new file mode 100644 index 000000000..e69de29bb diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-005_datetime-20240607T125530/behav/ExperimentEvents/config.yaml b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-005_datetime-20240607T125530/behav/ExperimentEvents/config.yaml new file mode 100644 index 000000000..b30771dda --- /dev/null +++ b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-005_datetime-20240607T125530/behav/ExperimentEvents/config.yaml @@ -0,0 +1,183 @@ +base_tunnel: + eye_fov: + fov: 100 + fov_shift: 57 + speed_gain: 1.5 + wall_length: 4.5 + wall_model: walls.egg + wall_spacing: 9 +card: + position: + - 0.9 + - 0.9 + size: + - 0.2 + - 0.2 +daqChannel: + finalV1: Dev1/port1/line0 + finalV2: Dev1/port1/line3 + fiveV: Dev1/port2/line3 + mo1: Dev1/port0/line0 + mo2: Dev1/port2/line6 + odour0: Dev1/port2/line4 + odour1: Dev1/port0/line4 + odour2: Dev1/port1/line7 + odour3: Dev1/port0/line7 + odour4: Dev1/port1/line6 + odour5: Dev1/port0/line2 + odour6: Dev1/port2/line5 + odour7: Dev1/port0/line3 + odour8: Dev1/port1/line4 + odour9: Dev1/port0/line5 + spout1: + chan: Dev1/ai10 + max_value: 10 + min_value: 0 + threshold: 1 + threeV: Dev1/port1/line1 + threeV2: Dev1/port1/line2 + valve1: Dev2/port0/line1 +flip_tunnel: + continuous_corridor: true + goals: + - - 20 + - 36 + - - 74 + - 90 + - - 155 + - 171 + - - 263 + - 279 + io_module: nidaq + landmarks: + - - 20 + - 36 + - - 47 + - 63 + - - 74 + - 90 + - - 101 + - 117 + - - 128 + - 144 + - - 155 + - 171 + - - 182 + - 198 + - - 209 + - 225 + - - 236 + - 252 + - - 263 + - 279 + landmarks_sequence: + - 0 + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + length: 288 + manual_reward_with_space: true + margin_start: 9 + modd1_odours: + - 1 + - 3 + - 5 + - 7 + - 9 + neutral_texture: grey.png + odour_diffs: + flush: 1 + flush_other: 0.75 + flush_same: 1.5 + odour_overlap: 0.05 + reward_distance: 9 + reward_length: + assist: 0.3 + correct: 0.3 + manual: 0.1 + wrong: 0.3 + reward_prob: 1 + reward_tone_length: 1 + sleep_time: 0 + sound_at_A: true + sound_dir: examples/sounds/ + sounds: + 0: 6kHz_tone.ogg + speed_limit: 15 + stimulus_onset: 12 +goal_ids: +- 1 +- 0 +- 2 +- 0 +- 0 +- 3 +- 0 +- 0 +- 0 +- 4 +inputs: + speed: + chan: Dev1/ctr0 + diameter: 0.197 + error_value: 4000000000 + pulses_per_rev: 1000 + threshold: 0.001 +logger: + date_time: '2025-05-16 14:11:49.094411' + experiment_start_time: '14:11' + foldername: Z:\public\projects\AtApSuKuSaRe_20250129_HFScohort2\TAA0000066\TrainingData\250516 +monitor: + dual_monitor: true + monitor1: + width: 1920 + monitor2: + height: 1080 + width: 1920 + monitor3: + height: 1080 + width: 1920 +outputs: {} +sequence_task: + protocol: olfactory_support_l1 + rulename: olfactory_support +walls_sequence: +- random_dots.png +- random_dots.png +- random_dots.png +- logs.png +- logs.png +- random_dots.png +- grating1.jpg +- grating1.jpg +- random_dots.png +- tiles.png +- tiles.png +- random_dots.png +- big_light_rectangles.png +- big_light_rectangles.png +- random_dots.png +- grass.png +- grass.png +- random_dots.png +- big_dark_rectangles.png +- big_dark_rectangles.png +- random_dots.png +- leaves.png +- leaves.png +- random_dots.png +- waves.png +- waves.png +- random_dots.png +- bark.png +- bark.png +- random_dots.png +- big_light_circles.png +- big_light_circles.png +- random_dots.png diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-005_datetime-20240607T125530/behav/ExperimentEvents/position_log .csv b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-005_datetime-20240607T125530/behav/ExperimentEvents/position_log .csv new file mode 100644 index 000000000..e69de29bb diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-005_datetime-20240607T125530/behav/SessionSettings/session.yaml b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-005_datetime-20240607T125530/behav/SessionSettings/session.yaml new file mode 100644 index 000000000..f298dc806 --- /dev/null +++ b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-005_datetime-20240607T125530/behav/SessionSettings/session.yaml @@ -0,0 +1,49 @@ +# yaml-language-server: $schema=schema.json + +animalId: plimbo +drawUntilPositionOffset: 5 +drawSetsWhenDistanceFromEndReached: 20 +availableRewardPositions: 4 +corridorWidth: 2 + +trial: { + randomiseSequence: true, + offsets: [0, 1, 5], + landmarks: [ + [{size: 3, texture: grating1, odour: odour1, rewardSequencePosition: 0}], + [{size: 3, texture: logs, odour: odour0, rewardSequencePosition: -1}, {size: 3, texture: bark, odour: odour0, rewardSequencePosition: -1}], + [{size: 3, texture: grating1, odour: odour2, rewardSequencePosition: 1}], + [{size: 3, texture: logs, odour: odour0, rewardSequencePosition: -1}, {size: 3, texture: bark, odour: odour0, rewardSequencePosition: -1}], + [{size: 3, texture: grating1, odour: odour3, rewardSequencePosition: 2}], + [{size: 3, texture: logs, odour: odour0, rewardSequencePosition: -1}, {size: 3, texture: bark, odour: odour0, rewardSequencePosition: -1}], + [{size: 3, texture: grating1, odour: odour4, rewardSequencePosition: 3}], + [{size: 3, texture: logs, odour: odour0, rewardSequencePosition: -1}, {size: 3, texture: bark, odour: odour0, rewardSequencePosition: -1}], + ] +} + +olfactometerCommands: [ + { + alias: odour1, + prepareCommand: [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + releaseCommand: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0], + flushCommand: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] + }, + { + alias: odour2, + prepareCommand: [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + releaseCommand: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0], + flushCommand: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] + }, + { + alias: odour3, + prepareCommand: [0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0], + releaseCommand: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1], + flushCommand: [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] + }, + { + alias: odour4, + prepareCommand: [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0], + releaseCommand: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + flushCommand: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + } +] diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-005_datetime-20240607T125530/behav/VideoData/Camera1_2025-05-16T14_11_17.avi b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-005_datetime-20240607T125530/behav/VideoData/Camera1_2025-05-16T14_11_17.avi new file mode 100644 index 000000000..e69de29bb diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-005_datetime-20240607T125530/behav/VideoData/Camera2_2025-05-16T14_11_17.avi b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-005_datetime-20240607T125530/behav/VideoData/Camera2_2025-05-16T14_11_17.avi new file mode 100644 index 000000000..e69de29bb diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-006_datetime-20240607T125530/behav/SessionSettings/session.yaml b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-006_datetime-20240607T125530/behav/SessionSettings/session.yaml new file mode 100644 index 000000000..f298dc806 --- /dev/null +++ b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-006_datetime-20240607T125530/behav/SessionSettings/session.yaml @@ -0,0 +1,49 @@ +# yaml-language-server: $schema=schema.json + +animalId: plimbo +drawUntilPositionOffset: 5 +drawSetsWhenDistanceFromEndReached: 20 +availableRewardPositions: 4 +corridorWidth: 2 + +trial: { + randomiseSequence: true, + offsets: [0, 1, 5], + landmarks: [ + [{size: 3, texture: grating1, odour: odour1, rewardSequencePosition: 0}], + [{size: 3, texture: logs, odour: odour0, rewardSequencePosition: -1}, {size: 3, texture: bark, odour: odour0, rewardSequencePosition: -1}], + [{size: 3, texture: grating1, odour: odour2, rewardSequencePosition: 1}], + [{size: 3, texture: logs, odour: odour0, rewardSequencePosition: -1}, {size: 3, texture: bark, odour: odour0, rewardSequencePosition: -1}], + [{size: 3, texture: grating1, odour: odour3, rewardSequencePosition: 2}], + [{size: 3, texture: logs, odour: odour0, rewardSequencePosition: -1}, {size: 3, texture: bark, odour: odour0, rewardSequencePosition: -1}], + [{size: 3, texture: grating1, odour: odour4, rewardSequencePosition: 3}], + [{size: 3, texture: logs, odour: odour0, rewardSequencePosition: -1}, {size: 3, texture: bark, odour: odour0, rewardSequencePosition: -1}], + ] +} + +olfactometerCommands: [ + { + alias: odour1, + prepareCommand: [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + releaseCommand: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0], + flushCommand: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] + }, + { + alias: odour2, + prepareCommand: [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + releaseCommand: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0], + flushCommand: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] + }, + { + alias: odour3, + prepareCommand: [0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0], + releaseCommand: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1], + flushCommand: [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] + }, + { + alias: odour4, + prepareCommand: [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0], + releaseCommand: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + flushCommand: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + } +] diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-007_datetime-20240607T125530/behav/ExperimentEvents/20250516_M8_ci.bin b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-007_datetime-20240607T125530/behav/ExperimentEvents/20250516_M8_ci.bin new file mode 100644 index 000000000..e69de29bb diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-007_datetime-20240607T125530/behav/ExperimentEvents/20250516_M8_ci_.bin b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-007_datetime-20240607T125530/behav/ExperimentEvents/20250516_M8_ci_.bin new file mode 100644 index 000000000..e69de29bb diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-007_datetime-20240607T125530/behav/ExperimentEvents/config.yaml b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-007_datetime-20240607T125530/behav/ExperimentEvents/config.yaml new file mode 100644 index 000000000..b30771dda --- /dev/null +++ b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-007_datetime-20240607T125530/behav/ExperimentEvents/config.yaml @@ -0,0 +1,183 @@ +base_tunnel: + eye_fov: + fov: 100 + fov_shift: 57 + speed_gain: 1.5 + wall_length: 4.5 + wall_model: walls.egg + wall_spacing: 9 +card: + position: + - 0.9 + - 0.9 + size: + - 0.2 + - 0.2 +daqChannel: + finalV1: Dev1/port1/line0 + finalV2: Dev1/port1/line3 + fiveV: Dev1/port2/line3 + mo1: Dev1/port0/line0 + mo2: Dev1/port2/line6 + odour0: Dev1/port2/line4 + odour1: Dev1/port0/line4 + odour2: Dev1/port1/line7 + odour3: Dev1/port0/line7 + odour4: Dev1/port1/line6 + odour5: Dev1/port0/line2 + odour6: Dev1/port2/line5 + odour7: Dev1/port0/line3 + odour8: Dev1/port1/line4 + odour9: Dev1/port0/line5 + spout1: + chan: Dev1/ai10 + max_value: 10 + min_value: 0 + threshold: 1 + threeV: Dev1/port1/line1 + threeV2: Dev1/port1/line2 + valve1: Dev2/port0/line1 +flip_tunnel: + continuous_corridor: true + goals: + - - 20 + - 36 + - - 74 + - 90 + - - 155 + - 171 + - - 263 + - 279 + io_module: nidaq + landmarks: + - - 20 + - 36 + - - 47 + - 63 + - - 74 + - 90 + - - 101 + - 117 + - - 128 + - 144 + - - 155 + - 171 + - - 182 + - 198 + - - 209 + - 225 + - - 236 + - 252 + - - 263 + - 279 + landmarks_sequence: + - 0 + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + length: 288 + manual_reward_with_space: true + margin_start: 9 + modd1_odours: + - 1 + - 3 + - 5 + - 7 + - 9 + neutral_texture: grey.png + odour_diffs: + flush: 1 + flush_other: 0.75 + flush_same: 1.5 + odour_overlap: 0.05 + reward_distance: 9 + reward_length: + assist: 0.3 + correct: 0.3 + manual: 0.1 + wrong: 0.3 + reward_prob: 1 + reward_tone_length: 1 + sleep_time: 0 + sound_at_A: true + sound_dir: examples/sounds/ + sounds: + 0: 6kHz_tone.ogg + speed_limit: 15 + stimulus_onset: 12 +goal_ids: +- 1 +- 0 +- 2 +- 0 +- 0 +- 3 +- 0 +- 0 +- 0 +- 4 +inputs: + speed: + chan: Dev1/ctr0 + diameter: 0.197 + error_value: 4000000000 + pulses_per_rev: 1000 + threshold: 0.001 +logger: + date_time: '2025-05-16 14:11:49.094411' + experiment_start_time: '14:11' + foldername: Z:\public\projects\AtApSuKuSaRe_20250129_HFScohort2\TAA0000066\TrainingData\250516 +monitor: + dual_monitor: true + monitor1: + width: 1920 + monitor2: + height: 1080 + width: 1920 + monitor3: + height: 1080 + width: 1920 +outputs: {} +sequence_task: + protocol: olfactory_support_l1 + rulename: olfactory_support +walls_sequence: +- random_dots.png +- random_dots.png +- random_dots.png +- logs.png +- logs.png +- random_dots.png +- grating1.jpg +- grating1.jpg +- random_dots.png +- tiles.png +- tiles.png +- random_dots.png +- big_light_rectangles.png +- big_light_rectangles.png +- random_dots.png +- grass.png +- grass.png +- random_dots.png +- big_dark_rectangles.png +- big_dark_rectangles.png +- random_dots.png +- leaves.png +- leaves.png +- random_dots.png +- waves.png +- waves.png +- random_dots.png +- bark.png +- bark.png +- random_dots.png +- big_light_circles.png +- big_light_circles.png +- random_dots.png diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-007_datetime-20240607T125530/behav/ExperimentEvents/position_log .csv b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-007_datetime-20240607T125530/behav/ExperimentEvents/position_log .csv new file mode 100644 index 000000000..e69de29bb diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-007_datetime-20240607T125530/behav/SessionSettings/session.yaml b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-007_datetime-20240607T125530/behav/SessionSettings/session.yaml new file mode 100644 index 000000000..f298dc806 --- /dev/null +++ b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-007_datetime-20240607T125530/behav/SessionSettings/session.yaml @@ -0,0 +1,49 @@ +# yaml-language-server: $schema=schema.json + +animalId: plimbo +drawUntilPositionOffset: 5 +drawSetsWhenDistanceFromEndReached: 20 +availableRewardPositions: 4 +corridorWidth: 2 + +trial: { + randomiseSequence: true, + offsets: [0, 1, 5], + landmarks: [ + [{size: 3, texture: grating1, odour: odour1, rewardSequencePosition: 0}], + [{size: 3, texture: logs, odour: odour0, rewardSequencePosition: -1}, {size: 3, texture: bark, odour: odour0, rewardSequencePosition: -1}], + [{size: 3, texture: grating1, odour: odour2, rewardSequencePosition: 1}], + [{size: 3, texture: logs, odour: odour0, rewardSequencePosition: -1}, {size: 3, texture: bark, odour: odour0, rewardSequencePosition: -1}], + [{size: 3, texture: grating1, odour: odour3, rewardSequencePosition: 2}], + [{size: 3, texture: logs, odour: odour0, rewardSequencePosition: -1}, {size: 3, texture: bark, odour: odour0, rewardSequencePosition: -1}], + [{size: 3, texture: grating1, odour: odour4, rewardSequencePosition: 3}], + [{size: 3, texture: logs, odour: odour0, rewardSequencePosition: -1}, {size: 3, texture: bark, odour: odour0, rewardSequencePosition: -1}], + ] +} + +olfactometerCommands: [ + { + alias: odour1, + prepareCommand: [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + releaseCommand: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0], + flushCommand: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] + }, + { + alias: odour2, + prepareCommand: [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + releaseCommand: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0], + flushCommand: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] + }, + { + alias: odour3, + prepareCommand: [0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0], + releaseCommand: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1], + flushCommand: [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] + }, + { + alias: odour4, + prepareCommand: [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0], + releaseCommand: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + flushCommand: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + } +] diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-007_datetime-20240607T125530/behav/VideoData/Camera1_2025-05-16T14_11_17.avi b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-007_datetime-20240607T125530/behav/VideoData/Camera1_2025-05-16T14_11_17.avi new file mode 100644 index 000000000..e69de29bb diff --git a/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-007_datetime-20240607T125530/behav/VideoData/Camera2_2025-05-16T14_11_17.avi b/datashuttle/converters/example_project/rawdata/sub-001_id-plimbo/ses-007_datetime-20240607T125530/behav/VideoData/Camera2_2025-05-16T14_11_17.avi new file mode 100644 index 000000000..e69de29bb diff --git a/datashuttle/converters/run_convert_sumiya.py b/datashuttle/converters/run_convert_sumiya.py new file mode 100644 index 000000000..1f79d4e61 --- /dev/null +++ b/datashuttle/converters/run_convert_sumiya.py @@ -0,0 +1,14 @@ +from pathlib import Path + +from datashuttle import DataShuttle + +project = DataShuttle("example_project") + +this_file_path = Path(__file__) + +if not project.cfg: # TODO: make a convenience func for this + project.make_config_file( + this_file_path.parent / "example_project", + ) + +project.convert(Path(this_file_path)) diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index fcd17d427..49c3675e6 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -2,6 +2,7 @@ import copy import glob +import importlib import json import os import shutil @@ -832,8 +833,8 @@ def write_public_key(self, filepath: str) -> None: def make_config_file( self, - local_path: str, - central_path: str | None = None, + local_path: Path | str, + central_path: Path | str | None = None, connection_method: str | None = None, central_host_id: Optional[str] = None, central_host_username: Optional[str] = None, @@ -992,6 +993,63 @@ def get_existing_projects() -> List[Path]: """ return getters.get_existing_project_paths() + @check_configs_set + def get_sub_names_from_key_value_pair( + self, key, value, include_central=False + ): + """TODO: PLACEHOLDER.""" + all_subs = folders.search_project_for_sub_or_ses_names( + self.cfg, "rawdata", None, "sub-*", include_central=include_central + ) + + all_subject_names = [] + + for sub in all_subs["local"]: + try: + sub_value = utils.get_values_from_bids_formatted_name( + [sub], key + )[0] + except NeuroBlueprintError: # check error + utils.warn( + f"The key: {key} was not found in subject name: {sub}", + log=False, + ) + continue + + if sub_value == value: + all_subject_names.append(sub) + + return all_subject_names + + def log_and_message( + self, message + ): # TODO: fix naming, maybe def message(log=True) is better + # TODO: assert a logger is running + """TODO: PLACEHOLDER.""" + utils.log_and_message(message) + + def log_and_raise_error(self, message, exception): + """TODO: PLACEHOLDER.""" + utils.log_and_raise_error(message, exception) + + def convert(self, custom_converter_path: Path): + """TODO: PLACEHOLDER.""" + if isinstance(custom_converter_path, str): + custom_converter_path = Path(custom_converter_path) + + spec = importlib.util.spec_from_file_location( + "mod", custom_converter_path + ) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + converter_func = getattr(mod, "converter_func") + + self._start_log(f"convert_{custom_converter_path.name}") + + converter_func(self) + + ds_logger.close_log_filehandler() + @check_configs_set def get_next_sub( self,