Drop missing values in HARP csv (#37)

Author: alejoe91

src/aind_ephys_transformation/ephys_job.py

@@ -377,7 +377,7 @@ def _get_read_blocks(self) -> Iterator[dict]:  # noqa: C901
             timestamps = []
             for amplifier_dataset in amplifier_datasets_to_compress:
                 recording = si.read_binary(amplifier_dataset, **binary_info)
-
+                recording_samples = recording.get_num_samples()
                 # unsigned to signed
                 recording = spre.unsigned_to_signed(
                     recording, bit_depth=adc_depth
@@ -391,8 +391,6 @@ def _get_read_blocks(self) -> Iterator[dict]:  # noqa: C901
                 # update cumulative start frame
                 cumulative_start_frame += recording.get_num_frames()
 
-                recording_list.append(recording)
-
                 # align timestamps based on harp sync data
                 clock_dataset_name = amplifier_dataset.name.replace(
                     "AmplifierData", "Clock"
@@ -414,11 +412,24 @@ def _get_read_blocks(self) -> Iterator[dict]:  # noqa: C901
                     clock_data = np.memmap(
                         clock_dataset, dtype="uint64", mode="r"
                     )
-                    harp_data = pd.read_csv(harp_sync_dataset)
+                    harp_data = pd.read_csv(harp_sync_dataset, index_col=False)
+                    # Drop rows with missing values (NaNs/zeros)
+                    harp_data = harp_data[harp_data != 0].dropna()
                     timestamps_chunk = self._sync_chronic_timestamps(
                         clock_data, harp_data, fs=recording.sampling_frequency
                     )
+                    timestamps_samples = len(timestamps_chunk)
+                    if timestamps_samples != recording_samples:
+                        logging.warning(  # pragma: no cover
+                            f"Number of timestamps ({timestamps_samples}) "
+                            f"does not match recording samples "
+                            f"({recording_samples}). "
+                            f"Skipping chunk {amplifier_dataset.name}!"
+                        )
+                        continue  # pragma: no cover
+
                     timestamps.append(timestamps_chunk)
+                recording_list.append(recording)
 
             # concatenate recordings
             recording_concatenated = si.concatenate_recordings(recording_list)

tests/resources/chronic-test-data/OnixHarpSyncData/OnixHarpSyncData_2025-05-13T19-00-00.csv

@@ -1,10 +1,10 @@
-,Seconds,Value.Clock
-0,10.0,10
-1,11.0,20
-2,12.0,30
-3,13.0,40
-4,14.0,50
-5,15.0,60
-6,16.0,70
-7,17.0,80
-8,18.0,90
+Seconds,Value.Clock
+10.0,10
+11.0,20
+12.0,30
+13.0,40
+14.0,50
+15.0,60
+16.0,70
+17.0,80
+18.0,90

tests/resources/chronic-test-data/OnixHarpSyncData/OnixHarpSyncData_2025-05-13T20-00-00.csv

@@ -1,10 +1,10 @@
-,Seconds,Value.Clock
-0,20.0,110
-1,21.0,120
-2,22.0,130
-3,23.0,140
-4,24.0,150
-5,25.0,160
-6,26.0,170
-7,27.0,180
-8,28.0,190
+Seconds,Value.Clock
+20.0,110
+21.0,120
+22.0,130
+23.0,140
+24.0,150
+25.0,160
+26.0,170
+27.0,180
+28.0,190

tests/resources/chronic-test-data/OnixHarpSyncData/OnixHarpSyncData_2025-05-13T21-00-00.csv

@@ -1,10 +1,10 @@
-,Seconds,Value.Clock
-0,30.0,210
-1,31.0,220
-2,32.0,230
-3,33.0,240
-4,34.0,250
-5,35.0,260
-6,36.0,270
-7,37.0,280
-8,38.0,290
+Seconds,Value.Clock
+30.0,210
+31.0,220
+32.0,230
+33.0,240
+34.0,250
+35.0,260
+36.0,270
+37.0,280
+38.0,290