Fix aplose2raven (#263)

* Partial fix of raven time related issue for duty cycled data

* apply duty cycle corr

* fix test_utils

* fix aplose2raven and adapt test_utils

* ruff fix

* fix aplose2raven for det_end in OFF duty cycle phase

---------

Co-authored-by: Maëlle TORTEROTOT <maelle.torterotot@ensta.fr>

Author: MaelleTtrt

src/osekit/utils/formatting_utils.py

@@ -1,14 +1,13 @@
 from __future__ import annotations
 
 import numpy as np
-import pandas as pd
-
+from pandas import DataFrame, Timedelta, Timestamp
 
 def aplose2raven(
-    aplose_result: pd.DataFrame,
-    audio_datetimes: list[pd.Timestamp],
+    aplose_result: DataFrame,
+    audio_datetimes: list[Timestamp],
     audio_durations: list[float],
-) -> pd.DataFrame:
+) -> DataFrame:
     r"""Format an APLOSE result DataFrame to a Raven result DataFrame.
 
     The list of audio files and durations considered for the Raven campaign should be
@@ -53,35 +52,65 @@ def aplose2raven(
         - 1
     )
 
+    # Add beg datetime of the wavfile
+    aplose_result["wav_timestamp"] = [audio_datetimes[i] for i in index_detection]
+
     # time differences between consecutive datetimes and add wav_duration
     filename_diff = [td.total_seconds() for td in np.diff(audio_datetimes).tolist()]
     adjust = [0]
-    adjust.extend([t1 - t2 for (t1, t2) in zip(audio_durations[:-1], filename_diff)])
+    adjust.extend([t1 - t2 for (t1, t2) in zip(audio_durations[:-1], filename_diff, strict=False)])
     cumsum_adjust = list(np.cumsum(adjust))
 
     # adjusted datetimes to match Raven annoying functioning
-    begin_datetime_adjusted = [
-        det + pd.Timedelta(seconds=cumsum_adjust[ind])
-        for (det, ind) in zip(aplose_result["start_datetime"], index_detection)
-    ]
-    end_datetime_adjusted = [
-        det + pd.Timedelta(seconds=cumsum_adjust[ind])
-        for (det, ind) in zip(aplose_result["end_datetime"], index_detection)
-    ]
+    begin_datetime_adjusted = []
+    end_datetime_adjusted = []
+    for (beg_det, end_det, beg_wav, ind) in (zip(aplose_result["start_datetime"], aplose_result["end_datetime"],
+                          aplose_result["wav_timestamp"], index_detection, strict=False)):
+        """
+        For duty cycled data, if the aplose_result detections were reshaped (eg : to 60-second duration),
+        the start or end of the detection might virtually be located in a OFF duty cycle phase.
+        This would cause issue in Raven, because the OFF part are not represented,
+        and the detection start will be located on the previous wav file.
+        The following 'if' conditions apply the appropriate correction to make the Raven box (1)starts or (2) ends
+         at the appropriate timing in Raven (ie at the begining or end of a wav file).
+        """
+
+        if (beg_wav + Timedelta(seconds=audio_durations[ind])) < beg_det < (beg_wav + Timedelta(seconds = filename_diff[ind])):
+            corr_dur = (audio_datetimes[ind + 1] - beg_det).total_seconds()
+            begin_datetime_adjusted.append(beg_det + Timedelta(seconds=cumsum_adjust[ind + 1]) + Timedelta(seconds=corr_dur))
+            end_datetime_adjusted.append(end_det + Timedelta(seconds=cumsum_adjust[ind + 1]))
+        elif (beg_wav + Timedelta(seconds=audio_durations[ind])) < end_det < (beg_wav + Timedelta(seconds = filename_diff[ind])):
+            begin_datetime_adjusted.append(
+                beg_det + Timedelta(seconds=cumsum_adjust[ind])
+            )
+            corr_dur = (end_det-beg_det).total_seconds() - ((beg_wav + Timedelta(seconds=audio_durations[ind])) -beg_det).total_seconds()
+            end_datetime_adjusted.append(end_det + Timedelta(seconds=cumsum_adjust[ind]) - Timedelta(seconds=corr_dur))
+
+        else:
+            # Else, apply normal raven time correction
+            begin_datetime_adjusted.append(
+                beg_det + Timedelta(seconds=cumsum_adjust[ind])
+            )
+            end_datetime_adjusted.append(
+                end_det + Timedelta(seconds=cumsum_adjust[ind])
+            )
+
+    # Convert the datetimes to seconds from the start of first wav (raven format)
     begin_time_adjusted = [
         (d - audio_datetimes[0]).total_seconds() for d in begin_datetime_adjusted
     ]
     end_time_adjusted = [
         (d - audio_datetimes[0]).total_seconds() for d in end_datetime_adjusted
     ]
-
-    raven_result = pd.DataFrame()
+    # Build corrected Raven selection table
+    raven_result = DataFrame()
     raven_result["Selection"] = list(range(1, len(aplose_result) + 1))
     raven_result["View"] = [1] * len(aplose_result)
     raven_result["Channel"] = [1] * len(aplose_result)
     raven_result["Begin Time (s)"] = begin_time_adjusted
     raven_result["End Time (s)"] = end_time_adjusted
     raven_result["Low Freq (Hz)"] = aplose_result["start_frequency"]
     raven_result["High Freq (Hz)"] = aplose_result["end_frequency"]
+    raven_result["Begin Date Time Real"] = aplose_result["start_datetime"]
 
     return raven_result

tests/test_utils.py

@@ -20,33 +20,35 @@
 def aplose_dataframe() -> pd.DataFrame:
     data = pd.DataFrame(
         {
-            "dataset": ["dataset_test", "dataset_test", "dataset_test"],
-            "filename": ["file1.wav", "file2.wav", "file3.wav"],
-            "start_time": [0, 0, 5.9],
-            "end_time": [60, 60, 8.1],
-            "start_frequency": [0, 0, 18500.0],
-            "end_frequency": [96000, 96000, 53000.0],
-            "annotation": ["boat", "boat", "boat"],
-            "annotator": ["bbjuni", "bbjuni", "bbjuni"],
+            "dataset": ["dataset_test", "dataset_test", "dataset_test", "dataset_test"],
+            "filename": ["file1.wav", "file2.wav", "file3.wav", "file4.wav"],
+            "start_time": [0, 0, 5.9, 0],
+            "end_time": [30, 30, 8.1, 30],
+            "start_frequency": [0, 0, 18500.0, 0],
+            "end_frequency": [96000, 96000, 53000.0, 96000],
+            "annotation": ["boat", "boat", "boat", "boat"],
+            "annotator": ["bbjuni", "bbjuni", "bbjuni", "bbjuni"],
             "start_datetime": [
                 pd.Timestamp("2020-05-29T11:30:00.000+00:00"),
                 pd.Timestamp("2020-05-29T11:31:00.000+00:00"),
                 pd.Timestamp("2020-05-29T11:31:05.900+00:00"),
+                pd.Timestamp("2020-05-29T11:32:50.000+00:00"),
             ],
             "end_datetime": [
-                pd.Timestamp("2020-05-29T11:31:00.000+00:00"),
-                pd.Timestamp("2020-05-29T11:32:00.000+00:00"),
-                pd.Timestamp("2020-05-29T11:32:08.100+00:00"),
+                pd.Timestamp("2020-05-29T11:30:30.000+00:00"),
+                pd.Timestamp("2020-05-29T11:31:30.000+00:00"),
+                pd.Timestamp("2020-05-29T11:31:08.100+00:00"),
+                pd.Timestamp("2020-05-29T11:33:20.000+00:00"),
             ],
-            "is_box": [0, 0, 1],
+            "is_box": [0, 0, 1, 0],
         },
     )
 
     return data.reset_index(drop=True)
 
 
 @pytest.fixture
-def raven_timestamps() -> list:
+def audio_timestamps() -> list:
     return list(
         pd.date_range(
             start="2020-05-29T11:30:00.000+00:00",
@@ -57,31 +59,32 @@ def raven_timestamps() -> list:
 
 
 @pytest.fixture
-def raven_durations(raven_timestamps: pytest.fixture) -> list:
-    return [60] * len(raven_timestamps)
+def audio_durations(audio_timestamps: pytest.fixture) -> list:
+    return [30] * len(audio_timestamps)
 
 
 @pytest.mark.unit
 def test_aplose2raven(
     aplose_dataframe: pytest.fixture,
-    raven_timestamps: pytest.fixture,
-    raven_durations: pytest.fixture,
+    audio_timestamps: pytest.fixture,
+    audio_durations: pytest.fixture,
 ) -> None:
     raven_dataframe = aplose2raven(
         aplose_result=aplose_dataframe,
-        audio_datetimes=raven_timestamps,
-        audio_durations=raven_durations,
+        audio_datetimes=audio_timestamps,
+        audio_durations=audio_durations,
     )
 
     expected_raven_dataframe = pd.DataFrame(
         {
-            "Selection": [1, 2, 3],
-            "View": [1, 1, 1],
-            "Channel": [1, 1, 1],
-            "Begin Time (s)": [0.0, 60.0, 65.9],
-            "End Time (s)": [60.0, 120.0, 128.1],
-            "Low Freq (Hz)": [0.0, 0.0, 18500.0],
-            "High Freq (Hz)": [96000.0, 96000.0, 53000.0],
+            "Selection": [1, 2, 3, 4],
+            "View": [1, 1, 1, 1],
+            "Channel": [1, 1, 1, 1],
+            "Begin Time (s)": [0.0, 30.0, 35.9, 90.0],
+            "End Time (s)": [30.0, 60.0, 38.1, 110.0],
+            "Low Freq (Hz)": [0.0, 0.0, 18500.0, 0.0],
+            "High Freq (Hz)": [96000.0, 96000.0, 53000.0, 96000.0],
+            "Begin Date Time Real": aplose_dataframe["start_datetime"],
         },
     )