Fix aplose2raven (#272)

* Partial fix of raven time related issue for duty cycled data

* apply duty cycle corr

* fix test_utils

* fix aplose2raven and adapt test_utils

* ruff fix

* fix aplose2raven for det_end in OFF duty cycle phase

* rename variables

* improve comment section

* quick fix

* adapt varible names in test

* fix - case: detection in last audio

---------

Co-authored-by: Maëlle TORTEROTOT <maelle.torterotot@ensta.fr>
Co-authored-by: Mathieu Dupont <108517594+mathieudpnt@users.noreply.github.com>

Author: 3 people

src/osekit/utils/formatting_utils.py

@@ -3,10 +3,11 @@
 import numpy as np
 from pandas import DataFrame, Timedelta, Timestamp
 
+
 def aplose2raven(
     aplose_result: DataFrame,
-    audio_datetimes: list[Timestamp],
-    audio_durations: list[float],
+    list_audio_begin_time: list[Timestamp],
+    audio_durations: list[Timedelta],
 ) -> DataFrame:
     r"""Format an APLOSE result DataFrame to a Raven result DataFrame.
 
@@ -19,89 +20,141 @@ def aplose2raven(
     aplose_result: Dataframe,
         APLOSE formatted result DataFrame.
 
-    audio_datetimes: list[pd.Timestamp]
-        list of tz-aware timestamps from considered audio files.
+    list_audio_begin_time: list[Timestamp]
+        list of tz-aware timestamps from considered audio files begin time.
 
-    audio_durations: list[float]
-        list of all considered audio file durations in seconds.
+    audio_durations: list[Timedelta]
+        list of all considered audio file durations.
 
     Returns
     -------
     Raven formatted DataFrame.
 
     Example of use
     --------------
-    aplose_file = Path("path/to/aplose/result/file")
-    timestamp_list = list(filenames)
-    duration_list = list(durations)
-
-    aplose_result = (
-        pd.read_csv(aplose_file, parse_dates=["start_datetime", "end_datetime"])
-        .sort_values("start_datetime")
-        .reset_index(drop=True)
-    )
-    raven_result = aplose2raven(aplose_result, filename_list, duration_list)
+    >>> from pathlib import Path
+    >>> from pandas import read_csv
+    >>> from osekit.core_api.audio_dataset import AudioDataset
+    >>> from osekit.utils.formatting_utils import aplose2raven
+
+    >>> dataset_folder = Path(r"path\to\audio\folder")
+    >>> dataset = AudioDataset.from_folder(dataset_folder,
+    >>>                                    strptime_format="strptime_format",
+    >>>                                    timezone='utc',
+    >>>                                   )
 
-    # export to Raven format: tab-separated files with a txt extension
-    raven_result.to_csv('path/to/result/file.txt', sep='\t', index=False)
+    >>> begin_list = sorted([f.begin for f in list(dataset.files)])
+    >>> duration_list = sorted([f.duration for f in list(dataset.files)])
+
+    >>> csv = Path(r"path\to\result\csv")
+    >>> df = read_csv(csv,
+    >>>               parse_dates=["start_datetime", "end_datetime"]
+    >>>               ).sort_values("start_datetime")
+    >>>                .reset_index(drop=True)
+
+    >>> df_raven = aplose2raven(df, begin_list, duration_list)
+    >>> raven_result.to_csv('path/to/result/file.txt', sep='\t', index=False)
 
     """
-    # index of the corresponding wav file for each detection
+    # index of the corresponding audio file for each detection
     index_detection = (
-        np.searchsorted(audio_datetimes, aplose_result["start_datetime"], side="right")
+        np.searchsorted(list_audio_begin_time,
+                        aplose_result["start_datetime"],
+                        side="right"
+                        )
         - 1
     )
 
-    # Add beg datetime of the wavfile
-    aplose_result["wav_timestamp"] = [audio_datetimes[i] for i in index_detection]
-
-    # time differences between consecutive datetimes and add wav_duration
-    filename_diff = [td.total_seconds() for td in np.diff(audio_datetimes).tolist()]
-    adjust = [0]
-    adjust.extend([t1 - t2 for (t1, t2) in zip(audio_durations[:-1], filename_diff, strict=False)])
-    cumsum_adjust = list(np.cumsum(adjust))
+    """
+    The following time adjustment is necessary because Raven does not account
+    for the duty cycle, nor for any potential offset between the end of one
+    file and the start of the next. To ensure that detection timestamps in
+    APLOSE format align with the spectrograms displayed by Raven, a correction
+    of the number of seconds is required, since the software only uses the
+    elapsed time from the beginning of the first file to generate the bounding boxes.
+    """
 
-    # adjusted datetimes to match Raven annoying functioning
-    begin_datetime_adjusted = []
-    end_datetime_adjusted = []
-    for (beg_det, end_det, beg_wav, ind) in (zip(aplose_result["start_datetime"], aplose_result["end_datetime"],
-                          aplose_result["wav_timestamp"], index_detection, strict=False)):
+    # Add the begin time of the audio file corresponding to each detection
+    aplose_result["wav_timestamp"] = [list_audio_begin_time[i] for i in index_detection]
+
+    # Compute the time gaps between consecutive audio file begin time
+    audio_begin_timegap = list(np.diff(list_audio_begin_time).tolist())
+
+    # Adjustment values: difference between each file's duration
+    # and the gap until the next file.
+    # (Required to account for potential gaps/overlaps between files)
+    adjustment_values = [Timedelta(0)]
+    adjustment_values.extend(
+        [t1 - t2 for (t1, t2) in zip(audio_durations[:-1],
+        audio_begin_timegap, strict=False)
+                              ]
+                             )
+
+    # Cumulative adjustment in seconds, to realign all detection timestamps consistently
+    cumsum_adjust = list(np.cumsum(adjustment_values))
+
+    detection_begin_datetime_adjusted = []
+    detection_end_datetime_adjusted = []
+    for i in range(len(aplose_result)):
+        detection_begin_time = aplose_result["start_datetime"].iloc[i]
+        detection_end_time = aplose_result["end_datetime"].iloc[i]
+        audio_begin_time = aplose_result["wav_timestamp"].iloc[i]
+        ind = index_detection[i]
         """
-        For duty cycled data, if the aplose_result detections were reshaped (eg : to 60-second duration),
+        For duty cycled data, if aplose_result detections were reshaped (eg to 60s duration),
         the start or end of the detection might virtually be located in a OFF duty cycle phase.
         This would cause issue in Raven, because the OFF part are not represented,
-        and the detection start will be located on the previous wav file.
-        The following 'if' conditions apply the appropriate correction to make the Raven box (1)starts or (2) ends
-         at the appropriate timing in Raven (ie at the begining or end of a wav file).
+        and the detection start will be located on the previous audio file.
+        The 2 following 'if' conditions apply the appropriate correction
+        to make the Raven box (1)starts or (2) ends.
+        at the appropriate timing in Raven (ie at the begining or end of an audio file).
         """
 
-        if (beg_wav + Timedelta(seconds=audio_durations[ind])) < beg_det < (beg_wav + Timedelta(seconds = filename_diff[ind])):
-            corr_dur = (audio_datetimes[ind + 1] - beg_det).total_seconds()
-            begin_datetime_adjusted.append(beg_det + Timedelta(seconds=cumsum_adjust[ind + 1]) + Timedelta(seconds=corr_dur))
-            end_datetime_adjusted.append(end_det + Timedelta(seconds=cumsum_adjust[ind + 1]))
-        elif (beg_wav + Timedelta(seconds=audio_durations[ind])) < end_det < (beg_wav + Timedelta(seconds = filename_diff[ind])):
-            begin_datetime_adjusted.append(
-                beg_det + Timedelta(seconds=cumsum_adjust[ind])
+        audio_begin_time_adjusted = audio_begin_time + audio_durations[ind]
+
+        if ind < len(audio_begin_timegap):
+            next_audio_begin_time_adjusted = audio_begin_time + audio_begin_timegap[ind]
+        else:
+            next_audio_begin_time_adjusted += audio_durations[ind]
+
+
+        if audio_begin_time_adjusted < detection_begin_time < next_audio_begin_time_adjusted:
+            correction_duration = (list_audio_begin_time[ind + 1] - detection_begin_time)
+            detection_begin_datetime_adjusted.append(detection_begin_time
+                                                     + cumsum_adjust[ind + 1]
+                                                     + correction_duration
+                                                     )
+            detection_end_datetime_adjusted.append(detection_end_time
+                                                  + cumsum_adjust[ind + 1]
+                                                   )
+        elif audio_begin_time_adjusted < detection_end_time < next_audio_begin_time_adjusted:
+            detection_begin_datetime_adjusted.append(
+                detection_begin_time + cumsum_adjust[ind]
             )
-            corr_dur = (end_det-beg_det).total_seconds() - ((beg_wav + Timedelta(seconds=audio_durations[ind])) -beg_det).total_seconds()
-            end_datetime_adjusted.append(end_det + Timedelta(seconds=cumsum_adjust[ind]) - Timedelta(seconds=corr_dur))
+            correction_duration = ((detection_end_time-detection_begin_time) -
+                                   ((audio_begin_time + audio_durations[ind])
+                                    - detection_begin_time))
+            detection_end_datetime_adjusted.append(detection_end_time +
+                                                   cumsum_adjust[ind] -
+                                                   correction_duration)
 
         else:
-            # Else, apply normal raven time correction
-            begin_datetime_adjusted.append(
-                beg_det + Timedelta(seconds=cumsum_adjust[ind])
+            # Else, apply normal Raven time correction
+            detection_begin_datetime_adjusted.append(
+                detection_begin_time + cumsum_adjust[ind]
             )
-            end_datetime_adjusted.append(
-                end_det + Timedelta(seconds=cumsum_adjust[ind])
+            detection_end_datetime_adjusted.append(
+                detection_end_time + cumsum_adjust[ind]
             )
 
-    # Convert the datetimes to seconds from the start of first wav (raven format)
+    # Convert the datetimes to seconds from the start of first audio (raven format)
     begin_time_adjusted = [
-        (d - audio_datetimes[0]).total_seconds() for d in begin_datetime_adjusted
+        (d - list_audio_begin_time[0]).total_seconds() for d in detection_begin_datetime_adjusted
     ]
     end_time_adjusted = [
-        (d - audio_datetimes[0]).total_seconds() for d in end_datetime_adjusted
+        (d - list_audio_begin_time[0]).total_seconds() for d in detection_end_datetime_adjusted
     ]
+
     # Build corrected Raven selection table
     raven_result = DataFrame()
     raven_result["Selection"] = list(range(1, len(aplose_result) + 1))

tests/test_utils.py

@@ -14,6 +14,7 @@
 )
 from osekit.utils.formatting_utils import aplose2raven
 from osekit.utils.path_utils import move_tree
+from pandas import Timedelta
 
 
 @pytest.fixture
@@ -60,7 +61,7 @@ def audio_timestamps() -> list:
 
 @pytest.fixture
 def audio_durations(audio_timestamps: pytest.fixture) -> list:
-    return [30] * len(audio_timestamps)
+    return [Timedelta("30s")] * len(audio_timestamps)
 
 
 @pytest.mark.unit
@@ -71,7 +72,7 @@ def test_aplose2raven(
 ) -> None:
     raven_dataframe = aplose2raven(
         aplose_result=aplose_dataframe,
-        audio_datetimes=audio_timestamps,
+        list_audio_begin_time=audio_timestamps,
         audio_durations=audio_durations,
     )