changes to validation

Author: Aser Abdelhakeem

src/sed/loader/cfel/buffer_handler.py

@@ -9,6 +9,7 @@
 from sed.loader.cfel.dataframe import DataFrameCreator
 from sed.loader.flash.buffer_handler import BufferFilePaths
 from sed.loader.flash.buffer_handler import BufferHandler as BaseBufferHandler
+from sed.loader.flash.utils import InvalidFileError
 from sed.loader.flash.utils import get_channels
 from sed.loader.flash.utils import get_dtypes
 
@@ -32,6 +33,18 @@ def __init__(
         """
         super().__init__(config)
 
+    def _validate_h5_files(self, config, h5_paths: list[Path]) -> list[Path]:
+        valid_h5_paths = []
+        for h5_path in h5_paths:
+            try:
+                dfc = DataFrameCreator(config_dataframe=config, h5_path=h5_path)
+                dfc.validate_channel_keys()
+                valid_h5_paths.append(h5_path)
+            except InvalidFileError as e:
+                logger.info(f"Skipping invalid file: {h5_path.stem}\n{e}")
+
+        return valid_h5_paths    
+
     def _save_buffer_file(self, paths: dict[str, Path]) -> None:
         """Creates the electron and timed buffer files from the raw H5 file."""
         logger.debug(f"Processing file: {paths['raw'].stem}")

src/sed/loader/cfel/dataframe.py

@@ -165,10 +165,10 @@ def df_timestamp(self) -> pd.DataFrame:
 
     def validate_channel_keys(self) -> None:
         """
-        Validates if the index and dataset keys for all channels in the config exist in the h5 file.
+        Validates if the dataset keys for all channels in the config exist in the h5 file.
 
         Raises:
-            InvalidFileError: If the index or dataset keys are missing in the h5 file.
+            InvalidFileError: If the dataset keys are missing in the h5 file.
         """
         invalid_channels = []
         for channel in self._config["channels"]:
@@ -182,7 +182,7 @@ def validate_channel_keys(self) -> None:
     @property
     def df(self) -> pd.DataFrame:
         """
-        Joins the 'per_electron', 'per_pulse', and 'per_train' using concat operation,
+        Joins the 'per_electron', 'per_pulse' using concat operation,
         returning a single dataframe.
 
         Returns:
@@ -201,7 +201,7 @@ def df(self) -> pd.DataFrame:
     @property
     def df_timed(self) -> pd.DataFrame:
         """
-        Joins the 'per_electron', 'per_pulse', and 'per_train' using concat operation,
+        Joins the 'per_electron', 'per_pulse' using concat operation,
         returning a single dataframe.
 
         Returns:

src/sed/loader/cfel/loader.py

@@ -289,6 +289,8 @@ def get_elapsed_time_from_fid(fid):
             try:
                 fid = str(fid)  # Ensure the key is a string
                 time_stamps = file_statistics[fid]["columns"][time_stamp_alias]
+                print(f"Time stamp max: {time_stamps['max']}")
+                print(f"Time stamp min: {time_stamps['min']}")
                 elapsed_time = time_stamps["max"] - time_stamps["min"]
             except KeyError as exc:
                 raise KeyError(