SXT bugfixes and improvements (#801)

Various changes needed to make the SXT workflow smoother:

    Use olefile directly and remove the txrm2tiff dependency
    Read energy, tilt limits and tilt count from the txrm metadata and store them in the DataCollection ispyb table
    Skip processing of reference txrm files
    Allow processing of nested folders
    Set session file to None and then tag the DCG with the unaltered metadata path
    Set the data collection comment as any part of the file name before _angle

The remaining issue is how to set the rsyncers correctly, which we need to discuss

Author: stephen-riggs

pyproject.toml

@@ -72,7 +72,8 @@ smartem = [
     "smartem-decisions[backend]",
 ]
 sxt = [
-    "txrm2tiff",
+    "numpy<3",
+    "olefile",
 ]
 [project.urls]
 Bug-Tracker = "https://github.com/DiamondLightSource/python-murfey/issues"

src/murfey/client/context.py

@@ -76,6 +76,7 @@ def ensure_dcg_exists(
     token: str,
 ) -> str | None:
     """Create  a data collection group"""
+    session_file: Path | None = None
     if collection_type == "tomo":
         experiment_type_id = 36
         session_file = metadata_source / "Session.dm"
@@ -97,13 +98,18 @@ def ensure_dcg_exists(
                 logger.warning(f"Get EPU session hook failed: {e}")
     elif collection_type == "sxt":
         experiment_type_id = 47
-        session_file = metadata_source / "Session.dm"
         source_visit_dir = metadata_source.parent
     else:
         logger.error(f"Unknown collection type {collection_type}")
         return None
 
-    if not session_file.is_file():
+    if session_file is None:
+        dcg_tag = "/".join(metadata_source.parts).replace("//", "/")
+        dcg_data = {
+            "experiment_type_id": experiment_type_id,
+            "tag": dcg_tag,
+        }
+    elif not session_file.is_file():
         logger.warning(f"Cannot find session file {str(session_file)}")
         dcg_tag = "/".join(
             [part for part in metadata_source.parts if part != environment.visit]

src/murfey/client/contexts/sxt.py

@@ -2,10 +2,8 @@
 from pathlib import Path
 from typing import Any
 
-from txrm2tiff.inspector import Inspector
-from txrm2tiff.txrm import open_txrm
-from txrm2tiff.txrm_functions.general import read_stream
-from txrm2tiff.xradia_properties.enums import XrmDataTypes
+import numpy as np
+from olefile import OleFileIO
 
 from murfey.client.context import (
     Context,
@@ -46,12 +44,9 @@ def register_sxt_data_collection(
             )
             return
         try:
-            metadata_source = (
-                self._basepath.parent / environment.visit / self._basepath.name
-            )
             ensure_dcg_exists(
                 collection_type="sxt",
-                metadata_source=metadata_source,
+                metadata_source=self._basepath,
                 environment=environment,
                 machine_config=self._machine_config,
                 token=self._token,
@@ -66,12 +61,18 @@ def register_sxt_data_collection(
                 "source": str(self._basepath),
                 "tag": tilt_series,
                 "pixel_size_on_image": str(
-                    data_collection_parameters.get("pixel_size", 100)
-                ),
+                    round(data_collection_parameters.get("pixel_size", 100), 2) * 1e-10
+                ),  # expected in metres
                 "image_size_x": data_collection_parameters.get("image_size_x", 0),
                 "image_size_y": data_collection_parameters.get("image_size_y", 0),
                 "magnification": data_collection_parameters.get("magnification", 0),
+                "energy": data_collection_parameters.get("energy"),
                 "voltage": 0,
+                "axis_start": data_collection_parameters.get("minimum_angle"),
+                "axis_end": data_collection_parameters.get("maximum_angle"),
+                "tilt_series_length": data_collection_parameters.get(
+                    "tilt_series_length"
+                ),
             }
             capture_post(
                 base_url=str(environment.url.geturl()),
@@ -127,84 +128,103 @@ def post_transfer(
                 return False
 
             # Read the tilt angles and pixel size from the txrm
-            metadata = {
+            metadata: dict[str, Any] = {
                 "source": str(self._basepath),
                 "tilt_series_tag": transferred_file.stem,
             }
-            with open_txrm(
-                transferred_file, load_images=False, load_reference=False, strict=False
-            ) as txrm:
-                inspector = Inspector(txrm)
-                angles = read_stream(
-                    inspector.txrm.ole,
-                    "ImageInfo/Angles",
-                    XrmDataTypes.XRM_FLOAT,
-                    strict=True,
-                )
-                if angles:
+            with OleFileIO(str(transferred_file)) as txrm_ole:
+                if txrm_ole.exists("ReferenceData/Image"):
+                    metadata["has_reference"] = True
+
+                if txrm_ole.exists("ImageInfo/Angles"):
+                    angles = np.frombuffer(
+                        txrm_ole.openstream("ImageInfo/Angles").getvalue(), np.float32
+                    ).tolist()
                     metadata["minimum_angle"] = min(angles)
                     metadata["maximum_angle"] = max(angles)
 
-                pixel_size_txrm = read_stream(
-                    inspector.txrm.ole,
-                    "ImageInfo/PixelSize",
-                    XrmDataTypes.XRM_FLOAT,
-                    strict=True,
-                )
-                if pixel_size_txrm:
+                if txrm_ole.exists("ImageInfo/PixelSize"):
+                    pixel_size_txrm = np.frombuffer(
+                        txrm_ole.openstream("ImageInfo/PixelSize").getvalue(),
+                        np.float32,
+                    ).tolist()
                     metadata["pixel_size"] = pixel_size_txrm[0] * 1e4
 
-                image_width_txrm = read_stream(
-                    inspector.txrm.ole,
-                    "ImageInfo/ImageWidth",
-                    XrmDataTypes.XRM_INT,
-                    strict=True,
-                )
-                if image_width_txrm:
+                if txrm_ole.exists("ImageInfo/ImageWidth"):
+                    image_width_txrm = np.frombuffer(
+                        txrm_ole.openstream("ImageInfo/ImageWidth").getvalue(), np.int32
+                    ).tolist()
                     metadata["image_size_x"] = image_width_txrm[0]
 
-                image_height_txrm = read_stream(
-                    inspector.txrm.ole,
-                    "ImageInfo/ImageHeight",
-                    XrmDataTypes.XRM_INT,
-                    strict=True,
-                )
-                if image_height_txrm:
+                if txrm_ole.exists("ImageInfo/ImageHeight"):
+                    image_height_txrm = np.frombuffer(
+                        txrm_ole.openstream("ImageInfo/ImageHeight").getvalue(),
+                        np.int32,
+                    ).tolist()
                     metadata["image_size_y"] = image_height_txrm[0]
 
-                exposure_time_txrm = read_stream(
-                    inspector.txrm.ole,
-                    "ImageInfo/ExpTimes",
-                    XrmDataTypes.XRM_FLOAT,
-                    strict=True,
-                )
-                if exposure_time_txrm:
+                if txrm_ole.exists("ImageInfo/ExpTimes"):
+                    exposure_time_txrm = np.frombuffer(
+                        txrm_ole.openstream("ImageInfo/ExpTimes").getvalue(), np.float32
+                    ).tolist()
                     metadata["exposure_time"] = exposure_time_txrm[0]
 
-                magnification_txrm = read_stream(
-                    inspector.txrm.ole,
-                    "ImageInfo/XrayMagnification",
-                    XrmDataTypes.XRM_FLOAT,
-                    strict=True,
-                )
-                if magnification_txrm:
+                if txrm_ole.exists("ImageInfo/XrayMagnification"):
+                    magnification_txrm = np.frombuffer(
+                        txrm_ole.openstream("ImageInfo/XrayMagnification").getvalue(),
+                        np.float32,
+                    ).tolist()
                     metadata["magnification"] = magnification_txrm[0]
 
-                tilt_count_txrm = read_stream(
-                    inspector.txrm.ole,
-                    "ImageInfo/ImagesTaken",
-                    XrmDataTypes.XRM_INT,
-                    strict=True,
-                )
-                if tilt_count_txrm:
-                    metadata["tilt_count"] = tilt_count_txrm[0]
+                if txrm_ole.exists("ImageInfo/ImagesTaken"):
+                    tilt_count_txrm = np.frombuffer(
+                        txrm_ole.openstream("ImageInfo/ImagesTaken").getvalue(),
+                        np.int32,
+                    ).tolist()
+                    metadata["tilt_series_length"] = tilt_count_txrm[0]
+
+                if txrm_ole.exists("PositionInfo/AxisNames") and txrm_ole.exists(
+                    "PositionInfo/MotorPositions"
+                ):
+                    # The ImageInfo/Energy field is empty
+                    # Instead it needs extracting from the PositionInfo list
+                    axis_names = [
+                        i
+                        for i in txrm_ole.openstream("PositionInfo/AxisNames")
+                        .read()
+                        .decode("ascii")
+                        .split("\x00")
+                        if i
+                    ]
+                    axis_values = np.frombuffer(
+                        txrm_ole.openstream("PositionInfo/MotorPositions").getvalue(),
+                        np.float32,
+                    )
+                    if "Energy" in axis_names:
+                        energy_index = list(np.array(axis_names) == "Energy").index(
+                            True
+                        )
+                        metadata["energy"] = int(round(axis_values[energy_index]))
+
+            if not metadata.get("has_reference", False):
+                logger.debug(f"Reference image {transferred_file} not processed")
+                return True
 
+            visit_index = transferred_file.parent.parts.index(environment.visit)
+            destination_search_dir = "/".join(
+                transferred_file.parts[: visit_index + 2]
+            ).replace("//", "/")
             self.register_sxt_data_collection(
                 tilt_series=transferred_file.stem,
                 data_collection_parameters=metadata,
                 file_extension=transferred_file.suffix,
-                image_directory=environment.default_destinations.get(
-                    transferred_file.parent, transferred_file.parent
+                image_directory=str(
+                    Path(
+                        environment.default_destinations.get(
+                            Path(destination_search_dir), destination_search_dir
+                        )
+                    )
+                    / transferred_file.parent.relative_to(destination_search_dir)
                 ),
                 environment=environment,
             )
@@ -227,11 +247,15 @@ def post_transfer(
                 visit_name=environment.visit,
                 session_id=environment.murfey_session,
                 data={
-                    "session_id": environment.murfey_session,
                     "tag": transferred_file.stem,
-                    "source": str(transferred_file.parent),
-                    "pixel_size": metadata.get("pixel_size", 100),
+                    "source": destination_search_dir,
+                    "pixel_size": round(
+                        metadata.get("pixel_size", 100), 2
+                    ),  # angstroms
                     "tilt_offset": midpoint(angles),
+                    "tilt_series_length": metadata.get(
+                        "tilt_series_length", len(angles)
+                    ),
                     "txrm": str(file_transferred_to),
                 },
             )

src/murfey/server/api/workflow.py

@@ -290,11 +290,15 @@ class DCParameters(BaseModel):
     tag: str
     source: str
     magnification: float
-    total_exposed_dose: Optional[float] = None
-    c2aperture: Optional[float] = None
-    exposure_time: Optional[float] = None
-    slit_width: Optional[float] = None
+    total_exposed_dose: float | None = None
+    c2aperture: float | None = None
+    exposure_time: float | None = None
+    slit_width: float | None = None
     phase_plate: bool = False
+    energy: float | None = None
+    axis_start: float | None = None
+    axis_end: float | None = None
+    tilt_series_length: int | None = None
     data_collection_tag: str = ""
 
 
@@ -321,6 +325,7 @@ def start_dc(
         "image_directory": str(rsync_basepath / dc_params.image_directory),
         "start_time": str(datetime.now()),
         "voltage": dc_params.voltage,
+        "energy": dc_params.energy,
         "pixel_size": str(float(dc_params.pixel_size_on_image) * 1e9),
         "image_suffix": dc_params.file_extension,
         "experiment_type": dc_params.experiment_type,
@@ -335,6 +340,9 @@ def start_dc(
         "exposure_time": dc_params.exposure_time,
         "slit_width": dc_params.slit_width,
         "phase_plate": dc_params.phase_plate,
+        "axis_start": dc_params.axis_start,
+        "axis_end": dc_params.axis_end,
+        "tilt_series_length": dc_params.tilt_series_length,
         "session_id": session_id,
     }
 

src/murfey/workflows/register_data_collection.py

@@ -65,6 +65,7 @@ def run(message: dict, murfey_db: SQLModelSession) -> dict[str, bool]:
                 imageDirectory=message["image_directory"],
                 imageSuffix=message["image_suffix"],
                 voltage=message["voltage"],
+                wavelength=message["energy"],
                 dataCollectionGroupId=dcgid,
                 pixelSizeOnImage=message["pixel_size"],
                 imageSizeX=message["image_size_x"],
@@ -75,13 +76,20 @@ def run(message: dict, murfey_db: SQLModelSession) -> dict[str, bool]:
                 totalExposedDose=message.get("total_exposed_dose"),
                 c2aperture=message.get("c2aperture"),
                 phasePlate=int(message.get("phase_plate", 0)),
+                axisStart=message.get("axis_start"),
+                axisEnd=message.get("axis_end"),
+                numberOfImages=message.get("tilt_series_length"),
             )
             dcid = _transport_object.do_insert_data_collection(
                 record,
                 tag=(
                     message.get("tag")
                     if message["experiment_type"] == "tomography"
-                    else ""
+                    else (
+                        message.get("tag", "").split("_angle")[0]
+                        if message["experiment_type"] == "sxt"
+                        else ""
+                    )
                 ),
             ).get("return_value", None)
             if dcid is None:

src/murfey/workflows/sxt/process_sxt_tilt_series.py

@@ -22,7 +22,6 @@
 
 
 class SXTTiltSeriesInfo(BaseModel):
-    session_id: int
     tag: str
     source: str
     txrm: str
@@ -39,19 +38,22 @@ def process_sxt_tilt_series_workflow(
 ):
     tilt_series_query = murfey_db.exec(
         select(TiltSeries)
-        .where(TiltSeries.session_id == tilt_series_info.session_id)
+        .where(TiltSeries.session_id == session_id)
         .where(TiltSeries.tag == tilt_series_info.tag)
         .where(TiltSeries.rsync_source == tilt_series_info.source)
     ).all()
     if tilt_series_query:
         tilt_series = tilt_series_query[0]
+        if tilt_series.processing_requested:
+            logger.info(f"Tilt series {tilt_series.tag} has already been processed")
+            return
     else:
         tilt_series = TiltSeries(
             session_id=session_id,
             tag=tilt_series_info.tag,
             rsync_source=tilt_series_info.source,
             tilt_series_length=tilt_series_info.tilt_series_length,
-            processing_requested=True,
+            processing_requested=False,
         )
         murfey_db.add(tilt_series)
         murfey_db.commit()
@@ -114,3 +116,6 @@ def process_sxt_tilt_series_workflow(
         logger.info(
             f"No transport object found. Zocalo message would be {sanitise(str(zocalo_message))}"
         )
+    tilt_series.processing_requested = True
+    murfey_db.add(tilt_series)
+    murfey_db.commit()