Match dcgs for tomo multigrid (#819)

The data collection groups for atlas, metadata and fractions were not being matched up correctly for tomo multigrid.

This is due to the metadata being in Supervisor/Sample and the data in Supervisor_Sample

This resolves this issue by always using the metadata form with a slash. In the tomo context it checks the existence of the different metadata locations and determines the metadata_source and group_tag from this.

Also catches errors from being unable to read grid squares on a tomo atlas, and not copying the batch positions list.

Author: stephen-riggs

src/murfey/client/contexts/atlas.py

@@ -162,7 +162,11 @@ def post_transfer_epu(
 
         elif environment and transferred_file.name == "Atlas.dm":
             # Register all grid squares on this atlas
-            gs_pix_positions = get_grid_square_atlas_positions(transferred_file)
+            try:
+                gs_pix_positions = get_grid_square_atlas_positions(transferred_file)
+            except KeyError:
+                logger.info("Unable to read grid square locations from Atlas.dm")
+                return
             for gs, pos_data in gs_pix_positions.items():
                 if pos_data:
                     capture_post(

src/murfey/client/contexts/tomo.py

@@ -93,6 +93,7 @@ def __init__(
         self._data_collection_stash: list = []
         self._processing_job_stash: dict = {}
         self._lock: RLock = RLock()
+        self._group_tag: str = str(self._basepath)
 
     def register_tomography_data_collections(
         self,
@@ -109,6 +110,20 @@ def register_tomography_data_collections(
             metadata_source = (
                 self._basepath.parent / environment.visit / self._basepath.name
             )
+            multigrid_metadata_source = (
+                self._basepath.parent
+                / environment.visit
+                / "_".join(self._basepath.name.split("_")[:-1])
+                / self._basepath.name.split("_")[-1]
+            )
+            if not metadata_source.is_dir() and multigrid_metadata_source.is_dir():
+                # If the multigrid path exists, tags need to replace the last _ with /
+                metadata_source = multigrid_metadata_source
+                self._group_tag = str(
+                    self._basepath.parent
+                    / "_".join(self._basepath.name.split("_")[:-1])
+                    / self._basepath.name.split("_")[-1]
+                )
             ensure_dcg_exists(
                 collection_type="tomo",
                 metadata_source=metadata_source,
@@ -125,7 +140,7 @@ def register_tomography_data_collections(
                         "acquisition_software": self._acquisition_software,
                         "image_directory": image_directory,
                         "data_collection_tag": tilt_series,
-                        "source": str(self._basepath),
+                        "source": self._group_tag,
                         "tag": tilt_series,
                     }
                     if (
@@ -178,7 +193,7 @@ def register_tomography_data_collections(
                                 session_id=environment.murfey_session,
                                 data={
                                     "tag": tilt_series,
-                                    "source": str(self._basepath),
+                                    "source": self._group_tag,
                                     "recipe": recipe,
                                     "experiment_type": "tomography",
                                 },
@@ -390,7 +405,7 @@ def _add_tilt(
                 "voltage": self.data_collection_parameters.get("voltage", 300),
                 "eer_fractionation_file": eer_fractionation_file,
                 "tag": tilt_series,
-                "group_tag": str(self._basepath),
+                "group_tag": self._group_tag,
             }
             capture_post(
                 base_url=str(environment.url.geturl()),
@@ -607,7 +622,7 @@ def gather_metadata(
                 environment.dose_per_frame if environment else None
             )
             mdoc_metadata["source"] = str(self._basepath)
-            mdoc_metadata["tag"] = str(self._basepath)
+            mdoc_metadata["tag"] = self._group_tag
             mdoc_metadata["tilt_series_tag"] = metadata_file.stem
             mdoc_metadata["exposure_time"] = float(mdoc_data_block["ExposureTime"])
             slit_width = mdoc_data_block["FilterSlitAndLoss"][0]

src/murfey/client/contexts/tomo_metadata.py

@@ -226,11 +226,14 @@ def post_transfer(
 
         elif transferred_file.name == "BatchPositionsList.xml":
             logger.info("Tomography session batch positions list found")
-            shutil.copy(
-                transferred_file,
-                transferred_file.parent
-                / f"{transferred_file.stem}-{datetime.now().strftime('%Y-%m-%d-%H-%M-%S')}{transferred_file.suffix}",
-            )
+            try:
+                shutil.copy(
+                    transferred_file,
+                    transferred_file.parent
+                    / f"{transferred_file.stem}-{datetime.now().strftime('%Y-%m-%d-%H-%M-%S')}{transferred_file.suffix}",
+                )
+            except PermissionError:
+                logger.warning("Unable to copy batch positions list")
             dcg_tag = ensure_dcg_exists(
                 collection_type="tomo",
                 metadata_source=metadata_source,

src/murfey/client/multigrid_control.py

@@ -520,7 +520,6 @@ def _start_dc(self, metadata_json):
                 k: None if v == "None" else v for k, v in metadata_json.items()
             }
         if isinstance(context, TomographyContext):
-            source = Path(metadata_json["source"])
             context.register_tomography_data_collections(
                 file_extension=metadata_json["file_extension"],
                 image_directory=str(self._environment.default_destinations[source]),
@@ -565,7 +564,7 @@ def _start_dc(self, metadata_json):
                 instrument_name=self._environment.instrument_name,
                 visit_name=self._environment.visit,
                 session_id=self._environment.murfey_session,
-                data={"rsync_source": str(source)},
+                data={"rsync_source": metadata_json.get("tag", "")},
             )
             log.info("Tomography processing flushed")