Merge branch 'main' into dcg-source-fix

Author: stephen-riggs

.github/workflows/test.yml

@@ -3,10 +3,11 @@ name: Build and test
 on: [push, pull_request]
 
 env:
-  ISPYB_DATABASE_SCHEMA: 4.8.0
+  ISPYB_DATABASE_SCHEMA: 4.11.0
   # Installs from GitHub
   # Versions: https://github.com/DiamondLightSource/ispyb-database/tags
   # Previous version(s):
+  # 4.8.0
   # 4.2.1 # released 2024-08-19
   # 4.1.0 # released 2024-03-26
 

src/murfey/client/analyser.py

@@ -65,7 +65,6 @@ def __init__(
                 str(environment.url.geturl()),
                 self._token,
                 instrument_name=environment.instrument_name,
-                demo=environment.demo,
             )
             if environment
             else {}
@@ -398,7 +397,9 @@ def _xml_file(self, data_file: Path) -> Path:
         base_dir, mid_dir = find_longest_data_directory(data_file, data_directories)
         if not base_dir:
             return data_file.with_suffix(".xml")
-        return base_dir / self._environment.visit / mid_dir / file_name
+        # Add the visit directory to the file path and return it
+        # The file is moved from a location where the visit name is not part of its path
+        return base_dir / self._environment.visit / (mid_dir or "") / file_name
 
     def enqueue(self, rsyncer: RSyncerUpdate):
         if not self._stopping and rsyncer.outcome == TransferResult.SUCCESS:

src/murfey/client/context.py

@@ -20,7 +20,6 @@ def _atlas_destination(
         str(environment.url.geturl()),
         token,
         instrument_name=environment.instrument_name,
-        demo=environment.demo,
     )
     for i, destination_part in enumerate(
         Path(environment.default_destinations[source]).parts

src/murfey/client/contexts/clem.py

@@ -28,7 +28,6 @@ def _file_transferred_to(
         str(environment.url.geturl()),
         token,
         instrument_name=environment.instrument_name,
-        demo=environment.demo,
     )
 
     # Construct destination path

src/murfey/client/contexts/spa.py

@@ -3,7 +3,7 @@
 import logging
 from itertools import count
 from pathlib import Path
-from typing import Any, Dict, List, Optional, OrderedDict, Tuple
+from typing import Any, Optional, OrderedDict
 
 import xmltodict
 
@@ -33,7 +33,6 @@ def _file_transferred_to(
         str(environment.url.geturl()),
         token,
         instrument_name=environment.instrument_name,
-        demo=environment.demo,
     )
     if environment.visit in environment.default_destinations[source]:
         return (
@@ -50,10 +49,10 @@ def _file_transferred_to(
 
 
 def _grid_square_metadata_file(
-    f: Path, data_directories: List[Path], visit: str, grid_square: int
+    f: Path, data_directories: list[Path], visit: str, grid_square: int
 ) -> Path:
     base_dir, mid_dir = find_longest_data_directory(f, data_directories)
-    if not base_dir:
+    if not base_dir or not mid_dir:
         raise ValueError(f"Could not determine grid square metadata path for {f}")
     metadata_file = (
         base_dir
@@ -113,7 +112,7 @@ def __init__(self, acquisition_software: str, basepath: Path, token: str):
         super().__init__("SPA", acquisition_software, token)
         self._basepath = basepath
         self._processing_job_stash: dict = {}
-        self._foil_holes: Dict[int, List[int]] = {}
+        self._foil_holes: dict[int, list[int]] = {}
 
     def gather_metadata(
         self, metadata_file: Path, environment: MurfeyInstanceEnvironment | None = None
@@ -287,7 +286,7 @@ def _position_analysis(
             and self._foil_holes.get(grid_square) is None
         ):
             self._foil_holes[grid_square] = []
-            gs_pix_position: Tuple[
+            gs_pix_position: tuple[
                 Optional[int],
                 Optional[int],
                 Optional[float],
@@ -453,7 +452,6 @@ def post_transfer(
                             str(environment.url.geturl()),
                             self._token,
                             instrument_name=environment.instrument_name,
-                            demo=environment.demo,
                         )
                     else:
                         machine_config = {}
@@ -586,7 +584,7 @@ def _register_processing_job(
         self,
         tag: str,
         environment: MurfeyInstanceEnvironment,
-        parameters: Dict[str, Any] | None = None,
+        parameters: dict[str, Any] | None = None,
     ):
         return
 

src/murfey/client/contexts/tomo.py

@@ -191,7 +191,6 @@ def _file_transferred_to(
             str(environment.url.geturl()),
             self._token,
             instrument_name=environment.instrument_name,
-            demo=environment.demo,
         )
         if environment.visit in environment.default_destinations[source]:
             return (
@@ -465,7 +464,6 @@ def post_transfer(
                             str(environment.url.geturl()),
                             self._token,
                             instrument_name=environment.instrument_name,
-                            demo=environment.demo,
                         )
                     else:
                         machine_config = {}

src/murfey/client/destinations.py

@@ -15,17 +15,19 @@
 
 def find_longest_data_directory(
     match_path: Path, data_directories: list[str] | list[Path]
-):
+) -> tuple[Path | None, Path | None]:
     """
     Determine the longest path in the data_directories list
     which the match path is relative to
     """
     base_dir: Path | None = None
     mid_dir: Path | None = None
     for dd in data_directories:
-        dd_base = str(Path(dd).absolute())
-        if str(match_path).startswith(str(dd)) and len(dd_base) > len(str(base_dir)):
-            base_dir = Path(dd_base)
+        dd_base = Path(dd).absolute()
+        if match_path.absolute().is_relative_to(dd_base) and len(dd_base.parts) > (
+            len(base_dir.parts) if base_dir else 0
+        ):
+            base_dir = dd_base
             mid_dir = match_path.absolute().relative_to(Path(base_dir)).parent
     return base_dir, mid_dir
 

src/murfey/client/instance_environment.py

@@ -40,7 +40,6 @@ class MurfeyInstanceEnvironment(BaseModel):
     default_destinations: Dict[Path, str] = {}
     destination_registry: Dict[str, str] = {}
     watchers: Dict[Path, DirWatcher] = {}
-    demo: bool = False
     movies: Dict[Path, MovieTracker] = {}
     movie_tilt_pair: Dict[Path, str] = {}
     tilt_angles: Dict[str, List[List[str]]] = {}

src/murfey/client/multigrid_control.py

@@ -37,7 +37,6 @@ class MultigridController:
     murfey_url: str = "http://localhost:8000"
     rsync_url: str = ""
     rsync_module: str = "data"
-    demo: bool = False
     finalising: bool = False
     dormant: bool = False
     multigrid_watcher_active: bool = True
@@ -68,7 +67,6 @@ def __post_init__(self):
             client_id=0,
             murfey_session=self.session_id,
             software_versions=machine_data.get("software_versions", {}),
-            demo=self.demo,
             visit=self.visit,
             dose_per_frame=self.data_collection_parameters.get("dose_per_frame"),
             gain_ref=self.data_collection_parameters.get("gain_ref"),
@@ -80,7 +78,6 @@ def __post_init__(self):
             str(self._environment.url.geturl()),
             self.token,
             instrument_name=self._environment.instrument_name,
-            demo=self._environment.demo,
         )
         self._data_suffixes = (".mrc", ".tiff", ".tif", ".eer")
         self._data_substrings = [
@@ -430,7 +427,7 @@ def rsync_result(update: RSyncerUpdate):
                     "source": str(source),
                     "destination": destination,
                     "session_id": self.session_id,
-                    "transferring": self.do_transfer or self._environment.demo,
+                    "transferring": self.do_transfer,
                     "tag": tag,
                 }
                 capture_post(

src/murfey/instrument_server/api.py

@@ -173,7 +173,6 @@ def setup_multigrid_watcher(
         watcher_spec.instrument_name,
         session_id,
         murfey_url=_get_murfey_url(),
-        demo=True,
         do_transfer=True,
         processing_enabled=not watcher_spec.skip_existing_processing,
         _machine_config=machine_config,