Initial code to insert searchmaps into murfey and ispyb databases

Author: stephen-riggs

src/murfey/client/contexts/tomo_metadata.py

@@ -152,9 +152,6 @@ def post_transfer(
                     "Binning"
                 ]["a:x"]
             )
-            readout_area = sm_data["MicroscopeImage"]["microscopeData"]["acquisition"][
-                "camera"
-            ]["ReadoutArea"]
 
             # Get the stage transformation
             sm_transformations = sm_data["MicroscopeImage"]["CustomData"][
@@ -222,14 +219,6 @@ def post_transfer(
                     "tag": visitless_source,
                     "x_stage_position": float(stage_position["X"]),
                     "y_stage_position": float(stage_position["Y"]),
-                    "readout_area_x": readout_area[0],
-                    "readout_area_y": readout_area[1],
-                    "thumbnail_size_x": int(
-                        (512 / max(readout_area)) * readout_area[0]
-                    ),
-                    "thumbnail_size_y": int(
-                        (512 / max(readout_area)) * readout_area[1]
-                    ),
                     "pixel_size": sm_pixel_size,
                     "image": str(image_path),
                     "binning": sm_binning,
@@ -248,7 +237,7 @@ def post_transfer(
             if not visitless_source:
                 return
 
-            # This bit gets SearchMap location on Atlas
+            # This bit gets SearchMap size
             sm_width = int(sm_data["TileSetXml"]["ImageSize"]["a:width"])
             sm_height = int(sm_data["TileSetXml"]["ImageSize"]["a:height"])
 
@@ -286,8 +275,8 @@ def post_transfer(
                     bp_url,
                     json={
                         "tag": visitless_source,
-                        "stage_position_x": batch_stage_location_x,
-                        "stage_position_y": batch_stage_location_y,
+                        "x_stage_position": batch_stage_location_x,
+                        "y_stage_position": batch_stage_location_y,
                         "search_map": search_map_name,
                     },
                 )
@@ -309,8 +298,8 @@ def post_transfer(
                             bp_url,
                             json={
                                 "tag": visitless_source,
-                                "stage_position_x": beamshift_position_x,
-                                "stage_position_y": beamshift_position_y,
+                                "x_stage_position": beamshift_position_x,
+                                "y_stage_position": beamshift_position_y,
                                 "search_map": search_map_name,
                             },
                         )

src/murfey/server/api/session_control.py

@@ -46,12 +46,18 @@
     Session,
 )
 from murfey.util.models import (
+    BatchPositionParameters,
     ClientInfo,
     FoilHoleParameters,
     GridSquareParameters,
     RsyncerInfo,
+    SearchMapParameters,
     Visit,
 )
+from murfey.util.tomo_metadata import (
+    register_batch_position_in_database,
+    register_search_map_in_database,
+)
 from murfey.workflows.spa.flush_spa_preprocess import (
     register_foil_hole as _register_foil_hole,
 )
@@ -364,6 +370,33 @@ def register_foil_hole(
     return _register_foil_hole(session_id, gs_name, foil_hole_params, db)
 
 
+tomography_router = APIRouter(
+    prefix="/session_control/tomography",
+    dependencies=[Depends(validate_instrument_token)],
+    tags=["Session Control: Tomography"],
+)
+
+
+@tomography_router.post("/sessions/{session_id}/search_map/{sm_name}")
+def register_search_map(
+    session_id: MurfeySessionID,
+    sm_name: str,
+    search_map_params: SearchMapParameters,
+    db=murfey_db,
+):
+    return register_search_map_in_database(session_id, sm_name, search_map_params, db)
+
+
+@tomography_router.post("/sessions/{session_id}/batch_position/{batch_name}")
+def register_batch_position(
+    session_id: MurfeySessionID,
+    batch_name: str,
+    batch_params: BatchPositionParameters,
+    db=murfey_db,
+):
+    return register_batch_position_in_database(session_id, batch_name, batch_params, db)
+
+
 correlative_router = APIRouter(
     prefix="/session_control/correlative",
     dependencies=[Depends(validate_instrument_token)],

src/murfey/server/api/workflow.py

@@ -81,7 +81,10 @@ class DCGroupParameters(BaseModel):
     tag: str
     atlas: str = ""
     sample: Optional[int] = None
-    atlas_pixel_size: int = 0
+    atlas_pixel_size: float = 0
+    atlas_size_x: int = 0
+    atlas_size_y: int = 0
+    atlas_binning: int = 1
 
 
 @router.post("/visits/{visit_name}/{session_id}/register_data_collection_group")
@@ -116,6 +119,9 @@ def register_dc_group(
                         "atlas": dcg_params.atlas,
                         "sample": dcg_params.sample,
                         "atlas_pixel_size": dcg_params.atlas_pixel_size,
+                        "atlas_size_x": dcg_params.atlas_size_x,
+                        "atlas_size_y": dcg_params.atlas_size_y,
+                        "atlas_binning": dcg_params.atlas_binning,
                         "dcgid": dcg_murfey[0].id,
                         "session_id": session_id,
                     },

src/murfey/server/ispyb.py

@@ -32,7 +32,11 @@
 
 from murfey.util import sanitise
 from murfey.util.config import get_security_config
-from murfey.util.models import FoilHoleParameters, GridSquareParameters
+from murfey.util.models import (
+    FoilHoleParameters,
+    GridSquareParameters,
+    SearchMapParameters,
+)
 
 log = logging.getLogger("murfey.server.ispyb")
 security_config = get_security_config()
@@ -388,6 +392,90 @@ def do_update_foil_hole(
             )
         return {"success": False, "return_value": None}
 
+    def do_insert_search_map(
+        self,
+        atlas_id: int,
+        search_map_parameters: SearchMapParameters,
+    ):
+        if (
+            search_map_parameters.pixel_size
+            and search_map_parameters.height
+            and search_map_parameters.height_on_atlas
+        ):
+            search_map_parameters.pixel_size *= (
+                search_map_parameters.height / search_map_parameters.height_on_atlas
+            )
+        record = GridSquare(
+            atlasId=atlas_id,
+            gridSquareImage=search_map_parameters.image,
+            pixelLocationX=search_map_parameters.x_location,
+            pixelLocationY=search_map_parameters.y_location,
+            height=search_map_parameters.height_on_atlas,
+            width=search_map_parameters.width_on_atlas,
+            stageLocationX=search_map_parameters.x_stage_position,
+            stageLocationY=search_map_parameters.y_stage_position,
+            pixelSize=search_map_parameters.pixel_size,
+        )
+        try:
+            with ISPyBSession() as db:
+                db.add(record)
+                db.commit()
+                log.info(f"Created SearchMap (GridSquare) {record.gridSquareId}")
+                return {"success": True, "return_value": record.gridSquareId}
+        except ispyb.ISPyBException as e:
+            log.error(
+                "Inserting SearchMap (GridSquare) entry caused exception '%s'.",
+                e,
+                exc_info=True,
+            )
+        return {"success": False, "return_value": None}
+
+    def do_update_search_map(
+        self, search_map_id, search_map_parameters: SearchMapParameters
+    ):
+        try:
+            with ISPyBSession() as db:
+                grid_square = (
+                    db.query(GridSquare)
+                    .filter(GridSquare.gridSquareId == search_map_id)
+                    .one()
+                )
+                if (
+                    search_map_parameters.pixel_size
+                    and search_map_parameters.height
+                    and search_map_parameters.height_on_atlas
+                ):
+                    search_map_parameters.pixel_size *= (
+                        search_map_parameters.height
+                        / search_map_parameters.height_on_atlas
+                    )
+                if search_map_parameters.image:
+                    grid_square.gridSquareImage = search_map_parameters.image
+                if search_map_parameters.x_location:
+                    grid_square.pixelLocationX = search_map_parameters.x_location
+                if search_map_parameters.y_location:
+                    grid_square.pixelLocationY = search_map_parameters.y_location
+                if search_map_parameters.height_on_atlas:
+                    grid_square.height = search_map_parameters.height_on_atlas
+                if search_map_parameters.width_on_atlas:
+                    grid_square.width = search_map_parameters.width_on_atlas
+                if search_map_parameters.x_stage_position:
+                    grid_square.stageLocationX = search_map_parameters.x_stage_position
+                if search_map_parameters.y_stage_position:
+                    grid_square.stageLocationY = search_map_parameters.y_stage_position
+                if search_map_parameters.pixel_size:
+                    grid_square.pixelSize = search_map_parameters.pixel_size
+                db.add(grid_square)
+                db.commit()
+                return {"success": True, "return_value": grid_square.gridSquareId}
+        except ispyb.ISPyBException as e:
+            log.error(
+                "Updating SearchMap (GridSquare) entry caused exception '%s'.",
+                e,
+                exc_info=True,
+            )
+        return {"success": False, "return_value": None}
+
     def send(self, queue: str, message: dict, new_connection: bool = False):
         if self.transport:
             if not self.transport.is_connected():

src/murfey/util/db.py

@@ -372,6 +372,7 @@ class DataCollectionGroup(SQLModel, table=True):  # type: ignore
     atlas_id: Optional[int] = None
     atlas_pixel_size: Optional[float] = None
     atlas: str = ""
+    atlas_binning: Optional[int] = None
     sample: Optional[int] = None
     session: Optional[Session] = Relationship(back_populates="data_collection_groups")
     data_collections: List["DataCollection"] = Relationship(
@@ -601,6 +602,24 @@ class FoilHole(SQLModel, table=True):  # type: ignore
     )
 
 
+class SearchMap(SQLModel, table=True):  # type: ignore
+    id: Optional[int] = Field(primary_key=True, default=None)
+    session_id: int = Field(foreign_key="session.id")
+    name: str
+    tag: str
+    x_location: Optional[float] = None
+    y_location: Optional[float] = None
+    x_stage_position: Optional[float] = None
+    y_stage_position: Optional[float] = None
+    pixel_size: Optional[float] = None
+    image: str = ""
+    binning: Optional[float] = None
+    reference_matrix: Optional[dict[str, float]] = None
+    stage_correction: Optional[dict[str, float]] = None
+    image_shift_correction: Optional[dict[str, float]] = None
+    session: Optional[Session] = Relationship(back_populates="grid_squares")
+
+
 class Movie(SQLModel, table=True):  # type: ignore
     murfey_id: int = Field(primary_key=True, foreign_key="murfeyledger.id")
     foil_hole_id: int = Field(foreign_key="foilhole.id", nullable=True, default=None)

src/murfey/util/models.py

@@ -136,6 +136,31 @@ class FoilHoleParameters(BaseModel):
     diameter: Optional[float] = None
 
 
+class SearchMapParameters:
+    tag: str
+    x_location: Optional[float] = None
+    y_location: Optional[float] = None
+    x_stage_position: Optional[float] = None
+    y_stage_position: Optional[float] = None
+    pixel_size: Optional[float] = None
+    image: Optional[str] = None
+    binning: Optional[float] = None
+    reference_matrix: Optional[dict[str, float]] = None
+    stage_correction: Optional[dict[str, float]] = None
+    image_shift_correction: Optional[dict[str, float]] = None
+    height: Optional[int] = None
+    width: Optional[int] = None
+    height_on_atlas: Optional[int] = None
+    width_on_atlas: Optional[int] = None
+
+
+class BatchPositionParameters:
+    tag: str
+    x_stage_position: float
+    Y_stage_position: float
+    search_map: str
+
+
 class MultigridWatcherSetup(BaseModel):
     source: Path
     skip_existing_processing: bool = False