Skip to content

Commit 39c27c6

Browse files
tieneupintieneupin
authored
Register FIB milling progress (#815)
* Bumped required 'ispyb' version * Updated ISPyB schema version * Added new Murfey workflow module for registering FIB milling progress that can be run through the 'feedback_callback' function * Added new TransportManager functions to insert and update MillingStep database entries * Moved 'DataCollectionGroup' and 'GridSquare' to the 'General' section, and added a new table to record a Murfey-side copy of ISPyB's 'MillingStep' table * Added logic to register FIB milling metadata in ISPyB and Murfey * Add integrated test for the FIB milling progress workflow
1 parent 3613f6a commit 39c27c6

9 files changed

Lines changed: 1310 additions & 91 deletions

File tree

‎.github/workflows/test.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,11 @@ name: Build and test
33
on: [push, pull_request]
44

55
env:
6-
ISPYB_DATABASE_SCHEMA: 4.11.0
6+
ISPYB_DATABASE_SCHEMA: 5.1.0
77
# Installs from GitHub
88
# Versions: https://github.com/DiamondLightSource/ispyb-database/tags
99
# Previous version(s):
10+
# 4.11.0
1011
# 4.8.0
1112
# 4.2.1 # released 2024-08-19
1213
# 4.1.0 # released 2024-03-26

‎pyproject.toml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ developer = [
5555
server = [
5656
"cryptography",
5757
"graypy",
58-
"ispyb>=10.2.4", # Responsible for setting requirements for SQLAlchemy and mysql-connector-python;
58+
"ispyb>=12.1.0", # Responsible for setting requirements for SQLAlchemy and mysql-connector-python;
5959
"jinja2",
6060
"mrcfile",
6161
"numpy<3",
@@ -117,6 +117,7 @@ TomographyMetadataContext = "murfey.client.contexts.tomo_metadata:TomographyMeta
117117
"data_collection_group" = "murfey.workflows.register_data_collection_group:run"
118118
"experiment_type_update" = "murfey.workflows.register_experiment_type_update:run"
119119
"fib.register_atlas" = "murfey.workflows.fib.register_atlas:run"
120+
"fib.register_milling_progress" = "murfey.workflows.fib.register_milling_progress:run"
120121
"pato" = "murfey.workflows.notifications:notification_setup"
121122
"picked_particles" = "murfey.workflows.spa.picking:particles_picked"
122123
"picked_tomogram" = "murfey.workflows.tomo.picking:picked_tomogram"

‎src/murfey/server/api/workflow_fib.py‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
import json
21
import logging
32
import os
43
from pathlib import Path
@@ -7,7 +6,7 @@
76
import PIL.Image
87
from fastapi import APIRouter, Depends
98
from pydantic import BaseModel
10-
from sqlmodel import Session, select
9+
from sqlmodel import select
1110

1211
import murfey.util.db as MurfeyDB
1312
from murfey.server import _transport_object
@@ -36,7 +35,7 @@ def register_fib_atlas(
3635
fib_atlas: FIBAtlasFile,
3736
):
3837
if _transport_object is None:
39-
logger.error("No Transport Manager object was set up")
38+
logger.error("No TransportManager object was set up")
4039
return None
4140
_transport_object.send(
4241
_transport_object.feedback_queue,
@@ -52,11 +51,17 @@ def register_fib_atlas(
5251
def register_fib_milling_progress(
5352
session_id: int,
5453
site_info: LamellaSiteInfo,
55-
db: Session = murfey_db,
5654
):
57-
logger.debug(
58-
"Received the following FIB metadata for registration:\n"
59-
f"{json.dumps(site_info.model_dump(exclude_none=True), indent=2, default=str)}"
55+
if _transport_object is None:
56+
logger.error("No TransportManager object was set up")
57+
return None
58+
_transport_object.send(
59+
_transport_object.feedback_queue,
60+
{
61+
"register": "fib.register_milling_progress",
62+
"session_id": session_id,
63+
"site_info": site_info.model_dump(exclude_none=True),
64+
},
6065
)
6166

6267

‎src/murfey/server/ispyb.py‎

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@
2020
DataCollectionGroup,
2121
FoilHole,
2222
GridSquare,
23+
MillingStep,
24+
MillingStepName,
2325
ProcessingJob,
2426
ProcessingJobParameter,
2527
Proposal,
@@ -710,6 +712,164 @@ def do_update_processing_status(self, record: AutoProcProgram, **kwargs):
710712
)
711713
return {"success": False, "return_value": None}
712714

715+
def do_insert_milling_step(
716+
self,
717+
# MillingStepName identifiers
718+
recipe_name: str,
719+
activity_name: str,
720+
grid_square_id: int,
721+
# Values
722+
is_enabled: bool | None = None,
723+
status: str | None = None,
724+
execution_time: float | None = None,
725+
stage_x: float | None = None,
726+
stage_y: float | None = None,
727+
stage_z: float | None = None,
728+
rotation: float | None = None,
729+
tilt_alpha: float | None = None,
730+
beam_type: str | None = None,
731+
beam_voltage: float | None = None,
732+
beam_current: float | None = None,
733+
milling_angle: float | None = None,
734+
depth_correction: float | None = None,
735+
lamella_offset: float | None = None,
736+
trench_height_front: float | None = None,
737+
trench_height_rear: float | None = None,
738+
width_overlap_front_left: float | None = None,
739+
width_overlap_front_right: float | None = None,
740+
width_overlap_rear_left: float | None = None,
741+
width_overlap_rear_right: float | None = None,
742+
):
743+
try:
744+
with ISPyBSession() as db:
745+
# Find the ID of this MillingStep
746+
milling_step_name = (
747+
db.query(MillingStepName)
748+
.filter(
749+
MillingStepName.recipe == recipe_name,
750+
MillingStepName.step == activity_name,
751+
)
752+
.one()
753+
)
754+
record = MillingStep(
755+
# IDs
756+
millingStepNameId=milling_step_name.millingStepNameId,
757+
gridSquareId=grid_square_id,
758+
# Values
759+
isEnabled=is_enabled,
760+
status=status,
761+
executionTime=execution_time,
762+
stageX=stage_x,
763+
stageY=stage_y,
764+
stageZ=stage_z,
765+
rotation=rotation,
766+
alphaTilt=tilt_alpha,
767+
beamType=beam_type,
768+
beamVoltage=beam_voltage,
769+
beamCurrent=beam_current,
770+
millingAngle=milling_angle,
771+
depthCorrection=depth_correction,
772+
lamellaOffset=lamella_offset,
773+
trenchHeightFront=trench_height_front,
774+
trenchHeightRear=trench_height_rear,
775+
widthOverlapFrontLeft=width_overlap_front_left,
776+
widthOverlapFrontRight=width_overlap_front_right,
777+
widthOverlapRearLeft=width_overlap_rear_left,
778+
widthOverlapRearRight=width_overlap_rear_right,
779+
)
780+
db.add(record)
781+
db.commit()
782+
log.info(f"Created MillingStep {record.millingStepId}")
783+
return {"success": True, "return_value": record.millingStepId}
784+
except ispyb.ISPyBException as e:
785+
log.error(
786+
"Insert MillingStep entry caused exception '%s'.",
787+
e,
788+
exc_info=True,
789+
)
790+
return {"success": False, "return_value": None}
791+
792+
def do_update_milling_step(
793+
self,
794+
milling_step_id: int,
795+
is_enabled: bool | None = None,
796+
status: str | None = None,
797+
execution_time: float | None = None,
798+
stage_x: float | None = None,
799+
stage_y: float | None = None,
800+
stage_z: float | None = None,
801+
rotation: float | None = None,
802+
tilt_alpha: float | None = None,
803+
beam_type: str | None = None,
804+
beam_voltage: float | None = None,
805+
beam_current: float | None = None,
806+
milling_angle: float | None = None,
807+
depth_correction: float | None = None,
808+
lamella_offset: float | None = None,
809+
trench_height_front: float | None = None,
810+
trench_height_rear: float | None = None,
811+
width_overlap_front_left: float | None = None,
812+
width_overlap_front_right: float | None = None,
813+
width_overlap_rear_left: float | None = None,
814+
width_overlap_rear_right: float | None = None,
815+
):
816+
try:
817+
with ISPyBSession() as db:
818+
milling_step = (
819+
db.query(MillingStep)
820+
.filter(MillingStep.millingStepId == milling_step_id)
821+
.one()
822+
)
823+
milling_step.isEnabled = is_enabled or milling_step.isEnabled
824+
milling_step.status = status or milling_step.status
825+
milling_step.executionTime = (
826+
execution_time or milling_step.executionTime
827+
)
828+
milling_step.stageX = stage_x or milling_step.stageX
829+
milling_step.stageY = stage_y or milling_step.stageY
830+
milling_step.stageZ = stage_z or milling_step.stageZ
831+
milling_step.rotation = rotation or milling_step.rotation
832+
milling_step.alphaTilt = tilt_alpha or milling_step.alphaTilt
833+
milling_step.beamType = beam_type or milling_step.beamType
834+
milling_step.beamVoltage = beam_voltage or milling_step.beamVoltage
835+
milling_step.beamCurrent = beam_current or milling_step.beamCurrent
836+
milling_step.millingAngle = milling_angle or milling_step.millingAngle
837+
milling_step.depthCorrection = (
838+
depth_correction or milling_step.depthCorrection
839+
)
840+
milling_step.lamellaOffset = (
841+
lamella_offset or milling_step.lamellaOffset
842+
)
843+
milling_step.trenchHeightFront = (
844+
trench_height_front or milling_step.trenchHeightFront
845+
)
846+
milling_step.trenchHeightRear = (
847+
trench_height_rear or milling_step.trenchHeightRear
848+
)
849+
milling_step.widthOverlapFrontLeft = (
850+
width_overlap_front_left or milling_step.widthOverlapFrontLeft
851+
)
852+
milling_step.widthOverlapFrontRight = (
853+
width_overlap_front_right or milling_step.widthOverlapFrontRight
854+
)
855+
milling_step.widthOverlapRearLeft = (
856+
width_overlap_rear_left or milling_step.widthOverlapRearLeft
857+
)
858+
milling_step.widthOverlapRearRight = (
859+
width_overlap_rear_right or milling_step.widthOverlapRearRight
860+
)
861+
862+
db.add(milling_step)
863+
db.commit()
864+
return {"success": True, "return_value": milling_step.millingStepId}
865+
except ispyb.ISPyBException as e:
866+
log.error(
867+
"Updating MillingStep entry caused exception '%s'.",
868+
e,
869+
exc_info=True,
870+
)
871+
return {"success": False, "return_value": None}
872+
713873
def do_buffer_lookup(self, app_id: int, uuid: int) -> Optional[int]:
714874
with ISPyBSession() as db:
715875
buffer_objects = (

0 commit comments

Comments
 (0)