From e01e08ecbad50cb1d7620876bf85499a9a998524 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Mon, 16 Mar 2026 10:28:43 +0000 Subject: [PATCH 01/14] Move tomo align to be named as aretomo --- .../{tomo_align.py => tomo_align_aretomo.py} | 23 +-- ...n_slurm.py => tomo_align_aretomo_slurm.py} | 10 +- tests/recipes/test_recipes.py | 4 +- ...mo_align.py => test_tomo_align_aretomo.py} | 142 +++++++++--------- ...rm.py => test_tomo_align_aretomo_slurm.py} | 54 +++---- 5 files changed, 118 insertions(+), 115 deletions(-) rename src/cryoemservices/services/{tomo_align.py => tomo_align_aretomo.py} (98%) rename src/cryoemservices/services/{tomo_align_slurm.py => tomo_align_aretomo_slurm.py} (96%) rename tests/services/{test_tomo_align.py => test_tomo_align_aretomo.py} (91%) rename tests/services/{test_tomo_align_slurm.py => test_tomo_align_aretomo_slurm.py} (91%) diff --git a/src/cryoemservices/services/tomo_align.py b/src/cryoemservices/services/tomo_align_aretomo.py similarity index 98% rename from src/cryoemservices/services/tomo_align.py rename to src/cryoemservices/services/tomo_align_aretomo.py index 45a875f3..73f156bb 100644 --- a/src/cryoemservices/services/tomo_align.py +++ b/src/cryoemservices/services/tomo_align_aretomo.py @@ -96,7 +96,7 @@ def create_tilt_stack(input_file_list_of_lists: List[Any], stack_file: Path): mrc.header.cella.z *= len(input_file_list_of_lists) -class TomoParameters(BaseModel): +class AreTomoParameters(BaseModel): aretomo_version: Literal[2, 3] = 3 stack_file: str = Field(..., min_length=1) pixel_size: float @@ -166,14 +166,13 @@ def check_list_of_lists_is_not_empty(cls, v): raise ValueError("input_file_list is not a list of lists") -class TomoAlign(CommonService): +class AreTomoAlign(CommonService): """ - A service for grouping and aligning tomography tilt-series - with Newstack and AreTomo2 or AreTomo3 + A service for grouping and aligning tomography tilt-series with AreTomo2 or AreTomo3 """ # Logger name - _logger_name = "cryoemservices.services.tomo_align" + _logger_name = "cryoemservices.services.tomo_align_aretomo" # Job name job_type = "relion.reconstructtomograms" @@ -207,7 +206,7 @@ def initializing(self): ) @staticmethod - def check_visit(tomo_params: TomoParameters): + def check_visit(tomo_params: AreTomoParameters): return True def parse_tomo_output(self, tomo_stdout: str): @@ -278,11 +277,13 @@ def tomo_align(self, rw, header: dict, message: dict): try: if isinstance(message, dict): - tomo_params = TomoParameters( + tomo_params = AreTomoParameters( **{**rw.recipe_step.get("parameters", {}), **message} ) else: - tomo_params = TomoParameters(**{**rw.recipe_step.get("parameters", {})}) + tomo_params = AreTomoParameters( + **{**rw.recipe_step.get("parameters", {})} + ) except (ValidationError, TypeError) as e: self.log.warning( f"TomoAlign parameter validation failed for message: {message} " @@ -815,7 +816,7 @@ def assemble_aretomo3_command( self, aretomo_executable: str, input_file: str, - tomo_parameters: TomoParameters, + tomo_parameters: AreTomoParameters, ): """ Assemble the command to run AreTomo3, using a base command with @@ -901,7 +902,7 @@ def assemble_aretomo2_command( self, aretomo_executable: str, input_file: str, - tomo_parameters: TomoParameters, + tomo_parameters: AreTomoParameters, aretomo_output_path: Path, angle_file: Path, ): @@ -970,7 +971,7 @@ def assemble_aretomo2_command( def aretomo( self, - tomo_parameters: TomoParameters, + tomo_parameters: AreTomoParameters, aretomo_output_path: Path, angle_file: Path, ): diff --git a/src/cryoemservices/services/tomo_align_slurm.py b/src/cryoemservices/services/tomo_align_aretomo_slurm.py similarity index 96% rename from src/cryoemservices/services/tomo_align_slurm.py rename to src/cryoemservices/services/tomo_align_aretomo_slurm.py index 4b8a662a..8cd9def0 100644 --- a/src/cryoemservices/services/tomo_align_slurm.py +++ b/src/cryoemservices/services/tomo_align_aretomo_slurm.py @@ -10,7 +10,7 @@ import requests -from cryoemservices.services.tomo_align import TomoAlign, TomoParameters +from cryoemservices.services.tomo_align_aretomo import AreTomoAlign, AreTomoParameters from cryoemservices.util.slurm_submission import slurm_submission_for_services @@ -85,13 +85,13 @@ def get_iris_state(logger, wait=True) -> str: return "unknown" -class TomoAlignSlurm(TomoAlign): +class AreTomoAlignSlurm(AreTomoAlign): """ A service for submitting AreTomo2 jobs to a slurm cluster via RestAPI """ # Logger name - _logger_name = "cryoemservices.services.tomo_align_slurm" + _logger_name = "cryoemservices.services.tomo_align_aretomo_slurm" def initializing(self): if not get_iris_state(self.log): @@ -99,7 +99,7 @@ def initializing(self): super().initializing() @staticmethod - def check_visit(tomo_params: TomoParameters): + def check_visit(tomo_params: AreTomoParameters): # Requeue visits that should not be sent via slurm visit_search = re.search( "/[a-z]{2}[0-9]{5}-[0-9]{1,3}/", tomo_params.stack_file @@ -128,7 +128,7 @@ def parse_tomo_output_file(self, tomo_output_file: Path): def aretomo( self, - tomo_parameters: TomoParameters, + tomo_parameters: AreTomoParameters, aretomo_output_path: Path, angle_file: Path, ): diff --git a/tests/recipes/test_recipes.py b/tests/recipes/test_recipes.py index 462379b7..a0f45a48 100644 --- a/tests/recipes/test_recipes.py +++ b/tests/recipes/test_recipes.py @@ -19,7 +19,7 @@ from cryoemservices.services.postprocess import PostProcessParameters from cryoemservices.services.select_classes import SelectClassesParameters from cryoemservices.services.select_particles import SelectParticlesParameters -from cryoemservices.services.tomo_align import TomoParameters +from cryoemservices.services.tomo_align_aretomo import AreTomoParameters from cryoemservices.wrappers.class2d_wrapper import Class2DParameters from cryoemservices.wrappers.class3d_wrapper import Class3DParameters from cryoemservices.wrappers.clem_align_and_merge import AlignAndMergeParameters @@ -89,7 +89,7 @@ class MurfeyParameters(BaseModel): "RefineWrapper": RefineParameters, "SelectClasses": SelectClassesParameters, "SelectParticles": SelectParticlesParameters, - "TomoAlign": TomoParameters, + "TomoAlign": AreTomoParameters, } diff --git a/tests/services/test_tomo_align.py b/tests/services/test_tomo_align_aretomo.py similarity index 91% rename from tests/services/test_tomo_align.py rename to tests/services/test_tomo_align_aretomo.py index 5d8a52e7..626390d8 100644 --- a/tests/services/test_tomo_align.py +++ b/tests/services/test_tomo_align_aretomo.py @@ -11,7 +11,7 @@ import pytest from workflows.transport.offline_transport import OfflineTransport -from cryoemservices.services import tomo_align +from cryoemservices.services import tomo_align_aretomo from cryoemservices.util.relion_service_options import RelionServiceOptions @@ -28,11 +28,11 @@ def offline_transport(mocker): return transport -@mock.patch("cryoemservices.services.tomo_align.subprocess.run") -@mock.patch("cryoemservices.services.tomo_align.mrcfile") -@mock.patch("cryoemservices.services.tomo_align.create_tilt_stack") -@mock.patch("cryoemservices.services.tomo_align.resize_tomogram") -@mock.patch("cryoemservices.services.tomo_align.rotate_tomogram") +@mock.patch("cryoemservices.services.tomo_align_aretomo.subprocess.run") +@mock.patch("cryoemservices.services.tomo_align_aretomo.mrcfile") +@mock.patch("cryoemservices.services.tomo_align_aretomo.create_tilt_stack") +@mock.patch("cryoemservices.services.tomo_align_aretomo.resize_tomogram") +@mock.patch("cryoemservices.services.tomo_align_aretomo.rotate_tomogram") def test_tomo_align_service_file_list_aretomo3( mock_rotate, mock_resize, @@ -43,7 +43,7 @@ def test_tomo_align_service_file_list_aretomo3( tmp_path, ): """ - Send a test message to TomoAlign (AreTomo3) + Send a test message to AreTomoAlign (AreTomo3) This should call the mock subprocess then send messages on to the denoising, ispyb_connector and images services. """ @@ -104,7 +104,7 @@ def test_tomo_align_service_file_list_aretomo3( (tmp_path / "MotionCorr/job002/Movies/Position_1_001_0.0.mrc").touch() # Set up the mock service - service = tomo_align.TomoAlign( + service = tomo_align_aretomo.AreTomoAlign( environment={"queue": ""}, transport=offline_transport ) service.initializing() @@ -371,9 +371,9 @@ def write_aretomo_outputs(command, capture_output): offline_transport.send.assert_any_call("success", {}) -@mock.patch("cryoemservices.services.tomo_align.subprocess.run") -@mock.patch("cryoemservices.services.tomo_align.mrcfile") -@mock.patch("cryoemservices.services.tomo_align.create_tilt_stack") +@mock.patch("cryoemservices.services.tomo_align_aretomo.subprocess.run") +@mock.patch("cryoemservices.services.tomo_align_aretomo.mrcfile") +@mock.patch("cryoemservices.services.tomo_align_aretomo.create_tilt_stack") def test_tomo_align_service_file_list_aretomo2( mock_tilt_stack, mock_mrcfile, @@ -382,7 +382,7 @@ def test_tomo_align_service_file_list_aretomo2( tmp_path, ): """ - Send a test message to TomoAlign (AreTomo2) + Send a test message to AreTomoAlign (AreTomo2) This should call the mock subprocess then send messages on to the denoising, ispyb_connector and images services. """ @@ -440,7 +440,7 @@ def test_tomo_align_service_file_list_aretomo2( (tmp_path / "MotionCorr/job002/Movies/Position_1_001_0.0.mrc").touch() # Set up the mock service - service = tomo_align.TomoAlign( + service = tomo_align_aretomo.AreTomoAlign( environment={"queue": ""}, transport=offline_transport ) service.initializing() @@ -692,11 +692,11 @@ def write_aretomo_outputs(command, capture_output): offline_transport.send.assert_any_call("success", {}) -@mock.patch("cryoemservices.services.tomo_align.subprocess.run") -@mock.patch("cryoemservices.services.tomo_align.mrcfile") -@mock.patch("cryoemservices.services.tomo_align.create_tilt_stack") -@mock.patch("cryoemservices.services.tomo_align.resize_tomogram") -@mock.patch("cryoemservices.services.tomo_align.rotate_tomogram") +@mock.patch("cryoemservices.services.tomo_align_aretomo.subprocess.run") +@mock.patch("cryoemservices.services.tomo_align_aretomo.mrcfile") +@mock.patch("cryoemservices.services.tomo_align_aretomo.create_tilt_stack") +@mock.patch("cryoemservices.services.tomo_align_aretomo.resize_tomogram") +@mock.patch("cryoemservices.services.tomo_align_aretomo.rotate_tomogram") def test_tomo_align_service_file_list_repeated_tilt( mock_rotate, mock_resize, @@ -707,7 +707,7 @@ def test_tomo_align_service_file_list_repeated_tilt( tmp_path, ): """ - Send a test message to TomoAlign with a duplicated tilt angle + Send a test message to AreTomoAlign with a duplicated tilt angle Only the newest one of the duplicated tilts should be used """ mock_mrcfile.open().__enter__().header = MrcFileHeader(3000, 4000) @@ -746,7 +746,7 @@ def test_tomo_align_service_file_list_repeated_tilt( (tmp_path / "MotionCorr/job002/Movies/Position_1_003_0.0.mrc").touch() # Set up the mock service - service = tomo_align.TomoAlign( + service = tomo_align_aretomo.AreTomoAlign( environment={"queue": ""}, transport=offline_transport ) service.initializing() @@ -835,11 +835,11 @@ def write_aretomo_outputs(command, capture_output): offline_transport.send.assert_any_call("success", {}) -@mock.patch("cryoemservices.services.tomo_align.subprocess.run") -@mock.patch("cryoemservices.services.tomo_align.mrcfile") -@mock.patch("cryoemservices.services.tomo_align.create_tilt_stack") -@mock.patch("cryoemservices.services.tomo_align.resize_tomogram") -@mock.patch("cryoemservices.services.tomo_align.rotate_tomogram") +@mock.patch("cryoemservices.services.tomo_align_aretomo.subprocess.run") +@mock.patch("cryoemservices.services.tomo_align_aretomo.mrcfile") +@mock.patch("cryoemservices.services.tomo_align_aretomo.create_tilt_stack") +@mock.patch("cryoemservices.services.tomo_align_aretomo.resize_tomogram") +@mock.patch("cryoemservices.services.tomo_align_aretomo.rotate_tomogram") def test_tomo_align_service_file_list_zero_rotation( mock_rotate, mock_resize, @@ -850,7 +850,7 @@ def test_tomo_align_service_file_list_zero_rotation( tmp_path, ): """ - Send a test message to TomoAlign with a tilt axis of zero to test rotation of volume + Send a test message to AreTomoAlign with a tilt axis of zero to test rotation of volume """ mock_mrcfile.open().__enter__().header = MrcFileHeader(3000, 4000) @@ -876,7 +876,7 @@ def test_tomo_align_service_file_list_zero_rotation( (tmp_path / "MotionCorr/job002/Movies/Position_1_001_0.0.mrc").touch() # Set up the mock service - service = tomo_align.TomoAlign( + service = tomo_align_aretomo.AreTomoAlign( environment={"queue": ""}, transport=offline_transport ) service.initializing() @@ -947,11 +947,11 @@ def write_aretomo_outputs(command, capture_output): ) -@mock.patch("cryoemservices.services.tomo_align.subprocess.run") -@mock.patch("cryoemservices.services.tomo_align.mrcfile") -@mock.patch("cryoemservices.services.tomo_align.create_tilt_stack") -@mock.patch("cryoemservices.services.tomo_align.resize_tomogram") -@mock.patch("cryoemservices.services.tomo_align.rotate_tomogram") +@mock.patch("cryoemservices.services.tomo_align_aretomo.subprocess.run") +@mock.patch("cryoemservices.services.tomo_align_aretomo.mrcfile") +@mock.patch("cryoemservices.services.tomo_align_aretomo.create_tilt_stack") +@mock.patch("cryoemservices.services.tomo_align_aretomo.resize_tomogram") +@mock.patch("cryoemservices.services.tomo_align_aretomo.rotate_tomogram") def test_tomo_align_service_file_list_bad_tilts( mock_rotate, mock_resize, @@ -962,7 +962,7 @@ def test_tomo_align_service_file_list_bad_tilts( tmp_path, ): """ - Send a test message to TomoAlign with a tilts with bad motion correction + Send a test message to AreTomoAlign with a tilts with bad motion correction This tilt should be removed """ mock_mrcfile.open().__enter__().header = MrcFileHeader(3000, 4000) @@ -1016,7 +1016,7 @@ def test_tomo_align_service_file_list_bad_tilts( (tmp_path / f"MotionCorr/job002/Movies/Position_1_00{i}_0.0.mrc").touch() # Set up the mock service - service = tomo_align.TomoAlign( + service = tomo_align_aretomo.AreTomoAlign( environment={"queue": ""}, transport=offline_transport ) service.initializing() @@ -1110,11 +1110,11 @@ def write_aretomo_outputs(command, capture_output): offline_transport.send.assert_any_call("success", {}) -@mock.patch("cryoemservices.services.tomo_align.subprocess.run") -@mock.patch("cryoemservices.services.tomo_align.mrcfile") -@mock.patch("cryoemservices.services.tomo_align.create_tilt_stack") -@mock.patch("cryoemservices.services.tomo_align.resize_tomogram") -@mock.patch("cryoemservices.services.tomo_align.rotate_tomogram") +@mock.patch("cryoemservices.services.tomo_align_aretomo.subprocess.run") +@mock.patch("cryoemservices.services.tomo_align_aretomo.mrcfile") +@mock.patch("cryoemservices.services.tomo_align_aretomo.create_tilt_stack") +@mock.patch("cryoemservices.services.tomo_align_aretomo.resize_tomogram") +@mock.patch("cryoemservices.services.tomo_align_aretomo.rotate_tomogram") def test_tomo_align_service_file_list_rerun( mock_rotate, mock_resize, @@ -1125,7 +1125,7 @@ def test_tomo_align_service_file_list_rerun( tmp_path, ): """ - Send a test message to TomoAlign for a rerun tomogram + Send a test message to AreTomoAlign for a rerun tomogram This should call the mock subprocess then send messages on to the denoising, ispyb_connector and images services. Should not do a node creator send @@ -1162,7 +1162,7 @@ def test_tomo_align_service_file_list_rerun( (tmp_path / "MotionCorr/job002/Movies/Position_1_001_0.0.mrc").touch() # Set up the mock service - service = tomo_align.TomoAlign( + service = tomo_align_aretomo.AreTomoAlign( environment={"queue": ""}, transport=offline_transport ) service.initializing() @@ -1343,11 +1343,11 @@ def write_aretomo_outputs(command, capture_output): offline_transport.send.assert_any_call("success", {}) -@mock.patch("cryoemservices.services.tomo_align.subprocess.run") -@mock.patch("cryoemservices.services.tomo_align.mrcfile") -@mock.patch("cryoemservices.services.tomo_align.create_tilt_stack") -@mock.patch("cryoemservices.services.tomo_align.resize_tomogram") -@mock.patch("cryoemservices.services.tomo_align.rotate_tomogram") +@mock.patch("cryoemservices.services.tomo_align_aretomo.subprocess.run") +@mock.patch("cryoemservices.services.tomo_align_aretomo.mrcfile") +@mock.patch("cryoemservices.services.tomo_align_aretomo.create_tilt_stack") +@mock.patch("cryoemservices.services.tomo_align_aretomo.resize_tomogram") +@mock.patch("cryoemservices.services.tomo_align_aretomo.rotate_tomogram") def test_tomo_align_service_path_pattern( mock_rotate, mock_resize, @@ -1358,7 +1358,7 @@ def test_tomo_align_service_path_pattern( tmp_path, ): """ - Send a test message to TomoAlign + Send a test message to AreTomoAlign This should call the mock subprocess then send messages on to the denoising, ispyb_connector and images services. """ @@ -1413,7 +1413,7 @@ def test_tomo_align_service_path_pattern( output_relion_options["vol_z"] = 800 # Set up the mock service - service = tomo_align.TomoAlign( + service = tomo_align_aretomo.AreTomoAlign( environment={"queue": ""}, transport=offline_transport ) service.initializing() @@ -1544,10 +1544,10 @@ def write_aretomo_outputs(command, capture_output): offline_transport.send.assert_any_call("success", {}) -@mock.patch("cryoemservices.services.tomo_align.subprocess.run") -@mock.patch("cryoemservices.services.tomo_align.mrcfile") -@mock.patch("cryoemservices.services.tomo_align.create_tilt_stack") -@mock.patch("cryoemservices.services.tomo_align.resize_tomogram") +@mock.patch("cryoemservices.services.tomo_align_aretomo.subprocess.run") +@mock.patch("cryoemservices.services.tomo_align_aretomo.mrcfile") +@mock.patch("cryoemservices.services.tomo_align_aretomo.create_tilt_stack") +@mock.patch("cryoemservices.services.tomo_align_aretomo.resize_tomogram") def test_tomo_align_service_dark_images( mock_resize, mock_tilt_stack, @@ -1557,7 +1557,7 @@ def test_tomo_align_service_dark_images( tmp_path, ): """ - Send a test message to TomoAlign for a case with dark images which are removed + Send a test message to AreTomoAlign for a case with dark images which are removed """ mock_mrcfile.open().__enter__().header = MrcFileHeader(3000, 4000) @@ -1601,7 +1601,7 @@ def test_tomo_align_service_dark_images( (tmp_path / f"MotionCorr/job002/Movies/Position_1_00{i}_0.0.mrc").touch() # Set up the mock service - service = tomo_align.TomoAlign( + service = tomo_align_aretomo.AreTomoAlign( environment={"queue": ""}, transport=offline_transport ) service.initializing() @@ -1805,9 +1805,9 @@ def write_aretomo_outputs(command, capture_output): offline_transport.send.assert_any_call("success", {}) -@mock.patch("cryoemservices.services.tomo_align.subprocess.run") -@mock.patch("cryoemservices.services.tomo_align.mrcfile") -@mock.patch("cryoemservices.services.tomo_align.create_tilt_stack") +@mock.patch("cryoemservices.services.tomo_align_aretomo.subprocess.run") +@mock.patch("cryoemservices.services.tomo_align_aretomo.mrcfile") +@mock.patch("cryoemservices.services.tomo_align_aretomo.create_tilt_stack") def test_tomo_align_service_all_dark( mock_tilt_stack, mock_mrcfile, @@ -1816,7 +1816,7 @@ def test_tomo_align_service_all_dark( tmp_path, ): """ - Send a test message to TomoAlign for a case where all images are dark + Send a test message to AreTomoAlign for a case where all images are dark """ mock_mrcfile.open().__enter__().header = MrcFileHeader(3000, 4000) @@ -1849,7 +1849,7 @@ def test_tomo_align_service_all_dark( (tmp_path / f"MotionCorr/job002/Movies/Position_1_00{i}_0.0.mrc").touch() # Set up the mock service - service = tomo_align.TomoAlign( + service = tomo_align_aretomo.AreTomoAlign( environment={"queue": ""}, transport=offline_transport ) service.initializing() @@ -1947,9 +1947,9 @@ def write_aretomo_outputs(command, capture_output): offline_transport.send.assert_any_call("success", {}) -@mock.patch("cryoemservices.services.tomo_align.subprocess.run") -@mock.patch("cryoemservices.services.tomo_align.mrcfile") -@mock.patch("cryoemservices.services.tomo_align.create_tilt_stack") +@mock.patch("cryoemservices.services.tomo_align_aretomo.subprocess.run") +@mock.patch("cryoemservices.services.tomo_align_aretomo.mrcfile") +@mock.patch("cryoemservices.services.tomo_align_aretomo.create_tilt_stack") def test_tomo_align_service_fail_case( mock_tilt_stack, mock_mrcfile, @@ -1958,7 +1958,7 @@ def test_tomo_align_service_fail_case( tmp_path, ): """ - Send a test message to TomoAlign with a simulated failure of AreTomo3 + Send a test message to AreTomoAlign with a simulated failure of AreTomo3 """ mock_mrcfile.open().__enter__().header = MrcFileHeader(3000, 4000) @@ -2000,7 +2000,7 @@ def test_tomo_align_service_fail_case( (tmp_path / "MotionCorr/job002/Movies/Position_1_001_0.0.mrc").touch() # Set up the mock service - service = tomo_align.TomoAlign( + service = tomo_align_aretomo.AreTomoAlign( environment={"queue": ""}, transport=offline_transport ) service.initializing() @@ -2084,12 +2084,12 @@ def test_parse_tomo_align_output(offline_transport): Send test lines to the output parser to check the rotations and offsets are being read in """ - service = tomo_align.TomoAlign( + service = tomo_align_aretomo.AreTomoAlign( environment={"queue": ""}, transport=offline_transport ) service.initializing() - tomo_align.TomoAlign.parse_tomo_output( + tomo_align_aretomo.AreTomoAlign.parse_tomo_output( service, "Rot center Z 100.0 200.0 300.0\n" "Rot center Z 150.0 250.0 350.0\n" @@ -2111,7 +2111,7 @@ def test_resize_tomogram(tmp_path): mrc.header.mz = 4 mrc.header.cella = (100, 50, 20) - tomo_align.resize_tomogram(tmp_path / "test.mrc", 2) + tomo_align_aretomo.resize_tomogram(tmp_path / "test.mrc", 2) with mrcfile.open(tmp_path / "test.mrc") as mrc: data = mrc.data @@ -2135,7 +2135,7 @@ def test_rotate_tomogram_axis90(tmp_path): mrc.header.mz = 8 mrc.header.cella = (100, 50, 20) - tomo_align.rotate_tomogram(tmp_path / "test.mrc", 85) + tomo_align_aretomo.rotate_tomogram(tmp_path / "test.mrc", 85) with mrcfile.open(tmp_path / "test.mrc") as mrc: data = mrc.data @@ -2159,7 +2159,7 @@ def test_rotate_tomogram_axis0(tmp_path): mrc.header.mz = 8 mrc.header.cella = (100, 50, 20) - tomo_align.rotate_tomogram(tmp_path / "test.mrc", 5) + tomo_align_aretomo.rotate_tomogram(tmp_path / "test.mrc", 5) with mrcfile.open(tmp_path / "test.mrc") as mrc: data = mrc.data @@ -2186,7 +2186,9 @@ def test_create_stack_file(tmp_path): mrc.header.mz = 1 mrc.header.cella = (100, 50, 20) - tomo_align.create_tilt_stack(input_file_list_of_lists, tmp_path / "output_file.mrc") + tomo_align_aretomo.create_tilt_stack( + input_file_list_of_lists, tmp_path / "output_file.mrc" + ) assert (tmp_path / "output_file.mrc").is_file() with mrcfile.open(tmp_path / "output_file.mrc") as mrc: diff --git a/tests/services/test_tomo_align_slurm.py b/tests/services/test_tomo_align_aretomo_slurm.py similarity index 91% rename from tests/services/test_tomo_align_slurm.py rename to tests/services/test_tomo_align_aretomo_slurm.py index 11330320..2997f136 100644 --- a/tests/services/test_tomo_align_slurm.py +++ b/tests/services/test_tomo_align_aretomo_slurm.py @@ -8,7 +8,7 @@ from requests import Response from workflows.transport.offline_transport import OfflineTransport -from cryoemservices.services import tomo_align_slurm +from cryoemservices.services import tomo_align_aretomo_slurm from tests.test_utils.config import cluster_submission_configuration @@ -27,14 +27,14 @@ def offline_transport(mocker): @mock.patch("cryoemservices.util.slurm_submission.requests") -@mock.patch("cryoemservices.services.tomo_align.mrcfile") -@mock.patch("cryoemservices.services.tomo_align.create_tilt_stack") -@mock.patch("cryoemservices.services.tomo_align.resize_tomogram") -@mock.patch("cryoemservices.services.tomo_align.rotate_tomogram") -@mock.patch("cryoemservices.services.tomo_align_slurm.transfer_files") -@mock.patch("cryoemservices.services.tomo_align_slurm.retrieve_files") -@mock.patch("cryoemservices.services.tomo_align_slurm.get_iris_state") -def test_tomo_align_slurm_service_aretomo3( +@mock.patch("cryoemservices.services.tomo_align_aretomo.mrcfile") +@mock.patch("cryoemservices.services.tomo_align_aretomo.create_tilt_stack") +@mock.patch("cryoemservices.services.tomo_align_aretomo.resize_tomogram") +@mock.patch("cryoemservices.services.tomo_align_aretomo.rotate_tomogram") +@mock.patch("cryoemservices.services.tomo_align_aretomo_slurm.transfer_files") +@mock.patch("cryoemservices.services.tomo_align_aretomo_slurm.retrieve_files") +@mock.patch("cryoemservices.services.tomo_align_aretomo_slurm.get_iris_state") +def test_tomo_align_aretomo_slurm_service_aretomo3( mock_iris_state, mock_retrieve, mock_transfer, @@ -116,7 +116,7 @@ def test_tomo_align_slurm_service_aretomo3( cluster_submission_configuration(tmp_path) # Set up the mock service - service = tomo_align_slurm.TomoAlignSlurm( + service = tomo_align_aretomo_slurm.AreTomoAlignSlurm( environment={ "config": f"{tmp_path}/config.yaml", "slurm_cluster": "default", @@ -259,12 +259,12 @@ def test_tomo_align_slurm_service_aretomo3( @mock.patch("cryoemservices.util.slurm_submission.requests") -@mock.patch("cryoemservices.services.tomo_align.mrcfile") -@mock.patch("cryoemservices.services.tomo_align.create_tilt_stack") -@mock.patch("cryoemservices.services.tomo_align_slurm.transfer_files") -@mock.patch("cryoemservices.services.tomo_align_slurm.retrieve_files") -@mock.patch("cryoemservices.services.tomo_align_slurm.get_iris_state") -def test_tomo_align_slurm_service_aretomo2( +@mock.patch("cryoemservices.services.tomo_align_aretomo.mrcfile") +@mock.patch("cryoemservices.services.tomo_align_aretomo.create_tilt_stack") +@mock.patch("cryoemservices.services.tomo_align_aretomo_slurm.transfer_files") +@mock.patch("cryoemservices.services.tomo_align_aretomo_slurm.retrieve_files") +@mock.patch("cryoemservices.services.tomo_align_aretomo_slurm.get_iris_state") +def test_tomo_align_aretomo_slurm_service_aretomo2( mock_iris_state, mock_retrieve, mock_transfer, @@ -330,7 +330,7 @@ def test_tomo_align_slurm_service_aretomo2( cluster_submission_configuration(tmp_path) # Set up the mock service - service = tomo_align_slurm.TomoAlignSlurm( + service = tomo_align_aretomo_slurm.AreTomoAlignSlurm( environment={ "config": f"{tmp_path}/config.yaml", "slurm_cluster": "default", @@ -476,9 +476,9 @@ def test_tomo_align_slurm_service_aretomo2( ) -@mock.patch("cryoemservices.services.tomo_align_slurm.get_iris_state") +@mock.patch("cryoemservices.services.tomo_align_aretomo_slurm.get_iris_state") @pytest.mark.parametrize("test_params", visit_validation_matrix) -def test_tomo_align_slurm_service_reject_visits( +def test_tomo_align_aretomo_slurm_service_reject_visits( mock_iris_state, test_params, offline_transport, @@ -506,7 +506,7 @@ def test_tomo_align_slurm_service_reject_visits( } # Set up the mock service - service = tomo_align_slurm.TomoAlignSlurm( + service = tomo_align_aretomo_slurm.AreTomoAlignSlurm( environment={ "config": f"{tmp_path}/config.yaml", "slurm_cluster": "default", @@ -527,13 +527,13 @@ def test_tomo_align_slurm_service_reject_visits( offline_transport.nack.assert_called_once_with(header, requeue=True) -@mock.patch("cryoemservices.services.tomo_align_slurm.get_iris_state") +@mock.patch("cryoemservices.services.tomo_align_aretomo_slurm.get_iris_state") def test_parse_tomo_align_output(mock_iris_state, offline_transport, tmp_path): """ Send test lines to the output parser to check the rotations and offsets are being read in """ - service = tomo_align_slurm.TomoAlignSlurm( + service = tomo_align_aretomo_slurm.AreTomoAlignSlurm( environment={"queue": ""}, transport=offline_transport ) service.initializing() @@ -547,7 +547,7 @@ def test_parse_tomo_align_output(mock_iris_state, offline_transport, tmp_path): "Best tilt axis: 57, Score: 0.07568\n" ) - tomo_align_slurm.TomoAlignSlurm.parse_tomo_output_file( + tomo_align_aretomo_slurm.AreTomoAlignSlurm.parse_tomo_output_file( service, tmp_path / "tomo_output.txt" ) assert service.rot_centre_z_list == ["300.0", "350.0"] @@ -559,7 +559,7 @@ def test_transfer_files(tmp_path): """Test that existing files can be transferred, and non-existant files are not""" (tmp_path / "to_transfer").mkdir() (tmp_path / "to_transfer/file_exists").touch() - transferred_files = tomo_align_slurm.transfer_files( + transferred_files = tomo_align_aretomo_slurm.transfer_files( [ tmp_path / "to_transfer/file_exists", tmp_path / "to_transfer/file_does_not_exist", @@ -580,7 +580,7 @@ def test_retrieve_files(tmp_path): (tmp_path / "remote_system/job_dir/different_basepath").touch() (tmp_path / "remote_system/job_dir/file_imod_dir/imod_file").touch() - tomo_align_slurm.retrieve_files( + tomo_align_aretomo_slurm.retrieve_files( job_directory=tmp_path / "local_system/job_dir", files_to_skip=[ tmp_path / "local_system/job_dir/file_to_ignore", @@ -630,10 +630,10 @@ def test_get_iris_state(mock_sleep, mock_requests_get, test_params: tuple[str, i mock_logger = mock.Mock() if output_colour != "red": - returned_colour = tomo_align_slurm.get_iris_state(mock_logger) + returned_colour = tomo_align_aretomo_slurm.get_iris_state(mock_logger) assert returned_colour == output_colour else: - assert not tomo_align_slurm.get_iris_state(mock_logger) + assert not tomo_align_aretomo_slurm.get_iris_state(mock_logger) mock_requests_get.assert_called_with( "https://iristrafficlights.diamond.ac.uk/status" ) From c1efed6f520890da561bce7f6943e5c932199ae5 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 27 Mar 2026 15:45:16 +0000 Subject: [PATCH 02/14] Add imod tomo service --- .../services/tomo_align_imod.py | 362 ++++++++++++++++++ 1 file changed, 362 insertions(+) create mode 100644 src/cryoemservices/services/tomo_align_imod.py diff --git a/src/cryoemservices/services/tomo_align_imod.py b/src/cryoemservices/services/tomo_align_imod.py new file mode 100644 index 00000000..888bbc1f --- /dev/null +++ b/src/cryoemservices/services/tomo_align_imod.py @@ -0,0 +1,362 @@ +import subprocess +from pathlib import Path +from typing import Any, List, Optional + +import mrcfile +from pydantic import BaseModel, Field, ValidationError +from txrm2tiff.main import convert_and_save +from workflows.recipe import wrap_subscribe + +from cryoemservices.services.common_service import CommonService +from cryoemservices.util.models import MockRW +from cryoemservices.util.relion_service_options import ( + RelionServiceOptions, + update_relion_options, +) + + +class ImodTomoParameters(BaseModel): + stack_file: str = Field(..., min_length=1) + txrm_file: str = Field(..., min_length=1) + pixel_size: float + vol_z: int = 700 + out_bin: int = 1 + tilt_axis: float = 0 + bead_size: int = 250 + bead_count: int = 4 + wbp: int = 1 + sirt: int = 1 + sirt_leave_iterations: int = 5 + patch: int = 0 + patch_size: int = 200 + patch_overlap: float = 0.5 + flip_vol: int = 0 + flip_vol_post_reconstruction: bool = True + manual_tilt_offset: Optional[float] = None + cpus: int = 1 + relion_options: RelionServiceOptions + + +class ImodTomoAlign(CommonService): + """ + A service for grouping and aligning tomography tilt-series with Imod + """ + + # Logger name + _logger_name = "cryoemservices.services.tomo_align_imod" + + # Job name + job_type = "relion.reconstructtomograms" + + # Values to extract for ISPyB + input_file_list_of_lists: List[Any] + refined_tilts: List[float] + x_shift: List[float] + y_shift: List[float] + rot_centre_z_list: List[str] + tilt_offset: Optional[float] = None + thickness_pixels: int | None = None + rot: float | None = None + mag: float | None = None + alignment_quality: Optional[float] = None + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.refined_tilts = [] + self.rot_centre_z_list = [] + + def initializing(self): + """Subscribe to a queue. Received messages must be acknowledged.""" + self.log.info("TomoAlign service starting") + wrap_subscribe( + self._transport, + self._environment["queue"] or "tomo_align", + self.tomo_align, + acknowledgement=True, + allow_non_recipe_messages=True, + ) + + def tomo_align(self, rw, header: dict, message: dict): + """Main function which interprets and processes received messages""" + if not rw: + self.log.info("Received a simple message") + if not isinstance(message, dict): + self.log.error("Rejected invalid simple message") + self._transport.nack(header) + return + + # Create a wrapper-like object that can be passed to functions + # as if a recipe wrapper was present. + rw = MockRW(self._transport) + rw.recipe_step = {"parameters": message} + + try: + if isinstance(message, dict): + tomo_params = ImodTomoParameters( + **{**rw.recipe_step.get("parameters", {}), **message} + ) + else: + tomo_params = ImodTomoParameters( + **{**rw.recipe_step.get("parameters", {})} + ) + except (ValidationError, TypeError) as e: + self.log.warning( + f"TomoAlign parameter validation failed for message: {message} " + f"and recipe parameters: {rw.recipe_step.get('parameters', {})} " + f"with exception: {e}" + ) + rw.transport.nack(header) + return + + # TODO + aln_file = "dummy" + rot_centre_z = 0 + shift_plot_suffix = "_xy_shift_plot.json" + + # Update the relion options + tomo_params.relion_options = update_relion_options( + tomo_params.relion_options, dict(tomo_params) + ) + + # Do txrm conversion + self.log.info(f"Input file {tomo_params.txrm_file}") + tifftomo = Path(tomo_params.stack_file).with_suffix(".tiff") + convert_and_save(tomo_params.txrm_file, str(tifftomo), custom_reference=None) + subprocess.run(["tif2mrc", str(tifftomo), tomo_params.stack_file]) + tifftomo.unlink(missing_ok=True) + if not Path(tomo_params.stack_file).is_file(): + self.log.error( + f"Converting {tomo_params.txrm_file} to {tomo_params.stack_file} failed" + ) + rw.transport.nack(header) + return + + # Find the input image dimensions + with mrcfile.open(self.input_file_list_of_lists[0][0]) as mrc: + mrc_header = mrc.header + + # Run batchruntomo + adoc_file = write_batch_directive_file(tomo_params) + imod_output_path = Path(tomo_params.stack_file).with_suffix("_Vol.rec") + imod_result = subprocess.run( + [ + "batchruntomo", + "-directive", + str(adoc_file), + "-cpus", + str(tomo_params.cpus), + ] + ) + if imod_result.returncode or not imod_output_path.is_file(): + self.log.error( + f"batchruntomo failed with exitcode {imod_result.returncode}:\n" + + imod_result.stderr.decode("utf8", "replace") + ) + # Update failure processing status + rw.send_to("failure", {}) + rw.transport.nack(header) + return + + # Insert tomogram into ispyb + ispyb_command_list: list[dict] = [ + { + "ispyb_command": "insert_tomogram", + "volume_file": str(imod_output_path), + "stack_file": tomo_params.stack_file, + "size_x": int(mrc_header.nx / tomo_params.out_bin), + "size_y": int(mrc_header.ny / tomo_params.out_bin), + "size_z": int(tomo_params.vol_z / tomo_params.out_bin), + "pixel_spacing": tomo_params.pixel_size, + "tilt_angle_offset": str( + self.tilt_offset or tomo_params.manual_tilt_offset + ), + "z_shift": rot_centre_z, + "file_directory": str(imod_output_path.parent), + "central_slice_image": imod_output_path.name + "_Vol_thumbnail.jpeg", + "tomogram_movie": imod_output_path.name + "_Vol_movie.png", + "xy_shift_plot": imod_output_path.name + shift_plot_suffix, + "proj_xy": imod_output_path.name + "_Vol_projXY.jpeg", + "proj_xz": imod_output_path.name + "_Vol_projXZ.jpeg", + "alignment_quality": str(self.alignment_quality), + }, + { + "ispyb_command": "insert_processed_tomogram", + "file_path": tomo_params.stack_file, + "processing_type": "Stack", + }, + { + "ispyb_command": "insert_processed_tomogram", + "file_path": f"{imod_output_path.parent}/{imod_output_path.name}_alignment.jpeg", + "processing_type": "Alignment", + }, + ] + + # Forward results to images service + self.log.info(f"Sending to images service {tomo_params.stack_file}") + rw.send_to( + "images", + { + "image_command": "tilt_series_alignment", + "file": tomo_params.stack_file, + "aln_file": str(aln_file), + "pixel_size": tomo_params.pixel_size, + }, + ) + rw.send_to( + "images", + { + "image_command": "mrc_central_slice", + "file": tomo_params.stack_file, + }, + ) + rw.send_to( + "images", + { + "image_command": "mrc_to_apng", + "file": tomo_params.stack_file, + }, + ) + self.log.info(f"Sending to images service {imod_output_path}") + rw.send_to( + "images", + { + "image_command": "mrc_central_slice", + "file": str(imod_output_path), + }, + ) + rw.send_to( + "images", + { + "image_command": "mrc_to_apng", + "file": str(imod_output_path), + }, + ) + + self.log.info("Sending to images service for XY and XZ projections") + side_projection = ( + "YZ" + if tomo_params.tilt_axis is not None and -45 < tomo_params.tilt_axis < 45 + else "XZ" + ) + for projection_type in ["XY", side_projection]: + images_call_params: dict[str, str | float] = { + "image_command": "mrc_projection", + "file": str(imod_output_path), + "projection": projection_type, + "pixel_spacing": tomo_params.pixel_size, + } + if projection_type in ["XZ", "YZ"] and self.thickness_pixels: + images_call_params["thickness_ang"] = ( + self.thickness_pixels * tomo_params.pixel_size + ) + rw.send_to("images", images_call_params) + + # Forward results to denoise service + self.log.info(f"Sending to denoise service {imod_output_path}") + rw.send_to( + "denoise", + { + "volume": str(imod_output_path), + "output_dir": str(imod_output_path.parent.parent.parent / "Denoise"), + "relion_options": dict(tomo_params.relion_options), + }, + ) + + # Insert tomogram into ispyb + ispyb_parameters = { + "ispyb_command": "multipart_message", + "ispyb_command_list": ispyb_command_list, + } + self.log.info(f"Sending to ispyb {ispyb_parameters}") + rw.send_to("ispyb_connector", ispyb_parameters) + + # Remove any temporary files + for tmp_file in imod_output_path.parent.glob( + f"{Path(tomo_params.stack_file).stem}*~" + ): + tmp_file.unlink() + + # Update success processing status + rw.send_to("success", {}) + self.log.info(f"Done tomogram alignment for {tomo_params.stack_file}") + rw.transport.ack(header) + + +def write_batch_directive_file(tomo_params: ImodTomoParameters): + adoc_file = Path(tomo_params.stack_file).with_suffix(".adoc") + with open(adoc_file, "w") as adoc: + # Commands for copytomocoms + adoc.write(f"setupset.datasetDirectory={adoc_file.parent}\n") + adoc.write(f"setupset.copyarg.name={Path(tomo_params.stack_file).stem}\n") + "setupset.copyarg.userawtlt=1" + # "setupset.copyarg.dual=0" + adoc.write(f"setupset.copyarg.pixel={tomo_params.pixel_size / 10}\n") + adoc.write(f"setupset.copyarg.gold={tomo_params.beam_size}\n") + adoc.write(f"setupset.copyarg.rotation={tomo_params.tilt_axis}\n") + # "setupset.copyarg.extract=0" + + # Preprocessing + adoc.write("runtime.Preprocessing.any.removeXrays=1\n") + + # Coarse Alignment + # if tomo_params.patch: + # "comparam.prenewst.newstack.BinByFactor=2" + + # Tracking Choices + adoc.write(f"runtime.Fiducials.any.trackingMethod={tomo_params.patch}\n") + adoc.write("runtime.Fiducials.any.seedingMethod=1\n") + + # Beadtracking + if not tomo_params.patch: + adoc.write("comparam.track.beadtrack.LightBeads=0\n") + adoc.write("comparam.track.beadtrack.LocalAreaTracking=1\n") + adoc.write("comparam.track.beadtrack.SobelFilterCentering=1\n") + adoc.write("comparam.track.beadtrack.KernelSigmaForSobel=1.5\n") + adoc.write("runtime.BeadTracking.any.numberOfRuns=2\n") + + # Auto Seed Finding + if not tomo_params.patch: + adoc.write( + f"comparam.autofidseed.autofidseed.TargetNumberOfBeads={tomo_params.bead_count}\n" + ) + adoc.write("comparam.autofidseed.autofidseed.AdjustSizes=1\n") + adoc.write("comparam.autofidseed.autofidseed.TwoSurfaces=0\n") + + # RAPTOR Parameters + + # Patch Tracking + if tomo_params.patch: + adoc.write( + f"comparam.xcorr_pt.tiltxcorr.SizeOfPatchesXandY={tomo_params.patch_size} {tomo_params.patch_size}\n" + ) + adoc.write( + f"comparam.xcorr_pt.tiltxcorr.OverlapOfPatchesXandY={tomo_params.patch_overlap} {tomo_params.patch_overlap}\n" + ) + adoc.write("comparam.xcorr_pt.tiltxcorr.IterateCorrelations=1\n") + adoc.write("runtime.PatchTracking.any.adjustTiltAngles=1\n") + + # Alignment + adoc.write("comparam.align.tiltalign.SurfacesToAnalyze=1\n") + adoc.write("comparam.align.tiltalign.LocalAlignments=1\n") + adoc.write("comparam.align.tiltalign.MagOption=3\n") + adoc.write("comparam.align.tiltalign.TiltOption=5\n") + adoc.write("comparam.align.tiltalign.RotOption=3\n") + adoc.write("comparam.align.tiltalign.RobustFitting=1\n") + + # Aligned Stack Parameters + adoc.write("runtime.AlignedStack.any.linearInterpolation=1\n") + adoc.write(f"runtime.AlignedStack.any.binByFactor={tomo_params.out_bin}\n") + + # Reconstruction + SIRT Parameters + adoc.write(f"comparam.tilt.tilt.THICKNESS={tomo_params.vol_z}\n") + "comparam.tilt.tilt.LOG=" + adoc.write(f"runtime.Reconstruction.any.useSirt={tomo_params.sirt}\n") + adoc.write(f"runtime.Reconstruction.any.doBackprojAlso={tomo_params.wbp}\n") + if tomo_params.sirt: + adoc.write( + f"comparam.sirtsetup.sirtsetup.LeaveIterations={tomo_params.sirt_leave_iterations}\n" + ) + + # Postprocessing + adoc.write(f"runtime.Trimvol.any.reorient={tomo_params.flip_vol}\n") + return adoc_file From a3990bdbd9c14f52abd3ad98337e5d21e9092ce5 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 27 Mar 2026 15:47:03 +0000 Subject: [PATCH 03/14] Alter pyproject names --- pyproject.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 609fbb28..cf3e5a87 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,6 +85,8 @@ torch = [ [project.entry-points."ccpem_pipeliner.jobs"] "combine_star_files_job" = "cryoemservices.pipeliner_plugins.combine_star_job:ProcessStarFiles" [project.entry-points."cryoemservices.services"] + AreTomoAlign = "cryoemservices.services.tomo_align_aretomo:AreTomoAlign" + AreTomoAlignSlurm = "cryoemservices.services.tomo_align_aretomo_slurm:AreTomoAlignSlurm" BFactor = "cryoemservices.services.bfactor_setup:BFactor" CLEMAlignAndMerge = "cryoemservices.services.clem_align_and_merge:AlignAndMergeService" CLEMProcessRawLIFs = "cryoemservices.services.clem_process_raw_lifs:ProcessRawLIFsService" @@ -102,6 +104,7 @@ torch = [ ExtractClass = "cryoemservices.services.extract_class:ExtractClass" IceBreaker = "cryoemservices.services.icebreaker:IceBreaker" Images = "cryoemservices.services.images:Images" + ImodTomoAlign = "cryoemservices.services.tomo_align_imod:ImodTomoAlign" MembrainSeg = "cryoemservices.services.membrain_seg:MembrainSeg" MotionCorr = "cryoemservices.services.motioncorr:MotionCorr" MurfeyDBConnector = "cryoemservices.services.murfey_db_connector:MurfeyDBConnector" @@ -111,8 +114,6 @@ torch = [ Refine3D = "cryoemservices.services.refine3d:Refine3D" SelectClasses = "cryoemservices.services.select_classes:SelectClasses" SelectParticles = "cryoemservices.services.select_particles:SelectParticles" - TomoAlign = "cryoemservices.services.tomo_align:TomoAlign" - TomoAlignSlurm = "cryoemservices.services.tomo_align_slurm:TomoAlignSlurm" TopazPick = "cryoemservices.services.topaz_pick:TopazPick" [project.entry-points."cryoemservices.services.images.plugins"] "mrc_central_slice" = "cryoemservices.services.images_plugins:mrc_central_slice" From 9de305acbe9112265b994a49eafb68a30dbcadc9 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 27 Mar 2026 16:19:14 +0000 Subject: [PATCH 04/14] Bugfixes --- src/cryoemservices/services/tomo_align_imod.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/cryoemservices/services/tomo_align_imod.py b/src/cryoemservices/services/tomo_align_imod.py index 888bbc1f..72e5ecd6 100644 --- a/src/cryoemservices/services/tomo_align_imod.py +++ b/src/cryoemservices/services/tomo_align_imod.py @@ -1,6 +1,6 @@ import subprocess from pathlib import Path -from typing import Any, List, Optional +from typing import List, Optional import mrcfile from pydantic import BaseModel, Field, ValidationError @@ -49,7 +49,6 @@ class ImodTomoAlign(CommonService): job_type = "relion.reconstructtomograms" # Values to extract for ISPyB - input_file_list_of_lists: List[Any] refined_tilts: List[float] x_shift: List[float] y_shift: List[float] @@ -132,12 +131,15 @@ def tomo_align(self, rw, header: dict, message: dict): return # Find the input image dimensions - with mrcfile.open(self.input_file_list_of_lists[0][0]) as mrc: + with mrcfile.open(tomo_params.stack_file) as mrc: mrc_header = mrc.header # Run batchruntomo adoc_file = write_batch_directive_file(tomo_params) - imod_output_path = Path(tomo_params.stack_file).with_suffix("_Vol.rec") + imod_output_path = ( + Path(tomo_params.stack_file).parent + / f"{Path(tomo_params.stack_file).name}_Vol.rec" + ) imod_result = subprocess.run( [ "batchruntomo", @@ -291,7 +293,7 @@ def write_batch_directive_file(tomo_params: ImodTomoParameters): "setupset.copyarg.userawtlt=1" # "setupset.copyarg.dual=0" adoc.write(f"setupset.copyarg.pixel={tomo_params.pixel_size / 10}\n") - adoc.write(f"setupset.copyarg.gold={tomo_params.beam_size}\n") + adoc.write(f"setupset.copyarg.gold={tomo_params.bead_size}\n") adoc.write(f"setupset.copyarg.rotation={tomo_params.tilt_axis}\n") # "setupset.copyarg.extract=0" From 76367ca83ff02318cc6fc89f2846e678debe7af9 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Tue, 31 Mar 2026 14:03:40 +0100 Subject: [PATCH 05/14] Update imod script --- .../services/tomo_align_imod.py | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/src/cryoemservices/services/tomo_align_imod.py b/src/cryoemservices/services/tomo_align_imod.py index 72e5ecd6..569fd9f2 100644 --- a/src/cryoemservices/services/tomo_align_imod.py +++ b/src/cryoemservices/services/tomo_align_imod.py @@ -4,7 +4,11 @@ import mrcfile from pydantic import BaseModel, Field, ValidationError +from txrm2tiff.inspector import Inspector from txrm2tiff.main import convert_and_save +from txrm2tiff.txrm import open_txrm +from txrm2tiff.txrm_functions.general import read_stream +from txrm2tiff.xradia_properties.enums import XrmDataTypes from workflows.recipe import wrap_subscribe from cryoemservices.services.common_service import CommonService @@ -27,7 +31,7 @@ class ImodTomoParameters(BaseModel): wbp: int = 1 sirt: int = 1 sirt_leave_iterations: int = 5 - patch: int = 0 + patch: int = 1 patch_size: int = 200 patch_overlap: float = 0.5 flip_vol: int = 0 @@ -130,6 +134,25 @@ def tomo_align(self, rw, header: dict, message: dict): rw.transport.nack(header) return + # Generate angles file + with open_txrm( + tomo_params.txrm_file, load_images=False, load_reference=False, strict=False + ) as txrm: + inspector = Inspector(txrm) + angles = read_stream( + inspector.txrm.ole, + "ImageInfo/Angles", + XrmDataTypes.XRM_FLOAT, + strict=True, + ) + with open( + Path(tomo_params.stack_file).parent + / f"{Path(tomo_params.stack_file).stem}.rawtlt", + "w", + ) as angles_file: + for ang in angles: + angles_file.write(f"{ang}\n") + # Find the input image dimensions with mrcfile.open(tomo_params.stack_file) as mrc: mrc_header = mrc.header @@ -147,11 +170,13 @@ def tomo_align(self, rw, header: dict, message: dict): str(adoc_file), "-cpus", str(tomo_params.cpus), - ] + ], + capture_output=True, ) if imod_result.returncode or not imod_output_path.is_file(): self.log.error( f"batchruntomo failed with exitcode {imod_result.returncode}:\n" + + imod_result.stdout.decode("utf8", "replace") + imod_result.stderr.decode("utf8", "replace") ) # Update failure processing status @@ -290,7 +315,7 @@ def write_batch_directive_file(tomo_params: ImodTomoParameters): # Commands for copytomocoms adoc.write(f"setupset.datasetDirectory={adoc_file.parent}\n") adoc.write(f"setupset.copyarg.name={Path(tomo_params.stack_file).stem}\n") - "setupset.copyarg.userawtlt=1" + adoc.write("setupset.copyarg.userawtlt=1\n") # "setupset.copyarg.dual=0" adoc.write(f"setupset.copyarg.pixel={tomo_params.pixel_size / 10}\n") adoc.write(f"setupset.copyarg.gold={tomo_params.bead_size}\n") From af3a4305f78d2241e80b49e94eff1d39ecab9dc0 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Wed, 15 Apr 2026 09:50:30 +0100 Subject: [PATCH 06/14] A few parameter changes --- src/cryoemservices/services/tomo_align_imod.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/cryoemservices/services/tomo_align_imod.py b/src/cryoemservices/services/tomo_align_imod.py index 569fd9f2..55b696fa 100644 --- a/src/cryoemservices/services/tomo_align_imod.py +++ b/src/cryoemservices/services/tomo_align_imod.py @@ -34,10 +34,10 @@ class ImodTomoParameters(BaseModel): patch: int = 1 patch_size: int = 200 patch_overlap: float = 0.5 - flip_vol: int = 0 + flip_vol: int = 1 flip_vol_post_reconstruction: bool = True manual_tilt_offset: Optional[float] = None - cpus: int = 1 + cpus: int = 2 relion_options: RelionServiceOptions @@ -161,7 +161,7 @@ def tomo_align(self, rw, header: dict, message: dict): adoc_file = write_batch_directive_file(tomo_params) imod_output_path = ( Path(tomo_params.stack_file).parent - / f"{Path(tomo_params.stack_file).name}_Vol.rec" + / f"{Path(tomo_params.stack_file).name}_rec.mrc" ) imod_result = subprocess.run( [ @@ -315,19 +315,20 @@ def write_batch_directive_file(tomo_params: ImodTomoParameters): # Commands for copytomocoms adoc.write(f"setupset.datasetDirectory={adoc_file.parent}\n") adoc.write(f"setupset.copyarg.name={Path(tomo_params.stack_file).stem}\n") + adoc.write(f"setupset.copyarg.stackext={Path(tomo_params.stack_file).suffix}\n") adoc.write("setupset.copyarg.userawtlt=1\n") - # "setupset.copyarg.dual=0" + adoc.write("setupset.copyarg.dual=0\n") adoc.write(f"setupset.copyarg.pixel={tomo_params.pixel_size / 10}\n") adoc.write(f"setupset.copyarg.gold={tomo_params.bead_size}\n") adoc.write(f"setupset.copyarg.rotation={tomo_params.tilt_axis}\n") - # "setupset.copyarg.extract=0" + adoc.write("setupset.copyarg.extract=0\n") # Preprocessing adoc.write("runtime.Preprocessing.any.removeXrays=1\n") # Coarse Alignment # if tomo_params.patch: - # "comparam.prenewst.newstack.BinByFactor=2" + # adoc.write("comparam.prenewst.newstack.BinByFactor=2\n") # Tracking Choices adoc.write(f"runtime.Fiducials.any.trackingMethod={tomo_params.patch}\n") @@ -376,7 +377,7 @@ def write_batch_directive_file(tomo_params: ImodTomoParameters): # Reconstruction + SIRT Parameters adoc.write(f"comparam.tilt.tilt.THICKNESS={tomo_params.vol_z}\n") - "comparam.tilt.tilt.LOG=" + adoc.write("comparam.tilt.tilt.LOG=\n") adoc.write(f"runtime.Reconstruction.any.useSirt={tomo_params.sirt}\n") adoc.write(f"runtime.Reconstruction.any.doBackprojAlso={tomo_params.wbp}\n") if tomo_params.sirt: From cc019ff30e6a733d003857ee5373da3ac1036db8 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Wed, 15 Apr 2026 13:51:22 +0100 Subject: [PATCH 07/14] stem not name --- src/cryoemservices/services/tomo_align_imod.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cryoemservices/services/tomo_align_imod.py b/src/cryoemservices/services/tomo_align_imod.py index 55b696fa..85ab40e7 100644 --- a/src/cryoemservices/services/tomo_align_imod.py +++ b/src/cryoemservices/services/tomo_align_imod.py @@ -161,7 +161,7 @@ def tomo_align(self, rw, header: dict, message: dict): adoc_file = write_batch_directive_file(tomo_params) imod_output_path = ( Path(tomo_params.stack_file).parent - / f"{Path(tomo_params.stack_file).name}_rec.mrc" + / f"{Path(tomo_params.stack_file).stem}_rec.mrc" ) imod_result = subprocess.run( [ From ce0f9ff17b82df09294e512f3149a2032273d6ca Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Wed, 17 Jun 2026 11:22:55 +0100 Subject: [PATCH 08/14] Add imod recipes --- recipes/ispyb/sxt-imod-beads-wbp.json | 101 ++++++++++++++++++ recipes/ispyb/sxt-imod-patch-wbp.json | 101 ++++++++++++++++++ .../services/tomo_align_imod.py | 13 ++- 3 files changed, 208 insertions(+), 7 deletions(-) create mode 100644 recipes/ispyb/sxt-imod-beads-wbp.json create mode 100644 recipes/ispyb/sxt-imod-patch-wbp.json diff --git a/recipes/ispyb/sxt-imod-beads-wbp.json b/recipes/ispyb/sxt-imod-beads-wbp.json new file mode 100644 index 00000000..2fb40539 --- /dev/null +++ b/recipes/ispyb/sxt-imod-beads-wbp.json @@ -0,0 +1,101 @@ +{ + "1": { + "output": 2, + "parameters": { + "dcid": "{dcid}", + "ispyb_command": "insert_tomogram", + "program_id": "{appid}", + "store_result": "ispyb_tomogram_id" + }, + "queue": "ispyb_connector", + "service": "EMISPyB" + }, + "2": { + "output": { + "denoise": 5, + "failure": 3, + "images": 6, + "ispyb_connector": 4, + "movie": 6, + "projxy": 6, + "projxz": 6, + "success": 7 + }, + "parameters": { + "manual_tilt_offset": "{manual_tilt_offset}", + "out_bin": 1, + "patch": 0, + "pixel_size": "{pixel_size}", + "relion_options": {}, + "sirt": 0, + "stack_file": "{stack_file}", + "tilt_axis": 0, + "txrm_file": "{txrm_file}", + "wbp": 1 + }, + "queue": "tomo_align_imod", + "service": "ImodTomoAlign" + }, + "3": { + "parameters": { + "ispyb_command": "update_processing_status", + "status_message": "processing failure", + "program_id": "{appid}", + "status": "failure" + }, + "queue": "ispyb_connector", + "service": "EMISPyB" + }, + "4": { + "parameters": { + "dcid": "{dcid}", + "ispyb_command": "multipart_message", + "program_id": "{appid}", + "tomogram_id": "$ispyb_tomogram_id" + }, + "queue": "ispyb_connector", + "service": "EMISPyB" + }, + "5": { + "output": { + "images": 6, + "ispyb_connector": 9, + "movie": 6, + "segmentation": 8 + }, + "queue": "denoise", + "service": "Denoise" + }, + "6": { + "queue": "images", + "service": "Images" + }, + "7": { + "parameters": { + "ispyb_command": "update_processing_status", + "status_message": "processing successful", + "program_id": "{appid}", + "status": "success" + }, + "queue": "ispyb_connector", + "service": "EMISPyB" + }, + "8": { + "output": { + "images": 6, + "ispyb_connector": 9, + "movie": 6 + }, + "queue": "segmentation", + "service": "MembrainSeg" + }, + "9": { + "parameters": { + "ispyb_command": "insert_processed_tomogram", + "tomogram_id": "$ispyb_tomogram_id" + }, + "queue": "ispyb_connector", + "service": "EMISPyB" + }, + "start": [[1, []]] +} diff --git a/recipes/ispyb/sxt-imod-patch-wbp.json b/recipes/ispyb/sxt-imod-patch-wbp.json new file mode 100644 index 00000000..a663d60e --- /dev/null +++ b/recipes/ispyb/sxt-imod-patch-wbp.json @@ -0,0 +1,101 @@ +{ + "1": { + "output": 2, + "parameters": { + "dcid": "{dcid}", + "ispyb_command": "insert_tomogram", + "program_id": "{appid}", + "store_result": "ispyb_tomogram_id" + }, + "queue": "ispyb_connector", + "service": "EMISPyB" + }, + "2": { + "output": { + "denoise": 5, + "failure": 3, + "images": 6, + "ispyb_connector": 4, + "movie": 6, + "projxy": 6, + "projxz": 6, + "success": 7 + }, + "parameters": { + "manual_tilt_offset": "{manual_tilt_offset}", + "out_bin": 1, + "patch": 1, + "pixel_size": "{pixel_size}", + "relion_options": {}, + "sirt": 0, + "stack_file": "{stack_file}", + "tilt_axis": 0, + "txrm_file": "{txrm_file}", + "wbp": 1 + }, + "queue": "tomo_align_imod", + "service": "ImodTomoAlign" + }, + "3": { + "parameters": { + "ispyb_command": "update_processing_status", + "status_message": "processing failure", + "program_id": "{appid}", + "status": "failure" + }, + "queue": "ispyb_connector", + "service": "EMISPyB" + }, + "4": { + "parameters": { + "dcid": "{dcid}", + "ispyb_command": "multipart_message", + "program_id": "{appid}", + "tomogram_id": "$ispyb_tomogram_id" + }, + "queue": "ispyb_connector", + "service": "EMISPyB" + }, + "5": { + "output": { + "images": 6, + "ispyb_connector": 9, + "movie": 6, + "segmentation": 8 + }, + "queue": "denoise", + "service": "Denoise" + }, + "6": { + "queue": "images", + "service": "Images" + }, + "7": { + "parameters": { + "ispyb_command": "update_processing_status", + "status_message": "processing successful", + "program_id": "{appid}", + "status": "success" + }, + "queue": "ispyb_connector", + "service": "EMISPyB" + }, + "8": { + "output": { + "images": 6, + "ispyb_connector": 9, + "movie": 6 + }, + "queue": "segmentation", + "service": "MembrainSeg" + }, + "9": { + "parameters": { + "ispyb_command": "insert_processed_tomogram", + "tomogram_id": "$ispyb_tomogram_id" + }, + "queue": "ispyb_connector", + "service": "EMISPyB" + }, + "start": [[1, []]] +} diff --git a/src/cryoemservices/services/tomo_align_imod.py b/src/cryoemservices/services/tomo_align_imod.py index 85ab40e7..569fe7f4 100644 --- a/src/cryoemservices/services/tomo_align_imod.py +++ b/src/cryoemservices/services/tomo_align_imod.py @@ -35,9 +35,8 @@ class ImodTomoParameters(BaseModel): patch_size: int = 200 patch_overlap: float = 0.5 flip_vol: int = 1 - flip_vol_post_reconstruction: bool = True manual_tilt_offset: Optional[float] = None - cpus: int = 2 + cpus: int = 4 relion_options: RelionServiceOptions @@ -85,7 +84,7 @@ def tomo_align(self, rw, header: dict, message: dict): self.log.info("Received a simple message") if not isinstance(message, dict): self.log.error("Rejected invalid simple message") - self._transport.nack(header) + self._reject_message(header, requeue=False) return # Create a wrapper-like object that can be passed to functions @@ -108,7 +107,7 @@ def tomo_align(self, rw, header: dict, message: dict): f"and recipe parameters: {rw.recipe_step.get('parameters', {})} " f"with exception: {e}" ) - rw.transport.nack(header) + self._reject_message(header, rw.transport, requeue=False) return # TODO @@ -131,7 +130,7 @@ def tomo_align(self, rw, header: dict, message: dict): self.log.error( f"Converting {tomo_params.txrm_file} to {tomo_params.stack_file} failed" ) - rw.transport.nack(header) + self._reject_message(header, rw.transport) return # Generate angles file @@ -181,7 +180,7 @@ def tomo_align(self, rw, header: dict, message: dict): ) # Update failure processing status rw.send_to("failure", {}) - rw.transport.nack(header) + self._reject_message(header, rw.transport) return # Insert tomogram into ispyb @@ -284,7 +283,7 @@ def tomo_align(self, rw, header: dict, message: dict): "denoise", { "volume": str(imod_output_path), - "output_dir": str(imod_output_path.parent.parent.parent / "Denoise"), + "output_dir": str(imod_output_path.parent.parent / "Denoise"), "relion_options": dict(tomo_params.relion_options), }, ) From cfbb3179a4f4789226d5ee52b0027660dab97cce Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Wed, 17 Jun 2026 11:38:09 +0100 Subject: [PATCH 09/14] Renaming --- recipes/ispyb/em-tomo-align.json | 2 +- recipes/ispyb/sxt-aretomo.json | 2 +- src/cryoemservices/services/denoise_slurm.py | 2 +- tests/recipes/test_recipes.py | 4 +++- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/recipes/ispyb/em-tomo-align.json b/recipes/ispyb/em-tomo-align.json index c5708948..504d6250 100644 --- a/recipes/ispyb/em-tomo-align.json +++ b/recipes/ispyb/em-tomo-align.json @@ -38,7 +38,7 @@ "tilt_axis": "{tilt_axis}" }, "queue": "tomo_align", - "service": "TomoAlign" + "service": "AreTomoAlign" }, "3": { "parameters": { diff --git a/recipes/ispyb/sxt-aretomo.json b/recipes/ispyb/sxt-aretomo.json index d7aa58d7..a1b2c1d8 100644 --- a/recipes/ispyb/sxt-aretomo.json +++ b/recipes/ispyb/sxt-aretomo.json @@ -33,7 +33,7 @@ "wbp": 1 }, "queue": "tomo_align", - "service": "TomoAlign" + "service": "AreTomoAlign" }, "3": { "parameters": { diff --git a/src/cryoemservices/services/denoise_slurm.py b/src/cryoemservices/services/denoise_slurm.py index 07daa552..93a61e55 100644 --- a/src/cryoemservices/services/denoise_slurm.py +++ b/src/cryoemservices/services/denoise_slurm.py @@ -7,7 +7,7 @@ from typing import List from cryoemservices.services.denoise import Denoise, DenoiseParameters -from cryoemservices.services.tomo_align_slurm import ( +from cryoemservices.services.tomo_align_aretomo_slurm import ( get_iris_state, retrieve_files, transfer_files, diff --git a/tests/recipes/test_recipes.py b/tests/recipes/test_recipes.py index a0f45a48..39470e7c 100644 --- a/tests/recipes/test_recipes.py +++ b/tests/recipes/test_recipes.py @@ -20,6 +20,7 @@ from cryoemservices.services.select_classes import SelectClassesParameters from cryoemservices.services.select_particles import SelectParticlesParameters from cryoemservices.services.tomo_align_aretomo import AreTomoParameters +from cryoemservices.services.tomo_align_imod import ImodTomoParameters from cryoemservices.wrappers.class2d_wrapper import Class2DParameters from cryoemservices.wrappers.class3d_wrapper import Class3DParameters from cryoemservices.wrappers.clem_align_and_merge import AlignAndMergeParameters @@ -62,6 +63,7 @@ class MurfeyParameters(BaseModel): FIXTURE_DIR = Path(__file__).parent.parent.parent.resolve() known_services = { + "AreTomoAlign": AreTomoParameters, "BFactor": BFactorParameters, "Class2D": Class2DParameters, "Class3D": Class3DParameters, @@ -79,6 +81,7 @@ class MurfeyParameters(BaseModel): "ExtractClass": ExtractClassParameters, "IceBreaker": IceBreakerParameters, "Images": BaseModel, + "ImodTomoAlign": ImodTomoParameters, "MembrainSeg": MembrainSegParameters, "MotionCorr": MotionCorrParameters, "Murfey": MurfeyParameters, @@ -89,7 +92,6 @@ class MurfeyParameters(BaseModel): "RefineWrapper": RefineParameters, "SelectClasses": SelectClassesParameters, "SelectParticles": SelectParticlesParameters, - "TomoAlign": AreTomoParameters, } From 9d28536e79e1203443c27d573b534bb72be022fd Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Wed, 17 Jun 2026 11:47:16 +0100 Subject: [PATCH 10/14] Update tests --- tests/services/test_tomo_align_aretomo.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/services/test_tomo_align_aretomo.py b/tests/services/test_tomo_align_aretomo.py index 544dab58..28dd4439 100644 --- a/tests/services/test_tomo_align_aretomo.py +++ b/tests/services/test_tomo_align_aretomo.py @@ -2080,12 +2080,12 @@ def write_aretomo_outputs(command, capture_output): offline_transport.send.assert_any_call("failure", {}) -@mock.patch("cryoemservices.services.tomo_align.subprocess.run") -@mock.patch("cryoemservices.services.tomo_align.mrcfile") -@mock.patch("cryoemservices.services.tomo_align.convert_and_save") -@mock.patch("cryoemservices.services.tomo_align.OleFileIO") -@mock.patch("cryoemservices.services.tomo_align.resize_tomogram") -@mock.patch("cryoemservices.services.tomo_align.rotate_tomogram") +@mock.patch("cryoemservices.services.tomo_align_aretomo.subprocess.run") +@mock.patch("cryoemservices.services.tomo_align_aretomo.mrcfile") +@mock.patch("cryoemservices.services.tomo_align_aretomo.convert_and_save") +@mock.patch("cryoemservices.services.tomo_align_aretomo.OleFileIO") +@mock.patch("cryoemservices.services.tomo_align_aretomo.resize_tomogram") +@mock.patch("cryoemservices.services.tomo_align_aretomo.rotate_tomogram") def test_tomo_align_service_txrm( mock_rotate, mock_resize, @@ -2131,7 +2131,7 @@ def test_tomo_align_service_txrm( (tmp_path / "tilt_stack.txrm").touch() # Set up the mock service - service = tomo_align.TomoAlign( + service = tomo_align_aretomo.AreTomoAlign( environment={"queue": ""}, transport=offline_transport ) service.initializing() From c44e53d68277ee22cab5ecce41d2dcbc04d0439e Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Wed, 17 Jun 2026 17:12:59 +0100 Subject: [PATCH 11/14] Test some of the imod service --- .../services/tomo_align_imod.py | 79 ++--- tests/services/test_tomo_align_imod.py | 301 ++++++++++++++++++ 2 files changed, 327 insertions(+), 53 deletions(-) create mode 100644 tests/services/test_tomo_align_imod.py diff --git a/src/cryoemservices/services/tomo_align_imod.py b/src/cryoemservices/services/tomo_align_imod.py index 569fe7f4..73797e44 100644 --- a/src/cryoemservices/services/tomo_align_imod.py +++ b/src/cryoemservices/services/tomo_align_imod.py @@ -1,27 +1,22 @@ import subprocess from pathlib import Path -from typing import List, Optional +from typing import Optional import mrcfile +import numpy as np +from olefile import OleFileIO from pydantic import BaseModel, Field, ValidationError -from txrm2tiff.inspector import Inspector from txrm2tiff.main import convert_and_save -from txrm2tiff.txrm import open_txrm -from txrm2tiff.txrm_functions.general import read_stream -from txrm2tiff.xradia_properties.enums import XrmDataTypes from workflows.recipe import wrap_subscribe from cryoemservices.services.common_service import CommonService from cryoemservices.util.models import MockRW -from cryoemservices.util.relion_service_options import ( - RelionServiceOptions, - update_relion_options, -) class ImodTomoParameters(BaseModel): stack_file: str = Field(..., min_length=1) txrm_file: str = Field(..., min_length=1) + xrm_reference: str | None = None pixel_size: float vol_z: int = 700 out_bin: int = 1 @@ -37,7 +32,6 @@ class ImodTomoParameters(BaseModel): flip_vol: int = 1 manual_tilt_offset: Optional[float] = None cpus: int = 4 - relion_options: RelionServiceOptions class ImodTomoAlign(CommonService): @@ -51,17 +45,6 @@ class ImodTomoAlign(CommonService): # Job name job_type = "relion.reconstructtomograms" - # Values to extract for ISPyB - refined_tilts: List[float] - x_shift: List[float] - y_shift: List[float] - rot_centre_z_list: List[str] - tilt_offset: Optional[float] = None - thickness_pixels: int | None = None - rot: float | None = None - mag: float | None = None - alignment_quality: Optional[float] = None - def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.refined_tilts = [] @@ -112,18 +95,16 @@ def tomo_align(self, rw, header: dict, message: dict): # TODO aln_file = "dummy" - rot_centre_z = 0 shift_plot_suffix = "_xy_shift_plot.json" - # Update the relion options - tomo_params.relion_options = update_relion_options( - tomo_params.relion_options, dict(tomo_params) - ) - # Do txrm conversion self.log.info(f"Input file {tomo_params.txrm_file}") tifftomo = Path(tomo_params.stack_file).with_suffix(".tiff") - convert_and_save(tomo_params.txrm_file, str(tifftomo), custom_reference=None) + convert_and_save( + tomo_params.txrm_file, + str(tifftomo), + custom_reference=tomo_params.xrm_reference or None, + ) subprocess.run(["tif2mrc", str(tifftomo), tomo_params.stack_file]) tifftomo.unlink(missing_ok=True) if not Path(tomo_params.stack_file).is_file(): @@ -134,16 +115,12 @@ def tomo_align(self, rw, header: dict, message: dict): return # Generate angles file - with open_txrm( - tomo_params.txrm_file, load_images=False, load_reference=False, strict=False - ) as txrm: - inspector = Inspector(txrm) - angles = read_stream( - inspector.txrm.ole, - "ImageInfo/Angles", - XrmDataTypes.XRM_FLOAT, - strict=True, - ) + with OleFileIO(tomo_params.txrm_file) as txrm_ole: + if txrm_ole.exists("ImageInfo/Angles"): + angles = np.frombuffer( + txrm_ole.openstream("ImageInfo/Angles").getvalue(), np.float32 + ).tolist() + angles = dict(enumerate(angles)) with open( Path(tomo_params.stack_file).parent / f"{Path(tomo_params.stack_file).stem}.rawtlt", @@ -187,23 +164,19 @@ def tomo_align(self, rw, header: dict, message: dict): ispyb_command_list: list[dict] = [ { "ispyb_command": "insert_tomogram", - "volume_file": str(imod_output_path), + "volume_file": str(imod_output_path.name), "stack_file": tomo_params.stack_file, "size_x": int(mrc_header.nx / tomo_params.out_bin), "size_y": int(mrc_header.ny / tomo_params.out_bin), "size_z": int(tomo_params.vol_z / tomo_params.out_bin), "pixel_spacing": tomo_params.pixel_size, - "tilt_angle_offset": str( - self.tilt_offset or tomo_params.manual_tilt_offset - ), - "z_shift": rot_centre_z, + "z_shift": 0, "file_directory": str(imod_output_path.parent), - "central_slice_image": imod_output_path.name + "_Vol_thumbnail.jpeg", - "tomogram_movie": imod_output_path.name + "_Vol_movie.png", - "xy_shift_plot": imod_output_path.name + shift_plot_suffix, - "proj_xy": imod_output_path.name + "_Vol_projXY.jpeg", - "proj_xz": imod_output_path.name + "_Vol_projXZ.jpeg", - "alignment_quality": str(self.alignment_quality), + "central_slice_image": imod_output_path.stem + "_thumbnail.jpeg", + "tomogram_movie": imod_output_path.stem + "_movie.png", + "xy_shift_plot": imod_output_path.stem + shift_plot_suffix, + "proj_xy": imod_output_path.stem + "_projXY.jpeg", + "proj_xz": imod_output_path.stem + "_projYZ.jpeg", }, { "ispyb_command": "insert_processed_tomogram", @@ -212,7 +185,7 @@ def tomo_align(self, rw, header: dict, message: dict): }, { "ispyb_command": "insert_processed_tomogram", - "file_path": f"{imod_output_path.parent}/{imod_output_path.name}_alignment.jpeg", + "file_path": f"{imod_output_path.parent}/{imod_output_path.stem}_alignment.jpeg", "processing_type": "Alignment", }, ] @@ -271,9 +244,9 @@ def tomo_align(self, rw, header: dict, message: dict): "projection": projection_type, "pixel_spacing": tomo_params.pixel_size, } - if projection_type in ["XZ", "YZ"] and self.thickness_pixels: + if projection_type in ["XZ", "YZ"]: images_call_params["thickness_ang"] = ( - self.thickness_pixels * tomo_params.pixel_size + tomo_params.vol_z * tomo_params.pixel_size ) rw.send_to("images", images_call_params) @@ -284,7 +257,7 @@ def tomo_align(self, rw, header: dict, message: dict): { "volume": str(imod_output_path), "output_dir": str(imod_output_path.parent.parent / "Denoise"), - "relion_options": dict(tomo_params.relion_options), + "relion_options": {}, }, ) diff --git a/tests/services/test_tomo_align_imod.py b/tests/services/test_tomo_align_imod.py new file mode 100644 index 00000000..5395797f --- /dev/null +++ b/tests/services/test_tomo_align_imod.py @@ -0,0 +1,301 @@ +from subprocess import CompletedProcess +from unittest import mock + +import numpy as np +import pytest +from workflows.transport.offline_transport import OfflineTransport + +from cryoemservices.services import tomo_align_imod + + +@pytest.fixture +def offline_transport(mocker): + transport = OfflineTransport() + mocker.spy(transport, "send") + return transport + + +class MrcFileHeader: + def __init__(self, nx, ny, nz=1): + self.nx = nx + self.ny = ny + self.nz = nz + + +@mock.patch("cryoemservices.services.tomo_align_imod.subprocess.run") +@mock.patch("cryoemservices.services.tomo_align_imod.mrcfile") +@mock.patch("cryoemservices.services.tomo_align_imod.OleFileIO") +@mock.patch("cryoemservices.services.tomo_align_imod.convert_and_save") +def test_tomo_align_imod( + mock_convert_and_save, + mock_ole_file, + mock_mrcfile, + mock_subprocess, + offline_transport, + tmp_path, +): + """ + Send a test message to ImodTomoAlign + This should call the mock subprocess then send messages on to + the denoising, ispyb_connector and images services. + """ + mock_mrcfile.open().__enter__().header = MrcFileHeader(nx=4000, ny=3000, nz=600) + mock_ole_file().__enter__().exists.return_value = True + mock_ole_file().__enter__().openstream().getvalue.return_value = np.array( + [0.01, 0.3, 0.5], dtype=np.float32 + ).tobytes() + mock_subprocess.return_value = CompletedProcess( + "", + returncode=0, + stdout="stdout".encode("ascii"), + stderr="stderr".encode("ascii"), + ) + + header = { + "message-id": mock.sentinel, + "subscription": mock.sentinel, + } + tomo_align_test_message = { + "stack_file": f"{tmp_path}/recipe/Tomograms/test_stack.mrc", + "txrm_file": f"{tmp_path}/test.txrm", + "xrm_reference": f"{tmp_path}/ref.xrm", + "pixel_size": 106, + "vol_z": 500, + "out_bin": 1, + "cpus": 2, + } + + # Touch the expected input/output and stack files + (tmp_path / "test.txrm").touch() + (tmp_path / "recipe/Tomograms").mkdir(parents=True) + (tmp_path / "recipe/Tomograms/test_stack.mrc").touch() + (tmp_path / "recipe/Tomograms/test_stack_rec.mrc").touch() + + # Set up the mock service + service = tomo_align_imod.ImodTomoAlign( + environment={"queue": ""}, transport=offline_transport + ) + service.initializing() + + # Send a message to the service + service.tomo_align(None, header=header, message=tomo_align_test_message) + + # Check conversion + mock_convert_and_save.assert_called_once_with( + f"{tmp_path}/test.txrm", + f"{tmp_path}/recipe/Tomograms/test_stack.tiff", + custom_reference=f"{tmp_path}/ref.xrm", + ) + mock_subprocess.assert_any_call( + [ + "tif2mrc", + f"{tmp_path}/recipe/Tomograms/test_stack.tiff", + f"{tmp_path}/recipe/Tomograms/test_stack.mrc", + ] + ) + + # Check txrm file reading + mock_ole_file.assert_any_call(f"{tmp_path}/test.txrm") + mock_ole_file().__enter__().exists.assert_any_call("ImageInfo/Angles") + mock_ole_file().__enter__().openstream.assert_any_call("ImageInfo/Angles") + assert (tmp_path / "recipe/Tomograms/test_stack.rawtlt").is_file() + + # Check batchruntomo command + mock_subprocess.assert_any_call( + [ + "batchruntomo", + "-directive", + f"{tmp_path}/recipe/Tomograms/test_stack.adoc", + "-cpus", + "2", + ], + capture_output=True, + ) + + """# Check the shift plot + with open( + tmp_path / "recipe/Tomograms/test_stack_rec.mrc/test_stack_xy_shift_plot.json" + ) as shift_plot: + shift_data = json.load(shift_plot) + assert shift_data["data"][0]["x"] == [1.2] + assert shift_data["data"][0]["y"] == [2.3] +""" + # Check that the correct messages were sent + assert offline_transport.send.call_count == 10 + offline_transport.send.assert_any_call( + "images", + { + "image_command": "tilt_series_alignment", + "file": tomo_align_test_message["stack_file"], + "aln_file": "dummy", # f"{tmp_path}/Tomograms/job006/tomograms/test_stack.aln", + "pixel_size": tomo_align_test_message["pixel_size"], + }, + ) + offline_transport.send.assert_any_call( + "images", + { + "image_command": "mrc_central_slice", + "file": tomo_align_test_message["stack_file"], + }, + ) + offline_transport.send.assert_any_call( + "images", + { + "image_command": "mrc_to_apng", + "file": tomo_align_test_message["stack_file"], + }, + ) + offline_transport.send.assert_any_call( + "images", + { + "image_command": "mrc_central_slice", + "file": f"{tmp_path}/recipe/Tomograms/test_stack_rec.mrc", + }, + ) + offline_transport.send.assert_any_call( + "images", + { + "image_command": "mrc_to_apng", + "file": f"{tmp_path}/recipe/Tomograms/test_stack_rec.mrc", + }, + ) + offline_transport.send.assert_any_call( + "images", + { + "image_command": "mrc_projection", + "file": f"{tmp_path}/recipe/Tomograms/test_stack_rec.mrc", + "projection": "XY", + "pixel_spacing": 106, + }, + ) + offline_transport.send.assert_any_call( + "images", + { + "image_command": "mrc_projection", + "file": f"{tmp_path}/recipe/Tomograms/test_stack_rec.mrc", + "projection": "YZ", + "pixel_spacing": 106, + "thickness_ang": 500 * 106, + }, + ) + offline_transport.send.assert_any_call( + "denoise", + { + "volume": f"{tmp_path}/recipe/Tomograms/test_stack_rec.mrc", + "output_dir": f"{tmp_path}/recipe/Denoise", + "relion_options": {}, + }, + ) + offline_transport.send.assert_any_call( + "ispyb_connector", + { + "ispyb_command": "multipart_message", + "ispyb_command_list": [ + { + "ispyb_command": "insert_tomogram", + "volume_file": "test_stack_rec.mrc", + "stack_file": tomo_align_test_message["stack_file"], + "size_x": 4000, + "size_y": 3000, + "size_z": 500, + "pixel_spacing": 106.0, + "z_shift": 0, + "file_directory": f"{tmp_path}/recipe/Tomograms", + "central_slice_image": "test_stack_rec_thumbnail.jpeg", + "tomogram_movie": "test_stack_rec_movie.png", + "xy_shift_plot": "test_stack_rec_xy_shift_plot.json", + "proj_xy": "test_stack_rec_projXY.jpeg", + "proj_xz": "test_stack_rec_projYZ.jpeg", + }, + { + "ispyb_command": "insert_processed_tomogram", + "file_path": tomo_align_test_message["stack_file"], + "processing_type": "Stack", + }, + { + "ispyb_command": "insert_processed_tomogram", + "file_path": f"{tmp_path}/recipe/Tomograms/test_stack_rec_alignment.jpeg", + "processing_type": "Alignment", + }, + ], + }, + ) + offline_transport.send.assert_any_call("success", {}) + + +def test_write_batch_directive_patch_wbp(tmp_path): + returned_adoc = tomo_align_imod.write_batch_directive_file( + tomo_align_imod.ImodTomoParameters( + **{ + "stack_file": f"{tmp_path}/stack.mrc", + "txrm_file": f"{tmp_path}/stack.txrm", + "pixel_size": 10.5, + "patch": 1, + "patch_size": 2, + "patch_overlap": 0.7, + "vol_z": 500, + "wbp": 1, + "sirt": 0, + "sirt_leave_iterations": 10, + } + ) + ) + assert returned_adoc == tmp_path / "stack.adoc" + with open(returned_adoc) as f: + adoc_lines = f.readlines() + + # Check some of the basic lines + assert f"setupset.datasetDirectory={tmp_path}\n" in adoc_lines + assert "setupset.copyarg.name=stack\n" in adoc_lines + assert "setupset.copyarg.pixel=1.05\n" in adoc_lines + assert "comparam.tilt.tilt.THICKNESS=500\n" in adoc_lines + assert "runtime.Reconstruction.any.useSirt=0\n" in adoc_lines + assert "runtime.Reconstruction.any.doBackprojAlso=1\n" in adoc_lines + + # Check patch lines + assert "comparam.xcorr_pt.tiltxcorr.SizeOfPatchesXandY=2 2\n" in adoc_lines + assert "comparam.xcorr_pt.tiltxcorr.OverlapOfPatchesXandY=0.7 0.7\n" in adoc_lines + + # Check lines not present + assert "comparam.autofidseed.autofidseed.AdjustSizes=1\n" not in adoc_lines + assert "comparam.sirtsetup.sirtsetup.LeaveIterations=10\n" not in adoc_lines + + +def test_write_batch_directive_beads_sirt(tmp_path): + returned_adoc = tomo_align_imod.write_batch_directive_file( + tomo_align_imod.ImodTomoParameters( + **{ + "stack_file": f"{tmp_path}/stack.mrc", + "txrm_file": f"{tmp_path}/stack.txrm", + "bead_count": 7, + "pixel_size": 10.5, + "patch": 0, + "patch_size": 2, + "patch_overlap": 4, + "vol_z": 500, + "wbp": 0, + "sirt": 1, + "sirt_leave_iterations": 10, + } + ) + ) + assert returned_adoc == tmp_path / "stack.adoc" + with open(returned_adoc) as f: + adoc_lines = f.readlines() + + # Check some of the basic lines + assert f"setupset.datasetDirectory={tmp_path}\n" in adoc_lines + assert "setupset.copyarg.name=stack\n" in adoc_lines + assert "setupset.copyarg.pixel=1.05\n" in adoc_lines + assert "comparam.tilt.tilt.THICKNESS=500\n" in adoc_lines + assert "runtime.Reconstruction.any.useSirt=1\n" in adoc_lines + assert "runtime.Reconstruction.any.doBackprojAlso=0\n" in adoc_lines + + # Check bead and sirt lines + assert "comparam.track.beadtrack.LightBeads=0\n" in adoc_lines + assert "comparam.autofidseed.autofidseed.TargetNumberOfBeads=7\n" in adoc_lines + assert "comparam.sirtsetup.sirtsetup.LeaveIterations=10\n" in adoc_lines + + # Check lines not present + assert "comparam.xcorr_pt.tiltxcorr.IterateCorrelations=1\n" not in adoc_lines + assert "comparam.xcorr_pt.tiltxcorr.OverlapOfPatchesXandY=4 4\n" not in adoc_lines From c538ff219506a491972f4584ccc6cf9b45cafe47 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 18 Jun 2026 09:02:35 +0100 Subject: [PATCH 12/14] Don't need relion options for sxt --- recipes/ispyb/sxt-imod-beads-wbp.json | 1 - recipes/ispyb/sxt-imod-patch-wbp.json | 1 - 2 files changed, 2 deletions(-) diff --git a/recipes/ispyb/sxt-imod-beads-wbp.json b/recipes/ispyb/sxt-imod-beads-wbp.json index 2fb40539..b143a323 100644 --- a/recipes/ispyb/sxt-imod-beads-wbp.json +++ b/recipes/ispyb/sxt-imod-beads-wbp.json @@ -26,7 +26,6 @@ "out_bin": 1, "patch": 0, "pixel_size": "{pixel_size}", - "relion_options": {}, "sirt": 0, "stack_file": "{stack_file}", "tilt_axis": 0, diff --git a/recipes/ispyb/sxt-imod-patch-wbp.json b/recipes/ispyb/sxt-imod-patch-wbp.json index a663d60e..a0e82f04 100644 --- a/recipes/ispyb/sxt-imod-patch-wbp.json +++ b/recipes/ispyb/sxt-imod-patch-wbp.json @@ -26,7 +26,6 @@ "out_bin": 1, "patch": 1, "pixel_size": "{pixel_size}", - "relion_options": {}, "sirt": 0, "stack_file": "{stack_file}", "tilt_axis": 0, From 8c5999d92a8aeb3dcc596e8d4d29919168dd076b Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 19 Jun 2026 10:38:01 +0100 Subject: [PATCH 13/14] Reversion to imod 4.9 --- .../services/tomo_align_aretomo.py | 2 +- .../services/tomo_align_imod.py | 32 ++++++++++++------- tests/services/test_tomo_align_aretomo.py | 3 +- tests/services/test_tomo_align_imod.py | 31 +++++++++--------- 4 files changed, 40 insertions(+), 28 deletions(-) diff --git a/src/cryoemservices/services/tomo_align_aretomo.py b/src/cryoemservices/services/tomo_align_aretomo.py index c69adb77..df4de6a4 100644 --- a/src/cryoemservices/services/tomo_align_aretomo.py +++ b/src/cryoemservices/services/tomo_align_aretomo.py @@ -882,7 +882,7 @@ def convert_txrm_to_stack( txrm_file, str(tifftomo), custom_reference=xrm_reference or None ) # Let this run, and check later if the output file exists - subprocess.run(["tif2mrc", str(tifftomo), stack_file]) + subprocess.run(["tif2mrc", str(tifftomo), stack_file], capture_output=True) tifftomo.unlink(missing_ok=True) return pixel_size_angstroms diff --git a/src/cryoemservices/services/tomo_align_imod.py b/src/cryoemservices/services/tomo_align_imod.py index 73797e44..7ea386f3 100644 --- a/src/cryoemservices/services/tomo_align_imod.py +++ b/src/cryoemservices/services/tomo_align_imod.py @@ -55,7 +55,7 @@ def initializing(self): self.log.info("TomoAlign service starting") wrap_subscribe( self._transport, - self._environment["queue"] or "tomo_align", + self._environment["queue"] or "tomo_align_imod", self.tomo_align, acknowledgement=True, allow_non_recipe_messages=True, @@ -105,7 +105,9 @@ def tomo_align(self, rw, header: dict, message: dict): str(tifftomo), custom_reference=tomo_params.xrm_reference or None, ) - subprocess.run(["tif2mrc", str(tifftomo), tomo_params.stack_file]) + subprocess.run( + ["tif2mrc", str(tifftomo), tomo_params.stack_file], capture_output=True + ) tifftomo.unlink(missing_ok=True) if not Path(tomo_params.stack_file).is_file(): self.log.error( @@ -126,10 +128,11 @@ def tomo_align(self, rw, header: dict, message: dict): / f"{Path(tomo_params.stack_file).stem}.rawtlt", "w", ) as angles_file: - for ang in angles: + for ang in angles.values(): angles_file.write(f"{ang}\n") # Find the input image dimensions + self.log.info(f"Converted {tomo_params.txrm_file} to mrc format") with mrcfile.open(tomo_params.stack_file) as mrc: mrc_header = mrc.header @@ -137,7 +140,7 @@ def tomo_align(self, rw, header: dict, message: dict): adoc_file = write_batch_directive_file(tomo_params) imod_output_path = ( Path(tomo_params.stack_file).parent - / f"{Path(tomo_params.stack_file).stem}_rec.mrc" + / f"{Path(tomo_params.stack_file).stem}.rec" ) imod_result = subprocess.run( [ @@ -149,15 +152,23 @@ def tomo_align(self, rw, header: dict, message: dict): ], capture_output=True, ) - if imod_result.returncode or not imod_output_path.is_file(): + if imod_result.returncode: self.log.error( - f"batchruntomo failed with exitcode {imod_result.returncode}:\n" + f"batchruntomo failed with exitcode {imod_result.returncode}" + ) + # Update failure processing status, then try again + rw.send_to("failure", {}) + self._reject_message(header, rw.transport) + return + elif not imod_output_path.is_file(): + self.log.error( + f"batchruntomo did not produce the output file {imod_output_path}\n" + imod_result.stdout.decode("utf8", "replace") + imod_result.stderr.decode("utf8", "replace") ) - # Update failure processing status + # Update failure processing status, but do not try again as return was 0 rw.send_to("failure", {}) - self._reject_message(header, rw.transport) + self._reject_message(header, rw.transport, requeue=False) return # Insert tomogram into ispyb @@ -287,7 +298,6 @@ def write_batch_directive_file(tomo_params: ImodTomoParameters): # Commands for copytomocoms adoc.write(f"setupset.datasetDirectory={adoc_file.parent}\n") adoc.write(f"setupset.copyarg.name={Path(tomo_params.stack_file).stem}\n") - adoc.write(f"setupset.copyarg.stackext={Path(tomo_params.stack_file).suffix}\n") adoc.write("setupset.copyarg.userawtlt=1\n") adoc.write("setupset.copyarg.dual=0\n") adoc.write(f"setupset.copyarg.pixel={tomo_params.pixel_size / 10}\n") @@ -299,8 +309,8 @@ def write_batch_directive_file(tomo_params: ImodTomoParameters): adoc.write("runtime.Preprocessing.any.removeXrays=1\n") # Coarse Alignment - # if tomo_params.patch: - # adoc.write("comparam.prenewst.newstack.BinByFactor=2\n") + if tomo_params.patch: + adoc.write("comparam.prenewst.newstack.BinByFactor=2\n") # Tracking Choices adoc.write(f"runtime.Fiducials.any.trackingMethod={tomo_params.patch}\n") diff --git a/tests/services/test_tomo_align_aretomo.py b/tests/services/test_tomo_align_aretomo.py index 28dd4439..dc72f68a 100644 --- a/tests/services/test_tomo_align_aretomo.py +++ b/tests/services/test_tomo_align_aretomo.py @@ -2219,7 +2219,8 @@ def write_aretomo_outputs(command, capture_output: bool = False): "tif2mrc", f"{tmp_path}/Tomograms/stack.tiff", f"{tmp_path}/Tomograms/stack.mrc", - ] + ], + capture_output=True, ) mock_subprocess.assert_any_call( aretomo_command, diff --git a/tests/services/test_tomo_align_imod.py b/tests/services/test_tomo_align_imod.py index 5395797f..f4c285cb 100644 --- a/tests/services/test_tomo_align_imod.py +++ b/tests/services/test_tomo_align_imod.py @@ -69,7 +69,7 @@ def test_tomo_align_imod( (tmp_path / "test.txrm").touch() (tmp_path / "recipe/Tomograms").mkdir(parents=True) (tmp_path / "recipe/Tomograms/test_stack.mrc").touch() - (tmp_path / "recipe/Tomograms/test_stack_rec.mrc").touch() + (tmp_path / "recipe/Tomograms/test_stack.rec").touch() # Set up the mock service service = tomo_align_imod.ImodTomoAlign( @@ -91,7 +91,8 @@ def test_tomo_align_imod( "tif2mrc", f"{tmp_path}/recipe/Tomograms/test_stack.tiff", f"{tmp_path}/recipe/Tomograms/test_stack.mrc", - ] + ], + capture_output=True, ) # Check txrm file reading @@ -114,7 +115,7 @@ def test_tomo_align_imod( """# Check the shift plot with open( - tmp_path / "recipe/Tomograms/test_stack_rec.mrc/test_stack_xy_shift_plot.json" + tmp_path / "recipe/Tomograms/test_stack.rec/test_stack_xy_shift_plot.json" ) as shift_plot: shift_data = json.load(shift_plot) assert shift_data["data"][0]["x"] == [1.2] @@ -149,21 +150,21 @@ def test_tomo_align_imod( "images", { "image_command": "mrc_central_slice", - "file": f"{tmp_path}/recipe/Tomograms/test_stack_rec.mrc", + "file": f"{tmp_path}/recipe/Tomograms/test_stack.rec", }, ) offline_transport.send.assert_any_call( "images", { "image_command": "mrc_to_apng", - "file": f"{tmp_path}/recipe/Tomograms/test_stack_rec.mrc", + "file": f"{tmp_path}/recipe/Tomograms/test_stack.rec", }, ) offline_transport.send.assert_any_call( "images", { "image_command": "mrc_projection", - "file": f"{tmp_path}/recipe/Tomograms/test_stack_rec.mrc", + "file": f"{tmp_path}/recipe/Tomograms/test_stack.rec", "projection": "XY", "pixel_spacing": 106, }, @@ -172,7 +173,7 @@ def test_tomo_align_imod( "images", { "image_command": "mrc_projection", - "file": f"{tmp_path}/recipe/Tomograms/test_stack_rec.mrc", + "file": f"{tmp_path}/recipe/Tomograms/test_stack.rec", "projection": "YZ", "pixel_spacing": 106, "thickness_ang": 500 * 106, @@ -181,7 +182,7 @@ def test_tomo_align_imod( offline_transport.send.assert_any_call( "denoise", { - "volume": f"{tmp_path}/recipe/Tomograms/test_stack_rec.mrc", + "volume": f"{tmp_path}/recipe/Tomograms/test_stack.rec", "output_dir": f"{tmp_path}/recipe/Denoise", "relion_options": {}, }, @@ -193,7 +194,7 @@ def test_tomo_align_imod( "ispyb_command_list": [ { "ispyb_command": "insert_tomogram", - "volume_file": "test_stack_rec.mrc", + "volume_file": "test_stack.rec", "stack_file": tomo_align_test_message["stack_file"], "size_x": 4000, "size_y": 3000, @@ -201,11 +202,11 @@ def test_tomo_align_imod( "pixel_spacing": 106.0, "z_shift": 0, "file_directory": f"{tmp_path}/recipe/Tomograms", - "central_slice_image": "test_stack_rec_thumbnail.jpeg", - "tomogram_movie": "test_stack_rec_movie.png", - "xy_shift_plot": "test_stack_rec_xy_shift_plot.json", - "proj_xy": "test_stack_rec_projXY.jpeg", - "proj_xz": "test_stack_rec_projYZ.jpeg", + "central_slice_image": "test_stack_thumbnail.jpeg", + "tomogram_movie": "test_stack_movie.png", + "xy_shift_plot": "test_stack_xy_shift_plot.json", + "proj_xy": "test_stack_projXY.jpeg", + "proj_xz": "test_stack_projYZ.jpeg", }, { "ispyb_command": "insert_processed_tomogram", @@ -214,7 +215,7 @@ def test_tomo_align_imod( }, { "ispyb_command": "insert_processed_tomogram", - "file_path": f"{tmp_path}/recipe/Tomograms/test_stack_rec_alignment.jpeg", + "file_path": f"{tmp_path}/recipe/Tomograms/test_stack_alignment.jpeg", "processing_type": "Alignment", }, ], From 0f53457a007a706d32a3c2ccf38942ddf099887b Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 19 Jun 2026 10:42:30 +0100 Subject: [PATCH 14/14] remove tilt series alignment calls --- src/cryoemservices/services/tomo_align_imod.py | 5 ++++- tests/services/test_tomo_align_imod.py | 6 ++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/cryoemservices/services/tomo_align_imod.py b/src/cryoemservices/services/tomo_align_imod.py index 7ea386f3..b692b3da 100644 --- a/src/cryoemservices/services/tomo_align_imod.py +++ b/src/cryoemservices/services/tomo_align_imod.py @@ -94,7 +94,7 @@ def tomo_align(self, rw, header: dict, message: dict): return # TODO - aln_file = "dummy" + # aln_file = "dummy" shift_plot_suffix = "_xy_shift_plot.json" # Do txrm conversion @@ -203,6 +203,8 @@ def tomo_align(self, rw, header: dict, message: dict): # Forward results to images service self.log.info(f"Sending to images service {tomo_params.stack_file}") + # TODO + """ rw.send_to( "images", { @@ -212,6 +214,7 @@ def tomo_align(self, rw, header: dict, message: dict): "pixel_size": tomo_params.pixel_size, }, ) + """ rw.send_to( "images", { diff --git a/tests/services/test_tomo_align_imod.py b/tests/services/test_tomo_align_imod.py index f4c285cb..e1e20e95 100644 --- a/tests/services/test_tomo_align_imod.py +++ b/tests/services/test_tomo_align_imod.py @@ -120,9 +120,10 @@ def test_tomo_align_imod( shift_data = json.load(shift_plot) assert shift_data["data"][0]["x"] == [1.2] assert shift_data["data"][0]["y"] == [2.3] -""" + """ # Check that the correct messages were sent - assert offline_transport.send.call_count == 10 + assert offline_transport.send.call_count == 9 # 10 + """ offline_transport.send.assert_any_call( "images", { @@ -132,6 +133,7 @@ def test_tomo_align_imod( "pixel_size": tomo_align_test_message["pixel_size"], }, ) + """ offline_transport.send.assert_any_call( "images", {