Skip to content

Commit ac79902

Browse files
authored
Add scaffolding for image correlation service (#272)
* Added placeholder service module for the new image alignment service * Added test module for the new service * Added service to Helm chart
1 parent 0ba2096 commit ac79902

8 files changed

Lines changed: 273 additions & 0 deletions

File tree

Helm/Chart.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ dependencies:
88
- name: clem_process_raw_lifs
99
- name: clem_process_raw_tiffs
1010
- name: cluster_submission
11+
- name: correlative_align_images
1112
- name: cryolo
1213
- name: cryolo_gpu
1314
- name: ctffind
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
apiVersion: v2
2+
name: correlative_align_images
3+
description: A Helm chart for the alignment of images
4+
version: 1.3.7
Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: correlative-align-images
5+
namespace: {{ .Values.global.namespace }}
6+
spec:
7+
replicas: {{ .Values.replicas }}
8+
selector:
9+
matchLabels:
10+
app: correlative-align-images
11+
template:
12+
metadata:
13+
labels:
14+
app: correlative-align-images
15+
spec:
16+
securityContext:
17+
runAsUser: {{ .Values.global.runAsUser }}
18+
runAsGroup: {{ .Values.global.runAsGroup }}
19+
containers:
20+
- name: correlative-align-images-runner
21+
image: {{ .Values.image }}
22+
imagePullPolicy: Always
23+
resources:
24+
requests:
25+
cpu: {{ .Values.cpuRequest }}
26+
limits:
27+
cpu: {{ .Values.cpuLimit }}
28+
memory: {{ .Values.memoryLimit }}
29+
command: ["/bin/sh"]
30+
args:
31+
- -c
32+
- >-
33+
{{ .Values.command }}
34+
volumeMounts:
35+
- name: config-file
36+
mountPath: /cryoemservices/config
37+
- name: secrets
38+
mountPath: /cryoemservices/secrets
39+
{{- if .Values.global.extraGlobalVolumeMounts }}
40+
{{- toYaml .Values.global.extraGlobalVolumeMounts | nindent 12 }}
41+
{{- end }}
42+
{{- if .Values.extraVolumeMounts }}
43+
{{- toYaml .Values.extraVolumeMounts | nindent 12 }}
44+
{{- end }}
45+
volumes:
46+
- name: config-file
47+
configMap:
48+
name: {{ .Values.global.configMap }}
49+
- name: secrets
50+
projected:
51+
defaultMode: 0444
52+
sources:
53+
- secret:
54+
name: {{ .Values.global.rmqSecretName }}
55+
{{- if .Values.global.extraGlobalVolumes }}
56+
{{- toYaml .Values.global.extraGlobalVolumes | nindent 8 }}
57+
{{- end }}
58+
{{- if .Values.extraVolumes }}
59+
{{- toYaml .Values.extraVolumes | nindent 8 }}
60+
{{- end }}
61+
{{- if .Values.global.tolerations }}
62+
tolerations:
63+
{{- toYaml .Values.global.tolerations | nindent 8 }}
64+
{{- end }}
65+
{{- if .Values.global.nodeSelector }}
66+
nodeSelector:
67+
{{- toYaml .Values.global.nodeSelector | nindent 8 }}
68+
{{- end }}
69+
{{- if .Values.global.imagePullSecrets }}
70+
imagePullSecrets:
71+
- name: {{ .Values.global.imagePullSecrets }}
72+
{{- end }}
73+
74+
{{- if .Values.scaleOnQueueLength }}
75+
---
76+
apiVersion: keda.sh/v1alpha1
77+
kind: ScaledObject
78+
metadata:
79+
name: correlative-align-images
80+
namespace: {{ .Values.global.namespace }}
81+
spec:
82+
scaleTargetRef:
83+
name: correlative-align-images
84+
triggers:
85+
- type: rabbitmq
86+
metadata:
87+
host: {{ .Values.global.rmqHost }}
88+
queueName: correlative.align_images
89+
mode: QueueLength
90+
value: "{{ .Values.queueLengthTrigger }}"
91+
minReplicaCount: {{ .Values.minReplicaCount }}
92+
maxReplicaCount: {{ .Values.maxReplicaCount }}
93+
{{- end }}

Helm/values.yaml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,18 @@ cluster_submission:
7676
memoryLimit: 256Mi
7777
scaleOnQueueLength: false
7878

79+
correlative_align_images:
80+
replicas: 0
81+
image: gcr.io/image/path
82+
command: cryoemservices.service -s CorrelativeAlignImages -c /cryoemservices/config/cryoemservices_config.yaml
83+
cpuRequest: "4"
84+
cpuLimit: "4"
85+
memoryLimit: 16Gi
86+
scaleOnQueueLength: true
87+
queueLengthTrigger: "4"
88+
minReplicaCount: 0
89+
maxReplicaCount: 4
90+
7991
cryolo:
8092
replicas: 1
8193
image: gcr.io/image/path

pyproject.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ torch = [
100100
Class2D = "cryoemservices.services.class2d:Class2D"
101101
Class3D = "cryoemservices.services.class3d:Class3D"
102102
ClusterSubmission = "cryoemservices.services.cluster_submission:ClusterSubmission"
103+
CorrelativeAlignImages = "cryoemservices.services.correlative_align_images:AlignImagesService"
103104
CrYOLO = "cryoemservices.services.cryolo:CrYOLO"
104105
Denoise = "cryoemservices.services.denoise:Denoise"
105106
DenoiseSlurm = "cryoemservices.services.denoise_slurm:DenoiseSlurm"
Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
from pathlib import Path
2+
from typing import Any
3+
4+
from pydantic import BaseModel, ValidationError
5+
from workflows.recipe import RecipeWrapper, wrap_subscribe
6+
7+
from cryoemservices.services.common_service import CommonService
8+
from cryoemservices.util.models import MockRW
9+
10+
11+
class AlignImagesParameters(BaseModel):
12+
id_ref: int # ISPyB Atlas atlasId
13+
image_ref: Path
14+
pixel_size_ref: float
15+
id_mov: int # ISPyB Atlas atlasId
16+
image_mov: Path
17+
pixel_size_mov: float
18+
19+
20+
class AlignImagesService(CommonService):
21+
"""
22+
A CryoEM service to align to images to one another
23+
"""
24+
25+
_logger_name = __name__
26+
27+
def initializing(self):
28+
"""Subscribe to a queue. Received messages must be acknowledged."""
29+
self.log.info("Correlative image alignment service starting")
30+
# Subscribe service to RMQ queue
31+
wrap_subscribe(
32+
self._transport,
33+
self._environment["queue"] or "correlative.align_images",
34+
self.call_align_images,
35+
acknowledgement=True,
36+
allow_non_recipe_messages=True,
37+
)
38+
39+
def call_align_images(
40+
self,
41+
rw: RecipeWrapper | None,
42+
header: dict[str, Any],
43+
message: dict[str, Any] | None,
44+
):
45+
"""Pass incoming message to the relevant plugin function."""
46+
# Encase message in ReceipeWrapper if none was provided
47+
if not rw:
48+
self.log.info("Received a simple message")
49+
if not isinstance(message, dict):
50+
self.log.error("Rejected invalid simple message")
51+
self._reject_message(header, requeue=False)
52+
return
53+
# Create a wrapper-like object to be passed to functions
54+
rw = MockRW(self._transport)
55+
rw.recipe_step = {"paramters": message}
56+
57+
try:
58+
if isinstance(message, dict):
59+
params = AlignImagesParameters(
60+
**{**rw.recipe_step.get("parameters", {}), **message}
61+
)
62+
else:
63+
params = AlignImagesParameters(
64+
**{**rw.recipe_step.get("parameters", {})}
65+
)
66+
except (ValidationError, TypeError) as e:
67+
self.log.error(
68+
f"AlignImagesParameters validation failed for message: {message} "
69+
f"and recipe parameters: {rw.recipe_step.get('parameters', {})} "
70+
f"with exception: {e}"
71+
)
72+
self._reject_message(header, transport=rw.transport, requeue=False)
73+
return
74+
75+
# Acknowledge receipt of parameters
76+
self.log.info(
77+
"Running image alignment with the following parameters:\n"
78+
f"{params.model_dump(mode='json')}"
79+
)
80+
81+
###############################################################################
82+
# Image alignment logic goes here
83+
###############################################################################
84+
85+
# Ack message after completion
86+
rw.transport.ack(header)
87+
return

tests/recipes/test_recipes.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
from pydantic import BaseModel
88

99
from cryoemservices.services.bfactor_setup import BFactorParameters
10+
from cryoemservices.services.correlative_align_images import AlignImagesParameters
1011
from cryoemservices.services.cryolo import CryoloParameters
1112
from cryoemservices.services.ctffind import CTFParameters
1213
from cryoemservices.services.denoise import DenoiseParameters
@@ -70,6 +71,7 @@ class MurfeyParameters(BaseModel):
7071
"CLEM-ALIGN-AND-MERGE": AlignAndMergeParameters,
7172
"CLEM-PROCESS-LIFS": ProcessRawLIFsParameters,
7273
"CLEM-PROCESS-TIFFS": ProcessRawTIFFsParameters,
74+
"CorrelativeAlignImages": AlignImagesParameters,
7375
"CrYOLO": CryoloParameters,
7476
"CTFFind": CTFParameters,
7577
"Denoise": DenoiseParameters,
Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
from pathlib import Path
2+
from unittest import mock
3+
from unittest.mock import MagicMock
4+
5+
import pytest
6+
from pytest_mock import MockerFixture
7+
from workflows.transport.offline_transport import OfflineTransport
8+
9+
from cryoemservices.services.correlative_align_images import (
10+
AlignImagesParameters,
11+
AlignImagesService,
12+
)
13+
from cryoemservices.util.models import MockRW
14+
15+
16+
@pytest.fixture
17+
def offline_transport(mocker: MockerFixture):
18+
transport = OfflineTransport()
19+
mocker.spy(transport, "send")
20+
return transport
21+
22+
23+
@pytest.mark.parametrize(
24+
"use_recwrap",
25+
(
26+
True,
27+
False,
28+
),
29+
)
30+
def test_align_images_service(
31+
tmp_path: Path,
32+
offline_transport: OfflineTransport,
33+
use_recwrap: bool,
34+
):
35+
# Set up the message parameters
36+
header = {
37+
"message-id": mock.sentinel,
38+
"subscription": mock.sentinel,
39+
}
40+
align_images_test_message = {
41+
"id_ref": 1,
42+
"image_ref": str(tmp_path / "ref.png"),
43+
"pixel_size_ref": 1e-6,
44+
"id_mov": 1,
45+
"image_mov": str(tmp_path / "mov.png"),
46+
"pixel_size_mov": 1e-6,
47+
}
48+
params = AlignImagesParameters(**align_images_test_message)
49+
50+
# Set up and run the service
51+
service = AlignImagesService(environment={"queue": ""}, transport=offline_transport)
52+
service.log = MagicMock() # Mock the logger to evaluate calls
53+
service.initializing()
54+
if use_recwrap:
55+
recwrap = MockRW(offline_transport)
56+
recwrap.recipe_step = {"parameters": align_images_test_message}
57+
service.call_align_images(
58+
recwrap,
59+
header=header,
60+
message=None,
61+
)
62+
else:
63+
service.call_align_images(
64+
None,
65+
header=header,
66+
message=align_images_test_message,
67+
)
68+
69+
# Check that the main block in the function was run
70+
service.log.info.assert_called_with(
71+
"Running image alignment with the following parameters:\n"
72+
f"{params.model_dump(mode='json')}"
73+
)

0 commit comments

Comments
 (0)