Skip to content

Commit 4d764e4

Browse files
chaoran-chenchaoran-chen
committed
prototyping
1 parent baaed78 commit 4d764e4

97 files changed

Lines changed: 14160 additions & 76 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎kubernetes/loculus/templates/_common-metadata.tpl‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -337,10 +337,35 @@ organisms:
337337
{{- end }}
338338
{{- end }}
339339
{{- end }}
340-
{{- end }}
340+
{{- end }}
341341
referenceGenomes:
342342
{{ $instance.referenceGenomes | toYaml | nindent 6 }}
343343
{{- end }}
344+
{{- if and $.Values.overview $.Values.overview.enabled }}
345+
{{- $overviewSchema := $.Values.overview.schema | include "loculus.patchMetadataSchema" | fromYaml }}
346+
{{- $overviewReferenceGenomes := $.Values.overview.referenceGenomes | default (list) }}
347+
overview:
348+
key: {{ $.Values.overview.key | quote }}
349+
displayName: {{ $.Values.overview.displayName | quote }}
350+
schema:
351+
organismName: {{ quote $overviewSchema.organismName }}
352+
loadSequencesAutomatically: {{ $overviewSchema.loadSequencesAutomatically | default false }}
353+
{{- include "loculus.submissionDataTypes" $overviewSchema | nindent 4 }}
354+
primaryKey: accessionVersion
355+
inputFields:
356+
{{ $overviewSchema.inputFields | default list | toYaml | nindent 6 }}
357+
metadata:
358+
{{- $args := dict "metadata" (concat $commonMetadata ($overviewSchema.metadata | default list)) "referenceGenomes" $overviewReferenceGenomes }}
359+
{{ $metadata := include "loculus.generateWebsiteMetadata" $args | fromYaml }}
360+
{{ $metadata.fields | toYaml | nindent 6 }}
361+
{{- omit ($overviewSchema.website | default dict) "multiFieldSearches" | toYaml | nindent 4 }}
362+
{{- if $overviewSchema.website.multiFieldSearches }}
363+
multiFieldSearches:
364+
{{ $overviewSchema.website.multiFieldSearches | toYaml | nindent 6 }}
365+
{{- end }}
366+
referenceGenomes:
367+
{{ $overviewReferenceGenomes | toYaml | nindent 4 }}
368+
{{- end }}
344369
{{- end }}
345370

346371
{{- define "loculus.standardWebsiteMetadata" }}

‎kubernetes/loculus/templates/_urls.tpl‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,10 @@
5757
{{- $key := $item.key }}
5858
"{{ $key }}": "{{ if not $.Values.disableWebsite }}http://{{ template "loculus.lapisServiceName" $key }}:8080{{ else -}}http://{{ $.Values.localHost }}:8080/{{ $key }}{{ end }}"
5959
{{ end }}
60+
{{- if and $.Values.overview $.Values.overview.enabled }}
61+
{{- $key := $.Values.overview.key }}
62+
"{{ $key }}": "{{ if not $.Values.disableWebsite }}http://{{ template "loculus.lapisServiceName" $key }}:8080{{ else -}}http://{{ $.Values.localHost }}:8080/{{ $key }}{{ end }}"
63+
{{- end }}
6064
{{ end }}
6165

6266
{{/* generates external LAPIS urls from { config, host } */}}
@@ -67,6 +71,9 @@
6771
"{{ $key -}}": "{{ $lapisUrlTemplate | replace "%organism%" $key }}"
6872
{{- end }}
6973
{{ end }}
74+
{{- if and .config.overview .config.overview.enabled }}
75+
"{{ .config.overview.key -}}": "{{ $lapisUrlTemplate | replace "%organism%" .config.overview.key }}"
76+
{{- end }}
7077
{{ end }}
7178

7279
{{/* generates the LAPIS service name for a given organism key */}}

‎kubernetes/loculus/templates/lapis-ingress.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@
66
{{- range $_, $item := $enabledOrganismsList -}}
77
{{- $organismKeys = append $organismKeys $item.key -}}
88
{{- end -}}
9+
{{- if and $.Values.overview $.Values.overview.enabled -}}
10+
{{- $organismKeys = append $organismKeys $.Values.overview.key -}}
11+
{{- end -}}
912
apiVersion: {{ $traefikApiVersion }}
1013
kind: Middleware
1114
metadata:

‎kubernetes/loculus/templates/lapis-service.yaml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,3 +16,21 @@ spec:
1616
protocol: TCP
1717
name: http
1818
{{- end }}
19+
{{- if and .Values.overview .Values.overview.enabled }}
20+
{{- $key := .Values.overview.key }}
21+
---
22+
apiVersion: v1
23+
kind: Service
24+
metadata:
25+
name: {{ template "loculus.lapisServiceName" $key }}
26+
spec:
27+
type: ClusterIP
28+
selector:
29+
app: loculus
30+
component: lapis-{{ $key }}
31+
ports:
32+
- port: 8080
33+
targetPort: 8080
34+
protocol: TCP
35+
name: http
36+
{{- end }}

‎kubernetes/loculus/templates/lapis-silo-database-config.yaml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,3 +34,28 @@ data:
3434
reference_genomes.json: |
3535
{{ include "loculus.mergeReferenceGenomes" $organismContent.referenceGenomes | fromYaml | toJson }}
3636
{{- end }}
37+
{{- if and .Values.overview .Values.overview.enabled }}
38+
{{- $overviewReferenceGenomes := .Values.overview.referenceGenomes | default (list) }}
39+
---
40+
apiVersion: v1
41+
kind: ConfigMap
42+
metadata:
43+
name: lapis-silo-database-config-{{ .Values.overview.key }}
44+
data:
45+
{{- $args := dict
46+
"schema" (.Values.overview.schema | include "loculus.patchMetadataSchema" | fromYaml)
47+
"commonMetadata" $commonMetadata
48+
"referenceGenomes" $overviewReferenceGenomes
49+
}}
50+
database_config.yaml: |
51+
{{ include "loculus.siloDatabaseConfig" $args | nindent 4 }}
52+
53+
preprocessing_config.yaml: |
54+
inputDirectory: /preprocessing/input
55+
outputDirectory: /preprocessing/output
56+
ndjsonInputFilename: data.ndjson.zst
57+
referenceGenomeFilename: reference_genomes.json
58+
59+
reference_genomes.json: |
60+
{{ include "loculus.mergeReferenceGenomes" $overviewReferenceGenomes | fromYaml | toJson }}
61+
{{- end }}

‎kubernetes/loculus/templates/overview-lapis-deployment.yaml‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
{{- if and .Values.overview .Values.overview.enabled }}
2+
{{- $dockerTag := include "loculus.dockerTag" .Values }}
3+
{{- $key := .Values.overview.key }}
4+
---
5+
apiVersion: apps/v1
6+
kind: Deployment
7+
metadata:
8+
name: loculus-lapis-{{ $key }}
9+
spec:
10+
replicas: {{ $.Values.replicas.lapis | default 1 }}
11+
selector:
12+
matchLabels:
13+
app: loculus
14+
component: lapis-{{ $key }}
15+
template:
16+
metadata:
17+
annotations:
18+
valuesHash: {{ .Values | toJson | sha256sum | quote }}
19+
labels:
20+
app: loculus
21+
component: lapis-{{ $key }}
22+
spec:
23+
{{- include "possiblePriorityClassName" $ | nindent 6 }}
24+
initContainers:
25+
{{- include "loculus.configProcessor" (dict "name" "lapis-silo-database-config" "dockerTag" $dockerTag "imagePullPolicy" $.Values.imagePullPolicy) | nindent 8 }}
26+
containers:
27+
- name: lapis
28+
image: "{{ $.Values.images.lapis.repository }}:{{ $.Values.images.lapis.tag }}"
29+
imagePullPolicy: "{{ $.Values.images.lapis.pullPolicy | default $.Values.imagePullPolicy }}"
30+
{{- include "loculus.resources" (list "lapis" $.Values $key) | nindent 10 }}
31+
ports:
32+
- containerPort: 8080
33+
args:
34+
- "--silo.url=http://loculus-silo-service-{{ $key }}:8081"
35+
env:
36+
- name: JVM_OPTS
37+
value: -XX:+UseContainerSupport -XX:MaxRAMPercentage=75.0 -XX:+UseG1GC -XX:MaxHeapFreeRatio=5 -XX:MinHeapFreeRatio=2 -XX:MaxGCPauseMillis=100
38+
volumeMounts:
39+
- name: lapis-silo-database-config-processed
40+
mountPath: /workspace/database_config.yaml
41+
subPath: database_config.yaml
42+
- name: lapis-silo-database-config-processed
43+
mountPath: /workspace/reference_genomes.json
44+
subPath: reference_genomes.json
45+
startupProbe:
46+
httpGet:
47+
path: /actuator/health
48+
port: 8080
49+
periodSeconds: 5
50+
failureThreshold: 36
51+
readinessProbe:
52+
httpGet:
53+
path: /sample/info
54+
port: 8080
55+
periodSeconds: 10
56+
failureThreshold: 3
57+
timeoutSeconds: 5
58+
livenessProbe:
59+
httpGet:
60+
path: /actuator/health
61+
port: 8080
62+
periodSeconds: 10
63+
failureThreshold: 3
64+
timeoutSeconds: 5
65+
volumes:
66+
{{- include "loculus.configVolume" (dict "name" "lapis-silo-database-config" "configmap" (printf "lapis-silo-database-config-%s" $key)) | nindent 8 }}
67+
{{- end }}

‎kubernetes/loculus/templates/overview-silo-deployment.yaml‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
{{- if and .Values.overview .Values.overview.enabled }}
2+
{{- $dockerTag := include "loculus.dockerTag" .Values }}
3+
{{- $key := .Values.overview.key }}
4+
{{- $commonMetadata := (include "loculus.commonMetadata" . | fromYaml).fields }}
5+
{{- $overviewSchema := .Values.overview.schema | include "loculus.patchMetadataSchema" | fromYaml }}
6+
{{- $metadataFields := list }}
7+
{{- range (concat $commonMetadata ($overviewSchema.metadata | default list)) }}
8+
{{- $metadataFields = append $metadataFields .name }}
9+
{{- end }}
10+
{{- $backendUrls := dict }}
11+
{{- $displayNames := dict }}
12+
{{- range $_, $item := (include "loculus.enabledOrganisms" . | fromJson).organisms }}
13+
{{- $organismKey := $item.key }}
14+
{{- $organismContent := $item.contents }}
15+
{{- if $.Values.disableBackend }}
16+
{{- $backendUrls = set $backendUrls $organismKey (printf "http://host.k3d.internal:8079/%s" $organismKey) }}
17+
{{- else }}
18+
{{- $backendUrls = set $backendUrls $organismKey (printf "http://loculus-backend-service:8079/%s" $organismKey) }}
19+
{{- end }}
20+
{{- $displayNames = set $displayNames $organismKey $organismContent.schema.organismName }}
21+
{{- end }}
22+
---
23+
apiVersion: apps/v1
24+
kind: Deployment
25+
metadata:
26+
name: loculus-silo-{{ $key }}
27+
annotations:
28+
argocd.argoproj.io/sync-options: Replace=true{{ if (and (not $.Values.developmentDatabasePersistence) $.Values.runDevelopmentMainDatabase) }},Force=true{{ end }}
29+
spec:
30+
replicas: 1
31+
selector:
32+
matchLabels:
33+
app: loculus
34+
component: silo-{{ $key }}
35+
template:
36+
metadata:
37+
annotations:
38+
valuesHash: {{ .Values | toJson | sha256sum | quote }}
39+
labels:
40+
app: loculus
41+
component: silo-{{ $key }}
42+
spec:
43+
{{- include "possiblePriorityClassName" $ | nindent 6 }}
44+
initContainers:
45+
{{- include "loculus.configProcessor" (dict "name" "lapis-silo-database-config" "dockerTag" $dockerTag "imagePullPolicy" $.Values.imagePullPolicy) | nindent 8 }}
46+
containers:
47+
- name: silo
48+
image: "{{ $.Values.images.loculusSilo.repository }}:{{ $.Values.images.loculusSilo.tag | default $dockerTag }}"
49+
command: ["/usr/local/bin/silo"]
50+
imagePullPolicy: {{ $.Values.imagePullPolicy }}
51+
{{- include "loculus.resources" (list "silo" $.Values $key) | nindent 10 }}
52+
env:
53+
- name: SPDLOG_LEVEL
54+
value: "debug"
55+
- name: SILO_DATA_DIRECTORY
56+
value: "/data/"
57+
ports:
58+
- containerPort: 8081
59+
args:
60+
- "api"
61+
- "--api-threads-for-http-connections"
62+
- {{ default 16 (($.Values.silo).apiThreadsForHttpConnections) | quote }}
63+
- "--api-max-queued-http-connections"
64+
- "1000"
65+
- "--query-materialization-cutoff"
66+
- "3276"
67+
volumeMounts:
68+
- name: lapis-silo-shared-data
69+
mountPath: /data
70+
readinessProbe:
71+
httpGet:
72+
path: /info
73+
port: 8081
74+
initialDelaySeconds: 30
75+
periodSeconds: 10
76+
failureThreshold: 3
77+
timeoutSeconds: 5
78+
livenessProbe:
79+
httpGet:
80+
path: /health
81+
port: 8081
82+
initialDelaySeconds: 30
83+
periodSeconds: 10
84+
failureThreshold: 3
85+
timeoutSeconds: 5
86+
- name: silo-importer
87+
image: "{{ $.Values.images.loculusSilo.repository }}:{{ $.Values.images.loculusSilo.tag | default $dockerTag }}"
88+
imagePullPolicy: "{{ $.Values.images.loculusSilo.pullPolicy }}"
89+
{{- include "loculus.resources" (list "silo-importer" $.Values) | nindent 10 }}
90+
env:
91+
- name: SILO_IMPORT_MODE
92+
value: "overview"
93+
- name: OVERVIEW_BACKEND_BASE_URLS
94+
value: {{ $backendUrls | toJson | quote }}
95+
- name: OVERVIEW_ORGANISM_DISPLAY_NAMES
96+
value: {{ $displayNames | toJson | quote }}
97+
- name: OVERVIEW_CLADE_FIELD_CANDIDATES
98+
value: {{ $.Values.overview.cladeFieldCandidates | default dict | toJson | quote }}
99+
- name: OVERVIEW_METADATA_FIELDS
100+
value: {{ $metadataFields | toJson | quote }}
101+
- name: SILO_RUN_TIMEOUT_SECONDS
102+
value: {{ $.Values.siloImport.siloTimeoutSeconds | quote }}
103+
- name: HARD_REFRESH_INTERVAL
104+
value: {{ $.Values.siloImport.hardRefreshIntervalSeconds | quote }}
105+
- name: SILO_IMPORT_POLL_INTERVAL_SECONDS
106+
value: {{ $.Values.siloImport.pollIntervalSeconds | quote }}
107+
- name: PATH_TO_SILO_BINARY
108+
value: "/usr/local/bin/silo"
109+
- name: PREPROCESSING_CONFIG
110+
value: "/app/preprocessing_config.yaml"
111+
volumeMounts:
112+
- name: lapis-silo-database-config-processed
113+
mountPath: /preprocessing/input/reference_genomes.json
114+
subPath: reference_genomes.json
115+
- name: lapis-silo-database-config-processed
116+
mountPath: /preprocessing/input/database_config.yaml
117+
subPath: database_config.yaml
118+
- name: lapis-silo-database-config-processed
119+
mountPath: /app/preprocessing_config.yaml
120+
subPath: preprocessing_config.yaml
121+
- name: lapis-silo-shared-data
122+
mountPath: /preprocessing/output
123+
- name: lapis-silo-input-data-cache
124+
mountPath: /preprocessing/input
125+
volumes:
126+
{{- include "loculus.configVolume" (dict "name" "lapis-silo-database-config" "configmap" (printf "lapis-silo-database-config-%s" $key)) | nindent 8 }}
127+
- name: lapis-silo-shared-data
128+
emptyDir: {}
129+
- name: lapis-silo-input-data-cache
130+
emptyDir: {}
131+
{{- end }}

‎kubernetes/loculus/templates/silo-service.yaml‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,4 +15,22 @@ spec:
1515
targetPort: 8081
1616
protocol: TCP
1717
name: http
18-
{{- end }}
18+
{{- end }}
19+
{{- if and .Values.overview .Values.overview.enabled }}
20+
{{- $key := .Values.overview.key }}
21+
---
22+
apiVersion: v1
23+
kind: Service
24+
metadata:
25+
name: loculus-silo-service-{{ $key }}
26+
spec:
27+
type: ClusterIP
28+
selector:
29+
app: loculus
30+
component: silo-{{ $key }}
31+
ports:
32+
- port: 8081
33+
targetPort: 8081
34+
protocol: TCP
35+
name: http
36+
{{- end }}

0 commit comments

Comments
 (0)