Skip to content

Commit bf3ee9a

Browse files
simusimu
committed
Optionally set COMMODORE_CATALOG_COMPILE_PROCESSES based on cluster's CPU limit
Users can opt-in to setting each CI job's environment variable `COMMODORE_CATALOG_COMPILE_PROCESSES` based on the cluster's CPU limit by setting the environment variable `COMMODORE_PROCESSES_FROM_CPU_LIMIT` to a non-empty value in their GitLab CI config. For CI configurations that opt-in to this feature, we compute the Commodore process count based on the cluster's CPU limit by rounding down to the next nearest integer, clamped at 1. Currently, the CPU limit parser supports unsuffixed CPU limits and milli-CPU CPU limits and will raise an error for other CPU limit configurations. For CI configurations that don't opt-in to this feature, `COMMODORE_CATALOG_COMPILE_PROCESSES` is set to `0` which allows Commodore to auto-detect the number of worker processes to use, preserving the Commodore v1.33.1 and older behavior.
1 parent 013bdd9 commit bf3ee9a

9 files changed

Lines changed: 138 additions & 7 deletions

‎gitlab/README.md‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,3 +129,17 @@ Example:
129129
variables:
130130
MEMORY_LIMITS: "c-my-cluster=4Gi c-my-other-cluster=4Gi"
131131
```
132+
133+
### Configure number of Commodore worker processes based on CI job CPU limits
134+
135+
Commodore v1.34.0 and newer allow users to configure the number of worker processes that are used to compile the catalog.
136+
137+
> [!NOTE]
138+
> This configuration option doesn't have an impact on the number of threads used by reclass-rs to render the Reclass inventory.
139+
140+
Users can opt-in to setting the number of Commodore worker processes based on each cluster's CI job CPU limits.
141+
To opt-in, users need to set environment variable `COMMODORE_PROCESSES_FROM_CPU_LIMIT` to a non-empty value in their GitLab CI configuration.
142+
When opting in, the Job generator sets the number of worker processes for each cluster's CI job to the maximum of the cluster's CPU limit and rounding it down to the next integer and 1.
143+
We use 1 as the lower bound, since flag value 0 configures Commodore to auto-detect the number of worker processes.
144+
145+
For users that don't opt-in, the Job generator configures Commodore to auto-detect the number of worker processes, preserving the Commodore v1.33.1 behavior.

‎gitlab/bin/step-render-pipeline‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,5 +20,6 @@ jsonnet \
2020
--ext-str cluster_catalog_urls="${cluster_catalog_urls}" \
2121
--ext-str server_fqdn="${CI_SERVER_FQDN}" \
2222
--ext-str server_ssh_host="${CI_SERVER_SHELL_SSH_HOST}" \
23+
--ext-str commodore_proc_count="${COMMODORE_PROCESSES_FROM_CPU_LIMIT:-}" \
2324
/opt/commodore-compile.jsonnet \
2425
>generated-commodore-compile.yml

‎gitlab/commodore-compile.jsonnet‎

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,35 @@ local gitInsteadOf(cluster) =
103103
'git config --add --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@%(gitlab_fqdn)s".insteadOf ssh://git@%(ssh_hostport)s' % gitlab_params,
104104
] + catalogInsteadOf;
105105

106+
local cpu_limit(cluster) = std.get(cpu_limits, cluster, '2');
107+
108+
// Compute Commodore process count based on CPU limit for the cluster by
109+
// rounding down to the next nearest integer, clamped at 1.
110+
local proc_count(cluster) =
111+
local parseK8sCPU(cl) =
112+
local val = if std.endsWith(cl, 'm') then
113+
{
114+
isMilli: true,
115+
num: cl[:-1],
116+
}
117+
else {
118+
isMilli: false,
119+
num: cl,
120+
};
121+
local clnum = std.parseYaml(val.num);
122+
if std.isString(clnum) then
123+
error 'Failed to parse K8s CPU limit %s: the parser currently only supports unsuffixed values and milli-CPU values' % cl
124+
else
125+
if val.isMilli then clnum / 1000 else clnum;
126+
if std.extVar('commodore_proc_count') != '' then (
127+
local cl = parseK8sCPU(cpu_limit(cluster));
128+
if cl >= 1 then std.floor(cl) else 1
129+
) else
130+
// Commodore auto-selects the number of worker processes when the value of
131+
// the flag/envvar is 0. This matches the behavior of Commodore v1.33.1
132+
// and older.
133+
0;
134+
106135
local compile_job(cluster) =
107136
{
108137
stage: 'build',
@@ -114,8 +143,9 @@ local compile_job(cluster) =
114143
{
115144
KUBERNETES_MEMORY_LIMIT: std.get(memory_limits, cluster, '3Gi'),
116145
KUBERNETES_MEMORY_REQUEST: std.get(memory_requests, cluster, '3Gi'),
117-
KUBERNETES_CPU_LIMIT: std.get(cpu_limits, cluster, '2'),
146+
KUBERNETES_CPU_LIMIT: cpu_limit(cluster),
118147
KUBERNETES_CPU_REQUEST: std.get(cpu_requests, cluster, '800m'),
148+
COMMODORE_CATALOG_COMPILE_PROCESSES: proc_count(cluster),
119149
},
120150
before_script:
121151
[
@@ -149,8 +179,9 @@ local deploy_job(cluster) =
149179
{
150180
KUBERNETES_MEMORY_LIMIT: std.get(memory_limits, cluster, '3Gi'),
151181
KUBERNETES_MEMORY_REQUEST: std.get(memory_requests, cluster, '3Gi'),
152-
KUBERNETES_CPU_LIMIT: std.get(cpu_limits, cluster, '2'),
182+
KUBERNETES_CPU_LIMIT: cpu_limit(cluster),
153183
KUBERNETES_CPU_REQUEST: std.get(cpu_requests, cluster, '800m'),
184+
COMMODORE_CATALOG_COMPILE_PROCESSES: proc_count(cluster),
154185
},
155186
image:
156187
{

‎gitlab/tests/golden/custom-ssh-port.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
],
3030
"stage": "build",
3131
"variables": {
32+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 0,
3233
"KUBERNETES_CPU_LIMIT": "2",
3334
"KUBERNETES_CPU_REQUEST": "800m",
3435
"KUBERNETES_MEMORY_LIMIT": "3Gi",
@@ -58,6 +59,7 @@
5859
],
5960
"stage": "deploy",
6061
"variables": {
62+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 0,
6163
"KUBERNETES_CPU_LIMIT": "2",
6264
"KUBERNETES_CPU_REQUEST": "800m",
6365
"KUBERNETES_MEMORY_LIMIT": "3Gi",

‎gitlab/tests/golden/default.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
],
3030
"stage": "build",
3131
"variables": {
32+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 0,
3233
"KUBERNETES_CPU_LIMIT": "2",
3334
"KUBERNETES_CPU_REQUEST": "800m",
3435
"KUBERNETES_MEMORY_LIMIT": "3Gi",
@@ -58,6 +59,7 @@
5859
],
5960
"stage": "deploy",
6061
"variables": {
62+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 0,
6163
"KUBERNETES_CPU_LIMIT": "2",
6264
"KUBERNETES_CPU_REQUEST": "800m",
6365
"KUBERNETES_MEMORY_LIMIT": "3Gi",

‎gitlab/tests/golden/external-catalog.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
],
3030
"stage": "build",
3131
"variables": {
32+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 0,
3233
"KUBERNETES_CPU_LIMIT": "2",
3334
"KUBERNETES_CPU_REQUEST": "800m",
3435
"KUBERNETES_MEMORY_LIMIT": "3Gi",
@@ -58,6 +59,7 @@
5859
],
5960
"stage": "deploy",
6061
"variables": {
62+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 0,
6163
"KUBERNETES_CPU_LIMIT": "2",
6264
"KUBERNETES_CPU_REQUEST": "800m",
6365
"KUBERNETES_MEMORY_LIMIT": "3Gi",
@@ -94,6 +96,7 @@
9496
],
9597
"stage": "build",
9698
"variables": {
99+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 0,
97100
"KUBERNETES_CPU_LIMIT": "2",
98101
"KUBERNETES_CPU_REQUEST": "800m",
99102
"KUBERNETES_MEMORY_LIMIT": "3Gi",
@@ -123,6 +126,7 @@
123126
],
124127
"stage": "deploy",
125128
"variables": {
129+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 0,
126130
"KUBERNETES_CPU_LIMIT": "2",
127131
"KUBERNETES_CPU_REQUEST": "800m",
128132
"KUBERNETES_MEMORY_LIMIT": "3Gi",
@@ -157,6 +161,7 @@
157161
],
158162
"stage": "build",
159163
"variables": {
164+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 0,
160165
"KUBERNETES_CPU_LIMIT": "2",
161166
"KUBERNETES_CPU_REQUEST": "800m",
162167
"KUBERNETES_MEMORY_LIMIT": "3Gi",
@@ -184,6 +189,7 @@
184189
],
185190
"stage": "deploy",
186191
"variables": {
192+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 0,
187193
"KUBERNETES_CPU_LIMIT": "2",
188194
"KUBERNETES_CPU_REQUEST": "800m",
189195
"KUBERNETES_MEMORY_LIMIT": "3Gi",

‎gitlab/tests/golden/k8s-resources.yml‎

Lines changed: 75 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
],
3030
"stage": "build",
3131
"variables": {
32+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 2,
3233
"KUBERNETES_CPU_LIMIT": "2500m",
3334
"KUBERNETES_CPU_REQUEST": "800m",
3435
"KUBERNETES_MEMORY_LIMIT": "3Gi",
@@ -58,12 +59,80 @@
5859
],
5960
"stage": "deploy",
6061
"variables": {
62+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 2,
6163
"KUBERNETES_CPU_LIMIT": "2500m",
6264
"KUBERNETES_CPU_REQUEST": "800m",
6365
"KUBERNETES_MEMORY_LIMIT": "3Gi",
6466
"KUBERNETES_MEMORY_REQUEST": "3Gi"
6567
}
6668
},
69+
"c-cluster-id-1000_compile": {
70+
"artifacts": {
71+
"expire_in": "1 week",
72+
"paths": [
73+
"diff.txt"
74+
]
75+
},
76+
"before_script": [
77+
"install --directory --mode=0700 ~/.ssh",
78+
"echo \"$SSH_KNOWN_HOSTS\" >> ~/.ssh/known_hosts",
79+
"echo \"$SSH_CONFIG\" >> ~/.ssh/config"
80+
],
81+
"image": {
82+
"name": "docker.io/projectsyn/commodore:v1.34.0"
83+
},
84+
"rules": [
85+
{
86+
"if": "$CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH"
87+
}
88+
],
89+
"script": [
90+
"git config --add --global url.\"https://gitlab-ci-token:${CI_JOB_TOKEN}@git.vshn.net:80\".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}",
91+
"git config --add --global url.\"https://gitlab-ci-token:${CI_JOB_TOKEN}@git.vshn.net:80\".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}${CI_SERVER_SHELL_SSH_PORT:+:${CI_SERVER_SHELL_SSH_PORT}}",
92+
"git config --add --global url.\"https://gitlab-ci-token:${ACCESS_TOKEN_c_cluster_id_1000}@git.vshn.net:80/cluster-catalogs/c-cluster-id-1000.git\".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}/cluster-catalogs/c-cluster-id-1000.git",
93+
"git config --add --global url.\"https://gitlab-ci-token:${ACCESS_TOKEN_c_cluster_id_1000}@git.vshn.net:80/cluster-catalogs/c-cluster-id-1000.git\".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}${CI_SERVER_SHELL_SSH_PORT:+:${CI_SERVER_SHELL_SSH_PORT}}/cluster-catalogs/c-cluster-id-1000.git",
94+
"/usr/local/bin/entrypoint.sh commodore catalog compile --tenant-repo-revision-override $CI_COMMIT_SHA c-cluster-id-1000",
95+
"(cd catalog/ && git --no-pager diff --staged --output ../diff.txt)"
96+
],
97+
"stage": "build",
98+
"variables": {
99+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 1,
100+
"KUBERNETES_CPU_LIMIT": "800m",
101+
"KUBERNETES_CPU_REQUEST": "800m",
102+
"KUBERNETES_MEMORY_LIMIT": "3Gi",
103+
"KUBERNETES_MEMORY_REQUEST": "3Gi"
104+
}
105+
},
106+
"c-cluster-id-1000_deploy": {
107+
"before_script": [
108+
"install --directory --mode=0700 ~/.ssh",
109+
"echo \"$SSH_KNOWN_HOSTS\" >> ~/.ssh/known_hosts",
110+
"echo \"$SSH_CONFIG\" >> ~/.ssh/config"
111+
],
112+
"image": {
113+
"name": "docker.io/projectsyn/commodore:v1.34.0"
114+
},
115+
"rules": [
116+
{
117+
"if": "$CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH"
118+
}
119+
],
120+
"script": [
121+
"git config --add --global url.\"https://gitlab-ci-token:${CI_JOB_TOKEN}@git.vshn.net:80\".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}",
122+
"git config --add --global url.\"https://gitlab-ci-token:${CI_JOB_TOKEN}@git.vshn.net:80\".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}${CI_SERVER_SHELL_SSH_PORT:+:${CI_SERVER_SHELL_SSH_PORT}}",
123+
"git config --add --global url.\"https://gitlab-ci-token:${ACCESS_TOKEN_c_cluster_id_1000}@git.vshn.net:80/cluster-catalogs/c-cluster-id-1000.git\".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}/cluster-catalogs/c-cluster-id-1000.git",
124+
"git config --add --global url.\"https://gitlab-ci-token:${ACCESS_TOKEN_c_cluster_id_1000}@git.vshn.net:80/cluster-catalogs/c-cluster-id-1000.git\".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}${CI_SERVER_SHELL_SSH_PORT:+:${CI_SERVER_SHELL_SSH_PORT}}/cluster-catalogs/c-cluster-id-1000.git",
125+
"/usr/local/bin/entrypoint.sh commodore catalog compile --push c-cluster-id-1000"
126+
],
127+
"stage": "deploy",
128+
"variables": {
129+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 1,
130+
"KUBERNETES_CPU_LIMIT": "800m",
131+
"KUBERNETES_CPU_REQUEST": "800m",
132+
"KUBERNETES_MEMORY_LIMIT": "3Gi",
133+
"KUBERNETES_MEMORY_REQUEST": "3Gi"
134+
}
135+
},
67136
"c-cluster-id-1234_compile": {
68137
"artifacts": {
69138
"expire_in": "1 week",
@@ -94,7 +163,8 @@
94163
],
95164
"stage": "build",
96165
"variables": {
97-
"KUBERNETES_CPU_LIMIT": "2",
166+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 1,
167+
"KUBERNETES_CPU_LIMIT": "1.6",
98168
"KUBERNETES_CPU_REQUEST": "1200m",
99169
"KUBERNETES_MEMORY_LIMIT": "3Gi",
100170
"KUBERNETES_MEMORY_REQUEST": "2Gi"
@@ -123,7 +193,8 @@
123193
],
124194
"stage": "deploy",
125195
"variables": {
126-
"KUBERNETES_CPU_LIMIT": "2",
196+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 1,
197+
"KUBERNETES_CPU_LIMIT": "1.6",
127198
"KUBERNETES_CPU_REQUEST": "1200m",
128199
"KUBERNETES_MEMORY_LIMIT": "3Gi",
129200
"KUBERNETES_MEMORY_REQUEST": "2Gi"
@@ -159,6 +230,7 @@
159230
],
160231
"stage": "build",
161232
"variables": {
233+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 2,
162234
"KUBERNETES_CPU_LIMIT": "2",
163235
"KUBERNETES_CPU_REQUEST": "2",
164236
"KUBERNETES_MEMORY_LIMIT": "3Gi",
@@ -188,6 +260,7 @@
188260
],
189261
"stage": "deploy",
190262
"variables": {
263+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 2,
191264
"KUBERNETES_CPU_LIMIT": "2",
192265
"KUBERNETES_CPU_REQUEST": "2",
193266
"KUBERNETES_MEMORY_LIMIT": "3Gi",

‎gitlab/tests/k8s-resources.env‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
1-
CLUSTERS="c-cluster-id-1234 c-cluster-id-5678 c-cluster-id-0099"
2-
CLUSTER_CATALOG_URLS="c-cluster-id-1234=ssh://git@git.vshn.net/cluster-catalogs/c-cluster-id-1234.git c-cluster-id-5678=ssh://git@git.vshn.net/cluster-catalogs/c-cluster-id-5678.git c-cluster-id-0099=ssh://git@git.vshn.net/cluster-catalogs/c-cluster-id-0099.git "
1+
CLUSTERS="c-cluster-id-1234 c-cluster-id-5678 c-cluster-id-0099 c-cluster-id-1000"
2+
CLUSTER_CATALOG_URLS="c-cluster-id-1234=ssh://git@git.vshn.net/cluster-catalogs/c-cluster-id-1234.git c-cluster-id-5678=ssh://git@git.vshn.net/cluster-catalogs/c-cluster-id-5678.git c-cluster-id-0099=ssh://git@git.vshn.net/cluster-catalogs/c-cluster-id-0099.git c-cluster-id-1000=ssh://git@git.vshn.net/cluster-catalogs/c-cluster-id-1000.git "
33
MEMORY_LIMITS="c-cluster-id-1234=3Gi"
44
MEMORY_REQUESTS="c-cluster-id-1234=2Gi"
55
CPU_REQUESTS="c-cluster-id-1234=1200m c-cluster-id-5678=2"
6-
CPU_LIMITS="c-cluster-id-0099=2500m"
6+
CPU_LIMITS="c-cluster-id-1234=1.6 c-cluster-id-0099=2500m c-cluster-id-1000=800m"
7+
COMMODORE_PROCESSES_FROM_CPU_LIMIT="true"

‎gitlab/tests/run-instance.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,5 @@ jsonnet --ext-str clusters="$CLUSTERS" \
1616
--ext-str memory_requests="${MEMORY_REQUESTS:-}" \
1717
--ext-str cpu_limits="${CPU_LIMITS:-}" \
1818
--ext-str cpu_requests="${CPU_REQUESTS:-}" \
19+
--ext-str commodore_proc_count="${COMMODORE_PROCESSES_FROM_CPU_LIMIT:-}" \
1920
commodore-compile.jsonnet

0 commit comments

Comments
 (0)