Skip to content

Commit ab336dc

Browse files
simusimu
authored
Merge pull request #42 from projectsyn/feat/more-resource-tuning
Optionally set `COMMODORE_CATALOG_COMPILE_PROCESSES` based on cluster's CPU limit
2 parents 013bdd9 + c4a201d commit ab336dc

9 files changed

Lines changed: 169 additions & 7 deletions

‎gitlab/README.md‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,3 +129,17 @@ Example:
129129
variables:
130130
MEMORY_LIMITS: "c-my-cluster=4Gi c-my-other-cluster=4Gi"
131131
```
132+
133+
### Configure number of Commodore worker processes based on CI job CPU limits
134+
135+
Commodore v1.34.0 and newer allow users to configure the number of worker processes that are used to compile the catalog.
136+
137+
> [!NOTE]
138+
> This configuration option doesn't have an impact on the number of threads used by reclass-rs to render the Reclass inventory.
139+
140+
Users can opt-in to setting the number of Commodore worker processes based on each cluster's CI job CPU limits.
141+
To opt-in, users need to set environment variable `COMMODORE_PROCESSES_FROM_CPU_LIMIT` to a non-empty value in their GitLab CI configuration.
142+
When opting in, the Job generator sets the number of worker processes for each cluster's CI job to the maximum of the cluster's CPU limit rounded down to the next integer and 1.
143+
We use 1 as the lower bound, since flag value 0 configures Commodore to auto-detect the number of worker processes.
144+
145+
For users that don't opt-in, the Job generator configures Commodore to auto-detect the number of worker processes, preserving the Commodore v1.33.1 behavior.

‎gitlab/bin/step-render-pipeline‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,5 +20,6 @@ jsonnet \
2020
--ext-str cluster_catalog_urls="${cluster_catalog_urls}" \
2121
--ext-str server_fqdn="${CI_SERVER_FQDN}" \
2222
--ext-str server_ssh_host="${CI_SERVER_SHELL_SSH_HOST}" \
23+
--ext-str commodore_proc_count_from_cpu_limit="${COMMODORE_PROCESSES_FROM_CPU_LIMIT:-}" \
2324
/opt/commodore-compile.jsonnet \
2425
>generated-commodore-compile.yml

‎gitlab/commodore-compile.jsonnet‎

Lines changed: 64 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,66 @@ local gitInsteadOf(cluster) =
103103
'git config --add --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@%(gitlab_fqdn)s".insteadOf ssh://git@%(ssh_hostport)s' % gitlab_params,
104104
] + catalogInsteadOf;
105105

106+
local cpu_limit(cluster) = std.get(cpu_limits, cluster, '2');
107+
108+
// K8s quantity parser.
109+
// Supported suffices as listed in `kubectl explain pod.spec.containers.resources.requests`.
110+
local parseK8sQuantity(q) =
111+
local decimalSIMap = {
112+
m: 1 / 1000,
113+
k: 1000,
114+
M: 1000 * 1000,
115+
G: 1000 * 1000 * 1000,
116+
T: 1000 * 1000 * 1000 * 1000,
117+
P: 1000 * 1000 * 1000 * 1000 * 1000,
118+
E: 1000 * 1000 * 1000 * 1000 * 1000 * 1000,
119+
};
120+
local binarySIMap = {
121+
Ki: 1024,
122+
Mi: 1024 * 1024,
123+
Gi: 1024 * 1024 * 1024,
124+
Ti: 1024 * 1024 * 1024 * 1024,
125+
Pi: 1024 * 1024 * 1024 * 1024 * 1024,
126+
Ei: 1024 * 1024 * 1024 * 1024 * 1024 * 1024,
127+
};
128+
local qlen = std.length(q);
129+
local decSuffix = if qlen > 1 then q[-1:] else '';
130+
local binSuffix = if qlen > 2 then q[-2:] else '';
131+
local val =
132+
if std.member(std.objectFields(decimalSIMap), decSuffix) then
133+
{
134+
scale: decimalSIMap[decSuffix],
135+
num: q[:-1],
136+
}
137+
else if std.member(std.objectFields(binarySIMap), binSuffix) then
138+
{
139+
scale: binarySIMap[binSuffix],
140+
num: q[:-2],
141+
}
142+
else
143+
{
144+
scale: 1,
145+
num: q,
146+
};
147+
148+
local num = std.parseYaml(val.num);
149+
if !std.isNumber(num) then
150+
error 'Failed to parse K8s quantity %s: parser got %s' % [ q, val ]
151+
else
152+
num * val.scale;
153+
154+
// Compute Commodore process count based on CPU limit for the cluster by
155+
// rounding down to the next nearest integer, clamped at 1.
156+
local proc_count(cluster) =
157+
if std.extVar('commodore_proc_count_from_cpu_limit') != '' then (
158+
local cl = parseK8sQuantity(cpu_limit(cluster));
159+
if cl >= 1 then std.floor(cl) else 1
160+
) else
161+
// Commodore auto-selects the number of worker processes when the value of
162+
// the flag/envvar is 0. This matches the behavior of Commodore v1.33.1
163+
// and older.
164+
0;
165+
106166
local compile_job(cluster) =
107167
{
108168
stage: 'build',
@@ -114,8 +174,9 @@ local compile_job(cluster) =
114174
{
115175
KUBERNETES_MEMORY_LIMIT: std.get(memory_limits, cluster, '3Gi'),
116176
KUBERNETES_MEMORY_REQUEST: std.get(memory_requests, cluster, '3Gi'),
117-
KUBERNETES_CPU_LIMIT: std.get(cpu_limits, cluster, '2'),
177+
KUBERNETES_CPU_LIMIT: cpu_limit(cluster),
118178
KUBERNETES_CPU_REQUEST: std.get(cpu_requests, cluster, '800m'),
179+
COMMODORE_CATALOG_COMPILE_PROCESSES: proc_count(cluster),
119180
},
120181
before_script:
121182
[
@@ -149,8 +210,9 @@ local deploy_job(cluster) =
149210
{
150211
KUBERNETES_MEMORY_LIMIT: std.get(memory_limits, cluster, '3Gi'),
151212
KUBERNETES_MEMORY_REQUEST: std.get(memory_requests, cluster, '3Gi'),
152-
KUBERNETES_CPU_LIMIT: std.get(cpu_limits, cluster, '2'),
213+
KUBERNETES_CPU_LIMIT: cpu_limit(cluster),
153214
KUBERNETES_CPU_REQUEST: std.get(cpu_requests, cluster, '800m'),
215+
COMMODORE_CATALOG_COMPILE_PROCESSES: proc_count(cluster),
154216
},
155217
image:
156218
{

‎gitlab/tests/golden/custom-ssh-port.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
],
3030
"stage": "build",
3131
"variables": {
32+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 0,
3233
"KUBERNETES_CPU_LIMIT": "2",
3334
"KUBERNETES_CPU_REQUEST": "800m",
3435
"KUBERNETES_MEMORY_LIMIT": "3Gi",
@@ -58,6 +59,7 @@
5859
],
5960
"stage": "deploy",
6061
"variables": {
62+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 0,
6163
"KUBERNETES_CPU_LIMIT": "2",
6264
"KUBERNETES_CPU_REQUEST": "800m",
6365
"KUBERNETES_MEMORY_LIMIT": "3Gi",

‎gitlab/tests/golden/default.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
],
3030
"stage": "build",
3131
"variables": {
32+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 0,
3233
"KUBERNETES_CPU_LIMIT": "2",
3334
"KUBERNETES_CPU_REQUEST": "800m",
3435
"KUBERNETES_MEMORY_LIMIT": "3Gi",
@@ -58,6 +59,7 @@
5859
],
5960
"stage": "deploy",
6061
"variables": {
62+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 0,
6163
"KUBERNETES_CPU_LIMIT": "2",
6264
"KUBERNETES_CPU_REQUEST": "800m",
6365
"KUBERNETES_MEMORY_LIMIT": "3Gi",

‎gitlab/tests/golden/external-catalog.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
],
3030
"stage": "build",
3131
"variables": {
32+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 0,
3233
"KUBERNETES_CPU_LIMIT": "2",
3334
"KUBERNETES_CPU_REQUEST": "800m",
3435
"KUBERNETES_MEMORY_LIMIT": "3Gi",
@@ -58,6 +59,7 @@
5859
],
5960
"stage": "deploy",
6061
"variables": {
62+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 0,
6163
"KUBERNETES_CPU_LIMIT": "2",
6264
"KUBERNETES_CPU_REQUEST": "800m",
6365
"KUBERNETES_MEMORY_LIMIT": "3Gi",
@@ -94,6 +96,7 @@
9496
],
9597
"stage": "build",
9698
"variables": {
99+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 0,
97100
"KUBERNETES_CPU_LIMIT": "2",
98101
"KUBERNETES_CPU_REQUEST": "800m",
99102
"KUBERNETES_MEMORY_LIMIT": "3Gi",
@@ -123,6 +126,7 @@
123126
],
124127
"stage": "deploy",
125128
"variables": {
129+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 0,
126130
"KUBERNETES_CPU_LIMIT": "2",
127131
"KUBERNETES_CPU_REQUEST": "800m",
128132
"KUBERNETES_MEMORY_LIMIT": "3Gi",
@@ -157,6 +161,7 @@
157161
],
158162
"stage": "build",
159163
"variables": {
164+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 0,
160165
"KUBERNETES_CPU_LIMIT": "2",
161166
"KUBERNETES_CPU_REQUEST": "800m",
162167
"KUBERNETES_MEMORY_LIMIT": "3Gi",
@@ -184,6 +189,7 @@
184189
],
185190
"stage": "deploy",
186191
"variables": {
192+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 0,
187193
"KUBERNETES_CPU_LIMIT": "2",
188194
"KUBERNETES_CPU_REQUEST": "800m",
189195
"KUBERNETES_MEMORY_LIMIT": "3Gi",

‎gitlab/tests/golden/k8s-resources.yml‎

Lines changed: 75 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
],
3030
"stage": "build",
3131
"variables": {
32+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 2,
3233
"KUBERNETES_CPU_LIMIT": "2500m",
3334
"KUBERNETES_CPU_REQUEST": "800m",
3435
"KUBERNETES_MEMORY_LIMIT": "3Gi",
@@ -58,12 +59,80 @@
5859
],
5960
"stage": "deploy",
6061
"variables": {
62+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 2,
6163
"KUBERNETES_CPU_LIMIT": "2500m",
6264
"KUBERNETES_CPU_REQUEST": "800m",
6365
"KUBERNETES_MEMORY_LIMIT": "3Gi",
6466
"KUBERNETES_MEMORY_REQUEST": "3Gi"
6567
}
6668
},
69+
"c-cluster-id-1000_compile": {
70+
"artifacts": {
71+
"expire_in": "1 week",
72+
"paths": [
73+
"diff.txt"
74+
]
75+
},
76+
"before_script": [
77+
"install --directory --mode=0700 ~/.ssh",
78+
"echo \"$SSH_KNOWN_HOSTS\" >> ~/.ssh/known_hosts",
79+
"echo \"$SSH_CONFIG\" >> ~/.ssh/config"
80+
],
81+
"image": {
82+
"name": "docker.io/projectsyn/commodore:v1.34.0"
83+
},
84+
"rules": [
85+
{
86+
"if": "$CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH"
87+
}
88+
],
89+
"script": [
90+
"git config --add --global url.\"https://gitlab-ci-token:${CI_JOB_TOKEN}@git.vshn.net:80\".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}",
91+
"git config --add --global url.\"https://gitlab-ci-token:${CI_JOB_TOKEN}@git.vshn.net:80\".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}${CI_SERVER_SHELL_SSH_PORT:+:${CI_SERVER_SHELL_SSH_PORT}}",
92+
"git config --add --global url.\"https://gitlab-ci-token:${ACCESS_TOKEN_c_cluster_id_1000}@git.vshn.net:80/cluster-catalogs/c-cluster-id-1000.git\".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}/cluster-catalogs/c-cluster-id-1000.git",
93+
"git config --add --global url.\"https://gitlab-ci-token:${ACCESS_TOKEN_c_cluster_id_1000}@git.vshn.net:80/cluster-catalogs/c-cluster-id-1000.git\".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}${CI_SERVER_SHELL_SSH_PORT:+:${CI_SERVER_SHELL_SSH_PORT}}/cluster-catalogs/c-cluster-id-1000.git",
94+
"/usr/local/bin/entrypoint.sh commodore catalog compile --tenant-repo-revision-override $CI_COMMIT_SHA c-cluster-id-1000",
95+
"(cd catalog/ && git --no-pager diff --staged --output ../diff.txt)"
96+
],
97+
"stage": "build",
98+
"variables": {
99+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 1,
100+
"KUBERNETES_CPU_LIMIT": "800m",
101+
"KUBERNETES_CPU_REQUEST": "800m",
102+
"KUBERNETES_MEMORY_LIMIT": "3Gi",
103+
"KUBERNETES_MEMORY_REQUEST": "3Gi"
104+
}
105+
},
106+
"c-cluster-id-1000_deploy": {
107+
"before_script": [
108+
"install --directory --mode=0700 ~/.ssh",
109+
"echo \"$SSH_KNOWN_HOSTS\" >> ~/.ssh/known_hosts",
110+
"echo \"$SSH_CONFIG\" >> ~/.ssh/config"
111+
],
112+
"image": {
113+
"name": "docker.io/projectsyn/commodore:v1.34.0"
114+
},
115+
"rules": [
116+
{
117+
"if": "$CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH"
118+
}
119+
],
120+
"script": [
121+
"git config --add --global url.\"https://gitlab-ci-token:${CI_JOB_TOKEN}@git.vshn.net:80\".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}",
122+
"git config --add --global url.\"https://gitlab-ci-token:${CI_JOB_TOKEN}@git.vshn.net:80\".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}${CI_SERVER_SHELL_SSH_PORT:+:${CI_SERVER_SHELL_SSH_PORT}}",
123+
"git config --add --global url.\"https://gitlab-ci-token:${ACCESS_TOKEN_c_cluster_id_1000}@git.vshn.net:80/cluster-catalogs/c-cluster-id-1000.git\".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}/cluster-catalogs/c-cluster-id-1000.git",
124+
"git config --add --global url.\"https://gitlab-ci-token:${ACCESS_TOKEN_c_cluster_id_1000}@git.vshn.net:80/cluster-catalogs/c-cluster-id-1000.git\".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}${CI_SERVER_SHELL_SSH_PORT:+:${CI_SERVER_SHELL_SSH_PORT}}/cluster-catalogs/c-cluster-id-1000.git",
125+
"/usr/local/bin/entrypoint.sh commodore catalog compile --push c-cluster-id-1000"
126+
],
127+
"stage": "deploy",
128+
"variables": {
129+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 1,
130+
"KUBERNETES_CPU_LIMIT": "800m",
131+
"KUBERNETES_CPU_REQUEST": "800m",
132+
"KUBERNETES_MEMORY_LIMIT": "3Gi",
133+
"KUBERNETES_MEMORY_REQUEST": "3Gi"
134+
}
135+
},
67136
"c-cluster-id-1234_compile": {
68137
"artifacts": {
69138
"expire_in": "1 week",
@@ -94,7 +163,8 @@
94163
],
95164
"stage": "build",
96165
"variables": {
97-
"KUBERNETES_CPU_LIMIT": "2",
166+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 1,
167+
"KUBERNETES_CPU_LIMIT": "1.6",
98168
"KUBERNETES_CPU_REQUEST": "1200m",
99169
"KUBERNETES_MEMORY_LIMIT": "3Gi",
100170
"KUBERNETES_MEMORY_REQUEST": "2Gi"
@@ -123,7 +193,8 @@
123193
],
124194
"stage": "deploy",
125195
"variables": {
126-
"KUBERNETES_CPU_LIMIT": "2",
196+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 1,
197+
"KUBERNETES_CPU_LIMIT": "1.6",
127198
"KUBERNETES_CPU_REQUEST": "1200m",
128199
"KUBERNETES_MEMORY_LIMIT": "3Gi",
129200
"KUBERNETES_MEMORY_REQUEST": "2Gi"
@@ -159,6 +230,7 @@
159230
],
160231
"stage": "build",
161232
"variables": {
233+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 2,
162234
"KUBERNETES_CPU_LIMIT": "2",
163235
"KUBERNETES_CPU_REQUEST": "2",
164236
"KUBERNETES_MEMORY_LIMIT": "3Gi",
@@ -188,6 +260,7 @@
188260
],
189261
"stage": "deploy",
190262
"variables": {
263+
"COMMODORE_CATALOG_COMPILE_PROCESSES": 2,
191264
"KUBERNETES_CPU_LIMIT": "2",
192265
"KUBERNETES_CPU_REQUEST": "2",
193266
"KUBERNETES_MEMORY_LIMIT": "3Gi",

‎gitlab/tests/k8s-resources.env‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
1-
CLUSTERS="c-cluster-id-1234 c-cluster-id-5678 c-cluster-id-0099"
2-
CLUSTER_CATALOG_URLS="c-cluster-id-1234=ssh://git@git.vshn.net/cluster-catalogs/c-cluster-id-1234.git c-cluster-id-5678=ssh://git@git.vshn.net/cluster-catalogs/c-cluster-id-5678.git c-cluster-id-0099=ssh://git@git.vshn.net/cluster-catalogs/c-cluster-id-0099.git "
1+
CLUSTERS="c-cluster-id-1234 c-cluster-id-5678 c-cluster-id-0099 c-cluster-id-1000"
2+
CLUSTER_CATALOG_URLS="c-cluster-id-1234=ssh://git@git.vshn.net/cluster-catalogs/c-cluster-id-1234.git c-cluster-id-5678=ssh://git@git.vshn.net/cluster-catalogs/c-cluster-id-5678.git c-cluster-id-0099=ssh://git@git.vshn.net/cluster-catalogs/c-cluster-id-0099.git c-cluster-id-1000=ssh://git@git.vshn.net/cluster-catalogs/c-cluster-id-1000.git "
33
MEMORY_LIMITS="c-cluster-id-1234=3Gi"
44
MEMORY_REQUESTS="c-cluster-id-1234=2Gi"
55
CPU_REQUESTS="c-cluster-id-1234=1200m c-cluster-id-5678=2"
6-
CPU_LIMITS="c-cluster-id-0099=2500m"
6+
CPU_LIMITS="c-cluster-id-1234=1.6 c-cluster-id-0099=2500m c-cluster-id-1000=800m"
7+
COMMODORE_PROCESSES_FROM_CPU_LIMIT="true"

‎gitlab/tests/run-instance.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,5 @@ jsonnet --ext-str clusters="$CLUSTERS" \
1616
--ext-str memory_requests="${MEMORY_REQUESTS:-}" \
1717
--ext-str cpu_limits="${CPU_LIMITS:-}" \
1818
--ext-str cpu_requests="${CPU_REQUESTS:-}" \
19+
--ext-str commodore_proc_count_from_cpu_limit="${COMMODORE_PROCESSES_FROM_CPU_LIMIT:-}" \
1920
commodore-compile.jsonnet

0 commit comments

Comments
 (0)