Skip to content

Commit e2ad88d

Browse files
committed
Merge branch '2722-integrate-with-knative-allowing-specifying-pglambda-type-in-sgstream-target-with-source-code' into 'main'
Resolves #2722, #1027, #2854, #2861, #2862 Closes #2722, #1027, #2854, #2861, and #2862 See merge request ongresinc/stackgres!1598
2 parents a3637d0 + aa97825 commit e2ad88d

135 files changed

Lines changed: 10708 additions & 467 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ local.properties
9292

9393
# Annotation Processing
9494
.apt_generated
95+
.apt_generated_tests
9596

9697
.sts4-cache/
9798

.gitlab-ci/deploy.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,14 @@ deploy operator openshift bundle image:
163163
IMAGE_PUSH_TAG_SUFFIX: "-openshift"
164164
TARGET_REGISTRIES: "quay.io"
165165

166+
deploy pglambda javascript image:
167+
extends: .deploy image
168+
variables:
169+
IMAGE_NAME: "stackgres/pglambdas"
170+
IMAGE_TAG_SUFFIX: "-javascript"
171+
IMAGE_PUSH_TAG_SUFFIX: "-javascript"
172+
TARGET_REGISTRIES: "quay.io"
173+
166174
deploy helm packages and templates:
167175
image: $CI_REGISTRY/$SG_CI_PROJECT_PATH/$SG_CI_IMAGE_NAME
168176
stage: deploy

.gitlab-ci/image-build.yml

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -462,3 +462,51 @@ build operator openshift bundle image multi archs:
462462
extends: .build operator bundle image multi archs
463463
variables:
464464
IMAGE_SUFFIX: '-openshift'
465+
466+
.build pglambda javascript image amd64:
467+
extends: .build image
468+
tags:
469+
- stackgres-runner-v2
470+
- linux-amd64
471+
variables:
472+
IMAGE_SUFFIX: '-javascript-amd64'
473+
IMAGE_BASE_NAME: stackgres/pglambda
474+
475+
.build pglambda javascript image arm64:
476+
extends: .build image
477+
dependencies:
478+
- build arm64
479+
tags:
480+
- stackgres-runner-v2
481+
- linux-arm64
482+
variables:
483+
IMAGE_SUFFIX: '-javascript-arm64'
484+
IMAGE_BASE_NAME: stackgres/pglambda
485+
only:
486+
variables:
487+
- $SKIP_ARM != "true" && $CI_COMMIT_TAG && $CI_COMMIT_TAG !~ /^latest-.*$/
488+
- $DO_ARM
489+
490+
build pglambda javascript image amd64:
491+
extends: .build pglambda javascript image amd64
492+
variables:
493+
LOCK_FILE: stackgres-pglambda-javascript-build-image
494+
IMAGE_MODULE_NAME: pglambda-javascript-image
495+
496+
build pglambda javascript image arm64:
497+
extends: .build pglambda javascript image arm64
498+
variables:
499+
LOCK_FILE: stackgres-pglambda-javascript-build-image
500+
IMAGE_MODULE_NAME: pglambda-javascript-image
501+
502+
.build pglambda javascript image multi archs:
503+
extends: .build image multi archs
504+
tags:
505+
- stackgres-runner-v2
506+
variables:
507+
IMAGE_BASE_NAME: stackgres/pglambda
508+
509+
build pglambda javascript image multi archs:
510+
extends: .build pglambda javascript image multi archs
511+
variables:
512+
IMAGE_SUFFIX: '-javascript'

stackgres-k8s/ci/build/config.yml

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
ubi9_minimal_image: &ubi9_minimal_image registry.access.redhat.com/ubi9-minimal:9.4-1194
77
admin_ui_build_image: &admin_ui_build_image registry.access.redhat.com/ubi8/nodejs-20:1-50.1720405266
88
admin_ui_nginx_image: &admin_ui_nginx_image registry.access.redhat.com/ubi8/nginx-124:1-16
9+
pglambda_nodejs_image: &pglambda_nodejs_image registry.access.redhat.com/ubi8/nodejs-18:1-114.1720405264
910
platforms:
1011
- linux/x86_64
1112
- linux/aarch64
@@ -39,10 +40,12 @@ modules:
3940
build_commands:
4041
- 'sh stackgres-k8s/ci/build/update-build-pom.sh'
4142
- 'stackgres-k8s/src/mvnw $MAVEN_CLI_OPTS clean install -D skipTests -f "$MODULE_PATH/pom.xml.build" -N'
43+
- 'stackgres-k8s/src/mvnw $MAVEN_CLI_OPTS clean install -D skipTests -f "$MODULE_PATH/pom.xml.build" -pl checks'
4244
cache:
4345
- .m2/
4446
artifacts:
4547
- .m2/repository/io/stackgres/stackgres-parent
48+
- .m2/repository/io/stackgres/stackgres-checks
4649
operator-framework-java: &operator-framework-java
4750
<<: *parent-java
4851
path: stackgres-k8s/src/operator-framework/.
@@ -51,6 +54,9 @@ modules:
5154
- sh stackgres-k8s/ci/build/redact-version.sh operator-framework
5255
sources:
5356
- stackgres-k8s/src/operator-framework/src/main
57+
build_commands:
58+
- 'sh stackgres-k8s/ci/build/update-build-pom.sh'
59+
- 'stackgres-k8s/src/mvnw $MAVEN_CLI_OPTS clean install -D skipTests -f "$MODULE_PATH/pom.xml.build" -N'
5460
artifacts:
5561
- stackgres-k8s/src/operator-framework/target
5662
- .m2/repository/io/stackgres/stackgres-operator-framework
@@ -238,8 +244,8 @@ modules:
238244
- export HOME=/project
239245
- npm install --verbose --unsafe-perm
240246
- |
241-
! grep -q 'later-cov' node_modules/prettycron/node_modules/later/index.js \
242-
|| sed -i 's/^require\(.*\);$/require(".\/later");/' node_modules/prettycron/node_modules/later/index.js
247+
! grep -q 'LATER_COV' node_modules/prettycron/node_modules/later/index.js \
248+
|| sed -i "s#^ *require\(.*LATER_COV.*\); *\$#require('./later');#" node_modules/prettycron/node_modules/later/index.js
243249
- npm run build
244250
- sh $SHELL_XTRACE build.sh
245251
cache:
@@ -697,6 +703,22 @@ modules:
697703
operator-release-openshift-bundle-image:
698704
<<: *operator-openshift-bundle-image
699705
type: operator-release-bundle-image
706+
pglambda-javascript-image:
707+
type: pglambda
708+
platform_dependent: true
709+
path: stackgres-k8s/src/stream
710+
target_image: *pglambda_nodejs_image
711+
dockerfile:
712+
path: stackgres-k8s/src/stream/src/main/docker/pglambda/javascript/Dockerfile
713+
args:
714+
BASE_IMAGE: $TARGET_IMAGE_NAME
715+
seds:
716+
- "s#^\\( *COPY\\) \\+'#\\1 './$MODULE_PATH/#"
717+
artifacts:
718+
0: stackgres-k8s/src/stream/src/main/docker/LICENCE
719+
1: stackgres-k8s/src/stream/src/main/docker/pglambda/javascript/index.js
720+
2: stackgres-k8s/src/stream/src/main/docker/pglambda/javascript/package.json
721+
3: stackgres-k8s/src/stream/src/main/docker/pglambda/javascript/package-lock.json.out
700722
stages:
701723
- parent-java: null
702724
- operator-framework-java: parent-java
@@ -762,3 +784,4 @@ stages:
762784
- operator-release-bundle-image: operator-release-bundle
763785
- operator-release-openshift-bundle: helm-release-packages
764786
- operator-release-openshift-bundle-image: operator-release-openshift-bundle
787+
- pglambda-javascript-image: null

stackgres-k8s/ci/test/e2e-run-all-tests-gitlab.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ set -e
1717
export E2E_SHELL="${E2E_SHELL:-sh}"
1818
export E2E_ENV="${E2E_ENV:-kind}"
1919
export E2E_PARALLELISM="${E2E_PARALLELISM:-32}"
20-
export K8S_VERSION="${K8S_VERSION:-1.22}"
20+
export K8S_VERSION="${K8S_VERSION:-1.24}"
2121
export K8S_FROM_DIND=true
2222
export K8S_REUSE="${K8S_REUSE:-false}"
2323
# shellcheck disable=SC2155

stackgres-k8s/e2e/envs/kind

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -56,9 +56,8 @@ update_k8s_config() {
5656
then
5757
local CONTAINER_NAME
5858
CONTAINER_NAME="$(cat /proc/self/cgroup | grep '^1:name' | cut -d / -f 3)"
59-
CONTAINER_NAME="${CONTAINER_NAME:-$(docker ps --filter name=^$(cat /etc/hostname) --format '{{.Names}}' 2>/dev/null)}"
60-
CONTAINER_NAME="${CONTAINER_NAME:-$(cat /etc/hostname)}"
61-
CONTAINER_NAME="${CONTAINER_NAME:-$(hostname)}"
59+
CONTAINER_NAME="${CONTAINER_NAME:-$(docker ps --filter name="^$(cat /etc/hostname)" --format '{{.Names}}')}"
60+
CONTAINER_NAME="${CONTAINER_NAME:-$(docker ps --filter id="$(cat /etc/hostname)" --format '{{.ID}}')}"
6261
docker inspect "$CONTAINER_NAME" \
6362
-f '{{ range $key,$value := .NetworkSettings.Networks }}{{ printf "%s\n" $key }}{{ end }}' \
6463
| grep -q '^kind$' \

stackgres-k8s/e2e/resources/knative-v1.14.1.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6966,6 +6966,7 @@ metadata:
69666966
annotations:
69676967
knative.dev/example-checksum: "ed77183a"
69686968
data:
6969+
registries-skipping-tag-resolving: "kind.local,ko.local,dev.local,quay.io,docker.io"
69696970
# This is the Go import path for the binary that is containerized
69706971
# and substituted here.
69716972
queue-sidecar-image: gcr.io/knative-releases/knative.dev/serving/cmd/queue@sha256:55e7d89003b3fb97ddfaf9d65326a7ccc69b437773c3eccab43cece771bc1eda
Lines changed: 201 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,201 @@
1+
#!/bin/sh
2+
3+
check_pgbench() {
4+
local DBOPS_NAME="${1:-$DBOPS_NAME}"
5+
6+
assert_dbops_running "$DBOPS_NAME" "$CLUSTER_NAMESPACE"
7+
8+
assert_dbops_completion "$DBOPS_NAME" "$CLUSTER_NAMESPACE"
9+
10+
if [ "$(kubectl get -n "$CLUSTER_NAMESPACE" job \
11+
-l "stackgres.io/dbops-name=$DBOPS_NAME,stackgres.io/db-ops=true" \
12+
-o name 2>/dev/null | wc -l)" = 1 ]
13+
then
14+
success "pgbench job was not removed after completion."
15+
else
16+
fail "pgbench job was removed after completion."
17+
fi
18+
19+
if kubectl get -n "$CLUSTER_NAMESPACE" sgdbops "$DBOPS_NAME" \
20+
--template '{{ .status.benchmark.pgbench.scaleFactor }}' | grep -x '[0-9]\+\(\.[0-9]\+\)\?'
21+
then
22+
success "pgbench returned the scale factor."
23+
else
24+
fail "pgbench did not returned the scale factor."
25+
fi
26+
27+
if kubectl get -n "$CLUSTER_NAMESPACE" sgdbops "$DBOPS_NAME" \
28+
--template '{{ .status.benchmark.pgbench.transactionsProcessed }}' | grep -x '[0-9]\+\(\.[0-9]\+\)\?'
29+
then
30+
success "pgbench returned the transactions processed."
31+
else
32+
fail "pgbench did not returned the transactions processed."
33+
fi
34+
35+
if kubectl get -n "$CLUSTER_NAMESPACE" sgdbops "$DBOPS_NAME" \
36+
--template '{{ .status.benchmark.pgbench.latency.average.value }}' | grep -x '[0-9]\+\(\.[0-9]\+\)\?'
37+
then
38+
success "pgbench returned the average latency."
39+
else
40+
fail "pgbench did not returned the average latency."
41+
fi
42+
43+
if [ "$(kubectl get -n "$CLUSTER_NAMESPACE" sgdbops "$DBOPS_NAME" \
44+
--template '{{ .status.benchmark.pgbench.latency.average.unit }}')" \
45+
= "ms" ]
46+
then
47+
success "pgbench returned the average latency unit"
48+
else
49+
fail "pgbench did not returned the average latency unit."
50+
fi
51+
52+
if kubectl get -n "$CLUSTER_NAMESPACE" sgdbops "$DBOPS_NAME" \
53+
--template '{{ .status.benchmark.pgbench.latency.standardDeviation.value}}' | grep -x '[0-9]\+\(\.[0-9]\+\)\?'
54+
then
55+
success "pgbench returned the latency standard deviation."
56+
else
57+
fail "pgbench did not returned the latency standard deviation."
58+
fi
59+
60+
if [ "$(kubectl get -n "$CLUSTER_NAMESPACE" sgdbops "$DBOPS_NAME" \
61+
--template '{{ .status.benchmark.pgbench.latency.standardDeviation.unit}}')" \
62+
= "ms" ]
63+
then
64+
success "pgbench returned the latency standard deviation unit."
65+
else
66+
fail "pgbench did not returned the latency standard deviation unit."
67+
fi
68+
69+
if [ "${E2E_POSTGRES_VERSION%%.*}" -le 13 ]
70+
then
71+
if kubectl get -n "$CLUSTER_NAMESPACE" sgdbops "$DBOPS_NAME" \
72+
--template '{{ .status.benchmark.pgbench.transactionsPerSecond.includingConnectionsEstablishing.value }}' | grep -x '[0-9]\+\(\.[0-9]\+\)\?'
73+
then
74+
success "pgbench returned the tps including connections establishing."
75+
else
76+
fail "pgbench did not returned the tps including connections establishing."
77+
fi
78+
79+
if [ "$(kubectl get -n "$CLUSTER_NAMESPACE" sgdbops "$DBOPS_NAME" \
80+
--template '{{ .status.benchmark.pgbench.transactionsPerSecond.includingConnectionsEstablishing.unit }}')" \
81+
= "tps" ]
82+
then
83+
success "pgbench returned the tps including connections establishing unit."
84+
else
85+
fail "pgbench did not returned the tps including connections establishing unit."
86+
fi
87+
fi
88+
89+
if kubectl get -n "$CLUSTER_NAMESPACE" sgdbops "$DBOPS_NAME" \
90+
--template '{{ .status.benchmark.pgbench.transactionsPerSecond.excludingConnectionsEstablishing.value }}' | grep -x '[0-9]\+\(\.[0-9]\+\)\?'
91+
then
92+
success "pgbench returned the tps excluding connections establishing."
93+
else
94+
fail "pgbench did not returned the tps excluding connections establishing."
95+
fi
96+
97+
if [ "$(kubectl get -n "$CLUSTER_NAMESPACE" sgdbops "$DBOPS_NAME" \
98+
--template '{{ .status.benchmark.pgbench.transactionsPerSecond.excludingConnectionsEstablishing.unit }}')" \
99+
= "tps" ]
100+
then
101+
success "pgbench returned the tps excluding connections establishing unit."
102+
else
103+
fail "pgbench did not returned the tps excluding connections establishing unit."
104+
fi
105+
106+
if [ "$(kubectl get -n "$CLUSTER_NAMESPACE" sgdbops "$DBOPS_NAME" \
107+
--template '{{ .status.benchmark.pgbench.transactionsPerSecond.overTime.values | len }}')" -gt 0 ]
108+
then
109+
success "pgbench returned the tps over time values."
110+
else
111+
fail "pgbench did not returned the tps over time values."
112+
fi
113+
114+
if kubectl get -n "$CLUSTER_NAMESPACE" sgdbops "$DBOPS_NAME" \
115+
--template '{{ (index .status.benchmark.pgbench.transactionsPerSecond.overTime.values 0) }}' | grep -x '[0-9]\+\(\.[0-9]\+\)\?'
116+
then
117+
success "pgbench returned the tps over time value."
118+
else
119+
fail "pgbench did not returned the tps over time value."
120+
fi
121+
122+
if [ "$(kubectl get -n "$CLUSTER_NAMESPACE" sgdbops "$DBOPS_NAME" \
123+
--template '{{ .status.benchmark.pgbench.transactionsPerSecond.overTime.valuesUnit }}')" = 'tps' ]
124+
then
125+
success "pgbench returned the tps over time values unit."
126+
else
127+
fail "pgbench did not returned the tps over time values unit."
128+
fi
129+
130+
if kubectl get -n "$CLUSTER_NAMESPACE" sgdbops "$DBOPS_NAME" \
131+
--template '{{ .status.benchmark.pgbench.transactionsPerSecond.overTime.intervalDuration }}' | grep -x '[0-9]\+\(\.[0-9]\+\)\?'
132+
then
133+
success "pgbench returned the tps over time interval duration."
134+
else
135+
fail "pgbench did not returned the tps over time interval duration."
136+
fi
137+
138+
if [ "$(kubectl get -n "$CLUSTER_NAMESPACE" sgdbops "$DBOPS_NAME" \
139+
--template '{{ .status.benchmark.pgbench.transactionsPerSecond.overTime.intervalDurationUnit }}')" = 'ms' ]
140+
then
141+
success "pgbench returned the tps over time interval duration unit."
142+
else
143+
fail "pgbench did not returned the tps over time interval duration unit."
144+
fi
145+
146+
if [ "$(kubectl get -n "$CLUSTER_NAMESPACE" sgdbops "$DBOPS_NAME" \
147+
--template '{{ .status.benchmark.pgbench.statements | len }}')" -gt 0 ]
148+
then
149+
success "pgbench returned the statements."
150+
else
151+
fail "pgbench did not returned the statements."
152+
fi
153+
154+
if kubectl get -n "$CLUSTER_NAMESPACE" sgdbops "$DBOPS_NAME" \
155+
--template '{{ (index .status.benchmark.pgbench.statements 0).command }}' | grep .
156+
then
157+
success "pgbench returned the statements command."
158+
else
159+
fail "pgbench did not returned the statements command."
160+
fi
161+
162+
if kubectl get -n "$CLUSTER_NAMESPACE" sgdbops "$DBOPS_NAME" \
163+
--template '{{ (index .status.benchmark.pgbench.statements 0).latency }}' | grep -x '[0-9]\+\(\.[0-9]\+\)\?'
164+
then
165+
success "pgbench returned the statements latency."
166+
else
167+
fail "pgbench did not returned the statements latency."
168+
fi
169+
170+
if [ "$(kubectl get -n "$CLUSTER_NAMESPACE" sgdbops "$DBOPS_NAME" \
171+
--template '{{ (index .status.benchmark.pgbench.statements 0).unit }}')" = ms ]
172+
then
173+
success "pgbench returned the statements unit."
174+
else
175+
fail "pgbench did not returned the statements unit."
176+
fi
177+
178+
if kubectl get -n "$CLUSTER_NAMESPACE" sgdbops "$DBOPS_NAME" \
179+
--template '{{ (index .status.benchmark.pgbench.statements 0).script }}' | grep -x '[0-9]\+'
180+
then
181+
success "pgbench returned the statements script."
182+
else
183+
fail "pgbench did not returned the statements script."
184+
fi
185+
186+
if kubectl get -n "$CLUSTER_NAMESPACE" sgdbops "$DBOPS_NAME" \
187+
--template '{{ .status.benchmark.pgbench.hdrHistogram }}' | grep -x '[0-9A-Za-z+/=]\+'
188+
then
189+
success "pgbench returned the HdrHistogram."
190+
else
191+
fail "pgbench did not returned the HdrHistogram."
192+
fi
193+
194+
RESULT="$(run_query -i 0 -p 5432 -q "SELECT EXISTS(SELECT * FROM pg_database WHERE datname LIKE 'pgbench-%')")"
195+
if [ "$RESULT" = "f" ]
196+
then
197+
success "pgbench database was dropped."
198+
else
199+
fail "pgbench database was not dropped."
200+
fi
201+
}

stackgres-k8s/e2e/spec/abstract/prometheus-integration

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,9 +39,17 @@ check_grafana_secret() {
3939

4040
check_exporter_queries() {
4141
local METRIC
42-
for METRIC in $(yq -r 'keys[]' "$PROJECT_PATH"/stackgres-k8s/src/operator/src/main/resources/prometheus-postgres-exporter/queries-1.22.yaml)
42+
local QUERIES_FILE=queries.yaml
43+
local VERSION_AS_NUMBER VERSION_1_12_0
44+
VERSION_AS_NUMBER="$(get_version_as_number "$STACKGRES_VERSION")"
45+
VERSION_1_12_0="$(get_version_as_number 1.12.0)"
46+
if [ "$VERSION_AS_NUMBER" -le "$VERSION_1_12_0" ]
47+
then
48+
QUERIES_FILE=queries-1.22.yaml
49+
fi
50+
for METRIC in $(yq -r 'keys[]' "$PROJECT_PATH"/stackgres-k8s/src/operator/src/main/resources/prometheus-postgres-exporter/"$QUERIES_FILE")
4351
do
44-
QUERY="$(yq -r ".[\"$METRIC\"].query" "$PROJECT_PATH"/stackgres-k8s/src/operator/src/main/resources/prometheus-postgres-exporter/queries-1.22.yaml)"
52+
QUERY="$(yq -r ".[\"$METRIC\"].query" "$PROJECT_PATH"/stackgres-k8s/src/operator/src/main/resources/prometheus-postgres-exporter/"$QUERIES_FILE")"
4553
if wait_until eval 'kubectl exec -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-0" -c postgres-util -- \
4654
psql -v ON_ERROR_STOP=ON -c "$QUERY" > /dev/null'
4755
then

0 commit comments

Comments
 (0)