Merge branch '2806-sgdbops-restart-minor-version-upgrade-and-security-upgrade-always-fail-on-retry' into 'main'

Resolve "Implement SGStream to integrate with Postgres CDC and CloudEvents", "Implement migration in SGStream from an SGCluster or an external Postgres instance to ..." and "SGDbOps restart, minor version upgrade and security upgrade always fail on retry"

Closes #2718, #2814, #2806, and #2834

See merge request ongresinc/stackgres!1593

Author: teoincontatto

.gitignore

@@ -124,6 +124,7 @@ hs_err_pid*
 
 ## ignoring target file
 target/
+!**/src/main/**/target/
 dist/
 
 # maven-release-plugin

.gitlab-ci.yml

@@ -1,7 +1,7 @@
 image: null
 
 variables:
-  SG_CI_IMAGE_NAME: "ci:1.25"
+  SG_CI_IMAGE_NAME: "ci:1.26"
   BUILDER_IMAGE_NAME: "builder:1.8"
   SKIP_NATIVE: "false"
   SKIP_E2E: "false"
@@ -16,7 +16,7 @@ variables:
   RESTORE_CACHE_ATTEMPTS: 3
   E2E_SHELL: sh
   E2E_ENV: kind
-  K8S_VERSION: "1.22"
+  K8S_VERSION: "1.24"
   LATEST: "false"
   BUILD_UID: 0
 

.gitlab-ci/build.yml

@@ -15,6 +15,7 @@
         --extract jobs-java-test:'stackgres-k8s/src/jobs/target/surefire-reports/TEST-*.xml' \
         --extract cluster-controller-java-test:'stackgres-k8s/src/cluster-controller/target/surefire-reports/TEST-*.xml' \
         --extract distributedlogs-controller-java-test:'stackgres-k8s/src/distributedlogs-controller/target/surefire-reports/TEST-*.xml' \
+        --extract stream-java-test:'stackgres-k8s/src/stream/target/surefire-reports/TEST-*.xml' \
         --extract restapi-java:'stackgres-k8s/src/api-web/target/swagger*' \
         parent-java \
         operator-framework-java operator-framework-java-test \
@@ -24,7 +25,9 @@
         restapi-java restapi-java-test \
         jobs-java jobs-java-test \
         cluster-controller-java cluster-controller-java-test \
-        distributedlogs-controller-java distributedlogs-controller-java-test
+        distributedlogs-controller-java distributedlogs-controller-java-test \
+        stream-java stream-java-test
+      cp stackgres-k8s/ci/build/target/junit-build.hashes.xml."$(cat stackgres-k8s/ci/build/target/build_hash)" stackgres-k8s/ci/build/target/junit-build.hashes.xml
   retry:
     max: 2
     when: [ unknown_failure, api_failure, stuck_or_timeout_failure, runner_system_failure, runner_unsupported,

.gitlab-ci/deploy.yml

@@ -96,6 +96,12 @@ deploy distributedlogs-controller jvm image:
   variables:
     IMAGE_NAME: "stackgres/distributedlogs-controller"
 
+deploy stream jvm image:
+  extends: .deploy jvm image
+  variables:
+    IMAGE_NAME: "stackgres/stream"
+    TARGET_REGISTRIES: "quay.io"
+
 .deploy native image:
   extends: .deploy image
   variables:
@@ -131,6 +137,12 @@ deploy distributedlogs-controller native image:
   variables:
     IMAGE_NAME: "stackgres/distributedlogs-controller"
 
+.deploy stream native image:
+  extends: .deploy native image
+  variables:
+    IMAGE_NAME: "stackgres/stream"
+    TARGET_REGISTRIES: "quay.io"
+
 deploy admin-ui image:
   extends: .deploy native image
   variables:

.gitlab-ci/image-build.yml

@@ -118,6 +118,13 @@ build distributedlogs-controller jvm image amd64:
     LOCK_FILE: stackgres-distributedlogs-controller-build-jvm-image
     IMAGE_MODULE_NAME: distributedlogs-controller-jvm-image
 
+build stream jvm image amd64:
+  extends: .build jvm image amd64
+  variables:
+    IMAGE_BASE_NAME: stackgres/stream
+    LOCK_FILE: stackgres-stream-build-jvm-image
+    IMAGE_MODULE_NAME: stream-jvm-image
+
 build operator native image amd64:
   extends: .build native image amd64
   variables:
@@ -153,6 +160,13 @@ build distributedlogs-controller native image amd64:
     LOCK_FILE: stackgres-distributedlogs-controller-build-native-image
     IMAGE_MODULE_NAME: distributedlogs-controller-native-image
 
+.build stream native image amd64:
+  extends: .build native image amd64
+  variables:
+    IMAGE_BASE_NAME: stackgres/stream
+    LOCK_FILE: stackgres-stream-build-native-image
+    IMAGE_MODULE_NAME: stream-native-image
+
 build admin-ui image amd64:
   extends: .build image
   dependencies:
@@ -229,6 +243,13 @@ build distributedlogs-controller jvm image arm64:
     LOCK_FILE: stackgres-distributedlogs-controller-build-jvm-image
     IMAGE_MODULE_NAME: distributedlogs-controller-jvm-image
 
+build stream jvm image arm64:
+  extends: .build jvm image arm64
+  variables:
+    IMAGE_BASE_NAME: stackgres/stream
+    LOCK_FILE: stackgres-stream-build-jvm-image
+    IMAGE_MODULE_NAME: stream-jvm-image
+
 build operator native image arm64:
   extends: .build native image arm64
   variables:
@@ -264,6 +285,13 @@ build distributedlogs-controller native image arm64:
     LOCK_FILE: stackgres-distributedlogs-controller-build-native-image
     IMAGE_MODULE_NAME: distributedlogs-controller-native-image
 
+.build stream native image arm64:
+  extends: .build native image arm64
+  variables:
+    IMAGE_BASE_NAME: stackgres/stream
+    LOCK_FILE: stackgres-stream-build-native-image
+    IMAGE_MODULE_NAME: stream-native-image
+
 build admin-ui image arm64:
   extends: .build image
   dependencies:
@@ -325,6 +353,11 @@ build distributedlogs-controller jvm image multi archs:
   variables:
     IMAGE_BASE_NAME: stackgres/distributedlogs-controller
 
+build stream jvm image multi archs:
+  extends: .build jvm image multi archs
+  variables:
+    IMAGE_BASE_NAME: stackgres/stream
+
 build operator native image multi archs:
   extends: .build native image multi archs
   variables:
@@ -350,6 +383,11 @@ build distributedlogs-controller native image multi archs:
   variables:
     IMAGE_BASE_NAME: stackgres/distributedlogs-controller
 
+.build stream native image multi archs:
+  extends: .build native image multi archs
+  variables:
+    IMAGE_BASE_NAME: stackgres/stream
+
 build admin-ui image multi archs:
   extends: .build image multi archs
   tags:

.gitlab-ci/image-release.yml

@@ -125,6 +125,13 @@ release distributedlogs-controller jvm image amd64:
     LOCK_FILE: stackgres-distributedlogs-controller-release-jvm-release-image
     IMAGE_MODULE_NAME: distributedlogs-controller-jvm-release-image
 
+release stream jvm image amd64:
+  extends: .release jvm image amd64
+  variables:
+    IMAGE_BASE_NAME: stackgres/stream
+    LOCK_FILE: stackgres-stream-release-jvm-release-image
+    IMAGE_MODULE_NAME: stream-jvm-release-image
+
 release operator native image amd64:
   extends: .release native image amd64
   variables:
@@ -160,6 +167,13 @@ release distributedlogs-controller native image amd64:
     LOCK_FILE: stackgres-distributedlogs-controller-release-native-release-image
     IMAGE_MODULE_NAME: distributedlogs-controller-native-release-image
 
+.release stream native image amd64:
+  extends: .release native image amd64
+  variables:
+    IMAGE_BASE_NAME: stackgres/stream
+    LOCK_FILE: stackgres-stream-release-native-release-image
+    IMAGE_MODULE_NAME: stream-native-release-image
+
 release admin-ui image amd64:
   extends: .release image
   dependencies:
@@ -244,6 +258,13 @@ release distributedlogs-controller jvm image arm64:
     LOCK_FILE: stackgres-distributedlogs-controller-release-jvm-release-image
     IMAGE_MODULE_NAME: distributedlogs-controller-jvm-release-image
 
+release stream jvm image arm64:
+  extends: .release jvm image arm64
+  variables:
+    IMAGE_BASE_NAME: stackgres/stream
+    LOCK_FILE: stackgres-stream-release-jvm-release-image
+    IMAGE_MODULE_NAME: stream-jvm-release-image
+
 release operator native image arm64:
   extends: .release native image arm64
   variables:
@@ -279,6 +300,13 @@ release distributedlogs-controller native image arm64:
     LOCK_FILE: stackgres-distributedlogs-controller-release-native-release-image
     IMAGE_MODULE_NAME: distributedlogs-controller-native-release-image
 
+.release stream native image arm64:
+  extends: .release native image arm64
+  variables:
+    IMAGE_BASE_NAME: stackgres/stream
+    LOCK_FILE: stackgres-stream-release-native-release-image
+    IMAGE_MODULE_NAME: stream-native-release-image
+
 release admin-ui image arm64:
   extends: .release image
   dependencies:
@@ -348,6 +376,11 @@ release distributedlogs-controller jvm image multi archs:
   variables:
     IMAGE_BASE_NAME: stackgres/distributedlogs-controller
 
+release stream jvm image multi archs:
+  extends: .release jvm image multi archs
+  variables:
+    IMAGE_BASE_NAME: stackgres/stream
+
 release operator native image multi archs:
   extends: .release native image multi archs
   variables:
@@ -373,6 +406,11 @@ release distributedlogs-controller native image multi archs:
   variables:
     IMAGE_BASE_NAME: stackgres/distributedlogs-controller
 
+.release stream native image multi archs:
+  extends: .release native image multi archs
+  variables:
+    IMAGE_BASE_NAME: stackgres/stream
+
 release admin-ui image multi archs:
   extends: .release image multi archs
   tags:

.gitlab-ci/native-build.yml

@@ -80,6 +80,13 @@ build distributedlogs-controller native executable amd64:
     MAVEN_EXTRA_ARGS: "-pl .,common,operator-framework,test-util,distributedlogs-controller"
     MODULE: "distributedlogs-controller-native"
 
+.build stream native executable amd64:
+  extends: .build native executable amd64
+  variables:
+    NATIVE_IMAGE_EXTRA_PREFIX: "-stream"
+    MAVEN_EXTRA_ARGS: "-pl .,common,operator-framework,test-util,stream"
+    MODULE: "stream-native"
+
 .build native executable arm64:
   extends: .build native executable
   dependencies:
@@ -127,3 +134,10 @@ build distributedlogs-controller native executable arm64:
     NATIVE_IMAGE_EXTRA_PREFIX: "-distributedlogs-controller"
     MAVEN_EXTRA_ARGS: "-pl .,common,operator-framework,test-util,distributedlogs-controller"
     MODULE: "distributedlogs-controller-native"
+
+.build stream native executable arm64:
+  extends: .build native executable arm64
+  variables:
+    NATIVE_IMAGE_EXTRA_PREFIX: "-stream"
+    MAVEN_EXTRA_ARGS: "-pl .,common,operator-framework,test-util,stream"
+    MODULE: "stream-native"

doc/content/en/06-crd-reference/14-sgshardeddbops/_index.template.md

@@ -28,11 +28,11 @@ kind: SGShardedDbOps
 metadata:
   name: restart
 spec:
- sgShardedCluster: my-cluster
- op: restart
- maxRetries: 1
- restart:
-   mode: InPlace
+  sgShardedCluster: my-cluster
+  op: restart
+  maxRetries: 1
+  restart:
+    mode: InPlace
 ```
 
 {{% include "generated/SGShardedDbOps.md" %}}

doc/content/en/06-crd-reference/15-sgstream/_index.template.md

@@ -0,0 +1,44 @@
+---
+title: SGStream
+weight: 14
+url: /reference/crd/sgstream
+description: Details about SGStream
+showToc: true
+---
+
+___
+**Kind:** SGStream
+
+**listKind:** SGStreamList
+
+**plural:** sgstreams
+
+**singular:** sgstream
+
+**shortNames** sgstr
+___
+
+The `SGStream` custom resource represents a stream of Change Data Capture (CDC) events from a source database to a target service.
+
+**Example:**
+
+```yaml
+apiVersion: stackgres.io/v1
+kind: SGStream
+metadata:
+  name: cloudevent
+spec:
+  source:
+    type: SGCluster
+    sgCluster:
+      name: my-cluster
+  target:
+    type: CloudEvent
+    cloudEvent:
+      binding: http
+      format: json
+      http:
+        url: cloudevent-service
+```
+
+{{% include "generated/SGStream.md" %}}

doc/content/en/08-stackgres-web-api/01-error-types/_index.md

@@ -32,6 +32,7 @@ That means that all error messages follow this structure:
 | [forbidden-configuration-deletion](#forbidden-configuration-deletion) | A resource that the cluster depends on is attempted to be deleted.                                                  |
 | [forbidden-configuration-update](#forbidden-configuration-update)     | A resource that the cluster depends on is attempted to be updated.                                                  |
 | [forbidden-cluster-update](#forbidden-cluster-update)                 | A certain cluster property that must not be updated is attempted to be updated.                                                     |
+| [forbidden-stream-update](#forbidden-stream-update)                   | A certain stream property that must not be updated is attempted to be updated.                                                     |
 | [invalid-storage-class](#invalid-storage-class)                       | The StackGres cluster refers to a storage class that doesn't exist.                                               |
 | [constraint-violation](#constraint-violation)                         | One of the resource properties that is created or updated violates its syntactic rules.                            |
 | [forbidden-authorization](#forbidden-authorization)                   | You don't have permission to access the Kubernetes resource based on the RBAC rules.                                   |