test: Test creating snapshots in S3

siegfriedweber · siegfriedweber · commit 2d1699ed57c0 · 2025-10-21T18:05:55.000+02:00
diff --git a/tests/templates/kuttl/snapshot-s3/00-patch-ns.yaml b/tests/templates/kuttl/snapshot-s3/00-patch-ns.yaml
@@ -0,0 +1,15 @@
+# see https://github.com/stackabletech/issues/issues/566
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |
+      kubectl patch namespace $NAMESPACE --patch='
+        {
+          "metadata": {
+            "labels": {
+              "pod-security.kubernetes.io/enforce": "privileged"
+            }
+          }
+        }'
+    timeout: 120
diff --git a/tests/templates/kuttl/snapshot-s3/01-rbac.yaml b/tests/templates/kuttl/snapshot-s3/01-rbac.yaml
@@ -0,0 +1,31 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: test-service-account
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: test-role
+rules:
+  - apiGroups:
+      - security.openshift.io
+    resources:
+      - securitycontextconstraints
+    resourceNames:
+      - privileged
+    verbs:
+      - use
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: test-role-binding
+subjects:
+  - kind: ServiceAccount
+    name: test-service-account
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: test-role
diff --git a/tests/templates/kuttl/snapshot-s3/02-assert.yaml.j2 b/tests/templates/kuttl/snapshot-s3/02-assert.yaml.j2
@@ -0,0 +1,10 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: vector-aggregator-discovery
+{% endif %}
diff --git a/tests/templates/kuttl/snapshot-s3/02-install-vector-aggregator-discovery-config-map.yaml.j2 b/tests/templates/kuttl/snapshot-s3/02-install-vector-aggregator-discovery-config-map.yaml.j2
@@ -0,0 +1,9 @@
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: vector-aggregator-discovery
+data:
+  ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }}
+{% endif %}
diff --git a/tests/templates/kuttl/snapshot-s3/10-assert.yaml b/tests/templates/kuttl/snapshot-s3/10-assert.yaml
@@ -0,0 +1,8 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: s3-credentials
diff --git a/tests/templates/kuttl/snapshot-s3/10-install-s3-credentials-secret.yaml b/tests/templates/kuttl/snapshot-s3/10-install-s3-credentials-secret.yaml
@@ -0,0 +1,8 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: s3-credentials
+stringData:
+  s3.client.default.access_key: opensearchAccessKey
+  s3.client.default.secret_key: opensearchSecretKey
diff --git a/tests/templates/kuttl/snapshot-s3/11-assert.yaml b/tests/templates/kuttl/snapshot-s3/11-assert.yaml
@@ -0,0 +1,12 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 600
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: minio
+status:
+  readyReplicas: 1
+  replicas: 1
diff --git a/tests/templates/kuttl/snapshot-s3/11-install-minio.yaml.j2 b/tests/templates/kuttl/snapshot-s3/11-install-minio.yaml.j2
@@ -0,0 +1,10 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: >-
+      helm install minio oci://registry-1.docker.io/bitnamicharts/minio
+      --version 17.0.21
+      --values 11_minio-values.yaml
+      --namespace $NAMESPACE
+    timeout: 240
diff --git a/tests/templates/kuttl/snapshot-s3/11_minio-values.yaml.j2 b/tests/templates/kuttl/snapshot-s3/11_minio-values.yaml.j2
@@ -0,0 +1,60 @@
+---
+global:
+  security:
+    allowInsecureImages: true
+
+image:
+  repository: bitnamilegacy/minio
+
+defaultInitContainers:
+  volumePermissions:
+    image:
+      repository: bitnamilegacy/os-shell
+
+console:
+  image:
+    repository: bitnamilegacy/minio-object-browser
+
+persistence:
+  enabled: false # "false" means, that an emptyDir is used instead of a persistentVolumeClaim
+  size: 64Mi
+
+provisioning:
+  enabled: true
+  buckets:
+    - name: opensearch
+  users:
+    - username: integrationtest
+      password: integrationtest
+      policies:
+        - readwrite
+        - consoleAdmin
+        - diagnostics
+  extraCommands:
+    - >-
+      mc admin user svcacct add
+      --access-key "$(cat /tmp/s3-credentials/s3.client.default.access_key)"
+      --secret-key "$(cat /tmp/s3-credentials/s3.client.default.secret_key)"
+      provisioning integrationtest
+  extraVolumeMounts:
+    - name: s3-credentials
+      mountPath: /tmp/s3-credentials
+  extraVolumes:
+    - name: s3-credentials
+      secret:
+        secretName: s3-credentials
+  resources:
+    requests:
+      memory: 1Gi
+      cpu: 512m
+    limits:
+      memory: 1Gi
+      cpu: "1"
+
+resources:
+  requests:
+    memory: 1Gi
+    cpu: 512m
+  limits:
+    memory: 1Gi
+    cpu: "1"
diff --git a/tests/templates/kuttl/snapshot-s3/20-assert.yaml.j2 b/tests/templates/kuttl/snapshot-s3/20-assert.yaml.j2
@@ -0,0 +1,12 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 600
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: opensearch-nodes-default
+status:
+  readyReplicas: 1
+  replicas: 1
diff --git a/tests/templates/kuttl/snapshot-s3/20-install-opensearch.yaml.j2 b/tests/templates/kuttl/snapshot-s3/20-install-opensearch.yaml.j2
@@ -0,0 +1,212 @@
+---
+apiVersion: opensearch.stackable.tech/v1alpha1
+kind: OpenSearchCluster
+metadata:
+  name: opensearch
+spec:
+  image:
+{% if test_scenario['values']['opensearch'].find(",") > 0 %}
+    custom: "{{ test_scenario['values']['opensearch'].split(',')[1] }}"
+    productVersion: "{{ test_scenario['values']['opensearch'].split(',')[0] }}"
+{% else %}
+    productVersion: "{{ test_scenario['values']['opensearch'] }}"
+{% endif %}
+    pullPolicy: IfNotPresent
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+  clusterConfig:
+    vectorAggregatorConfigMapName: vector-aggregator-discovery
+{% endif %}
+  nodes:
+    config:
+      logging:
+        enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
+      listenerClass: external-stable
+    roleGroups:
+      default:
+        replicas: 1
+    configOverrides:
+      opensearch.yml:
+        # Disable memory mapping in this test; If memory mapping were activated, the kernel setting
+        # vm.max_map_count would have to be increased to 262144 on the node.
+        node.store.allow_mmap: "false"
+        # Disable the disk allocation decider in this test; Otherwise the test depends on the disk
+        # usage of the node and if the relative watermark set in
+        # `cluster.routing.allocation.disk.watermark.high` is reached then the security index could
+        # not be created even if enough disk space would be available.
+        cluster.routing.allocation.disk.threshold_enabled: "false"
+        plugins.security.allow_default_init_securityindex: "true"
+        plugins.security.ssl.transport.enabled: "true"
+        plugins.security.ssl.transport.pemcert_filepath: /stackable/opensearch/config/tls/tls.crt
+        plugins.security.ssl.transport.pemkey_filepath: /stackable/opensearch/config/tls/tls.key
+        plugins.security.ssl.transport.pemtrustedcas_filepath: /stackable/opensearch/config/tls/ca.crt
+        plugins.security.ssl.http.enabled: "true"
+        plugins.security.ssl.http.pemcert_filepath: /stackable/opensearch/config/tls/tls.crt
+        plugins.security.ssl.http.pemkey_filepath: /stackable/opensearch/config/tls/tls.key
+        plugins.security.ssl.http.pemtrustedcas_filepath: /stackable/opensearch/config/tls/ca.crt
+        s3.client.default.endpoint: http://minio:9000/
+        s3.client.default.protocol: http
+        s3.client.default.region: unused # but required
+        s3.client.default.path_style_access: "true"
+    podOverrides:
+      spec:
+        initContainers:
+          - name: init-keystore
+            # TODO Use somehow the image of the opensearch container
+            image: oci.stackable.tech/sdp/opensearch:3.1.0-stackable0.0.0-dev
+            command:
+              - /bin/bash
+              - -euxo
+              - pipefail
+              - -c
+            args:
+              - |
+                bin/opensearch-keystore create
+
+                for i in keystore-secrets/*; do
+                  key=$(basename $i)
+                  bin/opensearch-keystore add-file "$key" "$i"
+                done
+
+                cp --archive config/opensearch.keystore initialized-keystore
+            volumeMounts:
+              - name: keystore
+                mountPath: /stackable/opensearch/initialized-keystore
+                readOnly: false
+              - name: keystore-secrets
+                mountPath: /stackable/opensearch/keystore-secrets
+                readOnly: true
+        containers:
+          - name: opensearch
+            volumeMounts:
+              - name: security-config
+                mountPath: /stackable/opensearch/config/opensearch-security
+                readOnly: true
+              - name: tls
+                mountPath: /stackable/opensearch/config/tls
+                readOnly: true
+              - name: keystore
+                mountPath: /stackable/opensearch/config/opensearch.keystore
+                subPath: opensearch.keystore
+                readOnly: true
+        volumes:
+          - name: keystore
+            emptyDir:
+              sizeLimit: 1Mi
+          - name: keystore-secrets
+            secret:
+              secretName: s3-credentials
+              defaultMode: 0o660
+          - name: security-config
+            secret:
+              secretName: opensearch-security-config
+              defaultMode: 0o660
+          - name: tls
+            ephemeral:
+              volumeClaimTemplate:
+                metadata:
+                  annotations:
+                    secrets.stackable.tech/scope: node,pod,service=opensearch,service=opensearch-nodes-default-headless
+                    secrets.stackable.tech/class: tls
+                spec:
+                  storageClassName: secrets.stackable.tech
+                  accessModes:
+                    - ReadWriteOnce
+                  resources:
+                    requests:
+                      storage: "1"
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: opensearch-security-config
+stringData:
+  action_groups.yml: |
+    ---
+    _meta:
+      type: actiongroups
+      config_version: 2
+  allowlist.yml: |
+    ---
+    _meta:
+      type: allowlist
+      config_version: 2
+
+    config:
+      enabled: false
+  audit.yml: |
+    ---
+    _meta:
+      type: audit
+      config_version: 2
+
+    config:
+      enabled: false
+  config.yml: |
+    ---
+    _meta:
+      type: config
+      config_version: 2
+
+    config:
+      dynamic:
+        authc:
+          basic_internal_auth_domain:
+            description: Authenticate via HTTP Basic against internal users database
+            http_enabled: true
+            transport_enabled: true
+            order: 1
+            http_authenticator:
+              type: basic
+              challenge: true
+            authentication_backend:
+              type: intern
+        authz: {}
+  internal_users.yml: |
+    ---
+    # The hash value is a bcrypt hash and can be generated with plugin/tools/hash.sh
+
+    _meta:
+      type: internalusers
+      config_version: 2
+
+    admin:
+      hash: $2y$10$xRtHZFJ9QhG9GcYhRpAGpufCZYsk//nxsuel5URh0GWEBgmiI4Q/e
+      reserved: true
+      backend_roles:
+        - admin
+      description: OpenSearch admin user
+
+    kibanaserver:
+      hash: $2y$10$vPgQ/6ilKDM5utawBqxoR.7euhVQ0qeGl8mPTeKhmFT475WUDrfQS
+      reserved: true
+      description: OpenSearch Dashboards user
+  nodes_dn.yml: |
+    ---
+    _meta:
+      type: nodesdn
+      config_version: 2
+  roles.yml: |
+    ---
+    _meta:
+      type: roles
+      config_version: 2
+  roles_mapping.yml: |
+    ---
+    _meta:
+      type: rolesmapping
+      config_version: 2
+
+    all_access:
+      reserved: false
+      backend_roles:
+        - admin
+
+    kibana_server:
+      reserved: true
+      users:
+        - kibanaserver
+  tenants.yml: |
+    ---
+    _meta:
+      type: tenants
+      config_version: 2
diff --git a/tests/templates/kuttl/snapshot-s3/30-assert.yaml b/tests/templates/kuttl/snapshot-s3/30-assert.yaml
diff --git a/tests/templates/kuttl/snapshot-s3/30-test-snapshot.yaml b/tests/templates/kuttl/snapshot-s3/30-test-snapshot.yaml
diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml
