Merge remote-tracking branch 'origin/main' into feat/snapshot-s3

Author: labrenbe

CHANGELOG.md

@@ -6,9 +6,11 @@ All notable changes to this project will be documented in this file.
 
 ### Added
 
+- Allow the configuration of TLS for the HTTP and TRANSPORT ports with the operator ([#55]).
 - Add the role group as a node attribute ([#63]).
 - Allow adding entries to the OpenSearch keystore ([#76]).
 
+[#55]: https://github.com/stackabletech/opensearch-operator/pull/55
 [#63]: https://github.com/stackabletech/opensearch-operator/pull/63
 [#76]: https://github.com/stackabletech/opensearch-operator/pull/76
 

deploy/helm/opensearch-operator/crds/crds.yaml

@@ -31,6 +31,9 @@ spec:
                 clusterConfig:
                   default:
                     keystore: []
+                    tls:
+                      internalSecretClass: tls
+                      serverSecretClass: tls
                   description: Configuration that applies to all roles and role groups
                   properties:
                     keystore:
@@ -67,6 +70,36 @@ spec:
                           - secretKeyRef
                         type: object
                       type: array
+                    tls:
+                      default:
+                        internalSecretClass: tls
+                        serverSecretClass: tls
+                      description: TLS configuration options for the server (REST API) and internal communication (transport).
+                      properties:
+                        internalSecretClass:
+                          default: tls
+                          description: |-
+                            Only affects internal communication (transport). Used for mutual verification between OpenSearch nodes.
+                            This setting controls:
+                            - Which cert the servers should use to authenticate themselves against other servers
+                            - Which ca.crt to use when validating the other server
+                          maxLength: 253
+                          minLength: 1
+                          pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$
+                          type: string
+                        serverSecretClass:
+                          default: tls
+                          description: |-
+                            Only affects client connections to the REST API.
+                            This setting controls:
+                            - If TLS encryption is used at all
+                            - Which cert the servers should use to authenticate themselves against the client
+                          maxLength: 253
+                          minLength: 1
+                          nullable: true
+                          pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$
+                          type: string
+                      type: object
                     vectorAggregatorConfigMapName:
                       description: |-
                         Name of the Vector aggregator [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery).
@@ -340,6 +373,14 @@ spec:
                             type: string
                           nullable: true
                           type: array
+                        requestedSecretLifetime:
+                          description: |-
+                            Request secret (currently only autoTls certificates) lifetime from the secret operator, e.g. `7d`, or `30d`.
+                            This can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate.
+
+                            Defaults to 1d.
+                          nullable: true
+                          type: string
                         resources:
                           default:
                             cpu:
@@ -689,6 +730,14 @@ spec:
                                   type: string
                                 nullable: true
                                 type: array
+                              requestedSecretLifetime:
+                                description: |-
+                                  Request secret (currently only autoTls certificates) lifetime from the secret operator, e.g. `7d`, or `30d`.
+                                  This can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate.
+
+                                  Defaults to 1d.
+                                nullable: true
+                                type: string
                               resources:
                                 default:
                                   cpu:

docs/modules/opensearch/examples/getting_started/opensearch.yaml

@@ -14,14 +14,6 @@ spec:
       opensearch.yml:
         plugins.security.allow_default_init_securityindex: "true"
         plugins.security.restapi.roles_enabled: all_access
-        plugins.security.ssl.transport.enabled: "true"
-        plugins.security.ssl.transport.pemcert_filepath: /stackable/opensearch/config/tls/tls.crt
-        plugins.security.ssl.transport.pemkey_filepath: /stackable/opensearch/config/tls/tls.key
-        plugins.security.ssl.transport.pemtrustedcas_filepath: /stackable/opensearch/config/tls/ca.crt
-        plugins.security.ssl.http.enabled: "true"
-        plugins.security.ssl.http.pemcert_filepath: /stackable/opensearch/config/tls/tls.crt
-        plugins.security.ssl.http.pemkey_filepath: /stackable/opensearch/config/tls/tls.key
-        plugins.security.ssl.http.pemtrustedcas_filepath: /stackable/opensearch/config/tls/ca.crt
     podOverrides:
       spec:
         containers:
@@ -30,25 +22,8 @@ spec:
               - name: security-config
                 mountPath: /stackable/opensearch/config/opensearch-security
                 readOnly: true
-              - name: tls
-                mountPath: /stackable/opensearch/config/tls
-                readOnly: true
         volumes:
           - name: security-config
             secret:
               secretName: opensearch-security-config
               defaultMode: 0o660
-          - name: tls
-            ephemeral:
-              volumeClaimTemplate:
-                metadata:
-                  annotations:
-                    secrets.stackable.tech/class: tls
-                    secrets.stackable.tech/scope: node,pod,service=simple-opensearch,service=simple-opensearch-nodes-default,service=simple-opensearch-nodes-default-headless
-                spec:
-                  storageClassName: secrets.stackable.tech
-                  accessModes:
-                    - ReadWriteOnce
-                  resources:
-                    requests:
-                      storage: "1"

docs/modules/opensearch/pages/usage-guide/security.adoc

@@ -0,0 +1,40 @@
+= Security
+:description: Configure TLS encryption for OpenSearch with the Stackable Operator.
+
+== TLS
+
+The internal and client communication at the REST API can be encrypted with TLS.
+This requires the xref:secret-operator:index.adoc[Secret Operator] to be running in the Kubernetes cluster providing certificates.
+The used certificates can be changed in a cluster-wide config and are configured using xref:secret-operator:secretclass.adoc[SecretClasses].
+TLS encryption on the REST API may be disabled, while it is always enabled for the internal communication between nodes using the `transport` port.
+
+[source,yaml]
+----
+---
+apiVersion: opensearch.stackable.tech/v1alpha1
+kind: OpenSearchCluster
+metadata:
+  name: opensearch
+spec:
+  image:
+    productVersion: 3.1.0
+  clusterConfig:
+    tls:
+      serverSecretClass: tls # <1>
+      internalSecretClass: opensearch-internal-tls # <2>
+  nodes:
+    config:
+      requestedSecretLifetime: 7d # <3>
+    roleGroups:
+      default:
+        replicas: 3
+----
+<1> The `spec.clusterConfig.tls.serverSecretClass` refers to the client-to-server encryption at the REST API.
+Defaults to the `tls` SecretClass and can be disabled by setting `serverSecretClass` to `null`.
+<2> The `spec.clusterConfig.tls.internalSecretClass` refers to the internal encryption between OpenSearch nodes using mTLS (transport).
+Defaults to the `tls` SecretClass and can't be disabled.
+<3> The lifetime for autoTls certificates generated by the secret operator.
+Only a lifetime up to the `maxCertificateLifetime` setting in the SecretClass is applied.
+
+Important: The operator sets the configuration `plugins.security.nodes_dn` to `["CN=generated certificate for pod"]` which provides weak authentication between nodes.
+If you want to increase security and use certificates which identify the OpenSearch nodes specifically, you must also adapt the `plugins.security.nodes_dn` setting via configOverrides.

docs/modules/opensearch/partials/nav.adoc

@@ -11,6 +11,7 @@
 ** xref:opensearch:usage-guide/opensearch-dashboards.adoc[]
 ** xref:opensearch:usage-guide/scaling.adoc[]
 ** xref:opensearch:usage-guide/keystore.adoc[]
+** xref:opensearch:usage-guide/security.adoc[]
 ** xref:opensearch:usage-guide/operations/index.adoc[]
 *** xref:opensearch:usage-guide/operations/cluster-operations.adoc[]
 *** xref:opensearch:usage-guide/operations/pod-placement.adoc[]

rust/operator-binary/src/controller.rs

@@ -134,6 +134,7 @@ pub struct ValidatedOpenSearchConfig {
     pub listener_class: ListenerClassName,
     pub logging: ValidatedLogging,
     pub node_roles: NodeRoles,
+    pub requested_secret_lifetime: Duration,
     pub resources: OpenSearchNodeResources,
     pub termination_grace_period_seconds: i64,
 }
@@ -169,6 +170,7 @@ pub struct ValidatedCluster {
     pub uid: Uid,
     pub role_config: GenericRoleConfig,
     pub role_group_configs: BTreeMap<RoleGroupName, OpenSearchRoleGroupConfig>,
+    pub tls_config: v1alpha1::OpenSearchTls,
     pub keystores: Vec<v1alpha1::OpenSearchKeystore>,
 }
 
@@ -182,6 +184,7 @@ impl ValidatedCluster {
         uid: impl Into<Uid>,
         role_config: GenericRoleConfig,
         role_group_configs: BTreeMap<RoleGroupName, OpenSearchRoleGroupConfig>,
+        tls_config: v1alpha1::OpenSearchTls,
         keystores: Vec<v1alpha1::OpenSearchKeystore>,
     ) -> Self {
         let uid = uid.into();
@@ -199,6 +202,7 @@ impl ValidatedCluster {
             uid,
             role_config,
             role_group_configs,
+            tls_config,
             keystores,
         }
     }
@@ -379,6 +383,7 @@ mod tests {
         kvp::LabelValue,
         product_logging::spec::AutomaticContainerLogConfig,
         role_utils::GenericRoleConfig,
+        shared::time::Duration,
     };
     use uuid::uuid;
 
@@ -504,6 +509,7 @@ mod tests {
                 ),
             ]
             .into(),
+            v1alpha1::OpenSearchTls::default(),
             vec![],
         )
     }
@@ -524,6 +530,8 @@ mod tests {
                     vector_container: None,
                 },
                 node_roles: NodeRoles(node_roles.to_vec()),
+                requested_secret_lifetime: Duration::from_str("1d")
+                    .expect("should be a valid duration"),
                 resources: OpenSearchNodeResources::default(),
                 termination_grace_period_seconds: 120,
             },

rust/operator-binary/src/controller/build.rs

@@ -68,6 +68,7 @@ mod tests {
         kvp::LabelValue,
         product_logging::spec::AutomaticContainerLogConfig,
         role_utils::GenericRoleConfig,
+        shared::time::Duration,
     };
     use uuid::uuid;
 
@@ -197,6 +198,7 @@ mod tests {
                 ),
             ]
             .into(),
+            v1alpha1::OpenSearchTls::default(),
             vec![],
         )
     }
@@ -217,6 +219,8 @@ mod tests {
                     vector_container: None,
                 },
                 node_roles: NodeRoles(node_roles.to_vec()),
+                requested_secret_lifetime: Duration::from_str("1d")
+                    .expect("should be a valid duration"),
                 resources: OpenSearchNodeResources::default(),
                 termination_grace_period_seconds: 120,
             },