Nidx: Support for S3 endpoint TLS validation option (#3644)

* Nidx: Support for S3 endpoint TLS validation option

* Update helm values/docs

* Fix bool option

* Fix bool option

* simpler

---------

Co-authored-by: Javier Torres <nino@progress.com>

Author: zepellin

charts/nidx/README.md

@@ -23,8 +23,8 @@ nidx chart
 | imagePullPolicy | string | `"IfNotPresent"` | Image pull policy |
 | containerRegistry | string | `"CONTAINER_REGISTRY_TO_REPLACE"` | Container registry (e.g. docker.io/nuclia) |
 | image | string | `"IMAGE_TO_REPLACE"` | Image name (without registry eg. nidx:latest) |
-| env | object | `{}` |  |
-| envFrom | object | `{}` |  |
+| env | object | `{}` | Global environment variables to add to all containers |
+| envFrom | object | `{}` | Global environment variables mount to add to all containers |
 | podAnnotations | object | `{}` | Global pod annotations to add to all pods |
 | podLabels | object | `{"nidxMetrics":"enabled"}` | Global pod labels to add to all pods |
 | maintenance | bool | `false` | Enable maintenance mode, which disables writes while keeping searchers active |

charts/nidx/values.yaml

@@ -13,7 +13,58 @@ containerRegistry: CONTAINER_REGISTRY_TO_REPLACE
 # -- Image name (without registry eg. nidx:latest)
 image: IMAGE_TO_REPLACE
 
+# -- Global environment variables to add to all containers
 env: {}
+  # -- Object store backend type for indexer.
+  # INDEXER__OBJECT_STORE: s3
+  # -- S3 bucket name used by indexer.
+  # INDEXER__BUCKET: ndb-index-env-name
+  # -- S3 region used by indexer.
+  # INDEXER__REGION_NAME: env-name
+  # -- Custom S3 endpoint URL for indexer.
+  # INDEXER__ENDPOINT: https://<url>:9021
+  # -- Enable or disable TLS certificate verification for indexer S3 connections.
+  # INDEXER__ALLOW_INVALID_CERTIFICATES: "true"
+  # -- S3 access key for indexer (if not using IAM or default credentials).
+  # INDEXER__CLIENT_ID: "your-access-key"
+  # -- S3 secret key for indexer (if not using IAM or default credentials).
+  # INDEXER__CLIENT_SECRET: "your-secret-key"
+  # -- GCP service account credentials for indexer (base64-encoded JSON).
+  # INDEXER__BASE64_CREDS: "base64-encoded-json"
+  # -- Azure Blob container URL for indexer.
+  # INDEXER__CONTAINER_URL: "https://account.blob.core.windows.net/container"
+  # -- Maximum number of concurrent indexer object store requests.
+  # INDEXER__MAX_REQUESTS: "10"
+  # -- Request timeout in seconds for indexer object store operations.
+  # INDEXER__TIMEOUT: "120"
+  # -- Azure storage account key for indexer.
+  # INDEXER__ACCOUNT_KEY: "your-azure-account-key"
+  # -- Object store backend type for storage.
+  # STORAGE__OBJECT_STORE: s3
+  # -- S3 bucket name used by storage.
+  # STORAGE__BUCKET: ndb-nidx-env-name
+  # -- S3 region used by storage.
+  # STORAGE__REGION_NAME: env-name
+  # -- Maximum number of concurrent storage object store requests.
+  # STORAGE__MAX_REQUESTS: "10"
+  # -- Request timeout in seconds for storage object store operations.
+  # STORAGE__TIMEOUT: "120"
+  # -- Custom S3 endpoint URL for storage.
+  # STORAGE__ENDPOINT: https://<url>:9021
+  # -- Enable or disable TLS certificate verification for storage S3 connections.
+  # STORAGE__ALLOW_INVALID_CERTIFICATES: "true"
+  # -- S3 access key for storage (if not using IAM or default credentials).
+  # STORAGE__CLIENT_ID: "your-access-key"
+  # -- S3 secret key for storage (if not using IAM or default credentials).
+  # STORAGE__CLIENT_SECRET: "your-secret-key"
+  # -- GCP service account credentials for storage (base64-encoded JSON).
+  # STORAGE__BASE64_CREDS: "base64-encoded-json"
+  # -- Azure Blob container URL for storage.
+  # STORAGE__CONTAINER_URL: "https://account.blob.core.windows.net/container"
+  # -- Azure storage account key for storage.
+  # STORAGE__ACCOUNT_KEY: "your-azure-account-key"
+
+# -- Global environment variables mount to add to all containers
 envFrom: {}
 
 # -- Global pod annotations to add to all pods

nidx/src/settings.rs

@@ -56,6 +56,8 @@ pub enum ObjectStoreKind {
         client_secret: Option<String>,
         region_name: String,
         endpoint: Option<String>,
+        #[serde(default, deserialize_with = "deserialize_bool")]
+        allow_invalid_certificates: Option<bool>,
     },
     Azure {
         container_url: String,
@@ -70,6 +72,12 @@ fn deserialize_u64<'de, D: Deserializer<'de>>(deserializer: D) -> Result<Option<
     ))
 }
 
+fn deserialize_bool<'de, D: Deserializer<'de>>(deserializer: D) -> Result<Option<bool>, D::Error> {
+    Ok(Some(
+        String::deserialize(deserializer)?.parse().expect("Expected a bool"),
+    ))
+}
+
 #[derive(Clone, Deserialize, Debug)]
 pub struct ObjectStoreConfig {
     #[serde(flatten)]
@@ -115,6 +123,7 @@ impl ObjectStoreConfig {
                 client_secret,
                 region_name,
                 endpoint,
+                allow_invalid_certificates,
             } => {
                 let mut builder = AmazonS3Builder::from_env()
                     .with_region(region_name.clone())
@@ -130,8 +139,15 @@ impl ObjectStoreConfig {
                     // This is needed for minio compatibility
                     builder = builder.with_endpoint(endpoint.clone().unwrap()).with_allow_http(true);
                 }
-                if let Some(t) = self.timeout {
-                    builder = builder.with_client_options(ClientOptions::new().with_timeout(Duration::from_secs(t)));
+                if self.timeout.is_some() || allow_invalid_certificates.is_some() {
+                    let mut options = ClientOptions::new();
+                    if let Some(t) = self.timeout {
+                        options = options.with_timeout(Duration::from_secs(t));
+                    }
+                    if let Some(allow_invalid_certificates) = allow_invalid_certificates {
+                        options = options.with_allow_invalid_certificates(*allow_invalid_certificates);
+                    }
+                    builder = builder.with_client_options(options);
                 }
                 Box::new(builder.build().unwrap())
             }
@@ -422,6 +438,8 @@ impl Settings {
 mod tests {
     use std::collections::HashMap;
 
+    use serde_json::json;
+
     use super::*;
 
     #[test]
@@ -442,4 +460,45 @@ mod tests {
             LogMergeSettings::default().min_number_of_segments
         );
     }
+
+    #[test]
+    fn test_s3_allow_invalid_certificates_default_is_none() {
+        let raw = json!({
+            "object_store": "s3",
+            "bucket": "bucket",
+            "region_name": "us-east-1"
+        });
+        let config: ObjectStoreConfig = serde_json::from_value(raw).unwrap();
+
+        match config.kind {
+            ObjectStoreKind::S3 {
+                allow_invalid_certificates,
+                ..
+            } => {
+                assert_eq!(allow_invalid_certificates, None);
+            }
+            _ => panic!("Expected s3 object store kind"),
+        }
+    }
+
+    #[test]
+    fn test_s3_allow_invalid_certificates_enabled() {
+        let raw = json!({
+            "object_store": "s3",
+            "bucket": "bucket",
+            "region_name": "us-east-1",
+            "allow_invalid_certificates": "true"
+        });
+        let config: ObjectStoreConfig = serde_json::from_value(raw).unwrap();
+
+        match config.kind {
+            ObjectStoreKind::S3 {
+                allow_invalid_certificates,
+                ..
+            } => {
+                assert_eq!(allow_invalid_certificates, Some(true));
+            }
+            _ => panic!("Expected s3 object store kind"),
+        }
+    }
 }