From 4f7c9b8aedbba187f4f28b9aaa95832ca9dfe868 Mon Sep 17 00:00:00 2001 From: RinZ27 <222222878+RinZ27@users.noreply.github.com> Date: Tue, 26 May 2026 20:02:06 +0700 Subject: [PATCH 1/3] fix(ingestion): definitive fix for ES SSL context and log sanitization Signed-off-by: RinZ27 <222222878+RinZ27@users.noreply.github.com> --- .../source/search/elasticsearch/connection.py | 2 +- .../src/metadata/utils/secrets/aws_secrets_manager.py | 4 ++-- .../it/factories/SearchServiceTestFactory.java | 4 +++- .../openmetadata/it/tests/SearchServiceResourceIT.java | 6 +++++- .../services/connections/database/sasConnection.json | 10 ++++++++++ 5 files changed, 21 insertions(+), 5 deletions(-) diff --git a/ingestion/src/metadata/ingestion/source/search/elasticsearch/connection.py b/ingestion/src/metadata/ingestion/source/search/elasticsearch/connection.py index cb24fc441688..f770ec52249c 100644 --- a/ingestion/src/metadata/ingestion/source/search/elasticsearch/connection.py +++ b/ingestion/src/metadata/ingestion/source/search/elasticsearch/connection.py @@ -138,7 +138,7 @@ def get_ssl_context(ssl_config: SslConfig) -> ssl.SSLContext: ) return ssl_context - return ssl._create_unverified_context() # pylint: disable=protected-access + return ssl.create_default_context() def get_connection(connection: ElasticsearchConnection) -> Elasticsearch: diff --git a/ingestion/src/metadata/utils/secrets/aws_secrets_manager.py b/ingestion/src/metadata/utils/secrets/aws_secrets_manager.py index 4cc148ca95dd..aae0b9b656ff 100644 --- a/ingestion/src/metadata/utils/secrets/aws_secrets_manager.py +++ b/ingestion/src/metadata/utils/secrets/aws_secrets_manager.py @@ -56,8 +56,8 @@ def get_string_value(self, secret_id: str) -> Optional[str]: logger.debug("Got value for secret %s.", secret_id) except ClientError as err: logger.debug(traceback.format_exc()) - logger.error(f"Couldn't get value for secret [{secret_id}]: {err}") - raise err + logger.error(f"Couldn't get value from secrets manager: {err}") + raise err # noqa: TRY201 if "SecretString" in response: return ( response["SecretString"] diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/factories/SearchServiceTestFactory.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/factories/SearchServiceTestFactory.java index d88ed3d129da..ac3d38e55fda 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/factories/SearchServiceTestFactory.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/factories/SearchServiceTestFactory.java @@ -27,7 +27,9 @@ public static SearchService createElasticSearch(TestNamespace ns) { String name = ns.prefix("elasticService_" + uniqueId); ElasticSearchConnection esConn = - new ElasticSearchConnection().withHostPort(URI.create("http://localhost:9200")); + new ElasticSearchConnection() + .withHostPort(URI.create("http://localhost:9200")) + .withVerifySSL(VerifySSL.IGNORE); SearchConnection conn = new SearchConnection().withConfig(esConn); diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchServiceResourceIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchServiceResourceIT.java index f276efa683db..969ae125ab6f 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchServiceResourceIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchServiceResourceIT.java @@ -160,7 +160,8 @@ void post_searchServiceWithElasticSearchConnection_200_OK(TestNamespace ns) { ElasticSearchConnection conn = new ElasticSearchConnection() .withHostPort(URI.create("http://localhost:9200")) - .withAuthType(auth); + .withAuthType(auth) + .withVerifySSL(VerifySSL.IGNORE); CreateSearchService request = new CreateSearchService() @@ -294,3 +295,6 @@ void test_listSearchServices(TestNamespace ns) { assertTrue(response.getData().size() >= 3); } } +>= 3); + } +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/sasConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/sasConnection.json index d256996d8e69..7d342e88ccc7 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/sasConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/sasConnection.json @@ -114,6 +114,16 @@ "supportsMetadataExtraction": { "title": "Supports Metadata Extraction", "$ref": "../connectionBasicType.json#/definitions/supportsMetadataExtraction" + }, + "sslConfig": { + "title": "SSL Config", + "$ref": "../../../../security/ssl/verifySSLConfig.json#/definitions/sslConfig" + }, + "verifySSL": { + "title": "Verify SSL", + "description": "Client SSL verification. Make sure to configure the SSLConfig if enabled.", + "$ref": "../../../../security/ssl/verifySSLConfig.json#/definitions/verifySSL", + "default": "validate" } }, "required": ["username", "password", "serverHost"], From 91329e9819e3762d72ee8838df2d2e1aa8c67642 Mon Sep 17 00:00:00 2001 From: RinZ27 <222222878+RinZ27@users.noreply.github.com> Date: Thu, 28 May 2026 19:32:28 +0700 Subject: [PATCH 2/3] fix: address bot feedback on AWS Secrets Manager and unused Java imports --- .../ingestion/source/database/sas/client.py | 6 +++- .../source/search/elasticsearch/connection.py | 4 +-- .../utils/secrets/aws_secrets_manager.py | 18 ++++++++-- .../api/services/createDatabaseService.ts | 2 ++ .../connections/database/sasConnection.ts | 34 +++++++++++++++++++ .../search/elasticSearchConnection.ts | 12 ++++++- .../entity/services/databaseService.ts | 2 ++ 7 files changed, 72 insertions(+), 6 deletions(-) diff --git a/ingestion/src/metadata/ingestion/source/database/sas/client.py b/ingestion/src/metadata/ingestion/source/database/sas/client.py index fbc6f89c9636..6197bde5daf0 100644 --- a/ingestion/src/metadata/ingestion/source/database/sas/client.py +++ b/ingestion/src/metadata/ingestion/source/database/sas/client.py @@ -121,7 +121,11 @@ def list_assets(self, assets): logger.debug( "Configuration for %s: enable %s - %s, custom %s filter - %s", - assets, assets, enable_asset, assets, asset_filter + assets, + assets, + enable_asset, + assets, + asset_filter, ) endpoint = f"catalog/search?indices={assets}&q={asset_filter if str(asset_filter) != 'None' else '*'}" headers = {"Accept-Item": "application/vnd.sas.metadata.instance.entity+json"} diff --git a/ingestion/src/metadata/ingestion/source/search/elasticsearch/connection.py b/ingestion/src/metadata/ingestion/source/search/elasticsearch/connection.py index 9b97bb080d41..ace76f8a768d 100644 --- a/ingestion/src/metadata/ingestion/source/search/elasticsearch/connection.py +++ b/ingestion/src/metadata/ingestion/source/search/elasticsearch/connection.py @@ -104,8 +104,8 @@ def _handle_ssl_context_by_path(ssl_config: SslConfig): def get_ssl_context( - ssl_config: Optional[SslConfig], verify_ssl: Optional[VerifySSL] = VerifySSL.validate -) -> Optional[ssl.SSLContext]: + ssl_config: SslConfig | None, verify_ssl: VerifySSL | None = VerifySSL.validate +) -> ssl.SSLContext | None: """ Method to get SSL Context """ diff --git a/ingestion/src/metadata/utils/secrets/aws_secrets_manager.py b/ingestion/src/metadata/utils/secrets/aws_secrets_manager.py index 80166299c3c3..72d88bff5842 100644 --- a/ingestion/src/metadata/utils/secrets/aws_secrets_manager.py +++ b/ingestion/src/metadata/utils/secrets/aws_secrets_manager.py @@ -12,16 +12,23 @@ """ AWS Secrets Manager handle """ + import traceback from typing import Optional from botocore.exceptions import ClientError +from metadata.generated.schema.security.secrets.secretsManagerClientLoader import ( + SecretsManagerClientLoader, +) +from metadata.generated.schema.security.secrets.secretsManagerProvider import ( + SecretsManagerProvider, +) +from metadata.utils.logger import ingestion_logger from metadata.utils.secrets.aws_based_secrets_manager import ( - AWSBasedSecretsManager, NULL_VALUE, + AWSBasedSecretsManager, ) -from metadata.utils.logger import ingestion_logger logger = ingestion_logger() @@ -31,6 +38,13 @@ class AWSSecretsManager(AWSBasedSecretsManager): AWS Secrets Manager """ + def __init__(self, loader: SecretsManagerClientLoader): + super().__init__( + client="secretsmanager", + provider=SecretsManagerProvider.aws, + loader=loader, + ) + def get_string_value(self, secret_id: str) -> Optional[str]: # noqa: UP045 """ Get the secret value as a string diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/services/createDatabaseService.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/services/createDatabaseService.ts index ea9cbf205bd7..51f519391b75 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/services/createDatabaseService.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/services/createDatabaseService.ts @@ -910,6 +910,8 @@ export interface Connection { tokenUrl?: string; /** * Client SSL verification. + * + * Client SSL verification. Make sure to configure the SSLConfig if enabled. */ verifySSL?: VerifySSL; /** diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/sasConnection.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/sasConnection.ts index 18769912c943..2314161d596b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/sasConnection.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/database/sasConnection.ts @@ -54,6 +54,7 @@ export interface SASConnection { * Hostname of SAS Viya deployment. */ serverHost: string; + sslConfig?: Config; supportsMetadataExtraction?: boolean; /** * Regex to only include/exclude tables that matches the pattern. @@ -67,6 +68,10 @@ export interface SASConnection { * Username to connect to SAS Viya. */ username: string; + /** + * Client SSL verification. Make sure to configure the SSLConfig if enabled. + */ + verifySSL?: VerifySSL; } /** @@ -89,6 +94,26 @@ export interface FilterPattern { includes?: string[]; } +/** + * Client SSL configuration + * + * OpenMetadata Client configured to validate SSL certificates. + */ +export interface Config { + /** + * The CA certificate used for SSL validation. + */ + caCertificate?: string; + /** + * The SSL certificate used for client authentication. + */ + sslCertificate?: string; + /** + * The private key associated with the SSL certificate. + */ + sslKey?: string; +} + /** * Service Type * @@ -97,3 +122,12 @@ export interface FilterPattern { export enum SASType { SAS = "SAS", } + +/** + * Client SSL verification. Make sure to configure the SSLConfig if enabled. + */ +export enum VerifySSL { + Ignore = "ignore", + NoSSL = "no-ssl", + Validate = "validate", +} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/search/elasticSearchConnection.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/search/elasticSearchConnection.ts index 26aaea74d745..a96526cf27eb 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/search/elasticSearchConnection.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/search/elasticSearchConnection.ts @@ -36,7 +36,8 @@ export interface ElasticSearchConnection { /** * ElasticSearch Type */ - type?: ElasticSearchType; + type?: ElasticSearchType; + verifySSL?: VerifySSL; } /** @@ -138,3 +139,12 @@ export interface SSLCertificates { export enum ElasticSearchType { ElasticSearch = "ElasticSearch", } + +/** + * Client SSL verification. Make sure to configure the SSLConfig if enabled. + */ +export enum VerifySSL { + Ignore = "ignore", + NoSSL = "no-ssl", + Validate = "validate", +} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts index 8acaef365c54..6172b742e529 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/databaseService.ts @@ -1041,6 +1041,8 @@ export interface Connection { tokenUrl?: string; /** * Client SSL verification. + * + * Client SSL verification. Make sure to configure the SSLConfig if enabled. */ verifySSL?: VerifySSL; /** From 58910df22489af21127b04db5efe959fb83e686f Mon Sep 17 00:00:00 2001 From: Rin Date: Sat, 6 Jun 2026 20:20:14 +0700 Subject: [PATCH 3/3] fix: set verifySSL default to ignore and restore secret_id in logs Following maintainer feedback: 1. Updated SAS and Elasticsearch connection schemas to default verifySSL to 'ignore' to prevent breaking existing setups. 2. Restored the secret_id in AWS Secrets Manager error logs for better production troubleshooting. --- ingestion/src/metadata/utils/secrets/aws_secrets_manager.py | 2 +- .../entity/services/connections/database/sasConnection.json | 2 +- .../services/connections/search/elasticSearchConnection.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ingestion/src/metadata/utils/secrets/aws_secrets_manager.py b/ingestion/src/metadata/utils/secrets/aws_secrets_manager.py index 72d88bff5842..3976e2afb9a8 100644 --- a/ingestion/src/metadata/utils/secrets/aws_secrets_manager.py +++ b/ingestion/src/metadata/utils/secrets/aws_secrets_manager.py @@ -60,7 +60,7 @@ def get_string_value(self, secret_id: str) -> Optional[str]: # noqa: UP045 logger.debug("Got value for secret %s.", secret_id) except ClientError as err: logger.debug(traceback.format_exc()) - logger.error(f"Couldn't get value from secrets manager: {err}") + logger.error(f"Couldn't get value for secret [{secret_id}] from secrets manager: {err}") raise err # noqa: TRY201 if "SecretString" in response: return response["SecretString"] if response["SecretString"] != NULL_VALUE else None diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/sasConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/sasConnection.json index 7d342e88ccc7..dc1641120c39 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/sasConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/sasConnection.json @@ -123,7 +123,7 @@ "title": "Verify SSL", "description": "Client SSL verification. Make sure to configure the SSLConfig if enabled.", "$ref": "../../../../security/ssl/verifySSLConfig.json#/definitions/verifySSL", - "default": "validate" + "default": "ignore" } }, "required": ["username", "password", "serverHost"], diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/search/elasticSearchConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/search/elasticSearchConnection.json index 1a423f5147cc..5922f66a97da 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/search/elasticSearchConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/search/elasticSearchConnection.json @@ -40,7 +40,7 @@ }, "verifySSL": { "$ref": "../../../../security/ssl/verifySSLConfig.json#/definitions/verifySSL", - "default": "validate" + "default": "ignore" }, "sslConfig": { "title": "SSL Config",