refactor: improve ingestion safety and log sanitization

Author: RinZ27

ingestion/src/metadata/ingestion/source/database/sas/client.py

@@ -24,6 +24,8 @@
 
 logger = ingestion_logger()
 
+SAS_CLI_AUTH_HEADER = "Basic c2FzLmNsaTo="
+
 
 class SASClient:
     """
@@ -41,7 +43,7 @@ def __init__(self, config: SASConnection):
             auth_token=self.get_auth_token,
             api_version="",
             allow_redirects=True,
-            verify=False,
+            verify=True,
         )
         self.client = TrackedREST(client_config, source_name="sas")
         # custom setting
@@ -170,14 +172,19 @@ def get_token(self, base_url, user, password):
         payload = {"grant_type": "password", "username": user, "password": password}
         headers = {
             "Content-type": "application/x-www-form-urlencoded",
-            "Authorization": "Basic c2FzLmNsaTo=",
+            "Authorization": SAS_CLI_AUTH_HEADER,
         }
         url = base_url + endpoint
         response = requests.request(
-            "POST", url, headers=headers, data=payload, verify=False, timeout=10
+            "POST", url, headers=headers, data=payload, verify=True, timeout=10
         )
-        text_response = response.json()
-        logger.info(
-            f"this is user: {user}, password: {password}, text: {text_response}"
+        logger.debug(
+            "Token request for user: %s completed with status: %s",
+            user,
+            response.status_code,
         )
-        return response.json()["access_token"]
+        response.raise_for_status()
+        token = response.json().get("access_token")
+        if not token:
+            raise APIError(f"Failed to retrieve access_token from SAS. Response: {response.text}")
+        return token

ingestion/src/metadata/ingestion/source/search/elasticsearch/connection.py

@@ -12,6 +12,7 @@
 """
 Source connection handler
 """
+
 import ssl
 from pathlib import Path
 from typing import Optional
@@ -138,7 +139,7 @@ def get_ssl_context(ssl_config: SslConfig) -> ssl.SSLContext:
         )
         return ssl_context
 
-    return ssl._create_unverified_context()  # pylint: disable=protected-access
+    return ssl.create_default_context()
 
 
 def get_connection(connection: ElasticsearchConnection) -> Elasticsearch:
@@ -154,9 +155,11 @@ def get_connection(connection: ElasticsearchConnection) -> Elasticsearch:
     ):
         basic_auth = (
             connection.authType.username,
-            connection.authType.password.get_secret_value()
-            if connection.authType.password
-            else None,
+            (
+                connection.authType.password.get_secret_value()
+                if connection.authType.password
+                else None
+            ),
         )
 
     if isinstance(connection.authType, ApiKeyAuthentication):

ingestion/src/metadata/utils/secrets/aws_secrets_manager.py

@@ -12,6 +12,7 @@
 """
 Secrets manager implementation using AWS Secrets Manager
 """
+
 import traceback
 from typing import Optional
 
@@ -53,17 +54,15 @@ def get_string_value(self, secret_id: str) -> Optional[str]:
         try:
             kwargs = {"SecretId": secret_id}
             response = self.client.get_secret_value(**kwargs)
-            logger.debug("Got value for secret %s.", secret_id)
+            logger.debug("Successfully retrieved value from secrets manager.")
         except ClientError as err:
             logger.debug(traceback.format_exc())
-            logger.error(f"Couldn't get value for secret [{secret_id}]: {err}")
+            logger.error(f"Couldn't get value from secrets manager: {err}")
             raise err
         if "SecretString" in response:
             return (
                 response["SecretString"]
                 if response["SecretString"] != NULL_VALUE
                 else None
             )
-        raise ValueError(
-            f"SecretString for secret [{secret_id}] not present in the response."
-        )
+        raise ValueError("SecretString not present in the response.")