diff --git a/ingestion/src/metadata/clients/domo_client.py b/ingestion/src/metadata/clients/domo_client.py index 1d0996484c60..2c9957963c1b 100644 --- a/ingestion/src/metadata/clients/domo_client.py +++ b/ingestion/src/metadata/clients/domo_client.py @@ -130,7 +130,7 @@ def get_chart_details(self, page_id) -> Optional[DomoChartDetails]: ) except Exception as exc: - logger.warning(f"Error while getting details for Card {page_id} - {exc}") + logger.error(f"Error while getting details for Card {page_id} - {exc}") logger.debug(traceback.format_exc()) return None @@ -140,7 +140,7 @@ def get_pipelines(self): response = self.client.get(path=WORKFLOW_URL, headers=HEADERS) return response except Exception as exc: - logger.warning(f"Error while getting pipelines - {exc}") + logger.error(f"Error while getting pipelines - {exc}") logger.debug(traceback.format_exc()) return [] @@ -164,7 +164,7 @@ def test_list_cards(self) -> None: self.client.get(path="content/v1/cards", headers=HEADERS) except Exception as exc: logger.debug(traceback.format_exc()) - logger.warning(f"Error listing cards due to [{exc}]") + logger.error(f"Error listing cards due to [{exc}]") raise exc diff --git a/ingestion/src/metadata/data_quality/validations/column/base/columnValueLengthsToBeBetween.py b/ingestion/src/metadata/data_quality/validations/column/base/columnValueLengthsToBeBetween.py index 6b72ad68e1fb..956bcbd94c7c 100644 --- a/ingestion/src/metadata/data_quality/validations/column/base/columnValueLengthsToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/column/base/columnValueLengthsToBeBetween.py @@ -75,7 +75,7 @@ def _run_validation(self) -> TestCaseResult: except (ValueError, RuntimeError) as exc: msg = f"Error computing {self.test_case.fullyQualifiedName}: {exc}" # type: ignore logger.debug(traceback.format_exc()) - logger.warning(msg) + logger.error(msg) return self.get_test_case_result_object( self.execution_date, TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/data_quality/validations/column/base/columnValueMaxToBeBetween.py b/ingestion/src/metadata/data_quality/validations/column/base/columnValueMaxToBeBetween.py index 24ae55fa237e..14fa80bd5a4d 100644 --- a/ingestion/src/metadata/data_quality/validations/column/base/columnValueMaxToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/column/base/columnValueMaxToBeBetween.py @@ -68,7 +68,7 @@ def _run_validation(self) -> TestCaseResult: except (ValueError, RuntimeError) as exc: msg = f"Error computing {self.test_case.fullyQualifiedName}: {exc}" # type: ignore logger.debug(traceback.format_exc()) - logger.warning(msg) + logger.error(msg) return self.get_test_case_result_object( self.execution_date, TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/data_quality/validations/column/base/columnValueMeanToBeBetween.py b/ingestion/src/metadata/data_quality/validations/column/base/columnValueMeanToBeBetween.py index 6212755a37b1..ff93f6048baf 100644 --- a/ingestion/src/metadata/data_quality/validations/column/base/columnValueMeanToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/column/base/columnValueMeanToBeBetween.py @@ -68,7 +68,7 @@ def _run_validation(self) -> TestCaseResult: except (ValueError, RuntimeError) as exc: msg = f"Error computing {self.test_case.fullyQualifiedName}: {exc}" # type: ignore logger.debug(traceback.format_exc()) - logger.warning(msg) + logger.error(msg) return self.get_test_case_result_object( self.execution_date, TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/data_quality/validations/column/base/columnValueMedianToBeBetween.py b/ingestion/src/metadata/data_quality/validations/column/base/columnValueMedianToBeBetween.py index 4ccabb675434..9e7bfc652d19 100644 --- a/ingestion/src/metadata/data_quality/validations/column/base/columnValueMedianToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/column/base/columnValueMedianToBeBetween.py @@ -68,7 +68,7 @@ def _run_validation(self) -> TestCaseResult: except (ValueError, RuntimeError) as exc: msg = f"Error computing {self.test_case.fullyQualifiedName}: {exc}" # type: ignore logger.debug(traceback.format_exc()) - logger.warning(msg) + logger.error(msg) return self.get_test_case_result_object( self.execution_date, TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/data_quality/validations/column/base/columnValueMinToBeBetween.py b/ingestion/src/metadata/data_quality/validations/column/base/columnValueMinToBeBetween.py index abb6c13405d2..2faa22d07749 100644 --- a/ingestion/src/metadata/data_quality/validations/column/base/columnValueMinToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/column/base/columnValueMinToBeBetween.py @@ -65,7 +65,7 @@ def _run_validation(self) -> TestCaseResult: except (ValueError, RuntimeError) as exc: msg = f"Error computing {self.test_case.fullyQualifiedName}: {exc}" # type: ignore logger.debug(traceback.format_exc()) - logger.warning(msg) + logger.error(msg) return self.get_test_case_result_object( self.execution_date, TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/data_quality/validations/column/base/columnValueStdDevToBeBetween.py b/ingestion/src/metadata/data_quality/validations/column/base/columnValueStdDevToBeBetween.py index 28feb7ef9b12..7277afa3ba02 100644 --- a/ingestion/src/metadata/data_quality/validations/column/base/columnValueStdDevToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/column/base/columnValueStdDevToBeBetween.py @@ -70,7 +70,7 @@ def _run_validation(self) -> TestCaseResult: except (ValueError, RuntimeError) as exc: msg = f"Error computing {self.test_case.fullyQualifiedName}: {exc}" # type: ignore logger.debug(traceback.format_exc()) - logger.warning(msg) + logger.error(msg) return self.get_test_case_result_object( self.execution_date, TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/data_quality/validations/column/base/columnValuesMissingCount.py b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesMissingCount.py index 746ee0b3c062..f0c4cb61d440 100644 --- a/ingestion/src/metadata/data_quality/validations/column/base/columnValuesMissingCount.py +++ b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesMissingCount.py @@ -84,7 +84,7 @@ def _run_validation(self) -> TestCaseResult: except (ValueError, RuntimeError) as exc: msg = f"Error computing {self.test_case.fullyQualifiedName}: {exc}" # type: ignore logger.debug(traceback.format_exc()) - logger.warning(msg) + logger.error(msg) return self.get_test_case_result_object( self.execution_date, TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/data_quality/validations/column/base/columnValuesSumToBeBetween.py b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesSumToBeBetween.py index 9bd07e324fa3..98127a195e90 100644 --- a/ingestion/src/metadata/data_quality/validations/column/base/columnValuesSumToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesSumToBeBetween.py @@ -68,7 +68,7 @@ def _run_validation(self) -> TestCaseResult: except (ValueError, RuntimeError) as exc: msg = f"Error computing {self.test_case.fullyQualifiedName}: {exc}" # type: ignore logger.debug(traceback.format_exc()) - logger.warning(msg) + logger.error(msg) return self.get_test_case_result_object( self.execution_date, TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeAtExpectedLocation.py b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeAtExpectedLocation.py index ef8acfe39b77..09e7ce9ba9cf 100644 --- a/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeAtExpectedLocation.py +++ b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeAtExpectedLocation.py @@ -230,7 +230,7 @@ def _run_validation(self) -> TestCaseResult: except (ValueError, RuntimeError) as exc: msg = f"Error computing {self.test_case.fullyQualifiedName}: {exc}" # type: ignore logger.debug(traceback.format_exc()) - logger.warning(msg) + logger.error(msg) return self.get_test_case_result_object( self.execution_date, TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeBetween.py b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeBetween.py index c79b50604c2f..58143dc574e2 100644 --- a/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeBetween.py @@ -80,7 +80,7 @@ def _run_validation(self) -> TestCaseResult: except (ValueError, RuntimeError) as exc: msg = f"Error computing {self.test_case.fullyQualifiedName}: {exc}" # type: ignore logger.debug(traceback.format_exc()) - logger.warning(msg) + logger.error(msg) return self.get_test_case_result_object( self.execution_date, TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeInSet.py b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeInSet.py index 7fc0db2278d2..ad08098d7a21 100644 --- a/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeInSet.py +++ b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeInSet.py @@ -73,7 +73,7 @@ def _run_validation(self) -> TestCaseResult: except (ValueError, RuntimeError) as exc: msg = f"Error computing {self.test_case.fullyQualifiedName}: {exc}" # type: ignore logger.debug(traceback.format_exc()) - logger.warning(msg) + logger.error(msg) return self.get_test_case_result_object( self.execution_date, TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeNotInSet.py b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeNotInSet.py index aae2589c6bee..b8bf1913e23b 100644 --- a/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeNotInSet.py +++ b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeNotInSet.py @@ -68,7 +68,7 @@ def _run_validation(self) -> TestCaseResult: except (ValueError, RuntimeError) as exc: msg = f"Error computing {self.test_case.name} for {get_table_fqn(self.test_case.entityLink.root)}: {exc}" logger.debug(traceback.format_exc()) - logger.warning(msg) + logger.error(msg) return self.get_test_case_result_object( self.execution_date, TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeNotNull.py b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeNotNull.py index 907d57cee006..dc10a76884e4 100644 --- a/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeNotNull.py +++ b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeNotNull.py @@ -65,7 +65,7 @@ def _run_validation(self) -> TestCaseResult: except (ValueError, RuntimeError) as exc: msg = f"Error computing {self.test_case.fullyQualifiedName}: {exc}" # type: ignore logger.debug(traceback.format_exc()) - logger.warning(msg) + logger.error(msg) return self.get_test_case_result_object( self.execution_date, TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeUnique.py b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeUnique.py index 89e203c0bdb9..c1a199114b58 100644 --- a/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeUnique.py +++ b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeUnique.py @@ -153,7 +153,7 @@ def _run_validation(self) -> TestCaseResult: except (ValueError, RuntimeError) as exc: msg = f"Error computing {self.test_case.fullyQualifiedName}: {exc}" # type: ignore logger.debug(traceback.format_exc()) - logger.warning(msg) + logger.error(msg) return self.get_test_case_result_object( self.execution_date, TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToMatchRegex.py b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToMatchRegex.py index ada8bb5ba534..121c16dda5b5 100644 --- a/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToMatchRegex.py +++ b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToMatchRegex.py @@ -72,7 +72,7 @@ def _run_validation(self) -> TestCaseResult: except (ValueError, RuntimeError) as exc: msg = f"Error computing {self.test_case.fullyQualifiedName}: {exc}" # type: ignore logger.debug(traceback.format_exc()) - logger.warning(msg) + logger.error(msg) return self.get_test_case_result_object( self.execution_date, TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToNotMatchRegex.py b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToNotMatchRegex.py index a97e43a3c83b..549dd364931b 100644 --- a/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToNotMatchRegex.py +++ b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToNotMatchRegex.py @@ -69,7 +69,7 @@ def _run_validation(self) -> TestCaseResult: except (ValueError, RuntimeError) as exc: msg = f"Error computing {self.test_case.fullyQualifiedName}: {exc}" # type: ignore logger.debug(traceback.format_exc()) - logger.warning(msg) + logger.error(msg) return self.get_test_case_result_object( self.execution_date, TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/data_quality/validations/table/base/tableColumnCountToBeBetween.py b/ingestion/src/metadata/data_quality/validations/table/base/tableColumnCountToBeBetween.py index f201c1cc8566..7782ea8a023a 100644 --- a/ingestion/src/metadata/data_quality/validations/table/base/tableColumnCountToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/table/base/tableColumnCountToBeBetween.py @@ -46,7 +46,7 @@ def _run_validation(self) -> TestCaseResult: except Exception as exc: msg = f"Error computing {self.test_case.fullyQualifiedName}: {exc}" # type: ignore logger.debug(traceback.format_exc()) - logger.warning(msg) + logger.error(msg) return self.get_test_case_result_object( self.execution_date, TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/data_quality/validations/table/base/tableColumnCountToEqual.py b/ingestion/src/metadata/data_quality/validations/table/base/tableColumnCountToEqual.py index d11311e04fab..7354b00e1128 100644 --- a/ingestion/src/metadata/data_quality/validations/table/base/tableColumnCountToEqual.py +++ b/ingestion/src/metadata/data_quality/validations/table/base/tableColumnCountToEqual.py @@ -46,7 +46,7 @@ def _run_validation(self) -> TestCaseResult: except Exception as exc: msg = f"Error computing {self.test_case.fullyQualifiedName}: {exc}" # type: ignore logger.debug(traceback.format_exc()) - logger.warning(msg) + logger.error(msg) return self.get_test_case_result_object( self.execution_date, TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/data_quality/validations/table/base/tableColumnNameToExist.py b/ingestion/src/metadata/data_quality/validations/table/base/tableColumnNameToExist.py index ada1bba5266b..19f236da4f73 100644 --- a/ingestion/src/metadata/data_quality/validations/table/base/tableColumnNameToExist.py +++ b/ingestion/src/metadata/data_quality/validations/table/base/tableColumnNameToExist.py @@ -46,7 +46,7 @@ def _run_validation(self) -> TestCaseResult: except Exception as exc: msg = f"Error computing {self.test_case.fullyQualifiedName}: {exc}" # type: ignore logger.debug(traceback.format_exc()) - logger.warning(msg) + logger.error(msg) return self.get_test_case_result_object( self.execution_date, TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/data_quality/validations/table/base/tableColumnToMatchSet.py b/ingestion/src/metadata/data_quality/validations/table/base/tableColumnToMatchSet.py index 53f5685e9e33..61c1889d2d66 100644 --- a/ingestion/src/metadata/data_quality/validations/table/base/tableColumnToMatchSet.py +++ b/ingestion/src/metadata/data_quality/validations/table/base/tableColumnToMatchSet.py @@ -51,7 +51,7 @@ def _run_validation(self) -> TestCaseResult: except Exception as exc: msg = f"Error computing {self.test_case.fullyQualifiedName}: {exc}" # type: ignore logger.debug(traceback.format_exc()) - logger.warning(msg) + logger.error(msg) return self.get_test_case_result_object( self.execution_date, TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/data_quality/validations/table/base/tableCustomSQLQuery.py b/ingestion/src/metadata/data_quality/validations/table/base/tableCustomSQLQuery.py index 4f86e7a1742b..cbe9135d8405 100644 --- a/ingestion/src/metadata/data_quality/validations/table/base/tableCustomSQLQuery.py +++ b/ingestion/src/metadata/data_quality/validations/table/base/tableCustomSQLQuery.py @@ -85,7 +85,7 @@ def _run_validation(self) -> TestCaseResult: except Exception as exc: msg = f"Error computing {self.test_case.fullyQualifiedName}: {exc}" # type: ignore logger.debug(traceback.format_exc()) - logger.warning(msg) + logger.error(msg) return self.get_test_case_result_object( self.execution_date, TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/data_quality/validations/table/base/tableRowCountToBeBetween.py b/ingestion/src/metadata/data_quality/validations/table/base/tableRowCountToBeBetween.py index bb701ae22fdd..f516fbd358ef 100644 --- a/ingestion/src/metadata/data_quality/validations/table/base/tableRowCountToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/table/base/tableRowCountToBeBetween.py @@ -47,7 +47,7 @@ def _run_validation(self) -> TestCaseResult: except ValueError as exc: msg = f"Error computing {self.test_case.fullyQualifiedName}: {exc}" # type: ignore logger.debug(traceback.format_exc()) - logger.warning(msg) + logger.error(msg) return self.get_test_case_result_object( self.execution_date, TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/data_quality/validations/table/base/tableRowCountToEqual.py b/ingestion/src/metadata/data_quality/validations/table/base/tableRowCountToEqual.py index be311fe243b1..735bbd241722 100644 --- a/ingestion/src/metadata/data_quality/validations/table/base/tableRowCountToEqual.py +++ b/ingestion/src/metadata/data_quality/validations/table/base/tableRowCountToEqual.py @@ -47,7 +47,7 @@ def _run_validation(self) -> TestCaseResult: except ValueError as exc: msg = f"Error computing {self.test_case.fullyQualifiedName}: {exc}" # type: ignore logger.debug(traceback.format_exc()) - logger.warning(msg) + logger.error(msg) return self.get_test_case_result_object( self.execution_date, TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/data_quality/validations/table/base/tableRowInsertedCountToBeBetween.py b/ingestion/src/metadata/data_quality/validations/table/base/tableRowInsertedCountToBeBetween.py index 73914fa98405..95614b5884d2 100644 --- a/ingestion/src/metadata/data_quality/validations/table/base/tableRowInsertedCountToBeBetween.py +++ b/ingestion/src/metadata/data_quality/validations/table/base/tableRowInsertedCountToBeBetween.py @@ -67,7 +67,7 @@ def _run_validation(self) -> TestCaseResult: except Exception as exc: msg = f"Error computing {self.test_case.name}: {exc}" # type: ignore logger.debug(traceback.format_exc()) - logger.warning(msg) + logger.error(msg) return self.get_test_case_result_object( self.execution_date, TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableDiff.py b/ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableDiff.py index a8b865f9651d..9a78e06cffae 100644 --- a/ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableDiff.py +++ b/ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableDiff.py @@ -211,7 +211,7 @@ def _run_validation(self): ) return result except UnsupportedDialectError as e: - logger.warning(f"[Data Diff]: Unsupported dialect: {e}") + logger.error(f"[Data Diff]: Unsupported dialect: {e}") result = TestCaseResult( timestamp=self.execution_date, # type: ignore testCaseStatus=TestCaseStatus.Aborted, diff --git a/ingestion/src/metadata/ingestion/bulksink/metadata_usage.py b/ingestion/src/metadata/ingestion/bulksink/metadata_usage.py index d96d8a455b79..595a17710baf 100644 --- a/ingestion/src/metadata/ingestion/bulksink/metadata_usage.py +++ b/ingestion/src/metadata/ingestion/bulksink/metadata_usage.py @@ -389,6 +389,6 @@ def close(self): self.metadata.compute_percentile(Database, self.today) except APIError as err: logger.debug(traceback.format_exc()) - logger.warning(f"Failed to publish compute.percentile: {err}") + logger.error(f"Failed to publish compute.percentile: {err}") self.metadata.close() diff --git a/ingestion/src/metadata/ingestion/connections/test_connections.py b/ingestion/src/metadata/ingestion/connections/test_connections.py index 3007fc89444f..4f06257b4998 100644 --- a/ingestion/src/metadata/ingestion/connections/test_connections.py +++ b/ingestion/src/metadata/ingestion/connections/test_connections.py @@ -136,7 +136,7 @@ def _test_connection_steps_automation_workflow( ) except Exception as err: logger.debug(traceback.format_exc()) - logger.warning(f"{step.name}-{err}") + logger.error(f"{step.name}-{err}") test_connection_result.steps.append( TestConnectionStepResult( name=step.name, diff --git a/ingestion/src/metadata/ingestion/ometa/mixins/container_mixin.py b/ingestion/src/metadata/ingestion/ometa/mixins/container_mixin.py index d291c65dbcec..c3af8d77362a 100644 --- a/ingestion/src/metadata/ingestion/ometa/mixins/container_mixin.py +++ b/ingestion/src/metadata/ingestion/ometa/mixins/container_mixin.py @@ -74,7 +74,7 @@ def _parse_response(self, resp: dict, container_fqn: str) -> Optional[TableData] return TableData(**resp["sampleData"]) except UnicodeError as err: logger.debug(traceback.format_exc()) - logger.warning(f"Cannot parse response from {container_fqn} due to {err}") + logger.error(f"Cannot parse response from {container_fqn} due to {err}") return None def ingest_container_sample_data(self, container: Container, sample_data: TableData) -> Optional[TableData]: diff --git a/ingestion/src/metadata/ingestion/source/api/rest/parser.py b/ingestion/src/metadata/ingestion/source/api/rest/parser.py index bb3598c74c01..36074466757d 100644 --- a/ingestion/src/metadata/ingestion/source/api/rest/parser.py +++ b/ingestion/src/metadata/ingestion/source/api/rest/parser.py @@ -55,12 +55,12 @@ def parse_openapi_schema(response: Response) -> Dict[str, Any]: try: return json.loads(content) except json.JSONDecodeError as e: - logger.warning(f"Failed to parse as JSON despite content-type: {e}") + logger.error(f"Failed to parse as JSON despite content-type: {e}") elif "yaml" in content_type or "yml" in content_type: try: return yaml.safe_load(content) except yaml.YAMLError as e: - logger.warning(f"Failed to parse as YAML despite content-type: {e}") + logger.error(f"Failed to parse as YAML despite content-type: {e}") # If content-type is not definitive or parsing failed, try both formats diff --git a/ingestion/src/metadata/ingestion/source/dashboard/metabase/client.py b/ingestion/src/metadata/ingestion/source/dashboard/metabase/client.py index 94fb6c8036cf..cf39f6284363 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/metabase/client.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/metabase/client.py @@ -127,7 +127,7 @@ def get_dashboards_list(self, collections: List[MetabaseCollection]) -> List[Met dashboards.extend(dashboard_list.data) except Exception: logger.debug(traceback.format_exc()) - logger.warning("Failed to fetch the dashboard list") + logger.error("Failed to fetch the dashboard list") return dashboards def get_dashboards_list_test_conn(self, collections: List[MetabaseCollection]) -> List[MetabaseDashboard]: @@ -162,7 +162,7 @@ def get_collections_list(self) -> List[MetabaseCollection]: return collection_list.collections except Exception: logger.debug(traceback.format_exc()) - logger.warning("Failed to fetch the collections list") + logger.error("Failed to fetch the collections list") return [] def get_charts_dict(self) -> Dict: @@ -176,7 +176,7 @@ def get_charts_dict(self) -> Dict: return charts_dict except Exception as e: logger.debug(traceback.format_exc()) - logger.warning(f"Failed to fetch the cards : {e}") + logger.error(f"Failed to fetch the cards : {e}") return {} def _create_default_dashboard_details(self, orphan_charts_id: List) -> MetabaseDashboardDetails: @@ -233,7 +233,7 @@ def get_dashboard_details( return self._process_dashboard_response(resp_dashboard, charts_dict, dashboard_id) except Exception: logger.debug(traceback.format_exc()) - logger.warning(f"Failed to fetch the dashboard with id: {dashboard_id}") + logger.error(f"Failed to fetch the dashboard with id: {dashboard_id}") return None def get_database(self, database_id: str) -> Optional[MetabaseDatabase]: @@ -248,7 +248,7 @@ def get_database(self, database_id: str) -> Optional[MetabaseDatabase]: return MetabaseDatabase(**resp_database) except Exception: logger.debug(traceback.format_exc()) - logger.warning(f"Failed to fetch the database with id: {database_id}") + logger.error(f"Failed to fetch the database with id: {database_id}") return None def get_table(self, table_id: str) -> Optional[MetabaseTable]: @@ -263,7 +263,7 @@ def get_table(self, table_id: str) -> Optional[MetabaseTable]: return MetabaseTable(**resp_table) except Exception: logger.debug(traceback.format_exc()) - logger.warning(f"Failed to fetch the table with id: {table_id}") + logger.error(f"Failed to fetch the table with id: {table_id}") return None def get_user_details(self, user_id: str) -> Optional[MetabaseUser]: @@ -278,5 +278,5 @@ def get_user_details(self, user_id: str) -> Optional[MetabaseUser]: return MetabaseUser(**resp_table) except Exception: logger.debug(traceback.format_exc()) - logger.warning(f"Failed to fetch the user with id: {user_id}") + logger.error(f"Failed to fetch the user with id: {user_id}") return None diff --git a/ingestion/src/metadata/ingestion/source/dashboard/microstrategy/client.py b/ingestion/src/metadata/ingestion/source/dashboard/microstrategy/client.py index 8441e32d9092..d754924e5846 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/microstrategy/client.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/microstrategy/client.py @@ -136,7 +136,7 @@ def close_api_session(self) -> None: except Exception as exc: logger.debug(traceback.format_exc()) - logger.warning(f"Failed to close the api sesison due to [{exc}]") + logger.error(f"Failed to close the api sesison due to [{exc}]") def is_project_name(self) -> bool: return bool(self.config.projectName) @@ -155,7 +155,7 @@ def get_projects_list(self) -> List[MstrProject]: except Exception as exc: logger.debug(traceback.format_exc()) - logger.warning(f"Failed to fetch the project list due to [{exc}]") + logger.error(f"Failed to fetch the project list due to [{exc}]") return [] @@ -173,7 +173,7 @@ def get_project_by_name(self) -> Optional[MstrProject]: except Exception: logger.debug(traceback.format_exc()) - logger.warning("Failed to fetch the project list") + logger.error("Failed to fetch the project list") return None @@ -205,7 +205,7 @@ def get_search_results_list(self, project_id, object_type) -> List[MstrSearchRes except Exception: logger.debug(traceback.format_exc()) - logger.warning("Failed to fetch the Search Result list") + logger.error("Failed to fetch the Search Result list") return [] @@ -225,7 +225,7 @@ def get_dashboards_list(self, project_id, project_name) -> List[MstrDashboard]: except Exception: logger.debug(traceback.format_exc()) - logger.warning("Failed to fetch the dashboard list") + logger.error("Failed to fetch the dashboard list") return [] @@ -241,7 +241,7 @@ def get_dashboard_details(self, project_id, project_name, dashboard_id) -> Optio except Exception: logger.debug(traceback.format_exc()) - logger.warning(f"Failed to fetch the dashboard with id: {dashboard_id}") + logger.error(f"Failed to fetch the dashboard with id: {dashboard_id}") return None @@ -260,6 +260,6 @@ def get_cube_sql_details(self, project_id: str, cube_id: str) -> Optional[str]: except Exception: logger.debug(traceback.format_exc()) - logger.warning(f"Failed to fetch the cube with id: {cube_id}") + logger.error(f"Failed to fetch the cube with id: {cube_id}") return None diff --git a/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/client.py b/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/client.py index fb96ccee73dd..ff1170654ec7 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/client.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/client.py @@ -124,7 +124,7 @@ def get_dashboard_charts(self, dashboard_id: str) -> List[QlikSheet]: return data.result.qLayout.qAppObjectList.qItems except Exception: logger.debug(traceback.format_exc()) - logger.warning("Failed to fetch the dashboard charts") + logger.error("Failed to fetch the dashboard charts") return [] def get_dashboards_list(self) -> Iterable[QlikApp]: @@ -144,7 +144,7 @@ def get_dashboards_list(self) -> Iterable[QlikApp]: break except Exception: logger.debug(traceback.format_exc()) - logger.warning("Failed to fetch the app list") + logger.error("Failed to fetch the app list") def get_dashboards_list_test_conn(self) -> Iterable[QlikApp]: resp_apps = self.client.get("/v1/items?resourceType=app") @@ -164,7 +164,7 @@ def get_dashboard_details(self, dashboard_id: str) -> Optional[QlikApp]: return QlikApp(**resp_dashboard.get("attributes")) except Exception: logger.debug(traceback.format_exc()) - logger.warning(f"Failed to fetch the dashboard with id: {dashboard_id}") + logger.error(f"Failed to fetch the dashboard with id: {dashboard_id}") return None def get_dashboard_models(self) -> List[QlikTable]: @@ -194,7 +194,7 @@ def get_dashboard_models(self) -> List[QlikTable]: return parsed_datamodels except Exception: logger.debug(traceback.format_exc()) - logger.warning("Failed to fetch the dashboard datamodels") + logger.error("Failed to fetch the dashboard datamodels") return [] def get_projects_list(self) -> Iterable[QlikSpace]: @@ -214,7 +214,7 @@ def get_projects_list(self) -> Iterable[QlikSpace]: break except Exception: logger.debug(traceback.format_exc()) - logger.warning("Failed to fetch the space list") + logger.error("Failed to fetch the space list") def get_script_tables(self) -> Optional[List[QlikTable]]: """Get script tables from the dashboard script""" @@ -234,7 +234,7 @@ def get_script_tables(self) -> Optional[List[QlikTable]]: return script_tables except Exception: logger.debug(traceback.format_exc()) - logger.warning("Failed to fetch the script tables") + logger.error("Failed to fetch the script tables") return script_tables def get_data_files(self) -> List[QlikDataFile]: @@ -246,5 +246,5 @@ def get_data_files(self) -> List[QlikDataFile]: data_files = parsed_resp.data or [] except Exception: logger.debug(traceback.format_exc()) - logger.warning("Failed to fetch data files from api `/v1/data-files`") + logger.error("Failed to fetch data files from api `/v1/data-files`") return data_files diff --git a/ingestion/src/metadata/ingestion/source/dashboard/qliksense/client.py b/ingestion/src/metadata/ingestion/source/dashboard/qliksense/client.py index ddc1c202c35f..a3e544613a56 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/qliksense/client.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/qliksense/client.py @@ -156,7 +156,7 @@ def get_dashboards_list(self, create_new_socket: bool = True) -> List[QlikDashbo return dashboard_result.result.qDocList except Exception: logger.debug(traceback.format_exc()) - logger.warning("Failed to fetch the dashboard list") + logger.error("Failed to fetch the dashboard list") return [] def get_dashboard_charts(self, dashboard_id: str) -> List[QlikSheet]: @@ -172,7 +172,7 @@ def get_dashboard_charts(self, dashboard_id: str) -> List[QlikSheet]: return data.result.qLayout.qAppObjectList.qItems except Exception: logger.debug(traceback.format_exc()) - logger.warning("Failed to fetch the dashboard charts") + logger.error("Failed to fetch the dashboard charts") return [] def _get_tables_via_get_tables_and_keys(self) -> Optional[List[QlikTable]]: @@ -238,7 +238,7 @@ def get_dashboard_models(self) -> List[QlikTable]: return self._get_tables_via_load_model() except Exception: logger.debug(traceback.format_exc()) - logger.warning("Failed to fetch the dashboard datamodels") + logger.error("Failed to fetch the dashboard datamodels") return [] def get_script(self) -> Optional[str]: @@ -253,7 +253,7 @@ def get_script(self) -> Optional[str]: return script_result.result.qScript except Exception: logger.debug(traceback.format_exc()) - logger.warning("Failed to fetch the app load script") + logger.error("Failed to fetch the app load script") return None def get_script_tables(self) -> Dict[str, Set[str]]: @@ -302,5 +302,5 @@ def get_dashboard_for_test_connection(self): return QlikDashboardResult(**json.loads(resp)) except ValidationError: logger.debug(traceback.format_exc()) - logger.warning("Failed to fetch the dashboard datamodels") + logger.error("Failed to fetch the dashboard datamodels") return None diff --git a/ingestion/src/metadata/ingestion/source/dashboard/superset/db_source.py b/ingestion/src/metadata/ingestion/source/dashboard/superset/db_source.py index 972266df6528..2cc83e6264f3 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/superset/db_source.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/superset/db_source.py @@ -95,7 +95,7 @@ def prepare(self): self.all_charts[chart_detail.id] = chart_detail except Exception as err: logger.debug(traceback.format_exc()) - logger.warning(f"Failed to fetch chart list due to - {err}]") + logger.error(f"Failed to fetch chart list due to - {err}]") def get_column_list(self, table_id: Optional[int]) -> Iterable[FetchChart]: try: diff --git a/ingestion/src/metadata/ingestion/source/database/common_db_source.py b/ingestion/src/metadata/ingestion/source/database/common_db_source.py index 84ae89ac3c97..76e513516f11 100644 --- a/ingestion/src/metadata/ingestion/source/database/common_db_source.py +++ b/ingestion/src/metadata/ingestion/source/database/common_db_source.py @@ -179,7 +179,7 @@ def _release_engine(self) -> None: try: self.engine.dispose() except Exception as exc: # pylint: disable=broad-except - logger.warning(f"Failed to dispose engine: {exc}") + logger.error(f"Failed to dispose engine: {exc}") self.engine = None self.connection_obj = None diff --git a/ingestion/src/metadata/ingestion/source/database/external_table_lineage_mixin.py b/ingestion/src/metadata/ingestion/source/database/external_table_lineage_mixin.py index 03ca68f3b62a..0763c82823ff 100644 --- a/ingestion/src/metadata/ingestion/source/database/external_table_lineage_mixin.py +++ b/ingestion/src/metadata/ingestion/source/database/external_table_lineage_mixin.py @@ -86,7 +86,7 @@ def yield_external_table_lineage(self) -> Iterable[AddLineageRequest]: ) ) except Exception as exc: - logger.warning(f"Failed to yield external table lineage due to - {exc}") + logger.error(f"Failed to yield external table lineage due to - {exc}") logger.debug(traceback.format_exc()) def _get_data_model_column_fqn(self, data_model_entity: ContainerDataModel, column: str) -> Optional[str]: diff --git a/ingestion/src/metadata/ingestion/source/database/lineage_source.py b/ingestion/src/metadata/ingestion/source/database/lineage_source.py index 5dfdc6ea1060..ef41a3c8a1fe 100644 --- a/ingestion/src/metadata/ingestion/source/database/lineage_source.py +++ b/ingestion/src/metadata/ingestion/source/database/lineage_source.py @@ -214,7 +214,7 @@ def process_queue_items(): try: yield from process_queue_items() except Exception as exc: - logger.warning(f"Error processing queue: {exc}") + logger.error(f"Error processing queue: {exc}") logger.debug(traceback.format_exc()) # Check for completed or timed-out processes @@ -248,7 +248,7 @@ def process_queue_items(): # check if there are more than MAX_ACTIVE_TIMED_OUT_THREADS if len(active_timed_out_threads) > MAX_ACTIVE_TIMED_OUT_THREADS: remaining_chunks = sum(1 for _ in chunk_iter) - logger.warning( + logger.error( f"There are more than {MAX_ACTIVE_TIMED_OUT_THREADS} active timed out threads, " f"skipping remaining {remaining_chunks}/{completed_chunks + remaining_chunks} chunks. " ) @@ -271,7 +271,7 @@ def process_queue_items(): try: yield from process_queue_items() except Exception as exc: - logger.warning(f"Error processing queue: {exc}") + logger.error(f"Error processing queue: {exc}") logger.debug(traceback.format_exc()) logger.info( diff --git a/ingestion/src/metadata/ingestion/source/drive/sftp/metadata.py b/ingestion/src/metadata/ingestion/source/drive/sftp/metadata.py index 200cdd436110..cef950f06bce 100644 --- a/ingestion/src/metadata/ingestion/source/drive/sftp/metadata.py +++ b/ingestion/src/metadata/ingestion/source/drive/sftp/metadata.py @@ -696,7 +696,7 @@ def _ingest_sample_data_for_file( else: logger.warning(f"Could not find file entity to ingest sample data: {file_fqn}") except Exception as e: - logger.warning(f"Failed to ingest sample data for file {file_name}: {e}") + logger.error(f"Failed to ingest sample data for file {file_name}: {e}") logger.debug(traceback.format_exc()) def _get_csv_separator(self, filename: str) -> str: @@ -773,6 +773,6 @@ def _extract_csv_schema( return columns, sample_data except Exception as e: - logger.warning(f"Failed to extract CSV schema from {file_path}: {e}") + logger.error(f"Failed to extract CSV schema from {file_path}: {e}") logger.debug(traceback.format_exc()) return None, None diff --git a/ingestion/src/metadata/ingestion/source/mcp/client.py b/ingestion/src/metadata/ingestion/source/mcp/client.py index e49fd5ca7904..abe0075a96e1 100644 --- a/ingestion/src/metadata/ingestion/source/mcp/client.py +++ b/ingestion/src/metadata/ingestion/source/mcp/client.py @@ -146,7 +146,7 @@ def _handle_response_line(self, line: str) -> None: try: response = json.loads(line) except json.JSONDecodeError as e: - logger.warning(f"Failed to parse MCP response: {e}") + logger.error(f"Failed to parse MCP response: {e}") return msg_id = response.get("id") if msg_id is None: @@ -287,7 +287,7 @@ def send_notification(self, method: str, params: Optional[Dict] = None) -> None: timeout=self.timeout, ) except Exception as e: - logger.warning(f"Failed to send notification '{method}': {e}") + logger.error(f"Failed to send notification '{method}': {e}") def send_request(self, method: str, params: Optional[Dict] = None) -> Dict[str, Any]: """Send a JSON-RPC request via HTTP POST""" @@ -469,7 +469,7 @@ def parse_claude_desktop_config(config_path: str, config: Optional[Dict] = None) with open(path, "r", encoding="utf-8") as f: config = json.load(f) except json.JSONDecodeError as e: - logger.warning(f"Failed to parse config file {config_path}: {e}") + logger.error(f"Failed to parse config file {config_path}: {e}") return [] servers = [] @@ -514,7 +514,7 @@ def parse_vscode_config(config_path: str, config: Optional[Dict] = None) -> List with open(path, "r", encoding="utf-8") as f: config = json.load(f) except json.JSONDecodeError as e: - logger.warning(f"Failed to parse VS Code settings {config_path}: {e}") + logger.error(f"Failed to parse VS Code settings {config_path}: {e}") return [] servers = [] diff --git a/ingestion/src/metadata/ingestion/source/mcp/connection.py b/ingestion/src/metadata/ingestion/source/mcp/connection.py index 2b6bcd2f9d15..ae26b6de7cfa 100644 --- a/ingestion/src/metadata/ingestion/source/mcp/connection.py +++ b/ingestion/src/metadata/ingestion/source/mcp/connection.py @@ -133,7 +133,7 @@ def test_server_connection(self, server: McpServerInfo) -> bool: client = self.connect_to_server(server) return True except McpProtocolError as e: - logger.warning(f"Failed to connect to MCP server '{server.name}': {e}") + logger.error(f"Failed to connect to MCP server '{server.name}': {e}") return False except Exception as e: logger.warning(f"Unexpected error connecting to MCP server '{server.name}': {e}") diff --git a/ingestion/src/metadata/ingestion/source/messaging/common_broker_source.py b/ingestion/src/metadata/ingestion/source/messaging/common_broker_source.py index f89a0de963e7..dbdaa005fbc3 100644 --- a/ingestion/src/metadata/ingestion/source/messaging/common_broker_source.py +++ b/ingestion/src/metadata/ingestion/source/messaging/common_broker_source.py @@ -201,7 +201,7 @@ def _get_schema_text_with_references(self, schema) -> Optional[str]: return schema_text except Exception as exc: logger.debug(traceback.format_exc()) - logger.warning(f"Failed to get schema with references: {exc}") + logger.error(f"Failed to get schema with references: {exc}") return None def _parse_topic_metadata(self, topic_name: str) -> Optional[Schema]: diff --git a/ingestion/src/metadata/ingestion/source/messaging/pubsub/metadata.py b/ingestion/src/metadata/ingestion/source/messaging/pubsub/metadata.py index 4933d95b609c..56c132e99c2e 100644 --- a/ingestion/src/metadata/ingestion/source/messaging/pubsub/metadata.py +++ b/ingestion/src/metadata/ingestion/source/messaging/pubsub/metadata.py @@ -204,10 +204,10 @@ def _get_topic_subscriptions(self, topic_name: str) -> List[PubSubSubscription]: ) except Exception as err: logger.debug(traceback.format_exc()) - logger.warning(f"Failed to get subscription {sub_path}: {err}") + logger.error(f"Failed to get subscription {sub_path}: {err}") except Exception as err: logger.debug(traceback.format_exc()) - logger.warning(f"Failed to list subscriptions for {topic_name}: {err}") + logger.error(f"Failed to list subscriptions for {topic_name}: {err}") return subscriptions def _get_schema_info(self, schema_name: str) -> Optional[PubSubSchemaInfo]: @@ -227,7 +227,7 @@ def _get_schema_info(self, schema_name: str) -> Optional[PubSubSchemaInfo]: ) except Exception as err: logger.debug(traceback.format_exc()) - logger.warning(f"Failed to get schema {schema_name}: {err}") + logger.error(f"Failed to get schema {schema_name}: {err}") return None def yield_topic(self, topic_details: BrokerTopicDetails) -> Iterable[Either[CreateTopicRequest]]: @@ -345,7 +345,7 @@ def _parse_schema(self, topic_name: str, schema_text: str, schema_type: SchemaTy return load_parser_fn(topic_name, schema_text) except Exception as exc: logger.debug(traceback.format_exc()) - logger.warning(f"Failed to parse schema for {topic_name}: {exc}") + logger.error(f"Failed to parse schema for {topic_name}: {exc}") return None def yield_topic_sample_data(self, topic_details: BrokerTopicDetails) -> Iterable[Either[OMetaTopicSampleData]]: diff --git a/ingestion/src/metadata/ingestion/source/pipeline/airbyte/metadata.py b/ingestion/src/metadata/ingestion/source/pipeline/airbyte/metadata.py index a1a19f53e9c5..aa7f533ebe39 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/airbyte/metadata.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/airbyte/metadata.py @@ -224,14 +224,14 @@ def _yield_pipeline_status_cloud( start_dt = datetime.fromisoformat(job.startTime.replace("Z", "+00:00")) created_at = datetime_to_timestamp(start_dt, milliseconds=True) except (ValueError, AttributeError) as exc: - logger.warning(f"Failed to parse startTime: {exc}") + logger.error(f"Failed to parse startTime: {exc}") if job.lastUpdatedAt: try: end_dt = datetime.fromisoformat(job.lastUpdatedAt.replace("Z", "+00:00")) ended_at = datetime_to_timestamp(end_dt, milliseconds=True) except (ValueError, AttributeError) as exc: - logger.warning(f"Failed to parse lastUpdatedAt: {exc}") + logger.error(f"Failed to parse lastUpdatedAt: {exc}") task_status = [ TaskStatus( diff --git a/ingestion/src/metadata/ingestion/source/pipeline/airflow/api/client.py b/ingestion/src/metadata/ingestion/source/pipeline/airflow/api/client.py index bbcec7e89b56..28afec0bf341 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/airflow/api/client.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/airflow/api/client.py @@ -146,7 +146,7 @@ def _parse_response(self, response): try: return response.json() except Exception as exc: - logger.warning(f"Failed to parse JSON response: {exc}") + logger.error(f"Failed to parse JSON response: {exc}") logger.warning(f"Response content type: {response.headers.get('content-type')}") logger.debug(f"Response status code: {response.status_code}") logger.debug(f"Response text: {response.text[:500]}") diff --git a/ingestion/src/metadata/ingestion/source/pipeline/dbtcloud/client.py b/ingestion/src/metadata/ingestion/source/pipeline/dbtcloud/client.py index b401f0fea593..4ac0798f6405 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/dbtcloud/client.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/dbtcloud/client.py @@ -250,7 +250,7 @@ def get_runs(self, job_id: int) -> Iterable[DBTRun]: except Exception as exc: logger.debug(traceback.format_exc()) - logger.warning(f"Unable to get run info :{exc}") + logger.error(f"Unable to get run info :{exc}") def get_models_with_lineage( self, job_id: int, run_id: int @@ -277,5 +277,5 @@ def get_models_with_lineage( except Exception as exc: logger.debug(traceback.format_exc()) - logger.warning(f"Unable to get models with lineage info: {exc}") + logger.error(f"Unable to get models with lineage info: {exc}") return None, None, None diff --git a/ingestion/src/metadata/ingestion/source/pipeline/domopipeline/metadata.py b/ingestion/src/metadata/ingestion/source/pipeline/domopipeline/metadata.py index f55aaee054f0..0630018527c2 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/domopipeline/metadata.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/domopipeline/metadata.py @@ -183,5 +183,5 @@ def get_source_url( ) except Exception as exc: logger.debug(traceback.format_exc()) - logger.warning(f"Unable to get source url for {pipeline_id}: {exc}") + logger.error(f"Unable to get source url for {pipeline_id}: {exc}") return None diff --git a/ingestion/src/metadata/ingestion/source/pipeline/gluepipeline/metadata.py b/ingestion/src/metadata/ingestion/source/pipeline/gluepipeline/metadata.py index 04b71b6c08fe..9abf7a79cb0e 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/gluepipeline/metadata.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/gluepipeline/metadata.py @@ -278,7 +278,7 @@ def _download_s3_script(self, s3_uri: str) -> Optional[str]: response = self.s3_client.get_object(Bucket=bucket, Key=key) return response["Body"].read().decode("utf-8") except Exception as exc: - logger.warning(f"Failed to download script from {s3_uri}: {exc}") + logger.error(f"Failed to download script from {s3_uri}: {exc}") logger.debug(traceback.format_exc()) return None diff --git a/ingestion/src/metadata/ingestion/source/pipeline/kafkaconnect/client.py b/ingestion/src/metadata/ingestion/source/pipeline/kafkaconnect/client.py index 86696782ff69..681dbca7bbaf 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/kafkaconnect/client.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/kafkaconnect/client.py @@ -235,7 +235,7 @@ def get_connector_plugins(self) -> Optional[dict]: return self.client.list_connector_plugins() except Exception as exc: logger.debug(traceback.format_exc()) - logger.warning(f"Unable to get connector plugins {exc}") + logger.error(f"Unable to get connector plugins {exc}") def get_connector_config(self, connector: str) -> Optional[dict]: """ @@ -319,7 +319,7 @@ def extract_column_mappings(self, connector_config: dict) -> Optional[List[Kafka except (KeyError, AttributeError, ValueError) as exc: logger.debug(traceback.format_exc()) - logger.warning(f"Unable to extract column mappings: {exc}") + logger.error(f"Unable to extract column mappings: {exc}") return None @@ -366,7 +366,7 @@ def get_connector_topics(self, connector: str) -> Optional[List[KafkaConnectTopi return topics except Exception as exc: logger.debug(traceback.format_exc()) - logger.warning(f"Unable to get connector Topics {exc}") + logger.error(f"Unable to get connector Topics {exc}") return None diff --git a/ingestion/src/metadata/ingestion/source/pipeline/kafkaconnect/metadata.py b/ingestion/src/metadata/ingestion/source/pipeline/kafkaconnect/metadata.py index dff6cab05853..feaa1031c59d 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/kafkaconnect/metadata.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/kafkaconnect/metadata.py @@ -213,7 +213,7 @@ def find_database_service_by_hostname(self, service_type: str, hostname: str) -> except Exception as exc: logger.debug(traceback.format_exc()) - logger.warning(f"Unable to find database service by hostname: {exc}") + logger.error(f"Unable to find database service by hostname: {exc}") return None def find_messaging_service_by_brokers(self, brokers: str) -> Optional[str]: @@ -264,7 +264,7 @@ def find_messaging_service_by_brokers(self, brokers: str) -> Optional[str]: except Exception as exc: logger.debug(traceback.format_exc()) - logger.warning(f"Unable to find messaging service by brokers: {exc}") + logger.error(f"Unable to find messaging service by brokers: {exc}") return None def get_service_from_connector_config( @@ -593,7 +593,7 @@ def get_dataset_entity( except Exception as exc: logger.debug(traceback.format_exc()) - logger.warning(f"Unable to get dataset entity {exc}") + logger.error(f"Unable to get dataset entity {exc}") return None @@ -900,7 +900,7 @@ def build_column_lineage( except Exception as exc: logger.debug(traceback.format_exc()) - logger.warning(f"Unable to build column lineage: {exc}") + logger.error(f"Unable to build column lineage: {exc}") return None @@ -973,7 +973,7 @@ def _search_topics_by_prefix( except Exception as exc: logger.debug(traceback.format_exc()) - logger.warning(f"Unable to search topics by prefix: {exc}") + logger.error(f"Unable to search topics by prefix: {exc}") return topics_found @@ -1024,7 +1024,7 @@ def _search_topics_by_regex( logger.warning(f"Invalid regex pattern '{topics_regex}': {exc}") except Exception as exc: logger.debug(traceback.format_exc()) - logger.warning(f"Unable to search topics by regex: {exc}") + logger.error(f"Unable to search topics by regex: {exc}") return topics_found diff --git a/ingestion/src/metadata/ingestion/source/pipeline/spline/metadata.py b/ingestion/src/metadata/ingestion/source/pipeline/spline/metadata.py index bd67ce9c751d..c9146ba330d5 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/spline/metadata.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/spline/metadata.py @@ -138,7 +138,7 @@ def _get_table_from_datasource_name(self, datasource: str) -> Optional[Table]: except Exception as exc: logger.debug(traceback.format_exc()) - logger.warning(f"failed to parse datasource details due to: {exc}") + logger.error(f"failed to parse datasource details due to: {exc}") return None diff --git a/ingestion/src/metadata/ingestion/source/pipeline/spline/utils.py b/ingestion/src/metadata/ingestion/source/pipeline/spline/utils.py index 9a0644f132be..7a3f22fbdba0 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/spline/utils.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/spline/utils.py @@ -34,7 +34,7 @@ def parse_dbfs_path(path: str) -> Optional[str]: try: return path.split("/")[-1] except Exception as exc: - logger.warning(f"Failed to parse dbfs: {exc}") + logger.error(f"Failed to parse dbfs: {exc}") logger.error(traceback.format_exc()) return None @@ -71,7 +71,7 @@ def parse_jdbc_url(url: str) -> Tuple[Optional[str], Optional[str], Optional[str return DEFAULT_DATABASE, database, table except Exception as exc: - logger.warning(f"Failed to parse jdbc url: {exc}") + logger.error(f"Failed to parse jdbc url: {exc}") logger.error(traceback.format_exc()) return None, None, None diff --git a/ingestion/src/metadata/ingestion/source/search/elasticsearch/parser.py b/ingestion/src/metadata/ingestion/source/search/elasticsearch/parser.py index 00b57533c6d8..0add16b2846a 100644 --- a/ingestion/src/metadata/ingestion/source/search/elasticsearch/parser.py +++ b/ingestion/src/metadata/ingestion/source/search/elasticsearch/parser.py @@ -52,6 +52,6 @@ def parse_es_index_mapping(mapping: dict) -> Optional[List[SearchIndexField]]: ) except Exception as exc: # pylint: disable=broad-except logger.debug(traceback.format_exc()) - logger.warning(f"Unable to parse the index properties: {exc}") + logger.error(f"Unable to parse the index properties: {exc}") return field_models diff --git a/ingestion/src/metadata/ingestion/source/search/opensearch/parser.py b/ingestion/src/metadata/ingestion/source/search/opensearch/parser.py index bb61edbf98bc..ae4a66cdc9f6 100644 --- a/ingestion/src/metadata/ingestion/source/search/opensearch/parser.py +++ b/ingestion/src/metadata/ingestion/source/search/opensearch/parser.py @@ -60,6 +60,6 @@ def parse_os_index_mapping(mapping: dict) -> Optional[List[SearchIndexField]]: ) except Exception as exc: # pylint: disable=broad-except logger.debug(traceback.format_exc()) - logger.warning(f"Unable to parse the index properties: {exc}") + logger.error(f"Unable to parse the index properties: {exc}") return field_models diff --git a/ingestion/src/metadata/ingestion/source/storage/s3/metadata.py b/ingestion/src/metadata/ingestion/source/storage/s3/metadata.py index bc4121af6a61..bf72a65fe5d1 100644 --- a/ingestion/src/metadata/ingestion/source/storage/s3/metadata.py +++ b/ingestion/src/metadata/ingestion/source/storage/s3/metadata.py @@ -688,7 +688,7 @@ def get_aws_bucket_region(self, bucket_name: str) -> str: region = region_resp.get("LocationConstraint") except Exception: logger.debug(traceback.format_exc()) - logger.warning(f"Unable to get the region for bucket: {bucket_name}") + logger.error(f"Unable to get the region for bucket: {bucket_name}") return region or self.service_connection.awsConfig.awsRegion def _get_bucket_source_url(self, bucket_name: str) -> Optional[str]: diff --git a/ingestion/src/metadata/pii/scanners/ner_scanner.py b/ingestion/src/metadata/pii/scanners/ner_scanner.py index 3f71b1bbe369..73a2eb5cd36d 100644 --- a/ingestion/src/metadata/pii/scanners/ner_scanner.py +++ b/ingestion/src/metadata/pii/scanners/ner_scanner.py @@ -112,7 +112,7 @@ def scan(self, data: List[Any]) -> Optional[TagAndConfidence]: try: self.process_data(row=row, entities_score=entities_score) except Exception as exc: - logger.warning(f"Unknown error while processing {row} - {exc}") + logger.error(f"Unknown error while processing {row} - {exc}") logger.debug(traceback.format_exc()) if entities_score: diff --git a/ingestion/tests/unit/source/mcp/test_mcp_client.py b/ingestion/tests/unit/source/mcp/test_mcp_client.py index 4df8f6be28c7..5822087a87ce 100644 --- a/ingestion/tests/unit/source/mcp/test_mcp_client.py +++ b/ingestion/tests/unit/source/mcp/test_mcp_client.py @@ -175,8 +175,8 @@ def test_send_notification_logs_on_failure(self, mock_post): with patch("metadata.ingestion.source.mcp.client.logger") as mock_logger: transport.send_notification("notifications/initialized", {}) - mock_logger.warning.assert_called_once() - assert "server down" in str(mock_logger.warning.call_args) + mock_logger.error.assert_called_once() + assert "server down" in str(mock_logger.error.call_args) class TestMcpClient: