Merge branch 'main' into feat/add-testsuites-to-createtestcase-api

Author: TeddyCr

.github/workflows/py-cli-e2e-tests.yml

@@ -283,18 +283,18 @@ jobs:
 
       - name: Generate report
         run: |
+          # Copy coverage artifacts into ingestion/ so paths resolve relative to it
           for folder in artifacts/coverage-*; do
-            cp -rT $folder/ . ;
+            cp -rT $folder/ ingestion/ ;
           done
-          mkdir ingestion/junit
+          mkdir -p ingestion/junit
           for folder in artifacts/tests-*; do
             cp -rT $folder/ ingestion/junit ;
           done
           source env/bin/activate
-          coverage combine --rcfile=ingestion/pyproject.toml --keep -a .coverage*
-          coverage xml --rcfile=ingestion/pyproject.toml --data-file=.coverage -o ingestion/coverage.xml
-          sed -e "s/$(python -c "import site; import os; from pathlib import Path; print(os.path.relpath(site.getsitepackages()[0], str(Path.cwd())).replace('/','\/'))")/src/g" ingestion/coverage.xml >> ingestion/ci-coverage.xml
-          sed -i 's/src\/metadata/\/github\/workspace\/ingestion\/src\/metadata/g' ingestion/ci-coverage.xml
+          cd ingestion
+          coverage combine --rcfile=pyproject.toml --keep -a .coverage*
+          coverage xml --rcfile=pyproject.toml --data-file=.coverage
         shell: bash
 
       - name: Push Results to Sonar

ingestion/src/metadata/ingestion/api/step.py

@@ -23,7 +23,7 @@
 from metadata.ingestion.api.models import Either, Entity, StackTraceError
 from metadata.ingestion.api.status import Status
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
-from metadata.utils.logger import ingestion_logger
+from metadata.utils.logger import StatusWarningHandler, ingestion_logger
 from metadata.utils.operation_metrics import OperationMetricsState
 from metadata.utils.progress_tracker import ProgressTrackerState
 
@@ -45,6 +45,24 @@ class Step(ABC, Closeable):
 
     def __init__(self):
         self.status = Status()
+        self._warning_handler = StatusWarningHandler(self.status)
+
+    def _activate_handler(self) -> None:
+        """Attach the warning handler to the ingestion logger.
+
+        Called at the start of each run() so that warnings emitted
+        during step execution are captured in the step's Status.
+        Must be paired with _deactivate_handler in a finally block.
+        """
+        ingestion_logger().addHandler(self._warning_handler)
+
+    def _deactivate_handler(self) -> None:
+        """Remove the warning handler from the ingestion logger.
+
+        Called in the finally block of run() to ensure the handler
+        does not leak across step boundaries.
+        """
+        ingestion_logger().removeHandler(self._warning_handler)
 
     @classmethod
     @abstractmethod
@@ -128,6 +146,7 @@ def run(self, record: Entity) -> Optional[Entity]:
         """
         Run the step and handle the status and exceptions
         """
+        self._activate_handler()
         try:
             result: Either = self._run(record)
             if result:
@@ -162,6 +181,8 @@ def run(self, record: Entity) -> Optional[Entity]:
                     name="Unhandled", error=error, stackTrace=traceback.format_exc()
                 )
             )
+        finally:
+            self._deactivate_handler()
 
         return None
 
@@ -186,6 +207,7 @@ def run(self, record: Entity) -> None:
         """
         Run the step and handle the status and exceptions.
         """
+        self._activate_handler()
         try:
             for result in self._run(record):
                 if result.left is not None:
@@ -217,6 +239,8 @@ def run(self, record: Entity) -> None:
                     name="Unhandled", error=error, stackTrace=traceback.format_exc()
                 )
             )
+        finally:
+            self._deactivate_handler()
 
 
 class IterStep(Step, ABC):
@@ -233,6 +257,7 @@ def run(self) -> Iterable[Optional[Entity]]:
         Note that we are overwriting the default run implementation
         in order to create a generator with `yield`.
         """
+        self._activate_handler()
         try:
             for result in self._iter():
                 if result.left is not None:
@@ -266,6 +291,8 @@ def run(self) -> Iterable[Optional[Entity]]:
                     name="Unhandled", error=error, stackTrace=traceback.format_exc()
                 )
             )
+        finally:
+            self._deactivate_handler()
 
 
 class BulkStep(Step, ABC):

ingestion/src/metadata/ingestion/api/steps.py

@@ -94,3 +94,15 @@ class BulkSink(BulkStep, ABC):
     @property
     def name(self) -> str:
         return "BulkSink"
+
+    def run(self) -> None:
+        """Wrap the abstract run() with the warning handler lifecycle.
+
+        BulkStep.run() is abstract and overridden directly by
+        concrete subclasses, so the handler must be scoped here.
+        """
+        self._activate_handler()
+        try:
+            super().run()
+        finally:
+            self._deactivate_handler()

ingestion/src/metadata/ingestion/ometa/ometa_api.py

@@ -429,6 +429,13 @@ def update_file_name(create: Type[C], file_name: str) -> str:
         """
         Update the filename for services and schemas
         """
+        explicit_file_name_map = {
+            "CreateTableProfileRequest": "table",
+            "CreateTestCaseResult": "basic",
+        }
+        if create.__name__ in explicit_file_name_map:
+            return explicit_file_name_map[create.__name__]
+
         if "service" in create.__name__.lower():
             return file_name.replace("service", "Service")
 

ingestion/src/metadata/ingestion/source/database/azuresql/metadata.py

@@ -43,7 +43,7 @@
 
 logger = ingestion_logger()
 
-ischema_names = update_mssql_ischema_names(ischema_names)
+update_mssql_ischema_names(ischema_names)
 
 MSDialect.get_table_comment = get_table_comment
 MSDialect.get_view_definition = get_view_definition