Merge branch 'main' into MINOR-Fix-Logging

Author: TeddyCr

.pre-commit-config.yaml

@@ -2,28 +2,30 @@ default_language_version:
   python: python3
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.3.0
+    rev: v5.0.0
     hooks:
       - id: check-json
-        exclude: vscode
-  - repo: https://github.com/hadialqattan/pycln
-    rev: v2.5.0
+        # TODO: investigate and fix or remove the excluded files. The first
+        # three carry real JSON issues (duplicate keys, malformed/empty
+        # content) that pre-commit-hooks v2.3.0 didn't catch; v5.0.0 does.
+        # The last is an intentionally malformed test fixture.
+        exclude: |
+          (?x)^(
+            .*vscode.*|
+            openmetadata-spec/src/main/resources/rdf/contexts/dataAsset\.jsonld|
+            ingestion/examples/sample_data/pipelines/tasks\.json|
+            openmetadata-service/src/main/resources/dataInsights/opensearch/indexSettingsTemplate\.json|
+            openmetadata-ui/src/main/resources/ui/playwright/test-data/odcs-examples/invalid-malformed\.json
+          )$
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.15.12
     hooks:
-      - id: pycln
+      - id: ruff-check
         files: ^(ingestion|openmetadata-airflow-apis)/
-        args: [ "--config", "ingestion/pyproject.toml" ]
-  - repo: https://github.com/timothycrosley/isort
-    rev: 5.12.0
-    hooks:
-      - id: isort
-        files: ^(ingestion|openmetadata-airflow-apis)/
-        args: [ "--settings-file", "ingestion/pyproject.toml" ]
-  - repo: https://github.com/ambv/black
-    rev: 22.3.0
-    hooks:
-      - id: black
+        args: ["--fix", "--config", "ingestion/pyproject.toml"]
+      - id: ruff-format
         files: ^(ingestion|openmetadata-airflow-apis)/
-        args: [ "--config", "ingestion/pyproject.toml" ]
+        args: ["--config", "ingestion/pyproject.toml"]
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: v2.5.1
     hooks:

ingestion/Makefile

@@ -62,16 +62,14 @@ precommit_install:  ## Install the project's precommit hooks from .pre-commit-co
 	pre-commit install
 
 .PHONY: py_format
-py_format:  ## Run black and isort to format the Python codebase
-	pycln $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --config $(INGESTION_DIR)/pyproject.toml
-	isort $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --settings-file $(INGESTION_DIR)/pyproject.toml
-	black $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --config $(INGESTION_DIR)/pyproject.toml
+py_format:  ## Run ruff to lint-fix and format the Python codebase
+	ruff check --fix $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --config $(INGESTION_DIR)/pyproject.toml
+	ruff format $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --config $(INGESTION_DIR)/pyproject.toml
 
 .PHONY: py_format_check
 py_format_check:  ## Check if Python sources are correctly formatted
-	pycln $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --diff --config $(INGESTION_DIR)/pyproject.toml
-	isort --check-only $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --settings-file $(INGESTION_DIR)/pyproject.toml
-	black --check --diff $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --config $(INGESTION_DIR)/pyproject.toml
+	ruff check $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --config $(INGESTION_DIR)/pyproject.toml
+	ruff format --check $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --config $(INGESTION_DIR)/pyproject.toml
 	PYTHONPATH="${PYTHONPATH}:$(INGESTION_DIR)/plugins" pylint --rcfile=$(INGESTION_DIR)/pyproject.toml --fail-under=10 $(PY_SOURCE)/metadata || (echo "PyLint error code $$?"; exit 1)
 
 .PHONY: unit_ingestion

ingestion/examples/airflow/dags/airflow_docker_operator.py

@@ -11,6 +11,7 @@
 """
 You can run this DAG from the default OM installation
 """
+
 from datetime import datetime
 
 from airflow import models

ingestion/examples/airflow/dags/airflow_lineage_example.py

@@ -29,7 +29,6 @@
 This is an example to demonstrate on how to configure a Airflow DAG's inlets and outlets.
 """
 
-
 from datetime import datetime, timedelta
 
 from airflow.decorators import dag, task
@@ -76,9 +75,7 @@ def generate_data():
 
     # Example 2: Using OMEntity objects
     @task(
-        inlets=[
-            OMEntity(entity=Container, fqn="s3_storage_sample.transactions", key="test")
-        ],
+        inlets=[OMEntity(entity=Container, fqn="s3_storage_sample.transactions", key="test")],
         outlets=[
             OMEntity(
                 entity=Table,

ingestion/examples/airflow/dags/airflow_metadata_extraction.py

@@ -15,6 +15,7 @@
 is to connect to the underlying database, retrieve the information
 and push it to OpenMetadata.
 """
+
 from datetime import datetime, timedelta
 
 import yaml

ingestion/examples/airflow/test_dags/airflow_lineage_operator.py

@@ -14,6 +14,7 @@
 For this DAG to run properly we expected an OpenMetadata
 Airflow connection named `openmetadata_conn_id`.
 """
+
 from datetime import datetime
 from textwrap import dedent
 
@@ -152,13 +153,7 @@ def get_airflow_headers():
     t1 = BashOperator(
         task_id="print_date",
         bash_command="date",
-        outlets=[
-            {
-                "tables": [
-                    "test-service-table-lineage.test-db.test-schema.lineage-test-outlet"
-                ]
-            }
-        ],
+        outlets=[{"tables": ["test-service-table-lineage.test-db.test-schema.lineage-test-outlet"]}],
     )
 
     t2 = BashOperator(
@@ -176,9 +171,7 @@ def get_airflow_headers():
         ],
     )
 
-    dag.doc_md = (
-        __doc__  # providing that you have a docstring at the beginning of the DAG
-    )
+    dag.doc_md = __doc__  # providing that you have a docstring at the beginning of the DAG
     dag.doc_md = """
     This is a documentation placed anywhere
     """  # otherwise, type it like this

ingestion/noxfile.py

@@ -11,6 +11,7 @@
 """
 Nox sessions for testing and formatting checks.
 """
+
 import os
 
 import nox
@@ -62,9 +63,7 @@ def lint(session):
     #   Some work is required to import plugins correctly
 
 
-@nox.session(
-    name="unit", reuse_venv=True, venv_backend="uv|venv", python=get_python_versions()
-)
+@nox.session(name="unit", reuse_venv=True, venv_backend="uv|venv", python=get_python_versions())
 def unit(session):
     session.install(".[all-dev-env, test-unit]")
     # TODO: we need to install pip so that spaCy can install its dependencies