From 83691c19fee0a6a08f0bffb9fd1cbd440ab3ba02 Mon Sep 17 00:00:00 2001 From: M Abulazm Date: Mon, 4 May 2026 14:49:18 +0200 Subject: [PATCH 01/79] Migrates the reconcile integration tests to Databricks away from local spark (#2394) ### What does this PR do? Migrates the reconcile integration tests off local Spark and onto the Databricks `spark` fixture (UC), and removes the no-longer-needed `local_test_run` plumbing that existed only to support that local-Spark path. ### Relevant implementation details - Replaces the local `mock_spark` fixture (Spark Connect to `sc://localhost`) with the `spark` fixture from `databricks-labs-pytester`, so all tests run against Databricks Spark with UC. `DELTA_FAILED_TO_MERGE_FIELDS` from the hand-rolled `report_tables_schema`). - Drops the `local_test_run` flag - Metadata tables are always addressed as `catalog.schema.`. Tests updated accordingly (every assertion now reads `{catalog}.{schema}.MAIN` and derived from a real `make_schema` + `make_volume` - Skips tests that depend on currently-unavailable infra (Snowflake) ### Caveats/things to watch out for when reviewing: - Integration tests now require a Databricks workspace + UC. They will not run against a bare local Spark anymore. - snowflake infra is not working and respective integration tests are ignored ### Functionality - [ ] added relevant user documentation - [ ] added new CLI command - [ ] modified existing command: `databricks labs lakebridge ...` ### Tests - [x] manually tested - [ ] added unit tests - [x] added integration tests --------- Co-authored-by: Andrew Snare Co-authored-by: Andrew Snare --- .github/scripts/setup_spark_remote.sh | 97 ---- .github/workflows/acceptance.yml | 9 - Makefile | 5 +- pyproject.toml | 2 +- .../lakebridge/reconcile/recon_capture.py | 21 +- .../trigger_recon_aggregate_service.py | 12 +- .../reconcile/trigger_recon_service.py | 15 +- tests/conftest.py | 9 +- tests/integration/conftest.py | 10 - tests/integration/reconcile/conftest.py | 36 +- .../connectors/test_mock_data_source.py | 14 +- .../reconcile/connectors/test_read_schema.py | 25 +- .../reconcile/query_builder/test_execute.py | 543 +++++++++--------- .../query_builder/test_sampling_query.py | 15 +- .../test_aggregates_recon_capture.py | 56 +- .../reconcile/test_aggregates_reconcile.py | 28 +- tests/integration/reconcile/test_compare.py | 64 +-- .../reconcile/test_oracle_reconcile.py | 13 +- .../reconcile/test_recon_capture.py | 150 ++--- tests/integration/reconcile/test_recon_e2e.py | 4 + tests/integration/reconcile/test_sampler.py | 24 +- .../reconcile/test_schema_compare.py | 15 +- 22 files changed, 483 insertions(+), 684 deletions(-) delete mode 100755 .github/scripts/setup_spark_remote.sh diff --git a/.github/scripts/setup_spark_remote.sh b/.github/scripts/setup_spark_remote.sh deleted file mode 100755 index 5323744e29..0000000000 --- a/.github/scripts/setup_spark_remote.sh +++ /dev/null @@ -1,97 +0,0 @@ -#!/bin/bash -# -# TODO: Replace this script with a GHA service container on the workflow. -# -# Set up Apache Spark in local Connect mode for integration tests. -# -# All downloads are pinned to exact versions and verified against embedded -# cryptographic checksums before use. -# -set -eu - -mkdir -p "$HOME"/spark -cd "$HOME"/spark || exit 1 - -# Spark Connect server still points to 3.5.5 -spark_version="3.5.5" -spark="spark-${spark_version}-bin-hadoop3" -spark_connect="spark-connect_2.12" -spark_sha256='8daa3f7fb0af2670fe11beb8a2ac79d908a534d7298353ec4746025b102d5e31' - -mssql_jdbc_version="1.4.0" -mssql_jdbc="spark-mssql-connector_2.12-${mssql_jdbc_version}-BETA" -mssql_jdbc_sha256="1057e93d946dffd2ecac1c11bcc40fdf51110c5a99e9e7379a9568584cc3de7f" -ORACLE_JDBC_VERSION="19.28.0.0" -SNOWFLAKE_JDBC_VERSION="3.26.1" -SNOWFLAKE_SPARK_VERSION="2.11.2-spark_3.3" - -mkdir -p "${spark}" -SERVER_SCRIPT=$HOME/spark/${spark}/sbin/start-connect-server.sh -JARS_DIR=$HOME/spark/${spark}/jars - -if [ -f "${SERVER_SCRIPT}" ]; then - printf "Spark %s already exists\n" "${spark_version}" -else - spark_tarball="${spark}.tgz" - if [ ! -f "${spark_tarball}" ]; then - printf "Downloading Spark %s...\n" "${spark_version}" - curl -fsSL "https://archive.apache.org/dist/spark/spark-${spark_version}/${spark_tarball}" -o "${spark_tarball}" - fi - printf '%s %s\n' "${spark_sha256}" "${spark_tarball}" | sha256sum -c >/dev/null - tar -xf "${spark_tarball}" -fi - -printf "Downloading JDBC JARs and dependencies via Maven...\n" - -for artifact in \ - com.microsoft.azure:adal4j:1.6.4:jar \ - com.nimbusds:oauth2-oidc-sdk:6.5:jar \ - com.google.code.gson:gson:2.8.0:jar \ - net.minidev:json-smart:1.3.1:jar \ - com.nimbusds:nimbus-jose-jwt:8.2.1:jar \ - org.slf4j:slf4j-api:1.7.21:jar \ - com.microsoft.sqlserver:mssql-jdbc:6.4.0.jre8:jar \ - com.oracle.database.jdbc:ojdbc8:${ORACLE_JDBC_VERSION}:jar \ - net.snowflake:snowflake-jdbc:${SNOWFLAKE_JDBC_VERSION}:jar \ - net.snowflake:spark-snowflake_2.12:${SNOWFLAKE_SPARK_VERSION}:jar -do - mvn dependency:copy -q --strict-checksums -DoutputDirectory="$JARS_DIR" -Dartifact="${artifact}" & -done -wait - -# The sql-spark-connector is not on Maven Central; download from GitHub. -curl -fsSL -o "$JARS_DIR/${mssql_jdbc}.jar" \ - "https://github.com/microsoft/sql-spark-connector/releases/download/v${mssql_jdbc_version}/${mssql_jdbc}.jar" -printf '%s %s\n' "${mssql_jdbc_sha256}" "$JARS_DIR/${mssql_jdbc}.jar" | sha256sum -c >/dev/null - -# --- Start Spark Connect server --- - -rm -rf "${HOME}"/spark/"${spark}"/spark-warehouse -printf "Cleared old spark warehouse default directory\n" - -cd "${spark}" || exit 1 -result=$(${SERVER_SCRIPT} --packages "org.apache.spark:${spark_connect}:${spark_version}" > "$HOME"/spark/log.out; echo $?) - -if [ "$result" -ne 0 ]; then - count=$(tail "${HOME}"/spark/log.out | grep -c "SparkConnectServer running as process") - if [ "${count}" == "0" ]; then - printf "Failed to start the server\n" - exit 1 - fi - # Wait for the server to start by pinging localhost:4040 - printf "Waiting for the server to start...\n" - for i in {1..30}; do - if nc -z localhost 4040; then - printf "Server is up and running\n" - break - fi - printf "Server not yet available, retrying in 5 seconds...\n" - sleep 5 - done - - if ! nc -z localhost 4040; then - printf "Failed to start the server within the expected time\n" - exit 1 - fi -fi -printf "Started the Server\n" diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml index c4804507f1..64247175c1 100644 --- a/.github/workflows/acceptance.yml +++ b/.github/workflows/acceptance.yml @@ -47,15 +47,6 @@ jobs: chmod +x $GITHUB_WORKSPACE/.github/scripts/setup_mssql_odbc.sh $GITHUB_WORKSPACE/.github/scripts/setup_mssql_odbc.sh - # TODO: Migrate tests to use Databricks clusters instead of Spark local mode - # Disabled for now, because access to archives.apache.org is blocked; a new approach will be needed for this. - # (The integration tests will still run, but many will fail due to this.) - - name: Setup spark - if: false - run: | - chmod +x $GITHUB_WORKSPACE/.github/scripts/setup_spark_remote.sh - $GITHUB_WORKSPACE/.github/scripts/setup_spark_remote.sh - - name: Run integration tests uses: databrickslabs/sandbox/acceptance@3313d06ce86227537b3f37f5974f7eecb2a8e59a # acceptance/v0.4.4 with: diff --git a/Makefile b/Makefile index d3c9da0ed4..21f5189c0d 100644 --- a/Makefile +++ b/Makefile @@ -37,13 +37,10 @@ fmt: $(UV_RUN) mypy --disable-error-code 'annotation-unchecked' . $(UV_RUN) pylint --output-format=colorized -j 0 src tests -setup_spark_remote: - .github/scripts/setup_spark_remote.sh - test: $(UV_TEST) --cov-report=xml tests/unit -integration: setup_spark_remote +integration: $(UV_TEST) tests/integration coverage: diff --git a/pyproject.toml b/pyproject.toml index 922062167a..77f56d080b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -480,7 +480,7 @@ max-bool-expr = 5 max-branches = 20 # Maximum number of locals for function / method body. -max-locals = 20 +max-locals = 21 # Maximum number of parents for a class (see R0901). max-parents = 7 diff --git a/src/databricks/labs/lakebridge/reconcile/recon_capture.py b/src/databricks/labs/lakebridge/reconcile/recon_capture.py index f9425cc3a2..2d665fa12d 100644 --- a/src/databricks/labs/lakebridge/reconcile/recon_capture.py +++ b/src/databricks/labs/lakebridge/reconcile/recon_capture.py @@ -2,7 +2,7 @@ import os import tempfile import uuid -from datetime import datetime +from datetime import datetime, timezone from functools import reduce, cached_property from pathlib import Path @@ -126,9 +126,8 @@ def generate_final_reconcile_output( recon_id: str, spark: SparkSession, metadata_config: ReconcileMetadataConfig = ReconcileMetadataConfig(), - local_test_run: bool = False, ) -> ReconcileOutput: - _db_prefix = metadata_config.schema if local_test_run else f"{metadata_config.catalog}.{metadata_config.schema}" + _db_prefix = f"{metadata_config.catalog}.{metadata_config.schema}" recon_df = spark.sql( f""" SELECT @@ -196,9 +195,8 @@ def generate_final_reconcile_aggregate_output( recon_id: str, spark: SparkSession, metadata_config: ReconcileMetadataConfig = ReconcileMetadataConfig(), - local_test_run: bool = False, ) -> ReconcileOutput: - _db_prefix = "default" if local_test_run else f"{metadata_config.catalog}.{metadata_config.schema}" + _db_prefix = f"{metadata_config.catalog}.{metadata_config.schema}" recon_df = spark.sql( f""" SELECT source_table, @@ -262,7 +260,6 @@ def __init__( ws: WorkspaceClient, spark: SparkSession, metadata_config: ReconcileMetadataConfig = ReconcileMetadataConfig(), - local_test_run: bool = False, ): self.database_config = database_config self.recon_id = recon_id @@ -270,9 +267,7 @@ def __init__( self.source_dialect = source_dialect self.ws = ws self.spark = spark - self._db_prefix = ( - metadata_config.schema if local_test_run else f"{metadata_config.catalog}.{metadata_config.schema}" - ) + self._db_prefix = f"{metadata_config.catalog}.{metadata_config.schema}" def _generate_recon_main_id( self, @@ -384,7 +379,7 @@ def _insert_into_metrics_table( if data_reconcile_output.exception is not None: exception_msg = data_reconcile_output.exception.replace("'", '').replace('"', '') - insertion_time = str(datetime.now()) + insertion_time = str(datetime.now(tz=timezone.utc)) mismatch_columns = [] if data_reconcile_output.mismatch and data_reconcile_output.mismatch.mismatch_columns: mismatch_columns = data_reconcile_output.mismatch.mismatch_columns @@ -441,7 +436,7 @@ def _create_map_column( df.withColumn("recon_table_id", lit(recon_table_id)) .withColumn("recon_type", lit(recon_type)) .withColumn("status", lit(status)) - .withColumn("inserted_ts", lit(datetime.now())) + .withColumn("inserted_ts", lit(datetime.now(tz=timezone.utc))) ) return ( df.groupBy("recon_table_id", "recon_type", "status", "inserted_ts") @@ -557,7 +552,7 @@ def _insert_aggregates_into_metrics_table( if agg_data.exception is not None: exception_msg = agg_data.exception.replace("'", '').replace('"', '') - insertion_time = str(datetime.now()) + insertion_time = str(datetime.now(tz=timezone.utc)) # If there is any exception while running the Query, # each rule is stored, with the Exception message in the metrics table @@ -667,7 +662,7 @@ def _insert_into_rules_table(self, recon_table_id: int, reconcile_agg_output_lis rule_query = agg_output.rule.get_rule_query(rule_id) rule_df_list.append( self.spark.sql(rule_query) - .withColumn("inserted_ts", lit(datetime.now())) + .withColumn("inserted_ts", lit(datetime.now(tz=timezone.utc))) .select("rule_id", "rule_type", "rule_info", "inserted_ts") ) diff --git a/src/databricks/labs/lakebridge/reconcile/trigger_recon_aggregate_service.py b/src/databricks/labs/lakebridge/reconcile/trigger_recon_aggregate_service.py index 1573f48e8e..94682512b5 100644 --- a/src/databricks/labs/lakebridge/reconcile/trigger_recon_aggregate_service.py +++ b/src/databricks/labs/lakebridge/reconcile/trigger_recon_aggregate_service.py @@ -1,5 +1,5 @@ import logging -from datetime import datetime +from datetime import datetime, timezone from pyspark.sql import SparkSession from databricks.sdk import WorkspaceClient @@ -31,11 +31,8 @@ def trigger_recon_aggregates( spark: SparkSession, table_recon: TableRecon, reconcile_config: ReconcileConfig, - local_test_run: bool = False, ) -> ReconcileOutput: - reconciler, recon_capture = TriggerReconService.create_recon_dependencies( - ws, spark, reconcile_config, local_test_run - ) + reconciler, recon_capture = TriggerReconService.create_recon_dependencies(ws, spark, reconcile_config) try: for table_conf in table_recon.tables: @@ -48,7 +45,6 @@ def trigger_recon_aggregates( recon_id=recon_capture.recon_id, spark=spark, metadata_config=reconcile_config.metadata_config, - local_test_run=local_test_run, ), report_type="aggregate", ) @@ -68,7 +64,7 @@ def recon_aggregate_one( if not normalized_table_conf.aggregates: raise ValueError("Aggregates must be defined for Aggregates Reconciliation") - recon_process_duration = ReconcileProcessDuration(start_ts=str(datetime.now()), end_ts=None) + recon_process_duration = ReconcileProcessDuration(start_ts=str(datetime.now(tz=timezone.utc)), end_ts=None) try: src_schema, tgt_schema = TriggerReconService.get_schemas( reconciler.source, reconciler.target, normalized_table_conf, reconcile_config.database_config, True @@ -82,7 +78,7 @@ def recon_aggregate_one( AggregateQueryOutput(reconcile_output=DataReconcileOutput(exception=str(e)), rule=None) ] - recon_process_duration.end_ts = str(datetime.now()) + recon_process_duration.end_ts = str(datetime.now(tz=timezone.utc)) recon_capture.store_aggregates_metrics( reconcile_agg_output_list=table_reconcile_agg_output_list, diff --git a/src/databricks/labs/lakebridge/reconcile/trigger_recon_service.py b/src/databricks/labs/lakebridge/reconcile/trigger_recon_service.py index 913768a128..3438cea994 100644 --- a/src/databricks/labs/lakebridge/reconcile/trigger_recon_service.py +++ b/src/databricks/labs/lakebridge/reconcile/trigger_recon_service.py @@ -1,5 +1,5 @@ import logging -from datetime import datetime +from datetime import datetime, timezone from uuid import uuid4 from pyspark.errors import PySparkException @@ -42,11 +42,8 @@ def trigger_recon( spark: SparkSession, table_recon: TableRecon, reconcile_config: ReconcileConfig, - local_test_run: bool = False, ) -> ReconcileOutput: - reconciler, recon_capture = TriggerReconService.create_recon_dependencies( - ws, spark, reconcile_config, local_test_run - ) + reconciler, recon_capture = TriggerReconService.create_recon_dependencies(ws, spark, reconcile_config) try: for table_conf in table_recon.tables: @@ -57,7 +54,6 @@ def trigger_recon( recon_id=recon_capture.recon_id, spark=spark, metadata_config=reconcile_config.metadata_config, - local_test_run=local_test_run, ), reconcile_config.report_type, ) @@ -69,7 +65,7 @@ def trigger_recon( @staticmethod def create_recon_dependencies( - ws: WorkspaceClient, spark: SparkSession, reconcile_config: ReconcileConfig, local_test_run: bool = False + ws: WorkspaceClient, spark: SparkSession, reconcile_config: ReconcileConfig ) -> tuple[Reconciliation, ReconCapture]: ws_client: WorkspaceClient = verify_workspace_client(ws) @@ -107,7 +103,6 @@ def create_recon_dependencies( ws=ws_client, spark=spark, metadata_config=reconcile_config.metadata_config, - local_test_run=local_test_run, ) return reconciler, recon_capture @@ -139,7 +134,7 @@ def recon_one( @staticmethod def _do_recon_one(reconciler: Reconciliation, reconcile_config: ReconcileConfig, table_conf: Table): - recon_process_duration = ReconcileProcessDuration(start_ts=str(datetime.now()), end_ts=None) + recon_process_duration = ReconcileProcessDuration(start_ts=str(datetime.now(tz=timezone.utc)), end_ts=None) schema_reconcile_output = SchemaReconcileOutput(is_valid=True) data_reconcile_output = DataReconcileOutput() @@ -168,7 +163,7 @@ def _do_recon_one(reconciler: Reconciliation, reconcile_config: ReconcileConfig, ) logger.info(f"Reconciliation for '{reconciler.report_type}' report completed.") - recon_process_duration.end_ts = str(datetime.now()) + recon_process_duration.end_ts = str(datetime.now(tz=timezone.utc)) return schema_reconcile_output, data_reconcile_output, recon_process_duration @staticmethod diff --git a/tests/conftest.py b/tests/conftest.py index 86f78b5059..c9acef9a09 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,7 +9,6 @@ LongType, StringType, TimestampType, - IntegerType, BooleanType, ArrayType, MapType, @@ -196,8 +195,8 @@ def report_tables_schema(): "row_comparison", StructType( [ - StructField("missing_in_source", IntegerType()), - StructField("missing_in_target", IntegerType()), + StructField("missing_in_source", LongType()), + StructField("missing_in_target", LongType()), ] ), ), @@ -205,8 +204,8 @@ def report_tables_schema(): "column_comparison", StructType( [ - StructField("absolute_mismatch", IntegerType()), - StructField("threshold_mismatch", IntegerType()), + StructField("absolute_mismatch", LongType()), + StructField("threshold_mismatch", LongType()), StructField("mismatch_columns", StringType()), ] ), diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 2c46d1935c..4e1437eadb 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -5,7 +5,6 @@ from uuid import UUID import pytest -from pyspark.sql import SparkSession from databricks.labs.blueprint.paths import WorkspacePath from databricks.labs.blueprint.wheels import ProductInfo @@ -56,15 +55,6 @@ def get_logger(): return logger -@pytest.fixture(scope="session") -def mock_spark() -> SparkSession: - """ - Method helps to create spark session - :return: returns the spark session - """ - return SparkSession.builder.appName("Remorph Reconcile Test").remote("sc://localhost").getOrCreate() - - @pytest.fixture() def sandbox_sqlserver_config() -> JsonObject: env = TestEnvGetter(True) diff --git a/tests/integration/reconcile/conftest.py b/tests/integration/reconcile/conftest.py index 205f5c644b..61e785ed2a 100644 --- a/tests/integration/reconcile/conftest.py +++ b/tests/integration/reconcile/conftest.py @@ -1,7 +1,6 @@ import json import logging import tempfile -import uuid from collections.abc import Generator from contextlib import contextmanager from dataclasses import asdict @@ -108,23 +107,18 @@ def recon_tables(ws: WorkspaceClient, recon_schema: SchemaInfo, make_table) -> t @pytest.fixture -def recon_metadata(mock_spark, report_tables_schema) -> Generator[ReconcileMetadataConfig, None, None]: - rand = uuid.uuid4().hex - schema = f"recon_schema_{rand}" - mock_spark.sql(f"CREATE SCHEMA {schema}") - main_schema, metrics_schema, details_schema = report_tables_schema - - mock_spark.createDataFrame(data=[], schema=main_schema).write.saveAsTable(f"{schema}.MAIN") - mock_spark.createDataFrame(data=[], schema=metrics_schema).write.saveAsTable(f"{schema}.METRICS") - mock_spark.createDataFrame(data=[], schema=details_schema).write.saveAsTable(f"{schema}.DETAILS") +def recon_metadata(spark, recon_schema, make_volume, report_tables_schema) -> ReconcileMetadataConfig: + assert recon_schema.catalog_name + assert recon_schema.name - yield ReconcileMetadataConfig( - catalog=f"recon_catalog_{rand}", - schema=schema, - volume=f"recon_volume_{rand}", - ) + prefix = f"{recon_schema.catalog_name}.{recon_schema.name}" + main_schema, metrics_schema, details_schema = report_tables_schema + spark.createDataFrame(data=[], schema=main_schema).write.saveAsTable(f"{prefix}.MAIN") + spark.createDataFrame(data=[], schema=metrics_schema).write.saveAsTable(f"{prefix}.METRICS") + spark.createDataFrame(data=[], schema=details_schema).write.saveAsTable(f"{prefix}.DETAILS") - mock_spark.sql(f"DROP SCHEMA {schema} CASCADE") + volume = make_volume(catalog_name=recon_schema.catalog_name, schema_name=recon_schema.name, name=recon_schema.name) + return ReconcileMetadataConfig(catalog=recon_schema.catalog_name, schema=recon_schema.name, volume=volume.name) @pytest.fixture @@ -186,9 +180,9 @@ def recon_cluster(make_cluster) -> ClusterDetails: @pytest.fixture -def databricks_recon_config(recon_cluster: ClusterDetails, recon_schema: SchemaInfo, make_volume) -> ReconcileConfig: - volume = make_volume(catalog_name=recon_schema.catalog_name, schema_name=recon_schema.name, name=recon_schema.name) - +def databricks_recon_config( + recon_cluster: ClusterDetails, recon_schema: SchemaInfo, recon_metadata: ReconcileMetadataConfig +) -> ReconcileConfig: deployment_overrides = ReconcileJobConfig( existing_cluster_id=recon_cluster.cluster_id or "bogus", tags={"lakebridge": "reconcile_test"}, @@ -207,9 +201,7 @@ def databricks_recon_config(recon_cluster: ClusterDetails, recon_schema: SchemaI target_catalog=recon_schema.catalog_name, target_schema=recon_schema.name, ), - metadata_config=ReconcileMetadataConfig( - catalog=recon_schema.catalog_name, schema=recon_schema.name, volume=volume.name - ), + metadata_config=recon_metadata, job_overrides=deployment_overrides, ) diff --git a/tests/integration/reconcile/connectors/test_mock_data_source.py b/tests/integration/reconcile/connectors/test_mock_data_source.py index 765ff371c4..8a0bc9f12c 100644 --- a/tests/integration/reconcile/connectors/test_mock_data_source.py +++ b/tests/integration/reconcile/connectors/test_mock_data_source.py @@ -12,13 +12,13 @@ TABLE = "employee" -def test_mock_data_source_happy(mock_spark): +def test_mock_data_source_happy(spark): dataframe_repository = { ( "org", "data", "select * from employee", - ): mock_spark.createDataFrame( + ): spark.createDataFrame( [ Row(emp_id="1", emp_name="name-1", sal=100), Row(emp_id="2", emp_name="name-2", sal=200), @@ -37,7 +37,7 @@ def test_mock_data_source_happy(mock_spark): data_source = MockDataSource(dataframe_repository, schema_repository) actual_data = data_source.read_data(CATALOG, SCHEMA, TABLE, "select * from employee", None) - expected_data = mock_spark.createDataFrame( + expected_data = spark.createDataFrame( [ Row(emp_id="1", emp_name="name-1", sal=100), Row(emp_id="2", emp_name="name-2", sal=200), @@ -54,7 +54,7 @@ def test_mock_data_source_happy(mock_spark): ] -def test_mock_data_source_fail(mock_spark): +def test_mock_data_source_fail(spark): data_source = MockDataSource({}, {}, Exception("TABLE NOT FOUND")) with pytest.raises( DataSourceRuntimeException, @@ -70,13 +70,13 @@ def test_mock_data_source_fail(mock_spark): data_source.get_schema(CATALOG, SCHEMA, "unknown") -def test_mock_data_source_no_catalog(mock_spark): +def test_mock_data_source_no_catalog(spark): dataframe_repository = { ( "", "data", "select * from employee", - ): mock_spark.createDataFrame( + ): spark.createDataFrame( [ Row(emp_id="1", emp_name="name-1", sal=100), Row(emp_id="2", emp_name="name-2", sal=200), @@ -95,7 +95,7 @@ def test_mock_data_source_no_catalog(mock_spark): data_source = MockDataSource(dataframe_repository, schema_repository) actual_data = data_source.read_data(None, SCHEMA, TABLE, "select * from employee", None) - expected_data = mock_spark.createDataFrame( + expected_data = spark.createDataFrame( [ Row(emp_id="1", emp_name="name-1", sal=100), Row(emp_id="2", emp_name="name-2", sal=200), diff --git a/tests/integration/reconcile/connectors/test_read_schema.py b/tests/integration/reconcile/connectors/test_read_schema.py index 26b9514202..7fa6e722f1 100644 --- a/tests/integration/reconcile/connectors/test_read_schema.py +++ b/tests/integration/reconcile/connectors/test_read_schema.py @@ -81,29 +81,29 @@ def _get_snowflake_options(self): return opts -def test_sql_server_read_schema_happy(mock_spark: SparkSession) -> None: +def test_sql_server_read_schema_happy(spark: SparkSession) -> None: mock_ws = create_autospec(WorkspaceClient) - connector = TSQLServerDataSourceUnderTest(mock_spark, mock_ws) + connector = TSQLServerDataSourceUnderTest(spark, mock_ws) columns = connector.get_schema("labs_azure_sandbox_remorph", "dbo", "reconcile_in") assert columns -def test_databricks_read_schema_happy(mock_spark: SparkSession) -> None: +def test_databricks_read_schema_happy(spark: SparkSession) -> None: mock_ws = create_autospec(WorkspaceClient) - connector = DatabricksDataSource(get_dialect("databricks"), mock_spark, mock_ws, "my_secret") + connector = DatabricksDataSource(get_dialect("databricks"), spark, mock_ws, "my_secret") random_view = f"test_view_{uuid.uuid4().hex}" try: - mock_spark.sql("CREATE DATABASE IF NOT EXISTS my_test_db") - mock_spark.sql("CREATE TABLE IF NOT EXISTS my_test_db.my_test_table (id INT, name STRING) USING parquet") - df = mock_spark.sql("SELECT * FROM my_test_db.my_test_table") + spark.sql("CREATE DATABASE IF NOT EXISTS my_test_db") + spark.sql("CREATE TABLE IF NOT EXISTS my_test_db.my_test_table (id INT, name STRING) USING parquet") + df = spark.sql("SELECT * FROM my_test_db.my_test_table") df.createGlobalTempView(random_view) columns = connector.get_schema(None, "global_temp", random_view) assert columns finally: - assert mock_spark.catalog.dropGlobalTempView(random_view) + assert spark.catalog.dropGlobalTempView(random_view) def test_databricks_read_schema_happy_sandbox( @@ -124,17 +124,18 @@ def test_databricks_read_schema_happy_sandbox( # 1. Deploy Oracle Free # 2. Add credentials to the test env getter @pytest.mark.skip(reason="Not Ready! Deploy Infra") -def test_oracle_read_schema_happy(mock_spark: SparkSession) -> None: +def test_oracle_read_schema_happy(spark: SparkSession) -> None: mock_ws = create_autospec(WorkspaceClient) - connector = OracleDataSourceUnderTest(mock_spark, mock_ws) + connector = OracleDataSourceUnderTest(spark, mock_ws) columns = connector.get_schema(None, "SYSTEM", "help") assert columns -def test_snowflake_read_schema_happy(mock_spark: SparkSession) -> None: +@pytest.mark.xfail(reason="Snowflake account unavailable", strict=True) +def test_snowflake_read_schema_happy(spark: SparkSession) -> None: mock_ws = create_autospec(WorkspaceClient) - connector = SnowflakeDataSourceUnderTest(mock_spark, mock_ws) + connector = SnowflakeDataSourceUnderTest(spark, mock_ws) columns = connector.get_schema('remorph', "sandbox", "diamonds") assert columns diff --git a/tests/integration/reconcile/query_builder/test_execute.py b/tests/integration/reconcile/query_builder/test_execute.py index 0d3f7ad308..bde1a7c44d 100644 --- a/tests/integration/reconcile/query_builder/test_execute.py +++ b/tests/integration/reconcile/query_builder/test_execute.py @@ -1,6 +1,6 @@ from pathlib import Path from dataclasses import dataclass -from datetime import datetime +from datetime import datetime, timezone from unittest.mock import patch, MagicMock from uuid import UUID @@ -43,6 +43,8 @@ SRC_TABLE = "supplier" TGT_TABLE = "target_supplier" +MOCK_TIMESTAMP = datetime(2024, 5, 23, 9, 21, 25, 122185, tzinfo=timezone.utc) + @dataclass class SamplingQueries: @@ -98,7 +100,7 @@ class QueryStore: @pytest.fixture -def query_store(mock_spark): +def query_store(spark): source_hash_query = "SELECT LOWER(SHA2(TRIM(s_address) || TRIM(s_name) || COALESCE(TRIM(`s_nationkey`), '_null_recon_') || TRIM(s_phone) || COALESCE(TRIM(`s_suppkey`), '_null_recon_'), 256)) AS hash_value_recon, `s_nationkey` AS `s_nationkey`, `s_suppkey` AS `s_suppkey` FROM :tbl WHERE s_name = 't' AND s_address = 'a'" target_hash_query = "SELECT LOWER(SHA2(TRIM(s_address_t) || TRIM(s_name) || COALESCE(TRIM(`s_nationkey_t`), '_null_recon_') || TRIM(s_phone_t) || COALESCE(TRIM(`s_suppkey_t`), '_null_recon_'), 256)) AS hash_value_recon, `s_nationkey_t` AS `s_nationkey`, `s_suppkey_t` AS `s_suppkey` FROM :tbl WHERE s_name = 't' AND s_address_t = 'a'" source_mismatch_query = "WITH recon AS (SELECT CAST(22 AS number) AS `s_nationkey`, CAST(2 AS number) AS `s_suppkey`), src AS (SELECT TRIM(s_address) AS `s_address`, TRIM(s_name) AS `s_name`, COALESCE(TRIM(`s_nationkey`), '_null_recon_') AS `s_nationkey`, TRIM(s_phone) AS `s_phone`, COALESCE(TRIM(`s_suppkey`), '_null_recon_') AS `s_suppkey` FROM :tbl WHERE s_name = 't' AND s_address = 'a') SELECT src.`s_address`, src.`s_name`, src.`s_nationkey`, src.`s_phone`, src.`s_suppkey` FROM src INNER JOIN recon AS recon ON src.`s_nationkey` = recon.`s_nationkey` AND src.`s_suppkey` = recon.`s_suppkey`" @@ -151,7 +153,7 @@ def query_store(mock_spark): def test_reconcile_data_with_mismatches_and_missing( - mock_spark, + spark, normalized_table_conf_with_opts, table_schema_ansi_ansi, query_store, @@ -164,20 +166,20 @@ def test_reconcile_data_with_mismatches_and_missing( CATALOG, SCHEMA, query_store.hash_queries.source_hash_query, - ): mock_spark.createDataFrame( + ): spark.createDataFrame( [ Row(hash_value_recon="a1b", s_nationkey=11, s_suppkey=1), Row(hash_value_recon="c2d", s_nationkey=22, s_suppkey=2), Row(hash_value_recon="e3g", s_nationkey=33, s_suppkey=3), ] ), - (CATALOG, SCHEMA, query_store.mismatch_queries.source_mismatch_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, query_store.mismatch_queries.source_mismatch_query): spark.createDataFrame( [Row(s_address="address-2", s_name="name-2", s_nationkey=22, s_phone="222-2", s_suppkey=2)] ), - (CATALOG, SCHEMA, query_store.missing_queries.target_missing_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, query_store.missing_queries.target_missing_query): spark.createDataFrame( [Row(s_address="address-3", s_name="name-3", s_nationkey=33, s_phone="333", s_suppkey=3)] ), - (CATALOG, SCHEMA, query_store.threshold_queries.source_threshold_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, query_store.threshold_queries.source_threshold_query): spark.createDataFrame( [Row(s_nationkey=11, s_suppkey=1, s_acctbal=100)] ), } @@ -187,7 +189,7 @@ def test_reconcile_data_with_mismatches_and_missing( CATALOG, SCHEMA, query_store.hash_queries.target_hash_query, - ): mock_spark.createDataFrame( + ): spark.createDataFrame( [ Row(hash_value_recon="a1b", s_nationkey=11, s_suppkey=1), Row(hash_value_recon="c2de", s_nationkey=22, s_suppkey=2), @@ -198,23 +200,23 @@ def test_reconcile_data_with_mismatches_and_missing( CATALOG, SCHEMA, query_store.sampling_queries.target_sampling_query, - ): mock_spark.createDataFrame( + ): spark.createDataFrame( [ Row(hash_value_recon="a1b", s_nationkey=11, s_suppkey=1), Row(hash_value_recon="c2de", s_nationkey=22, s_suppkey=2), Row(hash_value_recon="k4l", s_nationkey=44, s_suppkey=4), ] ), - (CATALOG, SCHEMA, query_store.mismatch_queries.target_mismatch_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, query_store.mismatch_queries.target_mismatch_query): spark.createDataFrame( [Row(s_address="address-22", s_name="name-2", s_nationkey=22, s_phone="222", s_suppkey=2)] ), - (CATALOG, SCHEMA, query_store.missing_queries.source_missing_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, query_store.missing_queries.source_missing_query): spark.createDataFrame( [Row(s_address="address-4", s_name="name-4", s_nationkey=44, s_phone="444", s_suppkey=4)] ), - (CATALOG, SCHEMA, query_store.threshold_queries.target_threshold_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, query_store.threshold_queries.target_threshold_query): spark.createDataFrame( [Row(s_nationkey=11, s_suppkey=1, s_acctbal=210)] ), - (CATALOG, SCHEMA, query_store.threshold_queries.threshold_comparison_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, query_store.threshold_queries.threshold_comparison_query): spark.createDataFrame( [ Row( s_acctbal_source=100, @@ -233,7 +235,7 @@ def test_reconcile_data_with_mismatches_and_missing( target_catalog=CATALOG, target_schema=SCHEMA, ) - schema_comparator = SchemaCompare(mock_spark) + schema_comparator = SchemaCompare(spark) source = MockDataSource(source_dataframe_repository, source_schema_repository) target = MockDataSource(target_dataframe_repository, target_schema_repository) actual_data_reconcile = Reconciliation( @@ -243,7 +245,7 @@ def test_reconcile_data_with_mismatches_and_missing( "data", schema_comparator, get_dialect("databricks"), - mock_spark, + spark, recon_metadata, FakeReconIntermediatePersist(), ).reconcile_data(normalized_table_conf_with_opts, src_schema, tgt_schema) @@ -251,14 +253,14 @@ def test_reconcile_data_with_mismatches_and_missing( mismatch_count=1, missing_in_src_count=1, missing_in_tgt_count=1, - missing_in_src=mock_spark.createDataFrame( + missing_in_src=spark.createDataFrame( [Row(s_address="address-4", s_name="name-4", s_nationkey=44, s_phone="444", s_suppkey=4)] ), - missing_in_tgt=mock_spark.createDataFrame( + missing_in_tgt=spark.createDataFrame( [Row(s_address="address-3", s_name="name-3", s_nationkey=33, s_phone="333", s_suppkey=3)] ), mismatch=MismatchOutput( - mismatch_df=mock_spark.createDataFrame( + mismatch_df=spark.createDataFrame( [ Row( s_suppkey=2, @@ -278,7 +280,7 @@ def test_reconcile_data_with_mismatches_and_missing( mismatch_columns=["s_address", "s_phone"], ), threshold_output=ThresholdOutput( - threshold_df=mock_spark.createDataFrame( + threshold_df=spark.createDataFrame( [ Row( s_acctbal_source=100, @@ -313,11 +315,11 @@ def test_reconcile_data_with_mismatches_and_missing( "data", schema_comparator, get_dialect("databricks"), - mock_spark, + spark, recon_metadata, FakeReconIntermediatePersist(), ).reconcile_schema(src_schema, tgt_schema, normalized_table_conf_with_opts) - expected_schema_reconcile = mock_spark.createDataFrame( + expected_schema_reconcile = spark.createDataFrame( [ Row( source_column="s_suppkey", @@ -368,7 +370,7 @@ def test_reconcile_data_with_mismatches_and_missing( assert actual_data_reconcile.threshold_output.threshold_df is not None assertDataFrameEqual( actual_data_reconcile.threshold_output.threshold_df, - mock_spark.createDataFrame( + spark.createDataFrame( [ Row( s_acctbal_source=100, @@ -384,7 +386,7 @@ def test_reconcile_data_with_mismatches_and_missing( def test_reconcile_data_without_mismatches_and_missing( - mock_spark, + spark, normalized_table_conf_with_opts, table_schema_ansi_ansi, query_store, @@ -397,13 +399,13 @@ def test_reconcile_data_without_mismatches_and_missing( CATALOG, SCHEMA, query_store.hash_queries.source_hash_query, - ): mock_spark.createDataFrame( + ): spark.createDataFrame( [ Row(hash_value_recon="a1b", s_nationkey=11, s_suppkey=1), Row(hash_value_recon="c2d", s_nationkey=22, s_suppkey=2), ] ), - (CATALOG, SCHEMA, query_store.threshold_queries.source_threshold_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, query_store.threshold_queries.source_threshold_query): spark.createDataFrame( [Row(s_nationkey=11, s_suppkey=1, s_acctbal=100)] ), } @@ -413,16 +415,16 @@ def test_reconcile_data_without_mismatches_and_missing( CATALOG, SCHEMA, query_store.hash_queries.target_hash_query, - ): mock_spark.createDataFrame( + ): spark.createDataFrame( [ Row(hash_value_recon="a1b", s_nationkey=11, s_suppkey=1), Row(hash_value_recon="c2d", s_nationkey=22, s_suppkey=2), ] ), - (CATALOG, SCHEMA, query_store.threshold_queries.target_threshold_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, query_store.threshold_queries.target_threshold_query): spark.createDataFrame( [Row(s_nationkey=11, s_suppkey=1, s_acctbal=110)] ), - (CATALOG, SCHEMA, query_store.threshold_queries.threshold_comparison_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, query_store.threshold_queries.threshold_comparison_query): spark.createDataFrame( [ Row( s_acctbal_source=100, @@ -441,7 +443,7 @@ def test_reconcile_data_without_mismatches_and_missing( target_catalog=CATALOG, target_schema=SCHEMA, ) - schema_comparator = SchemaCompare(mock_spark) + schema_comparator = SchemaCompare(spark) source = MockDataSource(source_dataframe_repository, source_schema_repository) target = MockDataSource(target_dataframe_repository, target_schema_repository) actual = Reconciliation( @@ -451,7 +453,7 @@ def test_reconcile_data_without_mismatches_and_missing( "data", schema_comparator, get_dialect("databricks"), - mock_spark, + spark, recon_metadata, FakeReconIntermediatePersist(), ).reconcile_data(normalized_table_conf_with_opts, src_schema, tgt_schema) @@ -466,7 +468,7 @@ def test_reconcile_data_without_mismatches_and_missing( def test_reconcile_data_with_mismatch_and_no_missing( - mock_spark, + spark, normalized_table_conf_with_opts, table_schema_ansi_ansi, query_store, @@ -481,13 +483,13 @@ def test_reconcile_data_with_mismatch_and_no_missing( CATALOG, SCHEMA, query_store.hash_queries.source_hash_query, - ): mock_spark.createDataFrame( + ): spark.createDataFrame( [ Row(hash_value_recon="a1b", s_nationkey=11, s_suppkey=1), Row(hash_value_recon="c2d", s_nationkey=22, s_suppkey=2), ] ), - (CATALOG, SCHEMA, query_store.mismatch_queries.source_mismatch_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, query_store.mismatch_queries.source_mismatch_query): spark.createDataFrame( [Row(s_address="address-2", s_name="name-2", s_nationkey=22, s_phone="222-2", s_suppkey=2)] ), } @@ -497,7 +499,7 @@ def test_reconcile_data_with_mismatch_and_no_missing( CATALOG, SCHEMA, query_store.hash_queries.target_hash_query, - ): mock_spark.createDataFrame( + ): spark.createDataFrame( [ Row(hash_value_recon="a1b", s_nationkey=11, s_suppkey=1), Row(hash_value_recon="c2de", s_nationkey=22, s_suppkey=2), @@ -507,13 +509,13 @@ def test_reconcile_data_with_mismatch_and_no_missing( CATALOG, SCHEMA, query_store.sampling_queries.target_sampling_query, - ): mock_spark.createDataFrame( + ): spark.createDataFrame( [ Row(hash_value_recon="a1b", s_nationkey=11, s_suppkey=1), Row(hash_value_recon="c2de", s_nationkey=22, s_suppkey=2), ] ), - (CATALOG, SCHEMA, query_store.mismatch_queries.target_mismatch_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, query_store.mismatch_queries.target_mismatch_query): spark.createDataFrame( [Row(s_address="address-22", s_name="name-2", s_nationkey=22, s_phone="222", s_suppkey=2)] ), } @@ -524,7 +526,7 @@ def test_reconcile_data_with_mismatch_and_no_missing( target_catalog=CATALOG, target_schema=SCHEMA, ) - schema_comparator = SchemaCompare(mock_spark) + schema_comparator = SchemaCompare(spark) source = MockDataSource(source_dataframe_repository, source_schema_repository) target = MockDataSource(target_dataframe_repository, target_schema_repository) actual = Reconciliation( @@ -534,7 +536,7 @@ def test_reconcile_data_with_mismatch_and_no_missing( "data", schema_comparator, get_dialect("databricks"), - mock_spark, + spark, recon_metadata, FakeReconIntermediatePersist(), ).reconcile_data(normalized_table_conf_with_opts, src_schema, tgt_schema) @@ -545,7 +547,7 @@ def test_reconcile_data_with_mismatch_and_no_missing( missing_in_src=None, missing_in_tgt=None, mismatch=MismatchOutput( - mismatch_df=mock_spark.createDataFrame( + mismatch_df=spark.createDataFrame( [ Row( s_suppkey=2, @@ -577,7 +579,7 @@ def test_reconcile_data_with_mismatch_and_no_missing( def test_reconcile_data_missing_and_no_mismatch( - mock_spark, + spark, normalized_table_conf_with_opts, table_schema_ansi_ansi, query_store, @@ -592,14 +594,14 @@ def test_reconcile_data_missing_and_no_mismatch( CATALOG, SCHEMA, query_store.hash_queries.source_hash_query, - ): mock_spark.createDataFrame( + ): spark.createDataFrame( [ Row(hash_value_recon="a1b", s_nationkey=11, s_suppkey=1), Row(hash_value_recon="c2d", s_nationkey=22, s_suppkey=2), Row(hash_value_recon="e3g", s_nationkey=33, s_suppkey=3), ] ), - (CATALOG, SCHEMA, query_store.missing_queries.target_missing_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, query_store.missing_queries.target_missing_query): spark.createDataFrame( [Row(s_address="address-3", s_name="name-3", s_nationkey=33, s_phone="333", s_suppkey=3)] ), } @@ -609,14 +611,14 @@ def test_reconcile_data_missing_and_no_mismatch( CATALOG, SCHEMA, query_store.hash_queries.target_hash_query, - ): mock_spark.createDataFrame( + ): spark.createDataFrame( [ Row(hash_value_recon="a1b", s_nationkey=11, s_suppkey=1), Row(hash_value_recon="c2d", s_nationkey=22, s_suppkey=2), Row(hash_value_recon="k4l", s_nationkey=44, s_suppkey=4), ] ), - (CATALOG, SCHEMA, query_store.missing_queries.source_missing_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, query_store.missing_queries.source_missing_query): spark.createDataFrame( [Row(s_address="address-4", s_name="name-4", s_nationkey=44, s_phone="444", s_suppkey=4)] ), } @@ -627,7 +629,7 @@ def test_reconcile_data_missing_and_no_mismatch( target_catalog=CATALOG, target_schema=SCHEMA, ) - schema_comparator = SchemaCompare(mock_spark) + schema_comparator = SchemaCompare(spark) source = MockDataSource(source_dataframe_repository, source_schema_repository) target = MockDataSource(target_dataframe_repository, target_schema_repository) actual = Reconciliation( @@ -637,7 +639,7 @@ def test_reconcile_data_missing_and_no_mismatch( "data", schema_comparator, get_dialect("databricks"), - mock_spark, + spark, recon_metadata, FakeReconIntermediatePersist(), ).reconcile_data(normalized_table_conf_with_opts, src_schema, tgt_schema) @@ -645,10 +647,10 @@ def test_reconcile_data_missing_and_no_mismatch( mismatch_count=0, missing_in_src_count=1, missing_in_tgt_count=1, - missing_in_src=mock_spark.createDataFrame( + missing_in_src=spark.createDataFrame( [Row(s_address="address-4", s_name="name-4", s_nationkey=44, s_phone="444", s_suppkey=4)] ), - missing_in_tgt=mock_spark.createDataFrame( + missing_in_tgt=spark.createDataFrame( [Row(s_address="address-3", s_name="name-3", s_nationkey=33, s_phone="333", s_suppkey=3)] ), mismatch=MismatchOutput(), @@ -667,7 +669,7 @@ def mock_for_report_type_data( table_schema_ansi_ansi, query_store, recon_metadata, - mock_spark, + spark, ): normalized_table_conf_with_opts.drop_columns = ["`s_acctbal`"] normalized_table_conf_with_opts.column_thresholds = None @@ -680,20 +682,20 @@ def mock_for_report_type_data( CATALOG, SCHEMA, query_store.hash_queries.source_hash_query, - ): mock_spark.createDataFrame( + ): spark.createDataFrame( [ Row(hash_value_recon="a1b", s_nationkey=11, s_suppkey=1), Row(hash_value_recon="c2d", s_nationkey=22, s_suppkey=2), Row(hash_value_recon="e3g", s_nationkey=33, s_suppkey=3), ] ), - (CATALOG, SCHEMA, query_store.mismatch_queries.source_mismatch_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, query_store.mismatch_queries.source_mismatch_query): spark.createDataFrame( [Row(s_address="address-2", s_name="name-2", s_nationkey=22, s_phone="222-2", s_suppkey=2)] ), - (CATALOG, SCHEMA, query_store.missing_queries.target_missing_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, query_store.missing_queries.target_missing_query): spark.createDataFrame( [Row(s_address="address-3", s_name="name-3", s_nationkey=33, s_phone="333", s_suppkey=3)] ), - (CATALOG, SCHEMA, query_store.record_count_queries.source_record_count_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, query_store.record_count_queries.source_record_count_query): spark.createDataFrame( [Row(count=3)] ), } @@ -703,7 +705,7 @@ def mock_for_report_type_data( CATALOG, SCHEMA, query_store.hash_queries.target_hash_query, - ): mock_spark.createDataFrame( + ): spark.createDataFrame( [ Row(hash_value_recon="a1b", s_nationkey=11, s_suppkey=1), Row(hash_value_recon="c2de", s_nationkey=22, s_suppkey=2), @@ -714,20 +716,20 @@ def mock_for_report_type_data( CATALOG, SCHEMA, query_store.sampling_queries.target_sampling_query, - ): mock_spark.createDataFrame( + ): spark.createDataFrame( [ Row(hash_value_recon="a1b", s_nationkey=11, s_suppkey=1), Row(hash_value_recon="c2de", s_nationkey=22, s_suppkey=2), Row(hash_value_recon="k4l", s_nationkey=44, s_suppkey=4), ] ), - (CATALOG, SCHEMA, query_store.mismatch_queries.target_mismatch_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, query_store.mismatch_queries.target_mismatch_query): spark.createDataFrame( [Row(s_address="address-22", s_name="name-2", s_nationkey=22, s_phone="222", s_suppkey=2)] ), - (CATALOG, SCHEMA, query_store.missing_queries.source_missing_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, query_store.missing_queries.source_missing_query): spark.createDataFrame( [Row(s_address="address-4", s_name="name-4", s_nationkey=44, s_phone="444", s_suppkey=4)] ), - (CATALOG, SCHEMA, query_store.record_count_queries.target_record_count_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, query_store.record_count_queries.target_record_count_query): spark.createDataFrame( [Row(count=3)] ), } @@ -750,10 +752,11 @@ def mock_for_report_type_data( def test_recon_for_report_type_is_data( - mock_workspace_client, mock_spark, report_tables_schema, mock_for_report_type_data, tmp_path: Path, recon_id: UUID + mock_workspace_client, spark, report_tables_schema, mock_for_report_type_data, tmp_path: Path, recon_id: UUID ): recon_schema, metrics_schema, details_schema = report_tables_schema table_recon, source, target, reconcile_config_data = mock_for_report_type_data + catalog = reconcile_config_data.metadata_config.catalog schema = reconcile_config_data.metadata_config.schema with ( patch("databricks.labs.lakebridge.reconcile.trigger_recon_service.datetime") as mock_datetime, @@ -765,49 +768,49 @@ def test_recon_for_report_type_is_data( ), patch( "databricks.labs.lakebridge.reconcile.recon_capture.ReconCapture._generate_recon_main_id", - return_value=11111, + return_value=11111111111, ), ): - mock_datetime.now.return_value = datetime(2024, 5, 23, 9, 21, 25, 122185) - recon_datetime.now.return_value = datetime(2024, 5, 23, 9, 21, 25, 122185) + mock_datetime.now.return_value = MOCK_TIMESTAMP + recon_datetime.now.return_value = MOCK_TIMESTAMP reconcile_output = TriggerReconService.trigger_recon( - mock_workspace_client, mock_spark, table_recon, reconcile_config_data, local_test_run=True + mock_workspace_client, spark, table_recon, reconcile_config_data ) assert reconcile_output.recon_id == recon_id.hex - expected_remorph_recon = mock_spark.createDataFrame( + expected_remorph_recon = spark.createDataFrame( data=[ ( - 11111, + 11111111111, recon_id.hex, "Databricks", ("org", "data", "supplier"), ("org", "data", "target_supplier"), "data", "reconcile", - datetime(2024, 5, 23, 9, 21, 25, 122185), - datetime(2024, 5, 23, 9, 21, 25, 122185), + MOCK_TIMESTAMP, + MOCK_TIMESTAMP, ) ], schema=recon_schema, ) - expected_remorph_recon_metrics = mock_spark.createDataFrame( + expected_remorph_recon_metrics = spark.createDataFrame( data=[ ( - 11111, + 11111111111, (3, 3, (1, 1), (1, 0, "s_address,s_phone"), None), (False, "remorph", ""), - datetime(2024, 5, 23, 9, 21, 25, 122185), + MOCK_TIMESTAMP, ) ], schema=metrics_schema, ) - expected_remorph_recon_details = mock_spark.createDataFrame( + expected_remorph_recon_details = spark.createDataFrame( data=[ ( - 11111, + 11111111111, "mismatch", False, [ @@ -825,10 +828,10 @@ def test_recon_for_report_type_is_data( "s_phone_match": "false", } ], - datetime(2024, 5, 23, 9, 21, 25, 122185), + MOCK_TIMESTAMP, ), ( - 11111, + 11111111111, "missing_in_source", False, [ @@ -840,10 +843,10 @@ def test_recon_for_report_type_is_data( "s_suppkey": "4", } ], - datetime(2024, 5, 23, 9, 21, 25, 122185), + MOCK_TIMESTAMP, ), ( - 11111, + 11111111111, "missing_in_target", False, [ @@ -855,18 +858,20 @@ def test_recon_for_report_type_is_data( "s_suppkey": "3", } ], - datetime(2024, 5, 23, 9, 21, 25, 122185), + MOCK_TIMESTAMP, ), ], schema=details_schema, ) - assertDataFrameEqual(mock_spark.sql(f"SELECT * FROM {schema}.MAIN"), expected_remorph_recon, ignoreNullable=True) assertDataFrameEqual( - mock_spark.sql(f"SELECT * FROM {schema}.METRICS"), expected_remorph_recon_metrics, ignoreNullable=True + spark.sql(f"SELECT * FROM {catalog}.{schema}.MAIN"), expected_remorph_recon, ignoreNullable=True + ) + assertDataFrameEqual( + spark.sql(f"SELECT * FROM {catalog}.{schema}.METRICS"), expected_remorph_recon_metrics, ignoreNullable=True ) assertDataFrameEqual( - mock_spark.sql(f"SELECT * FROM {schema}.DETAILS"), expected_remorph_recon_details, ignoreNullable=True + spark.sql(f"SELECT * FROM {catalog}.{schema}.DETAILS"), expected_remorph_recon_details, ignoreNullable=True ) @@ -875,7 +880,7 @@ def mock_for_report_type_schema( normalized_table_conf_with_opts, table_schema_ansi_ansi, query_store, - mock_spark, + spark, recon_metadata: ReconcileMetadataConfig, ): table_recon = TableRecon( @@ -887,20 +892,20 @@ def mock_for_report_type_schema( CATALOG, SCHEMA, query_store.hash_queries.source_hash_query, - ): mock_spark.createDataFrame( + ): spark.createDataFrame( [ Row(hash_value_recon="a1b", s_nationkey=11, s_suppkey=1), Row(hash_value_recon="c2d", s_nationkey=22, s_suppkey=2), Row(hash_value_recon="e3g", s_nationkey=33, s_suppkey=3), ] ), - (CATALOG, SCHEMA, query_store.mismatch_queries.source_mismatch_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, query_store.mismatch_queries.source_mismatch_query): spark.createDataFrame( [Row(s_address="address-2", s_name="name-2", s_nationkey=22, s_phone="222-2", s_suppkey=2)] ), - (CATALOG, SCHEMA, query_store.missing_queries.target_missing_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, query_store.missing_queries.target_missing_query): spark.createDataFrame( [Row(s_address="address-3", s_name="name-3", s_nationkey=33, s_phone="333", s_suppkey=3)] ), - (CATALOG, SCHEMA, query_store.record_count_queries.source_record_count_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, query_store.record_count_queries.source_record_count_query): spark.createDataFrame( [Row(count=3)] ), } @@ -911,20 +916,20 @@ def mock_for_report_type_schema( CATALOG, SCHEMA, query_store.hash_queries.target_hash_query, - ): mock_spark.createDataFrame( + ): spark.createDataFrame( [ Row(hash_value_recon="a1b", s_nationkey=11, s_suppkey=1), Row(hash_value_recon="c2de", s_nationkey=22, s_suppkey=2), Row(hash_value_recon="k4l", s_nationkey=44, s_suppkey=4), ] ), - (CATALOG, SCHEMA, query_store.mismatch_queries.target_mismatch_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, query_store.mismatch_queries.target_mismatch_query): spark.createDataFrame( [Row(s_address="address-22", s_name="name-2", s_nationkey=22, s_phone="222", s_suppkey=2)] ), - (CATALOG, SCHEMA, query_store.missing_queries.source_missing_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, query_store.missing_queries.source_missing_query): spark.createDataFrame( [Row(s_address="address-4", s_name="name-4", s_nationkey=44, s_phone="444", s_suppkey=4)] ), - (CATALOG, SCHEMA, query_store.record_count_queries.target_record_count_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, query_store.record_count_queries.target_record_count_query): spark.createDataFrame( [Row(count=3)] ), } @@ -947,10 +952,11 @@ def mock_for_report_type_schema( def test_recon_for_report_type_schema( - mock_workspace_client, mock_spark, report_tables_schema, mock_for_report_type_schema, tmp_path: Path, recon_id: UUID + mock_workspace_client, spark, report_tables_schema, mock_for_report_type_schema, tmp_path: Path, recon_id: UUID ): recon_schema, metrics_schema, details_schema = report_tables_schema table_recon, source, target, reconcile_config_schema = mock_for_report_type_schema + catalog = reconcile_config_schema.metadata_config.catalog schema = reconcile_config_schema.metadata_config.schema with ( patch("databricks.labs.lakebridge.reconcile.trigger_recon_service.datetime") as mock_datetime, @@ -959,39 +965,46 @@ def test_recon_for_report_type_schema( patch("databricks.labs.lakebridge.reconcile.trigger_recon_service.uuid4", return_value=recon_id), patch( "databricks.labs.lakebridge.reconcile.recon_capture.ReconCapture._generate_recon_main_id", - return_value=22222, + return_value=22222222222, ), ): - mock_datetime.now.return_value = datetime(2024, 5, 23, 9, 21, 25, 122185) - recon_datetime.now.return_value = datetime(2024, 5, 23, 9, 21, 25, 122185) + mock_datetime.now.return_value = MOCK_TIMESTAMP + recon_datetime.now.return_value = MOCK_TIMESTAMP final_reconcile_output = TriggerReconService.trigger_recon( - mock_workspace_client, mock_spark, table_recon, reconcile_config_schema, local_test_run=True + mock_workspace_client, spark, table_recon, reconcile_config_schema ) - expected_remorph_recon = mock_spark.createDataFrame( + expected_remorph_recon = spark.createDataFrame( data=[ ( - 22222, + 22222222222, recon_id.hex, "Databricks", ("org", "data", "supplier"), ("org", "data", "target_supplier"), "schema", "reconcile", - datetime(2024, 5, 23, 9, 21, 25, 122185), - datetime(2024, 5, 23, 9, 21, 25, 122185), + MOCK_TIMESTAMP, + MOCK_TIMESTAMP, ) ], schema=recon_schema, ) - expected_remorph_recon_metrics = mock_spark.createDataFrame( - data=[(22222, (0, 0, None, None, True), (True, "remorph", ""), datetime(2024, 5, 23, 9, 21, 25, 122185))], + expected_remorph_recon_metrics = spark.createDataFrame( + data=[ + ( + 22222222222, + (0, 0, None, None, True), + (True, "remorph", ""), + MOCK_TIMESTAMP, + ) + ], schema=metrics_schema, ) - expected_remorph_recon_details = mock_spark.createDataFrame( + expected_remorph_recon_details = spark.createDataFrame( data=[ ( - 22222, + 22222222222, "schema", True, [ @@ -1038,18 +1051,20 @@ def test_recon_for_report_type_schema( "is_valid": "true", }, ], - datetime(2024, 5, 23, 9, 21, 25, 122185), + MOCK_TIMESTAMP, ) ], schema=details_schema, ) - assertDataFrameEqual(mock_spark.sql(f"SELECT * FROM {schema}.MAIN"), expected_remorph_recon, ignoreNullable=True) assertDataFrameEqual( - mock_spark.sql(f"SELECT * FROM {schema}.METRICS"), expected_remorph_recon_metrics, ignoreNullable=True + spark.sql(f"SELECT * FROM {catalog}.{schema}.MAIN"), expected_remorph_recon, ignoreNullable=True + ) + assertDataFrameEqual( + spark.sql(f"SELECT * FROM {catalog}.{schema}.METRICS"), expected_remorph_recon_metrics, ignoreNullable=True ) assertDataFrameEqual( - mock_spark.sql(f"SELECT * FROM {schema}.DETAILS"), expected_remorph_recon_details, ignoreNullable=True + spark.sql(f"SELECT * FROM {catalog}.{schema}.DETAILS"), expected_remorph_recon_details, ignoreNullable=True ) assert final_reconcile_output.recon_id == recon_id.hex @@ -1060,7 +1075,7 @@ def mock_for_report_type_all( mock_workspace_client, normalized_table_conf_with_opts, table_schema_oracle_ansi, - mock_spark, + spark, query_store, recon_metadata, ): @@ -1076,24 +1091,24 @@ def mock_for_report_type_all( CATALOG, SCHEMA, snowflake_query_store.hash_queries.source_hash_query, - ): mock_spark.createDataFrame( + ): spark.createDataFrame( [ Row(hash_value_recon="a1b", s_nationkey=11, s_suppkey=1), Row(hash_value_recon="c2d", s_nationkey=22, s_suppkey=2), Row(hash_value_recon="e3g", s_nationkey=33, s_suppkey=3), ] ), - (CATALOG, SCHEMA, snowflake_query_store.mismatch_queries.source_mismatch_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, snowflake_query_store.mismatch_queries.source_mismatch_query): spark.createDataFrame( [Row(s_address="address-2", s_name="name-2", s_nationkey=22, s_phone="222-2", s_suppkey=2)] ), - (CATALOG, SCHEMA, snowflake_query_store.missing_queries.target_missing_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, snowflake_query_store.missing_queries.target_missing_query): spark.createDataFrame( [Row(s_address="address-3", s_name="name-3", s_nationkey=33, s_phone="333", s_suppkey=3)] ), ( CATALOG, SCHEMA, snowflake_query_store.record_count_queries.source_record_count_query, - ): mock_spark.createDataFrame([Row(count=3)]), + ): spark.createDataFrame([Row(count=3)]), } source_schema_repository = {(CATALOG, SCHEMA, SRC_TABLE): src_schema} @@ -1102,7 +1117,7 @@ def mock_for_report_type_all( CATALOG, SCHEMA, snowflake_query_store.hash_queries.target_hash_query, - ): mock_spark.createDataFrame( + ): spark.createDataFrame( [ Row(hash_value_recon="a1b", s_nationkey=11, s_suppkey=1), Row(hash_value_recon="c2de", s_nationkey=22, s_suppkey=2), @@ -1113,24 +1128,24 @@ def mock_for_report_type_all( CATALOG, SCHEMA, snowflake_query_store.sampling_queries.target_sampling_query, - ): mock_spark.createDataFrame( + ): spark.createDataFrame( [ Row(hash_value_recon="a1b", s_nationkey=11, s_suppkey=1), Row(hash_value_recon="c2de", s_nationkey=22, s_suppkey=2), Row(hash_value_recon="k4l", s_nationkey=44, s_suppkey=4), ] ), - (CATALOG, SCHEMA, snowflake_query_store.mismatch_queries.target_mismatch_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, snowflake_query_store.mismatch_queries.target_mismatch_query): spark.createDataFrame( [Row(s_address="address-22", s_name="name-2", s_nationkey=22, s_phone="222", s_suppkey=2)] ), - (CATALOG, SCHEMA, snowflake_query_store.missing_queries.source_missing_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, snowflake_query_store.missing_queries.source_missing_query): spark.createDataFrame( [Row(s_address="address-4", s_name="name-4", s_nationkey=44, s_phone="444", s_suppkey=4)] ), ( CATALOG, SCHEMA, snowflake_query_store.record_count_queries.target_record_count_query, - ): mock_spark.createDataFrame([Row(count=3)]), + ): spark.createDataFrame([Row(count=3)]), } target_schema_repository = {(CATALOG, SCHEMA, TGT_TABLE): tgt_schema} @@ -1154,13 +1169,14 @@ def mock_for_report_type_all( @pytest.mark.skip(reason="Will be fixed in a following PR") def test_recon_for_report_type_all( mock_workspace_client, - mock_spark, + spark, report_tables_schema, mock_for_report_type_all, tmp_path: Path, ): recon_schema, metrics_schema, details_schema = report_tables_schema table_recon, source, target, reconcile_config_all = mock_for_report_type_all + catalog = reconcile_config_all.metadata_config.catalog schema = reconcile_config_all.metadata_config.schema with ( @@ -1173,50 +1189,48 @@ def test_recon_for_report_type_all( ), patch( "databricks.labs.lakebridge.reconcile.recon_capture.ReconCapture._generate_recon_main_id", - return_value=33333, + return_value=33333333333, ), patch("databricks.labs.lakebridge.reconcile.utils.generate_volume_path", return_value=str(tmp_path)), ): - mock_datetime.now.return_value = datetime(2024, 5, 23, 9, 21, 25, 122185) - recon_datetime.now.return_value = datetime(2024, 5, 23, 9, 21, 25, 122185) + mock_datetime.now.return_value = MOCK_TIMESTAMP + recon_datetime.now.return_value = MOCK_TIMESTAMP with pytest.raises(ReconciliationException) as exc_info: - TriggerReconService.trigger_recon( - mock_workspace_client, mock_spark, table_recon, reconcile_config_all, local_test_run=True - ) + TriggerReconService.trigger_recon(mock_workspace_client, spark, table_recon, reconcile_config_all) if exc_info.value.reconcile_output is not None: assert exc_info.value.reconcile_output.recon_id == "00112233-4455-6677-8899-aabbccddeeff" - expected_remorph_recon = mock_spark.createDataFrame( + expected_remorph_recon = spark.createDataFrame( data=[ ( - 33333, + 33333333333, "00112233-4455-6677-8899-aabbccddeeff", "Snowflake", ("org", "data", "supplier"), ("org", "data", "target_supplier"), "all", "reconcile", - datetime(2024, 5, 23, 9, 21, 25, 122185), - datetime(2024, 5, 23, 9, 21, 25, 122185), + MOCK_TIMESTAMP, + MOCK_TIMESTAMP, ) ], schema=recon_schema, ) - expected_remorph_recon_metrics = mock_spark.createDataFrame( + expected_remorph_recon_metrics = spark.createDataFrame( data=[ ( - 33333, + 33333333333, (3, 3, (1, 1), (1, 0, "s_address,s_phone"), False), (False, "remorph", ""), - datetime(2024, 5, 23, 9, 21, 25, 122185), + MOCK_TIMESTAMP, ) ], schema=metrics_schema, ) - expected_remorph_recon_details = mock_spark.createDataFrame( + expected_remorph_recon_details = spark.createDataFrame( data=[ ( - 33333, + 33333333333, "mismatch", False, [ @@ -1234,10 +1248,10 @@ def test_recon_for_report_type_all( "s_phone_match": "false", } ], - datetime(2024, 5, 23, 9, 21, 25, 122185), + MOCK_TIMESTAMP, ), ( - 33333, + 33333333333, "missing_in_source", False, [ @@ -1249,10 +1263,10 @@ def test_recon_for_report_type_all( "s_suppkey": "4", } ], - datetime(2024, 5, 23, 9, 21, 25, 122185), + MOCK_TIMESTAMP, ), ( - 33333, + 33333333333, "missing_in_target", False, [ @@ -1264,10 +1278,10 @@ def test_recon_for_report_type_all( "s_suppkey": "3", } ], - datetime(2024, 5, 23, 9, 21, 25, 122185), + MOCK_TIMESTAMP, ), ( - 33333, + 33333333333, "schema", False, [ @@ -1307,24 +1321,26 @@ def test_recon_for_report_type_all( "is_valid": "false", }, ], - datetime(2024, 5, 23, 9, 21, 25, 122185), + MOCK_TIMESTAMP, ), ], schema=details_schema, ) - assertDataFrameEqual(mock_spark.sql(f"SELECT * FROM {schema}.MAIN"), expected_remorph_recon, ignoreNullable=True) assertDataFrameEqual( - mock_spark.sql(f"SELECT * FROM {schema}.METRICS"), expected_remorph_recon_metrics, ignoreNullable=True + spark.sql(f"SELECT * FROM {catalog}.{schema}.MAIN"), expected_remorph_recon, ignoreNullable=True + ) + assertDataFrameEqual( + spark.sql(f"SELECT * FROM {catalog}.{schema}.METRICS"), expected_remorph_recon_metrics, ignoreNullable=True ) assertDataFrameEqual( - mock_spark.sql(f"SELECT * FROM {schema}.DETAILS"), expected_remorph_recon_details, ignoreNullable=True + spark.sql(f"SELECT * FROM {catalog}.{schema}.DETAILS"), expected_remorph_recon_details, ignoreNullable=True ) @pytest.fixture def mock_for_report_type_row( - normalized_table_conf_with_opts, table_schema_ansi_ansi, mock_spark, query_store, recon_metadata + normalized_table_conf_with_opts, table_schema_ansi_ansi, spark, query_store, recon_metadata ): normalized_table_conf_with_opts.drop_columns = ["`s_acctbal`"] normalized_table_conf_with_opts.column_thresholds = None @@ -1337,7 +1353,7 @@ def mock_for_report_type_row( CATALOG, SCHEMA, query_store.row_queries.source_row_query, - ): mock_spark.createDataFrame( + ): spark.createDataFrame( [ Row( hash_value_recon="a1b", @@ -1365,7 +1381,7 @@ def mock_for_report_type_row( ), ] ), - (CATALOG, SCHEMA, query_store.record_count_queries.source_record_count_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, query_store.record_count_queries.source_record_count_query): spark.createDataFrame( [Row(count=3)] ), } @@ -1376,7 +1392,7 @@ def mock_for_report_type_row( CATALOG, SCHEMA, query_store.row_queries.target_row_query, - ): mock_spark.createDataFrame( + ): spark.createDataFrame( [ Row( hash_value_recon="a1b", @@ -1404,7 +1420,7 @@ def mock_for_report_type_row( ), ] ), - (CATALOG, SCHEMA, query_store.record_count_queries.target_record_count_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, query_store.record_count_queries.target_record_count_query): spark.createDataFrame( [Row(count=3)] ), } @@ -1431,13 +1447,14 @@ def mock_for_report_type_row( @pytest.mark.skip(reason="Will be fixed in a following PR") def test_recon_for_report_type_is_row( mock_workspace_client, - mock_spark, + spark, mock_for_report_type_row, report_tables_schema, tmp_path: Path, ): recon_schema, metrics_schema, details_schema = report_tables_schema source, target, table_recon, reconcile_config_row = mock_for_report_type_row + catalog = reconcile_config_row.metadata_config.catalog schema = reconcile_config_row.metadata_config.schema with ( patch("databricks.labs.lakebridge.reconcile.trigger_recon_service.datetime") as mock_datetime, @@ -1449,51 +1466,49 @@ def test_recon_for_report_type_is_row( ), patch( "databricks.labs.lakebridge.reconcile.recon_capture.ReconCapture._generate_recon_main_id", - return_value=33333, + return_value=33333333333, ), patch("databricks.labs.lakebridge.reconcile.utils.generate_volume_path", return_value=str(tmp_path)), ): - mock_datetime.now.return_value = datetime(2024, 5, 23, 9, 21, 25, 122185) - recon_datetime.now.return_value = datetime(2024, 5, 23, 9, 21, 25, 122185) + mock_datetime.now.return_value = MOCK_TIMESTAMP + recon_datetime.now.return_value = MOCK_TIMESTAMP with pytest.raises(ReconciliationException) as exc_info: - TriggerReconService.trigger_recon( - mock_workspace_client, mock_spark, table_recon, reconcile_config_row, local_test_run=True - ) + TriggerReconService.trigger_recon(mock_workspace_client, spark, table_recon, reconcile_config_row) if exc_info.value.reconcile_output is not None: assert exc_info.value.reconcile_output.recon_id == "00112233-4455-6677-8899-aabbccddeeff" - expected_remorph_recon = mock_spark.createDataFrame( + expected_remorph_recon = spark.createDataFrame( data=[ ( - 33333, + 33333333333, "00112233-4455-6677-8899-aabbccddeeff", "Snowflake", ("org", "data", "supplier"), ("org", "data", "target_supplier"), "row", "reconcile", - datetime(2024, 5, 23, 9, 21, 25, 122185), - datetime(2024, 5, 23, 9, 21, 25, 122185), + MOCK_TIMESTAMP, + MOCK_TIMESTAMP, ) ], schema=recon_schema, ) - expected_remorph_recon_metrics = mock_spark.createDataFrame( + expected_remorph_recon_metrics = spark.createDataFrame( data=[ ( - 33333, + 33333333333, (3, 3, (2, 2), None, None), (False, "remorph", ""), - datetime(2024, 5, 23, 9, 21, 25, 122185), + MOCK_TIMESTAMP, ) ], schema=metrics_schema, ) - expected_remorph_recon_details = mock_spark.createDataFrame( + expected_remorph_recon_details = spark.createDataFrame( data=[ ( - 33333, + 33333333333, "missing_in_source", False, [ @@ -1512,10 +1527,10 @@ def test_recon_for_report_type_is_row( 's_suppkey': '4', }, ], - datetime(2024, 5, 23, 9, 21, 25, 122185), + MOCK_TIMESTAMP, ), ( - 33333, + 33333333333, "missing_in_target", False, [ @@ -1534,18 +1549,20 @@ def test_recon_for_report_type_is_row( 's_suppkey': '3', }, ], - datetime(2024, 5, 23, 9, 21, 25, 122185), + MOCK_TIMESTAMP, ), ], schema=details_schema, ) - assertDataFrameEqual(mock_spark.sql(f"SELECT * FROM {schema}.MAIN"), expected_remorph_recon, ignoreNullable=True) assertDataFrameEqual( - mock_spark.sql(f"SELECT * FROM {schema}.METRICS"), expected_remorph_recon_metrics, ignoreNullable=True + spark.sql(f"SELECT * FROM {catalog}.{schema}.MAIN"), expected_remorph_recon, ignoreNullable=True ) assertDataFrameEqual( - mock_spark.sql(f"SELECT * FROM {schema}.DETAILS"), expected_remorph_recon_details, ignoreNullable=True + spark.sql(f"SELECT * FROM {catalog}.{schema}.METRICS"), expected_remorph_recon_metrics, ignoreNullable=True + ) + assertDataFrameEqual( + spark.sql(f"SELECT * FROM {catalog}.{schema}.DETAILS"), expected_remorph_recon_details, ignoreNullable=True ) @@ -1576,10 +1593,11 @@ def mock_for_recon_exception(normalized_table_conf_with_opts, recon_metadata): def test_schema_recon_with_data_source_exception( - mock_workspace_client, mock_spark, report_tables_schema, mock_for_recon_exception, tmp_path: Path, recon_id: UUID + mock_workspace_client, spark, report_tables_schema, mock_for_recon_exception, tmp_path: Path, recon_id: UUID ): recon_schema, metrics_schema, details_schema = report_tables_schema table_recon, source, target, reconcile_config_exception = mock_for_recon_exception + catalog = reconcile_config_exception.metadata_config.catalog schema = reconcile_config_exception.metadata_config.schema reconcile_config_exception.report_type = "schema" with ( @@ -1592,65 +1610,66 @@ def test_schema_recon_with_data_source_exception( ), patch( "databricks.labs.lakebridge.reconcile.recon_capture.ReconCapture._generate_recon_main_id", - return_value=33333, + return_value=33333333333, ), pytest.raises(ReconciliationException, match=recon_id.hex), ): - mock_datetime.now.return_value = datetime(2024, 5, 23, 9, 21, 25, 122185) - recon_datetime.now.return_value = datetime(2024, 5, 23, 9, 21, 25, 122185) - TriggerReconService.trigger_recon( - mock_workspace_client, mock_spark, table_recon, reconcile_config_exception, local_test_run=True - ) + mock_datetime.now.return_value = MOCK_TIMESTAMP + recon_datetime.now.return_value = MOCK_TIMESTAMP + TriggerReconService.trigger_recon(mock_workspace_client, spark, table_recon, reconcile_config_exception) - expected_remorph_recon = mock_spark.createDataFrame( + expected_remorph_recon = spark.createDataFrame( data=[ ( - 33333, + 33333333333, recon_id.hex, "Snowflake", ("org", "data", "supplier"), ("org", "data", "target_supplier"), "schema", "reconcile", - datetime(2024, 5, 23, 9, 21, 25, 122185), - datetime(2024, 5, 23, 9, 21, 25, 122185), + MOCK_TIMESTAMP, + MOCK_TIMESTAMP, ) ], schema=recon_schema, ) - expected_remorph_recon_metrics = mock_spark.createDataFrame( + expected_remorph_recon_metrics = spark.createDataFrame( data=[ ( - 33333, + 33333333333, (0, 0, None, None, None), ( False, "remorph", "Runtime exception occurred while fetching schema using (org, data, supplier) : Mock Exception", ), - datetime(2024, 5, 23, 9, 21, 25, 122185), + MOCK_TIMESTAMP, ) ], schema=metrics_schema, ) - expected_remorph_recon_details = mock_spark.createDataFrame(data=[], schema=details_schema) + expected_remorph_recon_details = spark.createDataFrame(data=[], schema=details_schema) - assertDataFrameEqual(mock_spark.sql(f"SELECT * FROM {schema}.MAIN"), expected_remorph_recon, ignoreNullable=True) assertDataFrameEqual( - mock_spark.sql(f"SELECT * FROM {schema}.METRICS"), expected_remorph_recon_metrics, ignoreNullable=True + spark.sql(f"SELECT * FROM {catalog}.{schema}.MAIN"), expected_remorph_recon, ignoreNullable=True ) assertDataFrameEqual( - mock_spark.sql(f"SELECT * FROM {schema}.DETAILS"), expected_remorph_recon_details, ignoreNullable=True + spark.sql(f"SELECT * FROM {catalog}.{schema}.METRICS"), expected_remorph_recon_metrics, ignoreNullable=True + ) + assertDataFrameEqual( + spark.sql(f"SELECT * FROM {catalog}.{schema}.DETAILS"), expected_remorph_recon_details, ignoreNullable=True ) def test_schema_recon_with_general_exception( - mock_workspace_client, mock_spark, report_tables_schema, mock_for_report_type_schema, tmp_path: Path, recon_id: UUID + mock_workspace_client, spark, report_tables_schema, mock_for_report_type_schema, tmp_path: Path, recon_id: UUID ): recon_schema, metrics_schema, details_schema = report_tables_schema table_recon, source, target, reconcile_config_schema = mock_for_report_type_schema reconcile_config_schema.data_source = "snowflake" reconcile_config_schema.secret_scope = "remorph_snowflake" + catalog = reconcile_config_schema.metadata_config.catalog schema = reconcile_config_schema.metadata_config.schema with ( patch("databricks.labs.lakebridge.reconcile.trigger_recon_service.datetime") as mock_datetime, @@ -1662,7 +1681,7 @@ def test_schema_recon_with_general_exception( ), patch( "databricks.labs.lakebridge.reconcile.recon_capture.ReconCapture._generate_recon_main_id", - return_value=33333, + return_value=33333333333, ), patch( "databricks.labs.lakebridge.reconcile.reconciliation.Reconciliation.reconcile_schema" @@ -1670,59 +1689,60 @@ def test_schema_recon_with_general_exception( pytest.raises(ReconciliationException, match=recon_id.hex), ): schema_source_mock.side_effect = PySparkException("Unknown Error") - mock_datetime.now.return_value = datetime(2024, 5, 23, 9, 21, 25, 122185) - recon_datetime.now.return_value = datetime(2024, 5, 23, 9, 21, 25, 122185) - TriggerReconService.trigger_recon( - mock_workspace_client, mock_spark, table_recon, reconcile_config_schema, local_test_run=True - ) + mock_datetime.now.return_value = MOCK_TIMESTAMP + recon_datetime.now.return_value = MOCK_TIMESTAMP + TriggerReconService.trigger_recon(mock_workspace_client, spark, table_recon, reconcile_config_schema) - expected_remorph_recon = mock_spark.createDataFrame( + expected_remorph_recon = spark.createDataFrame( data=[ ( - 33333, + 33333333333, recon_id.hex, "Snowflake", ("org", "data", "supplier"), ("org", "data", "target_supplier"), "schema", "reconcile", - datetime(2024, 5, 23, 9, 21, 25, 122185), - datetime(2024, 5, 23, 9, 21, 25, 122185), + MOCK_TIMESTAMP, + MOCK_TIMESTAMP, ) ], schema=recon_schema, ) - expected_remorph_recon_metrics = mock_spark.createDataFrame( + expected_remorph_recon_metrics = spark.createDataFrame( data=[ ( - 33333, + 33333333333, (0, 0, None, None, None), ( False, "remorph", "Unknown Error", ), - datetime(2024, 5, 23, 9, 21, 25, 122185), + MOCK_TIMESTAMP, ) ], schema=metrics_schema, ) - expected_remorph_recon_details = mock_spark.createDataFrame(data=[], schema=details_schema) + expected_remorph_recon_details = spark.createDataFrame(data=[], schema=details_schema) - assertDataFrameEqual(mock_spark.sql(f"SELECT * FROM {schema}.MAIN"), expected_remorph_recon, ignoreNullable=True) assertDataFrameEqual( - mock_spark.sql(f"SELECT * FROM {schema}.METRICS"), expected_remorph_recon_metrics, ignoreNullable=True + spark.sql(f"SELECT * FROM {catalog}.{schema}.MAIN"), expected_remorph_recon, ignoreNullable=True ) assertDataFrameEqual( - mock_spark.sql(f"SELECT * FROM {schema}.DETAILS"), expected_remorph_recon_details, ignoreNullable=True + spark.sql(f"SELECT * FROM {catalog}.{schema}.METRICS"), expected_remorph_recon_metrics, ignoreNullable=True + ) + assertDataFrameEqual( + spark.sql(f"SELECT * FROM {catalog}.{schema}.DETAILS"), expected_remorph_recon_details, ignoreNullable=True ) def test_data_recon_with_general_exception( - mock_workspace_client, mock_spark, report_tables_schema, mock_for_report_type_schema, tmp_path: Path, recon_id: UUID + mock_workspace_client, spark, report_tables_schema, mock_for_report_type_schema, tmp_path: Path, recon_id: UUID ): recon_schema, metrics_schema, details_schema = report_tables_schema table_recon, source, target, reconcile_config = mock_for_report_type_schema + catalog = reconcile_config.metadata_config.catalog schema = reconcile_config.metadata_config.schema reconcile_config.data_source = "snowflake" reconcile_config.secret_scope = "remorph_snowflake" @@ -1737,65 +1757,66 @@ def test_data_recon_with_general_exception( ), patch( "databricks.labs.lakebridge.reconcile.recon_capture.ReconCapture._generate_recon_main_id", - return_value=33333, + return_value=33333333333, ), patch("databricks.labs.lakebridge.reconcile.reconciliation.Reconciliation.reconcile_data") as data_source_mock, pytest.raises(ReconciliationException, match=recon_id.hex), ): data_source_mock.side_effect = DataSourceRuntimeException("Unknown Error") - mock_datetime.now.return_value = datetime(2024, 5, 23, 9, 21, 25, 122185) - recon_datetime.now.return_value = datetime(2024, 5, 23, 9, 21, 25, 122185) - TriggerReconService.trigger_recon( - mock_workspace_client, mock_spark, table_recon, reconcile_config, local_test_run=True - ) + mock_datetime.now.return_value = MOCK_TIMESTAMP + recon_datetime.now.return_value = MOCK_TIMESTAMP + TriggerReconService.trigger_recon(mock_workspace_client, spark, table_recon, reconcile_config) - expected_remorph_recon = mock_spark.createDataFrame( + expected_remorph_recon = spark.createDataFrame( data=[ ( - 33333, + 33333333333, recon_id.hex, "Snowflake", ("org", "data", "supplier"), ("org", "data", "target_supplier"), "data", "reconcile", - datetime(2024, 5, 23, 9, 21, 25, 122185), - datetime(2024, 5, 23, 9, 21, 25, 122185), + MOCK_TIMESTAMP, + MOCK_TIMESTAMP, ) ], schema=recon_schema, ) - expected_remorph_recon_metrics = mock_spark.createDataFrame( + expected_remorph_recon_metrics = spark.createDataFrame( data=[ ( - 33333, + 33333333333, (3, 3, None, None, None), ( False, "remorph", "Unknown Error", ), - datetime(2024, 5, 23, 9, 21, 25, 122185), + MOCK_TIMESTAMP, ) ], schema=metrics_schema, ) - expected_remorph_recon_details = mock_spark.createDataFrame(data=[], schema=details_schema) + expected_remorph_recon_details = spark.createDataFrame(data=[], schema=details_schema) - assertDataFrameEqual(mock_spark.sql(f"SELECT * FROM {schema}.MAIN"), expected_remorph_recon, ignoreNullable=True) assertDataFrameEqual( - mock_spark.sql(f"SELECT * FROM {schema}.METRICS"), expected_remorph_recon_metrics, ignoreNullable=True + spark.sql(f"SELECT * FROM {catalog}.{schema}.MAIN"), expected_remorph_recon, ignoreNullable=True ) assertDataFrameEqual( - mock_spark.sql(f"SELECT * FROM {schema}.DETAILS"), expected_remorph_recon_details, ignoreNullable=True + spark.sql(f"SELECT * FROM {catalog}.{schema}.METRICS"), expected_remorph_recon_metrics, ignoreNullable=True + ) + assertDataFrameEqual( + spark.sql(f"SELECT * FROM {catalog}.{schema}.DETAILS"), expected_remorph_recon_details, ignoreNullable=True ) def test_data_recon_with_source_exception( - mock_workspace_client, mock_spark, report_tables_schema, mock_for_report_type_schema, tmp_path: Path, recon_id: UUID + mock_workspace_client, spark, report_tables_schema, mock_for_report_type_schema, tmp_path: Path, recon_id: UUID ): recon_schema, metrics_schema, details_schema = report_tables_schema table_recon, source, target, reconcile_config = mock_for_report_type_schema + catalog = reconcile_config.metadata_config.catalog schema = reconcile_config.metadata_config.schema reconcile_config.data_source = "snowflake" reconcile_config.secret_scope = "remorph_snowflake" @@ -1810,74 +1831,74 @@ def test_data_recon_with_source_exception( ), patch( "databricks.labs.lakebridge.reconcile.recon_capture.ReconCapture._generate_recon_main_id", - return_value=33333, + return_value=33333333333, ), patch("databricks.labs.lakebridge.reconcile.reconciliation.Reconciliation.reconcile_data") as data_source_mock, pytest.raises(ReconciliationException, match=recon_id.hex), ): data_source_mock.side_effect = DataSourceRuntimeException("Source Runtime Error") - mock_datetime.now.return_value = datetime(2024, 5, 23, 9, 21, 25, 122185) - recon_datetime.now.return_value = datetime(2024, 5, 23, 9, 21, 25, 122185) - TriggerReconService.trigger_recon( - mock_workspace_client, mock_spark, table_recon, reconcile_config, local_test_run=True - ) + mock_datetime.now.return_value = MOCK_TIMESTAMP + recon_datetime.now.return_value = MOCK_TIMESTAMP + TriggerReconService.trigger_recon(mock_workspace_client, spark, table_recon, reconcile_config) - expected_remorph_recon = mock_spark.createDataFrame( + expected_remorph_recon = spark.createDataFrame( data=[ ( - 33333, + 33333333333, recon_id.hex, "Snowflake", ("org", "data", "supplier"), ("org", "data", "target_supplier"), "data", "reconcile", - datetime(2024, 5, 23, 9, 21, 25, 122185), - datetime(2024, 5, 23, 9, 21, 25, 122185), + MOCK_TIMESTAMP, + MOCK_TIMESTAMP, ) ], schema=recon_schema, ) - expected_remorph_recon_metrics = mock_spark.createDataFrame( + expected_remorph_recon_metrics = spark.createDataFrame( data=[ ( - 33333, + 33333333333, (3, 3, None, None, None), ( False, "remorph", "Source Runtime Error", ), - datetime(2024, 5, 23, 9, 21, 25, 122185), + MOCK_TIMESTAMP, ) ], schema=metrics_schema, ) - expected_remorph_recon_details = mock_spark.createDataFrame(data=[], schema=details_schema) + expected_remorph_recon_details = spark.createDataFrame(data=[], schema=details_schema) - assertDataFrameEqual(mock_spark.sql(f"SELECT * FROM {schema}.MAIN"), expected_remorph_recon, ignoreNullable=True) assertDataFrameEqual( - mock_spark.sql(f"SELECT * FROM {schema}.METRICS"), expected_remorph_recon_metrics, ignoreNullable=True + spark.sql(f"SELECT * FROM {catalog}.{schema}.MAIN"), expected_remorph_recon, ignoreNullable=True ) assertDataFrameEqual( - mock_spark.sql(f"SELECT * FROM {schema}.DETAILS"), expected_remorph_recon_details, ignoreNullable=True + spark.sql(f"SELECT * FROM {catalog}.{schema}.METRICS"), expected_remorph_recon_metrics, ignoreNullable=True + ) + assertDataFrameEqual( + spark.sql(f"SELECT * FROM {catalog}.{schema}.DETAILS"), expected_remorph_recon_details, ignoreNullable=True ) -def test_initialise_data_source(mock_workspace_client, mock_spark): +def test_initialise_data_source(mock_workspace_client, spark): src_engine = get_dialect("snowflake") secret_scope = "test" - source, target = initialise_data_source(mock_workspace_client, mock_spark, src_engine, secret_scope) + source, target = initialise_data_source(mock_workspace_client, spark, src_engine, secret_scope) - snowflake_data_source = SnowflakeDataSource(src_engine, mock_spark, mock_workspace_client, secret_scope).__class__ - databricks_data_source = DatabricksDataSource(src_engine, mock_spark, mock_workspace_client, secret_scope).__class__ + snowflake_data_source = SnowflakeDataSource(src_engine, spark, mock_workspace_client, secret_scope).__class__ + databricks_data_source = DatabricksDataSource(src_engine, spark, mock_workspace_client, secret_scope).__class__ assert isinstance(source, snowflake_data_source) assert isinstance(target, databricks_data_source) -def test_recon_for_wrong_report_type(mock_workspace_client, mock_spark, mock_for_report_type_row): +def test_recon_for_wrong_report_type(mock_workspace_client, spark, mock_for_report_type_row): source, target, table_recon, reconcile_config = mock_for_report_type_row reconcile_config.report_type = "ro" with ( @@ -1890,19 +1911,17 @@ def test_recon_for_wrong_report_type(mock_workspace_client, mock_spark, mock_for ), patch( "databricks.labs.lakebridge.reconcile.recon_capture.ReconCapture._generate_recon_main_id", - return_value=33333, + return_value=33333333333, ), pytest.raises(InvalidInputException), ): - mock_datetime.now.return_value = datetime(2024, 5, 23, 9, 21, 25, 122185) - recon_datetime.now.return_value = datetime(2024, 5, 23, 9, 21, 25, 122185) - TriggerReconService.trigger_recon( - mock_workspace_client, mock_spark, table_recon, reconcile_config, local_test_run=True - ) + mock_datetime.now.return_value = MOCK_TIMESTAMP + recon_datetime.now.return_value = MOCK_TIMESTAMP + TriggerReconService.trigger_recon(mock_workspace_client, spark, table_recon, reconcile_config) def test_reconcile_data_with_threshold_and_row_report_type( - mock_spark, normalized_table_conf_with_opts, table_schema_ansi_ansi, query_store, tmp_path: Path + spark, normalized_table_conf_with_opts, table_schema_ansi_ansi, query_store, tmp_path: Path ): src_schema, tgt_schema = table_schema_ansi_ansi source_dataframe_repository = { @@ -1910,13 +1929,13 @@ def test_reconcile_data_with_threshold_and_row_report_type( CATALOG, SCHEMA, query_store.row_queries.source_row_query, - ): mock_spark.createDataFrame( + ): spark.createDataFrame( [ Row(hash_value_recon="a1b", s_nationkey=11, s_suppkey=1), Row(hash_value_recon="c2d", s_nationkey=22, s_suppkey=2), ] ), - (CATALOG, SCHEMA, query_store.threshold_queries.source_threshold_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, query_store.threshold_queries.source_threshold_query): spark.createDataFrame( [Row(s_nationkey=11, s_suppkey=1, s_acctbal=100)] ), } @@ -1927,16 +1946,16 @@ def test_reconcile_data_with_threshold_and_row_report_type( CATALOG, SCHEMA, query_store.row_queries.target_row_query, - ): mock_spark.createDataFrame( + ): spark.createDataFrame( [ Row(hash_value_recon="a1b", s_nationkey=11, s_suppkey=1), Row(hash_value_recon="c2d", s_nationkey=22, s_suppkey=2), ] ), - (CATALOG, SCHEMA, query_store.threshold_queries.target_threshold_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, query_store.threshold_queries.target_threshold_query): spark.createDataFrame( [Row(s_nationkey=11, s_suppkey=1, s_acctbal=110)] ), - (CATALOG, SCHEMA, query_store.threshold_queries.threshold_comparison_query): mock_spark.createDataFrame( + (CATALOG, SCHEMA, query_store.threshold_queries.threshold_comparison_query): spark.createDataFrame( [ Row( s_acctbal_source=100, @@ -1956,7 +1975,7 @@ def test_reconcile_data_with_threshold_and_row_report_type( target_catalog=CATALOG, target_schema=SCHEMA, ) - schema_comparator = SchemaCompare(mock_spark) + schema_comparator = SchemaCompare(spark) source = MockDataSource(source_dataframe_repository, source_schema_repository) target = MockDataSource(target_dataframe_repository, target_schema_repository) @@ -1967,7 +1986,7 @@ def test_reconcile_data_with_threshold_and_row_report_type( "row", schema_comparator, get_dialect("databricks"), - mock_spark, + spark, ReconcileMetadataConfig(), FakeReconIntermediatePersist(), ).reconcile_data(normalized_table_conf_with_opts, src_schema, tgt_schema) @@ -1982,7 +2001,7 @@ def test_reconcile_data_with_threshold_and_row_report_type( @patch('databricks.labs.lakebridge.reconcile.recon_capture.generate_final_reconcile_output') def test_recon_output_without_exception(mock_gen_final_recon_output): mock_workspace_client = MagicMock() - mock_spark = MagicMock() + spark = MagicMock() mock_table_recon = MagicMock() mock_gen_final_recon_output.return_value = ReconcileOutput( recon_id="00112233-4455-6677-8899-aabbccddeeff", @@ -2015,7 +2034,7 @@ def test_recon_output_without_exception(mock_gen_final_recon_output): try: TriggerReconService.trigger_recon( mock_workspace_client, - mock_spark, + spark, mock_table_recon, reconcile_config, ) diff --git a/tests/integration/reconcile/query_builder/test_sampling_query.py b/tests/integration/reconcile/query_builder/test_sampling_query.py index eaf7082773..18c66e7ebb 100644 --- a/tests/integration/reconcile/query_builder/test_sampling_query.py +++ b/tests/integration/reconcile/query_builder/test_sampling_query.py @@ -14,9 +14,8 @@ def test_build_query_for_snowflake_src( - mock_spark, table_conf, table_schema_oracle_ansi, fake_oracle_datasource, fake_databricks_datasource + spark, table_conf, table_schema_oracle_ansi, fake_oracle_datasource, fake_databricks_datasource ): - spark = mock_spark sch, sch_with_alias = table_schema_oracle_ansi df_schema = StructType( [ @@ -97,14 +96,13 @@ def test_build_query_for_snowflake_src( def test_build_query_for_oracle_src( - mock_spark, + spark, table_conf, table_schema_oracle_ansi, normalized_column_mapping, fake_oracle_datasource, fake_databricks_datasource, ): - spark = mock_spark _, sch_with_alias = table_schema_oracle_ansi df_schema = StructType( [ @@ -186,8 +184,7 @@ def test_build_query_for_oracle_src( assert tgt_actual == tgt_expected -def test_build_query_for_databricks_src(mock_spark, table_conf, fake_databricks_datasource): - spark = mock_spark +def test_build_query_for_databricks_src(spark, table_conf, fake_databricks_datasource): df_schema = StructType( [ StructField('s_suppkey', IntegerType()), @@ -234,9 +231,8 @@ def test_build_query_for_databricks_src(mock_spark, table_conf, fake_databricks_ def test_build_query_for_snowflake_without_transformations( - mock_spark, table_conf, table_schema_oracle_ansi, fake_oracle_datasource, fake_databricks_datasource + spark, table_conf, table_schema_oracle_ansi, fake_oracle_datasource, fake_databricks_datasource ): - spark = mock_spark sch, sch_with_alias = table_schema_oracle_ansi df_schema = StructType( [ @@ -314,9 +310,8 @@ def test_build_query_for_snowflake_without_transformations( def test_build_query_for_snowflake_src_for_non_integer_primary_keys( - mock_spark, table_conf, fake_oracle_datasource, fake_databricks_datasource + spark, table_conf, fake_oracle_datasource, fake_databricks_datasource ): - spark = mock_spark sch = [ oracle_schema_fixture_factory("s_suppkey", "varchar"), oracle_schema_fixture_factory("s_name", "varchar"), diff --git a/tests/integration/reconcile/test_aggregates_recon_capture.py b/tests/integration/reconcile/test_aggregates_recon_capture.py index ff41d71bdd..2b743bdbf6 100644 --- a/tests/integration/reconcile/test_aggregates_recon_capture.py +++ b/tests/integration/reconcile/test_aggregates_recon_capture.py @@ -1,5 +1,4 @@ -import datetime -from pathlib import Path +from datetime import datetime, timezone from pyspark.sql import Row, SparkSession @@ -17,24 +16,12 @@ from tests.unit.conftest import get_dialect -def remove_directory_recursively(directory_path): - path = Path(directory_path) - if path.is_dir(): - for item in path.iterdir(): - if item.is_dir(): - remove_directory_recursively(item) - else: - item.unlink() - path.rmdir() - - def agg_data_prep(spark: SparkSession): table_conf = Table(source_name="supplier", target_name="target_supplier") reconcile_process_duration = ReconcileProcessDuration( - start_ts=str(datetime.datetime.now()), end_ts=str(datetime.datetime.now()) + start_ts=str(datetime.now(tz=timezone.utc)), end_ts=str(datetime.now(tz=timezone.utc)) ) - # Prepare output dataclasses agg_reconcile_output = [ AggregateQueryOutput( rule=expected_rule_output()["count"], reconcile_output=expected_reconcile_output_dict(spark)["count"] @@ -44,31 +31,18 @@ def agg_data_prep(spark: SparkSession): ), ] - # Drop old data - spark.sql("DROP TABLE IF EXISTS DEFAULT.main") - spark.sql("DROP TABLE IF EXISTS DEFAULT.aggregate_rules") - spark.sql("DROP TABLE IF EXISTS DEFAULT.aggregate_metrics") - spark.sql("DROP TABLE IF EXISTS DEFAULT.aggregate_details") - - # Get the warehouse location - warehouse_location = spark.conf.get("spark.sql.warehouse.dir") - - if warehouse_location and Path(warehouse_location.lstrip('file:')).exists(): - tables = ["main", "aggregate_rules", "aggregate_metrics", "aggregate_details"] - for table in tables: - remove_directory_recursively(f"{warehouse_location.lstrip('file:')}/{table}") - return agg_reconcile_output, table_conf, reconcile_process_duration -def test_aggregates_reconcile_store_aggregate_metrics(mock_workspace_client, mock_spark): +def test_aggregates_reconcile_store_aggregate_metrics( + mock_workspace_client, spark, recon_metadata: ReconcileMetadataConfig +): database_config = DatabaseConfig( "source_test_schema", "target_test_catalog", "target_test_schema", "source_test_catalog" ) source_type = get_dialect("snowflake") - spark = mock_spark - agg_reconcile_output, table_conf, reconcile_process_duration = agg_data_prep(mock_spark) + agg_reconcile_output, table_conf, reconcile_process_duration = agg_data_prep(spark) recon_id = "999fygdrs-dbb7-489f-bad1-6a7e8f4821b1" @@ -79,15 +53,14 @@ def test_aggregates_reconcile_store_aggregate_metrics(mock_workspace_client, moc source_type, mock_workspace_client, spark, - metadata_config=ReconcileMetadataConfig(schema="default"), - local_test_run=True, + metadata_config=recon_metadata, ) recon_capture.store_aggregates_metrics(table_conf, reconcile_process_duration, agg_reconcile_output) - # Check if the tables are created + table_prefix = f"{recon_metadata.catalog}.{recon_metadata.schema}" # assert main table data - remorph_reconcile_df = spark.sql("select * from DEFAULT.main") + remorph_reconcile_df = spark.sql(f"select * from {table_prefix}.main") assert remorph_reconcile_df.count() == 1 if remorph_reconcile_df.first(): @@ -98,7 +71,7 @@ def test_aggregates_reconcile_store_aggregate_metrics(mock_workspace_client, moc assert main.get("operation_name") == "aggregates-reconcile" # assert rules data - agg_reconcile_rules_df = spark.sql("select * from DEFAULT.aggregate_rules") + agg_reconcile_rules_df = spark.sql(f"select * from {table_prefix}.aggregate_rules") assert agg_reconcile_rules_df.count() == 2 assert agg_reconcile_rules_df.select("rule_type").distinct().count() == 1 @@ -109,7 +82,7 @@ def test_aggregates_reconcile_store_aggregate_metrics(mock_workspace_client, moc assert rule["rule_info"].keys() == {"agg_type", "agg_column", "group_by_columns"} # assert metrics - agg_reconcile_metrics_df = spark.sql("select * from DEFAULT.aggregate_metrics") + agg_reconcile_metrics_df = spark.sql(f"select * from {table_prefix}.aggregate_metrics") assert agg_reconcile_metrics_df.count() == 2 if agg_reconcile_metrics_df.first(): @@ -118,7 +91,7 @@ def test_aggregates_reconcile_store_aggregate_metrics(mock_workspace_client, moc assert metric.get("recon_metrics").asDict().keys() == {"mismatch", "missing_in_source", "missing_in_target"} # assert details - agg_reconcile_details_df = spark.sql("select * from DEFAULT.aggregate_details") + agg_reconcile_details_df = spark.sql(f"select * from {table_prefix}.aggregate_details") assert agg_reconcile_details_df.count() == 6 assert agg_reconcile_details_df.select("recon_type").distinct().count() == 3 @@ -130,9 +103,8 @@ def test_aggregates_reconcile_store_aggregate_metrics(mock_workspace_client, moc reconcile_output = generate_final_reconcile_aggregate_output( recon_id=recon_id, - spark=mock_spark, - metadata_config=ReconcileMetadataConfig(schema="default"), - local_test_run=True, + spark=spark, + metadata_config=recon_metadata, ) assert len(reconcile_output.results) == 1 assert not reconcile_output.results[0].exception_message diff --git a/tests/integration/reconcile/test_aggregates_reconcile.py b/tests/integration/reconcile/test_aggregates_reconcile.py index a4203bb851..ffcfdb39d4 100644 --- a/tests/integration/reconcile/test_aggregates_reconcile.py +++ b/tests/integration/reconcile/test_aggregates_reconcile.py @@ -46,7 +46,7 @@ class AggregateQueryStore: @pytest.fixture -def query_store(mock_spark: SparkSession) -> AggregateQueryStore: +def query_store() -> AggregateQueryStore: agg_queries = AggregateQueries( source_agg_query="SELECT min(`s_acctbal`) AS `source_min_s_acctbal` FROM :tbl WHERE s_name = 't' AND s_address = 'a'", target_agg_query="SELECT min(`s_acctbal_t`) AS `target_min_s_acctbal` FROM :tbl WHERE s_name = 't' AND s_address_t = 'a'", @@ -58,7 +58,7 @@ def query_store(mock_spark: SparkSession) -> AggregateQueryStore: @pytest.fixture -def query_store_special_char(mock_spark: SparkSession) -> AggregateQueryStore: +def query_store_special_char() -> AggregateQueryStore: agg_queries = AggregateQueries( source_agg_query=""" SELECT sum("s_acctbal") AS "source_sum_s_acctbal", count(TRIM(s_name)) AS "source_count_s_name", min("$carat$") AS "source_min_$carat$", max("$carat$") AS "source_max_$carat$", "s_nationkey" AS "source_group_by_s_nationkey" FROM :tbl WHERE s_name = 't' AND s_address = 'a' GROUP BY "s_nationkey" """.strip(), target_agg_query="SELECT sum(`s_acctbal_t`) AS `target_sum_s_acctbal`, count(TRIM(s_name)) AS `target_count_s_name`, min(`$carat$`) AS `target_min_$carat$`, max(`$carat$`) AS `target_max_$carat$`, `s_nationkey_t` AS `target_group_by_s_nationkey` FROM :tbl WHERE s_name = 't' AND s_address_t = 'a' GROUP BY `s_nationkey_t`", @@ -70,7 +70,7 @@ def query_store_special_char(mock_spark: SparkSession) -> AggregateQueryStore: def test_reconcile_aggregate_data_missing_records( - mock_spark: SparkSession, + spark: SparkSession, normalized_table_conf_with_opts: Table, table_schema_ansi_ansi: tuple[list[Schema], list[Schema]], query_store: AggregateQueryStore, @@ -86,7 +86,7 @@ def test_reconcile_aggregate_data_missing_records( CATALOG, SCHEMA, query_store.agg_queries.source_agg_query, - ): mock_spark.createDataFrame( + ): spark.createDataFrame( [ Row(source_min_s_acctbal=11), ] @@ -99,7 +99,7 @@ def test_reconcile_aggregate_data_missing_records( CATALOG, SCHEMA, query_store.agg_queries.target_agg_query, - ): mock_spark.createDataFrame( + ): spark.createDataFrame( [ Row(target_min_s_acctbal=10), ] @@ -120,9 +120,9 @@ def test_reconcile_aggregate_data_missing_records( target, database_config, "", - SchemaCompare(mock_spark), + SchemaCompare(spark), get_dialect("databricks"), - mock_spark, + spark, ReconcileMetadataConfig(), FakeReconIntermediatePersist(), ).reconcile_aggregates(normalized_table_conf_with_opts, src_schema, tgt_schema) @@ -148,7 +148,7 @@ def test_reconcile_aggregate_data_missing_records( missing_in_tgt_count=0, mismatch=MismatchOutput( mismatch_columns=None, - mismatch_df=mock_spark.createDataFrame( + mismatch_df=spark.createDataFrame( [ Row( source_min_s_acctbal=11, @@ -289,7 +289,7 @@ def _compare_reconcile_output( def test_reconcile_aggregate_data_mismatch_and_missing_records( - mock_spark: SparkSession, + spark: SparkSession, normalized_table_conf_with_opts: Table, table_schema_oracle_ansi: tuple[list[Schema], list[Schema]], query_store_special_char: AggregateQueryStore, @@ -321,7 +321,7 @@ def test_reconcile_aggregate_data_mismatch_and_missing_records( CATALOG, SCHEMA, query_store_special_char.agg_queries.source_agg_query, - ): mock_spark.createDataFrame( + ): spark.createDataFrame( [ source_df_model(101, 13, 1, 2, 11), source_df_model(23, 11, 0, 1, 12), @@ -343,7 +343,7 @@ def test_reconcile_aggregate_data_mismatch_and_missing_records( CATALOG, SCHEMA, query_store_special_char.agg_queries.target_agg_query, - ): mock_spark.createDataFrame( + ): spark.createDataFrame( [ target_df_model(101, 13, 1, 2, 11), target_df_model(43, 9, 0, 1, 12), @@ -366,9 +366,9 @@ def test_reconcile_aggregate_data_mismatch_and_missing_records( target, db_config, "", - SchemaCompare(mock_spark), + SchemaCompare(spark), get_dialect("snowflake"), - mock_spark, + spark, ReconcileMetadataConfig(), FakeReconIntermediatePersist(), ).reconcile_aggregates(normalized_table_conf_with_opts, src_schema, tgt_schema) @@ -394,7 +394,7 @@ def test_reconcile_aggregate_data_mismatch_and_missing_records( # Reconcile Output validations _compare_reconcile_output( - actual.reconcile_output, expected_reconcile_output_dict(mock_spark).get(actual.rule.agg_type) + actual.reconcile_output, expected_reconcile_output_dict(spark).get(actual.rule.agg_type) ) diff --git a/tests/integration/reconcile/test_compare.py b/tests/integration/reconcile/test_compare.py index 7b1733de6f..d4c6f39b7e 100644 --- a/tests/integration/reconcile/test_compare.py +++ b/tests/integration/reconcile/test_compare.py @@ -16,10 +16,10 @@ def test_compare_data_for_report_all( - mock_spark, + spark, tmp_path: Path, ): - source = mock_spark.createDataFrame( + source = spark.createDataFrame( [ Row(s_suppkey=1, s_nationkey=11, hash_value_recon='1a1'), Row(s_suppkey=2, s_nationkey=22, hash_value_recon='2b2'), @@ -27,7 +27,7 @@ def test_compare_data_for_report_all( Row(s_suppkey=5, s_nationkey=55, hash_value_recon='5e5'), ] ) - target = mock_spark.createDataFrame( + target = spark.createDataFrame( [ Row(s_suppkey=1, s_nationkey=11, hash_value_recon='1a1'), Row(s_suppkey=2, s_nationkey=22, hash_value_recon='2b4'), @@ -36,9 +36,9 @@ def test_compare_data_for_report_all( ] ) - mismatch = MismatchOutput(mismatch_df=mock_spark.createDataFrame([Row(s_suppkey=2, s_nationkey=22)])) - missing_in_src = mock_spark.createDataFrame([Row(s_suppkey=4, s_nationkey=44), Row(s_suppkey=5, s_nationkey=56)]) - missing_in_tgt = mock_spark.createDataFrame([Row(s_suppkey=3, s_nationkey=33), Row(s_suppkey=5, s_nationkey=55)]) + mismatch = MismatchOutput(mismatch_df=spark.createDataFrame([Row(s_suppkey=2, s_nationkey=22)])) + missing_in_src = spark.createDataFrame([Row(s_suppkey=4, s_nationkey=44), Row(s_suppkey=5, s_nationkey=56)]) + missing_in_tgt = spark.createDataFrame([Row(s_suppkey=3, s_nationkey=33), Row(s_suppkey=5, s_nationkey=55)]) actual = reconcile_data( source=source, @@ -67,8 +67,8 @@ def test_compare_data_for_report_all( assertDataFrameEqual(actual.missing_in_tgt, expected.missing_in_tgt) -def test_compare_data_for_report_hash(mock_spark, tmp_path: Path): - source = mock_spark.createDataFrame( +def test_compare_data_for_report_hash(spark, tmp_path: Path): + source = spark.createDataFrame( [ Row(s_suppkey=1, s_nationkey=11, hash_value_recon='1a1'), Row(s_suppkey=2, s_nationkey=22, hash_value_recon='2b2'), @@ -76,7 +76,7 @@ def test_compare_data_for_report_hash(mock_spark, tmp_path: Path): Row(s_suppkey=5, s_nationkey=55, hash_value_recon='5e5'), ] ) - target = mock_spark.createDataFrame( + target = spark.createDataFrame( [ Row(s_suppkey=1, s_nationkey=11, hash_value_recon='1a1'), Row(s_suppkey=2, s_nationkey=22, hash_value_recon='2b4'), @@ -85,10 +85,10 @@ def test_compare_data_for_report_hash(mock_spark, tmp_path: Path): ] ) - missing_in_src = mock_spark.createDataFrame( + missing_in_src = spark.createDataFrame( [Row(s_suppkey=2, s_nationkey=22), Row(s_suppkey=4, s_nationkey=44), Row(s_suppkey=5, s_nationkey=56)] ) - missing_in_tgt = mock_spark.createDataFrame( + missing_in_tgt = spark.createDataFrame( [Row(s_suppkey=2, s_nationkey=22), Row(s_suppkey=3, s_nationkey=33), Row(s_suppkey=5, s_nationkey=55)] ) @@ -118,16 +118,16 @@ def test_compare_data_for_report_hash(mock_spark, tmp_path: Path): assertDataFrameEqual(actual.missing_in_tgt, expected.missing_in_tgt) -def test_capture_mismatch_data_and_cols(mock_spark): +def test_capture_mismatch_data_and_cols(spark): # these mock dataframes are expected to contain only mismatched rows. Hence, the matching rows between source and target are removed for this test-case. - source = mock_spark.createDataFrame( + source = spark.createDataFrame( [ Row(s_suppkey=2, s_nationkey=22, s_name='supp-22', s_address='a-2', s_phone='ph-2', s_acctbal=200), Row(s_suppkey=3, s_nationkey=33, s_name='supp-3', s_address='a-3', s_phone='ph-3', s_acctbal=300), Row(s_suppkey=5, s_nationkey=55, s_name='supp-5', s_address='a-5', s_phone='ph-5', s_acctbal=400), ] ) - target = mock_spark.createDataFrame( + target = spark.createDataFrame( [ Row(s_suppkey=2, s_nationkey=22, s_name='supp-2', s_address='a-2', s_phone='ph-2', s_acctbal=2000), Row(s_suppkey=3, s_nationkey=33, s_name='supp-33', s_address='a-3', s_phone='ph-3', s_acctbal=300), @@ -137,7 +137,7 @@ def test_capture_mismatch_data_and_cols(mock_spark): actual = capture_mismatch_data_and_columns(source=source, target=target, key_columns=["s_suppkey", "s_nationkey"]) - expected_df = mock_spark.createDataFrame( + expected_df = spark.createDataFrame( [ Row( s_suppkey=2, @@ -180,15 +180,15 @@ def test_capture_mismatch_data_and_cols(mock_spark): assert sorted(actual.mismatch_columns) == ['s_acctbal', 's_name'] -def test_capture_mismatch_data_and_cols_no_mismatch(mock_spark): +def test_capture_mismatch_data_and_cols_no_mismatch(spark): # this is to test the behaviour of the function `capture_mismatch_data_and_columns` when there is no mismatch in the dataframes. - source = mock_spark.createDataFrame( + source = spark.createDataFrame( [ Row(s_suppkey=1, s_nationkey=11, s_name='supp-1', s_address='a-1', s_phone='ph-1', s_acctbal=100), ] ) - target = mock_spark.createDataFrame( + target = spark.createDataFrame( [ Row(s_suppkey=1, s_nationkey=11, s_name='supp-1', s_address='a-1', s_phone='ph-1', s_acctbal=100), ] @@ -196,7 +196,7 @@ def test_capture_mismatch_data_and_cols_no_mismatch(mock_spark): actual = capture_mismatch_data_and_columns(source=source, target=target, key_columns=["s_suppkey", "s_nationkey"]) - expected_df = mock_spark.createDataFrame( + expected_df = spark.createDataFrame( [ Row( s_suppkey=1, @@ -223,8 +223,8 @@ def test_capture_mismatch_data_and_cols_no_mismatch(mock_spark): assert sorted(actual.mismatch_columns) == [] -def test_capture_mismatch_data_and_cols_fail(mock_spark): - source = mock_spark.createDataFrame( +def test_capture_mismatch_data_and_cols_fail(spark): + source = spark.createDataFrame( [ Row(s_suppkey=1, s_nationkey=11, s_name='supp-1', s_address='a-1', s_phone='ph-1', s_acctbal=100), Row(s_suppkey=2, s_nationkey=22, s_name='supp-22', s_address='a-2', s_phone='ph-2', s_acctbal=200), @@ -232,7 +232,7 @@ def test_capture_mismatch_data_and_cols_fail(mock_spark): Row(s_suppkey=5, s_nationkey=55, s_name='supp-5', s_address='a-5', s_phone='ph-5', s_acctbal=400), ] ) - target = mock_spark.createDataFrame( + target = spark.createDataFrame( [ Row(s_suppkey=1), Row(s_suppkey=2), @@ -251,10 +251,10 @@ def test_capture_mismatch_data_and_cols_fail(mock_spark): ) -def test_compare_data_special_column_names(mock_spark, tmp_path: Path): +def test_compare_data_special_column_names(spark, tmp_path: Path): model_with_hash = Row("s`supp#", "s_nation#", "hash_value_recon") model = Row("s`supp#", "s_nation#") - source = mock_spark.createDataFrame( + source = spark.createDataFrame( [ model_with_hash(1, 11, '1a1'), model_with_hash(2, 22, '2b2'), @@ -262,7 +262,7 @@ def test_compare_data_special_column_names(mock_spark, tmp_path: Path): model_with_hash(5, 55, '5e5'), ] ) - target = mock_spark.createDataFrame( + target = spark.createDataFrame( [ model_with_hash(1, 11, '1a1'), model_with_hash(2, 22, '2b4'), @@ -271,9 +271,9 @@ def test_compare_data_special_column_names(mock_spark, tmp_path: Path): ] ) - mismatch = MismatchOutput(mismatch_df=mock_spark.createDataFrame([model(2, 22)])) - missing_in_src = mock_spark.createDataFrame([model(4, 44), model(5, 56)]) - missing_in_tgt = mock_spark.createDataFrame([model(3, 33), model(5, 55)]) + mismatch = MismatchOutput(mismatch_df=spark.createDataFrame([model(2, 22)])) + missing_in_src = spark.createDataFrame([model(4, 44), model(5, 56)]) + missing_in_tgt = spark.createDataFrame([model(3, 33), model(5, 55)]) actual = reconcile_data( source=source, @@ -302,17 +302,17 @@ def test_compare_data_special_column_names(mock_spark, tmp_path: Path): assertDataFrameEqual(actual.missing_in_tgt, expected.missing_in_tgt) -def test_capture_mismatch_data_and_cols_special_column_names(mock_spark): +def test_capture_mismatch_data_and_cols_special_column_names(spark): model = Row("s`supp#", "s_nation#", "s`name") expected_model = Row("s`supp#", "s_nation#", "s`name_base", "s`name_compare", "s`name_match") - source = mock_spark.createDataFrame( + source = spark.createDataFrame( [ model(2, 22, '2b2'), model(3, 33, '3c3'), model(5, 55, '5e5'), ] ) - target = mock_spark.createDataFrame( + target = spark.createDataFrame( [ model(2, 22, '2b4'), model(3, 33, '3c3'), @@ -322,7 +322,7 @@ def test_capture_mismatch_data_and_cols_special_column_names(mock_spark): actual = capture_mismatch_data_and_columns(source=source, target=target, key_columns=["`s``supp#`", "`s_nation#`"]) - expected_df = mock_spark.createDataFrame( + expected_df = spark.createDataFrame( [ expected_model( 2, diff --git a/tests/integration/reconcile/test_oracle_reconcile.py b/tests/integration/reconcile/test_oracle_reconcile.py index 234936a1cb..cb7a3aac11 100644 --- a/tests/integration/reconcile/test_oracle_reconcile.py +++ b/tests/integration/reconcile/test_oracle_reconcile.py @@ -35,13 +35,13 @@ def read_data( @pytest.mark.skip(reason="Requires Oracle DB running locally and a databricks cluster to connect to.") -def test_oracle_db_reconcile(mock_spark, mock_workspace_client, tmp_path): +def test_oracle_db_reconcile(spark, mock_workspace_client, tmp_path): test_env = TestEnvGetter(True) cluster = test_env.get("TEST_USER_ISOLATION_CLUSTER_ID") host = test_env.get("DATABRICKS_HOST") databricks = DatabricksSession.builder.host(host).clusterId(cluster).getOrCreate() - databricks_data_source = DatabricksDataSourceUnderTest(databricks, mock_workspace_client, mock_spark) - oracle_data_source = OracleDataSourceUnderTest(mock_spark, mock_workspace_client) + databricks_data_source = DatabricksDataSourceUnderTest(databricks, mock_workspace_client, spark) + oracle_data_source = OracleDataSourceUnderTest(spark, mock_workspace_client) report = "row" db_config = DatabaseConfig( source_schema="SYSTEM", @@ -60,9 +60,9 @@ def test_oracle_db_reconcile(mock_spark, mock_workspace_client, tmp_path): target=databricks_data_source, database_config=db_config, report_type=report, - schema_comparator=SchemaCompare(mock_spark), + schema_comparator=SchemaCompare(spark), source_engine=get_dialect("oracle"), - spark=mock_spark, + spark=spark, metadata_config=ReconcileMetadataConfig(catalog="tmp", schema="reconcile"), intermediate_persist=FakeReconIntermediatePersist(), ) @@ -72,9 +72,8 @@ def test_oracle_db_reconcile(mock_spark, mock_workspace_client, tmp_path): report_type=report, source_dialect=get_dialect("oracle"), ws=mock_workspace_client, - spark=mock_spark, + spark=spark, metadata_config=ReconcileMetadataConfig(catalog="tmp", schema="reconcile"), - local_test_run=True, ) with patch("databricks.labs.lakebridge.reconcile.utils.generate_volume_path", return_value=str(tmp_path)): _, data_reconcile_output = TriggerReconService.recon_one( diff --git a/tests/integration/reconcile/test_recon_capture.py b/tests/integration/reconcile/test_recon_capture.py index 628f1ca190..7d7f8c23c3 100644 --- a/tests/integration/reconcile/test_recon_capture.py +++ b/tests/integration/reconcile/test_recon_capture.py @@ -1,4 +1,4 @@ -import datetime +from datetime import datetime, timezone import json import tempfile @@ -112,7 +112,7 @@ def data_prep(spark: SparkSession): schema_output = SchemaReconcileOutput(is_valid=True, compare_df=schema_df) table_conf = Table(source_name="supplier", target_name="target_supplier") reconcile_process = ReconcileProcessDuration( - start_ts=str(datetime.datetime.now()), end_ts=str(datetime.datetime.now()) + start_ts=str(datetime.now(tz=timezone.utc)), end_ts=str(datetime.now(tz=timezone.utc)) ) row_count = ReconcileRecordCount(source=5, target=5) @@ -120,13 +120,12 @@ def data_prep(spark: SparkSession): return reconcile_output, schema_output, table_conf, reconcile_process, row_count -def test_recon_capture_start_snowflake_all(mock_workspace_client, mock_spark, recon_metadata): +def test_recon_capture_start_snowflake_all(mock_workspace_client, spark, recon_metadata): database_config = DatabaseConfig( "source_test_schema", "target_test_catalog", "target_test_schema", "source_test_catalog" ) ws = mock_workspace_client source_type = get_dialect("snowflake") - spark = mock_spark reconcile_output, schema_output, table_conf, reconcile_process, row_count = data_prep(spark) recon_capture = ReconCapture( database_config, @@ -136,7 +135,6 @@ def test_recon_capture_start_snowflake_all(mock_workspace_client, mock_spark, re ws, spark, metadata_config=recon_metadata, - local_test_run=True, ) recon_capture.start( data_reconcile_output=reconcile_output, @@ -147,7 +145,7 @@ def test_recon_capture_start_snowflake_all(mock_workspace_client, mock_spark, re ) # assert main - remorph_recon_df = spark.sql(f"select * from {recon_metadata.schema}.main") + remorph_recon_df = spark.sql(f"select * from {recon_metadata.catalog}.{recon_metadata.schema}.main") row = remorph_recon_df.collect()[0] assert remorph_recon_df.count() == 1 assert row.recon_id == "73b44582-dbb7-489f-bad1-6a7e8f4821b1" @@ -161,7 +159,7 @@ def test_recon_capture_start_snowflake_all(mock_workspace_client, mock_spark, re assert row.source_type == "Snowflake" # assert metrics - remorph_recon_metrics_df = spark.sql(f"select * from {recon_metadata.schema}.metrics") + remorph_recon_metrics_df = spark.sql(f"select * from {recon_metadata.catalog}.{recon_metadata.schema}.metrics") row = remorph_recon_metrics_df.collect()[0] assert remorph_recon_metrics_df.count() == 1 assert row.recon_metrics.source_record_count == 5 @@ -177,7 +175,7 @@ def test_recon_capture_start_snowflake_all(mock_workspace_client, mock_spark, re assert row.run_metrics.exception_message == "" # assert details - remorph_recon_details_df = spark.sql(f"select * from {recon_metadata.schema}.details") + remorph_recon_details_df = spark.sql(f"select * from {recon_metadata.catalog}.{recon_metadata.schema}.details") assert remorph_recon_details_df.count() == 5 assert remorph_recon_details_df.select("recon_type").distinct().count() == 5 assert ( @@ -207,11 +205,10 @@ def test_recon_capture_start_snowflake_all(mock_workspace_client, mock_spark, re assert rows[4].status is False -def test_test_recon_capture_start_databricks_data(mock_workspace_client, mock_spark, recon_metadata): +def test_test_recon_capture_start_databricks_data(mock_workspace_client, spark, recon_metadata): database_config = DatabaseConfig("source_test_schema", "target_test_catalog", "target_test_schema") ws = mock_workspace_client source_type = get_dialect("databricks") - spark = mock_spark recon_capture = ReconCapture( database_config, "73b44582-dbb7-489f-bad1-6a7e8f4821b1", @@ -220,7 +217,6 @@ def test_test_recon_capture_start_databricks_data(mock_workspace_client, mock_sp ws, spark, metadata_config=recon_metadata, - local_test_run=True, ) reconcile_output, schema_output, table_conf, reconcile_process, row_count = data_prep(spark) schema_output.compare_df = None @@ -234,7 +230,7 @@ def test_test_recon_capture_start_databricks_data(mock_workspace_client, mock_sp ) # assert main - remorph_recon_df = spark.sql(f"select * from {recon_metadata.schema}.main") + remorph_recon_df = spark.sql(f"select * from {recon_metadata.catalog}.{recon_metadata.schema}.main") row = remorph_recon_df.collect()[0] assert remorph_recon_df.count() == 1 assert row.source_table.catalog is None @@ -242,24 +238,23 @@ def test_test_recon_capture_start_databricks_data(mock_workspace_client, mock_sp assert row.source_type == "Databricks" # assert metrics - remorph_recon_metrics_df = spark.sql(f"select * from {recon_metadata.schema}.metrics") + remorph_recon_metrics_df = spark.sql(f"select * from {recon_metadata.catalog}.{recon_metadata.schema}.metrics") row = remorph_recon_metrics_df.collect()[0] assert row.recon_metrics.schema_comparison is None assert row.run_metrics.status is False # assert details - remorph_recon_details_df = spark.sql(f"select * from {recon_metadata.schema}.details") + remorph_recon_details_df = spark.sql(f"select * from {recon_metadata.catalog}.{recon_metadata.schema}.details") assert remorph_recon_details_df.count() == 4 assert remorph_recon_details_df.select("recon_type").distinct().count() == 4 -def test_test_recon_capture_start_databricks_row(mock_workspace_client, mock_spark, recon_metadata): +def test_test_recon_capture_start_databricks_row(mock_workspace_client, spark, recon_metadata): database_config = DatabaseConfig( "source_test_schema", "target_test_catalog", "target_test_schema", "source_test_catalog" ) ws = mock_workspace_client source_type = get_dialect("databricks") - spark = mock_spark recon_capture = ReconCapture( database_config, "73b44582-dbb7-489f-bad1-6a7e8f4821b1", @@ -268,7 +263,6 @@ def test_test_recon_capture_start_databricks_row(mock_workspace_client, mock_spa ws, spark, metadata_config=recon_metadata, - local_test_run=True, ) reconcile_output, schema_output, table_conf, reconcile_process, row_count = data_prep(spark) reconcile_output.mismatch_count = 0 @@ -285,32 +279,31 @@ def test_test_recon_capture_start_databricks_row(mock_workspace_client, mock_spa ) # assert main - remorph_recon_df = spark.sql(f"select * from {recon_metadata.schema}.main") + remorph_recon_df = spark.sql(f"select * from {recon_metadata.catalog}.{recon_metadata.schema}.main") row = remorph_recon_df.collect()[0] assert remorph_recon_df.count() == 1 assert row.report_type == "row" assert row.source_type == "Databricks" # assert metrics - remorph_recon_metrics_df = spark.sql(f"select * from {recon_metadata.schema}.metrics") + remorph_recon_metrics_df = spark.sql(f"select * from {recon_metadata.catalog}.{recon_metadata.schema}.metrics") row = remorph_recon_metrics_df.collect()[0] assert row.recon_metrics.column_comparison is None assert row.recon_metrics.schema_comparison is None assert row.run_metrics.status is False # assert details - remorph_recon_details_df = spark.sql(f"select * from {recon_metadata.schema}.details") + remorph_recon_details_df = spark.sql(f"select * from {recon_metadata.catalog}.{recon_metadata.schema}.details") assert remorph_recon_details_df.count() == 2 assert remorph_recon_details_df.select("recon_type").distinct().count() == 2 -def test_recon_capture_start_oracle_schema(mock_workspace_client, mock_spark, recon_metadata): +def test_recon_capture_start_oracle_schema(mock_workspace_client, spark, recon_metadata): database_config = DatabaseConfig( "source_test_schema", "target_test_catalog", "target_test_schema", "source_test_catalog" ) ws = mock_workspace_client source_type = get_dialect("oracle") - spark = mock_spark recon_capture = ReconCapture( database_config, "73b44582-dbb7-489f-bad1-6a7e8f4821b1", @@ -319,7 +312,6 @@ def test_recon_capture_start_oracle_schema(mock_workspace_client, mock_spark, re ws, spark, metadata_config=recon_metadata, - local_test_run=True, ) reconcile_output, schema_output, table_conf, reconcile_process, row_count = data_prep(spark) reconcile_output.threshold_output = ThresholdOutput() @@ -337,14 +329,14 @@ def test_recon_capture_start_oracle_schema(mock_workspace_client, mock_spark, re ) # assert main - remorph_recon_df = spark.sql(f"select * from {recon_metadata.schema}.main") + remorph_recon_df = spark.sql(f"select * from {recon_metadata.catalog}.{recon_metadata.schema}.main") row = remorph_recon_df.collect()[0] assert remorph_recon_df.count() == 1 assert row.report_type == "schema" assert row.source_type == "Oracle" # assert metrics - remorph_recon_metrics_df = spark.sql(f"select * from {recon_metadata.schema}.metrics") + remorph_recon_metrics_df = spark.sql(f"select * from {recon_metadata.catalog}.{recon_metadata.schema}.metrics") row = remorph_recon_metrics_df.collect()[0] assert row.recon_metrics.row_comparison is None assert row.recon_metrics.column_comparison is None @@ -352,18 +344,17 @@ def test_recon_capture_start_oracle_schema(mock_workspace_client, mock_spark, re assert row.run_metrics.status is True # assert details - remorph_recon_details_df = spark.sql(f"select * from {recon_metadata.schema}.details") + remorph_recon_details_df = spark.sql(f"select * from {recon_metadata.catalog}.{recon_metadata.schema}.details") assert remorph_recon_details_df.count() == 1 assert remorph_recon_details_df.select("recon_type").distinct().count() == 1 -def test_recon_capture_start_oracle_with_exception(mock_workspace_client, mock_spark, recon_metadata): +def test_recon_capture_start_oracle_with_exception(mock_workspace_client, spark, recon_metadata): database_config = DatabaseConfig( "source_test_schema", "target_test_catalog", "target_test_schema", "source_test_catalog" ) ws = mock_workspace_client source_type = get_dialect("oracle") - spark = mock_spark recon_capture = ReconCapture( database_config, "73b44582-dbb7-489f-bad1-6a7e8f4821b1", @@ -372,7 +363,6 @@ def test_recon_capture_start_oracle_with_exception(mock_workspace_client, mock_s ws, spark, metadata_config=recon_metadata, - local_test_run=True, ) reconcile_output, schema_output, table_conf, reconcile_process, row_count = data_prep(spark) reconcile_output.threshold_output = ThresholdOutput() @@ -391,27 +381,26 @@ def test_recon_capture_start_oracle_with_exception(mock_workspace_client, mock_s ) # assert main - remorph_recon_df = spark.sql(f"select * from {recon_metadata.schema}.main") + remorph_recon_df = spark.sql(f"select * from {recon_metadata.catalog}.{recon_metadata.schema}.main") row = remorph_recon_df.collect()[0] assert remorph_recon_df.count() == 1 assert row.report_type == "all" assert row.source_type == "Oracle" # assert metrics - remorph_recon_metrics_df = spark.sql(f"select * from {recon_metadata.schema}.metrics") + remorph_recon_metrics_df = spark.sql(f"select * from {recon_metadata.catalog}.{recon_metadata.schema}.metrics") row = remorph_recon_metrics_df.collect()[0] assert row.recon_metrics.schema_comparison is None assert row.run_metrics.status is False assert row.run_metrics.exception_message == "Test exception" -def test_recon_capture_start_with_exception(mock_workspace_client, mock_spark): +def test_recon_capture_start_with_exception(mock_workspace_client, spark): database_config = DatabaseConfig( "source_test_schema", "target_test_catalog", "target_test_schema", "source_test_catalog" ) ws = mock_workspace_client source_type = get_dialect("snowflake") - spark = mock_spark recon_capture = ReconCapture( database_config, "73b44582-dbb7-489f-bad1-6a7e8f4821b1", @@ -431,7 +420,7 @@ def test_recon_capture_start_with_exception(mock_workspace_client, mock_spark): ) -def test_generate_final_reconcile_output_row(mock_workspace_client, mock_spark, recon_metadata): +def test_generate_final_reconcile_output_row(mock_workspace_client, spark, recon_metadata): database_config = DatabaseConfig( "source_test_schema", "target_test_catalog", @@ -439,7 +428,6 @@ def test_generate_final_reconcile_output_row(mock_workspace_client, mock_spark, ) ws = mock_workspace_client source_type = get_dialect("databricks") - spark = mock_spark recon_capture = ReconCapture( database_config, "73b44582-dbb7-489f-bad1-6a7e8f4821b1", @@ -448,7 +436,6 @@ def test_generate_final_reconcile_output_row(mock_workspace_client, mock_spark, ws, spark, metadata_config=recon_metadata, - local_test_run=True, ) reconcile_output, schema_output, table_conf, reconcile_process, row_count = data_prep(spark) recon_capture.start( @@ -461,9 +448,8 @@ def test_generate_final_reconcile_output_row(mock_workspace_client, mock_spark, final_output = generate_final_reconcile_output( "73b44582-dbb7-489f-bad1-6a7e8f4821b1", - mock_spark, + spark, metadata_config=recon_metadata, - local_test_run=True, ) assert final_output == ReconcileOutput( @@ -479,7 +465,7 @@ def test_generate_final_reconcile_output_row(mock_workspace_client, mock_spark, ) -def test_generate_final_reconcile_output_data(mock_workspace_client, mock_spark, recon_metadata): +def test_generate_final_reconcile_output_data(mock_workspace_client, spark, recon_metadata): database_config = DatabaseConfig( "source_test_schema", "target_test_catalog", @@ -487,7 +473,6 @@ def test_generate_final_reconcile_output_data(mock_workspace_client, mock_spark, ) ws = mock_workspace_client source_type = get_dialect("databricks") - spark = mock_spark recon_capture = ReconCapture( database_config, "73b44582-dbb7-489f-bad1-6a7e8f4821b1", @@ -496,7 +481,6 @@ def test_generate_final_reconcile_output_data(mock_workspace_client, mock_spark, ws, spark, metadata_config=recon_metadata, - local_test_run=True, ) reconcile_output, schema_output, table_conf, reconcile_process, row_count = data_prep(spark) recon_capture.start( @@ -509,9 +493,8 @@ def test_generate_final_reconcile_output_data(mock_workspace_client, mock_spark, final_output = generate_final_reconcile_output( "73b44582-dbb7-489f-bad1-6a7e8f4821b1", - mock_spark, + spark, metadata_config=recon_metadata, - local_test_run=True, ) assert final_output == ReconcileOutput( @@ -527,7 +510,7 @@ def test_generate_final_reconcile_output_data(mock_workspace_client, mock_spark, ) -def test_generate_final_reconcile_output_schema(mock_workspace_client, mock_spark, recon_metadata): +def test_generate_final_reconcile_output_schema(mock_workspace_client, spark, recon_metadata): database_config = DatabaseConfig( "source_test_schema", "target_test_catalog", @@ -535,7 +518,6 @@ def test_generate_final_reconcile_output_schema(mock_workspace_client, mock_spar ) ws = mock_workspace_client source_type = get_dialect("databricks") - spark = mock_spark recon_capture = ReconCapture( database_config, "73b44582-dbb7-489f-bad1-6a7e8f4821b1", @@ -544,7 +526,6 @@ def test_generate_final_reconcile_output_schema(mock_workspace_client, mock_spar ws, spark, metadata_config=recon_metadata, - local_test_run=True, ) reconcile_output, schema_output, table_conf, reconcile_process, row_count = data_prep(spark) recon_capture.start( @@ -557,9 +538,8 @@ def test_generate_final_reconcile_output_schema(mock_workspace_client, mock_spar final_output = generate_final_reconcile_output( "73b44582-dbb7-489f-bad1-6a7e8f4821b1", - mock_spark, + spark, metadata_config=recon_metadata, - local_test_run=True, ) assert final_output == ReconcileOutput( @@ -575,7 +555,7 @@ def test_generate_final_reconcile_output_schema(mock_workspace_client, mock_spar ) -def test_generate_final_reconcile_output_all(mock_workspace_client, mock_spark, recon_metadata): +def test_generate_final_reconcile_output_all(mock_workspace_client, spark, recon_metadata): database_config = DatabaseConfig( "source_test_schema", "target_test_catalog", @@ -583,7 +563,6 @@ def test_generate_final_reconcile_output_all(mock_workspace_client, mock_spark, ) ws = mock_workspace_client source_type = get_dialect("databricks") - spark = mock_spark recon_capture = ReconCapture( database_config, "73b44582-dbb7-489f-bad1-6a7e8f4821b1", @@ -592,7 +571,6 @@ def test_generate_final_reconcile_output_all(mock_workspace_client, mock_spark, ws, spark, metadata_config=recon_metadata, - local_test_run=True, ) reconcile_output, schema_output, table_conf, reconcile_process, row_count = data_prep(spark) @@ -606,9 +584,8 @@ def test_generate_final_reconcile_output_all(mock_workspace_client, mock_spark, final_output = generate_final_reconcile_output( "73b44582-dbb7-489f-bad1-6a7e8f4821b1", - mock_spark, + spark, metadata_config=recon_metadata, - local_test_run=True, ) assert final_output == ReconcileOutput( @@ -624,7 +601,7 @@ def test_generate_final_reconcile_output_all(mock_workspace_client, mock_spark, ) -def test_generate_final_reconcile_output_exception(mock_workspace_client, mock_spark, recon_metadata): +def test_generate_final_reconcile_output_exception(mock_workspace_client, spark, recon_metadata): database_config = DatabaseConfig( "source_test_schema", "target_test_catalog", @@ -632,7 +609,6 @@ def test_generate_final_reconcile_output_exception(mock_workspace_client, mock_s ) ws = mock_workspace_client source_type = get_dialect("databricks") - spark = mock_spark recon_capture = ReconCapture( database_config, "73b44582-dbb7-489f-bad1-6a7e8f4821b1", @@ -641,7 +617,6 @@ def test_generate_final_reconcile_output_exception(mock_workspace_client, mock_s ws, spark, metadata_config=recon_metadata, - local_test_run=True, ) reconcile_output, schema_output, table_conf, reconcile_process, row_count = data_prep(spark) reconcile_output.exception = "Test exception" @@ -656,9 +631,8 @@ def test_generate_final_reconcile_output_exception(mock_workspace_client, mock_s final_output = generate_final_reconcile_output( "73b44582-dbb7-489f-bad1-6a7e8f4821b1", - mock_spark, + spark, metadata_config=recon_metadata, - local_test_run=True, ) assert final_output == ReconcileOutput( @@ -674,13 +648,12 @@ def test_generate_final_reconcile_output_exception(mock_workspace_client, mock_s ) -def test_apply_threshold_for_mismatch_with_true_absolute(mock_workspace_client, mock_spark, recon_metadata): +def test_apply_threshold_for_mismatch_with_true_absolute(mock_workspace_client, spark, recon_metadata): database_config = DatabaseConfig( "source_test_schema", "target_test_catalog", "target_test_schema", "source_test_catalog" ) ws = mock_workspace_client source_type = get_dialect("snowflake") - spark = mock_spark reconcile_output, schema_output, table_conf, reconcile_process, row_count = data_prep(spark) reconcile_output.missing_in_src_count = 0 reconcile_output.missing_in_tgt_count = 0 @@ -697,7 +670,6 @@ def test_apply_threshold_for_mismatch_with_true_absolute(mock_workspace_client, ws, spark, metadata_config=recon_metadata, - local_test_run=True, ) recon_capture.start( data_reconcile_output=reconcile_output, @@ -708,18 +680,17 @@ def test_apply_threshold_for_mismatch_with_true_absolute(mock_workspace_client, ) # assert metrics - remorph_recon_metrics_df = spark.sql(f"select * from {recon_metadata.schema}.metrics") + remorph_recon_metrics_df = spark.sql(f"select * from {recon_metadata.catalog}.{recon_metadata.schema}.metrics") row = remorph_recon_metrics_df.collect()[0] assert row.run_metrics.status is True -def test_apply_threshold_for_mismatch_with_missing(mock_workspace_client, mock_spark, recon_metadata): +def test_apply_threshold_for_mismatch_with_missing(mock_workspace_client, spark, recon_metadata): database_config = DatabaseConfig( "source_test_schema", "target_test_catalog", "target_test_schema", "source_test_catalog" ) ws = mock_workspace_client source_type = get_dialect("snowflake") - spark = mock_spark reconcile_output, schema_output, table_conf, reconcile_process, row_count = data_prep(spark) table_conf.table_thresholds = [ TableThresholds(lower_bound="0", upper_bound="4", model="mismatch"), @@ -732,7 +703,6 @@ def test_apply_threshold_for_mismatch_with_missing(mock_workspace_client, mock_s ws, spark, metadata_config=recon_metadata, - local_test_run=True, ) recon_capture.start( @@ -743,18 +713,17 @@ def test_apply_threshold_for_mismatch_with_missing(mock_workspace_client, mock_s record_count=row_count, ) # assert metrics - remorph_recon_metrics_df = spark.sql(f"select * from {recon_metadata.schema}.metrics") + remorph_recon_metrics_df = spark.sql(f"select * from {recon_metadata.catalog}.{recon_metadata.schema}.metrics") row = remorph_recon_metrics_df.collect()[0] assert row.run_metrics.status is False -def test_apply_threshold_for_mismatch_with_schema_fail(mock_workspace_client, mock_spark, recon_metadata): +def test_apply_threshold_for_mismatch_with_schema_fail(mock_workspace_client, spark, recon_metadata): database_config = DatabaseConfig( "source_test_schema", "target_test_catalog", "target_test_schema", "source_test_catalog" ) ws = mock_workspace_client source_type = get_dialect("snowflake") - spark = mock_spark reconcile_output, schema_output, table_conf, reconcile_process, row_count = data_prep(spark) table_conf.table_thresholds = [ TableThresholds(lower_bound="0", upper_bound="4", model="mismatch"), @@ -767,7 +736,6 @@ def test_apply_threshold_for_mismatch_with_schema_fail(mock_workspace_client, mo ws, spark, metadata_config=recon_metadata, - local_test_run=True, ) reconcile_output.missing_in_src_count = 0 @@ -782,18 +750,17 @@ def test_apply_threshold_for_mismatch_with_schema_fail(mock_workspace_client, mo record_count=row_count, ) # assert metrics - remorph_recon_metrics_df = spark.sql(f"select * from {recon_metadata.schema}.metrics") + remorph_recon_metrics_df = spark.sql(f"select * from {recon_metadata.catalog}.{recon_metadata.schema}.metrics") row = remorph_recon_metrics_df.collect()[0] assert row.run_metrics.status is False -def test_apply_threshold_for_mismatch_with_wrong_absolute_bound(mock_workspace_client, mock_spark, recon_metadata): +def test_apply_threshold_for_mismatch_with_wrong_absolute_bound(mock_workspace_client, spark, recon_metadata): database_config = DatabaseConfig( "source_test_schema", "target_test_catalog", "target_test_schema", "source_test_catalog" ) ws = mock_workspace_client source_type = get_dialect("snowflake") - spark = mock_spark reconcile_output, schema_output, table_conf, reconcile_process, row_count = data_prep(spark) table_conf.table_thresholds = [ TableThresholds(lower_bound="0", upper_bound="1", model="mismatch"), @@ -811,7 +778,6 @@ def test_apply_threshold_for_mismatch_with_wrong_absolute_bound(mock_workspace_c ws, spark, metadata_config=recon_metadata, - local_test_run=True, ) recon_capture.start( data_reconcile_output=reconcile_output, @@ -822,18 +788,17 @@ def test_apply_threshold_for_mismatch_with_wrong_absolute_bound(mock_workspace_c ) # assert metrics - remorph_recon_metrics_df = spark.sql(f"select * from {recon_metadata.schema}.metrics") + remorph_recon_metrics_df = spark.sql(f"select * from {recon_metadata.catalog}.{recon_metadata.schema}.metrics") row = remorph_recon_metrics_df.collect()[0] assert row.run_metrics.status is False -def test_apply_threshold_for_mismatch_with_wrong_percentage_bound(mock_workspace_client, mock_spark, recon_metadata): +def test_apply_threshold_for_mismatch_with_wrong_percentage_bound(mock_workspace_client, spark, recon_metadata): database_config = DatabaseConfig( "source_test_schema", "target_test_catalog", "target_test_schema", "source_test_catalog" ) ws = mock_workspace_client source_type = get_dialect("snowflake") - spark = mock_spark reconcile_output, schema_output, table_conf, reconcile_process, row_count = data_prep(spark) table_conf.table_thresholds = [ TableThresholds(lower_bound="0%", upper_bound="20%", model="mismatch"), @@ -851,7 +816,6 @@ def test_apply_threshold_for_mismatch_with_wrong_percentage_bound(mock_workspace ws, spark, metadata_config=recon_metadata, - local_test_run=True, ) recon_capture.start( data_reconcile_output=reconcile_output, @@ -862,18 +826,17 @@ def test_apply_threshold_for_mismatch_with_wrong_percentage_bound(mock_workspace ) # assert metrics - remorph_recon_metrics_df = spark.sql(f"select * from {recon_metadata.schema}.metrics") + remorph_recon_metrics_df = spark.sql(f"select * from {recon_metadata.catalog}.{recon_metadata.schema}.metrics") row = remorph_recon_metrics_df.collect()[0] assert row.run_metrics.status is False -def test_apply_threshold_for_mismatch_with_true_percentage_bound(mock_workspace_client, mock_spark, recon_metadata): +def test_apply_threshold_for_mismatch_with_true_percentage_bound(mock_workspace_client, spark, recon_metadata): database_config = DatabaseConfig( "source_test_schema", "target_test_catalog", "target_test_schema", "source_test_catalog" ) ws = mock_workspace_client source_type = get_dialect("snowflake") - spark = mock_spark reconcile_output, schema_output, table_conf, reconcile_process, row_count = data_prep(spark) table_conf.table_thresholds = [ TableThresholds(lower_bound="0%", upper_bound="90%", model="mismatch"), @@ -890,7 +853,6 @@ def test_apply_threshold_for_mismatch_with_true_percentage_bound(mock_workspace_ ws, spark, metadata_config=recon_metadata, - local_test_run=True, ) recon_capture.start( data_reconcile_output=reconcile_output, @@ -901,18 +863,17 @@ def test_apply_threshold_for_mismatch_with_true_percentage_bound(mock_workspace_ ) # assert metrics - remorph_recon_metrics_df = spark.sql(f"select * from {recon_metadata.schema}.metrics") + remorph_recon_metrics_df = spark.sql(f"select * from {recon_metadata.catalog}.{recon_metadata.schema}.metrics") row = remorph_recon_metrics_df.collect()[0] assert row.run_metrics.status is True -def test_apply_threshold_for_mismatch_with_invalid_bounds(mock_workspace_client, mock_spark): +def test_apply_threshold_for_mismatch_with_invalid_bounds(mock_workspace_client, spark): database_config = DatabaseConfig( "source_test_schema", "target_test_catalog", "target_test_schema", "source_test_catalog" ) ws = mock_workspace_client source_type = get_dialect("snowflake") - spark = mock_spark reconcile_output, schema_output, table_conf, reconcile_process, row_count = data_prep(spark) reconcile_output.missing_in_src_count = 0 reconcile_output.missing_in_tgt_count = 0 @@ -927,7 +888,6 @@ def test_apply_threshold_for_mismatch_with_invalid_bounds(mock_workspace_client, ws, spark, metadata_config=ReconcileMetadataConfig(schema="default"), - local_test_run=True, ) with pytest.raises(TableThresholdBoundsException): table_conf.table_thresholds = [ @@ -954,15 +914,12 @@ def test_apply_threshold_for_mismatch_with_invalid_bounds(mock_workspace_client, ) -def test_apply_threshold_for_only_threshold_mismatch_with_true_absolute( - mock_workspace_client, mock_spark, recon_metadata -): +def test_apply_threshold_for_only_threshold_mismatch_with_true_absolute(mock_workspace_client, spark, recon_metadata): database_config = DatabaseConfig( "source_test_schema", "target_test_catalog", "target_test_schema", "source_test_catalog" ) ws = mock_workspace_client source_type = get_dialect("snowflake") - spark = mock_spark reconcile_output, schema_output, table_conf, reconcile_process, row_count = data_prep(spark) reconcile_output.mismatch_count = 0 reconcile_output.missing_in_src_count = 0 @@ -980,7 +937,6 @@ def test_apply_threshold_for_only_threshold_mismatch_with_true_absolute( ws, spark, metadata_config=recon_metadata, - local_test_run=True, ) recon_capture.start( data_reconcile_output=reconcile_output, @@ -991,7 +947,7 @@ def test_apply_threshold_for_only_threshold_mismatch_with_true_absolute( ) # assert metrics - remorph_recon_metrics_df = spark.sql(f"select * from {recon_metadata.schema}.metrics") + remorph_recon_metrics_df = spark.sql(f"select * from {recon_metadata.catalog}.{recon_metadata.schema}.metrics") row = remorph_recon_metrics_df.collect()[0] assert row.run_metrics.status is True @@ -1006,24 +962,24 @@ def format(self): return self._format -def test_is_databricks_false(mock_spark): +def test_is_databricks_false(spark): conf = ReconcileMetadataConfig() - persist = ReconIntermediatePersistUnderTest(mock_spark, conf) + persist = ReconIntermediatePersistUnderTest(spark, conf) assert persist.is_databricks is False -def test_dir_uses_tempfile(mock_spark): +def test_dir_uses_tempfile(spark): conf = ReconcileMetadataConfig() - persist = ReconIntermediatePersistUnderTest(mock_spark, conf) + persist = ReconIntermediatePersistUnderTest(spark, conf) expected = tempfile.gettempdir() assert str(persist.base_dir).startswith(expected) -def test_format_uses_parquet(mock_spark): +def test_format_uses_parquet(spark): conf = ReconcileMetadataConfig() - persist = ReconIntermediatePersistUnderTest(mock_spark, conf) + persist = ReconIntermediatePersistUnderTest(spark, conf) assert persist.format == "parquet" diff --git a/tests/integration/reconcile/test_recon_e2e.py b/tests/integration/reconcile/test_recon_e2e.py index 61bf6070f3..be9ef88bf6 100644 --- a/tests/integration/reconcile/test_recon_e2e.py +++ b/tests/integration/reconcile/test_recon_e2e.py @@ -1,6 +1,7 @@ import re import logging +import pytest from databricks.sdk.service.jobs import TerminationTypeType from databricks.sdk.core import DatabricksError @@ -12,6 +13,8 @@ logger = logging.getLogger(__name__) +pytestmark = pytest.mark.timeout(1800) + def _debug_run_output(ctx: ApplicationContext, run_id: int) -> None: _ansi_escape = re.compile(r"\x1B\[[0-?]*[ -/]*[@-~]") @@ -76,6 +79,7 @@ def test_recon_sql_server_job_succeeds( _run_recon_e2e_spec(app_ctx) +@pytest.mark.xfail(reason="Snowflake account unavailable", strict=True) def test_recon_snowflake_job_succeeds( application_ctx: ApplicationContext, snowflake_recon_config: ReconcileConfig, diff --git a/tests/integration/reconcile/test_sampler.py b/tests/integration/reconcile/test_sampler.py index 15d15074ad..2d6775da43 100644 --- a/tests/integration/reconcile/test_sampler.py +++ b/tests/integration/reconcile/test_sampler.py @@ -12,11 +12,11 @@ _MAX_SAMPLE_COUNT = 400 -def test_random_sampler_count(mock_spark): - keys_df = mock_spark.createDataFrame([Row(key=1), Row(key=2), Row(key=3), Row(key=4), Row(key=5)]) +def test_random_sampler_count(spark): + keys_df = spark.createDataFrame([Row(key=1), Row(key=2), Row(key=3), Row(key=4), Row(key=5)]) keys_df_count = keys_df.count() - target_table_df = mock_spark.createDataFrame( + target_table_df = spark.createDataFrame( [ Row(key=1, state="NC", age=25), Row(key=2, state="NC", age=30), @@ -51,11 +51,11 @@ def test_random_sampler_count(mock_spark): ) -def test_random_sampler_negative_count(mock_spark): - keys_df = mock_spark.createDataFrame([Row(key=1), Row(key=2), Row(key=3), Row(key=4), Row(key=5)]) +def test_random_sampler_negative_count(spark): + keys_df = spark.createDataFrame([Row(key=1), Row(key=2), Row(key=3), Row(key=4), Row(key=5)]) keys_df_count = keys_df.count() - target_table_df = mock_spark.createDataFrame( + target_table_df = spark.createDataFrame( [ Row(key=1, state="NC", age=25), Row(key=2, state="NC", age=30), @@ -100,11 +100,11 @@ def test_random_sampler_invalid_fraction(): ) -def test_stratified_sampler_count(mock_spark): - keys_df = mock_spark.createDataFrame([Row(key=1), Row(key=2), Row(key=3), Row(key=4), Row(key=5)]) +def test_stratified_sampler_count(spark): + keys_df = spark.createDataFrame([Row(key=1), Row(key=2), Row(key=3), Row(key=4), Row(key=5)]) keys_df_count = keys_df.count() - target_table_df = mock_spark.createDataFrame( + target_table_df = spark.createDataFrame( [ Row(key=1, state="NC", age=25), Row(key=2, state="NC", age=30), @@ -174,11 +174,11 @@ def test_stratified_sampler_missing_columns_buckets(): ) -def test_stratified_sampler_negative_count(mock_spark): - keys_df = mock_spark.createDataFrame([Row(key=1), Row(key=2), Row(key=3), Row(key=4), Row(key=5)]) +def test_stratified_sampler_negative_count(spark): + keys_df = spark.createDataFrame([Row(key=1), Row(key=2), Row(key=3), Row(key=4), Row(key=5)]) keys_df_count = keys_df.count() - target_table_df = mock_spark.createDataFrame( + target_table_df = spark.createDataFrame( [ Row(key=1, state="NC", age=25), Row(key=2, state="NC", age=30), diff --git a/tests/integration/reconcile/test_schema_compare.py b/tests/integration/reconcile/test_schema_compare.py index fce7d61604..e957553e5a 100644 --- a/tests/integration/reconcile/test_schema_compare.py +++ b/tests/integration/reconcile/test_schema_compare.py @@ -277,9 +277,8 @@ def schemas(): } -def test_snowflake_schema_compare(schemas, mock_spark): +def test_snowflake_schema_compare(schemas, spark): src_schema, tgt_schema = schemas["snowflake_databricks_schema"] - spark = mock_spark table_conf = Table( source_name="supplier", target_name="supplier", @@ -303,9 +302,8 @@ def test_snowflake_schema_compare(schemas, mock_spark): assert df.filter("is_valid = 'false'").count() == 1 -def test_databricks_schema_compare(schemas, mock_spark): +def test_databricks_schema_compare(schemas, spark): src_schema, tgt_schema = schemas["databricks_databricks_schema"] - spark = mock_spark table_conf = Table( source_name="supplier", target_name="supplier", @@ -344,9 +342,8 @@ def test_databricks_schema_compare(schemas, mock_spark): assert df.filter("is_valid = 'false'").count() == 1 -def test_oracle_schema_compare(schemas, mock_spark): +def test_oracle_schema_compare(schemas, spark): src_schema, tgt_schema = schemas["oracle_databricks_schema"] - spark = mock_spark table_conf = Table( source_name="supplier", target_name="supplier", @@ -370,9 +367,8 @@ def test_oracle_schema_compare(schemas, mock_spark): assert df.filter("is_valid = 'false'").count() == 0 -def test_tsql_schema_compare(schemas, mock_spark): +def test_tsql_schema_compare(schemas, spark): src_schema, tgt_schema = schemas["tsql_databricks_schema"] - spark = mock_spark table_conf = Table( source_name="supplier", target_name="supplier", @@ -395,7 +391,7 @@ def test_tsql_schema_compare(schemas, mock_spark): assert df.filter("is_valid = 'false'").count() == 3 -def test_schema_compare(mock_spark): +def test_schema_compare(spark): src_schema = [ schema_fixture_factory("col1", "int", "`col1`", "`col1`"), schema_fixture_factory("col2", "string", "`col2`", "`col2`"), @@ -404,7 +400,6 @@ def test_schema_compare(mock_spark): schema_fixture_factory("col1", "int", "`col1`", "`col1`"), schema_fixture_factory("col2", "string", "`col2`", "`col2`"), ] - spark = mock_spark table_conf = Table( source_name="supplier", target_name="supplier", From f9d7cbed32b150e9179160270763d0b276025f49 Mon Sep 17 00:00:00 2001 From: Andrew Snare Date: Tue, 5 May 2026 11:56:45 +0200 Subject: [PATCH 02/79] Implement minimum support for maven mirrors during `install-transpile` (#2405) ## Changes This PR implements minimal support for using a Maven mirror to install morpheus, instead of requiring direct access to Maven Central. This has been implemented by allowing an environment variable, `LAKEBRIDGE_MAVEN_URL` to override the repository URL so that a mirror is used instead of Maven Central. If credentials are required to access the mirror, they can be specified in `~/.netrc`. (Alternately, `NETRC` can be set in the environment to specify the location of this file.) ### Relevant implementation details Support for `~/.netrc` (and `NETRC`) is provided automatically by the `requests` library that we use to issue HTTP calls against the maven repository. ### Notes for reviewers This PR also syncs the `jfrog-auth` action to the latest version: this sets up the netrc file needed for this to work during CI. Lots of integration tests are still failing on `main`. Relevant integration tests for this PR include: - `test_gets_maven_artifact_version` - `test_downloads_from_maven` - `test_installs_and_runs_maven_morpheus` - `test_transpiles_all_dbt_project_files` - `test_transpile_sql_file` _Documentation updates will follow._ ### Linked issues Resolves #2359 ### Functionality - modified existing command: `databricks labs lakebridge install-transpile` ### Tests - manually tested - added unit tests - existing integration tests --- .github/actions/jfrog-auth/action.yml | 107 ++++++++++++++---- .github/actions/jfrog-auth/jfrog-auth | 2 +- .github/workflows/acceptance.yml | 3 +- .../labs/lakebridge/transpiler/installers.py | 90 +++++++++++---- .../integration/transpile/test_installers.py | 10 +- tests/unit/transpiler/test_installers.py | 22 +++- 6 files changed, 179 insertions(+), 55 deletions(-) diff --git a/.github/actions/jfrog-auth/action.yml b/.github/actions/jfrog-auth/action.yml index bf7b809abb..d4979d227b 100644 --- a/.github/actions/jfrog-auth/action.yml +++ b/.github/actions/jfrog-auth/action.yml @@ -1,8 +1,8 @@ name: 'Authenticate for JFrog' description: 'Authenticate with JFrog using OIDC based on the GitHub repository.' # Some things to note: -# - Run this _after_ installing any tools that need to use JFrog; auth is configured all the (supported) tools that it -# detects. +# - Run this _after_ installing any tools that need to use JFrog; auth is configured for all the (supported) tools that +# it detects. # - Where possible we avoid exposing tokens in environment variables, preferring to write them into files instead. # (Tokens in environment variables tend to be more exposed and easier to leak than those written into files.) # @@ -18,17 +18,22 @@ runs: name: Authenticate against JFrog shell: bash run: | + if [[ -z "${ACTIONS_ID_TOKEN_REQUEST_URL}" ]] || [[ -z "${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" ]] + then + printf '::error::%s\n' 'This action uses OIDC: job must have "id-token: write" permission' + exit 1 + fi "${GITHUB_ACTION_PATH}/jfrog-auth" "${ACTIONS_ID_TOKEN_REQUEST_URL}" "${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" - id: detect-cmds - name: Detecting python package/project managers. + name: Detecting package/project managers. shell: bash run: | - for cmd in bun mvn npm pip3 uv + for cmd in bun coursier mvn npm pip3 sbt uv do - command -v "${cmd}" > /dev/null && found=true || found=false - printf '::debug::%s\n' "Found ${cmd}: ${found}" - printf '%s=%s\n' "command_${cmd}" "${found}" >> "${GITHUB_OUTPUT}" + command -v "${cmd}" > /dev/null && found=true || found=false + printf '::debug::%s\n' "Found ${cmd}: ${found}" + printf '%s=%s\n' "command_${cmd}" "${found}" >> "${GITHUB_OUTPUT}" done - name: Configure bun for JFrog @@ -38,7 +43,7 @@ runs: JFROG_ACCESS_TOKEN: "${{ steps.jfrog-auth.outputs.jfrog-access-token }}" run: | umask 077 - cat > ~/.bunfig.toml << EOF + cat > ~/.bunfig.toml << 'EOF' [install] registry = { url = "https://databricks.jfrog.io/artifactory/api/npm/db-npm/", token = "$jfrog_access_token" } EOF @@ -51,24 +56,28 @@ runs: # There are currently the following issues with JFrog: # - The default set of CAs doesn't seem to cover the ones used by our JFrog instance. # - The JSON metadata returned for some NPM artefacts can be invalid JSON. - printf '::warning::%s\n' 'JFrog has compatibility issues with bun; it probably won't work.' + printf '::warning::%s\n' 'JFrog has compatibility issues with bun; it will probably not work.' - - name: Configure pip for JFrog - if: "${{ steps.detect-cmds.outputs.command_pip3 == 'true' }}" + - name: Configure coursier for JFrog + # Note: SBT bootstrapping uses Coursier internally. + if: "${{ steps.detect-cmds.outputs.command_coursier == 'true' || + steps.detect-cmds.outputs.command_sbt == 'true' }}" shell: bash env: JFROG_ACCESS_TOKEN: "${{ steps.jfrog-auth.outputs.jfrog-access-token }}" run: | umask 077 - cat > "$RUNNER_TEMP/.pip.conf" < "${RUNNER_TEMP}/.coursier-credentials.properties" << EOF + jfrog.host=databricks.jfrog.io + jfrog.realm=Artifactory Realm + jfrog.username=gha-service-account + jfrog.password=${JFROG_ACCESS_TOKEN} EOF - printf '%s=%s\n' 'PIP_CONFIG_FILE' "${RUNNER_TEMP}/.pip.conf" >> "${GITHUB_ENV}" - printf '::debug::%s\n' 'Configured JFrog access for pip.' + printf '%s=%s\n' 'COURSIER_CREDENTIALS' "${RUNNER_TEMP}/.coursier-credentials.properties" >> "${GITHUB_ENV}" + printf '::debug::%s\n' 'Configured JFrog access for Coursier.' - name: Configure Maven for JFrog - if: "${{ steps.detect-cmds.outputs.command_uv == 'true' }}" + if: "${{ steps.detect-cmds.outputs.command_mvn == 'true' }}" shell: bash env: JFROG_ACCESS_TOKEN: "${{ steps.jfrog-auth.outputs.jfrog-access-token }}" @@ -95,17 +104,20 @@ runs: EOF printf '::debug::%s\n' 'Configured JFrog access for maven.' - - name: Configure uv for JFrog - if: "${{ steps.detect-cmds.outputs.command_uv == 'true' }}" + - name: Configure netrc for JFrog + if: "${{ steps.detect-cmds.outputs.command_pip3 == 'true' || + steps.detect-cmds.outputs.command_uv == 'true' }}" shell: bash env: JFROG_ACCESS_TOKEN: "${{ steps.jfrog-auth.outputs.jfrog-access-token }}" - UV_INDEX_URL: 'https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple' run: | - uv auth login "${UV_INDEX_URL}" --username gha-service-account --password "${JFROG_ACCESS_TOKEN}" - printf '%s=%s\n' 'UV_INDEX_URL' "${UV_INDEX_URL}" >> "${GITHUB_ENV}" - printf '%s=%s\n' 'UV_FROZEN' '1' >> "${GITHUB_ENV}" - printf '::debug::%s\n' 'Configured JFrog access for uv.' + umask 077 + cat > "${RUNNER_TEMP}/.netrc" << EOF + machine databricks.jfrog.io + login gha-service-account + password ${JFROG_ACCESS_TOKEN} + EOF + printf '%s=%s\n' 'NETRC' "${RUNNER_TEMP}/.netrc" >> "${GITHUB_ENV}" - name: Configure npm/yarn (classic) for JFrog if: "${{ steps.detect-cmds.outputs.command_npm == 'true' }}" @@ -121,3 +133,50 @@ runs: //databricks.jfrog.io/artifactory/api/npm/db-npm/:_authToken=${JFROG_ACCESS_TOKEN} EOF printf '::debug::%s\n' 'Configured JFrog access for npm/yarn (classic).' + + - name: Configure pip for JFrog + if: "${{ steps.detect-cmds.outputs.command_pip3 == 'true' }}" + shell: bash + run: | + cat > "${RUNNER_TEMP}/.pip.conf" << 'EOF' + [global] + index-url = https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + EOF + printf '%s=%s\n' 'PIP_CONFIG_FILE' "${RUNNER_TEMP}/.pip.conf" >> "${GITHUB_ENV}" + printf '::debug::%s\n' 'Configured JFrog access for pip.' + + - name: Configure sbt for JFrog + if: "${{ steps.detect-cmds.outputs.command_sbt == 'true' }}" + shell: bash + env: + JFROG_ACCESS_TOKEN: "${{ steps.jfrog-auth.outputs.jfrog-access-token }}" + run: | + umask 077 + mkdir -p ~/.sbt/1.0 + cat > ~/.sbt/repositories << 'EOF' + [repositories] + local + databricks-jfrog: https://databricks.jfrog.io/artifactory/db-maven/ + EOF + + cat > "${RUNNER_TEMP}/.sbt.credentials" << EOF + realm=Artifactory Realm + host=databricks.jfrog.io + user=gha-service-account + password=${JFROG_ACCESS_TOKEN} + EOF + + cat > ~/.sbt/1.0/global.sbt << 'EOF' + credentials += Credentials(file(sys.env("RUNNER_TEMP")) / ".sbt.credentials") + EOF + printf '::debug::%s\n' 'Configured JFrog access for SBT.' + + - name: Configure uv for JFrog + if: "${{ steps.detect-cmds.outputs.command_uv == 'true' }}" + shell: bash + env: + UV_INDEX_URL: 'https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple' + run: | + printf '%s=%s\n' 'UV_INDEX_URL' "${UV_INDEX_URL}" >> "${GITHUB_ENV}" + printf '%s=%s\n' 'UV_FROZEN' '1' >> "${GITHUB_ENV}" + printf '::debug::%s\n' 'Configured JFrog access for uv.' diff --git a/.github/actions/jfrog-auth/jfrog-auth b/.github/actions/jfrog-auth/jfrog-auth index 6a5ab856a4..8c5533487d 100755 --- a/.github/actions/jfrog-auth/jfrog-auth +++ b/.github/actions/jfrog-auth/jfrog-auth @@ -22,7 +22,7 @@ printf '::add-mask::%s\n' "${_id_token}" # Step 2: Exchange it for the JFrog access token. # printf '::debug::%s\n' "Exchanging OIDC identifier token for JFrog access token..." -_access_token=$(curl -sLS \ +_access_token=$(curl -fsSL \ --json "{\"grant_type\": \"urn:ietf:params:oauth:grant-type:token-exchange\", \"subject_token_type\":\"urn:ietf:params:oauth:token-type:id_token\", \"subject_token\": \"${_id_token}\", \"provider_name\": \"github-actions\"}" \ "https://databricks.jfrog.io/access/api/v1/oidc/token" | jq -r .access_token) diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml index 64247175c1..ff93fa2c42 100644 --- a/.github/workflows/acceptance.yml +++ b/.github/workflows/acceptance.yml @@ -54,7 +54,8 @@ jobs: directory: ${{ github.workspace }} timeout: 2h env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }} ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + LAKEBRIDGE_MAVEN_URL: "https://databricks.jfrog.io/artifactory/db-maven" TEST_ENV: 'ACCEPTANCE' diff --git a/src/databricks/labs/lakebridge/transpiler/installers.py b/src/databricks/labs/lakebridge/transpiler/installers.py index 57375317a2..48815328c1 100644 --- a/src/databricks/labs/lakebridge/transpiler/installers.py +++ b/src/databricks/labs/lakebridge/transpiler/installers.py @@ -9,10 +9,11 @@ import sys import venv import xml.etree.ElementTree as ET +from abc import ABC, abstractmethod from pathlib import Path from shutil import rmtree from types import SimpleNamespace -from typing import Literal +from typing import Literal, ClassVar from zipfile import ZipFile import requests @@ -296,38 +297,65 @@ def _run_custom_installer(self, installer_path: Path) -> None: subprocess.run(args, stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr, cwd=self._install_path, check=True) -class MavenInstaller(ArtifactInstaller): +class MavenClient(ABC): + @abstractmethod + def get_current_maven_artifact_version(self, group_id: str, artifact_id: str) -> str | None: ... + + @abstractmethod + def download_artifact_from_maven( + self, + group_id: str, + artifact_id: str, + version: str, + target: Path, + classifier: str | None = None, + extension: str = "jar", + ) -> bool: ... + + @classmethod + def default(cls) -> "MavenClient": + return MavenLite() + + +class MavenLite(MavenClient): + """Lightweight non-mvn-based client for installing artifacts from Maven Central. + + This client uses direct HTTP calls, instead of the mvn command-line tool. + + By default Maven Central will be used: https://repo.maven.apache.org/maven2 + This can be overridden by setting LAKEBRIDGE_MAVEN_URL in the environment to a mirror. (Credentials will be sourced + if necessary via ~/.netrc, or the alternate location referred to by the NETRC environment variable.) + """ + # Maven Central, base URL. - _maven_central_repo: str = "https://repo.maven.apache.org/maven2/" + _maven_central_repo: ClassVar[str] = "https://repo.maven.apache.org/maven2" @classmethod - def _artifact_base_url(cls, group_id: str, artifact_id: str) -> str: + def _normalise_maven_url(cls, url: str) -> str: + # Ensure URL ends with '/' so that concatenation builds a path. + return url if url.endswith("/") else f"{url}/" + + def _artifact_base_url(self, group_id: str, artifact_id: str) -> str: """Construct the base URL for a Maven artifact.""" # Reference: https://maven.apache.org/repositories/layout.html group_path = group_id.replace(".", "/") - return f"{cls._maven_central_repo}{group_path}/{artifact_id}/" + return f"{self._maven_repo_url}{group_path}/{artifact_id}/" - @classmethod - def artifact_metadata_url(cls, group_id: str, artifact_id: str) -> str: + def artifact_metadata_url(self, group_id: str, artifact_id: str) -> str: """Get the metadata URL for a Maven artifact.""" - # TODO: Unit test this method. - return f"{cls._artifact_base_url(group_id, artifact_id)}maven-metadata.xml" + return f"{self._artifact_base_url(group_id, artifact_id)}maven-metadata.xml" - @classmethod def artifact_url( - cls, group_id: str, artifact_id: str, version: str, classifier: str | None = None, extension: str = "jar" + self, group_id: str, artifact_id: str, version: str, classifier: str | None = None, extension: str = "jar" ) -> str: """Get the URL for a versioned Maven artifact.""" - # TODO: Unit test this method, including classifier and extension. _classifier = f"-{classifier}" if classifier else "" - artifact_base_url = cls._artifact_base_url(group_id, artifact_id) + artifact_base_url = self._artifact_base_url(group_id, artifact_id) return f"{artifact_base_url}{version}/{artifact_id}-{version}{_classifier}.{extension}" - @classmethod - def get_current_maven_artifact_version(cls, group_id: str, artifact_id: str) -> str | None: - url = cls.artifact_metadata_url(group_id, artifact_id) + def get_current_maven_artifact_version(self, group_id: str, artifact_id: str) -> str | None: + url = self.artifact_metadata_url(group_id, artifact_id) try: - # TODO: Use a user-agent that identifies this application. response = requests.get(url, timeout=_DEFAULT_HTTP_TIMEOUT) response.raise_for_status() # Content will be XML. @@ -336,7 +364,7 @@ def get_current_maven_artifact_version(cls, group_id: str, artifact_id: str) -> logger.error(f"Error while fetching maven metadata: {group_id}:{artifact_id}", exc_info=e) return None logger.debug(f"Maven metadata for {group_id}:{artifact_id}: {text}") - return cls._extract_latest_release_version(text) + return self._extract_latest_release_version(text) @classmethod def _extract_latest_release_version(cls, maven_metadata: str) -> str | None: @@ -350,9 +378,8 @@ def _extract_latest_release_version(cls, maven_metadata: str) -> str | None: return version return root.findtext("./versioning/versions/version[last()]") - @classmethod def download_artifact_from_maven( - cls, + self, group_id: str, artifact_id: str, version: str, @@ -363,7 +390,7 @@ def download_artifact_from_maven( if target.exists(): logger.warning(f"Skipping download of {group_id}:{artifact_id}:{version}; target already exists: {target}") return True - url = cls.artifact_url(group_id, artifact_id, version, classifier, extension) + url = self.artifact_url(group_id, artifact_id, version, classifier, extension) tmp_target = target.parent / f".{target.name}.download" try: # TODO: Use a user-agent that identifies this application. @@ -382,16 +409,31 @@ def download_artifact_from_maven( logger.info(f"Successfully installed: {group_id}:{artifact_id}:{version}") return True + def __init__(self, maven_repo_url: str | None = None) -> None: + if maven_repo_url is None: + if (maven_repo_url := os.environ.get("LAKEBRIDGE_MAVEN_URL")) is not None: + logger.debug(f"Using LAKEBRIDGE_MAVEN_URL override for maven artifacts: {maven_repo_url}") + else: + maven_repo_url = self._maven_central_repo + self._maven_repo_url = self._normalise_maven_url(maven_repo_url) + + +class MavenInstaller(ArtifactInstaller): def __init__( self, repository: TranspilerRepository, artifact_id: str, group_id: str, artifact: Path | None = None, + *, + maven_client: MavenClient | None = None, ) -> None: super().__init__(repository, artifact_id) + if maven_client is None: + maven_client = MavenClient.default() self._group_id = group_id self._artifact = artifact + self._maven_client = maven_client def install(self) -> Path | None: return self._install_checking_versions() @@ -400,7 +442,7 @@ def _install_checking_versions(self) -> Path | None: if self._artifact: latest_version = self.get_local_artifact_version(self._artifact) else: - latest_version = self.get_current_maven_artifact_version(self._group_id, self._artifact_id) + latest_version = self._maven_client.get_current_maven_artifact_version(self._group_id, self._artifact_id) if latest_version is None: logger.warning(f"Could not determine the latest version of Databricks {self._artifact_id} transpiler") logger.error(f"Failed to install transpiler: Databricks {self._artifact_id} transpiler") @@ -416,7 +458,9 @@ def _install_version(self, version: str) -> bool: if self._artifact: logger.debug(f"Copying: {self._artifact} -> {jar_file_path}") shutil.copyfile(self._artifact, jar_file_path) - elif not self.download_artifact_from_maven(self._group_id, self._artifact_id, version, jar_file_path): + elif not self._maven_client.download_artifact_from_maven( + self._group_id, self._artifact_id, version, jar_file_path + ): logger.error(f"Failed to install Databricks {self._artifact_id} transpiler (v{version})") return False self._copy_lsp_config(jar_file_path) diff --git a/tests/integration/transpile/test_installers.py b/tests/integration/transpile/test_installers.py index 25e98e75e0..ecd4922978 100644 --- a/tests/integration/transpile/test_installers.py +++ b/tests/integration/transpile/test_installers.py @@ -2,20 +2,20 @@ import pytest -from databricks.labs.lakebridge.transpiler.installers import MavenInstaller, MorpheusInstaller, WheelInstaller - -# TODO: These should run as part of the integration tests, not a separate test suite. +from databricks.labs.lakebridge.transpiler.installers import MavenLite, WheelInstaller, MorpheusInstaller def test_gets_maven_artifact_version() -> None: - version = MavenInstaller.get_current_maven_artifact_version("com.databricks", "databricks-connect") + client = MavenLite() + version = client.get_current_maven_artifact_version("com.databricks", "databricks-connect") assert version is not None check_valid_version(version) def test_downloads_from_maven(tmp_path: Path) -> None: + client = MavenLite() pom_path = tmp_path / "pom.xml" - success = MavenInstaller.download_artifact_from_maven( + success = client.download_artifact_from_maven( "com.databricks", "databricks-connect", "16.0.0", pom_path, extension="pom" ) assert success diff --git a/tests/unit/transpiler/test_installers.py b/tests/unit/transpiler/test_installers.py index 5724787484..cc40074b81 100644 --- a/tests/unit/transpiler/test_installers.py +++ b/tests/unit/transpiler/test_installers.py @@ -6,7 +6,7 @@ import pytest -from databricks.labs.lakebridge.transpiler.installers import ArtifactInstaller, MorpheusInstaller +from databricks.labs.lakebridge.transpiler.installers import ArtifactInstaller, MavenLite, MorpheusInstaller def test_store_product_state(tmp_path) -> None: @@ -40,6 +40,26 @@ def store_artifact_state(cls, product_path: Path, version: str) -> None: assert stored_state == expected_state +def test_maven_lite_artifact_metadata_url() -> None: + repo = "https://maven-proxy.some.example.com/prefix" + group_id, artifactid = ("org.apache.maven", "apache-maven") + url = MavenLite(repo).artifact_metadata_url(group_id, artifactid) + assert url == "https://maven-proxy.some.example.com/prefix/org/apache/maven/apache-maven/maven-metadata.xml" + + +def test_maven_lite_artifact_url() -> None: + repo = "https://maven-proxy.some.example.com/prefix" + group_id, artifactid = ("org.apache.maven", "apache-maven") + version = "3.8.4" + classifier = "bin" + extension = "tar.gz" + url = MavenLite(repo).artifact_url(group_id, artifactid, version, classifier, extension) + assert ( + url + == "https://maven-proxy.some.example.com/prefix/org/apache/maven/apache-maven/3.8.4/apache-maven-3.8.4-bin.tar.gz" + ) + + @pytest.fixture def no_java(monkeypatch: pytest.MonkeyPatch) -> None: """Ensure that (temporarily) no 'java' binary can be found in the environment.""" From 0ba44e683e041a587bec1fbd3f235fe7cdcdcc21 Mon Sep 17 00:00:00 2001 From: SundarShankar89 <72757199+sundarshankar89@users.noreply.github.com> Date: Wed, 6 May 2026 18:33:17 +0530 Subject: [PATCH 03/79] Fix test-profiler-connection to exit non-zero on failure (#2342) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary The `test-profiler-connection` CLI command always exited 0 on failure because `logger.fatal()` only logs — it doesn't terminate the process. The blueprint framework's `except Exception` catch-all swallowed everything else. This replaces all three broken error handlers with `raise SystemExit(...)` ### Functionality - [ ] added relevant user documentation - [ ] added new CLI command - [ ] modified existing command: `databricks labs lakebridge ...` - [ ] ... +add your own ### Tests - [x] manually tested - [ ] added unit tests - [x] added integration tests --- src/databricks/labs/lakebridge/cli.py | 15 ++++---- .../cli/test_profiler_connection_cli.py | 36 ++++++++++++------- 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/src/databricks/labs/lakebridge/cli.py b/src/databricks/labs/lakebridge/cli.py index 034a404f5a..872e7e6163 100644 --- a/src/databricks/labs/lakebridge/cli.py +++ b/src/databricks/labs/lakebridge/cli.py @@ -1103,10 +1103,9 @@ def test_profiler_connection( raw_config = cred_manager.get_credentials(source_tech) except KeyError as e: logger.error(f"Credential configuration error: {e}") - logger.fatal( + raise SystemExit( f"Invalid credentials for {source_tech}. Please run `databricks labs lakebridge configure-database-profiler`." - ) - return + ) from e try: _test_database_connection(source_tech, raw_config) @@ -1114,13 +1113,11 @@ def test_profiler_connection( logger.error(f"Failed to connect to the source system: {e}") error_msg = str(e).lower() if any(pattern in error_msg for pattern in ("im002", "odbc driver not found", "can't open lib")): - logger.fatal("Missing ODBC driver, Please install pre-req. Exiting...") - else: - logger.fatal("Connection validation failed. Exiting...") - except Exception as e: # pylint: disable=broad-exception-caught - # Catch all exceptions to provide user-friendly error messages for CLI + raise SystemExit("Missing ODBC driver, Please install pre-req. Exiting...") from e + raise SystemExit("Connection validation failed. Exiting...") from e + except Exception as e: # noqa: BLE001 logger.error(f"Unexpected error during connection test: {e}") - logger.fatal("Connection test failed. Exiting...") + raise SystemExit("Connection test failed. Exiting...") from e if __name__ == "__main__": diff --git a/tests/integration/cli/test_profiler_connection_cli.py b/tests/integration/cli/test_profiler_connection_cli.py index 04f333bc84..6ae70188ec 100644 --- a/tests/integration/cli/test_profiler_connection_cli.py +++ b/tests/integration/cli/test_profiler_connection_cli.py @@ -15,9 +15,10 @@ def _create_credentials_file( tmp_path: Path, *, exclude_serverless: bool | None = None, + exclude_dedicated: bool | None = None, invalid_server: bool = False, invalid_driver: bool = False, - missing_workspace: bool = False, + missing_source_key: bool = False, use_same_serverless_endpoint: bool = False, ) -> Path: cred_path = tmp_path / ".credentials.yml" @@ -32,12 +33,14 @@ def _create_credentials_file( if exclude_serverless is not None: profiler["exclude_serverless_sql_pool"] = exclude_serverless + if exclude_dedicated is not None: + profiler["exclude_dedicated_sql_pools"] = exclude_dedicated if invalid_server: workspace["dedicated_sql_endpoint"] = "invalid-server.database.windows.net" if invalid_driver: workspace["driver"] = "ODBC Driver 999 for SQL Server" - if missing_workspace: - credentials["synapse"] = {"jdbc": synapse["jdbc"], "profiler": profiler} + if missing_source_key: + del credentials["synapse"] if use_same_serverless_endpoint: workspace["serverless_sql_endpoint"] = workspace["dedicated_sql_endpoint"] @@ -87,18 +90,27 @@ def test_profiler_connection_invalid_source_technology( check_connection(w=ws, source_tech="mssql", cred_file_path=str(cred_path)) -def test_profiler_connection_invalid_config_errors( +@pytest.mark.parametrize( + ("cred_kwargs", "expected_msg"), + [ + ({"exclude_serverless": True, "invalid_driver": True}, "Missing ODBC driver"), + ({"exclude_serverless": True, "invalid_server": True}, "Connection validation failed"), + ({"exclude_serverless": True, "exclude_dedicated": True}, "Connection test failed"), + ({"missing_source_key": True}, "Invalid credentials"), + ], + ids=["odbc-driver-missing", "invalid-server", "all-pools-excluded", "missing-source-key"], +) +def test_profiler_connection_error_cases( sandbox_synapse_cred_config: JsonObject, tmp_path: Path, ws: WorkspaceClient, - caplog: pytest.LogCaptureFixture, + cred_kwargs: dict, + expected_msg: str, ) -> None: - """Test error handling when ODBC driver is missing.""" - cred_path = _create_credentials_file( - sandbox_synapse_cred_config, tmp_path, exclude_serverless=True, invalid_driver=True - ) + """Test that each failure mode raises SystemExit with the appropriate message.""" + cred_path = _create_credentials_file(sandbox_synapse_cred_config, tmp_path, **cred_kwargs) - check_connection(w=ws, source_tech="synapse", cred_file_path=str(cred_path)) + with pytest.raises(SystemExit) as exc_info: + check_connection(w=ws, source_tech="synapse", cred_file_path=str(cred_path)) - assert "Missing ODBC driver" in caplog.text - assert "Please install pre-req" in caplog.text + assert expected_msg in str(exc_info.value) From ae2c29b6f631bc30f8591125cdefc4bc632d3674 Mon Sep 17 00:00:00 2001 From: M Abulazm Date: Wed, 6 May 2026 18:38:52 +0200 Subject: [PATCH 04/79] Add foreign connections as reconcile source (#2362) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Changes ### What does this PR do? **Replaced direct JDBC connections with Databricks Unity Catalog remote_query().** Previously, reconciliation connected to external databases (Oracle, Snowflake, SQL Server) by managing JDBC credentials through Databricks secret scopes — each connector built JDBC URLs, handled authentication (passwords, PEM keys), and executed queries directly. Now, connections are managed through **UC Connections** — a centralized Databricks-native way to connect to foreign data sources. The connectors call `remote_query()` via Spark SQL, and Databricks handles authentication and connectivity. ### Why - **No more secret management** — credentials are managed in Unity Catalog, not in application code or secret scopes - **Simpler configuration** — users provide a UC connection name instead of configuring secret scopes with individual credential keys - **Centralized governance** — UC Connections are auditable, permission-controlled, and managed through standard Databricks tooling ### Config change (v1 → v2) ```yaml # Before (v1) data_source: oracle secret_scope: remorph_oracle database_config: source_schema: HR target_catalog: main target_schema: reconcile_target # After (v2) source: dialect: oracle catalog: ORCL schema: HR uc_connection_name: my_oracle_connection target: catalog: main schema: reconcile_target Existing v1 configs are auto-migrated on load. ``` ### Functionality - [ ] added relevant user documentation - [ ] added new CLI command - [ ] modified existing command: `databricks labs lakebridge ...` - [ ] ... +add your own ### Tests - [ ] manually tested - [ ] added unit tests - [ ] added integration tests --------- Co-authored-by: Claude Opus 4.7 (1M context) --- docs/lakebridge/docs/faq.mdx | 41 --- docs/lakebridge/docs/reconcile/index.mdx | 129 ++------- .../docs/reconcile/recon_notebook.mdx | 78 ++--- .../docs/reconcile/reconcile_automation.mdx | 129 --------- .../reconcile/reconcile_configuration.mdx | 180 ++++++------ .../static/lakebridge_reconcile/README.html | 46 --- .../static/lakebridge_reconcile/index.html | 61 ---- .../lakebridge_recon_main.html | 46 --- .../recon_wrapper_nb.html | 46 --- ...wflake_transformation_query_generator.html | 46 --- .../static/lakebridge_reconciliation.dbc | Bin 19201 -> 0 bytes src/databricks/labs/lakebridge/config.py | 68 ++++- .../labs/lakebridge/deployment/job.py | 2 +- .../labs/lakebridge/deployment/recon.py | 4 - src/databricks/labs/lakebridge/install.py | 59 ++-- .../reconcile/connectors/data_source.py | 4 +- .../reconcile/connectors/databricks.py | 5 +- .../reconcile/connectors/jdbc_reader.py | 41 --- .../lakebridge/reconcile/connectors/oracle.py | 64 +---- .../connectors/remote_query_reader.py | 47 +++ .../reconcile/connectors/secrets.py | 49 ---- .../reconcile/connectors/snowflake.py | 120 +------- .../reconcile/connectors/source_adapter.py | 13 +- .../lakebridge/reconcile/connectors/tsql.py | 64 +---- .../labs/lakebridge/reconcile/execute.py | 8 +- .../normalize_recon_config_service.py | 6 +- .../reconcile/query_builder/sampling_query.py | 65 ++++- .../lakebridge/reconcile/recon_capture.py | 5 +- .../labs/lakebridge/reconcile/recon_config.py | 12 +- .../reconcile/trigger_recon_service.py | 12 +- .../labs/lakebridge/reconcile/utils.py | 13 +- tests/conftest.py | 2 +- tests/integration/conftest.py | 19 ++ tests/integration/reconcile/conftest.py | 97 ++++--- .../reconcile/connectors/test_read_schema.py | 96 +------ .../reconcile/query_builder/test_execute.py | 121 ++++---- .../query_builder/test_sampling_query.py | 62 +++- .../reconcile/test_oracle_reconcile.py | 74 ++++- .../reconcile/test_recon_capture.py | 47 +-- tests/integration/reconcile/test_recon_e2e.py | 1 - tests/unit/deployment/test_installation.py | 49 ++-- tests/unit/deployment/test_job.py | 34 ++- tests/unit/deployment/test_recon.py | 35 ++- .../reconcile/connectors/test_databricks.py | 27 +- .../unit/reconcile/connectors/test_oracle.py | 145 ++++------ .../connectors/test_remote_query_reader.py | 73 +++++ .../unit/reconcile/connectors/test_secrets.py | 65 ----- .../reconcile/connectors/test_snowflake.py | 272 ++++-------------- .../reconcile/connectors/test_sql_server.py | 121 +++----- .../query_builder/test_threshold_query.py | 2 +- tests/unit/reconcile/test_source_adapter.py | 9 +- tests/unit/test_config.py | 124 +++++++- tests/unit/test_install.py | 207 +++++++------ 53 files changed, 1257 insertions(+), 1888 deletions(-) delete mode 100644 docs/lakebridge/docs/reconcile/reconcile_automation.mdx delete mode 100644 docs/lakebridge/static/lakebridge_reconcile/README.html delete mode 100644 docs/lakebridge/static/lakebridge_reconcile/index.html delete mode 100644 docs/lakebridge/static/lakebridge_reconcile/lakebridge_recon_main.html delete mode 100644 docs/lakebridge/static/lakebridge_reconcile/recon_wrapper_nb.html delete mode 100644 docs/lakebridge/static/lakebridge_reconcile/snowflake_transformation_query_generator.html delete mode 100644 docs/lakebridge/static/lakebridge_reconciliation.dbc delete mode 100644 src/databricks/labs/lakebridge/reconcile/connectors/jdbc_reader.py create mode 100644 src/databricks/labs/lakebridge/reconcile/connectors/remote_query_reader.py delete mode 100644 src/databricks/labs/lakebridge/reconcile/connectors/secrets.py create mode 100644 tests/unit/reconcile/connectors/test_remote_query_reader.py delete mode 100644 tests/unit/reconcile/connectors/test_secrets.py diff --git a/docs/lakebridge/docs/faq.mdx b/docs/lakebridge/docs/faq.mdx index 1144bb73f2..022a638c36 100644 --- a/docs/lakebridge/docs/faq.mdx +++ b/docs/lakebridge/docs/faq.mdx @@ -14,44 +14,3 @@ apt update && apt install -y curl sudo unzip #install databricks cli curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/v0.242.0/install.sh | sudo sh ``` ----- -## Reconcile -### 1. Guidance for Oracle as a source - -### Driver - -#### Option 1 -import CodeBlock from '@theme/CodeBlock'; - -* **Download `ojdbc8.jar` from Oracle:** -Visit the [official Oracle website](https://www.oracle.com/database/technologies/appdev/jdbc-downloads.html) to -acquire the `ojdbc8.jar` JAR file. This file is crucial for establishing connectivity between Databricks and Oracle -databases. - -* **Install the JAR file on Databricks:** -Upon completing the download, install the JAR file onto your Databricks cluster. Refer -to [this page](https://docs.databricks.com/en/libraries/cluster-libraries.html) -For comprehensive instructions on uploading a JAR file, Python egg, or Python wheel to your Databricks workspace. - -#### Option 2 - -* **Install ojdbc8 library from Maven:** -Follow [this guide](https://docs.databricks.com/en/libraries/package-repositories.html#maven-or-spark-package) to -install the Maven library on a cluster. Refer -to [this document](https://mvnrepository.com/artifact/com.oracle.database.jdbc/ojdbc8) for obtaining the Maven -coordinates. - -This installation is a necessary step to enable seamless comparison between Oracle and Databricks, ensuring that the -required Oracle JDBC functionality is readily available within the Databricks environment. - - -### 2. Commonly Used Custom Transformations - -| source_type | data_type | source_transformation | target_transformation | source_value_example | target_value_example | comments | -|-------------|---------------|--------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------|-------------------------|-------------------------|---------------------------------------------------------------------------------------------| -| Oracle | number(10,5) | "trim(to_char(coalesce(col_name,0.0), ’99990.99999’))" | "cast(coalesce(col_name,0.0) as decimal(10,5))" | 1.00 | 1.00000 | this can be used for any precision and scale by adjusting accordingly in the transformation | -| Snowflake | array | "array_to_string(array_compact(col_name),’,’)" | "concat_ws(’,’, col_name)" | [1,undefined,2] | [1,2] | in case of removing "undefined" during migration(converts sparse array to dense array) | -| Snowflake | array | "array_to_string(array_sort(array_compact(col_name), true, true),’,’)" | "concat_ws(’,’, col_name)" | [2,undefined,1] | [1,2] | in case of removing "undefined" during migration and want to sort the array | -| Snowflake | timestamp_ntz | "date_part(epoch_second,col_name)" | "unix_timestamp(col_name)" | 2020-01-01 00:00:00.000 | 2020-01-01 00:00:00.000 | convert timestamp_ntz to epoch for getting a match between Snowflake and data bricks | - - diff --git a/docs/lakebridge/docs/reconcile/index.mdx b/docs/lakebridge/docs/reconcile/index.mdx index 07a12bd2bf..8cc8ff7235 100644 --- a/docs/lakebridge/docs/reconcile/index.mdx +++ b/docs/lakebridge/docs/reconcile/index.mdx @@ -10,101 +10,47 @@ import TabItem from '@theme/TabItem'; This tool empowers users to efficiently identify discrepancies and variations in data when comparing the source with the Databricks target. -### Execution Pre-Set Up ->1. Setup the configuration file: +## Execution Pre-Set Up +### 1. Setup the configuration file: Once the [installation](/installation.mdx#configure-reconcile) is done, a folder named **.lakebridge** will be created in the user workspace's home folder. To process the reconciliation for specific table sources, we must create a config file that gives the detailed required configurations for the table-specific ones. The file name should be in the format as below and created inside the **.lakebridge** folder. ``` -recon_config___.json +recon_config___.json -Note: For CATALOG_OR_SCHEMA , if CATALOG exists then CATALOG else SCHEMA +Note: For UNITY_CATALOG_CONNECTION_NAME_OR_CATALOG , if databricks source then CATALOG else connection name ``` eg: -| source_type | catalog_or_schema | report_type | file_name | +| source_type | connection_name_or_catalog | report_type | file_name | |-------------|-------------------|-------------|---------------------------------------| | databricks | tpch | all | recon_config_databricks_tpch_all.json | -| source1 | tpch | row | recon_config_source1_tpch_row.json | -| source2 | tpch | schema | recon_config_source2_tpch_schema.json | +| source1 | conn1 | row | recon_config_source1_conn1_row.json | +| source2 | conn2 | schema | recon_config_source2_conn2_schema.json | Refer to [Reconcile Configuration Guide](reconcile_configuration) for detailed instructions and [example configurations](example_config) -> 2. Setup the connection properties - -Lakebridge-Reconcile manages connection properties by utilizing secrets stored in the Databricks workspace. -Below is the default secret naming convention for managing connection properties. - -**Note: When both the source and target are Databricks, a secret scope is not required.** - -**Default Secret Scope:** lakebridge_data_source - -| source | scope | -|---------------|-----------------------| -| snowflake | lakebridge_snowflake | -| oracle | lakebridge_oracle | -| databricks | lakebridge_databricks | -| mssql | lakebridge_mssql | -| synapse | lakebridge_synapse | - -Below are the connection properties required for each source: - - - - ```yaml - sfUrl = https://[acount_name].snowflakecomputing.com - account = [acount_name] - sfUser = [user] - sfPassword = [password] - sfDatabase = [database] - sfSchema = [schema] - sfWarehouse = [warehouse_name] - sfRole = [role_name] - pem_private_key = [pkcs8_pem_private_key] - pem_private_key_password = [pkcs8_pem_private_key] - ``` - - :::note - For Snowflake authentication, either sfPassword or pem_private_key is required. - Priority is given to pem_private_key, and if it is not found, sfPassword will be used. - If neither is available, an exception will be raised. - - When using an encrypted pem_private_key, you'll need to provide the pem_private_key_password. - This password is used to decrypt the private key for authentication. - ::: - - - ```yaml - user = [user] - password = [password] - host = [host] - port = [port] - database = [database/SID] - ``` - - - ```yaml - user = [user] - password = [password] - host = [host] - port = [port] - database = [database/SID] - encrypt = [true/false] - trustServerCertificate = [true/false] - ``` - - - ->3. Databricks permissions required +### 2. Setup the source connection + +Follow the official Databricks docs to: +- [Create a connection](https://docs.databricks.com/aws/en/query-federation/remote-queries#create-a-connection) +- [Grant connection access](https://docs.databricks.com/aws/en/query-federation/remote-queries#grant-connection-access) + +:::note +You do not have to create a foreign catalog. +::: + + +### 3. Databricks permissions required - User configuring reconcile must have permission to create Data Warehouses - Additionally, the user must have `USE CATALOG` and `CREATE SCHEMA` permission in order to deploy metadata tables and dashboards that are created as part of the Reconcile output. If there is a pre-existing schema, the 'create volumes' permission is also required. ->4. Serverless cluster support +### 4. Serverless cluster support Reconcile automatically detects the cluster type and optimizes intermediate data persistence accordingly: - **On Serverless clusters**: Reconcile uses Unity Catalog volumes for intermediate data persistence @@ -117,37 +63,6 @@ Reconcile automatically detects the cluster type and optimizes intermediate data ::: -### Execution -You can execute the reconciliation process by executing the below command in a notebook cell. - - -``` python -from databricks.labs.lakebridge import __version__ -from databricks.sdk import WorkspaceClient - -from databricks.labs.lakebridge.reconcile.trigger_recon_service import TriggerReconService -from databricks.labs.lakebridge.reconcile.exception import ReconciliationException - -ws = WorkspaceClient(product="lakebridge", product_version=__version__) - - -try: - result = TriggerReconService.trigger_recon( - ws = ws, - spark = spark, # notebook spark session - table_recon = table_recon, # previously created - reconcile_config = reconcile_config # previously created - ) - print(result.recon_id) - print(result) -except ReconciliationException as e: - recon_id = e.reconcile_output.recon_id - print(f" Failed : {recon_id}") - print(e) -except Exception as e: - print(e.with_traceback) - raise e - print(f"Exception : {str(e)}") -``` +## Running Reconcile -For more details, refer to the [Reconcile Notebook](recon_notebook.mdx) documentation. +Please refer to the [Reconcile Notebook](recon_notebook.mdx) documentation. diff --git a/docs/lakebridge/docs/reconcile/recon_notebook.mdx b/docs/lakebridge/docs/reconcile/recon_notebook.mdx index 2e3b6de0c4..b4bdd7d7ad 100644 --- a/docs/lakebridge/docs/reconcile/recon_notebook.mdx +++ b/docs/lakebridge/docs/reconcile/recon_notebook.mdx @@ -41,10 +41,11 @@ Import all the necessary modules. from databricks.sdk import WorkspaceClient from databricks.labs.lakebridge.config import ( - DatabaseConfig, ReconcileConfig, ReconcileMetadataConfig, - TableRecon + TableRecon, + SourceConnectionConfig, + TargetConnectionConfig ) from databricks.labs.lakebridge.reconcile.recon_config import ( Table, @@ -67,29 +68,35 @@ We use the class `ReconcileConfig` to configure the properties required for reco ```python @dataclass class ReconcileConfig: - data_source: str report_type: str - secret_scope: str - database_config: DatabaseConfig + source: SourceConnectionConfig + target: TargetConnectionConfig metadata_config: ReconcileMetadataConfig ``` Parameters: -- `data_source`: The data source to be reconciled. Supported values: `snowflake`, `oracle`, `mssql`, `synapse`, `databricks`. - `report_type`: The type of report to be generated. Available report types are `schema`, `row`, `data` or `all`. For details check [here](./dataflow_example.mdx). -- `secret_scope`: The secret scope name used to store the connection credentials for the source database system. -- `database_config`: The database configuration for connecting to the source database. expects a `DatabaseConfig` object. - - `source_schema`: The source schema name. - - `target_catalog`: The target catalog name. - - `target_schema`: The target schema name (Databricks). - - `source_catalog`: The source catalog name. (Optional and is applied to the source system that support catalog). +- `source`: The configuration for connecting to the source database to be reconciled. + - `dialect`: The dialect of the source. Supported values: `snowflake`, `oracle`, `mssql`, `synapse`, `databricks`. + - `catalog`: The source database/catalog name. catalog is used for consistency in naming + - `schema`: The source schema name. + - `uc_connection_name`: the connection name for the source as configured in workspace `Connections`. Not allowed for `databricks` ```python @dataclass -class DatabaseConfig: - source_schema: str - target_catalog: str - target_schema: str - source_catalog: str | None = None +class SourceConnectionConfig: + dialect: str + catalog: str + schema: str + uc_connection_name: str | None = None +``` +- `target`: The specs of the target databricks catalog to be reconciled. + - `catalog`: The target catalog name. + - `schema`: The target schema name. +```python +@dataclass +class TargetConnectionConfig: + catalog: str + schema: str ``` - `metadata_config`: The metadata configuration. Reconcile uses this catalog & Schema on Databricks to store all the backend metadata details for reconciliation. expects a `ReconcileMetadataConfig` object. @@ -105,28 +112,33 @@ class ReconcileMetadataConfig: If not set the default values will be used to store the metadata. The default resources are created during the installation of Lakebridge. + An Example of configuring the Reconcile properties: ```python from databricks.labs.lakebridge.config import ( - DatabaseConfig, ReconcileConfig, - ReconcileMetadataConfig + ReconcileMetadataConfig, + SourceConnectionConfig, + TargetConnectionConfig, ) -reconcile_config = ReconcileConfig( - data_source = "snowflake", - report_type = "all", - secret_scope = "snowflake-credential", - database_config= DatabaseConfig(source_catalog="source_sf_catalog", - source_schema="source_sf_schema", - target_catalog="target_databricks_catalog", - target_schema="target_databricks_schema" - ), - metadata_config = ReconcileMetadataConfig( +ReconcileConfig( + report_type="all", + source=SourceConnectionConfig( + dialect="snowflake", + catalog="source_sf_catalog", + schema="source_sf_schema", + uc_connection_name="source_connection_name" + ), + target=TargetConnectionConfig( + catalog="target_databricks_catalog", + schema="target_databricks_schema", + ), + metadata_config = ReconcileMetadataConfig( catalog = "lakebridge_metadata", schema= "reconcile" - ) + ), ) ``` @@ -140,10 +152,6 @@ class TableRecon: tables: list[Table] ``` -:::note -Schema and catalog information are configured using `DatabaseConfig` inside `ReconcileConfig` (see above). -::: - An Example Table properties for reconciliation: ```python @@ -189,7 +197,7 @@ table_recon = TableRecon( ) ], jdbc_reader_options=JdbcReaderOptions( - number_partitions=50, + num_partitions=50, partition_column="lct_nbr", lower_bound="1", upper_bound="50000", diff --git a/docs/lakebridge/docs/reconcile/reconcile_automation.mdx b/docs/lakebridge/docs/reconcile/reconcile_automation.mdx deleted file mode 100644 index 80fc668940..0000000000 --- a/docs/lakebridge/docs/reconcile/reconcile_automation.mdx +++ /dev/null @@ -1,129 +0,0 @@ ---- -sidebar_position: 5 -title: Reconcile Automation ---- - -import useBaseUrl from '@docusaurus/useBaseUrl'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - -## Overview - -The purpose of this utility is to automate table reconciliation based on provided table configurations. -- It ensures a streamlined comparison of tables, applying necessary transformations and computing reconciliation results efficiently. -- The utility also provides lookup tables which can be configured to provide custom inputs including - - the source/target tables - - transformations to be applied, - - thresholds to be set - -## Pre-requisites - -- The Lakebridge Recon tool should be configured through CLI to create the catalog (the name of the catalog can be customized during installation) -- A volume is created inside `.` -- Ensure `table_configs` table is created inside `.` with the below DDL. This table will store the configs for the tables that needs to be validated. -```sql -CREATE TABLE ..table_configs ( - label STRING, - source_catalog STRING, - source_schema STRING, - source_table STRING, - databricks_catalog STRING, - databricks_schema STRING, - databricks_table STRING, - primary_key ARRAY, - source_filters STRING, - databricks_filters STRING, - federated_source_catalog STRING, - select_columns STRING, - drop_columns STRING -) USING DELTA; -``` - -| Column Name | Description | -|------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **label** | The label to be used for grouping validation runs. | -| **source_catalog** | The source catalog name. | -| **source_schema** | The source schema name. | -| **source_table** | The source table name. | -| **databricks_catalog** | The databricks catalog name. | -| **databricks_schema** | The databricks schema name. | -| **databricks_table** | The databricks table name. | -| **primary_key** | `` The primary key columns to be used as an array. This will initiate the lakebridge reconcile job to run the comparison including primary keys. If unspecified runs a row level comparison. | -| **source_filters** | `` The filters to be applied on the source table. This will initiate the lakebridge reconcile job to run the comparison including filters. | -| **databricks_filters** | `` The filters to be applied on the databricks table. This will initiate the lakebridge reconcile job to run the comparison including filters. | -| **federated_source_catalog** | `` The federated source catalog name, if applicable to pull some metadata for the table references | -| **select_columns** | `` The columns to be selected from the source table. This will initiate the lakebridge reconcile job to run the comparison including selected columns. | -| **drop_columns** | `` The columns to be dropped from the source table. This will initiate the lakebridge reconcile job to run the comparison including dropped columns. | - -- Ensure `table_recon_summary` table is created inside `.` with the below DDL. This table will store the summary results of the validated tables. -```sql -CREATE TABLE ..table_recon_summary ( - timestamp TIMESTAMP, - label STRING, - databricks_catalog STRING, - databricks_schema STRING, - databicks_table STRING, - status STRING, - recon_id STRING, - row_status STRING, - column_status STRING, - schema_status STRING, - error STRING -) USING DELTA; -``` -| Column Name | Description | -|------------------------|-----------------------------------------------------| -| **timestamp** | The timestamp when the validation was run. | -| **label** | The label to be used for grouping validation runs. | -| **databricks_catalog** | The databricks catalog name. | -| **databricks_schema** | The databricks schema name. | -| **databricks_table** | The databricks table name. | -| **status** | The status of the validation. | -| **recon_id** | The reconciliation ID generated for the validation. | -| **row_status** | The status of the row level validation. | -| **column_status** | The status of the column level validation. | -| **schema_status** | The status of the schema level validation. | -| **error** | The error message, if any, during the validation. | - -## [Notebook Details](#notebook-details) - -import LakebridgeTabs from '@site/src/components/ReconcileTabs'; - - - -Link to the notebook - Unzip the downloaded file and upload the notebook file to your Databricks workspace. - -The utility consists of three key Databricks notebooks: - -#### recon_wrapper_nb: -- Acts as the main orchestrator. -- Reads the table configurations and triggers reconciliation for each table. -#### lakebridge_recon_main: -- Core reconciliation utility. -- Performs row, column and schema level comparisons. -- Computes reconciliation ID, status, and results. -#### transformation_query_generator: -- A source system-specific transformation script. -- Applies transformations based on source and databricks column data types. -- Enables efficient hash computation for reconciliation. -- This is a variable script based on the customer's source system. The one provided in the repository is for snowflake as the source system. - -## Parameters to Configure -To run the utility, the following parameters must be set: - -- `label`: The label from the table table_configs which will be used as a filter for validating only selected tables with the specific label. -- `remorph_catalog`: The catalog configured through CLI. -- `remorph_schema`: The schema configured through CLI. -- `remorph_config_table`: The table configs created as a part of the pre-requisites. -- `secret_scope`: The Databricks secret scope for accessing the source system. Refer to the Lakebridge documentation for the specific keys required to be configured as per the source system. -- `source_system`: The source system against which reconciliation is performed. -- `table_recon_summary`: The target summary table created as a part of the pre-requisites. - -## Points to Note - -The notebook `_transformation_query_generator` needs to be created or modified as per the customer's source system -The following rules are applied while performing validation. This can be customized as per customer's use case by doing the necessary changes in the [notebook](#notebook-details) -- Given that the data types across platform would vary and fail the schema level checks, the overall validation will still be marked as passed based on the row and column level checks. -- In case the primary keys are not configured in the table_configs, only row level checks will be performed to pass the validation. - diff --git a/docs/lakebridge/docs/reconcile/reconcile_configuration.mdx b/docs/lakebridge/docs/reconcile/reconcile_configuration.mdx index 16cebaba1a..7f5243ad17 100644 --- a/docs/lakebridge/docs/reconcile/reconcile_configuration.mdx +++ b/docs/lakebridge/docs/reconcile/reconcile_configuration.mdx @@ -18,50 +18,6 @@ import CodeBlock from '@theme/CodeBlock'; [[back to top](#types-of-report-supported)] -## Report Type-Flow Chart ---- -```mermaid -flowchart TD - REPORT_TYPE --> DATA - REPORT_TYPE --> SCHEMA - REPORT_TYPE --> ROW - REPORT_TYPE --> ALL -``` - ---- - -```mermaid -flowchart TD - SCHEMA --> SCHEMA_VALIDATION -``` ---- - -```mermaid -flowchart TD - ROW --> MISSING_IN_SRC - ROW --> MISSING_IN_TGT -``` ---- - -```mermaid -flowchart TD - DATA --> MISMATCH_ROWS - DATA --> MISSING_IN_SRC - DATA --> MISSING_IN_TGT -``` ---- - -```mermaid -flowchart TD - ALL --> MISMATCH_ROWS - ALL --> MISSING_IN_SRC - ALL --> MISSING_IN_TGT - ALL --> SCHEMA_VALIDATION -``` ---- - -[[back to top](#types-of-report-supported)] - ## Supported Source System | Source | Schema | Row | Data | All | @@ -74,51 +30,52 @@ flowchart TD [[back to top](#types-of-report-supported)] ## TABLE Config Json filename: -The config file must be named as `recon_config_[DATA_SOURCE]_[SOURCE_CATALOG_OR_SCHEMA]_[REPORT_TYPE].json` and should be placed in the Lakebridge root directory `.lakebridge` within the Databricks Workspace. +The config file must be named as `recon_config_[DATA_SOURCE]_[SOURCE_UNITY_CATALOG_CONNECTION_NAME]_[REPORT_TYPE].json` and should be placed in the Lakebridge root directory `.lakebridge` within the Databricks Workspace. > The filename pattern would remain the same for all the data_sources. ``` shell -recon_config_${DATA_SOURCE}_${SOURCE_CATALOG_OR_SCHEMA}_${REPORT_TYPE}.json +recon_config_${DATA_SOURCE}_${SOURCE_UNITY_CATALOG_CONNECTION_NAME}_${REPORT_TYPE}.json ``` Please find the `Table Recon` filename examples below for the `Snowflake`, `Oracle`, `` and `Databricks` source systems. - ``` yaml title="recon_config_snowflake_sample_data_all.json" - database_config: - source_catalog: sample_data - source_schema: default + ``` yaml title="recon_config_snowflake_example_connection_snowflake_all.json" + source: + dialect: snowflake + catalog: sample_data + schema: default + uc_connection_name: example_connection_snowflake ... metadata_config: ... - data_source: snowflake report_type: all ... ``` - ``` yaml title="recon_config_oracle_orc_data.json" - database_config: - source_schema: orc + ``` yaml title="recon_config_oracle_example_connection_oracle_data.json" + source: + dialect: oracle + uc_connection_name: example_connection_oracle ... metadata_config: ... - data_source: oracle report_type: data ... ``` - ``` yaml title="recon_config_tsql_silver_data.json" + ``` yaml title="recon_config_tsql_example_connection_mssql_data.json" database_config: - source_schema: silver + dialect: mssql + uc_connection_name: example_connection_mssql ... metadata_config: ... - data_source: mssql report_type: data ... ``` @@ -127,11 +84,11 @@ Please find the `Table Recon` filename examples below for the `Snowflake`, `Orac ``` yaml title="recon_config_databricks_hms_schema.json" database_config: - source_schema: hms + dialect: databricks + catalog: hms ... metadata_config: ... - data_source: databricks report_type: schema ... ``` @@ -139,8 +96,8 @@ Please find the `Table Recon` filename examples below for the `Snowflake`, `Orac -> **Note:** the filename must be created in the same case as [SOURCE_CATALOG_OR_SCHEMA] is defined. -> For example, if the source schema is defined as `ORC` in the `reconcile` config, the filename should be `recon_config_oracle_ORC_data.json`. +> **Note:** the filename must be created in the same case as `uc_connection_name` is defined. Or `catalog` for databricks sources +> For example, if the source connection name is defined as `ORC_connection` in the `reconcile` config, the filename should be `recon_config_oracle_ORC_connection_data.json`. [[back to top](#types-of-report-supported)] @@ -198,7 +155,7 @@ Please find the `Table Recon` filename examples below for the `Snowflake`, `Orac | `target_name` | `string` | Target table name | Required | {`{"target_name": "product"}`} | | `aggregates` | `list[Aggregate]` | List of aggregation rules (see [Aggregate](#aggregate)) | Optional | {`{"aggregates": {"type": "MAX", "agg_columns": ["price"]}}`} | | `join_columns` | `list[string]` | Primary key columns | Optional | {`{"join_columns": ["product_id", "order_id"]}`} | -| `jdbc_reader_options` | `object` | JDBC read parallelization configuration | Optional | {`{"jdbc_reader_options": {"number_partitions": 10, "partition_column": "id", "fetch_size": 1000}}`} | +| `jdbc_reader_options` | `object` | JDBC read parallelization configuration | Optional | {`{"jdbc_reader_options": {"num_partitions": 10, "lower_bound": : 1, upper_bound: 99, "partition_column": "id"}}`}| | `select_columns` | `list[string]` | Columns to include in reconciliation | Optional | {`{"select_columns": ["id", "name", "price"]}`} | | `drop_columns` | `list[string]` | Columns to exclude from reconciliation | Optional | {`{"drop_columns": ["temp_sku"]}`} | | `column_mapping` | `list[ColumnMapping]` | Source-target column mapping (see [column_mapping](#column-mapping)) | Optional | {`{"column_mapping": {"source_name": "id", "target_name": "product_id"}}`} | @@ -216,21 +173,21 @@ Please find the `Table Recon` filename examples below for the `Snowflake`, `Orac ```python @dataclass class JdbcReaderOptions: - number_partitions: int + num_partitions: int partition_column: str lower_bound: str upper_bound: str - fetch_size: int = 100 + fetchsize: int = 0 ``` ```json "jdbc_reader_options": { - "number_partitions": "", + "num_partitions": "", "partition_column": "", "lower_bound": "", "upper_bound": "", - "fetch_size": "" + "fetchsize": "" } ``` @@ -238,24 +195,13 @@ Please find the `Table Recon` filename examples below for the `Snowflake`, `Orac | field_name | data_type | description | required/optional | example_value | |-------------------|-----------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------|---------------| -| number_partitions | string | the number of partitions for reading input data in parallel | required | "200" | +| num_partitions | string | the number of partitions for reading input data in parallel | required | "200" | | partition_column | string | Int/date/timestamp parameter defining the column used for partitioning, typically the primary key of the source table. Note that this parameter accepts only one column, which is especially crucial when dealing with a composite primary key. In such cases, provide the column with higher cardinality. | required | "employee_id" | | upper_bound | string | integer or date or timestamp without time zone value as string), that should be set appropriately (usually the maximum value in case of non-skew data) so the data read from the source should be approximately equally distributed | required | "1" | | lower_bound | string | integer or date or timestamp without time zone value as string), that should be set appropriately (usually the minimum value in case of non-skew data) so the data read from the source should be approximately equally distributed | required | "100000" | -| fetch_size | string | This parameter influences the number of rows fetched per round-trip between Spark and the JDBC database, optimising data retrieval performance. Adjusting this option significantly impacts the efficiency of data extraction, controlling the volume of data retrieved in each fetch operation. More details on configuring fetch size can be found [here](https://docs.databricks.com/en/connect/external-systems/jdbc.html#control-number-of-rows-fetched-per-query) | optional(default="100") | "10000" | +| fetchsize | string | This parameter influences the number of rows fetched per round-trip between Spark and the JDBC database, optimising data retrieval performance. Adjusting this option significantly impacts the efficiency of data extraction, controlling the volume of data retrieved in each fetch operation. More details on configuring fetch size can be found [here](https://docs.databricks.com/en/connect/external-systems/jdbc.html#control-number-of-rows-fetched-per-query) | optional(default="0") | "10000" | -:::tip -#### Key Considerations for Oracle JDBC Reader Options: -For Oracle source, the following Spark Configurations are automatically set: -```json -"oracle.jdbc.mapDateToTimestamp": "False", -"sessionInitStatement": "BEGIN dbms_session.set_nls('nls_date_format', '''YYYY-MM-DD''');dbms_session.set_nls('nls_timestamp_format', '''YYYY-MM-DD HH24:MI:SS''');END;" -``` - -While configuring Recon for Oracle source, the above options should be taken into consideration. -::: - [[back to top](#types-of-report-supported)] ## Column Mapping @@ -609,6 +555,50 @@ on top of this. [[back to top](#types-of-report-supported)] +## Report Type-Flow Chart +--- +```mermaid +flowchart TD + REPORT_TYPE --> DATA + REPORT_TYPE --> SCHEMA + REPORT_TYPE --> ROW + REPORT_TYPE --> ALL +``` + +--- + +```mermaid +flowchart TD + SCHEMA --> SCHEMA_VALIDATION +``` +--- + +```mermaid +flowchart TD + ROW --> MISSING_IN_SRC + ROW --> MISSING_IN_TGT +``` +--- + +```mermaid +flowchart TD + DATA --> MISMATCH_ROWS + DATA --> MISSING_IN_SRC + DATA --> MISSING_IN_TGT +``` +--- + +```mermaid +flowchart TD + ALL --> MISMATCH_ROWS + ALL --> MISSING_IN_SRC + ALL --> MISSING_IN_TGT + ALL --> SCHEMA_VALIDATION +``` +--- + +[[back to top](#types-of-report-supported)] + ## Aggregates Reconciliation Aggregates Reconcile is an utility to streamline the reconciliation process, specific aggregate metric is compared @@ -643,21 +633,6 @@ between source and target data residing on Databricks. [[back to top](#types-of-report-supported)] -### Flow Chart - -```mermaid -flowchart TD - Aggregates-Reconcile --> MISMATCH_ROWS - Aggregates-Reconcile --> MISSING_IN_SRC - Aggregates-Reconcile --> MISSING_IN_TGT -``` - - -[[back to aggregates-reconciliation](#aggregates-reconciliation)] - -[[back to top](#types-of-report-supported)] - - ### Aggregate @@ -759,5 +734,20 @@ Key Considerations: [[back to top](#types-of-report-supported)] +### Flow Chart + +```mermaid +flowchart TD + Aggregates-Reconcile --> MISMATCH_ROWS + Aggregates-Reconcile --> MISSING_IN_SRC + Aggregates-Reconcile --> MISSING_IN_TGT +``` + + +[[back to aggregates-reconciliation](#aggregates-reconciliation)] + +[[back to top](#types-of-report-supported)] + + diff --git a/docs/lakebridge/static/lakebridge_reconcile/README.html b/docs/lakebridge/static/lakebridge_reconcile/README.html deleted file mode 100644 index efd1d5ff6a..0000000000 --- a/docs/lakebridge/static/lakebridge_reconcile/README.html +++ /dev/null @@ -1,46 +0,0 @@ - - - - - -README - Databricks - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/docs/lakebridge/static/lakebridge_reconcile/index.html b/docs/lakebridge/static/lakebridge_reconcile/index.html deleted file mode 100644 index ebfacf8f8b..0000000000 --- a/docs/lakebridge/static/lakebridge_reconcile/index.html +++ /dev/null @@ -1,61 +0,0 @@ - - - - - - -lakebridge_reconciliation - Databricks - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/docs/lakebridge/static/lakebridge_reconcile/lakebridge_recon_main.html b/docs/lakebridge/static/lakebridge_reconcile/lakebridge_recon_main.html deleted file mode 100644 index 3f4ab49382..0000000000 --- a/docs/lakebridge/static/lakebridge_reconcile/lakebridge_recon_main.html +++ /dev/null @@ -1,46 +0,0 @@ - - - - - -lakebridge_recon_main - Databricks - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/docs/lakebridge/static/lakebridge_reconcile/recon_wrapper_nb.html b/docs/lakebridge/static/lakebridge_reconcile/recon_wrapper_nb.html deleted file mode 100644 index 83e2ab4db6..0000000000 --- a/docs/lakebridge/static/lakebridge_reconcile/recon_wrapper_nb.html +++ /dev/null @@ -1,46 +0,0 @@ - - - - - -recon_wrapper_nb - Databricks - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/docs/lakebridge/static/lakebridge_reconcile/snowflake_transformation_query_generator.html b/docs/lakebridge/static/lakebridge_reconcile/snowflake_transformation_query_generator.html deleted file mode 100644 index 5fe987f104..0000000000 --- a/docs/lakebridge/static/lakebridge_reconcile/snowflake_transformation_query_generator.html +++ /dev/null @@ -1,46 +0,0 @@ - - - - - -snowflake_transformation_query_generator - Databricks - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/docs/lakebridge/static/lakebridge_reconciliation.dbc b/docs/lakebridge/static/lakebridge_reconciliation.dbc deleted file mode 100644 index a320ebc2b3f8e7e74f04789719b701e16f3b7523..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 19201 zcmaI718i?y_bpu8HcoBZwr$(CZMUbk-JaUEZR=OJr`CO*_uiZP&zF4f&fYs~uQBJE zBiUIqBO_UgGN52+Ku}OnK-lV#Iza#9f&~HtvNN)`GBaQQBq8euFD1M%4%rf+Ue#T!GRl%#DysPE--x~QF!=c2_T8> zVAx1EWd~Yn(~I&Ts%Q8kfnU^gx*jo##-N3O`Jt6zMS>+^imLQKy}AmYJj6kb9L>SwNF(;EnC|Dj~%OR=aq>ZeTbntDI&o^CmWoRA(W{HvK}Q%7(JW zITb0TaZCcpQQdK+o5G1Iv*s@1SGGmg{mszXT4Y&PXZ0;f$eVk0_7OYzGEiDg$&8`4 zF;+yc6VI8PMDtE!`o_%;H#U#>=vPIWEq8my7n)hgb*?`JM?nv+o>yUDT{4by=lbCR0tfpBGjs^+XNcJFg+owz_1{VovV3637v@Fy{ZeTUAI;X`af@=d zP1>7eCNqpJqJ6SknheU}bSx!`<3vgZt6}Va|4t7i=Y=VttLtec$QG^!O9w_Jy1_G% zTe$t=lqb3Nnt&%k-}@T0lhcpBAj&O_ylzT>-vMPGF0t8QVGJlj%B9{se+`2s5Q5V- zUEdPdtcJoQL6NF%kU5na@XfS+9yPZA7pa%Pn1zRQ5 z!v9et5JQrbp&n|nN=+=i>CRz;vqPu!6LsCOh0Z017)Jl(=?J9bF$J}KBKgRy-U~~? zXT5O$*n=v$L=;923-lT11?STj=W09_#(OtIn{cx${0+Ostf1-d5mIKwf!@4# z{GvV7MwGo`d6D}kR-c8^s%M6%h|{eF1#y??6DW1Hd;sunlqqDJW&ycLzEe**?L~h^*$B{0s4=r|2Fnddso=CE8Sv)M{vP&<$DL*f+&V9XyoeteOzb*&cyJ zJVUgBc<%+GF@Y`xxm1%U!B|zi!&01Ep)O+C2$jJ)#lyThVUGyP+qT~UK1cUFMK{j% zgG149^fN*L^q01pD=O;tq~xXA3{~euo%Wou@N4jGs8f^uu`fSRpSlW}9B>JUcK(7q z32CnYZ)4MLeQ!c`L#30skjqOpdbRb2Bsi%a0le}KeF@ECj(-i47&f$0J2AK9O z@teM-DJ5+_TqMcYOI_1@Gp@*Qd*7xX&$)TtjJPo5`P^^EoaxVv_xLB3v+0C2oHezL zzUMi1FL(wFX|y#8rUm`sjG2WBCjnk_O}kxip21|@I`I?ifP&B zD+W=?p82ORB+Mbfj|>l7l`H&buM0|S-L%JNf1s^unnfQh~NbL))I3Z$`H;-Pywn4x8MbmA4f< z9nLxZ^Iw-w&?om6_}+<29eIW`SwRU%2gliPB`b--pO`?Lt+Q}9s%9uMS1eLHn(0kj zvcYXyOl}Huv_=63qB;FRTks)b(WQG)c+|dd$9{O;X%OKvLeVp=Fa7ET#K{4BqT{2m zvEhXd1DZ^!l559eGWDSe<1w73rt5eNcht}Pmvx+Gubo)6nG<=j=?_=?${OwxlSF+B zopC5OHmOBwwbz#Qm+M;glj&8zw+5n0bAcU0MW{qw#zbh>g3${6>-#Gb+zl%b#3td! z#Wf|&tN{KM(pE1T?Grsq-R4>wwMA8?_Et++3o7M%{kFgArBT}6MJj1Q*w&=b>}bgCa$!0 zUs0#bz+lIva#5bYQ;xbtAGqR&9yD?{o9$i|@a=;u2@q<~MCXivvcO$*8QGHkeKI6Q zFp?5ZZ`O)kg#v)F>k?GT+t?_|;zknt8$nCswA}b~ zYX?As_KYp${Fimbi}gUSHbEq#pd$0`_TqGsTGX6H#7ULC=~qC=KV00rd-$*mOtmG0 z)qej%T->Z8#Qb2wu3T*o{(J9n1my}$IZjNCDD4}0wJzV&iH)J| zbYvw0P2*!y(#=f^*>7b*RPQCpa;ufT6DJ2oBL0Z#-SYO(x|zD}nQf!NY!1y2{-Wqc zOH*`QQA}rr&)^Xc)2DbKXk>8LLy8v@L*_jk-4iNmUpCCJH!?-G9fIN79Bc>r z<3le{q`nhtZ+H;t*cgSLqnh4dYr1S;^?^Elaett{Rb{>m`yj2gcCf8Ss6l4!{)n@ zygjD7k+S+2rwE}QAcF-?f`}MG{iD__y#9^gKVH$+hnj&c^s*mti4m}{ABb`CWSV7A zi5>c9?*FX8`*PTUIoI=_cKE-8{{vV-D6r(&0sdJWQ}|KX^8XEM#m|BO$+J>{KGw2GsuAW>{gQJnb^Fx-Z5x-2Q?Fu9Ey&J^xoY3pHm; zs{cx+p6uc<86gxwg4K@AS|5FC%iuBkCi<#vRvhG@`5$lqHePB3KUGXkKpj@M2rF$} zd_rmB?{7t}n=w`hlV@dSV5906m2%G5p9?bqey8eGX+43b5r2bT$W5|cW*nizz1AJ!iB_{`O=+RDkX0SB9 zJTO7rt04)5F_N@LEI~yyPTsEpGK=`0+!~8YgT2@?Yu471Rp@SqmcQLV#g1m}I8=>V za}`H&F;t_asBS~ki_7ia+U#2m`6S7_iL%?Y9c^P+fz~xmPP5-L)(5(A3C$f0bw3{7 zE9VISIep6RHYH1=8yfRLNuz=)u28|;3KFqA7oP>?|_%sP1JAVYOl!0tYY>?F>{Rj`~_VU8=79EZ(x`vfE16>*rF^Z}79(IA< zMh6(dpNpQ~9t~i~{rq4a5iLn6edC#t?jsYSWS=I^4zR5tU`sCW`DgE5p)T{J>rxvx>`Du_neNIK?aCh3 zP(~pJX!Agu=yZkVuNv-`zUy7-!uEgY-T7m7?}6WX3|#(tz2y(6+Vk`5i^-@+-v6Ti zXCOjN>DdRndmDU)!Q1xga*EM?n!Kj_&CHhZzmxux|3CNNKcO#O|H*~@k0{tEoBba( z{?AbCf7bnH?ms*5Umjw^A9}oF#{Vg4T)P)mRIypGdMnI!4D8_1=+hO$ASQLri`ZPC zPQra|9Sn?{@Y3&jPk;q6_#oNBrZf=G0LU_nX{|FvrV4kkuDhU07x1KZx94^-PW3E> zR`lFcGu^`5D00Bfs>mM4OuD2lf-C17YeXBlpEukw^d{Zg^shy6U>;mw)fI79TS75g zY7x^LO}8Xj)B&|(U&I@FNa7;GB_)2=JGKT=Tkhx->If-cMn?Ygw`%)1F}xpc(WnT> z`VJ_$pC2SDT@qHr!%?%2xUSWhV$}x?LcejS{?4_OdhsX(gTJ}7wztF7=HGU^otXYO?|zx1cudG+y03sr^Moq9pjpx~t_o^x?PYV@O1{kjhn`W_LYlRJNb^6%=fSr=Hw~E}lV%=qQ^c7Cg zO)MPp1q`Q*!~&Wg!9{v~-6cGFmyB%j>j2`yj{ZU2!1(37eoX0FS!X-}4ja7j6oF7# zu$bBtZPOzyvXC;Fch%8)E5!%5^C}xuBm5yQEI8e^4`nm%C)@d-?B=zyP4R?+EM-5_ zj`wlleR<-~fpAvQt!1`_MLoHpbO`9Uox zcmBT_b5j4+>T|>fro;Eik<} z`>pX7e{^9%BQKi8&mX2kG<}HMvHQVLRO2iqqnIhFTybKRfkt4Mwc25~K z+dM1+CG-jAiZWpA^vwy!Kn{i*t_~j_2MQ|`p(7$=|Kvtun9X*?oS&h7LP!5`6cjMs zHHGhWnPgxdUe3VVo@I=JvmhB1P0LI2Jfhk5-?B5^tNZB=w^;y1sPM)~GmO?A}a-6<*JoVfUh)EW+8{4s$ z?IK;v=;kx~OA3=Yhk|p+rZhb|%|j(q^gTu-%@22%6>Nd$dA@x!lG@HCDTexS#X=~C zrG%A;m9_U~sMffIdz!1a_b4wSeiSN)vpXM1{>SnhB1tZIj+FieD^R=f7*#iz$dSsQ zs;y|?PJ#Hu^-Ige#~%B=mzK2~)WM z4q~;=5pX#a)rCmFJ53rDap7McmzzaKdJXzsuTMkCsnP>Rb*RrwBPfqlE%6;4$EG!mNFu|qJO#K;EUm0Pj z6b1_&mtaGam|fh&65B5gUKoZJYjZt0C5!B3A$ksI)KE!xskUtJo{a-UX8rBl6aywJJ_D$>t-1%p-1sj$PjEJJBXpfe+ups zNRYG`31`iqMYnEZ$M3OQs6Smdj_2Puly91NwuGc18wttl1B-GFpbbcY0+`B=z74`l z2z6)lNBZ;?{>Tc9{&|blEcLe3M))+Nj$W)c_$IXhj>tb;NBQ`9>HB#k{eeLk{*J^H zyNL<*EU?NHpUY2w56EH-4m?8j&KEeo-+TS5ssGlO-}r~pDyUjwiPQFdTQ^ragNAAKTp;Bob92Fsh zVfy3siG`#Zks9UILv-l7wX`&eQ^7(R`L%sp8h!3J~*bF zaJZ}<Pg1jqT6FV;aAhkh$QL<7)OS$ zKvgd2qT&e|3!5bXQlZ>OMPS@vB@`~?W7;N4lP}`Y_&;14sDr*y_UJypv8S8X;;7(3 zvjR=1HM0@~5RGj_k$fpER~Oq!s$HBQb!J9_B=FUuUd}1eTKYj198htL)8M2FVBU|a zE=WS1zpq~I}w1fk||-#`5jWgOk{LL-g|-v|0?#O-f1}H zgMABFeoBkAc*(lxC1Tf56~$t>_9gb>ChO7)Z5isejc49}1I;gQ$Sg=Z6wWNK7a(rG!6OL$<N}h`+>t%)^k~#F57=azCI-o^Jlii zAO?97LjRRa1v80ZdjX#H)P2p;Z-dM#-hzKoM#Rs~JO`97HZJ%}B`mhE?zp@ya5^Qp z84J4_23yYC*Tx&dkxp1#8N34q2X&9pQ`u5I9IpTEtZyyUmY7w&g->h6k$EVKY6VrBQ%XW#%lB4YopyQ=piboGy% zzVpsGomE}k?Rqm3{@(B_G&!BpEVa>0+Tu#Yh3rVoknRrB$j%~~;p>@Vn(Vx^Q%fx$ z#Kt1e?rSW!i#Jb+0{B=$f&5F@wVmnRmbmB;m4QoxFl^zxbWphRSiksDlQ&B$kXW-gTLfERe=h%d6fBXp^)Zi3brX)sO)HrH5lB3 zK?L_|{~$OUKU$0f6#pR=XV(MOa)<*|0pu>L0*9}hm@laE?|iLYTC|RK*Z2y$+y1sRJF&4`-1%$0^usl5%X?0uZ8SsP#?=RjhtO@*zL3!F&3?m&7WFiW8y z^(GhYvI!-OT8$g5E6M-jCfSGYdCD!qZ!JOHVnt6?XMqxVhoGF*Bn0(o=JAXA+IR+Brj9y5(ob%3qE{kX!4@m?cpz*uUuO1m zTVv6I@_GY@0P;u~g1k!^szc4ICLsh6zse`1A9ng)GO$ zlSTWuaa<5~dq_9Jmg!ZG>3JdD3{$W2Xy;elkAp=Qc+m#+2Xj;c=ZHYd3GP z%~lD%kHxH~zUg{{`&}X(<5Tgxjynr-?AECIIh21v_N0CX`Y4iriW9_ickO$1$V9j2mL-o2)aM^h-)0% z)kSl*Q3`zgQfF*tcY$A4QddqyEnu{<;_k;Izhai1C{6TnfDTR2%9}65rUueOYkb3| z9uif$2UL)aqrvB3MN}Fa+#r??%@Qgj-dQ$wlQ#}7%3)McLmrp7S!^gUL*I);@r$de z{kA1O7;&rkv7L!aGvm;BvdpVI<&F{|bSD*Ye-v{rj>)IQ46#=5v?=Sf7CZJ@+*qB9iOE<|>~?ik zj}do$@aW2flUGKYPF7i3FLmi9$V)|+=q&LVB*R^cR8z^6>xA&HAb;kMNOx_Jzpl2*BQcafD$0G9S4DLxsTnDo#;>3hH$#3d#v0 zcjNmMzgvWIO?bM}dy76Z!sOpXczY1fdD|5JqsO|(p|H8lkb>Z#Lpa-hJN3I8Reo;I z-!d1!O+g^XgI<1@Z7x161@TrOH`l;%+u=5=)_lNpmSg*3X)Iwz-S5^%DzU4_FMfY` zP%}rlPm08Ksw2no%zHj1h@~B)z_Hq5aX-{=H7NLsOz8)~s4-;}Sl3BwC}d~~=S9L^ z*XIO?85p86uh5ef{(=pkKyUT;3l##r%~AF~;b2>?Yxp;ZV)QsckzA z&x?{Cn$pD*e+nAQ5lR#3{u04R>idi*x#I|GcADEiAIxpU_Wa{x9fXhg?+piMW74$} zz@(x67TV|45daMK>;)ZJ<8c{xv$c=**}OLJrzk~3*^*wyFysm^XBgIY$}n5PE}+vY zAah0ck52j7Pnyo;+0p7m*O9tEy=4iL-Ek8*N4NnRV6rXUJG?dV@tfkrWmf@ zB6GPDC+vDKw1`})fq%Q}mWoR~OfGAoeIZico7C+e0Vv-w zh23s381UGNpt6$Uo~)uuo2>YyXir>*aAXlF8~kTB$DU2KZz2<<##1`}HLq+hHK6^u zeE#p29owMTQV+k$ZSEjMx^oEYju${GcTwxvD0HyOJVrwBHCHN~py|!k&JXCnw$hZu z8st7Q5D<*-f3uY+{;RE|EG8r>C-#3$BW*i}El!kgn$Vb$YmD@2jjc&^aOWt8wTKd2 zQhUUh>tBVUljOf*w3VYTX&&|*UW+h!Z#y*^K$bHW(^JFW6a_*q}A=K z(JgThlza7$g2x9Y>hrx}bFl9%{w73(n9T_Cj$5L2syH0BS7LG-Wq~2g8XqFzl$c8M zCd=w7l3bQ-rdvC8nYX9+YK>kfbqg!dltj^d`WbG$pmaEb5rT}WAGi&N*S2_edfDSSD$&P2( zShMzL-h}XghB)<8MQcG4Oz$xwPgYKOiLCZiY_!%?j>b0CABWK^&vemyvxbXv!jb;s z$iyTI#|hMZjWK{Rz=tV-ho>*MuSZXJ&OGG1AQMSfSDDJqH->vI0<_AMgY6u13x7j+ zKNY9^rG&>|c0e(GN<6)v-yHH7m;RJAruW1P5SCZ4dtaZ1zhL^^O#Py61u!IjQ=L1a&0G^ zj#8_x+RjUY1#kVDSXcG+?n^S?9U$?_wFAx7%6WFuMrzKJm^h9|#}EOtiH*OOtmIs6 zwmM&iZEUl~86AQ_J=oY7&r0lpFGbd_a6!=+_%I&}U9+URR*ogz>`_>G;S?Wg)=Hkn z8hk2;f-@i?Ophrdg{aztQH%?C1t4Nj72mZX?;@;KT7y;L(tViSj9=MeHv-jyILpaJ zxi{eqmEXO8!AJIJ{ES@B@R72fR;QS8?ZtHZcP?Q+8x}$eTm*+{HOchDEqMuF*{&>Ag{_hXx^6kNf&^_`NdD z=BP)wbYKSNW8t-2UdbT{S6hv`aBBosMQtbXz4wd)VPx5D&^9=il-R@~nRFvmloj{G z^p5khsff=i#vRR*Kkj~5%0%tYm*e5E6&_E}<){%(m&shVuinCQKH4?nNDL;!{G4Hh z^J~PRNTViG6aYrrqeX0e*^WBN9Tug_OfQWfcwdC-6|@AUamY|{G`(fM_!w%2Mif>{ zL`oDZv*N5K@W2oj50_jBd~Vv5OSGN#k&cr9omI_D#a4;Tk(h-ZE;2A{P_|Aj@Oh?b zKFBh?F;yt>@%kL-CKUhX+;{~NKLs}6S-4MQduJ#3h9Ks(F;c)__-7d0TiO>VRDVCV zmY)uK)0aWF6enIlMoaZWCl6tJ-R4&S0weU3mBE^85o%!nR=-gnP-n~VRH;u ztT0t*YY~we%bIQ~O{%b>bML3(D~W3~!LnBg)O!+?3EECiE2{l<*>?VRsx>b*=Glu{?cHs$|q($)!7KOM&3xX`aTRs&?I24kejJe92IL?}0 zj{SX<6F%;qUjDC>mZZLoHX0Kz@Uu%6>wN4aC6FgfW2Xd{w~d&Z#8@}{Hr^uPQarp4ZQ15K*}opCJOdQoO%yli`I4tARwcR=i@55 z1tcCw)mX1+yII$2i*s>0e1GRu=$7BdWX#V1M{2XBp8FAcpNY@;9c#eA{`( z5G9Q1o;RY*|1=bpC}#rg+taZFzL^So$?3sOmxXg1&hmW?Md$%(3Aw;dgotfyKsdPo zZD)y&+X0q`I_rYMx(d|N8t^FIUz-=b^~>w+K@@#*Ez0w#U7TGtU2`eev(cMO zlot0jGYccqy_E;2ZPd}kmqV}ERd+tKW*Sxby#VT|d=|*m#Uj1rr|RT)zCM+}kAl~Z$g&2ebthMbQX)02Cl|5L$2s_cyqj?M4L`^eFN_kGGJPL~%S9~6 z-OnSB$D9}17}GqiaL5q$GH!RkH(P-#KbD6IZnYme!Rl_z8;6&1I6dTw;^X@6c1OVf z6{LRjfsV0n?RQz{a!dRf>CV%I*#aEyAHd8pHrR^&ke?pxVQ2+xYM;(fAaN5PkHb#JmI)6ie_-OS&xBr9O4Yty#?90L^L05f-2=BinH_ zsL@kRSM>Aix8nFrPxFeuH@efvuhi2_Rx1c0KxshiM!+e}`S+|lu%P4t_O{*lzOFyi z)Y2Y~SE}RT1i_9ip@S+@kdNv+wXQytPn8`8}j3?e8Ycso7s4NdF z)24OV8H|KK3&33v$1`y`$C8Y3I>$OBn#@y@7zIX!99f-*+8=VO2nV7jLPzm&NJ{d4 z8OI;A;F+MvMsk!j9wBo$4Kg9*t|#zP;4h(#h6(MP8;mg(S?p4@AHpT8pj%GAYIk%n<&k1T1ST*Fnu6AE+`?VMV#sQ2Zwut+O2?Sr z^`MFDpD{j3ddfymVV{StD+(HKs$H$5jH+W*6#gkEz8W9=XA_ncJrfv zn^=xoasn3pbUY5y6`T=Bjesd$2`6n*K zzV`mplqJ5f^ZEHi;eY8r@8+>O1kz-nuXG9buwVn);(&+rdme^p$o0yU1>VO3&yueAs!EkKkMg3C z>TvT%6|pu?sU|bb%B&Mg`a~&a8a=U4u9J>af0fi?TKGan7*vLt5hQT2BcL%#_@k|bteKA{A8yH%qSU)#C%jQb zvQeqi@`7-p$FGO`(i8GEMW&F4g_Es4`aS*b9KGJ$9bG-$SU?&*R2m1Tl=C$Psq^F- zwqv};{HJ*k*C+TEc2Y8|9=)C?v;^4^-rPYjS6SXA-KSD9+@B^o$d_%)?@OaGWpqn*^lz&b#XuU%=j;5w_>Z80_ihU&ini`u= z@R^gsw4u>7bm)~_n=_ad5zv81`&+&0a_on&RM&+0=oNcb(NIyuD|6?CMXH~C82AVJ zs03wqHknno_i^ZNKC+dF8969;nga%txO^d_c53$vN`h-)2~r_ew1JOdU}KLadK&(m z8$$@#m{k+KXOv%#u{Pt22<+Rfk}TJ|VS&^;4=b_sKJ!DVzO-)Bx%`EgzH|u@L_Y1{ z$D=q^n%x1{?_N&C+}Vff?J9|KtDFH?hBvxDw+$}Xy1qhF98c`KUA&Rqz}3!*(b87w zE01eRc)LHRLHI%NQFp*gnx((f$BV3mH#m&F3YN*7Sar3b{wO%U(~su!`Tk%GZpwM~ zy3*%zP*PhTtE)tMV{^XBOzcDm(M4}L2Z^bOse$TLF%Y>s_6RnW_35WnHMWmhNz-;3 z6QUK)skx%cXdabT279i!x$dgcMYYkXEb6T}lR?Mz%00)_XrpV@vd~rFK!N zf`+vXw5GqInfma5v0oOjQhj3l!3|2M;E0RlKZtG{e(JCDqsaUOXg;IA631UC%C~Fe zXo_2b=aoVXO3?XG$*SMQ= zWE0a&FK*P3a%3y5WzE=!e1!7YAYOAui%>K#HE(I_yYghsA2xAy*QMa#dz8R8bhRnus#xG$>*iyHSf)Dv8xdSP!}jM#T(zE3*1*Tm9v5=6YK3KUcv9l^b&Lza!!F4;H>2lZjy#Lrly2j3A}QK zTL52GODsqs=H$Nt1ZME6W|mBDF=9b&@O1(fQx_IfWk?Ct(Og|2u(drA+0T>D6|$xW zJuw0mRafSUZq2u(BOei7`0$#5n&PR@Jb5*3(pk87_y5YoEfju>ES?5`PixN@GDdRh z)U?WQES1@hmm6)Tw1(0TS3k|ujeUe#Fq~g(i)-GnR>%Fm(*UnHT0&UCHdb*O6acS^ z<*7F7g+D}c%d&Efg2+*=m|cz0pStdW$;mULVU^|Jp;+@0&@8U-L&)TLda= z@RmxxLGaG*3LwSKDxq#Fq0F+j&k;8>zqkAbjCpojxi#H3_zD5u=h43v^~X zO8p*T&)X=F7s%0=gRbGtsPLXvQ)kU`L~?@~se<;S&z>I>M{VOY$|7s*@s~Wi0DFM7 z1+5+2qR0WgxBogOf&Cx?Ydy*l!V)4E`sIp8xB58+LM^xvf<6qU=`ysQv<1hA-qs0Y z&-rmYtNS}6CY|MVub0P1%y*xih|3^v6}>8&myJXCjW7C&t77T)U7atDMS-cx{dt;j79_`#(+cULPQ8TXL?K2iTXfI zGzOOSJ>g}qFK%O;nEi2QA%vix#Dh$xwu$`^#RN^wVyxAFneNG4vioKuY*avg%vxfu zRLJEFs?0%S|6CvzxqWWQQR%^`VDAhKXl3s2)3;0U36b>AE>Y^qcDeuY)=4E0&uf_@ z9<$L)_7tPLtFbD~edn+C%Xhq_cGKXrc@;}R=b|q%&SjeuwqZU2!L}I z%lx;GN3-iTZpbrq-!r6} zZ>ZFZF}V2JIyMMZD;9en`)uXn;&WOg5*&bwBm_(y5 zj4u?4zI$>2cHWK1_A%XDUMWXjNG!CR&?Zy8h-nyAxZhY+)?V_in^&0A(aofNC=KMA zys)N&w&Ll&74KD;4qZjE1~KmQ#Du@AtbZj%Rq^ZOELeU18N!hvXluxkGIY|Ae~#Nd zWQ_r3b}?)MUlUdPC6*Zjg^qUlvGwpsUW6o!s+-o^ljWj#1S@(#o^Ty ztI&fWdJ^PP-}pT3JRB+xvOUgBhS_8kn;WHzZ@7Eh*go*;eT=m_KD^Z{ck@a9IQk1? z#J#>UrHOpit_>|d{7O$`u(VJN(9IWxYhP6JS7^Vo1_ zBa>S((@QqpVMSA^de$4MJk;iP_A<@luCI!pCJ|}#TFC3{q^=wnG&k8}Ycyey_6P#A zto2)qUpIpzDi5z0&~PxixKJCJ_sFNEoiIP%mig(GT%zRS;|jWsnGQZK@sEqrjpV7^ zSq!f5zli|0>QdmLhwn&n#m*JgA0B%}Rgm|==XV~iY<>To(lZ&HV+%b8(v$;V!#MUh z^6)SXGR6ZRU8Il+Ni)}DiiKvVD)g4T4E^|aKvfqnkdJ#&?1HwAMLXgJ!n#fWvSh>< zhC>@vLdfYjj>Rw}7)hc-d1OLmnbwy$WH1l1_ABWY9_ksytS+Wmvwbw58P1MDI@XZj zGn533{nUIQDHoN0ag+}o75D92Rr&ZR*+~BvT9RDg!iR+~FtDdAtfile_h&l@br~9jvLPbaPuAZ$a!`=ggBvy>{UNJlF>6p>h4#7z$}Y7D7rd;=3DT`vc+4 z{FEqwjCQ$#iN=Ikw^FU4spRHB?4}6O6O$79b>(zp`Ae5%6X=PrE@dVMWne#AUj}6i z;H$ll+^p)dia?2Ue+Cf0y;%&82AWuB|cLa9F8^-6?*;co4{GLKi~B)dMt zBI|R_dqZ+Z*}y9Sr=5Dp?o(SS#DdN4*LFgd?_NqaM{W%zqF2=)Uj*v6!720-v^;nk za0WzI^jfD6_!U)$90C+KxfX+7f5JU}u$`>Ex|dr0CWfivYu*)5zrON>+;0c|{u~|O z$R*qBW!LKJn-PaDjJVyTErl4bTLe`jl6W|X+?zXcK7x?q8^)f;nvKPNM~gf`y`4h= zNugY7rMTSk52LiJ>QK7n3)fwi%-g(T)$4?Zi33UQs=4U(4>*<3hQO!D1cT8#G=n5p zMyX3L(Tl~0Z>dS7wJIpFq(15<-runt>()9HSm`Y=Ze5*C$p%pLHHt9>=EuM*JEfaPj8Klt( zJf#RsEYFP>(XEiP$dKcG)Si(N&S&#dqc=h3be@eg-YZo8TW~+*^ujso>x11Ex2!}ZU zc93VJF0_#3H0pPA6Kv9|M3u1rMqQi63}U{0WmCfCAdoM8lN8bMR$WvYK6A?zlk|jy ztoEh3Kt%v}+d`JyUBul1e$R_(cftOJjpyjyP)BMui*z)4%D9}#$4(wY&S&)kEf!u7 zQ!L+Eye1Ypx0lz6VGes4eg!f3H%9C&peX-q8=o6ae6Z^)AqMIyqD5& zUa?X-EPZws_rJoDd#^qh^z+uPcC=@`{!ez|VYXS17M!x|G_#zvCDHOp`<%p09jBCD z)jPSo{l30=ZuWAP$tfwnpC(41uzI-Lz|-1e)z;q`@)y&;xGLDoynP&^@I8xF>4d57 zg8dq{>!wdsTfWO~N6(6@>e|c8pKRyc-ElSU*&@wT&8o&*zP3e6oM+AX9XBQ6q4Gs% z*M74~*X;16xSgkj( z++7p7Az+DOH=~%#0UQ428|pijZt1;r$c^o-B!_CJevEpZ@0IB{`odEhYXmLcJDq5K z{VO9xx3)T7ESh80r3u01r*y8D+;R-O)PCrE@~X+V;#Tfb_UzdFXOjQkElW&QLgpP@ z`qQcT&k?RB)1zKH9A3(EPx+^P=AXEW@{;_3BJ(^^aj_*c_?PenXZ~Y8vSao>lO>(8 zFP4AI>Szp!)|Oo}FYnZ;3B^}6;~&~MSR8FyzoSof(wxO{b3VUZ=lM2L=1$XIo(0U7 zUToI?r$1by@=1^9@24x<%F0;(KlvCiyXf=!XRb#VHT+X4)|;aF=a&|Dk+@+0v~Rx` zt7!RI9H0Hv^+NJ$jcr}O_|ENT)7`&(x^+dj)r_0(-!HmU{&Z2Cs(0NI#Vt=)o|NKN zQQUbtymrTmi9PJE^I79;C;YzX_vUw@Z}i$_ix1ZWH#>GFv-@3M7_!4IHg^j<+y1s| zZU4B64bR4^-{|EN;E`A{@5FuIXrZZF#U~3j{wefWXqI?s{p+i*<3FF?EHY#7@Abzv z#YYod5@QYaMOTU`$|7G#U8v&(uQp}8K5@~#{cly{#=qCO zuhbvDU%T|x!%4uAseSc-ew8Xd6JB_Id(+|aV`mHtESTGkLBwY3BRTv}+iy81KKO8a zN8p(Q4|!zjPUI%D@ufc~0F`Nvj-Sz$ui#cFIHD2du(n)deeZaB? zq~{!%_{1|U-6mc`8D=;zZ)!o=?{zhyEU>*WBHGoqJ5&Ti#(uti#`=i9)lz*thxIM@ ztn#wz54rk~_voe#YTk>wzx-8v=*7x;z4Yo;lO>;pwl(g1cWckeS4#PQuED~x?(=tD zzL+@S!^<*{9cL6C@(7)S5^tUv?6y2p=C|+gTwylO`kv||V7c}?hG`QD*f`U7vYyth zJ9DRL;Uo#Imp2zol2DoBct{dNL?RWa2=jE#CsZE$%xYldVbR=T>|xQpLT3?_{o~#w zm?&5ZN^B0jvN^)l5UL(7^W$@4i}$2;v-@nHrg~1Q5EatBcPXX0ocE-w)wF{QM}WS~ zV&!ySv^Z*$#nV{dBYR$HnB`mTI<2i|c4pZ=T1xkSpl?1EHKg~6~g|oR~5>FNI~6^zIsLa z#;zo&dN!{A3mqQzp)=*T z&8kfHzwCFMk)+M@{@#^iDyMD>EoFWA-qB9TAVV>5Hm~cyRURKVI`cNYc^Rvxe1G%a zFHg0d8dpt>Sg_3NRmdct(AipDYi1n1uv2tP72C%XjQl^QweQgI(c9;eIATo6w?FfsF6xUDXoG){l{}>N^+}6LEpf%+JbX z)5mYOA<{)pQ0-Xzl0B(%<%)xO=5W4Zm%V|J+u@Jye1fTIF?#x2B7I{_zv{nZJ xI@Iu4i?VVD-B9GX1Wjllzz$%@;SO?i8RY4W0B=?{kU4BXD8j_RaNPsM0|2buLWuwX diff --git a/src/databricks/labs/lakebridge/config.py b/src/databricks/labs/lakebridge/config.py index 874f0b14e2..235fc6ce26 100644 --- a/src/databricks/labs/lakebridge/config.py +++ b/src/databricks/labs/lakebridge/config.py @@ -223,12 +223,33 @@ def v1_migrate(cls, raw: dict[str, Any]) -> dict[str, Any]: return raw -@dataclass +@dataclass(frozen=True) class DatabaseConfig: + """TODO remove. this was kept for backwards compatibility while migrating to ReconcileConfig v2""" + + source_catalog: str source_schema: str target_catalog: str target_schema: str - source_catalog: str | None = None + + +@dataclass +class SourceConnectionConfig: + dialect: str + catalog: str + schema: str + uc_connection_name: str | None = None + + def __post_init__(self): + self.dialect = self.dialect.lower() + if self.dialect != "databricks" and not self.uc_connection_name: + raise ValueError(f"uc_connection_name is required for non-databricks sources (dialect={self.dialect})") + + +@dataclass +class TargetConnectionConfig: + catalog: str + schema: str @dataclass @@ -260,15 +281,50 @@ class ReconcileJobConfig: @dataclass class ReconcileConfig: __file__ = "reconcile.yml" - __version__ = 1 + __version__ = 2 - data_source: str report_type: str - secret_scope: str - database_config: DatabaseConfig + source: SourceConnectionConfig + target: TargetConnectionConfig metadata_config: ReconcileMetadataConfig job_overrides: ReconcileJobConfig | None = None + @classmethod + def v1_migrate(cls, raw: dict[str, Any]) -> dict[str, Any]: + db_config = raw.pop("database_config") + dialect = raw.pop("data_source") + # Drop fields no longer recognised by the v2 schema. + raw.pop("secret_scope", None) + raw.pop("tables", None) + + source = { + "dialect": dialect, + # source_catalog was optional in v1 (Oracle service name could be omitted); default Oracle to ORCL. + "catalog": db_config.get("source_catalog") or ("ORCL" if dialect == "oracle" else ""), + "schema": db_config["source_schema"], + } + if dialect != "databricks": + # Marker consumed by upgrade_reconcile_config_if_needed; user must supply the real UC connection name. + source["uc_connection_name"] = "TODO" + + raw["source"] = source + raw["target"] = { + "catalog": db_config["target_catalog"], + "schema": db_config["target_schema"], + } + raw["version"] = 2 + return raw + + @property + def database_config(self) -> DatabaseConfig: + """TODO remove. this was kept for backwards compatibility while migrating to ReconcileConfig v2""" + return DatabaseConfig( + source_catalog=self.source.catalog, + source_schema=self.source.schema, + target_catalog=self.target.catalog, + target_schema=self.target.schema, + ) + @dataclass class ProfilerDashboardMetadataConfig: diff --git a/src/databricks/labs/lakebridge/deployment/job.py b/src/databricks/labs/lakebridge/deployment/job.py index cfb86309e2..58b0662810 100644 --- a/src/databricks/labs/lakebridge/deployment/job.py +++ b/src/databricks/labs/lakebridge/deployment/job.py @@ -107,7 +107,7 @@ def _job_recon_task( compute.Library(whl=lakebridge_wheel_path), ] - if recon_config.data_source == ReconSourceType.ORACLE.value: + if recon_config.source.dialect == ReconSourceType.ORACLE.value: # TODO: Automatically fetch a version list for `ojdbc8` oracle_driver_version = "23.4.0.24.05" libraries.append( diff --git a/src/databricks/labs/lakebridge/deployment/recon.py b/src/databricks/labs/lakebridge/deployment/recon.py index 98235eb677..6d999d5095 100644 --- a/src/databricks/labs/lakebridge/deployment/recon.py +++ b/src/databricks/labs/lakebridge/deployment/recon.py @@ -60,10 +60,6 @@ def uninstall(self, recon_config: ReconcileConfig | None): f"Won't remove reconcile metadata schema `{recon_config.metadata_config.schema}` " f"from catalog `{recon_config.metadata_config.catalog}`. Please remove it and the tables inside manually." ) - logging.info( - f"Won't remove configured reconcile secret scope `{recon_config.secret_scope}`. " - f"Please remove it manually." - ) def _deploy_tables(self, recon_config: ReconcileConfig): logger.info("Deploying reconciliation metadata tables.") diff --git a/src/databricks/labs/lakebridge/install.py b/src/databricks/labs/lakebridge/install.py index 18fd00831e..061f33208d 100644 --- a/src/databricks/labs/lakebridge/install.py +++ b/src/databricks/labs/lakebridge/install.py @@ -20,10 +20,11 @@ from databricks.labs.lakebridge.__about__ import __version__ from databricks.labs.lakebridge.cli import lakebridge from databricks.labs.lakebridge.config import ( - DatabaseConfig, ReconcileConfig, LakebridgeConfiguration, ReconcileMetadataConfig, + SourceConnectionConfig, + TargetConnectionConfig, TranspileConfig, ProfilerDashboardConfig, ProfilerDashboardMetadataConfig, @@ -344,46 +345,48 @@ def _prompt_for_new_reconcile_installation(self) -> ReconcileConfig: report_type = self._prompts.choice( "Select the report type:", [report_type.value for report_type in ReconReportType] ) - scope_name = self._prompts.question( - f"Enter Secret scope name to store `{data_source.capitalize()}` connection details / secrets", - default=f"remorph_{data_source}", - ) - db_config = self._prompt_for_reconcile_database_config(data_source) + source_config = self._prompt_for_source_connection_config(data_source) + target_config = self._prompt_for_target_connection_config() metadata_config = self._prompt_for_reconcile_metadata_config() return ReconcileConfig( - data_source=data_source, report_type=report_type, - secret_scope=scope_name, - database_config=db_config, + source=source_config, + target=target_config, metadata_config=metadata_config, ) - def _prompt_for_reconcile_database_config(self, source) -> DatabaseConfig: - source_catalog = None - if source == ReconSourceType.SNOWFLAKE.value: - source_catalog = self._prompts.question(f"Enter source catalog name for `{source.capitalize()}`") - if source == ReconSourceType.DATABRICKS.value: - source_catalog = self._prompts.question( - f"Enter source catalog name for `{source.capitalize()}`", default="hive_metastore" - ) + def _prompt_for_source_connection_config(self, dialect: str) -> SourceConnectionConfig: + uc_connection_name: str | None = None + if dialect != ReconSourceType.DATABRICKS.value: + uc_connection_name = self._prompts.question(f"Enter Unity Catalog {dialect.capitalize()} connection name") - schema_prompt = f"Enter source schema name for `{source.capitalize()}`" - if source in {ReconSourceType.ORACLE.value}: - schema_prompt = f"Enter source database name for `{source.capitalize()}`" + if dialect == ReconSourceType.ORACLE.value: + catalog = self._prompts.question("Enter Oracle service name") + elif dialect == ReconSourceType.DATABRICKS.value: + catalog = self._prompts.question("Enter source Databricks catalog name", default="hive_metastore") + else: + catalog = self._prompts.question(f"Enter {dialect.capitalize()} database name") - source_schema = self._prompts.question(schema_prompt) - target_catalog = self._prompts.question("Enter target catalog name for Databricks") - target_schema = self._prompts.question("Enter target schema name for Databricks") + schema_prompt = f"Enter source {dialect.capitalize()} schema name" + if dialect == ReconSourceType.ORACLE.value: + schema_prompt = "Enter Oracle database name" - return DatabaseConfig( - source_schema=source_schema, - target_catalog=target_catalog, - target_schema=target_schema, - source_catalog=source_catalog, + schema = self._prompts.question(schema_prompt) + + return SourceConnectionConfig( + dialect=dialect, + catalog=catalog, + schema=schema, + uc_connection_name=uc_connection_name, ) + def _prompt_for_target_connection_config(self) -> TargetConnectionConfig: + target_catalog = self._prompts.question("Enter target Databricks catalog name") + target_schema = self._prompts.question("Enter target Databricks schema name") + return TargetConnectionConfig(catalog=target_catalog, schema=target_schema) + def _prompt_for_reconcile_metadata_config(self) -> ReconcileMetadataConfig: logger.info("Configuring reconcile metadata.") catalog = self._configure_catalog() diff --git a/src/databricks/labs/lakebridge/reconcile/connectors/data_source.py b/src/databricks/labs/lakebridge/reconcile/connectors/data_source.py index 9294768b77..f8318ef558 100644 --- a/src/databricks/labs/lakebridge/reconcile/connectors/data_source.py +++ b/src/databricks/labs/lakebridge/reconcile/connectors/data_source.py @@ -16,7 +16,7 @@ class DataSource(ABC): @abstractmethod def read_data( self, - catalog: str | None, + catalog: str, schema: str, table: str, query: str, @@ -27,7 +27,7 @@ def read_data( @abstractmethod def get_schema( self, - catalog: str | None, + catalog: str, schema: str, table: str, normalize: bool = True, diff --git a/src/databricks/labs/lakebridge/reconcile/connectors/databricks.py b/src/databricks/labs/lakebridge/reconcile/connectors/databricks.py index 89d05b3e4c..c83e3feb8f 100644 --- a/src/databricks/labs/lakebridge/reconcile/connectors/databricks.py +++ b/src/databricks/labs/lakebridge/reconcile/connectors/databricks.py @@ -9,7 +9,6 @@ from databricks.labs.lakebridge.reconcile.connectors.data_source import DataSource from databricks.labs.lakebridge.reconcile.connectors.models import NormalizedIdentifier -from databricks.labs.lakebridge.reconcile.connectors.secrets import SecretsMixin from databricks.labs.lakebridge.reconcile.connectors.dialect_utils import DialectUtils from databricks.labs.lakebridge.reconcile.recon_config import JdbcReaderOptions, Schema from databricks.sdk import WorkspaceClient @@ -36,7 +35,7 @@ def _get_schema_query(catalog: str, schema: str, table: str): return re.sub(r'\s+', ' ', query) -class DatabricksDataSource(DataSource, SecretsMixin): +class DatabricksDataSource(DataSource): _IDENTIFIER_DELIMITER = "`" def __init__( @@ -44,12 +43,10 @@ def __init__( engine: Dialect, spark: SparkSession, ws: WorkspaceClient, - secret_scope: str, ): self._engine = engine self._spark = spark self._ws = ws - self._secret_scope = secret_scope def read_data( self, diff --git a/src/databricks/labs/lakebridge/reconcile/connectors/jdbc_reader.py b/src/databricks/labs/lakebridge/reconcile/connectors/jdbc_reader.py deleted file mode 100644 index 3b9ec6b1e4..0000000000 --- a/src/databricks/labs/lakebridge/reconcile/connectors/jdbc_reader.py +++ /dev/null @@ -1,41 +0,0 @@ -from typing import Any - -from pyspark.sql import SparkSession - -from databricks.labs.lakebridge.reconcile.recon_config import JdbcReaderOptions - - -class JDBCReaderMixin: - _spark: SparkSession - - # TODO update the url - def _get_jdbc_reader(self, query, jdbc_url, driver, additional_options: dict | None = None): - driver_class = { - "oracle": "oracle.jdbc.OracleDriver", - "snowflake": "net.snowflake.client.jdbc.SnowflakeDriver", - "sqlserver": "com.microsoft.sqlserver.jdbc.SQLServerDriver", - } - reader = ( - self._spark.read.format("jdbc") - .option("url", jdbc_url) - .option("driver", driver_class.get(driver, driver)) - .option("dbtable", f"({query}) tmp") - ) - if isinstance(additional_options, dict): - reader = reader.options(**additional_options) - return reader - - @staticmethod - def _get_jdbc_reader_options(options: JdbcReaderOptions): - option_dict: dict[str, Any] = {} - if options.number_partitions: - option_dict["numPartitions"] = options.number_partitions - if options.partition_column: - option_dict["partitionColumn"] = options.partition_column - if options.lower_bound: - option_dict["lowerBound"] = options.lower_bound - if options.upper_bound: - option_dict["upperBound"] = options.upper_bound - if options.fetch_size: - option_dict["fetchsize"] = options.fetch_size - return option_dict diff --git a/src/databricks/labs/lakebridge/reconcile/connectors/oracle.py b/src/databricks/labs/lakebridge/reconcile/connectors/oracle.py index 355b140526..b0b1a0fec7 100644 --- a/src/databricks/labs/lakebridge/reconcile/connectors/oracle.py +++ b/src/databricks/labs/lakebridge/reconcile/connectors/oracle.py @@ -1,26 +1,22 @@ import re import logging -from collections.abc import Mapping from datetime import datetime from pyspark.errors import PySparkException -from pyspark.sql import DataFrame, DataFrameReader, SparkSession +from pyspark.sql import DataFrame from pyspark.sql.functions import col from sqlglot import Dialect from databricks.labs.lakebridge.reconcile.connectors.data_source import DataSource -from databricks.labs.lakebridge.reconcile.connectors.jdbc_reader import JDBCReaderMixin from databricks.labs.lakebridge.reconcile.connectors.models import NormalizedIdentifier -from databricks.labs.lakebridge.reconcile.connectors.secrets import SecretsMixin +from databricks.labs.lakebridge.reconcile.connectors.remote_query_reader import RemoteQueryReader from databricks.labs.lakebridge.reconcile.connectors.dialect_utils import DialectUtils -from databricks.labs.lakebridge.reconcile.recon_config import JdbcReaderOptions, Schema, OptionalPrimitiveType -from databricks.sdk import WorkspaceClient +from databricks.labs.lakebridge.reconcile.recon_config import JdbcReaderOptions, Schema logger = logging.getLogger(__name__) -class OracleDataSource(DataSource, SecretsMixin, JDBCReaderMixin): - _DRIVER = "oracle" +class OracleDataSource(DataSource): _IDENTIFIER_DELIMITER = "\"" _SCHEMA_QUERY = """select column_name, case when (data_precision is not null and data_scale <> 0) @@ -38,25 +34,14 @@ class OracleDataSource(DataSource, SecretsMixin, JDBCReaderMixin): def __init__( self, engine: Dialect, - spark: SparkSession, - ws: WorkspaceClient, - secret_scope: str, + reader: RemoteQueryReader, ): self._engine = engine - self._spark = spark - self._ws = ws - self._secret_scope = secret_scope - - @property - def get_jdbc_url(self) -> str: - return ( - f"jdbc:{OracleDataSource._DRIVER}:thin:@//{self._get_secret('host')}" - f":{self._get_secret('port')}/{self._get_secret('database')}" - ) + self._reader = reader def read_data( self, - catalog: str | None, + catalog: str, schema: str, table: str, query: str, @@ -64,21 +49,15 @@ def read_data( ) -> DataFrame: table_query = query.replace(":tbl", f"{schema}.{table}") try: - if options is None: - return self.reader(table_query, self._get_timestamp_options()).load() - reader_options = self._get_jdbc_reader_options(options) | self._get_timestamp_options() - df = self.reader(table_query, reader_options).load() - logger.warning(f"Fetching data using query: \n`{table_query}`") - - # Convert all column names to lower case - df = df.select([col(c).alias(c.lower()) for c in df.columns]) - return df + logger.info(f"Fetching data using query: \n`{table_query}`") + df = self._reader.read_data(table_query, catalog, "service_name", "dbtable", options) + return df.select([col(c).alias(c.lower()) for c in df.columns]) except (RuntimeError, PySparkException) as e: return self.log_and_throw_exception(e, "data", table_query) def get_schema( self, - catalog: str | None, + catalog: str, schema: str, table: str, normalize: bool = True, @@ -91,7 +70,7 @@ def get_schema( try: logger.debug(f"Fetching schema using query: \n`{schema_query}`") logger.info(f"Fetching Schema: Started at: {datetime.now()}") - df = self.reader(schema_query).load() + df = self._reader.read_data(schema_query, catalog, "service_name", "query") schema_metadata = df.select([col(c).alias(c.lower()) for c in df.columns]).collect() logger.info(f"Schema fetched successfully. Completed at: {datetime.now()}") logger.debug(f"schema_metadata: {schema_metadata}") @@ -99,25 +78,6 @@ def get_schema( except (RuntimeError, PySparkException) as e: return self.log_and_throw_exception(e, "schema", schema_query) - @staticmethod - def _get_timestamp_options() -> dict[str, str]: - return { - "oracle.jdbc.mapDateToTimestamp": "false", - "sessionInitStatement": "BEGIN dbms_session.set_nls('nls_date_format', " - "'''YYYY-MM-DD''');dbms_session.set_nls('nls_timestamp_format', '''YYYY-MM-DD " - "HH24:MI:SS''');END;", - } - - def reader(self, query: str, options: Mapping[str, OptionalPrimitiveType] | None = None) -> DataFrameReader: - if options is None: - options = {} - user = self._get_secret('user') - password = self._get_secret('password') - logger.debug(f"Using user: {user} to connect to Oracle") - return self._get_jdbc_reader( - query, self.get_jdbc_url, OracleDataSource._DRIVER, {**options, "user": user, "password": password} - ) - def normalize_identifier(self, identifier: str) -> NormalizedIdentifier: normalized = DialectUtils.normalize_identifier( identifier, diff --git a/src/databricks/labs/lakebridge/reconcile/connectors/remote_query_reader.py b/src/databricks/labs/lakebridge/reconcile/connectors/remote_query_reader.py new file mode 100644 index 0000000000..281ee58a43 --- /dev/null +++ b/src/databricks/labs/lakebridge/reconcile/connectors/remote_query_reader.py @@ -0,0 +1,47 @@ +from dataclasses import asdict + +from pyspark.sql import DataFrame, SparkSession + +from databricks.labs.lakebridge.reconcile.connectors.dialect_utils import DialectUtils +from databricks.labs.lakebridge.reconcile.recon_config import JdbcReaderOptions + + +class RemoteQueryReader: + + def __init__(self, spark: SparkSession, connection_name: str): + self._spark = spark + self._connection_name = connection_name + + def read_data( + self, + source_query: str, + catalog: str, + catalog_key: str = "database", + source_query_key: str = "query", + options: JdbcReaderOptions | None = None, + ) -> DataFrame: + query_options = self._build_options(catalog, catalog_key, options) + query = self._build_query(query_options, source_query, source_query_key) + return self._spark.sql(query) + + def _build_query(self, query_options: str, source_query: str, source_query_key: str) -> str: + escaped = source_query.replace("'", r"\'") + return ( + f"SELECT * FROM remote_query('{self._connection_name}', {source_query_key} => '{escaped}', {query_options})" + ) + + @staticmethod + def _build_options(catalog: str, catalog_key: str, options: JdbcReaderOptions | None = None) -> str: + def camelcase(underscored): + parts = underscored.split('_') + return parts[0] + ''.join(word.capitalize() for word in parts[1:]) + + def encode(key, value): + if key == "partition_column": + value = DialectUtils.unnormalize_identifier(value) # revert to original value without backticks + return f"{camelcase(key)} => '{value}'" + + opts = {catalog_key: catalog, **(asdict(options) if options else {})} + encoded = [encode(k, v) for k, v in opts.items()] + + return ", ".join(encoded) diff --git a/src/databricks/labs/lakebridge/reconcile/connectors/secrets.py b/src/databricks/labs/lakebridge/reconcile/connectors/secrets.py deleted file mode 100644 index daa213afc8..0000000000 --- a/src/databricks/labs/lakebridge/reconcile/connectors/secrets.py +++ /dev/null @@ -1,49 +0,0 @@ -import base64 -import logging - -from databricks.sdk import WorkspaceClient -from databricks.sdk.errors import NotFound - -logger = logging.getLogger(__name__) - - -# TODO use CredentialManager to allow for changing secret provider for tests -class SecretsMixin: - _ws: WorkspaceClient - _secret_scope: str - - def _get_secret_or_none(self, secret_key: str) -> str | None: - """ - Get the secret value given a secret scope & secret key. Log a warning if secret does not exist - Used To ensure backwards compatibility when supporting new secrets - """ - try: - # Return the decoded secret value in string format - return self._get_secret(secret_key) - except NotFound as e: - logger.warning(f"Secret not found: key={secret_key}") - logger.debug("Secret lookup failed", exc_info=e) - return None - - def _get_secret(self, secret_key: str) -> str: - """Get the secret value given a secret scope & secret key. - - Raises: - NotFound: The secret could not be found. - UnicodeDecodeError: The secret value was not Base64-encoded UTF-8. - """ - try: - # Return the decoded secret value in string format - secret = self._ws.secrets.get_secret(self._secret_scope, secret_key) - assert secret.value is not None - return base64.b64decode(secret.value).decode("utf-8") - except NotFound as e: - raise NotFound(f'Secret does not exist with scope: {self._secret_scope} and key: {secret_key} : {e}') from e - except UnicodeDecodeError as e: - raise UnicodeDecodeError( - "utf-8", - secret_key.encode(), - 0, - 1, - f"Secret {self._secret_scope}/{secret_key} has Base64 bytes that cannot be decoded to utf-8 string: {e}.", - ) from e diff --git a/src/databricks/labs/lakebridge/reconcile/connectors/snowflake.py b/src/databricks/labs/lakebridge/reconcile/connectors/snowflake.py index e66751d29b..5333a9e3fa 100644 --- a/src/databricks/labs/lakebridge/reconcile/connectors/snowflake.py +++ b/src/databricks/labs/lakebridge/reconcile/connectors/snowflake.py @@ -3,26 +3,20 @@ from datetime import datetime from pyspark.errors import PySparkException -from pyspark.sql import DataFrame, DataFrameReader, SparkSession +from pyspark.sql import DataFrame from pyspark.sql.functions import col from sqlglot import Dialect -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import serialization from databricks.labs.lakebridge.reconcile.connectors.data_source import DataSource -from databricks.labs.lakebridge.reconcile.connectors.jdbc_reader import JDBCReaderMixin from databricks.labs.lakebridge.reconcile.connectors.models import NormalizedIdentifier -from databricks.labs.lakebridge.reconcile.connectors.secrets import SecretsMixin +from databricks.labs.lakebridge.reconcile.connectors.remote_query_reader import RemoteQueryReader from databricks.labs.lakebridge.reconcile.connectors.dialect_utils import DialectUtils -from databricks.labs.lakebridge.reconcile.exception import InvalidSnowflakePemPrivateKey from databricks.labs.lakebridge.reconcile.recon_config import JdbcReaderOptions, Schema -from databricks.sdk import WorkspaceClient -from databricks.sdk.errors import NotFound logger = logging.getLogger(__name__) -class SnowflakeDataSource(DataSource, SecretsMixin, JDBCReaderMixin): +class SnowflakeDataSource(DataSource): _DRIVER = "snowflake" _IDENTIFIER_DELIMITER = "\"" @@ -54,34 +48,14 @@ class SnowflakeDataSource(DataSource, SecretsMixin, JDBCReaderMixin): def __init__( self, engine: Dialect, - spark: SparkSession, - ws: WorkspaceClient, - secret_scope: str, + reader: RemoteQueryReader, ): self._engine = engine - self._spark = spark - self._ws = ws - self._secret_scope = secret_scope - - @property - def get_jdbc_url(self) -> str: - try: - sf_password = self._get_secret('sfPassword') - except (NotFound, KeyError) as e: - message = "sfPassword is mandatory for jdbc connectivity with Snowflake." - logger.error(message) - raise NotFound(message) from e - - return ( - f"jdbc:{SnowflakeDataSource._DRIVER}://{self._get_secret('sfAccount')}.snowflakecomputing.com" - f"/?user={self._get_secret('sfUser')}&password={sf_password}" - f"&db={self._get_secret('sfDatabase')}&schema={self._get_secret('sfSchema')}" - f"&warehouse={self._get_secret('sfWarehouse')}&role={self._get_secret('sfRole')}" - ) + self._reader = reader def read_data( self, - catalog: str | None, + catalog: str, schema: str, table: str, query: str, @@ -89,22 +63,15 @@ def read_data( ) -> DataFrame: table_query = query.replace(":tbl", f"{catalog}.{schema}.{table}") try: - if options is None: - df = self.reader(table_query).load() - else: - options = self._get_jdbc_reader_options(options) - df = ( - self._get_jdbc_reader(table_query, self.get_jdbc_url, SnowflakeDataSource._DRIVER) - .options(**options) - .load() - ) + logger.info(f"Fetching data using query: \n`{table_query}`") + df = self._reader.read_data(table_query, catalog, "database", "query") return df.select([col(column).alias(column.lower()) for column in df.columns]) except (RuntimeError, PySparkException) as e: return self.log_and_throw_exception(e, "data", table_query) def get_schema( self, - catalog: str | None, + catalog: str, schema: str, table: str, normalize: bool = True, @@ -124,80 +91,13 @@ def get_schema( try: logger.debug(f"Fetching schema using query: \n`{schema_query}`") logger.info(f"Fetching Schema: Started at: {datetime.now()}") - df = self.reader(schema_query).load() + df = self._reader.read_data(schema_query, catalog, "database", "query") schema_metadata = df.select([col(c).alias(c.lower()) for c in df.columns]).collect() logger.info(f"Schema fetched successfully. Completed at: {datetime.now()}") return [self._map_meta_column(field, normalize) for field in schema_metadata] except (RuntimeError, PySparkException) as e: return self.log_and_throw_exception(e, "schema", schema_query) - def reader(self, query: str) -> DataFrameReader: - options = self._get_snowflake_options() - return self._spark.read.format("snowflake").option("dbtable", f"({query}) as tmp").options(**options) - - # TODO cache this method using @functools.cache - # Pay attention to https://pylint.pycqa.org/en/latest/user_guide/messages/warning/method-cache-max-size-none.html - def _get_snowflake_options(self): - options = { - "sfUrl": self._get_secret('sfUrl'), - "sfUser": self._get_secret('sfUser'), - "sfDatabase": self._get_secret('sfDatabase'), - "sfSchema": self._get_secret('sfSchema'), - "sfWarehouse": self._get_secret('sfWarehouse'), - "sfRole": self._get_secret('sfRole'), - } - options = options | self._get_snowflake_auth_options() - - return options - - def _get_snowflake_auth_options(self): - try: - key = SnowflakeDataSource._get_private_key( - self._get_secret('pem_private_key'), self._get_secret_or_none('pem_private_key_password') - ) - return {"pem_private_key": key} - except (NotFound, KeyError): - logger.warning("pem_private_key not found. Checking for sfPassword") - try: - password = self._get_secret('sfPassword') - return {"sfPassword": password} - except (NotFound, KeyError) as e: - message = "sfPassword and pem_private_key not found. Either one is required for snowflake auth." - logger.error(message) - raise NotFound(message) from e - - @staticmethod - def _get_private_key(pem_private_key: str, pem_private_key_password: str | None) -> str: - try: - private_key_bytes = pem_private_key.encode("UTF-8") - password_bytes = pem_private_key_password.encode("UTF-8") if pem_private_key_password else None - except UnicodeEncodeError as e: - message = f"Invalid pem key and/or pem password: unable to encode. --> {e}" - logger.error(message) - raise ValueError(message) from e - - try: - p_key = serialization.load_pem_private_key( - private_key_bytes, - password_bytes, - backend=default_backend(), - ) - pkb = p_key.private_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PrivateFormat.PKCS8, - encryption_algorithm=serialization.NoEncryption(), - ) - pkb_str = pkb.decode("UTF-8") - # Remove the first and last lines (BEGIN/END markers) - private_key_pem_lines = pkb_str.strip().split('\n')[1:-1] - # Join the lines to form the base64 encoded string - private_key_pem_str = ''.join(private_key_pem_lines) - return private_key_pem_str - except Exception as e: - message = f"Failed to load or process the provided PEM private key. --> {e}" - logger.error(message) - raise InvalidSnowflakePemPrivateKey(message) from e - def normalize_identifier(self, identifier: str) -> NormalizedIdentifier: normalized = DialectUtils.normalize_identifier( identifier, diff --git a/src/databricks/labs/lakebridge/reconcile/connectors/source_adapter.py b/src/databricks/labs/lakebridge/reconcile/connectors/source_adapter.py index b8c7af2c92..932a8f0399 100644 --- a/src/databricks/labs/lakebridge/reconcile/connectors/source_adapter.py +++ b/src/databricks/labs/lakebridge/reconcile/connectors/source_adapter.py @@ -4,6 +4,7 @@ from databricks.labs.lakebridge.reconcile.connectors.data_source import DataSource from databricks.labs.lakebridge.reconcile.connectors.databricks import DatabricksDataSource from databricks.labs.lakebridge.reconcile.connectors.oracle import OracleDataSource +from databricks.labs.lakebridge.reconcile.connectors.remote_query_reader import RemoteQueryReader from databricks.labs.lakebridge.reconcile.connectors.snowflake import SnowflakeDataSource from databricks.labs.lakebridge.reconcile.connectors.tsql import TSQLServerDataSource from databricks.labs.lakebridge.transpiler.sqlglot.generator.databricks import Databricks @@ -13,18 +14,20 @@ from databricks.sdk import WorkspaceClient +# TODO add checks connection exists def create_adapter( engine: Dialect, spark: SparkSession, ws: WorkspaceClient, - secret_scope: str, + connection_name: str, ) -> DataSource: + reader = RemoteQueryReader(spark, connection_name) if isinstance(engine, Snowflake): - return SnowflakeDataSource(engine, spark, ws, secret_scope) + return SnowflakeDataSource(engine, reader) if isinstance(engine, Oracle): - return OracleDataSource(engine, spark, ws, secret_scope) + return OracleDataSource(engine, reader) if isinstance(engine, Databricks): - return DatabricksDataSource(engine, spark, ws, secret_scope) + return DatabricksDataSource(engine, spark, ws) if isinstance(engine, Tsql): - return TSQLServerDataSource(engine, spark, ws, secret_scope) + return TSQLServerDataSource(engine, reader) raise ValueError(f"Unsupported source type --> {engine}") diff --git a/src/databricks/labs/lakebridge/reconcile/connectors/tsql.py b/src/databricks/labs/lakebridge/reconcile/connectors/tsql.py index 3b3394441a..62b7e3f8fe 100644 --- a/src/databricks/labs/lakebridge/reconcile/connectors/tsql.py +++ b/src/databricks/labs/lakebridge/reconcile/connectors/tsql.py @@ -1,20 +1,17 @@ import re import logging from datetime import datetime -from collections.abc import Mapping from pyspark.errors import PySparkException -from pyspark.sql import DataFrame, DataFrameReader, SparkSession +from pyspark.sql import DataFrame from pyspark.sql.functions import col from sqlglot import Dialect from databricks.labs.lakebridge.reconcile.connectors.data_source import DataSource -from databricks.labs.lakebridge.reconcile.connectors.jdbc_reader import JDBCReaderMixin from databricks.labs.lakebridge.reconcile.connectors.models import NormalizedIdentifier -from databricks.labs.lakebridge.reconcile.connectors.secrets import SecretsMixin +from databricks.labs.lakebridge.reconcile.connectors.remote_query_reader import RemoteQueryReader from databricks.labs.lakebridge.reconcile.connectors.dialect_utils import DialectUtils -from databricks.labs.lakebridge.reconcile.recon_config import JdbcReaderOptions, Schema, OptionalPrimitiveType -from databricks.sdk import WorkspaceClient +from databricks.labs.lakebridge.reconcile.recon_config import JdbcReaderOptions, Schema logger = logging.getLogger(__name__) @@ -50,62 +47,36 @@ """ -class TSQLServerDataSource(DataSource, SecretsMixin, JDBCReaderMixin): +class TSQLServerDataSource(DataSource): _DRIVER = "sqlserver" _IDENTIFIER_DELIMITER = {"prefix": "[", "suffix": "]"} def __init__( self, engine: Dialect, - spark: SparkSession, - ws: WorkspaceClient, - secret_scope: str, + reader: RemoteQueryReader, ): self._engine = engine - self._spark = spark - self._ws = ws - self._secret_scope = secret_scope - - @property - def get_jdbc_url(self) -> str: - # Construct the JDBC URL - return ( - f"jdbc:{self._DRIVER}://{self._get_secret('host')}:{self._get_secret('port')};" - f"databaseName={self._get_secret('database')};" - f"encrypt={self._get_secret('encrypt')};" - f"trustServerCertificate={self._get_secret('trustServerCertificate')};" - ) + self._reader = reader def read_data( self, - catalog: str | None, + catalog: str, schema: str, table: str, query: str, options: JdbcReaderOptions | None, ) -> DataFrame: - table_query = query.replace(":tbl", f"{catalog}.{schema}.{self.normalize_identifier(table).source_normalized}") - with_clause_pattern = re.compile(r'WITH\s+.*?\)\s*(?=SELECT)', re.IGNORECASE | re.DOTALL) - match = with_clause_pattern.search(table_query) - if match: - prepare_query_string = match.group(0) - query = table_query.replace(match.group(0), '') - else: - query = table_query - prepare_query_string = "" + table_query = query.replace(":tbl", f"{schema}.{self.normalize_identifier(table).source_normalized}") try: - if options is None: - df = self.reader(query, {"prepareQuery": prepare_query_string}).load() - else: - spark_options = self._get_jdbc_reader_options(options) - df = self.reader(table_query, spark_options).load() + df = self._reader.read_data(table_query, catalog, "database", "query", options) return df.select([col(column).alias(column.lower()) for column in df.columns]) except (RuntimeError, PySparkException) as e: return self.log_and_throw_exception(e, "data", table_query) def get_schema( self, - catalog: str | None, + catalog: str, schema: str, table: str, normalize: bool = True, @@ -125,26 +96,13 @@ def get_schema( try: logger.debug(f"Fetching schema using query: \n`{schema_query}`") logger.info(f"Fetching Schema: Started at: {datetime.now()}") - df = self.reader(schema_query).load() + df = self._reader.read_data(schema_query, catalog, "database", "query") schema_metadata = df.select([col(c).alias(c.lower()) for c in df.columns]).collect() logger.info(f"Schema fetched successfully. Completed at: {datetime.now()}") return [self._map_meta_column(field, normalize) for field in schema_metadata] except (RuntimeError, PySparkException) as e: return self.log_and_throw_exception(e, "schema", schema_query) - def reader(self, query: str, options: Mapping[str, OptionalPrimitiveType] | None = None) -> DataFrameReader: - if options is None: - options = {} - - creds = self._get_user_password() - return self._get_jdbc_reader(query, self.get_jdbc_url, self._DRIVER, {**options, **creds}) - - def _get_user_password(self) -> Mapping[str, str]: - return { - "user": self._get_secret("user"), - "password": self._get_secret("password"), - } - def normalize_identifier(self, identifier: str) -> NormalizedIdentifier: return DialectUtils.normalize_identifier( TSQLServerDataSource._normalize_quotes(identifier), diff --git a/src/databricks/labs/lakebridge/reconcile/execute.py b/src/databricks/labs/lakebridge/reconcile/execute.py index 448df0e30b..39b7d42c83 100644 --- a/src/databricks/labs/lakebridge/reconcile/execute.py +++ b/src/databricks/labs/lakebridge/reconcile/execute.py @@ -44,12 +44,10 @@ def main(*argv: str) -> None: reconcile_config = installation.load(ReconcileConfig) - catalog_or_schema = ( - reconcile_config.database_config.source_catalog - if reconcile_config.database_config.source_catalog - else reconcile_config.database_config.source_schema + connection_or_catalog = reconcile_config.source.uc_connection_name or reconcile_config.source.catalog + filename = ( + f"recon_config_{reconcile_config.source.dialect}_{connection_or_catalog}_{reconcile_config.report_type}.json" ) - filename = f"recon_config_{reconcile_config.data_source}_{catalog_or_schema}_{reconcile_config.report_type}.json" logger.info(f"Loading {filename} from Databricks Workspace...") diff --git a/src/databricks/labs/lakebridge/reconcile/normalize_recon_config_service.py b/src/databricks/labs/lakebridge/reconcile/normalize_recon_config_service.py index 257a8b9bb3..1d17b94f76 100644 --- a/src/databricks/labs/lakebridge/reconcile/normalize_recon_config_service.py +++ b/src/databricks/labs/lakebridge/reconcile/normalize_recon_config_service.py @@ -123,11 +123,7 @@ def _normalize_col_threshold(self, threshold: ColumnThresholds): def _normalize_jdbc_options(self, table: Table): if table.jdbc_reader_options: normalized = dataclasses.replace(table.jdbc_reader_options) - normalized.partition_column = ( - self.source.normalize_identifier(normalized.partition_column).ansi_normalized - if normalized.partition_column - else None - ) + normalized.partition_column = self.source.normalize_identifier(normalized.partition_column).ansi_normalized table.jdbc_reader_options = normalized return table diff --git a/src/databricks/labs/lakebridge/reconcile/query_builder/sampling_query.py b/src/databricks/labs/lakebridge/reconcile/query_builder/sampling_query.py index 8ed27b33f7..00adc1d50f 100644 --- a/src/databricks/labs/lakebridge/reconcile/query_builder/sampling_query.py +++ b/src/databricks/labs/lakebridge/reconcile/query_builder/sampling_query.py @@ -3,6 +3,7 @@ import sqlglot.expressions as exp from pyspark.sql import DataFrame from sqlglot import select +from sqlglot.dialects.tsql import TSQL from databricks.labs.lakebridge.reconcile.connectors.dialect_utils import DialectUtils from databricks.labs.lakebridge.transpiler.sqlglot.dialect_utils import get_key_from_dialect @@ -53,7 +54,6 @@ def build_query(self, df: DataFrame): else: key_cols = sorted(self.table_conf.get_tgt_to_src_col_mapping_list(join_columns)) keys_df = df.select(*key_cols) - with_clause = self._get_with_clause(keys_df) cols = sorted((join_columns | self.select_columns) - self.threshold_columns - self.drop_columns) @@ -76,15 +76,29 @@ def build_query(self, df: DataFrame): for col in sorted(self.table_conf.get_tgt_to_src_col_mapping_list(cols)) ] - join_clause = self._get_join_clause(key_cols) + if isinstance(self.engine, TSQL): + # T-SQL rejects `WITH cte AS (...)` inside a derived table, and Spark JDBC wraps the + # query as `SELECT * FROM (...) WHERE 1=0` for output-schema resolution. Emit two + # derived tables joined directly so the wrapper produces valid T-SQL. + recon_subquery = self._get_recon_values_subquery(keys_df, key_cols) + on_expr = self._get_join_clause(key_cols).args["on"] + query = ( + select(*with_select) + .from_(query_sql.subquery(alias="src")) + .join(recon_subquery, on=on_expr, join_type="inner") + .sql(dialect=self.engine) + ) + else: + with_clause = self._get_with_clause(keys_df) + join_clause = self._get_join_clause(key_cols) + query = ( + with_clause.with_(alias="src", as_=query_sql) + .select(*with_select) + .from_("src") + .join(join_clause) + .sql(dialect=self.engine) + ) - query = ( - with_clause.with_(alias="src", as_=query_sql) - .select(*with_select) - .from_("src") - .join(join_clause) - .sql(dialect=self.engine) - ) logger.info(f"Sampling Query for {self.layer}: {query}") return query @@ -128,3 +142,36 @@ def _get_with_clause(self, df: DataFrame) -> exp.Select: union_res.append(select(*row_select)) union_statements = _union_concat(union_res, union_res[0], 0) return exp.Select().with_(alias='recon', as_=union_statements) + + def _get_recon_values_subquery(self, df: DataFrame, key_cols: list[str]) -> exp.Subquery: + column_types_dict = {str(f.name).lower(): f.dataType for f in df.schema.fields} + orig_types_dict = { + schema.column_name: schema.data_type + for schema in self.schema + if schema.column_name not in self.user_transformations + } + tuples: list[exp.Expression] = [] + for row in df.collect(): + row_values: list[exp.Expression] = [] + for col, value in zip(df.columns, row): + if value is not None: + row_values.append( + build_literal( + this=str(value), + is_string=_get_is_string(column_types_dict, col), + cast=orig_types_dict.get(DialectUtils.ansi_normalize_identifier(col)), + quoted=True and self._is_add_quotes, + ) + ) + else: + row_values.append(exp.Null()) + tuples.append(exp.Tuple(expressions=row_values)) + + column_idents = [ + exp.to_identifier(DialectUtils.unnormalize_identifier(col), quoted=True and self._is_add_quotes) + for col in key_cols + ] + return exp.Subquery( + this=exp.Values(expressions=tuples), + alias=exp.TableAlias(this=exp.to_identifier("recon"), columns=column_idents), + ) diff --git a/src/databricks/labs/lakebridge/reconcile/recon_capture.py b/src/databricks/labs/lakebridge/reconcile/recon_capture.py index 2d665fa12d..4ef602c3da 100644 --- a/src/databricks/labs/lakebridge/reconcile/recon_capture.py +++ b/src/databricks/labs/lakebridge/reconcile/recon_capture.py @@ -75,7 +75,10 @@ def base_dir(self) -> Path: @cached_property def is_serverless(self) -> bool: - is_serverless = os.getenv("IS_SERVERLESS", "").lower() == "true" + is_serverless = ( + os.getenv("IS_SERVERLESS", "").lower() == "true" + or os.getenv("DATABRICKS_SERVERLESS_COMPUTE_ID", "").lower() == "auto" + ) return is_serverless @property diff --git a/src/databricks/labs/lakebridge/reconcile/recon_config.py b/src/databricks/labs/lakebridge/reconcile/recon_config.py index 9879a0dc24..8cf9201582 100644 --- a/src/databricks/labs/lakebridge/reconcile/recon_config.py +++ b/src/databricks/labs/lakebridge/reconcile/recon_config.py @@ -82,12 +82,12 @@ def __post_init__(self): @dataclass -class JdbcReaderOptions: - number_partitions: int | None = None - partition_column: str | None = None - lower_bound: str | None = None - upper_bound: str | None = None - fetch_size: int = 100 +class JdbcReaderOptions: # This class follows the same naming as db `remote_query` options + partition_column: str # if used, other props here have to be set as well + num_partitions: int + lower_bound: str + upper_bound: str + fetchsize: int = 0 # this uses driver default def __post_init__(self): self.partition_column = self.partition_column.lower() if self.partition_column else None diff --git a/src/databricks/labs/lakebridge/reconcile/trigger_recon_service.py b/src/databricks/labs/lakebridge/reconcile/trigger_recon_service.py index 3438cea994..7eb79e59d3 100644 --- a/src/databricks/labs/lakebridge/reconcile/trigger_recon_service.py +++ b/src/databricks/labs/lakebridge/reconcile/trigger_recon_service.py @@ -71,14 +71,16 @@ def create_recon_dependencies( # validate the report type report_type = reconcile_config.report_type.lower() - logger.info(f"report_type: {report_type}, data_source: {reconcile_config.data_source} ") + source_dialect = reconcile_config.source.dialect + logger.info(f"report_type: {report_type}, data_source: {source_dialect} ") utils.validate_input(report_type, _RECON_REPORT_TYPES, "Invalid report type") + # validate the connection source, target = utils.initialise_data_source( - engine=reconcile_config.data_source, + source_dialect=reconcile_config.source.dialect, spark=spark, ws=ws_client, - secret_scope=reconcile_config.secret_scope, + connection_name=reconcile_config.source.uc_connection_name, ) recon_id = uuid4().hex @@ -89,7 +91,7 @@ def create_recon_dependencies( reconcile_config.database_config, report_type, SchemaCompare(spark=spark), - get_dialect(reconcile_config.data_source), + get_dialect(source_dialect), spark, metadata_config=reconcile_config.metadata_config, intermediate_persist=ReconIntermediatePersist(spark, reconcile_config.metadata_config), @@ -99,7 +101,7 @@ def create_recon_dependencies( database_config=reconcile_config.database_config, recon_id=recon_id, report_type=report_type, - source_dialect=get_dialect(reconcile_config.data_source), + source_dialect=get_dialect(source_dialect), ws=ws_client, spark=spark, metadata_config=reconcile_config.metadata_config, diff --git a/src/databricks/labs/lakebridge/reconcile/utils.py b/src/databricks/labs/lakebridge/reconcile/utils.py index 5c4c268378..73b5cd8931 100644 --- a/src/databricks/labs/lakebridge/reconcile/utils.py +++ b/src/databricks/labs/lakebridge/reconcile/utils.py @@ -14,11 +14,16 @@ def initialise_data_source( ws: WorkspaceClient, spark: SparkSession, - engine: str, - secret_scope: str, + source_dialect: str, + connection_name: str | None, ): - source = create_adapter(engine=get_dialect(engine), spark=spark, ws=ws, secret_scope=secret_scope) - target = create_adapter(engine=get_dialect("databricks"), spark=spark, ws=ws, secret_scope=secret_scope) + if not connection_name: + validate_input(source_dialect, {"databricks"}, "Please configure connection name") + source = create_adapter(engine=get_dialect("databricks"), spark=spark, ws=ws, connection_name="databricks") + else: + source = create_adapter(engine=get_dialect(source_dialect), spark=spark, ws=ws, connection_name=connection_name) + + target = create_adapter(engine=get_dialect("databricks"), spark=spark, ws=ws, connection_name="databricks") return source, target diff --git a/tests/conftest.py b/tests/conftest.py index c9acef9a09..742adaed87 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -81,7 +81,7 @@ def table_conf_with_opts(column_mapping): source_name="supplier", target_name="target_supplier", jdbc_reader_options=JdbcReaderOptions( - number_partitions=100, partition_column="s_nationkey", lower_bound="0", upper_bound="100" + num_partitions=100, partition_column="s_nationkey", lower_bound="0", upper_bound="100" ), join_columns=["s_suppkey", "s_nationkey"], select_columns=["s_suppkey", "s_name", "s_address", "s_phone", "s_acctbal", "s_nationkey"], diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 4e1437eadb..b5d662f89d 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -23,6 +23,25 @@ logger = logging.getLogger(__name__) +@pytest.fixture(scope="session") +def test_env() -> TestEnvGetter: + """Reusable :class:`TestEnvGetter` for reading values from ``~/.databricks/debug-env.json``.""" + return TestEnvGetter(True) + + +@pytest.fixture(autouse=True) +def remap_cluster_id_to_dqx(monkeypatch, debug_env) -> None: + """Point the Databricks SDK at the DQX cluster used by reconcile integration tests. + + Depends on ``debug_env`` so that the ``.env``-loaded value of + ``DATABRICKS_DQX_CLUSTER_ID`` is visible in ``os.environ`` before we read it. + Uses ``monkeypatch.setenv`` so the substitution is reverted after each test. + """ + dqx_cluster_id = debug_env.get("DATABRICKS_DQX_CLUSTER_ID") + if dqx_cluster_id: + monkeypatch.setenv("DATABRICKS_CLUSTER_ID", dqx_cluster_id) + + class MockApplicationContext(ApplicationContext): """A mock application context that uses a unique installation path.""" diff --git a/tests/integration/reconcile/conftest.py b/tests/integration/reconcile/conftest.py index 61e785ed2a..4e9b7c49b3 100644 --- a/tests/integration/reconcile/conftest.py +++ b/tests/integration/reconcile/conftest.py @@ -14,20 +14,19 @@ from databricks.sdk import WorkspaceClient from databricks.sdk.errors.platform import PermissionDenied from databricks.sdk.service.catalog import TableInfo, SchemaInfo -from databricks.sdk.service.compute import DataSecurityMode, Kind, ClusterDetails from databricks.labs.lakebridge.config import ( - DatabaseConfig, LakebridgeConfiguration, ReconcileConfig, ReconcileJobConfig, ReconcileMetadataConfig, + SourceConnectionConfig, + TargetConnectionConfig, TableRecon, ) from databricks.labs.lakebridge.contexts.application import ApplicationContext from databricks.labs.lakebridge.reconcile.recon_capture import AbstractReconIntermediatePersist from databricks.labs.lakebridge.reconcile.recon_config import Table -from tests.integration.debug_envgetter import TestEnvGetter logger = logging.getLogger(__name__) @@ -48,11 +47,13 @@ (0.31, 'Good', 'J', 'SI2', '2000-01-01'); \ """ +TSQL_CONNECTION = "sqlserver_sandbox" TSQL_CATALOG = "labs_azure_sandbox_remorph" TSQL_SCHEMA = "dbo" -TSQL_TABLE = "diamonds_big_column" -SNOWFLAKE_CATALOG = "REMORPH" -SNOWFLAKE_SCHEMA = "SANDBOX" +TSQL_TABLE = "diamonds" +SNOWFLAKE_CONNECTION = "sf_sandbox" +SNOWFLAKE_CATALOG = "INTEGRATION" +SNOWFLAKE_SCHEMA = "LAKEBRIDGE" SNOWFLAKE_TABLE = "DIAMONDS" @@ -77,7 +78,7 @@ def recon_schema(recon_catalog, make_schema) -> SchemaInfo: @pytest.fixture -def recon_tables(ws: WorkspaceClient, recon_schema: SchemaInfo, make_table) -> tuple[TableInfo, TableInfo]: +def recon_tables(ws: WorkspaceClient, recon_schema: SchemaInfo, make_table, test_env) -> tuple[TableInfo, TableInfo]: src_table = make_table( catalog_name=recon_schema.catalog_name, schema_name=recon_schema.name, columns=DIAMONDS_COLUMNS ) @@ -86,7 +87,6 @@ def recon_tables(ws: WorkspaceClient, recon_schema: SchemaInfo, make_table) -> t ) logger.info(f"Created recon tables {src_table.name}, {tgt_table.name} in schema {recon_schema.name}") - test_env = TestEnvGetter(True) warehouse = test_env.get("TEST_DEFAULT_WAREHOUSE_ID") for tbl in (src_table, tgt_table): @@ -171,20 +171,16 @@ def snowflake_recon_table_config(recon_schema: SchemaInfo, recon_tables: tuple[T @pytest.fixture -def recon_cluster(make_cluster) -> ClusterDetails: - return make_cluster( - data_security_mode=DataSecurityMode.DATA_SECURITY_MODE_AUTO, - kind=Kind.CLASSIC_PREVIEW, - num_workers=2, - ).result() +def recon_cluster(test_env) -> str: + return test_env.get("DATABRICKS_DQX_CLUSTER_ID") @pytest.fixture def databricks_recon_config( - recon_cluster: ClusterDetails, recon_schema: SchemaInfo, recon_metadata: ReconcileMetadataConfig + recon_cluster: str, recon_schema: SchemaInfo, recon_metadata: ReconcileMetadataConfig ) -> ReconcileConfig: deployment_overrides = ReconcileJobConfig( - existing_cluster_id=recon_cluster.cluster_id or "bogus", + existing_cluster_id=recon_cluster, tags={"lakebridge": "reconcile_test"}, ) logger.info(f"Using recon job overrides: {deployment_overrides}") @@ -192,14 +188,15 @@ def databricks_recon_config( assert recon_schema.catalog_name assert recon_schema.name return ReconcileConfig( - data_source="databricks", report_type="all", - secret_scope="NOT_NEEDED", - database_config=DatabaseConfig( - source_catalog=recon_schema.catalog_name, - source_schema=recon_schema.name, - target_catalog=recon_schema.catalog_name, - target_schema=recon_schema.name, + source=SourceConnectionConfig( + dialect="databricks", + catalog=recon_schema.catalog_name, + schema=recon_schema.name, + ), + target=TargetConnectionConfig( + catalog=recon_schema.catalog_name, + schema=recon_schema.name, ), metadata_config=recon_metadata, job_overrides=deployment_overrides, @@ -207,11 +204,11 @@ def databricks_recon_config( @pytest.fixture -def tsql_recon_config(recon_cluster: ClusterDetails, recon_schema: SchemaInfo, make_volume) -> ReconcileConfig: +def tsql_recon_config(recon_cluster: str, recon_schema: SchemaInfo, make_volume) -> ReconcileConfig: volume = make_volume(catalog_name=recon_schema.catalog_name, schema_name=recon_schema.name, name=recon_schema.name) deployment_overrides = ReconcileJobConfig( - existing_cluster_id=recon_cluster.cluster_id or "bogus", + existing_cluster_id=recon_cluster, tags={"lakebridge": "reconcile_test"}, ) logger.info(f"Using recon job overrides: {deployment_overrides}") @@ -219,14 +216,16 @@ def tsql_recon_config(recon_cluster: ClusterDetails, recon_schema: SchemaInfo, m assert recon_schema.catalog_name assert recon_schema.name return ReconcileConfig( - data_source="tsql", - report_type="row", - secret_scope="labs_azure_sandbox_sql_server_secrets", - database_config=DatabaseConfig( - source_catalog=TSQL_CATALOG, - source_schema=TSQL_SCHEMA, - target_catalog=recon_schema.catalog_name, - target_schema=recon_schema.name, + report_type="all", + source=SourceConnectionConfig( + dialect="tsql", + catalog=TSQL_CATALOG, + schema=TSQL_SCHEMA, + uc_connection_name=TSQL_CONNECTION, + ), + target=TargetConnectionConfig( + catalog=recon_schema.catalog_name, + schema=recon_schema.name, ), metadata_config=ReconcileMetadataConfig( catalog=recon_schema.catalog_name, schema=recon_schema.name, volume=volume.name @@ -236,11 +235,11 @@ def tsql_recon_config(recon_cluster: ClusterDetails, recon_schema: SchemaInfo, m @pytest.fixture -def snowflake_recon_config(recon_cluster: ClusterDetails, recon_schema: SchemaInfo, make_volume) -> ReconcileConfig: +def snowflake_recon_config(recon_cluster: str, recon_schema: SchemaInfo, make_volume) -> ReconcileConfig: volume = make_volume(catalog_name=recon_schema.catalog_name, schema_name=recon_schema.name, name=recon_schema.name) deployment_overrides = ReconcileJobConfig( - existing_cluster_id=recon_cluster.cluster_id or "bogus", + existing_cluster_id=recon_cluster, tags={"lakebridge": "reconcile_test"}, ) logger.info(f"Using recon job overrides: {deployment_overrides}") @@ -248,14 +247,16 @@ def snowflake_recon_config(recon_cluster: ClusterDetails, recon_schema: SchemaIn assert recon_schema.catalog_name assert recon_schema.name return ReconcileConfig( - data_source="snowflake", report_type="all", - secret_scope="labs_snowflake_sandbox_secrets", - database_config=DatabaseConfig( - source_catalog=SNOWFLAKE_CATALOG, - source_schema=SNOWFLAKE_SCHEMA, - target_catalog=recon_schema.catalog_name, - target_schema=recon_schema.name, + source=SourceConnectionConfig( + dialect="snowflake", + catalog=SNOWFLAKE_CATALOG, + schema=SNOWFLAKE_SCHEMA, + uc_connection_name=SNOWFLAKE_CONNECTION, + ), + target=TargetConnectionConfig( + catalog=recon_schema.catalog_name, + schema=recon_schema.name, ), metadata_config=ReconcileMetadataConfig( catalog=recon_schema.catalog_name, schema=recon_schema.name, volume=volume.name @@ -265,12 +266,8 @@ def snowflake_recon_config(recon_cluster: ClusterDetails, recon_schema: SchemaIn def recon_config_filename(recon_config: ReconcileConfig) -> str: - source_catalog_or_schema = ( - recon_config.database_config.source_catalog - if recon_config.database_config.source_catalog - else recon_config.database_config.source_schema - ) - return f"recon_config_{recon_config.data_source}_{source_catalog_or_schema}_{recon_config.report_type}.json" + connection_or_catalog = recon_config.source.uc_connection_name or recon_config.source.catalog + return f"recon_config_{recon_config.source.dialect}_{connection_or_catalog}_{recon_config.report_type}.json" @contextmanager @@ -305,7 +302,9 @@ def base_dir(self) -> Path: @property def is_serverless(self) -> bool: - return False + return True + # treat everything as serverless to avoid using cache completely. + # the fallback to cache is stubbed as well def write_and_read_df_with_volumes( self, diff --git a/tests/integration/reconcile/connectors/test_read_schema.py b/tests/integration/reconcile/connectors/test_read_schema.py index 7fa6e722f1..2e659a46dc 100644 --- a/tests/integration/reconcile/connectors/test_read_schema.py +++ b/tests/integration/reconcile/connectors/test_read_schema.py @@ -1,89 +1,23 @@ -from collections.abc import Mapping from unittest.mock import create_autospec import uuid import pytest -from pyspark.sql import DataFrameReader, SparkSession - +from pyspark.sql import SparkSession +from databricks.sdk import WorkspaceClient from databricks.sdk.service.catalog import TableInfo from databricks.labs.lakebridge.reconcile.connectors.databricks import DatabricksDataSource -from databricks.labs.lakebridge.reconcile.connectors.oracle import OracleDataSource +from databricks.labs.lakebridge.reconcile.connectors.remote_query_reader import RemoteQueryReader from databricks.labs.lakebridge.reconcile.connectors.snowflake import SnowflakeDataSource from databricks.labs.lakebridge.reconcile.connectors.tsql import TSQLServerDataSource -from databricks.labs.lakebridge.reconcile.recon_config import OptionalPrimitiveType from databricks.labs.lakebridge.transpiler.sqlglot.dialect_utils import get_dialect - -from databricks.sdk import WorkspaceClient - -from tests.integration.debug_envgetter import TestEnvGetter, parse_snowflake_jdbc_url - - -class TSQLServerDataSourceUnderTest(TSQLServerDataSource): - def __init__(self, spark, ws): - super().__init__(get_dialect("tsql"), spark, ws, "secret_scope") - self._test_env = TestEnvGetter(True) - - @property - def get_jdbc_url(self) -> str: - return self._test_env.get("TEST_TSQL_JDBC") - - def _get_user_password(self) -> dict: - user = self._test_env.get("TEST_TSQL_USER") - password = self._test_env.get("TEST_TSQL_PASS") - return {"user": user, "password": password} - - -class OracleDataSourceUnderTest(OracleDataSource): - def __init__(self, spark, ws): - super().__init__(get_dialect("oracle"), spark, ws, "secret_scope") - self._test_env = TestEnvGetter(False) - - @property - def get_jdbc_url(self) -> str: - return self._test_env.get("TEST_ORACLE_JDBC") - - def reader(self, query: str, options: Mapping[str, OptionalPrimitiveType] | None = None) -> DataFrameReader: - if options is None: - options = {} - user = self._test_env.get("TEST_ORACLE_USER") - password = self._test_env.get("TEST_ORACLE_PASSWORD") - return self._get_jdbc_reader( - query, self.get_jdbc_url, OracleDataSource._DRIVER, {**options, "user": user, "password": password} - ) - - -class SnowflakeDataSourceUnderTest(SnowflakeDataSource): - def __init__(self, spark, ws): - super().__init__(get_dialect("snowflake"), spark, ws, "secret_scope") - self._test_env = TestEnvGetter(True) - - @property - def get_jdbc_url(self) -> str: - raise NotImplementedError("Not needed for this test") - - def reader(self, query: str) -> DataFrameReader: - options = self._get_snowflake_options() - return self._spark.read.format("snowflake").option("dbtable", f"({query}) as tmp").options(**options) - - def _get_snowflake_options(self): - parsed = parse_snowflake_jdbc_url(self._test_env.get("TEST_SNOWFLAKE_JDBC")) - opts = { - "sfURL": parsed.get("url"), - "sfUser": parsed.get("user"), - "sfDatabase": parsed.get("db"), - "sfSchema": parsed.get("schema"), - "sfWarehouse": parsed.get("warehouse"), - "sfRole": "LABS", - "pem_private_key": SnowflakeDataSource._get_private_key( - self._test_env.get("TEST_SNOWFLAKE_PRIVATE_KEY"), None - ), - } - return opts +from tests.integration.reconcile.test_oracle_reconcile import OracleDataSourceUnderTest +from tests.integration.debug_envgetter import TestEnvGetter def test_sql_server_read_schema_happy(spark: SparkSession) -> None: - mock_ws = create_autospec(WorkspaceClient) - connector = TSQLServerDataSourceUnderTest(spark, mock_ws) + connection = "sqlserver_sandbox" + reader = RemoteQueryReader(spark, connection) + connector = TSQLServerDataSource(get_dialect("tsql"), reader) columns = connector.get_schema("labs_azure_sandbox_remorph", "dbo", "reconcile_in") assert columns @@ -91,7 +25,7 @@ def test_sql_server_read_schema_happy(spark: SparkSession) -> None: def test_databricks_read_schema_happy(spark: SparkSession) -> None: mock_ws = create_autospec(WorkspaceClient) - connector = DatabricksDataSource(get_dialect("databricks"), spark, mock_ws, "my_secret") + connector = DatabricksDataSource(get_dialect("databricks"), spark, mock_ws) random_view = f"test_view_{uuid.uuid4().hex}" try: @@ -110,7 +44,7 @@ def test_databricks_read_schema_happy_sandbox( spark: SparkSession, ws: WorkspaceClient, recon_tables: tuple[TableInfo, TableInfo] ) -> None: test_table, _ = recon_tables - connector = DatabricksDataSource(get_dialect("databricks"), spark, ws, "my_secret") + connector = DatabricksDataSource(get_dialect("databricks"), spark, ws) assert test_table.catalog_name assert test_table.schema_name @@ -125,17 +59,17 @@ def test_databricks_read_schema_happy_sandbox( # 2. Add credentials to the test env getter @pytest.mark.skip(reason="Not Ready! Deploy Infra") def test_oracle_read_schema_happy(spark: SparkSession) -> None: - mock_ws = create_autospec(WorkspaceClient) - connector = OracleDataSourceUnderTest(spark, mock_ws) + connector = OracleDataSourceUnderTest(spark) - columns = connector.get_schema(None, "SYSTEM", "help") + columns = connector.get_schema("ORCL", "SYSTEM", "help") assert columns @pytest.mark.xfail(reason="Snowflake account unavailable", strict=True) def test_snowflake_read_schema_happy(spark: SparkSession) -> None: - mock_ws = create_autospec(WorkspaceClient) - connector = SnowflakeDataSourceUnderTest(spark, mock_ws) + connection = TestEnvGetter(False).get("TEST_SNOWFLAKE_CONNECTION") + reader = RemoteQueryReader(spark, connection) + connector = SnowflakeDataSource(get_dialect("snowflake"), reader) columns = connector.get_schema('remorph', "sandbox", "diamonds") assert columns diff --git a/tests/integration/reconcile/query_builder/test_execute.py b/tests/integration/reconcile/query_builder/test_execute.py index bde1a7c44d..40178e2047 100644 --- a/tests/integration/reconcile/query_builder/test_execute.py +++ b/tests/integration/reconcile/query_builder/test_execute.py @@ -14,14 +14,16 @@ TableRecon, ReconcileMetadataConfig, ReconcileConfig, + SourceConnectionConfig, + TargetConnectionConfig, ) +from databricks.labs.lakebridge.reconcile.connectors.databricks import DatabricksDataSource +from databricks.labs.lakebridge.reconcile.connectors.snowflake import SnowflakeDataSource from databricks.labs.lakebridge.reconcile.reconciliation import Reconciliation from databricks.labs.lakebridge.reconcile.trigger_recon_service import TriggerReconService from databricks.labs.lakebridge.reconcile.utils import initialise_data_source from databricks.labs.lakebridge.transpiler.sqlglot.dialect_utils import get_dialect from databricks.labs.lakebridge.reconcile.connectors.data_source import MockDataSource -from databricks.labs.lakebridge.reconcile.connectors.databricks import DatabricksDataSource -from databricks.labs.lakebridge.reconcile.connectors.snowflake import SnowflakeDataSource from databricks.labs.lakebridge.reconcile.exception import ( DataSourceRuntimeException, InvalidInputException, @@ -737,14 +739,15 @@ def mock_for_report_type_data( source = MockDataSource(source_dataframe_repository, source_schema_repository) target = MockDataSource(target_dataframe_repository, target_schema_repository) reconcile_config_data = ReconcileConfig( - data_source="databricks", report_type="data", - secret_scope="remorph_databricks", - database_config=DatabaseConfig( - source_catalog=CATALOG, - source_schema=SCHEMA, - target_catalog=CATALOG, - target_schema=SCHEMA, + source=SourceConnectionConfig( + dialect="databricks", + catalog=CATALOG, + schema=SCHEMA, + ), + target=TargetConnectionConfig( + catalog=CATALOG, + schema=SCHEMA, ), metadata_config=recon_metadata, ) @@ -937,14 +940,15 @@ def mock_for_report_type_schema( source = MockDataSource(source_dataframe_repository, source_schema_repository) target = MockDataSource(target_dataframe_repository, target_schema_repository) reconcile_config_schema = ReconcileConfig( - data_source="databricks", report_type="schema", - secret_scope="remorph_databricks", - database_config=DatabaseConfig( - source_catalog=CATALOG, - source_schema=SCHEMA, - target_catalog=CATALOG, - target_schema=SCHEMA, + source=SourceConnectionConfig( + dialect="databricks", + catalog=CATALOG, + schema=SCHEMA, + ), + target=TargetConnectionConfig( + catalog=CATALOG, + schema=SCHEMA, ), metadata_config=recon_metadata, ) @@ -1152,14 +1156,16 @@ def mock_for_report_type_all( source = MockDataSource(source_dataframe_repository, source_schema_repository) target = MockDataSource(target_dataframe_repository, target_schema_repository) reconcile_config_all = ReconcileConfig( - data_source="snowflake", report_type="all", - secret_scope="remorph_snowflake", - database_config=DatabaseConfig( - source_catalog=CATALOG, - source_schema=SCHEMA, - target_catalog=CATALOG, - target_schema=SCHEMA, + source=SourceConnectionConfig( + dialect="snowflake", + catalog=CATALOG, + schema=SCHEMA, + uc_connection_name="remorph_snowflake", + ), + target=TargetConnectionConfig( + catalog=CATALOG, + schema=SCHEMA, ), metadata_config=recon_metadata, ) @@ -1429,14 +1435,16 @@ def mock_for_report_type_row( source = MockDataSource(source_dataframe_repository, source_schema_repository) target = MockDataSource(target_dataframe_repository, target_schema_repository) reconcile_config_row = ReconcileConfig( - data_source="snowflake", report_type="row", - secret_scope="remorph_snowflake", - database_config=DatabaseConfig( - source_catalog=CATALOG, - source_schema=SCHEMA, - target_catalog=CATALOG, - target_schema=SCHEMA, + source=SourceConnectionConfig( + dialect="snowflake", + catalog=CATALOG, + schema=SCHEMA, + uc_connection_name="remorph_snowflake", + ), + target=TargetConnectionConfig( + catalog=CATALOG, + schema=SCHEMA, ), metadata_config=recon_metadata, ) @@ -1577,14 +1585,16 @@ def mock_for_recon_exception(normalized_table_conf_with_opts, recon_metadata): source = MockDataSource({}, {}) target = MockDataSource({}, {}) reconcile_config_exception = ReconcileConfig( - data_source="snowflake", report_type="all", - secret_scope="remorph_snowflake", - database_config=DatabaseConfig( - source_catalog=CATALOG, - source_schema=SCHEMA, - target_catalog=CATALOG, - target_schema=SCHEMA, + source=SourceConnectionConfig( + dialect="snowflake", + catalog=CATALOG, + schema=SCHEMA, + uc_connection_name="remorph_snowflake", + ), + target=TargetConnectionConfig( + catalog=CATALOG, + schema=SCHEMA, ), metadata_config=recon_metadata, ) @@ -1667,8 +1677,7 @@ def test_schema_recon_with_general_exception( ): recon_schema, metrics_schema, details_schema = report_tables_schema table_recon, source, target, reconcile_config_schema = mock_for_report_type_schema - reconcile_config_schema.data_source = "snowflake" - reconcile_config_schema.secret_scope = "remorph_snowflake" + reconcile_config_schema.source.dialect = "snowflake" catalog = reconcile_config_schema.metadata_config.catalog schema = reconcile_config_schema.metadata_config.schema with ( @@ -1744,8 +1753,7 @@ def test_data_recon_with_general_exception( table_recon, source, target, reconcile_config = mock_for_report_type_schema catalog = reconcile_config.metadata_config.catalog schema = reconcile_config.metadata_config.schema - reconcile_config.data_source = "snowflake" - reconcile_config.secret_scope = "remorph_snowflake" + reconcile_config.source.dialect = "snowflake" reconcile_config.report_type = "data" with ( patch("databricks.labs.lakebridge.reconcile.trigger_recon_service.datetime") as mock_datetime, @@ -1818,8 +1826,7 @@ def test_data_recon_with_source_exception( table_recon, source, target, reconcile_config = mock_for_report_type_schema catalog = reconcile_config.metadata_config.catalog schema = reconcile_config.metadata_config.schema - reconcile_config.data_source = "snowflake" - reconcile_config.secret_scope = "remorph_snowflake" + reconcile_config.source.dialect = "snowflake" reconcile_config.report_type = "data" with ( patch("databricks.labs.lakebridge.reconcile.trigger_recon_service.datetime") as mock_datetime, @@ -1886,16 +1893,12 @@ def test_data_recon_with_source_exception( def test_initialise_data_source(mock_workspace_client, spark): - src_engine = get_dialect("snowflake") - secret_scope = "test" - - source, target = initialise_data_source(mock_workspace_client, spark, src_engine, secret_scope) + conn = "test" - snowflake_data_source = SnowflakeDataSource(src_engine, spark, mock_workspace_client, secret_scope).__class__ - databricks_data_source = DatabricksDataSource(src_engine, spark, mock_workspace_client, secret_scope).__class__ + source, target = initialise_data_source(mock_workspace_client, spark, "snowflake", conn) - assert isinstance(source, snowflake_data_source) - assert isinstance(target, databricks_data_source) + assert isinstance(source, SnowflakeDataSource) + assert isinstance(target, DatabricksDataSource) def test_recon_for_wrong_report_type(mock_workspace_client, spark, mock_for_report_type_row): @@ -2019,14 +2022,16 @@ def test_recon_output_without_exception(mock_gen_final_recon_output): ], ) reconcile_config = ReconcileConfig( - data_source="snowflake", report_type="all", - secret_scope="remorph_snowflake", - database_config=DatabaseConfig( - source_catalog=CATALOG, - source_schema=SCHEMA, - target_catalog=CATALOG, - target_schema=SCHEMA, + source=SourceConnectionConfig( + dialect="snowflake", + catalog=CATALOG, + schema=SCHEMA, + uc_connection_name="remorph_snowflake", + ), + target=TargetConnectionConfig( + catalog=CATALOG, + schema=SCHEMA, ), metadata_config=ReconcileMetadataConfig(), ) diff --git a/tests/integration/reconcile/query_builder/test_sampling_query.py b/tests/integration/reconcile/query_builder/test_sampling_query.py index 18c66e7ebb..bb904b1432 100644 --- a/tests/integration/reconcile/query_builder/test_sampling_query.py +++ b/tests/integration/reconcile/query_builder/test_sampling_query.py @@ -1,5 +1,8 @@ -from pyspark.sql.types import IntegerType, StringType, StructField, StructType +from datetime import date +from pyspark.sql.types import IntegerType, StringType, StructField, StructType, DoubleType, DateType + +from databricks.labs.lakebridge.reconcile.normalize_recon_config_service import NormalizeReconConfigService from databricks.labs.lakebridge.transpiler.sqlglot.dialect_utils import get_dialect from databricks.labs.lakebridge.reconcile.query_builder.sampling_query import ( SamplingQueryBuilder, @@ -7,10 +10,11 @@ from databricks.labs.lakebridge.reconcile.recon_config import ( ColumnMapping, Filters, + Table, Transformation, ) -from tests.conftest import oracle_schema_fixture_factory, ansi_schema_fixture_factory +from tests.conftest import oracle_schema_fixture_factory, ansi_schema_fixture_factory, tsql_schema_fixture_factory def test_build_query_for_snowflake_src( @@ -309,6 +313,60 @@ def test_build_query_for_snowflake_without_transformations( assert tgt_actual == tgt_expected +def test_build_query_for_tsql(spark, fake_tsql_datasource, fake_databricks_datasource): + src_schema = [ + tsql_schema_fixture_factory("carat", "double"), + tsql_schema_fixture_factory("clarity", "string"), + tsql_schema_fixture_factory("color", "string"), + tsql_schema_fixture_factory("cut", "string"), + tsql_schema_fixture_factory("mined_at", "date"), + ] + + table_conf = Table( + source_name="diamonds", + target_name="diamonds", + join_columns=["color", "clarity"], + ) + + normalize_service = NormalizeReconConfigService(fake_tsql_datasource, fake_databricks_datasource) + normalized_conf = normalize_service.normalize_recon_table_config(table_conf) + + df_schema = StructType( + [ + StructField('carat', DoubleType()), + StructField('clarity', StringType()), + StructField('color', StringType()), + StructField('cut', StringType()), + StructField('mined_at', DateType()), + ] + ) + df = spark.createDataFrame( + [ + (0.23, 'SI2', 'E', 'Ideal', date(2000, 1, 1)), + (0.21, 'VS1', 'E', 'Premium', date(2000, 1, 1)), + ], + schema=df_schema, + ) + + src_actual = SamplingQueryBuilder( + normalized_conf, src_schema, "source", get_dialect("tsql"), fake_tsql_datasource + ).build_query(df) + + src_expected = ( + "SELECT src.[carat], src.[clarity], src.[color], src.[cut], src.[mined_at] FROM " + "(SELECT COALESCE(TRIM(CAST([carat] AS VARCHAR(MAX))), '_null_recon_') AS [carat], " + "COALESCE(TRIM(CAST([clarity] AS VARCHAR(MAX))), '_null_recon_') AS [clarity], " + "COALESCE(TRIM(CAST([color] AS VARCHAR(MAX))), '_null_recon_') AS [color], " + "COALESCE(TRIM(CAST([cut] AS VARCHAR(MAX))), '_null_recon_') AS [cut], " + "COALESCE(CONVERT(VARCHAR(10), [mined_at], 101), '1900-01-01') AS [mined_at] FROM :tbl) AS src " + "INNER JOIN (VALUES (CAST('SI2' AS string), CAST('E' AS string)), " + "(CAST('VS1' AS string), CAST('E' AS string))) AS recon([clarity], [color]) " + "ON src.[clarity] = recon.[clarity] AND src.[color] = recon.[color]" + ) + + assert src_actual == src_expected + + def test_build_query_for_snowflake_src_for_non_integer_primary_keys( spark, table_conf, fake_oracle_datasource, fake_databricks_datasource ): diff --git a/tests/integration/reconcile/test_oracle_reconcile.py b/tests/integration/reconcile/test_oracle_reconcile.py index cb7a3aac11..28cd320fb8 100644 --- a/tests/integration/reconcile/test_oracle_reconcile.py +++ b/tests/integration/reconcile/test_oracle_reconcile.py @@ -1,25 +1,73 @@ from unittest.mock import patch import pytest -from pyspark.sql import DataFrame - +from pyspark.sql import DataFrame, DataFrameReader from databricks.connect import DatabricksSession -from databricks.labs.lakebridge.config import DatabaseConfig, ReconcileMetadataConfig, ReconcileConfig +from databricks.labs.lakebridge.config import ( + DatabaseConfig, + ReconcileMetadataConfig, + ReconcileConfig, + SourceConnectionConfig, + TargetConnectionConfig, +) from databricks.labs.lakebridge.reconcile.connectors.databricks import DatabricksDataSource +from databricks.labs.lakebridge.reconcile.connectors.remote_query_reader import RemoteQueryReader from databricks.labs.lakebridge.reconcile.recon_capture import ReconCapture from databricks.labs.lakebridge.reconcile.recon_config import Table, JdbcReaderOptions +from databricks.labs.lakebridge.reconcile.connectors.oracle import OracleDataSource +from databricks.labs.lakebridge.reconcile.recon_config import Schema from databricks.labs.lakebridge.reconcile.reconciliation import Reconciliation from databricks.labs.lakebridge.reconcile.schema_compare import SchemaCompare from databricks.labs.lakebridge.reconcile.trigger_recon_service import TriggerReconService from databricks.labs.lakebridge.transpiler.sqlglot.dialect_utils import get_dialect from tests.integration.reconcile.conftest import FakeReconIntermediatePersist from tests.integration.debug_envgetter import TestEnvGetter -from tests.integration.reconcile.connectors.test_read_schema import OracleDataSourceUnderTest + + +class OracleDataSourceUnderTest(OracleDataSource): + def __init__(self, spark): + # reader is unused — this subclass fully overrides read_data/get_schema with JDBC + reader = RemoteQueryReader(spark, "NOT USED") + super().__init__(get_dialect("oracle"), reader) + self._spark = spark + self._test_env = TestEnvGetter(False) + + @property + def _get_jdbc_url(self) -> str: + return self._test_env.get("TEST_ORACLE_JDBC") + + def _jdbc_reader(self, query: str) -> DataFrameReader: + user = self._test_env.get("TEST_ORACLE_USER") + password = self._test_env.get("TEST_ORACLE_PASSWORD") + return self._spark.read.format("JDBC").options( + **{"driver": "", "url": self._get_jdbc_url, "user": user, "password": password, "dbtable": query} + ) + + def read_data( + self, + catalog: str, + schema: str, + table: str, + query: str, + options: JdbcReaderOptions | None, + ): + table_query = query.replace(":tbl", table) + return self._jdbc_reader(table_query).load().collect() + + def get_schema( + self, + catalog: str, + schema: str, + table: str, + normalize: bool = True, + ) -> list[Schema]: + rows = self._jdbc_reader(OracleDataSource._SCHEMA_QUERY).load().collect() + return [self._map_meta_column(r, normalize) for r in rows] class DatabricksDataSourceUnderTest(DatabricksDataSource): def __init__(self, databricks, ws, local_spark): - super().__init__(get_dialect("databricks"), databricks, ws, "not used") + super().__init__(get_dialect("databricks"), databricks, ws) self._local_spark = local_spark def read_data( @@ -41,18 +89,26 @@ def test_oracle_db_reconcile(spark, mock_workspace_client, tmp_path): host = test_env.get("DATABRICKS_HOST") databricks = DatabricksSession.builder.host(host).clusterId(cluster).getOrCreate() databricks_data_source = DatabricksDataSourceUnderTest(databricks, mock_workspace_client, spark) - oracle_data_source = OracleDataSourceUnderTest(spark, mock_workspace_client) + oracle_data_source = OracleDataSourceUnderTest(spark) report = "row" db_config = DatabaseConfig( + source_catalog="", source_schema="SYSTEM", target_catalog="main", target_schema="lakebridge", ) reconcile_config = ReconcileConfig( - data_source="oracle", report_type=report, - secret_scope="not used", - database_config=db_config, + source=SourceConnectionConfig( + dialect="oracle", + catalog="", + schema="SYSTEM", + uc_connection_name="not used", + ), + target=TargetConnectionConfig( + catalog="main", + schema="lakebridge", + ), metadata_config=ReconcileMetadataConfig(catalog="tmp", schema="reconcile"), ) recon = Reconciliation( diff --git a/tests/integration/reconcile/test_recon_capture.py b/tests/integration/reconcile/test_recon_capture.py index 7d7f8c23c3..492cf4d362 100644 --- a/tests/integration/reconcile/test_recon_capture.py +++ b/tests/integration/reconcile/test_recon_capture.py @@ -122,7 +122,7 @@ def data_prep(spark: SparkSession): def test_recon_capture_start_snowflake_all(mock_workspace_client, spark, recon_metadata): database_config = DatabaseConfig( - "source_test_schema", "target_test_catalog", "target_test_schema", "source_test_catalog" + "source_test_catalog", "source_test_schema", "target_test_catalog", "target_test_schema" ) ws = mock_workspace_client source_type = get_dialect("snowflake") @@ -206,7 +206,9 @@ def test_recon_capture_start_snowflake_all(mock_workspace_client, spark, recon_m def test_test_recon_capture_start_databricks_data(mock_workspace_client, spark, recon_metadata): - database_config = DatabaseConfig("source_test_schema", "target_test_catalog", "target_test_schema") + database_config = DatabaseConfig( + "source_test_catalog", "source_test_schema", "target_test_catalog", "target_test_schema" + ) ws = mock_workspace_client source_type = get_dialect("databricks") recon_capture = ReconCapture( @@ -233,7 +235,7 @@ def test_test_recon_capture_start_databricks_data(mock_workspace_client, spark, remorph_recon_df = spark.sql(f"select * from {recon_metadata.catalog}.{recon_metadata.schema}.main") row = remorph_recon_df.collect()[0] assert remorph_recon_df.count() == 1 - assert row.source_table.catalog is None + assert row.source_table.catalog == "source_test_catalog" assert row.report_type == "data" assert row.source_type == "Databricks" @@ -251,7 +253,7 @@ def test_test_recon_capture_start_databricks_data(mock_workspace_client, spark, def test_test_recon_capture_start_databricks_row(mock_workspace_client, spark, recon_metadata): database_config = DatabaseConfig( - "source_test_schema", "target_test_catalog", "target_test_schema", "source_test_catalog" + "source_test_catalog", "source_test_schema", "target_test_catalog", "target_test_schema" ) ws = mock_workspace_client source_type = get_dialect("databricks") @@ -300,7 +302,7 @@ def test_test_recon_capture_start_databricks_row(mock_workspace_client, spark, r def test_recon_capture_start_oracle_schema(mock_workspace_client, spark, recon_metadata): database_config = DatabaseConfig( - "source_test_schema", "target_test_catalog", "target_test_schema", "source_test_catalog" + "source_test_catalog", "source_test_schema", "target_test_catalog", "target_test_schema" ) ws = mock_workspace_client source_type = get_dialect("oracle") @@ -351,7 +353,7 @@ def test_recon_capture_start_oracle_schema(mock_workspace_client, spark, recon_m def test_recon_capture_start_oracle_with_exception(mock_workspace_client, spark, recon_metadata): database_config = DatabaseConfig( - "source_test_schema", "target_test_catalog", "target_test_schema", "source_test_catalog" + "source_test_catalog", "source_test_schema", "target_test_catalog", "target_test_schema" ) ws = mock_workspace_client source_type = get_dialect("oracle") @@ -397,7 +399,7 @@ def test_recon_capture_start_oracle_with_exception(mock_workspace_client, spark, def test_recon_capture_start_with_exception(mock_workspace_client, spark): database_config = DatabaseConfig( - "source_test_schema", "target_test_catalog", "target_test_schema", "source_test_catalog" + "source_test_catalog", "source_test_schema", "target_test_catalog", "target_test_schema" ) ws = mock_workspace_client source_type = get_dialect("snowflake") @@ -422,6 +424,7 @@ def test_recon_capture_start_with_exception(mock_workspace_client, spark): def test_generate_final_reconcile_output_row(mock_workspace_client, spark, recon_metadata): database_config = DatabaseConfig( + "source_test_catalog", "source_test_schema", "target_test_catalog", "target_test_schema", @@ -457,7 +460,7 @@ def test_generate_final_reconcile_output_row(mock_workspace_client, spark, recon results=[ ReconcileTableOutput( target_table_name='target_test_catalog.target_test_schema.target_supplier', - source_table_name='source_test_schema.supplier', + source_table_name='source_test_catalog.source_test_schema.supplier', status=StatusOutput(row=False, column=None, schema=None), exception_message='', ) @@ -467,6 +470,7 @@ def test_generate_final_reconcile_output_row(mock_workspace_client, spark, recon def test_generate_final_reconcile_output_data(mock_workspace_client, spark, recon_metadata): database_config = DatabaseConfig( + "source_test_catalog", "source_test_schema", "target_test_catalog", "target_test_schema", @@ -502,7 +506,7 @@ def test_generate_final_reconcile_output_data(mock_workspace_client, spark, reco results=[ ReconcileTableOutput( target_table_name='target_test_catalog.target_test_schema.target_supplier', - source_table_name='source_test_schema.supplier', + source_table_name='source_test_catalog.source_test_schema.supplier', status=StatusOutput(row=False, column=False, schema=None), exception_message='', ) @@ -512,6 +516,7 @@ def test_generate_final_reconcile_output_data(mock_workspace_client, spark, reco def test_generate_final_reconcile_output_schema(mock_workspace_client, spark, recon_metadata): database_config = DatabaseConfig( + "source_test_catalog", "source_test_schema", "target_test_catalog", "target_test_schema", @@ -547,7 +552,7 @@ def test_generate_final_reconcile_output_schema(mock_workspace_client, spark, re results=[ ReconcileTableOutput( target_table_name='target_test_catalog.target_test_schema.target_supplier', - source_table_name='source_test_schema.supplier', + source_table_name='source_test_catalog.source_test_schema.supplier', status=StatusOutput(row=None, column=None, schema=True), exception_message='', ) @@ -557,6 +562,7 @@ def test_generate_final_reconcile_output_schema(mock_workspace_client, spark, re def test_generate_final_reconcile_output_all(mock_workspace_client, spark, recon_metadata): database_config = DatabaseConfig( + "source_test_catalog", "source_test_schema", "target_test_catalog", "target_test_schema", @@ -593,7 +599,7 @@ def test_generate_final_reconcile_output_all(mock_workspace_client, spark, recon results=[ ReconcileTableOutput( target_table_name='target_test_catalog.target_test_schema.target_supplier', - source_table_name='source_test_schema.supplier', + source_table_name='source_test_catalog.source_test_schema.supplier', status=StatusOutput(row=False, column=False, schema=True), exception_message='', ) @@ -603,6 +609,7 @@ def test_generate_final_reconcile_output_all(mock_workspace_client, spark, recon def test_generate_final_reconcile_output_exception(mock_workspace_client, spark, recon_metadata): database_config = DatabaseConfig( + "source_test_catalog", "source_test_schema", "target_test_catalog", "target_test_schema", @@ -640,7 +647,7 @@ def test_generate_final_reconcile_output_exception(mock_workspace_client, spark, results=[ ReconcileTableOutput( target_table_name='target_test_catalog.target_test_schema.target_supplier', - source_table_name='source_test_schema.supplier', + source_table_name='source_test_catalog.source_test_schema.supplier', status=StatusOutput(row=None, column=None, schema=None), exception_message='Test exception', ) @@ -650,7 +657,7 @@ def test_generate_final_reconcile_output_exception(mock_workspace_client, spark, def test_apply_threshold_for_mismatch_with_true_absolute(mock_workspace_client, spark, recon_metadata): database_config = DatabaseConfig( - "source_test_schema", "target_test_catalog", "target_test_schema", "source_test_catalog" + "source_test_catalog", "source_test_schema", "target_test_catalog", "target_test_schema" ) ws = mock_workspace_client source_type = get_dialect("snowflake") @@ -687,7 +694,7 @@ def test_apply_threshold_for_mismatch_with_true_absolute(mock_workspace_client, def test_apply_threshold_for_mismatch_with_missing(mock_workspace_client, spark, recon_metadata): database_config = DatabaseConfig( - "source_test_schema", "target_test_catalog", "target_test_schema", "source_test_catalog" + "source_test_catalog", "source_test_schema", "target_test_catalog", "target_test_schema" ) ws = mock_workspace_client source_type = get_dialect("snowflake") @@ -720,7 +727,7 @@ def test_apply_threshold_for_mismatch_with_missing(mock_workspace_client, spark, def test_apply_threshold_for_mismatch_with_schema_fail(mock_workspace_client, spark, recon_metadata): database_config = DatabaseConfig( - "source_test_schema", "target_test_catalog", "target_test_schema", "source_test_catalog" + "source_test_catalog", "source_test_schema", "target_test_catalog", "target_test_schema" ) ws = mock_workspace_client source_type = get_dialect("snowflake") @@ -757,7 +764,7 @@ def test_apply_threshold_for_mismatch_with_schema_fail(mock_workspace_client, sp def test_apply_threshold_for_mismatch_with_wrong_absolute_bound(mock_workspace_client, spark, recon_metadata): database_config = DatabaseConfig( - "source_test_schema", "target_test_catalog", "target_test_schema", "source_test_catalog" + "source_test_catalog", "source_test_schema", "target_test_catalog", "target_test_schema" ) ws = mock_workspace_client source_type = get_dialect("snowflake") @@ -795,7 +802,7 @@ def test_apply_threshold_for_mismatch_with_wrong_absolute_bound(mock_workspace_c def test_apply_threshold_for_mismatch_with_wrong_percentage_bound(mock_workspace_client, spark, recon_metadata): database_config = DatabaseConfig( - "source_test_schema", "target_test_catalog", "target_test_schema", "source_test_catalog" + "source_test_catalog", "source_test_schema", "target_test_catalog", "target_test_schema" ) ws = mock_workspace_client source_type = get_dialect("snowflake") @@ -833,7 +840,7 @@ def test_apply_threshold_for_mismatch_with_wrong_percentage_bound(mock_workspace def test_apply_threshold_for_mismatch_with_true_percentage_bound(mock_workspace_client, spark, recon_metadata): database_config = DatabaseConfig( - "source_test_schema", "target_test_catalog", "target_test_schema", "source_test_catalog" + "source_test_catalog", "source_test_schema", "target_test_catalog", "target_test_schema" ) ws = mock_workspace_client source_type = get_dialect("snowflake") @@ -870,7 +877,7 @@ def test_apply_threshold_for_mismatch_with_true_percentage_bound(mock_workspace_ def test_apply_threshold_for_mismatch_with_invalid_bounds(mock_workspace_client, spark): database_config = DatabaseConfig( - "source_test_schema", "target_test_catalog", "target_test_schema", "source_test_catalog" + "source_test_catalog", "source_test_schema", "target_test_catalog", "target_test_schema" ) ws = mock_workspace_client source_type = get_dialect("snowflake") @@ -916,7 +923,7 @@ def test_apply_threshold_for_mismatch_with_invalid_bounds(mock_workspace_client, def test_apply_threshold_for_only_threshold_mismatch_with_true_absolute(mock_workspace_client, spark, recon_metadata): database_config = DatabaseConfig( - "source_test_schema", "target_test_catalog", "target_test_schema", "source_test_catalog" + "source_test_catalog", "source_test_schema", "target_test_catalog", "target_test_schema" ) ws = mock_workspace_client source_type = get_dialect("snowflake") diff --git a/tests/integration/reconcile/test_recon_e2e.py b/tests/integration/reconcile/test_recon_e2e.py index be9ef88bf6..5ee1329233 100644 --- a/tests/integration/reconcile/test_recon_e2e.py +++ b/tests/integration/reconcile/test_recon_e2e.py @@ -79,7 +79,6 @@ def test_recon_sql_server_job_succeeds( _run_recon_e2e_spec(app_ctx) -@pytest.mark.xfail(reason="Snowflake account unavailable", strict=True) def test_recon_snowflake_job_succeeds( application_ctx: ApplicationContext, snowflake_recon_config: ReconcileConfig, diff --git a/tests/unit/deployment/test_installation.py b/tests/unit/deployment/test_installation.py index 6e749e27ac..b00960c77f 100644 --- a/tests/unit/deployment/test_installation.py +++ b/tests/unit/deployment/test_installation.py @@ -13,7 +13,8 @@ TranspileConfig, LakebridgeConfiguration, ReconcileConfig, - DatabaseConfig, + SourceConnectionConfig, + TargetConnectionConfig, ReconcileMetadataConfig, ProfilerDashboardConfig, ProfilerDashboardMetadataConfig, @@ -51,13 +52,16 @@ def test_install_all(ws): schema_name="transpiler6", ) reconcile_config = ReconcileConfig( - data_source="oracle", report_type="all", - secret_scope="remorph_oracle6", - database_config=DatabaseConfig( - source_schema="tpch_sf10006", - target_catalog="tpch6", - target_schema="1000gb6", + source=SourceConnectionConfig( + dialect="oracle", + catalog="ORCL6", + schema="tpch_sf10006", + uc_connection_name="remorph_oracle6", + ), + target=TargetConnectionConfig( + catalog="tpch6", + schema="1000gb6", ), metadata_config=ReconcileMetadataConfig( catalog="remorph6", @@ -115,13 +119,16 @@ def test_recon_component_installation(ws): upgrades = create_autospec(Upgrades) reconcile_config = ReconcileConfig( - data_source="oracle", report_type="all", - secret_scope="remorph_oracle8", - database_config=DatabaseConfig( - source_schema="tpch_sf10008", - target_catalog="tpch8", - target_schema="1000gb8", + source=SourceConnectionConfig( + dialect="oracle", + catalog="ORCL8", + schema="tpch_sf10008", + uc_connection_name="remorph_oracle8", + ), + target=TargetConnectionConfig( + catalog="tpch8", + schema="1000gb8", ), metadata_config=ReconcileMetadataConfig( catalog="remorph8", @@ -168,14 +175,16 @@ def test_uninstall_configs_exist(ws): ) reconcile_config = ReconcileConfig( - data_source="snowflake", report_type="all", - secret_scope="remorph_snowflake1", - database_config=DatabaseConfig( - source_catalog="snowflake_sample_data1", - source_schema="tpch_sf10001", - target_catalog="tpch1", - target_schema="1000gb1", + source=SourceConnectionConfig( + dialect="snowflake", + catalog="snowflake_sample_data1", + schema="tpch_sf10001", + uc_connection_name="remorph_snowflake1", + ), + target=TargetConnectionConfig( + catalog="tpch1", + schema="1000gb1", ), metadata_config=ReconcileMetadataConfig( catalog="remorph1", diff --git a/tests/unit/deployment/test_job.py b/tests/unit/deployment/test_job.py index 6c01f1927f..514eeb9305 100644 --- a/tests/unit/deployment/test_job.py +++ b/tests/unit/deployment/test_job.py @@ -11,8 +11,9 @@ from databricks.labs.lakebridge.config import ( LakebridgeConfiguration, ReconcileConfig, - DatabaseConfig, ReconcileMetadataConfig, + SourceConnectionConfig, + TargetConnectionConfig, ProfilerDashboardConfig, ProfilerDashboardMetadataConfig, ) @@ -22,13 +23,16 @@ @pytest.fixture def oracle_recon_config() -> ReconcileConfig: return ReconcileConfig( - data_source="oracle", report_type="all", - secret_scope="remorph_oracle9", - database_config=DatabaseConfig( - source_schema="tpch_sf10009", - target_catalog="tpch9", - target_schema="1000gb9", + source=SourceConnectionConfig( + dialect="oracle", + catalog="ORCL", + schema="tpch_sf10009", + uc_connection_name="remorph_oracle9", + ), + target=TargetConnectionConfig( + catalog="tpch9", + schema="1000gb9", ), metadata_config=ReconcileMetadataConfig( catalog="remorph9", @@ -41,14 +45,16 @@ def oracle_recon_config() -> ReconcileConfig: @pytest.fixture def snowflake_recon_config() -> ReconcileConfig: return ReconcileConfig( - data_source="snowflake", report_type="all", - secret_scope="remorph_snowflake9", - database_config=DatabaseConfig( - source_schema="tpch_sf10009", - target_catalog="tpch9", - target_schema="1000gb9", - source_catalog="snowflake_sample_data9", + source=SourceConnectionConfig( + dialect="snowflake", + catalog="snowflake_sample_data9", + schema="tpch_sf10009", + uc_connection_name="remorph_snowflake9", + ), + target=TargetConnectionConfig( + catalog="tpch9", + schema="1000gb9", ), metadata_config=ReconcileMetadataConfig( catalog="remorph9", diff --git a/tests/unit/deployment/test_recon.py b/tests/unit/deployment/test_recon.py index f55c62b757..c1a4482b5c 100644 --- a/tests/unit/deployment/test_recon.py +++ b/tests/unit/deployment/test_recon.py @@ -11,7 +11,8 @@ from databricks.labs.lakebridge.config import ( LakebridgeConfiguration, ReconcileConfig, - DatabaseConfig, + SourceConnectionConfig, + TargetConnectionConfig, ReconcileMetadataConfig, ) from databricks.labs.lakebridge.deployment.dashboard import DashboardDeployment @@ -54,14 +55,16 @@ def test_install_missing_config(ws): def test_install(ws): reconcile_config = ReconcileConfig( - data_source="snowflake", report_type="all", - secret_scope="remorph_snowflake4", - database_config=DatabaseConfig( - source_catalog="snowflake_sample_data4", - source_schema="tpch_sf10004", - target_catalog="tpch4", - target_schema="1000gb4", + source=SourceConnectionConfig( + dialect="snowflake", + catalog="snowflake_sample_data4", + schema="tpch_sf10004", + uc_connection_name="remorph_snowflake4", + ), + target=TargetConnectionConfig( + catalog="tpch4", + schema="1000gb4", ), metadata_config=ReconcileMetadataConfig( catalog="remorph4", @@ -147,14 +150,16 @@ def test_uninstall_missing_config(ws): def test_uninstall(ws): recon_config = ReconcileConfig( - data_source="snowflake", report_type="all", - secret_scope="remorph_snowflake5", - database_config=DatabaseConfig( - source_catalog="snowflake_sample_data5", - source_schema="tpch_sf10005", - target_catalog="tpch5", - target_schema="1000gb5", + source=SourceConnectionConfig( + dialect="snowflake", + catalog="snowflake_sample_data5", + schema="tpch_sf10005", + uc_connection_name="remorph_snowflake5", + ), + target=TargetConnectionConfig( + catalog="tpch5", + schema="1000gb5", ), metadata_config=ReconcileMetadataConfig( catalog="remorph5", diff --git a/tests/unit/reconcile/connectors/test_databricks.py b/tests/unit/reconcile/connectors/test_databricks.py index 7f89612e85..e21b850e1c 100644 --- a/tests/unit/reconcile/connectors/test_databricks.py +++ b/tests/unit/reconcile/connectors/test_databricks.py @@ -17,16 +17,15 @@ def initial_setup(): # Define the source, workspace, and scope engine = get_dialect("databricks") ws = create_autospec(WorkspaceClient) - scope = "scope" - return engine, spark, ws, scope + return engine, spark, ws def test_get_schema(): # initial setup - engine, spark, ws, scope = initial_setup() + engine, spark, ws = initial_setup() # catalog as catalog - ddds = DatabricksDataSource(engine, spark, ws, scope) + ddds = DatabricksDataSource(engine, spark, ws) ddds.get_schema("catalog", "schema", "supplier") spark.sql.assert_called_with( re.sub( @@ -56,10 +55,10 @@ def test_get_schema(): def test_read_data_from_uc(): # initial setup - engine, spark, ws, scope = initial_setup() + engine, spark, ws = initial_setup() # create object for DatabricksDataSource - ddds = DatabricksDataSource(engine, spark, ws, scope) + ddds = DatabricksDataSource(engine, spark, ws) # Test with query ddds.read_data("org", "data", "employee", "select id as id, name as name from :tbl", None) @@ -72,10 +71,10 @@ def test_read_data_from_uc(): def test_read_data_from_hive(): # initial setup - engine, spark, ws, scope = initial_setup() + engine, spark, ws = initial_setup() # create object for DatabricksDataSource - ddds = DatabricksDataSource(engine, spark, ws, scope) + ddds = DatabricksDataSource(engine, spark, ws) # Test with query ddds.read_data("hive_metastore", "data", "employee", "select id as id, name as name from :tbl", None) @@ -88,10 +87,10 @@ def test_read_data_from_hive(): def test_read_data_exception_handling(): # initial setup - engine, spark, ws, scope = initial_setup() + engine, spark, ws = initial_setup() # create object for DatabricksDataSource - ddds = DatabricksDataSource(engine, spark, ws, scope) + ddds = DatabricksDataSource(engine, spark, ws) spark.sql.side_effect = RuntimeError("Test Exception") with pytest.raises( @@ -104,10 +103,10 @@ def test_read_data_exception_handling(): def test_get_schema_exception_handling(): # initial setup - engine, spark, ws, scope = initial_setup() + engine, spark, ws = initial_setup() # create object for DatabricksDataSource - ddds = DatabricksDataSource(engine, spark, ws, scope) + ddds = DatabricksDataSource(engine, spark, ws) spark.sql.side_effect = RuntimeError("Test Exception") with pytest.raises(DataSourceRuntimeException) as exception: ddds.get_schema("org", "data", "employee") @@ -121,8 +120,8 @@ def test_get_schema_exception_handling(): def test_normalize_identifier(): - engine, spark, ws, scope = initial_setup() - data_source = DatabricksDataSource(engine, spark, ws, scope) + engine, spark, ws = initial_setup() + data_source = DatabricksDataSource(engine, spark, ws) assert data_source.normalize_identifier("a") == NormalizedIdentifier("`a`", '`a`') assert data_source.normalize_identifier('`b`') == NormalizedIdentifier("`b`", '`b`') diff --git a/tests/unit/reconcile/connectors/test_oracle.py b/tests/unit/reconcile/connectors/test_oracle.py index 07ee7e1d2f..167fa9eab5 100644 --- a/tests/unit/reconcile/connectors/test_oracle.py +++ b/tests/unit/reconcile/connectors/test_oracle.py @@ -1,60 +1,34 @@ -import base64 import re -from unittest.mock import MagicMock, create_autospec +from unittest.mock import create_autospec import pytest from databricks.labs.lakebridge.reconcile.connectors.models import NormalizedIdentifier from databricks.labs.lakebridge.transpiler.sqlglot.dialect_utils import get_dialect from databricks.labs.lakebridge.reconcile.connectors.oracle import OracleDataSource +from databricks.labs.lakebridge.reconcile.connectors.remote_query_reader import RemoteQueryReader from databricks.labs.lakebridge.reconcile.exception import DataSourceRuntimeException from databricks.labs.lakebridge.reconcile.recon_config import JdbcReaderOptions, Table -from databricks.sdk import WorkspaceClient -from databricks.sdk.service.workspace import GetSecretResponse - - -def mock_secret(scope, key): - secret_mock = { - "scope": { - 'user': GetSecretResponse(key='user', value=base64.b64encode(bytes('my_user', 'utf-8')).decode('utf-8')), - 'password': GetSecretResponse( - key='password', value=base64.b64encode(bytes('my_password', 'utf-8')).decode('utf-8') - ), - 'host': GetSecretResponse(key='host', value=base64.b64encode(bytes('my_host', 'utf-8')).decode('utf-8')), - 'port': GetSecretResponse(key='port', value=base64.b64encode(bytes('777', 'utf-8')).decode('utf-8')), - 'database': GetSecretResponse( - key='database', value=base64.b64encode(bytes('my_database', 'utf-8')).decode('utf-8') - ), - } - } - - return secret_mock[scope][key] def initial_setup(): - pyspark_sql_session = MagicMock() - spark = pyspark_sql_session.SparkSession.builder.getOrCreate() - - # Define the source, workspace, and scope engine = get_dialect("oracle") - ws = create_autospec(WorkspaceClient) - scope = "scope" - ws.secrets.get_secret.side_effect = mock_secret - return engine, spark, ws, scope + reader = create_autospec(RemoteQueryReader) + return engine, reader def test_read_data_with_options(): # initial setup - engine, spark, ws, scope = initial_setup() + engine, reader = initial_setup() - # create object for SnowflakeDataSource - ords = OracleDataSource(engine, spark, ws, scope) + # create object for OracleDataSource + ords = OracleDataSource(engine, reader) # Create a Tables configuration object with JDBC reader options table_conf = Table( source_name="supplier", target_name="supplier", jdbc_reader_options=JdbcReaderOptions( - number_partitions=50, partition_column="s_nationkey", lower_bound="0", upper_bound="100" + num_partitions=50, partition_column="s_nationkey", lower_bound="0", upper_bound="100" ), join_columns=None, select_columns=None, @@ -66,69 +40,56 @@ def test_read_data_with_options(): ) # Call the read_data method with the Tables configuration - ords.read_data(None, "data", "employee", "select 1 from :tbl", table_conf.jdbc_reader_options) + ords.read_data("ORCL", "data", "employee", "select 1 from :tbl", table_conf.jdbc_reader_options) - # spark assertions - spark.read.format.assert_called_with("jdbc") - spark.read.format().option.assert_called_with( - "url", - "jdbc:oracle:thin:@//my_host:777/my_database", + # reader assertions — verify the query passed to remote_query reader + reader.read_data.assert_called_once_with( + "select 1 from data.employee", + "ORCL", + "service_name", + "dbtable", + table_conf.jdbc_reader_options, ) - spark.read.format().option().option.assert_called_with("driver", "oracle.jdbc.OracleDriver") - spark.read.format().option().option().option.assert_called_with("dbtable", "(select 1 from data.employee) tmp") - jdbc_actual_args = spark.read.format().option().option().option().options.call_args.kwargs - jdbc_expected_args = { - "numPartitions": 50, - "partitionColumn": "s_nationkey", - "lowerBound": '0', - "upperBound": "100", - "fetchsize": 100, - "oracle.jdbc.mapDateToTimestamp": "false", - "sessionInitStatement": r"BEGIN dbms_session.set_nls('nls_date_format', " - r"'''YYYY-MM-DD''');dbms_session.set_nls('nls_timestamp_format', '''YYYY-MM-DD " - r"HH24:MI:SS''');END;", - "user": "my_user", - "password": "my_password", - } - assert jdbc_actual_args == jdbc_expected_args - spark.read.format().option().option().option().options().load.assert_called_once() def test_get_schema(): # initial setup - engine, spark, ws, scope = initial_setup() + engine, reader = initial_setup() - # create object for SnowflakeDataSource - ords = OracleDataSource(engine, spark, ws, scope) + # create object for OracleDataSource + ords = OracleDataSource(engine, reader) # call test method - ords.get_schema(None, "data", "employee") - # spark assertions - spark.read.format.assert_called_with("jdbc") - spark.read.format().option().option().option.assert_called_with( - "dbtable", - re.sub( - r'\s+', - ' ', - r"""(select column_name, case when (data_precision is not null - and data_scale <> 0) - then data_type || '(' || data_precision || ',' || data_scale || ')' - when (data_precision is not null and data_scale = 0) - then data_type || '(' || data_precision || ')' - when data_precision is null and (lower(data_type) in ('date') or - lower(data_type) like 'timestamp%') then data_type - when CHAR_LENGTH = 0 then data_type - else data_type || '(' || CHAR_LENGTH || ')' - end data_type - FROM ALL_TAB_COLUMNS - WHERE lower(TABLE_NAME) = 'employee' and lower(owner) = 'data') tmp""", - ), + ords.get_schema("ORCL", "data", "employee") + # reader assertions — verify the schema query passed to remote_query reader + reader.read_data.assert_called_once() + call_args = reader.read_data.call_args + source_query = call_args[0][0] + expected_query = re.sub( + r'\s+', + ' ', + r"""select column_name, case when (data_precision is not null + and data_scale <> 0) + then data_type || '(' || data_precision || ',' || data_scale || ')' + when (data_precision is not null and data_scale = 0) + then data_type || '(' || data_precision || ')' + when data_precision is null and (lower(data_type) in ('date') or + lower(data_type) like 'timestamp%') then data_type + when CHAR_LENGTH = 0 then data_type + else data_type || '(' || CHAR_LENGTH || ')' + end data_type + FROM ALL_TAB_COLUMNS + WHERE lower(TABLE_NAME) = 'employee' and lower(owner) = 'data'""", ) + assert source_query == expected_query + assert call_args[0][1] == "ORCL" + assert call_args[0][2] == "service_name" + assert call_args[0][3] == "query" def test_read_data_exception_handling(): # initial setup - engine, spark, ws, scope = initial_setup() - ords = OracleDataSource(engine, spark, ws, scope) + engine, reader = initial_setup() + ords = OracleDataSource(engine, reader) # Create a Tables configuration object table_conf = Table( source_name="supplier", @@ -143,22 +104,22 @@ def test_read_data_exception_handling(): filters=None, ) - spark.read.format().option().option().option().options().load.side_effect = RuntimeError("Test Exception") + reader.read_data.side_effect = RuntimeError("Test Exception") # Call the read_data method with the Tables configuration and assert that a PySparkException is raised with pytest.raises( DataSourceRuntimeException, match="Runtime exception occurred while fetching data using select 1 from data.employee : Test Exception", ): - ords.read_data(None, "data", "employee", "select 1 from :tbl", table_conf.jdbc_reader_options) + ords.read_data("ORCL", "data", "employee", "select 1 from :tbl", table_conf.jdbc_reader_options) def test_get_schema_exception_handling(): # initial setup - engine, spark, ws, scope = initial_setup() - ords = OracleDataSource(engine, spark, ws, scope) + engine, reader = initial_setup() + ords = OracleDataSource(engine, reader) - spark.read.format().option().option().option().options().load.side_effect = RuntimeError("Test Exception") + reader.read_data.side_effect = RuntimeError("Test Exception") # Call the get_schema method with predefined table, schema, and catalog names and assert that a PySparkException # is raised @@ -177,13 +138,13 @@ def test_get_schema_exception_handling(): FROM ALL_TAB_COLUMNS WHERE lower(TABLE_NAME) = 'employee' and lower(owner) = 'data' """, ): - ords.get_schema(None, "data", "employee") + ords.get_schema("ORCL", "data", "employee") @pytest.mark.skip("Turned off till we can handle case sensitivity.") def test_normalize_identifier(): - engine, spark, ws, scope = initial_setup() - data_source = OracleDataSource(engine, spark, ws, scope) + engine, reader = initial_setup() + data_source = OracleDataSource(engine, reader) assert data_source.normalize_identifier("a") == NormalizedIdentifier("`a`", '"a"') assert data_source.normalize_identifier('"b"') == NormalizedIdentifier("`b`", '"b"') diff --git a/tests/unit/reconcile/connectors/test_remote_query_reader.py b/tests/unit/reconcile/connectors/test_remote_query_reader.py new file mode 100644 index 0000000000..1ae3c95967 --- /dev/null +++ b/tests/unit/reconcile/connectors/test_remote_query_reader.py @@ -0,0 +1,73 @@ +from unittest.mock import MagicMock + +from databricks.labs.lakebridge.reconcile.connectors.remote_query_reader import RemoteQueryReader +from databricks.labs.lakebridge.reconcile.recon_config import JdbcReaderOptions + + +def test_read_data_simple_query(): + spark = MagicMock() + reader = RemoteQueryReader(spark, "my_connection") + + reader.read_data("SELECT * FROM employees", "my_db", "database") + + spark.sql.assert_called_once_with( + "SELECT * FROM remote_query('my_connection', query => 'SELECT * FROM employees', database => 'my_db')" + ) + + +def test_read_data_with_service_name_catalog_key(): + spark = MagicMock() + reader = RemoteQueryReader(spark, "oracle_conn") + + reader.read_data("SELECT 1 FROM dual", "ORCL", "service_name", "dbtable") + + spark.sql.assert_called_once_with( + "SELECT * FROM remote_query('oracle_conn', dbtable => 'SELECT 1 FROM dual', serviceName => 'ORCL')" + ) + + +def test_read_data_with_options(): + spark = MagicMock() + reader = RemoteQueryReader(spark, "tsql_conn") + options = JdbcReaderOptions( + num_partitions=10, + partition_column="id", + lower_bound="0", + upper_bound="1000", + fetchsize=500, + ) + + reader.read_data("SELECT * FROM orders", "my_db", "database", "dbtable", options) + + spark.sql.assert_called_once_with( + "SELECT * FROM remote_query('tsql_conn', " + "dbtable => 'SELECT * FROM orders', " + "database => 'my_db', " + "partitionColumn => 'id', " + "numPartitions => '10', " + "lowerBound => '0', " + "upperBound => '1000', " + "fetchsize => '500')" + ) + + +def test_read_data_escapes_single_quotes(): + spark = MagicMock() + reader = RemoteQueryReader(spark, "my_conn") + + reader.read_data("SELECT * FROM t WHERE name = 'val'", "db", "database") + + spark.sql.assert_called_once_with( + r"SELECT * FROM remote_query('my_conn', query => 'SELECT * FROM t WHERE name = \'val\'', database => 'db')" + ) + + +def test_read_data_without_options(): + spark = MagicMock() + reader = RemoteQueryReader(spark, "sf_conn") + + reader.read_data("SELECT col1 FROM schema.table", "MY_DB", "database", "query") + + spark.sql.assert_called_once_with( + "SELECT * FROM remote_query('sf_conn', query => 'SELECT col1 FROM schema.table', database => 'MY_DB')" + ) diff --git a/tests/unit/reconcile/connectors/test_secrets.py b/tests/unit/reconcile/connectors/test_secrets.py deleted file mode 100644 index dea7515b09..0000000000 --- a/tests/unit/reconcile/connectors/test_secrets.py +++ /dev/null @@ -1,65 +0,0 @@ -import base64 -from unittest.mock import create_autospec - -import pytest - -from databricks.labs.lakebridge.reconcile.connectors.secrets import SecretsMixin -from databricks.sdk import WorkspaceClient -from databricks.sdk.errors import NotFound -from databricks.sdk.service.workspace import GetSecretResponse - - -class SecretsMixinUnderTest(SecretsMixin): - def __init__(self, ws: WorkspaceClient, secret_scope: str): - self._ws = ws - self._secret_scope = secret_scope - - def get_secret(self, secret_key: str) -> str: - return self._get_secret(secret_key) - - def get_secret_or_none(self, secret_key: str) -> str | None: - return self._get_secret_or_none(secret_key) - - -def mock_secret(scope, key): - secret_mock = { - "scope": { - 'user_name': GetSecretResponse( - key='user_name', value=base64.b64encode(bytes('my_user', 'utf-8')).decode('utf-8') - ), - 'password': GetSecretResponse( - key='password', value=base64.b64encode(bytes('my_password', 'utf-8')).decode('utf-8') - ), - } - } - - return secret_mock.get(scope).get(key) - - -def test_get_secrets_happy(): - ws = create_autospec(WorkspaceClient) - ws.secrets.get_secret.side_effect = mock_secret - - sut = SecretsMixinUnderTest(ws, "scope") - - assert sut.get_secret("user_name") == "my_user" - assert sut.get_secret_or_none("user_name") == "my_user" - assert sut.get_secret("password") == "my_password" - assert sut.get_secret_or_none("password") == "my_password" - - -def test_get_secrets_not_found_exception(): - ws = create_autospec(WorkspaceClient) - ws.secrets.get_secret.side_effect = NotFound("Test Exception") - sut = SecretsMixinUnderTest(ws, "scope") - - with pytest.raises(NotFound, match="Secret does not exist with scope: scope and key: unknown : Test Exception"): - sut.get_secret("unknown") - - -def test_get_secrets_not_found_swallow(): - ws = create_autospec(WorkspaceClient) - ws.secrets.get_secret.side_effect = NotFound("Test Exception") - sut = SecretsMixinUnderTest(ws, "scope") - - assert sut.get_secret_or_none("unknown") is None diff --git a/tests/unit/reconcile/connectors/test_snowflake.py b/tests/unit/reconcile/connectors/test_snowflake.py index 114aa42f2a..dfcc0b78e6 100644 --- a/tests/unit/reconcile/connectors/test_snowflake.py +++ b/tests/unit/reconcile/connectors/test_snowflake.py @@ -1,133 +1,28 @@ -import base64 import re -from unittest.mock import MagicMock, create_autospec +from unittest.mock import create_autospec import pytest -from cryptography.hazmat.primitives.asymmetric import rsa -from cryptography.hazmat.primitives import serialization from databricks.labs.lakebridge.reconcile.connectors.models import NormalizedIdentifier from databricks.labs.lakebridge.transpiler.sqlglot.dialect_utils import get_dialect from databricks.labs.lakebridge.reconcile.connectors.snowflake import SnowflakeDataSource -from databricks.labs.lakebridge.reconcile.exception import DataSourceRuntimeException, InvalidSnowflakePemPrivateKey +from databricks.labs.lakebridge.reconcile.connectors.remote_query_reader import RemoteQueryReader +from databricks.labs.lakebridge.reconcile.exception import DataSourceRuntimeException from databricks.labs.lakebridge.reconcile.recon_config import JdbcReaderOptions, Table -from databricks.sdk import WorkspaceClient -from databricks.sdk.service.workspace import GetSecretResponse -from databricks.sdk.errors import NotFound - - -def mock_secret(scope, key): - secret_mock = { - "scope": { - 'sfAccount': GetSecretResponse( - key='sfAccount', value=base64.b64encode(bytes('my_account', 'utf-8')).decode('utf-8') - ), - 'sfUser': GetSecretResponse( - key='sfUser', value=base64.b64encode(bytes('my_user', 'utf-8')).decode('utf-8') - ), - 'sfPassword': GetSecretResponse( - key='sfPassword', value=base64.b64encode(bytes('my_password', 'utf-8')).decode('utf-8') - ), - 'sfDatabase': GetSecretResponse( - key='sfDatabase', value=base64.b64encode(bytes('my_database', 'utf-8')).decode('utf-8') - ), - 'sfSchema': GetSecretResponse( - key='sfSchema', value=base64.b64encode(bytes('my_schema', 'utf-8')).decode('utf-8') - ), - 'sfWarehouse': GetSecretResponse( - key='sfWarehouse', value=base64.b64encode(bytes('my_warehouse', 'utf-8')).decode('utf-8') - ), - 'sfRole': GetSecretResponse( - key='sfRole', value=base64.b64encode(bytes('my_role', 'utf-8')).decode('utf-8') - ), - 'sfUrl': GetSecretResponse(key='sfUrl', value=base64.b64encode(bytes('my_url', 'utf-8')).decode('utf-8')), - } - } - - return secret_mock[scope][key] - - -def generate_pkcs8_pem_key(malformed=False): - private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048) - pem_key = private_key.private_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PrivateFormat.PKCS8, - encryption_algorithm=serialization.NoEncryption(), - ).decode('utf-8') - return pem_key[:50] + "MALFORMED" + pem_key[60:] if malformed else pem_key - - -def mock_private_key_secret(scope, key): - if key == 'pem_private_key': - return GetSecretResponse(key=key, value=base64.b64encode(generate_pkcs8_pem_key().encode()).decode()) - if key == 'pem_private_key_password': - return GetSecretResponse(key=key, value=b''.decode()) - return mock_secret(scope, key) - - -def mock_malformed_private_key_secret(scope, key): - if key == 'pem_private_key': - return GetSecretResponse(key=key, value=base64.b64encode(generate_pkcs8_pem_key(True).encode()).decode()) - if key == 'pem_private_key_password': - return GetSecretResponse(key=key, value=b''.decode()) - return mock_secret(scope, key) - - -def mock_no_auth_key_secret(scope, key): - if key in {'pem_private_key', 'sfPassword'}: - raise NotFound("Secret not found") - return mock_secret(scope, key) def initial_setup(): - pyspark_sql_session = MagicMock() - spark = pyspark_sql_session.SparkSession.builder.getOrCreate() - - # Define the source, workspace, and scope engine = get_dialect("snowflake") - ws = create_autospec(WorkspaceClient) - scope = "scope" - ws.secrets.get_secret.side_effect = mock_secret - return engine, spark, ws, scope - - -def test_get_jdbc_url_happy(): - # initial setup - engine, spark, ws, scope = initial_setup() - # create object for SnowflakeDataSource - dfds = SnowflakeDataSource(engine, spark, ws, scope) - url = dfds.get_jdbc_url - # Assert that the URL is generated correctly - assert url == ( - "jdbc:snowflake://my_account.snowflakecomputing.com" - "/?user=my_user&password=my_password" - "&db=my_database&schema=my_schema" - "&warehouse=my_warehouse&role=my_role" - ) - - -def test_get_jdbc_url_fail(): - # initial setup - engine, spark, ws, scope = initial_setup() - ws.secrets.get_secret.side_effect = mock_secret - # create object for SnowflakeDataSource - dfds = SnowflakeDataSource(engine, spark, ws, scope) - url = dfds.get_jdbc_url - # Assert that the URL is generated correctly - assert url == ( - "jdbc:snowflake://my_account.snowflakecomputing.com" - "/?user=my_user&password=my_password" - "&db=my_database&schema=my_schema" - "&warehouse=my_warehouse&role=my_role" - ) + reader = create_autospec(RemoteQueryReader) + return engine, reader def test_read_data_with_out_options(): # initial setup - engine, spark, ws, scope = initial_setup() + engine, reader = initial_setup() # create object for SnowflakeDataSource - dfds = SnowflakeDataSource(engine, spark, ws, scope) + dfds = SnowflakeDataSource(engine, reader) # Create a Tables configuration object with no JDBC reader options table_conf = Table( source_name="supplier", @@ -137,33 +32,27 @@ def test_read_data_with_out_options(): # Call the read_data method with the Tables configuration dfds.read_data("org", "data", "employee", "select 1 from :tbl", table_conf.jdbc_reader_options) - # spark assertions - spark.read.format.assert_called_with("snowflake") - spark.read.format().option.assert_called_with("dbtable", "(select 1 from org.data.employee) as tmp") - spark.read.format().option().options.assert_called_with( - sfUrl="my_url", - sfUser="my_user", - sfPassword="my_password", - sfDatabase="my_database", - sfSchema="my_schema", - sfWarehouse="my_warehouse", - sfRole="my_role", + # reader assertions — verify the query passed to remote_query reader + reader.read_data.assert_called_once_with( + "select 1 from org.data.employee", + "org", + "database", + "query", ) - spark.read.format().option().options().load.assert_called_once() def test_read_data_with_options(): # initial setup - engine, spark, ws, scope = initial_setup() + engine, reader = initial_setup() # create object for SnowflakeDataSource - dfds = SnowflakeDataSource(engine, spark, ws, scope) + dfds = SnowflakeDataSource(engine, reader) # Create a Tables configuration object with JDBC reader options table_conf = Table( source_name="supplier", target_name="supplier", jdbc_reader_options=JdbcReaderOptions( - number_partitions=100, partition_column="s_nationkey", lower_bound="0", upper_bound="100" + num_partitions=100, partition_column="s_nationkey", lower_bound="0", upper_bound="100" ), select_columns=None, drop_columns=None, @@ -177,59 +66,46 @@ def test_read_data_with_options(): # Call the read_data method with the Tables configuration dfds.read_data("org", "data", "employee", "select 1 from :tbl", table_conf.jdbc_reader_options) - # spark assertions - spark.read.format.assert_called_with("jdbc") - spark.read.format().option.assert_called_with( - "url", - "jdbc:snowflake://my_account.snowflakecomputing.com/?user=my_user&password=" - "my_password&db=my_database&schema=my_schema&warehouse=my_warehouse&role=my_role", + # reader assertions — verify the query passed to remote_query reader + # Note: SnowflakeDataSource.read_data does not pass options to reader.read_data (options are unused for snowflake) + reader.read_data.assert_called_once_with( + "select 1 from org.data.employee", + "org", + "database", + "query", ) - spark.read.format().option().option.assert_called_with("driver", "net.snowflake.client.jdbc.SnowflakeDriver") - spark.read.format().option().option().option.assert_called_with("dbtable", "(select 1 from org.data.employee) tmp") - spark.read.format().option().option().option().options.assert_called_with( - numPartitions=100, partitionColumn='s_nationkey', lowerBound='0', upperBound='100', fetchsize=100 - ) - spark.read.format().option().option().option().options().load.assert_called_once() def test_get_schema(): # initial setup - engine, spark, ws, scope = initial_setup() - # Mocking get secret method to return the required values + engine, reader = initial_setup() # create object for SnowflakeDataSource - dfds = SnowflakeDataSource(engine, spark, ws, scope) + dfds = SnowflakeDataSource(engine, reader) # call test method dfds.get_schema("catalog", "schema", "supplier") - # spark assertions - spark.read.format.assert_called_with("snowflake") - spark.read.format().option.assert_called_with( - "dbtable", - re.sub( - r'\s+', - ' ', - """(select column_name, case when numeric_precision is not null and numeric_scale is not null then + # reader assertions — verify the schema query passed to remote_query reader + reader.read_data.assert_called_once() + call_args = reader.read_data.call_args + source_query = call_args[0][0] + expected_query = re.sub( + r'\s+', + ' ', + """select column_name, case when numeric_precision is not null and numeric_scale is not null then concat(data_type, '(', numeric_precision, ',' , numeric_scale, ')') when lower(data_type) = 'text' then concat('varchar', '(', CHARACTER_MAXIMUM_LENGTH, ')') else data_type end as data_type from catalog.INFORMATION_SCHEMA.COLUMNS where lower(table_name)='supplier' and table_schema = 'SCHEMA' - order by ordinal_position) as tmp""", - ), + order by ordinal_position""", ) - spark.read.format().option().options.assert_called_with( - sfUrl="my_url", - sfUser="my_user", - sfPassword="my_password", - sfDatabase="my_database", - sfSchema="my_schema", - sfWarehouse="my_warehouse", - sfRole="my_role", - ) - spark.read.format().option().options().load.assert_called_once() + assert source_query == expected_query + assert call_args[0][1] == "catalog" + assert call_args[0][2] == "database" + assert call_args[0][3] == "query" def test_read_data_exception_handling(): # initial setup - engine, spark, ws, scope = initial_setup() - dfds = SnowflakeDataSource(engine, spark, ws, scope) + engine, reader = initial_setup() + dfds = SnowflakeDataSource(engine, reader) # Create a Tables configuration object table_conf = Table( source_name="supplier", @@ -244,7 +120,7 @@ def test_read_data_exception_handling(): filters=None, ) - spark.read.format().option().options().load.side_effect = RuntimeError("Test Exception") + reader.read_data.side_effect = RuntimeError("Test Exception") # Call the read_data method with the Tables configuration and assert that a PySparkException is raised with pytest.raises( @@ -256,72 +132,32 @@ def test_read_data_exception_handling(): def test_get_schema_exception_handling(): # initial setup - engine, spark, ws, scope = initial_setup() + engine, reader = initial_setup() - dfds = SnowflakeDataSource(engine, spark, ws, scope) + dfds = SnowflakeDataSource(engine, reader) - spark.read.format().option().options().load.side_effect = RuntimeError("Test Exception") + reader.read_data.side_effect = RuntimeError("Test Exception") # Call the get_schema method with predefined table, schema, and catalog names and assert that a PySparkException # is raised with pytest.raises( DataSourceRuntimeException, - match=r"Runtime exception occurred while fetching schema using select column_name, case when numeric_precision " - "is not null and numeric_scale is not null then concat\\(data_type, '\\(', numeric_precision, ',' , " - "numeric_scale, '\\)'\\) when lower\\(data_type\\) = 'text' then concat\\('varchar', '\\(', " - "CHARACTER_MAXIMUM_LENGTH, '\\)'\\) else data_type end as data_type from catalog.INFORMATION_SCHEMA.COLUMNS " - "where lower\\(table_name\\)='supplier' and table_schema = 'SCHEMA' order by ordinal_position : Test " - "Exception", + match=re.escape( + "Runtime exception occurred while fetching schema using select column_name, case when numeric_precision " + "is not null and numeric_scale is not null then concat(data_type, '(', numeric_precision, ',' , " + "numeric_scale, ')') when lower(data_type) = 'text' then concat('varchar', '(', " + "CHARACTER_MAXIMUM_LENGTH, ')') else data_type end as data_type from catalog.INFORMATION_SCHEMA.COLUMNS " + "where lower(table_name)='supplier' and table_schema = 'SCHEMA' order by ordinal_position : Test " + "Exception" + ), ): dfds.get_schema("catalog", "schema", "supplier") -def test_read_data_without_options_private_key(): - engine, spark, ws, scope = initial_setup() - ws.secrets.get_secret.side_effect = mock_private_key_secret - dfds = SnowflakeDataSource(engine, spark, ws, scope) - table_conf = Table(source_name="supplier", target_name="supplier") - dfds.read_data("org", "data", "employee", "select 1 from :tbl", table_conf.jdbc_reader_options) - spark.read.format.assert_called_with("snowflake") - spark.read.format().option.assert_called_with("dbtable", "(select 1 from org.data.employee) as tmp") - expected_options = { - "sfUrl": "my_url", - "sfUser": "my_user", - "sfDatabase": "my_database", - "sfSchema": "my_schema", - "sfWarehouse": "my_warehouse", - "sfRole": "my_role", - } - actual_options = spark.read.format().option().options.call_args[1] - actual_options.pop("pem_private_key", None) - assert actual_options == expected_options - spark.read.format().option().options().load.assert_called_once() - - -def test_read_data_without_options_malformed_private_key(): - engine, spark, ws, scope = initial_setup() - ws.secrets.get_secret.side_effect = mock_malformed_private_key_secret - dfds = SnowflakeDataSource(engine, spark, ws, scope) - table_conf = Table(source_name="supplier", target_name="supplier") - with pytest.raises(InvalidSnowflakePemPrivateKey, match="Failed to load or process the provided PEM private key."): - dfds.read_data("org", "data", "employee", "select 1 from :tbl", table_conf.jdbc_reader_options) - - -def test_read_data_without_any_auth(): - engine, spark, ws, scope = initial_setup() - ws.secrets.get_secret.side_effect = mock_no_auth_key_secret - dfds = SnowflakeDataSource(engine, spark, ws, scope) - table_conf = Table(source_name="supplier", target_name="supplier") - with pytest.raises( - NotFound, match='sfPassword and pem_private_key not found. Either one is required for snowflake auth.' - ): - dfds.read_data("org", "data", "employee", "select 1 from :tbl", table_conf.jdbc_reader_options) - - @pytest.mark.skip("Turned off till we can handle case sensitivity.") def test_normalize_identifier(): - engine, spark, ws, scope = initial_setup() - data_source = SnowflakeDataSource(engine, spark, ws, scope) + engine, reader = initial_setup() + data_source = SnowflakeDataSource(engine, reader) assert data_source.normalize_identifier("a") == NormalizedIdentifier("`a`", '"a"') assert data_source.normalize_identifier('"b"') == NormalizedIdentifier("`b`", '"b"') diff --git a/tests/unit/reconcile/connectors/test_sql_server.py b/tests/unit/reconcile/connectors/test_sql_server.py index 175a91086b..938272dcb7 100644 --- a/tests/unit/reconcile/connectors/test_sql_server.py +++ b/tests/unit/reconcile/connectors/test_sql_server.py @@ -1,76 +1,34 @@ -import base64 import re -from unittest.mock import MagicMock, create_autospec +from unittest.mock import create_autospec import pytest from databricks.labs.lakebridge.reconcile.connectors.models import NormalizedIdentifier from databricks.labs.lakebridge.transpiler.sqlglot.dialect_utils import get_dialect from databricks.labs.lakebridge.reconcile.connectors.tsql import TSQLServerDataSource +from databricks.labs.lakebridge.reconcile.connectors.remote_query_reader import RemoteQueryReader from databricks.labs.lakebridge.reconcile.exception import DataSourceRuntimeException from databricks.labs.lakebridge.reconcile.recon_config import JdbcReaderOptions, Table -from databricks.sdk import WorkspaceClient -from databricks.sdk.service.workspace import GetSecretResponse - - -def mock_secret(scope, key): - scope_secret_mock = { - "scope": { - 'user': GetSecretResponse(key='user', value=base64.b64encode('my_user'.encode('utf-8')).decode('utf-8')), - 'password': GetSecretResponse( - key='password', value=base64.b64encode(bytes('my_password', 'utf-8')).decode('utf-8') - ), - 'host': GetSecretResponse(key='host', value=base64.b64encode(bytes('my_host', 'utf-8')).decode('utf-8')), - 'port': GetSecretResponse(key='port', value=base64.b64encode(bytes('777', 'utf-8')).decode('utf-8')), - 'database': GetSecretResponse( - key='database', value=base64.b64encode(bytes('my_database', 'utf-8')).decode('utf-8') - ), - 'encrypt': GetSecretResponse(key='encrypt', value=base64.b64encode(bytes('true', 'utf-8')).decode('utf-8')), - 'trustServerCertificate': GetSecretResponse( - key='trustServerCertificate', value=base64.b64encode(bytes('true', 'utf-8')).decode('utf-8') - ), - } - } - - return scope_secret_mock[scope][key] def initial_setup(): - pyspark_sql_session = MagicMock() - spark = pyspark_sql_session.SparkSession.builder.getOrCreate() - - # Define the source, workspace, and scope engine = get_dialect("tsql") - ws = create_autospec(WorkspaceClient) - scope = "scope" - ws.secrets.get_secret.side_effect = mock_secret - return engine, spark, ws, scope - - -def test_get_jdbc_url_happy(): - # initial setup - engine, spark, ws, scope = initial_setup() - # create object for TSQLServerDataSource - data_source = TSQLServerDataSource(engine, spark, ws, scope) - url = data_source.get_jdbc_url - # Assert that the URL is generated correctly - assert url == ( - """jdbc:sqlserver://my_host:777;databaseName=my_database;encrypt=true;trustServerCertificate=true;""" - ) + reader = create_autospec(RemoteQueryReader) + return engine, reader def test_read_data_with_options(): # initial setup - engine, spark, ws, scope = initial_setup() + engine, reader = initial_setup() # create object for MSSQLServerDataSource - data_source = TSQLServerDataSource(engine, spark, ws, scope) + data_source = TSQLServerDataSource(engine, reader) # Create a Tables configuration object with JDBC reader options table_conf = Table( source_name="src_supplier", target_name="tgt_supplier", jdbc_reader_options=JdbcReaderOptions( - number_partitions=100, partition_column="s_partition_key", lower_bound="0", upper_bound="100" + num_partitions=100, partition_column="s_partition_key", lower_bound="0", upper_bound="100" ), ) @@ -79,45 +37,31 @@ def test_read_data_with_options(): "org", "data", "employee", "WITH tmp AS (SELECT * from :tbl) select 1 from tmp", table_conf.jdbc_reader_options ) - # spark assertions - spark.read.format.assert_called_with("jdbc") - spark.read.format().option.assert_called_with( - "url", - "jdbc:sqlserver://my_host:777;databaseName=my_database;encrypt=true;trustServerCertificate=true;", - ) - spark.read.format().option().option.assert_called_with("driver", "com.microsoft.sqlserver.jdbc.SQLServerDriver") - spark.read.format().option().option().option.assert_called_with( - "dbtable", "(WITH tmp AS (SELECT * from org.data.[employee]) select 1 from tmp) tmp" + # reader assertions — verify the query passed to remote_query reader + reader.read_data.assert_called_once_with( + "WITH tmp AS (SELECT * from data.[employee]) select 1 from tmp", + "org", + "database", + "query", + table_conf.jdbc_reader_options, ) - actual_args = spark.read.format().option().option().option().options.call_args.kwargs - expected_args = { - "numPartitions": 100, - "partitionColumn": "s_partition_key", - "lowerBound": '0', - "upperBound": "100", - "fetchsize": 100, - "user": "my_user", - "password": "my_password", - } - assert actual_args == expected_args - spark.read.format().option().option().option().options().load.assert_called_once() def test_get_schema(): # initial setup - engine, spark, ws, scope = initial_setup() + engine, reader = initial_setup() # Mocking get secret method to return the required values - data_source = TSQLServerDataSource(engine, spark, ws, scope) + data_source = TSQLServerDataSource(engine, reader) # call test method data_source.get_schema("org", "schema", "supplier") - # spark assertions - spark.read.format.assert_called_with("jdbc") - spark.read.format().option().option().option.assert_called_with( - "dbtable", - re.sub( - r'\s+', - ' ', - r"""(SELECT + # reader assertions — verify the schema query passed to remote_query reader + reader.read_data.assert_called_once() + call_args = reader.read_data.call_args + source_query = call_args[0][0] + expected_query = re.sub( + r'\s+', + ' ', + r"""SELECT COLUMN_NAME AS 'column_name', CASE WHEN DATA_TYPE IN ('int', 'bigint') @@ -145,17 +89,20 @@ def test_get_schema(): WHERE LOWER(TABLE_NAME) = LOWER('supplier') AND LOWER(TABLE_SCHEMA) = LOWER('schema') AND LOWER(TABLE_CATALOG) = LOWER('org') - ) tmp""", - ), + """, ) + assert source_query == expected_query + assert call_args[0][1] == "org" + assert call_args[0][2] == "database" + assert call_args[0][3] == "query" def test_get_schema_exception_handling(): # initial setup - engine, spark, ws, scope = initial_setup() - data_source = TSQLServerDataSource(engine, spark, ws, scope) + engine, reader = initial_setup() + data_source = TSQLServerDataSource(engine, reader) - spark.read.format().option().option().option().options().load.side_effect = RuntimeError("Test Exception") + reader.read_data.side_effect = RuntimeError("Test Exception") # Call the get_schema method with predefined table, schema, and catalog names and assert that a PySparkException # is raised @@ -169,8 +116,8 @@ def test_get_schema_exception_handling(): def test_normalize_identifier(): - engine, spark, ws, scope = initial_setup() - data_source = TSQLServerDataSource(engine, spark, ws, scope) + engine, reader = initial_setup() + data_source = TSQLServerDataSource(engine, reader) assert data_source.normalize_identifier("a") == NormalizedIdentifier("`a`", "[a]") assert data_source.normalize_identifier('"b"') == NormalizedIdentifier("`b`", "[b]") diff --git a/tests/unit/reconcile/query_builder/test_threshold_query.py b/tests/unit/reconcile/query_builder/test_threshold_query.py index 74a90b0076..292735356e 100644 --- a/tests/unit/reconcile/query_builder/test_threshold_query.py +++ b/tests/unit/reconcile/query_builder/test_threshold_query.py @@ -112,7 +112,7 @@ def test_build_threshold_query_with_multiple_threshold( ): table_conf = normalized_table_conf_with_opts table_conf.jdbc_reader_options = JdbcReaderOptions( - number_partitions=100, partition_column="`s_phone`", lower_bound="0", upper_bound="100" + num_partitions=100, partition_column="`s_phone`", lower_bound="0", upper_bound="100" ) table_conf.column_thresholds = [ ColumnThresholds(column_name="`s_acctbal`", lower_bound="5%", upper_bound="-5%", type="float"), diff --git a/tests/unit/reconcile/test_source_adapter.py b/tests/unit/reconcile/test_source_adapter.py index 5a9cc4032d..f245c7c767 100644 --- a/tests/unit/reconcile/test_source_adapter.py +++ b/tests/unit/reconcile/test_source_adapter.py @@ -18,9 +18,8 @@ def test_create_adapter_for_snowflake_dialect(): scope = "scope" data_source = create_adapter(engine, spark, ws, scope) - snowflake_data_source = SnowflakeDataSource(engine, spark, ws, scope).__class__ - assert isinstance(data_source, snowflake_data_source) + assert isinstance(data_source, SnowflakeDataSource) def test_create_adapter_for_oracle_dialect(): @@ -30,9 +29,8 @@ def test_create_adapter_for_oracle_dialect(): scope = "scope" data_source = create_adapter(engine, spark, ws, scope) - oracle_data_source = OracleDataSource(engine, spark, ws, scope).__class__ - assert isinstance(data_source, oracle_data_source) + assert isinstance(data_source, OracleDataSource) def test_create_adapter_for_databricks_dialect(): @@ -42,9 +40,8 @@ def test_create_adapter_for_databricks_dialect(): scope = "scope" data_source = create_adapter(engine, spark, ws, scope) - databricks_data_source = DatabricksDataSource(engine, spark, ws, scope).__class__ - assert isinstance(data_source, databricks_data_source) + assert isinstance(data_source, DatabricksDataSource) def test_raise_exception_for_unknown_dialect(): diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 3202bb86f3..979e744415 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -1,6 +1,13 @@ from databricks.labs.blueprint.installation import MockInstallation -from databricks.labs.lakebridge.config import TranspileConfig, TableRecon +from databricks.labs.lakebridge.config import ( + ReconcileConfig, + ReconcileMetadataConfig, + SourceConnectionConfig, + TableRecon, + TargetConnectionConfig, + TranspileConfig, +) from databricks.labs.lakebridge.reconcile.recon_config import Table @@ -95,3 +102,118 @@ def test_reconcile_table_config_default_serialization() -> None: loaded = installation.load(TableRecon) assert loaded.tables == config.tables + + +def test_reconcile_v1_migrate_non_databricks() -> None: + """v1 reconcile.yml for an external source migrates to v2 with a placeholder UC connection name.""" + installation = MockInstallation( + { + "reconcile.yml": { + "data_source": "snowflake", + "secret_scope": "remorph_snowflake", + "report_type": "all", + "database_config": { + "source_catalog": "snowflake_sample_data", + "source_schema": "tpch_sf1000", + "target_catalog": "tpch", + "target_schema": "1000gb", + }, + "metadata_config": {"catalog": "remorph", "schema": "reconcile", "volume": "reconcile_volume"}, + "version": 1, + } + } + ) + + loaded = installation.load(ReconcileConfig) + + assert loaded == ReconcileConfig( + report_type="all", + source=SourceConnectionConfig( + dialect="snowflake", + catalog="snowflake_sample_data", + schema="tpch_sf1000", + uc_connection_name="TODO", + ), + target=TargetConnectionConfig(catalog="tpch", schema="1000gb"), + metadata_config=ReconcileMetadataConfig(catalog="remorph", schema="reconcile", volume="reconcile_volume"), + ) + + +def test_reconcile_v1_migrate_databricks() -> None: + """v1 reconcile.yml for databricks source migrates without setting a UC connection name.""" + installation = MockInstallation( + { + "reconcile.yml": { + "data_source": "databricks", + "secret_scope": "remorph_databricks", + "report_type": "all", + "database_config": { + "source_catalog": "src_catalog", + "source_schema": "src_schema", + "target_catalog": "tgt_catalog", + "target_schema": "tgt_schema", + }, + "metadata_config": {"catalog": "remorph", "schema": "reconcile", "volume": "reconcile_volume"}, + "version": 1, + } + } + ) + + loaded = installation.load(ReconcileConfig) + + assert loaded.source.dialect == "databricks" + assert loaded.source.uc_connection_name is None + assert loaded.source.catalog == "src_catalog" + + +def test_reconcile_v1_migrate_oracle_without_source_catalog() -> None: + """In v1 source_catalog was optional for Oracle (service name); migrate defaults to ORCL.""" + installation = MockInstallation( + { + "reconcile.yml": { + "data_source": "oracle", + "secret_scope": "remorph_oracle", + "report_type": "all", + "database_config": { + "source_schema": "HR", + "target_catalog": "tgt_catalog", + "target_schema": "tgt_schema", + }, + "metadata_config": {"catalog": "remorph", "schema": "reconcile", "volume": "reconcile_volume"}, + "version": 1, + } + } + ) + + loaded = installation.load(ReconcileConfig) + + assert loaded.source.dialect == "oracle" + assert loaded.source.catalog == "ORCL" + assert loaded.source.uc_connection_name == "TODO" + + +def test_reconcile_v1_migrate_drops_orphan_fields() -> None: + """secret_scope and the legacy tables field are not retained in the migrated config.""" + installation = MockInstallation( + { + "reconcile.yml": { + "data_source": "snowflake", + "secret_scope": "remorph_snowflake", + "report_type": "all", + "database_config": { + "source_catalog": "src", + "source_schema": "sch", + "target_catalog": "tgt", + "target_schema": "tsch", + }, + "tables": {"filter_type": "all", "tables_list": ["*"]}, + "metadata_config": {"catalog": "remorph", "schema": "reconcile", "volume": "reconcile_volume"}, + "version": 1, + } + } + ) + + # Should load cleanly without choking on orphan fields. + loaded = installation.load(ReconcileConfig) + assert loaded.source.dialect == "snowflake" + assert loaded.target.catalog == "tgt" diff --git a/tests/unit/test_install.py b/tests/unit/test_install.py index a6b0a942b1..8173e8cf4c 100644 --- a/tests/unit/test_install.py +++ b/tests/unit/test_install.py @@ -10,12 +10,13 @@ from databricks.labs.blueprint.tui import MockPrompts from databricks.labs.blueprint.wheels import ProductInfo, WheelsV2 from databricks.labs.lakebridge.config import ( - DatabaseConfig, LSPConfigOptionV1, LSPPromptMethod, LakebridgeConfiguration, ReconcileConfig, ReconcileMetadataConfig, + SourceConnectionConfig, + TargetConnectionConfig, TranspileConfig, ProfilerDashboardConfig, ProfilerDashboardMetadataConfig, @@ -607,10 +608,11 @@ def test_configure_reconcile_installation_config_error_continue_install(ws: Work { r"Select the Data Source": str(RECONCILE_DATA_SOURCES.index("oracle")), r"Select the report type": str(RECONCILE_REPORT_TYPES.index("all")), - r"Enter Secret scope name to store .* connection details / secrets": "remorph_oracle", - r"Enter source database name for .*": "tpch_sf1000", - r"Enter target catalog name for Databricks": "tpch", - r"Enter target schema name for Databricks": "1000gb", + r"Enter Unity Catalog .* connection name": "my_oracle_conn", + r"Enter Oracle service name": "ORCL", + r"Enter .* database name": "tpch_sf1000", + r"Enter target Databricks catalog name": "tpch", + r"Enter target Databricks schema name": "1000gb", r"Open .* in the browser?": "no", } ) @@ -619,18 +621,22 @@ def test_configure_reconcile_installation_config_error_continue_install(ws: Work "reconcile.yml": { "source_dialect": "oracle", # Invalid key "report_type": "all", - "secret_scope": "remorph_oracle", - "database_config": { - "source_schema": "tpch_sf1000", - "target_catalog": "tpch", - "target_schema": "1000gb", + "source": { + "dialect": "oracle", + "catalog": "ORCL", + "schema": "tpch_sf1000", + "uc_connection_name": "my_oracle_conn", + }, + "target": { + "catalog": "tpch", + "schema": "1000gb", }, "metadata_config": { "catalog": "remorph", "schema": "reconcile", "volume": "reconcile_volume", }, - "version": 1, + "version": 2, } } ) @@ -661,13 +667,16 @@ def test_configure_reconcile_installation_config_error_continue_install(ws: Work expected_config = LakebridgeConfiguration( reconcile=ReconcileConfig( - data_source="oracle", report_type="all", - secret_scope="remorph_oracle", - database_config=DatabaseConfig( - source_schema="tpch_sf1000", - target_catalog="tpch", - target_schema="1000gb", + source=SourceConnectionConfig( + dialect="oracle", + catalog="ORCL", + schema="tpch_sf1000", + uc_connection_name="my_oracle_conn", + ), + target=TargetConnectionConfig( + catalog="tpch", + schema="1000gb", ), metadata_config=ReconcileMetadataConfig( catalog="remorph", @@ -682,20 +691,23 @@ def test_configure_reconcile_installation_config_error_continue_install(ws: Work installation.assert_file_written( "reconcile.yml", { - "data_source": "oracle", "report_type": "all", - "secret_scope": "remorph_oracle", - "database_config": { - "source_schema": "tpch_sf1000", - "target_catalog": "tpch", - "target_schema": "1000gb", + "source": { + "dialect": "oracle", + "catalog": "ORCL", + "schema": "tpch_sf1000", + "uc_connection_name": "my_oracle_conn", + }, + "target": { + "catalog": "tpch", + "schema": "1000gb", }, "metadata_config": { "catalog": "remorph", "schema": "reconcile", "volume": "reconcile_volume", }, - "version": 1, + "version": 2, }, ) @@ -706,11 +718,11 @@ def test_configure_reconcile_no_existing_installation(ws: WorkspaceClient) -> No { r"Select the Data Source": str(RECONCILE_DATA_SOURCES.index("snowflake")), r"Select the report type": str(RECONCILE_REPORT_TYPES.index("all")), - r"Enter Secret scope name to store .* connection details / secrets": "remorph_snowflake", - r"Enter source catalog name for .*": "snowflake_sample_data", - r"Enter source schema name for .*": "tpch_sf1000", - r"Enter target catalog name for Databricks": "tpch", - r"Enter target schema name for Databricks": "1000gb", + r"Enter Unity Catalog .* connection name": "my_snowflake_conn", + r"Enter .* database name": "snowflake_sample_data", + r"Enter .* schema name": "tpch_sf1000", + r"Enter target Databricks catalog name": "tpch", + r"Enter target Databricks schema name": "1000gb", r"Open .* in the browser?": "yes", } ) @@ -741,14 +753,16 @@ def test_configure_reconcile_no_existing_installation(ws: WorkspaceClient) -> No expected_config = LakebridgeConfiguration( reconcile=ReconcileConfig( - data_source="snowflake", report_type="all", - secret_scope="remorph_snowflake", - database_config=DatabaseConfig( - source_schema="tpch_sf1000", - target_catalog="tpch", - target_schema="1000gb", - source_catalog="snowflake_sample_data", + source=SourceConnectionConfig( + dialect="snowflake", + catalog="snowflake_sample_data", + schema="tpch_sf1000", + uc_connection_name="my_snowflake_conn", + ), + target=TargetConnectionConfig( + catalog="tpch", + schema="1000gb", ), metadata_config=ReconcileMetadataConfig( catalog="remorph", @@ -763,21 +777,23 @@ def test_configure_reconcile_no_existing_installation(ws: WorkspaceClient) -> No installation.assert_file_written( "reconcile.yml", { - "data_source": "snowflake", "report_type": "all", - "secret_scope": "remorph_snowflake", - "database_config": { - "source_catalog": "snowflake_sample_data", - "source_schema": "tpch_sf1000", - "target_catalog": "tpch", - "target_schema": "1000gb", + "source": { + "dialect": "snowflake", + "catalog": "snowflake_sample_data", + "schema": "tpch_sf1000", + "uc_connection_name": "my_snowflake_conn", + }, + "target": { + "catalog": "tpch", + "schema": "1000gb", }, "metadata_config": { "catalog": "remorph", "schema": "reconcile", "volume": "reconcile_volume", }, - "version": 1, + "version": 2, }, ) @@ -787,12 +803,11 @@ def test_configure_reconcile_databricks_no_existing_installation(ws: WorkspaceCl prompts = MockPrompts( { r"Select the Data Source": str(RECONCILE_DATA_SOURCES.index("databricks")), - r"Enter Secret scope name to store .* connection details / secrets": "remorph_databricks", r"Select the report type": str(RECONCILE_REPORT_TYPES.index("all")), - r"Enter source catalog name for .*": "databricks_catalog", - r"Enter source schema name for .*": "some_schema", - r"Enter target catalog name for Databricks": "tpch", - r"Enter target schema name for Databricks": "1000gb", + r"Enter .* catalog name": "databricks_catalog", + r"Enter .* schema name": "some_schema", + r"Enter target Databricks catalog name": "tpch", + r"Enter target Databricks schema name": "1000gb", r"Open .* in the browser?": "yes", } ) @@ -823,14 +838,15 @@ def test_configure_reconcile_databricks_no_existing_installation(ws: WorkspaceCl expected_config = LakebridgeConfiguration( reconcile=ReconcileConfig( - data_source="databricks", report_type="all", - secret_scope="remorph_databricks", - database_config=DatabaseConfig( - source_schema="some_schema", - target_catalog="tpch", - target_schema="1000gb", - source_catalog="databricks_catalog", + source=SourceConnectionConfig( + dialect="databricks", + catalog="databricks_catalog", + schema="some_schema", + ), + target=TargetConnectionConfig( + catalog="tpch", + schema="1000gb", ), metadata_config=ReconcileMetadataConfig( catalog="remorph", @@ -845,21 +861,22 @@ def test_configure_reconcile_databricks_no_existing_installation(ws: WorkspaceCl installation.assert_file_written( "reconcile.yml", { - "data_source": "databricks", "report_type": "all", - "secret_scope": "remorph_databricks", - "database_config": { - "source_catalog": "databricks_catalog", - "source_schema": "some_schema", - "target_catalog": "tpch", - "target_schema": "1000gb", + "source": { + "dialect": "databricks", + "catalog": "databricks_catalog", + "schema": "some_schema", + }, + "target": { + "catalog": "tpch", + "schema": "1000gb", }, "metadata_config": { "catalog": "remorph", "schema": "reconcile", "volume": "reconcile_volume", }, - "version": 1, + "version": 2, }, ) @@ -880,11 +897,11 @@ def test_configure_all_override_installation( r"Open .* in the browser?": "no", r"Select the Data Source": str(RECONCILE_DATA_SOURCES.index("snowflake")), r"Select the report type": str(RECONCILE_REPORT_TYPES.index("all")), - r"Enter Secret scope name to store .* connection details / secrets": "remorph_snowflake", - r"Enter source catalog name for .*": "snowflake_sample_data", - r"Enter source schema name for .*": "tpch_sf1000", - r"Enter target catalog name for Databricks": "tpch", - r"Enter target schema name for Databricks": "1000gb", + r"Enter Unity Catalog .* connection name": "my_snowflake_conn", + r"Enter .* database name": "snowflake_sample_data", + r"Enter .* schema name": "tpch_sf1000", + r"Enter target Databricks catalog name": "tpch", + r"Enter target Databricks schema name": "1000gb", # Profiler Configuration Prompts r"Select the source technology": "0", r"Enter the path to the profiler extract file:": "", @@ -906,21 +923,23 @@ def test_configure_all_override_installation( "version": 3, }, "reconcile.yml": { - "data_source": "snowflake", "report_type": "all", - "secret_scope": "remorph_snowflake", - "database_config": { - "source_catalog": "snowflake_sample_data", - "source_schema": "tpch_sf1000", - "target_catalog": "tpch", - "target_schema": "1000gb", + "source": { + "dialect": "snowflake", + "catalog": "snowflake_sample_data", + "schema": "tpch_sf1000", + "uc_connection_name": "my_snowflake_conn", + }, + "target": { + "catalog": "tpch", + "schema": "1000gb", }, "metadata_config": { "catalog": "remorph", "schema": "reconcile", "volume": "reconcile_volume", }, - "version": 1, + "version": 2, }, } ) @@ -963,14 +982,16 @@ def test_configure_all_override_installation( ) expected_reconcile_config = ReconcileConfig( - data_source="snowflake", report_type="all", - secret_scope="remorph_snowflake", - database_config=DatabaseConfig( - source_schema="tpch_sf1000", - target_catalog="tpch", - target_schema="1000gb", - source_catalog="snowflake_sample_data", + source=SourceConnectionConfig( + dialect="snowflake", + catalog="snowflake_sample_data", + schema="tpch_sf1000", + uc_connection_name="my_snowflake_conn", + ), + target=TargetConnectionConfig( + catalog="tpch", + schema="1000gb", ), metadata_config=ReconcileMetadataConfig( catalog="remorph", @@ -1015,21 +1036,23 @@ def test_configure_all_override_installation( installation.assert_file_written( "reconcile.yml", { - "data_source": "snowflake", "report_type": "all", - "secret_scope": "remorph_snowflake", - "database_config": { - "source_catalog": "snowflake_sample_data", - "source_schema": "tpch_sf1000", - "target_catalog": "tpch", - "target_schema": "1000gb", + "source": { + "dialect": "snowflake", + "catalog": "snowflake_sample_data", + "schema": "tpch_sf1000", + "uc_connection_name": "my_snowflake_conn", + }, + "target": { + "catalog": "tpch", + "schema": "1000gb", }, "metadata_config": { "catalog": "remorph", "schema": "reconcile", "volume": "reconcile_volume", }, - "version": 1, + "version": 2, }, ) From 1c32cbbb05db4e7ada5f7f8a25c192fb30834d50 Mon Sep 17 00:00:00 2001 From: Andrew Snare Date: Fri, 8 May 2026 16:08:29 +0200 Subject: [PATCH 05/79] Fix project packaging for source and wheel distributions (#2419) ## Changes This PR updates the configuration used to build the source and wheel distributions from this repository. With these changes: - The source distribution is correct; - The wheel can be built from either the source distribution (or directly from the project repository). As a bonus a lot of unintended resources (documentation) are no longer included in the wheel that we build. ### Relevant implementation details The build configuration was previously incorrect, and worked coincidentally when building a wheel directly from this repository but not when building from the source distribution. A quick summary of the gory details: - A source distribution is intended to facilitate, in a reproducible manner: a) testing the package; b) building a binary wheel distribution. - Wheels can be built _either_ from a source distribution _or_ from the project repository. - Previously, using hatch, the wheel was produced directly from the project repository. A source distribution was produced, but its layout was different: a wheel built from this was bogus and didn't contain the intended package sources. - Now, using uv, the wheel is produced in a 2-step process: 1. Build the source distribution; 2. Build the wheel from the source distribution. This exposed the problem with the source distribution. With these changes building the wheel directly from the project repo or the source distribution yield the same results. This also provided an opportunity to ensure that both the source distribution and wheel only include the intended files from the repository. ### Tests - manually tested --- pyproject.toml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 77f56d080b..e56c47c3d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,9 +57,20 @@ requires = [ ] build-backend = "hatchling.build" -[tool.hatch.build] -sources = ["src"] -include = ["src"] +[tool.hatch.build.targets.sdist] +only-include = [ + "/.build-constraints.txt", + "/README.*", + "/LICEN[CS]E", + "/NOTICE", + "/pyproject.toml", + "/src/", + "/tests/", + "/uv.lock", +] + +[tool.hatch.build.targets.wheel] +packages = ["src/databricks"] [tool.hatch.version] path = "src/databricks/labs/lakebridge/__about__.py" From 37bb1b59ac8086a7f9b8770f44c61da516684a22 Mon Sep 17 00:00:00 2001 From: Will Girten <47335283+goodwillpunning@users.noreply.github.com> Date: Mon, 11 May 2026 03:37:31 -0400 Subject: [PATCH 06/79] Add SQL Server Profiler (#2151) ## Changes ### What does this PR do? This PR adds a new profiler for `mssql`. ### Relevant implementation details This profiler closely follows the implementation of the Azure Synapse profiler. However, this implementation provides a `last_execution_time` param for all `mssql` server queries, allowing for a future scheduler component to hook in an repeat queries. ### Caveats/things to watch out for when reviewing: This profiler will not support on-prem `mssql` servers. ### Linked issues N/A ### Functionality - [ ] added relevant user documentation - [ ] added new CLI command - [ ] modified existing command: `databricks labs lakebridge ...` - [X] added new profiler component - [ ] ... +add your own ### Tests - [X] manually tested - [ ] added unit tests - [ ] added integration tests --------- Co-authored-by: M Abulazm --- .../docs/assessment/profiler/index.mdx | 1 + .../docs/assessment/profiler/mssql.mdx | 261 ++++++++++++ pyproject.toml | 5 + .../labs/lakebridge/assessments/_constants.py | 3 +- .../assessments/configure_assessment.py | 17 +- .../resources/assessments/mssql/__init__.py | 0 .../assessments/mssql/activity_extract.py | 75 ++++ .../resources/assessments/mssql/columns.sql | 29 ++ .../assessments/mssql/common/__init__.py | 0 .../assessments/mssql/common/connector.py | 22 + .../assessments/mssql/common/functions.py | 9 + .../assessments/mssql/common/queries.py | 385 ++++++++++++++++++ .../assessments/mssql/cpu_utilization.sql | 32 ++ .../assessments/mssql/database_sizes.sql | 20 + .../resources/assessments/mssql/databases.sql | 12 + .../assessments/mssql/indexed_views.sql | 17 + .../assessments/mssql/info_extract.py | 109 +++++ .../assessments/mssql/pipeline_config.yml | 38 ++ .../assessments/mssql/procedure_stats.sql | 17 + .../assessments/mssql/query_stats.sql | 64 +++ .../resources/assessments/mssql/routines.sql | 28 ++ .../resources/assessments/mssql/sessions.sql | 25 ++ .../resources/assessments/mssql/sys_info.sql | 9 + .../assessments/mssql/table_sizes.sql | 36 ++ .../resources/assessments/mssql/tables.sql | 10 + .../resources/assessments/mssql/views.sql | 12 + .../synapse/common/duckdb_helpers.py | 15 + .../cli/test_profiler_connection_cli.py | 3 +- tests/unit/assessment/test_assessment.py | 22 +- tests/unit/test_install.py | 4 +- uv.lock | 261 +++++++----- 31 files changed, 1422 insertions(+), 119 deletions(-) create mode 100644 docs/lakebridge/docs/assessment/profiler/mssql.mdx create mode 100644 src/databricks/labs/lakebridge/resources/assessments/mssql/__init__.py create mode 100644 src/databricks/labs/lakebridge/resources/assessments/mssql/activity_extract.py create mode 100644 src/databricks/labs/lakebridge/resources/assessments/mssql/columns.sql create mode 100644 src/databricks/labs/lakebridge/resources/assessments/mssql/common/__init__.py create mode 100644 src/databricks/labs/lakebridge/resources/assessments/mssql/common/connector.py create mode 100644 src/databricks/labs/lakebridge/resources/assessments/mssql/common/functions.py create mode 100644 src/databricks/labs/lakebridge/resources/assessments/mssql/common/queries.py create mode 100644 src/databricks/labs/lakebridge/resources/assessments/mssql/cpu_utilization.sql create mode 100644 src/databricks/labs/lakebridge/resources/assessments/mssql/database_sizes.sql create mode 100644 src/databricks/labs/lakebridge/resources/assessments/mssql/databases.sql create mode 100644 src/databricks/labs/lakebridge/resources/assessments/mssql/indexed_views.sql create mode 100644 src/databricks/labs/lakebridge/resources/assessments/mssql/info_extract.py create mode 100644 src/databricks/labs/lakebridge/resources/assessments/mssql/pipeline_config.yml create mode 100644 src/databricks/labs/lakebridge/resources/assessments/mssql/procedure_stats.sql create mode 100644 src/databricks/labs/lakebridge/resources/assessments/mssql/query_stats.sql create mode 100644 src/databricks/labs/lakebridge/resources/assessments/mssql/routines.sql create mode 100644 src/databricks/labs/lakebridge/resources/assessments/mssql/sessions.sql create mode 100644 src/databricks/labs/lakebridge/resources/assessments/mssql/sys_info.sql create mode 100644 src/databricks/labs/lakebridge/resources/assessments/mssql/table_sizes.sql create mode 100644 src/databricks/labs/lakebridge/resources/assessments/mssql/tables.sql create mode 100644 src/databricks/labs/lakebridge/resources/assessments/mssql/views.sql diff --git a/docs/lakebridge/docs/assessment/profiler/index.mdx b/docs/lakebridge/docs/assessment/profiler/index.mdx index d0e13b7f15..d3bd836d6a 100644 --- a/docs/lakebridge/docs/assessment/profiler/index.mdx +++ b/docs/lakebridge/docs/assessment/profiler/index.mdx @@ -33,6 +33,7 @@ Key capabilities: | Source Platform | Configuration Status | |:---------------:|:-------------------:| | Azure Synapse | ✅ | +| Microsoft SQL Server | ✅ | ## Configure Profiler diff --git a/docs/lakebridge/docs/assessment/profiler/mssql.mdx b/docs/lakebridge/docs/assessment/profiler/mssql.mdx new file mode 100644 index 0000000000..066881c664 --- /dev/null +++ b/docs/lakebridge/docs/assessment/profiler/mssql.mdx @@ -0,0 +1,261 @@ +--- +sidebar_position: 2 +title: MSSQL Profiler Details +--- +import Admonition from '@theme/Admonition'; + +# MSSQL Profiler Details + +- [Prerequisites](#prerequisites) +- [Profiled Tables and Views](#profiled-tables-and-views) +- [Configure Connection to MSSQL](#configure-connection-to-mssql) +- [Execute the Profiler](#execute-the-profiler) + +## Prerequisites + +### 1. Download +- ODBC driver for SQL Server (https://learn.microsoft.com/en-us/sql/connect/odbc/download-odbc-driver-for-sql-server?view=sql-server-ver18) +- (Windows only) Visual C++ (https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist?view=msvc-170) + +### 2. SQL Server Authentication + +:::warning Attention: +Multi-factor Authentication (MFA) is **not** supported. The MSSQL profiler connects via ODBC, which does not support MFA. +You must use **SQL Authentication** (username and password) to connect to the SQL Server instance. +::: + +### 3. Required Database Permissions + +The SQL user configured for the profiler must have read access to the system catalog views and Dynamic Management Views (DMVs) +listed in [Profiled Tables and Views](#profiled-tables-and-views). + +Permissions differ depending on whether you are running against an **on-premises SQL Server** or **Azure SQL Database**. + +#### On-Premises SQL Server + +On a self-hosted SQL Server instance, a server-level grant is sufficient: + +```sql +-- Grant access to Dynamic Management Views (DMVs) +GRANT VIEW SERVER STATE TO []; + +-- Grant access to object definitions (routines, views) +GRANT VIEW DEFINITION TO []; + +-- Grant read access to INFORMATION_SCHEMA views +GRANT SELECT ON SCHEMA::INFORMATION_SCHEMA TO []; +``` + +:::tip +`VIEW SERVER STATE` is a server-level permission required to query `sys.dm_*` Dynamic Management Views. +`VIEW DEFINITION` allows the user to see the definitions of stored procedures and views. +These grants should be executed by a `sysadmin` or a login with `CONTROL SERVER` permission. +::: + +#### Azure SQL Database + +:::warning Attention: +`VIEW SERVER STATE` is **not supported** on Azure SQL Database. You must use `VIEW DATABASE STATE` instead, +which must be granted **per database** — including the `master` database. +::: + +First, identify your target database(s) by connecting as an admin and running: + +```sql +SELECT name FROM sys.databases WHERE database_id > 4; +``` + +Then grant permissions in both `master` and each target database: + +```sql +-- In the master database (required for server-level DMVs like sys.databases, sys.dm_os_sys_info) +USE master; +CREATE USER [] FROM LOGIN []; +GRANT VIEW DATABASE STATE TO []; + +-- In each target database +USE []; +CREATE USER [] FROM LOGIN []; +GRANT VIEW DATABASE STATE TO []; +GRANT VIEW DEFINITION TO []; +GRANT SELECT ON SCHEMA::INFORMATION_SCHEMA TO []; +``` + +:::tip +On Azure SQL Database, `VIEW DATABASE STATE` is scoped to a single database. The profiler queries +server-level DMVs (e.g., `sys.databases`, `sys.dm_os_sys_info`) in the `master` database context, so +the user must have `VIEW DATABASE STATE` granted there in addition to the target database(s). +::: + +## Profiled Tables and Views + +The MSSQL profiler executes queries against the following system tables and DMVs. The results are organized into +two extraction steps: **schema metadata** and **activity metrics**. + +### Schema Metadata + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
QuerySource Table(s)Description
System Infosys.dm_os_sys_infoInstance-level metadata including memory, CPU count, and scheduler configuration.
Databasessys.databasesLists all user databases (excluding system databases) with IDs, names, collation, and creation dates.
TablesINFORMATION_SCHEMA.TABLESExtracts table definitions and types from each database.
ViewsINFORMATION_SCHEMA.VIEWSExtracts view definitions (SQL text is redacted for security).
ColumnsINFORMATION_SCHEMA.COLUMNSColumn-level metadata including data types, nullability, precision, collation, and domain information.
Indexed Viewssys.views, sys.indexesIdentifies views that have clustered or non-clustered indexes, with index type and IDs.
RoutinesINFORMATION_SCHEMA.ROUTINESStored procedures and user-defined functions (routine definitions are redacted for security).
Database Sizessys.database_filesDatabase file metadata including current size, free space, and maximum configured size (in MB).
Table Sizessys.dm_db_partition_stats, sys.objectsPer-table storage metrics: row counts, reserved/used/unused space, and data vs. index space breakdown.
+ +### Activity Metrics + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
QuerySource Table(s)Description
Query Statssys.dm_exec_query_stats, sys.dm_exec_sql_textRecently executed queries classified by command type (QUERY, DML, DDL, ROUTINE, TRANSACTION_CONTROL, OTHER) with execution count, duration, CPU time, and row counts.
Procedure Statssys.dm_exec_procedure_statsStored procedure execution statistics including execution counts, total CPU time, and elapsed time.
Sessionssys.dm_exec_sessionsActive user sessions with login info (hashed), program names, CPU/memory usage, and request timing.
CPU Utilizationsys.dm_os_ring_buffers, sys.dm_os_sys_infoCPU utilization over time, including system idle percentage and SQL Server process utilization.
+ +## Configure Connection to MSSQL + +Run the following command to configure the profiler connection to your SQL Server instance: + +```console +databricks labs lakebridge configure-database-profiler + +Please select the source system you want to configure +[0] synapse +[1] mssql +Enter a number between 0 and 1: 1 + +(local | env) +local means values are read as plain text +env means values are read from environment variables, and fall back to plain text if not variable is not found + +Enter secret vault type (local | env) +[0] env +[1] local +Enter a number between 0 and 1: 1 +Enter fetch size (default: 1000): +Enter login timeout (seconds) (default: 30): +Enter the fully-qualified server name: my-server.database.windows.net +Enter the port details: 1433 +Enter the SQL username: profiler_user +Enter the SQL password: +Enter timezone (e.g. America/New_York) (default: UTC): +Enter the ODBC driver installed locally (default: ODBC Driver 18 for SQL Server): +Do you want to test the connection to mssql? (yes/no): yes +``` + +### Configuration Parameters + +| Parameter | Description | Default | +|:----------|:------------|:--------| +| **Secret vault type** | `local` for plain text values, `env` to read from environment variables | — | +| **Fetch size** | Number of rows fetched per batch from the source | `1000` | +| **Login timeout** | Connection timeout in seconds | `30` | +| **Server name** | Fully-qualified SQL Server hostname | — | +| **Port** | SQL Server port number | — | +| **SQL username** | SQL Authentication username | — | +| **SQL password** | SQL Authentication password (hidden input) | — | +| **Timezone** | Timezone for timestamp normalization | `UTC` | +| **ODBC driver** | Locally installed ODBC driver name | `ODBC Driver 18 for SQL Server` | + +## Execute the Profiler + +Once configured, run the profiler to extract metadata and activity metrics from your SQL Server instance: + +```bash +databricks labs lakebridge execute-database-profiler --source-tech mssql +``` + +The profiler will: +1. Connect to your SQL Server instance using the configured credentials +2. Execute the schema metadata and activity metric extraction queries +3. Store the results in a local DuckDB extract file + +### Publish Profiler Summary Dashboard + +After executing the profiler, you can upload the results and deploy a summary dashboard to your Databricks workspace: + +```bash +databricks labs lakebridge create-profiler-dashboard \ + --extract-file /tmp/data/mssql_assessment/profiler_extract.db \ + --source-tech mssql \ + --volume-path /Volumes/lakebridge_profiler/profiler_runs \ + --catalog-name lakebridge_profiler \ + --schema-name profiler_runs +``` + +Refer to the [Profiler Guide](../#publish-profiler-summary-dashboard) for full details on dashboard deployment options. + +[Back to Configure Profiler](../#configure-profiler) diff --git a/pyproject.toml b/pyproject.toml index e56c47c3d1..f2e6bf29cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,6 +93,7 @@ lint = [ "pandas-stubs~=2.3.0.250703", # Azure SDK dependencies for linting resources/assessments folder "azure-identity~=1.19.0", + "azure-mgmt-sql~=3.0.1", "azure-monitor-query~=1.4.0", "azure-synapse-artifacts~=0.20.0", ] @@ -158,6 +159,10 @@ asyncio_default_fixture_loop_scope="function" [tool.mypy] exclude = ["tests/resources/.*"] +[[tool.mypy.overrides]] +module = "azure.mgmt.sql.*" +ignore_missing_imports = true + [tool.black] target-version = ["py310"] line-length = 120 diff --git a/src/databricks/labs/lakebridge/assessments/_constants.py b/src/databricks/labs/lakebridge/assessments/_constants.py index 9a793b19d1..e1299ccbae 100644 --- a/src/databricks/labs/lakebridge/assessments/_constants.py +++ b/src/databricks/labs/lakebridge/assessments/_constants.py @@ -5,10 +5,11 @@ PLATFORM_TO_SOURCE_TECHNOLOGY_CFG = { "synapse": "src/databricks/labs/lakebridge/resources/assessments/synapse/pipeline_config.yml", + "mssql": "src/databricks/labs/lakebridge/resources/assessments/mssql/pipeline_config.yml", } # TODO modify this PLATFORM_TO_SOURCE_TECHNOLOGY.keys() once all platforms are supported -PROFILER_SOURCE_SYSTEM = ["synapse"] +PROFILER_SOURCE_SYSTEM = ["synapse", "mssql"] # This flag indicates whether a connector is required for the source system when pipeline is trigger diff --git a/src/databricks/labs/lakebridge/assessments/configure_assessment.py b/src/databricks/labs/lakebridge/assessments/configure_assessment.py index a9b82b865a..cbd6cdf9a1 100644 --- a/src/databricks/labs/lakebridge/assessments/configure_assessment.py +++ b/src/databricks/labs/lakebridge/assessments/configure_assessment.py @@ -86,18 +86,21 @@ def _configure_credentials(self) -> str: secret_vault_type = str(self.prompts.choice("Enter secret vault type (local | env)", ["local", "env"])).lower() secret_vault_name = None - logger.info("Please refer to the documentation to understand the difference between local and env.") - credential = { "secret_vault_type": secret_vault_type, "secret_vault_name": secret_vault_name, source: { - "database": self.prompts.question("Enter the database name"), - "driver": self.prompts.question("Enter the driver details"), - "server": self.prompts.question("Enter the server or host details"), + "auth_type": "sql_authentication", + "fetch_size": self.prompts.question("Enter fetch size", default="1000"), + "login_timeout": self.prompts.question("Enter login timeout (seconds)", default="30"), + "server": self.prompts.question("Enter the fully-qualified server name"), "port": int(self.prompts.question("Enter the port details", valid_number=True)), - "user": self.prompts.question("Enter the user details"), - "password": self.prompts.password("Enter the password details"), + "user": self.prompts.question("Enter the SQL username"), + "password": self.prompts.password("Enter the SQL password"), + "tz_info": self.prompts.question("Enter timezone (e.g. America/New_York)", default="UTC"), + "driver": self.prompts.question( + "Enter the ODBC driver installed locally", default="ODBC Driver 18 for SQL Server" + ), }, } diff --git a/src/databricks/labs/lakebridge/resources/assessments/mssql/__init__.py b/src/databricks/labs/lakebridge/resources/assessments/mssql/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/databricks/labs/lakebridge/resources/assessments/mssql/activity_extract.py b/src/databricks/labs/lakebridge/resources/assessments/mssql/activity_extract.py new file mode 100644 index 0000000000..99ece1e8bf --- /dev/null +++ b/src/databricks/labs/lakebridge/resources/assessments/mssql/activity_extract.py @@ -0,0 +1,75 @@ +import json +import sys + +from databricks.labs.blueprint.entrypoint import get_logger + +from databricks.labs.lakebridge.connections.credential_manager import create_credential_manager +from databricks.labs.lakebridge.assessments import PRODUCT_NAME +from databricks.labs.lakebridge.resources.assessments.mssql.common.connector import get_sqlserver_reader +from databricks.labs.lakebridge.resources.assessments.mssql.common.queries import MSSQLQueries +from databricks.labs.lakebridge.resources.assessments.synapse.common.duckdb_helpers import save_resultset_to_db +from databricks.labs.lakebridge.resources.assessments.synapse.common.functions import arguments_loader + +logger = get_logger(__file__) + + +def execute(): + + db_path, creds_file = arguments_loader(desc="MSSQL Server Activity Extract Script") + cred_manager = create_credential_manager(PRODUCT_NAME, creds_file) + mssql_settings = cred_manager.get_credentials("mssql") + auth_type = mssql_settings.get("auth_type", "sql_authentication") + server_name = mssql_settings.get("server", "") + try: + + # TODO: get the last time the profiler was executed + # For now, we'll default to None, but this will eventually need + # input from a scheduler component. + last_execution_time = None + mode = "overwrite" + + # Extract activity metrics + logger.info(f"Extracting activity metrics for: {server_name}") + print(f"Extracting activity metrics for: {server_name}") + connection = get_sqlserver_reader( + mssql_settings, db_name="master", server_name=server_name, auth_type=auth_type + ) + + # Query stats + table_name = "query_stats" + table_query = MSSQLQueries.get_query_stats(last_execution_time) + logger.info(f"Loading '{table_name}' for SQL server: {server_name}") + result = connection.fetch(table_query) + save_resultset_to_db(result, f"mssql_{table_name}", db_path, mode=mode) + + # Stored procedure stats + table_name = "proc_stats" + table_query = MSSQLQueries.get_procedure_stats(last_execution_time) + logger.info(f"Loading '{table_name}' for SQL server: {server_name}") + result = connection.fetch(table_query) + save_resultset_to_db(result, f"mssql_{table_name}", db_path, mode=mode) + + # Session info + table_name = "sessions" + table_query = MSSQLQueries.get_sessions(last_execution_time) + logger.info(f"Loading '{table_name}' for SQL server: {server_name}") + result = connection.fetch(table_query) + save_resultset_to_db(result, f"mssql_{table_name}", db_path, mode=mode) + + # CPU Utilization + table_name = "cpu_utilization" + table_query = MSSQLQueries.get_cpu_utilization(last_execution_time) + logger.info(f"Loading '{table_name}' for SQL server: {server_name}") + result = connection.fetch(table_query) + save_resultset_to_db(result, f"mssql_{table_name}", db_path, mode=mode) + + print(json.dumps({"status": "success", "message": "All data loaded successfully loaded successfully"})) + + except Exception as e: + logger.error(f"Failed to extract activity info for SQL server: {str(e)}") + print(json.dumps({"status": "error", "message": str(e)}), file=sys.stderr) + sys.exit(1) + + +if __name__ == '__main__': + execute() diff --git a/src/databricks/labs/lakebridge/resources/assessments/mssql/columns.sql b/src/databricks/labs/lakebridge/resources/assessments/mssql/columns.sql new file mode 100644 index 0000000000..255340fc6c --- /dev/null +++ b/src/databricks/labs/lakebridge/resources/assessments/mssql/columns.sql @@ -0,0 +1,29 @@ +/** + * Retrieves column-level metadata for all tables and views in the specified + * database by querying INFORMATION_SCHEMA.COLUMNS. Returns column attributes + * along with a timestamp indicating when the data was extracted. + */ +SELECT table_catalog, + table_schema, + table_name, + column_name, + ordinal_position, + column_default, + is_nullable, + data_type, + character_maximum_length, + character_octet_length, + numeric_precision, + numeric_precision_radix, + numeric_scale, + datetime_precision, + character_set_catalog, + character_set_schema, + character_set_name, + collation_catalog, + collation_schema, + collation_name, + domain_catalog, + domain_schema, + domain_name +FROM information_schema.columns; diff --git a/src/databricks/labs/lakebridge/resources/assessments/mssql/common/__init__.py b/src/databricks/labs/lakebridge/resources/assessments/mssql/common/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/databricks/labs/lakebridge/resources/assessments/mssql/common/connector.py b/src/databricks/labs/lakebridge/resources/assessments/mssql/common/connector.py new file mode 100644 index 0000000000..4b5f2eb496 --- /dev/null +++ b/src/databricks/labs/lakebridge/resources/assessments/mssql/common/connector.py @@ -0,0 +1,22 @@ +from databricks.labs.lakebridge.connections.database_manager import DatabaseManager + + +def get_sqlserver_reader( + input_cred: dict, + db_name: str, + *, + server_name: str, + auth_type: str = 'sql_authentication', +) -> DatabaseManager: + config = { + "driver": input_cred['driver'], + "server": server_name, + "database": db_name, + "user": input_cred['user'], + "password": input_cred['password'], + "port": input_cred.get('port', 1433), + "auth_type": auth_type, + } + source = "mssql" + + return DatabaseManager(source, config) diff --git a/src/databricks/labs/lakebridge/resources/assessments/mssql/common/functions.py b/src/databricks/labs/lakebridge/resources/assessments/mssql/common/functions.py new file mode 100644 index 0000000000..33e63e446e --- /dev/null +++ b/src/databricks/labs/lakebridge/resources/assessments/mssql/common/functions.py @@ -0,0 +1,9 @@ +from azure.identity import DefaultAzureCredential +from azure.mgmt.sql import SqlManagementClient + + +def create_msql_sql_client(config: dict) -> SqlManagementClient: + """ + Creates an Azure SQL management client for the provided subscription using the default Azure credential. + """ + return SqlManagementClient(credential=DefaultAzureCredential(), subscription_id=config["subscription_id"]) diff --git a/src/databricks/labs/lakebridge/resources/assessments/mssql/common/queries.py b/src/databricks/labs/lakebridge/resources/assessments/mssql/common/queries.py new file mode 100644 index 0000000000..7ba18dc7fe --- /dev/null +++ b/src/databricks/labs/lakebridge/resources/assessments/mssql/common/queries.py @@ -0,0 +1,385 @@ +class MSSQLQueries: + + @staticmethod + def get_query_stats(last_execution_time: str | None) -> str: + """ + Retrieves and classifies recently executed SQL statements from `sys.dm_exec_query_stats`. + Includes execution metrics (count, duration, CPU time, rows) and categorizes each statement as QUERY, DML, DDL, + ROUTINE, TRANSACTION_CONTROL, or OTHER based on its command type. + """ + predicate = ( + f"WHERE qs.last_execution_time > CAST('{last_execution_time}' AS DATETIME2(6))" + if last_execution_time + else "" + ) + + return f""" + with query_stats as ( + SELECT + CONVERT(VARCHAR(64), HASHBYTES('SHA2_256', qs.sql_handle), 1) as sql_handle, + st.dbid, + qs.creation_time, + qs.last_execution_time, + qs.execution_count, + qs.total_worker_time, + qs.total_elapsed_time, + qs.total_rows, + SUBSTRING(st.text, (qs.statement_start_offset/2) + 1, + ((CASE statement_end_offset + WHEN -1 THEN DATALENGTH(st.text) + ELSE qs.statement_end_offset END + - qs.statement_start_offset)/2) + 1) AS statement_text + FROM sys.dm_exec_query_stats AS qs + CROSS APPLY sys.dm_exec_sql_text(qs.sql_handle) AS st + {predicate} + ), + query_stats_ex as ( + select + dbid, + creation_time, + last_execution_time, + execution_count, + total_worker_time, + total_elapsed_time, + total_rows, + UPPER(SUBSTRING(LTRIM(RTRIM(statement_text)), 1, 40)) as command + from query_stats + ) + SELECT + *, + CASE + WHEN command like 'SELECT%' THEN 'QUERY' + WHEN command like 'WITH%' THEN 'QUERY' + WHEN command like 'INSERT%' THEN 'DML' + WHEN command like 'UPDATE%' THEN 'DML' + WHEN command like 'MERGE%' THEN 'DML' + WHEN command like 'DELETE%' THEN 'DML' + WHEN command like 'TRUNCATE%' THEN 'DML' + WHEN command like 'COPY%' THEN 'DML' + WHEN command like 'IF%' THEN 'DML' + WHEN command like 'BEGIN%' THEN 'DML' + WHEN command like 'DECLARE%' THEN 'DML' + WHEN command like 'BUILDREPLICATEDTABLECACHE%' THEN 'DML' + WHEN command like 'CREATE%' THEN 'DDL' + WHEN command like 'DROP%' THEN 'DDL' + WHEN command like 'ALTER%' THEN 'DDL' + WHEN command like 'EXEC%' THEN 'ROUTINE' + WHEN command like 'EXECUTE %' THEN 'ROUTINE' + WHEN command like 'BEGIN%TRAN%' THEN 'TRANSACTION_CONTROL' + WHEN command like 'END%TRAN%' THEN 'TRANSACTION_CONTROL' + WHEN command like 'COMMIT%' THEN 'TRANSACTION_CONTROL' + WHEN command like 'ROLLBACK%' THEN 'TRANSACTION_CONTROL' + ELSE 'OTHER' + END as command_type, + SYSDATETIME() as extract_ts + FROM query_stats_ex + ORDER BY last_execution_time + """ + + @staticmethod + def get_procedure_stats(last_execution_time: str | None): + """ + Retrieves execution statistics for stored procedures from `sys.dm_exec_procedure_stats`, + including execution counts, total CPU and elapsed time, last execution timestamp, + and maps object and database IDs to their names. Results are ordered by most recent execution. + """ + predicate = ( + f"WHERE last_execution_time > CAST('{last_execution_time}' AS DATETIME2(6))" if last_execution_time else "" + ) + return f""" + SELECT + database_id, + DB_NAME(database_id) AS db_name, + object_id, + OBJECT_NAME(object_id, database_id) AS object_name, + type, + last_execution_time, + execution_count, + total_worker_time, + total_elapsed_time, + SYSDATETIME() as extract_ts + FROM + sys.dm_exec_procedure_stats + {predicate} + ORDER BY + last_execution_time + """ + + @staticmethod + def get_sessions(last_execution_time: str | None): + """ + Retrieves active user session details from `sys.dm_exec_sessions`, including login info, + program and client names, CPU and memory usage, request timing, row counts, and database context. + Excludes system sessions and orders results by the end time of the last request. + """ + predicate = ( + f"AND last_request_end_time > CAST('{last_execution_time}' AS DATETIME2(6))" if last_execution_time else "" + ) + return f""" + SELECT + session_id, + login_time, + program_name, + client_interface_name, + CONVERT(VARCHAR(64), HASHBYTES('SHA2_256', login_name), 1) as login_name, + status, + cpu_time, + memory_usage, + total_scheduled_time, + total_elapsed_time, + last_request_start_time, + last_request_end_time, + is_user_process, + row_count, + database_id, + DB_NAME(database_id) AS db_name, + SYSDATETIME() as extract_ts + FROM + sys.dm_exec_sessions + WHERE + is_user_process <> 0 {predicate} + ORDER BY + last_request_end_time + """ + + @staticmethod + def get_cpu_utilization(last_execution_time: str | None): + """ + Extracts SQL Server CPU and system utilization metrics from `sys.dm_os_ring_buffers`, + including system idle and SQL process utilization over time. + """ + predicate = f"WHERE EventTime > CAST('{last_execution_time}' AS DATETIME2(6))" if last_execution_time else "" + return f""" + WITH process_utilization_info + AS (SELECT record.value('(./Record/@id)[1]', 'int') + AS record_id, + [timestamp], + record.value('(./Record/SchedulerMonitorEvent/SystemHealth/SystemIdle)[1]', 'int') AS SystemIdle, + record.value('(./Record/SchedulerMonitorEvent/SystemHealth/ProcessUtilization)[1]', 'int') AS SQLProcessUtilization + FROM (SELECT [timestamp], + CONVERT(XML, record) AS record + FROM sys.dm_os_ring_buffers + WHERE ring_buffer_type = N'RING_BUFFER_SCHEDULER_MONITOR' + AND record LIKE '%%') X), + os_sysinfo + AS (SELECT TOP 1 ms_ticks + FROM sys.dm_os_sys_info), + cpu_utilization + AS (SELECT record_id, + Dateadd (ms, ( [timestamp] - ms_ticks ), Getdate()) AS EventTime, + systemidle, + sqlprocessutilization + FROM process_utilization_info + CROSS JOIN os_sysinfo) + SELECT *, + Sysdatetime() AS extract_ts + FROM cpu_utilization + {predicate} + ORDER BY eventtime + """ + + @staticmethod + def get_sys_info(): + """ + Retrieves system-level information from SQL Server using sys.dm_os_sys_info. + Returns details about memory, CPU, scheduler count, and other OS-related + metadata for the SQL Server instance, along with a timestamp indicating when + the data was extracted. + """ + return """ + SELECT *, + Sysdatetime() AS extract_ts + FROM sys.dm_os_sys_info + """ + + @staticmethod + def get_databases(): + """ + Retrieve metadata for all user databases, excluding system databases. + Returns each database's ID, name, collation, creation date, + and a timestamp indicating when the data was extracted. + """ + return """ + SELECT DB_ID(NAME) AS db_id, + NAME, + collation_name, + create_date, + SYSDATETIME() AS extract_ts + FROM sys.databases + WHERE NAME NOT IN ( 'master', 'tempdb', 'model', 'msdb' ); + """ + + @staticmethod + def get_tables(): + """ + Retrieves metadata for all tables in the specified database by querying + INFORMATION_SCHEMA.TABLES. Returns table definitions along with a timestamp + indicating when the data was extracted. + """ + return """ + SELECT + TABLE_CATALOG, + TABLE_SCHEMA, + TABLE_NAME, + TABLE_TYPE + FROM INFORMATION_SCHEMA.TABLES ; + """ + + @staticmethod + def get_views(): + """ + Retrieves metadata for all views in the specified database by querying + `INFORMATION_SCHEMA.VIEWS`. Returns view definitions along with a timestamp + indicating when the data was extracted. + """ + return """ + SELECT + TABLE_CATALOG, + TABLE_SCHEMA, + TABLE_NAME, + CHECK_OPTION, + IS_UPDATABLE, + '[REDACTED]' as VIEW_DEFINITION + FROM INFORMATION_SCHEMA.VIEWS + """ + + @staticmethod + def get_columns(): + """ + Retrieves column-level metadata for all tables and views in the specified + database by querying INFORMATION_SCHEMA.COLUMNS. Returns column attributes + along with a timestamp indicating when the data was extracted. + """ + return """ + SELECT + TABLE_CATALOG, + TABLE_SCHEMA, + TABLE_NAME, + COLUMN_NAME, + ORDINAL_POSITION, + COLUMN_DEFAULT, + IS_NULLABLE, + DATA_TYPE, + CHARACTER_MAXIMUM_LENGTH, + CHARACTER_OCTET_LENGTH, + NUMERIC_PRECISION, + NUMERIC_PRECISION_RADIX, + NUMERIC_SCALE, + DATETIME_PRECISION, + CHARACTER_SET_CATALOG, + CHARACTER_SET_SCHEMA, + CHARACTER_SET_NAME, + COLLATION_CATALOG, + COLLATION_SCHEMA, + COLLATION_NAME, + DOMAIN_CATALOG, + DOMAIN_SCHEMA, + DOMAIN_NAME + FROM INFORMATION_SCHEMA.COLUMNS ; + """ + + @staticmethod + def get_indexed_views(): + """ + Retrieves metadata for all indexed views in the specified database by joining + `sys.views` with `sys.indexes`. Returns view details for those with a clustered + index (index_id = 1) along with a timestamp indicating when the data was extracted. + """ + return """ + SELECT + v.[name] AS indexed_view_name, + s.[name] AS schema_name, + i.[name] AS index_name, + i.[type_desc] AS index_type, + i.[index_id], + SYSDATETIME() as extract_ts + FROM sys.views AS v + JOIN sys.schemas AS s + ON v.[schema_id] = s.[schema_id] + JOIN sys.indexes AS i + ON v.[object_id] = i.[object_id] + WHERE i.[index_id] = 1; + """ + + @staticmethod + def get_routines(): + """ + Retrieves metadata for all routines (stored procedures and functions) in the + specified database by querying INFORMATION_SCHEMA.ROUTINES. Returns routine + details along with a timestamp indicating when the data was extracted. + """ + return """ + SELECT + CREATED, + DATA_TYPE, + IS_DETERMINISTIC, + IS_IMPLICITLY_INVOCABLE, + IS_NULL_CALL, + IS_USER_DEFINED_CAST, + LAST_ALTERED, + MAX_DYNAMIC_RESULT_SETS, + NUMERIC_PRECISION, + NUMERIC_PRECISION_RADIX, + NUMERIC_SCALE, + ROUTINE_BODY, + ROUTINE_CATALOG, + '[REDACTED]' as ROUTINE_DEFINITION, + ROUTINE_NAME, + ROUTINE_SCHEMA, + ROUTINE_TYPE, + SCHEMA_LEVEL_ROUTINE, + SPECIFIC_CATALOG, + SPECIFIC_NAME, + SPECIFIC_SCHEMA, + SQL_DATA_ACCESS + FROM information_schema.routines + """ + + @staticmethod + def get_db_sizes(): + """ + Retrieves metadata for all data files (type = 0) in the specified database + from sys.database_files. Returns file name, type, current size, free space, + maximum size, and a timestamp indicating when the data was extracted. + """ + return """ + SELECT + DB_NAME() AS database_name, + name AS FileName, + type_desc, + size/128.0 AS CurrentSizeMB, + size/128.0 - CAST(FILEPROPERTY(name, 'SpaceUsed') AS int)/128.0 AS FreeSpaceInMB, + max_size as MaxSize, + SYSDATETIME() as extract_ts + FROM sys.database_files WHERE type=0 + """ + + @staticmethod + def get_table_sizes(): + """ + Retrieves storage and row count statistics for all user tables in the specified + database by querying sys.dm_db_partition_stats and sys.objects. Returns table + name, total rows, reserved, used, and unused space (MB), breakdown of data vs. + index space, and a timestamp indicating when the data was extracted. + """ + return """ + SELECT + o.name AS TableName, + SUM(ps.row_count) AS [RowCount], + SUM(ps.reserved_page_count) * 8 / 1024 AS ReservedMB, + SUM(ps.used_page_count) * 8 / 1024 AS UsedMB, + (SUM(ps.reserved_page_count) - SUM(ps.used_page_count)) * 8 / 1024 AS UnusedMB, + SUM(CASE + WHEN ps.index_id < 2 THEN ps.in_row_data_page_count + ps.lob_used_page_count + ps.row_overflow_used_page_count + ELSE 0 + END) * 8 / 1024 AS DataMB, + SUM(CASE + WHEN ps.index_id >= 2 THEN ps.in_row_data_page_count + ELSE 0 + END) * 8 / 1024 AS IndexMB, + SYSDATETIME() as extract_ts + FROM sys.dm_db_partition_stats AS ps + JOIN sys.objects AS o ON ps.object_id = o.object_id + WHERE o.type = 'U' + GROUP BY schema_name(o.schema_id), o.name + """ diff --git a/src/databricks/labs/lakebridge/resources/assessments/mssql/cpu_utilization.sql b/src/databricks/labs/lakebridge/resources/assessments/mssql/cpu_utilization.sql new file mode 100644 index 0000000000..8f768495b6 --- /dev/null +++ b/src/databricks/labs/lakebridge/resources/assessments/mssql/cpu_utilization.sql @@ -0,0 +1,32 @@ +/** + * Extracts SQL Server CPU and system utilization metrics from `sys.dm_os_ring_buffers`, + * including system idle and SQL process utilization over time. + */ +WITH process_utilization_info + AS (SELECT record.value('(./Record/@id)[1]', 'int') + AS record_id, + [timestamp], +record.value('(./Record/SchedulerMonitorEvent/SystemHealth/SystemIdle)[1]', +'int') + AS SystemIdle, +record.value('(./Record/SchedulerMonitorEvent/SystemHealth/ProcessUtilization)[1]', 'int') AS SQLProcessUtilization + FROM (SELECT [timestamp], + CONVERT(XML, record) AS record + FROM sys.dm_os_ring_buffers + WHERE ring_buffer_type = N'RING_BUFFER_SCHEDULER_MONITOR' + AND record LIKE '%%') X), + os_sysinfo + AS (SELECT TOP 1 ms_ticks + FROM sys.dm_os_sys_info), + cpu_utilization + AS (SELECT record_id, + Dateadd (ms, ( [timestamp] - ms_ticks ), Getdate()) AS EventTime + , + systemidle, + sqlprocessutilization + FROM process_utilization_info + CROSS JOIN os_sysinfo) +SELECT *, + Sysdatetime() AS extract_ts +FROM cpu_utilization +ORDER BY eventtime diff --git a/src/databricks/labs/lakebridge/resources/assessments/mssql/database_sizes.sql b/src/databricks/labs/lakebridge/resources/assessments/mssql/database_sizes.sql new file mode 100644 index 0000000000..5cf09fbfd3 --- /dev/null +++ b/src/databricks/labs/lakebridge/resources/assessments/mssql/database_sizes.sql @@ -0,0 +1,20 @@ +/** + * Retrieves metadata for all data files (type = 0) in the specified database + * from sys.database_files. Returns file name, type, current size, free space, + * maximum size, and a timestamp indicating when the data was extracted. + */ +SELECT Db_name() AS + database_name, + NAME AS + FileName, + type_desc, + size / 128.0 AS + CurrentSizeMB, + size / 128.0 - Cast(Fileproperty(NAME, 'SpaceUsed') AS INT) / 128.0 AS + FreeSpaceInMB, + max_size AS + MaxSize, + Sysdatetime() AS + extract_ts +FROM sys.database_files +WHERE type = 0; diff --git a/src/databricks/labs/lakebridge/resources/assessments/mssql/databases.sql b/src/databricks/labs/lakebridge/resources/assessments/mssql/databases.sql new file mode 100644 index 0000000000..c7bfff2f9b --- /dev/null +++ b/src/databricks/labs/lakebridge/resources/assessments/mssql/databases.sql @@ -0,0 +1,12 @@ +/** + * Retrieve metadata for all user databases, excluding system databases. + * Returns each database's ID, name, collation, creation date, + * and a timestamp indicating when the data was extracted. + */ +SELECT Db_id(NAME) AS db_id, + NAME, + collation_name, + create_date, + Sysdatetime() AS extract_ts +FROM sys.databases +WHERE NAME NOT IN ( 'master', 'tempdb', 'model', 'msdb' ); diff --git a/src/databricks/labs/lakebridge/resources/assessments/mssql/indexed_views.sql b/src/databricks/labs/lakebridge/resources/assessments/mssql/indexed_views.sql new file mode 100644 index 0000000000..ddb62a9944 --- /dev/null +++ b/src/databricks/labs/lakebridge/resources/assessments/mssql/indexed_views.sql @@ -0,0 +1,17 @@ +/** + * Retrieves metadata for all indexed views in the specified database by joining + * `sys.views` with `sys.indexes`. Returns view details for those with a clustered + * index (index_id = 1) along with a timestamp indicating when the data was extracted. + */ +SELECT v.[name] AS indexed_view_name, + s.[name] AS schema_name, + i.[name] AS index_name, + i.[type_desc] AS index_type, + i.[index_id], + Sysdatetime() AS extract_ts +FROM sys.views AS v + JOIN sys.schemas AS s + ON v.[schema_id] = s.[schema_id] + JOIN sys.indexes AS i + ON v.[object_id] = i.[object_id] +WHERE i.[index_id] = 1; diff --git a/src/databricks/labs/lakebridge/resources/assessments/mssql/info_extract.py b/src/databricks/labs/lakebridge/resources/assessments/mssql/info_extract.py new file mode 100644 index 0000000000..89abbdadd1 --- /dev/null +++ b/src/databricks/labs/lakebridge/resources/assessments/mssql/info_extract.py @@ -0,0 +1,109 @@ +import json +import sys + +from databricks.labs.blueprint.entrypoint import get_logger + +from databricks.labs.lakebridge.connections.credential_manager import create_credential_manager +from databricks.labs.lakebridge.assessments import PRODUCT_NAME +from databricks.labs.lakebridge.resources.assessments.mssql.common.connector import get_sqlserver_reader +from databricks.labs.lakebridge.resources.assessments.mssql.common.queries import MSSQLQueries +from databricks.labs.lakebridge.resources.assessments.synapse.common.duckdb_helpers import save_resultset_to_db +from databricks.labs.lakebridge.resources.assessments.synapse.common.functions import arguments_loader + +logger = get_logger(__file__) + + +def execute(): + db_path, creds_file = arguments_loader(desc="MSSQL Server Info Extract Script") + cred_manager = create_credential_manager(PRODUCT_NAME, creds_file) + mssql_settings = cred_manager.get_credentials("mssql") + auth_type = mssql_settings.get("auth_type", "sql_authentication") + server_name = mssql_settings.get("server", "") + + try: + # TODO: get the last time the profiler was executed + # For now, we'll default to None, but this will eventually need + # input from a scheduler component. + mode = "overwrite" + + # Extract info metrics + logger.info(f"Extracting info metrics for: {server_name}") + print(f"Extracting info metrics for: {server_name}") + connection = get_sqlserver_reader( + mssql_settings, db_name="master", server_name=server_name, auth_type=auth_type + ) + + # System info + table_name = "sys_info" + table_query = MSSQLQueries.get_sys_info() + logger.info(f"Loading '{table_name}' for SQL server: {server_name}") + print(f"Loading '{table_name}' for SQL server: {server_name}") + result = connection.fetch(table_query) + save_resultset_to_db(result, f"mssql_{table_name}", db_path, mode=mode) + + # Databases + table_name = "databases" + table_query = MSSQLQueries.get_databases() + logger.info(f"Loading '{table_name}' for SQL server: {server_name}") + result = connection.fetch(table_query) + save_resultset_to_db(result, f"mssql_{table_name}", db_path, mode=mode) + + # Tables + table_name = "tables" + table_query = MSSQLQueries.get_tables() + logger.info(f"Loading '{table_name}' for SQL server: {server_name}") + result = connection.fetch(table_query) + save_resultset_to_db(result, f"mssql_{table_name}", db_path, mode=mode) + + # Views + table_name = "views" + table_query = MSSQLQueries.get_views() + logger.info(f"Loading '{table_name}' for SQL server: {server_name}") + result = connection.fetch(table_query) + save_resultset_to_db(result, f"mssql_{table_name}", db_path, mode=mode) + + # Columns + table_name = "columns" + table_query = MSSQLQueries.get_columns() + logger.info(f"Loading '{table_name}' for SQL server: {server_name}") + result = connection.fetch(table_query) + save_resultset_to_db(result, f"mssql_{table_name}", db_path, mode=mode) + + # Indexed views + table_name = "indexed_views" + table_query = MSSQLQueries.get_indexed_views() + logger.info(f"Loading '{table_name}' for SQL server: {server_name}") + result = connection.fetch(table_query) + save_resultset_to_db(result, f"mssql_{table_name}", db_path, mode=mode) + + # Routines + table_name = "routines" + table_query = MSSQLQueries.get_routines() + logger.info(f"Loading '{table_name}' for SQL server: {server_name}") + result = connection.fetch(table_query) + save_resultset_to_db(result, f"mssql_{table_name}", db_path, mode=mode) + + # Database sizes + table_name = "db_sizes" + table_query = MSSQLQueries.get_db_sizes() + logger.info(f"Loading '{table_name}' for SQL server: {server_name}") + result = connection.fetch(table_query) + save_resultset_to_db(result, f"mssql_{table_name}", db_path, mode=mode) + + # Table sizes + table_name = "table_sizes" + table_query = MSSQLQueries.get_table_sizes() + logger.info(f"Loading '{table_name}' for SQL server: {server_name}") + result = connection.fetch(table_query) + save_resultset_to_db(result, f"mssql_{table_name}", db_path, mode=mode) + + print(json.dumps({"status": "success", "message": "All data loaded successfully loaded successfully"})) + + except Exception as e: + logger.error(f"Failed to execute info extract for SQL server: {str(e)}") + print(json.dumps({"status": "error", "message": str(e)}), file=sys.stderr) + sys.exit(1) + + +if __name__ == '__main__': + execute() diff --git a/src/databricks/labs/lakebridge/resources/assessments/mssql/pipeline_config.yml b/src/databricks/labs/lakebridge/resources/assessments/mssql/pipeline_config.yml new file mode 100644 index 0000000000..4d69689729 --- /dev/null +++ b/src/databricks/labs/lakebridge/resources/assessments/mssql/pipeline_config.yml @@ -0,0 +1,38 @@ +name: mssql_assessment +version: "1.0" +extract_folder: "/tmp/data/mssql_assessment" +steps: + - name: activity_extract + type: python + extract_source: src/databricks/labs/lakebridge/resources/assessments/mssql/activity_extract.py + mode: overwrite + frequency: once + flag: active + dependencies: + - pandas + - duckdb + - databricks-sdk + - databricks-labs-blueprint[yaml]>=0.10.1 + - sqlalchemy + - azure-synapse-artifacts + - azure-mgmt-sql + - azure-identity + - azure-monitor-query==1.3.0b1 + - pyodbc + - name: info_extract + type: python + extract_source: src/databricks/labs/lakebridge/resources/assessments/mssql/info_extract.py + mode: overwrite + frequency: once + flag: active + dependencies: + - pandas + - duckdb + - databricks-sdk + - databricks-labs-blueprint[yaml]>=0.10.1 + - sqlalchemy + - azure-synapse-artifacts + - azure-mgmt-sql + - azure-identity + - azure-monitor-query==1.3.0b1 + - pyodbc diff --git a/src/databricks/labs/lakebridge/resources/assessments/mssql/procedure_stats.sql b/src/databricks/labs/lakebridge/resources/assessments/mssql/procedure_stats.sql new file mode 100644 index 0000000000..d056291b6d --- /dev/null +++ b/src/databricks/labs/lakebridge/resources/assessments/mssql/procedure_stats.sql @@ -0,0 +1,17 @@ +/** + * Retrieves execution statistics for stored procedures from `sys.dm_exec_procedure_stats`, + * including execution counts, total CPU and elapsed time, last execution timestamp, + * and maps object and database IDs to their names. Results are ordered by most recent execution. + */ +SELECT database_id, + Db_name(database_id) AS db_name, + object_id, + Object_name(object_id, database_id) AS object_name, + type, + last_execution_time, + execution_count, + total_worker_time, + total_elapsed_time, + Sysdatetime() AS extract_ts +FROM sys.dm_exec_procedure_stats +ORDER BY last_execution_time diff --git a/src/databricks/labs/lakebridge/resources/assessments/mssql/query_stats.sql b/src/databricks/labs/lakebridge/resources/assessments/mssql/query_stats.sql new file mode 100644 index 0000000000..30c292b1e0 --- /dev/null +++ b/src/databricks/labs/lakebridge/resources/assessments/mssql/query_stats.sql @@ -0,0 +1,64 @@ +/** + * Retrieves and classifies recently executed SQL statements from `sys.dm_exec_query_stats`. + * Includes execution metrics (count, duration, CPU time, rows) and categorizes each statement + * as QUERY, DML, DDL, ROUTINE, TRANSACTION_CONTROL, or OTHER based on its command type. + */ +with query_stats as ( + SELECT + CONVERT(VARCHAR(64), HASHBYTES('SHA2_256', qs.sql_handle), 1) as sql_handle, + st.dbid, + qs.creation_time, + qs.last_execution_time, + qs.execution_count, + qs.total_worker_time, + qs.total_elapsed_time, + qs.total_rows, + SUBSTRING(st.text, (qs.statement_start_offset/2) + 1, + ((CASE statement_end_offset + WHEN -1 THEN DATALENGTH(st.text) + ELSE qs.statement_end_offset END + - qs.statement_start_offset)/2) + 1) AS statement_text + FROM sys.dm_exec_query_stats AS qs + CROSS APPLY sys.dm_exec_sql_text(qs.sql_handle) AS st +), + query_stats_ex as ( + select + dbid, + creation_time, + last_execution_time, + execution_count, + total_worker_time, + total_elapsed_time, + total_rows, + UPPER(SUBSTRING(LTRIM(RTRIM(statement_text)), 1, 40)) as command + from query_stats + ) + SELECT + *, + CASE + WHEN command like 'SELECT%' THEN 'QUERY' + WHEN command like 'WITH%' THEN 'QUERY' + WHEN command like 'INSERT%' THEN 'DML' + WHEN command like 'UPDATE%' THEN 'DML' + WHEN command like 'MERGE%' THEN 'DML' + WHEN command like 'DELETE%' THEN 'DML' + WHEN command like 'TRUNCATE%' THEN 'DML' + WHEN command like 'COPY%' THEN 'DML' + WHEN command like 'IF%' THEN 'DML' + WHEN command like 'BEGIN%' THEN 'DML' + WHEN command like 'DECLARE%' THEN 'DML' + WHEN command like 'BUILDREPLICATEDTABLECACHE%' THEN 'DML' + WHEN command like 'CREATE%' THEN 'DDL' + WHEN command like 'DROP%' THEN 'DDL' + WHEN command like 'ALTER%' THEN 'DDL' + WHEN command like 'EXEC%' THEN 'ROUTINE' + WHEN command like 'EXECUTE %' THEN 'ROUTINE' + WHEN command like 'BEGIN%TRAN%' THEN 'TRANSACTION_CONTROL' + WHEN command like 'END%TRAN%' THEN 'TRANSACTION_CONTROL' + WHEN command like 'COMMIT%' THEN 'TRANSACTION_CONTROL' + WHEN command like 'ROLLBACK%' THEN 'TRANSACTION_CONTROL' + ELSE 'OTHER' + END as command_type, + SYSDATETIME() as extract_ts + FROM query_stats_ex + ORDER BY last_execution_time diff --git a/src/databricks/labs/lakebridge/resources/assessments/mssql/routines.sql b/src/databricks/labs/lakebridge/resources/assessments/mssql/routines.sql new file mode 100644 index 0000000000..3d6af32173 --- /dev/null +++ b/src/databricks/labs/lakebridge/resources/assessments/mssql/routines.sql @@ -0,0 +1,28 @@ +/** + * Retrieves metadata for all routines (stored procedures and functions) in the + * specified database by querying INFORMATION_SCHEMA.ROUTINES. Returns routine + * details along with a timestamp indicating when the data was extracted. + */ +SELECT created, + data_type, + is_deterministic, + is_implicitly_invocable, + is_null_call, + is_user_defined_cast, + last_altered, + max_dynamic_result_sets, + numeric_precision, + numeric_precision_radix, + numeric_scale, + routine_body, + routine_catalog, + '[REDACTED]' AS ROUTINE_DEFINITION, + routine_name, + routine_schema, + routine_type, + schema_level_routine, + specific_catalog, + specific_name, + specific_schema, + sql_data_access +FROM information_schema.routines; diff --git a/src/databricks/labs/lakebridge/resources/assessments/mssql/sessions.sql b/src/databricks/labs/lakebridge/resources/assessments/mssql/sessions.sql new file mode 100644 index 0000000000..fa8a74e387 --- /dev/null +++ b/src/databricks/labs/lakebridge/resources/assessments/mssql/sessions.sql @@ -0,0 +1,25 @@ +/** + * Retrieves active user session details from `sys.dm_exec_sessions`, including login info, + * program and client names, CPU and memory usage, request timing, row counts, and database context. + * Excludes system sessions and orders results by the end time of the last request. + */ +SELECT session_id, + login_time, + program_name, + client_interface_name, + CONVERT(VARCHAR(64), Hashbytes('SHA2_256', login_name), 1) AS login_name, + status, + cpu_time, + memory_usage, + total_scheduled_time, + total_elapsed_time, + last_request_start_time, + last_request_end_time, + is_user_process, + row_count, + database_id, + Db_name(database_id) AS db_name, + Sysdatetime() AS extract_ts +FROM sys.dm_exec_sessions +WHERE is_user_process <> 0 {predicate} +ORDER BY last_request_end_time diff --git a/src/databricks/labs/lakebridge/resources/assessments/mssql/sys_info.sql b/src/databricks/labs/lakebridge/resources/assessments/mssql/sys_info.sql new file mode 100644 index 0000000000..82eae5e4b6 --- /dev/null +++ b/src/databricks/labs/lakebridge/resources/assessments/mssql/sys_info.sql @@ -0,0 +1,9 @@ +/** + * Retrieves system-level information from SQL Server using sys.dm_os_sys_info. + * Returns details about memory, CPU, scheduler count, and other OS-related + * metadata for the SQL Server instance, along with a timestamp indicating when + * the data was extracted. + */ +SELECT *, + Sysdatetime() AS extract_ts + FROM sys.dm_os_sys_info diff --git a/src/databricks/labs/lakebridge/resources/assessments/mssql/table_sizes.sql b/src/databricks/labs/lakebridge/resources/assessments/mssql/table_sizes.sql new file mode 100644 index 0000000000..927f3920cb --- /dev/null +++ b/src/databricks/labs/lakebridge/resources/assessments/mssql/table_sizes.sql @@ -0,0 +1,36 @@ +/** + * Retrieves storage and row count statistics for all user tables in the specified + * database by querying sys.dm_db_partition_stats and sys.objects. Returns table + * name, total rows, reserved, used, and unused space (MB), breakdown of data vs. + * index space, and a timestamp indicating when the data was extracted. + */ +SELECT o.NAME AS + TableName, + Sum(ps.row_count) AS + [RowCount], + Sum(ps.reserved_page_count) * 8 / 1024 AS + ReservedMB, + Sum(ps.used_page_count) * 8 / 1024 AS + UsedMB, + ( Sum(ps.reserved_page_count) - Sum(ps.used_page_count) ) * 8 / 1024 AS + UnusedMB, + Sum(CASE + WHEN ps.index_id < 2 THEN ps.in_row_data_page_count + + ps.lob_used_page_count + + ps.row_overflow_used_page_count + ELSE 0 + END) * 8 / 1024 AS + DataMB, + Sum(CASE + WHEN ps.index_id >= 2 THEN ps.in_row_data_page_count + ELSE 0 + END) * 8 / 1024 AS + IndexMB, + Sysdatetime() AS + extract_ts +FROM sys.dm_db_partition_stats AS ps + JOIN sys.objects AS o + ON ps.object_id = o.object_id +WHERE o.type = 'U' +GROUP BY Schema_name(o.schema_id), + o.NAME; diff --git a/src/databricks/labs/lakebridge/resources/assessments/mssql/tables.sql b/src/databricks/labs/lakebridge/resources/assessments/mssql/tables.sql new file mode 100644 index 0000000000..67306bb42f --- /dev/null +++ b/src/databricks/labs/lakebridge/resources/assessments/mssql/tables.sql @@ -0,0 +1,10 @@ +/** + * Retrieves metadata for all tables in the specified database by querying + * INFORMATION_SCHEMA.TABLES. Returns table definitions along with a timestamp + * indicating when the data was extracted. + */ +SELECT table_catalog, + table_schema, + table_name, + table_type +FROM information_schema.tables; diff --git a/src/databricks/labs/lakebridge/resources/assessments/mssql/views.sql b/src/databricks/labs/lakebridge/resources/assessments/mssql/views.sql new file mode 100644 index 0000000000..667757bd74 --- /dev/null +++ b/src/databricks/labs/lakebridge/resources/assessments/mssql/views.sql @@ -0,0 +1,12 @@ +/** + * Retrieves metadata for all views in the specified database by querying + * `INFORMATION_SCHEMA.VIEWS`. Returns view definitions along with a timestamp + * indicating when the data was extracted. + */ +SELECT table_catalog, + table_schema, + table_name, + check_option, + is_updatable, + '[REDACTED]' AS VIEW_DEFINITION +FROM information_schema.views diff --git a/src/databricks/labs/lakebridge/resources/assessments/synapse/common/duckdb_helpers.py b/src/databricks/labs/lakebridge/resources/assessments/synapse/common/duckdb_helpers.py index 00b53293a8..4ea994fab4 100644 --- a/src/databricks/labs/lakebridge/resources/assessments/synapse/common/duckdb_helpers.py +++ b/src/databricks/labs/lakebridge/resources/assessments/synapse/common/duckdb_helpers.py @@ -67,6 +67,21 @@ def save_resultset_to_db( "serverless_requests_history": "STATUS STRING, TRANSACTION_ID BIGINT, DISTRIBUTED_STATEMENT_ID STRING, QUERY_HASH STRING, LOGIN_NAME STRING, START_TIME STRING, ERROR_CODE INTEGER, REJECTED_ROWS_PATH STRING, END_TIME STRING, COMMAND STRING, QUERY_TEXT STRING, TOTAL_ELAPSED_TIME_MS BIGINT, DATA_PROCESSED_MB BIGINT, ERROR STRING", # Data processed info "serverless_data_processed": "DATA_PROCESSED_MB BIGINT, TYPE STRING, POOL_NAME STRING, EXTRACT_TS STRING", + # MSSQL activity extract + "mssql_query_stats": "DBID VARCHAR, CREATION_TIME TIMESTAMP, LAST_EXECUTION_TIME TIMESTAMP, EXECUTION_COUNT BIGINT, TOTAL_WORKER_TIME BIGINT, TOTAL_ELAPSED_TIME BIGINT, TOTAL_ROWS BIGINT, COMMAND VARCHAR, COMMAND_TYPE VARCHAR, EXTRACT_TS TIMESTAMP", + "mssql_proc_stats": "DATABASE_ID BIGINT, DB_NAME VARCHAR, OBJECT_ID BIGINT, OBJECT_NAME VARCHAR, TYPE VARCHAR, LAST_EXECUTION_TIME TIMESTAMP, EXECUTION_COUNT BIGINT, TOTAL_WORKER_TIME BIGINT, TOTAL_ELAPSED_TIME BIGINT, EXTRACT_TS TIMESTAMP", + "mssql_sessions": "SESSION_ID BIGINT, LOGIN_TIME TIMESTAMP, PROGRAM_NAME VARCHAR, CLIENT_INTERFACE_NAME VARCHAR, LOGIN_NAME VARCHAR, STATUS VARCHAR, CPU_TIME BIGINT, MEMORY_USAGE BIGINT, TOTAL_SCHEDULED_TIME BIGINT, TOTAL_ELAPSED_TIME BIGINT, LAST_REQUEST_START_TIME TIMESTAMP, LAST_REQUEST_END_TIME TIMESTAMP, IS_USER_PROCESS BOOLEAN, ROW_COUNT BIGINT, DATABASE_ID BIGINT, DB_NAME VARCHAR, EXTRACT_TS TIMESTAMP", + "mssql_cpu_utilization": "RECORD_ID BIGINT, EVENTTIME TIMESTAMP, SYSTEMIDLE BIGINT, SQLPROCESSUTILIZATION BIGINT, EXTRACT_TS TIMESTAMP", + # MSSQL info extract + "mssql_sys_info": "CPU_TICKS BIGINT, MS_TICKS BIGINT, CPU_COUNT BIGINT, HYPERTHREAD_RATIO BIGINT, PHYSICAL_MEMORY_KB BIGINT, VIRTUAL_MEMORY_KB BIGINT, COMMITTED_KB BIGINT, COMMITTED_TARGET_KB BIGINT, VISIBLE_TARGET_KB BIGINT, STACK_SIZE_IN_BYTES BIGINT, OS_QUANTUM BIGINT, OS_ERROR_MODE BIGINT, OS_PRIORITY_CLASS BIGINT, MAX_WORKERS_COUNT BIGINT, SCHEDULER_COUNT BIGINT, SCHEDULER_TOTAL_COUNT BIGINT, DEADLOCK_MONITOR_SERIAL_NUMBER BIGINT, SQLSERVER_START_TIME_MS_TICKS BIGINT, SQLSERVER_START_TIME TIMESTAMP, AFFINITY_TYPE BIGINT, AFFINITY_TYPE_DESC VARCHAR, PROCESS_KERNEL_TIME_MS BIGINT, PROCESS_USER_TIME_MS BIGINT, TIME_SOURCE BIGINT, TIME_SOURCE_DESC VARCHAR, VIRTUAL_MACHINE_TYPE BIGINT, VIRTUAL_MACHINE_TYPE_DESC VARCHAR, SOFTNUMA_CONFIGURATION BIGINT, SOFTNUMA_CONFIGURATION_DESC VARCHAR, PROCESS_PHYSICAL_AFFINITY VARCHAR, SQL_MEMORY_MODEL BIGINT, SQL_MEMORY_MODEL_DESC VARCHAR, SOCKET_COUNT BIGINT, CORES_PER_SOCKET BIGINT, NUMA_NODE_COUNT BIGINT, CONTAINER_TYPE BIGINT, CONTAINER_TYPE_DESC VARCHAR, EXTRACT_TS TIMESTAMP", + "mssql_databases": "DB_ID VARCHAR, NAME VARCHAR, COLLATION_NAME VARCHAR, CREATE_DATE TIMESTAMP, EXTRACT_TS TIMESTAMP", + "mssql_tables": "TABLE_CATALOG VARCHAR, TABLE_SCHEMA VARCHAR, TABLE_NAME VARCHAR, TABLE_TYPE VARCHAR", + "mssql_views": "TABLE_CATALOG VARCHAR, TABLE_SCHEMA VARCHAR, TABLE_NAME VARCHAR, CHECK_OPTION VARCHAR, IS_UPDATABLE VARCHAR, VIEW_DEFINITION VARCHAR", + "mssql_columns": "TABLE_CATALOG VARCHAR, TABLE_SCHEMA VARCHAR, TABLE_NAME VARCHAR, COLUMN_NAME VARCHAR, ORDINAL_POSITION BIGINT, COLUMN_DEFAULT VARCHAR, IS_NULLABLE VARCHAR, DATA_TYPE VARCHAR, CHARACTER_MAXIMUM_LENGTH DOUBLE, CHARACTER_OCTET_LENGTH DOUBLE, NUMERIC_PRECISION DOUBLE, NUMERIC_PRECISION_RADIX DOUBLE, NUMERIC_SCALE DOUBLE, DATETIME_PRECISION DOUBLE, CHARACTER_SET_CATALOG VARCHAR, CHARACTER_SET_SCHEMA VARCHAR, CHARACTER_SET_NAME VARCHAR, COLLATION_CATALOG VARCHAR, COLLATION_SCHEMA VARCHAR, COLLATION_NAME VARCHAR, DOMAIN_CATALOG VARCHAR, DOMAIN_SCHEMA VARCHAR, DOMAIN_NAME VARCHAR", + "mssql_indexed_views": "INDEXED_VIEW_NAME VARCHAR, SCHEMA_NAME VARCHAR, INDEX_NAME VARCHAR, INDEX_TYPE VARCHAR, INDEX_ID VARCHAR, EXTRACT_TS VARCHAR", + "mssql_routines": "CREATED TIMESTAMP, DATA_TYPE VARCHAR, IS_DETERMINISTIC VARCHAR, IS_IMPLICITLY_INVOCABLE VARCHAR, IS_NULL_CALL VARCHAR, IS_USER_DEFINED_CAST VARCHAR, LAST_ALTERED TIMESTAMP, MAX_DYNAMIC_RESULT_SETS BIGINT, NUMERIC_PRECISION DOUBLE, NUMERIC_PRECISION_RADIX DOUBLE, NUMERIC_SCALE DOUBLE, ROUTINE_BODY VARCHAR, ROUTINE_CATALOG VARCHAR, ROUTINE_DEFINITION VARCHAR, ROUTINE_NAME VARCHAR, ROUTINE_SCHEMA VARCHAR, ROUTINE_TYPE VARCHAR, SCHEMA_LEVEL_ROUTINE VARCHAR, SPECIFIC_CATALOG VARCHAR, SPECIFIC_NAME VARCHAR, SPECIFIC_SCHEMA VARCHAR, SQL_DATA_ACCESS VARCHAR", + "mssql_db_sizes": "DATABASE_NAME VARCHAR, FILENAME VARCHAR, TYPE_DESC VARCHAR, CURRENTSIZEMB VARCHAR, FREESPACEINMB VARCHAR, MAXSIZE BIGINT, EXTRACT_TS TIMESTAMP", + "mssql_table_sizes": "TABLENAME VARCHAR, ROWCOUNT BIGINT, RESERVEDMB BIGINT, USEDMB BIGINT, UNUSEDMB BIGINT, DATAMB BIGINT, INDEXMB BIGINT, EXTRACT_TS TIMESTAMP", } try: columns = list(result.columns) diff --git a/tests/integration/cli/test_profiler_connection_cli.py b/tests/integration/cli/test_profiler_connection_cli.py index 6ae70188ec..a72580f45d 100644 --- a/tests/integration/cli/test_profiler_connection_cli.py +++ b/tests/integration/cli/test_profiler_connection_cli.py @@ -85,9 +85,8 @@ def test_profiler_connection_invalid_source_technology( """Test error handling for unsupported source technology.""" cred_path = _create_credentials_file(sandbox_synapse_cred_config, tmp_path, exclude_serverless=True) - # mssql is not in PROFILER_SOURCE_SYSTEM with pytest.raises(ValueError, match="Invalid source technology"): - check_connection(w=ws, source_tech="mssql", cred_file_path=str(cred_path)) + check_connection(w=ws, source_tech="bogus", cred_file_path=str(cred_path)) @pytest.mark.parametrize( diff --git a/tests/unit/assessment/test_assessment.py b/tests/unit/assessment/test_assessment.py index 0a9f589790..deaba08b25 100644 --- a/tests/unit/assessment/test_assessment.py +++ b/tests/unit/assessment/test_assessment.py @@ -12,12 +12,15 @@ def test_configure_sqlserver_credentials(tmp_path): { r"Enter secret vault type \(local \| env\)": sorted(['local', 'env']).index("env"), r"Enter the database name": "TEST_TSQL_JDBC", - r"Enter the driver details": "ODBC Driver 18 for SQL Server", - r"Enter the server or host details": "TEST_TSQL_JDBC", + r"Enter the ODBC driver installed locally.*": "ODBC Driver 18 for SQL Server", + r"Enter the fully-qualified server name": "URL", r"Enter the port details": "1433", - r"Enter the user details": "TEST_TSQL_USER", - r"Enter the password details": "TEST_TSQL_PASS", - r"Do you want to test the connection to mssql?": "no", + r"Enter the SQL username": "TEST_TSQL_USER", + r"Enter the SQL password": "TEST_TSQL_PASS", + r"Do you want to test the connection to mssql?.*": "no", + r"Enter fetch size": "4000", + r"Enter timezone.*": "UTC", + r"Enter login timeout.*": 5, } ) file = tmp_path / ".credentials.yml" @@ -30,12 +33,15 @@ def test_configure_sqlserver_credentials(tmp_path): 'secret_vault_type': 'env', 'secret_vault_name': None, 'mssql': { - 'database': 'TEST_TSQL_JDBC', + 'auth_type': 'sql_authentication', 'driver': 'ODBC Driver 18 for SQL Server', - 'server': 'TEST_TSQL_JDBC', + 'fetch_size': '4000', + 'login_timeout': 5, + 'password': 'TEST_TSQL_PASS', 'port': 1433, + 'server': 'URL', + 'tz_info': 'UTC', 'user': 'TEST_TSQL_USER', - 'password': 'TEST_TSQL_PASS', }, } diff --git a/tests/unit/test_install.py b/tests/unit/test_install.py index 8173e8cf4c..9bdfe5c562 100644 --- a/tests/unit/test_install.py +++ b/tests/unit/test_install.py @@ -881,7 +881,7 @@ def test_configure_reconcile_databricks_no_existing_installation(ws: WorkspaceCl ) -def test_configure_all_override_installation( +def test_configure_all_override_installation( # FIXME ws_installer: Callable[..., WorkspaceInstaller], ws: WorkspaceClient, ) -> None: @@ -1001,7 +1001,7 @@ def test_configure_all_override_installation( ) expected_profiler_dash_config = ProfilerDashboardConfig( - source_tech="synapse", + source_tech="mssql", extract_file_path=str( Path("~/.databricks/labs/lakebridge_profilers/synapse_assessment/profiler_extract.db").expanduser() ), diff --git a/uv.lock b/uv.lock index 362e3072cf..95566fca52 100644 --- a/uv.lock +++ b/uv.lock @@ -8,23 +8,30 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-04-07T11:03:49.852608Z" +exclude-newer = "2026-04-29T10:08:14.097671Z" exclude-newer-span = "P7D" [options.exclude-newer-package] databricks-labs-lsql = false -databricks-labs-pytester = false databricks-labs-blueprint = false +hatchling = false databricks-switch-plugin = false databricks-connect = false -databricks-sdk = false +trove-classifiers = false databricks-bb-analyzer = false +databricks-labs-pytester = false +pathspec = false +tomli = false +packaging = false +databricks-sdk = false +pluggy = false databricks-labs-pylint = false +editables = false [[package]] name = "argcomplete" version = "3.6.3" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/38/61/0b9ae6399dd4a58d8c1b1dc5a27d6f2808023d0b5dd3104bb99f45a33ff6/argcomplete-3.6.3.tar.gz", hash = "sha256:62e8ed4fd6a45864acc8235409461b72c9a28ee785a2011cc5eb78318786c89c", size = 73754, upload-time = "2025-10-20T03:33:34.741Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/74/f5/9373290775639cb67a2fce7f629a1c240dce9f12fe927bc32b2736e16dfc/argcomplete-3.6.3-py3-none-any.whl", hash = "sha256:f5007b3a600ccac5d25bbce33089211dfd49eab4a7718da3f10e3082525a92ce", size = 43846, upload-time = "2025-10-20T03:33:33.021Z" }, @@ -33,7 +40,7 @@ wheels = [ [[package]] name = "astroid" version = "3.2.4" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] @@ -45,7 +52,7 @@ wheels = [ [[package]] name = "attrs" version = "26.1.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, @@ -54,7 +61,7 @@ wheels = [ [[package]] name = "azure-common" version = "1.1.28" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/3e/71/f6f71a276e2e69264a97ad39ef850dca0a04fce67b12570730cb38d0ccac/azure-common-1.1.28.zip", hash = "sha256:4ac0cd3214e36b6a1b6a442686722a5d8cc449603aa833f3f0f40bda836704a3", size = 20914, upload-time = "2022-02-03T19:39:44.373Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/62/55/7f118b9c1b23ec15ca05d15a578d8207aa1706bc6f7c87218efffbbf875d/azure_common-1.1.28-py2.py3-none-any.whl", hash = "sha256:5c12d3dcf4ec20599ca6b0d3e09e86e146353d443e7fcc050c9a19c1f9df20ad", size = 14462, upload-time = "2022-02-03T19:39:42.417Z" }, @@ -63,7 +70,7 @@ wheels = [ [[package]] name = "azure-core" version = "1.39.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "requests" }, { name = "typing-extensions" }, @@ -76,7 +83,7 @@ wheels = [ [[package]] name = "azure-identity" version = "1.19.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "azure-core" }, { name = "cryptography" }, @@ -92,7 +99,7 @@ wheels = [ [[package]] name = "azure-mgmt-core" version = "1.6.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "azure-core" }, ] @@ -101,10 +108,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/26/c79f962fd3172b577b6f38685724de58b6b4337a51d3aad316a43a4558c6/azure_mgmt_core-1.6.0-py3-none-any.whl", hash = "sha256:0460d11e85c408b71c727ee1981f74432bc641bb25dfcf1bb4e90a49e776dbc4", size = 29310, upload-time = "2025-07-03T02:02:25.203Z" }, ] +[[package]] +name = "azure-mgmt-sql" +version = "3.0.1" +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +dependencies = [ + { name = "azure-common" }, + { name = "azure-mgmt-core" }, + { name = "msrest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3f/af/398c57d15064ea23475076cd087b1a143b66d33a029e6e47c4688ca32310/azure-mgmt-sql-3.0.1.zip", hash = "sha256:129042cc011225e27aee6ef2697d585fa5722e5d1aeb0038af6ad2451a285457", size = 996625, upload-time = "2021-07-16T02:07:56.492Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/46/db6dd21a237c32eb747c4a1790a6b0aafe1f52c55e07b90f732231027d47/azure_mgmt_sql-3.0.1-py2.py3-none-any.whl", hash = "sha256:1d1dd940d4d41be4ee319aad626341251572a5bf4a2addec71779432d9a1381f", size = 912881, upload-time = "2021-07-16T02:07:53.079Z" }, +] + [[package]] name = "azure-monitor-query" version = "1.4.1" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "azure-core" }, { name = "isodate" }, @@ -118,7 +139,7 @@ wheels = [ [[package]] name = "azure-synapse-artifacts" version = "0.20.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "azure-common" }, { name = "azure-mgmt-core" }, @@ -133,7 +154,7 @@ wheels = [ [[package]] name = "backports-asyncio-runner" version = "1.2.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/8e/ff/70dca7d7cb1cbc0edb2c6cc0c38b65cba36cccc491eca64cabd5fe7f8670/backports_asyncio_runner-1.2.0.tar.gz", hash = "sha256:a5aa7b2b7d8f8bfcaa2b57313f70792df84e32a2a746f585213373f900b42162", size = 69893, upload-time = "2025-07-02T02:27:15.685Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/a0/59/76ab57e3fe74484f48a53f8e337171b4a2349e506eabe136d7e01d059086/backports_asyncio_runner-1.2.0-py3-none-any.whl", hash = "sha256:0da0a936a8aeb554eccb426dc55af3ba63bcdc69fa1a600b5bb305413a4477b5", size = 12313, upload-time = "2025-07-02T02:27:14.263Z" }, @@ -142,7 +163,7 @@ wheels = [ [[package]] name = "black" version = "25.9.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "click" }, { name = "mypy-extensions" }, @@ -177,7 +198,7 @@ wheels = [ [[package]] name = "cattrs" version = "25.3.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "attrs" }, { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, @@ -191,7 +212,7 @@ wheels = [ [[package]] name = "certifi" version = "2026.2.25" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/af/2d/7bf41579a8986e348fa033a31cdd0e4121114f6bce2457e8876010b092dd/certifi-2026.2.25.tar.gz", hash = "sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7", size = 155029, upload-time = "2026-02-25T02:54:17.342Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/9a/3c/c17fb3ca2d9c3acff52e30b309f538586f9f5b9c9cf454f3845fc9af4881/certifi-2026.2.25-py3-none-any.whl", hash = "sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa", size = 153684, upload-time = "2026-02-25T02:54:15.766Z" }, @@ -200,7 +221,7 @@ wheels = [ [[package]] name = "cffi" version = "2.0.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "pycparser", marker = "implementation_name != 'PyPy'" }, ] @@ -260,7 +281,7 @@ wheels = [ [[package]] name = "charset-normalizer" version = "3.4.7" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/e7/a1/67fe25fac3c7642725500a3f6cfe5821ad557c3abb11c9d20d12c7008d3e/charset_normalizer-3.4.7.tar.gz", hash = "sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5", size = 144271, upload-time = "2026-04-02T09:28:39.342Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/26/08/0f303cb0b529e456bb116f2d50565a482694fbb94340bf56d44677e7ed03/charset_normalizer-3.4.7-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cdd68a1fb318e290a2077696b7eb7a21a49163c455979c639bf5a5dcdc46617d", size = 315182, upload-time = "2026-04-02T09:25:40.673Z" }, @@ -333,7 +354,7 @@ wheels = [ [[package]] name = "click" version = "8.3.2" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, ] @@ -345,7 +366,7 @@ wheels = [ [[package]] name = "colorama" version = "0.4.6" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, @@ -354,7 +375,7 @@ wheels = [ [[package]] name = "coverage" version = "7.10.7" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/51/26/d22c300112504f5f9a9fd2297ce33c35f3d353e4aeb987c8419453b2a7c2/coverage-7.10.7.tar.gz", hash = "sha256:f4ab143ab113be368a3e9b795f9cd7906c5ef407d6173fe9675a902e1fffc239", size = 827704, upload-time = "2025-09-21T20:03:56.815Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/e5/6c/3a3f7a46888e69d18abe3ccc6fe4cb16cccb1e6a2f99698931dafca489e6/coverage-7.10.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:fc04cc7a3db33664e0c2d10eb8990ff6b3536f6842c9590ae8da4c614b9ed05a", size = 217987, upload-time = "2025-09-21T20:00:57.218Z" }, @@ -432,7 +453,7 @@ toml = [ [[package]] name = "cryptography" version = "46.0.6" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, { name = "typing-extensions", marker = "python_full_version < '3.11'" }, @@ -478,7 +499,7 @@ wheels = [ [[package]] name = "databricks-bb-analyzer" version = "0.3.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "jsonschema" }, ] @@ -489,7 +510,7 @@ wheels = [ [[package]] name = "databricks-connect" version = "15.1.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "databricks-sdk" }, { name = "googleapis-common-protos" }, @@ -509,7 +530,7 @@ wheels = [ [[package]] name = "databricks-labs-blueprint" version = "0.12.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "databricks-sdk" }, ] @@ -547,6 +568,7 @@ dependencies = [ [package.dev-dependencies] dev = [ { name = "azure-identity" }, + { name = "azure-mgmt-sql" }, { name = "azure-monitor-query" }, { name = "azure-synapse-artifacts" }, { name = "black" }, @@ -572,6 +594,7 @@ dev = [ ] lint = [ { name = "azure-identity" }, + { name = "azure-mgmt-sql" }, { name = "azure-monitor-query" }, { name = "azure-synapse-artifacts" }, { name = "black" }, @@ -615,7 +638,7 @@ yq = [ [package.metadata] requires-dist = [ { name = "cryptography", specifier = ">=44.0.2,<46.1.0" }, - { name = "databricks-bb-analyzer", specifier = "~=0.3.0,!=0.3.1" }, + { name = "databricks-bb-analyzer", specifier = "~=0.3.0" }, { name = "databricks-labs-blueprint", extras = ["yaml"], specifier = ">=0.12.0,<0.13.0" }, { name = "databricks-labs-lsql", specifier = "==0.16.0" }, { name = "databricks-sdk", specifier = "~=0.85.0" }, @@ -634,6 +657,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ { name = "azure-identity", specifier = "~=1.19.0" }, + { name = "azure-mgmt-sql", specifier = "~=3.0.1" }, { name = "azure-monitor-query", specifier = "~=1.4.0" }, { name = "azure-synapse-artifacts", specifier = "~=0.20.0" }, { name = "black", specifier = "~=25.9.0" }, @@ -659,6 +683,7 @@ dev = [ ] lint = [ { name = "azure-identity", specifier = "~=1.19.0" }, + { name = "azure-mgmt-sql", specifier = "~=3.0.1" }, { name = "azure-monitor-query", specifier = "~=1.4.0" }, { name = "azure-synapse-artifacts", specifier = "~=0.20.0" }, { name = "black", specifier = "~=25.9.0" }, @@ -698,7 +723,7 @@ yq = [{ name = "yq", specifier = "~=3.4.3" }] [[package]] name = "databricks-labs-lsql" version = "0.16.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "databricks-labs-blueprint", extra = ["yaml"] }, { name = "databricks-sdk" }, @@ -712,7 +737,7 @@ wheels = [ [[package]] name = "databricks-labs-pylint" version = "0.5.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "astroid" }, { name = "databricks-sdk" }, @@ -727,7 +752,7 @@ wheels = [ [[package]] name = "databricks-labs-pytester" version = "0.7.4" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "databricks-labs-lsql" }, { name = "databricks-sdk" }, @@ -741,7 +766,7 @@ wheels = [ [[package]] name = "databricks-sdk" version = "0.85.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "google-auth" }, { name = "protobuf" }, @@ -755,7 +780,7 @@ wheels = [ [[package]] name = "databricks-switch-plugin" version = "0.1.8" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/b3/27/531c2fd0c787752ee6b27cfa95ee04ee5197f700343fcc8f0356788c03b4/databricks_switch_plugin-0.1.8.tar.gz", hash = "sha256:c7dece48c1490c56782adf335b366bad0401ea06350f06041d607fe810129b35", size = 93389, upload-time = "2026-03-24T03:47:17.53Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/4c/0b/f0d3a4ef90aad8f7b66be4cafb7aebc3003b449be0f79cf8b9ce02eb68aa/databricks_switch_plugin-0.1.8-py3-none-any.whl", hash = "sha256:c4a6ee8e3c0603a97ec3180d7bd2a952c5bc9eb9fe3c0316b09a0eca095980c8", size = 135427, upload-time = "2026-03-24T03:47:15.705Z" }, @@ -764,7 +789,7 @@ wheels = [ [[package]] name = "dill" version = "0.4.1" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/81/e1/56027a71e31b02ddc53c7d65b01e68edf64dea2932122fe7746a516f75d5/dill-0.4.1.tar.gz", hash = "sha256:423092df4182177d4d8ba8290c8a5b640c66ab35ec7da59ccfa00f6fa3eea5fa", size = 187315, upload-time = "2026-01-19T02:36:56.85Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/1e/77/dc8c558f7593132cf8fefec57c4f60c83b16941c574ac5f619abb3ae7933/dill-0.4.1-py3-none-any.whl", hash = "sha256:1e1ce33e978ae97fcfcff5638477032b801c46c7c65cf717f95fbc2248f79a9d", size = 120019, upload-time = "2026-01-19T02:36:55.663Z" }, @@ -773,7 +798,7 @@ wheels = [ [[package]] name = "duckdb" version = "1.4.4" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/36/9d/ab66a06e416d71b7bdcb9904cdf8d4db3379ef632bb8e9495646702d9718/duckdb-1.4.4.tar.gz", hash = "sha256:8bba52fd2acb67668a4615ee17ee51814124223de836d9e2fdcbc4c9021b3d3c", size = 18419763, upload-time = "2026-01-26T11:50:37.68Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/a2/9f/67a75f1e88f84946909826fa7aadd0c4b0dc067f24956142751fd9d59fe6/duckdb-1.4.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e870a441cb1c41d556205deb665749f26347ed13b3a247b53714f5d589596977", size = 28884338, upload-time = "2026-01-26T11:48:41.591Z" }, @@ -808,7 +833,7 @@ wheels = [ [[package]] name = "eradicate" version = "2.3.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/7a/e1/665186aedea2d6ebf0415cf97c0629c8123a721e7afc417deeade5598215/eradicate-2.3.0.tar.gz", hash = "sha256:06df115be3b87d0fc1c483db22a2ebb12bcf40585722810d809cc770f5031c37", size = 8536, upload-time = "2023-06-09T06:31:41.814Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/90/c2/533e1338429aeba1f089566a2314d69d3e78ab57a73006f16a923bf2b24c/eradicate-2.3.0-py3-none-any.whl", hash = "sha256:2b29b3dd27171f209e4ddd8204b70c02f0682ae95eecb353f10e8d72b149c63e", size = 6113, upload-time = "2023-06-09T06:31:40.209Z" }, @@ -817,7 +842,7 @@ wheels = [ [[package]] name = "exceptiongroup" version = "1.3.1" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] @@ -829,7 +854,7 @@ wheels = [ [[package]] name = "execnet" version = "2.1.2" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, @@ -838,7 +863,7 @@ wheels = [ [[package]] name = "faker" version = "40.12.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "tzdata", marker = "sys_platform == 'win32'" }, ] @@ -850,7 +875,7 @@ wheels = [ [[package]] name = "google-auth" version = "2.49.1" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "cryptography" }, { name = "pyasn1-modules" }, @@ -863,7 +888,7 @@ wheels = [ [[package]] name = "googleapis-common-protos" version = "1.74.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "protobuf" }, ] @@ -875,7 +900,7 @@ wheels = [ [[package]] name = "greenlet" version = "3.3.2" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/a3/51/1664f6b78fc6ebbd98019a1fd730e83fa78f2db7058f72b1463d3612b8db/greenlet-3.3.2.tar.gz", hash = "sha256:2eaf067fc6d886931c7962e8c6bede15d2f01965560f3359b27c80bde2d151f2", size = 188267, upload-time = "2026-02-20T20:54:15.531Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/38/3f/9859f655d11901e7b2996c6e3d33e0caa9a1d4572c3bc61ed0faa64b2f4c/greenlet-3.3.2-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:9bc885b89709d901859cf95179ec9f6bb67a3d2bb1f0e88456461bd4b7f8fd0d", size = 277747, upload-time = "2026-02-20T20:16:21.325Z" }, @@ -914,7 +939,7 @@ wheels = [ [[package]] name = "grpcio" version = "1.80.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "typing-extensions" }, ] @@ -965,7 +990,7 @@ wheels = [ [[package]] name = "grpcio-status" version = "1.80.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "googleapis-common-protos" }, { name = "grpcio" }, @@ -979,7 +1004,7 @@ wheels = [ [[package]] name = "idna" version = "3.11" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, @@ -988,7 +1013,7 @@ wheels = [ [[package]] name = "iniconfig" version = "2.3.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, @@ -997,7 +1022,7 @@ wheels = [ [[package]] name = "isodate" version = "0.7.2" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/54/4d/e940025e2ce31a8ce1202635910747e5a87cc3a6a6bb2d00973375014749/isodate-0.7.2.tar.gz", hash = "sha256:4cd1aa0f43ca76f4a6c6c0292a85f40b35ec2e43e315b59f06e6d32171a953e6", size = 29705, upload-time = "2024-10-08T23:04:11.5Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl", hash = "sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15", size = 22320, upload-time = "2024-10-08T23:04:09.501Z" }, @@ -1006,7 +1031,7 @@ wheels = [ [[package]] name = "isort" version = "5.13.2" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/87/f9/c1eb8635a24e87ade2efce21e3ce8cd6b8630bb685ddc9cdaca1349b2eb5/isort-5.13.2.tar.gz", hash = "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109", size = 175303, upload-time = "2023-12-13T20:37:26.124Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/d1/b3/8def84f539e7d2289a02f0524b944b15d7c75dab7628bedf1c4f0992029c/isort-5.13.2-py3-none-any.whl", hash = "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6", size = 92310, upload-time = "2023-12-13T20:37:23.244Z" }, @@ -1015,7 +1040,7 @@ wheels = [ [[package]] name = "jsonschema" version = "4.0.1" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "attrs" }, { name = "pyrsistent" }, @@ -1028,7 +1053,7 @@ wheels = [ [[package]] name = "lsprotocol" version = "2025.0.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "attrs" }, { name = "cattrs" }, @@ -1041,7 +1066,7 @@ wheels = [ [[package]] name = "mccabe" version = "0.7.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/e7/ff/0ffefdcac38932a54d2b5eed4e0ba8a408f215002cd178ad1df0f2806ff8/mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", size = 9658, upload-time = "2022-01-24T01:14:51.113Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/27/1a/1f68f9ba0c207934b35b86a8ca3aad8395a3d6dd7921c0686e23853ff5a9/mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e", size = 7350, upload-time = "2022-01-24T01:14:49.62Z" }, @@ -1050,7 +1075,7 @@ wheels = [ [[package]] name = "msal" version = "1.35.1" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "cryptography" }, { name = "pyjwt", extra = ["crypto"] }, @@ -1064,7 +1089,7 @@ wheels = [ [[package]] name = "msal-extensions" version = "1.3.1" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "msal" }, ] @@ -1073,10 +1098,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5e/75/bd9b7bb966668920f06b200e84454c8f3566b102183bc55c5473d96cb2b9/msal_extensions-1.3.1-py3-none-any.whl", hash = "sha256:96d3de4d034504e969ac5e85bae8106c8373b5c6568e4c8fa7af2eca9dbe6bca", size = 20583, upload-time = "2025-03-14T23:51:03.016Z" }, ] +[[package]] +name = "msrest" +version = "0.7.1" +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +dependencies = [ + { name = "azure-core" }, + { name = "certifi" }, + { name = "isodate" }, + { name = "requests" }, + { name = "requests-oauthlib" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/68/77/8397c8fb8fc257d8ea0fa66f8068e073278c65f05acb17dcb22a02bfdc42/msrest-0.7.1.zip", hash = "sha256:6e7661f46f3afd88b75667b7187a92829924446c7ea1d169be8c4bb7eeb788b9", size = 175332, upload-time = "2022-06-13T22:41:25.111Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/cf/f2966a2638144491f8696c27320d5219f48a072715075d168b31d3237720/msrest-0.7.1-py3-none-any.whl", hash = "sha256:21120a810e1233e5e6cc7fe40b474eeb4ec6f757a15d7cf86702c369f9567c32", size = 85384, upload-time = "2022-06-13T22:41:22.42Z" }, +] + [[package]] name = "mypy" version = "1.18.2" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "mypy-extensions" }, { name = "pathspec" }, @@ -1115,7 +1156,7 @@ wheels = [ [[package]] name = "mypy-extensions" version = "1.1.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, @@ -1124,7 +1165,7 @@ wheels = [ [[package]] name = "numpy" version = "1.26.4" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/65/6e/09db70a523a96d25e115e71cc56a6f9031e7b8cd166c1ac8438307c14058/numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010", size = 15786129, upload-time = "2024-02-06T00:26:44.495Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/a7/94/ace0fdea5241a27d13543ee117cbc65868e82213fb31a8eb7fe9ff23f313/numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0", size = 20631468, upload-time = "2024-02-05T23:48:01.194Z" }, @@ -1153,10 +1194,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/2e/86f24451c2d530c88daf997cb8d6ac622c1d40d19f5a031ed68a4b73a374/numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818", size = 15517754, upload-time = "2024-02-05T23:58:36.364Z" }, ] +[[package]] +name = "oauthlib" +version = "3.3.1" +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/5f/19930f824ffeb0ad4372da4812c50edbd1434f678c90c2733e1188edfc63/oauthlib-3.3.1.tar.gz", hash = "sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9", size = 185918, upload-time = "2025-06-19T22:48:08.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/9c/92789c596b8df838baa98fa71844d84283302f7604ed565dafe5a6b5041a/oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1", size = 160065, upload-time = "2025-06-19T22:48:06.508Z" }, +] + [[package]] name = "packaging" version = "26.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/65/ee/299d360cdc32edc7d2cf530f3accf79c4fca01e96ffc950d8a52213bd8e4/packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", size = 143416, upload-time = "2026-01-21T20:50:39.064Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" }, @@ -1165,7 +1215,7 @@ wheels = [ [[package]] name = "pandas" version = "2.3.3" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "numpy" }, { name = "python-dateutil" }, @@ -1213,7 +1263,7 @@ wheels = [ [[package]] name = "pandas-stubs" version = "2.3.0.250703" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "numpy" }, { name = "types-pytz" }, @@ -1226,7 +1276,7 @@ wheels = [ [[package]] name = "pathspec" version = "1.0.4" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/fa/36/e27608899f9b8d4dff0617b2d9ab17ca5608956ca44461ac14ac48b44015/pathspec-1.0.4.tar.gz", hash = "sha256:0210e2ae8a21a9137c0d470578cb0e595af87edaa6ebf12ff176f14a02e0e645", size = 131200, upload-time = "2026-01-27T03:59:46.938Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/ef/3c/2c197d226f9ea224a9ab8d197933f9da0ae0aac5b6e0f884e2b8d9c8e9f7/pathspec-1.0.4-py3-none-any.whl", hash = "sha256:fb6ae2fd4e7c921a165808a552060e722767cfa526f99ca5156ed2ce45a5c723", size = 55206, upload-time = "2026-01-27T03:59:45.137Z" }, @@ -1235,7 +1285,7 @@ wheels = [ [[package]] name = "pip" version = "26.0.1" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/48/83/0d7d4e9efe3344b8e2fe25d93be44f64b65364d3c8d7bc6dc90198d5422e/pip-26.0.1.tar.gz", hash = "sha256:c4037d8a277c89b320abe636d59f91e6d0922d08a05b60e85e53b296613346d8", size = 1812747, upload-time = "2026-02-05T02:20:18.702Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/de/f0/c81e05b613866b76d2d1066490adf1a3dbc4ee9d9c839961c3fc8a6997af/pip-26.0.1-py3-none-any.whl", hash = "sha256:bdb1b08f4274833d62c1aa29e20907365a2ceb950410df15fc9521bad440122b", size = 1787723, upload-time = "2026-02-05T02:20:16.416Z" }, @@ -1244,7 +1294,7 @@ wheels = [ [[package]] name = "platformdirs" version = "4.9.4" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/19/56/8d4c30c8a1d07013911a8fdbd8f89440ef9f08d07a1b50ab8ca8be5a20f9/platformdirs-4.9.4.tar.gz", hash = "sha256:1ec356301b7dc906d83f371c8f487070e99d3ccf9e501686456394622a01a934", size = 28737, upload-time = "2026-03-05T18:34:13.271Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/63/d7/97f7e3a6abb67d8080dd406fd4df842c2be0efaf712d1c899c32a075027c/platformdirs-4.9.4-py3-none-any.whl", hash = "sha256:68a9a4619a666ea6439f2ff250c12a853cd1cbd5158d258bd824a7df6be2f868", size = 21216, upload-time = "2026-03-05T18:34:12.172Z" }, @@ -1253,7 +1303,7 @@ wheels = [ [[package]] name = "pluggy" version = "1.6.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, @@ -1262,7 +1312,7 @@ wheels = [ [[package]] name = "protobuf" version = "6.33.6" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/66/70/e908e9c5e52ef7c3a6c7902c9dfbb34c7e29c25d2f81ade3856445fd5c94/protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135", size = 444531, upload-time = "2026-03-18T19:05:00.988Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/fc/9f/2f509339e89cfa6f6a4c4ff50438db9ca488dec341f7e454adad60150b00/protobuf-6.33.6-cp310-abi3-win32.whl", hash = "sha256:7d29d9b65f8afef196f8334e80d6bc1d5d4adedb449971fefd3723824e6e77d3", size = 425739, upload-time = "2026-03-18T19:04:48.373Z" }, @@ -1277,7 +1327,7 @@ wheels = [ [[package]] name = "py4j" version = "0.10.9.7" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/1e/f2/b34255180c72c36ff7097f7c2cdca02abcbd89f5eebf7c7c41262a9a0637/py4j-0.10.9.7.tar.gz", hash = "sha256:0b6e5315bb3ada5cf62ac651d107bb2ebc02def3dee9d9548e3baac644ea8dbb", size = 1508234, upload-time = "2022-08-12T22:49:09.792Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/10/30/a58b32568f1623aaad7db22aa9eafc4c6c194b429ff35bdc55ca2726da47/py4j-0.10.9.7-py2.py3-none-any.whl", hash = "sha256:85defdfd2b2376eb3abf5ca6474b51ab7e0de341c75a02f46dc9b5976f5a5c1b", size = 200481, upload-time = "2022-08-12T22:49:07.05Z" }, @@ -1286,7 +1336,7 @@ wheels = [ [[package]] name = "pyarrow" version = "23.0.1" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/88/22/134986a4cc224d593c1afde5494d18ff629393d74cc2eddb176669f234a4/pyarrow-23.0.1.tar.gz", hash = "sha256:b8c5873e33440b2bc2f4a79d2b47017a89c5a24116c055625e6f2ee50523f019", size = 1167336, upload-time = "2026-02-16T10:14:12.39Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/bc/a8/24e5dc6855f50a62936ceb004e6e9645e4219a8065f304145d7fb8a79d5d/pyarrow-23.0.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:3fab8f82571844eb3c460f90a75583801d14ca0cc32b1acc8c361650e006fd56", size = 34307390, upload-time = "2026-02-16T10:08:08.654Z" }, @@ -1329,7 +1379,7 @@ wheels = [ [[package]] name = "pyasn1" version = "0.6.3" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/5c/5f/6583902b6f79b399c9c40674ac384fd9cd77805f9e6205075f828ef11fb2/pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf", size = 148685, upload-time = "2026-03-17T01:06:53.382Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/5d/a0/7d793dce3fa811fe047d6ae2431c672364b462850c6235ae306c0efd025f/pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde", size = 83997, upload-time = "2026-03-17T01:06:52.036Z" }, @@ -1338,7 +1388,7 @@ wheels = [ [[package]] name = "pyasn1-modules" version = "0.4.2" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "pyasn1" }, ] @@ -1350,7 +1400,7 @@ wheels = [ [[package]] name = "pycparser" version = "3.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492, upload-time = "2026-01-21T14:26:51.89Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, @@ -1359,7 +1409,7 @@ wheels = [ [[package]] name = "pygls" version = "2.0.1" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "attrs" }, { name = "cattrs" }, @@ -1373,7 +1423,7 @@ wheels = [ [[package]] name = "pygments" version = "2.20.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, @@ -1382,7 +1432,7 @@ wheels = [ [[package]] name = "pyjwt" version = "2.12.1" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] @@ -1399,7 +1449,7 @@ crypto = [ [[package]] name = "pylint" version = "3.2.7" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "astroid" }, { name = "colorama", marker = "sys_platform == 'win32'" }, @@ -1418,7 +1468,7 @@ wheels = [ [[package]] name = "pylint-pytest" version = "2.0.0a0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "pylint" }, { name = "pytest" }, @@ -1431,7 +1481,7 @@ wheels = [ [[package]] name = "pyodbc" version = "5.3.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/8f/85/44b10070a769a56bd910009bb185c0c0a82daff8d567cd1a116d7d730c7d/pyodbc-5.3.0.tar.gz", hash = "sha256:2fe0e063d8fb66efd0ac6dc39236c4de1a45f17c33eaded0d553d21c199f4d05", size = 121770, upload-time = "2025-10-17T18:04:09.43Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/be/cd/d0ac9e8963cf43f3c0e8ebd284cd9c5d0e17457be76c35abe4998b7b6df2/pyodbc-5.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6682cdec78f1302d0c559422c8e00991668e039ed63dece8bf99ef62173376a5", size = 71888, upload-time = "2025-10-17T18:02:58.285Z" }, @@ -1475,7 +1525,7 @@ wheels = [ [[package]] name = "pyrsistent" version = "0.20.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/ce/3a/5031723c09068e9c8c2f0bc25c3a9245f2b1d1aea8396c787a408f2b95ca/pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4", size = 103642, upload-time = "2023-10-25T21:06:56.342Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/c7/19/c343b14061907b629b765444b6436b160e2bd4184d17d4804bbe6381f6be/pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce", size = 83416, upload-time = "2023-10-25T21:06:04.579Z" }, @@ -1502,7 +1552,7 @@ wheels = [ [[package]] name = "pytest" version = "8.4.2" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, @@ -1520,7 +1570,7 @@ wheels = [ [[package]] name = "pytest-asyncio" version = "1.2.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "backports-asyncio-runner", marker = "python_full_version < '3.11'" }, { name = "pytest" }, @@ -1534,7 +1584,7 @@ wheels = [ [[package]] name = "pytest-cov" version = "7.0.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "coverage", extra = ["toml"] }, { name = "pluggy" }, @@ -1548,7 +1598,7 @@ wheels = [ [[package]] name = "pytest-timeout" version = "2.4.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "pytest" }, ] @@ -1560,7 +1610,7 @@ wheels = [ [[package]] name = "pytest-xdist" version = "3.8.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "execnet" }, { name = "pytest" }, @@ -1573,7 +1623,7 @@ wheels = [ [[package]] name = "python-dateutil" version = "2.9.0.post0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "six" }, ] @@ -1585,7 +1635,7 @@ wheels = [ [[package]] name = "pytokens" version = "0.4.1" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/b6/34/b4e015b99031667a7b960f888889c5bd34ef585c85e1cb56a594b92836ac/pytokens-0.4.1.tar.gz", hash = "sha256:292052fe80923aae2260c073f822ceba21f3872ced9a68bb7953b348e561179a", size = 23015, upload-time = "2026-01-30T01:03:45.924Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/42/24/f206113e05cb8ef51b3850e7ef88f20da6f4bf932190ceb48bd3da103e10/pytokens-0.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2a44ed93ea23415c54f3face3b65ef2b844d96aeb3455b8a69b3df6beab6acc5", size = 161522, upload-time = "2026-01-30T01:02:50.393Z" }, @@ -1614,7 +1664,7 @@ wheels = [ [[package]] name = "pytz" version = "2026.1.post1" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/56/db/b8721d71d945e6a8ac63c0fc900b2067181dbb50805958d4d4661cf7d277/pytz-2026.1.post1.tar.gz", hash = "sha256:3378dde6a0c3d26719182142c56e60c7f9af7e968076f31aae569d72a0358ee1", size = 321088, upload-time = "2026-03-03T07:47:50.683Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/10/99/781fe0c827be2742bcc775efefccb3b048a3a9c6ce9aec0cbf4a101677e5/pytz-2026.1.post1-py2.py3-none-any.whl", hash = "sha256:f2fd16142fda348286a75e1a524be810bb05d444e5a081f37f7affc635035f7a", size = 510489, upload-time = "2026-03-03T07:47:49.167Z" }, @@ -1623,7 +1673,7 @@ wheels = [ [[package]] name = "pyyaml" version = "6.0.3" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/f4/a0/39350dd17dd6d6c6507025c0e53aef67a9293a6d37d3511f23ea510d5800/pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b", size = 184227, upload-time = "2025-09-25T21:31:46.04Z" }, @@ -1669,7 +1719,7 @@ wheels = [ [[package]] name = "requests" version = "2.33.1" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "certifi" }, { name = "charset-normalizer" }, @@ -1681,10 +1731,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d7/8e/7540e8a2036f79a125c1d2ebadf69ed7901608859186c856fa0388ef4197/requests-2.33.1-py3-none-any.whl", hash = "sha256:4e6d1ef462f3626a1f0a0a9c42dd93c63bad33f9f1c1937509b8c5c8718ab56a", size = 64947, upload-time = "2026-03-30T16:09:13.83Z" }, ] +[[package]] +name = "requests-oauthlib" +version = "2.0.0" +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +dependencies = [ + { name = "oauthlib" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/42/f2/05f29bc3913aea15eb670be136045bf5c5bbf4b99ecb839da9b422bb2c85/requests-oauthlib-2.0.0.tar.gz", hash = "sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9", size = 55650, upload-time = "2024-03-22T20:32:29.939Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36", size = 24179, upload-time = "2024-03-22T20:32:28.055Z" }, +] + [[package]] name = "ruff" version = "0.13.3" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/c7/8e/f9f9ca747fea8e3ac954e3690d4698c9737c23b51731d02df999c150b1c9/ruff-0.13.3.tar.gz", hash = "sha256:5b0ba0db740eefdfbcce4299f49e9eaefc643d4d007749d77d047c2bab19908e", size = 5438533, upload-time = "2025-10-02T19:29:31.582Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/d2/33/8f7163553481466a92656d35dea9331095122bb84cf98210bef597dd2ecd/ruff-0.13.3-py3-none-linux_armv6l.whl", hash = "sha256:311860a4c5e19189c89d035638f500c1e191d283d0cc2f1600c8c80d6dcd430c", size = 12484040, upload-time = "2025-10-02T19:28:49.199Z" }, @@ -1710,7 +1773,7 @@ wheels = [ [[package]] name = "six" version = "1.17.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, @@ -1719,7 +1782,7 @@ wheels = [ [[package]] name = "sqlalchemy" version = "2.0.49" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, { name = "typing-extensions" }, @@ -1766,7 +1829,7 @@ wheels = [ [[package]] name = "sqlglot" version = "28.5.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/bf/8c/a4d24b6103305467506c1dea9c3ca8dc92773a91bae246c2517c256a0cf9/sqlglot-28.5.0.tar.gz", hash = "sha256:b3213b3e867dcc306074f1c90480aeee89a0e635cf0dfe70eb4a3af7b61972e6", size = 5652688, upload-time = "2025-12-17T23:38:00.121Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/ec/f7/6a7effd2526f64bcb0d2264c0dbebc7f8508add3f2c0748540d1448a24a3/sqlglot-28.5.0-py3-none-any.whl", hash = "sha256:5798bfdb6e9bc36c964e6c64d7222624d98b2631cc20f44628a82eba7cf7b4bf", size = 561086, upload-time = "2025-12-17T23:37:57.972Z" }, @@ -1775,7 +1838,7 @@ wheels = [ [[package]] name = "standard-distutils" version = "3.11.9" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/a8/da/c33a5b982f16b41fbe400863d6124377623c33401c34c0814b8302b1c5d0/standard_distutils-3.11.9.tar.gz", hash = "sha256:37d6c9f0f0321ed3c9c923e54b0fdbafa3d7bf556821de55fd8158d7f440deb5", size = 209658, upload-time = "2024-10-30T02:06:21.49Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/fc/1b/b8068593d405562f19ef8dd0b1858709db05fe44c9d25d1dbecc9e1666f8/standard_distutils-3.11.9-py3-none-any.whl", hash = "sha256:9ba3300167f1c95d5a24e9ac2eb323ad6cb146ec0bea703851a7720e8ffd7878", size = 259359, upload-time = "2024-10-30T02:06:19.595Z" }, @@ -1784,7 +1847,7 @@ wheels = [ [[package]] name = "tomli" version = "2.4.1" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/22/de/48c59722572767841493b26183a0d1cc411d54fd759c5607c4590b6563a6/tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f", size = 17543, upload-time = "2026-03-25T20:22:03.828Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/f4/11/db3d5885d8528263d8adc260bb2d28ebf1270b96e98f0e0268d32b8d9900/tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30", size = 154704, upload-time = "2026-03-25T20:21:10.473Z" }, @@ -1820,7 +1883,7 @@ wheels = [ [[package]] name = "tomlkit" version = "0.14.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/c3/af/14b24e41977adb296d6bd1fb59402cf7d60ce364f90c890bd2ec65c43b5a/tomlkit-0.14.0.tar.gz", hash = "sha256:cf00efca415dbd57575befb1f6634c4f42d2d87dbba376128adb42c121b87064", size = 187167, upload-time = "2026-01-13T01:14:53.304Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/b5/11/87d6d29fb5d237229d67973a6c9e06e048f01cf4994dee194ab0ea841814/tomlkit-0.14.0-py3-none-any.whl", hash = "sha256:592064ed85b40fa213469f81ac584f67a4f2992509a7c3ea2d632208623a3680", size = 39310, upload-time = "2026-01-13T01:14:51.965Z" }, @@ -1829,7 +1892,7 @@ wheels = [ [[package]] name = "types-pytz" version = "2025.2.0.20251108" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/40/ff/c047ddc68c803b46470a357454ef76f4acd8c1088f5cc4891cdd909bfcf6/types_pytz-2025.2.0.20251108.tar.gz", hash = "sha256:fca87917836ae843f07129567b74c1929f1870610681b4c92cb86a3df5817bdb", size = 10961, upload-time = "2025-11-08T02:55:57.001Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/e7/c1/56ef16bf5dcd255155cc736d276efa6ae0a5c26fd685e28f0412a4013c01/types_pytz-2025.2.0.20251108-py3-none-any.whl", hash = "sha256:0f1c9792cab4eb0e46c52f8845c8f77cf1e313cb3d68bf826aa867fe4717d91c", size = 10116, upload-time = "2025-11-08T02:55:56.194Z" }, @@ -1838,7 +1901,7 @@ wheels = [ [[package]] name = "types-pyyaml" version = "6.0.12.20250915" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/7e/69/3c51b36d04da19b92f9e815be12753125bd8bc247ba0470a982e6979e71c/types_pyyaml-6.0.12.20250915.tar.gz", hash = "sha256:0f8b54a528c303f0e6f7165687dd33fafa81c807fcac23f632b63aa624ced1d3", size = 17522, upload-time = "2025-09-15T03:01:00.728Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/bd/e0/1eed384f02555dde685fff1a1ac805c1c7dcb6dd019c916fe659b1c1f9ec/types_pyyaml-6.0.12.20250915-py3-none-any.whl", hash = "sha256:e7d4d9e064e89a3b3cae120b4990cd370874d2bf12fa5f46c97018dd5d3c9ab6", size = 20338, upload-time = "2025-09-15T03:00:59.218Z" }, @@ -1847,7 +1910,7 @@ wheels = [ [[package]] name = "types-requests" version = "2.33.0.20260402" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "urllib3" }, ] @@ -1859,7 +1922,7 @@ wheels = [ [[package]] name = "typing-extensions" version = "4.15.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, @@ -1868,7 +1931,7 @@ wheels = [ [[package]] name = "tzdata" version = "2026.1" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/19/f5/cd531b2d15a671a40c0f66cf06bc3570a12cd56eef98960068ebbad1bf5a/tzdata-2026.1.tar.gz", hash = "sha256:67658a1903c75917309e753fdc349ac0efd8c27db7a0cb406a25be4840f87f98", size = 197639, upload-time = "2026-04-03T11:25:22.002Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/b0/70/d460bd685a170790ec89317e9bd33047988e4bce507b831f5db771e142de/tzdata-2026.1-py2.py3-none-any.whl", hash = "sha256:4b1d2be7ac37ceafd7327b961aa3a54e467efbdb563a23655fbfe0d39cfc42a9", size = 348952, upload-time = "2026-04-03T11:25:20.313Z" }, @@ -1877,7 +1940,7 @@ wheels = [ [[package]] name = "urllib3" version = "2.6.3" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6661c76461e25f024d5a38a46f04aaca912426a2b1d3/urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", size = 435556, upload-time = "2026-01-07T16:24:43.925Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" }, @@ -1886,7 +1949,7 @@ wheels = [ [[package]] name = "xmltodict" version = "1.0.4" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/19/70/80f3b7c10d2630aa66414bf23d210386700aa390547278c789afa994fd7e/xmltodict-1.0.4.tar.gz", hash = "sha256:6d94c9f834dd9e44514162799d344d815a3a4faec913717a9ecbfa5be1bb8e61", size = 26124, upload-time = "2026-02-22T02:21:22.074Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/38/34/98a2f52245f4d47be93b580dae5f9861ef58977d73a79eb47c58f1ad1f3a/xmltodict-1.0.4-py3-none-any.whl", hash = "sha256:a4a00d300b0e1c59fc2bfccb53d7b2e88c32f200df138a0dd2229f842497026a", size = 13580, upload-time = "2026-02-22T02:21:21.039Z" }, @@ -1895,7 +1958,7 @@ wheels = [ [[package]] name = "yq" version = "3.4.3" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } dependencies = [ { name = "argcomplete" }, { name = "pyyaml" }, From 2f0075757ea5be1c42e7a1e00b63f00ab08422dd Mon Sep 17 00:00:00 2001 From: Noritaka Sekiyama Date: Tue, 12 May 2026 00:52:29 +0900 Subject: [PATCH 07/79] Fix reconcile schema fetch failure on Foreign Catalogs (Lakehouse Federation) (#2422) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Fixes #2366 - Foreign Catalogs (created via Lakehouse Federation) lack the Databricks-specific `full_data_type` column in `information_schema.columns`, causing `UNRESOLVED_COLUMN` errors for all reconcile report types (`schema`, `data`, `row`, `all`) - Add `DatabricksNonUnityCatalogDataSource(DatabricksDataSource)` that overrides only `get_schema` to use `DESCRIBE TABLE`, covering hive_metastore, global_temp views, and Foreign Catalogs - `DatabricksDataSource` is scoped to native Unity Catalog only (`information_schema.columns` with `full_data_type`) - `source_adapter.create_adapter` dispatches via `is_target`: target → `DatabricksDataSource`, source → `DatabricksNonUnityCatalogDataSource` - `catalog` parameter on `get_schema` / `read_data` tightened from `str | None` to `str` (now required by `SourceConnectionConfig`) ## Test plan - [x] Existing unit tests pass (1159) - [x] New / updated unit tests: `test_get_schema_uses_information_schema`, `test_get_schema_non_uc_uses_describe_table`, `test_get_schema_non_uc_foreign_catalog`, `test_get_schema_information_schema_exception_handling`, `test_get_schema_describe_exception_handling` - [x] Source adapter tests: `test_create_adapter_for_databricks_dialect_source`, `test_create_adapter_for_databricks_dialect_target` - [x] Integration test: Reconcile with Foreign Catalog (Lakebase PostgreSQL via Lakehouse Federation) as direct source --- Reopened from #2367 on an upstream branch to bypass the fork-PR OIDC restriction on JFrog auth (CI cannot run on fork PRs). All review comments and history are preserved on the original PR. --- .../reconcile/connectors/databricks.py | 51 +++++++++--- .../reconcile/connectors/source_adapter.py | 10 ++- .../labs/lakebridge/reconcile/utils.py | 4 +- .../reconcile/connectors/test_read_schema.py | 11 ++- .../reconcile/test_oracle_reconcile.py | 2 +- .../reconcile/connectors/test_databricks.py | 77 +++++++++++++------ tests/unit/reconcile/test_source_adapter.py | 34 +++++--- 7 files changed, 137 insertions(+), 52 deletions(-) diff --git a/src/databricks/labs/lakebridge/reconcile/connectors/databricks.py b/src/databricks/labs/lakebridge/reconcile/connectors/databricks.py index c83e3feb8f..2ca7ae65d5 100644 --- a/src/databricks/labs/lakebridge/reconcile/connectors/databricks.py +++ b/src/databricks/labs/lakebridge/reconcile/connectors/databricks.py @@ -16,14 +16,13 @@ logger = logging.getLogger(__name__) -def _get_schema_query(catalog: str, schema: str, table: str): - # TODO: Ensure that the target_catalog in the configuration is not set to "hive_metastore". The source_catalog - # can only be set to "hive_metastore" if the source type is "databricks". +def _get_describe_query(catalog: str, schema: str, table: str): if schema == "global_temp": return f"describe table global_temp.{table}" - if catalog == "hive_metastore": - return f"describe table {catalog}.{schema}.{table}" + return f"describe table {catalog}.{schema}.{table}" + +def _get_information_schema_query(catalog: str, schema: str, table: str): query = f"""select lower(column_name) as col_name, full_data_type as data_type @@ -36,6 +35,12 @@ def _get_schema_query(catalog: str, schema: str, table: str): class DatabricksDataSource(DataSource): + """Databricks data source backed by Unity Catalog `information_schema`. + + Use `DatabricksNonUnityCatalogDataSource` for hive_metastore, global views, + or Foreign Catalogs + """ + _IDENTIFIER_DELIMITER = "`" def __init__( @@ -50,17 +55,16 @@ def __init__( def read_data( self, - catalog: str | None, + catalog: str, schema: str, table: str, query: str, options: JdbcReaderOptions | None, ) -> DataFrame: - namespace_catalog = "hive_metastore" if not catalog else catalog if schema == "global_temp": namespace_catalog = "global_temp" else: - namespace_catalog = f"{namespace_catalog}.{schema}" + namespace_catalog = f"{catalog}.{schema}" table_with_namespace = f"{namespace_catalog}.{table}" table_query = query.replace(":tbl", table_with_namespace) try: @@ -71,13 +75,12 @@ def read_data( def get_schema( self, - catalog: str | None, + catalog: str, schema: str, table: str, normalize: bool = True, ) -> list[Schema]: - catalog_str = catalog if catalog else "hive_metastore" - schema_query = _get_schema_query(catalog_str, schema, table) + schema_query = _get_information_schema_query(catalog, schema, table) try: logger.debug(f"Fetching schema using query: \n`{schema_query}`") logger.info(f"Fetching Schema: Started at: {datetime.now()}") @@ -99,3 +102,29 @@ def normalize_identifier(self, identifier: str) -> NormalizedIdentifier: source_start_delimiter=DatabricksDataSource._IDENTIFIER_DELIMITER, source_end_delimiter=DatabricksDataSource._IDENTIFIER_DELIMITER, ) + + +class DatabricksNonUnityCatalogDataSource(DatabricksDataSource): + + def get_schema( + self, + catalog: str, + schema: str, + table: str, + normalize: bool = True, + ) -> list[Schema]: + schema_query = _get_describe_query(catalog, schema, table) + try: + logger.debug(f"Fetching schema using query: \n`{schema_query}`") + logger.info(f"Fetching Schema: Started at: {datetime.now()}") + schema_metadata = ( + self._spark.sql(schema_query) + .selectExpr("col_name as column_name", "data_type") + .where("column_name not like '#%'") + .distinct() + .collect() + ) + logger.info(f"Schema fetched successfully. Completed at: {datetime.now()}") + return [self._map_meta_column(field, normalize) for field in schema_metadata] + except (RuntimeError, PySparkException) as e: + return self.log_and_throw_exception(e, "schema", schema_query) diff --git a/src/databricks/labs/lakebridge/reconcile/connectors/source_adapter.py b/src/databricks/labs/lakebridge/reconcile/connectors/source_adapter.py index 932a8f0399..8a64a3a4e8 100644 --- a/src/databricks/labs/lakebridge/reconcile/connectors/source_adapter.py +++ b/src/databricks/labs/lakebridge/reconcile/connectors/source_adapter.py @@ -2,7 +2,10 @@ from sqlglot import Dialect from databricks.labs.lakebridge.reconcile.connectors.data_source import DataSource -from databricks.labs.lakebridge.reconcile.connectors.databricks import DatabricksDataSource +from databricks.labs.lakebridge.reconcile.connectors.databricks import ( + DatabricksDataSource, + DatabricksNonUnityCatalogDataSource, +) from databricks.labs.lakebridge.reconcile.connectors.oracle import OracleDataSource from databricks.labs.lakebridge.reconcile.connectors.remote_query_reader import RemoteQueryReader from databricks.labs.lakebridge.reconcile.connectors.snowflake import SnowflakeDataSource @@ -20,6 +23,7 @@ def create_adapter( spark: SparkSession, ws: WorkspaceClient, connection_name: str, + is_target: bool = False, ) -> DataSource: reader = RemoteQueryReader(spark, connection_name) if isinstance(engine, Snowflake): @@ -27,7 +31,9 @@ def create_adapter( if isinstance(engine, Oracle): return OracleDataSource(engine, reader) if isinstance(engine, Databricks): - return DatabricksDataSource(engine, spark, ws) + if is_target: + return DatabricksDataSource(engine, spark, ws) + return DatabricksNonUnityCatalogDataSource(engine, spark, ws) if isinstance(engine, Tsql): return TSQLServerDataSource(engine, reader) raise ValueError(f"Unsupported source type --> {engine}") diff --git a/src/databricks/labs/lakebridge/reconcile/utils.py b/src/databricks/labs/lakebridge/reconcile/utils.py index 73b5cd8931..e3c4b0275a 100644 --- a/src/databricks/labs/lakebridge/reconcile/utils.py +++ b/src/databricks/labs/lakebridge/reconcile/utils.py @@ -23,7 +23,9 @@ def initialise_data_source( else: source = create_adapter(engine=get_dialect(source_dialect), spark=spark, ws=ws, connection_name=connection_name) - target = create_adapter(engine=get_dialect("databricks"), spark=spark, ws=ws, connection_name="databricks") + target = create_adapter( + engine=get_dialect("databricks"), spark=spark, ws=ws, connection_name="databricks", is_target=True + ) return source, target diff --git a/tests/integration/reconcile/connectors/test_read_schema.py b/tests/integration/reconcile/connectors/test_read_schema.py index 2e659a46dc..96311453df 100644 --- a/tests/integration/reconcile/connectors/test_read_schema.py +++ b/tests/integration/reconcile/connectors/test_read_schema.py @@ -5,7 +5,10 @@ from pyspark.sql import SparkSession from databricks.sdk import WorkspaceClient from databricks.sdk.service.catalog import TableInfo -from databricks.labs.lakebridge.reconcile.connectors.databricks import DatabricksDataSource +from databricks.labs.lakebridge.reconcile.connectors.databricks import ( + DatabricksDataSource, + DatabricksNonUnityCatalogDataSource, +) from databricks.labs.lakebridge.reconcile.connectors.remote_query_reader import RemoteQueryReader from databricks.labs.lakebridge.reconcile.connectors.snowflake import SnowflakeDataSource from databricks.labs.lakebridge.reconcile.connectors.tsql import TSQLServerDataSource @@ -25,7 +28,8 @@ def test_sql_server_read_schema_happy(spark: SparkSession) -> None: def test_databricks_read_schema_happy(spark: SparkSession) -> None: mock_ws = create_autospec(WorkspaceClient) - connector = DatabricksDataSource(get_dialect("databricks"), spark, mock_ws) + # global_temp views are not in Unity Catalog's information_schema, so use the non-UC variant. + connector = DatabricksNonUnityCatalogDataSource(get_dialect("databricks"), spark, mock_ws) random_view = f"test_view_{uuid.uuid4().hex}" try: @@ -33,7 +37,8 @@ def test_databricks_read_schema_happy(spark: SparkSession) -> None: spark.sql("CREATE TABLE IF NOT EXISTS my_test_db.my_test_table (id INT, name STRING) USING parquet") df = spark.sql("SELECT * FROM my_test_db.my_test_table") df.createGlobalTempView(random_view) - columns = connector.get_schema(None, "global_temp", random_view) + # global_temp short-circuits the catalog, so the value here is ignored by the DESCRIBE query. + columns = connector.get_schema("hive_metastore", "global_temp", random_view) assert columns finally: diff --git a/tests/integration/reconcile/test_oracle_reconcile.py b/tests/integration/reconcile/test_oracle_reconcile.py index 28cd320fb8..9e91d0e56b 100644 --- a/tests/integration/reconcile/test_oracle_reconcile.py +++ b/tests/integration/reconcile/test_oracle_reconcile.py @@ -72,7 +72,7 @@ def __init__(self, databricks, ws, local_spark): def read_data( self, - catalog: str | None, + catalog: str, schema: str, table: str, query: str, diff --git a/tests/unit/reconcile/connectors/test_databricks.py b/tests/unit/reconcile/connectors/test_databricks.py index e21b850e1c..512b4dd6c1 100644 --- a/tests/unit/reconcile/connectors/test_databricks.py +++ b/tests/unit/reconcile/connectors/test_databricks.py @@ -5,7 +5,10 @@ from databricks.labs.lakebridge.reconcile.connectors.models import NormalizedIdentifier from databricks.labs.lakebridge.transpiler.sqlglot.dialect_utils import get_dialect -from databricks.labs.lakebridge.reconcile.connectors.databricks import DatabricksDataSource +from databricks.labs.lakebridge.reconcile.connectors.databricks import ( + DatabricksDataSource, + DatabricksNonUnityCatalogDataSource, +) from databricks.labs.lakebridge.reconcile.exception import DataSourceRuntimeException from databricks.sdk import WorkspaceClient @@ -14,18 +17,16 @@ def initial_setup(): pyspark_sql_session = MagicMock() spark = pyspark_sql_session.SparkSession.builder.getOrCreate() - # Define the source, workspace, and scope engine = get_dialect("databricks") ws = create_autospec(WorkspaceClient) return engine, spark, ws -def test_get_schema(): - # initial setup +def test_get_schema_uses_information_schema(): + """DatabricksDataSource always uses information_schema (UC native catalogs only).""" engine, spark, ws = initial_setup() - - # catalog as catalog ddds = DatabricksDataSource(engine, spark, ws) + ddds.get_schema("catalog", "schema", "supplier") spark.sql.assert_called_with( re.sub( @@ -40,24 +41,32 @@ def test_get_schema(): spark.sql().selectExpr.assert_called_with("col_name as column_name", "data_type") spark.sql().selectExpr().where.assert_called_with("column_name not like '#%'") - # hive_metastore as catalog + +def test_get_schema_non_uc_uses_describe_table(): + """DatabricksNonUnityCatalogDataSource always uses DESCRIBE TABLE for hive, global_temp, and Foreign Catalogs.""" + engine, spark, ws = initial_setup() + ddds = DatabricksNonUnityCatalogDataSource(engine, spark, ws) + + # UC catalog — non-UC variant still uses DESCRIBE TABLE (caller chooses the variant) + ddds.get_schema("catalog", "schema", "supplier") + spark.sql.assert_called_with("describe table catalog.schema.supplier") + + # hive_metastore ddds.get_schema("hive_metastore", "schema", "supplier") - spark.sql.assert_called_with(re.sub(r'\s+', ' ', """describe table hive_metastore.schema.supplier""")) - spark.sql().selectExpr.assert_called_with("col_name as column_name", "data_type") - spark.sql().selectExpr().where.assert_called_with("column_name not like '#%'") + spark.sql.assert_called_with("describe table hive_metastore.schema.supplier") - # global_temp as schema with hive_metastore + # global_temp ddds.get_schema("hive_metastore", "global_temp", "supplier") - spark.sql.assert_called_with(re.sub(r'\s+', ' ', """describe table global_temp.supplier""")) - spark.sql().selectExpr.assert_called_with("col_name as column_name", "data_type") - spark.sql().selectExpr().where.assert_called_with("column_name not like '#%'") + spark.sql.assert_called_with("describe table global_temp.supplier") + + # Foreign Catalog (Lakehouse Federation) + ddds.get_schema("foreign_catalog", "public", "customers") + spark.sql.assert_called_with("describe table foreign_catalog.public.customers") def test_read_data_from_uc(): - # initial setup engine, spark, ws = initial_setup() - # create object for DatabricksDataSource ddds = DatabricksDataSource(engine, spark, ws) # Test with query @@ -70,11 +79,9 @@ def test_read_data_from_uc(): def test_read_data_from_hive(): - # initial setup engine, spark, ws = initial_setup() - # create object for DatabricksDataSource - ddds = DatabricksDataSource(engine, spark, ws) + ddds = DatabricksNonUnityCatalogDataSource(engine, spark, ws) # Test with query ddds.read_data("hive_metastore", "data", "employee", "select id as id, name as name from :tbl", None) @@ -86,10 +93,8 @@ def test_read_data_from_hive(): def test_read_data_exception_handling(): - # initial setup engine, spark, ws = initial_setup() - # create object for DatabricksDataSource ddds = DatabricksDataSource(engine, spark, ws) spark.sql.side_effect = RuntimeError("Test Exception") @@ -101,11 +106,10 @@ def test_read_data_exception_handling(): ddds.read_data("org", "data", "employee", "select id as id, ename as name from :tbl", None) -def test_get_schema_exception_handling(): - # initial setup +def test_get_schema_information_schema_exception_handling(): + """DatabricksDataSource schema fetch exception includes the information_schema query.""" engine, spark, ws = initial_setup() - # create object for DatabricksDataSource ddds = DatabricksDataSource(engine, spark, ws) spark.sql.side_effect = RuntimeError("Test Exception") with pytest.raises(DataSourceRuntimeException) as exception: @@ -119,6 +123,31 @@ def test_get_schema_exception_handling(): ) +def test_get_schema_describe_exception_handling(): + """DatabricksNonUnityCatalogDataSource schema fetch exception includes the DESCRIBE TABLE query.""" + engine, spark, ws = initial_setup() + + ddds = DatabricksNonUnityCatalogDataSource(engine, spark, ws) + spark.sql.side_effect = RuntimeError("Test Exception") + with pytest.raises(DataSourceRuntimeException) as exception: + ddds.get_schema("org", "data", "employee") + + assert "describe table org.data.employee" in str(exception.value) + assert "Test Exception" in str(exception.value) + + +def test_get_schema_non_uc_foreign_catalog(): + """DatabricksNonUnityCatalogDataSource uses DESCRIBE TABLE for Foreign Catalogs without fallback.""" + engine, spark, ws = initial_setup() + ddds = DatabricksNonUnityCatalogDataSource(engine, spark, ws) + + ddds.get_schema("foreign_catalog", "public", "customers") + + # Only one SQL call — no fallback needed since the non-UC variant always uses DESCRIBE TABLE + assert spark.sql.call_count == 1 + spark.sql.assert_called_with("describe table foreign_catalog.public.customers") + + def test_normalize_identifier(): engine, spark, ws = initial_setup() data_source = DatabricksDataSource(engine, spark, ws) diff --git a/tests/unit/reconcile/test_source_adapter.py b/tests/unit/reconcile/test_source_adapter.py index f245c7c767..360bc2fc09 100644 --- a/tests/unit/reconcile/test_source_adapter.py +++ b/tests/unit/reconcile/test_source_adapter.py @@ -4,7 +4,10 @@ from databricks.connect import DatabricksSession from databricks.labs.lakebridge.transpiler.sqlglot.dialect_utils import get_dialect -from databricks.labs.lakebridge.reconcile.connectors.databricks import DatabricksDataSource +from databricks.labs.lakebridge.reconcile.connectors.databricks import ( + DatabricksDataSource, + DatabricksNonUnityCatalogDataSource, +) from databricks.labs.lakebridge.reconcile.connectors.oracle import OracleDataSource from databricks.labs.lakebridge.reconcile.connectors.snowflake import SnowflakeDataSource from databricks.labs.lakebridge.reconcile.connectors.source_adapter import create_adapter @@ -15,9 +18,9 @@ def test_create_adapter_for_snowflake_dialect(): spark = create_autospec(DatabricksSession) engine = get_dialect("snowflake") ws = create_autospec(WorkspaceClient) - scope = "scope" + connection_name = "snowflake" - data_source = create_adapter(engine, spark, ws, scope) + data_source = create_adapter(engine, spark, ws, connection_name) assert isinstance(data_source, SnowflakeDataSource) @@ -26,29 +29,40 @@ def test_create_adapter_for_oracle_dialect(): spark = create_autospec(DatabricksSession) engine = get_dialect("oracle") ws = create_autospec(WorkspaceClient) - scope = "scope" + connection_name = "oracle" - data_source = create_adapter(engine, spark, ws, scope) + data_source = create_adapter(engine, spark, ws, connection_name) assert isinstance(data_source, OracleDataSource) -def test_create_adapter_for_databricks_dialect(): +def test_create_adapter_for_databricks_dialect_source(): spark = create_autospec(DatabricksSession) engine = get_dialect("databricks") ws = create_autospec(WorkspaceClient) - scope = "scope" + connection_name = "databricks" - data_source = create_adapter(engine, spark, ws, scope) + data_source = create_adapter(engine, spark, ws, connection_name) + assert isinstance(data_source, DatabricksNonUnityCatalogDataSource) + +def test_create_adapter_for_databricks_dialect_target(): + spark = create_autospec(DatabricksSession) + engine = get_dialect("databricks") + ws = create_autospec(WorkspaceClient) + connection_name = "databricks" + + data_source = create_adapter(engine, spark, ws, connection_name, is_target=True) assert isinstance(data_source, DatabricksDataSource) + # Target uses the base class directly, not the non-UC subclass + assert not isinstance(data_source, DatabricksNonUnityCatalogDataSource) def test_raise_exception_for_unknown_dialect(): spark = create_autospec(DatabricksSession) engine = get_dialect("trino") ws = create_autospec(WorkspaceClient) - scope = "scope" + connection_name = "trino" with pytest.raises(ValueError, match=f"Unsupported source type --> {engine}"): - create_adapter(engine, spark, ws, scope) + create_adapter(engine, spark, ws, connection_name) From ec57e799e4804a89da87554008851885fc2bcceb Mon Sep 17 00:00:00 2001 From: Bishwajit <147722855+bishwajit-db@users.noreply.github.com> Date: Mon, 11 May 2026 23:27:30 +0530 Subject: [PATCH 08/79] Add Redshift connector to Recon (#2339) Add Redshift connector to Recon --------- Co-authored-by: M Abulazm --- .../docs/reconcile/recon_notebook.mdx | 2 +- .../reconcile/reconcile_configuration.mdx | 1 + .../labs/lakebridge/deployment/job.py | 10 -- .../reconcile/connectors/redshift.py | 94 +++++++++++++ .../reconcile/connectors/source_adapter.py | 4 + .../labs/lakebridge/reconcile/constants.py | 1 + .../query_builder/expression_generator.py | 30 +++++ tests/conftest.py | 36 +++++ tests/integration/reconcile/conftest.py | 51 +++++++ .../reconcile/connectors/test_read_schema.py | 10 ++ tests/integration/reconcile/test_recon_e2e.py | 11 ++ .../reconcile/test_schema_compare.py | 68 +++++++++- tests/unit/helpers/test_recon_config_utils.py | 2 +- .../reconcile/connectors/test_redshift.py | 127 ++++++++++++++++++ .../query_builder/test_hash_query.py | 39 ++++++ tests/unit/reconcile/test_source_adapter.py | 12 ++ 16 files changed, 485 insertions(+), 13 deletions(-) create mode 100644 src/databricks/labs/lakebridge/reconcile/connectors/redshift.py create mode 100644 tests/unit/reconcile/connectors/test_redshift.py diff --git a/docs/lakebridge/docs/reconcile/recon_notebook.mdx b/docs/lakebridge/docs/reconcile/recon_notebook.mdx index b4bdd7d7ad..0a60249f7b 100644 --- a/docs/lakebridge/docs/reconcile/recon_notebook.mdx +++ b/docs/lakebridge/docs/reconcile/recon_notebook.mdx @@ -77,7 +77,7 @@ Parameters: - `report_type`: The type of report to be generated. Available report types are `schema`, `row`, `data` or `all`. For details check [here](./dataflow_example.mdx). - `source`: The configuration for connecting to the source database to be reconciled. - - `dialect`: The dialect of the source. Supported values: `snowflake`, `oracle`, `mssql`, `synapse`, `databricks`. + - `dialect`: The dialect of the source. Supported values: `snowflake`, `oracle`, `mssql`, `synapse`, `databricks`, `redshift`. - `catalog`: The source database/catalog name. catalog is used for consistency in naming - `schema`: The source schema name. - `uc_connection_name`: the connection name for the source as configured in workspace `Connections`. Not allowed for `databricks` diff --git a/docs/lakebridge/docs/reconcile/reconcile_configuration.mdx b/docs/lakebridge/docs/reconcile/reconcile_configuration.mdx index 7f5243ad17..6f03a1cc45 100644 --- a/docs/lakebridge/docs/reconcile/reconcile_configuration.mdx +++ b/docs/lakebridge/docs/reconcile/reconcile_configuration.mdx @@ -26,6 +26,7 @@ import CodeBlock from '@theme/CodeBlock'; | Snowflake | Yes | Yes | Yes | Yes | | MS SQL Server (incl. Synapse) | Yes | Yes | Yes | Yes | | Databricks | Yes | Yes | Yes | Yes | +| Redshift | Yes | Yes | Yes | Yes | [[back to top](#types-of-report-supported)] diff --git a/src/databricks/labs/lakebridge/deployment/job.py b/src/databricks/labs/lakebridge/deployment/job.py index 58b0662810..9167adf383 100644 --- a/src/databricks/labs/lakebridge/deployment/job.py +++ b/src/databricks/labs/lakebridge/deployment/job.py @@ -16,7 +16,6 @@ ) from databricks.labs.lakebridge.config import ReconcileConfig, ProfilerDashboardConfig from databricks.labs.lakebridge.deployment.dashboard import ProfilerDashboardManager -from databricks.labs.lakebridge.reconcile.constants import ReconSourceType logger = logging.getLogger(__name__) @@ -107,15 +106,6 @@ def _job_recon_task( compute.Library(whl=lakebridge_wheel_path), ] - if recon_config.source.dialect == ReconSourceType.ORACLE.value: - # TODO: Automatically fetch a version list for `ojdbc8` - oracle_driver_version = "23.4.0.24.05" - libraries.append( - compute.Library( - maven=compute.MavenLibrary(f"com.oracle.database.jdbc:ojdbc8:{oracle_driver_version}"), - ), - ) - task = Task( task_key=task_key, description=description, diff --git a/src/databricks/labs/lakebridge/reconcile/connectors/redshift.py b/src/databricks/labs/lakebridge/reconcile/connectors/redshift.py new file mode 100644 index 0000000000..4940110059 --- /dev/null +++ b/src/databricks/labs/lakebridge/reconcile/connectors/redshift.py @@ -0,0 +1,94 @@ +import re +import logging +from datetime import datetime + +from pyspark.errors import PySparkException +from pyspark.sql import DataFrame +from pyspark.sql.functions import col +from sqlglot import Dialect + +from databricks.labs.lakebridge.reconcile.connectors.data_source import DataSource +from databricks.labs.lakebridge.reconcile.connectors.models import NormalizedIdentifier +from databricks.labs.lakebridge.reconcile.connectors.remote_query_reader import RemoteQueryReader +from databricks.labs.lakebridge.reconcile.connectors.dialect_utils import DialectUtils +from databricks.labs.lakebridge.reconcile.recon_config import JdbcReaderOptions, Schema + +logger = logging.getLogger(__name__) + + +class RedshiftDataSource(DataSource): + _IDENTIFIER_DELIMITER = "\"" + _SCHEMA_QUERY = """SELECT + column_name, + CASE + WHEN data_type = 'numeric' AND numeric_precision IS NOT NULL + THEN 'decimal(' || numeric_precision || ',' || numeric_scale || ')' + WHEN data_type = 'character varying' AND character_maximum_length IS NOT NULL + THEN 'varchar(' || character_maximum_length || ')' + WHEN data_type = 'character' AND character_maximum_length IS NOT NULL + THEN 'char(' || character_maximum_length || ')' + WHEN data_type IN ('binary varying') + THEN 'binary' + ELSE data_type + END AS data_type + FROM + information_schema.columns + WHERE + LOWER(table_name) = LOWER('{table}') + AND LOWER(table_schema) = LOWER('{schema}') + ORDER BY ordinal_position + """ + + def __init__( + self, + engine: Dialect, + reader: RemoteQueryReader, + ): + self._engine = engine + self._reader = reader + + def read_data( + self, + catalog: str, + schema: str, + table: str, + query: str, + options: JdbcReaderOptions | None, + ) -> DataFrame: + # Redshift dialect in SQLGlot converts :tbl to %(tbl)s (PostgreSQL parameter syntax) + table_query = query.replace("%(tbl)s", f"{schema}.{table}").replace(":tbl", f"{schema}.{table}") + try: + logger.info(f"Fetching data using query: \n`{table_query}`") + df = self._reader.read_data(table_query, catalog, "database", "query", options) + return df.select([col(c).alias(c.lower()) for c in df.columns]) + except (RuntimeError, PySparkException) as e: + return self.log_and_throw_exception(e, "data", table_query) + + def get_schema( + self, + catalog: str, + schema: str, + table: str, + normalize: bool = True, + ) -> list[Schema]: + schema_query = re.sub( + r'\s+', + ' ', + RedshiftDataSource._SCHEMA_QUERY.format(schema=schema, table=table), + ) + try: + logger.debug(f"Fetching schema using query: \n`{schema_query}`") + logger.info(f"Fetching Schema: Started at: {datetime.now()}") + df = self._reader.read_data(schema_query, catalog, "database", "query") + schema_metadata = df.select([col(c).alias(c.lower()) for c in df.columns]).collect() + logger.info(f"Schema fetched successfully. Completed at: {datetime.now()}") + return [self._map_meta_column(field, normalize) for field in schema_metadata] + except (RuntimeError, PySparkException) as e: + return self.log_and_throw_exception(e, "schema", schema_query) + + def normalize_identifier(self, identifier: str) -> NormalizedIdentifier: + return DialectUtils.normalize_identifier( + identifier, + source_start_delimiter=RedshiftDataSource._IDENTIFIER_DELIMITER, + source_end_delimiter=RedshiftDataSource._IDENTIFIER_DELIMITER, + ) diff --git a/src/databricks/labs/lakebridge/reconcile/connectors/source_adapter.py b/src/databricks/labs/lakebridge/reconcile/connectors/source_adapter.py index 8a64a3a4e8..9e74f95b3c 100644 --- a/src/databricks/labs/lakebridge/reconcile/connectors/source_adapter.py +++ b/src/databricks/labs/lakebridge/reconcile/connectors/source_adapter.py @@ -1,5 +1,6 @@ from pyspark.sql import SparkSession from sqlglot import Dialect +from sqlglot.dialects.redshift import Redshift from databricks.labs.lakebridge.reconcile.connectors.data_source import DataSource from databricks.labs.lakebridge.reconcile.connectors.databricks import ( @@ -7,6 +8,7 @@ DatabricksNonUnityCatalogDataSource, ) from databricks.labs.lakebridge.reconcile.connectors.oracle import OracleDataSource +from databricks.labs.lakebridge.reconcile.connectors.redshift import RedshiftDataSource from databricks.labs.lakebridge.reconcile.connectors.remote_query_reader import RemoteQueryReader from databricks.labs.lakebridge.reconcile.connectors.snowflake import SnowflakeDataSource from databricks.labs.lakebridge.reconcile.connectors.tsql import TSQLServerDataSource @@ -36,4 +38,6 @@ def create_adapter( return DatabricksNonUnityCatalogDataSource(engine, spark, ws) if isinstance(engine, Tsql): return TSQLServerDataSource(engine, reader) + if isinstance(engine, Redshift): + return RedshiftDataSource(engine, reader) raise ValueError(f"Unsupported source type --> {engine}") diff --git a/src/databricks/labs/lakebridge/reconcile/constants.py b/src/databricks/labs/lakebridge/reconcile/constants.py index d9489d8dd9..bf249b627b 100644 --- a/src/databricks/labs/lakebridge/reconcile/constants.py +++ b/src/databricks/labs/lakebridge/reconcile/constants.py @@ -20,6 +20,7 @@ class ReconSourceType(AutoName): ORACLE = auto() SNOWFLAKE = auto() SYNAPSE = auto() + REDSHIFT = auto() class ReconReportType(AutoName): diff --git a/src/databricks/labs/lakebridge/reconcile/query_builder/expression_generator.py b/src/databricks/labs/lakebridge/reconcile/query_builder/expression_generator.py index fb63560938..fb0244513e 100644 --- a/src/databricks/labs/lakebridge/reconcile/query_builder/expression_generator.py +++ b/src/databricks/labs/lakebridge/reconcile/query_builder/expression_generator.py @@ -273,6 +273,32 @@ def _get_is_string(column_types_dict: dict[str, DataType], column_name: str) -> partial(anonymous, func="COALESCE(CONVERT(VARCHAR(23), {0}, 120), '1900-01-01 00:00:00')") ], }, + "redshift": { + exp.DataType.Type.SUPER.value: [ + partial(anonymous, func="COALESCE(JSON_SERIALIZE({}), '_null_recon_')", dialect=get_dialect("redshift")) + ], + exp.DataType.Type.DATE.value: [ + partial( + anonymous, + func="COALESCE(TO_CHAR({}, 'YYYY-MM-DD'), '_null_recon_')", + dialect=get_dialect("redshift"), + ) + ], + exp.DataType.Type.TIMESTAMP.value: [ + partial( + anonymous, + func="COALESCE(TO_CHAR({}, 'YYYY-MM-DD HH24:MI:SS.US'), '_null_recon_')", + dialect=get_dialect("redshift"), + ) + ], + exp.DataType.Type.TIMESTAMPTZ.value: [ + partial( + anonymous, + func="COALESCE(TO_CHAR({}, 'YYYY-MM-DD HH24:MI:SS.US'), '_null_recon_')", + dialect=get_dialect("redshift"), + ) + ], + }, } sha256_partial = partial(sha2, num_bits="256", is_expr=True) @@ -306,4 +332,8 @@ def _get_is_string(column_types_dict: dict[str, DataType], column_name: str) -> ), target=sha256_partial, ), + get_dialect("redshift"): HashAlgoMapping( + source=sha256_partial, + target=sha256_partial, + ), } diff --git a/tests/conftest.py b/tests/conftest.py index 742adaed87..79eece26c9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -282,6 +282,16 @@ def tsql_schema_fixture_factory(column_name: str, data_type: str) -> Schema: ) +def redshift_schema_fixture_factory(column_name: str, data_type: str) -> Schema: + norm = DialectUtils.normalize_identifier(column_name, "\"", "\"") + return schema_fixture_factory( + norm.ansi_normalized, + data_type, + norm.ansi_normalized, + norm.source_normalized, + ) + + def ansi_schema_fixture_factory(column_name: str, data_type: str) -> Schema: ansi = DialectUtils.ansi_normalize_identifier(column_name) return schema_fixture_factory( @@ -341,6 +351,11 @@ def fake_databricks_datasource() -> FakeDataSource: return FakeDataSource("`", "`") +@pytest.fixture +def fake_redshift_datasource() -> FakeDataSource: + return FakeDataSource('"', '"') + + @pytest.fixture def fake_tsql_datasource() -> FakeDataSource: return FakeDataSource("[", "]") @@ -370,6 +385,19 @@ def snowflake_table_conf_with_opts(normalize_config_service: NormalizeReconConfi return conf +@pytest.fixture +def redshift_table_conf_with_opts(normalize_config_service: NormalizeReconConfigService, table_conf_with_opts): + conf = normalize_config_service.normalize_recon_table_config(table_conf_with_opts) + conf.transformations = [ # SQL has to be valid + Transformation(column_name="`s_address`", source="trim(\"s_address\")", target="trim(`s_address_t`)"), + Transformation(column_name="`s_phone`", source="trim(\"s_phone\")", target="trim(`s_phone_t`)"), + Transformation(column_name="`s_name`", source="trim(\"s_name\")", target="trim(`s_name`)"), + ] + if conf.filters: + conf.filters.source = "\"s_name\"='t' and \"s_address\"='a'" + return conf + + @pytest.fixture def tsql_table_conf_with_opts(normalize_config_service: NormalizeReconConfigService, table_conf_with_opts): conf = normalize_config_service.normalize_recon_table_config(table_conf_with_opts) @@ -400,6 +428,14 @@ def table_schema_ansi_ansi(table_schema): return src_schema, tgt_schema +@pytest.fixture +def table_schema_redshift_ansi(table_schema): + src_schema, tgt_schema = table_schema + src_schema = [redshift_schema_fixture_factory(s.column_name, s.data_type) for s in src_schema] + tgt_schema = [ansi_schema_fixture_factory(s.column_name, s.data_type) for s in tgt_schema] + return src_schema, tgt_schema + + @pytest.fixture def table_schema_tsql_ansi(table_schema): src_schema, tgt_schema = table_schema diff --git a/tests/integration/reconcile/conftest.py b/tests/integration/reconcile/conftest.py index 4e9b7c49b3..4a3e39fbe9 100644 --- a/tests/integration/reconcile/conftest.py +++ b/tests/integration/reconcile/conftest.py @@ -55,6 +55,10 @@ SNOWFLAKE_CATALOG = "INTEGRATION" SNOWFLAKE_SCHEMA = "LAKEBRIDGE" SNOWFLAKE_TABLE = "DIAMONDS" +REDSHIFT_CONNECTION = "sandbox_labs_tool_redshift" +REDSHIFT_CATALOG = "labs" +REDSHIFT_SCHEMA = "lakebridge" +REDSHIFT_TABLE = "diamonds" @pytest.fixture @@ -265,6 +269,53 @@ def snowflake_recon_config(recon_cluster: str, recon_schema: SchemaInfo, make_vo ) +@pytest.fixture +def redshift_recon_table_config(recon_schema: SchemaInfo, recon_tables: tuple[TableInfo, TableInfo]) -> TableRecon: + (_, tgt_table) = recon_tables + assert tgt_table.name + + return TableRecon( + [ + Table( + source_name=REDSHIFT_TABLE, + target_name=tgt_table.name, + join_columns=["color", "clarity"], + ) + ] + ) + + +@pytest.fixture +def redshift_recon_config(recon_cluster: str, recon_schema: SchemaInfo, make_volume) -> ReconcileConfig: + volume = make_volume(catalog_name=recon_schema.catalog_name, schema_name=recon_schema.name, name=recon_schema.name) + + deployment_overrides = ReconcileJobConfig( + existing_cluster_id=recon_cluster, + tags={"lakebridge": "reconcile_test"}, + ) + logger.info(f"Using recon job overrides: {deployment_overrides}") + + assert recon_schema.catalog_name + assert recon_schema.name + return ReconcileConfig( + report_type="all", + source=SourceConnectionConfig( + dialect="redshift", + catalog=REDSHIFT_CATALOG, + schema=REDSHIFT_SCHEMA, + uc_connection_name=REDSHIFT_CONNECTION, + ), + target=TargetConnectionConfig( + catalog=recon_schema.catalog_name, + schema=recon_schema.name, + ), + metadata_config=ReconcileMetadataConfig( + catalog=recon_schema.catalog_name, schema=recon_schema.name, volume=volume.name + ), + job_overrides=deployment_overrides, + ) + + def recon_config_filename(recon_config: ReconcileConfig) -> str: connection_or_catalog = recon_config.source.uc_connection_name or recon_config.source.catalog return f"recon_config_{recon_config.source.dialect}_{connection_or_catalog}_{recon_config.report_type}.json" diff --git a/tests/integration/reconcile/connectors/test_read_schema.py b/tests/integration/reconcile/connectors/test_read_schema.py index 96311453df..6617f78676 100644 --- a/tests/integration/reconcile/connectors/test_read_schema.py +++ b/tests/integration/reconcile/connectors/test_read_schema.py @@ -9,6 +9,7 @@ DatabricksDataSource, DatabricksNonUnityCatalogDataSource, ) +from databricks.labs.lakebridge.reconcile.connectors.redshift import RedshiftDataSource from databricks.labs.lakebridge.reconcile.connectors.remote_query_reader import RemoteQueryReader from databricks.labs.lakebridge.reconcile.connectors.snowflake import SnowflakeDataSource from databricks.labs.lakebridge.reconcile.connectors.tsql import TSQLServerDataSource @@ -70,6 +71,15 @@ def test_oracle_read_schema_happy(spark: SparkSession) -> None: assert columns +def test_redshift_read_schema_happy(spark: SparkSession) -> None: + connection = "sandbox_labs_tool_redshift" + reader = RemoteQueryReader(spark, connection) + connector = RedshiftDataSource(get_dialect("redshift"), reader) + + columns = connector.get_schema("labs", "lakebridge", "diamonds") + assert columns + + @pytest.mark.xfail(reason="Snowflake account unavailable", strict=True) def test_snowflake_read_schema_happy(spark: SparkSession) -> None: connection = TestEnvGetter(False).get("TEST_SNOWFLAKE_CONNECTION") diff --git a/tests/integration/reconcile/test_recon_e2e.py b/tests/integration/reconcile/test_recon_e2e.py index 5ee1329233..70f02f0427 100644 --- a/tests/integration/reconcile/test_recon_e2e.py +++ b/tests/integration/reconcile/test_recon_e2e.py @@ -88,3 +88,14 @@ def test_recon_snowflake_job_succeeds( application_ctx, snowflake_recon_config, snowflake_recon_table_config ) as app_ctx: _run_recon_e2e_spec(app_ctx) + + +def test_recon_redshift_job_succeeds( + application_ctx: ApplicationContext, + redshift_recon_config: ReconcileConfig, + redshift_recon_table_config: TableRecon, +) -> None: + with generate_recon_application_context( + application_ctx, redshift_recon_config, redshift_recon_table_config + ) as app_ctx: + _run_recon_e2e_spec(app_ctx) diff --git a/tests/integration/reconcile/test_schema_compare.py b/tests/integration/reconcile/test_schema_compare.py index e957553e5a..9dbca11ec6 100644 --- a/tests/integration/reconcile/test_schema_compare.py +++ b/tests/integration/reconcile/test_schema_compare.py @@ -4,7 +4,12 @@ from databricks.labs.lakebridge.reconcile.recon_config import ColumnMapping, Table from databricks.labs.lakebridge.reconcile.schema_compare import SchemaCompare -from tests.conftest import schema_fixture_factory, tsql_schema_fixture_factory, ansi_schema_fixture_factory +from tests.conftest import ( + schema_fixture_factory, + tsql_schema_fixture_factory, + ansi_schema_fixture_factory, + redshift_schema_fixture_factory, +) def snowflake_databricks_schema(): @@ -267,6 +272,46 @@ def tsql_databricks_schema(): return src_schema, tgt_schema +def redshift_databricks_schema(): + src_schema = [ + redshift_schema_fixture_factory("col_smallint", "smallint"), + redshift_schema_fixture_factory("col_integer", "integer"), + redshift_schema_fixture_factory("col_bigint", "bigint"), + redshift_schema_fixture_factory("col_decimal", "decimal(10,2)"), + redshift_schema_fixture_factory("col_numeric", "decimal(38,0)"), + redshift_schema_fixture_factory("col_real", "real"), + redshift_schema_fixture_factory("col_double", "double precision"), + redshift_schema_fixture_factory("col_varchar", "varchar(100)"), + redshift_schema_fixture_factory("col_varchar_max", "varchar(65535)"), + redshift_schema_fixture_factory("col_binary", "binary"), + redshift_schema_fixture_factory("col_date", "date"), + redshift_schema_fixture_factory("col_timestamp", "timestamp without time zone"), + redshift_schema_fixture_factory("col_timestamptz", "timestamp with time zone"), + redshift_schema_fixture_factory("col_boolean", "boolean"), + schema_fixture_factory("`col Escaped`", "varchar(50)", source_delimiter='"'), + schema_fixture_factory('"col escaped2"', "integer", source_delimiter='"'), + ] + tgt_schema = [ + ansi_schema_fixture_factory("col_smallint", "smallint"), + ansi_schema_fixture_factory("col_integer", "int"), + ansi_schema_fixture_factory("col_bigint", "bigint"), + ansi_schema_fixture_factory("col_decimal", "decimal(10,2)"), + ansi_schema_fixture_factory("col_numeric", "decimal(38,0)"), + ansi_schema_fixture_factory("col_real", "float"), + ansi_schema_fixture_factory("col_double", "double"), + ansi_schema_fixture_factory("col_varchar", "string"), + ansi_schema_fixture_factory("col_varchar_max", "string"), + ansi_schema_fixture_factory("col_binary", "binary"), + ansi_schema_fixture_factory("col_date", "date"), + ansi_schema_fixture_factory("col_timestamp", "timestamp"), + ansi_schema_fixture_factory("col_timestamptz", "timestamp"), + ansi_schema_fixture_factory("col_boolean", "boolean"), + ansi_schema_fixture_factory("`col Escaped`", "string"), + ansi_schema_fixture_factory("`col escaped2`", "int"), + ] + return src_schema, tgt_schema + + @pytest.fixture def schemas(): return { @@ -274,6 +319,7 @@ def schemas(): "databricks_databricks_schema": databricks_databricks_schema(), "oracle_databricks_schema": oracle_databricks_schema(), "tsql_databricks_schema": tsql_databricks_schema(), + "redshift_databricks_schema": redshift_databricks_schema(), } @@ -422,3 +468,23 @@ def test_schema_compare(spark): assert df.count() == 2 assert df.filter("is_valid = 'true'").count() == 2 assert df.filter("is_valid = 'false'").count() == 0 + + +def test_redshift_schema_compare(schemas, spark): + src_schema, tgt_schema = schemas["redshift_databricks_schema"] + table_conf = Table( + source_name="supplier", + target_name="supplier", + ) + + schema_compare_output = SchemaCompare(spark).compare( + src_schema, + tgt_schema, + get_dialect("redshift"), + table_conf, + ) + df = schema_compare_output.compare_df + assert schema_compare_output.is_valid + assert df.count() == 16 + assert df.filter("is_valid = 'true'").count() == 16 + assert df.filter("is_valid = 'false'").count() == 0 diff --git a/tests/unit/helpers/test_recon_config_utils.py b/tests/unit/helpers/test_recon_config_utils.py index 84558295b3..e0097b8b7b 100644 --- a/tests/unit/helpers/test_recon_config_utils.py +++ b/tests/unit/helpers/test_recon_config_utils.py @@ -7,7 +7,7 @@ from databricks.sdk.errors.platform import ResourceDoesNotExist from databricks.sdk.service.workspace import SecretScope -SOURCE_DICT = {"databricks": "0", "mssql": "1", "oracle": "2", "snowflake": "3", "synapse": "4"} +SOURCE_DICT = {"databricks": "0", "mssql": "1", "oracle": "2", "redshift": "3", "snowflake": "4", "synapse": "5"} SCOPE_NAME = "dummy_scope" diff --git a/tests/unit/reconcile/connectors/test_redshift.py b/tests/unit/reconcile/connectors/test_redshift.py new file mode 100644 index 0000000000..3c209b5e5c --- /dev/null +++ b/tests/unit/reconcile/connectors/test_redshift.py @@ -0,0 +1,127 @@ +import re +from unittest.mock import create_autospec + +import pytest + +from databricks.labs.lakebridge.reconcile.connectors.models import NormalizedIdentifier +from databricks.labs.lakebridge.transpiler.sqlglot.dialect_utils import get_dialect +from databricks.labs.lakebridge.reconcile.connectors.redshift import RedshiftDataSource +from databricks.labs.lakebridge.reconcile.connectors.remote_query_reader import RemoteQueryReader +from databricks.labs.lakebridge.reconcile.exception import DataSourceRuntimeException +from databricks.labs.lakebridge.reconcile.recon_config import JdbcReaderOptions + + +def initial_setup(): + engine = get_dialect("redshift") + reader = create_autospec(RemoteQueryReader) + return engine, reader + + +def test_read_data_with_options(): + engine, reader = initial_setup() + + rds = RedshiftDataSource(engine, reader) + options = JdbcReaderOptions(num_partitions=50, partition_column="s_nationkey", lower_bound="0", upper_bound="100") + + rds.read_data("dev", "data", "employee", "select 1 from :tbl", options) + + reader.read_data.assert_called_once_with( + "select 1 from data.employee", + "dev", + "database", + "query", + options, + ) + + +def test_read_data_replaces_postgres_param_syntax(): + """SQLGlot's Redshift dialect emits :tbl as %(tbl)s; the connector must handle both.""" + engine, reader = initial_setup() + rds = RedshiftDataSource(engine, reader) + + rds.read_data("dev", "data", "employee", "select 1 from %(tbl)s", None) + + reader.read_data.assert_called_once_with( + "select 1 from data.employee", + "dev", + "database", + "query", + None, + ) + + +def test_get_schema(): + engine, reader = initial_setup() + rds = RedshiftDataSource(engine, reader) + + rds.get_schema("dev", "data", "employee") + + reader.read_data.assert_called_once() + call_args = reader.read_data.call_args + source_query = call_args[0][0] + expected_query = re.sub( + r'\s+', + ' ', + """SELECT + column_name, + CASE + WHEN data_type = 'numeric' AND numeric_precision IS NOT NULL + THEN 'decimal(' || numeric_precision || ',' || numeric_scale || ')' + WHEN data_type = 'character varying' AND character_maximum_length IS NOT NULL + THEN 'varchar(' || character_maximum_length || ')' + WHEN data_type = 'character' AND character_maximum_length IS NOT NULL + THEN 'char(' || character_maximum_length || ')' + WHEN data_type IN ('binary varying') + THEN 'binary' + ELSE data_type + END AS data_type + FROM + information_schema.columns + WHERE + LOWER(table_name) = LOWER('employee') + AND LOWER(table_schema) = LOWER('data') + ORDER BY ordinal_position + """, + ) + assert source_query == expected_query + assert call_args[0][1] == "dev" + assert call_args[0][2] == "database" + assert call_args[0][3] == "query" + + +def test_read_data_exception_handling(): + engine, reader = initial_setup() + rds = RedshiftDataSource(engine, reader) + + reader.read_data.side_effect = RuntimeError("Test Exception") + + with pytest.raises( + DataSourceRuntimeException, + match="Runtime exception occurred while fetching data using select 1 from data.employee : Test Exception", + ): + rds.read_data("dev", "data", "employee", "select 1 from :tbl", None) + + +def test_get_schema_exception_handling(): + engine, reader = initial_setup() + rds = RedshiftDataSource(engine, reader) + + reader.read_data.side_effect = RuntimeError("Test Exception") + + with pytest.raises( + DataSourceRuntimeException, + match="Runtime exception occurred while fetching schema", + ): + rds.get_schema("dev", "data", "employee") + + +def test_normalize_identifier(): + engine, reader = initial_setup() + data_source = RedshiftDataSource(engine, reader) + + assert data_source.normalize_identifier("a") == NormalizedIdentifier("`a`", '"a"') + assert data_source.normalize_identifier('"b"') == NormalizedIdentifier("`b`", '"b"') + assert data_source.normalize_identifier('"`e`f`"') == NormalizedIdentifier("```e``f```", '"`e`f`"') + assert data_source.normalize_identifier('" g h "') == NormalizedIdentifier("` g h `", '" g h "') + assert data_source.normalize_identifier('"""j""k"""') == NormalizedIdentifier('`"j"k"`', '"""j""k"""') + assert data_source.normalize_identifier('"j""k"') == NormalizedIdentifier('`j"k`', '"j""k"') diff --git a/tests/unit/reconcile/query_builder/test_hash_query.py b/tests/unit/reconcile/query_builder/test_hash_query.py index 1b0f98e708..18bb972107 100644 --- a/tests/unit/reconcile/query_builder/test_hash_query.py +++ b/tests/unit/reconcile/query_builder/test_hash_query.py @@ -119,6 +119,45 @@ def test_hash_query_builder_for_databricks_src( assert tgt_actual == tgt_expected +def test_hash_query_builder_for_redshift_src( + redshift_table_conf_with_opts, + table_schema_redshift_ansi, + fake_redshift_datasource, + fake_databricks_datasource, +): + src_schema, tgt_schema = table_schema_redshift_ansi + src_actual = HashQueryBuilder( + redshift_table_conf_with_opts, + src_schema, + "source", + get_dialect("redshift"), + fake_redshift_datasource, + ).build_query(report_type="data") + src_expected = ( + "SELECT LOWER(SHA2(TRIM(\"s_address\") || TRIM(\"s_name\") || COALESCE(TRIM(\"s_nationkey\"), '_null_recon_') || " + "TRIM(\"s_phone\") || COALESCE(TRIM(\"s_suppkey\"), '_null_recon_'), 256)) AS hash_value_recon, \"s_nationkey\" AS " + "\"s_nationkey\", " + "\"s_suppkey\" AS \"s_suppkey\" FROM %(tbl)s WHERE \"s_name\" = 't' AND \"s_address\" = 'a'" + ) + + tgt_actual = HashQueryBuilder( + redshift_table_conf_with_opts, + tgt_schema, + "target", + get_dialect("databricks"), + fake_databricks_datasource, + ).build_query(report_type="data") + tgt_expected = ( + "SELECT LOWER(SHA2(TRIM(`s_address_t`) || TRIM(`s_name`) || COALESCE(TRIM(`s_nationkey_t`), '_null_recon_') || " + "TRIM(`s_phone_t`) || COALESCE(TRIM(`s_suppkey_t`), '_null_recon_'), 256)) AS hash_value_recon, `s_nationkey_t` AS " + "`s_nationkey`, " + "`s_suppkey_t` AS `s_suppkey` FROM :tbl WHERE s_name = 't' AND s_address_t = 'a'" + ) + + assert src_actual == src_expected + assert tgt_actual == tgt_expected + + def test_hash_query_builder_for_tsql_src( tsql_table_conf_with_opts, table_schema_tsql_ansi, diff --git a/tests/unit/reconcile/test_source_adapter.py b/tests/unit/reconcile/test_source_adapter.py index 360bc2fc09..0fe268ec79 100644 --- a/tests/unit/reconcile/test_source_adapter.py +++ b/tests/unit/reconcile/test_source_adapter.py @@ -9,6 +9,7 @@ DatabricksNonUnityCatalogDataSource, ) from databricks.labs.lakebridge.reconcile.connectors.oracle import OracleDataSource +from databricks.labs.lakebridge.reconcile.connectors.redshift import RedshiftDataSource from databricks.labs.lakebridge.reconcile.connectors.snowflake import SnowflakeDataSource from databricks.labs.lakebridge.reconcile.connectors.source_adapter import create_adapter from databricks.sdk import WorkspaceClient @@ -58,6 +59,17 @@ def test_create_adapter_for_databricks_dialect_target(): assert not isinstance(data_source, DatabricksNonUnityCatalogDataSource) +def test_create_adapter_for_redshift_dialect(): + spark = create_autospec(DatabricksSession) + engine = get_dialect("redshift") + ws = create_autospec(WorkspaceClient) + scope = "scope" + + data_source = create_adapter(engine, spark, ws, scope) + + assert isinstance(data_source, RedshiftDataSource) + + def test_raise_exception_for_unknown_dialect(): spark = create_autospec(DatabricksSession) engine = get_dialect("trino") From 535c3900238ba6625177830a4c5b54f069e81b41 Mon Sep 17 00:00:00 2001 From: Guenia Izquierdo Date: Tue, 12 May 2026 06:28:41 -0400 Subject: [PATCH 09/79] Revamp documentation (#2365) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Summary This PR revamps the Lakebridge documentation with a focus on clarity, structure, and first-time user experience. - **New pages:** Added a Getting Started tutorial (end-to-end SQL Server → Databricks SQL walkthrough), a tool selection decision guide (Choosing Tools), a dedicated Morpheus transpiler page, and a Switch architecture page split out from the Switch index - **Restructured Reconcile docs**: Consolidated from 5 files (~1,400 lines) to 3 (~750 lines). The index was rewritten as a concise overview with a report-type comparison table. The former reconcile_configuration.mdx and example_config.mdx were merged into a single Configuration Reference page; recon_notebook.mdx and reconcile_automation.mdx were merged into a single Running Reconcile page covering both CLI and notebook execution paths - **Restructured SSIS docs**: Moved the three SSIS pages (ssis.mdx, ssis_supported_components.mdx, ssis_examples.mdx) into a dedicated ssis/ subfolder, creating a collapsible SSIS category in the sidebar consistent with how other multi-page sections are structured; updated all cross-links between the pages - **Other restructured pages**: Rewrote Installation for brevity; expanded FAQ from 3 to 25+ questions; merged the Transpile overview into the Transpile index; split Switch docs into a quickstart index and a separate architecture page - **Deduplication**: Removed install-transpile and configure-reconcile commands from Getting Started (they now live exclusively in Installation); Getting Started Step 3 is reframed as configuration guidance with a back-link to Installation, and Step 5 runs reconcile directly - **Navigation overhaul**: Reordered the sidebar to match the actual user journey: Installation → Getting Started → Choosing Tools → Assessment → Transpile → Reconcile → SQL Splitter → FAQ — with Installation moved before Getting Started since it is a prerequisite. Resolved a position conflict between Installation and Choosing Tools which were both set to sidebar_position: 2 --------- Co-authored-by: Andrew Snare Co-authored-by: M Abulazm --- .../analyzer/complexity_scoring.mdx | 73 +- docs/lakebridge/docs/assessment/index.mdx | 11 +- .../assessment/profiler/dashboards/index.mdx | 4 - docs/lakebridge/docs/choosing_tools.mdx | 75 ++ docs/lakebridge/docs/dev/contributing.md | 4 + docs/lakebridge/docs/faq.mdx | 195 ++++- docs/lakebridge/docs/getting_started.mdx | 122 +++ docs/lakebridge/docs/installation.mdx | 321 ++------ .../docs/reconcile/configuration.mdx | 447 +++++++++++ .../docs/reconcile/example_config.mdx | 181 ----- docs/lakebridge/docs/reconcile/index.mdx | 94 ++- .../reconcile/reconcile_configuration.mdx | 754 ------------------ .../{recon_notebook.mdx => running.mdx} | 159 ++-- docs/lakebridge/docs/sql_splitter.mdx | 2 +- docs/lakebridge/docs/transpile/index.mdx | 63 +- docs/lakebridge/docs/transpile/overview.mdx | 170 ---- .../bladebridge/bladebridge_configuration.mdx | 58 +- .../pluggable_transpilers/morpheus/index.mdx | 135 ++++ .../switch/architecture.mdx | 156 ++++ .../pluggable_transpilers/switch/index.mdx | 164 +--- .../switch/switch_faqs.mdx | 77 -- .../docs/transpile/source_systems/index.mdx | 6 - .../docs/transpile/source_systems/ssis.mdx | 608 -------------- .../source_systems/ssis/examples.mdx | 341 ++++++++ .../transpile/source_systems/ssis/index.mdx | 87 ++ .../ssis/supported_components.mdx | 111 +++ docs/lakebridge/docusaurus.config.ts | 2 +- docs/lakebridge/src/pages/index.tsx | 2 +- 28 files changed, 2020 insertions(+), 2402 deletions(-) create mode 100644 docs/lakebridge/docs/choosing_tools.mdx create mode 100644 docs/lakebridge/docs/getting_started.mdx create mode 100644 docs/lakebridge/docs/reconcile/configuration.mdx delete mode 100644 docs/lakebridge/docs/reconcile/example_config.mdx delete mode 100644 docs/lakebridge/docs/reconcile/reconcile_configuration.mdx rename docs/lakebridge/docs/reconcile/{recon_notebook.mdx => running.mdx} (53%) delete mode 100644 docs/lakebridge/docs/transpile/overview.mdx create mode 100644 docs/lakebridge/docs/transpile/pluggable_transpilers/morpheus/index.mdx create mode 100644 docs/lakebridge/docs/transpile/pluggable_transpilers/switch/architecture.mdx delete mode 100644 docs/lakebridge/docs/transpile/pluggable_transpilers/switch/switch_faqs.mdx delete mode 100644 docs/lakebridge/docs/transpile/source_systems/ssis.mdx create mode 100644 docs/lakebridge/docs/transpile/source_systems/ssis/examples.mdx create mode 100644 docs/lakebridge/docs/transpile/source_systems/ssis/index.mdx create mode 100644 docs/lakebridge/docs/transpile/source_systems/ssis/supported_components.mdx diff --git a/docs/lakebridge/docs/assessment/analyzer/complexity_scoring.mdx b/docs/lakebridge/docs/assessment/analyzer/complexity_scoring.mdx index 16ec137513..a2a3ed1ce9 100644 --- a/docs/lakebridge/docs/assessment/analyzer/complexity_scoring.mdx +++ b/docs/lakebridge/docs/assessment/analyzer/complexity_scoring.mdx @@ -4,6 +4,31 @@ title: Complexity Scoring --- import useBaseUrl from '@docusaurus/useBaseUrl'; +## Overview + +The Lakebridge Analyzer assigns a complexity level to each SQL object or ETL job it processes. Complexity scores help you prioritize migration effort, estimate timelines, and identify which objects will require manual review after transpilation. + +Each object receives one of four levels: **LOW**, **MEDIUM**, **HIGH**, or **VERY HIGH**. + +## How to Interpret Your Scores + +| Level | What it means | Typical action | +|---|---|---| +| **LOW** | Simple object — few statements, no loops, no complex constructs | Transpile directly; little or no manual review expected | +| **MEDIUM** | Moderate complexity — some loops, 10–30 statements, limited use of advanced SQL | Transpile; review transpiler warnings before deploying | +| **HIGH** | High complexity — many loops, 30–50 statements, or advanced constructs (XML, PIVOT) | Plan for manual review after transpilation; test thoroughly | +| **VERY HIGH** | Highest complexity — heavy loop nesting, 50+ statements, or extreme PIVOT/XML usage | Allocate dedicated review time; consider using Switch for intent-aware conversion | + +**Planning guidance:** + +- If more than 50% of your objects score HIGH or VERY HIGH, plan for at least two manual review passes before deploying. +- Objects scored LOW and MEDIUM typically transpile cleanly with Morpheus or BladeBridge. +- HIGH and VERY HIGH objects are strong candidates for Switch, which handles business logic that deterministic transpilers struggle with. + +The scoring rules for each source system are detailed below. + +--- + ## SQL Code Analysis At the beginning of script analysis, mark a script with complexity level of **LOW** @@ -14,14 +39,14 @@ If any of the following conditions are true, then mark the job as **MEDIUM** com 4. Number of pivot statements between 1 and 3 5. Number of XML SQL statements between 1 and 3 -If any of the following conditions are true, then mark the job as **COMPLEX** complexity: +If any of the following conditions are true, then mark the job as **HIGH** complexity: 1. Number of loops greater than 5 2. Conventional Statement count greater than 30 3. Simple Statement count greater than 2000 4. Number of pivot statements greater than 3 5. Number of XML SQL statements greater than 3 -If any of the following conditions are true, then mark the job as **VERY COMPLEX** complexity: +If any of the following conditions are true, then mark the job as **VERY HIGH** complexity: 1. Number of loops greater than 8 2. Conventional Statement count greater than 50 3. Simple Statement count greater than 5000 @@ -47,7 +72,7 @@ If any of the following conditions are true, then mark the job as **MEDIUM** com 5. Number of targets > 1 6. Overall function call count >= 10 -If any of the following conditions are true, then mark the job as **COMPLEX** complexity: +If any of the following conditions are true, then mark the job as **HIGH** complexity: 1. Three MEDIUM breaks from the list above 2. Number of expressions with 5+ function calls between 5 and 7 3. Number of job components >= 20 @@ -55,8 +80,8 @@ If any of the following conditions are true, then mark the job as **COMPLEX** co 5. Complex or Unstructured nodes are being used (ChangeCapture, etc..) 6. Number of lookups between 7 and 14 -If any of the following conditions are true, then mark the job as **VERY COMPLEX** complexity: -1. Three COMPLEX breaks from the list above +If any of the following conditions are true, then mark the job as **VERY HIGH** complexity: +1. Three HIGH complexity breaks from the list above 2. Number of expressions with 5+ function calls > 7 3. Number of lookups > 15 4. Number of job components >= 50 @@ -73,15 +98,15 @@ If any of the following conditions are true, then mark the job as **MEDIUM** com 6. Overall function call count >= 10 -If any of the following conditions are true, then mark the job as **COMPLEX** complexity: +If any of the following conditions are true, then mark the job as **HIGH** complexity: 1. Three MEDIUM breaks from the list above 2. Number of expressions with 5+ function calls between 5 and 7 3. Number of job components >= 20 4. Overall function call count >= 20 5. Complex or Unstructured nodes are being used (ChangeCapture, etc..) -If any of the following conditions are true, then mark the job as **VERY COMPLEX** complexity: -1. Three COMPLEX breaks from the list above +If any of the following conditions are true, then mark the job as **VERY HIGH** complexity: +1. Three HIGH complexity breaks from the list above 2. Number of job components >= 50 ## SSIS Code Analysis @@ -94,14 +119,14 @@ If any of the following conditions are true, then mark the mapping as **MEDIUM** 4. Overall function call count >= 10 5. Number of package components >= 10 -If any of the following conditions are true, then mark the mapping as **COMPLEX** complexity: +If any of the following conditions are true, then mark the mapping as **HIGH** complexity: 1. Three MEDIUM breaks from the list above 2. Number of expressions with 5+ function calls between 5 and 7 3. Number of package components >= 20 4. Overall function call count >= 20 -If any of the following conditions are true, then mark the mapping as **VERY COMPLEX** complexity: -1. Three COMPLEX breaks from the list above +If any of the following conditions are true, then mark the mapping as **VERY HIGH** complexity: +1. Three HIGH complexity breaks from the list above 2. Number of expressions with 5+ function calls > 7 4. Number of job components >= 50 @@ -113,14 +138,14 @@ If any of the following conditions are true, then mark the mapping as **MEDIUM** 2. Overall function call count >= 10 3. Number of job components >= 10 -If any of the following conditions are true, then mark the mapping as **COMPLEX** complexity: +If any of the following conditions are true, then mark the mapping as **HIGH** complexity: 1. Three MEDIUM breaks from the list above 2. Number of expressions with 5+ function calls between 5 and 7 3. Number of package components >= 20 4. Overall function call count >= 20 -If any of the following conditions are true, then mark the mapping as **VERY COMPLEX** complexity: -1. Three COMPLEX breaks from the list above +If any of the following conditions are true, then mark the mapping as **VERY HIGH** complexity: +1. Three HIGH complexity breaks from the list above 2. Number of expressions with 5+ function calls > 7 4. Number of job components >= 50 @@ -132,14 +157,14 @@ If any of the following conditions are true, then mark the mapping as **MEDIUM** 2. Overall function call count >= 10 3. Number of job components >= 10 -If any of the following conditions are true, then mark the mapping as **COMPLEX** complexity: +If any of the following conditions are true, then mark the mapping as **HIGH** complexity: 1. Three MEDIUM breaks from the list above 2. Number of expressions with 5+ function calls between 5 and 7 3. Number of package components >= 20 4. Overall function call count >= 20 -If any of the following conditions are true, then mark the mapping as **VERY COMPLEX** complexity: -1. Three COMPLEX breaks from the list above +If any of the following conditions are true, then mark the mapping as **VERY HIGH** complexity: +1. Three HIGH complexity breaks from the list above 2. Number of expressions with 5+ function calls > 7 4. Number of job components >= 50 @@ -155,22 +180,22 @@ If any of the following conditions are true, then mark the script as **MEDIUM** 6. Count of SQL Procs categorized as MEDIUM > 0 7. SQL Proc count > 10 -If any of the following conditions are true, then mark the script as **COMPLEX**: +If any of the following conditions are true, then mark the script as **HIGH**: 1. Macro definition count > 7 2. Data block count > 15 3. number of statements inside macros and data blocks > 100 4. Conditional statement count > 20 5. 'DO' loop count > 10 -6. Count of SQL Procs categorized as COMPLEX > 0 +6. Count of SQL Procs categorized as HIGH complexity > 0 7. SQL Proc count > 20 -If any of the following conditions are true, then mark the script as **VERY COMPLEX**: +If any of the following conditions are true, then mark the script as **VERY HIGH**: 1. Macro definition count > 15 2. Data block count > 25 3. number of statements inside macros and data blocks > 150 4. Conditional statement count > 50 5. 'DO' loop count > 20 -6. Count of SQL Procs categorized as VERY COMPLEX > 0 +6. Count of SQL Procs categorized as VERY HIGH > 0 7. SQL Proc count > 40 ## Pentaho Code Analysis @@ -187,7 +212,7 @@ If any of the following conditions are true, then mark the mapping as **MEDIUM** 6. Overall function call count >= 10 7. Number of components (transformations) >= 10 -If any of the following conditions are true, then mark the mapping as **COMPLEX** complexity: +If any of the following conditions are true, then mark the mapping as **HIGH** complexity: 1. Three MEDIUM breaks from the list above 2. Number of expressions with 5+ function calls between 5 and 7 @@ -196,9 +221,9 @@ If any of the following conditions are true, then mark the mapping as **COMPLEX* 5. Complex or Unstructured nodes are being used (e.g. Normalizer) 6. Number of lookups between 7 and 14 -If any of the following conditions are true, then mark the mapping as **VERY COMPLEX** complexity: +If any of the following conditions are true, then mark the mapping as **VERY HIGH** complexity: -1. Three COMPLEX breaks from the list above +1. Three HIGH complexity breaks from the list above 2. Number of expressions with 5+ function calls > 7 3. Number of lookups > 15 4. Number of job components >= 50 diff --git a/docs/lakebridge/docs/assessment/index.mdx b/docs/lakebridge/docs/assessment/index.mdx index 15b934edd3..8ac6b7cde4 100644 --- a/docs/lakebridge/docs/assessment/index.mdx +++ b/docs/lakebridge/docs/assessment/index.mdx @@ -1,13 +1,14 @@ --- -sidebar_position: 3 +sidebar_position: 4 title: Assessment Guide --- ## Table of Contents -| Guide | Description | -|-------------------------------|-------------------------------------------------------------------| -| [Profiler Guide](./profiler/) | Instructions for running the Profiler component. | -| [Analyzer Guide](./analyzer/) | Learn how to use the Analyzer for metadata scanning and insights. | +| Guide | Description | +|----------------------------------------|----------------------------------------------------------------------------------------| +| [SQL Splitter](/docs/sql_splitter) | (Optional) Split monolithic SQL files into individual objects before assessing or transpiling. | +| [Profiler Guide](./profiler/) | Instructions for running the Profiler component. | +| [Analyzer Guide](./analyzer/) | Learn how to use the Analyzer for metadata scanning and insights. | diff --git a/docs/lakebridge/docs/assessment/profiler/dashboards/index.mdx b/docs/lakebridge/docs/assessment/profiler/dashboards/index.mdx index 69e3eae83c..1004ca94a1 100644 --- a/docs/lakebridge/docs/assessment/profiler/dashboards/index.mdx +++ b/docs/lakebridge/docs/assessment/profiler/dashboards/index.mdx @@ -33,10 +33,6 @@ that loads the extract data into Delta tables. The result is a live dashboard yo | Source Platform | Configuration Status | |:---------------:|:-------------------:| | Azure Synapse | ✅ | -| Amazon Redshift | ❌ | -| Oracle | 🔸 Coming Soon | -| Microsoft SQL Server | 🔸 Coming Soon | -| Snowflake | 🔸 Coming Soon | ## Running the Command diff --git a/docs/lakebridge/docs/choosing_tools.mdx b/docs/lakebridge/docs/choosing_tools.mdx new file mode 100644 index 0000000000..233a52b409 --- /dev/null +++ b/docs/lakebridge/docs/choosing_tools.mdx @@ -0,0 +1,75 @@ +--- +sidebar_position: 3 +title: Which Tool Do I Use? +--- + +# Which Tool Should I Use? + +Lakebridge provides multiple tools for assessment and transpilation. Use this guide to choose the right one for your migration. + +--- + +## Assessment: Profiler vs Analyzer + +| | Profiler | Analyzer | +|---|---|---| +| **Purpose** | Understand data volumes, query complexity, and workload patterns | Analyze SQL code complexity and compatibility | +| **Input** | Live database connections | SQL code files (local filesystem) | +| **Output** | Interactive dashboard with usage metrics | Per-object complexity report (LOW / MEDIUM / HIGH / VERY HIGH) | +| **When to use** | Early-stage scoping, executive reporting, sizing estimates | Pre-migration planning, review current state of source code and its compatibility | +| **Requires DB connection** | Yes | No | + +**Quick decision:** +- If you need to **scope the migration and communicate to stakeholders**, use the **Profiler**. +- If you have SQL files and want to know **how hard each one will be to transpile**, use the **Analyzer**. +- **Run both** when you have access to the database — Profiler for sizing, Analyzer for developer planning. + +--- + +## Transpiler: BladeBridge vs Morpheus vs Switch {#transpiler-bladebridge-vs-morpheus-vs-switch} + +| | Morpheus | BladeBridge | Switch (Experimental) | +|---|---|---|---| +| **Approach** | AST-based (ANTLR grammar → IR tree → code gen) | Config/pattern-based converter | LLM-powered | +| **Best for** | SQL migration where correctness guarantees and reproducibility matter | ETL platforms + broader SQL dialect coverage | Complex logic, unsupported constructs, or any format via custom prompts | +| **Source dialects** | MSSql (SQL Server, Azure SQL, RDS for SQL Server), Snowflake (including dbt repointing), Synapse (Azure Synapse Analytics dedicated SQL pools) | DataStage, SSIS, Oracle, Teradata, Netezza, Synapse, Redshift, and more | Any SQL dialect or programming language. Either via built-in (MSSQL, MySQL, Netezza, Oracle, PostgreSQL, Redshift, Snowflake, Teradata, Airflow, Python, Scala) or custom YAML prompts | +| **Output formats** | Databricks SQL only | Databricks SQL, SparkSQL, PySpark, notebooks, workflow definitions | Python/SQL notebooks; any text-based format | +| **Accuracy** | Strong guarantee: errors or warns rather than silently producing wrong output | Deterministic and repeatable; extensible via config | Semantic/intent-aware; handles edge cases deterministic transpilers miss | +| **Speed** | Fast — local execution, no API calls | Fast — local execution, no API calls | Slower — requires LLM API calls per file | +| **Cost** | Free | Free | Token costs apply (Databricks Foundation Model API) | +| **Scalability** | Thousands of files locally | Thousands of files locally | Rate-limited by model serving; scales via Lakeflow Jobs | +| **Configuration** | Simple: `source-dialect` flag + I/O paths | Simple: `source-dialect` flag + I/O paths + Optional JSON Config override files | YAML prompt files; configurable model endpoint | +| **Extensibility** | Not extensible (fixed grammar) | Fully extensible via JSON config | Fully extensible via custom YAML prompts | + +### When to choose Morpheus + +Use Morpheus when: +- You are migrating SQL code from **SQL Server, Snowflake, or Azure Synapse Analytics** +- You need **correctness guarantees** — you want to know immediately when a construct cannot be translated, rather than getting silently wrong output +- You do not need ETL platform support + +### When to choose BladeBridge + +Use BladeBridge when: +- You are migrating **ETL workloads** (DataStage, SSIS, others) +- You need **broad SQL dialect coverage** (Oracle, Teradata, Netezza, Redshift) with customizable configurations + +### When to choose Switch + +Use Switch when: +- You have **stored procedures with complex business logic** that rely heavily on context and intent +- Your source dialect is **not covered by Morpheus or BladeBridge** (Switch supports any format via custom YAML prompts) +- You want **Python notebook output** for logic that is difficult to express in pure SQL +- You want to convert non-SQL sources (Python scripts, Airflow DAGs, etc.) + +:::note +Switch is currently **Experimental**. Generated notebooks may require manual adjustments. +::: + +--- + +## Still not sure? + +Start with **Morpheus** if your source is SQL Server, Snowflake, or Synapse. Its correctness guarantee makes it the lowest-risk choice when it supports your dialect. + +For everything else, check the [supported dialects table](/docs/transpile/#supported-dialects) and pick the transpiler that covers your source. diff --git a/docs/lakebridge/docs/dev/contributing.md b/docs/lakebridge/docs/dev/contributing.md index 08d07a7403..645ef28b36 100644 --- a/docs/lakebridge/docs/dev/contributing.md +++ b/docs/lakebridge/docs/dev/contributing.md @@ -175,6 +175,10 @@ pull request checks do pass, before your code is reviewed by others: make lint test ``` +## Developer Documentation + +The developer documentation covers local setup, code organization, JVM proxy configuration, and troubleshooting the development environment. + ## First contribution Here are the example steps to submit your first contribution: diff --git a/docs/lakebridge/docs/faq.mdx b/docs/lakebridge/docs/faq.mdx index 022a638c36..22e230892c 100644 --- a/docs/lakebridge/docs/faq.mdx +++ b/docs/lakebridge/docs/faq.mdx @@ -1,16 +1,199 @@ --- sidebar_position: 8 --- +import CodeBlock from '@theme/CodeBlock'; + # FAQs -## Installation -### 1. Install Databricks CLI on Linux without brew +## General + +### What is Lakebridge? + +Lakebridge is a Databricks toolkit for migrating data workloads to Databricks. It covers three phases: +1. **Assessment** — analyze your existing SQL or ETL environment and code to understand complexity and effort +2. **Transpilation** — convert source SQL or ETL code to Databricks-compatible SQL, PySpark, or notebooks +3. **Reconciliation** — validate that the migrated data matches the source + +### Do I need to run all three phases? + +No. Use the phases you need: +- **Assess only** if you are scoping a migration before committing to it +- **Transpile only** if you are migrating code and do not need pre-migration analysis or post-migration validation +- **Reconcile only** if you have already migrated data and want to validate row/column fidelity + +### Which source systems are supported? + +See the [supported dialects table](/docs/transpile/#supported-dialects) for a full list. In summary: +- **SQL sources:** SQL Server, Snowflake, Azure Synapse Analytics, Oracle, Teradata, Netezza, Redshift, MySQL, PostgreSQL +- **ETL sources:** DataStage, SSIS, others +- **Other:** Airflow, Python, Scala (via Switch) + +--- + +## Assessment + +### Profiler vs Analyzer — which should I use? + +See [Which Tool Do I Use? — Profiler vs Analyzer](/docs/choosing_tools#assessment-profiler-vs-analyzer). + +Short answer: use the Profiler for executive-level scoping; use the Analyzer for per-file migration planning. + +### How do I interpret complexity scores? + +The Analyzer classifies each SQL object as LOW, MEDIUM, HIGH, or VERY HIGH. As a rule of thumb: +- **LOW / MEDIUM** objects typically transpile cleanly with no manual review required +- **HIGH** objects should be reviewed after transpilation for any warnings +- **VERY HIGH** objects (>50% of total) suggest you should plan for 2+ manual review passes before deploying + +See [Complexity Scoring](/docs/assessment/analyzer/complexity_scoring) for the exact thresholds by source system. + +### Should I run the SQL Splitter before the Analyzer? + +Yes. The Analyzer works at the individual-object level. If your SQL files mix multiple objects (stored procedures, tables, views), split them first with the [SQL Splitter](/docs/sql_splitter) so the Analyzer produces per-object granularity. + +--- + +## Transpilation + +### Morpheus {#morpheus} + +#### Which dialects does Morpheus support? + +Morpheus supports `mssql` (SQL Server, Azure SQL, RDS for SQL Server), `snowflake` (including dbt repointing), and `synapse` (Azure Synapse Analytics dedicated SQL pools). It does **not** support Redshift, Oracle, or ETL platforms. + +For other dialects, use [BladeBridge](/docs/transpile/pluggable_transpilers/bladebridge) or [Switch](/docs/transpile/pluggable_transpilers/switch). + +#### What is the difference between an error and a warning in Morpheus output? + +- **Error** — Morpheus knows that the transpiled output cannot be guaranteed to produce equivalent results. Manual fix required before deploying. +- **Warning** — Morpheus could not confirm equivalence but the output *may* still be correct (a conservative false-negative). Review the flagged section and test against source data. + +A file transpiled with **no errors and no warnings** carries a full correctness guarantee. + +#### My file transpiled with warnings but the output looks correct. Is it safe to use? + +Possibly. Morpheus is conservative — it warns when it cannot *guarantee* correctness, even if the actual output is correct. Test the transpiled file against your source data. If the results match, it is safe to deploy. + +#### What should I do when a file fails to parse? + +Parsing errors are very rare and indicate either malformed input SQL or a gap in the Morpheus ANTLR grammar. Verify that the input file is valid SQL. If it is, file a bug at [GitHub Issues](https://github.com/databrickslabs/lakebridge/issues). + +### BladeBridge + +#### Which dialects does BladeBridge support? + +BladeBridge supports SQL dialects (Oracle, Teradata, Netezza, SQL Server, Synapse, Redshift) and ETL platforms (DataStage, SSIS). See the [supported dialects table](/docs/transpile/#supported-dialects). + +#### What is the `target-tech` parameter? + +`target-tech` controls the output format: `SQL` (Databricks SQL), `SPARKSQL` (SparkSQL notebooks), or `PYSPARK` (PySpark notebooks). It only applies to ETL sources — SQL dialects always output Databricks SQL. + +Available options per ETL dialect: +- `datastage`: `SPARKSQL` or `PYSPARK` +- `ssis`: `SPARKSQL` only (PYSPARK not available) + +#### How do I customize BladeBridge output? + +Use a custom JSON override file. Pass it via `--overrides-file` or set it during `install-transpile`. See [BladeBridge Configuration](/docs/transpile/pluggable_transpilers/bladebridge/bladebridge_configuration) for the full config reference. + +### Switch + +#### Can Switch support my source system if it's not in the built-in list? + +Yes. Switch uses LLMs to convert arbitrary source formats through custom YAML prompts. If your dialect is not in the [built-in list](/docs/transpile/pluggable_transpilers/switch#source-format-support), create a custom prompt YAML: +- Start with a similar built-in dialect's YAML as a template +- Add examples specific to your source dialect +- Reference [SQLGlot dialects](https://github.com/tobymao/sqlglot/tree/main/sqlglot/dialects) for dialect-specific patterns + +#### My Switch files show status "Not converted". What does that mean? + +The file exceeded the `token_count_threshold` and was skipped. Solutions: +- Split the file into smaller parts +- Increase `token_count_threshold` in `switch_config.yml` if your model supports it + +#### My Switch files show "Converted with errors". How do I fix them? + +The LLM converted the file but the output has syntax errors. Options: +1. Review the `error_details` column in the conversion result table +2. Increase `max_fix_attempts` in `switch_config.yml` for more automatic correction attempts +3. Fix errors manually in the output notebook + +#### Switch exported some files but they weren't written to the output directory. + +Check the `export_error` column in the result table. Common causes: +- **Size limit:** Notebooks >10MB cannot be written. Split the converted content manually. +- **Permissions:** Verify your user has write access to the workspace output path. +- **Invalid path:** Output paths must start with `/Workspace/`. + +--- + +## Reconciliation + +### Commonly used custom transformations + +| source_type | data_type | source_transformation | target_transformation | comments | +|---|---|---|---|---| +| Oracle | number(10,5) | "trim(to_char(coalesce(col_name,0.0), '99990.99999'))" | "cast(coalesce(col_name,0.0) as decimal(10,5))" | Adjust precision/scale as needed | +| Snowflake | array | "array_to_string(array_compact(col_name),',')" | "concat_ws(',', col_name)" | Removes undefined during migration | +| Snowflake | array | "array_to_string(array_sort(array_compact(col_name), true, true),',')" | "concat_ws(',', col_name)" | Removes undefined and sorts | +| Snowflake | timestamp_ntz | "date_part(epoch_second,col_name)" | "unix_timestamp(col_name)" | Convert timestamp_ntz to epoch | + +### How do I reconcile tables when column names differ between source and target? + +Use `column_mapping` in your table config: + +```json +"column_mapping": [ + { "source_name": "dept_id", "target_name": "department_id" } +] +``` + +See [Configuration Reference — Column Mapping](/docs/reconcile/configuration#column-mapping). + +### How do I exclude specific columns from reconciliation? + +Use `drop_columns`: + +```json +"drop_columns": ["audit_timestamp", "etl_load_date"] +``` + +--- + +## Troubleshooting + +### Install Databricks CLI on Linux without brew + ```bash #!/usr/bin/env bash - -#install dependencies apt update && apt install -y curl sudo unzip +curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/v0.299.0/install.sh | sudo sh +``` -#install databricks cli -curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/v0.242.0/install.sh | sudo sh +### The `configure-reconcile` command fails because I can't create SQL warehouses or clusters. + +Add your existing warehouse or cluster ID to your Databricks CLI profile: + +``` +[profile-name] +host = +warehouse_id = +cluster_id = ``` + +### Transpilation produces output files with headers but no SQL. + +This means the entire file failed to parse. Check that: +1. The input file is valid SQL for the declared `--source-dialect` +2. The file is not empty +3. If using Morpheus, the input SQL matches a supported dialect (`mssql`, `snowflake`, or `synapse`) + +### The `install-transpile` command fails with a download error. + +Lakebridge downloads transpiler components from GitHub, Maven Central, and PyPI. If you are in a restricted network: +1. Whitelist the required endpoints (see [Installation — Prerequisites](/docs/installation#prerequisites)) +2. Or set up a private artifact mirror (Artifactory, Nexus) and configure it as the download source + +### How do I report a bug or request a feature? + +Open an issue at [github.com/databrickslabs/lakebridge/issues](https://github.com/databrickslabs/lakebridge/issues). For Switch-specific issues, include the LLM model name and a sample of the input that failed to convert. diff --git a/docs/lakebridge/docs/getting_started.mdx b/docs/lakebridge/docs/getting_started.mdx new file mode 100644 index 0000000000..d191460e52 --- /dev/null +++ b/docs/lakebridge/docs/getting_started.mdx @@ -0,0 +1,122 @@ +--- +sidebar_position: 2 +title: Getting Started +--- + +# Getting Started with Lakebridge + +This guide walks you through a complete end-to-end migration using **SQL Server as the source** and **Databricks SQL as the target**. You will assess your SQL code, transpile it, and validate the output — in about 15 minutes. + +**Prerequisites:** Lakebridge must already be installed. If not, see [Installation](/docs/installation) first. + +--- + +## _(Optional)_ Step 1: Split Your SQL Files + +If your SQL code lives in monolithic files that contain multiple objects (stored procedures, tables, views, functions), it is recommended to split them first. + +```bash +./sqlsplit -d /path/to/your/sql -o /path/to/split/output +``` + +- `-d` accepts a directory of `.sql` files +- `-o` must be a directory that already exists +- Output is organized into subfolders: `/PROCEDURE`, `/FUNCTION`, `/TABLE`, `/VIEW` + +See [SQL Splitter](/docs/sql_splitter) for full usage and download. + +:::tip +Splitting first gives the Analyzer more granular, per-object results and gives the transpiler a cleaner input. +::: + +--- + +## _(Optional)_ Step 2: Assess Your Code + +Run the Analyzer on the (optionally split) files to understand complexity and identify patterns the transpiler may need help with. +```bash +databricks labs lakebridge analyze +``` + +When prompted, enter the following details: +- **Input path:** `/path/to/split/output` +- **Source dialect:** `mssql` +- **Report file:** `/path/to/transpiled/output/output.xlsx` +- **Source technology:** Select the number corresponding to `MS SQL Server` + +The Analyzer produces a complexity report saved as an Excel file (`.xlsx`). Objects flagged as `HIGH` or `VERY HIGH` +complexity may need manual review after transpilation. + +See [Assessment Guide](/docs/assessment) for more details. + +--- + +## Step 3: Configure Transpilation + +During `install-transpile` (covered in [Installation](/docs/installation#install-transpile)), you are prompted for your migration parameters. For this _Getting Started_ guide, the values to use are: + +- **Source dialect:** Select the number corresponding to `mssql` +- **Input path:** `/path/to/split/output` +- **Output directory:** `/path/to/transpiled/output` +- **Transpiler:** Select the number corresponding to `Morpheus` (both Morpheus and BladeBridge support SQL Server, but this guide assumes Morpheus) + +If you need to change these settings, re-run `install-transpile`. + +Morpheus is the recommended transpiler for SQL Server migrations — it provides strong correctness guarantees and warns you when it cannot fully guarantee equivalence. + +See [Which transpiler should I use?](/docs/choosing_tools#transpiler-bladebridge-vs-morpheus-vs-switch) if you are migrating from a different source. + +--- + +## Step 4: Transpile + +Run the transpilation (if params already configured in previous step): + +```bash +databricks labs lakebridge transpile +``` + +Or pass all parameters inline: + +```bash +databricks labs lakebridge transpile \ + --source-dialect mssql \ + --input-source /path/to/split/output \ + --output-folder /path/to/transpiled/output +``` + +**Reading the output:** + +In the command output, you will see a final summary with all files converted and whether any errors were found. + +You may also find inline comments in the output files prefixed with "FIXME" that you'll need to review. + +Files with errors still contain as much translated output as possible. Review the flagged positions and fix manually. + +--- + +## Step 5: Reconcile + +:::note Prerequisite +This step requires `configure-reconcile` to have been completed during installation. If you skipped it, run it now — see [Installation → Configure Reconcile](/docs/installation#configure-reconcile). +::: + +After deploying your transpiled SQL to Databricks, run the Reconciler to validate that the data output matches the source. + +```bash +databricks labs lakebridge reconcile +``` + +The Reconciler compares row counts, schema, and data values between your source system and the Databricks target. Results are written to a dashboard in your workspace. + +See [Reconcile Guide](/docs/reconcile) for full configuration details. + +--- + +## What's Next + +| Next step | Guide | +|-----------|-------| +| Understand transpiler differences | [Which Tool Do I Use?](/docs/choosing_tools) | +| Migrate ETL workloads | [Source Systems](/docs/transpile/source_systems/) | +| Customize transpiler output | [BladeBridge Configuration](/docs/transpile/pluggable_transpilers/bladebridge/bladebridge_configuration), [Switch Configuration](/docs/transpile/pluggable_transpilers/switch/customizing_switch/) | diff --git a/docs/lakebridge/docs/installation.mdx b/docs/lakebridge/docs/installation.mdx index 3a22ff28c0..bb1fcd6d0a 100644 --- a/docs/lakebridge/docs/installation.mdx +++ b/docs/lakebridge/docs/installation.mdx @@ -1,5 +1,5 @@ --- -sidebar_position: 2 +sidebar_position: 1 --- import useBaseUrl from '@docusaurus/useBaseUrl'; import Tabs from '@theme/Tabs'; @@ -7,335 +7,140 @@ import TabItem from '@theme/TabItem'; # Installation +## Prerequisites -## [Table of Contents](#table-of-contents) +| Requirement | Details | +|---|---| +| **Databricks workspace** | Any workspace (production, development, or [free trial](https://www.databricks.com/try-databricks)) | +| **Databricks CLI** | [Install here](https://docs.databricks.com/en/dev-tools/cli/install.html) and configure with PAT or Service Principal | +| **Python** | 3.10.1 – 3.13.x (Python 3.14 not supported) | +| **Java** | Java 11 or above (required for the Morpheus transpiler) | +| **Network access** | GitHub, Maven Central (`repo1.maven.org`), PyPI | -* [Pre-requisites](#pre-requisites) -* [Install Lakebridge](#install-lakebridge) -* [Install Transpile](#install-transpile) -* [Configure Reconcile](#configure-reconcile) +:::warning Restricted environments +Hardened & Security-Restricted Environments +If you are operating in a hardened environment with internet restrictions, firewall rules, or security policies, you must whitelist the following resources before installation: ----- -## Pre-requisites - -### 1. Databricks Workspace Requirements - -#### 1.1 Databricks Workspace Access -You must have access to a Databricks workspace to install and use Lakebridge: - -- **Production/Enterprise Workspace:** Recommended for production migrations -- **Development Workspace:** Suitable for testing and development -- **Free Databricks Workspace:** Available at [databricks.com/try-databricks](https://www.databricks.com/try-databricks) - Perfect for evaluation and learning +- **GitHub:** `github.com`, `raw.githubusercontent.com` - For Lakebridge source code +- **Maven Central:** `repo1.maven.org`, `central.sonatype.com` - For transpiler plugins +- **PyPI:** `pypi.org`, `files.pythonhosted.org` - For Python packages +- **Python Downloads:** `python.org` - If installing Python +- **Java Downloads:** `oracle.com` or OpenJDK mirrors - If installing Java -:::tip Pro Tip -For evaluation purposes, using a free Databricks workspace with a Personal Access Token is the fastest way to get started. This combination requires no enterprise approvals and can be set up in minutes. +Action Required: Contact your IT Security, CyberSecOps, or Infrastructure team to request whitelisting. Consider setting up a private repository/artifact mirror for organizations with strict internet access policies. ::: -#### 1.2 Databricks CLI Installation & Configuration +### Configure the Databricks CLI -**Install Databricks CLI** - Ensure that you have the Databricks Command-Line Interface (CLI) installed on your machine. Refer to the installation instructions provided for Linux, MacOS, and Windows, available [here](https://docs.databricks.com/en/dev-tools/cli/install.html#install-or-update-the-databricks-cli). +Install and authenticate the CLI: -Installing the Databricks CLI in different OS: - - macos-databricks-cli-install - - - windows-databricks-cli-install + + ```shell + brew tap databricks/tap + brew install databricks + ``` - + ```shell + curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/v0.299.0/install.sh + ``` + + + ```bash #!/usr/bin/env bash - - #install dependencies apt update && apt install -y curl sudo unzip - - #install databricks cli - curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/v0.242.0/install.sh | sudo sh + curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/v0.299.0/install.sh | sudo sh ``` -**Configure Databricks CLI** - Details can be found [here](https://docs.databricks.com/aws/en/dev-tools/cli/authentication#databricks-personal-access-token-authentication). +Authenticate the CLI: -**Authentication Options:** -- **Personal Access Token (PAT):** Recommended for individual users - Generate from User Settings → Developer → Access Tokens -- **Service Principal:** Recommended for automated/production deployments - Requires admin privileges to create - -**Profile Configuration:** - -Additionally, Lakebridge requires the profile used for the Databricks CLI to specify a cluster_id, to do this, you can either: -- Edit your `~/.databrickscfg` file directly and enter a `cluster_id` for the profile you're using or -- The flag `--configure-cluster` gives you the prompt to select the cluster_id from the available clusters on the workspace -specified on the selected profile. -```shell -databricks configure --host --configure-cluster --profile -``` -- Alternatively you can use the environment variable `DATABRICKS_CLUSTER_ID` to set the cluster id you would want to use -for your profile before running the `databricks configure` command. - -```shell -export DATABRICKS_CLUSTER_ID= -databricks configure --host --profile +```bash +databricks configure ``` -**Verification:** Run `databricks clusters list` to confirm connectivity +Verify connectivity: `databricks clusters list` -### 2. Software Requirements - -#### 2.1 Python - -**Version Required:** Python between 3.10.1 and 3.13.x (inclusive). - -- **Windows** - Install python from [here](https://www.python.org/downloads/). -- **MacOS/Unix** - Use [brew](https://formulae.brew.sh/formula/python@3.10) to install python in macOS/Unix machines - -**Check Python version on Windows, macOS, and Unix:** -check-python-version - -**Verification:** Run `python --version` to confirm installation - -**Note: Python 3.14 is not currently supported.** - -#### 2.2 Java - -**Version Required:** [Java 11 or above](https://www.oracle.com/java/technologies/downloads/) - -- **Recommended:** OpenJDK 11 or later (available via system package managers) -- **Purpose:** Required for the Morpheus transpiler component that powers code conversion -- **Verification:** Run `java -version` to confirm installation - -### 3. Network & Environment Access - -The following network endpoints must be accessible from your installation environment: - -#### 3.1 GitHub Access -Access to Databricks Labs GitHub repository for downloading Lakebridge. -- **Repository:** [github.com/databrickslabs/lakebridge](https://github.com/databrickslabs/lakebridge) -- **Purpose:** Download latest official Lakebridge installation packages - -#### 3.2 Maven Central Repository -Access to Maven Central for downloading transpiler plugins and dependencies. -- **Repository:** [central.sonatype.com](https://central.sonatype.com/artifact/com.databricks.labs/databricks-morph-plugin) -- **Purpose:** Download latest version of the transpiler plugins - -#### 3.3 PyPI (Python Package Index) -Access to PyPI for Python package dependencies. -- **Repository:** [pypi.org](https://pypi.org) -- **Purpose:** Install Python dependencies - -:::warning Hardened & Security-Restricted Environments -If you are operating in a hardened environment with internet restrictions, firewall rules, or security policies, you must whitelist the following resources **before installation**: - -- **GitHub:** `github.com`, `raw.githubusercontent.com` - For Lakebridge source code -- **Maven Central:** `repo1.maven.org`, `central.sonatype.com` - For transpiler plugins -- **PyPI:** `pypi.org`, `files.pythonhosted.org` - For Python packages -- **Python Downloads:** `python.org` - If installing Python -- **Java Downloads:** `oracle.com` or OpenJDK mirrors - If installing Java - -**Action Required:** Contact your IT Security, CyberSecOps, or Infrastructure team to request whitelisting. Consider setting up a **private repository/artifact mirror** for organizations with strict internet access policies. -::: - -#### 3.4 Private Repository Hosting (Optional) -For organizations with restricted internet access or security requirements: -- **Purpose:** Internal hosting and distribution of Lakebridge components -- **Options:** Artifactory, Nexus -- **Benefits:** Control over versions, security scanning, compliance validation - -### Pre-Installation Checklist - -Verify all prerequisites before proceeding: - -- ☐ Databricks workspace access confirmed (production, dev, or free trial) -- ☐ Databricks CLI installed on your machine -- ☐ Databricks CLI configured with PAT or Service Principal for your workspace -- ☐ CLI connectivity verified (`databricks clusters list` succeeds) -- ☐ Python between 3.10.1 and 3.13.x (inclusive) installed and verified (`python --version`) -- ☐ Java 11+ installed and verified (`java -version`) -- ☐ Network access to GitHub confirmed -- ☐ Network access to Maven Central confirmed -- ☐ Network access to PyPI confirmed -- ☐ (If applicable) All required resources whitelisted in restricted environments -- ☐ (If applicable) Private repository configured for package hosting - -[[back to top](#table-of-contents)] +--- ----- ## Install Lakebridge -Upon completing the environment setup, install Lakebridge by executing the following command: ```bash databricks labs install lakebridge ``` -This will install Lakebridge using the workspace details set in the DEFAULT profile. If you want to install it using a different profile, you can specify the profile name using the `--profile` flag. + +To use a specific profile: + ```bash databricks labs install lakebridge --profile ``` -To view all the profiles available, you can run the following command: -```bash -databricks auth profiles -``` lakebridge-install +Verify: -### Verify Installation -Verify the successful installation by executing the provided command; -confirmation of a successful installation is indicated when the displayed output aligns with the example below: - -Command: ```bash databricks labs lakebridge --help ``` -Should output: -```console -Code Transpiler and Data Reconciliation tool for Accelerating Data onboarding to Databricks from EDW, CDW and other ETL sources. - -Usage: - databricks labs lakebridge [command] - -Available Commands: - aggregates-reconcile Reconcile source and target data residing on Databricks using aggregated metrics - analyze Analyze existing non-Databricks database or ETL sources - configure-database-profiler Configure database profiler - configure-reconcile Configure 'reconcile' dependencies - describe-transpile Describe installed transpilers - install-transpile Install & optionally configure 'transpile' dependencies - reconcile Reconcile source and target data residing on Databricks - transpile Transpile SQL/ETL sources to Databricks-compatible code - -Flags: - -h, --help help for lakebridge - -Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) - -Use "databricks labs lakebridge [command] --help" for more information about a command. -``` - -[[back to top](#table-of-contents)] - ----- +--- ## Install Transpile -Upon completing the environment setup, you can install the out of the box transpilers by executing the following command. -This command will also prompt for the required configuration elements so that you don't need to include them in your command-line -call every time. - ```bash databricks labs lakebridge install-transpile ``` -transpile-install - +The command will prompt for your source dialect, input/output paths, and target technology. -:::tip -#### Override the default[Bladebridge] config: +To install Switch (the LLM transpiler): -There is an option for you to override the default config file that `Lakebridge` uses for converting source code from dialects like `datastage`, `synapse`, -`oracle` etc. During installation you may use your own custom config file and `Lakebridge` will override the config with the one you would provide. You can only setup this -override during installation. +```bash +databricks labs lakebridge install-transpile --include-llm-transpiler true +``` -Specify the config file to override the default[Bladebridge] config during installation: +:::tip Override the default BladeBridge config +During `install-transpile` you can supply a custom config file for BladeBridge: ``` -Specify the config file to override the default[Bladebridge] config - press for none (default: ): /custom_2databricks.json +Specify the config file to override the default[Bladebridge] config: /custom_config.json ``` ::: -### Verify Installation -Verify the successful installation by executing the provided command; confirmation of a successful installation is indicated when the displayed output aligns with the example output: +Verify: -Command: ```bash databricks labs lakebridge transpile --help ``` -Should output: -```console -Transpile SQL/ETL sources to Databricks-compatible code - -Usage: - databricks labs lakebridge transpile [flags] - -Flags: - --catalog-name name (Optional) Catalog name, only used when validating converted code - --error-file-path path (Optional) Local path where a log of conversion errors (if any) will be written - -h, --help help for transpile - --input-source path (Optional) Local path of the sources to be convert - --output-folder path (Optional) Local path where converted code will be written - --overrides-file path (Optional) Local path of a file containing transpiler overrides, if supported by the transpiler in use - --schema-name name (Optional) Schema name, only used when validating converted code - --skip-validation string (Optional) Whether to skip validating the output ('true') after conversion or not ('false') - --source-dialect string (Optional) The source dialect to use when performing conversion - --target-technology string (Optional) Target technology to use for code generation, if supported by the transpiler in use - --transpiler-config-path path (Optional) Local path to the configuration file of the transpiler to use for conversion - -Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) -``` - -[[back to top](#table-of-contents)] - ----- +--- ## Configure Reconcile -Once you're ready to reconcile your data, you need to configure the reconcile module. - -```commandline +```bash databricks labs lakebridge configure-reconcile ``` -reconcile-configure +The command will prompt for your source connection and Databricks catalog to reconcile, and install Lakebridge and create the required workspace resources to run Reconcile. -### SQL Warehouse for Reconcile +If you don't have permission to create SQL warehouses or clusters, add a `warehouse_id` or a `cluster_id` to your Databricks CLI profile: -While configuring the reconcile properties, lakebridge by default creates a SQL warehouse. lakebridge uses user profile to authenticate to any Databricks resource and hence -if the user running this command doesn't have permission to create SQL warehouse, the configure-reconcile would fail. In this case users can provide the -`warehouse_id` of an already created SQL warehouse that they have atleast CAN_USE permission on in the databricks profile (`~/.databrickscfg`) using which they -are running the lakebridge commands and lakebridge would use that warehouse to complete the reconcile configuration instead of trying to create a new one. - -This is how the profile would look like: - -```console +``` [profile-name] host = -... warehouse_id = +cluster_id = ``` +Verify: -### Verify Configuration -Verify the successful configuration by executing the provided command; confirmation of a successful configuration is indicated when the displayed output aligns with the example screenshot provided: - -Command: ```bash - databricks labs lakebridge reconcile --help - ``` - -Should output: -```console -Reconcile source and target data residing on Databricks - -Usage: - databricks labs lakebridge reconcile [flags] +databricks labs lakebridge reconcile --help +``` -Flags: - -h, --help help for reconcile +--- -Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) -``` +## Service Principal Setup (Optional) -[[back to top](#table-of-contents)] +For automated/production deployments, use a Service Principal instead of a Personal Access Token. See the [Databricks CLI authentication docs](https://docs.databricks.com/aws/en/dev-tools/cli/authentication) for setup instructions. diff --git a/docs/lakebridge/docs/reconcile/configuration.mdx b/docs/lakebridge/docs/reconcile/configuration.mdx new file mode 100644 index 0000000000..4218404573 --- /dev/null +++ b/docs/lakebridge/docs/reconcile/configuration.mdx @@ -0,0 +1,447 @@ +--- +sidebar_position: 1 +title: Configuration Reference +--- +import useBaseUrl from '@docusaurus/useBaseUrl'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; + +# Reconcile Configuration Reference + +This page covers the full configuration schema for Lakebridge Reconcile. For setup steps and prerequisites, see the [Reconcile Guide](/docs/reconcile). + +--- + +## Report Types + +| report type | description | key outputs | +|---|---|---| +| `schema` | Reconcile schema of source and target — validate that data types are the same or compatible | `schema_comparison`, `schema_difference` | +| `row` | Reconcile data at row level (hash value comparison). Use when there are no join columns. | `missing_in_src`, `missing_in_tgt` | +| `data` | Reconcile data at row and column level using `join_columns` to identify per-row, per-column mismatches | `mismatch_data`, `missing_in_src`, `missing_in_tgt`, `threshold_mismatch`, `mismatch_columns` | +| `all` | Combination of `data` + `schema` | All outputs above | + +See [Reconcile Data Flow Examples](/docs/reconcile/dataflow_example) for visualizations of each report type. + +--- + +## Config File Naming + +``` +recon_config___.json +``` + +Place the file in `.lakebridge/` in your Databricks workspace home folder. + +> The filename must match the case of `` exactly. For example, if the source connection is `conn-oracle-prod`, the filename is `recon_config_oracle_conn-oracle-prod_data.json`. + +**Examples by source:** + + + + ```yaml title="recon_config_snowflake_sample_data_all.json" + source: + dialect: snowflake + catalog: sample_data + schema: default + uc_connection_name: example_connection_snowflake + target: + catalog: migrated + schema: migrated + report_type: all + ``` + + + ```yaml title="recon_config_oracle_orc_data.json" + source: + dialect: oracle + uc_connection_name: example_connection_oracle + target: + catalog: migrated + schema: migrated + report_type: data + ``` + + + ```yaml title="recon_config_tsql_silver_data.json" + source: + dialect: mssql + uc_connection_name: example_connection_mssql + target: + catalog: migrated + schema: migrated + report_type: data + ``` + + + ```yaml title="recon_config_databricks_hms_schema.json" + source: + dialect: databricks + catalog: hive_metastore + schema: hr + target: + catalog: migrated + schema: migrated + ... + report_type: data + ``` + + + +--- + +## TABLE Config Schema + + + + ```python + @dataclass + class Table: + source_name: str + target_name: str + aggregates: list[Aggregate] | None = None + join_columns: list[str] | None = None + jdbc_reader_options: JdbcReaderOptions | None = None + select_columns: list[str] | None = None + drop_columns: list[str] | None = None + column_mapping: list[ColumnMapping] | None = None + transformations: list[Transformation] | None = None + column_thresholds: list[ColumnThresholds] | None = None + filters: Filters | None = None + table_thresholds: list[TableThresholds] | None = None + ``` + + + ```json + { + "source_name": "[SOURCE_NAME]", + "target_name": "[TARGET_NAME]", + "aggregates": null, + "join_columns": ["COLUMN_NAME_1", "COLUMN_NAME_2"], + "jdbc_reader_options": null, + "select_columns": null, + "drop_columns": null, + "column_mapping": null, + "transformation": null, + "column_thresholds": null, + "filters": null, + "table_thresholds": null + } + ``` + + + +### Configuration Parameters + +| Parameter | Type | Required | Description | +|---|---|---|---| +| `source_name` | string | Yes | Source table name | +| `target_name` | string | Yes | Target table name | +| `join_columns` | list[string] | No | Primary key columns for row-level joins | +| `aggregates` | list[Aggregate] | No | Aggregate reconciliation rules (see [Aggregates](#aggregates-reconciliation)) | +| `jdbc_reader_options` | object | No | JDBC read parallelization (see [JDBC Reader Options](#jdbc-reader-options)) | +| `select_columns` | list[string] | No | Columns to include in reconciliation | +| `drop_columns` | list[string] | No | Columns to exclude from reconciliation | +| `column_mapping` | list[ColumnMapping] | No | Source-to-target column name mapping | +| `transformations` | list[Transformation] | No | Column-level SQL transformation expressions | +| `column_thresholds` | list[ColumnThresholds] | No | Acceptable variance per column | +| `table_thresholds` | list[TableThresholds] | No | Acceptable mismatch rate at table level | +| `filters` | Filters | No | WHERE-clause filters for source and/or target | + +--- + +## JDBC Reader Options + +Use JDBC reader options to parallelize reads from large tables. + + + + ```python + @dataclass + class JdbcReaderOptions: + num_partitions: int + partition_column: str + lower_bound: str + upper_bound: str + fetchsize: int = 0 + ``` + + + ```json + "jdbc_reader_options": { + "num_partitions": 10, + "partition_column": "id", + "lower_bound": "1", + "upper_bound": "100000", + "fetchsize": 0 + } + ``` + + + +| Field | Type | Required | Description | +|---|---|---|---| +| `num_partitions` | int | Yes | Number of Spark partitions for parallel reads | +| `partition_column` | string | Yes | Column used for partitioning (choose high-cardinality column for composite keys) | +| `lower_bound` | string | Yes | Minimum value for partitioning | +| `upper_bound` | string | Yes | Maximum value for partitioning | +| `fetchsize` | int | No (default: 100) | Rows per JDBC round-trip | + +--- + +## Column Mapping + +Map source column names to target column names when they differ. + + + + ```python + @dataclass + class ColumnMapping: + source_name: str + target_name: str + ``` + + + ```json + "column_mapping": [ + { + "source_name": "dept_id", + "target_name": "department_id" + } + ] + ``` + + + +--- + +## Drop Columns + +Exclude specific columns from reconciliation (for `data` or `all` report types). + +```json +"drop_columns": ["comment", "audit_timestamp"] +``` + +--- + +## Transformations + +Apply SQL expressions to source or target columns before comparison. Use this when data types or formats differ between systems. + + + + ```python + @dataclass + class Transformation: + column_name: str + source: str + target: str | None = None + ``` + + + ```json + "transformations": [ + { + "column_name": "unit_price", + "source": "coalesce(cast(cast(unit_price as decimal(38,10)) as string), '_null_recon_')", + "target": "coalesce(cast(format_number(cast(unit_price as decimal(38, 10)), 10) as string), '_null_recon_')" + } + ] + ``` + + + +:::danger Handling Nulls +When you provide a transformation expression, Reconcile uses it as-is. **You must handle nulls explicitly** in the expression. Use `coalesce(..., '_null_recon_')` to avoid null mismatches. +::: + +:::tip Timestamp Columns +Convert timestamps to Unix epoch strings for cross-platform comparison: +```python +Transformation( + column_name='UPDATE_TIMESTAMP', + source="coalesce(cast(EXTRACT(epoch_millisecond FROM UPDATE_TIMESTAMP) as string), '_null_recon_')", + target="coalesce(cast(unix_millis(UPDATE_TIMESTAMP) as string), '_null_recon_')" +) +``` +::: + +--- + +## Column Thresholds + +Allow a bounded variance between source and target column values. + +```json +"column_thresholds": [ + { + "column_name": "price", + "lower_bound": "-5%", + "upper_bound": "5%", + "type": "float" + } +] +``` + +| Field | Type | Required | Description | +|---|---|---|---| +| `column_name` | string | Yes | Column to apply the threshold to | +| `lower_bound` | string | Yes | Lower bound of acceptable difference | +| `upper_bound` | string | Yes | Upper bound of acceptable difference | +| `type` | string | Yes | Column type (supports SQLGlot `DataType.NUMERIC_TYPES` and `DataType.TEMPORAL_TYPES`) | + +--- + +## Table Thresholds + +Allow a bounded mismatch rate at the table level. + +```json +"table_thresholds": [ + { + "lower_bound": "0%", + "upper_bound": "5%", + "model": "mismatch" + } +] +``` + +Bounds must be non-negative, and lower bound must not exceed upper bound. Only `"mismatch"` is supported for `model`. + +--- + +## Filters + +Apply WHERE-clause filters to source and/or target before reconciliation. + +```json +"filters": { + "source": "lower(dept_name) = 'finance'", + "target": "lower(department_name) = 'finance'" +} +``` + +:::note +Filters must use dialect-specific SQL expressions — they are applied directly without any transformation by Reconcile. +::: + +--- + +## Aggregates Reconciliation + +Reconcile aggregate metrics (MIN, MAX, COUNT, SUM, AVG, etc.) between source and target instead of comparing raw rows. + +### Supported Functions + +MIN, MAX, COUNT, SUM, AVG, MEAN, MODE, STDDEV, VARIANCE, MEDIAN + +### Aggregate Config + + + + ```python + @dataclass + class Aggregate: + agg_columns: list[str] + type: str + group_by_columns: list[str] | None = None + ``` + + + ```json + { + "type": "MIN", + "agg_columns": ["discount"], + "group_by_columns": ["p_id"] + } + ``` + + + +:::note +`select_columns` and `drop_columns` are ignored for aggregate reconciliation. `column_mapping`, `transformations`, `jdbc_reader_options`, and `filters` are applied. +::: + +--- + +## Complete Examples + +### Standard Reconcile Config + +```json +{ + "tables": [ + { + "source_name": "product_prod", + "target_name": "product", + "jdbc_reader_options": { + "num_partitions": 10, + "partition_column": "p_id", + "lower_bound": "0", + "upper_bound": "10000000" + }, + "join_columns": ["p_id"], + "drop_columns": ["comment"], + "column_mapping": [ + { "source_name": "p_id", "target_name": "product_id" }, + { "source_name": "p_name", "target_name": "product_name" } + ], + "transformations": [ + { + "column_name": "creation_date", + "source": "creation_date", + "target": "to_date(creation_date,'yyyy-mm-dd')" + } + ], + "column_thresholds": [ + { "column_name": "price", "lower_bound": "-50", "upper_bound": "50", "type": "float" } + ], + "table_thresholds": [ + { "lower_bound": "0%", "upper_bound": "5%", "model": "mismatch" } + ], + "filters": { + "source": "p_id > 0", + "target": "product_id > 0" + } + } + ] +} +``` + +### Aggregates Reconcile Config + +```json +{ + "tables": [ + { + "source_name": "product_prod", + "target_name": "product", + "join_columns": ["p_id"], + "aggregates": [ + { "type": "MIN", "agg_columns": ["discount"], "group_by_columns": ["p_id"] }, + { "type": "AVG", "agg_columns": ["discount"], "group_by_columns": ["p_id"] }, + { "type": "MAX", "agg_columns": ["p_id"], "group_by_columns": ["creation_date"] }, + { "type": "SUM", "agg_columns": ["p_id"] } + ], + "jdbc_reader_options": { + "num_partitions": 10, + "partition_column": "p_id", + "lower_bound": "0", + "upper_bound": "10000000" + } + } + ] +} +``` + +--- + +:::note Key Considerations +1. Column names are always converted to lowercase. +2. Case insensitivity and collation are not currently supported. +3. Always reference **source** column names in all configs, except `transformations` and `filters` — those use dialect-specific SQL applied directly. +4. When no user transformation is provided, Reconcile applies default transformations based on the source data type. +::: diff --git a/docs/lakebridge/docs/reconcile/example_config.mdx b/docs/lakebridge/docs/reconcile/example_config.mdx deleted file mode 100644 index 688c5b5755..0000000000 --- a/docs/lakebridge/docs/reconcile/example_config.mdx +++ /dev/null @@ -1,181 +0,0 @@ ---- -sidebar_position: 3 -title: Example Configuration ---- - -## Reconcile Config -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - -Consider the below tables that we want to reconcile: - -| category | catalog | schema | table_name | schema | primary_key | -|----------|----------------|---------------|--------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------| -| source | source_catalog | source_schema | product_prod | p_id INT,
p_name STRING
,
price NUMBER
,
discount DECIMAL(5,3)
,
offer DOUBLE
,
creation_date DATE
,
comment STRING
 | p_id | -| target | target_catalog | target_schema | product | product_id INT,
product_name STRING
,
price NUMBER
,
discount DECIMAL(5,3)
,
offer DOUBLE
,
creation_date DATE
,
comment STRING
 | product_id | - - - - - - :::note - Run with Drop,Join,Transformation,ColumnThresholds,Filter,JDBC ReaderOptions configs - ::: - ```json - { - "tables": [ - { - "source_name": "product_prod", - "target_name": "product", - "jdbc_reader_options": { - "number_partitions": 10, - "partition_column": "p_id", - "lower_bound": "0", - "upper_bound": "10000000" - }, - "join_columns": [ - "p_id" - ], - "drop_columns": [ - "comment" - ], - "column_mapping": [ - { - "source_name": "p_id", - "target_name": "product_id" - }, - { - "source_name": "p_name", - "target_name": "product_name" - } - ], - "transformations": [ - { - "column_name": "creation_date", - "source": "creation_date", - "target": "to_date(creation_date,'yyyy-mm-dd')" - } - ], - "column_thresholds": [ - { - "column_name": "price", - "upper_bound": "-50", - "lower_bound": "50", - "type": "float" - } - ], - "table_thresholds": [ - { - "lower_bound": "0%", - "upper_bound": "5%", - "model": "mismatch" - } - ], - "filters": { - "source": "p_id > 0", - "target": "product_id > 0" - } - } - ] - } - ``` - - - :::note - Aggregates-Reconcile run with Join, Column Mappings, Transformation, Filter and JDBC ReaderOptions configs - Even though the user provides the \`select_columns\` and \`drop_columns\`, those are not considered. - ::: - ```json - { - "tables": [ - { - "aggregates": [{ - "type": "MIN", - "agg_columns": ["discount"], - "group_by_columns": ["p_id"] - }, - { - "type": "AVG", - "agg_columns": ["discount"], - "group_by_columns": ["p_id"] - }, - { - "type": "MAX", - "agg_columns": ["p_id"], - "group_by_columns": ["creation_date"] - }, - { - "type": "MAX", - "agg_columns": ["p_name"] - }, - { - "type": "SUM", - "agg_columns": ["p_id"] - }, - { - "type": "MAX", - "agg_columns": ["creation_date"] - }, - { - "type": "MAX", - "agg_columns": ["p_id"], - "group_by_columns": ["creation_date"] - } - ], - "source_name": "product_prod", - "target_name": "product", - "jdbc_reader_options": { - "number_partitions": 10, - "partition_column": "p_id", - "lower_bound": "0", - "upper_bound": "10000000" - }, - "join_columns": [ - "p_id" - ], - "drop_columns": [ - "comment" - ], - "column_mapping": [ - { - "source_name": "p_id", - "target_name": "product_id" - }, - { - "source_name": "p_name", - "target_name": "product_name" - } - ], - "transformations": [ - { - "column_name": "creation_date", - "source": "creation_date", - "target": "to_date(creation_date,'yyyy-mm-dd')" - } - ], - "column_thresholds": [ - { - "column_name": "price", - "upper_bound": "-50", - "lower_bound": "50", - "type": "float" - } - ], - "table_thresholds": [ - { - "lower_bound": "0%", - "upper_bound": "5%", - "model": "mismatch" - } - ], - "filters": { - "source": "p_id > 0", - "target": "product_id > 0" - } - } - ] - } - ``` - - diff --git a/docs/lakebridge/docs/reconcile/index.mdx b/docs/lakebridge/docs/reconcile/index.mdx index 8cc8ff7235..1be5e46144 100644 --- a/docs/lakebridge/docs/reconcile/index.mdx +++ b/docs/lakebridge/docs/reconcile/index.mdx @@ -1,5 +1,5 @@ --- -sidebar_position: 5 +sidebar_position: 6 --- import useBaseUrl from '@docusaurus/useBaseUrl'; import Tabs from '@theme/Tabs'; @@ -7,50 +7,82 @@ import TabItem from '@theme/TabItem'; # Reconcile Guide -This tool empowers users to efficiently identify discrepancies and variations in data when comparing the source with the Databricks target. +Lakebridge Reconcile validates data fidelity after migration by comparing your source system against the Databricks target. It identifies discrepancies at the row, column, and schema level. +## What it does -## Execution Pre-Set Up -### 1. Setup the configuration file: +| Report type | What is compared | When to use | +|-------------|-----------------|-------------| +| `schema` | Column names and data types | Verify DDL migration is correct | +| `row` | Hash of each row (no join key needed) | Quick row-level check when there is no primary key | +| `data` | Row and column values via join columns | Full fidelity check with per-column mismatch detail | +| `all` | Both `data` + `schema` | Complete validation | -Once the [installation](/installation.mdx#configure-reconcile) is done, a folder named **.lakebridge** will be created in the user workspace's home folder. -To process the reconciliation for specific table sources, we must create a config file that gives the detailed required configurations for the table-specific ones. -The file name should be in the format as below and created inside the **.lakebridge** folder. -``` -recon_config___.json - -Note: For UNITY_CATALOG_CONNECTION_NAME_OR_CATALOG , if databricks source then CATALOG else connection name -``` - -eg: +## Supported Source Systems -| source_type | connection_name_or_catalog | report_type | file_name | -|-------------|-------------------|-------------|---------------------------------------| -| databricks | tpch | all | recon_config_databricks_tpch_all.json | -| source1 | conn1 | row | recon_config_source1_conn1_row.json | -| source2 | conn2 | schema | recon_config_source2_conn2_schema.json | +| Source | Schema | Row | Data | All | +|---|---|---|---|---| +| Oracle | Yes | Yes | Yes | Yes | +| Snowflake | Yes | Yes | Yes | Yes | +| SQL Server | Yes | Yes | Yes | Yes | +| Redshift | Yes | Yes | Yes | Yes | +| Databricks | Yes | Yes | Yes | Yes | +--- -Refer to [Reconcile Configuration Guide](reconcile_configuration) for detailed instructions and [example configurations](example_config) +## Setup -### 2. Setup the source connection +### Step 1: Setup the source connection Follow the official Databricks docs to: - [Create a connection](https://docs.databricks.com/aws/en/query-federation/remote-queries#create-a-connection) - [Grant connection access](https://docs.databricks.com/aws/en/query-federation/remote-queries#grant-connection-access) +- [Enable Databricks preview](https://docs.databricks.com/aws/en/admin/workspace-settings/manage-previews#-manage-workspace-level-previews) of `remote_query` feature :::note You do not have to create a foreign catalog. ::: +### Step 2: Run `configure-reconcile` -### 3. Databricks permissions required +If you haven't already, complete the initial setup: + +```bash +databricks labs lakebridge configure-reconcile +``` -- User configuring reconcile must have permission to create Data Warehouses -- Additionally, the user must have `USE CATALOG` and `CREATE SCHEMA` permission in order to deploy metadata tables and -dashboards that are created as part of the Reconcile output. If there is a pre-existing schema, the 'create volumes' permission is also required. +This sets up Lakebridge workspace resources, deploys the reconciliation dashboards and creates the config file. See [Installation → Configure Reconcile](/docs/installation#configure-reconcile) for details. -### 4. Serverless cluster support +### Config file +A reconcile config file is created under the path: +``` +/.lakebridge/recon_config___.json +``` +:::note +For `UNITY_CATALOG_CONNECTION_NAME_OR_CATALOG`: if the source is databricks then source catalog name is used else connection name is used +::: + +Examples: + +| source_type | connection_name_or_catalog | report_type | file_name | +|-------------|-------------------|-------------|---------------------------------------| +| databricks | tpch | all | recon_config_databricks_tpch_all.json | +| source1 | conn1 | row | recon_config_source1_conn1_row.json | +| source2 | conn2 | schema | recon_config_source2_conn2_schema.json | + + +See [Configuration Reference](/docs/reconcile/configuration) for the full schema and examples. + +### Required permissions + +The User configuring reconcile must have permission to: +- Create Data Warehouses +- Create Compute Clusters +- `USE CONNECTION` on the source connection +- `USE CATALOG` and `CREATE SCHEMA` on the target catalog +- `CREATE VOLUME` if using a pre-existing schema on a serverless cluster + +### Serverless cluster support Reconcile automatically detects the cluster type and optimizes intermediate data persistence accordingly: - **On Serverless clusters**: Reconcile uses Unity Catalog volumes for intermediate data persistence @@ -63,6 +95,12 @@ Reconcile automatically detects the cluster type and optimizes intermediate data ::: -## Running Reconcile +Reconcile automatically adapts to the cluster type: +- **Serverless clusters:** Uses Unity Catalog volumes for intermediate data persistence (`metadata_config.volume`) +- **Standard clusters:** Uses DataFrame caching + +--- + +## Run -Please refer to the [Reconcile Notebook](recon_notebook.mdx) documentation. +See [Running Reconcile](/docs/reconcile/running) for CLI execution, notebook usage, and automation. diff --git a/docs/lakebridge/docs/reconcile/reconcile_configuration.mdx b/docs/lakebridge/docs/reconcile/reconcile_configuration.mdx deleted file mode 100644 index 6f03a1cc45..0000000000 --- a/docs/lakebridge/docs/reconcile/reconcile_configuration.mdx +++ /dev/null @@ -1,754 +0,0 @@ ---- -sidebar_position: 1 -title: Configuration ---- -import useBaseUrl from '@docusaurus/useBaseUrl'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - -## Types of Report Supported - -| report type | sample visualisation | description | key outputs captured in the recon metrics tables | -|-------------|--------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **schema** | [schema](../dataflow_example#schema) | reconcile the schema of source and target.
- validate the datatype is same or compatible | - **schema_comparison**
- **schema_difference** | -| **row** | [row](../dataflow_example#row) | reconcile the data only at row level(hash value of the source row is matched with the hash value of the target).Preferred when there are no join columns identified between source and target. | - **missing_in_src**(sample rows that are available in target but missing in source + sample rows in the target that don't match with the source)
- **missing_in_tgt**(sample rows that are available in source but are missing in target + sample rows in the source that doesn't match with target)
**NOTE**: the report won't differentiate the mismatch and missing here. | -| **data** | [data](../dataflow_example#data) | reconcile the data at row and column level- ```join_columns``` will help us to identify mismatches at each row and column level | - **mismatch_data**(the sample data with mismatches captured at each column and row level )
- **missing_in_src**(sample rows that are available in target but missing in source)
- **missing_in_tgt**(sample rows that are available in source but are missing in target)
- **threshold_mismatch**(configured column will be reconciled based on percentile or threshold boundary or date boundary)
- **mismatch_columns**(consolidated list of columns that has mismatches in them)
| -| **all** | [all](../dataflow_example#all) | this is a combination of data + schema | - **data + schema outputs** | - -[[back to top](#types-of-report-supported)] - -## Supported Source System - -| Source | Schema | Row | Data | All | -|-------------------------------|--------|-----|------|-----| -| Oracle | Yes | Yes | Yes | Yes | -| Snowflake | Yes | Yes | Yes | Yes | -| MS SQL Server (incl. Synapse) | Yes | Yes | Yes | Yes | -| Databricks | Yes | Yes | Yes | Yes | -| Redshift | Yes | Yes | Yes | Yes | - -[[back to top](#types-of-report-supported)] - -## TABLE Config Json filename: -The config file must be named as `recon_config_[DATA_SOURCE]_[SOURCE_UNITY_CATALOG_CONNECTION_NAME]_[REPORT_TYPE].json` and should be placed in the Lakebridge root directory `.lakebridge` within the Databricks Workspace. - -> The filename pattern would remain the same for all the data_sources. - -``` shell -recon_config_${DATA_SOURCE}_${SOURCE_UNITY_CATALOG_CONNECTION_NAME}_${REPORT_TYPE}.json -``` - -Please find the `Table Recon` filename examples below for the `Snowflake`, `Oracle`, `` and `Databricks` source systems. - - - - ``` yaml title="recon_config_snowflake_example_connection_snowflake_all.json" - source: - dialect: snowflake - catalog: sample_data - schema: default - uc_connection_name: example_connection_snowflake - ... - metadata_config: - ... - report_type: all - ... - ``` - - - ``` yaml title="recon_config_oracle_example_connection_oracle_data.json" - source: - dialect: oracle - uc_connection_name: example_connection_oracle - ... - metadata_config: - ... - report_type: data - ... - ``` - - - - ``` yaml title="recon_config_tsql_example_connection_mssql_data.json" - database_config: - dialect: mssql - uc_connection_name: example_connection_mssql - ... - metadata_config: - ... - report_type: data - ... - ``` - - - - ``` yaml title="recon_config_databricks_hms_schema.json" - database_config: - dialect: databricks - catalog: hms - ... - metadata_config: - ... - report_type: schema - ... - ``` - - - - -> **Note:** the filename must be created in the same case as `uc_connection_name` is defined. Or `catalog` for databricks sources -> For example, if the source connection name is defined as `ORC_connection` in the `reconcile` config, the filename should be `recon_config_oracle_ORC_connection_data.json`. - - -[[back to top](#types-of-report-supported)] - - - -## TABLE Config Elements: - - - - ```python - @dataclass - class Table: - source_name: str - target_name: str - aggregates: list[Aggregate] | None = None - join_columns: list[str] | None = None - jdbc_reader_options: JdbcReaderOptions | None = None - select_columns: list[str] | None = None - drop_columns: list[str] | None = None - column_mapping: list[ColumnMapping] | None = None - transformations: list[Transformation] | None = None - column_thresholds: list[ColumnThresholds] | None = None - filters: Filters | None = None - table_thresholds: list[TableThresholds] | None = None - ``` - - - ```json - { - "source_name": "[SOURCE_NAME]", - "target_name": "[TARGET_NAME]", - "aggregates": null, - "join_columns": ["COLUMN_NAME_1","COLUMN_NAME_2"], - "jdbc_reader_options": null, - "select_columns": null, - "drop_columns": null, - "column_mapping": null, - "transformation": null, - "column_thresholds": null, - "filters": null, - "table_thresholds": null - } - ``` - - - - - -### Configuration Reference - -| Configuration Parameter | Data Type | Description | Requirement | Example Value | -|-------------------------|--------------------------|--------------------------------------------------------------------------------|-------------|---------------------------------------------------------------------------------------------------------------------------------------------------------| -| `source_name` | `string` | Source table name | Required | {`{"source_name": "product"}`} | -| `target_name` | `string` | Target table name | Required | {`{"target_name": "product"}`} | -| `aggregates` | `list[Aggregate]` | List of aggregation rules (see [Aggregate](#aggregate)) | Optional | {`{"aggregates": {"type": "MAX", "agg_columns": ["price"]}}`} | -| `join_columns` | `list[string]` | Primary key columns | Optional | {`{"join_columns": ["product_id", "order_id"]}`} | -| `jdbc_reader_options` | `object` | JDBC read parallelization configuration | Optional | {`{"jdbc_reader_options": {"num_partitions": 10, "lower_bound": : 1, upper_bound: 99, "partition_column": "id"}}`}| -| `select_columns` | `list[string]` | Columns to include in reconciliation | Optional | {`{"select_columns": ["id", "name", "price"]}`} | -| `drop_columns` | `list[string]` | Columns to exclude from reconciliation | Optional | {`{"drop_columns": ["temp_sku"]}`} | -| `column_mapping` | `list[ColumnMapping]` | Source-target column mapping (see [column_mapping](#column-mapping)) | Optional | {`{"column_mapping": {"source_name": "id", "target_name": "product_id"}}`} | -| `transformations` | `list[Transformations]` | Column transformation rules (see [transformations](#transformations)) | Optional | {`{"transformations": {"column_name": "address", "source": "TRIM(address)", "target": "LOWER(TRIM(address))"}}`} | -| `column_thresholds` | `list[ColumnThresholds]` | Column-level variance thresholds (see [column_thresholds](#column-thresholds)) | Optional | {`{"column_thresholds": {"column_name": "price", "lower_bound": "-5%", "upper_bound": "+10%"}}`} | -| `table_thresholds` | `list[TableThresholds]` | Table-level mismatch thresholds (see [table_thresholds](#table-thresholds)) | Optional | {`{"table_thresholds": {"model": "mismatch", "lower_bound": "0%", "upper_bound": "5%"}}`} | -| `filters` | `Filters` | Source/target filter expressions | Optional | {`{"filters": {"source": "quantity > 100", "target": "stock_quantity >= 100"}}`} | - -[[back to top](#types-of-report-supported)] - -### JDBC Reader Options - - - - ```python - @dataclass - class JdbcReaderOptions: - num_partitions: int - partition_column: str - lower_bound: str - upper_bound: str - fetchsize: int = 0 - ``` - - - ```json - "jdbc_reader_options": { - "num_partitions": "", - "partition_column": "", - "lower_bound": "", - "upper_bound": "", - "fetchsize": "" - } - ``` - - - -| field_name | data_type | description | required/optional | example_value | -|-------------------|-----------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------|---------------| -| num_partitions | string | the number of partitions for reading input data in parallel | required | "200" | -| partition_column | string | Int/date/timestamp parameter defining the column used for partitioning, typically the primary key of the source table. Note that this parameter accepts only one column, which is especially crucial when dealing with a composite primary key. In such cases, provide the column with higher cardinality. | required | "employee_id" | -| upper_bound | string | integer or date or timestamp without time zone value as string), that should be set appropriately (usually the maximum value in case of non-skew data) so the data read from the source should be approximately equally distributed | required | "1" | -| lower_bound | string | integer or date or timestamp without time zone value as string), that should be set appropriately (usually the minimum value in case of non-skew data) so the data read from the source should be approximately equally distributed | required | "100000" | -| fetchsize | string | This parameter influences the number of rows fetched per round-trip between Spark and the JDBC database, optimising data retrieval performance. Adjusting this option significantly impacts the efficiency of data extraction, controlling the volume of data retrieved in each fetch operation. More details on configuring fetch size can be found [here](https://docs.databricks.com/en/connect/external-systems/jdbc.html#control-number-of-rows-fetched-per-query) | optional(default="0") | "10000" | - - -[[back to top](#types-of-report-supported)] - -## Column Mapping - - - - ```python - @dataclass - class ColumnMapping: - source_name: str - target_name: str - ``` - - - ```json - "column_mapping": [ - { - "source_name": "", - "target_name": "" - } - ] - ``` - - - -| field_name | data_type | description | required/optional | example_value | -|-------------|-----------|--------------------|-------------------|-----------------| -| source_name | string | source column name | required | "dept_id" | -| target_name | string | target column name | required | "department_id" | - -[[back to top](#types-of-report-supported)] - - -## Drop Columns - -For Recon Type `all` or `data`, the `drop_columns` parameter is used to exclude columns from the reconciliation process. -using this config, you can specify the columns that you want to exclude from the reconciliation process. - - - - ```python - Table( - drop_columns = ["column_name1", "column_name2"] - ) - ``` - - - ```json - { - "drop_columns": [ "column_name1", "column_name2"] - } - ``` - - - -[[back to top](#types-of-report-supported)] - - -## Transformations - -You can apply custom transformations to the columns using the `transformations` parameter. If applied, Reconcile -uses these transformations to fetch the data from source and/or target before comparing them. You can write SQL -expressions in `source` & `target` fields to apply transformations. - -The class detail: -```python -@dataclass -class Transformation: - column_name: str - source: str - target: str | None = None - -``` - -Syntax for applying transformations: - - - - ```python - Table( - transformations = [ - Transformation( - column_name = "unit_price", - source = "coalesce(cast(cast(unit_price as decimal(38,10)) as string), '_null_recon_')", - target = "coalesce(cast(format_number(cast(unit_price as decimal(38, 10)), 10) as string), '_null_recon_')" - ) - ] - ) - ``` - - - ```json - "transformations": [ - { - "column_name": "[COLUMN_NAME]", - "source": "[TRANSFORMATION_EXPRESSION]", - "target": "[TRANSFORMATION_EXPRESSION]" - } - ] - ``` - - - -| field_name | data_type | description | required/optional | example_value | -|-------------|-----------|------------------------------------------------------------|-------------------|----------------------------------| -| column_name | string | the column name on which the transformation to be applied | required | "s_address" | -| source | string | the transformation sql expr to be applied on source column | required | "trim(s_address)" or "s_address" | -| target | string | the transformation sql expr to be applied on source column | required | "trim(s_address)" or "s_address" | - - - -:::note -Reconciliation also takes an udf in the transformation expr.Say for eg. we have a udf named sort_array_input() that takes an unsorted array as input and returns an array sorted.We can use that in transformation as below: -``` python -transformations=[Transformation(column_name)="array_col",source=sort_array_input(array_col),target=sort_array_input(array_col)] -``` -`NULL` values are defaulted to `_null_recon_` using the transformation expressions in these files: - -1. [expression_generator.py](https://github.com/databrickslabs/remorph/tree/main/src/databricks/labs/remorph/reconcile/query_builder/expression_generator.py) -2. [sampling_query.py](https://github.com/databrickslabs/remorph/tree/main/src/databricks/labs/remorph/reconcile/query_builder/sampling_query.py). -If User is looking for any specific behaviour, they can override these rules using [transformations](#transformations) accordingly. -::: - -:::danger -### Handling Nulls - -While applying transformations, make sure you handle the nulls explicitly in the transformation column. -While Reconcile takes care of null handling for all the other columns (including join keys and other keys that are not -included in drop_column list), when users introduce transformation columns, Reconcile uses those expressions as is. -So you would have to include null handling in the transformation expression itself. - -So if you are planning on using the below expression: -``` sql -cast(cast(scanout_units as decimal(38,10)) as string) -``` - -Use the below expression instead: - -``` sql -coalesce(cast(cast(scanout_units as decimal(38,10)) as string), '_null_recon_') -``` - - -### Handling Timestamp Columns - -Different systems handle timestamps differently. During Reconciliation it is important that we apply necessary transformation to the timestamp columns -on both source and target to make sure that they are reconciled correctly. A recommended approach to dealing with timestamps is to convert them to -corresponding `unix epoch` string values. SQL Functions like `epoch_millisecond` (Snowflake), [unix_millis](https://docs.databricks.com/aws/en/sql/language-manual/functions/unix_millis#:~:text=Returns%20the%20number%20of%20milliseconds,00%3A00%3A00%20UTC%20.) (Databricks) become very handy to transform -those timestamp values to unix epoch values. Then they can be converted to string which is safer when it comes to comparing timestamps across different systems. - - - - ```python - Transformation( - column_name= 'UPDATE_TIMESTAMP', - source= "coalesce(cast(EXTRACT(epoch_millisecond FROM UPDATE_TIMESTAMP) as string), '_null_recon_')", - target= "coalesce(cast(unix_millis(UPDATE_TIMESTAMP) as string), '_null_recon_')" - ), - ``` - - - ```json - "transformations": [ - { - "column_name": "UPDATE_TIMESTAMP", - "source": "coalesce(cast(EXTRACT(epoch_millisecond FROM UPDATE_TIMESTAMP) as string), '_null_recon_')", - "target": "coalesce(cast(unix_millis(UPDATE_TIMESTAMP) as string), '_null_recon_')" - } - ] - ``` - - -::: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Transformation Expressions
filenamefunction / variabletransformation_ruledescription
sampling_query.py_get_join_clausetransform(coalesce, default="_null_recon_", is_string=True)Applies the coalesce transformation function for String column and defaults to `_null_recon_` if column is NULL
expression_generator.pyDataType_transform_mapping(coalesce, default='_null_recon_', is_string=True)Default String column Transformation rule for all dialects. Applies the coalesce transformation function and defaults to `_null_recon_` if column is NULL
expression_generator.pyDataType_transform_mapping"oracle": DataType...NCHAR: ..."NVL(TRIM(TO_CHAR..,'_null_recon_')"Transformation rule for oracle dialect 'NCHAR' datatype. Applies TO_CHAR, TRIM transformation functions. If column is NULL, then defaults to `_null_recon_`
expression_generator.pyDataType_transform_mapping"oracle": DataType...NVARCHAR: ..."NVL(TRIM(TO_CHAR..,'_null_recon_')"Transformation rule for oracle dialect 'NVARCHAR' datatype. Applies TO_CHAR, TRIM transformation functions. If column is NULL, then defaults to `_null_recon_`
- -:::tip - -If you are applying any transformation for generating the Reconciliation report, it is recommended that you run -the queries with transformation expressions both on Source and target to ensure, they are syntactically and semantically -giving correct result. - -If you need help with the exact reconciliation queries that are getting generated after you apply the transformations, -you may search the reconciliation log with the following string: - -`“Hash query for [source or target]”` - -::: - -[[back to Transformations](#transformations)] - -[[back to top](#types-of-report-supported)] - -## Column Thresholds - - - - ```python - @dataclass - class ColumnThresholds: - column_name: str - lower_bound: str - upper_bound: str - type: str - ``` - - - ```json - "column_thresholds": [ - { - "column_name": "COLUMN_NAME", - "lower_bound": "LOWER_BOUND", - "upper_bound": "UPPER_BOUND", - "type": "DATA_TYPE" - } - - ] - ``` - - - - -| field_name | data_type | description | required/optional | example_value | -|-------------|-----------|-----------------------------------------------------------------------------------------------------------------|-------------------|--------------------| -| column_name | string | the column that should be considered for column threshold reconciliation | required | "product_discount" | -| lower_bound | string | the lower bound of the difference between the source value and the target value | required | "-5%" | -| upper_bound | string | the upper bound of the difference between the source value and the target value | required | "5%" | -| type | string | The user must specify the column type. Supports SQLGLOT `DataType.NUMERIC_TYPES` and `DataType.TEMPORAL_TYPES`. | required | "int" | - -[[back to top](#types-of-report-supported)] - -## Table Thresholds - - - - ```python - @dataclass - class TableThresholds: - lower_bound: str - upper_bound: str - model: str - ``` - - - ```json - "table_thresholds": [ - { - "lower_bound": "LOWER_BOUND", - "upper_bound": "UPPER_BOUND", - "model": "MODEL" - } - ] - ``` - - - -* The threshold bounds for the table must be non-negative, with the lower bound not exceeding the upper bound. - -| field_name | data_type | description | required/optional | example_value | -|-------------|-----------|------------------------------------------------------------------------------------------------------|-------------------|---------------| -| lower_bound | string | the lower bound of the difference between the source mismatch and the target mismatch count | required | 0% | -| upper_bound | string | the upper bound of the difference between the source mismatch and the target mismatch count | required | 5% | -| model | string | The user must specify on which table model it should be applied; for now, we support only "mismatch" | required | int | - -[[back to top](#types-of-report-supported)] - -## Filters - - - - ```python - @dataclass - class Filters: - source: str - target: str - ``` - - - ```json - "filters": { - "source": "FILTER_EXPRESSION", - "target": "FILTER_EXPRESSION" - } - ``` - - - -| field_name | data_type | description | required/optional | example_value | -|------------|-----------|---------------------------------------------------|------------------------|--------------------------------------------------------------------| -| source | string | the sql expression to filter the data from source | optional(default=None) | "lower(dept_name)='finance'" | -| target | string | the sql expression to filter the data from target | optional(default=None) | "lower(dept_name)='finance'" | - - - - -:::note Key Considerations: - -1. The column names are always converted to lowercase and considered for reconciliation. -2. Currently, it doesn't support case insensitivity and doesn't have collation support -3. Table Transformation internally considers the default value as the column value. It doesn't apply any default -transformations -if not provided. -```eg:Transformation(column_name="address",source_name=None,target_name="trim(s_address)")``` -For the given example, -the source transformation is None, so the raw value in the source is considered for reconciliation. -4. If no user transformation is provided for a given column in the configuration by default, depending on the source -data -type, our reconciler will apply -default transformation on both source and target to get the matching hash value in source and target. Please find the -detailed default transformations here. -5. Always the column reference to be source column names in all the configs, except **Transformations** and **Filters** -as these are dialect-specific SQL expressions that are applied directly in the SQL. -6. **Transformations** and **Filters** should always be in their respective dialect SQL expressions, and the -reconciler will not apply any logic -on top of this. -::: - -[[back to top](#types-of-report-supported)] - -## Report Type-Flow Chart ---- -```mermaid -flowchart TD - REPORT_TYPE --> DATA - REPORT_TYPE --> SCHEMA - REPORT_TYPE --> ROW - REPORT_TYPE --> ALL -``` - ---- - -```mermaid -flowchart TD - SCHEMA --> SCHEMA_VALIDATION -``` ---- - -```mermaid -flowchart TD - ROW --> MISSING_IN_SRC - ROW --> MISSING_IN_TGT -``` ---- - -```mermaid -flowchart TD - DATA --> MISMATCH_ROWS - DATA --> MISSING_IN_SRC - DATA --> MISSING_IN_TGT -``` ---- - -```mermaid -flowchart TD - ALL --> MISMATCH_ROWS - ALL --> MISSING_IN_SRC - ALL --> MISSING_IN_TGT - ALL --> SCHEMA_VALIDATION -``` ---- - -[[back to top](#types-of-report-supported)] - -## Aggregates Reconciliation - -Aggregates Reconcile is an utility to streamline the reconciliation process, specific aggregate metric is compared -between source and target data residing on Databricks. - -### Summary - -| operation_name | sample visualisation | description | key outputs captured in the recon metrics tables | -|--------------------------|-------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **aggregates-reconcile** | `[data]({useBaseUrl('docs/reconcile/aggregates_reconcile_visualisation.md#data')})` | reconciles the data for each aggregate metric `join_columns` are used to identify the mismatches at aggregated metric level | mismatch_data(sample data with mismatches captured at aggregated metric level ) missing_in_src(sample rows that are available in target but missing in source) missing_in_tgt(sample rows that are available in source but are missing in target) | - - -### Supported Aggregate Functions - - -| Aggregate Functions | -|-----------------------------------------------------------------------------------------------------------------------------------------------------------| -| **min** | -| **max** | -| **count** | -| **sum** | -| **avg** | -| **mean** | -| **mode** | -| **stddev** | -| **variance** | -| **median** | - - - -[[back to aggregates-reconciliation](#aggregates-reconciliation)] - -[[back to top](#types-of-report-supported)] - -### Aggregate - - - - ```python - @dataclass - class Aggregate: - agg_columns: list[str] - type: str - group_by_columns: list[str] | None = None - ``` - - - ```json - { - "type": "MIN", - "agg_columns": ["COLUMN_NAME_3"], - "group_by_columns": ["GROUP_COLUMN_NAME"] - } - ``` - - - -| field_name | data_type | description | required/optional | example_value | -|------------------|--------------|-----------------------------------------------------------------------|------------------------|------------------------| -| type | string | [Supported Aggregate Functions](#supported-aggregate-functions) | required | MIN | -| agg_columns | list[string] | list of columns names on which aggregate function needs to be applied | required | ["product_discount"] | -| group_by_columns | list[string] | list of column names on which grouping needs to be applied | optional(default=None) | ["product_id"] or None | - - - -[[back to aggregates-reconciliation](#aggregates-reconciliation)] - -[[back to top](#types-of-report-supported)] - - -### TABLE Config Examples: -Please refer [TABLE Config Elements](#table-config-elements) for Class and JSON configs. - - - - ```python - Table( - source_name="SOURCE_NAME", - target_name="TARGET_NAME", - join_columns=["COLUMN_NAME_1", "COLUMN_NAME_2"], - aggregates=[ - Aggregate( - agg_columns=["COLUMN_NAME_3"], - type="MIN", - group_by_columns=["GROUP_COLUMN_NAME"] - ), - Aggregate( - agg_columns=["COLUMN_NAME_4"], - type="MAX" - ) - ] - ) - ``` - - - ```json - { - "source_name": "SOURCE_NAME", - "target_name": "TARGET_NAME", - "join_columns": ["COLUMN_NAME_1", "COLUMN_NAME_2"], - "aggregates": [ - { - "type": "MIN", - "agg_columns": ["COLUMN_NAME_3"], - "group_by_columns": ["GROUP_COLUMN_NAME"] - }, - { - "type": "MAX", - "agg_columns": ["COLUMN_NAME_4"] - } - ] - } - ``` - - - - -:::note -Key Considerations: - -1. The aggregate column names, group by columns and type are always converted to lowercase and considered for reconciliation. -2. Currently, it doesn't support aggregates on window function using the OVER clause. -3. It doesn't support case insensitivity and does not have collation support -4. The queries with “group by” column(s) are compared based on the same group by columns. -5. The queries without “group by” column(s) are compared row-to-row. -6. Existing features like `column_mapping`, `transformations`, `JDBCReaderOptions` and `filters` are leveraged for the aggregate metric reconciliation. -7. Existing `select_columns` and `drop_columns` are not considered for the aggregate metric reconciliation. -8. Even though the user provides the `select_columns` and `drop_columns`, those are not considered. -9. If Transformations are defined, those are applied to both the “aggregate columns” and “group by columns”. -::: - -[[back to aggregates-reconciliation](#aggregates-reconciliation)] - -[[back to top](#types-of-report-supported)] - -### Flow Chart - -```mermaid -flowchart TD - Aggregates-Reconcile --> MISMATCH_ROWS - Aggregates-Reconcile --> MISSING_IN_SRC - Aggregates-Reconcile --> MISSING_IN_TGT -``` - - -[[back to aggregates-reconciliation](#aggregates-reconciliation)] - -[[back to top](#types-of-report-supported)] - - - - diff --git a/docs/lakebridge/docs/reconcile/recon_notebook.mdx b/docs/lakebridge/docs/reconcile/running.mdx similarity index 53% rename from docs/lakebridge/docs/reconcile/recon_notebook.mdx rename to docs/lakebridge/docs/reconcile/running.mdx index 0a60249f7b..756c0549a4 100644 --- a/docs/lakebridge/docs/reconcile/recon_notebook.mdx +++ b/docs/lakebridge/docs/reconcile/running.mdx @@ -1,6 +1,6 @@ --- sidebar_position: 2 -title: Running Reconcile on Notebook +title: Running Reconcile --- import useBaseUrl from '@docusaurus/useBaseUrl'; @@ -8,23 +8,32 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import CodeBlock from '@theme/CodeBlock'; +# Running Reconcile -This page gives you a comprehensive guide on how to configure and run the Reconcile utility of Lakebridge on your Databricks -workspace using Databricks notebook. +This page covers how to run Lakebridge Reconcile via the CLI and via Databricks notebooks. For configuration options, see [Configuration Reference](/docs/reconcile/configuration). -## Installation +--- + +## CLI Execution + +After completing [setup](/docs/reconcile#setup), run reconciliation with: -Before running the Reconcile utility, you need to install the required packages. You can install the required packages -using the following command: +```bash +databricks labs lakebridge reconcile +``` + +Results are written to the reconciliation dashboard deployed in your workspace. + +--- + +## Notebook Execution + +Use the notebook approach when you need fine-grained control over reconcile configuration, or when running from within a Databricks notebook workflow. + +### Step 1: Install - - ```python - %pip install git+https://github.com/databrickslabs/lakebridge - dbutils.library.restartPython() - ``` - - + ```python %pip install databricks-labs-lakebridge dbutils.library.restartPython() @@ -32,10 +41,7 @@ using the following command: - -## Imports - -Import all the necessary modules. +### Step 2: Import ```python from databricks.sdk import WorkspaceClient @@ -61,12 +67,10 @@ from databricks.labs.lakebridge.reconcile.trigger_recon_aggregate_service import from databricks.labs.lakebridge.reconcile.exception import ReconciliationException ``` -## Configure Reconcile Properties - -We use the class `ReconcileConfig` to configure the properties required for reconciliation. +### Step 3: Configure ReconcileConfig +#### __Before the actual example, some details about how to configure:__ ```python -@dataclass class ReconcileConfig: report_type: str source: SourceConnectionConfig @@ -81,6 +85,7 @@ Parameters: - `catalog`: The source database/catalog name. catalog is used for consistency in naming - `schema`: The source schema name. - `uc_connection_name`: the connection name for the source as configured in workspace `Connections`. Not allowed for `databricks` + ```python @dataclass class SourceConnectionConfig: @@ -113,17 +118,10 @@ If not set the default values will be used to store the metadata. The default re of Lakebridge. -An Example of configuring the Reconcile properties: +#### __Now, an Example of configuring the ReconcileConfig properties that you can copy into the notebook:__ ```python -from databricks.labs.lakebridge.config import ( - ReconcileConfig, - ReconcileMetadataConfig, - SourceConnectionConfig, - TargetConnectionConfig, -) - -ReconcileConfig( +reconcile_config = ReconcileConfig( report_type="all", source=SourceConnectionConfig( dialect="snowflake", @@ -142,17 +140,7 @@ ReconcileConfig( ) ``` -## Configure Table Properties - -We use the class `TableRecon` to configure the tables to be reconciled. - -```python -@dataclass -class TableRecon: - tables: list[Table] -``` - -An Example Table properties for reconciliation: +### Step 4: Configure TableRecon ```python from databricks.labs.lakebridge.config import TableRecon @@ -172,97 +160,66 @@ table_recon = TableRecon( Table( source_name="source_table_name", target_name="target_table_name", - join_columns= ["store_id", "account_id"], # List of columns to join the source and target tables. + join_columns=["store_id", "account_id"], column_mapping=[ ColumnMapping(source_name="dept_id", target_name="department_id"), - ColumnMapping(source_name="cty_cd", target_name="country_code") ], column_thresholds=[ ColumnThresholds(column_name="unit_price", upper_bound="-5", lower_bound="5", type="float") ], - table_thresholds=[ - TableThresholds(lower_bound="0%", upper_bound="5%", model="mismatch") - ], + table_thresholds=[ + TableThresholds(lower_bound="0%", upper_bound="5%", model="mismatch") + ], transformations=[ Transformation( - column_name= 'inventory_units', - source= "coalesce(cast( cast(inventory_units as decimal(38,10)) as string),'_null_recon_')", - target= 'coalesce(replace(cast(format_number(cast(inventory_units as decimal(38, 10)), 10) as string),",", ""),"_null_recon_")', - ) - , - Transformation( - column_name= 'scanout_dollars', - source= "coalesce(cast( cast(scanout_dollars as decimal(38,10)) as string) ,'_null_recon_')", - target= 'coalesce(replace(cast(format_number(cast(scanout_dollars as decimal(38, 10)), 10) as string),",", ""),"_null_recon_")', - ) + column_name="inventory_units", + source="coalesce(cast(cast(inventory_units as decimal(38,10)) as string), '_null_recon_')", + target='coalesce(replace(cast(format_number(cast(inventory_units as decimal(38, 10)), 10) as string), ",", ""), "_null_recon_")' + ) ], jdbc_reader_options=JdbcReaderOptions( num_partitions=50, partition_column="lct_nbr", lower_bound="1", - upper_bound="50000", + upper_bound="50000" ), - aggregates=[ - Aggregate(agg_columns=["inventory_units"], type="count"), - Aggregate(agg_columns=["unit_price"], type="min"), - Aggregate(agg_columns=["unit_price"], type="max") - ], - filters= Filters( - source="lower(dept_name)='finance'", - target="lower(department_name)='finance'") + filters=Filters( + source="lower(dept_name)='finance'", + target="lower(department_name)='finance'" + ) ) ] ) - ``` -## Configure Dashboards using Notebook - -The recommended way is to use the CLI following the instructions [here](/installation.mdx#configure-reconcile). -However, if you want to configure the metadata tables and dashboards using a notebook, you can use the following notebook. - - - Open notebook in new tab - - -## Run Reconcile -To run Reconcile on the configured properties, use the `recon` method. you also need to pass a `WorkspaceClient` to `recon`. +### Step 5: Run ```python from databricks.labs.lakebridge import __version__ from databricks.sdk import WorkspaceClient - from databricks.labs.lakebridge.reconcile.trigger_recon_service import TriggerReconService from databricks.labs.lakebridge.reconcile.exception import ReconciliationException ws = WorkspaceClient(product="lakebridge", product_version=__version__) - try: - result = TriggerReconService.trigger_recon( - ws = ws, - spark = spark, # notebook spark session - table_recon = table_recon, # previously created - reconcile_config = reconcile_config # previously created - ) - print(result.recon_id) - print(result) - print("***************************") + result = TriggerReconService.trigger_recon( + ws=ws, + spark=spark, + table_recon=table_recon, + reconcile_config=reconcile_config + ) + print(result.recon_id) + print(result) except ReconciliationException as e: - recon_id = e.reconcile_output.recon_id - print(f" Failed : {recon_id}") + print(f"Failed: {e.reconcile_output.recon_id}") print(e) - print("***************************") -except Exception as e: - print(e.with_traceback) - raise e - print(f"Exception : {str(e)}") - print("***************************") - ``` -## Visualization +### Visualization + +After running, use the `recon_id` to drill into the results on the AI/BI Dashboard deployed in your workspace during installation. -When you [install](../installation.mdx) Lakebridge using databricks cli, it deploys an AI/BI Dashboard on your workspace. -This dashboard gives you a detailed report of all the reconciliation runs on your workspace. After every reconciliation run, you get a -`recon_id`. You can use this recon_id to drill down into the details of that particular reconciliation run on the dashboard. +:::note +Improved bulk automation tooling is in development. Check back for updates. +::: diff --git a/docs/lakebridge/docs/sql_splitter.mdx b/docs/lakebridge/docs/sql_splitter.mdx index f498fa1902..54b16fa604 100644 --- a/docs/lakebridge/docs/sql_splitter.mdx +++ b/docs/lakebridge/docs/sql_splitter.mdx @@ -1,5 +1,5 @@ --- -sidebar_position: 9 +sidebar_position: 7 title: SQL Splitter --- import useBaseUrl from '@docusaurus/useBaseUrl'; diff --git a/docs/lakebridge/docs/transpile/index.mdx b/docs/lakebridge/docs/transpile/index.mdx index ddd87568a0..dc369830b5 100644 --- a/docs/lakebridge/docs/transpile/index.mdx +++ b/docs/lakebridge/docs/transpile/index.mdx @@ -1,11 +1,35 @@ --- -sidebar_position: 4 +sidebar_position: 5 title: Transpile Guide --- import useBaseUrl from '@docusaurus/useBaseUrl'; # Transpile Guide +Lakebridge comes with three transpilation plugins: [Morpheus](/docs/transpile/pluggable_transpilers/morpheus), [BladeBridge](/docs/transpile/pluggable_transpilers/bladebridge), and [Switch](/docs/transpile/pluggable_transpilers/switch). Not sure which to use? See [Which Tool Do I Use?](/docs/choosing_tools#transpiler-bladebridge-vs-morpheus-vs-switch). + +## Transpiler Selection Guide + +Lakebridge offers both deterministic and LLM-powered transpilers, each optimized for different conversion scenarios. + +### Deterministic Conversion (BladeBridge & Morpheus) + +Deterministic transpilers excel in scenarios requiring consistency and speed: +- **Deterministic output with guaranteed syntax equivalence** — Every conversion produces the same predictable result +- **High-volume batch processing** — Efficiently handle thousands of files without API rate limits +- **Fast local execution without API dependencies** — Sub-minute processing with no external service calls +- **Production-grade SQL aligned with Databricks SQL evolution** — Leverages SQL Scripting, Stored Procedures, and latest DBSQL features + +### LLM-Powered Conversion (Switch) + +Switch is best suited for scenarios requiring semantic understanding and flexibility: +- **Complex logic requiring contextual understanding** — Stored procedures and business logic where intent matters more than syntax +- **Source formats not covered by deterministic transpilers** — Any SQL dialect or programming language through custom prompts +- **Extensible conversion through custom YAML prompts** — Adapt to proprietary or uncommon source formats +- **Python notebook output for SQL beyond ANSI SQL/PSM standards** — Complex transformations that benefit from procedural code + +--- + ## Supported dialects @@ -132,11 +156,28 @@ import useBaseUrl from '@docusaurus/useBaseUrl';
+## Execution Pre-Set Up +When you run `install-transpile`, you will be prompted for settings to use when transpiling your sources. You can +choose to provide these at the time of installation, or to provide them later as arguments when transpiling. + +The `transpile` command will trigger the conversion of the specified code. These settings provided during `install-transpile` can be overridden (or provided if unavailable) using the command-line options: + - `input-source`: The local filesystem path to the sources that should be transpiled. This must be provided if not set during `install-transpile`. + - `output-folder`: The local filesystem path where converted code will be written. This must be provided if not set during `install-transpile`. + - `source-dialect`: Dialect name (ex: snowflake, oracle, datastage, etc). This must be provided if not set during `install-transpile`. + - `overrides-file`: An optional local path to a JSON file containing custom overrides for the transpilation process, if the underlying transpiler supports this. (Refer to [this documentation](pluggable_transpilers/bladebridge/bladebridge_configuration) for more details on custom overrides.) + - `target-technology`: The target technology to use for conversion output. This must be provided if not set during `install-transpile` and the underlying transpiler requires it for the source dialect in use. +- `error-file-path`: The path to the file where a log of conversion errors will be stored. If not provided here or during `install-transpile` no error log will be written. + - `skip-validation`: Whether the transpiler will skip the validation of transpiled SQL sources. If not provided here or during `install-transpile` validation will be attempted by default. + - `catalog-name`: The name of the catalog in Databricks to use when validating transpiled SQL sources. If not specified, `remorph` will be used as the default catalog. + - `schema-name`: The name of the schema in Databricks to use when validating transpiled SQL sources. If not specified, `transpiler` will be used as the default schema. + - `transpiler-config-path`: This path of the configuration file for the transpiler to use for conversion. This is normally inferred from the source dialect or chosen during `install-transpile` if multiple transpilers support the source dialect. + + ## Verify Installation Verify the successful installation by executing the provided command; confirmation of a successful installation is indicated when the displayed output aligns with the example screenshot provided: Command: -```shell +```bash databricks labs lakebridge transpile --help ``` @@ -165,25 +206,9 @@ Global Flags: -t, --target string bundle target to use (if applicable) ``` -## Execution Pre-Set Up -When you run `install-transpile`, you will be prompted for settings to use when transpiling your sources. You can -choose to provide these at the time of installation, or to provide them later as arguments when transpiling. - -The `transpile` command will trigger the conversion of the specified code. These settings provided during `install-transpile` can be overridden (or provided if unavailable) using the command-line options: - - `input-source`: The local filesystem path to the sources that should be transpiled. This must be provided if not set during `install-transpile`. - - `output-folder`: The local filesystem path where converted code will be written. This must be provided if not set during `install-transpile`. - - `source-dialect`: Dialect name (ex: snowflake, oracle, datastage, etc). This must be provided if not set during `install-transpile`. - - `overrides-file`: An optional local path to a JSON file containing custom overrides for the transpilation process, if the underlying transpiler supports this. (Refer to [this documentation](pluggable_transpilers/bladebridge/bladebridge_configuration) for more details on custom overrides.) - - `target-technology`: The target technology to use for conversion output. This must be provided if not set during `install-transpile` and the underlying transpiler requires it for the source dialect in use. -- `error-file-path`: The path to the file where a log of conversion errors will be stored. If not provided here or during `install-transpile` no error log will be written. - - `skip-validation`: Whether the transpiler will skip the validation of transpiled SQL sources. If not provided here or during `install-transpile` validation will be attempted by default. - - `catalog-name`: The name of the catalog in Databricks to use when validating transpiled SQL sources. If not specified, `remorph` will be used as the default catalog. - - `schema-name`: The name of the schema in Databricks to use when validating transpiled SQL sources. If not specified, `transpiler` will be used as the default schema. - - `transpiler-config-path`: This path of the configuration file for the transpiler to use for conversion. This is normally inferred from the source dialect or chosen during `install-transpile` if multiple transpilers support the source dialect. - ## Execution Execute the below command to initialize the transpile process passing the arguments to the command directly in the call. -```shell +```bash databricks labs lakebridge transpile --transpiler-config-path --input-source --source-dialect --output-folder --skip-validation --catalog-name --schema-name ``` transpile-run diff --git a/docs/lakebridge/docs/transpile/overview.mdx b/docs/lakebridge/docs/transpile/overview.mdx deleted file mode 100644 index 1739202098..0000000000 --- a/docs/lakebridge/docs/transpile/overview.mdx +++ /dev/null @@ -1,170 +0,0 @@ ---- -sidebar_position: 1 -title: Transpilation overview ---- - -Lakebridge's transpilation module offers a pluggable mechanism for transpiling SQL files written in any supported source dialect into their equivalent in Databricks SQL. - -### Design Flow: -```mermaid -flowchart TD - A(Transpile CLI) e1@==> |Directory| B[Transpile All Files In Directory]; - A e2@==> |File| C[Transpile Single File] ; - B e3@==> D[List Files]; - C e4@==> E("LSP Pluggable Transpiler"); - D e5@==> E - E e6@==> |Parse Error| F(Failed Queries) - E e7@==> G{Skip Validations} - G e8@==> |Yes| H(Save Output) - G e9@==> |No| I{Validate} - I e10@==> |Success| H - I e11@==> |Fail| J(Flag, Capture) - J e12@==> H - e1@{ animate: true } - e2@{ animate: true } - e3@{ animate: true } - e4@{ animate: true } - e5@{ animate: true } - e6@{ animate: true } - e7@{ animate: true } - e8@{ animate: true } - e9@{ animate: true } - e10@{ animate: true } - e11@{ animate: true } - e12@{ animate: true } -``` - -When invoked with the following command line: - -```shell -databricks labs lakebridge transpile --source-dialect snowflake --input-source /path/to/input --output-folder /path/to/output -``` - -Lakebridge will (recursively) scan `/path/to/input`, looking for `.sql` files, and for each such file it will produce a corresponding file (with the same relative path) under `/path/to/output`. - -If for example we have the following `/path/to/input/foo/bar/silly.sql` file: - -```sql -SELECT 1; -``` - -The transpiled file will be generated at `/path/to/output/foo/bar/silly.sql`: - -```sql -/* - Successfully transpiled from /path/to/input/foo/bar/silly.sql -*/ -SELECT 1; -``` - -Notice how thee transpiler adds a header comment at the top of the transpile file. This header isn't very useful in such successful case, but it may come in handy when something went wrong during transpilation. - -As a matter of fact, transpilation could fail for various reasons: -- the input code may contain constructs that have no equivalent in Databricks SQL, -- the selected transpilation plugin may be missing an implementation (or have a bug), -- less likely, the input code may be incorrect, making the transpilation plugin unable to parse it (Lakebridge is built on the assumption that the code you feed it is correct). - -In such cases, the header comment will contain the list of errors encountered during transpilation in an attempt at making it easier for you to fix the transpiled code manually. - -Let see an example: - -```sql -/* - Failed transpilation of /path/to/input/broken.sql - - The following errors were found while transpiling: - - [3:3] Function GIBBERISH cannot be translated to Databricks SQL -*/ -SELECT - t1.name, - GIBBERISH(t1.comment) -FROM - t1; -``` - -Here the input code uses the (made up) `GIBBERISH` function that has no counterpart in Databricks SQL. The error is reported in the header comment with a specific position: `[3:5]` tells you that the problematic expression starts at the fifth character of the third line. - -It is important to note that the transpiler isn't always able to know whether it can't translate a given function because such translation isn't possible or because it is lacking the implementation of said translation. When reviewing such errors, if you think that it is the latter case, please file a bug report/feature request. - -## Plugins overview - -Out of the box, Lakebridge comes with two transpilation plugins: Morpheus and BladeBridge. Each one has its own set of capabilities and levels of guarantee that we describe below. - -### Morpheus - -Morpheus aims at offering strong guarantees about the code it produces. A file successfully transpiled by Morpheus (without any error or warning) is supposed to yield a result on Databricks that's equivalent[^1] to the result of running the corresponding input file on the source platform. - -As a consequence, whenever it finds itself unable to commit on such guarantee, Morpheus will emit either an error (when it knows that equivalent results cannot be guaranteed) or a warning (when it couldn't make sure that the guarantee holds). Being very conservative about this guarantee, Morpheus may emit warnings that are in fact false-negatives. In other words, a file that's successfully transpiled with warnings may still be correct and produce a result that's equivalent to its corresponding input on the source platform, but Morpheus was unable to guarantee it. - -Internally, Morpheus is built upon a custom parser (generated from a carefully crafted ANTLR grammar). The input text is first parsed into an intermediate representation tree. That tree then undergoes a series of transformations where structures that are specific to the input dialect are transformed to their Databricks equivalent. Finally, the transformed tree is fed to a code generator that produces and formats the final Databricks SQL text. - -Morpheus accumulates every error or warning encountered during the transformation and generation phases but continues processing the file. That way, even in the presence of errors, it keeps producing as much meaningful output as possible. - -However, errors encountered during the parsing phase, because they prevent it from building the initial tree, make it unable to produce any meaningful output. In such a case, the output file will contain the original text, along with the parsing errors. Such parsing error are very unlikely to happen though and should be considered as a bug. - -Finally, being built on the assumption that the input SQL is correct, Morpheus doesn't perform any semantical validation on the input code (such as type-checking). Therefore, the behaviour of code transpiled from a semantically incorrect input is unspecified. - - -[^1]: some statistical functions may be implemented on Databricks using an algorithm that slightly differs from the one used on the source platform, making the respective results non strictly identical to one another (although the differences wouldn't be statistically significant). Also, if no ordering is specified, rows may come in different order on Databricks and the source platform. In every other case, both results should be strictly identical. - -### BladeBridge - -**BladeBridge** is a flexible and extensible code conversion engine designed to accelerate modernization to Databricks. It supports a wide range of **ETL platforms** (e.g., DataStage) and **SQL-based systems** (e.g., Oracle, Teradata, Netezza), accepting inputs in the form of exported metadata and scripts. - -The converter generates **Databricks-compatible outputs** including: - -- **Notebooks** – with ETL logic translated into code cells -- **Table & view definitions** -- **Stored procedures** -- **Spark SQL** -- **PySpark scripts** -- **Workflow definitions** - -Internally, the execution is driven by **configuration files**, allowing users to define: - -- Source/target mappings -- Naming conventions -- Transformation preferences -- Output formats - -BladeBridge is fully **extensible** — new converters can be added, and the output structure can be **customized** to meet project-specific requirements. - -Whether you're migrating a single job or thousands, BladeBridge delivers predictable, repeatable results optimized for the Databricks ecosystem. - -More details about the BladeBridge converter [here](/docs/transpile/pluggable_transpilers/bladebridge) - -### Switch - -Switch is an LLM-powered Lakebridge transpiler that extends beyond SQL to convert various source formats into Databricks-compatible outputs. Using [Mosaic AI Model Serving](https://docs.databricks.com/aws/en/machine-learning/model-serving/), it understands code intent and semantics to handle complex transformations. - -Key characteristics: -- **LLM-powered extensibility** - Built-in prompts for multiple SQL dialects (T-SQL, Snowflake, Teradata, Oracle, etc.) and generic formats (Python, Scala, Airflow) -- **Custom prompt support** - Add YAML prompts to handle any source format not covered by built-in options -- **Flexible outputs** - Python notebooks with Spark SQL (default), SQL notebooks, or any text-based format -- **Databricks-native processing** - Executes as scalable Lakeflow Jobs with serverless compute -- **Model flexibility** - Uses Databricks Foundation Model APIs with configurable endpoint selection -- **Complex logic handling** - Excels at stored procedures and business logic beyond ANSI SQL/PSM standards - -Switch's LLM-based approach enables support for any source format through custom prompts. For built-in prompt details and custom prompt creation, see the [Switch documentation](/docs/transpile/pluggable_transpilers/switch). - ---- -Transpiler Selection Guide ---- - -Lakebridge offers both deterministic and LLM-powered transpilers out of the box, each optimized for different conversion scenarios. - -### Deterministic Conversion (BladeBridge & Morpheus) - -Deterministic transpilers excel in scenarios requiring consistency and speed: -- **Deterministic output with guaranteed syntax equivalence** - Every conversion produces the same predictable result -- **High-volume batch processing** - Efficiently handle thousands of files without API rate limits -- **Fast local execution without API dependencies** - Sub-minute processing with no external service calls -- **Production-grade SQL aligned with Databricks SQL evolution** - Leverages SQL Scripting, Stored Procedures, and latest DBSQL features - -### LLM-Powered Conversion (Switch) - -Switch is best suited for scenarios requiring semantic understanding and flexibility: -- **Complex logic requiring contextual understanding** - Stored procedures and business logic where intent matters more than syntax -- **Source formats not covered by deterministic transpilers** - Any SQL dialect or programming language through custom prompts -- **Extensible conversion through custom YAML prompts** - Adapt to proprietary or uncommon source formats -- **Python notebook output for SQL beyond ANSI SQL/PSM standards** - Complex transformations that benefit from procedural code diff --git a/docs/lakebridge/docs/transpile/pluggable_transpilers/bladebridge/bladebridge_configuration.mdx b/docs/lakebridge/docs/transpile/pluggable_transpilers/bladebridge/bladebridge_configuration.mdx index 731d478614..e6c0508e6a 100644 --- a/docs/lakebridge/docs/transpile/pluggable_transpilers/bladebridge/bladebridge_configuration.mdx +++ b/docs/lakebridge/docs/transpile/pluggable_transpilers/bladebridge/bladebridge_configuration.mdx @@ -7,7 +7,61 @@ import CodeBlock from '@theme/CodeBlock'; import Admonition from '@theme/Admonition'; -## Overview +## Configuration Overview + +BladeBridge uses a **two-layer configuration model**: + +| Layer | What it controls | How to set it | +|---|---|---| +| **Layer 1 — CLI parameters** | Source dialect, target output format, override file path | Set via `install-transpile` prompts or `transpile` CLI flags | +| **Layer 2 — JSON override files** | Conversion rules, naming conventions, source/target mappings | Create a custom JSON file and reference it via `--overrides-file` | + +You can use Layer 1 alone (most migrations), or combine both layers for advanced customization. + +--- + +## Layer 1: CLI Parameters + +These parameters are set when you run `install-transpile` (or passed directly to the `transpile` command): + +| Parameter | Required | Default | Description | +|---|---|---|---| +| `source-dialect` | Yes | `ansi` | Source SQL or ETL dialect (e.g., `mssql`, `oracle`, `datastage`, `ssis`) | +| `target-technology` | No | `SQL` | Output format: `SQL`, `SPARKSQL`, or `PYSPARK` | +| `overrides-file` | No | none | Path to a custom JSON config file (Layer 2) | + +### Dialect-specific `target-technology` constraints + +The `target-technology` prompt only appears for ETL source dialects. SQL dialects always output Databricks SQL: + +| Dialect | `target-technology` prompt? | Available choices | +|---|---|---| +| `datastage` | Yes | `SPARKSQL`, `PYSPARK` | +| `ssis` | Yes | `SPARKSQL` only | +| `mssql`, `oracle`, `synapse`, `netezza`, `redshift`, `teradata` | No | Outputs Databricks SQL only | + +### Minimal working example + +```bash +databricks labs lakebridge transpile \ + --source-dialect oracle \ + --input-source /path/to/sql \ + --output-folder /path/to/output +``` + +For ETL sources with target technology: + +```bash +databricks labs lakebridge transpile \ + --source-dialect datastage \ + --input-source /path/to/datastage/exports \ + --output-folder /path/to/output \ + --target-technology SPARKSQL +``` + +--- + +## Layer 2: JSON Override Files The **BladeBridge** transpiler relies heavily on rules defined inside configuration files provided with the converter. These configurations are comprised of a set of layered json files and code templates that drive the generation of output @@ -579,7 +633,7 @@ This section holds templates on read and write instructions for each system clas ### Native Database Connection Support -When converting ETL components that connect to external databases (e.g., Oracle, SQL Server, Redshift, Synapse), you can configure the converter to use native JDBC/ODBC connections instead of Databricks native connectors. This is useful when you need direct database access or when migrating from platforms like Informatica Cloud or DataStage. +When converting ETL components that connect to external databases (e.g., Oracle, SQL Server, Redshift, Synapse), you can configure the converter to use native JDBC/ODBC connections instead of Databricks native connectors. This is useful when you need direct database access or when migrating from platforms like DataStage. #### Configuration Flags diff --git a/docs/lakebridge/docs/transpile/pluggable_transpilers/morpheus/index.mdx b/docs/lakebridge/docs/transpile/pluggable_transpilers/morpheus/index.mdx new file mode 100644 index 0000000000..b0e6c1356e --- /dev/null +++ b/docs/lakebridge/docs/transpile/pluggable_transpilers/morpheus/index.mdx @@ -0,0 +1,135 @@ +--- +title: Morpheus +sidebar_position: 1 +--- + +# Morpheus + +**Morpheus** is Lakebridge's AST-based SQL transpiler. It provides strong correctness guarantees: a file that Morpheus transpiles without any errors or warnings is guaranteed to produce equivalent results on Databricks as the original file did on the source platform. + +:::tip Not sure which transpiler to use? +See [Which Tool Do I Use?](/docs/choosing_tools#transpiler-bladebridge-vs-morpheus-vs-switch) for a full comparison of Morpheus, BladeBridge, and Switch. +::: + +--- + +## Supported Source Dialects + +| Dialect key | Source systems | +|---|---| +| `mssql` | Microsoft SQL Server, Azure SQL Database, Azure SQL Managed Instance, Amazon RDS for SQL Server | +| `snowflake` | Snowflake (including dbt repointing) | +| `synapse` | Azure Synapse Analytics (dedicated SQL pools) | + +Morpheus targets **Databricks SQL** only. For ETL sources (DataStage, SSIS) or other SQL dialects (Oracle, Teradata, Redshift), use [BladeBridge](/docs/transpile/pluggable_transpilers/bladebridge) or [Switch](/docs/transpile/pluggable_transpilers/switch). + +--- + +## How Morpheus Works + +Morpheus uses a custom parser generated from a carefully crafted ANTLR grammar. The transpilation pipeline has three stages: + +1. **Parse** — The input SQL is parsed into an intermediate representation (IR) tree. Parsing errors prevent the tree from being built and will result in the original text being returned with error annotations. +2. **Transform** — The IR tree undergoes a series of transformations that convert source-dialect constructs into their Databricks SQL equivalents. +3. **Code gen** — The transformed tree is fed to a code generator that produces and formats the final Databricks SQL text. + +Morpheus accumulates every error and warning encountered during the transform and code-gen phases but continues processing the file. Even in the presence of errors, it produces as much meaningful output as possible. + +--- + +## Correctness Guarantee Model + +Morpheus distinguishes between **errors** and **warnings**: + +| Signal | Meaning | What to do | +|---|---|---| +| No errors, no warnings | Full correctness guarantee — output is equivalent to input on the source platform | Deploy with confidence | +| **Warning** | Morpheus could not confirm equivalence but the output may still be correct (a conservative false-negative) | Review the flagged section; test output against source data | +| **Error** | Morpheus knows equivalent results cannot be guaranteed for this construct | Manual fix required before deploying | + +:::note +Some statistical functions may be implemented on Databricks using a slightly different algorithm than the source platform. Row ordering may differ when no `ORDER BY` is specified. In all other cases, results are strictly identical. +::: + +Morpheus does not perform semantic validation (e.g., type-checking) on the input SQL. The behavior of code transpiled from semantically incorrect input is unspecified. + +--- + +## Usage + +### Install + +Morpheus is installed automatically when you run: + +```bash +databricks labs lakebridge install-transpile +``` + +Morpheus is fetched from [Maven Central](https://central.sonatype.com/) and installed at: +``` +.databricks/labs/remorph-transpilers/databricks-morph-plugin/ +``` + +### Run + +```bash +databricks labs lakebridge transpile \ + --source-dialect mssql \ + --input-source /path/to/sql/files \ + --output-folder /path/to/output +``` + +**Common flags:** + +| Flag | Description | Default | +|---|---|---| +| `--source-dialect` | Source SQL dialect: `mssql`, `snowflake`, or `synapse` | Required | +| `--input-source` | Local path to the SQL files to transpile | Required | +| `--output-folder` | Local path where transpiled files will be written | Required | +| `--skip-validation` | Skip Databricks SQL validation of output (`true`/`false`) | `false` | +| `--catalog-name` | Catalog to use when validating output | `remorph` | +| `--schema-name` | Schema to use when validating output | `transpiler` | + +### Reading the output + +Each transpiled file begins with a header comment: + +**Successful transpile:** +```sql +/* + Successfully transpiled from /path/to/input/my_proc.sql +*/ +CREATE PROCEDURE ... +``` + +**Transpile with errors/warnings:** +```sql +/* + Failed transpilation of /path/to/input/broken.sql + + The following errors were found while transpiling: + - [3:5] Function GIBBERISH cannot be translated to Databricks SQL +*/ +SELECT + t1.name, + GIBBERISH(t1.comment) +FROM t1; +``` + +The `[line:column]` notation tells you exactly where in the input file the untranslatable construct starts. + +--- + +## Troubleshooting + +**My file transpiles with warnings — is it safe to use?** + +Morpheus is conservative: it emits warnings when it cannot *guarantee* correctness, even if the output is actually correct. Review the flagged section and test the output against your source data. If the results match, the file is safe. + +**I see a parsing error — is that a bug?** + +Parsing errors are very unlikely when the input SQL is valid. If you see one, the input may be malformed, or there is a gap in the Morpheus ANTLR grammar. File a bug report at [GitHub](https://github.com/databrickslabs/lakebridge/issues). + +**Morpheus doesn't support my dialect.** + +Morpheus supports `mssql`, `snowflake`, and `synapse` only. For other dialects, use [BladeBridge](/docs/transpile/pluggable_transpilers/bladebridge) (Oracle, Teradata, Netezza, Redshift, DataStage, SSIS) or [Switch](/docs/transpile/pluggable_transpilers/switch) (any dialect via LLM). diff --git a/docs/lakebridge/docs/transpile/pluggable_transpilers/switch/architecture.mdx b/docs/lakebridge/docs/transpile/pluggable_transpilers/switch/architecture.mdx new file mode 100644 index 0000000000..cc5029da75 --- /dev/null +++ b/docs/lakebridge/docs/transpile/pluggable_transpilers/switch/architecture.mdx @@ -0,0 +1,156 @@ +--- +title: Switch Architecture +sidebar_position: 6 +--- + +import CodeBlock from '@theme/CodeBlock'; + +# Switch Architecture + +This page describes the internal architecture of Switch — how it executes as a Databricks Job and the processing pipeline it runs. This is useful for advanced users, contributors, and anyone debugging conversion failures. + +For getting started with Switch, see the [Switch guide](/docs/transpile/pluggable_transpilers/switch). + +--- + +## Overview + +When you run Switch via the CLI, it executes as a Databricks Job using a multi-stage processing pipeline. The main orchestration notebook (`00_main`) validates parameters and routes to the appropriate orchestrator based on `target_type`. + +```mermaid +flowchart TD + main["00_main (Entry Point)"] e1@==> decision{target_type?} + + decision e2@==>|notebook| notebookOrch["orchestrate_to_notebook (Full Processing Pipeline)"] + decision e3@==>|file| fileOrch["orchestrate_to_file (Simplified Processing Pipeline)"] + + notebookOrch e4@==> notebookFlow["7-Step Workflow: analyze → convert → validate → fix → split → export → sql_convert(optional)"] + fileOrch e5@==> fileFlow["3-Step Workflow: analyze → convert → export"] + + notebookFlow e6@==> notebookOutput[Python/SQL Notebooks] + fileFlow e7@==> fileOutput["Generic Files (YAML, JSON, etc.)"] + e1@{ animate: true } + e2@{ animate: true } + e3@{ animate: true } + e4@{ animate: true } + e5@{ animate: true } + e6@{ animate: true } + e7@{ animate: true } +``` + +--- + +## Notebook Conversion Flow + +For `target_type=notebook` or `target_type=sdp`, the `orchestrate_to_notebook` orchestrator runs a 7-step pipeline: + +```mermaid +flowchart TD + orchestrator[orchestrate_to_notebook] e1@==>|sequentially calls| processing + + subgraph processing ["Notebook Processing Workflow"] + direction TB + analyze[analyze_input_files] e2@==> convert[convert_with_llm] + + convert e8@==>|if SDP| validate_sdp[validate_sdp] + convert e3@==>|if NOT SDP| validate_nb[validate_python_notebook] + + validate_nb e4@==> fix[fix_syntax_with_llm] + validate_sdp e9@==> fix + + fix e5@==> split[split_code_into_cells] + split e6@==> export[export_to_notebook] + export -.-> sqlExport["convert_notebook_to_sql (Optional)"] + end + + processing <-->|Read & Write| table[Conversion Result Table] + + convert -.->|Uses| endpoint[Model Serving Endpoint] + fix -.->|Uses| endpoint + sqlExport -.->|Uses| endpoint + + export e7@==> notebooks[Python Notebooks] + sqlExport -.-> sqlNotebooks["SQL Notebooks (Optional Output)"] + + e1@{ animate: true } + e2@{ animate: true } + e3@{ animate: true } + e4@{ animate: true } + e5@{ animate: true } + e6@{ animate: true } + e7@{ animate: true } + e8@{ animate: true } + e9@{ animate: true } +``` + +--- + +## File Conversion Flow + +For `target_type=file`, the `orchestrate_to_file` orchestrator uses a simplified 3-step pipeline: + +```mermaid +flowchart TD + orchestrator[orchestrate_to_file] e1@==>|sequentially calls| processing + + subgraph processing ["File Processing Workflow"] + direction TB + analyze[analyze_input_files] e2@==> convert[convert_with_llm] + convert e3@==> export[export_to_file] + end + + processing <-->|Read & Write| table[Conversion Result Table] + convert -.->|Uses| endpoint[Model Serving Endpoint] + export e4@==> files["Generic Files (YAML, JSON, etc.)"] + e1@{ animate: true } + e2@{ animate: true } + e3@{ animate: true } + e4@{ animate: true } +``` + +File conversion skips syntax validation, error fixing, and cell splitting — it exports directly from converted content to the specified file format. + +--- + +## Processing Steps + +### analyze_input_files + +Scans the input directory recursively and stores all file contents, metadata, and analysis results in a timestamped Delta table. For SQL sources, creates preprocessed versions with comments removed and whitespace normalized. Counts tokens using model-specific tokenizers (Claude uses ~3.4 characters per token; OpenAI and other models use tiktoken). Files exceeding `token_count_threshold` are excluded from conversion. + +### convert_with_llm + +Loads conversion prompts (built-in or custom YAML) and sends file content to the configured model serving endpoint. Multiple files are processed concurrently (default: 4, controlled by `concurrency`). For SQL sources, generates Python code with `spark.sql()` calls. For generic sources, adapts to the specified target format. + +### validate_python_notebook + +Checks Python syntax using `ast.parse()` and validates SQL statements within `spark.sql()` calls using Spark's `EXPLAIN` command. Errors are recorded in the result table. + +### validate_sdp + +Validates Spark Declarative Pipeline output: + +| Step | Description | +|---|---| +| Export Notebook | Writes converted code to a temporary workspace notebook | +| Create Pipeline | Creates a temporary Spark Declarative Pipeline referencing the notebook | +| Update Pipeline | Runs a validation-only update to check for SDP syntax errors | +| Delete Pipeline | Cleans up the temporary pipeline | + +`TABLE_OR_VIEW_NOT_FOUND` errors are ignored. + +### fix_syntax_with_llm + +Sends error context back to the model serving endpoint for automatic correction. Repeats up to `max_fix_attempts` times (default: 1). Set to 0 to disable automatic fixing. + +### split_code_into_cells + +Transforms raw converted Python code into well-structured notebook cells. Splits at logical boundaries (imports, function definitions, major operations) and adds markdown cells for documentation. + +### export_to_notebook + +Creates Databricks-compatible `.py` notebooks in the output directory. Handles files up to 10MB and preserves the original directory structure. Includes metadata, source file references, and syntax check results as comments. + +### convert_notebook_to_sql (Optional) + +When `sql_output_dir` is specified, uses the model serving endpoint to convert Python notebooks to SQL notebook format. Some Python-specific logic may be lost in this conversion. diff --git a/docs/lakebridge/docs/transpile/pluggable_transpilers/switch/index.mdx b/docs/lakebridge/docs/transpile/pluggable_transpilers/switch/index.mdx index b2615dff93..4c615f70ea 100644 --- a/docs/lakebridge/docs/transpile/pluggable_transpilers/switch/index.mdx +++ b/docs/lakebridge/docs/transpile/pluggable_transpilers/switch/index.mdx @@ -34,7 +34,7 @@ Instead of parsing rules, Switch uses [Mosaic AI Model Serving](https://docs.dat - Enable extensible conversion through custom YAML prompts ### 2. Native Databricks Integration -Switch runs entirely within the Databricks workspace. You can find details about this architecture [here](#databricks-implementation-details) +Switch runs entirely within the Databricks workspace. You can find details about this architecture [here](/docs/transpile/pluggable_transpilers/switch/architecture) - **Jobs API**: Executes as scalable Databricks Jobs for batch processing - **Model Serving**: Direct integration with Databricks LLM endpoints, with concurrent processing for multiple files - **Delta Tables**: Tracks conversion progress and results @@ -198,16 +198,16 @@ Additional conversion parameters are managed in the Switch configuration file. Y | Parameter | Description | Default Value | Available Options | |-----------|-------------|---------------|-------------------| -| `target_type` | Output format type. `notebook` for Python notebooks with validation and error fixing, `file` for generic file formats, `sdp` for conversion from etl workloads to Spark Declarative Pipeline (SDP). See [Conversion Flow Overview](#conversion-flow-overview) for processing differences. | `notebook` | `notebook`, `file`, `sdp` | -| `source_format` | Source file format type. `sql` performs SQL comment removal and whitespace compression preprocessing before conversion. `generic` processes files as-is without preprocessing. Preprocessing affects token counting and conversion quality. See [analyze_input_files](#analyze_input_files) for preprocessing details. | `sql` | `sql`, `generic` | +| `target_type` | Output format type. `notebook` for Python notebooks with validation and error fixing, `file` for generic file formats, `sdp` for conversion from etl workloads to Spark Declarative Pipeline (SDP). See [Conversion Flow Overview](/docs/transpile/pluggable_transpilers/switch/architecture#overview) for processing differences. | `notebook` | `notebook`, `file`, `sdp` | +| `source_format` | Source file format type. `sql` performs SQL comment removal and whitespace compression preprocessing before conversion. `generic` processes files as-is without preprocessing. Preprocessing affects token counting and conversion quality. See [analyze_input_files](/docs/transpile/pluggable_transpilers/switch/architecture#analyze_input_files) for preprocessing details. | `sql` | `sql`, `generic` | | `comment_lang` | Language for generated comments. | `English` | `English`, `Japanese`, `Chinese`, `French`, `German`, `Italian`, `Korean`, `Portuguese`, `Spanish` | | `log_level` | Logging verbosity level. | `INFO` | `DEBUG`, `INFO`, `WARNING`, `ERROR` | | `token_count_threshold` | Maximum tokens per file for processing. Files exceeding this limit are automatically excluded from conversion. Adjust based on your model's context window and conversion complexity. See [Token Management](/docs/transpile/pluggable_transpilers/switch/customizing_switch#token-management) for detailed configuration guidelines and file splitting strategies. | `20000` | Any positive integer | | `concurrency` | Number of parallel LLM requests for processing multiple files simultaneously. Higher values improve throughput but may hit rate limits. Default is optimized for Claude models. See [Performance Optimization](/docs/transpile/pluggable_transpilers/switch/customizing_switch#performance-optimization) for scaling guidance and model-specific considerations. | `4` | Any positive integer | -| `max_fix_attempts` | Maximum number of automatic syntax error correction attempts per file. Each attempt sends error context back to the LLM for fixing. Set to 0 to skip automatic fixes. See [fix_syntax_with_llm](#fix_syntax_with_llm) for details on the error correction process. | `1` | 0 or any positive integer | +| `max_fix_attempts` | Maximum number of automatic syntax error correction attempts per file. Each attempt sends error context back to the LLM for fixing. Set to 0 to skip automatic fixes. See [fix_syntax_with_llm](/docs/transpile/pluggable_transpilers/switch/architecture#fix_syntax_with_llm) for details on the error correction process. | `1` | 0 or any positive integer | | `conversion_prompt_yaml` | Custom conversion prompt YAML file path. When specified, overrides the built-in prompt for the selected `--source-dialect`, enabling support for additional source formats or specialized conversion requirements. See [Customizable Prompts](/docs/transpile/pluggable_transpilers/switch/customizing_switch#customizable-prompts) for YAML structure and creation guide. | `null` | Full workspace path to YAML file | -| `output_extension` | File extension for output files when `target_type=file`. Required for non-notebook output formats like YAML workflows or JSON configurations. See [File Conversion Flow](#file-conversion-flow) for usage examples. | `null` | Any extension (e.g., `.yml`, `.json`) | -| `sql_output_dir` | (Experimental) When specified, triggers additional conversion of Python notebooks to SQL notebook format. This optional post-processing step may lose some Python-specific logic. See [convert_notebook_to_sql](#convert_notebook_to_sql-optional) for details on the SQL conversion process. | `null` | Full workspace path | +| `output_extension` | File extension for output files when `target_type=file`. Required for non-notebook output formats like YAML workflows or JSON configurations. See [File Conversion Flow](/docs/transpile/pluggable_transpilers/switch/architecture#file-conversion-flow) for usage examples. | `null` | Any extension (e.g., `.yml`, `.json`) | +| `sql_output_dir` | (Experimental) When specified, triggers additional conversion of Python notebooks to SQL notebook format. This optional post-processing step may lose some Python-specific logic. See [convert_notebook_to_sql](/docs/transpile/pluggable_transpilers/switch/architecture#convert_notebook_to_sql-optional) for details on the SQL conversion process. | `null` | Full workspace path | | `request_params` | Additional request parameters passed to the model serving endpoint. Use for advanced configurations like extended thinking mode or custom token limits. See [LLM Configuration](/docs/transpile/pluggable_transpilers/switch/customizing_switch#llm-configuration) for configuration examples including Claude's extended thinking mode. | `null` | JSON format string (e.g., `{"max_tokens": 64000}`) | | `sdp_language` | Control the language of converted SDP code, can only be "python" or "sql". | `python` | `python`, `sql` | @@ -308,155 +308,7 @@ databricks labs lakebridge llm-transpile \ ``` --- -## Databricks Implementation Details -When you run Switch via the CLI, it executes as Databricks Jobs using a sophisticated multi-stage processing pipeline. This section covers the internal architecture and configuration options. +## Internal Architecture -## Processing Architecture - -Switch executes as a Databricks Job that runs the main orchestration notebook (`00_main`), which routes to specialized orchestrators that coordinate the conversion pipeline: - -### Main Orchestration -The **`00_main`** notebook serves as the entry point when Switch is executed via Databricks Jobs API. It: -- Validates all input parameters from the job configuration -- Routes execution to the appropriate orchestrator based on `target_type` (notebook or file output) -- Handles orchestrator results and displays final conversion summary - -### Conversion Flow Overview - -Switch supports two target types with different processing workflows. The main entry point (`00_main`) routes to the appropriate orchestrator based on the `target_type` parameter: - -```mermaid -flowchart TD - main["00_main (Entry Point)"] e1@==> decision{target_type?} - - decision e2@==>|notebook| notebookOrch["orchestrate_to_notebook (Full Processing Pipeline)"] - decision e3@==>|file| fileOrch["orchestrate_to_file (Simplified Processing Pipeline)"] - - notebookOrch e4@==> notebookFlow["7-Step Workflow: analyze → convert → validate → fix → split → export → sql_convert(optional)"] - fileOrch e5@==> fileFlow["3-Step Workflow: analyze → convert → export"] - - notebookFlow e6@==> notebookOutput[Python/SQL Notebooks] - fileFlow e7@==> fileOutput["Generic Files (YAML, JSON, etc.)"] - e1@{ animate: true } - e2@{ animate: true } - e3@{ animate: true } - e4@{ animate: true } - e5@{ animate: true } - e6@{ animate: true } - e7@{ animate: true } -``` - -### Notebook Conversion Flow - -For `target_type=notebook` or `target_type=sdp`, the `orchestrate_to_notebook` orchestrator executes a comprehensive 7-step processing pipeline: - -```mermaid -flowchart TD - orchestrator[orchestrate_to_notebook] e1@==>|sequentially calls| processing - - subgraph processing ["Notebook Processing Workflow"] - direction TB - analyze[analyze_input_files] e2@==> convert[convert_with_llm] - - %% Branch: decide validation path - convert e8@==>|if SDP| validate_sdp[validate_sdp] - convert e3@==>|if NOT SDP| validate_nb[validate_python_notebook] - - %% Downstream connections - both validations flow to fix_syntax - validate_nb e4@==> fix[fix_syntax_with_llm] - validate_sdp e9@==> fix - - fix e5@==> split[split_code_into_cells] - split e6@==> export[export_to_notebook] - export -.-> sqlExport["convert_notebook_to_sql
(Optional)"] - end - - processing <-->|Read & Write| table[Conversion Result Table] - - convert -.->|Uses| endpoint[Model Serving Endpoint] - fix -.->|Uses| endpoint - sqlExport -.->|Uses| endpoint - - export e7@==> notebooks[Python Notebooks] - sqlExport -.-> sqlNotebooks["SQL Notebooks
(Optional Output)"] - - e1@{ animate: true } - e2@{ animate: true } - e3@{ animate: true } - e4@{ animate: true } - e5@{ animate: true } - e6@{ animate: true } - e7@{ animate: true } - e8@{ animate: true } - e9@{ animate: true } -``` - -### File Conversion Flow - -For `target_type=file`, the `orchestrate_to_file` orchestrator uses a simplified 3-step processing pipeline optimized for generic file output: - -```mermaid -flowchart TD - orchestrator[orchestrate_to_file] e1@==>|sequentially calls| processing - - subgraph processing ["File Processing Workflow"] - direction TB - analyze[analyze_input_files] e2@==> convert[convert_with_llm] - convert e3@==> export[export_to_file] - end - - processing <-->|Read & Write| table[Conversion Result Table] - - convert -.->|Uses| endpoint[Model Serving Endpoint] - - export e4@==> files["Generic Files (YAML, JSON, etc.)"] - e1@{ animate: true } - e2@{ animate: true } - e3@{ animate: true } - e4@{ animate: true } -``` - -**Key Differences:** -- **File conversion skips** syntax validation, error fixing, and cell splitting steps -- **Direct export** from converted content to specified file format with custom extension -- **Optimized** for non-notebook outputs like YAML workflows, JSON configurations, etc. - ---- - -## Processing Steps - -The following sections describe each processing step used in the workflows above: - -### analyze_input_files -Scans the input directory recursively and performs initial analysis. Stores all file contents, metadata, and analysis results in a timestamped Delta table. For SQL sources, creates preprocessed versions with comments removed and whitespace normalized in the table. Counts tokens using model-specific tokenizers (Claude uses ~3.4 characters per token, OpenAI and other models use tiktoken) to determine if files exceed the `token_count_threshold`. Files exceeding the threshold are excluded from conversion. - -### convert_with_llm -Loads conversion prompts (built-in or custom YAML) and sends file content to the configured model serving endpoint. Multiple files are processed concurrently (configurable, default: 4) for efficiency. The LLM transforms source code based on the conversion prompt, preserving business logic while adapting to Databricks patterns. For SQL sources, generates Python code with `spark.sql()` calls. For generic sources, adapts content to the specified target format. - -### validate_python_notebook -Performs syntax validation on the generated code. Python syntax is checked using `ast.parse()`, while SQL statements within `spark.sql()` calls are validated using Spark's `EXPLAIN` command. Any errors are recorded in the result table for potential fixing in the next step. - -### validate_sdp -Performs Spark Declarative Pipeline validation on the generated code. The validation process executes these steps sequentially: - -| Step | Description | -|------|-------------| -| Export Notebook | Writes the converted code to a temporary notebook in workspace | -| Create Pipeline | Creates a temporary Spark Declarative Pipeline referencing the notebook | -| Update Pipeline | Runs a validation-only update to check for SDP syntax errors | -| Delete Pipeline | Cleans up the temporary pipeline after validation | - -Note: `TABLE_OR_VIEW_NOT_FOUND` errors are ignored. - -### fix_syntax_with_llm -Attempts automatic error correction when syntax issues are detected. Sends error context back to the model serving endpoint, which suggests corrections. The validation and fix process repeats up to `max_fix_attempts` times (default: 1) until errors are resolved or the retry limit is reached. - -### split_code_into_cells -Transforms raw converted Python code into well-structured notebook cells. Analyzes code flow and dependencies, splitting content at logical boundaries like imports, function definitions, and major operations. Adds appropriate markdown cells for documentation and readability. - -### export_to_notebook -Creates Databricks-compatible `.py` notebooks in the specified output directory. Each notebook includes proper metadata, source file references, and any syntax check results as comments. Handles large files (up to 10MB) and preserves the original directory structure. - -### convert_notebook_to_sql (Optional) -When `sql_output_dir` is specified, this optional step uses the model serving endpoint to convert Python notebooks into SQL notebook format with Databricks SQL syntax. Useful for teams preferring SQL-only workflows, though some Python logic may be lost in the conversion process. +Switch runs as a Databricks Job using a multi-stage processing pipeline. For details on how the pipeline works internally (orchestration notebooks, processing steps, validation logic), see [Switch Architecture](/docs/transpile/pluggable_transpilers/switch/architecture). diff --git a/docs/lakebridge/docs/transpile/pluggable_transpilers/switch/switch_faqs.mdx b/docs/lakebridge/docs/transpile/pluggable_transpilers/switch/switch_faqs.mdx deleted file mode 100644 index 00c47cc9e5..0000000000 --- a/docs/lakebridge/docs/transpile/pluggable_transpilers/switch/switch_faqs.mdx +++ /dev/null @@ -1,77 +0,0 @@ ---- -title: Switch FAQs and Troubleshooting -sidebar_position: 7 ---- - -import CodeBlock from '@theme/CodeBlock'; - -## Can Switch Support My Source System? - -A common question is whether Switch can handle sources beyond the built-in conversion types. The answer is: **try it!** - -Switch already supports various source formats including SQL dialects (MySQL, Snowflake, Oracle, etc.), programming languages (Python scripts, Scala code), and workflows (Airflow DAGs). - -**For SQL-based sources**: Creating a custom prompt YAML file should work well for most SQL dialects. Since LLMs understand SQL syntax patterns, you can typically achieve good results by: -- Starting with a similar built-in dialect's YAML as a template -- Adding specific syntax examples from your source system -- Testing and iterating based on results - -**Tips for efficient prompt creation:** -- **Quick baseline creation**: Feed built-in prompts and your source dialect's representative features to an advanced LLM to quickly generate a baseline YAML configuration -- **Dialect-specific patterns**: Reference open source projects like [SQLGlot dialects](https://github.com/tobymao/sqlglot/tree/main/sqlglot/dialects) for insights into dialect-specific transformation patterns - -**For other source formats**: Switch's LLM-based architecture means it can potentially handle various conversions beyond the built-in types. Modern LLMs have strong comprehension capabilities across many languages and formats. You can experiment by: -- Creating custom prompts that define your source format -- Providing clear conversion examples in the few-shots section -- Testing with representative source samples - -Rather than waiting for additional built-in examples, we encourage experimentation with custom prompts. The flexibility of LLM-based conversion means many use cases are possible with the right prompt engineering. - ---- - -## Conversion Results and Troubleshooting - -### Understanding Conversion Results - -After your Switch job completes, review the conversion results displayed at the end of the `00_main` notebook execution. The results table shows the status of each input file: - -- **Successfully converted files**: Ready to use as Databricks notebooks -- **Files requiring attention**: May need manual review or re-processing - -If you encounter files that didn't convert successfully, here are the most common issues and their solutions: - -### Files Not Converting (Status: `Not converted`) - -These files were skipped during the conversion process, typically because they're too large for the model to process effectively. - -**Cause**: Input files exceed the token count threshold - -**Solutions**: -- Split large input files into smaller, more manageable parts -- Increase the `token_count_threshold` parameter if your LLM model can handle larger inputs - -### Conversion with Errors (Status: `Converted with errors`) - -These files were successfully processed by the LLM but the generated code contains syntax errors that need to be addressed. - -**Cause**: Files were converted but contain syntax errors - -**Solutions**: -- Review syntax error messages in the result table's error_details column -- Manually fix errors in the converted notebooks/files -- Increase `max_fix_attempts` for more automatic error correction attempts - -### Export Failures (Status: `Export failed` or `Not exported`) - -These files were converted successfully but couldn't be exported to the output directory. - -**Causes**: -- Content exceeds 10MB size limit of Databricks notebooks -- File system permissions issues -- Invalid output paths - -**Solutions**: -- Check the `export_error` column in the result table for specific error details -- For size issues: Manually split large converted content into smaller units -- For permission issues: Verify workspace access to the output directory -- For path issues: Ensure output directory paths are valid workspace locations diff --git a/docs/lakebridge/docs/transpile/source_systems/index.mdx b/docs/lakebridge/docs/transpile/source_systems/index.mdx index 022e09abf9..256527e1cd 100644 --- a/docs/lakebridge/docs/transpile/source_systems/index.mdx +++ b/docs/lakebridge/docs/transpile/source_systems/index.mdx @@ -46,9 +46,3 @@ In this section we'll have source-specific conversion documentation. Select your [📖 View DataStage Conversion Guide](./datastage) --- - -## Coming Soon - -Additional source system conversion guides will be added as they become available for all supported sources in Lakebridge - ---- diff --git a/docs/lakebridge/docs/transpile/source_systems/ssis.mdx b/docs/lakebridge/docs/transpile/source_systems/ssis.mdx deleted file mode 100644 index 51a671da35..0000000000 --- a/docs/lakebridge/docs/transpile/source_systems/ssis.mdx +++ /dev/null @@ -1,608 +0,0 @@ ---- -sidebar_position: 1 -title: SSIS - -toc_min_heading_level: 2 -toc_max_heading_level: 3 ---- -# Microsoft SSIS to Databricks - -## Conversion information -* **Transpiler**: BladeBridge -* **Available** target: Databricks notebooks (experimental) - -### Supported SSIS Versions - -- SQL Server 2012, 2014, 2016, 2017, 2019, 2022 -- Azure Data Factory SSIS Integration Runtime - -### Input Requirements - -Export your SSIS packages as DTSX files: - -1. **Solution Export**: Export from Visual Studio / SQL Server Data Tools (SSDT) -2. **File System Packages**: Direct DTSX file access -3. **SSISDB Export**: Extract from SSIS catalog - -**Export from SSISDB:** -```sql --- Extract package from SSISDB catalog -DECLARE @packageData VARBINARY(MAX) -SELECT @packageData = [packagedata] -FROM [SSISDB].[catalog].[packages] -WHERE [name] = 'YourPackageName' - --- Save to file system for conversion -``` - ---- - -## Components - -### Control Flow (Orchestration) - Supported - -| SSIS Component | Microsoft Name | Spark Equivalent | Notes | -|----------------|----------------|------------------|-------| -| **Data Flow Task** | Microsoft.DataFlowTask | Spark SQL temp views | Core data transformations | -| **Execute SQL Task** | Microsoft.ExecuteSQLTask | Spark SQL statements | SQL execution with parameter mapping | -| **Execute Package Task** | Microsoft.ExecutePackageTask | `dbutils.notebook.run()` | Nested notebook execution | -| **File System Task** | Microsoft.FileSystemTask | `dbutils.fs` commands | File operations (copy, move, delete, rename) | -| **Script Task** | Microsoft.ScriptTask | Python code with SQL | C#/VB.NET scripts converted to Python/SQL | -| **For Loop Container** | STOCK:FORLOOP | SQL iteration pattern | Iteration with counter | -| **Foreach Loop Container** | STOCK:FOREACHLOOP | SQL wildcard file reading | File/folder iteration | -| **Sequence Container** | STOCK:SEQUENCE | Function or notebook section | Grouping tasks | -| **Execute Process Task** | Microsoft.ExecuteProcess | `subprocess.run()` | External process execution | -| **Extensible File Task** | ExtensibleFileTask | `dbutils.fs` commands | Extended file operations | - -### Control Flow (Orchestration) - Unsupported - -The following Control Flow components are **not supported** and would require manual conversion or alternative approaches: - -| SSIS Component | Microsoft Name | Reason | -|----------------|----------------|--------| -| **Analysis Services Execute DDL** | Microsoft.AnalysisServicesExecuteDDLTask | SSAS-specific, requires manual migration to Delta/SQL | -| **Analysis Services Processing** | Microsoft.AnalysisServicesProcessingTask | SSAS-specific, requires manual migration to Delta/SQL | -| **Bulk Insert Task** | Microsoft.BulkInsertTask | Use Data Flow Task with JDBC/Delta instead | -| **Data Profiling Task** | Microsoft.DataProfilingTask | Use Databricks Data Profile UI or custom profiling code | -| **FTP Task** | Microsoft.FTPTask | Implement using Python `ftplib` or `dbutils.fs` | -| **Message Queue Task** | Microsoft.MessageQueueTask | Requires custom Kafka/Event Hub integration | -| **Send Mail Task** | Microsoft.SendMailTask | Implement using Databricks job notifications or Python `smtplib` | -| **Web Service Task** | Microsoft.WebServiceTask | Implement using Python `requests` library | -| **WMI Data Reader Task** | Microsoft.WMIDataReaderTask | Windows-specific, requires alternative monitoring solution | -| **XML Task** | Microsoft.XMLTask | Implement using Python `xml.etree` or PySpark XML functions | - -:::warning Conversion Difficulty -Components like **Script Task** contain **C# or VB.NET code bodies** that cannot be automatically converted. These require **manual code translation** from C#/VB to Python. The converter will preserve the logic structure but the actual implementation must be rewritten. -::: - ---- - -### Data Flow (Transformation) - Supported - -#### Sources - -| SSIS Component | Microsoft Name | Spark Equivalent | Notes | -|----------------|----------------|------------------|-------| -| **OLE DB Source** | Microsoft.OLEDBSource | `spark.read.format("jdbc")` | Database reads via JDBC with PySpark | -| **Flat File Source** | Microsoft.FlatFileSource | `spark.read.csv()` or SQL `csv.\`path\`` | CSV, delimited, fixed-width files | -| **Excel Source** | Microsoft.ExcelSource | `spark.read.format("excel")` | Excel file reads | -| **Raw File Source** | Microsoft.RawSource | `spark.read.format("parquet")` | SSIS raw files converted to Parquet | - -#### Transformations - -| SSIS Component | Microsoft Name | Spark Equivalent | Notes | -|----------------|----------------|------------------|-------| -| **Aggregate** | Microsoft.Aggregate | `GROUP BY` with aggregation | Sum, count, avg, min, max operations | -| **Audit** | Microsoft.Audit | Add columns in SELECT | Add audit columns (timestamp, user, etc.) | -| **Cache Transform** | Microsoft.Cache | Temp views or CTEs | Cache data for lookup operations | -| **Character Map** | Microsoft.CharacterMap | String functions | String transformations (upper, lower, etc.) | -| **Conditional Split** | Microsoft.ConditionalSplit | Multiple `WHERE` clauses | Route rows by conditions | -| **Copy Column** | Microsoft.CopyColumn | Column in SELECT | Duplicate columns | -| **Data Conversion** | Microsoft.DataConvert | `CAST()` | Type conversions | -| **Derived Column** | Microsoft.DerivedColumn | Calculated columns in SELECT | Column transformations and calculations | -| **Lookup** | Microsoft.Lookup | `LEFT JOIN` | Reference data lookup with caching | -| **Merge** | Microsoft.Merge | `UNION` | Merge sorted datasets | -| **Merge Join** | Microsoft.MergeJoin | `JOIN` | Sorted input joins | -| **Multicast** | Microsoft.Multicast | Temp view reuse | Send data to multiple outputs | -| **OLE DB Command** | Microsoft.OLEDBCommand | Row-by-row SQL execution | Row-by-row SQL execution | -| **Percentage Sampling** | Microsoft.PercentageSampling | `TABLESAMPLE` | Statistical sampling | -| **Pivot** | Microsoft.Pivot | `PIVOT` clause | Pivot operations | -| **Row Count** | Microsoft.RowCount | `COUNT(*)` | Count rows and store in variable | -| **Script Component** | Microsoft.SqlServer.Dts.Pipeline.ScriptComponentHost | SQL UDFs or CASE statements | Custom transformations | -| **Slowly Changing Dimension** | Microsoft.SCD | `MERGE` with Delta Lake | SCD Type 1, 2, 3 patterns | -| **Sort** | Microsoft.Sort | `ORDER BY` | Sorting operations | -| **Union All** | Microsoft.UnionAll | `UNION ALL` | Combine multiple datasets | -| **Unpivot** | Microsoft.UnPivot | `UNPIVOT` or `STACK()` | Unpivot operations | - -#### Destinations - -| SSIS Component | Microsoft Name | Spark Equivalent | Notes | -|----------------|----------------|------------------|-------| -| **OLE DB Destination** | Microsoft.OLEDBDestination | `INSERT INTO` SQL or Delta Lake | Database writes via SQL statements | -| **Flat File Destination** | Microsoft.FlatFileDestination | `df.write.csv()` or SQL INSERT | CSV and delimited file writes | -| **Excel Destination** | Microsoft.ExcelDestination | `df.write.format('excel')` | Excel file writes | -| **Raw File Destination** | Microsoft.RawDestination | `df.write.format('parquet')` | SSIS raw files converted to Parquet | - -### Data Flow (Transformation) - Unsupported - -The following Data Flow components are **not supported** and would require manual conversion: - -| SSIS Component | Microsoft Name | Reason | -|----------------|----------------|--------| -| **Export Column** | Microsoft.ExportColumn | Implement using custom UDF with file writing | -| **Import Column** | Microsoft.ImportColumn | Implement using custom UDF with file reading | - -:::warning Script Component Conversion Difficulty -The **Script Component** often contains **C# or VB.NET code bodies** with complex row-by-row processing logic. While the converter identifies these components, the actual C#/VB code **cannot be automatically converted**. - -**Manual conversion required:** -- Analyze the C#/VB logic -- Rewrite as SQL UDFs or CASE statements -- Test thoroughly as row-by-row logic may need redesign for set-based SQL processing -::: - ---- - -## Data Flow Conversion - -### Derived Column Transformation - -**SSIS Derived Column:** -``` -Derived Column Transformation - Columns: - FullName = FirstName + " " + LastName - AgeGroup = Age < 30 ? "Young" : (Age < 60 ? "Middle" : "Senior") - ProcessDate = GETDATE() -``` - -**Converted Spark SQL:** -```python -# Processing node Package\Data Flow Task\Derived Column, type DERIVED_COLUMN -# component nameDerived Column -Derived_Column_Add_Columns = f""" -SELECT - *, - CONCAT(FirstName, ' ', LastName) AS FullName, - CASE - WHEN Age < 30 THEN 'Young' - WHEN Age < 60 THEN 'Middle' - ELSE 'Senior' - END AS AgeGroup, - CURRENT_TIMESTAMP() AS ProcessDate -FROM source_table -""" -Derived_Column_Add_Columns = spark.sql(Derived_Column_Add_Columns) -Derived_Column_Add_Columns.createOrReplaceTempView('Derived_Column_Add_Columns') -``` - -### Conditional Split - -**SSIS Conditional Split:** -``` -Conditional Split Transformation - Outputs: - HighValue: Amount > 1000 - MediumValue: Amount > 100 AND Amount <= 1000 - LowValue: Default Output -``` - -**Converted Spark SQL:** -```python -# Processing node Package\Data Flow Task\Conditional Split, type CONDITIONAL_SPLIT -# component nameConditional Split - -# High Value output -Conditional_Split_High_Value = f""" -SELECT * FROM source_table -WHERE Amount > 1000 -""" -Conditional_Split_High_Value = spark.sql(Conditional_Split_High_Value) -Conditional_Split_High_Value.createOrReplaceTempView('Conditional_Split_High_Value') - -# Medium Value output -Conditional_Split_Medium_Value = f""" -SELECT * FROM source_table -WHERE Amount > 100 AND Amount <= 1000 -""" -Conditional_Split_Medium_Value = spark.sql(Conditional_Split_Medium_Value) -Conditional_Split_Medium_Value.createOrReplaceTempView('Conditional_Split_Medium_Value') - -# Low Value output (default) -Conditional_Split_Low_Value = f""" -SELECT * FROM source_table -WHERE Amount <= 100 -""" -Conditional_Split_Low_Value = spark.sql(Conditional_Split_Low_Value) -Conditional_Split_Low_Value.createOrReplaceTempView('Conditional_Split_Low_Value') -``` - -### Lookup Transformation - -**SSIS Lookup:** -``` -Lookup Transformation - Reference Table: DimCustomerType - Join Columns: CustomerTypeID = TypeID - Lookup Columns: TypeName, TypeDescription - Cache Mode: Full Cache -``` - -**Converted Spark SQL:** -```python -# Processing node Package\Data Flow Task\Lookup Customer Type, type LOOKUP -# component nameLookup Customer Type -Lookup_Customer_Type = f""" -SELECT - src.*, - ref.TypeName, - ref.TypeDescription -FROM source_table src -LEFT JOIN dim.customer_type ref - ON src.CustomerTypeID = ref.TypeID -""" -Lookup_Customer_Type = spark.sql(Lookup_Customer_Type) -Lookup_Customer_Type.createOrReplaceTempView('Lookup_Customer_Type') -``` - ---- - -## Variables and Expressions - -### SSIS Variables - -**SSIS Package Variables:** -``` -Variables: - User::MaxProcessDate (DateTime) - User::RowCount (Int32) - User::SourceFolder (String) -``` - -**Converted Spark SQL (with Python variables):** -```python -# SSIS Package Variables converted to Python variables -V_MaxProcessDate = f'2024-01-01' -V_SourceFolder = f'/mnt/source/' - -# Use in SQL transformations via spark.sql() -Incremental_Data_Query = f""" -SELECT * -FROM source_table -WHERE ProcessDate > '{V_MaxProcessDate}' -""" -spark.sql(Incremental_Data_Query) -``` - -### SSIS Expressions - -**SSIS Expression Language:** -``` -File Connection Manager Expression: - @[User::SourceFolder] + "customers_" + - (DT_WSTR, 8) DATEPART("yyyy", GETDATE()) + - RIGHT("0" + (DT_WSTR, 2) DATEPART("mm", GETDATE()), 2) + ".csv" -``` - -**Converted Spark SQL:** -```python -from datetime import datetime - -# SSIS Variables -V_SourceFolder = f'/mnt/source/' - -# Build dynamic file path -current_date = datetime.now() -year = current_date.strftime("%Y") -month = current_date.strftime("%m") -V_FilePath = f"{V_SourceFolder}customers_{year}{month}.csv" - -# Read from dynamic path using spark.sql() -Read_Customers_Data = f"""SELECT * FROM csv.`{V_FilePath}`""" -Read_Customers_Data = spark.sql(Read_Customers_Data) -Read_Customers_Data.createOrReplaceTempView('Read_Customers_Data') -``` - ---- - -## Control Flow Patterns - -### ForEach Loop Container - -**SSIS ForEach Loop:** -``` -ForEach Loop Container (File Enumerator) - Folder: C:\Data\Input\ - Files: *.csv - - Tasks: - - Data Flow: Process each file - - Execute SQL: Log processing -``` - -**Converted Spark SQL:** -```python -# SSIS Variables -V_InputFolder = f'/mnt/data/input/' - -# Process multiple CSV files using wildcard pattern -ForEach_Read_All_Files = f""" -SELECT - *, - input_file_name() AS source_file, - CURRENT_TIMESTAMP() AS process_date -FROM csv.`{V_InputFolder}*.csv` -""" -ForEach_Read_All_Files = spark.sql(ForEach_Read_All_Files) -ForEach_Read_All_Files.createOrReplaceTempView('ForEach_Read_All_Files') - -# Insert into target table -ForEach_Insert_Customer_Data = f""" -INSERT INTO staging.customer_data -SELECT * FROM ForEach_Read_All_Files -""" -spark.sql(ForEach_Insert_Customer_Data) - -# Log processing -ForEach_Log_Processing = f""" -INSERT INTO logs.processing_log -SELECT - source_file AS file_path, - COUNT(*) AS row_count, - MAX(process_date) AS process_time -FROM ForEach_Read_All_Files -GROUP BY source_file -""" -spark.sql(ForEach_Log_Processing) -``` - -### Execute SQL Task - -**SSIS Execute SQL Task:** -``` -Execute SQL Task - Connection: DW_Connection - SQL Statement: - TRUNCATE TABLE staging.customer_temp; - - INSERT INTO staging.customer_temp - SELECT * FROM staging.customer_stage - WHERE process_date >= ?; - - Parameter Mapping: - Variable: User::MaxProcessDate → Parameter 0 -``` - -**Converted Spark SQL:** -```python -# SSIS Variable -V_MaxProcessDate = f'2024-01-01' - -# Processing node Package\Execute SQL Task, type EXECUTE_SQL -# component nameExecute SQL Task -Execute_SQL_Truncate = f"""TRUNCATE TABLE staging.customer_temp""" -spark.sql(Execute_SQL_Truncate) - -Execute_SQL_Insert = f""" -INSERT INTO staging.customer_temp -SELECT * FROM staging.customer_stage -WHERE process_date >= '{V_MaxProcessDate}' -""" -spark.sql(Execute_SQL_Insert) -``` - ---- - -## Script Component Conversion - -### Script Task (Control Flow) - -**SSIS Script Task (C#):** -```csharp -public void Main() -{ - string sourceFolder = Dts.Variables["User::SourceFolder"].Value.ToString(); - DateTime processDate = DateTime.Now; - - int fileCount = Directory.GetFiles(sourceFolder, "*.csv").Length; - - Dts.Variables["User::FileCount"].Value = fileCount; - - Dts.TaskResult = (int)ScriptResults.Success; -} -``` - -**Converted Python (with Spark SQL):** -```python -from datetime import datetime - -# SSIS Variables -V_SourceFolder = f'/mnt/source/' -V_ProcessDate = datetime.now() - -# Count files (requires Python/dbutils for file system operations) -file_list = dbutils.fs.ls(V_SourceFolder) -V_FileCount = str(len([f for f in file_list if f.path.endswith('.csv')])) -``` - -### Script Component (Data Flow) - -**SSIS Script Component (transformation):** -```csharp -public override void Input0_ProcessInputRow(Input0Buffer Row) -{ - // Custom business logic - if (Row.Amount > 1000) - { - Row.PriorityFlag = "HIGH"; - Row.DiscountRate = 0.15; - } - else - { - Row.PriorityFlag = "NORMAL"; - Row.DiscountRate = 0.05; - } - - Row.ProcessedDate = DateTime.Now; -} -``` - -**Converted Spark SQL:** -```python -# Processing node Package\Data Flow Task\Script Component, type SCRIPT_COMPONENT -# component nameScript Component -Script_Component_Custom_Logic = f""" -SELECT - *, - CASE - WHEN Amount > 1000 THEN 'HIGH' - ELSE 'NORMAL' - END AS PriorityFlag, - CASE - WHEN Amount > 1000 THEN 0.15 - ELSE 0.05 - END AS DiscountRate, - CURRENT_TIMESTAMP() AS ProcessedDate -FROM source_table -""" -Script_Component_Custom_Logic = spark.sql(Script_Component_Custom_Logic) -Script_Component_Custom_Logic.createOrReplaceTempView('Script_Component_Custom_Logic') -``` - ---- - -## Package Configurations - -### Connection Managers - -**SSIS Connection Managers:** -```xml - - - SourceDB - - Data Source=source-server;Initial Catalog=SourceDB; - Integrated Security=True; - - - -``` - -**Converted Databricks:** -```python -# Processing node Package\Data Flow Task\OLE DB Source, type SOURCE -# component nameOLE DB Source - SourceDB Connection -Connection_Manager_SourceDB = spark.read \ - .format("jdbc") \ - .option("url", "source-db-url") \ - .option("dbtable", "dbo.Customers") \ - .option("user", "source-db-user") \ - .option("password", "source-db-password") \ - .load() -Connection_Manager_SourceDB.createOrReplaceTempView('Connection_Manager_SourceDB') -``` - ---- - -## Conversion Example - -### Complete SSIS Package - -**SSIS Package: [LoadCustomerData.dtsx](/downloads/LoadCustomerData.dtsx)** (download sample) - -Control Flow: -1. Execute SQL Task: Get Destination Table Name or File Name (with ProcessName parameter) -2. Execute SQL Task: Truncate Staging Table -3. Execute SQL Task: Load Customer Data -4. Execute SQL Task: Update Control Table (with ProcessName parameter) - -Parameters: -- ProcessName (String): "CustomerETL" - -**Converted Spark SQL Notebook:** - -```python -# Databricks notebook source - -ProcessName = f'CustomerETL' - -# COMMAND ---------- - -# Processing node Package\Get Destination Table Name or File Name, type EXECUTE_SQL -# component nameGet Destination Table Name or File Name -# input parameters : - # ProcessName -# output parameters : -Package_Get_Destination_Table_Name_or_File_Name = f"""DECLARE VARIABLE V_TblNm STRING; -call aud.spGetPackageDesTbl( {ProcessName}, V_TblNm ); - -SELECT V_TblNm AS DestinationTable;""" -spark.sql(Package_Get_Destination_Table_Name_or_File_Name) - -# COMMAND ---------- - -# Processing node Package\Truncate Staging Table, type EXECUTE_SQL -# component nameTruncate Staging Table -# input parameters : -# output parameters : -Package_Truncate_Staging_Table = f"""TRUNCATE TABLE staging.customers;""" -spark.sql(Package_Truncate_Staging_Table) - -# COMMAND ---------- - -# Processing node Package\Load Customer Data, type EXECUTE_SQL -# component nameLoad Customer Data -# input parameters : -# output parameters : -Package_Load_Customer_Data = f"""INSERT INTO staging.customers -(CustomerID, CustomerName, Email, Status, ProcessDate) -SELECT - CustomerID, - CustomerName, - Email, - Status, - current_timestamp() AS ProcessDate -FROM source.customers -WHERE Status = 'Active';""" -spark.sql(Package_Load_Customer_Data) - -# COMMAND ---------- - -# Processing node Package\Update Control Table, type EXECUTE_SQL -# component nameUpdate Control Table -# input parameters : - # ProcessName -# output parameters : -Package_Update_Control_Table = f"""INSERT INTO control.load_log -(process_name, rows_processed, process_date) -SELECT - {ProcessName} AS process_name, - COUNT(*) AS rows_processed, - current_timestamp() AS process_date -FROM dw.customers -WHERE ProcessDate >= CAST(current_timestamp() AS DATE);""" -spark.sql(Package_Update_Control_Table) -``` - ---- - -## Next Steps - -1. **Export SSIS packages** to DTSX files -2. **Run conversion** using Lakebridge CLI: - ```bash - databricks labs lakebridge transpile \ - --source-dialect ssis \ - --input-source /path/to/ssis/packages \ - --output-folder /output/sparksql \ - --target-technology sparksql - ``` -3. **Review generated notebooks** for conversion warnings -4. **Configure Databricks secrets** for connection strings -5. **Test with sample data** in Databricks -6. **Deploy workflows** to production - -For more information, see: -- [Main ETL Conversion Guide](../../) -- [BladeBridge Configuration](../../pluggable_transpilers/bladebridge/bladebridge_configuration) -- [Transpile CLI Reference](../../) diff --git a/docs/lakebridge/docs/transpile/source_systems/ssis/examples.mdx b/docs/lakebridge/docs/transpile/source_systems/ssis/examples.mdx new file mode 100644 index 0000000000..bbd69873c2 --- /dev/null +++ b/docs/lakebridge/docs/transpile/source_systems/ssis/examples.mdx @@ -0,0 +1,341 @@ +--- +sidebar_position: 3 +title: SSIS Conversion Examples +toc_min_heading_level: 2 +toc_max_heading_level: 3 +--- + +# SSIS Conversion Examples + +Before/after examples showing how SSIS constructs are converted to Databricks. For the component support matrix, see [SSIS Supported Components](/docs/transpile/source_systems/ssis/supported_components). + +--- + +## Data Flow Transformations + +### Derived Column + +**SSIS:** +``` +Derived Column Transformation + Columns: + FullName = FirstName + " " + LastName + AgeGroup = Age < 30 ? "Young" : (Age < 60 ? "Middle" : "Senior") + ProcessDate = GETDATE() +``` + +**Databricks:** +```python +Derived_Column_Add_Columns = f""" +SELECT + *, + CONCAT(FirstName, ' ', LastName) AS FullName, + CASE + WHEN Age < 30 THEN 'Young' + WHEN Age < 60 THEN 'Middle' + ELSE 'Senior' + END AS AgeGroup, + CURRENT_TIMESTAMP() AS ProcessDate +FROM source_table +""" +Derived_Column_Add_Columns = spark.sql(Derived_Column_Add_Columns) +Derived_Column_Add_Columns.createOrReplaceTempView('Derived_Column_Add_Columns') +``` + +--- + +### Conditional Split + +**SSIS:** +``` +Conditional Split Transformation + Outputs: + HighValue: Amount > 1000 + MediumValue: Amount > 100 AND Amount <= 1000 + LowValue: Default Output +``` + +**Databricks:** +```python +Conditional_Split_High_Value = f""" +SELECT * FROM source_table WHERE Amount > 1000 +""" +Conditional_Split_High_Value = spark.sql(Conditional_Split_High_Value) +Conditional_Split_High_Value.createOrReplaceTempView('Conditional_Split_High_Value') + +Conditional_Split_Medium_Value = f""" +SELECT * FROM source_table WHERE Amount > 100 AND Amount <= 1000 +""" +Conditional_Split_Medium_Value = spark.sql(Conditional_Split_Medium_Value) +Conditional_Split_Medium_Value.createOrReplaceTempView('Conditional_Split_Medium_Value') + +Conditional_Split_Low_Value = f""" +SELECT * FROM source_table WHERE Amount <= 100 +""" +Conditional_Split_Low_Value = spark.sql(Conditional_Split_Low_Value) +Conditional_Split_Low_Value.createOrReplaceTempView('Conditional_Split_Low_Value') +``` + +--- + +### Lookup Transformation + +**SSIS:** +``` +Lookup Transformation + Reference Table: DimCustomerType + Join Columns: CustomerTypeID = TypeID + Lookup Columns: TypeName, TypeDescription + Cache Mode: Full Cache +``` + +**Databricks:** +```python +Lookup_Customer_Type = f""" +SELECT + src.*, + ref.TypeName, + ref.TypeDescription +FROM source_table src +LEFT JOIN dim.customer_type ref + ON src.CustomerTypeID = ref.TypeID +""" +Lookup_Customer_Type = spark.sql(Lookup_Customer_Type) +Lookup_Customer_Type.createOrReplaceTempView('Lookup_Customer_Type') +``` + +--- + +## Variables and Expressions + +### SSIS Variables + +**SSIS:** +``` +Variables: + User::MaxProcessDate (DateTime) + User::RowCount (Int32) + User::SourceFolder (String) +``` + +**Databricks:** +```python +V_MaxProcessDate = f'2024-01-01' +V_SourceFolder = f'/mnt/source/' + +Incremental_Data_Query = f""" +SELECT * +FROM source_table +WHERE ProcessDate > '{V_MaxProcessDate}' +""" +spark.sql(Incremental_Data_Query) +``` + +--- + +### SSIS Expressions + +**SSIS:** +``` +@[User::SourceFolder] + "customers_" + +(DT_WSTR, 8) DATEPART("yyyy", GETDATE()) + +RIGHT("0" + (DT_WSTR, 2) DATEPART("mm", GETDATE()), 2) + ".csv" +``` + +**Databricks:** +```python +from datetime import datetime + +V_SourceFolder = f'/mnt/source/' +current_date = datetime.now() +year = current_date.strftime("%Y") +month = current_date.strftime("%m") +V_FilePath = f"{V_SourceFolder}customers_{year}{month}.csv" + +Read_Customers_Data = f"""SELECT * FROM csv.`{V_FilePath}`""" +Read_Customers_Data = spark.sql(Read_Customers_Data) +Read_Customers_Data.createOrReplaceTempView('Read_Customers_Data') +``` + +--- + +## Control Flow Patterns + +### ForEach Loop Container + +**SSIS:** +``` +ForEach Loop Container (File Enumerator) + Folder: C:\Data\Input\ + Files: *.csv + Tasks: Data Flow, Execute SQL (log processing) +``` + +**Databricks:** +```python +V_InputFolder = f'/mnt/data/input/' + +ForEach_Read_All_Files = f""" +SELECT + *, + input_file_name() AS source_file, + CURRENT_TIMESTAMP() AS process_date +FROM csv.`{V_InputFolder}*.csv` +""" +ForEach_Read_All_Files = spark.sql(ForEach_Read_All_Files) +ForEach_Read_All_Files.createOrReplaceTempView('ForEach_Read_All_Files') + +ForEach_Insert_Customer_Data = f""" +INSERT INTO staging.customer_data +SELECT * FROM ForEach_Read_All_Files +""" +spark.sql(ForEach_Insert_Customer_Data) + +ForEach_Log_Processing = f""" +INSERT INTO logs.processing_log +SELECT source_file AS file_path, COUNT(*) AS row_count, MAX(process_date) AS process_time +FROM ForEach_Read_All_Files +GROUP BY source_file +""" +spark.sql(ForEach_Log_Processing) +``` + +--- + +### Execute SQL Task + +**SSIS:** +``` +Execute SQL Task + SQL Statement: + TRUNCATE TABLE staging.customer_temp; + INSERT INTO staging.customer_temp + SELECT * FROM staging.customer_stage WHERE process_date >= ?; + Parameter Mapping: + User::MaxProcessDate → Parameter 0 +``` + +**Databricks:** +```python +V_MaxProcessDate = f'2024-01-01' + +Execute_SQL_Truncate = f"""TRUNCATE TABLE staging.customer_temp""" +spark.sql(Execute_SQL_Truncate) + +Execute_SQL_Insert = f""" +INSERT INTO staging.customer_temp +SELECT * FROM staging.customer_stage +WHERE process_date >= '{V_MaxProcessDate}' +""" +spark.sql(Execute_SQL_Insert) +``` + +--- + +## Script Component Conversion + +### Script Task (Control Flow) + +**SSIS (C#):** +```csharp +public void Main() +{ + string sourceFolder = Dts.Variables["User::SourceFolder"].Value.ToString(); + int fileCount = Directory.GetFiles(sourceFolder, "*.csv").Length; + Dts.Variables["User::FileCount"].Value = fileCount; + Dts.TaskResult = (int)ScriptResults.Success; +} +``` + +**Databricks (Python):** +```python +V_SourceFolder = f'/mnt/source/' +file_list = dbutils.fs.ls(V_SourceFolder) +V_FileCount = str(len([f for f in file_list if f.path.endswith('.csv')])) +``` + +--- + +### Script Component (Data Flow) + +**SSIS (C#):** +```csharp +public override void Input0_ProcessInputRow(Input0Buffer Row) +{ + if (Row.Amount > 1000) { + Row.PriorityFlag = "HIGH"; + Row.DiscountRate = 0.15; + } else { + Row.PriorityFlag = "NORMAL"; + Row.DiscountRate = 0.05; + } + Row.ProcessedDate = DateTime.Now; +} +``` + +**Databricks:** +```python +Script_Component_Custom_Logic = f""" +SELECT + *, + CASE WHEN Amount > 1000 THEN 'HIGH' ELSE 'NORMAL' END AS PriorityFlag, + CASE WHEN Amount > 1000 THEN 0.15 ELSE 0.05 END AS DiscountRate, + CURRENT_TIMESTAMP() AS ProcessedDate +FROM source_table +""" +Script_Component_Custom_Logic = spark.sql(Script_Component_Custom_Logic) +Script_Component_Custom_Logic.createOrReplaceTempView('Script_Component_Custom_Logic') +``` + +--- + +## Complete Package Example + +**SSIS Package: [LoadCustomerData.dtsx](/downloads/LoadCustomerData.dtsx)** + +Control Flow: +1. Execute SQL Task: Get Destination Table Name +2. Execute SQL Task: Truncate Staging Table +3. Execute SQL Task: Load Customer Data +4. Execute SQL Task: Update Control Table + +**Converted Databricks Notebook:** + +```python +# Databricks notebook source + +ProcessName = f'CustomerETL' + +# COMMAND ---------- +# Get Destination Table Name +Package_Get_Destination_Table_Name = f""" +DECLARE VARIABLE V_TblNm STRING; +CALL aud.spGetPackageDesTbl({ProcessName}, V_TblNm); +SELECT V_TblNm AS DestinationTable; +""" +spark.sql(Package_Get_Destination_Table_Name) + +# COMMAND ---------- +# Truncate Staging Table +spark.sql(f"""TRUNCATE TABLE staging.customers;""") + +# COMMAND ---------- +# Load Customer Data +Package_Load_Customer_Data = f""" +INSERT INTO staging.customers (CustomerID, CustomerName, Email, Status, ProcessDate) +SELECT CustomerID, CustomerName, Email, Status, current_timestamp() AS ProcessDate +FROM source.customers +WHERE Status = 'Active'; +""" +spark.sql(Package_Load_Customer_Data) + +# COMMAND ---------- +# Update Control Table +Package_Update_Control_Table = f""" +INSERT INTO control.load_log (process_name, rows_processed, process_date) +SELECT {ProcessName} AS process_name, COUNT(*) AS rows_processed, current_timestamp() AS process_date +FROM dw.customers +WHERE ProcessDate >= CAST(current_timestamp() AS DATE); +""" +spark.sql(Package_Update_Control_Table) +``` diff --git a/docs/lakebridge/docs/transpile/source_systems/ssis/index.mdx b/docs/lakebridge/docs/transpile/source_systems/ssis/index.mdx new file mode 100644 index 0000000000..e7f51c7767 --- /dev/null +++ b/docs/lakebridge/docs/transpile/source_systems/ssis/index.mdx @@ -0,0 +1,87 @@ +--- +sidebar_position: 1 +title: SSIS +toc_min_heading_level: 2 +toc_max_heading_level: 3 +--- + +# Microsoft SSIS to Databricks + +## Conversion Information + +- **Transpiler:** BladeBridge +- **Available target:** Databricks notebooks (experimental) + +### Supported SSIS Versions + +- SQL Server 2012, 2014, 2016, 2017, 2019, 2022 +- Azure Data Factory SSIS Integration Runtime + +### Input Requirements + +Export your SSIS packages as DTSX files: + +1. **Solution Export:** Export from Visual Studio / SQL Server Data Tools (SSDT) +2. **File System Packages:** Direct DTSX file access +3. **SSISDB Export:** Extract from SSIS catalog + +```sql +-- Extract package from SSISDB catalog +DECLARE @packageData VARBINARY(MAX) +SELECT @packageData = [packagedata] +FROM [SSISDB].[catalog].[packages] +WHERE [name] = 'YourPackageName' +-- Save to file system for conversion +``` + +--- + +## Running the Conversion + +```bash +databricks labs lakebridge transpile \ + --source-dialect ssis \ + --input-source /path/to/ssis/packages \ + --output-folder /output/sparksql \ + --target-technology sparksql +``` + +The transpiler recursively scans the input directory for `.dtsx` files and generates Databricks notebook equivalents in the output folder. + +:::warning Script Component Limitations +SSIS **Script Task** and **Script Component** contain C# or VB.NET code bodies that cannot be automatically converted. The converter preserves the logic structure, but the actual implementation must be rewritten in Python. See [Supported Components](/docs/transpile/source_systems/ssis/supported_components#control-flow-orchestration---unsupported) for details. +::: + +--- + +## What Gets Converted + +| SSIS Concept | Databricks Equivalent | +|---|---| +| Control Flow Tasks | Notebook cells / `dbutils.notebook.run()` | +| Data Flow Task | Spark SQL temp views | +| Variables | Python variables | +| SSIS Expressions | Python f-strings / `spark.sql()` calls | +| Connection Managers | JDBC `spark.read.format("jdbc")` | +| ForEach Loop | Wildcard file reads with `input_file_name()` | +| Script Task | Python with `dbutils` | + +For the full list of supported and unsupported components, see [SSIS Supported Components](/docs/transpile/source_systems/ssis/supported_components). + +For conversion examples with before/after code, see [SSIS Conversion Examples](/docs/transpile/source_systems/ssis/examples). + +--- + +## Next Steps + +1. Export SSIS packages to DTSX files +2. Run conversion (command above) +3. Review generated notebooks for conversion warnings +4. Configure Databricks secrets for connection strings +5. Test with sample data in Databricks +6. Deploy workflows to production + +For more information, see: +- [SSIS Supported Components](/docs/transpile/source_systems/ssis/supported_components) +- [SSIS Conversion Examples](/docs/transpile/source_systems/ssis/examples) +- [BladeBridge Configuration](/docs/transpile/pluggable_transpilers/bladebridge/bladebridge_configuration) diff --git a/docs/lakebridge/docs/transpile/source_systems/ssis/supported_components.mdx b/docs/lakebridge/docs/transpile/source_systems/ssis/supported_components.mdx new file mode 100644 index 0000000000..cc0e393813 --- /dev/null +++ b/docs/lakebridge/docs/transpile/source_systems/ssis/supported_components.mdx @@ -0,0 +1,111 @@ +--- +sidebar_position: 2 +title: SSIS Supported Components +toc_min_heading_level: 2 +toc_max_heading_level: 3 +--- + +# SSIS Supported Components + +This page lists all SSIS components and their Databricks equivalents. For usage and conversion examples, see [SSIS Conversion Examples](/docs/transpile/source_systems/ssis/examples). + +--- + +## Control Flow (Orchestration) - Supported + +| SSIS Component | Microsoft Name | Spark Equivalent | Notes | +|---|---|---|---| +| **Data Flow Task** | Microsoft.DataFlowTask | Spark SQL temp views | Core data transformations | +| **Execute SQL Task** | Microsoft.ExecuteSQLTask | Spark SQL statements | SQL execution with parameter mapping | +| **Execute Package Task** | Microsoft.ExecutePackageTask | `dbutils.notebook.run()` | Nested notebook execution | +| **File System Task** | Microsoft.FileSystemTask | `dbutils.fs` commands | File operations (copy, move, delete, rename) | +| **Script Task** | Microsoft.ScriptTask | Python code with SQL | C#/VB.NET scripts converted to Python/SQL | +| **For Loop Container** | STOCK:FORLOOP | SQL iteration pattern | Iteration with counter | +| **Foreach Loop Container** | STOCK:FOREACHLOOP | SQL wildcard file reading | File/folder iteration | +| **Sequence Container** | STOCK:SEQUENCE | Function or notebook section | Grouping tasks | +| **Execute Process Task** | Microsoft.ExecuteProcess | `subprocess.run()` | External process execution | +| **Extensible File Task** | ExtensibleFileTask | `dbutils.fs` commands | Extended file operations | + +--- + +## Control Flow (Orchestration) - Unsupported + +The following components are **not supported** and require manual conversion: + +| SSIS Component | Microsoft Name | Reason | +|---|---|---| +| **Analysis Services Execute DDL** | Microsoft.AnalysisServicesExecuteDDLTask | SSAS-specific; migrate to Delta/SQL manually | +| **Analysis Services Processing** | Microsoft.AnalysisServicesProcessingTask | SSAS-specific; migrate to Delta/SQL manually | +| **Bulk Insert Task** | Microsoft.BulkInsertTask | Use Data Flow Task with JDBC/Delta | +| **Data Profiling Task** | Microsoft.DataProfilingTask | Use Databricks Data Profile UI or custom profiling | +| **FTP Task** | Microsoft.FTPTask | Implement using Python `ftplib` or `dbutils.fs` | +| **Message Queue Task** | Microsoft.MessageQueueTask | Requires custom Kafka/Event Hub integration | +| **Send Mail Task** | Microsoft.SendMailTask | Use Databricks job notifications or Python `smtplib` | +| **Web Service Task** | Microsoft.WebServiceTask | Implement using Python `requests` | +| **WMI Data Reader Task** | Microsoft.WMIDataReaderTask | Windows-specific; requires alternative monitoring | +| **XML Task** | Microsoft.XMLTask | Implement using Python `xml.etree` or PySpark XML | + +:::warning Script Task Conversion +**Script Task** contains C# or VB.NET code bodies that cannot be automatically converted. The converter preserves the logic structure but the implementation must be rewritten in Python manually. +::: + +--- + +## Data Flow (Transformation) - Supported + +### Sources + +| SSIS Component | Microsoft Name | Spark Equivalent | Notes | +|---|---|---|---| +| **OLE DB Source** | Microsoft.OLEDBSource | `spark.read.format("jdbc")` | Database reads via JDBC | +| **Flat File Source** | Microsoft.FlatFileSource | `spark.read.csv()` or SQL `csv.\`path\`` | CSV, delimited, fixed-width files | +| **Excel Source** | Microsoft.ExcelSource | `spark.read.format("excel")` | Excel file reads | +| **Raw File Source** | Microsoft.RawSource | `spark.read.format("parquet")` | SSIS raw files converted to Parquet | + +### Transformations + +| SSIS Component | Microsoft Name | Spark Equivalent | Notes | +|---|---|---|---| +| **Aggregate** | Microsoft.Aggregate | `GROUP BY` with aggregation | Sum, count, avg, min, max | +| **Audit** | Microsoft.Audit | Add columns in SELECT | Add audit columns (timestamp, user, etc.) | +| **Cache Transform** | Microsoft.Cache | Temp views or CTEs | Cache data for lookups | +| **Character Map** | Microsoft.CharacterMap | String functions | upper, lower, etc. | +| **Conditional Split** | Microsoft.ConditionalSplit | Multiple `WHERE` clauses | Route rows by conditions | +| **Copy Column** | Microsoft.CopyColumn | Column in SELECT | Duplicate columns | +| **Data Conversion** | Microsoft.DataConvert | `CAST()` | Type conversions | +| **Derived Column** | Microsoft.DerivedColumn | Calculated columns in SELECT | Column transformations | +| **Lookup** | Microsoft.Lookup | `LEFT JOIN` | Reference data lookup | +| **Merge** | Microsoft.Merge | `UNION` | Merge sorted datasets | +| **Merge Join** | Microsoft.MergeJoin | `JOIN` | Sorted input joins | +| **Multicast** | Microsoft.Multicast | Temp view reuse | Send data to multiple outputs | +| **OLE DB Command** | Microsoft.OLEDBCommand | Row-by-row SQL execution | Row-level SQL | +| **Percentage Sampling** | Microsoft.PercentageSampling | `TABLESAMPLE` | Statistical sampling | +| **Pivot** | Microsoft.Pivot | `PIVOT` clause | Pivot operations | +| **Row Count** | Microsoft.RowCount | `COUNT(*)` | Count rows into variable | +| **Script Component** | Microsoft.SqlServer.Dts.Pipeline.ScriptComponentHost | SQL UDFs or CASE statements | Custom transformations | +| **Slowly Changing Dimension** | Microsoft.SCD | `MERGE` with Delta Lake | SCD Type 1, 2, 3 patterns | +| **Sort** | Microsoft.Sort | `ORDER BY` | Sorting | +| **Union All** | Microsoft.UnionAll | `UNION ALL` | Combine datasets | +| **Unpivot** | Microsoft.UnPivot | `UNPIVOT` or `STACK()` | Unpivot | + +### Destinations + +| SSIS Component | Microsoft Name | Spark Equivalent | Notes | +|---|---|---|---| +| **OLE DB Destination** | Microsoft.OLEDBDestination | `INSERT INTO` SQL or Delta Lake | Database writes | +| **Flat File Destination** | Microsoft.FlatFileDestination | `df.write.csv()` or SQL INSERT | CSV/delimited writes | +| **Excel Destination** | Microsoft.ExcelDestination | `df.write.format('excel')` | Excel writes | +| **Raw File Destination** | Microsoft.RawDestination | `df.write.format('parquet')` | Parquet writes | + +--- + +## Data Flow (Transformation) - Unsupported + +| SSIS Component | Microsoft Name | Reason | +|---|---|---| +| **Export Column** | Microsoft.ExportColumn | Implement using custom UDF with file writing | +| **Import Column** | Microsoft.ImportColumn | Implement using custom UDF with file reading | + +:::warning Script Component Conversion +The **Script Component** contains C# or VB.NET row-by-row processing logic that cannot be automatically converted. The converter identifies the component but the actual C#/VB code must be rewritten as SQL UDFs or CASE statements, then tested thoroughly since row-by-row logic often needs redesign for set-based SQL. +::: diff --git a/docs/lakebridge/docusaurus.config.ts b/docs/lakebridge/docusaurus.config.ts index be247da3b5..0c80bee2d4 100644 --- a/docs/lakebridge/docusaurus.config.ts +++ b/docs/lakebridge/docusaurus.config.ts @@ -196,7 +196,7 @@ const config: Config = { label: "Overview" }, { - to: '/docs/installation/', + to: '/docs/getting_started/', position: 'left', label: "Get Started" }, diff --git a/docs/lakebridge/src/pages/index.tsx b/docs/lakebridge/src/pages/index.tsx index 023c69064a..2a87fffa76 100644 --- a/docs/lakebridge/src/pages/index.tsx +++ b/docs/lakebridge/src/pages/index.tsx @@ -36,7 +36,7 @@ const Hero = () => { />