diff --git a/.build-constraints.txt b/.build-constraints.txt index 90731efa50..992756186a 100644 --- a/.build-constraints.txt +++ b/.build-constraints.txt @@ -1,13 +1,13 @@ hatchling==1.29.0 \ --hash=sha256:50af9343281f34785fab12da82e445ed987a6efb34fd8c2fc0f6e6630dbcc1b0 \ --hash=sha256:793c31816d952cee405b83488ce001c719f325d9cda69f1fc4cd750527640ea6 -packaging==26.0 \ - --hash=sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4 \ - --hash=sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529 +packaging==26.2 \ + --hash=sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e \ + --hash=sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661 # via hatchling -pathspec==1.0.4 \ - --hash=sha256:0210e2ae8a21a9137c0d470578cb0e595af87edaa6ebf12ff176f14a02e0e645 \ - --hash=sha256:fb6ae2fd4e7c921a165808a552060e722767cfa526f99ca5156ed2ce45a5c723 +pathspec==1.1.1 \ + --hash=sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a \ + --hash=sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189 # via hatchling pluggy==1.6.0 \ --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ @@ -62,7 +62,7 @@ tomli==2.4.1 ; python_full_version < '3.11' \ --hash=sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9 \ --hash=sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049 # via hatchling -trove-classifiers==2026.1.14.14 \ - --hash=sha256:00492545a1402b09d4858605ba190ea33243d361e2b01c9c296ce06b5c3325f3 \ - --hash=sha256:1f9553927f18d0513d8e5ff80ab8980b8202ce37ecae0e3274ed2ef11880e74d +trove-classifiers==2026.6.1.19 \ + --hash=sha256:ab4c4ec93cc4a4e7815fa759906e05e6bb3f2fbd92ea0f897288c6a43efd15b3 \ + --hash=sha256:c5132b4b61a829d11cfbd2d72e97f20a45ed6edb95e45c5efdeb5e00836b2745 # via hatchling diff --git a/.github/actions/jfrog-auth/action.yml b/.github/actions/jfrog-auth/action.yml index bf7b809abb..d4979d227b 100644 --- a/.github/actions/jfrog-auth/action.yml +++ b/.github/actions/jfrog-auth/action.yml @@ -1,8 +1,8 @@ name: 'Authenticate for JFrog' description: 'Authenticate with JFrog using OIDC based on the GitHub repository.' # Some things to note: -# - Run this _after_ installing any tools that need to use JFrog; auth is configured all the (supported) tools that it -# detects. +# - Run this _after_ installing any tools that need to use JFrog; auth is configured for all the (supported) tools that +# it detects. # - Where possible we avoid exposing tokens in environment variables, preferring to write them into files instead. # (Tokens in environment variables tend to be more exposed and easier to leak than those written into files.) # @@ -18,17 +18,22 @@ runs: name: Authenticate against JFrog shell: bash run: | + if [[ -z "${ACTIONS_ID_TOKEN_REQUEST_URL}" ]] || [[ -z "${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" ]] + then + printf '::error::%s\n' 'This action uses OIDC: job must have "id-token: write" permission' + exit 1 + fi "${GITHUB_ACTION_PATH}/jfrog-auth" "${ACTIONS_ID_TOKEN_REQUEST_URL}" "${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" - id: detect-cmds - name: Detecting python package/project managers. + name: Detecting package/project managers. shell: bash run: | - for cmd in bun mvn npm pip3 uv + for cmd in bun coursier mvn npm pip3 sbt uv do - command -v "${cmd}" > /dev/null && found=true || found=false - printf '::debug::%s\n' "Found ${cmd}: ${found}" - printf '%s=%s\n' "command_${cmd}" "${found}" >> "${GITHUB_OUTPUT}" + command -v "${cmd}" > /dev/null && found=true || found=false + printf '::debug::%s\n' "Found ${cmd}: ${found}" + printf '%s=%s\n' "command_${cmd}" "${found}" >> "${GITHUB_OUTPUT}" done - name: Configure bun for JFrog @@ -38,7 +43,7 @@ runs: JFROG_ACCESS_TOKEN: "${{ steps.jfrog-auth.outputs.jfrog-access-token }}" run: | umask 077 - cat > ~/.bunfig.toml << EOF + cat > ~/.bunfig.toml << 'EOF' [install] registry = { url = "https://databricks.jfrog.io/artifactory/api/npm/db-npm/", token = "$jfrog_access_token" } EOF @@ -51,24 +56,28 @@ runs: # There are currently the following issues with JFrog: # - The default set of CAs doesn't seem to cover the ones used by our JFrog instance. # - The JSON metadata returned for some NPM artefacts can be invalid JSON. - printf '::warning::%s\n' 'JFrog has compatibility issues with bun; it probably won't work.' + printf '::warning::%s\n' 'JFrog has compatibility issues with bun; it will probably not work.' - - name: Configure pip for JFrog - if: "${{ steps.detect-cmds.outputs.command_pip3 == 'true' }}" + - name: Configure coursier for JFrog + # Note: SBT bootstrapping uses Coursier internally. + if: "${{ steps.detect-cmds.outputs.command_coursier == 'true' || + steps.detect-cmds.outputs.command_sbt == 'true' }}" shell: bash env: JFROG_ACCESS_TOKEN: "${{ steps.jfrog-auth.outputs.jfrog-access-token }}" run: | umask 077 - cat > "$RUNNER_TEMP/.pip.conf" < "${RUNNER_TEMP}/.coursier-credentials.properties" << EOF + jfrog.host=databricks.jfrog.io + jfrog.realm=Artifactory Realm + jfrog.username=gha-service-account + jfrog.password=${JFROG_ACCESS_TOKEN} EOF - printf '%s=%s\n' 'PIP_CONFIG_FILE' "${RUNNER_TEMP}/.pip.conf" >> "${GITHUB_ENV}" - printf '::debug::%s\n' 'Configured JFrog access for pip.' + printf '%s=%s\n' 'COURSIER_CREDENTIALS' "${RUNNER_TEMP}/.coursier-credentials.properties" >> "${GITHUB_ENV}" + printf '::debug::%s\n' 'Configured JFrog access for Coursier.' - name: Configure Maven for JFrog - if: "${{ steps.detect-cmds.outputs.command_uv == 'true' }}" + if: "${{ steps.detect-cmds.outputs.command_mvn == 'true' }}" shell: bash env: JFROG_ACCESS_TOKEN: "${{ steps.jfrog-auth.outputs.jfrog-access-token }}" @@ -95,17 +104,20 @@ runs: EOF printf '::debug::%s\n' 'Configured JFrog access for maven.' - - name: Configure uv for JFrog - if: "${{ steps.detect-cmds.outputs.command_uv == 'true' }}" + - name: Configure netrc for JFrog + if: "${{ steps.detect-cmds.outputs.command_pip3 == 'true' || + steps.detect-cmds.outputs.command_uv == 'true' }}" shell: bash env: JFROG_ACCESS_TOKEN: "${{ steps.jfrog-auth.outputs.jfrog-access-token }}" - UV_INDEX_URL: 'https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple' run: | - uv auth login "${UV_INDEX_URL}" --username gha-service-account --password "${JFROG_ACCESS_TOKEN}" - printf '%s=%s\n' 'UV_INDEX_URL' "${UV_INDEX_URL}" >> "${GITHUB_ENV}" - printf '%s=%s\n' 'UV_FROZEN' '1' >> "${GITHUB_ENV}" - printf '::debug::%s\n' 'Configured JFrog access for uv.' + umask 077 + cat > "${RUNNER_TEMP}/.netrc" << EOF + machine databricks.jfrog.io + login gha-service-account + password ${JFROG_ACCESS_TOKEN} + EOF + printf '%s=%s\n' 'NETRC' "${RUNNER_TEMP}/.netrc" >> "${GITHUB_ENV}" - name: Configure npm/yarn (classic) for JFrog if: "${{ steps.detect-cmds.outputs.command_npm == 'true' }}" @@ -121,3 +133,50 @@ runs: //databricks.jfrog.io/artifactory/api/npm/db-npm/:_authToken=${JFROG_ACCESS_TOKEN} EOF printf '::debug::%s\n' 'Configured JFrog access for npm/yarn (classic).' + + - name: Configure pip for JFrog + if: "${{ steps.detect-cmds.outputs.command_pip3 == 'true' }}" + shell: bash + run: | + cat > "${RUNNER_TEMP}/.pip.conf" << 'EOF' + [global] + index-url = https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + EOF + printf '%s=%s\n' 'PIP_CONFIG_FILE' "${RUNNER_TEMP}/.pip.conf" >> "${GITHUB_ENV}" + printf '::debug::%s\n' 'Configured JFrog access for pip.' + + - name: Configure sbt for JFrog + if: "${{ steps.detect-cmds.outputs.command_sbt == 'true' }}" + shell: bash + env: + JFROG_ACCESS_TOKEN: "${{ steps.jfrog-auth.outputs.jfrog-access-token }}" + run: | + umask 077 + mkdir -p ~/.sbt/1.0 + cat > ~/.sbt/repositories << 'EOF' + [repositories] + local + databricks-jfrog: https://databricks.jfrog.io/artifactory/db-maven/ + EOF + + cat > "${RUNNER_TEMP}/.sbt.credentials" << EOF + realm=Artifactory Realm + host=databricks.jfrog.io + user=gha-service-account + password=${JFROG_ACCESS_TOKEN} + EOF + + cat > ~/.sbt/1.0/global.sbt << 'EOF' + credentials += Credentials(file(sys.env("RUNNER_TEMP")) / ".sbt.credentials") + EOF + printf '::debug::%s\n' 'Configured JFrog access for SBT.' + + - name: Configure uv for JFrog + if: "${{ steps.detect-cmds.outputs.command_uv == 'true' }}" + shell: bash + env: + UV_INDEX_URL: 'https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple' + run: | + printf '%s=%s\n' 'UV_INDEX_URL' "${UV_INDEX_URL}" >> "${GITHUB_ENV}" + printf '%s=%s\n' 'UV_FROZEN' '1' >> "${GITHUB_ENV}" + printf '::debug::%s\n' 'Configured JFrog access for uv.' diff --git a/.github/actions/jfrog-auth/jfrog-auth b/.github/actions/jfrog-auth/jfrog-auth index 6a5ab856a4..8c5533487d 100755 --- a/.github/actions/jfrog-auth/jfrog-auth +++ b/.github/actions/jfrog-auth/jfrog-auth @@ -22,7 +22,7 @@ printf '::add-mask::%s\n' "${_id_token}" # Step 2: Exchange it for the JFrog access token. # printf '::debug::%s\n' "Exchanging OIDC identifier token for JFrog access token..." -_access_token=$(curl -sLS \ +_access_token=$(curl -fsSL \ --json "{\"grant_type\": \"urn:ietf:params:oauth:grant-type:token-exchange\", \"subject_token_type\":\"urn:ietf:params:oauth:token-type:id_token\", \"subject_token\": \"${_id_token}\", \"provider_name\": \"github-actions\"}" \ "https://databricks.jfrog.io/access/api/v1/oidc/token" | jq -r .access_token) diff --git a/.github/scripts/lint-uv b/.github/scripts/lint-uv new file mode 100755 index 0000000000..8ee7a597f6 --- /dev/null +++ b/.github/scripts/lint-uv @@ -0,0 +1,143 @@ +#!/bin/sh +# +# Check the uv settings of the project, to ensure dependencies are being updated properly. +# +# The intent here is that: +# +# - Lock-files should only be updated via the `make lock-dependencies` target. +# - We only re-lock when: +# - Updating the project dependencies in some way. +# - Refreshing the locked versions, where that is the sole purpose of the PR. +# - Developer environment information should not be present in the lock-file; it should be usable by the public. +# +# The checks that we perform to enforce this are: +# - The only registry we should see in uv.lock is "https://pypi.org/simple". +# - The uv.lock file must be consistent with pyproject.toml. +# - If changes to uv.lock are present, there must be no other files changed in the PR _or_ pyproject.toml must also +# be updated. +# - The same applies to .build-constraints.txt. +# + +set -eu + +PROGNAME="${0##*/}" +die() { + _exitcode="$1" + shift + printf "%s: %s\n" "${PROGNAME}" "$*" 1>&2 + + exit "${_exitcode}" +} + +# Locate the root of the project. +PROJECT_ROOT="$(git rev-parse --show-toplevel)" + +# Set up a scratch directory for temporary files during the check. +SCRATCH="$(mktemp -d -t "${PROGNAME}.XXX")" +trap 'rm -fr "${SCRATCH}"' EXIT + +# This script handles the locked/frozen flags for uv itself. +unset UV_FROZEN UV_LOCKED + +tomlq() { + # Assumes typical project setup: tomlq is present (and locked). + uv run --frozen --quiet --group yq tomlq "$@" +} + +# +# Check 1: Verify that uv.lock refers only to PyPI as a registry. +# +check_lock_registry() { + # Collect all registry URLs present in uv.lock, except for the only one we allow. + tomlq --raw-output \ + '[.. | objects | .source.registry // empty] - ["https://pypi.org/simple"] | unique | .[]' \ + "${PROJECT_ROOT}/uv.lock" > "${SCRATCH}/foreign-registries.txt" + if [ -s "${SCRATCH}/foreign-registries.txt" ] + then + printf "%s: Detected non-PyPI registries:\n" "${PROGNAME}" + sed 's/^/ /' "${SCRATCH}/foreign-registries.txt" + die 1 "uv.lock may only contain PyPI registries." + fi +} +check_lock_registry + +# +# Check 2: Verify that uv.lock is consistent with pyproject.toml. +# + +detect_index_url() { + # Annoyingly, there is no way to query from uv the index URL it's going to use; the only approach is to use a fake + # project and ask to resolve, then dig the index URL out of the results. + mkdir -p "${SCRATCH}/uvtest" + cat > "${SCRATCH}/uvtest/pyproject.toml" << 'EOF' +[project] +name = "uvtest" +version = "0" +requires-python = ">=3.10" +dependencies = ["packaging"] +EOF + uv lock --quiet --project "${SCRATCH}/uvtest" + tomlq --raw-output 'first(.package[].source.registry // empty)' "${SCRATCH}/uvtest/uv.lock" +} + +check_lock_consistency() { + # We can't check uv.lock directly, because we can't reach the registry recorded in there. Instead we make a shadow + # version of this project and check that. + index_url="$(detect_index_url)" + [ -n "${index_url}" ] || die 2 "Could not detect the URL to use for PyPI." + mkdir -p "${SCRATCH}/shadow-project" + ln -s \ + "${PROJECT_ROOT}/README.md" \ + "${PROJECT_ROOT}/pyproject.toml" \ + "${PROJECT_ROOT}/src" \ + "${SCRATCH}/shadow-project/" + # shellcheck disable=SC2016 # $index_url is a jq variable, bound via --arg + tomlq --toml-output --arg index_url "${index_url}" \ + '(.. | objects | .source | select(.registry? == "https://pypi.org/simple")).registry = $index_url' \ + "${PROJECT_ROOT}/uv.lock" > "${SCRATCH}/shadow-project/uv.lock" + uv lock --project "${SCRATCH}/shadow-project" --check --quiet || + die 2 "Inconsistency detected between pyproject.toml and uv.lock." +} +check_lock_consistency + +# +# Check 3: Detect if uv.lock or .build-constraints.txt have been updated with pyproject.toml or on their own. +# + +base_rev() { + # A few situations to deal with, in order of priority: + # - In CI, we set BASE_SHA for pull requests and the merge queue. + # - In CI, we also have the push to main where there's no base: in this case we can use GITHUB_REF and diff will + # end up a no-op (because there are no differences). + # - For local dev, fall-back to `main` which by convention is always available. + _base_rev="${BASE_SHA:-${GITHUB_REF:-main}}" + git rev-parse --verify --quiet "${_base_rev}" > /dev/null || + die 3 "Base revision of repository is not available: ${_base_rev}" + printf "%s" "${_base_rev}" +} +merge_base() { + # Figure out the merge-base between this branch and verify it's available. (CI checkout might be shallow.) + _tip="$1" + _base_rev=$(base_rev) + if ! git merge-base "${_base_rev}" "${_tip}" 2> /dev/null + then + die 4 "Repository is incomplete, merge base between ${_base_rev} and ${_tip} is not available." + fi +} + +check_lockfile_updates() { + _merge_base=$(merge_base HEAD) + git diff "${_merge_base}..HEAD" --name-only > "${SCRATCH}/changed-files.txt" + + if grep -qxF -e 'uv.lock' -e '.build-constraints.txt' "${SCRATCH}/changed-files.txt" + then + # Lock-files changed: either pyproject.toml has to be changed, or nothing else is allowed. + if grep -vxF -e 'uv.lock' -e '.build-constraints.txt' "${SCRATCH}/changed-files.txt" > "${SCRATCH}/other-files.txt" && ! grep -qxF 'pyproject.toml' "${SCRATCH}/other-files.txt" + then + printf "%s: Detected changes to uv.lock and/or .build-constraints.txt mixed with changes to:\n" "${PROGNAME}" + sed -e 's/^/ /' "${SCRATCH}/other-files.txt" + die 5 "Lock-files must be updated either in conjunction with dependencies in pyproject.toml or on their own." + fi + fi +} +check_lockfile_updates diff --git a/.github/scripts/setup_local_oracle.sh b/.github/scripts/setup_local_oracle.sh deleted file mode 100644 index d0725c20bc..0000000000 --- a/.github/scripts/setup_local_oracle.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/bin/bash -# TODO: Replace this script with a GHA service container on the workflow. -set -Eeuo pipefail - -# Config -ORACLE_CONTAINER="${ORACLE_CONTAINER:-oracle-free}" -ORACLE_IMAGE="${ORACLE_IMAGE:-container-registry.oracle.com/database/free@sha256:481dbb4a1ea7cac6aadd354ff42b48fb7e4df955725158f237ad58c8fca1f458}" # latest-lite -ORACLE_PORT="${ORACLE_PORT:-1521}" -ORACLE_PWD="${ORACLE_PWD:?export ORACLE_PWD for SYS}" -ORACLE_SID="${ORACLE_SID:-FREEPDB1}" - -# Dependencies -command -v docker >/dev/null || { printf "Docker not installed\n" >&2; exit 2; } - -# Image present? -docker image inspect "${ORACLE_IMAGE}" >/dev/null 2>&1 || docker pull "${ORACLE_IMAGE}" - -# Start container if needed -if docker ps --format '{{.Names}}' | grep -qx "${ORACLE_CONTAINER}"; then - : -elif docker ps -a --format '{{.Names}}' | grep -qx "${ORACLE_CONTAINER}"; then - docker start "${ORACLE_CONTAINER}" >/dev/null -else - docker run --name "${ORACLE_CONTAINER}" \ - -p "${ORACLE_PORT}:1521" \ - -e ORACLE_PWD="${ORACLE_PWD}" \ - -d "${ORACLE_IMAGE}" >/dev/null - printf "Starting Oracle container...\n" -fi - -printf "Waiting up to 5 minutes for Oracle to be healthy...\n" -MAX_WAIT=300; WAIT_INTERVAL=5; waited=0 -while :; do - state="$(docker inspect -f '{{.State.Health.Status}}' "${ORACLE_CONTAINER}" 2>/dev/null || true)" - printf "health=%s waited=%ss\n" "${state:-unknown}" "${waited}" - [[ "$state" == "healthy" ]] && break - (( waited >= MAX_WAIT )) && { printf "ERROR: Oracle not healthy in 300s\n" >&2; exit 1; } - sleep "$WAIT_INTERVAL"; waited=$((waited + WAIT_INTERVAL)) -done -printf "Oracle is fully started.\n" - -# SQL bootstrap as SYSDBA, target SYSTEM schema -docker exec -i "${ORACLE_CONTAINER}" bash -lc \ -"sqlplus -L -s 'sys/${ORACLE_PWD}@localhost:${ORACLE_PORT}/${ORACLE_SID} as sysdba'" <<'SQL' -WHENEVER SQLERROR EXIT SQL.SQLCODE -SET ECHO ON FEEDBACK ON PAGESIZE 200 LINESIZE 32767 SERVEROUTPUT ON - --- reconcile queries executes DBMS_CRYPTO -GRANT EXECUTE ON DBMS_CRYPTO TO SYSTEM; - --- work in SYSTEM, not SYS -ALTER SESSION SET CURRENT_SCHEMA=SYSTEM; - --- create table if not exists (guard ORA-00955) -BEGIN - EXECUTE IMMEDIATE q'[ - CREATE TABLE SOURCE_TABLE ( - ID NUMBER(15,0), - DESCR CHAR(30 CHAR), - YEAR NUMBER(4,0), - DATEE DATE, - CONSTRAINT PK_SOURCE_TABLE PRIMARY KEY (ID) - ) - ]'; -EXCEPTION - WHEN OTHERS THEN - IF SQLCODE != -955 THEN RAISE; END IF; -END; -/ - --- truncate if exists -DECLARE n INTEGER; -BEGIN - SELECT COUNT(*) INTO n FROM USER_TABLES WHERE TABLE_NAME='SOURCE_TABLE'; - IF n=1 THEN EXECUTE IMMEDIATE 'TRUNCATE TABLE SOURCE_TABLE'; END IF; -END; -/ - --- idempotent load -MERGE INTO SOURCE_TABLE t -USING ( - SELECT 1001 ID, 'Cycle 1' DESCR, 2025 YEAR, DATE '2025-01-01' DATEE FROM DUAL UNION ALL - SELECT 1002, 'Cycle 2', 2025, DATE '2025-02-01' FROM DUAL UNION ALL - SELECT 1003, 'Cycle 3', 2025, DATE '2025-03-01' FROM DUAL UNION ALL - SELECT 1004, 'Cycle 4', 2025, DATE '2025-04-15' FROM DUAL UNION ALL - SELECT 1005, 'Cycle 5', 2025, DATE '2025-05-01' FROM DUAL -) s -ON (t.ID = s.ID) -WHEN MATCHED THEN UPDATE SET t.DESCR = RPAD(s.DESCR,30), t.YEAR = s.YEAR, t.DATEE = s.DATEE -WHEN NOT MATCHED THEN INSERT (ID, DESCR, YEAR, DATEE) - VALUES (s.ID, RPAD(s.DESCR,30), s.YEAR, s.DATEE); - -COMMIT; -SQL - -printf "Oracle DDL/DML completed.\n" diff --git a/.github/scripts/setup_mssql_odbc.sh b/.github/scripts/setup_mssql_odbc.sh deleted file mode 100644 index 1f29c0f6c0..0000000000 --- a/.github/scripts/setup_mssql_odbc.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# -# TODO: Replace with a job container that has msodbcsql18 pre-installed. -# -# Install the Microsoft ODBC driver for SQL Server. -# -# The FILE_MANIFEST from Microsoft is verified against a pinned checksum -# before it is trusted to verify the package repository configuration. -# -# Repurposed from https://github.com/Yarden-zamir/install-mssql-odbc -# -set -eu - -VERSION_ID="$(awk -F= '$1=="VERSION_ID"{gsub(/"/, "", $2); print $2}' /etc/os-release)" -PKG_NAME="packages-microsoft-prod.deb" - -# Pinned FILE_MANIFEST checksums per Ubuntu version. -case "${VERSION_ID}" in - 22.04) manifest_sha256="3fc171cfaeb40e8c2103267b28277c43de02269bd35ac7cb897b2bbd9a40f256" ;; - 24.04) manifest_sha256="52830b24a2c53300daa3991cb91997288e7440c3c66782ac003e4cf6c0271d04" ;; - *) printf "Unsupported Ubuntu version: %s\n" "${VERSION_ID}" >&2; exit 1 ;; -esac - -FILE_MANIFEST_URL="https://packages.microsoft.com/config/ubuntu/${VERSION_ID}/FILE_MANIFEST" -FILE_MANIFEST="hashes.txt" - -curl -fsSL -o "${FILE_MANIFEST}" "${FILE_MANIFEST_URL}" -printf '%s %s\n' "${manifest_sha256}" "${FILE_MANIFEST}" | sha256sum -c >/dev/null - -expected_hash="$(awk -F, -v pkg="${PKG_NAME}" '$1==pkg {print $2}' "${FILE_MANIFEST}")" - -curl -fsSL -O "https://packages.microsoft.com/config/ubuntu/${VERSION_ID}/${PKG_NAME}" -printf "%s *packages-microsoft-prod.deb\n" "${expected_hash}" | sha256sum -c - - -# Install the Microsoft package repository configuration file using dpkg. -# The --force-confold option ensures that existing configuration files are not overwritten. -# The DEBIAN_FRONTEND=noninteractive environment variable suppresses interactive prompts. -sudo DEBIAN_FRONTEND=noninteractive dpkg --force-confold -i packages-microsoft-prod.deb - -sudo apt-get update -sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 diff --git a/.github/scripts/setup_spark_remote.sh b/.github/scripts/setup_spark_remote.sh deleted file mode 100755 index 5323744e29..0000000000 --- a/.github/scripts/setup_spark_remote.sh +++ /dev/null @@ -1,97 +0,0 @@ -#!/bin/bash -# -# TODO: Replace this script with a GHA service container on the workflow. -# -# Set up Apache Spark in local Connect mode for integration tests. -# -# All downloads are pinned to exact versions and verified against embedded -# cryptographic checksums before use. -# -set -eu - -mkdir -p "$HOME"/spark -cd "$HOME"/spark || exit 1 - -# Spark Connect server still points to 3.5.5 -spark_version="3.5.5" -spark="spark-${spark_version}-bin-hadoop3" -spark_connect="spark-connect_2.12" -spark_sha256='8daa3f7fb0af2670fe11beb8a2ac79d908a534d7298353ec4746025b102d5e31' - -mssql_jdbc_version="1.4.0" -mssql_jdbc="spark-mssql-connector_2.12-${mssql_jdbc_version}-BETA" -mssql_jdbc_sha256="1057e93d946dffd2ecac1c11bcc40fdf51110c5a99e9e7379a9568584cc3de7f" -ORACLE_JDBC_VERSION="19.28.0.0" -SNOWFLAKE_JDBC_VERSION="3.26.1" -SNOWFLAKE_SPARK_VERSION="2.11.2-spark_3.3" - -mkdir -p "${spark}" -SERVER_SCRIPT=$HOME/spark/${spark}/sbin/start-connect-server.sh -JARS_DIR=$HOME/spark/${spark}/jars - -if [ -f "${SERVER_SCRIPT}" ]; then - printf "Spark %s already exists\n" "${spark_version}" -else - spark_tarball="${spark}.tgz" - if [ ! -f "${spark_tarball}" ]; then - printf "Downloading Spark %s...\n" "${spark_version}" - curl -fsSL "https://archive.apache.org/dist/spark/spark-${spark_version}/${spark_tarball}" -o "${spark_tarball}" - fi - printf '%s %s\n' "${spark_sha256}" "${spark_tarball}" | sha256sum -c >/dev/null - tar -xf "${spark_tarball}" -fi - -printf "Downloading JDBC JARs and dependencies via Maven...\n" - -for artifact in \ - com.microsoft.azure:adal4j:1.6.4:jar \ - com.nimbusds:oauth2-oidc-sdk:6.5:jar \ - com.google.code.gson:gson:2.8.0:jar \ - net.minidev:json-smart:1.3.1:jar \ - com.nimbusds:nimbus-jose-jwt:8.2.1:jar \ - org.slf4j:slf4j-api:1.7.21:jar \ - com.microsoft.sqlserver:mssql-jdbc:6.4.0.jre8:jar \ - com.oracle.database.jdbc:ojdbc8:${ORACLE_JDBC_VERSION}:jar \ - net.snowflake:snowflake-jdbc:${SNOWFLAKE_JDBC_VERSION}:jar \ - net.snowflake:spark-snowflake_2.12:${SNOWFLAKE_SPARK_VERSION}:jar -do - mvn dependency:copy -q --strict-checksums -DoutputDirectory="$JARS_DIR" -Dartifact="${artifact}" & -done -wait - -# The sql-spark-connector is not on Maven Central; download from GitHub. -curl -fsSL -o "$JARS_DIR/${mssql_jdbc}.jar" \ - "https://github.com/microsoft/sql-spark-connector/releases/download/v${mssql_jdbc_version}/${mssql_jdbc}.jar" -printf '%s %s\n' "${mssql_jdbc_sha256}" "$JARS_DIR/${mssql_jdbc}.jar" | sha256sum -c >/dev/null - -# --- Start Spark Connect server --- - -rm -rf "${HOME}"/spark/"${spark}"/spark-warehouse -printf "Cleared old spark warehouse default directory\n" - -cd "${spark}" || exit 1 -result=$(${SERVER_SCRIPT} --packages "org.apache.spark:${spark_connect}:${spark_version}" > "$HOME"/spark/log.out; echo $?) - -if [ "$result" -ne 0 ]; then - count=$(tail "${HOME}"/spark/log.out | grep -c "SparkConnectServer running as process") - if [ "${count}" == "0" ]; then - printf "Failed to start the server\n" - exit 1 - fi - # Wait for the server to start by pinging localhost:4040 - printf "Waiting for the server to start...\n" - for i in {1..30}; do - if nc -z localhost 4040; then - printf "Server is up and running\n" - break - fi - printf "Server not yet available, retrying in 5 seconds...\n" - sleep 5 - done - - if ! nc -z localhost 4040; then - printf "Failed to start the server within the expected time\n" - exit 1 - fi -fi -printf "Started the Server\n" diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml index c4804507f1..a18df39fde 100644 --- a/.github/workflows/acceptance.yml +++ b/.github/workflows/acceptance.yml @@ -18,7 +18,13 @@ concurrency: jobs: integration: - environment: tool + strategy: + fail-fast: false + matrix: + python: [ '3.10', '3.11', '3.12', '3.13', '3.14' ] + environment: + name: tool + deployment: false permissions: # Access to JFrog and the integration testing infrastructure. id-token: write @@ -27,6 +33,8 @@ jobs: runs-on: group: larger-runners labels: larger + env: + UV_PYTHON: "${{ matrix.python }}" steps: - name: Checkout Code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -39,23 +47,15 @@ jobs: version: "0.11.2" checksum: "7ac2ca0449c8d68dae9b99e635cd3bc9b22a4cb1de64b7c43716398447d42981" + - name: Set up JDK 21 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + with: + distribution: corretto + java-version: 21 + - name: Setup for JFrog uses: ./.github/actions/jfrog-auth - - name: Install MSSQL ODBC Driver - run: | - chmod +x $GITHUB_WORKSPACE/.github/scripts/setup_mssql_odbc.sh - $GITHUB_WORKSPACE/.github/scripts/setup_mssql_odbc.sh - - # TODO: Migrate tests to use Databricks clusters instead of Spark local mode - # Disabled for now, because access to archives.apache.org is blocked; a new approach will be needed for this. - # (The integration tests will still run, but many will fail due to this.) - - name: Setup spark - if: false - run: | - chmod +x $GITHUB_WORKSPACE/.github/scripts/setup_spark_remote.sh - $GITHUB_WORKSPACE/.github/scripts/setup_spark_remote.sh - - name: Run integration tests uses: databrickslabs/sandbox/acceptance@3313d06ce86227537b3f37f5974f7eecb2a8e59a # acceptance/v0.4.4 with: @@ -63,7 +63,8 @@ jobs: directory: ${{ github.workspace }} timeout: 2h env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }} ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + LAKEBRIDGE_MAVEN_URL: "https://databricks.jfrog.io/artifactory/db-maven" TEST_ENV: 'ACCEPTANCE' diff --git a/.github/workflows/docs-build.yml b/.github/workflows/docs-build.yml index cc7ff654b2..aa6c29aa1c 100644 --- a/.github/workflows/docs-build.yml +++ b/.github/workflows/docs-build.yml @@ -30,9 +30,9 @@ jobs: docs sparse-checkout-cone-mode: 'true' - - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: - node-version: 25.9.0 + node-version: 26.1.0 cache: yarn cache-dependency-path: docs/lakebridge/yarn.lock diff --git a/.github/workflows/docs-release.yml b/.github/workflows/docs-release.yml index 67a9080773..2ceafeaf42 100644 --- a/.github/workflows/docs-release.yml +++ b/.github/workflows/docs-release.yml @@ -15,7 +15,9 @@ jobs: runs-on: group: databrickslabs-protected-runner-group labels: linux-ubuntu-latest - environment: release + environment: + name: release + deployment: false permissions: # JFrog OIDC authentication. id-token: write @@ -27,9 +29,9 @@ jobs: docs sparse-checkout-cone-mode: 'true' - - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: - node-version: 25.9.0 + node-version: 26.1.0 cache: yarn cache-dependency-path: docs/lakebridge/yarn.lock @@ -43,7 +45,7 @@ jobs: run: make docs-build - name: Upload Build Artifact - uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4.0.0 + uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4.0.0 with: path: docs/lakebridge/build # The directory to upload as an artifact @@ -67,4 +69,4 @@ jobs: steps: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5 + uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5 diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 0a65860488..05cbda7e6a 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -19,12 +19,18 @@ permissions: jobs: test-python: + strategy: + fail-fast: false + matrix: + python: [ '3.10', '3.11', '3.12', '3.13', '3.14' ] runs-on: group: databrickslabs-protected-runner-group labels: linux-ubuntu-latest permissions: # JFrog OIDC authentication. id-token: write + env: + UV_PYTHON: "${{ matrix.python }}" steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -78,6 +84,49 @@ jobs: # Exit with status code 1 if there are differences (i.e. unformatted files) git diff --exit-code + lint-uv: + runs-on: + group: databrickslabs-protected-runner-group + labels: linux-ubuntu-latest + permissions: + # JFrog OIDC authentication. + id-token: write + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Deepening checkout to find merge-base + if: github.event.pull_request || github.event.merge_group + env: + BASE_SHA: "${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }}" + run: | + _depth=32 + _max=512 + until git merge-base "${BASE_SHA}" HEAD > /dev/null 2>&1 + do + if [[ "${_max}" -lt "${_depth}" ]] + then + # Give up deepening, just fetch the rest of the repo. + git fetch --unshallow origin "${BASE_SHA}" "${GITHUB_REF}" + break + fi + git fetch --deepen "${_depth}" origin "${BASE_SHA}" "${GITHUB_REF}" + _depth=$((_depth * 2)) + done + printf '%s=%s\n' 'BASE_SHA' "${BASE_SHA}" >> "${GITHUB_ENV}" + + - name: Setup uv + uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1 + with: + version: "0.11.2" + checksum: "7ac2ca0449c8d68dae9b99e635cd3bc9b22a4cb1de64b7c43716398447d42981" + + - name: Setup for JFrog + uses: ./.github/actions/jfrog-auth + + - name: Linting project lock-files + run: ./.github/scripts/lint-uv + python-no-pylint-disable: runs-on: group: databrickslabs-protected-runner-group @@ -92,7 +141,9 @@ jobs: - name: Verify no lint disabled in the new code run: | git fetch origin "${GITHUB_BASE_REF}:${GITHUB_BASE_REF}" - git diff --ignore-space-change "${GITHUB_BASE_REF}...HEAD" -- ':(glob,icase)**/*.py' > diff_data.txt + # Upgrade scripts are named vX.Y.Z_*.py, which trips pylint's invalid-name; they carry a + # file-level `# pylint: disable=invalid-name` by convention, so exclude them from this check. + git diff --ignore-space-change "${GITHUB_BASE_REF}...HEAD" -- ':(glob,icase)**/*.py' ':(exclude,glob,icase)src/databricks/labs/lakebridge/upgrades/*.py' > diff_data.txt python tests/unit/no_cheat.py diff_data.txt > cheats.txt COUNT=$(cat cheats.txt | wc -c) if [ ${COUNT} -gt 1 ]; then diff --git a/.gitignore b/.gitignore index 0a567a6c11..99db486ef3 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,7 @@ remorph_transpile/ .databricks-login.json .mypy_cache .env +/tests/resources/lsp_transpiler/test-lsp-server.log +.credentials.yml +.credentials.bak +.credentials.* diff --git a/CHANGELOG.md b/CHANGELOG.md index f8c2c2900e..c0791978f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,420 @@ # Version changelog +# Release v0.14.1 + +## Highlights + +- **Teradata is now a profiler source.** Teradata joins the set of supported profiler platforms, letting you assess a Teradata estate ahead of a migration to Databricks. + +- **Multi-database profiling for SQL Server.** The SQL Server profiler can now extract from multiple databases in a single run, rather than being scoped to one database at a time. + +- **BigQuery is now a reconcile source.** BigQuery can be used as a source platform for data-quality reconciliation, extending validation coverage for customers migrating off BigQuery. + +- **Java 21 is now required.** The Morpheus transpiler now requires Java 21 or above; `install-transpile` will prompt if an older runtime is detected. + +--- + +## Profilers + +### New Sources + +- **Teradata ([#2343](https://github.com/databrickslabs/lakebridge/pull/2343))** + Adds Teradata as a profiler source, wiring the interactive configurator, extraction queries, and local extract population so Teradata estates can be assessed ahead of migration. + +### Enhancements + +- **Multi-database profiling for SQL Server ([#2536](https://github.com/databrickslabs/lakebridge/pull/2536))** + The SQL Server profiler can now extract across multiple databases in a single execution instead of being limited to one database per run. + +- **Profiler variants in `execute-profiler` ([#2511](https://github.com/databrickslabs/lakebridge/pull/2511))** + Introduces the concept of a profiler *variant*: when a source supports more than one variant, the user is prompted to select one at execution time. + +- **Correct exit code on profiler error ([#2571](https://github.com/databrickslabs/lakebridge/pull/2571))** + The profiler now returns a non-zero exit code when it fails, so failures surface correctly in scripted and CI environments. + +- **BigQuery test connection and profiler config cleanup ([#2512](https://github.com/databrickslabs/lakebridge/pull/2512))** + Implements the previously missing BigQuery `test-connection` path and tidies up the profiler configuration. + +- **Suppress noisy `google.api_core` warnings ([#2538](https://github.com/databrickslabs/lakebridge/pull/2538))** + Silences an unhelpful warning emitted by `google.api_core` during profiling. + +### Snowflake Profiler + +- **Reduce output size by decoupling query metrics from `QUERY_TEXT` ([#2567](https://github.com/databrickslabs/lakebridge/pull/2567))** + Query metrics are no longer joined against the full `QUERY_TEXT`, substantially shrinking the profiler output. + +- **Exclude system databases from `automatic_clustering` ([#2534](https://github.com/databrickslabs/lakebridge/pull/2534))** + System databases are now excluded from the automatic-clustering extract, avoiding irrelevant/noisy results. + +### Synapse Profiler + +- **Fix failures for empty SQL pools and serverless routines schema ([#2572](https://github.com/databrickslabs/lakebridge/pull/2572))** + Resolves profiler failures against empty SQL pools and corrects the serverless routines schema. + +- **Correct column order for tables and views in `schemas.py` ([#2531](https://github.com/databrickslabs/lakebridge/pull/2531))** + Fixes the column ordering for tables and views in the Synapse profiler schema definitions. + +- **Add `WORKSPACE_NAME` to activity extract tables ([#2549](https://github.com/databrickslabs/lakebridge/pull/2549))** + Adds a `WORKSPACE_NAME` column to the Synapse activity extract tables. + +### BigQuery Profiler + +- **Add missing BigQuery profiler extract source ([#2551](https://github.com/databrickslabs/lakebridge/pull/2551))** + Adds the extract source that was missing from the BigQuery profiler pipeline. + +### Removed + +- **Profiler dashboard removed ([#2524](https://github.com/databrickslabs/lakebridge/pull/2524))** + The profiler dashboard command has been removed, it will now be part of the desktop app. + +--- + +## Converters + +### Morpheus + +#### Redshift Improvements + +- **`CREATE PROCEDURE` support (#967)** for ANSI/PL-pgSQL stored procedures, including `IN`/`OUT`/`INOUT` parameter modes, `NONATOMIC`, and `SECURITY INVOKER`/`DEFINER`. +- **SQL UDF bodies are now parsed and translated (#975, #976)** rather than emitted verbatim, and positional references (`$1`, `$2`, …) are resolved to parameter names via an IR rule. +- **`SELECT INTO` (#972)** now handles both meanings — table creation (`CREATE TABLE ... AS SELECT`) and variable assignment inside PL/pgSQL bodies. +- **Typed datetime literals (#953)** such as `DATE '...'`, `TIMESTAMP '...'`, and `INTERVAL '7' DAY` transpile to the correct `CAST(...)` forms. +- **Extra-argument function forms (#968, #970, #971)** for scalar (`NVL`, `TO_DATE`, `ARRAY_CONTAINS`, `ARRAY_EXCEPT`), spatial (`ST_*`), and `REGEXP_*` families are now translated correctly instead of failing with `WRONG_NUM_ARGS`. +- **`:=` procedure assignment and H3/spatial type alignment (#988).** +- **Repaired truncated/invalid SQL in `CREATE PROCEDURE` functional tests (#969).** + +#### Snowflake Improvements + +- **Geospatial (`ST_*`) conversions fixed (#980)** — `ST_MAKELINE`, `ST_COLLECT`, `ST_COVEREDBY`, and others, moved into the shared `CallMapper` so every source dialect reuses them. +- **`FROM` subqueries now accept CTEs and set-operations (#958).** +- **Untyped `ARRAY` defaults to `ARRAY` (#983)** instead of the invalid `ARRAY<>`. +- **`DESC` parsed as a synonym for `DESCRIBE` (#985).** + + +#### T-SQL Improvements + +- **`AT` allowed as a soft keyword / table alias (#1179)**, fixing a bug where a multi-CTE query was silently truncated. + +#### Cross-dialect Improvements + +- **Unified `CREATE PROCEDURE`, `CREATE TABLE` column, and `ALTER PROCEDURE` grammar (#977, #978, #981)**, fixing related T-SQL computed-column, constraint, and `COLLATE` bugs along the way. +- **Cursor statements consolidated (#962)** — `OPEN`, `CLOSE`, `FETCH`, `DECLARE CURSOR`, `DEALLOCATE` now share one scripting path across dialects. +- **`CURRENT_USER` translated to `SESSION_USER()` (#960)** consistently across dialects. +- **Single-argument `CONCAT` accepted (#959).** +- **Ambiguous trailing-comma grammar tightened plus an ANTLR diagnostic reporter (#966)** to reduce costly parser fallbacks. + +### Performance + +- **Trampolined `Transformation` monad (#963)** eliminates `StackOverflowError`s when transpiling very large procedures. + +--- + +## Reconcile + +- NEW! **BigQuery as a reconcile source ([#2527](https://github.com/databrickslabs/lakebridge/pull/2527))** + Adds BigQuery as a supported source platform for reconciliation. + +- **Configurable sample size via `sampling_options` ([#2497](https://github.com/databrickslabs/lakebridge/pull/2497))** + Adds a `sampling_options` block to the reconcile table config so users can control how many sample rows are captured per result set, with engine-aware upper bounds. + +- **Materialize intermediate DataFrames during reconcile ([#2535](https://github.com/databrickslabs/lakebridge/pull/2535))** + Intermediate DataFrames are now materialized during reconciliation, improving stability and performance on larger jobs. + +- **Refactor `DatabaseConfig` usages in reconcile runtime ([#2503](https://github.com/databrickslabs/lakebridge/pull/2503))** + Reconcile runtime internals now use explicit `SourceConnectionConfig`/`TargetConnectionConfig` objects instead of the shared `DatabaseConfig`. + +- **Use the pytester `ws` fixture in reconcile integration tests ([#2521](https://github.com/databrickslabs/lakebridge/pull/2521))** + Migrates reconcile integration tests to the real pytester `ws` fixture in place of a mocked workspace client. + +--- + +## Requirements + +- **Java 21 required for Morpheus ([#2507](https://github.com/databrickslabs/lakebridge/pull/2507))** + The Morpheus transpiler now requires Java 21 or above. `install-transpile` reports a clear message when an older Java runtime is detected. + +## New Contributors +* @dey-abhishek made their first contribution in https://github.com/databrickslabs/lakebridge/pull/2343 + +**Full Changelog**: https://github.com/databrickslabs/lakebridge/compare/v0.14.0...v0.14.1 + +-- + +# Release Notes — Lakebridge v0.14.0 + +## Highlights + +- **Five new profiler sources.** Snowflake, Redshift, BigQuery, Oracle, and Legacy SQL DW are now all supported profiler targets, significantly expanding the set of platforms Lakebridge can assess ahead of a migration. + +- **Switch now supports SAS.** A new built-in prompt converts SAS programs to PySpark, adding SAS to the growing list of languages Switch can migrate automatically. + +- **Reconcile now supports Teradata.** Teradata can now be used as a source platform for data quality reconciliation, enabling validation for customers migrating from Teradata to Databricks. + +- **Automatic reconciliation configuration.** A new `auto-configure-recon-tables` command discovers source and target tables and generates the initial reconcile configuration automatically, replacing a previously manual setup step. + +--- + +## Profilers + +### New Sources + +- **Redshift ([#2305](https://github.com/databrickslabs/lakebridge/pull/2305), [#2304](https://github.com/databrickslabs/lakebridge/pull/2304), [#2306](https://github.com/databrickslabs/lakebridge/pull/2306), [#2408](https://github.com/databrickslabs/lakebridge/pull/2408), [#2501](https://github.com/databrickslabs/lakebridge/pull/2501))** + Amazon Redshift is now a fully supported profiler source, covering all three deployment variants: provisioned, provisioned multi-AZ, and serverless. This includes credential flows (database password, federated user, AWS Secrets Manager ARN, temporary credentials with db user or IAM, with optional SSL), dedicated extraction queries and validation schemas for each variant, full CLI wiring, and user documentation. A bug in the serverless managed-storage aggregation was also fixed: the previous query summed hourly `sys_serverless_usage` snapshots, causing reported storage to grow linearly with the lookback window rather than reflecting the actual allocated amount. + +- **Snowflake ([#2420](https://github.com/databrickslabs/lakebridge/pull/2420), [#2499](https://github.com/databrickslabs/lakebridge/pull/2499))** + Adds Snowflake as a profiler source. The interactive configurator prompts for connection details and a Programmatic Access Token (PAT), then extracts warehouse usage, query history, storage, user activity, account info, and optional credits (pipe, autoclustering, materialized-view refresh) from `SNOWFLAKE.ACCOUNT_USAGE` into a timestamped DuckDB file. Post-extraction computations produce TCO summaries. A supplementary `rate_sheet` extract pulls the effective per-credit rate (90-day average) and account service tier from `SNOWFLAKE.ORGANIZATION_USAGE.RATE_SHEET_DAILY`, providing the inputs needed by the downstream TCO value model. + +- **BigQuery ([#2472](https://github.com/databrickslabs/lakebridge/pull/2472))** + Adds BigQuery as a profiler source. Running `configure-database-profiler` followed by `execute-database-profiler --source-tech bigquery` executes 16 region-qualified `INFORMATION_SCHEMA` queries against the customer's configured BigQuery project(s) and writes 12 analysis tables into a local DuckDB file at `~/.databricks/labs/lakebridge_profilers/bigquery_assessment/profiler_extract.db`. The pipeline structure mirrors existing source-techs. + +- **Oracle ([#2187](https://github.com/databrickslabs/lakebridge/pull/2187))** + Adds Oracle as a profiler source. Covers the interactive configuration dialog, extraction SQL scripts, and local DuckDB population from Oracle Database, with unit tests for all three components. + +- **Legacy SQL DW ([#2441](https://github.com/databrickslabs/lakebridge/pull/2441))** + Extends profiler coverage to legacy Azure SQL DW (pre-Synapse) deployments, adding the necessary extraction queries to assess this platform. + +### Enhancements + +- **SQL Server profiler switched to SQL scripts and shared DatabaseManager ([#2482](https://github.com/databrickslabs/lakebridge/pull/2482))** + The MSSQL profiler's activity and info extraction steps have been converted from per-step Python virtualenvs to in-process `sql`/`ddl` steps using the shared `DatabaseManager` connector. Thirteen SQL query/DDL pairs replace the previous Python scripts, eliminating the duplicated `get_sqlserver_reader` connector and aligning SQL Server with the architecture used by other profiler sources. + +- **User-configurable output folder ([#2488](https://github.com/databrickslabs/lakebridge/pull/2488))** + The profiler's DuckDB extract path is now exposed as a CLI flag (`--output-folder`) instead of being hardcoded per source in `pipeline_config.yml`. The default remains `~/.databricks/labs/lakebridge_profilers/_assessment`. Output filenames now include a timestamp (`profiler_extract_.db`) to prevent overwrites on repeated runs, and the absolute path to the extract is logged on successful completion. + +- **Custom credentials file path for `execute-database-profiler` ([#2494](https://github.com/databrickslabs/lakebridge/pull/2494))** + The `execute-database-profiler` command now accepts an optional `--cred-file-path` argument, letting users supply a non-default credentials file rather than the one written by `configure-database-profiler`. This makes it easier to manage multiple credential configurations or run the profiler in scripted, non-interactive environments. + +- **SQL Server: `TrustServerCertificate` support ([#2498](https://github.com/databrickslabs/lakebridge/pull/2498))** + The `configure-database-profiler` command for SQL Server now surfaces a `TrustServerCertificate` connection property, addressing a common customer request for environments where the server certificate cannot be validated. + +- **Fix: profiler steps no longer fail with `ModuleNotFoundError` ([#2485](https://github.com/databrickslabs/lakebridge/pull/2485))** + The Synapse and SQL Server profilers previously created a fresh virtualenv per pipeline step, installing only each step's declared dependencies. After `database_manager.py` began importing `redshift_connector` at module scope, every clean run of those profilers failed with `ModuleNotFoundError`. Profiler Python steps now run with the parent interpreter, inheriting all installed packages—faster, more robust, and immune to this class of transitive-import failures. + +--- + +## Converters + +### Morpheus + +#### T-SQL Improvements + +- **Better date and number formatting from T-SQL `CONVERT`** + T-SQL's `CONVERT` function accepts a style code to format dates and numbers as strings (e.g. `CONVERT(VARCHAR, myDate, 103)` for `dd/MM/yyyy`). Morpheus now correctly translates these style codes to the equivalent Databricks SQL expressions, unblocking a large class of previously untranslatable queries. + +- **Fix integer-as-date behavior from T-SQL** + T-SQL allows using `0` (or any integer) where a date is expected, treating it as "N days after 1 Jan 1900". Morpheus now replicates this behavior, preventing runtime type errors when running translated queries on Databricks. + +- **Fix variable declarations with `VARCHAR(N)` / `CHAR(N)` types** + T-SQL local variables declared as `VARCHAR(N)` or `CHAR(N)` were being passed through verbatim and failing at runtime in Databricks. They are now automatically translated to `STRING`, the correct equivalent type for variable declarations. + +- **Fix T-SQL variable assignment queries with `ORDER BY`** + T-SQL queries that assign a value to a variable (e.g. `SELECT @var = col FROM t ORDER BY col`) were being incorrectly structured during translation. The `ORDER BY` and similar clauses are now placed correctly in the output. + +- **Support `HASH JOIN` query hint in T-SQL** + T-SQL's `OPTION(HASH JOIN)` query hint, which instructs the database to use a specific join strategy, is now correctly parsed and handled during translation. + +#### Snowflake Improvements + +- **Translate `REGEXP_SUBSTR_ALL` to `REGEXP_EXTRACT_ALL`** + Snowflake's `REGEXP_SUBSTR_ALL` (returns all regex matches as an array) is now translated to its Databricks SQL equivalent `REGEXP_EXTRACT_ALL`. + +- **Translate binary hash functions (`MD5_BINARY`, `SHA1_BINARY`, `SHA2_BINARY`)** + Snowflake's binary digest functions are now translated to their Databricks SQL equivalents by wrapping the hex output with `UNHEX()`. + +- **Translate hex hash function synonyms (`MD5_HEX`, `SHA1_HEX`, `SHA2_HEX`)** + Snowflake's `*_HEX` hash function aliases are now directly mapped to their identically-behaved Databricks SQL counterparts. + +- **Translate `UNICODE` and `TRY_TO_DOUBLE`** + Snowflake's `UNICODE()` (returns the code point of the first character) is now mapped to `ASCII()` in Databricks SQL, and `TRY_TO_DOUBLE()` is mapped to `TRY_CAST(_ AS DOUBLE)`. The `UNICODE` fix also applies to T-SQL. + +- **Translate type-check functions (`IS_DATE`, `IS_DOUBLE`, `IS_REAL`, etc.)** + Snowflake functions that test whether a value inside a semi-structured (VARIANT) column holds a specific type are now translated where possible (e.g. `IS_DATE` → `TRY_CAST(v AS DATE) IS NOT NULL`). Those with no equivalent in Databricks (`IS_TIME`, `IS_TIMESTAMP_TZ`) are flagged with a clear migration note. + +- **Flag unsupported functions (`CHECK_XML`, `PARSE_XML`, `IS_ROLE_IN_SESSION`, etc.) with migration notes** + Seven Snowflake-specific functions with no Databricks equivalent now produce a clear "FIXME" comment in the output instead of silently passing through and failing at runtime. `IS_NULL_VALUE` is also correctly translated to `IS_VARIANT_NULL`. + +- **Flag 12 more Snowflake-only functions with migration notes** + Additional Snowflake admin, statistical, and VARIANT-inspection functions (including `COMPRESS`, `NORMAL`, `ZIPF`, `INVOKER_ROLE`, `IS_BOOLEAN`) now produce clear FIXME annotations instead of failing silently at runtime. + +- **Flag Snowflake `INFORMATION_SCHEMA` metadata functions with migration notes** + Snowflake monitoring/metadata functions called via `INFORMATION_SCHEMA` (like `PIPE_USAGE_HISTORY`, `MATERIALIZED_VIEW_REFRESH_HISTORY`) no longer cause an `UNRESOLVED_ROUTINE` crash — they are now flagged with a clear migration note explaining that no equivalent exists. + +- **Translate Snowflake's row generator pattern to `RANGE()`** + Snowflake's `TABLE(GENERATOR(ROWCOUNT => N))` pattern (used to generate N rows, often with `SEQ4()` to get row numbers) is now automatically translated to Databricks SQL's `RANGE(0, N)`. + +- **Improve parsing of `COPY INTO` commands** + The parser now correctly handles both forms of Snowflake's `COPY INTO` — loading data into a table and unloading data to an external location — laying groundwork for future full translation support. + +#### Cross-dialect Improvements + +- **Support for cursor-based SQL across all dialects** + Cursor statements (`DECLARE`, `OPEN`, `FETCH`, `CLOSE`, `DEALLOCATE`) are now supported for migration across T-SQL, Snowflake, and Redshift. + +- **Clearer error messages for untranslatable statements** + When a SQL statement can't be automatically translated, the output now includes the original SQL text and a meaningful explanation in the FIXME comment, rather than a generic confusing error message. + +--- + +### Switch + +#### New Source Format Support + +- **SAS code conversion** + New built-in prompt converts SAS programs (both inline `DATALINES` and external file patterns) to PySpark equivalents, with example input/output pairs included. + +- **Informatica ETL conversion** + New built-in prompt for migrating Informatica workflows to Lakeflow Spark Declarative Pipelines (SDP). Covers Source Qualifier, Expression, Router, Joiner, Lookup (connected and unconnected), Aggregator, Normalizer, Sequence Generator, and Update Strategy transformations to PySpark/Spark SQL. + +- **Custom ETL conversion** + A flexible companion template lets users define their own input/output specs and conversion logic for any ETL tool not covered by a dedicated prompt. + +#### New Reference Prompts + +- **Teradata stored procedures reference prompt** + Introduces a new category of reference prompts — field-tested, opt-in prompts users can point Switch at via `conversion_prompt_yaml`. This first entry converts Teradata stored procedures directly to Databricks SQL (no Python-notebook wrapping) and covers: `CREATE VOLATILE TABLE` + multiple `INSERT` → `CREATE OR REPLACE TEMPORARY VIEW ... UNION ALL`; `UPDATE...FROM` → `MERGE INTO`; multi-column `IN` deletes → `EXISTS`; epoch/timezone conversions; and Teradata-specific functions (`SYSLIB.OREPLACE`, `DELIMITEDCOUNT`, `DELIMITEDITEM`, etc.). + +#### Improved Built-in Prompts + +- **Snowflake** + Added conversion rules discovered during a customer dbt migration — `IFF`, `TRY_TO_*`, `PARSE_JSON`, `SELECT * EXCLUDE`, `QUALIFY`, `WITH RECURSIVE`, colon (`:`) JSON access, `TIMESTAMP_NTZ`, trailing comma removal, and a new date-format pattern mapping section (e.g. `HH24` → `HH`, `MI` → `mm`). + +- **Teradata** + Major expansion with rules found during BTEQ customer migrations. New coverage includes: BTEQ control commands (`.LOGON`, `.QUIT`, FastLoad, MultiLoad, FastExport statements); shell variable substitution via `dbutils.widgets` and Python f-strings; `(FORMAT)(CHAR)` shorthand cast patterns; `REPLACE VIEW` with column list, `RENAME TABLE`, `COLLECT STATISTICS`; `UPDATE...FROM` → `MERGE INTO`; extended DDL clause removal (`NO FALLBACK`, `COMPRESS`, `CASESPECIFIC`, `PARTITION BY RANGE_N`, etc.); and two new few-shot examples for BTEQ scripts with dynamic identifiers. + +- **Redshift** + Substantial update validated against ~30,000 real queries, producing more accurate and performant Databricks SQL output. + +#### Bug Fixes + +- **Serverless export failures** + Fixed a bug where exporting files or notebooks to subdirectories would fail on serverless compute with `ResourceDoesNotExist: The parent folder does not exist`. The root cause was `os.makedirs` only writing to the FUSE layer without materializing a real Workspace object. Replaced with `ws_client.workspace.mkdirs` in `export_to_file.py` and `export_to_notebook.py`, matching the pattern already used elsewhere in the codebase. + +#### Enhancements + +- **[#2256](https://github.com/databrickslabs/lakebridge/pull/2256) — Custom Switch configuration path for `llm-transpile`** + The `llm-transpile` CLI command now accepts an optional `--switch-config-path` parameter, allowing users to point Switch at a custom configuration file stored in their Databricks workspace (the path must start with `/Workspace/`). When omitted, Switch uses its default configuration as before. + +--- + +## Reconcile + +- **[#2456](https://github.com/databrickslabs/lakebridge/pull/2456) — Teradata support** + Reconciliation now supports Teradata as a source platform, enabling data quality validation for customers migrating from Teradata to Databricks. + +- **[#2465](https://github.com/databrickslabs/lakebridge/pull/2465) — Unified sampling query across all dialects** + The reconcile sampling query is now implemented as a single derived-table-join shape that works across all supported dialects. The previous implementation used a CTE for non-T-SQL paths and a `VALUES` derived table for T-SQL; the latter failed at runtime on Azure Synapse Dedicated SQL Pool, which does not accept `VALUES` as a derived table. + +- **[#2392](https://github.com/databrickslabs/lakebridge/pull/2392) — Auto-discovery of source/target tables** + A new CLI command, `databricks labs lakebridge auto-configure-recon-tables`, discovers tables in source and target systems and automatically generates column mappings for the reconcile configuration. Users are guided through two interactive stages: table discovery, then auto-configuration of the discovered tables. + +- **[#2504](https://github.com/databrickslabs/lakebridge/pull/2504) — Source system logging in recon commands** + Reconciliation commands now include the source system and report type in user-agent extras, improving observability and making it easier to trace usage across different source platforms. + +--- + +## Documentation + +- **[#2457](https://github.com/databrickslabs/lakebridge/pull/2457) — Document PyPI and Maven mirrors during installation** + The installation documentation now covers how to configure package mirrors and proxies when direct access to GitHub, PyPI, or Maven Central is unavailable — a common requirement in air-gapped or enterprise firewall environments. + +--- + +## General + +- **Python 3.14 support ([#2333](https://github.com/databrickslabs/lakebridge/pull/2333))** + The CLI and its dependencies now support Python 3.14. This required updating `mypy`, `pylint`, and `pytest` to versions compatible with the new release, along with minor code changes to satisfy the updated tooling. Test coverage now runs across all supported Python versions. Downstream dependencies (Blueprint, Bladebridge) had already released Python 3.14 support. + +- **Remove explicit `cryptography` dependency ([#2458](https://github.com/databrickslabs/lakebridge/pull/2458))** + The explicit dependency on the `cryptography` package has been dropped. It was previously used directly for Snowflake connection setup but has not been needed since that code was revised. The package remains available as a transitive dependency. + +## New Contributors +* @dgomez04 made their first contribution in https://github.com/databrickslabs/lakebridge/pull/2431 +* @ysmx-github made their first contribution in https://github.com/databrickslabs/lakebridge/pull/2305 +* @lolo115 made their first contribution in https://github.com/databrickslabs/lakebridge/pull/2187 +* @jneil17 made their first contribution in https://github.com/databrickslabs/lakebridge/pull/2420 +* @take60 made their first contribution in https://github.com/databrickslabs/lakebridge/pull/2472 + +**Full Changelog**: https://github.com/databrickslabs/lakebridge/compare/v0.13.0...v0.14.0 + +--- + +# Lakebridge v0.13.0 Release Notes + +## Highlights + +A few headline changes in this release worth calling out: + +1. **SQL Server profiler is now available**, extending assessment coverage to Microsoft SQL Server alongside the existing Azure Synapse support. +2. **Redshift reconciliation is now supported**. Redshift can be used as a source for all reconcile report types. +3. **Major Morpheus conversion improvements for T-SQL and Redshift**, including a full Redshift dialect rollout (DDL, DML, functions, operators) and a wide expansion of T-SQL function and resilience handling. + +This release also includes a **major overhaul of the documentation**, aimed at simplifying the structure and making the docs easier to follow. + +## Assessment + +### Profiler + +- Added a new SQL Server profiler that extends the existing assessment capabilities to Microsoft SQL Server, closely mirroring the Azure Synapse profiler design. The implementation exposes a `last_execution_time` parameter on all server queries, laying the groundwork for future incremental/scheduled extractions. On-prem SQL Server is not in scope. ([#2151](https://github.com/databrickslabs/lakebridge/pull/2151)) +- Updated the Azure Synapse Workspace profiler summary dashboard and introduced a new dashboard template for SQL Server. The Synapse template removes deprecated dashboard widget parameters, parameterizes table values so they can be set dynamically by Lakebridge, adds a dedicated-storage summary widget by SQL pool, renames datasets for clarity, fixes broken column references in SQL pool activity widgets, and reformats dataset queries. ([#2317](https://github.com/databrickslabs/lakebridge/pull/2317)) +- Reworked the `create-profiler-dashboard` CLI flow to bring it in line with the rest of the Lakebridge installer experience: clearer prompts for extract file location, UC catalog, schema, and volume; a helper to parse the extract path and UC volume upload location; and dashboard install/uninstall hooked into the standard Lakebridge installer/uninstaller. ([#2319](https://github.com/databrickslabs/lakebridge/pull/2319)) +- Fixed a bug that produced false positives in `test-profiler-connection`. ([#2342](https://github.com/databrickslabs/lakebridge/pull/2342)) + +## Converters + +### Morpheus + +#### Snowflake + +- Tightened parsing of `CREATE FILE FORMAT` statements, including format type options, with clear diagnostics for unsupported variants. + +#### Synapse / T-SQL + +- Improved resilience by silently dropping unsupported constructs (table hints such as `WITH (NOLOCK)` and `READPAST`, `UPDATE STATISTICS`/`CREATE STATISTICS`, and other unsupported `CREATE TABLE` options) with warnings so surrounding scripts keep transpiling instead of failing. +- Expanded function coverage with a batch of T-SQL "easy wins" (`DATEPART` unit aliases, `EOMONTH`, `ISNUMERIC`, `TIME → STRING` conversions) and richer date/time handling (`DATETRUNC`, `DATE_BUCKET`, `SYSDATETIME`, `DATENAME`, normalized `DATE_PART` units). +- Added Databricks-compatible translations for `PATINDEX` (to `REGEXP_INSTR`, converting SQL wildcards to regex), `IIF` (mapped to `IF` at parse time), and `FORMATMESSAGE` (graceful fallback with diagnostics for unsupported format specifiers). +- Migrated the T-SQL functional test suite to the new `eval`-based scenario runner, expanding executable coverage and removing legacy skips. + +#### Redshift + +- Added Redshift as a first-class source dialect: a full dialect mapping in the converter (parser, IR builder, generator), Language Server advertisement alongside Snowflake and T-SQL, and a published per-feature workplan. +- Implemented Redshift DDL and DML coverage, including `CREATE TABLE` (distribution, sort key, constraint clauses), `DELETE … REMOVE DUPLICATES`, complex literals (arrays, super values, composite forms), and the supportable portion of the `SUPER` type with explicit rejection diagnostics for the rest. +- Added wide function coverage across string, `VARBYTE`, window, admin, `OBJECT`, JSON, HLL, and math families, plus date/time (`DATEADD`, `DATE_CMP`, `TIMEZONE`, timezone comparisons, `TIMEOFDAY`, `LAST_DAY`, `MONTHS_BETWEEN`, `SYSDATE`, single-argument `TRUNC`), numeric coercion (`TEXT_TO_INT_ALT`, `TEXT_TO_NUMERIC_ALT`), hashing (`CHECKSUM`, `FARMHASH64`), and a staged `TO_TIMESTAMP` implementation. +- Implemented Redshift-specific operators including the `+` overload (so string and date arithmetic disambiguate correctly) and the `|/` (square root) and `|//` (cube root) prefix operators. +- Explicitly rejected `EXPLAIN_MODEL` since Databricks SQL has no equivalent ML-model explainer, surfacing actionable diagnostics rather than silent miscompilation. + +#### General + +- Broadened cross-dialect DDL with `CREATE FUNCTION` (as much as is feasible per dialect, with diagnostics for unsupported procedural features) and `CREATE SCHEMA` for Snowflake, T-SQL, and Redshift, both lowered to Databricks SQL. +- Expanded shared function and expression support: `CURRENT_USER` across all dialects, additional `TIMESTAMP`-related functions, raw strings as proper IR expressions (so they participate in type inference and round-trip cleanly), and `EXECUTE IMMEDIATE` usable as an expression (not just a statement). +- Added full coverage of H3 and ST spatial functions across every supported dialect, with normalized names and argument shapes. +- Improved grammar flexibility by allowing reserved keywords `DATABASE` and `PRIMARY` to be used as identifiers in unambiguous contexts, unblocking real-world schemas. +- Expanded `DELETE` to cover all Redshift, T-SQL, and Snowflake use cases (including dialect-specific extensions) and added `UPDATE`-to-`MERGE` lowering for Redshift and T-SQL when an update uses a join or source table. + +## Reconcile + +- Added a Redshift connector to reconcile so Redshift can be used as a source for data, row, schema, and full report types. ([#2339](https://github.com/databrickslabs/lakebridge/pull/2339)) +- Replaced direct JDBC connections (Oracle, Snowflake, SQL Server) with Databricks Unity Catalog `remote_query()` calls backed by **UC Connections**. Reconcile no longer manages JDBC URLs, secret scopes, or PEM keys directly — authentication and connectivity are handled by Databricks. This introduces a v2 configuration format that takes a `uc_connection_name` in place of `secret_scope`; existing v1 configs are auto-migrated on load. ([#2362](https://github.com/databrickslabs/lakebridge/pull/2362)) +- Fixed reconcile schema fetch failures on Foreign Catalogs created via Lakehouse Federation. Foreign catalogs lack the Databricks-specific `full_data_type` column in `information_schema.columns`, which previously caused `UNRESOLVED_COLUMN` errors for all report types (`schema`, `data`, `row`, `all`). A new `DatabricksNonUnityCatalogDataSource` now falls back to `DESCRIBE TABLE` and covers `hive_metastore`, `global_temp` views, and Foreign Catalogs, while the native `DatabricksDataSource` remains scoped to Unity Catalog tables. ([#2422](https://github.com/databrickslabs/lakebridge/pull/2422)) +- Fixed a T-SQL/Synapse reconciliation regression where switching to `VARCHAR(MAX)` in hash concatenation broke date/time columns: SQL Server accepts `VARCHAR(256) + DATE` via implicit conversion but rejects `VARCHAR(MAX) + DATE`. Temporal transforms now `CONVERT` `DATE`/`TIME`/`DATETIME` to `VARCHAR(10)`/`VARCHAR(12)`/`VARCHAR(23)` so all temporal columns produce `VARCHAR` output that concatenates safely in the hash input string. ([#2320](https://github.com/databrickslabs/lakebridge/pull/2320)) +- Improved Oracle reconcile coverage, fixed parsing of remote query options, and dropped the legacy Oracle test scripts and Docker harness that required heavy manual setup. ([#2433](https://github.com/databrickslabs/lakebridge/pull/2433)) + +## Installer + +- Added minimal support for using a Maven mirror when installing Morpheus via `install-transpile`. Setting `LAKEBRIDGE_MAVEN_URL` overrides the default repository URL, and credentials can be supplied through `~/.netrc` (or via `NETRC`) so `install-transpile` works in environments without direct Maven Central access. ([#2405](https://github.com/databrickslabs/lakebridge/pull/2405)) +- Updated the wheel installer used during `install-transpile` to look up version information via `pip` instead of issuing a direct HTTP call to PyPI. This allows `install-transpile` to work in environments where only a local PyPI mirror is available. ([#2404](https://github.com/databrickslabs/lakebridge/pull/2404)) + +## Documentation + +- Major revamp of the Lakebridge documentation focused on clarity, structure, and first-time user experience: added a new end-to-end **Getting Started** tutorial (SQL Server → Databricks SQL walkthrough), a **Choosing Tools** decision guide, a dedicated **Morpheus transpiler** page, and a split-out **Switch architecture** page. Reconcile docs were consolidated from 5 files (~1,400 lines) to 3 (~750 lines) with a new report-type comparison table and unified Configuration Reference and Running Reconcile pages. SSIS docs were moved into a dedicated subfolder with a collapsible sidebar category. The Installation page was rewritten for brevity, the FAQ expanded from 3 to 25+ questions, and the sidebar reordered to match the actual user journey: Installation → Getting Started → Choosing Tools → Assessment → Transpile → Reconcile → SQL Splitter → FAQ. ([#2365](https://github.com/databrickslabs/lakebridge/pull/2365)) +- Fixed the reconcile notebook documentation to match the v2 `TableRecon` API: the example now shows `TableRecon` as `tables: list[Table]` only (with `source_schema`, `target_catalog`, `target_schema`, and `source_catalog` configured via `DatabaseConfig` inside `ReconcileConfig`), corrected the location of `drop_columns` (it belongs on `Table`, not `TableRecon`), and added a migration note pointing users to `DatabaseConfig`. ([#2329](https://github.com/databrickslabs/lakebridge/pull/2329)) +- Patched documentation transitive dependencies via new `yarn` resolutions, clearing all 75 known `yarn audit` vulnerabilities (33 high, 37 moderate, 5 low). ([#2340](https://github.com/databrickslabs/lakebridge/pull/2340)) + +## Build & Packaging + +- Migrated the project from Hatch to `uv`: all `Makefile` targets and CI/CD workflows now use `uv`, third-party dependencies are locked via `uv.lock` and `.build-constraints.txt`, workflow permissions have been narrowed, contribution documentation has been updated, the legacy release workflow has been removed, and the docusaurus `Makefile` targets have been refactored so `npm`/`yarn` no longer run package scripts and the `yarn.lock` is no longer auto-updated. ([#2374](https://github.com/databrickslabs/lakebridge/pull/2374)) + +--- + ## 0.12.2 ## Assessment diff --git a/Makefile b/Makefile index d3c9da0ed4..55dbb9206b 100644 --- a/Makefile +++ b/Makefile @@ -37,13 +37,10 @@ fmt: $(UV_RUN) mypy --disable-error-code 'annotation-unchecked' . $(UV_RUN) pylint --output-format=colorized -j 0 src tests -setup_spark_remote: - .github/scripts/setup_spark_remote.sh - test: $(UV_TEST) --cov-report=xml tests/unit -integration: setup_spark_remote +integration: $(UV_TEST) tests/integration coverage: @@ -58,11 +55,14 @@ build: lock-dependencies: UV_FROZEN := 0 lock-dependencies: - uv lock + uv lock --upgrade $(UV_RUN) --group yq tomlq -r '.["build-system"].requires[]' pyproject.toml | \ - uv pip compile --generate-hashes --universal --no-header --quiet - > build-constraints-new.txt + uv pip compile --upgrade --generate-hashes --universal --no-header --quiet - > build-constraints-new.txt mv build-constraints-new.txt .build-constraints.txt - @perl -pi -e 's|registry = "https://[^"]*"|registry = "https://pypi.org/simple/"|g' uv.lock + @perl -pi \ + -e 's|registry = "https://[^"]*"|registry = "https://pypi.org/simple"|g;' \ + -e 's|url = "https://[^"]*/packages/([^"]*)"|url = "https://files.pythonhosted.org/packages/$$1"|g;' \ + uv.lock @printf 'Stripped registry references from uv.lock.\n' clean_coverage_dir: diff --git a/docs/lakebridge/docs/assessment/analyzer/complexity_scoring.mdx b/docs/lakebridge/docs/assessment/analyzer/complexity_scoring.mdx index 16ec137513..a2a3ed1ce9 100644 --- a/docs/lakebridge/docs/assessment/analyzer/complexity_scoring.mdx +++ b/docs/lakebridge/docs/assessment/analyzer/complexity_scoring.mdx @@ -4,6 +4,31 @@ title: Complexity Scoring --- import useBaseUrl from '@docusaurus/useBaseUrl'; +## Overview + +The Lakebridge Analyzer assigns a complexity level to each SQL object or ETL job it processes. Complexity scores help you prioritize migration effort, estimate timelines, and identify which objects will require manual review after transpilation. + +Each object receives one of four levels: **LOW**, **MEDIUM**, **HIGH**, or **VERY HIGH**. + +## How to Interpret Your Scores + +| Level | What it means | Typical action | +|---|---|---| +| **LOW** | Simple object — few statements, no loops, no complex constructs | Transpile directly; little or no manual review expected | +| **MEDIUM** | Moderate complexity — some loops, 10–30 statements, limited use of advanced SQL | Transpile; review transpiler warnings before deploying | +| **HIGH** | High complexity — many loops, 30–50 statements, or advanced constructs (XML, PIVOT) | Plan for manual review after transpilation; test thoroughly | +| **VERY HIGH** | Highest complexity — heavy loop nesting, 50+ statements, or extreme PIVOT/XML usage | Allocate dedicated review time; consider using Switch for intent-aware conversion | + +**Planning guidance:** + +- If more than 50% of your objects score HIGH or VERY HIGH, plan for at least two manual review passes before deploying. +- Objects scored LOW and MEDIUM typically transpile cleanly with Morpheus or BladeBridge. +- HIGH and VERY HIGH objects are strong candidates for Switch, which handles business logic that deterministic transpilers struggle with. + +The scoring rules for each source system are detailed below. + +--- + ## SQL Code Analysis At the beginning of script analysis, mark a script with complexity level of **LOW** @@ -14,14 +39,14 @@ If any of the following conditions are true, then mark the job as **MEDIUM** com 4. Number of pivot statements between 1 and 3 5. Number of XML SQL statements between 1 and 3 -If any of the following conditions are true, then mark the job as **COMPLEX** complexity: +If any of the following conditions are true, then mark the job as **HIGH** complexity: 1. Number of loops greater than 5 2. Conventional Statement count greater than 30 3. Simple Statement count greater than 2000 4. Number of pivot statements greater than 3 5. Number of XML SQL statements greater than 3 -If any of the following conditions are true, then mark the job as **VERY COMPLEX** complexity: +If any of the following conditions are true, then mark the job as **VERY HIGH** complexity: 1. Number of loops greater than 8 2. Conventional Statement count greater than 50 3. Simple Statement count greater than 5000 @@ -47,7 +72,7 @@ If any of the following conditions are true, then mark the job as **MEDIUM** com 5. Number of targets > 1 6. Overall function call count >= 10 -If any of the following conditions are true, then mark the job as **COMPLEX** complexity: +If any of the following conditions are true, then mark the job as **HIGH** complexity: 1. Three MEDIUM breaks from the list above 2. Number of expressions with 5+ function calls between 5 and 7 3. Number of job components >= 20 @@ -55,8 +80,8 @@ If any of the following conditions are true, then mark the job as **COMPLEX** co 5. Complex or Unstructured nodes are being used (ChangeCapture, etc..) 6. Number of lookups between 7 and 14 -If any of the following conditions are true, then mark the job as **VERY COMPLEX** complexity: -1. Three COMPLEX breaks from the list above +If any of the following conditions are true, then mark the job as **VERY HIGH** complexity: +1. Three HIGH complexity breaks from the list above 2. Number of expressions with 5+ function calls > 7 3. Number of lookups > 15 4. Number of job components >= 50 @@ -73,15 +98,15 @@ If any of the following conditions are true, then mark the job as **MEDIUM** com 6. Overall function call count >= 10 -If any of the following conditions are true, then mark the job as **COMPLEX** complexity: +If any of the following conditions are true, then mark the job as **HIGH** complexity: 1. Three MEDIUM breaks from the list above 2. Number of expressions with 5+ function calls between 5 and 7 3. Number of job components >= 20 4. Overall function call count >= 20 5. Complex or Unstructured nodes are being used (ChangeCapture, etc..) -If any of the following conditions are true, then mark the job as **VERY COMPLEX** complexity: -1. Three COMPLEX breaks from the list above +If any of the following conditions are true, then mark the job as **VERY HIGH** complexity: +1. Three HIGH complexity breaks from the list above 2. Number of job components >= 50 ## SSIS Code Analysis @@ -94,14 +119,14 @@ If any of the following conditions are true, then mark the mapping as **MEDIUM** 4. Overall function call count >= 10 5. Number of package components >= 10 -If any of the following conditions are true, then mark the mapping as **COMPLEX** complexity: +If any of the following conditions are true, then mark the mapping as **HIGH** complexity: 1. Three MEDIUM breaks from the list above 2. Number of expressions with 5+ function calls between 5 and 7 3. Number of package components >= 20 4. Overall function call count >= 20 -If any of the following conditions are true, then mark the mapping as **VERY COMPLEX** complexity: -1. Three COMPLEX breaks from the list above +If any of the following conditions are true, then mark the mapping as **VERY HIGH** complexity: +1. Three HIGH complexity breaks from the list above 2. Number of expressions with 5+ function calls > 7 4. Number of job components >= 50 @@ -113,14 +138,14 @@ If any of the following conditions are true, then mark the mapping as **MEDIUM** 2. Overall function call count >= 10 3. Number of job components >= 10 -If any of the following conditions are true, then mark the mapping as **COMPLEX** complexity: +If any of the following conditions are true, then mark the mapping as **HIGH** complexity: 1. Three MEDIUM breaks from the list above 2. Number of expressions with 5+ function calls between 5 and 7 3. Number of package components >= 20 4. Overall function call count >= 20 -If any of the following conditions are true, then mark the mapping as **VERY COMPLEX** complexity: -1. Three COMPLEX breaks from the list above +If any of the following conditions are true, then mark the mapping as **VERY HIGH** complexity: +1. Three HIGH complexity breaks from the list above 2. Number of expressions with 5+ function calls > 7 4. Number of job components >= 50 @@ -132,14 +157,14 @@ If any of the following conditions are true, then mark the mapping as **MEDIUM** 2. Overall function call count >= 10 3. Number of job components >= 10 -If any of the following conditions are true, then mark the mapping as **COMPLEX** complexity: +If any of the following conditions are true, then mark the mapping as **HIGH** complexity: 1. Three MEDIUM breaks from the list above 2. Number of expressions with 5+ function calls between 5 and 7 3. Number of package components >= 20 4. Overall function call count >= 20 -If any of the following conditions are true, then mark the mapping as **VERY COMPLEX** complexity: -1. Three COMPLEX breaks from the list above +If any of the following conditions are true, then mark the mapping as **VERY HIGH** complexity: +1. Three HIGH complexity breaks from the list above 2. Number of expressions with 5+ function calls > 7 4. Number of job components >= 50 @@ -155,22 +180,22 @@ If any of the following conditions are true, then mark the script as **MEDIUM** 6. Count of SQL Procs categorized as MEDIUM > 0 7. SQL Proc count > 10 -If any of the following conditions are true, then mark the script as **COMPLEX**: +If any of the following conditions are true, then mark the script as **HIGH**: 1. Macro definition count > 7 2. Data block count > 15 3. number of statements inside macros and data blocks > 100 4. Conditional statement count > 20 5. 'DO' loop count > 10 -6. Count of SQL Procs categorized as COMPLEX > 0 +6. Count of SQL Procs categorized as HIGH complexity > 0 7. SQL Proc count > 20 -If any of the following conditions are true, then mark the script as **VERY COMPLEX**: +If any of the following conditions are true, then mark the script as **VERY HIGH**: 1. Macro definition count > 15 2. Data block count > 25 3. number of statements inside macros and data blocks > 150 4. Conditional statement count > 50 5. 'DO' loop count > 20 -6. Count of SQL Procs categorized as VERY COMPLEX > 0 +6. Count of SQL Procs categorized as VERY HIGH > 0 7. SQL Proc count > 40 ## Pentaho Code Analysis @@ -187,7 +212,7 @@ If any of the following conditions are true, then mark the mapping as **MEDIUM** 6. Overall function call count >= 10 7. Number of components (transformations) >= 10 -If any of the following conditions are true, then mark the mapping as **COMPLEX** complexity: +If any of the following conditions are true, then mark the mapping as **HIGH** complexity: 1. Three MEDIUM breaks from the list above 2. Number of expressions with 5+ function calls between 5 and 7 @@ -196,9 +221,9 @@ If any of the following conditions are true, then mark the mapping as **COMPLEX* 5. Complex or Unstructured nodes are being used (e.g. Normalizer) 6. Number of lookups between 7 and 14 -If any of the following conditions are true, then mark the mapping as **VERY COMPLEX** complexity: +If any of the following conditions are true, then mark the mapping as **VERY HIGH** complexity: -1. Three COMPLEX breaks from the list above +1. Three HIGH complexity breaks from the list above 2. Number of expressions with 5+ function calls > 7 3. Number of lookups > 15 4. Number of job components >= 50 diff --git a/docs/lakebridge/docs/assessment/index.mdx b/docs/lakebridge/docs/assessment/index.mdx index 15b934edd3..8ac6b7cde4 100644 --- a/docs/lakebridge/docs/assessment/index.mdx +++ b/docs/lakebridge/docs/assessment/index.mdx @@ -1,13 +1,14 @@ --- -sidebar_position: 3 +sidebar_position: 4 title: Assessment Guide --- ## Table of Contents -| Guide | Description | -|-------------------------------|-------------------------------------------------------------------| -| [Profiler Guide](./profiler/) | Instructions for running the Profiler component. | -| [Analyzer Guide](./analyzer/) | Learn how to use the Analyzer for metadata scanning and insights. | +| Guide | Description | +|----------------------------------------|----------------------------------------------------------------------------------------| +| [SQL Splitter](/docs/sql_splitter) | (Optional) Split monolithic SQL files into individual objects before assessing or transpiling. | +| [Profiler Guide](./profiler/) | Instructions for running the Profiler component. | +| [Analyzer Guide](./analyzer/) | Learn how to use the Analyzer for metadata scanning and insights. | diff --git a/docs/lakebridge/docs/assessment/profiler/bigquery.mdx b/docs/lakebridge/docs/assessment/profiler/bigquery.mdx new file mode 100644 index 0000000000..2c1df25cd9 --- /dev/null +++ b/docs/lakebridge/docs/assessment/profiler/bigquery.mdx @@ -0,0 +1,127 @@ +--- +sidebar_position: 3 +title: BigQuery Profiler Details +--- +import Admonition from '@theme/Admonition'; + +# BigQuery Profiler Details + +- [Prerequisites](#prerequisites) +- [Configure Connection to BigQuery](#configure-connection-to-bigquery) +- [Execute the profiler](#execute-the-profiler) + +## Prerequisites + +### 1. Download +- gcloud CLI (https://cloud.google.com/sdk/docs/install) + +- No driver install required for the BigQuery profiler + +### 2. Authenticate to GCP + +The profiler authenticates with [Application Default Credentials (ADC)](https://docs.cloud.google.com/docs/authentication/application-default-credentials). The simplest setup is: + +```bash +gcloud auth application-default login +``` + +For more options, please review the docs. + +### 3. Required IAM permissions on the target BigQuery project(s) + +The executing identity needs the following permissions. `roles/bigquery.admin` covers all of them; for least-privilege deployments grant the discrete permissions below. + +``` +bigquery.jobs.create +bigquery.jobs.list +bigquery.datasets.get +bigquery.datasets.getIamPolicy +bigquery.tables.get +bigquery.tables.getData +bigquery.routines.list +bigquery.readSessions.create +bigquery.reservations.list # Editions customers only +bigquery.reservationAssignments.list # Editions customers only +bigquery.capacityCommitments.list # Editions customers only +``` + +### 4. INFORMATION_SCHEMA / region-scoped views accessed + +| Profiler aspect | Region-scoped view | Required permission | +|---|---|---| +| Job history (workload, timeline, fulfillment, jobs-by-reservation) | `region-{region}.INFORMATION_SCHEMA.JOBS_BY_PROJECT`, `JOBS_TIMELINE` | `bigquery.jobs.list` (job-owner sees own jobs; admin sees all) | +| Table storage | `region-{region}.INFORMATION_SCHEMA.TABLE_STORAGE_BY_PROJECT` | `bigquery.tables.get` + `bigquery.tables.getData` | +| Streaming inserts | `region-{region}.INFORMATION_SCHEMA.STREAMING_TIMELINE_BY_PROJECT` | `bigquery.jobs.list` | +| Write API | `region-{region}.INFORMATION_SCHEMA.WRITE_API_TIMELINE_BY_PROJECT` | `bigquery.jobs.list` | +| Reservation timeline | `region-{region}.INFORMATION_SCHEMA.RESERVATIONS_TIMELINE`, `RESERVATION_CHANGES` | `bigquery.reservations.list` | +| Capacity commitments | `region-{region}.INFORMATION_SCHEMA.CAPACITY_COMMITMENT_CHANGES_BY_PROJECT` | `bigquery.capacityCommitments.list` (admin project only) | + +:::warning Attention: +Skip the reservation/commitment permissions if the customer is on **Pay-as-you-go pricing** rather than BigQuery Editions. The corresponding SQL files will return empty results — harmless. You can also set `exclude_reservations_data: true` during configuration to skip those queries entirely (saves BigQuery slot time; the corresponding DuckDB tables are still created as empty stubs so the output schema stays consistent). +::: + +:::warning Attention: +Some queries (commitments, capacity commitments) only return data when run against the customer's **administrative project** — the one that owns the BigQuery Edition slot reservations. For non-admin projects they return empty. +::: + +## Configure Connection to BigQuery + +```console +databricks labs lakebridge configure-database-profiler + +Please select the source system you want to configure +[0] bigquery +[1] mssql +[2] synapse +Enter a number between 0 and 2: 0 + +(local | env) +local means values are read as plain text +env means values are read from environment variables, and fall back to plain text if not variable is not found + +Enter secret vault type (local | env) +[0] env +[1] local +Enter a number between 0 and 1: 1 +Please provide BigQuery connection settings: + +Enter BigQuery project and region pairs (Format: comma-separated project.region. Example: my-proj-a.us, my-proj-b.eu-west-1): customer-prod-1.us, customer-admin.us +Enter lookback window in days to profile (default: 180): +Enter max parallel SQLs per (project, region) iteration (default: 8): +Please configure profiler settings: +Exclude reservations and commitments data? (default: no): +Exclude streaming and write API summary? (default: no): +``` + +The resulting `~/.databricks/labs/lakebridge/.credentials.yml` looks like: + +```yaml +secret_vault_type: local +secret_vault_name: null +bigquery: + pairs: + - project: customer-prod-1 + region: us + - project: customer-admin + region: us + profiler: + profiling_window_days: 180 + max_parallel_sqls: 8 + redact_query_text: true + exclude_reservations_data: false + exclude_streaming_metrics: false +``` + +## Execute the profiler + +```bash +databricks labs lakebridge execute-database-profiler --source-tech bigquery +``` + +The pipeline runs `bq_metadata_extract` — connects to BigQuery and runs 16 vendored SQL queries serially across the configured `(project, region)` pairs and in parallel within each iteration (`max_parallel_sqls` workers). Results land in `~/.databricks/labs/lakebridge_profilers/bigquery_assessment/profiler_extract.db` (DuckDB). + +The final DuckDB contains **12 analysis tables** by default (one per `analysis_type`). + +Setting `exclude_reservations_data: true` skips the 6 reservation/commitment BigQuery queries — the corresponding DuckDB tables are still created (empty) to keep the output schema consistent. + +[Back to Configure Profiler](../#configure-profiler) diff --git a/docs/lakebridge/docs/assessment/profiler/dashboards/index.mdx b/docs/lakebridge/docs/assessment/profiler/dashboards/index.mdx deleted file mode 100644 index 69e3eae83c..0000000000 --- a/docs/lakebridge/docs/assessment/profiler/dashboards/index.mdx +++ /dev/null @@ -1,121 +0,0 @@ ---- -sidebar_position: 1 -title: Profiler Summary Dashboard ---- -import Admonition from '@theme/Admonition'; - -# Profiler Summary Dashboard - -:::warning Attention: -The Profiler Summary Dashboard is currently an Experimental feature in Lakebridge. For any feedback and/or issues, -feel free to reach out via Github issues. -::: - -## Overview - -The `visualize-profiler-results` command uploads a DuckDB profiler extract file to a Unity Catalog Volume, -deploys a source-specific Lakeview dashboard template to your Databricks workspace, and triggers an ingestion job -that loads the extract data into Delta tables. The result is a live dashboard you can open immediately in your browser. - -## Prerequisites - -- Unity Catalog must be enabled on your Databricks workspace. -- Your user account must have sufficient privileges on the target catalog, schema, and volume: - - `CREATE CATALOG` (or `USE CATALOG` on an existing catalog) - - `CREATE SCHEMA` (or `USE SCHEMA` on an existing schema) - - `CREATE VOLUME` (or `READ VOLUME` + `WRITE VOLUME` on an existing volume) -- At least one **PRO** or **SERVERLESS** SQL warehouse must be available in the workspace. -- A profiler extract file (`.db`) must already exist locally, produced by a previous - `execute-database-profiler` run. - -## Supported Sources - -| Source Platform | Configuration Status | -|:---------------:|:-------------------:| -| Azure Synapse | ✅ | -| Amazon Redshift | ❌ | -| Oracle | 🔸 Coming Soon | -| Microsoft SQL Server | 🔸 Coming Soon | -| Snowflake | 🔸 Coming Soon | - -## Running the Command - -```bash -databricks labs lakebridge visualize-profiler-results -``` - -The command is fully interactive — there are no CLI flags. It walks you through the following prompts: - -```console -# Shown only when a profiler dashboard is already installed -Do you want to override the existing installation? [y/n]: - -Enter the source tech: [synapse] -Enter the path to the profiler extract file: [/tmp/data/synapse_assessment/profiler_extract.db] -Enter catalog name [remorph]: -Enter schema name [profiler]: -Enter volume name [ingestion_volume]: - -# Shown if the catalog does not exist -Catalog `` doesn't exist. Create it? [y/n]: - -# Shown if the schema does not exist in the catalog -Schema `` doesn't exist in catalog ``. Create it? [y/n]: - -# Shown if the volume does not exist in the catalog and schema -Volume `` doesn't exist in catalog `` and schema ``. Create it? [y/n]: -``` - -Default values are shown in brackets `[…]`. Press **Enter** to accept a default. - -## What Happens After You Run - -1. **Extract uploaded** — the local file is uploaded to the UC Volume at - `/Volumes////`. -2. **Dashboard deployed** — a Lakeview dashboard is created from the source-specific template. - The placeholders `` `PROFILER_CATALOG` `` and `` `PROFILER_SCHEMA` `` in the template are substituted - with the catalog and schema names you provided. -3. **Ingestion job created and run** — a Databricks job named `"Profiler Dashboard Ingestion Job"` is - created in your workspace and immediately executed. It **overwrites** all profiler Delta tables with - the data from the extract file you specified. Existing table data is not preserved. -4. **Configuration saved** — your choices are persisted to `profiler_dashboard.yml` in the Lakebridge - install folder for future reference. - -## Data Behavior Details - -**Volume path resolution** -The extract file is uploaded to `/Volumes///`. If the resolved UC Volume path -already ends with a file suffix (e.g. `.db`), that path is used as-is. Otherwise the extract filename -is automatically appended to the volume path. - -**Re-running the command** - -:::caution Data overwrite — no append support -Each run **completely replaces** the profiler Delta tables with the contents of the extract file you -provide. Data from previous runs is not preserved or merged. If you have run `execute-database-profiler` -multiple times and want to keep historical data, you must manage that yourself before re-running this -command. -::: - -In addition to the table overwrite, any previously deployed profiler dashboard is automatically deleted -and replaced with the latest template. You will be prompted to confirm the override before any changes -are made. - -**Configuration persistence** -Answers to all prompts are saved as `profiler_dashboard.yml` in the Lakebridge install folder. On -subsequent runs the command detects this file and asks `Do you want to override the existing installation?` -before proceeding. - -## Error Reference - -| Condition | Logged Message | Remediation | -|-----------|----------------|-------------| -| Local extract file missing | `Local file not found: ` | Verify the path; run `execute-database-profiler` first to generate the extract | -| Volume path invalid | `Invalid Volume path provided` | Ensure the volume exists and that the path resolves to a location starting with `/Volumes/` | -| Insufficient permissions | `Insufficient privileges detected while accessing Volume path` | Grant `CREATE`/`WRITE` privileges on the catalog, schema, and volume to your user | -| Databricks internal error | `Internal Databricks error while uploading extract file` | Retry the command; contact Databricks support if the error persists | - -## Related Commands - -- [Configure Profiler](../#configure-profiler) — set up connection details before profiling -- [Execute Profiler](../#execute-profiler) — run the profiler to generate the extract file consumed by this command diff --git a/docs/lakebridge/docs/assessment/profiler/index.mdx b/docs/lakebridge/docs/assessment/profiler/index.mdx index d0e13b7f15..cb74faee0f 100644 --- a/docs/lakebridge/docs/assessment/profiler/index.mdx +++ b/docs/lakebridge/docs/assessment/profiler/index.mdx @@ -33,6 +33,13 @@ Key capabilities: | Source Platform | Configuration Status | |:---------------:|:-------------------:| | Azure Synapse | ✅ | +| Teradata | ✅ | +| Snowflake | ✅ | +| Microsoft SQL Server | ✅ | +| Legacy Synapse (Dedicated SQL Pool) | ✅ | +| Oracle | ✅ | +| Google BigQuery | ✅ | +| Amazon Redshift | ✅ | ## Configure Profiler @@ -84,7 +91,3 @@ The profiler can be run multiple times to capture different time periods or upda Each execution will create a timestamped snapshot of your source environment. ::: -## Publish Profiler Summary Dashboard - -Visualize your profiler results as a Lakeview dashboard deployed directly to your Databricks workspace. -See the full guide: [Profiler Summary Dashboard](./dashboards). diff --git a/docs/lakebridge/docs/assessment/profiler/legacy_synapse.mdx b/docs/lakebridge/docs/assessment/profiler/legacy_synapse.mdx new file mode 100644 index 0000000000..f360e42bd1 --- /dev/null +++ b/docs/lakebridge/docs/assessment/profiler/legacy_synapse.mdx @@ -0,0 +1,208 @@ +--- +sidebar_position: 3 +title: Legacy Synapse Profiler Details +--- +import Admonition from '@theme/Admonition'; + +# Legacy Synapse Profiler Details + +- [Prerequisites](#prerequisites) +- [Profiled Tables and Views](#profiled-tables-and-views) +- [Configure Connection to Legacy Synapse](#configure-connection-to-legacy-synapse) +- [Execute the Profiler](#execute-the-profiler) + +:::info +This profiler targets **Azure Synapse dedicated SQL pool** (formerly Azure SQL Data Warehouse). +For serverless SQL pools, Spark pools, and the Synapse workspace control plane, use the [Synapse profiler](./synapse.mdx) instead. +::: + +## Prerequisites + +### 1. Download + +No driver installation is required: the profiler connects with Microsoft's +[mssql-python](https://github.com/microsoft/mssql-python) driver, which bundles its own connectivity layer. + +### 2. Authentication + +The profiler supports the following authentication modes (`configure-database-profiler` prompts for one): + +| Auth method | Description | MFA-capable | +|---|---|---| +| `SqlPassword` | SQL Authentication — username + password from credentials file | No | +| `DefaultAzureCredential` | Entra ID via the Azure Identity credentials chain. Recommended for Azure-hosted targets | Yes | +| `ActiveDirectoryPassword` | Entra ID (Azure AD) username + password | No | +| `ActiveDirectoryServicePrincipal` | Service Principal — `AZURE_CLIENT_ID` / `AZURE_CLIENT_SECRET` env vars | No | + +:::warning +For `ActiveDirectoryServicePrincipal`, set `AZURE_CLIENT_ID` and `AZURE_CLIENT_SECRET` env vars before running the profiler. +For `DefaultAzureCredential`, run `az login` first; for unattended runs set `AZURE_TENANT_ID`, `AZURE_CLIENT_ID`, and `AZURE_CLIENT_SECRET`. +::: + + +### 3. Required Database Permissions + +The SQL user configured for the profiler must have read access to the system catalog views and PDW Dynamic Management Views +listed in [Profiled Tables and Views](#profiled-tables-and-views). + +Connect to the target dedicated SQL pool as an admin and run: + +```sql +-- Create a contained user from the server login +CREATE USER [] FROM LOGIN []; + +-- Grant access to PDW Dynamic Management Views (sys.dm_pdw_*) +GRANT VIEW DATABASE STATE TO []; + +-- Grant access to object definitions (routines, views) +GRANT VIEW DEFINITION TO []; + +-- Grant read access to INFORMATION_SCHEMA views +GRANT SELECT ON SCHEMA::INFORMATION_SCHEMA TO []; +``` + +:::tip +The profiler queries server-level catalog views such as `sys.databases` from the target database context. +`VIEW DATABASE STATE` covers the `sys.dm_pdw_*` DMVs used to extract sessions, requests, and per-node storage statistics. +::: + +## Profiled Tables and Views + +The profiler executes queries against the following system tables and PDW DMVs. The results are organized +into two extraction steps: **schema metadata** and **activity metrics**. + +### Schema Metadata + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
QuerySource Table(s)Description
Databasessys.databasesLists each database by name.
TablesINFORMATION_SCHEMA.TABLESExtracts table definitions and types from the dedicated SQL pool.
ViewsINFORMATION_SCHEMA.VIEWSExtracts view definitions (SQL text is redacted for security).
ColumnsINFORMATION_SCHEMA.COLUMNSColumn-level metadata including data types, nullability, precision, collation, and domain information.
RoutinesINFORMATION_SCHEMA.ROUTINESStored procedures and user-defined functions (routine definitions are redacted for security).
+ +### Activity Metrics + + + + + + + + + + + + + + + + + + + + + + + + + + +
QuerySource Table(s)Description
Requestssys.dm_pdw_exec_requestsRecently executed PDW requests with submit/start/end times, total elapsed time, status, resource class, and the originating command.
Sessionssys.dm_pdw_exec_sessionsActive user sessions with login info, app names, query counts, and transaction state. System sessions are excluded.
Storage Infosys.dm_pdw_nodes_db_partition_statsPer-compute-node storage usage: reserved and used space (MB), aggregated across all partitions on each node.
+ +## Configure Connection to Legacy Synapse + +Run the following command to configure the profiler connection to your dedicated SQL pool: + +```console +databricks labs lakebridge configure-database-profiler + +Please select the source system you want to configure +[0] synapse +[1] mssql +[2] legacy_synapse +Enter a number between 0 and 2: 2 + +(local | env) +local means values are read as plain text +env means values are read from environment variables, and fall back to plain text if not variable is not found + +Enter secret vault type (local | env) +[0] env +[1] local +Enter a number between 0 and 1: 1 +Select authentication method +[0] SqlPassword +[1] DefaultAzureCredential +[2] ActiveDirectoryPassword +[3] ActiveDirectoryServicePrincipal +Enter a number between 0 and 3: 0 +Enter the username: profiler_user +Enter the password: +Enter fetch size (default: 1000): +Enter login timeout (seconds) (default: 30): +Enter the fully-qualified server name: my-dw-server.database.windows.net +Enter the port details: 1433 +Enter the database name: my_pool +Trust server certificate (default: no): +Enter timezone (e.g. America/New_York) (default: UTC): +``` + +### Configuration Parameters + +| Parameter | Description | Default | +|:----------|:------------|:--------| +| **Secret vault type** | `local` for plain text values, `env` to read from environment variables | — | +| **Authentication method** | One of the four modes in the table above | `SqlPassword` | +| **Username / Password** | Required for `SqlPassword` and `ActiveDirectoryPassword`. Skipped for `DefaultAzureCredential` and `ActiveDirectoryServicePrincipal`. | — | +| **Fetch size** | Number of rows fetched per batch from the source | `1000` | +| **Login timeout** | Connection timeout in seconds | `30` | +| **Server name** | Fully-qualified hostname of the dedicated SQL pool | — | +| **Port** | Server port number | — | +| **Database** | The dedicated SQL pool name to connect to. Dedicated pools do not have a `master` database, so this must be set to the pool you want to profile. | — | +| **Trust server certificate** | Skip TLS certificate validation when connecting. Leave as `no` for Azure-hosted dedicated SQL pools, which present valid certificates. | `no` | +| **Timezone** | Timezone for timestamp normalization | `UTC` | + +## Execute the Profiler + +Once configured, run the profiler to extract metadata and activity metrics from your dedicated SQL pool: + +```bash +databricks labs lakebridge execute-database-profiler --source-tech legacy_synapse +``` + +The profiler will: +1. Connect to your dedicated SQL pool using the configured credentials +2. Execute the schema metadata and activity metric extraction queries +3. Store the results in a local DuckDB extract file + +[Back to Configure Profiler](../#configure-profiler) diff --git a/docs/lakebridge/docs/assessment/profiler/mssql.mdx b/docs/lakebridge/docs/assessment/profiler/mssql.mdx new file mode 100644 index 0000000000..f78cac0365 --- /dev/null +++ b/docs/lakebridge/docs/assessment/profiler/mssql.mdx @@ -0,0 +1,262 @@ +--- +sidebar_position: 2 +title: MSSQL Profiler Details +--- +import Admonition from '@theme/Admonition'; + +# MSSQL Profiler Details + +- [Prerequisites](#prerequisites) +- [Profiled Tables and Views](#profiled-tables-and-views) +- [Configure Connection to MSSQL](#configure-connection-to-mssql) +- [Execute the Profiler](#execute-the-profiler) + +## Prerequisites + +### 1. Download + +No driver installation is required: the profiler connects with Microsoft's +[mssql-python](https://github.com/microsoft/mssql-python) driver, which bundles its own connectivity layer. + +### 2. Authentication + +The profiler supports the following authentication modes (`configure-database-profiler` prompts for one): + +| Auth method | Description | MFA-capable | +|---|---|---| +| `SqlPassword` | SQL Authentication — username + password from credentials file | No | +| `DefaultAzureCredential` | Entra ID via the Azure Identity credentials chain. Recommended for Azure-hosted targets | Yes | +| `ActiveDirectoryPassword` | Entra ID (Azure AD) username + password | No | +| `ActiveDirectoryServicePrincipal` | Service Principal — `AZURE_CLIENT_ID` / `AZURE_CLIENT_SECRET` env vars | No | + +:::warning +For `ActiveDirectoryServicePrincipal`, set `AZURE_CLIENT_ID` and `AZURE_CLIENT_SECRET` env vars before running the profiler. +For `DefaultAzureCredential`, run `az login` first; for unattended runs set `AZURE_TENANT_ID`, `AZURE_CLIENT_ID`, and `AZURE_CLIENT_SECRET`. +::: + +### 3. Required Database Permissions + +The SQL user configured for the profiler must have read access to the system catalog views and Dynamic Management Views (DMVs) +listed in [Profiled Tables and Views](#profiled-tables-and-views). + +Permissions differ depending on whether you are running against an **on-premises SQL Server** or **Azure SQL Database**. + +#### On-Premises SQL Server + +On a self-hosted SQL Server instance, a server-level grant is sufficient: + +```sql +-- Grant access to Dynamic Management Views (DMVs) +GRANT VIEW SERVER STATE TO []; + +-- Grant access to object definitions (routines, views) +GRANT VIEW DEFINITION TO []; + +-- Grant read access to INFORMATION_SCHEMA views +GRANT SELECT ON SCHEMA::INFORMATION_SCHEMA TO []; +``` + +:::tip +`VIEW SERVER STATE` is a server-level permission required to query `sys.dm_*` Dynamic Management Views. +`VIEW DEFINITION` allows the user to see the definitions of stored procedures and views. +These grants should be executed by a `sysadmin` or a login with `CONTROL SERVER` permission. +::: + +#### Azure SQL Database + +:::warning Attention: +`VIEW SERVER STATE` is **not supported** on Azure SQL Database. You must use `VIEW DATABASE STATE` instead, +which must be granted **per database** — including the `master` database. +::: + +First, identify your target database(s) by connecting as an admin and running: + +```sql +SELECT name FROM sys.databases WHERE database_id > 4; +``` + +Then grant permissions in both `master` and each target database: + +```sql +-- In the master database (required for server-level DMVs like sys.databases, sys.dm_os_sys_info) +USE master; +CREATE USER [] FROM LOGIN []; +GRANT VIEW DATABASE STATE TO []; + +-- In each target database +USE []; +CREATE USER [] FROM LOGIN []; +GRANT VIEW DATABASE STATE TO []; +GRANT VIEW DEFINITION TO []; +GRANT SELECT ON SCHEMA::INFORMATION_SCHEMA TO []; +``` + +:::tip +On Azure SQL Database, `VIEW DATABASE STATE` is scoped to a single database. The profiler queries +server-level DMVs (e.g., `sys.databases`, `sys.dm_os_sys_info`) in the `master` database context, so +the user must have `VIEW DATABASE STATE` granted there in addition to the target database(s). +::: + +## Profiled Tables and Views + +The MSSQL profiler executes queries against the following system tables and DMVs. The results are organized into +two extraction steps: **schema metadata** and **activity metrics**. + +### Schema Metadata + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
QuerySource Table(s)Description
System Infosys.dm_os_sys_infoInstance-level metadata: CPU topology, memory, virtualization, and SQL Server start time.
Databasessys.databasesLists all user databases (excluding system databases) with IDs, names, collation, and creation dates.
TablesINFORMATION_SCHEMA.TABLESExtracts table definitions and types from each database.
ViewsINFORMATION_SCHEMA.VIEWSExtracts view definitions (SQL text is redacted for security).
ColumnsINFORMATION_SCHEMA.COLUMNSColumn-level metadata including data types, nullability, precision, collation, and domain information.
Indexed Viewssys.views, sys.indexesIdentifies views that have clustered or non-clustered indexes, with index type and IDs.
RoutinesINFORMATION_SCHEMA.ROUTINESStored procedures and user-defined functions (routine definitions are redacted for security).
Database Sizessys.database_filesDatabase file metadata including current size, free space, and maximum configured size (in MB).
Table Sizessys.dm_db_partition_stats, sys.objectsPer-table storage metrics: row counts, reserved/used/unused space, and data vs. index space breakdown.
+ +### Activity Metrics + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
QuerySource Table(s)Description
Query Statssys.dm_exec_query_stats, sys.dm_exec_sql_textRecently executed queries classified by command type (QUERY, DML, DDL, ROUTINE, TRANSACTION_CONTROL, OTHER) with execution count, duration, CPU time, and row counts.
Procedure Statssys.dm_exec_procedure_statsStored procedure execution statistics including execution counts, total CPU time, and elapsed time.
Sessionssys.dm_exec_sessionsActive user sessions with login info (hashed), program names, CPU/memory usage, and request timing.
CPU Utilizationsys.dm_os_ring_buffers, sys.dm_os_sys_infoCPU utilization over time, including system idle percentage and SQL Server process utilization.
+ +## Configure Connection to MSSQL + +Run the following command to configure the profiler connection to your SQL Server instance: + +```console +databricks labs lakebridge configure-database-profiler + +Please select the source system you want to configure +[0] synapse +[1] mssql +Enter a number between 0 and 1: 1 + +Enter secret vault type (local | env) +[0] env +[1] local +Enter a number between 0 and 1: 1 +Select authentication method +[0] SqlPassword +[1] DefaultAzureCredential +[2] ActiveDirectoryPassword +[3] ActiveDirectoryServicePrincipal +Enter a number between 0 and 3: 0 +Enter the username: profiler_user +Enter the password: +Enter fetch size (default: 1000): +Enter login timeout (seconds) (default: 30): +Enter the fully-qualified server name: my-server.database.windows.net +Enter the port details: 1433 +Enter the database name: MyAppDB +Trust server certificate (default: no): +Enter timezone (e.g. America/New_York) (default: UTC): +Do you want to test the connection to mssql? (yes/no): yes +``` + +For `DefaultAzureCredential` and `ActiveDirectoryServicePrincipal`, the `Enter the username` / `Enter the password` prompts are skipped — the identity is resolved at run time (`az login` or `AZURE_*` env vars). + +### Configuration Parameters + +| Parameter | Description | Default | +|:----------|:------------|:--------| +| **Secret vault type** | `local` for plain text values, `env` to read from environment variables | — | +| **Authentication method** | One of the four modes in the table above | `SqlPassword` | +| **Username / Password** | Required for `SqlPassword` and `ActiveDirectoryPassword`. Skipped for `DefaultAzureCredential` and `ActiveDirectoryServicePrincipal`. | — | +| **Fetch size** | Number of rows fetched per batch from the source | `1000` | +| **Login timeout** | Connection timeout in seconds | `30` | +| **Server name** | Fully-qualified SQL Server hostname | — | +| **Port** | SQL Server port number | — | +| **Database** | Database to connect to. `INFORMATION_SCHEMA` queries are scoped to this database; other databases on the instance are still discovered via `sys.databases`. | — | +| **Trust server certificate** | Skip TLS certificate validation when connecting. Set to `yes` only when the server uses a self-signed or untrusted certificate (e.g., local/dev SQL Server). Leave as `no` for Azure SQL Database and production servers with valid certificates. | `no` | +| **Timezone** | Timezone for timestamp normalization | `UTC` | + +## Execute the Profiler + +Once configured, run the profiler to extract metadata and activity metrics from your SQL Server instance: + +```bash +databricks labs lakebridge execute-database-profiler --source-tech mssql +``` + +The profiler will: +1. Connect to your SQL Server instance using the configured credentials +2. Execute the schema metadata and activity metric extraction queries +3. Store the results in a local DuckDB extract file + +[Back to Configure Profiler](../#configure-profiler) diff --git a/docs/lakebridge/docs/assessment/profiler/oracle.mdx b/docs/lakebridge/docs/assessment/profiler/oracle.mdx new file mode 100644 index 0000000000..6df2309bb7 --- /dev/null +++ b/docs/lakebridge/docs/assessment/profiler/oracle.mdx @@ -0,0 +1,203 @@ +--- +sidebar_position: 4 +title: Oracle Profiler Details +--- +import Admonition from '@theme/Admonition'; + +# Oracle Profiler Details + +- [Prerequisites](#prerequisites) +- [Profiled Tables and Views](#profiled-tables-and-views) +- [Configure Connection to Oracle](#configure-connection-to-oracle) +- [Execute the Profiler](#execute-the-profiler) + +:::info +The Oracle profiler targets **multitenant container databases (CDB)** and reads from `CDB_*`, `GV$*`, and AWR views. +It must be run against `CDB$ROOT`, not an individual PDB. +::: + +## Prerequisites + +### 1. Download + +No local Oracle client install is required. + +### 2. Network and Authentication + +:::warning Attention: +The profiler connects with **username and password over TCP** only. The following are **not supported**: + +- TCPS / TLS-encrypted listener connections +- Oracle Wallet (mTLS, externally identified users) +- Kerberos / OS authentication +- Multi-factor authentication + +The Oracle listener must be reachable from the host running `databricks labs lakebridge` on the configured TNS port. +::: + +### 3. Required Database Permissions + +The Oracle user configured for the profiler must be able to read the data dictionary and AWR views listed in +[Profiled Tables and Views](#profiled-tables-and-views). + +Connect to `CDB$ROOT` as `SYS` (or a DBA-privileged user) and run: + +```sql +-- Create a common profiler user (skip if reusing an existing account) +CREATE USER c##lakebridge_profiler IDENTIFIED BY CONTAINER=ALL; +GRANT CREATE SESSION TO c##lakebridge_profiler CONTAINER=ALL; +GRANT SET CONTAINER TO c##lakebridge_profiler CONTAINER=ALL; + +-- Catalog and AWR access +GRANT SELECT_CATALOG_ROLE TO c##lakebridge_profiler CONTAINER=ALL; +``` + +:::warning Diagnostic Pack license +The performance queries read from AWR (`CDB_HIST_*`), which requires an Oracle **Diagnostic Pack** license. AWR +views return data on any Enterprise Edition database whether or not the pack is licensed — querying them without +a license is an audit-time violation, not a runtime error. + +By running the profiler against an Oracle source you are asserting that your database is appropriately licensed. +::: + +## Profiled Tables and Views + +The Oracle profiler executes queries against the following Oracle data dictionary and AWR views. The results are +organized into two extraction steps: **configuration metadata** and **performance metrics**. + +### Configuration Metadata + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
QuerySource View(s)Description
Containersgv$containersCDB and PDB inventory across all RAC instances.
DB Featuresproduct_component_version, gv$instance, gv$pdbs, gv$osstatDatabase version, instance count, PDB list, and CPU/core/socket counts.
Instancegv$instanceInstance id, name, version, and database type.
Memory Evolutioncdb_hist_parameter, cdb_hist_snapshotHistorical SGA/PGA/buffer-cache parameter values across AWR snapshots.
PDB Objectscdb_objects, cdb_usersObject counts per PDB, owner, and object type (Oracle-maintained schemas excluded).
PDB Partitionscdb_tables, cdb_indexes, cdb_lobsPartitioned vs. non-partitioned counts for tables, indexes, and LOBs per PDB and owner.
Storagecdb_data_files, cdb_free_space, cdb_tablespacesAllocated, free, and max sizes per container, grouped into SYSTEM, UNDO, and USER_DATA.
+ +### Performance Metrics + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
QuerySource View(s)Description
CPU & Waitscdb_hist_active_sess_historyHourly wait-event and CPU-time breakdown per PDB and instance from ASH samples.
Foreground Session Evolutioncdb_hist_active_sess_history, cdb_usersPer-minute distinct foreground session counts by PDB, instance, and end-user.
CPU Heatmapcdb_hist_active_sess_history, gv$osstatHourly Average Active Sessions normalized by CPU core count.
SQL Textcdb_hist_sqlstat, cdb_hist_sqltext, audit_actionsSQL statements from AWR, classified into workload categories (BI/QUERY, ETL, DML, DDL, PL/SQL) with execution counts and total elapsed time per schema and PDB.
+ +## Configure Connection to Oracle + +Run the following command to configure the profiler connection to your Oracle CDB: + +```console +databricks labs lakebridge configure-database-profiler + +Please select the source system you want to configure +[0] synapse +[1] mssql +[2] oracle +Enter a number between 0 and 2: 2 + +(local | env) +local means values are read as plain text +env means values are read from environment variables, and fall back to plain text if not variable is not found + +Enter secret vault type (local | env) +[0] env +[1] local +Enter a number between 0 and 1: 1 +Enter the host details (Server name, IP address, SCAN Name): oracle-host.example.com +Enter the host port number (default: 1521): +Enter the service name (default: orcl): ORCLCDB +Enter user with privileges: +Enter user password: +``` + +### Configuration Parameters + +| Parameter | Description | Default | +|:----------|:------------|:--------| +| **Secret vault type** | `local` for plain text values, `env` to read from environment variables | — | +| **Host** | Oracle server hostname, IP, or RAC SCAN name | — | +| **Port** | Port the Oracle listener is bound to | `1521` | +| **Service name** | Service name registered in the Oracle listener (the CDB service) | `orcl` | +| **User** | Database user with the grants from [Required Database Permissions](#3-required-database-permissions). `SYSTEM` works out of the box but is over-privileged; the dedicated user above is recommended. | — | +| **Password** | Password for the database user (hidden input) | — | + +## Execute the Profiler + +Once configured, run the profiler to extract metadata and performance metrics from your Oracle CDB: + +```bash +databricks labs lakebridge execute-database-profiler --source-tech oracle +``` + +The profiler will: +1. Connect to your Oracle CDB using the configured credentials +2. Execute the configuration and performance metric extraction queries +3. Store the results in a local DuckDB extract file + +[Back to Configure Profiler](../#configure-profiler) diff --git a/docs/lakebridge/docs/assessment/profiler/redshift.mdx b/docs/lakebridge/docs/assessment/profiler/redshift.mdx new file mode 100644 index 0000000000..06d514f452 --- /dev/null +++ b/docs/lakebridge/docs/assessment/profiler/redshift.mdx @@ -0,0 +1,152 @@ +--- +sidebar_position: 2 +title: Amazon Redshift Profiler Details +--- +import Admonition from '@theme/Admonition'; + +# Amazon Redshift Profiler Details + +- [Prerequisites](#prerequisites) +- [Configure Connection to Redshift](#configure-connection-to-redshift) +- [Run the profiler](#run-the-profiler) +- [Profiler output](#profiler-output) + +## Prerequisites + +### 1. Environment + +- **Lakebridge CLI** installed and configured for your Databricks workspace (same as other profiler sources). + +### 2. Choose the Redshift deployment variant + +The profiler ships **three extract pipelines** — pick the one that matches your Redshift instance: + +| Variant | Use when | +|--------|-----------| +| **serverless** | Amazon Redshift Serverless | +| **provisioned** | Single-AZ provisioned cluster | +| **provisioned_multi_az** | Multi-AZ provisioned cluster | + +### 3. Network connectivity + +The machine running the profiler must reach the Redshift cluster **endpoint** (hostname) on the cluster port (default **5439**), subject to your security groups / VPC / routing rules. + +### 4. Authentication + +During configuration you choose an **authentication type** and where secrets are read from (**local**, **env**, or **file**): + +| Authentication type | Typical use | +|---------------------|-------------| +| **sql_authentication** | Native database user and password | +| **iam** | IAM-authenticated connection using AWS credential resolution | + +For IAM authentication you typically need: + +- **AWS credentials** available to the process (for example `AWS_PROFILE`, environment variables for keys, or instance/profile credentials). +- **IAM permissions** allowing Amazon Redshift credential APIs appropriate for your setup (for example `redshift:GetClusterCredentials` where applicable). + +Use **`local`** to store plaintext values in `~/.databricks/labs/lakebridge/.credentials.yml`, **`env`** to substitute values from environment variables (with fallback), or **`file`** to reuse an existing credential file when it already contains valid Redshift entries. + +### 5. Database privileges + +The profiler connects as your configured user and runs read-only extracts. The pipeline includes **source_ddl** steps that create a helper view **`query_view`** in the database (via `CREATE OR REPLACE VIEW`). The connecting user therefore needs permission to: + +- **Create (and replace) views** in the target database used for profiling. +- **Select** from the Amazon Redshift system relations used by the extracts (see below). + +**Provisioned clusters** (`provisioned` / `provisioned_multi_az`) — objects referenced by the bundled SQL include, among others: + +- `stl_query`, `stl_query_metrics` +- `stv_node_storage_capacity`, `stv_partitions` +- `sys_external_query_detail` + +**Serverless** (`serverless`) — examples include: + +- `sys_query_history`, `sys_query_detail` +- `sys_external_query_detail`, `sys_serverless_usage` + +Exact object access is determined by Amazon Redshift documentation for your edition; grant **minimum read** access consistent with those views/tables. + +:::tip +If you cannot grant broad catalog access, narrow to the relations used in the YAML pipeline for your variant under `resources/assessments/redshift//` in the Lakebridge package. +::: + +## Configure Connection to Redshift + +```bash +databricks labs lakebridge configure-database-profiler +``` + +Select one Redshift source variant when prompted (`redshift_serverless`, `redshift_provisioned`, or `redshift_provisioned_multi_az`). The wizard will ask for authentication type, credential source (`local` | `env` | `file`), and connection details — for password auth, for example: + +- Redshift cluster **endpoint** (host) +- **Port** (default 5439) +- **Database** name +- **User** and **password** + +For IAM auth, expect prompts for optional fields such as **DB user**, **cluster identifier**, **AWS profile**, and **region**. + +Example-style transcript (values are illustrative): + +```console +databricks labs lakebridge configure-database-profiler + +Please select the source system you want to configure +[0] redshift_provisioned +[1] redshift_provisioned_multi_az +[2] redshift_serverless +... +Enter a number ...: 0 + +Redshift authentication: sql_authentication or iam. + +Authentication type +[0] iam +[1] sql_authentication +Enter a number between 0 and 1: 1 +Credential source (local | env | file) +[0] local +[1] env +[2] file +... + +Enter the Redshift cluster endpoint (host): mycluster.abc123.us-east-1.redshift.amazonaws.com +Enter the port details (default: 5439): 5439 +Enter the database name: dev +Enter the user details: profiler_reader +Enter the password details: ******** + +Do you want to test the connection to redshift? [y/n]: y +``` + +## Run the profiler + +After configuration and a successful connection test (optional): + +```bash +databricks labs lakebridge execute-database-profiler --help +``` + +Run the profiler (interactive source selection): + +```bash +databricks labs lakebridge execute-database-profiler +``` + +You can pass the source explicitly: + +```bash +databricks labs lakebridge execute-database-profiler --source-tech redshift_provisioned +``` + +Execution will: + +1. Load `pipeline_config.yml` for the chosen source variant. +2. Run **source_ddl** steps on the cluster (including creating/updating **`query_view`**). +3. Run SQL extracts and persist results into a timestamped DuckDB file under the selected `--output-folder`. + +## Profiler output + +The profiler extract is written as a timestamped DuckDB file under the selected `--output-folder` and uploaded to the configured UC Volume for downstream analysis. + +[Back to Configure Profiler](../#configure-profiler) diff --git a/docs/lakebridge/docs/assessment/profiler/snowflake.mdx b/docs/lakebridge/docs/assessment/profiler/snowflake.mdx new file mode 100644 index 0000000000..fd64faa344 --- /dev/null +++ b/docs/lakebridge/docs/assessment/profiler/snowflake.mdx @@ -0,0 +1,54 @@ +--- +sidebar_position: 2 +title: Snowflake Profiler Details +--- +import Admonition from '@theme/Admonition'; + +# Snowflake Profiler Details + +## Prerequisites + +- A Snowflake user with permission to create Personal Access Tokens (PATs) and to query the `SNOWFLAKE.ACCOUNT_USAGE` schema (e.g., the `ACCOUNTADMIN` role or a role granted `IMPORTED PRIVILEGES ON DATABASE SNOWFLAKE`). +- The `rate_sheet` step also reads `SNOWFLAKE.ORGANIZATION_USAGE.RATE_SHEET_DAILY` for the effective credit rate and account tier. This is available to `ACCOUNTADMIN` in an ORGADMIN-enabled account (the first account in an organization is ORGADMIN-enabled by default, so single-account orgs are covered), or to a custom role granted the `ORGANIZATION_BILLING_VIEWER` database role. If the account isn't ORGADMIN-enabled — or the organization is on a reseller contract — the view returns no rows, so `rate_sheet` is skipped and the rest of the profile still completes. +- Network access from the machine running Lakebridge to your Snowflake account. + +## Authentication + +The Snowflake profiler authenticates with a **Programmatic Access Token (PAT)**. Follow Snowflake's official guide to generate one: + +[Snowflake docs: Programmatic access tokens](https://docs.snowflake.com/en/user-guide/programmatic-access-tokens#generating-a-programmatic-access-token) + +You'll need: + +- `account` — your Snowflake account identifier (e.g., `myorg-myaccount.snowflakecomputing.com`) +- `user` — Snowflake username +- `role`, `warehouse`, `database`, `schema` — typically `ACCOUNTADMIN`, `COMPUTE_WH`, `SNOWFLAKE`, `ACCOUNT_USAGE` +- The PAT itself (paste when prompted; it's stored under `pat` in the credentials file) + +If your account or user has a network policy, also see Snowflake's docs on [bypassing the network policy on a PAT](https://docs.snowflake.com/en/user-guide/programmatic-access-tokens) for short-term testing from a dynamic IP. + +## Configure and run the profiler + +```bash +databricks labs lakebridge configure-database-profiler +``` + +Pick **snowflake** when prompted, then paste the connection details and PAT. + +```bash +databricks labs lakebridge execute-database-profiler +``` + +The extract is written to a DuckDB file at `~/.databricks/labs/lakebridge_profilers/snowflake_assessment/profiler_extract.db`. It contains raw `SNOWFLAKE.ACCOUNT_USAGE` data — warehouse usage, query history, storage, and (when applicable) pipe / autoclustering / materialized-view refresh credits. By default each extract looks back 90 days. + +To keep extracts shareable on high-volume accounts, `query_history` includes metrics and `QUERY_TYPE` for every query in the window but **omits `QUERY_TEXT`**. A separate `query_samples` table holds a random sample of **10,000** rows **with** `QUERY_TEXT` (join to `query_history` on `QUERY_ID` for full metrics). Login history (`user_activity`) is not extracted. + +Inspect it with: + +```bash +cd ~/.databricks/labs/lakebridge_profilers/snowflake_assessment +duckdb profiler_extract.db "SHOW TABLES;" +duckdb profiler_extract.db "SELECT * FROM warehouse_usage LIMIT 10;" +duckdb profiler_extract.db "SELECT * FROM query_history LIMIT 10;" +duckdb profiler_extract.db "SELECT query_id, query_type, left(query_text, 80) FROM query_samples LIMIT 10;" +``` diff --git a/docs/lakebridge/docs/assessment/profiler/synapse.mdx b/docs/lakebridge/docs/assessment/profiler/synapse.mdx index 144ba1e38e..ef40783e89 100644 --- a/docs/lakebridge/docs/assessment/profiler/synapse.mdx +++ b/docs/lakebridge/docs/assessment/profiler/synapse.mdx @@ -12,50 +12,49 @@ import Admonition from '@theme/Admonition'; ## Prerequisites ### 1. Download -- Azure CLI (https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) -- ODBC driver for SQL Server (https://learn.microsoft.com/en-us/sql/connect/odbc/download-odbc-driver-for-sql-server?view=sql-server-ver17) -- (Windows only) Visual C++ (https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist?view=msvc-170) -### 2. Authenticate to Azure using Azure CLI -```bash -az login -``` +- **Azure CLI** — required for authenticating to the Azure Management REST API path (workspace metadata, pool listing, monitoring metrics). [Download](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli). -The profiler uses Azure SDK's `DefaultAzureCredential` which attempts authentication in this order: -1. **Environment Variables** (Service Principal): - - `AZURE_TENANT_ID` - - `AZURE_CLIENT_ID` - - `AZURE_CLIENT_SECRET` -2. **Managed Identity** (if running on Azure VM/App Service) -3. **Azure CLI** (from `az login`) -4. **Visual Studio Code** -5. **Azure PowerShell** +No database driver installation is required: the profiler connects to SQL pools with Microsoft's +[mssql-python](https://github.com/microsoft/mssql-python) driver, which bundles its own connectivity layer. -Ensure your user/service principal account has the required roles listed below. See [Azure documentation](https://learn.microsoft.com/en-us/cli/azure/authenticate-azure-cli?view=azure-cli-latest) for more details. +### 2. Authentication +:::info +This page covers a full Synapse Workspace. If you are profiling a standalone SQL Dedicated Pool (formerly Azure SQL DW), use the [Legacy Synapse profiler](./legacy_synapse.mdx) instead. +::: -### 3. Required Access to Synapse Workspace +The synapse profiler has **two auth paths**: -:::warning Attention: -Skip this prerequisite if you are using a standalone SQL Dedicated Pool (formerly Azure SQL DW) and NOT a Synapse Workspace -::: +#### Azure Management REST API (workspace metadata, pool listing, monitoring metrics) + +Uses Azure SDK's `DefaultAzureCredential`. Run `az login` first or set env vars `AZURE_TENANT_ID`, `AZURE_CLIENT_ID`, and `AZURE_CLIENT_SECRET`. + +The authenticated identity needs the **Synapse Artifact User** and **Monitoring Reader** roles on the workspace. +Granting Synapse Administrator alone is **not** enough — both roles must be explicitly assigned. +Assign from Synapse Workspace → Manage Access → Access Control and Synapse Workspace → IAM respectively. See [Azure documentation](https://learn.microsoft.com/en-us/azure/synapse-analytics/security/how-to-manage-synapse-rbac-role-assignments). -- Profiler uses the Python version of Azure SDK libraries to extract information about target Synapse Workspace. -- For making the Azure API calls using Azure SDK, the authenticated identity (user or service principal) needs the following role assignments. -Just giving the Synapse Administrator role is not enough. The below roles must be explicitly assigned. - - Synapse Artifact User. - - Assign from Synapse Workspace → Manage Access → Access Control [Refer to Azure documentation](https://learn.microsoft.com/en-us/azure/synapse-analytics/security/how-to-manage-synapse-rbac-role-assignments) - - Monitoring Reader - - Assign from Synapse Workspace → IAM +#### SQL connection to SQL pools -### 4. Setup user_id/password for ODBC connectivity -Create/Use a user with access to query the following tables in Synapse. +Picking authentication mode `DefaultAzureCredential` will use the same credentials as the previous section. The authenticated identity needs the right permissions on the sql pools listed in [section 3](#3-required-database-permissions). +Or pick one of the other authentication modes (`configure-database-profiler` prompts for one): -:::warning Attention: -The user should not have Multi-factor Authentication (MFA) enabled as ODBC does not support MFA +| Auth method | Description | MFA-capable | +|---|---|---| +| `SqlPassword` | SQL Authentication — username + password from credentials file | No | +| `DefaultAzureCredential` | Entra ID via the Azure Identity credentials chain. Recommended for Azure-hosted targets | Yes | +| `ActiveDirectoryPassword` | Entra ID (Azure AD) username + password | No | +| `ActiveDirectoryServicePrincipal` | Service Principal — `AZURE_CLIENT_ID` / `AZURE_CLIENT_SECRET` env vars | No | + +:::warning +For `ActiveDirectoryServicePrincipal`, set `AZURE_CLIENT_ID` and `AZURE_CLIENT_SECRET` env vars before running the profiler. +For `DefaultAzureCredential`, run `az login` first; for unattended runs set `AZURE_TENANT_ID`, `AZURE_CLIENT_ID`, and `AZURE_CLIENT_SECRET`. ::: -This user id needs to have read access (`SELECT grants`) on the following tables. The following permissions are required for Dynamic Management Views (DMVs): + +### 3. Required Database Permissions + +The SQL user configured for the profiler must have read access (`SELECT grants`) to the following tables. The following permissions are required for Dynamic Management Views (DMVs): - **`VIEW DATABASE STATE`** - Required for database-scoped DMVs (queries within specific databases) - **`VIEW SERVER STATE`** - Required for server-level DMVs (queries in the `master` database) @@ -143,22 +142,20 @@ Enter secret vault type (local | env) [0] env [1] local Enter a number between 0 and 1: 1 +Select authentication method +[0] SqlPassword +[1] DefaultAzureCredential +[2] ActiveDirectoryPassword +[3] ActiveDirectoryServicePrincipal +Enter a number between 0 and 3: 0 +Enter the username: user +Enter the password: Please provide Synapse Workspace settings: Enter Synapse workspace name: synapse -Enter SQL user: user -Enter SQL password: -Enter timezone (e.g. America/New_York) (default: UTC): -Enter the ODBC driver installed locally (default: ODBC Driver 18 for SQL Server): -Please provide Azure access settings: Enter development endpoint: synapse.endpoint -Please select JDBC authentication type: -Select authentication type -[0] ad_passwd_authentication -[1] spn_authentication -[2] sql_authentication -Enter a number between 0 and 2: 2 Enter fetch size (default: 1000): Enter login timeout (seconds) (default: 30): +Enter timezone (e.g. America/New_York) (default: UTC): Exclude serverless SQL pool from profiling? (default: no): Exclude dedicated SQL pools from profiling? (default: no): Exclude Spark pools from profiling? (default: no): diff --git a/docs/lakebridge/docs/assessment/profiler/teradata.mdx b/docs/lakebridge/docs/assessment/profiler/teradata.mdx new file mode 100644 index 0000000000..bdb728153e --- /dev/null +++ b/docs/lakebridge/docs/assessment/profiler/teradata.mdx @@ -0,0 +1,148 @@ +--- +sidebar_position: 2 +title: Teradata Profiler Details +--- +import Admonition from '@theme/Admonition'; + +# Teradata Profiler Details + +- [Prerequisites](#prerequisites) +- [Configure Connection to Teradata](#configure-connection-to-teradata) +- [Run the Profiler](#run-the-profiler) +- [Choosing a Profiler Variant](#choosing-a-profiler-variant) +- [SQL Text Redaction](#sql-text-redaction) +- [International Character Support](#international-character-support) + +## Prerequisites + +### 1. Teradata connectivity and driver + +- Network access to the Teradata system. +- No driver install required for the Teradata profiler. +- Access to the Teradata database with appropriate permissions. + +### 2. Required permissions on system views + +Grant `SELECT` access to: + +- `DBC.DBCInfoTbl` +- `DBC.ResUsageSpma` +- `DBC.DiskSpaceV` +- `DBC.TablesV` +- `DBC.FunctionsV` +- `DBC.ColumnsV` + +### 3. Optional PDCR access for longer-history analysis + +The `pdcr` variant uses Teradata PDCR tables for richer, aggregated history (up to 180 days). To use it, grant `SELECT` access to: + +- `PDCRINFO.DBQlogTbl_Hst` +- `PDCRINFO.DBQLSummaryTbl_Hst` +- `PDCRINFO.UserInfo` + + +If PDCR is not available in the environment, use the `core` variant, which relies only on the core `DBC`/DBQL tables. See [Choosing a Profiler Variant](#choosing-a-profiler-variant). + + +## Configure Connection to Teradata + +```console +databricks labs lakebridge configure-database-profiler + +Select the source technology +[0] bigquery +[1] legacy_synapse +[2] mssql +[3] oracle +[4] redshift +[5] snowflake +[6] synapse +[7] teradata +Enter a number between 0 and 7: 7 + +(local | env) +local means values are read as plain text +env means values are read from environment variables fall back to plain text if not variable is not found + +Enter secret vault type (local | env) +[0] env +[1] local +Enter a number between 0 and 1: 1 +Enter the Teradata server or host details: td-host.example.com +Enter the user details: dbc_user +Enter the password details: +Enter the default database name (default: DBC): +``` + +The generated credential block (written to `~/.databricks/labs/lakebridge/.credentials.yml`) includes: + +- `teradata.host` +- `teradata.user` +- `teradata.password` +- `teradata.database` + +## Run the Profiler + +The variant (`core` or `pdcr`) is selected at extraction time via `--variant`. Optionally verify connectivity first: + +```console +databricks labs lakebridge test-profiler-connection --source-tech teradata +``` + +Then run the extraction: + +```console +databricks labs lakebridge execute-database-profiler --source-tech teradata --variant core --output-folder /path/to/output +``` + +`--output-folder` is optional; if omitted you are prompted for it (defaulting to +`~/.databricks/labs/lakebridge_profilers/_assessment`). + +This executes the extraction steps defined in the selected variant's pipeline configuration and produces a local +DuckDB extract in the output folder, named: + +``` +profiler_extract___.db +``` + +The extract contains the system, storage, object, user-database, UDF, and workload tables gathered from the source. + +## Choosing a Profiler Variant + +The Teradata profiler ships as two variants. They share the same connection and credentials; choose the one that +matches the access available in your environment: + +| Variant | Workload source | Use when | +|---|---|---| +| `core` | Core `DBC` / DBQL tables | The baseline. Works on any Teradata system with the system-view grants above. | +| `pdcr` | PDCR aggregated history (`PDCRINFO`) | You have PDCR access and want richer, longer-history workload aggregates. | + +Both variants extract the same system, storage, object, user-database, and UDF tables; they differ only in the +workload extraction: + +- `core` extracts `td_dbql_core_info_extract`. +- `pdcr` extracts `td_pdcr_info_agg_extract` and `td_pdcr_sp_exe_info_agg_extract`. + +The `core` workload data is typically narrower (recent days from DBQL instead of long historical +aggregates), which can reduce long-horizon trend analysis and may underrepresent short-query aggregate metrics. + + +If you run the `pdcr` variant and the profiler fails with `Error 3802 ... Database 'PDCRINFO' does not exist`, +the PDCR repository is not present (or not accessible) in this environment. Re-run with `--source-tech teradata --variant core`, +which uses only the core `DBC`/DBQL tables. + + +## SQL Text Redaction + +The Teradata profiler does **not** export raw SQL query text. In the DBQL-core extract the `SQLTextInfo` +column is redacted to a fixed `[REDACTED]` value at extraction time, so query text never leaves the source +system. Workload analysis relies on statement classification and performance metrics rather than the raw SQL. + +## International Character Support + +Lakebridge preserves multilingual metadata (for example Japanese, Korean, and Thai text) in Teradata profiler extracts. + +- The `teradatasql` driver runs the session over UTF-8 and returns character data as already-decoded text, + so international characters are preserved end to end with no lossy client-side re-encoding. + +[Back to Configure Profiler](../#configure-profiler) diff --git a/docs/lakebridge/docs/choosing_tools.mdx b/docs/lakebridge/docs/choosing_tools.mdx new file mode 100644 index 0000000000..233a52b409 --- /dev/null +++ b/docs/lakebridge/docs/choosing_tools.mdx @@ -0,0 +1,75 @@ +--- +sidebar_position: 3 +title: Which Tool Do I Use? +--- + +# Which Tool Should I Use? + +Lakebridge provides multiple tools for assessment and transpilation. Use this guide to choose the right one for your migration. + +--- + +## Assessment: Profiler vs Analyzer + +| | Profiler | Analyzer | +|---|---|---| +| **Purpose** | Understand data volumes, query complexity, and workload patterns | Analyze SQL code complexity and compatibility | +| **Input** | Live database connections | SQL code files (local filesystem) | +| **Output** | Interactive dashboard with usage metrics | Per-object complexity report (LOW / MEDIUM / HIGH / VERY HIGH) | +| **When to use** | Early-stage scoping, executive reporting, sizing estimates | Pre-migration planning, review current state of source code and its compatibility | +| **Requires DB connection** | Yes | No | + +**Quick decision:** +- If you need to **scope the migration and communicate to stakeholders**, use the **Profiler**. +- If you have SQL files and want to know **how hard each one will be to transpile**, use the **Analyzer**. +- **Run both** when you have access to the database — Profiler for sizing, Analyzer for developer planning. + +--- + +## Transpiler: BladeBridge vs Morpheus vs Switch {#transpiler-bladebridge-vs-morpheus-vs-switch} + +| | Morpheus | BladeBridge | Switch (Experimental) | +|---|---|---|---| +| **Approach** | AST-based (ANTLR grammar → IR tree → code gen) | Config/pattern-based converter | LLM-powered | +| **Best for** | SQL migration where correctness guarantees and reproducibility matter | ETL platforms + broader SQL dialect coverage | Complex logic, unsupported constructs, or any format via custom prompts | +| **Source dialects** | MSSql (SQL Server, Azure SQL, RDS for SQL Server), Snowflake (including dbt repointing), Synapse (Azure Synapse Analytics dedicated SQL pools) | DataStage, SSIS, Oracle, Teradata, Netezza, Synapse, Redshift, and more | Any SQL dialect or programming language. Either via built-in (MSSQL, MySQL, Netezza, Oracle, PostgreSQL, Redshift, Snowflake, Teradata, Airflow, Python, Scala) or custom YAML prompts | +| **Output formats** | Databricks SQL only | Databricks SQL, SparkSQL, PySpark, notebooks, workflow definitions | Python/SQL notebooks; any text-based format | +| **Accuracy** | Strong guarantee: errors or warns rather than silently producing wrong output | Deterministic and repeatable; extensible via config | Semantic/intent-aware; handles edge cases deterministic transpilers miss | +| **Speed** | Fast — local execution, no API calls | Fast — local execution, no API calls | Slower — requires LLM API calls per file | +| **Cost** | Free | Free | Token costs apply (Databricks Foundation Model API) | +| **Scalability** | Thousands of files locally | Thousands of files locally | Rate-limited by model serving; scales via Lakeflow Jobs | +| **Configuration** | Simple: `source-dialect` flag + I/O paths | Simple: `source-dialect` flag + I/O paths + Optional JSON Config override files | YAML prompt files; configurable model endpoint | +| **Extensibility** | Not extensible (fixed grammar) | Fully extensible via JSON config | Fully extensible via custom YAML prompts | + +### When to choose Morpheus + +Use Morpheus when: +- You are migrating SQL code from **SQL Server, Snowflake, or Azure Synapse Analytics** +- You need **correctness guarantees** — you want to know immediately when a construct cannot be translated, rather than getting silently wrong output +- You do not need ETL platform support + +### When to choose BladeBridge + +Use BladeBridge when: +- You are migrating **ETL workloads** (DataStage, SSIS, others) +- You need **broad SQL dialect coverage** (Oracle, Teradata, Netezza, Redshift) with customizable configurations + +### When to choose Switch + +Use Switch when: +- You have **stored procedures with complex business logic** that rely heavily on context and intent +- Your source dialect is **not covered by Morpheus or BladeBridge** (Switch supports any format via custom YAML prompts) +- You want **Python notebook output** for logic that is difficult to express in pure SQL +- You want to convert non-SQL sources (Python scripts, Airflow DAGs, etc.) + +:::note +Switch is currently **Experimental**. Generated notebooks may require manual adjustments. +::: + +--- + +## Still not sure? + +Start with **Morpheus** if your source is SQL Server, Snowflake, or Synapse. Its correctness guarantee makes it the lowest-risk choice when it supports your dialect. + +For everything else, check the [supported dialects table](/docs/transpile/#supported-dialects) and pick the transpiler that covers your source. diff --git a/docs/lakebridge/docs/dev/contributing.md b/docs/lakebridge/docs/dev/contributing.md index 08d07a7403..645ef28b36 100644 --- a/docs/lakebridge/docs/dev/contributing.md +++ b/docs/lakebridge/docs/dev/contributing.md @@ -175,6 +175,10 @@ pull request checks do pass, before your code is reviewed by others: make lint test ``` +## Developer Documentation + +The developer documentation covers local setup, code organization, JVM proxy configuration, and troubleshooting the development environment. + ## First contribution Here are the example steps to submit your first contribution: diff --git a/docs/lakebridge/docs/faq.mdx b/docs/lakebridge/docs/faq.mdx index 1144bb73f2..c9681058bc 100644 --- a/docs/lakebridge/docs/faq.mdx +++ b/docs/lakebridge/docs/faq.mdx @@ -1,57 +1,199 @@ --- sidebar_position: 8 --- +import CodeBlock from '@theme/CodeBlock'; + # FAQs -## Installation -### 1. Install Databricks CLI on Linux without brew -```bash -#!/usr/bin/env bash +## General -#install dependencies -apt update && apt install -y curl sudo unzip +### What is Lakebridge? + +Lakebridge is a Databricks toolkit for migrating data workloads to Databricks. It covers three phases: +1. **Assessment** — analyze your existing SQL or ETL environment and code to understand complexity and effort +2. **Transpilation** — convert source SQL or ETL code to Databricks-compatible SQL, PySpark, or notebooks +3. **Reconciliation** — validate that the migrated data matches the source + +### Do I need to run all three phases? + +No. Use the phases you need: +- **Assess only** if you are scoping a migration before committing to it +- **Transpile only** if you are migrating code and do not need pre-migration analysis or post-migration validation +- **Reconcile only** if you have already migrated data and want to validate row/column fidelity + +### Which source systems are supported? + +See the [supported dialects table](/docs/transpile/#supported-dialects) for a full list. In summary: +- **SQL sources:** SQL Server, Snowflake, Azure Synapse Analytics, Oracle, Teradata, Netezza, Redshift, MySQL, PostgreSQL +- **ETL sources:** DataStage, SSIS, others +- **Other:** Airflow, Python, Scala (via Switch) + +--- + +## Assessment + +### Profiler vs Analyzer — which should I use? + +See [Which Tool Do I Use? — Profiler vs Analyzer](/docs/choosing_tools#assessment-profiler-vs-analyzer). + +Short answer: use the Profiler for executive-level scoping; use the Analyzer for per-file migration planning. + +### How do I interpret complexity scores? + +The Analyzer classifies each SQL object as LOW, MEDIUM, HIGH, or VERY HIGH. As a rule of thumb: +- **LOW / MEDIUM** objects typically transpile cleanly with no manual review required +- **HIGH** objects should be reviewed after transpilation for any warnings +- **VERY HIGH** objects (>50% of total) suggest you should plan for 2+ manual review passes before deploying + +See [Complexity Scoring](/docs/assessment/analyzer/complexity_scoring) for the exact thresholds by source system. + +### Should I run the SQL Splitter before the Analyzer? + +Yes. The Analyzer works at the individual-object level. If your SQL files mix multiple objects (stored procedures, tables, views), split them first with the [SQL Splitter](/docs/sql_splitter) so the Analyzer produces per-object granularity. + +--- + +## Transpilation + +### Morpheus {#morpheus} + +#### Which dialects does Morpheus support? + +Morpheus supports `mssql` (SQL Server, Azure SQL, RDS for SQL Server), `snowflake` (including dbt repointing), and `synapse` (Azure Synapse Analytics dedicated SQL pools). It does **not** support Redshift, Oracle, or ETL platforms. + +For other dialects, use [BladeBridge](/docs/transpile/pluggable_transpilers/bladebridge) or [Switch](/docs/transpile/pluggable_transpilers/switch). + +#### What is the difference between an error and a warning in Morpheus output? + +- **Error** — Morpheus knows that the transpiled output cannot be guaranteed to produce equivalent results. Manual fix required before deploying. +- **Warning** — Morpheus could not confirm equivalence but the output *may* still be correct (a conservative false-negative). Review the flagged section and test against source data. + +A file transpiled with **no errors and no warnings** carries a full correctness guarantee. + +#### My file transpiled with warnings but the output looks correct. Is it safe to use? + +Possibly. Morpheus is conservative — it warns when it cannot *guarantee* correctness, even if the actual output is correct. Test the transpiled file against your source data. If the results match, it is safe to deploy. + +#### What should I do when a file fails to parse? + +Parsing errors are very rare and indicate either malformed input SQL or a gap in the Morpheus ANTLR grammar. Verify that the input file is valid SQL. If it is, file a bug at [GitHub Issues](https://github.com/databrickslabs/lakebridge/issues). + +### BladeBridge + +#### Which dialects does BladeBridge support? -#install databricks cli -curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/v0.242.0/install.sh | sudo sh +BladeBridge supports SQL dialects (Oracle, Teradata, Netezza, SQL Server, Synapse, Redshift) and ETL platforms (DataStage, SSIS). See the [supported dialects table](/docs/transpile/#supported-dialects). + +#### What is the `target-tech` parameter? + +`target-tech` controls the output format: `SQL` (Databricks SQL), `SPARKSQL` (SparkSQL notebooks), or `PYSPARK` (PySpark notebooks). It only applies to ETL sources — SQL dialects always output Databricks SQL. + +Available options per ETL dialect: +- `datastage`: `SPARKSQL` or `PYSPARK` +- `ssis`: `SPARKSQL` only (PYSPARK not available) + +#### How do I customize BladeBridge output? + +Use a custom JSON override file. Pass it via `--overrides-file` or set it during `install-transpile`. See [BladeBridge Configuration](/docs/transpile/pluggable_transpilers/bladebridge/bladebridge_configuration) for the full config reference. + +### Switch + +#### Can Switch support my source system if it's not in the built-in list? + +Yes. Switch uses LLMs to convert arbitrary source formats through custom YAML prompts. If your dialect is not in the [built-in list](/docs/transpile/pluggable_transpilers/switch#source-format-support), create a custom prompt YAML: +- Start with a similar built-in dialect's YAML as a template +- Add examples specific to your source dialect +- Reference [SQLGlot dialects](https://github.com/tobymao/sqlglot/tree/main/sqlglot/dialects) for dialect-specific patterns + +#### My Switch files show status "Not converted". What does that mean? + +The file exceeded the `token_count_threshold` and was skipped. Solutions: +- Split the file into smaller parts +- Increase `token_count_threshold` in `switch_config.yml` if your model supports it + +#### My Switch files show "Converted with errors". How do I fix them? + +The LLM converted the file but the output has syntax errors. Options: +1. Review the `error_details` column in the conversion result table +2. Increase `max_fix_attempts` in `switch_config.yml` for more automatic correction attempts +3. Fix errors manually in the output notebook + +#### Switch exported some files but they weren't written to the output directory. + +Check the `export_error` column in the result table. Common causes: +- **Size limit:** Notebooks >10MB cannot be written. Split the converted content manually. +- **Permissions:** Verify your user has write access to the workspace output path. +- **Invalid path:** Output paths must start with `/Workspace/`. + +--- + +## Reconciliation + +### Commonly used custom transformations + +| source_type | data_type | source_transformation | target_transformation | comments | +|---|---|---|---|---| +| Oracle | number(10,5) | "trim(to_char(coalesce(col_name,0.0), '99990.99999'))" | "cast(coalesce(col_name,0.0) as decimal(10,5))" | Adjust precision/scale as needed | +| Snowflake | array | "array_to_string(array_compact(col_name),',')" | "concat_ws(',', col_name)" | Removes undefined during migration | +| Snowflake | array | "array_to_string(array_sort(array_compact(col_name), true, true),',')" | "concat_ws(',', col_name)" | Removes undefined and sorts | +| Snowflake | timestamp_ntz | "date_part(epoch_second,col_name)" | "unix_timestamp(col_name)" | Convert timestamp_ntz to epoch | + +### How do I reconcile tables when column names differ between source and target? + +Use `column_mapping` in your table config: + +```json +"column_mapping": [ + { "source_name": "dept_id", "target_name": "department_id" } +] ``` ----- -## Reconcile -### 1. Guidance for Oracle as a source -### Driver +See [Configuration Reference — Column Mapping](/docs/reconcile/configuration#column-mapping). -#### Option 1 -import CodeBlock from '@theme/CodeBlock'; +### How do I exclude specific columns from reconciliation? + +Use `drop_columns`: + +```json +"drop_columns": ["audit_timestamp", "etl_load_date"] +``` + +--- -* **Download `ojdbc8.jar` from Oracle:** -Visit the [official Oracle website](https://www.oracle.com/database/technologies/appdev/jdbc-downloads.html) to -acquire the `ojdbc8.jar` JAR file. This file is crucial for establishing connectivity between Databricks and Oracle -databases. +## Troubleshooting -* **Install the JAR file on Databricks:** -Upon completing the download, install the JAR file onto your Databricks cluster. Refer -to [this page](https://docs.databricks.com/en/libraries/cluster-libraries.html) -For comprehensive instructions on uploading a JAR file, Python egg, or Python wheel to your Databricks workspace. +### Install Databricks CLI on Linux without brew -#### Option 2 +```bash +#!/usr/bin/env bash +apt update && apt install -y curl sudo unzip +curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/v0.299.0/install.sh | sudo sh +``` -* **Install ojdbc8 library from Maven:** -Follow [this guide](https://docs.databricks.com/en/libraries/package-repositories.html#maven-or-spark-package) to -install the Maven library on a cluster. Refer -to [this document](https://mvnrepository.com/artifact/com.oracle.database.jdbc/ojdbc8) for obtaining the Maven -coordinates. +### The `configure-reconcile` command fails because I can't create SQL warehouses or clusters. + +Add your existing warehouse or cluster ID to your Databricks CLI profile: + +``` +[profile-name] +host = +warehouse_id = +cluster_id = +``` -This installation is a necessary step to enable seamless comparison between Oracle and Databricks, ensuring that the -required Oracle JDBC functionality is readily available within the Databricks environment. +### Transpilation produces output files with headers but no SQL. +This means the entire file failed to parse. Check that: +1. The input file is valid SQL for the declared `--source-dialect` +2. The file is not empty +3. If using Morpheus, the input SQL matches a supported dialect (`mssql`, `snowflake`, or `synapse`) -### 2. Commonly Used Custom Transformations +### The `install-transpile` command fails with a download error. -| source_type | data_type | source_transformation | target_transformation | source_value_example | target_value_example | comments | -|-------------|---------------|--------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------|-------------------------|-------------------------|---------------------------------------------------------------------------------------------| -| Oracle | number(10,5) | "trim(to_char(coalesce(col_name,0.0), ’99990.99999’))" | "cast(coalesce(col_name,0.0) as decimal(10,5))" | 1.00 | 1.00000 | this can be used for any precision and scale by adjusting accordingly in the transformation | -| Snowflake | array | "array_to_string(array_compact(col_name),’,’)" | "concat_ws(’,’, col_name)" | [1,undefined,2] | [1,2] | in case of removing "undefined" during migration(converts sparse array to dense array) | -| Snowflake | array | "array_to_string(array_sort(array_compact(col_name), true, true),’,’)" | "concat_ws(’,’, col_name)" | [2,undefined,1] | [1,2] | in case of removing "undefined" during migration and want to sort the array | -| Snowflake | timestamp_ntz | "date_part(epoch_second,col_name)" | "unix_timestamp(col_name)" | 2020-01-01 00:00:00.000 | 2020-01-01 00:00:00.000 | convert timestamp_ntz to epoch for getting a match between Snowflake and data bricks | +Lakebridge downloads transpiler components from GitHub, Maven Central, and PyPI. If you are in a restricted network: +1. Whitelist the required endpoints (see [Installation — Network access](/docs/installation#network-access-proxies-and-mirrors)) +2. Or set up a private artifact mirror (Artifactory, Nexus) and configure it as the download source +### How do I report a bug or request a feature? +Open an issue at [github.com/databrickslabs/lakebridge/issues](https://github.com/databrickslabs/lakebridge/issues). For Switch-specific issues, include the LLM model name and a sample of the input that failed to convert. diff --git a/docs/lakebridge/docs/getting_started.mdx b/docs/lakebridge/docs/getting_started.mdx new file mode 100644 index 0000000000..d191460e52 --- /dev/null +++ b/docs/lakebridge/docs/getting_started.mdx @@ -0,0 +1,122 @@ +--- +sidebar_position: 2 +title: Getting Started +--- + +# Getting Started with Lakebridge + +This guide walks you through a complete end-to-end migration using **SQL Server as the source** and **Databricks SQL as the target**. You will assess your SQL code, transpile it, and validate the output — in about 15 minutes. + +**Prerequisites:** Lakebridge must already be installed. If not, see [Installation](/docs/installation) first. + +--- + +## _(Optional)_ Step 1: Split Your SQL Files + +If your SQL code lives in monolithic files that contain multiple objects (stored procedures, tables, views, functions), it is recommended to split them first. + +```bash +./sqlsplit -d /path/to/your/sql -o /path/to/split/output +``` + +- `-d` accepts a directory of `.sql` files +- `-o` must be a directory that already exists +- Output is organized into subfolders: `/PROCEDURE`, `/FUNCTION`, `/TABLE`, `/VIEW` + +See [SQL Splitter](/docs/sql_splitter) for full usage and download. + +:::tip +Splitting first gives the Analyzer more granular, per-object results and gives the transpiler a cleaner input. +::: + +--- + +## _(Optional)_ Step 2: Assess Your Code + +Run the Analyzer on the (optionally split) files to understand complexity and identify patterns the transpiler may need help with. +```bash +databricks labs lakebridge analyze +``` + +When prompted, enter the following details: +- **Input path:** `/path/to/split/output` +- **Source dialect:** `mssql` +- **Report file:** `/path/to/transpiled/output/output.xlsx` +- **Source technology:** Select the number corresponding to `MS SQL Server` + +The Analyzer produces a complexity report saved as an Excel file (`.xlsx`). Objects flagged as `HIGH` or `VERY HIGH` +complexity may need manual review after transpilation. + +See [Assessment Guide](/docs/assessment) for more details. + +--- + +## Step 3: Configure Transpilation + +During `install-transpile` (covered in [Installation](/docs/installation#install-transpile)), you are prompted for your migration parameters. For this _Getting Started_ guide, the values to use are: + +- **Source dialect:** Select the number corresponding to `mssql` +- **Input path:** `/path/to/split/output` +- **Output directory:** `/path/to/transpiled/output` +- **Transpiler:** Select the number corresponding to `Morpheus` (both Morpheus and BladeBridge support SQL Server, but this guide assumes Morpheus) + +If you need to change these settings, re-run `install-transpile`. + +Morpheus is the recommended transpiler for SQL Server migrations — it provides strong correctness guarantees and warns you when it cannot fully guarantee equivalence. + +See [Which transpiler should I use?](/docs/choosing_tools#transpiler-bladebridge-vs-morpheus-vs-switch) if you are migrating from a different source. + +--- + +## Step 4: Transpile + +Run the transpilation (if params already configured in previous step): + +```bash +databricks labs lakebridge transpile +``` + +Or pass all parameters inline: + +```bash +databricks labs lakebridge transpile \ + --source-dialect mssql \ + --input-source /path/to/split/output \ + --output-folder /path/to/transpiled/output +``` + +**Reading the output:** + +In the command output, you will see a final summary with all files converted and whether any errors were found. + +You may also find inline comments in the output files prefixed with "FIXME" that you'll need to review. + +Files with errors still contain as much translated output as possible. Review the flagged positions and fix manually. + +--- + +## Step 5: Reconcile + +:::note Prerequisite +This step requires `configure-reconcile` to have been completed during installation. If you skipped it, run it now — see [Installation → Configure Reconcile](/docs/installation#configure-reconcile). +::: + +After deploying your transpiled SQL to Databricks, run the Reconciler to validate that the data output matches the source. + +```bash +databricks labs lakebridge reconcile +``` + +The Reconciler compares row counts, schema, and data values between your source system and the Databricks target. Results are written to a dashboard in your workspace. + +See [Reconcile Guide](/docs/reconcile) for full configuration details. + +--- + +## What's Next + +| Next step | Guide | +|-----------|-------| +| Understand transpiler differences | [Which Tool Do I Use?](/docs/choosing_tools) | +| Migrate ETL workloads | [Source Systems](/docs/transpile/source_systems/) | +| Customize transpiler output | [BladeBridge Configuration](/docs/transpile/pluggable_transpilers/bladebridge/bladebridge_configuration), [Switch Configuration](/docs/transpile/pluggable_transpilers/switch/customizing_switch/) | diff --git a/docs/lakebridge/docs/installation.mdx b/docs/lakebridge/docs/installation.mdx index 3a22ff28c0..dca64ce04f 100644 --- a/docs/lakebridge/docs/installation.mdx +++ b/docs/lakebridge/docs/installation.mdx @@ -1,5 +1,5 @@ --- -sidebar_position: 2 +sidebar_position: 1 --- import useBaseUrl from '@docusaurus/useBaseUrl'; import Tabs from '@theme/Tabs'; @@ -7,335 +7,337 @@ import TabItem from '@theme/TabItem'; # Installation +Lakebridge can be installed in two ways. Pick the path that fits your environment: -## [Table of Contents](#table-of-contents) +- **[Desktop App](#desktop-app)** — a graphical installer for macOS and Windows. Best if you prefer a UI and want the + fewest manual steps: it bundles the Databricks CLI and Python and installs Lakebridge for you. Available from + release **v0.14.0** onwards. +- **[Command-line (CLI)](#command-line-cli)** — install via the Databricks CLI. Best for automation, headless or + server environments, Linux, or if you already work in the terminal. -* [Pre-requisites](#pre-requisites) -* [Install Lakebridge](#install-lakebridge) -* [Install Transpile](#install-transpile) -* [Configure Reconcile](#configure-reconcile) +Both paths connect to a Databricks workspace, require **Java 21 or above** (for the Morpheus transpiler), and need +[network access](#network-access-proxies-and-mirrors) to GitHub, Maven Central, and PyPI. Each flow below is +self-contained — follow the one you chose. ----- -## Pre-requisites - -### 1. Databricks Workspace Requirements +--- -#### 1.1 Databricks Workspace Access -You must have access to a Databricks workspace to install and use Lakebridge: +## Desktop App -- **Production/Enterprise Workspace:** Recommended for production migrations -- **Development Workspace:** Suitable for testing and development -- **Free Databricks Workspace:** Available at [databricks.com/try-databricks](https://www.databricks.com/try-databricks) - Perfect for evaluation and learning +The desktop app is a graphical installer for macOS and Windows. The installer is published as an asset on each +[GitHub release](https://github.com/databrickslabs/lakebridge/releases), from **v0.14.0** onwards. -:::tip Pro Tip -For evaluation purposes, using a free Databricks workspace with a Personal Access Token is the fastest way to get started. This combination requires no enterprise approvals and can be set up in minutes. +:::note +The desktop app has its own version line (e.g. `1.3.2`), which is independent of the Lakebridge release tag it +ships with — so the version in the installer filename does not match the release version. ::: -#### 1.2 Databricks CLI Installation & Configuration - -**Install Databricks CLI** - Ensure that you have the Databricks Command-Line Interface (CLI) installed on your machine. Refer to the installation instructions provided for Linux, MacOS, and Windows, available [here](https://docs.databricks.com/en/dev-tools/cli/install.html#install-or-update-the-databricks-cli). - -Installing the Databricks CLI in different OS: - - - macos-databricks-cli-install - - - windows-databricks-cli-install - - - ```shell - #!/usr/bin/env bash - - #install dependencies - apt update && apt install -y curl sudo unzip - - #install databricks cli - curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/v0.242.0/install.sh | sudo sh - ``` - - - -**Configure Databricks CLI** - Details can be found [here](https://docs.databricks.com/aws/en/dev-tools/cli/authentication#databricks-personal-access-token-authentication). +### Requirements -**Authentication Options:** -- **Personal Access Token (PAT):** Recommended for individual users - Generate from User Settings → Developer → Access Tokens -- **Service Principal:** Recommended for automated/production deployments - Requires admin privileges to create +| Requirement | Details | +|--------------------------|-----------------------------------------------------------------------------------------------------| +| **Databricks workspace** | Any workspace (production, development, or [free trial](https://www.databricks.com/try-databricks)) | +| **Java** | Java 21 or above (required for the Morpheus transpiler) — [download](https://adoptium.net/temurin/releases) | +| **Network access** | GitHub, Maven Central, PyPI (see [Network access](#network-access-proxies-and-mirrors)) | -**Profile Configuration:** +:::note +Unlike the CLI installation, you do **not** need to pre-install the Databricks CLI, Python, or Lakebridge itself — +the app bundles the Databricks CLI and Python, and installs Lakebridge for you on first run. +::: -Additionally, Lakebridge requires the profile used for the Databricks CLI to specify a cluster_id, to do this, you can either: -- Edit your `~/.databrickscfg` file directly and enter a `cluster_id` for the profile you're using or -- The flag `--configure-cluster` gives you the prompt to select the cluster_id from the available clusters on the workspace -specified on the selected profile. -```shell -databricks configure --host --configure-cluster --profile -``` -- Alternatively you can use the environment variable `DATABRICKS_CLUSTER_ID` to set the cluster id you would want to use -for your profile before running the `databricks configure` command. +### Download -```shell -export DATABRICKS_CLUSTER_ID= -databricks configure --host --profile -``` - -**Verification:** Run `databricks clusters list` to confirm connectivity +Grab the installer for your platform from the [latest release](https://github.com/databrickslabs/lakebridge/releases/latest). +Choose the build that matches your operating system and CPU architecture: -### 2. Software Requirements +| Platform | Apple Silicon / ARM | Intel / AMD (x64) | +|----------|-----------------------------------|---------------------------------| +| macOS | `MacOS-Lakebridge--arm64.dmg` | `MacOS-Lakebridge--x64.dmg` | +| Windows | `Windows-Lakebridge--arm64.exe` | `Windows-Lakebridge--x64.exe` | -#### 2.1 Python +:::tip Which architecture? +- **macOS:** Apple Silicon (M1/M2/M3/M4) uses `arm64`; older Intel Macs use `x64`. +- **Windows:** most machines use `x64`; use `arm64` only for ARM-based devices (e.g. Surface Pro X). +::: -**Version Required:** Python between 3.10.1 and 3.13.x (inclusive). +### Install -- **Windows** - Install python from [here](https://www.python.org/downloads/). -- **MacOS/Unix** - Use [brew](https://formulae.brew.sh/formula/python@3.10) to install python in macOS/Unix machines + + + 1. Download the `.dmg` matching your Mac's architecture. + 2. Open the `.dmg` and drag **Lakebridge** into your **Applications** folder. + 3. Launch Lakebridge from Applications. + + + 1. Download the `.exe` matching your machine's architecture. + 2. Run the installer and follow the prompts. + 3. Launch Lakebridge from the Start menu. + + -**Check Python version on Windows, macOS, and Unix:** -check-python-version +### First run -**Verification:** Run `python --version` to confirm installation +On first launch, the app walks you through everything needed to get started: -**Note: Python 3.14 is not currently supported.** +1. **Check prerequisites** — the app verifies Java is installed at a supported version. If it is missing, + install [Java 21 or above](https://adoptium.net/temurin/releases) and re-check. +2. **Connect to Databricks** — enter your workspace URL and sign in. The app authenticates via OAuth in your + browser and configures the Databricks CLI profile for you. +3. **Install Lakebridge** — the app installs the Lakebridge CLI (and its bundled Databricks CLI and Python) + in the background. -#### 2.2 Java +Once these steps complete, you can run assessment, transpilation, and profiling directly from the app. -**Version Required:** [Java 11 or above](https://www.oracle.com/java/technologies/downloads/) +Installing Lakebridge using the desktop app -- **Recommended:** OpenJDK 11 or later (available via system package managers) -- **Purpose:** Required for the Morpheus transpiler component that powers code conversion -- **Verification:** Run `java -version` to confirm installation +--- -### 3. Network & Environment Access +## Command-line (CLI) -The following network endpoints must be accessible from your installation environment: +### Prerequisites -#### 3.1 GitHub Access -Access to Databricks Labs GitHub repository for downloading Lakebridge. -- **Repository:** [github.com/databrickslabs/lakebridge](https://github.com/databrickslabs/lakebridge) -- **Purpose:** Download latest official Lakebridge installation packages +| Requirement | Details | +|--------------------------|-----------------------------------------------------------------------------------------------------------------------| +| **Databricks workspace** | Any workspace (production, development, or [free trial](https://www.databricks.com/try-databricks)) | +| **Databricks CLI** | [Install here](https://docs.databricks.com/en/dev-tools/cli/install.html) and configure with PAT or Service Principal | +| **Python** | 3.10.1–3.14.x | +| **Java** | Java 21 or above (required for the Morpheus transpiler) | +| **Network access** | GitHub, Maven Central, PyPI (see [Network access](#network-access-proxies-and-mirrors)) | -#### 3.2 Maven Central Repository -Access to Maven Central for downloading transpiler plugins and dependencies. -- **Repository:** [central.sonatype.com](https://central.sonatype.com/artifact/com.databricks.labs/databricks-morph-plugin) -- **Purpose:** Download latest version of the transpiler plugins +#### Python and Java -#### 3.3 PyPI (Python Package Index) -Access to PyPI for Python package dependencies. -- **Repository:** [pypi.org](https://pypi.org) -- **Purpose:** Install Python dependencies +If necessary: + - Python can be obtained [here](https://python.org/); if installing on Windows, please ensure you install the 64-bit version. + - Java can be obtained [here](https://adoptium.net/temurin/releases); the current LTS release is recommended. -:::warning Hardened & Security-Restricted Environments -If you are operating in a hardened environment with internet restrictions, firewall rules, or security policies, you must whitelist the following resources **before installation**: +To verify these are installed and available, from the terminal the following should work and display the installed versions: -- **GitHub:** `github.com`, `raw.githubusercontent.com` - For Lakebridge source code -- **Maven Central:** `repo1.maven.org`, `central.sonatype.com` - For transpiler plugins -- **PyPI:** `pypi.org`, `files.pythonhosted.org` - For Python packages -- **Python Downloads:** `python.org` - If installing Python -- **Java Downloads:** `oracle.com` or OpenJDK mirrors - If installing Java +```console +python -V +java -version +``` -**Action Required:** Contact your IT Security, CyberSecOps, or Infrastructure team to request whitelisting. Consider setting up a **private repository/artifact mirror** for organizations with strict internet access policies. -::: +### Configure the Databricks CLI -#### 3.4 Private Repository Hosting (Optional) -For organizations with restricted internet access or security requirements: -- **Purpose:** Internal hosting and distribution of Lakebridge components -- **Options:** Artifactory, Nexus -- **Benefits:** Control over versions, security scanning, compliance validation +Install and authenticate the CLI: -### Pre-Installation Checklist + + + ```shell + brew tap databricks/tap + brew install databricks + ``` + + + ```shell + curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/v0.299.0/install.sh + ``` + + + ```bash + #!/usr/bin/env bash + apt update && apt install -y curl sudo unzip + curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/v0.299.0/install.sh | sudo sh + ``` + + -Verify all prerequisites before proceeding: +Authenticate the CLI: -- ☐ Databricks workspace access confirmed (production, dev, or free trial) -- ☐ Databricks CLI installed on your machine -- ☐ Databricks CLI configured with PAT or Service Principal for your workspace -- ☐ CLI connectivity verified (`databricks clusters list` succeeds) -- ☐ Python between 3.10.1 and 3.13.x (inclusive) installed and verified (`python --version`) -- ☐ Java 11+ installed and verified (`java -version`) -- ☐ Network access to GitHub confirmed -- ☐ Network access to Maven Central confirmed -- ☐ Network access to PyPI confirmed -- ☐ (If applicable) All required resources whitelisted in restricted environments -- ☐ (If applicable) Private repository configured for package hosting +```bash +databricks configure +``` -[[back to top](#table-of-contents)] +Verify connectivity: `databricks clusters list` ----- -## Install Lakebridge +### Install Lakebridge -Upon completing the environment setup, install Lakebridge by executing the following command: ```bash databricks labs install lakebridge ``` -This will install Lakebridge using the workspace details set in the DEFAULT profile. If you want to install it using a different profile, you can specify the profile name using the `--profile` flag. + +To use a specific profile: + ```bash databricks labs install lakebridge --profile ``` -To view all the profiles available, you can run the following command: -```bash -databricks auth profiles -``` lakebridge-install +Verify: -### Verify Installation -Verify the successful installation by executing the provided command; -confirmation of a successful installation is indicated when the displayed output aligns with the example below: - -Command: ```bash databricks labs lakebridge --help ``` -Should output: -```console -Code Transpiler and Data Reconciliation tool for Accelerating Data onboarding to Databricks from EDW, CDW and other ETL sources. - -Usage: - databricks labs lakebridge [command] - -Available Commands: - aggregates-reconcile Reconcile source and target data residing on Databricks using aggregated metrics - analyze Analyze existing non-Databricks database or ETL sources - configure-database-profiler Configure database profiler - configure-reconcile Configure 'reconcile' dependencies - describe-transpile Describe installed transpilers - install-transpile Install & optionally configure 'transpile' dependencies - reconcile Reconcile source and target data residing on Databricks - transpile Transpile SQL/ETL sources to Databricks-compatible code - -Flags: - -h, --help help for lakebridge - -Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) - -Use "databricks labs lakebridge [command] --help" for more information about a command. +### Install Transpile + +```bash +databricks labs lakebridge install-transpile ``` -[[back to top](#table-of-contents)] +The command will prompt for your source dialect, input/output paths, and target technology. ----- +To install Switch (the LLM transpiler): -## Install Transpile +```bash +databricks labs lakebridge install-transpile --include-llm-transpiler true +``` -Upon completing the environment setup, you can install the out of the box transpilers by executing the following command. -This command will also prompt for the required configuration elements so that you don't need to include them in your command-line -call every time. +:::tip Override the default BladeBridge config +During `install-transpile` you can supply a custom config file for BladeBridge: +``` +Specify the config file to override the default[Bladebridge] config: /custom_config.json +``` +::: + +Verify: ```bash -databricks labs lakebridge install-transpile +databricks labs lakebridge transpile --help ``` -transpile-install +### Configure Reconcile +```bash +databricks labs lakebridge configure-reconcile +``` -:::tip -#### Override the default[Bladebridge] config: +The command will prompt for your source connection and Databricks catalog to reconcile, and install Lakebridge and create the required workspace resources to run Reconcile. +Optionally, the command can discover the tables in your source and generate a base config to run reconcile. This autoconfiguration should be reviewed before running reconcile. -There is an option for you to override the default config file that `Lakebridge` uses for converting source code from dialects like `datastage`, `synapse`, -`oracle` etc. During installation you may use your own custom config file and `Lakebridge` will override the config with the one you would provide. You can only setup this -override during installation. +If you don't have permission to create SQL warehouses or clusters, add a `warehouse_id` or a `cluster_id` to your Databricks CLI profile: -Specify the config file to override the default[Bladebridge] config during installation: ``` -Specify the config file to override the default[Bladebridge] config - press for none (default: ): /custom_2databricks.json +[profile-name] +host = +warehouse_id = +cluster_id = ``` -::: -### Verify Installation -Verify the successful installation by executing the provided command; confirmation of a successful installation is indicated when the displayed output aligns with the example output: +Verify: -Command: ```bash -databricks labs lakebridge transpile --help +databricks labs lakebridge reconcile --help ``` -Should output: -```console -Transpile SQL/ETL sources to Databricks-compatible code - -Usage: - databricks labs lakebridge transpile [flags] - -Flags: - --catalog-name name (Optional) Catalog name, only used when validating converted code - --error-file-path path (Optional) Local path where a log of conversion errors (if any) will be written - -h, --help help for transpile - --input-source path (Optional) Local path of the sources to be convert - --output-folder path (Optional) Local path where converted code will be written - --overrides-file path (Optional) Local path of a file containing transpiler overrides, if supported by the transpiler in use - --schema-name name (Optional) Schema name, only used when validating converted code - --skip-validation string (Optional) Whether to skip validating the output ('true') after conversion or not ('false') - --source-dialect string (Optional) The source dialect to use when performing conversion - --target-technology string (Optional) Target technology to use for code generation, if supported by the transpiler in use - --transpiler-config-path path (Optional) Local path to the configuration file of the transpiler to use for conversion - -Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) -``` +:::note Upgrading from an older Reconcile schema +Upgrading Lakebridge migrates the reconcile `details` and `aggregate_details` +metadata tables to a new schema. The original tables are preserved as +`..details_backup` and `aggregate_details_backup`, so the raw +rows are never lost. -[[back to top](#table-of-contents)] +The migration is not automatically resumable. If it fails midway, rerun the +upgrade after restoring the original tables manually: ----- +- If `details` no longer exists, rename the backup back and rerun: + `ALTER TABLE ..details_backup RENAME TO ..details` +- If a new (empty or partial) `details` was already created, drop it, rename the + backup back as above, then rerun. -## Configure Reconcile +Apply the same steps to `aggregate_details` / `aggregate_details_backup`. Once +the migrated tables are verified, the `*_backup` tables can be dropped. +::: -Once you're ready to reconcile your data, you need to configure the reconcile module. +### Service Principal Setup (Optional) -```commandline -databricks labs lakebridge configure-reconcile -``` -reconcile-configure +For automated/production deployments, use a Service Principal instead of a Personal Access Token. See the [Databricks CLI authentication docs](https://docs.databricks.com/aws/en/dev-tools/cli/authentication) for setup instructions. +--- -### SQL Warehouse for Reconcile +## Network access, proxies, and mirrors + +Both installation paths need access to the following network resources to download Lakebridge and its transpiler plugins: + + + + + + + + + + + + + + + + + + + + + +
SiteHostsPurpose
GitHubgithub.com
+ raw.githubusercontent.com		Packages and metadata used for general installation and upgrades of Lakebridge.
Maven Centralrepo1.maven.orgInstalling and upgrading transpiler plugins.
PyPIpypi.org
+ files.pythonhosted.org
+ +Support for proxies or mirrors to access these Internet resources can be configured as shown below. + +### General HTTP proxy configuration + +To configure general HTTP proxy for network access, set an environment variable named `https_proxy` to the URL of the HTTPS proxy. -While configuring the reconcile properties, lakebridge by default creates a SQL warehouse. lakebridge uses user profile to authenticate to any Databricks resource and hence -if the user running this command doesn't have permission to create SQL warehouse, the configure-reconcile would fail. In this case users can provide the -`warehouse_id` of an already created SQL warehouse that they have atleast CAN_USE permission on in the databricks profile (`~/.databrickscfg`) using which they -are running the lakebridge commands and lakebridge would use that warehouse to complete the reconcile configuration instead of trying to create a new one. + + + ```bash + export https_proxy=http://my-proxy.example.com:3128/ + ``` + + + ```command + set https_proxy=http://my-proxy.example.com:3128/ + ``` + + + ```bash + export https_proxy=http://my-proxy.example.com:3128/ + ``` + + -This is how the profile would look like: +Contact your IT team if necessary for information on the URL to use. If authentication is needed, refer to the [section below](#proxy-or-mirror-authentication). -```console -[profile-name] -host = -... -warehouse_id = -``` +### Maven Central +If a local mirror should be used for downloading resources from Maven Central, set the `LAKEBRIDGE_MAVEN_URL` environment to the URL of the mirror. -### Verify Configuration -Verify the successful configuration by executing the provided command; confirmation of a successful configuration is indicated when the displayed output aligns with the example screenshot provided: + + + ```bash + export LAKEBRIDGE_MAVEN_URL=https://mirror.example.com/maven/releases/ + ``` + + + ```command + set LAKEBRIDGE_MAVEN_URL=https://mirror.example.com/maven/releases/ + ``` + + + ```bash + export LAKEBRIDGE_MAVEN_URL=https://mirror.example.com/maven/releases/ + ``` + + -Command: -```bash - databricks labs lakebridge reconcile --help - ``` +Contact your IT team if necessary for information on the URL to use. If authentication is needed, refer to the [section below](#proxy-or-mirror-authentication). -Should output: -```console -Reconcile source and target data residing on Databricks +### PyPI + +If a local mirror should be used for downloaded resources from PyPI, this needs to be configured with pip: +```shell +pip3 config --user set global.index-url https://mirror.example.com/pypi +``` -Usage: - databricks labs lakebridge reconcile [flags] +Contact your IT team if necessary for information on the URL to use. If authentication is needed, refer to the [section below](#proxy-or-mirror-authentication). -Flags: - -h, --help help for reconcile +### Proxy or Mirror Authentication -Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) +If authentication is needed to access a mirror or proxy, a `~/.netrc` file can be used to specify the credentials to use. The format is of the form: +```netrc +machine my-proxy.example.com +login bobby +password tAble5 ``` -[[back to top](#table-of-contents)] +Note that `my-proxy.example.com` should be the host from the URL (and not the entire URL). diff --git a/docs/lakebridge/docs/overview.mdx b/docs/lakebridge/docs/overview.mdx index 4931c14db5..8f8b07cc7f 100644 --- a/docs/lakebridge/docs/overview.mdx +++ b/docs/lakebridge/docs/overview.mdx @@ -86,10 +86,10 @@ Supported sources big query - + ✅ ✅ - + ✅ db2 @@ -114,7 +114,7 @@ Supported sources mssql - + ✅ ✅ ✅ ✅ @@ -135,7 +135,7 @@ Supported sources oracle - + ✅ ✅ ✅ ✅ @@ -159,7 +159,7 @@ Supported sources ✅ ✅ - + ✅ sap hana (calc views) @@ -170,7 +170,7 @@ Supported sources snowflake - + ✅ ✅ ✅ ✅ @@ -184,10 +184,10 @@ Supported sources teradata - ✅ ✅ - + ✅ + ✅ vertica diff --git a/docs/lakebridge/docs/reconcile/configuration.mdx b/docs/lakebridge/docs/reconcile/configuration.mdx new file mode 100644 index 0000000000..2024381537 --- /dev/null +++ b/docs/lakebridge/docs/reconcile/configuration.mdx @@ -0,0 +1,660 @@ +--- +sidebar_position: 1 +title: Configuration Reference +--- +import useBaseUrl from '@docusaurus/useBaseUrl'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; + +# Reconcile Configuration Reference + +This page covers the full configuration schema for Lakebridge Reconcile. For setup steps and prerequisites, see the [Reconcile Guide](/docs/reconcile). + +--- + +## Report Types + +| report type | description | key outputs | +|---|---|---| +| `schema` | Reconcile schema of source and target — validate that data types are the same or compatible | `schema_comparison`, `schema_difference` | +| `row` | Reconcile data at row level (hash value comparison). Use when there are no join columns. | `missing_in_src`, `missing_in_tgt` | +| `data` | Reconcile data at row and column level using `join_columns` to identify per-row, per-column mismatches | `mismatch_data`, `missing_in_src`, `missing_in_tgt`, `threshold_mismatch`, `mismatch_columns` | +| `all` | Combination of `data` + `schema` | All outputs above | + +See [Reconcile Data Flow Examples](/docs/reconcile/dataflow_example) for visualizations of each report type. + +--- + +## Config File Naming + +``` +recon_config___.json +``` + +Place the file in `.lakebridge/` in your Databricks workspace home folder. + +> The filename must match the case of `` exactly. For example, if the source connection is `conn-oracle-prod`, the filename is `recon_config_oracle_conn-oracle-prod_data.json`. + +**Examples by source:** + + + + ```yaml title="recon_config_snowflake_sample_data_all.json" + source: + dialect: snowflake + catalog: sample_data + schema: default + uc_connection_name: example_connection_snowflake + target: + catalog: migrated + schema: migrated + report_type: all + ``` + + + ```yaml title="recon_config_oracle_orc_data.json" + source: + dialect: oracle + uc_connection_name: example_connection_oracle + target: + catalog: migrated + schema: migrated + report_type: data + ``` + + + ```yaml title="recon_config_tsql_silver_data.json" + source: + dialect: mssql + uc_connection_name: example_connection_mssql + target: + catalog: migrated + schema: migrated + report_type: data + ``` + + + ```yaml title="recon_config_teradata_teradata_sandbox_all.json" + source: + dialect: teradata + catalog: DBC + schema: lakebridge_test + uc_connection_name: teradata_sandbox + target: + catalog: migrated + schema: migrated + report_type: all + ``` + + :::warning Required: install a hash UDF on the Teradata source + Teradata does not provide a portable cryptographic hash function in pure SQL, so **you + must install your own hash UDF on the Teradata source** before running any `row`, `data`, + or `all` report (the `schema` report type does not need it). Any deterministic hash + (SHA-256, MD5, etc.) works as long as the source and target sides produce the same string + for the same input. + + You must also tell Lakebridge how to call the UDF by setting + `hash_expression_overrides.source` on the recon config (see + [Hash Expression](#hash-expression)). + + + Refer to the Teradata documentation for how to create and register UDFs on your release + and platform. For a real-world example, Teradata publishes an MD5 message-digest UDF you + can install as-is or adapt: + [MD5 Message Digest UDF](https://downloads.teradata.com/download/extensibility/md5-message-digest-udf). + ::: + + + ```yaml title="recon_config_bigquery_sample_data_all.json" + source: + dialect: bigquery + catalog: bq_project_name + schema: sample_dataset + uc_connection_name: example_connection_bigquery + target: + catalog: migrated + schema: migrated + report_type: all + ``` + + :::warning Writable dataset required + Results materialize into the ``lakebridge_reconcile`` dataset. Make sure a dataset with this name + exist in the same region as the source data and be writable by the connection's service account. + ::: + + + ```yaml title="recon_config_databricks_hms_schema.json" + source: + dialect: databricks + catalog: hive_metastore + schema: hr + target: + catalog: migrated + schema: migrated + ... + report_type: data + ``` + + + +## TABLE Config Schema + + + + ```python + @dataclass + class Table: + source_name: str + target_name: str + aggregates: list[Aggregate] | None = None + join_columns: list[str] | None = None + jdbc_reader_options: JdbcReaderOptions | None = None + select_columns: list[str] | None = None + drop_columns: list[str] | None = None + column_mapping: list[ColumnMapping] | None = None + transformations: list[Transformation] | None = None + column_thresholds: list[ColumnThresholds] | None = None + filters: Filters | None = None + table_thresholds: list[TableThresholds] | None = None + sampling_options: SamplingOptions | None = None + ``` + + + ```json + { + "source_name": "[SOURCE_NAME]", + "target_name": "[TARGET_NAME]", + "aggregates": null, + "join_columns": ["COLUMN_NAME_1", "COLUMN_NAME_2"], + "jdbc_reader_options": null, + "select_columns": null, + "drop_columns": null, + "column_mapping": null, + "transformation": null, + "column_thresholds": null, + "filters": null, + "table_thresholds": null, + "sampling_options": null + } + ``` + + + +### Configuration Parameters + +| Parameter | Type | Required | Description | +|---|---|---|---| +| `source_name` | string | Yes | Source table name | +| `target_name` | string | Yes | Target table name | +| `join_columns` | list[string] | No | Primary key columns for row-level joins | +| `aggregates` | list[Aggregate] | No | Aggregate reconciliation rules (see [Aggregates](#aggregates-reconciliation)) | +| `jdbc_reader_options` | object | No | JDBC read parallelization (see [JDBC Reader Options](#jdbc-reader-options)) | +| `select_columns` | list[string] | No | Columns to include in reconciliation | +| `drop_columns` | list[string] | No | Columns to exclude from reconciliation | +| `column_mapping` | list[ColumnMapping] | No | Source-to-target column name mapping | +| `transformations` | list[Transformation] | No | Column-level SQL transformation expressions | +| `column_thresholds` | list[ColumnThresholds] | No | Acceptable variance per column | +| `table_thresholds` | list[TableThresholds] | No | Acceptable mismatch rate at table level | +| `filters` | Filters | No | WHERE-clause filters for source and/or target | +| `sampling_options` | SamplingOptions | No | Caps the rows pulled into each sample bucket — mismatch, missing-in-source, missing-in-target (see [Sampling Options](#sampling-options)) | + +--- + +## JDBC Reader Options + +Use JDBC reader options to parallelize reads from large tables. + + + + ```python + @dataclass + class JdbcReaderOptions: + num_partitions: int + partition_column: str + lower_bound: str + upper_bound: str + fetchsize: int = 0 + ``` + + + ```json + "jdbc_reader_options": { + "num_partitions": 10, + "partition_column": "id", + "lower_bound": "1", + "upper_bound": "100000", + "fetchsize": 0 + } + ``` + + + +| Field | Type | Required | Description | +|---|---|---|---| +| `num_partitions` | int | Yes | Number of Spark partitions for parallel reads | +| `partition_column` | string | Yes | Column used for partitioning (choose high-cardinality column for composite keys) | +| `lower_bound` | string | Yes | Minimum value for partitioning | +| `upper_bound` | string | Yes | Maximum value for partitioning | +| `fetchsize` | int | No (default: 100) | Rows per JDBC round-trip | + +--- + +## Column Mapping + +Map source column names to target column names when they differ. + + + + ```python + @dataclass + class ColumnMapping: + source_name: str + target_name: str + ``` + + + ```json + "column_mapping": [ + { + "source_name": "dept_id", + "target_name": "department_id" + } + ] + ``` + + + +--- + +## Drop Columns + +Exclude specific columns from reconciliation (for `data` or `all` report types). + +```json +"drop_columns": ["comment", "audit_timestamp"] +``` + +--- + +## Transformations + +Apply SQL expressions to source or target columns before comparison. Use this when data types or formats differ between systems. + + + + ```python + @dataclass + class Transformation: + column_name: str + source: str + target: str | None = None + ``` + + + ```json + "transformations": [ + { + "column_name": "unit_price", + "source": "coalesce(cast(cast(unit_price as decimal(38,10)) as string), '_null_recon_')", + "target": "coalesce(cast(format_number(cast(unit_price as decimal(38, 10)), 10) as string), '_null_recon_')" + } + ] + ``` + + + +:::danger Handling Nulls +When you provide a transformation expression, Reconcile uses it as-is. **You must handle nulls and cast to string explicitly** in the expression. +Use `coalesce(cast(... as string), '_null_recon_')` to avoid null mismatches and incompatibility with the internal transformations that rely on strings. +::: + +:::tip Timestamp Columns +Convert timestamps to Unix epoch strings for cross-platform comparison: +```python +Transformation( + column_name='UPDATE_TIMESTAMP', + source="coalesce(cast(EXTRACT(epoch_millisecond FROM UPDATE_TIMESTAMP) as string), '_null_recon_')", + target="coalesce(cast(unix_millis(UPDATE_TIMESTAMP) as string), '_null_recon_')" +) +``` +::: + +--- + +## Hash Expression + +Override the row-hash function with your own SQL. Each value is raw SQL containing a single +`{}` placeholder that Lakebridge replaces with the concatenated hash input. + +- `source` is **required** when the block is set: the source-side expression Lakebridge will + emit on the source dialect. +- `target` defaults to `sha2({}, 256)` (the target is always Databricks); override it if you + use a non-SHA-256 hash. + +The block itself is optional for most dialects, omit it to use the source dialect's default. +For Teradata sources the block is **mandatory**, since Teradata has no out-of-the-box +cryptographic hash. + + + + ```python + @dataclass + class HashExpressionOverrides: + source: str + target: str = "sha2({}, 256)" + ``` + + + ```yaml + hash_expression_overrides: + source: "my_sha256({})" + target: "sha2({}, 256)" + ``` + + + +:::note Same hash on both sides +The source and target expressions must produce the **same string for the same logical row** — +that is what the comparison query joins on. Lakebridge wraps both sides with `LOWER(...)` +automatically, so case alone won't cause mismatches; the bytes produced by the hash itself +must match across systems. +::: + +--- + +## Column Thresholds + +Allow a bounded variance between source and target column values. + +```json +"column_thresholds": [ + { + "column_name": "price", + "lower_bound": "-5%", + "upper_bound": "5%", + "type": "float" + } +] +``` + +| Field | Type | Required | Description | +|---|---|---|---| +| `column_name` | string | Yes | Column to apply the threshold to | +| `lower_bound` | string | Yes | Lower bound of acceptable difference | +| `upper_bound` | string | Yes | Upper bound of acceptable difference | +| `type` | string | Yes | Column type (supports SQLGlot `DataType.NUMERIC_TYPES` and `DataType.TEMPORAL_TYPES`) | + +--- + +## Table Thresholds + +Allow a bounded mismatch rate at the table level. + +```json +"table_thresholds": [ + { + "lower_bound": "0%", + "upper_bound": "5%", + "model": "mismatch" + } +] +``` + +Bounds must be non-negative, and lower bound must not exceed upper bound. Only `"mismatch"` is supported for `model`. + +--- + +## Filters + +Apply WHERE-clause filters to source and/or target before reconciliation. + +```json +"filters": { + "source": "lower(dept_name) = 'finance'", + "target": "lower(department_name) = 'finance'" +} +``` + +:::note +Filters must use dialect-specific SQL expressions — they are applied directly without any transformation by Reconcile. +::: + +--- + +## Sampling Options + +For `data` and `all` report types, Reconcile captures sample rows for each mismatch / missing-in-source / missing-in-target bucket. `sampling_options` controls **how many** rows are sampled and **how** they are drawn. + +:::note +`sampling_options` applies to **row-level** reconcile (`data` / `all` report types) only. +[Aggregates Reconciliation](#aggregates-reconciliation) always samples up to `50` +(`DEFAULT_SAMPLE_ROWS`) rows per missing-in-source / missing-in-target set and ignores +`sampling_options` (both the sampling method and size). +::: + +### SamplingSpecifications + +Controls the sample-size policy. + + + + ```python + @dataclass + class SamplingSpecifications: + type: SamplingSpecificationsType = SamplingSpecificationsType.COUNT + value: int | float = 50 + ``` + + + ```json + "specifications": { + "type": "count", + "value": 50 + } + ``` + + + +| field | type | description | default | +|---|---|---|---| +| `type` | `count` \| `fraction` | `count` = absolute row cap. `fraction` is reserved and currently disabled — passing it raises a config error. | `count` | +| `value` | int (float for fraction) | Maximum rows per sample set (mismatch / missing-in-source / missing-in-target). | `50` | + +**Engine-aware bounds on `value`:** + +Each source has a `max_sample_size` (a property on its connector): + +- **Databricks source:** honored up to `50_000`. +- **Non-Databricks source:** honored up to `400`. +- Values above the source's limit are capped to it with a `WARNING` log (`exceeds the limit of for ; capping to `). +- `value <= 0` falls back to `50`. + +### SamplingOptions + +Wraps `SamplingSpecifications` and selects the sampling method. + + + + ```python + @dataclass + class SamplingOptions: + method: SamplingOptionMethod = SamplingOptionMethod.RANDOM + specifications: SamplingSpecifications = field(default_factory=SamplingSpecifications) + stratified_columns: list[str] | None = None + stratified_buckets: int | None = None + ``` + + + ```json + "sampling_options": { + "method": "random", + "specifications": { "type": "count", "value": 50 }, + "stratified_columns": null, + "stratified_buckets": null + } + ``` + + + +| field | type | description | default | +|---|---|---|---| +| `method` | `random` \| `stratified` | Sampling strategy. | `random` | +| `specifications` | SamplingSpecifications | Size policy (see [above](#samplingspecifications)). | `count` / `50` | +| `stratified_columns` | list[string] | **Required when `method = stratified`.** Columns whose values define the strata. | `null` | +| `stratified_buckets` | int | **Required when `method = stratified`.** Number of strata buckets. Clamped to `[2, 50]`. | `null` | + +When `method = stratified`, rows are bucketed by `pmod(abs(hash(stratified_columns)), stratified_buckets)` and sampling proceeds within each bucket. Missing `stratified_columns` or `stratified_buckets` raises a `ValueError`. + +**Examples** + +Random sampling (Databricks source) — limit each sample set (mismatch, missing-in-source, missing-in-target) to 500 rows: +```json +"sampling_options": { + "method": "random", + "specifications": { "type": "count", "value": 500 } +} +``` + +Shortest form (relies on defaults of `random` and `count`): +```json +"sampling_options": { "specifications": { "value": 500 } } +``` + +Stratified sampling — 10 strata on `ss_store_sk`, up to 500 sample rows total (allocated proportionally across buckets): +```json +"sampling_options": { + "method": "stratified", + "specifications": { "type": "count", "value": 500 }, + "stratified_columns": ["ss_store_sk"], + "stratified_buckets": 10 +} +``` + +--- + +## Aggregates Reconciliation + +Reconcile aggregate metrics (MIN, MAX, COUNT, SUM, AVG, etc.) between source and target instead of comparing raw rows. + +:::note +Aggregates reconcile always samples up to `50` (`DEFAULT_SAMPLE_ROWS`) rows per +missing-in-source / missing-in-target set; [`sampling_options`](#sampling-options) does not +apply here. +::: + +### Supported Functions + +MIN, MAX, COUNT, SUM, AVG, MEAN, MODE, STDDEV, VARIANCE, MEDIAN + +### Aggregate Config + + + + ```python + @dataclass + class Aggregate: + agg_columns: list[str] + type: str + group_by_columns: list[str] | None = None + ``` + + + ```json + { + "type": "MIN", + "agg_columns": ["discount"], + "group_by_columns": ["p_id"] + } + ``` + + + +:::note +`select_columns` and `drop_columns` are ignored for aggregate reconciliation. `column_mapping`, `transformations`, `jdbc_reader_options`, and `filters` are applied. +::: + +--- + +## Complete Examples + +### Standard Reconcile Config + +```json +{ + "tables": [ + { + "source_name": "product_prod", + "target_name": "product", + "jdbc_reader_options": { + "num_partitions": 10, + "partition_column": "p_id", + "lower_bound": "0", + "upper_bound": "10000000" + }, + "join_columns": ["p_id"], + "drop_columns": ["comment"], + "column_mapping": [ + { "source_name": "p_id", "target_name": "product_id" }, + { "source_name": "p_name", "target_name": "product_name" } + ], + "transformations": [ + { + "column_name": "creation_date", + "source": "creation_date", + "target": "to_date(creation_date,'yyyy-mm-dd')" + } + ], + "column_thresholds": [ + { "column_name": "price", "lower_bound": "-50", "upper_bound": "50", "type": "float" } + ], + "table_thresholds": [ + { "lower_bound": "0%", "upper_bound": "5%", "model": "mismatch" } + ], + "filters": { + "source": "p_id > 0", + "target": "product_id > 0" + }, + "sampling_options": { + "method": "random", + "specifications": { "type": "count", "value": 100 } + } + } + ] +} +``` + +### Aggregates Reconcile Config + +```json +{ + "tables": [ + { + "source_name": "product_prod", + "target_name": "product", + "join_columns": ["p_id"], + "aggregates": [ + { "type": "MIN", "agg_columns": ["discount"], "group_by_columns": ["p_id"] }, + { "type": "AVG", "agg_columns": ["discount"], "group_by_columns": ["p_id"] }, + { "type": "MAX", "agg_columns": ["p_id"], "group_by_columns": ["creation_date"] }, + { "type": "SUM", "agg_columns": ["p_id"] } + ], + "jdbc_reader_options": { + "num_partitions": 10, + "partition_column": "p_id", + "lower_bound": "0", + "upper_bound": "10000000" + } + } + ] +} +``` + +--- + +:::note Key Considerations +1. Column names are always converted to lowercase. +2. Case insensitivity and collation are not currently supported. +3. Always reference **source** column names in all configs, except `transformations` and `filters` — those use dialect-specific SQL applied directly. +4. When no user transformation is provided, Reconcile applies default transformations based on the source data type. +::: diff --git a/docs/lakebridge/docs/reconcile/example_config.mdx b/docs/lakebridge/docs/reconcile/example_config.mdx deleted file mode 100644 index 688c5b5755..0000000000 --- a/docs/lakebridge/docs/reconcile/example_config.mdx +++ /dev/null @@ -1,181 +0,0 @@ ---- -sidebar_position: 3 -title: Example Configuration ---- - -## Reconcile Config -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - -Consider the below tables that we want to reconcile: - -| category | catalog | schema | table_name | schema | primary_key | -|----------|----------------|---------------|--------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------| -| source | source_catalog | source_schema | product_prod | p_id INT,
p_name STRING
,
price NUMBER
,
discount DECIMAL(5,3)
,
offer DOUBLE
,
creation_date DATE
,
comment STRING
 | p_id | -| target | target_catalog | target_schema | product | product_id INT,
product_name STRING
,
price NUMBER
,
discount DECIMAL(5,3)
,
offer DOUBLE
,
creation_date DATE
,
comment STRING
 | product_id | - - - - - - :::note - Run with Drop,Join,Transformation,ColumnThresholds,Filter,JDBC ReaderOptions configs - ::: - ```json - { - "tables": [ - { - "source_name": "product_prod", - "target_name": "product", - "jdbc_reader_options": { - "number_partitions": 10, - "partition_column": "p_id", - "lower_bound": "0", - "upper_bound": "10000000" - }, - "join_columns": [ - "p_id" - ], - "drop_columns": [ - "comment" - ], - "column_mapping": [ - { - "source_name": "p_id", - "target_name": "product_id" - }, - { - "source_name": "p_name", - "target_name": "product_name" - } - ], - "transformations": [ - { - "column_name": "creation_date", - "source": "creation_date", - "target": "to_date(creation_date,'yyyy-mm-dd')" - } - ], - "column_thresholds": [ - { - "column_name": "price", - "upper_bound": "-50", - "lower_bound": "50", - "type": "float" - } - ], - "table_thresholds": [ - { - "lower_bound": "0%", - "upper_bound": "5%", - "model": "mismatch" - } - ], - "filters": { - "source": "p_id > 0", - "target": "product_id > 0" - } - } - ] - } - ``` - - - :::note - Aggregates-Reconcile run with Join, Column Mappings, Transformation, Filter and JDBC ReaderOptions configs - Even though the user provides the \`select_columns\` and \`drop_columns\`, those are not considered. - ::: - ```json - { - "tables": [ - { - "aggregates": [{ - "type": "MIN", - "agg_columns": ["discount"], - "group_by_columns": ["p_id"] - }, - { - "type": "AVG", - "agg_columns": ["discount"], - "group_by_columns": ["p_id"] - }, - { - "type": "MAX", - "agg_columns": ["p_id"], - "group_by_columns": ["creation_date"] - }, - { - "type": "MAX", - "agg_columns": ["p_name"] - }, - { - "type": "SUM", - "agg_columns": ["p_id"] - }, - { - "type": "MAX", - "agg_columns": ["creation_date"] - }, - { - "type": "MAX", - "agg_columns": ["p_id"], - "group_by_columns": ["creation_date"] - } - ], - "source_name": "product_prod", - "target_name": "product", - "jdbc_reader_options": { - "number_partitions": 10, - "partition_column": "p_id", - "lower_bound": "0", - "upper_bound": "10000000" - }, - "join_columns": [ - "p_id" - ], - "drop_columns": [ - "comment" - ], - "column_mapping": [ - { - "source_name": "p_id", - "target_name": "product_id" - }, - { - "source_name": "p_name", - "target_name": "product_name" - } - ], - "transformations": [ - { - "column_name": "creation_date", - "source": "creation_date", - "target": "to_date(creation_date,'yyyy-mm-dd')" - } - ], - "column_thresholds": [ - { - "column_name": "price", - "upper_bound": "-50", - "lower_bound": "50", - "type": "float" - } - ], - "table_thresholds": [ - { - "lower_bound": "0%", - "upper_bound": "5%", - "model": "mismatch" - } - ], - "filters": { - "source": "p_id > 0", - "target": "product_id > 0" - } - } - ] - } - ``` - - diff --git a/docs/lakebridge/docs/reconcile/index.mdx b/docs/lakebridge/docs/reconcile/index.mdx index 07a12bd2bf..4b81446618 100644 --- a/docs/lakebridge/docs/reconcile/index.mdx +++ b/docs/lakebridge/docs/reconcile/index.mdx @@ -1,5 +1,5 @@ --- -sidebar_position: 5 +sidebar_position: 6 --- import useBaseUrl from '@docusaurus/useBaseUrl'; import Tabs from '@theme/Tabs'; @@ -7,147 +7,110 @@ import TabItem from '@theme/TabItem'; # Reconcile Guide -This tool empowers users to efficiently identify discrepancies and variations in data when comparing the source with the Databricks target. +Lakebridge Reconcile validates data fidelity after migration by comparing your source system against the Databricks target. It identifies discrepancies at the row, column, and schema level. +## What it does -### Execution Pre-Set Up ->1. Setup the configuration file: +| Report type | What is compared | When to use | +|-------------|-----------------|-------------| +| `schema` | Column names and data types | Verify DDL migration is correct | +| `row` | Hash of each row (no join key needed) | Quick row-level check when there is no primary key | +| `data` | Row and column values via join columns | Full fidelity check with per-column mismatch detail | +| `all` | Both `data` + `schema` | Complete validation | -Once the [installation](/installation.mdx#configure-reconcile) is done, a folder named **.lakebridge** will be created in the user workspace's home folder. -To process the reconciliation for specific table sources, we must create a config file that gives the detailed required configurations for the table-specific ones. -The file name should be in the format as below and created inside the **.lakebridge** folder. +## Supported Source Systems + +| Source | Schema | Row | Data | All | +|---|---|---|---|---| +| Oracle | Yes | Yes | Yes | Yes | +| Snowflake | Yes | Yes | Yes | Yes | +| SQL Server | Yes | Yes | Yes | Yes | +| Redshift | Yes | Yes | Yes | Yes | +| Teradata | Yes | Yes [^teradata-hash] | Yes [^teradata-hash] | Yes [^teradata-hash] | +| BigQuery | Yes [^bigquery-types] | Yes | Yes | Yes | +| Databricks | Yes | Yes | Yes | Yes | + +[^teradata-hash]: Teradata has no portable cryptographic hash in pure SQL, so row-hash report types (`row`, `data`, `all`) require a user-installed hash UDF on the source and an explicit `hash_expression_overrides.source` entry on the recon config. See [Hash Expression](/docs/reconcile/configuration#hash-expression) for wiring. + +[^bigquery-types]: BigQuery types with no equivalent in Databricks — `BIGNUMERIC`, `TIME`, `RANGE`, nested `JSON` such as `ARRAY`, or these nested inside `ARRAY`/`STRUCT` — are left as their original BigQuery type and may be reported as `schema` mismatches even when the data migrated correctly. + +--- + +## Setup + +### Step 1: Setup the source connection + +Follow the official Databricks docs to: +- [Create a connection](https://docs.databricks.com/aws/en/query-federation/remote-queries#create-a-connection) +- [Grant connection access](https://docs.databricks.com/aws/en/query-federation/remote-queries#grant-connection-access) +- [Enable Databricks preview](https://docs.databricks.com/aws/en/admin/workspace-settings/manage-previews#-manage-workspace-level-previews) of `remote_query` feature + +:::note +You do not have to create a foreign catalog. +::: + +### Step 2: Run `configure-reconcile` + +If you haven't already, complete the initial setup: + +```bash +databricks labs lakebridge configure-reconcile ``` -recon_config___.json -Note: For CATALOG_OR_SCHEMA , if CATALOG exists then CATALOG else SCHEMA +This sets up Lakebridge workspace resources. See [Installation → Configure Reconcile](/docs/installation#configure-reconcile) for details. + +### Config file +A reconcile config file is created under the path: ``` +/.lakebridge/recon_config___.json +``` +:::note +For `UNITY_CATALOG_CONNECTION_NAME_OR_CATALOG`: if the source is databricks then source catalog name is used else connection name is used +::: -eg: +Examples: -| source_type | catalog_or_schema | report_type | file_name | +| source_type | connection_name_or_catalog | report_type | file_name | |-------------|-------------------|-------------|---------------------------------------| | databricks | tpch | all | recon_config_databricks_tpch_all.json | -| source1 | tpch | row | recon_config_source1_tpch_row.json | -| source2 | tpch | schema | recon_config_source2_tpch_schema.json | - - -Refer to [Reconcile Configuration Guide](reconcile_configuration) for detailed instructions and [example configurations](example_config) - -> 2. Setup the connection properties - -Lakebridge-Reconcile manages connection properties by utilizing secrets stored in the Databricks workspace. -Below is the default secret naming convention for managing connection properties. - -**Note: When both the source and target are Databricks, a secret scope is not required.** - -**Default Secret Scope:** lakebridge_data_source - -| source | scope | -|---------------|-----------------------| -| snowflake | lakebridge_snowflake | -| oracle | lakebridge_oracle | -| databricks | lakebridge_databricks | -| mssql | lakebridge_mssql | -| synapse | lakebridge_synapse | - -Below are the connection properties required for each source: - - - - ```yaml - sfUrl = https://[acount_name].snowflakecomputing.com - account = [acount_name] - sfUser = [user] - sfPassword = [password] - sfDatabase = [database] - sfSchema = [schema] - sfWarehouse = [warehouse_name] - sfRole = [role_name] - pem_private_key = [pkcs8_pem_private_key] - pem_private_key_password = [pkcs8_pem_private_key] - ``` - - :::note - For Snowflake authentication, either sfPassword or pem_private_key is required. - Priority is given to pem_private_key, and if it is not found, sfPassword will be used. - If neither is available, an exception will be raised. - - When using an encrypted pem_private_key, you'll need to provide the pem_private_key_password. - This password is used to decrypt the private key for authentication. - ::: - - - ```yaml - user = [user] - password = [password] - host = [host] - port = [port] - database = [database/SID] - ``` - - - ```yaml - user = [user] - password = [password] - host = [host] - port = [port] - database = [database/SID] - encrypt = [true/false] - trustServerCertificate = [true/false] - ``` - - - ->3. Databricks permissions required - -- User configuring reconcile must have permission to create Data Warehouses -- Additionally, the user must have `USE CATALOG` and `CREATE SCHEMA` permission in order to deploy metadata tables and -dashboards that are created as part of the Reconcile output. If there is a pre-existing schema, the 'create volumes' permission is also required. - ->4. Serverless cluster support - -Reconcile automatically detects the cluster type and optimizes intermediate data persistence accordingly: -- **On Serverless clusters**: Reconcile uses Unity Catalog volumes for intermediate data persistence -- **On Standard clusters**: Reconcile uses DataFrame caching for better performance - -:::note -- On serverless clusters, the configured volume (from `metadata_config.volume`) is automatically used -- The volume must be created in the metadata catalog and schema specified in your `ReconcileMetadataConfig` -- Ensure you have the necessary permissions to write to the volume on serverless clusters -::: +| source1 | conn1 | row | recon_config_source1_conn1_row.json | +| source2 | conn2 | schema | recon_config_source2_conn2_schema.json | -### Execution -You can execute the reconciliation process by executing the below command in a notebook cell. +See [Configuration Reference](/docs/reconcile/configuration) for the full schema and examples. +### Required permissions -``` python -from databricks.labs.lakebridge import __version__ -from databricks.sdk import WorkspaceClient +The User configuring reconcile must have permission to: +- Create Data Warehouses +- `USE CONNECTION` on the source connection +- `USE CATALOG` and `CREATE SCHEMA` on the target catalog +- `CREATE VOLUME` if using a pre-existing schema -from databricks.labs.lakebridge.reconcile.trigger_recon_service import TriggerReconService -from databricks.labs.lakebridge.reconcile.exception import ReconciliationException +### Compute selection -ws = WorkspaceClient(product="lakebridge", product_version=__version__) +**The deployed reconcile job runs on serverless compute by default.** No cluster is created or managed for you. +To run the job on a classic cluster instead, create the cluster yourself and point +`job_overrides` at it in the reconcile config (`/.lakebridge/reconcile.yml`): -try: - result = TriggerReconService.trigger_recon( - ws = ws, - spark = spark, # notebook spark session - table_recon = table_recon, # previously created - reconcile_config = reconcile_config # previously created - ) - print(result.recon_id) - print(result) -except ReconciliationException as e: - recon_id = e.reconcile_output.recon_id - print(f" Failed : {recon_id}") - print(e) -except Exception as e: - print(e.with_traceback) - raise e - print(f"Exception : {str(e)}") +```yaml +job_overrides: + existing_cluster_id: "0714-000000-abcdefgh" ``` -For more details, refer to the [Reconcile Notebook](recon_notebook.mdx) documentation. +:::note +A classic cluster must run **Databricks Runtime 17.3 or above** that supports reading from Unity Catalog connections. +::: + +### Intermediate data persistence + +Reconcile automatically adapts to the compute type: +- **Serverless (default):** Uses Unity Catalog volumes for intermediate data persistence — the configured volume (from `metadata_config.volume`) is automatically used. The volume must exist in the metadata catalog and schema specified in your `ReconcileMetadataConfig`, and you need write permission on it. +- **Classic clusters:** Uses DataFrame caching for better performance + +--- + +## Run + +See [Running Reconcile](/docs/reconcile/running) for CLI execution, notebook usage, and automation. diff --git a/docs/lakebridge/docs/reconcile/recon_notebook.mdx b/docs/lakebridge/docs/reconcile/recon_notebook.mdx deleted file mode 100644 index 2e3b6de0c4..0000000000 --- a/docs/lakebridge/docs/reconcile/recon_notebook.mdx +++ /dev/null @@ -1,260 +0,0 @@ ---- -sidebar_position: 2 -title: Running Reconcile on Notebook ---- - -import useBaseUrl from '@docusaurus/useBaseUrl'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - - -This page gives you a comprehensive guide on how to configure and run the Reconcile utility of Lakebridge on your Databricks -workspace using Databricks notebook. - -## Installation - -Before running the Reconcile utility, you need to install the required packages. You can install the required packages -using the following command: - - - - ```python - %pip install git+https://github.com/databrickslabs/lakebridge - dbutils.library.restartPython() - ``` - - - ```python - %pip install databricks-labs-lakebridge - dbutils.library.restartPython() - ``` - - - - -## Imports - -Import all the necessary modules. - -```python -from databricks.sdk import WorkspaceClient - -from databricks.labs.lakebridge.config import ( - DatabaseConfig, - ReconcileConfig, - ReconcileMetadataConfig, - TableRecon -) -from databricks.labs.lakebridge.reconcile.recon_config import ( - Table, - ColumnMapping, - ColumnThresholds, - Transformation, - JdbcReaderOptions, - Aggregate, - Filters -) -from databricks.labs.lakebridge.reconcile.trigger_recon_service import TriggerReconService -from databricks.labs.lakebridge.reconcile.trigger_recon_aggregate_service import TriggerReconAggregateService -from databricks.labs.lakebridge.reconcile.exception import ReconciliationException -``` - -## Configure Reconcile Properties - -We use the class `ReconcileConfig` to configure the properties required for reconciliation. - -```python -@dataclass -class ReconcileConfig: - data_source: str - report_type: str - secret_scope: str - database_config: DatabaseConfig - metadata_config: ReconcileMetadataConfig -``` -Parameters: - -- `data_source`: The data source to be reconciled. Supported values: `snowflake`, `oracle`, `mssql`, `synapse`, `databricks`. -- `report_type`: The type of report to be generated. Available report types are `schema`, `row`, `data` or `all`. For details check [here](./dataflow_example.mdx). -- `secret_scope`: The secret scope name used to store the connection credentials for the source database system. -- `database_config`: The database configuration for connecting to the source database. expects a `DatabaseConfig` object. - - `source_schema`: The source schema name. - - `target_catalog`: The target catalog name. - - `target_schema`: The target schema name (Databricks). - - `source_catalog`: The source catalog name. (Optional and is applied to the source system that support catalog). -```python -@dataclass -class DatabaseConfig: - source_schema: str - target_catalog: str - target_schema: str - source_catalog: str | None = None -``` -- `metadata_config`: The metadata configuration. Reconcile uses this catalog & Schema on Databricks to store all the backend -metadata details for reconciliation. expects a `ReconcileMetadataConfig` object. - - `catalog`: The catalog name to store the metadata. - - `schema`: The schema name to store the metadata. -```python -@dataclass -class ReconcileMetadataConfig: - catalog: str = "lakebridge" - schema: str = "reconcile" - volume: str = "reconcile_volume" -``` -If not set the default values will be used to store the metadata. The default resources are created during the installation -of Lakebridge. - -An Example of configuring the Reconcile properties: - -```python -from databricks.labs.lakebridge.config import ( - DatabaseConfig, - ReconcileConfig, - ReconcileMetadataConfig -) - -reconcile_config = ReconcileConfig( - data_source = "snowflake", - report_type = "all", - secret_scope = "snowflake-credential", - database_config= DatabaseConfig(source_catalog="source_sf_catalog", - source_schema="source_sf_schema", - target_catalog="target_databricks_catalog", - target_schema="target_databricks_schema" - ), - metadata_config = ReconcileMetadataConfig( - catalog = "lakebridge_metadata", - schema= "reconcile" - ) - ) -``` - -## Configure Table Properties - -We use the class `TableRecon` to configure the tables to be reconciled. - -```python -@dataclass -class TableRecon: - tables: list[Table] -``` - -:::note -Schema and catalog information are configured using `DatabaseConfig` inside `ReconcileConfig` (see above). -::: - -An Example Table properties for reconciliation: - -```python -from databricks.labs.lakebridge.config import TableRecon -from databricks.labs.lakebridge.reconcile.recon_config import ( - Table, - ColumnMapping, - ColumnThresholds, - TableThresholds, - Transformation, - JdbcReaderOptions, - Aggregate, - Filters -) - -table_recon = TableRecon( - tables=[ - Table( - source_name="source_table_name", - target_name="target_table_name", - join_columns= ["store_id", "account_id"], # List of columns to join the source and target tables. - column_mapping=[ - ColumnMapping(source_name="dept_id", target_name="department_id"), - ColumnMapping(source_name="cty_cd", target_name="country_code") - ], - column_thresholds=[ - ColumnThresholds(column_name="unit_price", upper_bound="-5", lower_bound="5", type="float") - ], - table_thresholds=[ - TableThresholds(lower_bound="0%", upper_bound="5%", model="mismatch") - ], - transformations=[ - Transformation( - column_name= 'inventory_units', - source= "coalesce(cast( cast(inventory_units as decimal(38,10)) as string),'_null_recon_')", - target= 'coalesce(replace(cast(format_number(cast(inventory_units as decimal(38, 10)), 10) as string),",", ""),"_null_recon_")', - ) - , - Transformation( - column_name= 'scanout_dollars', - source= "coalesce(cast( cast(scanout_dollars as decimal(38,10)) as string) ,'_null_recon_')", - target= 'coalesce(replace(cast(format_number(cast(scanout_dollars as decimal(38, 10)), 10) as string),",", ""),"_null_recon_")', - ) - ], - jdbc_reader_options=JdbcReaderOptions( - number_partitions=50, - partition_column="lct_nbr", - lower_bound="1", - upper_bound="50000", - ), - aggregates=[ - Aggregate(agg_columns=["inventory_units"], type="count"), - Aggregate(agg_columns=["unit_price"], type="min"), - Aggregate(agg_columns=["unit_price"], type="max") - ], - filters= Filters( - source="lower(dept_name)='finance'", - target="lower(department_name)='finance'") - ) - ] -) - -``` -## Configure Dashboards using Notebook - -The recommended way is to use the CLI following the instructions [here](/installation.mdx#configure-reconcile). -However, if you want to configure the metadata tables and dashboards using a notebook, you can use the following notebook. - - - Open notebook in new tab - - -## Run Reconcile - -To run Reconcile on the configured properties, use the `recon` method. you also need to pass a `WorkspaceClient` to `recon`. - -```python -from databricks.labs.lakebridge import __version__ -from databricks.sdk import WorkspaceClient - -from databricks.labs.lakebridge.reconcile.trigger_recon_service import TriggerReconService -from databricks.labs.lakebridge.reconcile.exception import ReconciliationException - -ws = WorkspaceClient(product="lakebridge", product_version=__version__) - - -try: - result = TriggerReconService.trigger_recon( - ws = ws, - spark = spark, # notebook spark session - table_recon = table_recon, # previously created - reconcile_config = reconcile_config # previously created - ) - print(result.recon_id) - print(result) - print("***************************") -except ReconciliationException as e: - recon_id = e.reconcile_output.recon_id - print(f" Failed : {recon_id}") - print(e) - print("***************************") -except Exception as e: - print(e.with_traceback) - raise e - print(f"Exception : {str(e)}") - print("***************************") - -``` - -## Visualization - -When you [install](../installation.mdx) Lakebridge using databricks cli, it deploys an AI/BI Dashboard on your workspace. -This dashboard gives you a detailed report of all the reconciliation runs on your workspace. After every reconciliation run, you get a -`recon_id`. You can use this recon_id to drill down into the details of that particular reconciliation run on the dashboard. diff --git a/docs/lakebridge/docs/reconcile/reconcile_automation.mdx b/docs/lakebridge/docs/reconcile/reconcile_automation.mdx deleted file mode 100644 index 80fc668940..0000000000 --- a/docs/lakebridge/docs/reconcile/reconcile_automation.mdx +++ /dev/null @@ -1,129 +0,0 @@ ---- -sidebar_position: 5 -title: Reconcile Automation ---- - -import useBaseUrl from '@docusaurus/useBaseUrl'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - -## Overview - -The purpose of this utility is to automate table reconciliation based on provided table configurations. -- It ensures a streamlined comparison of tables, applying necessary transformations and computing reconciliation results efficiently. -- The utility also provides lookup tables which can be configured to provide custom inputs including - - the source/target tables - - transformations to be applied, - - thresholds to be set - -## Pre-requisites - -- The Lakebridge Recon tool should be configured through CLI to create the catalog (the name of the catalog can be customized during installation) -- A volume is created inside `.` -- Ensure `table_configs` table is created inside `.` with the below DDL. This table will store the configs for the tables that needs to be validated. -```sql -CREATE TABLE ..table_configs ( - label STRING, - source_catalog STRING, - source_schema STRING, - source_table STRING, - databricks_catalog STRING, - databricks_schema STRING, - databricks_table STRING, - primary_key ARRAY, - source_filters STRING, - databricks_filters STRING, - federated_source_catalog STRING, - select_columns STRING, - drop_columns STRING -) USING DELTA; -``` - -| Column Name | Description | -|------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **label** | The label to be used for grouping validation runs. | -| **source_catalog** | The source catalog name. | -| **source_schema** | The source schema name. | -| **source_table** | The source table name. | -| **databricks_catalog** | The databricks catalog name. | -| **databricks_schema** | The databricks schema name. | -| **databricks_table** | The databricks table name. | -| **primary_key** | `` The primary key columns to be used as an array. This will initiate the lakebridge reconcile job to run the comparison including primary keys. If unspecified runs a row level comparison. | -| **source_filters** | `` The filters to be applied on the source table. This will initiate the lakebridge reconcile job to run the comparison including filters. | -| **databricks_filters** | `` The filters to be applied on the databricks table. This will initiate the lakebridge reconcile job to run the comparison including filters. | -| **federated_source_catalog** | `` The federated source catalog name, if applicable to pull some metadata for the table references | -| **select_columns** | `` The columns to be selected from the source table. This will initiate the lakebridge reconcile job to run the comparison including selected columns. | -| **drop_columns** | `` The columns to be dropped from the source table. This will initiate the lakebridge reconcile job to run the comparison including dropped columns. | - -- Ensure `table_recon_summary` table is created inside `.` with the below DDL. This table will store the summary results of the validated tables. -```sql -CREATE TABLE ..table_recon_summary ( - timestamp TIMESTAMP, - label STRING, - databricks_catalog STRING, - databricks_schema STRING, - databicks_table STRING, - status STRING, - recon_id STRING, - row_status STRING, - column_status STRING, - schema_status STRING, - error STRING -) USING DELTA; -``` -| Column Name | Description | -|------------------------|-----------------------------------------------------| -| **timestamp** | The timestamp when the validation was run. | -| **label** | The label to be used for grouping validation runs. | -| **databricks_catalog** | The databricks catalog name. | -| **databricks_schema** | The databricks schema name. | -| **databricks_table** | The databricks table name. | -| **status** | The status of the validation. | -| **recon_id** | The reconciliation ID generated for the validation. | -| **row_status** | The status of the row level validation. | -| **column_status** | The status of the column level validation. | -| **schema_status** | The status of the schema level validation. | -| **error** | The error message, if any, during the validation. | - -## [Notebook Details](#notebook-details) - -import LakebridgeTabs from '@site/src/components/ReconcileTabs'; - - - -Link to the notebook - Unzip the downloaded file and upload the notebook file to your Databricks workspace. - -The utility consists of three key Databricks notebooks: - -#### recon_wrapper_nb: -- Acts as the main orchestrator. -- Reads the table configurations and triggers reconciliation for each table. -#### lakebridge_recon_main: -- Core reconciliation utility. -- Performs row, column and schema level comparisons. -- Computes reconciliation ID, status, and results. -#### transformation_query_generator: -- A source system-specific transformation script. -- Applies transformations based on source and databricks column data types. -- Enables efficient hash computation for reconciliation. -- This is a variable script based on the customer's source system. The one provided in the repository is for snowflake as the source system. - -## Parameters to Configure -To run the utility, the following parameters must be set: - -- `label`: The label from the table table_configs which will be used as a filter for validating only selected tables with the specific label. -- `remorph_catalog`: The catalog configured through CLI. -- `remorph_schema`: The schema configured through CLI. -- `remorph_config_table`: The table configs created as a part of the pre-requisites. -- `secret_scope`: The Databricks secret scope for accessing the source system. Refer to the Lakebridge documentation for the specific keys required to be configured as per the source system. -- `source_system`: The source system against which reconciliation is performed. -- `table_recon_summary`: The target summary table created as a part of the pre-requisites. - -## Points to Note - -The notebook `_transformation_query_generator` needs to be created or modified as per the customer's source system -The following rules are applied while performing validation. This can be customized as per customer's use case by doing the necessary changes in the [notebook](#notebook-details) -- Given that the data types across platform would vary and fail the schema level checks, the overall validation will still be marked as passed based on the row and column level checks. -- In case the primary keys are not configured in the table_configs, only row level checks will be performed to pass the validation. - diff --git a/docs/lakebridge/docs/reconcile/reconcile_configuration.mdx b/docs/lakebridge/docs/reconcile/reconcile_configuration.mdx deleted file mode 100644 index 16cebaba1a..0000000000 --- a/docs/lakebridge/docs/reconcile/reconcile_configuration.mdx +++ /dev/null @@ -1,763 +0,0 @@ ---- -sidebar_position: 1 -title: Configuration ---- -import useBaseUrl from '@docusaurus/useBaseUrl'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - -## Types of Report Supported - -| report type | sample visualisation | description | key outputs captured in the recon metrics tables | -|-------------|--------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **schema** | [schema](../dataflow_example#schema) | reconcile the schema of source and target.
- validate the datatype is same or compatible | - **schema_comparison**
- **schema_difference** | -| **row** | [row](../dataflow_example#row) | reconcile the data only at row level(hash value of the source row is matched with the hash value of the target).Preferred when there are no join columns identified between source and target. | - **missing_in_src**(sample rows that are available in target but missing in source + sample rows in the target that don't match with the source)
- **missing_in_tgt**(sample rows that are available in source but are missing in target + sample rows in the source that doesn't match with target)
**NOTE**: the report won't differentiate the mismatch and missing here. | -| **data** | [data](../dataflow_example#data) | reconcile the data at row and column level- ```join_columns``` will help us to identify mismatches at each row and column level | - **mismatch_data**(the sample data with mismatches captured at each column and row level )
- **missing_in_src**(sample rows that are available in target but missing in source)
- **missing_in_tgt**(sample rows that are available in source but are missing in target)
- **threshold_mismatch**(configured column will be reconciled based on percentile or threshold boundary or date boundary)
- **mismatch_columns**(consolidated list of columns that has mismatches in them)
| -| **all** | [all](../dataflow_example#all) | this is a combination of data + schema | - **data + schema outputs** | - -[[back to top](#types-of-report-supported)] - -## Report Type-Flow Chart ---- -```mermaid -flowchart TD - REPORT_TYPE --> DATA - REPORT_TYPE --> SCHEMA - REPORT_TYPE --> ROW - REPORT_TYPE --> ALL -``` - ---- - -```mermaid -flowchart TD - SCHEMA --> SCHEMA_VALIDATION -``` ---- - -```mermaid -flowchart TD - ROW --> MISSING_IN_SRC - ROW --> MISSING_IN_TGT -``` ---- - -```mermaid -flowchart TD - DATA --> MISMATCH_ROWS - DATA --> MISSING_IN_SRC - DATA --> MISSING_IN_TGT -``` ---- - -```mermaid -flowchart TD - ALL --> MISMATCH_ROWS - ALL --> MISSING_IN_SRC - ALL --> MISSING_IN_TGT - ALL --> SCHEMA_VALIDATION -``` ---- - -[[back to top](#types-of-report-supported)] - -## Supported Source System - -| Source | Schema | Row | Data | All | -|-------------------------------|--------|-----|------|-----| -| Oracle | Yes | Yes | Yes | Yes | -| Snowflake | Yes | Yes | Yes | Yes | -| MS SQL Server (incl. Synapse) | Yes | Yes | Yes | Yes | -| Databricks | Yes | Yes | Yes | Yes | - -[[back to top](#types-of-report-supported)] - -## TABLE Config Json filename: -The config file must be named as `recon_config_[DATA_SOURCE]_[SOURCE_CATALOG_OR_SCHEMA]_[REPORT_TYPE].json` and should be placed in the Lakebridge root directory `.lakebridge` within the Databricks Workspace. - -> The filename pattern would remain the same for all the data_sources. - -``` shell -recon_config_${DATA_SOURCE}_${SOURCE_CATALOG_OR_SCHEMA}_${REPORT_TYPE}.json -``` - -Please find the `Table Recon` filename examples below for the `Snowflake`, `Oracle`, `` and `Databricks` source systems. - - - - ``` yaml title="recon_config_snowflake_sample_data_all.json" - database_config: - source_catalog: sample_data - source_schema: default - ... - metadata_config: - ... - data_source: snowflake - report_type: all - ... - ``` - - - ``` yaml title="recon_config_oracle_orc_data.json" - database_config: - source_schema: orc - ... - metadata_config: - ... - data_source: oracle - report_type: data - ... - ``` - - - - ``` yaml title="recon_config_tsql_silver_data.json" - database_config: - source_schema: silver - ... - metadata_config: - ... - data_source: mssql - report_type: data - ... - ``` - - - - ``` yaml title="recon_config_databricks_hms_schema.json" - database_config: - source_schema: hms - ... - metadata_config: - ... - data_source: databricks - report_type: schema - ... - ``` - - - - -> **Note:** the filename must be created in the same case as [SOURCE_CATALOG_OR_SCHEMA] is defined. -> For example, if the source schema is defined as `ORC` in the `reconcile` config, the filename should be `recon_config_oracle_ORC_data.json`. - - -[[back to top](#types-of-report-supported)] - - - -## TABLE Config Elements: - - - - ```python - @dataclass - class Table: - source_name: str - target_name: str - aggregates: list[Aggregate] | None = None - join_columns: list[str] | None = None - jdbc_reader_options: JdbcReaderOptions | None = None - select_columns: list[str] | None = None - drop_columns: list[str] | None = None - column_mapping: list[ColumnMapping] | None = None - transformations: list[Transformation] | None = None - column_thresholds: list[ColumnThresholds] | None = None - filters: Filters | None = None - table_thresholds: list[TableThresholds] | None = None - ``` - - - ```json - { - "source_name": "[SOURCE_NAME]", - "target_name": "[TARGET_NAME]", - "aggregates": null, - "join_columns": ["COLUMN_NAME_1","COLUMN_NAME_2"], - "jdbc_reader_options": null, - "select_columns": null, - "drop_columns": null, - "column_mapping": null, - "transformation": null, - "column_thresholds": null, - "filters": null, - "table_thresholds": null - } - ``` - - - - - -### Configuration Reference - -| Configuration Parameter | Data Type | Description | Requirement | Example Value | -|-------------------------|--------------------------|--------------------------------------------------------------------------------|-------------|---------------------------------------------------------------------------------------------------------------------------------------------------------| -| `source_name` | `string` | Source table name | Required | {`{"source_name": "product"}`} | -| `target_name` | `string` | Target table name | Required | {`{"target_name": "product"}`} | -| `aggregates` | `list[Aggregate]` | List of aggregation rules (see [Aggregate](#aggregate)) | Optional | {`{"aggregates": {"type": "MAX", "agg_columns": ["price"]}}`} | -| `join_columns` | `list[string]` | Primary key columns | Optional | {`{"join_columns": ["product_id", "order_id"]}`} | -| `jdbc_reader_options` | `object` | JDBC read parallelization configuration | Optional | {`{"jdbc_reader_options": {"number_partitions": 10, "partition_column": "id", "fetch_size": 1000}}`} | -| `select_columns` | `list[string]` | Columns to include in reconciliation | Optional | {`{"select_columns": ["id", "name", "price"]}`} | -| `drop_columns` | `list[string]` | Columns to exclude from reconciliation | Optional | {`{"drop_columns": ["temp_sku"]}`} | -| `column_mapping` | `list[ColumnMapping]` | Source-target column mapping (see [column_mapping](#column-mapping)) | Optional | {`{"column_mapping": {"source_name": "id", "target_name": "product_id"}}`} | -| `transformations` | `list[Transformations]` | Column transformation rules (see [transformations](#transformations)) | Optional | {`{"transformations": {"column_name": "address", "source": "TRIM(address)", "target": "LOWER(TRIM(address))"}}`} | -| `column_thresholds` | `list[ColumnThresholds]` | Column-level variance thresholds (see [column_thresholds](#column-thresholds)) | Optional | {`{"column_thresholds": {"column_name": "price", "lower_bound": "-5%", "upper_bound": "+10%"}}`} | -| `table_thresholds` | `list[TableThresholds]` | Table-level mismatch thresholds (see [table_thresholds](#table-thresholds)) | Optional | {`{"table_thresholds": {"model": "mismatch", "lower_bound": "0%", "upper_bound": "5%"}}`} | -| `filters` | `Filters` | Source/target filter expressions | Optional | {`{"filters": {"source": "quantity > 100", "target": "stock_quantity >= 100"}}`} | - -[[back to top](#types-of-report-supported)] - -### JDBC Reader Options - - - - ```python - @dataclass - class JdbcReaderOptions: - number_partitions: int - partition_column: str - lower_bound: str - upper_bound: str - fetch_size: int = 100 - ``` - - - ```json - "jdbc_reader_options": { - "number_partitions": "", - "partition_column": "", - "lower_bound": "", - "upper_bound": "", - "fetch_size": "" - } - ``` - - - -| field_name | data_type | description | required/optional | example_value | -|-------------------|-----------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------|---------------| -| number_partitions | string | the number of partitions for reading input data in parallel | required | "200" | -| partition_column | string | Int/date/timestamp parameter defining the column used for partitioning, typically the primary key of the source table. Note that this parameter accepts only one column, which is especially crucial when dealing with a composite primary key. In such cases, provide the column with higher cardinality. | required | "employee_id" | -| upper_bound | string | integer or date or timestamp without time zone value as string), that should be set appropriately (usually the maximum value in case of non-skew data) so the data read from the source should be approximately equally distributed | required | "1" | -| lower_bound | string | integer or date or timestamp without time zone value as string), that should be set appropriately (usually the minimum value in case of non-skew data) so the data read from the source should be approximately equally distributed | required | "100000" | -| fetch_size | string | This parameter influences the number of rows fetched per round-trip between Spark and the JDBC database, optimising data retrieval performance. Adjusting this option significantly impacts the efficiency of data extraction, controlling the volume of data retrieved in each fetch operation. More details on configuring fetch size can be found [here](https://docs.databricks.com/en/connect/external-systems/jdbc.html#control-number-of-rows-fetched-per-query) | optional(default="100") | "10000" | - - -:::tip -#### Key Considerations for Oracle JDBC Reader Options: -For Oracle source, the following Spark Configurations are automatically set: -```json -"oracle.jdbc.mapDateToTimestamp": "False", -"sessionInitStatement": "BEGIN dbms_session.set_nls('nls_date_format', '''YYYY-MM-DD''');dbms_session.set_nls('nls_timestamp_format', '''YYYY-MM-DD HH24:MI:SS''');END;" -``` - -While configuring Recon for Oracle source, the above options should be taken into consideration. -::: - -[[back to top](#types-of-report-supported)] - -## Column Mapping - - - - ```python - @dataclass - class ColumnMapping: - source_name: str - target_name: str - ``` - - - ```json - "column_mapping": [ - { - "source_name": "", - "target_name": "" - } - ] - ``` - - - -| field_name | data_type | description | required/optional | example_value | -|-------------|-----------|--------------------|-------------------|-----------------| -| source_name | string | source column name | required | "dept_id" | -| target_name | string | target column name | required | "department_id" | - -[[back to top](#types-of-report-supported)] - - -## Drop Columns - -For Recon Type `all` or `data`, the `drop_columns` parameter is used to exclude columns from the reconciliation process. -using this config, you can specify the columns that you want to exclude from the reconciliation process. - - - - ```python - Table( - drop_columns = ["column_name1", "column_name2"] - ) - ``` - - - ```json - { - "drop_columns": [ "column_name1", "column_name2"] - } - ``` - - - -[[back to top](#types-of-report-supported)] - - -## Transformations - -You can apply custom transformations to the columns using the `transformations` parameter. If applied, Reconcile -uses these transformations to fetch the data from source and/or target before comparing them. You can write SQL -expressions in `source` & `target` fields to apply transformations. - -The class detail: -```python -@dataclass -class Transformation: - column_name: str - source: str - target: str | None = None - -``` - -Syntax for applying transformations: - - - - ```python - Table( - transformations = [ - Transformation( - column_name = "unit_price", - source = "coalesce(cast(cast(unit_price as decimal(38,10)) as string), '_null_recon_')", - target = "coalesce(cast(format_number(cast(unit_price as decimal(38, 10)), 10) as string), '_null_recon_')" - ) - ] - ) - ``` - - - ```json - "transformations": [ - { - "column_name": "[COLUMN_NAME]", - "source": "[TRANSFORMATION_EXPRESSION]", - "target": "[TRANSFORMATION_EXPRESSION]" - } - ] - ``` - - - -| field_name | data_type | description | required/optional | example_value | -|-------------|-----------|------------------------------------------------------------|-------------------|----------------------------------| -| column_name | string | the column name on which the transformation to be applied | required | "s_address" | -| source | string | the transformation sql expr to be applied on source column | required | "trim(s_address)" or "s_address" | -| target | string | the transformation sql expr to be applied on source column | required | "trim(s_address)" or "s_address" | - - - -:::note -Reconciliation also takes an udf in the transformation expr.Say for eg. we have a udf named sort_array_input() that takes an unsorted array as input and returns an array sorted.We can use that in transformation as below: -``` python -transformations=[Transformation(column_name)="array_col",source=sort_array_input(array_col),target=sort_array_input(array_col)] -``` -`NULL` values are defaulted to `_null_recon_` using the transformation expressions in these files: - -1. [expression_generator.py](https://github.com/databrickslabs/remorph/tree/main/src/databricks/labs/remorph/reconcile/query_builder/expression_generator.py) -2. [sampling_query.py](https://github.com/databrickslabs/remorph/tree/main/src/databricks/labs/remorph/reconcile/query_builder/sampling_query.py). -If User is looking for any specific behaviour, they can override these rules using [transformations](#transformations) accordingly. -::: - -:::danger -### Handling Nulls - -While applying transformations, make sure you handle the nulls explicitly in the transformation column. -While Reconcile takes care of null handling for all the other columns (including join keys and other keys that are not -included in drop_column list), when users introduce transformation columns, Reconcile uses those expressions as is. -So you would have to include null handling in the transformation expression itself. - -So if you are planning on using the below expression: -``` sql -cast(cast(scanout_units as decimal(38,10)) as string) -``` - -Use the below expression instead: - -``` sql -coalesce(cast(cast(scanout_units as decimal(38,10)) as string), '_null_recon_') -``` - - -### Handling Timestamp Columns - -Different systems handle timestamps differently. During Reconciliation it is important that we apply necessary transformation to the timestamp columns -on both source and target to make sure that they are reconciled correctly. A recommended approach to dealing with timestamps is to convert them to -corresponding `unix epoch` string values. SQL Functions like `epoch_millisecond` (Snowflake), [unix_millis](https://docs.databricks.com/aws/en/sql/language-manual/functions/unix_millis#:~:text=Returns%20the%20number%20of%20milliseconds,00%3A00%3A00%20UTC%20.) (Databricks) become very handy to transform -those timestamp values to unix epoch values. Then they can be converted to string which is safer when it comes to comparing timestamps across different systems. - - - - ```python - Transformation( - column_name= 'UPDATE_TIMESTAMP', - source= "coalesce(cast(EXTRACT(epoch_millisecond FROM UPDATE_TIMESTAMP) as string), '_null_recon_')", - target= "coalesce(cast(unix_millis(UPDATE_TIMESTAMP) as string), '_null_recon_')" - ), - ``` - - - ```json - "transformations": [ - { - "column_name": "UPDATE_TIMESTAMP", - "source": "coalesce(cast(EXTRACT(epoch_millisecond FROM UPDATE_TIMESTAMP) as string), '_null_recon_')", - "target": "coalesce(cast(unix_millis(UPDATE_TIMESTAMP) as string), '_null_recon_')" - } - ] - ``` - - -::: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Transformation Expressions
filenamefunction / variabletransformation_ruledescription
sampling_query.py_get_join_clausetransform(coalesce, default="_null_recon_", is_string=True)Applies the coalesce transformation function for String column and defaults to `_null_recon_` if column is NULL
expression_generator.pyDataType_transform_mapping(coalesce, default='_null_recon_', is_string=True)Default String column Transformation rule for all dialects. Applies the coalesce transformation function and defaults to `_null_recon_` if column is NULL
expression_generator.pyDataType_transform_mapping"oracle": DataType...NCHAR: ..."NVL(TRIM(TO_CHAR..,'_null_recon_')"Transformation rule for oracle dialect 'NCHAR' datatype. Applies TO_CHAR, TRIM transformation functions. If column is NULL, then defaults to `_null_recon_`
expression_generator.pyDataType_transform_mapping"oracle": DataType...NVARCHAR: ..."NVL(TRIM(TO_CHAR..,'_null_recon_')"Transformation rule for oracle dialect 'NVARCHAR' datatype. Applies TO_CHAR, TRIM transformation functions. If column is NULL, then defaults to `_null_recon_`
- -:::tip - -If you are applying any transformation for generating the Reconciliation report, it is recommended that you run -the queries with transformation expressions both on Source and target to ensure, they are syntactically and semantically -giving correct result. - -If you need help with the exact reconciliation queries that are getting generated after you apply the transformations, -you may search the reconciliation log with the following string: - -`“Hash query for [source or target]”` - -::: - -[[back to Transformations](#transformations)] - -[[back to top](#types-of-report-supported)] - -## Column Thresholds - - - - ```python - @dataclass - class ColumnThresholds: - column_name: str - lower_bound: str - upper_bound: str - type: str - ``` - - - ```json - "column_thresholds": [ - { - "column_name": "COLUMN_NAME", - "lower_bound": "LOWER_BOUND", - "upper_bound": "UPPER_BOUND", - "type": "DATA_TYPE" - } - - ] - ``` - - - - -| field_name | data_type | description | required/optional | example_value | -|-------------|-----------|-----------------------------------------------------------------------------------------------------------------|-------------------|--------------------| -| column_name | string | the column that should be considered for column threshold reconciliation | required | "product_discount" | -| lower_bound | string | the lower bound of the difference between the source value and the target value | required | "-5%" | -| upper_bound | string | the upper bound of the difference between the source value and the target value | required | "5%" | -| type | string | The user must specify the column type. Supports SQLGLOT `DataType.NUMERIC_TYPES` and `DataType.TEMPORAL_TYPES`. | required | "int" | - -[[back to top](#types-of-report-supported)] - -## Table Thresholds - - - - ```python - @dataclass - class TableThresholds: - lower_bound: str - upper_bound: str - model: str - ``` - - - ```json - "table_thresholds": [ - { - "lower_bound": "LOWER_BOUND", - "upper_bound": "UPPER_BOUND", - "model": "MODEL" - } - ] - ``` - - - -* The threshold bounds for the table must be non-negative, with the lower bound not exceeding the upper bound. - -| field_name | data_type | description | required/optional | example_value | -|-------------|-----------|------------------------------------------------------------------------------------------------------|-------------------|---------------| -| lower_bound | string | the lower bound of the difference between the source mismatch and the target mismatch count | required | 0% | -| upper_bound | string | the upper bound of the difference between the source mismatch and the target mismatch count | required | 5% | -| model | string | The user must specify on which table model it should be applied; for now, we support only "mismatch" | required | int | - -[[back to top](#types-of-report-supported)] - -## Filters - - - - ```python - @dataclass - class Filters: - source: str - target: str - ``` - - - ```json - "filters": { - "source": "FILTER_EXPRESSION", - "target": "FILTER_EXPRESSION" - } - ``` - - - -| field_name | data_type | description | required/optional | example_value | -|------------|-----------|---------------------------------------------------|------------------------|--------------------------------------------------------------------| -| source | string | the sql expression to filter the data from source | optional(default=None) | "lower(dept_name)='finance'" | -| target | string | the sql expression to filter the data from target | optional(default=None) | "lower(dept_name)='finance'" | - - - - -:::note Key Considerations: - -1. The column names are always converted to lowercase and considered for reconciliation. -2. Currently, it doesn't support case insensitivity and doesn't have collation support -3. Table Transformation internally considers the default value as the column value. It doesn't apply any default -transformations -if not provided. -```eg:Transformation(column_name="address",source_name=None,target_name="trim(s_address)")``` -For the given example, -the source transformation is None, so the raw value in the source is considered for reconciliation. -4. If no user transformation is provided for a given column in the configuration by default, depending on the source -data -type, our reconciler will apply -default transformation on both source and target to get the matching hash value in source and target. Please find the -detailed default transformations here. -5. Always the column reference to be source column names in all the configs, except **Transformations** and **Filters** -as these are dialect-specific SQL expressions that are applied directly in the SQL. -6. **Transformations** and **Filters** should always be in their respective dialect SQL expressions, and the -reconciler will not apply any logic -on top of this. -::: - -[[back to top](#types-of-report-supported)] - -## Aggregates Reconciliation - -Aggregates Reconcile is an utility to streamline the reconciliation process, specific aggregate metric is compared -between source and target data residing on Databricks. - -### Summary - -| operation_name | sample visualisation | description | key outputs captured in the recon metrics tables | -|--------------------------|-------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **aggregates-reconcile** | `[data]({useBaseUrl('docs/reconcile/aggregates_reconcile_visualisation.md#data')})` | reconciles the data for each aggregate metric `join_columns` are used to identify the mismatches at aggregated metric level | mismatch_data(sample data with mismatches captured at aggregated metric level ) missing_in_src(sample rows that are available in target but missing in source) missing_in_tgt(sample rows that are available in source but are missing in target) | - - -### Supported Aggregate Functions - - -| Aggregate Functions | -|-----------------------------------------------------------------------------------------------------------------------------------------------------------| -| **min** | -| **max** | -| **count** | -| **sum** | -| **avg** | -| **mean** | -| **mode** | -| **stddev** | -| **variance** | -| **median** | - - - -[[back to aggregates-reconciliation](#aggregates-reconciliation)] - -[[back to top](#types-of-report-supported)] - -### Flow Chart - -```mermaid -flowchart TD - Aggregates-Reconcile --> MISMATCH_ROWS - Aggregates-Reconcile --> MISSING_IN_SRC - Aggregates-Reconcile --> MISSING_IN_TGT -``` - - -[[back to aggregates-reconciliation](#aggregates-reconciliation)] - -[[back to top](#types-of-report-supported)] - - -### Aggregate - - - - ```python - @dataclass - class Aggregate: - agg_columns: list[str] - type: str - group_by_columns: list[str] | None = None - ``` - - - ```json - { - "type": "MIN", - "agg_columns": ["COLUMN_NAME_3"], - "group_by_columns": ["GROUP_COLUMN_NAME"] - } - ``` - - - -| field_name | data_type | description | required/optional | example_value | -|------------------|--------------|-----------------------------------------------------------------------|------------------------|------------------------| -| type | string | [Supported Aggregate Functions](#supported-aggregate-functions) | required | MIN | -| agg_columns | list[string] | list of columns names on which aggregate function needs to be applied | required | ["product_discount"] | -| group_by_columns | list[string] | list of column names on which grouping needs to be applied | optional(default=None) | ["product_id"] or None | - - - -[[back to aggregates-reconciliation](#aggregates-reconciliation)] - -[[back to top](#types-of-report-supported)] - - -### TABLE Config Examples: -Please refer [TABLE Config Elements](#table-config-elements) for Class and JSON configs. - - - - ```python - Table( - source_name="SOURCE_NAME", - target_name="TARGET_NAME", - join_columns=["COLUMN_NAME_1", "COLUMN_NAME_2"], - aggregates=[ - Aggregate( - agg_columns=["COLUMN_NAME_3"], - type="MIN", - group_by_columns=["GROUP_COLUMN_NAME"] - ), - Aggregate( - agg_columns=["COLUMN_NAME_4"], - type="MAX" - ) - ] - ) - ``` - - - ```json - { - "source_name": "SOURCE_NAME", - "target_name": "TARGET_NAME", - "join_columns": ["COLUMN_NAME_1", "COLUMN_NAME_2"], - "aggregates": [ - { - "type": "MIN", - "agg_columns": ["COLUMN_NAME_3"], - "group_by_columns": ["GROUP_COLUMN_NAME"] - }, - { - "type": "MAX", - "agg_columns": ["COLUMN_NAME_4"] - } - ] - } - ``` - - - - -:::note -Key Considerations: - -1. The aggregate column names, group by columns and type are always converted to lowercase and considered for reconciliation. -2. Currently, it doesn't support aggregates on window function using the OVER clause. -3. It doesn't support case insensitivity and does not have collation support -4. The queries with “group by” column(s) are compared based on the same group by columns. -5. The queries without “group by” column(s) are compared row-to-row. -6. Existing features like `column_mapping`, `transformations`, `JDBCReaderOptions` and `filters` are leveraged for the aggregate metric reconciliation. -7. Existing `select_columns` and `drop_columns` are not considered for the aggregate metric reconciliation. -8. Even though the user provides the `select_columns` and `drop_columns`, those are not considered. -9. If Transformations are defined, those are applied to both the “aggregate columns” and “group by columns”. -::: - -[[back to aggregates-reconciliation](#aggregates-reconciliation)] - -[[back to top](#types-of-report-supported)] - - - diff --git a/docs/lakebridge/docs/reconcile/running.mdx b/docs/lakebridge/docs/reconcile/running.mdx new file mode 100644 index 0000000000..18a3b22097 --- /dev/null +++ b/docs/lakebridge/docs/reconcile/running.mdx @@ -0,0 +1,360 @@ +--- +sidebar_position: 2 +title: Running Reconcile +--- + +import useBaseUrl from '@docusaurus/useBaseUrl'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; + +# Running Reconcile + +This page covers how to run Lakebridge Reconcile via the CLI and via Databricks notebooks. For configuration options, see [Configuration Reference](/docs/reconcile/configuration). + +--- + +## CLI Execution + +After completing [setup](/docs/reconcile#setup), run reconciliation with: + +```bash +databricks labs lakebridge reconcile +``` + +Results are written to the reconciliation dashboard deployed in your workspace. + +--- + +## Notebook Execution + +Use the notebook approach when you need fine-grained control over reconcile configuration, or when running from within a Databricks notebook workflow. + +### Step 1: Install + + + + ```python + %pip install databricks-labs-lakebridge + dbutils.library.restartPython() + ``` + + + +### Step 2: Import + +```python +from databricks.sdk import WorkspaceClient + +from databricks.labs.lakebridge.config import ( + ReconcileConfig, + ReconcileMetadataConfig, + TableRecon, + SourceConnectionConfig, + TargetConnectionConfig +) +from databricks.labs.lakebridge.reconcile.recon_config import ( + Table, + ColumnMapping, + ColumnThresholds, + Transformation, + JdbcReaderOptions, + Aggregate, + Filters +) +from databricks.labs.lakebridge.reconcile.trigger_recon_service import TriggerReconService +from databricks.labs.lakebridge.reconcile.trigger_recon_aggregate_service import TriggerReconAggregateService +from databricks.labs.lakebridge.reconcile.exception import ReconciliationException +``` + +### Step 3: Configure ReconcileConfig + +#### __Before the actual example, some details about how to configure:__ +```python +class ReconcileConfig: + report_type: str + source: SourceConnectionConfig + target: TargetConnectionConfig + metadata_config: ReconcileMetadataConfig +``` +Parameters: + +- `report_type`: The type of report to be generated. Available report types are `schema`, `row`, `data` or `all`. For details check [here](./dataflow_example.mdx). +- `source`: The configuration for connecting to the source database to be reconciled. + - `dialect`: The dialect of the source. Supported values: `snowflake`, `oracle`, `mssql`, `synapse`, `databricks`, `redshift`, `teradata`. + - `catalog`: The source database/catalog name. catalog is used for consistency in naming + - `schema`: The source schema name. + - `uc_connection_name`: the connection name for the source as configured in workspace `Connections`. Not allowed for `databricks` + +```python +@dataclass +class SourceConnectionConfig: + dialect: str + catalog: str + schema: str + uc_connection_name: str | None = None +``` +- `target`: The specs of the target databricks catalog to be reconciled. + - `catalog`: The target catalog name. + - `schema`: The target schema name. +```python +@dataclass +class TargetConnectionConfig: + catalog: str + schema: str +``` +- `metadata_config`: The metadata configuration. Reconcile uses this catalog & Schema on Databricks to store all the backend +metadata details for reconciliation. expects a `ReconcileMetadataConfig` object. + - `catalog`: The catalog name to store the metadata. + - `schema`: The schema name to store the metadata. +```python +@dataclass +class ReconcileMetadataConfig: + catalog: str = "lakebridge" + schema: str = "reconcile" + volume: str = "reconcile_volume" +``` +If not set the default values will be used to store the metadata. The default resources are created during the installation +of Lakebridge. + + +#### __Now, an Example of configuring the ReconcileConfig properties that you can copy into the notebook:__ + +```python +reconcile_config = ReconcileConfig( + report_type="all", + source=SourceConnectionConfig( + dialect="snowflake", + catalog="source_sf_catalog", + schema="source_sf_schema", + uc_connection_name="source_connection_name" + ), + target=TargetConnectionConfig( + catalog="target_databricks_catalog", + schema="target_databricks_schema", + ), + metadata_config = ReconcileMetadataConfig( + catalog = "lakebridge_metadata", + schema= "reconcile" + ), + ) +``` + +### Step 4: Configure TableRecon + +```python +from databricks.labs.lakebridge.config import TableRecon +from databricks.labs.lakebridge.reconcile.recon_config import ( + Table, + ColumnMapping, + ColumnThresholds, + TableThresholds, + Transformation, + JdbcReaderOptions, + Aggregate, + Filters +) + +table_recon = TableRecon( + tables=[ + Table( + source_name="source_table_name", + target_name="target_table_name", + join_columns=["store_id", "account_id"], + column_mapping=[ + ColumnMapping(source_name="dept_id", target_name="department_id"), + ], + column_thresholds=[ + ColumnThresholds(column_name="unit_price", upper_bound="-5", lower_bound="5", type="float") + ], + table_thresholds=[ + TableThresholds(lower_bound="0%", upper_bound="5%", model="mismatch") + ], + transformations=[ + Transformation( + column_name="inventory_units", + source="coalesce(cast(cast(inventory_units as decimal(38,10)) as string), '_null_recon_')", + target='coalesce(replace(cast(format_number(cast(inventory_units as decimal(38, 10)), 10) as string), ",", ""), "_null_recon_")' + ) + ], + jdbc_reader_options=JdbcReaderOptions( + num_partitions=50, + partition_column="lct_nbr", + lower_bound="1", + upper_bound="50000" + ), + filters=Filters( + source="lower(dept_name)='finance'", + target="lower(department_name)='finance'" + ) + ) + ] +) +``` + +### Step 5: Run + +```python +from databricks.labs.lakebridge import __version__ +from databricks.sdk import WorkspaceClient +from databricks.labs.lakebridge.reconcile.trigger_recon_service import TriggerReconService +from databricks.labs.lakebridge.reconcile.exception import ReconciliationException + +ws = WorkspaceClient(product="lakebridge", product_version=__version__) + +try: + result = TriggerReconService.trigger_recon( + ws=ws, + spark=spark, + table_recon=table_recon, + reconcile_config=reconcile_config + ) + print(result.recon_id) + print(result) +except ReconciliationException as e: + print(f"Failed: {e.reconcile_output.recon_id}") + print(e) +``` + +### Visualization + +After running, use the `recon_id` to drill into the results on the AI/BI Dashboard deployed in your workspace during installation. + +--- + +## Auto-configure Table Mappings + +When you have many tables to reconcile, you can skip writing the `TableRecon` by hand. The `auto-configure-recon-tables` command discovers source/target table pairs by matching their names, fills in `column_mapping` where source and target column names differ, and drops unmatched columns, and uploads the resulting reconcile config file to your install folder. + +This is the automated alternative to [Step 4: Configure TableRecon](#step-4-configure-tablerecon) above. + +### Prerequisites + +- CLI users: `configure-reconcile` has been run (the reconcile config file must exist in your install folder). +- Python users: [Step 3: Configure ReconcileConfig](#step-3-configure-reconcileconfig) + +### Run + +```bash +databricks labs lakebridge auto-configure-recon-tables +``` + +The matcher normalizes tables and columns names (case, underscores vs. hyphens, simple plural/singular) so `Customers` matches `customers`, `Order_Items` matches `order-items`, `orders` matches `order`. +Source tables the matcher cannot pair with a target table are **omitted** from the draft and logged so you can add them manually. + +When the workspace job finishes, the resulting `recon_config_<...>.json` is uploaded to your install folder. + +:::danger +Running auto-configure **overwrites** existing config if the user approves. Apply manual edits *after* running auto-configure. +::: + +### Python + +The auto-configure module exposes two automation entry points: + +- `discover_tables(...)` — matches table pairs from the configured catalogs and schemas. +- `auto_configure_tables(table_recon, ...)` — apply all registered configurers (column mapping today; join keys, transformations, ... as they're added) to each Table in `table_recon` and return the result. + +Both take the `reconcile_config` from [Step 3](#step-3-configure-reconcileconfig) and return `TableRecon`, that can be passed to `TriggerReconService.trigger_recon` ([Step 5](#step-5-run)): + +```python +from databricks.labs.lakebridge.reconcile.config_generator.execute import ( + auto_configure_tables, + discover_tables, +) + +# `reconcile_config` from Step 3 above. +discovered = discover_tables(reconcile_config=reconcile_config, spark=spark) +table_recon = auto_configure_tables(discovered, reconcile_config=reconcile_config, spark=spark) + +for t in table_recon.tables: + print(t.source_name, "→", t.target_name) +``` + +### Recommended flow: discover, review, then auto-configure + +The recommended pattern is to split discovery from auto-configuration so you can review the discovered pairs before any column mappings are filled in: + +1. **Discover** — call `discover_tables(...)` and inspect the returned `TableRecon`. +2. **Review** — drop pairs you don't want to reconcile, fix names the matcher got wrong, and add tables it couldn't auto-match (look for the warning in the logs). +3. **Auto-configure** — pass the curated `TableRecon` to `auto_configure_tables(curated, ...)`. + +```python +discovered = discover_tables(reconcile_config=reconcile_config, spark=spark) + +# Review/curate in memory — or save the file, edit it in the workspace, and load it back. +curated = TableRecon(tables=[t for t in discovered.tables if t.source_name != "audit_log"]) + +table_recon = auto_configure_tables(curated, reconcile_config=reconcile_config, spark=spark) +``` + +:::note +From the CLI this is two runs of `auto-configure-recon-tables`: + +1. **First run** — no existing file. Answer **yes** to "Discover tables now?", then **no** to + "Also run auto-configure in the same job (skips the recommended review step)?". This emits a + discover-only draft for you to review. +2. **Second run** — existing file. Answer **yes** to "Auto-configure and use existing table + mappings (no discovery)?" — auto-configure applies its configurers to your curated draft. + +If you'd rather discover and auto-configure in one job (and skip the review step), accept the +"Also run auto-configure in the same job" prompt during the first run. +::: + +### Plugging in a custom configurer + +`auto_configure_tables` and `auto_configure_table` both accept an `auto_configurers` parameter (defaults to `SUPPORTED_AUTO_CONFIGURERS`). Pass your own list of `TableAutoConfigurer` implementations to extend or replace the defaults — e.g. an LLM-driven mapper, a join-key inferrer, or a transformation suggester: + +```python +from databricks.labs.lakebridge.reconcile.config_generator.configure import TableAutoConfigurer +from databricks.labs.lakebridge.reconcile.config_generator.execute import ( + SUPPORTED_AUTO_CONFIGURERS, + auto_configure_tables, +) + +class MyConfigurer(TableAutoConfigurer): + def configure(self, table, ctx): + ... # inspect ctx.source_columns / ctx.target_columns and return an updated Table + return table + +auto_configure_tables( + table_recon, + reconcile_config=reconcile_config, + spark=spark, + auto_configurers=[*SUPPORTED_AUTO_CONFIGURERS, MyConfigurer()], +) +``` + +Each configurer in the list runs in order on each table; later configurers see the table as updated by earlier ones via the `ctx` they share. + +### Auto-configure a single table + +To run all supported configurers against just one table — e.g. after a column was added on the source — call `auto_configure_table`: + +```python +from databricks.labs.lakebridge.reconcile.config_generator.execute import auto_configure_table +from databricks.labs.lakebridge.reconcile.recon_config import Table + +configured = auto_configure_table( + table=Table(source_name="orders", target_name="orders"), + reconcile_config=reconcile_config, + spark=spark, +) + +print(configured) +``` + +### Review the output + +Auto-discovery fills in table pairs, `column_mapping`, `select_columns` (matched source columns when any source column couldn't be paired), and `drop_columns` (unmatched target columns). The following fields are not auto-discovered and must be added manually for the tables that need them: + +- `join_columns` — required for `data` and `all` report types. +- `column_thresholds`, `table_thresholds` — numeric tolerance bounds. +- `transformations` — column-level SQL transforms. +- `filters` — source/target WHERE clauses. + +See [Configuration Reference](/docs/reconcile/configuration) for the full schema. + +:::note +This command is experimental. Review the output before running reconcile. +::: diff --git a/docs/lakebridge/docs/sql_splitter.mdx b/docs/lakebridge/docs/sql_splitter.mdx index f498fa1902..54b16fa604 100644 --- a/docs/lakebridge/docs/sql_splitter.mdx +++ b/docs/lakebridge/docs/sql_splitter.mdx @@ -1,5 +1,5 @@ --- -sidebar_position: 9 +sidebar_position: 7 title: SQL Splitter --- import useBaseUrl from '@docusaurus/useBaseUrl'; diff --git a/docs/lakebridge/docs/transpile/index.mdx b/docs/lakebridge/docs/transpile/index.mdx index ddd87568a0..dc369830b5 100644 --- a/docs/lakebridge/docs/transpile/index.mdx +++ b/docs/lakebridge/docs/transpile/index.mdx @@ -1,11 +1,35 @@ --- -sidebar_position: 4 +sidebar_position: 5 title: Transpile Guide --- import useBaseUrl from '@docusaurus/useBaseUrl'; # Transpile Guide +Lakebridge comes with three transpilation plugins: [Morpheus](/docs/transpile/pluggable_transpilers/morpheus), [BladeBridge](/docs/transpile/pluggable_transpilers/bladebridge), and [Switch](/docs/transpile/pluggable_transpilers/switch). Not sure which to use? See [Which Tool Do I Use?](/docs/choosing_tools#transpiler-bladebridge-vs-morpheus-vs-switch). + +## Transpiler Selection Guide + +Lakebridge offers both deterministic and LLM-powered transpilers, each optimized for different conversion scenarios. + +### Deterministic Conversion (BladeBridge & Morpheus) + +Deterministic transpilers excel in scenarios requiring consistency and speed: +- **Deterministic output with guaranteed syntax equivalence** — Every conversion produces the same predictable result +- **High-volume batch processing** — Efficiently handle thousands of files without API rate limits +- **Fast local execution without API dependencies** — Sub-minute processing with no external service calls +- **Production-grade SQL aligned with Databricks SQL evolution** — Leverages SQL Scripting, Stored Procedures, and latest DBSQL features + +### LLM-Powered Conversion (Switch) + +Switch is best suited for scenarios requiring semantic understanding and flexibility: +- **Complex logic requiring contextual understanding** — Stored procedures and business logic where intent matters more than syntax +- **Source formats not covered by deterministic transpilers** — Any SQL dialect or programming language through custom prompts +- **Extensible conversion through custom YAML prompts** — Adapt to proprietary or uncommon source formats +- **Python notebook output for SQL beyond ANSI SQL/PSM standards** — Complex transformations that benefit from procedural code + +--- + ## Supported dialects @@ -132,11 +156,28 @@ import useBaseUrl from '@docusaurus/useBaseUrl';
+## Execution Pre-Set Up +When you run `install-transpile`, you will be prompted for settings to use when transpiling your sources. You can +choose to provide these at the time of installation, or to provide them later as arguments when transpiling. + +The `transpile` command will trigger the conversion of the specified code. These settings provided during `install-transpile` can be overridden (or provided if unavailable) using the command-line options: + - `input-source`: The local filesystem path to the sources that should be transpiled. This must be provided if not set during `install-transpile`. + - `output-folder`: The local filesystem path where converted code will be written. This must be provided if not set during `install-transpile`. + - `source-dialect`: Dialect name (ex: snowflake, oracle, datastage, etc). This must be provided if not set during `install-transpile`. + - `overrides-file`: An optional local path to a JSON file containing custom overrides for the transpilation process, if the underlying transpiler supports this. (Refer to [this documentation](pluggable_transpilers/bladebridge/bladebridge_configuration) for more details on custom overrides.) + - `target-technology`: The target technology to use for conversion output. This must be provided if not set during `install-transpile` and the underlying transpiler requires it for the source dialect in use. +- `error-file-path`: The path to the file where a log of conversion errors will be stored. If not provided here or during `install-transpile` no error log will be written. + - `skip-validation`: Whether the transpiler will skip the validation of transpiled SQL sources. If not provided here or during `install-transpile` validation will be attempted by default. + - `catalog-name`: The name of the catalog in Databricks to use when validating transpiled SQL sources. If not specified, `remorph` will be used as the default catalog. + - `schema-name`: The name of the schema in Databricks to use when validating transpiled SQL sources. If not specified, `transpiler` will be used as the default schema. + - `transpiler-config-path`: This path of the configuration file for the transpiler to use for conversion. This is normally inferred from the source dialect or chosen during `install-transpile` if multiple transpilers support the source dialect. + + ## Verify Installation Verify the successful installation by executing the provided command; confirmation of a successful installation is indicated when the displayed output aligns with the example screenshot provided: Command: -```shell +```bash databricks labs lakebridge transpile --help ``` @@ -165,25 +206,9 @@ Global Flags: -t, --target string bundle target to use (if applicable) ``` -## Execution Pre-Set Up -When you run `install-transpile`, you will be prompted for settings to use when transpiling your sources. You can -choose to provide these at the time of installation, or to provide them later as arguments when transpiling. - -The `transpile` command will trigger the conversion of the specified code. These settings provided during `install-transpile` can be overridden (or provided if unavailable) using the command-line options: - - `input-source`: The local filesystem path to the sources that should be transpiled. This must be provided if not set during `install-transpile`. - - `output-folder`: The local filesystem path where converted code will be written. This must be provided if not set during `install-transpile`. - - `source-dialect`: Dialect name (ex: snowflake, oracle, datastage, etc). This must be provided if not set during `install-transpile`. - - `overrides-file`: An optional local path to a JSON file containing custom overrides for the transpilation process, if the underlying transpiler supports this. (Refer to [this documentation](pluggable_transpilers/bladebridge/bladebridge_configuration) for more details on custom overrides.) - - `target-technology`: The target technology to use for conversion output. This must be provided if not set during `install-transpile` and the underlying transpiler requires it for the source dialect in use. -- `error-file-path`: The path to the file where a log of conversion errors will be stored. If not provided here or during `install-transpile` no error log will be written. - - `skip-validation`: Whether the transpiler will skip the validation of transpiled SQL sources. If not provided here or during `install-transpile` validation will be attempted by default. - - `catalog-name`: The name of the catalog in Databricks to use when validating transpiled SQL sources. If not specified, `remorph` will be used as the default catalog. - - `schema-name`: The name of the schema in Databricks to use when validating transpiled SQL sources. If not specified, `transpiler` will be used as the default schema. - - `transpiler-config-path`: This path of the configuration file for the transpiler to use for conversion. This is normally inferred from the source dialect or chosen during `install-transpile` if multiple transpilers support the source dialect. - ## Execution Execute the below command to initialize the transpile process passing the arguments to the command directly in the call. -```shell +```bash databricks labs lakebridge transpile --transpiler-config-path --input-source --source-dialect --output-folder --skip-validation --catalog-name --schema-name ``` transpile-run diff --git a/docs/lakebridge/docs/transpile/overview.mdx b/docs/lakebridge/docs/transpile/overview.mdx deleted file mode 100644 index 1739202098..0000000000 --- a/docs/lakebridge/docs/transpile/overview.mdx +++ /dev/null @@ -1,170 +0,0 @@ ---- -sidebar_position: 1 -title: Transpilation overview ---- - -Lakebridge's transpilation module offers a pluggable mechanism for transpiling SQL files written in any supported source dialect into their equivalent in Databricks SQL. - -### Design Flow: -```mermaid -flowchart TD - A(Transpile CLI) e1@==> |Directory| B[Transpile All Files In Directory]; - A e2@==> |File| C[Transpile Single File] ; - B e3@==> D[List Files]; - C e4@==> E("LSP Pluggable Transpiler"); - D e5@==> E - E e6@==> |Parse Error| F(Failed Queries) - E e7@==> G{Skip Validations} - G e8@==> |Yes| H(Save Output) - G e9@==> |No| I{Validate} - I e10@==> |Success| H - I e11@==> |Fail| J(Flag, Capture) - J e12@==> H - e1@{ animate: true } - e2@{ animate: true } - e3@{ animate: true } - e4@{ animate: true } - e5@{ animate: true } - e6@{ animate: true } - e7@{ animate: true } - e8@{ animate: true } - e9@{ animate: true } - e10@{ animate: true } - e11@{ animate: true } - e12@{ animate: true } -``` - -When invoked with the following command line: - -```shell -databricks labs lakebridge transpile --source-dialect snowflake --input-source /path/to/input --output-folder /path/to/output -``` - -Lakebridge will (recursively) scan `/path/to/input`, looking for `.sql` files, and for each such file it will produce a corresponding file (with the same relative path) under `/path/to/output`. - -If for example we have the following `/path/to/input/foo/bar/silly.sql` file: - -```sql -SELECT 1; -``` - -The transpiled file will be generated at `/path/to/output/foo/bar/silly.sql`: - -```sql -/* - Successfully transpiled from /path/to/input/foo/bar/silly.sql -*/ -SELECT 1; -``` - -Notice how thee transpiler adds a header comment at the top of the transpile file. This header isn't very useful in such successful case, but it may come in handy when something went wrong during transpilation. - -As a matter of fact, transpilation could fail for various reasons: -- the input code may contain constructs that have no equivalent in Databricks SQL, -- the selected transpilation plugin may be missing an implementation (or have a bug), -- less likely, the input code may be incorrect, making the transpilation plugin unable to parse it (Lakebridge is built on the assumption that the code you feed it is correct). - -In such cases, the header comment will contain the list of errors encountered during transpilation in an attempt at making it easier for you to fix the transpiled code manually. - -Let see an example: - -```sql -/* - Failed transpilation of /path/to/input/broken.sql - - The following errors were found while transpiling: - - [3:3] Function GIBBERISH cannot be translated to Databricks SQL -*/ -SELECT - t1.name, - GIBBERISH(t1.comment) -FROM - t1; -``` - -Here the input code uses the (made up) `GIBBERISH` function that has no counterpart in Databricks SQL. The error is reported in the header comment with a specific position: `[3:5]` tells you that the problematic expression starts at the fifth character of the third line. - -It is important to note that the transpiler isn't always able to know whether it can't translate a given function because such translation isn't possible or because it is lacking the implementation of said translation. When reviewing such errors, if you think that it is the latter case, please file a bug report/feature request. - -## Plugins overview - -Out of the box, Lakebridge comes with two transpilation plugins: Morpheus and BladeBridge. Each one has its own set of capabilities and levels of guarantee that we describe below. - -### Morpheus - -Morpheus aims at offering strong guarantees about the code it produces. A file successfully transpiled by Morpheus (without any error or warning) is supposed to yield a result on Databricks that's equivalent[^1] to the result of running the corresponding input file on the source platform. - -As a consequence, whenever it finds itself unable to commit on such guarantee, Morpheus will emit either an error (when it knows that equivalent results cannot be guaranteed) or a warning (when it couldn't make sure that the guarantee holds). Being very conservative about this guarantee, Morpheus may emit warnings that are in fact false-negatives. In other words, a file that's successfully transpiled with warnings may still be correct and produce a result that's equivalent to its corresponding input on the source platform, but Morpheus was unable to guarantee it. - -Internally, Morpheus is built upon a custom parser (generated from a carefully crafted ANTLR grammar). The input text is first parsed into an intermediate representation tree. That tree then undergoes a series of transformations where structures that are specific to the input dialect are transformed to their Databricks equivalent. Finally, the transformed tree is fed to a code generator that produces and formats the final Databricks SQL text. - -Morpheus accumulates every error or warning encountered during the transformation and generation phases but continues processing the file. That way, even in the presence of errors, it keeps producing as much meaningful output as possible. - -However, errors encountered during the parsing phase, because they prevent it from building the initial tree, make it unable to produce any meaningful output. In such a case, the output file will contain the original text, along with the parsing errors. Such parsing error are very unlikely to happen though and should be considered as a bug. - -Finally, being built on the assumption that the input SQL is correct, Morpheus doesn't perform any semantical validation on the input code (such as type-checking). Therefore, the behaviour of code transpiled from a semantically incorrect input is unspecified. - - -[^1]: some statistical functions may be implemented on Databricks using an algorithm that slightly differs from the one used on the source platform, making the respective results non strictly identical to one another (although the differences wouldn't be statistically significant). Also, if no ordering is specified, rows may come in different order on Databricks and the source platform. In every other case, both results should be strictly identical. - -### BladeBridge - -**BladeBridge** is a flexible and extensible code conversion engine designed to accelerate modernization to Databricks. It supports a wide range of **ETL platforms** (e.g., DataStage) and **SQL-based systems** (e.g., Oracle, Teradata, Netezza), accepting inputs in the form of exported metadata and scripts. - -The converter generates **Databricks-compatible outputs** including: - -- **Notebooks** – with ETL logic translated into code cells -- **Table & view definitions** -- **Stored procedures** -- **Spark SQL** -- **PySpark scripts** -- **Workflow definitions** - -Internally, the execution is driven by **configuration files**, allowing users to define: - -- Source/target mappings -- Naming conventions -- Transformation preferences -- Output formats - -BladeBridge is fully **extensible** — new converters can be added, and the output structure can be **customized** to meet project-specific requirements. - -Whether you're migrating a single job or thousands, BladeBridge delivers predictable, repeatable results optimized for the Databricks ecosystem. - -More details about the BladeBridge converter [here](/docs/transpile/pluggable_transpilers/bladebridge) - -### Switch - -Switch is an LLM-powered Lakebridge transpiler that extends beyond SQL to convert various source formats into Databricks-compatible outputs. Using [Mosaic AI Model Serving](https://docs.databricks.com/aws/en/machine-learning/model-serving/), it understands code intent and semantics to handle complex transformations. - -Key characteristics: -- **LLM-powered extensibility** - Built-in prompts for multiple SQL dialects (T-SQL, Snowflake, Teradata, Oracle, etc.) and generic formats (Python, Scala, Airflow) -- **Custom prompt support** - Add YAML prompts to handle any source format not covered by built-in options -- **Flexible outputs** - Python notebooks with Spark SQL (default), SQL notebooks, or any text-based format -- **Databricks-native processing** - Executes as scalable Lakeflow Jobs with serverless compute -- **Model flexibility** - Uses Databricks Foundation Model APIs with configurable endpoint selection -- **Complex logic handling** - Excels at stored procedures and business logic beyond ANSI SQL/PSM standards - -Switch's LLM-based approach enables support for any source format through custom prompts. For built-in prompt details and custom prompt creation, see the [Switch documentation](/docs/transpile/pluggable_transpilers/switch). - ---- -Transpiler Selection Guide ---- - -Lakebridge offers both deterministic and LLM-powered transpilers out of the box, each optimized for different conversion scenarios. - -### Deterministic Conversion (BladeBridge & Morpheus) - -Deterministic transpilers excel in scenarios requiring consistency and speed: -- **Deterministic output with guaranteed syntax equivalence** - Every conversion produces the same predictable result -- **High-volume batch processing** - Efficiently handle thousands of files without API rate limits -- **Fast local execution without API dependencies** - Sub-minute processing with no external service calls -- **Production-grade SQL aligned with Databricks SQL evolution** - Leverages SQL Scripting, Stored Procedures, and latest DBSQL features - -### LLM-Powered Conversion (Switch) - -Switch is best suited for scenarios requiring semantic understanding and flexibility: -- **Complex logic requiring contextual understanding** - Stored procedures and business logic where intent matters more than syntax -- **Source formats not covered by deterministic transpilers** - Any SQL dialect or programming language through custom prompts -- **Extensible conversion through custom YAML prompts** - Adapt to proprietary or uncommon source formats -- **Python notebook output for SQL beyond ANSI SQL/PSM standards** - Complex transformations that benefit from procedural code diff --git a/docs/lakebridge/docs/transpile/pluggable_transpilers/bladebridge/bladebridge_configuration.mdx b/docs/lakebridge/docs/transpile/pluggable_transpilers/bladebridge/bladebridge_configuration.mdx index 731d478614..e6c0508e6a 100644 --- a/docs/lakebridge/docs/transpile/pluggable_transpilers/bladebridge/bladebridge_configuration.mdx +++ b/docs/lakebridge/docs/transpile/pluggable_transpilers/bladebridge/bladebridge_configuration.mdx @@ -7,7 +7,61 @@ import CodeBlock from '@theme/CodeBlock'; import Admonition from '@theme/Admonition'; -## Overview +## Configuration Overview + +BladeBridge uses a **two-layer configuration model**: + +| Layer | What it controls | How to set it | +|---|---|---| +| **Layer 1 — CLI parameters** | Source dialect, target output format, override file path | Set via `install-transpile` prompts or `transpile` CLI flags | +| **Layer 2 — JSON override files** | Conversion rules, naming conventions, source/target mappings | Create a custom JSON file and reference it via `--overrides-file` | + +You can use Layer 1 alone (most migrations), or combine both layers for advanced customization. + +--- + +## Layer 1: CLI Parameters + +These parameters are set when you run `install-transpile` (or passed directly to the `transpile` command): + +| Parameter | Required | Default | Description | +|---|---|---|---| +| `source-dialect` | Yes | `ansi` | Source SQL or ETL dialect (e.g., `mssql`, `oracle`, `datastage`, `ssis`) | +| `target-technology` | No | `SQL` | Output format: `SQL`, `SPARKSQL`, or `PYSPARK` | +| `overrides-file` | No | none | Path to a custom JSON config file (Layer 2) | + +### Dialect-specific `target-technology` constraints + +The `target-technology` prompt only appears for ETL source dialects. SQL dialects always output Databricks SQL: + +| Dialect | `target-technology` prompt? | Available choices | +|---|---|---| +| `datastage` | Yes | `SPARKSQL`, `PYSPARK` | +| `ssis` | Yes | `SPARKSQL` only | +| `mssql`, `oracle`, `synapse`, `netezza`, `redshift`, `teradata` | No | Outputs Databricks SQL only | + +### Minimal working example + +```bash +databricks labs lakebridge transpile \ + --source-dialect oracle \ + --input-source /path/to/sql \ + --output-folder /path/to/output +``` + +For ETL sources with target technology: + +```bash +databricks labs lakebridge transpile \ + --source-dialect datastage \ + --input-source /path/to/datastage/exports \ + --output-folder /path/to/output \ + --target-technology SPARKSQL +``` + +--- + +## Layer 2: JSON Override Files The **BladeBridge** transpiler relies heavily on rules defined inside configuration files provided with the converter. These configurations are comprised of a set of layered json files and code templates that drive the generation of output @@ -579,7 +633,7 @@ This section holds templates on read and write instructions for each system clas ### Native Database Connection Support -When converting ETL components that connect to external databases (e.g., Oracle, SQL Server, Redshift, Synapse), you can configure the converter to use native JDBC/ODBC connections instead of Databricks native connectors. This is useful when you need direct database access or when migrating from platforms like Informatica Cloud or DataStage. +When converting ETL components that connect to external databases (e.g., Oracle, SQL Server, Redshift, Synapse), you can configure the converter to use native JDBC/ODBC connections instead of Databricks native connectors. This is useful when you need direct database access or when migrating from platforms like DataStage. #### Configuration Flags diff --git a/docs/lakebridge/docs/transpile/pluggable_transpilers/morpheus/index.mdx b/docs/lakebridge/docs/transpile/pluggable_transpilers/morpheus/index.mdx new file mode 100644 index 0000000000..b0e6c1356e --- /dev/null +++ b/docs/lakebridge/docs/transpile/pluggable_transpilers/morpheus/index.mdx @@ -0,0 +1,135 @@ +--- +title: Morpheus +sidebar_position: 1 +--- + +# Morpheus + +**Morpheus** is Lakebridge's AST-based SQL transpiler. It provides strong correctness guarantees: a file that Morpheus transpiles without any errors or warnings is guaranteed to produce equivalent results on Databricks as the original file did on the source platform. + +:::tip Not sure which transpiler to use? +See [Which Tool Do I Use?](/docs/choosing_tools#transpiler-bladebridge-vs-morpheus-vs-switch) for a full comparison of Morpheus, BladeBridge, and Switch. +::: + +--- + +## Supported Source Dialects + +| Dialect key | Source systems | +|---|---| +| `mssql` | Microsoft SQL Server, Azure SQL Database, Azure SQL Managed Instance, Amazon RDS for SQL Server | +| `snowflake` | Snowflake (including dbt repointing) | +| `synapse` | Azure Synapse Analytics (dedicated SQL pools) | + +Morpheus targets **Databricks SQL** only. For ETL sources (DataStage, SSIS) or other SQL dialects (Oracle, Teradata, Redshift), use [BladeBridge](/docs/transpile/pluggable_transpilers/bladebridge) or [Switch](/docs/transpile/pluggable_transpilers/switch). + +--- + +## How Morpheus Works + +Morpheus uses a custom parser generated from a carefully crafted ANTLR grammar. The transpilation pipeline has three stages: + +1. **Parse** — The input SQL is parsed into an intermediate representation (IR) tree. Parsing errors prevent the tree from being built and will result in the original text being returned with error annotations. +2. **Transform** — The IR tree undergoes a series of transformations that convert source-dialect constructs into their Databricks SQL equivalents. +3. **Code gen** — The transformed tree is fed to a code generator that produces and formats the final Databricks SQL text. + +Morpheus accumulates every error and warning encountered during the transform and code-gen phases but continues processing the file. Even in the presence of errors, it produces as much meaningful output as possible. + +--- + +## Correctness Guarantee Model + +Morpheus distinguishes between **errors** and **warnings**: + +| Signal | Meaning | What to do | +|---|---|---| +| No errors, no warnings | Full correctness guarantee — output is equivalent to input on the source platform | Deploy with confidence | +| **Warning** | Morpheus could not confirm equivalence but the output may still be correct (a conservative false-negative) | Review the flagged section; test output against source data | +| **Error** | Morpheus knows equivalent results cannot be guaranteed for this construct | Manual fix required before deploying | + +:::note +Some statistical functions may be implemented on Databricks using a slightly different algorithm than the source platform. Row ordering may differ when no `ORDER BY` is specified. In all other cases, results are strictly identical. +::: + +Morpheus does not perform semantic validation (e.g., type-checking) on the input SQL. The behavior of code transpiled from semantically incorrect input is unspecified. + +--- + +## Usage + +### Install + +Morpheus is installed automatically when you run: + +```bash +databricks labs lakebridge install-transpile +``` + +Morpheus is fetched from [Maven Central](https://central.sonatype.com/) and installed at: +``` +.databricks/labs/remorph-transpilers/databricks-morph-plugin/ +``` + +### Run + +```bash +databricks labs lakebridge transpile \ + --source-dialect mssql \ + --input-source /path/to/sql/files \ + --output-folder /path/to/output +``` + +**Common flags:** + +| Flag | Description | Default | +|---|---|---| +| `--source-dialect` | Source SQL dialect: `mssql`, `snowflake`, or `synapse` | Required | +| `--input-source` | Local path to the SQL files to transpile | Required | +| `--output-folder` | Local path where transpiled files will be written | Required | +| `--skip-validation` | Skip Databricks SQL validation of output (`true`/`false`) | `false` | +| `--catalog-name` | Catalog to use when validating output | `remorph` | +| `--schema-name` | Schema to use when validating output | `transpiler` | + +### Reading the output + +Each transpiled file begins with a header comment: + +**Successful transpile:** +```sql +/* + Successfully transpiled from /path/to/input/my_proc.sql +*/ +CREATE PROCEDURE ... +``` + +**Transpile with errors/warnings:** +```sql +/* + Failed transpilation of /path/to/input/broken.sql + + The following errors were found while transpiling: + - [3:5] Function GIBBERISH cannot be translated to Databricks SQL +*/ +SELECT + t1.name, + GIBBERISH(t1.comment) +FROM t1; +``` + +The `[line:column]` notation tells you exactly where in the input file the untranslatable construct starts. + +--- + +## Troubleshooting + +**My file transpiles with warnings — is it safe to use?** + +Morpheus is conservative: it emits warnings when it cannot *guarantee* correctness, even if the output is actually correct. Review the flagged section and test the output against your source data. If the results match, the file is safe. + +**I see a parsing error — is that a bug?** + +Parsing errors are very unlikely when the input SQL is valid. If you see one, the input may be malformed, or there is a gap in the Morpheus ANTLR grammar. File a bug report at [GitHub](https://github.com/databrickslabs/lakebridge/issues). + +**Morpheus doesn't support my dialect.** + +Morpheus supports `mssql`, `snowflake`, and `synapse` only. For other dialects, use [BladeBridge](/docs/transpile/pluggable_transpilers/bladebridge) (Oracle, Teradata, Netezza, Redshift, DataStage, SSIS) or [Switch](/docs/transpile/pluggable_transpilers/switch) (any dialect via LLM). diff --git a/docs/lakebridge/docs/transpile/pluggable_transpilers/switch/architecture.mdx b/docs/lakebridge/docs/transpile/pluggable_transpilers/switch/architecture.mdx new file mode 100644 index 0000000000..cc5029da75 --- /dev/null +++ b/docs/lakebridge/docs/transpile/pluggable_transpilers/switch/architecture.mdx @@ -0,0 +1,156 @@ +--- +title: Switch Architecture +sidebar_position: 6 +--- + +import CodeBlock from '@theme/CodeBlock'; + +# Switch Architecture + +This page describes the internal architecture of Switch — how it executes as a Databricks Job and the processing pipeline it runs. This is useful for advanced users, contributors, and anyone debugging conversion failures. + +For getting started with Switch, see the [Switch guide](/docs/transpile/pluggable_transpilers/switch). + +--- + +## Overview + +When you run Switch via the CLI, it executes as a Databricks Job using a multi-stage processing pipeline. The main orchestration notebook (`00_main`) validates parameters and routes to the appropriate orchestrator based on `target_type`. + +```mermaid +flowchart TD + main["00_main (Entry Point)"] e1@==> decision{target_type?} + + decision e2@==>|notebook| notebookOrch["orchestrate_to_notebook (Full Processing Pipeline)"] + decision e3@==>|file| fileOrch["orchestrate_to_file (Simplified Processing Pipeline)"] + + notebookOrch e4@==> notebookFlow["7-Step Workflow: analyze → convert → validate → fix → split → export → sql_convert(optional)"] + fileOrch e5@==> fileFlow["3-Step Workflow: analyze → convert → export"] + + notebookFlow e6@==> notebookOutput[Python/SQL Notebooks] + fileFlow e7@==> fileOutput["Generic Files (YAML, JSON, etc.)"] + e1@{ animate: true } + e2@{ animate: true } + e3@{ animate: true } + e4@{ animate: true } + e5@{ animate: true } + e6@{ animate: true } + e7@{ animate: true } +``` + +--- + +## Notebook Conversion Flow + +For `target_type=notebook` or `target_type=sdp`, the `orchestrate_to_notebook` orchestrator runs a 7-step pipeline: + +```mermaid +flowchart TD + orchestrator[orchestrate_to_notebook] e1@==>|sequentially calls| processing + + subgraph processing ["Notebook Processing Workflow"] + direction TB + analyze[analyze_input_files] e2@==> convert[convert_with_llm] + + convert e8@==>|if SDP| validate_sdp[validate_sdp] + convert e3@==>|if NOT SDP| validate_nb[validate_python_notebook] + + validate_nb e4@==> fix[fix_syntax_with_llm] + validate_sdp e9@==> fix + + fix e5@==> split[split_code_into_cells] + split e6@==> export[export_to_notebook] + export -.-> sqlExport["convert_notebook_to_sql (Optional)"] + end + + processing <-->|Read & Write| table[Conversion Result Table] + + convert -.->|Uses| endpoint[Model Serving Endpoint] + fix -.->|Uses| endpoint + sqlExport -.->|Uses| endpoint + + export e7@==> notebooks[Python Notebooks] + sqlExport -.-> sqlNotebooks["SQL Notebooks (Optional Output)"] + + e1@{ animate: true } + e2@{ animate: true } + e3@{ animate: true } + e4@{ animate: true } + e5@{ animate: true } + e6@{ animate: true } + e7@{ animate: true } + e8@{ animate: true } + e9@{ animate: true } +``` + +--- + +## File Conversion Flow + +For `target_type=file`, the `orchestrate_to_file` orchestrator uses a simplified 3-step pipeline: + +```mermaid +flowchart TD + orchestrator[orchestrate_to_file] e1@==>|sequentially calls| processing + + subgraph processing ["File Processing Workflow"] + direction TB + analyze[analyze_input_files] e2@==> convert[convert_with_llm] + convert e3@==> export[export_to_file] + end + + processing <-->|Read & Write| table[Conversion Result Table] + convert -.->|Uses| endpoint[Model Serving Endpoint] + export e4@==> files["Generic Files (YAML, JSON, etc.)"] + e1@{ animate: true } + e2@{ animate: true } + e3@{ animate: true } + e4@{ animate: true } +``` + +File conversion skips syntax validation, error fixing, and cell splitting — it exports directly from converted content to the specified file format. + +--- + +## Processing Steps + +### analyze_input_files + +Scans the input directory recursively and stores all file contents, metadata, and analysis results in a timestamped Delta table. For SQL sources, creates preprocessed versions with comments removed and whitespace normalized. Counts tokens using model-specific tokenizers (Claude uses ~3.4 characters per token; OpenAI and other models use tiktoken). Files exceeding `token_count_threshold` are excluded from conversion. + +### convert_with_llm + +Loads conversion prompts (built-in or custom YAML) and sends file content to the configured model serving endpoint. Multiple files are processed concurrently (default: 4, controlled by `concurrency`). For SQL sources, generates Python code with `spark.sql()` calls. For generic sources, adapts to the specified target format. + +### validate_python_notebook + +Checks Python syntax using `ast.parse()` and validates SQL statements within `spark.sql()` calls using Spark's `EXPLAIN` command. Errors are recorded in the result table. + +### validate_sdp + +Validates Spark Declarative Pipeline output: + +| Step | Description | +|---|---| +| Export Notebook | Writes converted code to a temporary workspace notebook | +| Create Pipeline | Creates a temporary Spark Declarative Pipeline referencing the notebook | +| Update Pipeline | Runs a validation-only update to check for SDP syntax errors | +| Delete Pipeline | Cleans up the temporary pipeline | + +`TABLE_OR_VIEW_NOT_FOUND` errors are ignored. + +### fix_syntax_with_llm + +Sends error context back to the model serving endpoint for automatic correction. Repeats up to `max_fix_attempts` times (default: 1). Set to 0 to disable automatic fixing. + +### split_code_into_cells + +Transforms raw converted Python code into well-structured notebook cells. Splits at logical boundaries (imports, function definitions, major operations) and adds markdown cells for documentation. + +### export_to_notebook + +Creates Databricks-compatible `.py` notebooks in the output directory. Handles files up to 10MB and preserves the original directory structure. Includes metadata, source file references, and syntax check results as comments. + +### convert_notebook_to_sql (Optional) + +When `sql_output_dir` is specified, uses the model serving endpoint to convert Python notebooks to SQL notebook format. Some Python-specific logic may be lost in this conversion. diff --git a/docs/lakebridge/docs/transpile/pluggable_transpilers/switch/index.mdx b/docs/lakebridge/docs/transpile/pluggable_transpilers/switch/index.mdx index b2615dff93..30ef768cbb 100644 --- a/docs/lakebridge/docs/transpile/pluggable_transpilers/switch/index.mdx +++ b/docs/lakebridge/docs/transpile/pluggable_transpilers/switch/index.mdx @@ -34,7 +34,7 @@ Instead of parsing rules, Switch uses [Mosaic AI Model Serving](https://docs.dat - Enable extensible conversion through custom YAML prompts ### 2. Native Databricks Integration -Switch runs entirely within the Databricks workspace. You can find details about this architecture [here](#databricks-implementation-details) +Switch runs entirely within the Databricks workspace. You can find details about this architecture [here](/docs/transpile/pluggable_transpilers/switch/architecture) - **Jobs API**: Executes as scalable Databricks Jobs for batch processing - **Model Serving**: Direct integration with Databricks LLM endpoints, with concurrent processing for multiple files - **Delta Tables**: Tracks conversion progress and results @@ -147,6 +147,7 @@ databricks labs lakebridge llm-transpile \ [--schema-name your_schema] \ [--volume your_volume] \ [--foundation-model your_foundation_model] \ + [--switch-config-path /Workspace/path/to/switch_config.yml] \ [--profile profile_name] ``` @@ -189,25 +190,28 @@ The `llm-transpile` command accepts the following parameters: | `--schema-name` | Optional (prompted, default: `switch`) | Schema within the catalog for Switch Delta tables and Volume | `your_schema` | | `--volume` | Optional (prompted, default: `switch_volume`) | Unity Catalog Volume for uploaded input source files | `your_volume` | | `--foundation-model` | Optional (prompted from available FM APIs) | Model serving endpoint name for conversions | `databricks-claude-sonnet-4-5` | +| `--switch-config-path` | Optional | Workspace path to a custom Switch configuration file. Overrides the default configuration file location. Must start with `/Workspace/`. See [Switch Configuration File](#switch-configuration-file) for available configuration parameters. | `/Workspace/Users/user/switch_config.yml` | ### Switch Configuration File Additional conversion parameters are managed in the Switch configuration file. You can edit this file directly in your workspace to customize Switch's conversion behavior. -**File location:** `/Workspace/Users/{user}/.lakebridge/switch/resources/switch_config.yml` +**Default file location:** `/Workspace/Users/{user}/.lakebridge/switch/resources/switch_config.yml` + +You can also specify a custom configuration file path using the `--switch-config-path` command-line parameter, which overrides the default location. | Parameter | Description | Default Value | Available Options | |-----------|-------------|---------------|-------------------| -| `target_type` | Output format type. `notebook` for Python notebooks with validation and error fixing, `file` for generic file formats, `sdp` for conversion from etl workloads to Spark Declarative Pipeline (SDP). See [Conversion Flow Overview](#conversion-flow-overview) for processing differences. | `notebook` | `notebook`, `file`, `sdp` | -| `source_format` | Source file format type. `sql` performs SQL comment removal and whitespace compression preprocessing before conversion. `generic` processes files as-is without preprocessing. Preprocessing affects token counting and conversion quality. See [analyze_input_files](#analyze_input_files) for preprocessing details. | `sql` | `sql`, `generic` | +| `target_type` | Output format type. `notebook` for Python notebooks with validation and error fixing, `file` for generic file formats, `sdp` for conversion from etl workloads to Spark Declarative Pipeline (SDP). See [Conversion Flow Overview](/docs/transpile/pluggable_transpilers/switch/architecture#overview) for processing differences. | `notebook` | `notebook`, `file`, `sdp` | +| `source_format` | Source file format type. `sql` performs SQL comment removal and whitespace compression preprocessing before conversion. `generic` processes files as-is without preprocessing. Preprocessing affects token counting and conversion quality. See [analyze_input_files](/docs/transpile/pluggable_transpilers/switch/architecture#analyze_input_files) for preprocessing details. | `sql` | `sql`, `generic` | | `comment_lang` | Language for generated comments. | `English` | `English`, `Japanese`, `Chinese`, `French`, `German`, `Italian`, `Korean`, `Portuguese`, `Spanish` | | `log_level` | Logging verbosity level. | `INFO` | `DEBUG`, `INFO`, `WARNING`, `ERROR` | | `token_count_threshold` | Maximum tokens per file for processing. Files exceeding this limit are automatically excluded from conversion. Adjust based on your model's context window and conversion complexity. See [Token Management](/docs/transpile/pluggable_transpilers/switch/customizing_switch#token-management) for detailed configuration guidelines and file splitting strategies. | `20000` | Any positive integer | | `concurrency` | Number of parallel LLM requests for processing multiple files simultaneously. Higher values improve throughput but may hit rate limits. Default is optimized for Claude models. See [Performance Optimization](/docs/transpile/pluggable_transpilers/switch/customizing_switch#performance-optimization) for scaling guidance and model-specific considerations. | `4` | Any positive integer | -| `max_fix_attempts` | Maximum number of automatic syntax error correction attempts per file. Each attempt sends error context back to the LLM for fixing. Set to 0 to skip automatic fixes. See [fix_syntax_with_llm](#fix_syntax_with_llm) for details on the error correction process. | `1` | 0 or any positive integer | +| `max_fix_attempts` | Maximum number of automatic syntax error correction attempts per file. Each attempt sends error context back to the LLM for fixing. Set to 0 to skip automatic fixes. See [fix_syntax_with_llm](/docs/transpile/pluggable_transpilers/switch/architecture#fix_syntax_with_llm) for details on the error correction process. | `1` | 0 or any positive integer | | `conversion_prompt_yaml` | Custom conversion prompt YAML file path. When specified, overrides the built-in prompt for the selected `--source-dialect`, enabling support for additional source formats or specialized conversion requirements. See [Customizable Prompts](/docs/transpile/pluggable_transpilers/switch/customizing_switch#customizable-prompts) for YAML structure and creation guide. | `null` | Full workspace path to YAML file | -| `output_extension` | File extension for output files when `target_type=file`. Required for non-notebook output formats like YAML workflows or JSON configurations. See [File Conversion Flow](#file-conversion-flow) for usage examples. | `null` | Any extension (e.g., `.yml`, `.json`) | -| `sql_output_dir` | (Experimental) When specified, triggers additional conversion of Python notebooks to SQL notebook format. This optional post-processing step may lose some Python-specific logic. See [convert_notebook_to_sql](#convert_notebook_to_sql-optional) for details on the SQL conversion process. | `null` | Full workspace path | +| `output_extension` | File extension for output files when `target_type=file`. Required for non-notebook output formats like YAML workflows or JSON configurations. See [File Conversion Flow](/docs/transpile/pluggable_transpilers/switch/architecture#file-conversion-flow) for usage examples. | `null` | Any extension (e.g., `.yml`, `.json`) | +| `sql_output_dir` | (Experimental) When specified, triggers additional conversion of Python notebooks to SQL notebook format. This optional post-processing step may lose some Python-specific logic. See [convert_notebook_to_sql](/docs/transpile/pluggable_transpilers/switch/architecture#convert_notebook_to_sql-optional) for details on the SQL conversion process. | `null` | Full workspace path | | `request_params` | Additional request parameters passed to the model serving endpoint. Use for advanced configurations like extended thinking mode or custom token limits. See [LLM Configuration](/docs/transpile/pluggable_transpilers/switch/customizing_switch#llm-configuration) for configuration examples including Claude's extended thinking mode. | `null` | JSON format string (e.g., `{"max_tokens": 64000}`) | | `sdp_language` | Control the language of converted SDP code, can only be "python" or "sql". | `python` | `python`, `sql` | @@ -308,155 +312,7 @@ databricks labs lakebridge llm-transpile \ ``` --- -## Databricks Implementation Details - -When you run Switch via the CLI, it executes as Databricks Jobs using a sophisticated multi-stage processing pipeline. This section covers the internal architecture and configuration options. - -## Processing Architecture - -Switch executes as a Databricks Job that runs the main orchestration notebook (`00_main`), which routes to specialized orchestrators that coordinate the conversion pipeline: - -### Main Orchestration -The **`00_main`** notebook serves as the entry point when Switch is executed via Databricks Jobs API. It: -- Validates all input parameters from the job configuration -- Routes execution to the appropriate orchestrator based on `target_type` (notebook or file output) -- Handles orchestrator results and displays final conversion summary - -### Conversion Flow Overview - -Switch supports two target types with different processing workflows. The main entry point (`00_main`) routes to the appropriate orchestrator based on the `target_type` parameter: - -```mermaid -flowchart TD - main["00_main (Entry Point)"] e1@==> decision{target_type?} - - decision e2@==>|notebook| notebookOrch["orchestrate_to_notebook (Full Processing Pipeline)"] - decision e3@==>|file| fileOrch["orchestrate_to_file (Simplified Processing Pipeline)"] - - notebookOrch e4@==> notebookFlow["7-Step Workflow: analyze → convert → validate → fix → split → export → sql_convert(optional)"] - fileOrch e5@==> fileFlow["3-Step Workflow: analyze → convert → export"] - - notebookFlow e6@==> notebookOutput[Python/SQL Notebooks] - fileFlow e7@==> fileOutput["Generic Files (YAML, JSON, etc.)"] - e1@{ animate: true } - e2@{ animate: true } - e3@{ animate: true } - e4@{ animate: true } - e5@{ animate: true } - e6@{ animate: true } - e7@{ animate: true } -``` - -### Notebook Conversion Flow - -For `target_type=notebook` or `target_type=sdp`, the `orchestrate_to_notebook` orchestrator executes a comprehensive 7-step processing pipeline: - -```mermaid -flowchart TD - orchestrator[orchestrate_to_notebook] e1@==>|sequentially calls| processing - - subgraph processing ["Notebook Processing Workflow"] - direction TB - analyze[analyze_input_files] e2@==> convert[convert_with_llm] - - %% Branch: decide validation path - convert e8@==>|if SDP| validate_sdp[validate_sdp] - convert e3@==>|if NOT SDP| validate_nb[validate_python_notebook] - - %% Downstream connections - both validations flow to fix_syntax - validate_nb e4@==> fix[fix_syntax_with_llm] - validate_sdp e9@==> fix - - fix e5@==> split[split_code_into_cells] - split e6@==> export[export_to_notebook] - export -.-> sqlExport["convert_notebook_to_sql
(Optional)"] - end - - processing <-->|Read & Write| table[Conversion Result Table] - - convert -.->|Uses| endpoint[Model Serving Endpoint] - fix -.->|Uses| endpoint - sqlExport -.->|Uses| endpoint - - export e7@==> notebooks[Python Notebooks] - sqlExport -.-> sqlNotebooks["SQL Notebooks
(Optional Output)"] - - e1@{ animate: true } - e2@{ animate: true } - e3@{ animate: true } - e4@{ animate: true } - e5@{ animate: true } - e6@{ animate: true } - e7@{ animate: true } - e8@{ animate: true } - e9@{ animate: true } -``` - -### File Conversion Flow - -For `target_type=file`, the `orchestrate_to_file` orchestrator uses a simplified 3-step processing pipeline optimized for generic file output: - -```mermaid -flowchart TD - orchestrator[orchestrate_to_file] e1@==>|sequentially calls| processing - - subgraph processing ["File Processing Workflow"] - direction TB - analyze[analyze_input_files] e2@==> convert[convert_with_llm] - convert e3@==> export[export_to_file] - end - - processing <-->|Read & Write| table[Conversion Result Table] - - convert -.->|Uses| endpoint[Model Serving Endpoint] - - export e4@==> files["Generic Files (YAML, JSON, etc.)"] - e1@{ animate: true } - e2@{ animate: true } - e3@{ animate: true } - e4@{ animate: true } -``` - -**Key Differences:** -- **File conversion skips** syntax validation, error fixing, and cell splitting steps -- **Direct export** from converted content to specified file format with custom extension -- **Optimized** for non-notebook outputs like YAML workflows, JSON configurations, etc. - ---- - -## Processing Steps - -The following sections describe each processing step used in the workflows above: - -### analyze_input_files -Scans the input directory recursively and performs initial analysis. Stores all file contents, metadata, and analysis results in a timestamped Delta table. For SQL sources, creates preprocessed versions with comments removed and whitespace normalized in the table. Counts tokens using model-specific tokenizers (Claude uses ~3.4 characters per token, OpenAI and other models use tiktoken) to determine if files exceed the `token_count_threshold`. Files exceeding the threshold are excluded from conversion. - -### convert_with_llm -Loads conversion prompts (built-in or custom YAML) and sends file content to the configured model serving endpoint. Multiple files are processed concurrently (configurable, default: 4) for efficiency. The LLM transforms source code based on the conversion prompt, preserving business logic while adapting to Databricks patterns. For SQL sources, generates Python code with `spark.sql()` calls. For generic sources, adapts content to the specified target format. - -### validate_python_notebook -Performs syntax validation on the generated code. Python syntax is checked using `ast.parse()`, while SQL statements within `spark.sql()` calls are validated using Spark's `EXPLAIN` command. Any errors are recorded in the result table for potential fixing in the next step. - -### validate_sdp -Performs Spark Declarative Pipeline validation on the generated code. The validation process executes these steps sequentially: - -| Step | Description | -|------|-------------| -| Export Notebook | Writes the converted code to a temporary notebook in workspace | -| Create Pipeline | Creates a temporary Spark Declarative Pipeline referencing the notebook | -| Update Pipeline | Runs a validation-only update to check for SDP syntax errors | -| Delete Pipeline | Cleans up the temporary pipeline after validation | - -Note: `TABLE_OR_VIEW_NOT_FOUND` errors are ignored. - -### fix_syntax_with_llm -Attempts automatic error correction when syntax issues are detected. Sends error context back to the model serving endpoint, which suggests corrections. The validation and fix process repeats up to `max_fix_attempts` times (default: 1) until errors are resolved or the retry limit is reached. - -### split_code_into_cells -Transforms raw converted Python code into well-structured notebook cells. Analyzes code flow and dependencies, splitting content at logical boundaries like imports, function definitions, and major operations. Adds appropriate markdown cells for documentation and readability. -### export_to_notebook -Creates Databricks-compatible `.py` notebooks in the specified output directory. Each notebook includes proper metadata, source file references, and any syntax check results as comments. Handles large files (up to 10MB) and preserves the original directory structure. +## Internal Architecture -### convert_notebook_to_sql (Optional) -When `sql_output_dir` is specified, this optional step uses the model serving endpoint to convert Python notebooks into SQL notebook format with Databricks SQL syntax. Useful for teams preferring SQL-only workflows, though some Python logic may be lost in the conversion process. +Switch runs as a Databricks Job using a multi-stage processing pipeline. For details on how the pipeline works internally (orchestration notebooks, processing steps, validation logic), see [Switch Architecture](/docs/transpile/pluggable_transpilers/switch/architecture). diff --git a/docs/lakebridge/docs/transpile/pluggable_transpilers/switch/switch_faqs.mdx b/docs/lakebridge/docs/transpile/pluggable_transpilers/switch/switch_faqs.mdx deleted file mode 100644 index 00c47cc9e5..0000000000 --- a/docs/lakebridge/docs/transpile/pluggable_transpilers/switch/switch_faqs.mdx +++ /dev/null @@ -1,77 +0,0 @@ ---- -title: Switch FAQs and Troubleshooting -sidebar_position: 7 ---- - -import CodeBlock from '@theme/CodeBlock'; - -## Can Switch Support My Source System? - -A common question is whether Switch can handle sources beyond the built-in conversion types. The answer is: **try it!** - -Switch already supports various source formats including SQL dialects (MySQL, Snowflake, Oracle, etc.), programming languages (Python scripts, Scala code), and workflows (Airflow DAGs). - -**For SQL-based sources**: Creating a custom prompt YAML file should work well for most SQL dialects. Since LLMs understand SQL syntax patterns, you can typically achieve good results by: -- Starting with a similar built-in dialect's YAML as a template -- Adding specific syntax examples from your source system -- Testing and iterating based on results - -**Tips for efficient prompt creation:** -- **Quick baseline creation**: Feed built-in prompts and your source dialect's representative features to an advanced LLM to quickly generate a baseline YAML configuration -- **Dialect-specific patterns**: Reference open source projects like [SQLGlot dialects](https://github.com/tobymao/sqlglot/tree/main/sqlglot/dialects) for insights into dialect-specific transformation patterns - -**For other source formats**: Switch's LLM-based architecture means it can potentially handle various conversions beyond the built-in types. Modern LLMs have strong comprehension capabilities across many languages and formats. You can experiment by: -- Creating custom prompts that define your source format -- Providing clear conversion examples in the few-shots section -- Testing with representative source samples - -Rather than waiting for additional built-in examples, we encourage experimentation with custom prompts. The flexibility of LLM-based conversion means many use cases are possible with the right prompt engineering. - ---- - -## Conversion Results and Troubleshooting - -### Understanding Conversion Results - -After your Switch job completes, review the conversion results displayed at the end of the `00_main` notebook execution. The results table shows the status of each input file: - -- **Successfully converted files**: Ready to use as Databricks notebooks -- **Files requiring attention**: May need manual review or re-processing - -If you encounter files that didn't convert successfully, here are the most common issues and their solutions: - -### Files Not Converting (Status: `Not converted`) - -These files were skipped during the conversion process, typically because they're too large for the model to process effectively. - -**Cause**: Input files exceed the token count threshold - -**Solutions**: -- Split large input files into smaller, more manageable parts -- Increase the `token_count_threshold` parameter if your LLM model can handle larger inputs - -### Conversion with Errors (Status: `Converted with errors`) - -These files were successfully processed by the LLM but the generated code contains syntax errors that need to be addressed. - -**Cause**: Files were converted but contain syntax errors - -**Solutions**: -- Review syntax error messages in the result table's error_details column -- Manually fix errors in the converted notebooks/files -- Increase `max_fix_attempts` for more automatic error correction attempts - -### Export Failures (Status: `Export failed` or `Not exported`) - -These files were converted successfully but couldn't be exported to the output directory. - -**Causes**: -- Content exceeds 10MB size limit of Databricks notebooks -- File system permissions issues -- Invalid output paths - -**Solutions**: -- Check the `export_error` column in the result table for specific error details -- For size issues: Manually split large converted content into smaller units -- For permission issues: Verify workspace access to the output directory -- For path issues: Ensure output directory paths are valid workspace locations diff --git a/docs/lakebridge/docs/transpile/source_systems/index.mdx b/docs/lakebridge/docs/transpile/source_systems/index.mdx index 022e09abf9..256527e1cd 100644 --- a/docs/lakebridge/docs/transpile/source_systems/index.mdx +++ b/docs/lakebridge/docs/transpile/source_systems/index.mdx @@ -46,9 +46,3 @@ In this section we'll have source-specific conversion documentation. Select your [📖 View DataStage Conversion Guide](./datastage) --- - -## Coming Soon - -Additional source system conversion guides will be added as they become available for all supported sources in Lakebridge - ---- diff --git a/docs/lakebridge/docs/transpile/source_systems/ssis.mdx b/docs/lakebridge/docs/transpile/source_systems/ssis.mdx deleted file mode 100644 index 51a671da35..0000000000 --- a/docs/lakebridge/docs/transpile/source_systems/ssis.mdx +++ /dev/null @@ -1,608 +0,0 @@ ---- -sidebar_position: 1 -title: SSIS - -toc_min_heading_level: 2 -toc_max_heading_level: 3 ---- -# Microsoft SSIS to Databricks - -## Conversion information -* **Transpiler**: BladeBridge -* **Available** target: Databricks notebooks (experimental) - -### Supported SSIS Versions - -- SQL Server 2012, 2014, 2016, 2017, 2019, 2022 -- Azure Data Factory SSIS Integration Runtime - -### Input Requirements - -Export your SSIS packages as DTSX files: - -1. **Solution Export**: Export from Visual Studio / SQL Server Data Tools (SSDT) -2. **File System Packages**: Direct DTSX file access -3. **SSISDB Export**: Extract from SSIS catalog - -**Export from SSISDB:** -```sql --- Extract package from SSISDB catalog -DECLARE @packageData VARBINARY(MAX) -SELECT @packageData = [packagedata] -FROM [SSISDB].[catalog].[packages] -WHERE [name] = 'YourPackageName' - --- Save to file system for conversion -``` - ---- - -## Components - -### Control Flow (Orchestration) - Supported - -| SSIS Component | Microsoft Name | Spark Equivalent | Notes | -|----------------|----------------|------------------|-------| -| **Data Flow Task** | Microsoft.DataFlowTask | Spark SQL temp views | Core data transformations | -| **Execute SQL Task** | Microsoft.ExecuteSQLTask | Spark SQL statements | SQL execution with parameter mapping | -| **Execute Package Task** | Microsoft.ExecutePackageTask | `dbutils.notebook.run()` | Nested notebook execution | -| **File System Task** | Microsoft.FileSystemTask | `dbutils.fs` commands | File operations (copy, move, delete, rename) | -| **Script Task** | Microsoft.ScriptTask | Python code with SQL | C#/VB.NET scripts converted to Python/SQL | -| **For Loop Container** | STOCK:FORLOOP | SQL iteration pattern | Iteration with counter | -| **Foreach Loop Container** | STOCK:FOREACHLOOP | SQL wildcard file reading | File/folder iteration | -| **Sequence Container** | STOCK:SEQUENCE | Function or notebook section | Grouping tasks | -| **Execute Process Task** | Microsoft.ExecuteProcess | `subprocess.run()` | External process execution | -| **Extensible File Task** | ExtensibleFileTask | `dbutils.fs` commands | Extended file operations | - -### Control Flow (Orchestration) - Unsupported - -The following Control Flow components are **not supported** and would require manual conversion or alternative approaches: - -| SSIS Component | Microsoft Name | Reason | -|----------------|----------------|--------| -| **Analysis Services Execute DDL** | Microsoft.AnalysisServicesExecuteDDLTask | SSAS-specific, requires manual migration to Delta/SQL | -| **Analysis Services Processing** | Microsoft.AnalysisServicesProcessingTask | SSAS-specific, requires manual migration to Delta/SQL | -| **Bulk Insert Task** | Microsoft.BulkInsertTask | Use Data Flow Task with JDBC/Delta instead | -| **Data Profiling Task** | Microsoft.DataProfilingTask | Use Databricks Data Profile UI or custom profiling code | -| **FTP Task** | Microsoft.FTPTask | Implement using Python `ftplib` or `dbutils.fs` | -| **Message Queue Task** | Microsoft.MessageQueueTask | Requires custom Kafka/Event Hub integration | -| **Send Mail Task** | Microsoft.SendMailTask | Implement using Databricks job notifications or Python `smtplib` | -| **Web Service Task** | Microsoft.WebServiceTask | Implement using Python `requests` library | -| **WMI Data Reader Task** | Microsoft.WMIDataReaderTask | Windows-specific, requires alternative monitoring solution | -| **XML Task** | Microsoft.XMLTask | Implement using Python `xml.etree` or PySpark XML functions | - -:::warning Conversion Difficulty -Components like **Script Task** contain **C# or VB.NET code bodies** that cannot be automatically converted. These require **manual code translation** from C#/VB to Python. The converter will preserve the logic structure but the actual implementation must be rewritten. -::: - ---- - -### Data Flow (Transformation) - Supported - -#### Sources - -| SSIS Component | Microsoft Name | Spark Equivalent | Notes | -|----------------|----------------|------------------|-------| -| **OLE DB Source** | Microsoft.OLEDBSource | `spark.read.format("jdbc")` | Database reads via JDBC with PySpark | -| **Flat File Source** | Microsoft.FlatFileSource | `spark.read.csv()` or SQL `csv.\`path\`` | CSV, delimited, fixed-width files | -| **Excel Source** | Microsoft.ExcelSource | `spark.read.format("excel")` | Excel file reads | -| **Raw File Source** | Microsoft.RawSource | `spark.read.format("parquet")` | SSIS raw files converted to Parquet | - -#### Transformations - -| SSIS Component | Microsoft Name | Spark Equivalent | Notes | -|----------------|----------------|------------------|-------| -| **Aggregate** | Microsoft.Aggregate | `GROUP BY` with aggregation | Sum, count, avg, min, max operations | -| **Audit** | Microsoft.Audit | Add columns in SELECT | Add audit columns (timestamp, user, etc.) | -| **Cache Transform** | Microsoft.Cache | Temp views or CTEs | Cache data for lookup operations | -| **Character Map** | Microsoft.CharacterMap | String functions | String transformations (upper, lower, etc.) | -| **Conditional Split** | Microsoft.ConditionalSplit | Multiple `WHERE` clauses | Route rows by conditions | -| **Copy Column** | Microsoft.CopyColumn | Column in SELECT | Duplicate columns | -| **Data Conversion** | Microsoft.DataConvert | `CAST()` | Type conversions | -| **Derived Column** | Microsoft.DerivedColumn | Calculated columns in SELECT | Column transformations and calculations | -| **Lookup** | Microsoft.Lookup | `LEFT JOIN` | Reference data lookup with caching | -| **Merge** | Microsoft.Merge | `UNION` | Merge sorted datasets | -| **Merge Join** | Microsoft.MergeJoin | `JOIN` | Sorted input joins | -| **Multicast** | Microsoft.Multicast | Temp view reuse | Send data to multiple outputs | -| **OLE DB Command** | Microsoft.OLEDBCommand | Row-by-row SQL execution | Row-by-row SQL execution | -| **Percentage Sampling** | Microsoft.PercentageSampling | `TABLESAMPLE` | Statistical sampling | -| **Pivot** | Microsoft.Pivot | `PIVOT` clause | Pivot operations | -| **Row Count** | Microsoft.RowCount | `COUNT(*)` | Count rows and store in variable | -| **Script Component** | Microsoft.SqlServer.Dts.Pipeline.ScriptComponentHost | SQL UDFs or CASE statements | Custom transformations | -| **Slowly Changing Dimension** | Microsoft.SCD | `MERGE` with Delta Lake | SCD Type 1, 2, 3 patterns | -| **Sort** | Microsoft.Sort | `ORDER BY` | Sorting operations | -| **Union All** | Microsoft.UnionAll | `UNION ALL` | Combine multiple datasets | -| **Unpivot** | Microsoft.UnPivot | `UNPIVOT` or `STACK()` | Unpivot operations | - -#### Destinations - -| SSIS Component | Microsoft Name | Spark Equivalent | Notes | -|----------------|----------------|------------------|-------| -| **OLE DB Destination** | Microsoft.OLEDBDestination | `INSERT INTO` SQL or Delta Lake | Database writes via SQL statements | -| **Flat File Destination** | Microsoft.FlatFileDestination | `df.write.csv()` or SQL INSERT | CSV and delimited file writes | -| **Excel Destination** | Microsoft.ExcelDestination | `df.write.format('excel')` | Excel file writes | -| **Raw File Destination** | Microsoft.RawDestination | `df.write.format('parquet')` | SSIS raw files converted to Parquet | - -### Data Flow (Transformation) - Unsupported - -The following Data Flow components are **not supported** and would require manual conversion: - -| SSIS Component | Microsoft Name | Reason | -|----------------|----------------|--------| -| **Export Column** | Microsoft.ExportColumn | Implement using custom UDF with file writing | -| **Import Column** | Microsoft.ImportColumn | Implement using custom UDF with file reading | - -:::warning Script Component Conversion Difficulty -The **Script Component** often contains **C# or VB.NET code bodies** with complex row-by-row processing logic. While the converter identifies these components, the actual C#/VB code **cannot be automatically converted**. - -**Manual conversion required:** -- Analyze the C#/VB logic -- Rewrite as SQL UDFs or CASE statements -- Test thoroughly as row-by-row logic may need redesign for set-based SQL processing -::: - ---- - -## Data Flow Conversion - -### Derived Column Transformation - -**SSIS Derived Column:** -``` -Derived Column Transformation - Columns: - FullName = FirstName + " " + LastName - AgeGroup = Age < 30 ? "Young" : (Age < 60 ? "Middle" : "Senior") - ProcessDate = GETDATE() -``` - -**Converted Spark SQL:** -```python -# Processing node Package\Data Flow Task\Derived Column, type DERIVED_COLUMN -# component nameDerived Column -Derived_Column_Add_Columns = f""" -SELECT - *, - CONCAT(FirstName, ' ', LastName) AS FullName, - CASE - WHEN Age < 30 THEN 'Young' - WHEN Age < 60 THEN 'Middle' - ELSE 'Senior' - END AS AgeGroup, - CURRENT_TIMESTAMP() AS ProcessDate -FROM source_table -""" -Derived_Column_Add_Columns = spark.sql(Derived_Column_Add_Columns) -Derived_Column_Add_Columns.createOrReplaceTempView('Derived_Column_Add_Columns') -``` - -### Conditional Split - -**SSIS Conditional Split:** -``` -Conditional Split Transformation - Outputs: - HighValue: Amount > 1000 - MediumValue: Amount > 100 AND Amount <= 1000 - LowValue: Default Output -``` - -**Converted Spark SQL:** -```python -# Processing node Package\Data Flow Task\Conditional Split, type CONDITIONAL_SPLIT -# component nameConditional Split - -# High Value output -Conditional_Split_High_Value = f""" -SELECT * FROM source_table -WHERE Amount > 1000 -""" -Conditional_Split_High_Value = spark.sql(Conditional_Split_High_Value) -Conditional_Split_High_Value.createOrReplaceTempView('Conditional_Split_High_Value') - -# Medium Value output -Conditional_Split_Medium_Value = f""" -SELECT * FROM source_table -WHERE Amount > 100 AND Amount <= 1000 -""" -Conditional_Split_Medium_Value = spark.sql(Conditional_Split_Medium_Value) -Conditional_Split_Medium_Value.createOrReplaceTempView('Conditional_Split_Medium_Value') - -# Low Value output (default) -Conditional_Split_Low_Value = f""" -SELECT * FROM source_table -WHERE Amount <= 100 -""" -Conditional_Split_Low_Value = spark.sql(Conditional_Split_Low_Value) -Conditional_Split_Low_Value.createOrReplaceTempView('Conditional_Split_Low_Value') -``` - -### Lookup Transformation - -**SSIS Lookup:** -``` -Lookup Transformation - Reference Table: DimCustomerType - Join Columns: CustomerTypeID = TypeID - Lookup Columns: TypeName, TypeDescription - Cache Mode: Full Cache -``` - -**Converted Spark SQL:** -```python -# Processing node Package\Data Flow Task\Lookup Customer Type, type LOOKUP -# component nameLookup Customer Type -Lookup_Customer_Type = f""" -SELECT - src.*, - ref.TypeName, - ref.TypeDescription -FROM source_table src -LEFT JOIN dim.customer_type ref - ON src.CustomerTypeID = ref.TypeID -""" -Lookup_Customer_Type = spark.sql(Lookup_Customer_Type) -Lookup_Customer_Type.createOrReplaceTempView('Lookup_Customer_Type') -``` - ---- - -## Variables and Expressions - -### SSIS Variables - -**SSIS Package Variables:** -``` -Variables: - User::MaxProcessDate (DateTime) - User::RowCount (Int32) - User::SourceFolder (String) -``` - -**Converted Spark SQL (with Python variables):** -```python -# SSIS Package Variables converted to Python variables -V_MaxProcessDate = f'2024-01-01' -V_SourceFolder = f'/mnt/source/' - -# Use in SQL transformations via spark.sql() -Incremental_Data_Query = f""" -SELECT * -FROM source_table -WHERE ProcessDate > '{V_MaxProcessDate}' -""" -spark.sql(Incremental_Data_Query) -``` - -### SSIS Expressions - -**SSIS Expression Language:** -``` -File Connection Manager Expression: - @[User::SourceFolder] + "customers_" + - (DT_WSTR, 8) DATEPART("yyyy", GETDATE()) + - RIGHT("0" + (DT_WSTR, 2) DATEPART("mm", GETDATE()), 2) + ".csv" -``` - -**Converted Spark SQL:** -```python -from datetime import datetime - -# SSIS Variables -V_SourceFolder = f'/mnt/source/' - -# Build dynamic file path -current_date = datetime.now() -year = current_date.strftime("%Y") -month = current_date.strftime("%m") -V_FilePath = f"{V_SourceFolder}customers_{year}{month}.csv" - -# Read from dynamic path using spark.sql() -Read_Customers_Data = f"""SELECT * FROM csv.`{V_FilePath}`""" -Read_Customers_Data = spark.sql(Read_Customers_Data) -Read_Customers_Data.createOrReplaceTempView('Read_Customers_Data') -``` - ---- - -## Control Flow Patterns - -### ForEach Loop Container - -**SSIS ForEach Loop:** -``` -ForEach Loop Container (File Enumerator) - Folder: C:\Data\Input\ - Files: *.csv - - Tasks: - - Data Flow: Process each file - - Execute SQL: Log processing -``` - -**Converted Spark SQL:** -```python -# SSIS Variables -V_InputFolder = f'/mnt/data/input/' - -# Process multiple CSV files using wildcard pattern -ForEach_Read_All_Files = f""" -SELECT - *, - input_file_name() AS source_file, - CURRENT_TIMESTAMP() AS process_date -FROM csv.`{V_InputFolder}*.csv` -""" -ForEach_Read_All_Files = spark.sql(ForEach_Read_All_Files) -ForEach_Read_All_Files.createOrReplaceTempView('ForEach_Read_All_Files') - -# Insert into target table -ForEach_Insert_Customer_Data = f""" -INSERT INTO staging.customer_data -SELECT * FROM ForEach_Read_All_Files -""" -spark.sql(ForEach_Insert_Customer_Data) - -# Log processing -ForEach_Log_Processing = f""" -INSERT INTO logs.processing_log -SELECT - source_file AS file_path, - COUNT(*) AS row_count, - MAX(process_date) AS process_time -FROM ForEach_Read_All_Files -GROUP BY source_file -""" -spark.sql(ForEach_Log_Processing) -``` - -### Execute SQL Task - -**SSIS Execute SQL Task:** -``` -Execute SQL Task - Connection: DW_Connection - SQL Statement: - TRUNCATE TABLE staging.customer_temp; - - INSERT INTO staging.customer_temp - SELECT * FROM staging.customer_stage - WHERE process_date >= ?; - - Parameter Mapping: - Variable: User::MaxProcessDate → Parameter 0 -``` - -**Converted Spark SQL:** -```python -# SSIS Variable -V_MaxProcessDate = f'2024-01-01' - -# Processing node Package\Execute SQL Task, type EXECUTE_SQL -# component nameExecute SQL Task -Execute_SQL_Truncate = f"""TRUNCATE TABLE staging.customer_temp""" -spark.sql(Execute_SQL_Truncate) - -Execute_SQL_Insert = f""" -INSERT INTO staging.customer_temp -SELECT * FROM staging.customer_stage -WHERE process_date >= '{V_MaxProcessDate}' -""" -spark.sql(Execute_SQL_Insert) -``` - ---- - -## Script Component Conversion - -### Script Task (Control Flow) - -**SSIS Script Task (C#):** -```csharp -public void Main() -{ - string sourceFolder = Dts.Variables["User::SourceFolder"].Value.ToString(); - DateTime processDate = DateTime.Now; - - int fileCount = Directory.GetFiles(sourceFolder, "*.csv").Length; - - Dts.Variables["User::FileCount"].Value = fileCount; - - Dts.TaskResult = (int)ScriptResults.Success; -} -``` - -**Converted Python (with Spark SQL):** -```python -from datetime import datetime - -# SSIS Variables -V_SourceFolder = f'/mnt/source/' -V_ProcessDate = datetime.now() - -# Count files (requires Python/dbutils for file system operations) -file_list = dbutils.fs.ls(V_SourceFolder) -V_FileCount = str(len([f for f in file_list if f.path.endswith('.csv')])) -``` - -### Script Component (Data Flow) - -**SSIS Script Component (transformation):** -```csharp -public override void Input0_ProcessInputRow(Input0Buffer Row) -{ - // Custom business logic - if (Row.Amount > 1000) - { - Row.PriorityFlag = "HIGH"; - Row.DiscountRate = 0.15; - } - else - { - Row.PriorityFlag = "NORMAL"; - Row.DiscountRate = 0.05; - } - - Row.ProcessedDate = DateTime.Now; -} -``` - -**Converted Spark SQL:** -```python -# Processing node Package\Data Flow Task\Script Component, type SCRIPT_COMPONENT -# component nameScript Component -Script_Component_Custom_Logic = f""" -SELECT - *, - CASE - WHEN Amount > 1000 THEN 'HIGH' - ELSE 'NORMAL' - END AS PriorityFlag, - CASE - WHEN Amount > 1000 THEN 0.15 - ELSE 0.05 - END AS DiscountRate, - CURRENT_TIMESTAMP() AS ProcessedDate -FROM source_table -""" -Script_Component_Custom_Logic = spark.sql(Script_Component_Custom_Logic) -Script_Component_Custom_Logic.createOrReplaceTempView('Script_Component_Custom_Logic') -``` - ---- - -## Package Configurations - -### Connection Managers - -**SSIS Connection Managers:** -```xml - - - SourceDB - - Data Source=source-server;Initial Catalog=SourceDB; - Integrated Security=True; - - - -``` - -**Converted Databricks:** -```python -# Processing node Package\Data Flow Task\OLE DB Source, type SOURCE -# component nameOLE DB Source - SourceDB Connection -Connection_Manager_SourceDB = spark.read \ - .format("jdbc") \ - .option("url", "source-db-url") \ - .option("dbtable", "dbo.Customers") \ - .option("user", "source-db-user") \ - .option("password", "source-db-password") \ - .load() -Connection_Manager_SourceDB.createOrReplaceTempView('Connection_Manager_SourceDB') -``` - ---- - -## Conversion Example - -### Complete SSIS Package - -**SSIS Package: [LoadCustomerData.dtsx](/downloads/LoadCustomerData.dtsx)** (download sample) - -Control Flow: -1. Execute SQL Task: Get Destination Table Name or File Name (with ProcessName parameter) -2. Execute SQL Task: Truncate Staging Table -3. Execute SQL Task: Load Customer Data -4. Execute SQL Task: Update Control Table (with ProcessName parameter) - -Parameters: -- ProcessName (String): "CustomerETL" - -**Converted Spark SQL Notebook:** - -```python -# Databricks notebook source - -ProcessName = f'CustomerETL' - -# COMMAND ---------- - -# Processing node Package\Get Destination Table Name or File Name, type EXECUTE_SQL -# component nameGet Destination Table Name or File Name -# input parameters : - # ProcessName -# output parameters : -Package_Get_Destination_Table_Name_or_File_Name = f"""DECLARE VARIABLE V_TblNm STRING; -call aud.spGetPackageDesTbl( {ProcessName}, V_TblNm ); - -SELECT V_TblNm AS DestinationTable;""" -spark.sql(Package_Get_Destination_Table_Name_or_File_Name) - -# COMMAND ---------- - -# Processing node Package\Truncate Staging Table, type EXECUTE_SQL -# component nameTruncate Staging Table -# input parameters : -# output parameters : -Package_Truncate_Staging_Table = f"""TRUNCATE TABLE staging.customers;""" -spark.sql(Package_Truncate_Staging_Table) - -# COMMAND ---------- - -# Processing node Package\Load Customer Data, type EXECUTE_SQL -# component nameLoad Customer Data -# input parameters : -# output parameters : -Package_Load_Customer_Data = f"""INSERT INTO staging.customers -(CustomerID, CustomerName, Email, Status, ProcessDate) -SELECT - CustomerID, - CustomerName, - Email, - Status, - current_timestamp() AS ProcessDate -FROM source.customers -WHERE Status = 'Active';""" -spark.sql(Package_Load_Customer_Data) - -# COMMAND ---------- - -# Processing node Package\Update Control Table, type EXECUTE_SQL -# component nameUpdate Control Table -# input parameters : - # ProcessName -# output parameters : -Package_Update_Control_Table = f"""INSERT INTO control.load_log -(process_name, rows_processed, process_date) -SELECT - {ProcessName} AS process_name, - COUNT(*) AS rows_processed, - current_timestamp() AS process_date -FROM dw.customers -WHERE ProcessDate >= CAST(current_timestamp() AS DATE);""" -spark.sql(Package_Update_Control_Table) -``` - ---- - -## Next Steps - -1. **Export SSIS packages** to DTSX files -2. **Run conversion** using Lakebridge CLI: - ```bash - databricks labs lakebridge transpile \ - --source-dialect ssis \ - --input-source /path/to/ssis/packages \ - --output-folder /output/sparksql \ - --target-technology sparksql - ``` -3. **Review generated notebooks** for conversion warnings -4. **Configure Databricks secrets** for connection strings -5. **Test with sample data** in Databricks -6. **Deploy workflows** to production - -For more information, see: -- [Main ETL Conversion Guide](../../) -- [BladeBridge Configuration](../../pluggable_transpilers/bladebridge/bladebridge_configuration) -- [Transpile CLI Reference](../../) diff --git a/docs/lakebridge/docs/transpile/source_systems/ssis/examples.mdx b/docs/lakebridge/docs/transpile/source_systems/ssis/examples.mdx new file mode 100644 index 0000000000..bbd69873c2 --- /dev/null +++ b/docs/lakebridge/docs/transpile/source_systems/ssis/examples.mdx @@ -0,0 +1,341 @@ +--- +sidebar_position: 3 +title: SSIS Conversion Examples +toc_min_heading_level: 2 +toc_max_heading_level: 3 +--- + +# SSIS Conversion Examples + +Before/after examples showing how SSIS constructs are converted to Databricks. For the component support matrix, see [SSIS Supported Components](/docs/transpile/source_systems/ssis/supported_components). + +--- + +## Data Flow Transformations + +### Derived Column + +**SSIS:** +``` +Derived Column Transformation + Columns: + FullName = FirstName + " " + LastName + AgeGroup = Age < 30 ? "Young" : (Age < 60 ? "Middle" : "Senior") + ProcessDate = GETDATE() +``` + +**Databricks:** +```python +Derived_Column_Add_Columns = f""" +SELECT + *, + CONCAT(FirstName, ' ', LastName) AS FullName, + CASE + WHEN Age < 30 THEN 'Young' + WHEN Age < 60 THEN 'Middle' + ELSE 'Senior' + END AS AgeGroup, + CURRENT_TIMESTAMP() AS ProcessDate +FROM source_table +""" +Derived_Column_Add_Columns = spark.sql(Derived_Column_Add_Columns) +Derived_Column_Add_Columns.createOrReplaceTempView('Derived_Column_Add_Columns') +``` + +--- + +### Conditional Split + +**SSIS:** +``` +Conditional Split Transformation + Outputs: + HighValue: Amount > 1000 + MediumValue: Amount > 100 AND Amount <= 1000 + LowValue: Default Output +``` + +**Databricks:** +```python +Conditional_Split_High_Value = f""" +SELECT * FROM source_table WHERE Amount > 1000 +""" +Conditional_Split_High_Value = spark.sql(Conditional_Split_High_Value) +Conditional_Split_High_Value.createOrReplaceTempView('Conditional_Split_High_Value') + +Conditional_Split_Medium_Value = f""" +SELECT * FROM source_table WHERE Amount > 100 AND Amount <= 1000 +""" +Conditional_Split_Medium_Value = spark.sql(Conditional_Split_Medium_Value) +Conditional_Split_Medium_Value.createOrReplaceTempView('Conditional_Split_Medium_Value') + +Conditional_Split_Low_Value = f""" +SELECT * FROM source_table WHERE Amount <= 100 +""" +Conditional_Split_Low_Value = spark.sql(Conditional_Split_Low_Value) +Conditional_Split_Low_Value.createOrReplaceTempView('Conditional_Split_Low_Value') +``` + +--- + +### Lookup Transformation + +**SSIS:** +``` +Lookup Transformation + Reference Table: DimCustomerType + Join Columns: CustomerTypeID = TypeID + Lookup Columns: TypeName, TypeDescription + Cache Mode: Full Cache +``` + +**Databricks:** +```python +Lookup_Customer_Type = f""" +SELECT + src.*, + ref.TypeName, + ref.TypeDescription +FROM source_table src +LEFT JOIN dim.customer_type ref + ON src.CustomerTypeID = ref.TypeID +""" +Lookup_Customer_Type = spark.sql(Lookup_Customer_Type) +Lookup_Customer_Type.createOrReplaceTempView('Lookup_Customer_Type') +``` + +--- + +## Variables and Expressions + +### SSIS Variables + +**SSIS:** +``` +Variables: + User::MaxProcessDate (DateTime) + User::RowCount (Int32) + User::SourceFolder (String) +``` + +**Databricks:** +```python +V_MaxProcessDate = f'2024-01-01' +V_SourceFolder = f'/mnt/source/' + +Incremental_Data_Query = f""" +SELECT * +FROM source_table +WHERE ProcessDate > '{V_MaxProcessDate}' +""" +spark.sql(Incremental_Data_Query) +``` + +--- + +### SSIS Expressions + +**SSIS:** +``` +@[User::SourceFolder] + "customers_" + +(DT_WSTR, 8) DATEPART("yyyy", GETDATE()) + +RIGHT("0" + (DT_WSTR, 2) DATEPART("mm", GETDATE()), 2) + ".csv" +``` + +**Databricks:** +```python +from datetime import datetime + +V_SourceFolder = f'/mnt/source/' +current_date = datetime.now() +year = current_date.strftime("%Y") +month = current_date.strftime("%m") +V_FilePath = f"{V_SourceFolder}customers_{year}{month}.csv" + +Read_Customers_Data = f"""SELECT * FROM csv.`{V_FilePath}`""" +Read_Customers_Data = spark.sql(Read_Customers_Data) +Read_Customers_Data.createOrReplaceTempView('Read_Customers_Data') +``` + +--- + +## Control Flow Patterns + +### ForEach Loop Container + +**SSIS:** +``` +ForEach Loop Container (File Enumerator) + Folder: C:\Data\Input\ + Files: *.csv + Tasks: Data Flow, Execute SQL (log processing) +``` + +**Databricks:** +```python +V_InputFolder = f'/mnt/data/input/' + +ForEach_Read_All_Files = f""" +SELECT + *, + input_file_name() AS source_file, + CURRENT_TIMESTAMP() AS process_date +FROM csv.`{V_InputFolder}*.csv` +""" +ForEach_Read_All_Files = spark.sql(ForEach_Read_All_Files) +ForEach_Read_All_Files.createOrReplaceTempView('ForEach_Read_All_Files') + +ForEach_Insert_Customer_Data = f""" +INSERT INTO staging.customer_data +SELECT * FROM ForEach_Read_All_Files +""" +spark.sql(ForEach_Insert_Customer_Data) + +ForEach_Log_Processing = f""" +INSERT INTO logs.processing_log +SELECT source_file AS file_path, COUNT(*) AS row_count, MAX(process_date) AS process_time +FROM ForEach_Read_All_Files +GROUP BY source_file +""" +spark.sql(ForEach_Log_Processing) +``` + +--- + +### Execute SQL Task + +**SSIS:** +``` +Execute SQL Task + SQL Statement: + TRUNCATE TABLE staging.customer_temp; + INSERT INTO staging.customer_temp + SELECT * FROM staging.customer_stage WHERE process_date >= ?; + Parameter Mapping: + User::MaxProcessDate → Parameter 0 +``` + +**Databricks:** +```python +V_MaxProcessDate = f'2024-01-01' + +Execute_SQL_Truncate = f"""TRUNCATE TABLE staging.customer_temp""" +spark.sql(Execute_SQL_Truncate) + +Execute_SQL_Insert = f""" +INSERT INTO staging.customer_temp +SELECT * FROM staging.customer_stage +WHERE process_date >= '{V_MaxProcessDate}' +""" +spark.sql(Execute_SQL_Insert) +``` + +--- + +## Script Component Conversion + +### Script Task (Control Flow) + +**SSIS (C#):** +```csharp +public void Main() +{ + string sourceFolder = Dts.Variables["User::SourceFolder"].Value.ToString(); + int fileCount = Directory.GetFiles(sourceFolder, "*.csv").Length; + Dts.Variables["User::FileCount"].Value = fileCount; + Dts.TaskResult = (int)ScriptResults.Success; +} +``` + +**Databricks (Python):** +```python +V_SourceFolder = f'/mnt/source/' +file_list = dbutils.fs.ls(V_SourceFolder) +V_FileCount = str(len([f for f in file_list if f.path.endswith('.csv')])) +``` + +--- + +### Script Component (Data Flow) + +**SSIS (C#):** +```csharp +public override void Input0_ProcessInputRow(Input0Buffer Row) +{ + if (Row.Amount > 1000) { + Row.PriorityFlag = "HIGH"; + Row.DiscountRate = 0.15; + } else { + Row.PriorityFlag = "NORMAL"; + Row.DiscountRate = 0.05; + } + Row.ProcessedDate = DateTime.Now; +} +``` + +**Databricks:** +```python +Script_Component_Custom_Logic = f""" +SELECT + *, + CASE WHEN Amount > 1000 THEN 'HIGH' ELSE 'NORMAL' END AS PriorityFlag, + CASE WHEN Amount > 1000 THEN 0.15 ELSE 0.05 END AS DiscountRate, + CURRENT_TIMESTAMP() AS ProcessedDate +FROM source_table +""" +Script_Component_Custom_Logic = spark.sql(Script_Component_Custom_Logic) +Script_Component_Custom_Logic.createOrReplaceTempView('Script_Component_Custom_Logic') +``` + +--- + +## Complete Package Example + +**SSIS Package: [LoadCustomerData.dtsx](/downloads/LoadCustomerData.dtsx)** + +Control Flow: +1. Execute SQL Task: Get Destination Table Name +2. Execute SQL Task: Truncate Staging Table +3. Execute SQL Task: Load Customer Data +4. Execute SQL Task: Update Control Table + +**Converted Databricks Notebook:** + +```python +# Databricks notebook source + +ProcessName = f'CustomerETL' + +# COMMAND ---------- +# Get Destination Table Name +Package_Get_Destination_Table_Name = f""" +DECLARE VARIABLE V_TblNm STRING; +CALL aud.spGetPackageDesTbl({ProcessName}, V_TblNm); +SELECT V_TblNm AS DestinationTable; +""" +spark.sql(Package_Get_Destination_Table_Name) + +# COMMAND ---------- +# Truncate Staging Table +spark.sql(f"""TRUNCATE TABLE staging.customers;""") + +# COMMAND ---------- +# Load Customer Data +Package_Load_Customer_Data = f""" +INSERT INTO staging.customers (CustomerID, CustomerName, Email, Status, ProcessDate) +SELECT CustomerID, CustomerName, Email, Status, current_timestamp() AS ProcessDate +FROM source.customers +WHERE Status = 'Active'; +""" +spark.sql(Package_Load_Customer_Data) + +# COMMAND ---------- +# Update Control Table +Package_Update_Control_Table = f""" +INSERT INTO control.load_log (process_name, rows_processed, process_date) +SELECT {ProcessName} AS process_name, COUNT(*) AS rows_processed, current_timestamp() AS process_date +FROM dw.customers +WHERE ProcessDate >= CAST(current_timestamp() AS DATE); +""" +spark.sql(Package_Update_Control_Table) +``` diff --git a/docs/lakebridge/docs/transpile/source_systems/ssis/index.mdx b/docs/lakebridge/docs/transpile/source_systems/ssis/index.mdx new file mode 100644 index 0000000000..e7f51c7767 --- /dev/null +++ b/docs/lakebridge/docs/transpile/source_systems/ssis/index.mdx @@ -0,0 +1,87 @@ +--- +sidebar_position: 1 +title: SSIS +toc_min_heading_level: 2 +toc_max_heading_level: 3 +--- + +# Microsoft SSIS to Databricks + +## Conversion Information + +- **Transpiler:** BladeBridge +- **Available target:** Databricks notebooks (experimental) + +### Supported SSIS Versions + +- SQL Server 2012, 2014, 2016, 2017, 2019, 2022 +- Azure Data Factory SSIS Integration Runtime + +### Input Requirements + +Export your SSIS packages as DTSX files: + +1. **Solution Export:** Export from Visual Studio / SQL Server Data Tools (SSDT) +2. **File System Packages:** Direct DTSX file access +3. **SSISDB Export:** Extract from SSIS catalog + +```sql +-- Extract package from SSISDB catalog +DECLARE @packageData VARBINARY(MAX) +SELECT @packageData = [packagedata] +FROM [SSISDB].[catalog].[packages] +WHERE [name] = 'YourPackageName' +-- Save to file system for conversion +``` + +--- + +## Running the Conversion + +```bash +databricks labs lakebridge transpile \ + --source-dialect ssis \ + --input-source /path/to/ssis/packages \ + --output-folder /output/sparksql \ + --target-technology sparksql +``` + +The transpiler recursively scans the input directory for `.dtsx` files and generates Databricks notebook equivalents in the output folder. + +:::warning Script Component Limitations +SSIS **Script Task** and **Script Component** contain C# or VB.NET code bodies that cannot be automatically converted. The converter preserves the logic structure, but the actual implementation must be rewritten in Python. See [Supported Components](/docs/transpile/source_systems/ssis/supported_components#control-flow-orchestration---unsupported) for details. +::: + +--- + +## What Gets Converted + +| SSIS Concept | Databricks Equivalent | +|---|---| +| Control Flow Tasks | Notebook cells / `dbutils.notebook.run()` | +| Data Flow Task | Spark SQL temp views | +| Variables | Python variables | +| SSIS Expressions | Python f-strings / `spark.sql()` calls | +| Connection Managers | JDBC `spark.read.format("jdbc")` | +| ForEach Loop | Wildcard file reads with `input_file_name()` | +| Script Task | Python with `dbutils` | + +For the full list of supported and unsupported components, see [SSIS Supported Components](/docs/transpile/source_systems/ssis/supported_components). + +For conversion examples with before/after code, see [SSIS Conversion Examples](/docs/transpile/source_systems/ssis/examples). + +--- + +## Next Steps + +1. Export SSIS packages to DTSX files +2. Run conversion (command above) +3. Review generated notebooks for conversion warnings +4. Configure Databricks secrets for connection strings +5. Test with sample data in Databricks +6. Deploy workflows to production + +For more information, see: +- [SSIS Supported Components](/docs/transpile/source_systems/ssis/supported_components) +- [SSIS Conversion Examples](/docs/transpile/source_systems/ssis/examples) +- [BladeBridge Configuration](/docs/transpile/pluggable_transpilers/bladebridge/bladebridge_configuration) diff --git a/docs/lakebridge/docs/transpile/source_systems/ssis/supported_components.mdx b/docs/lakebridge/docs/transpile/source_systems/ssis/supported_components.mdx new file mode 100644 index 0000000000..cc0e393813 --- /dev/null +++ b/docs/lakebridge/docs/transpile/source_systems/ssis/supported_components.mdx @@ -0,0 +1,111 @@ +--- +sidebar_position: 2 +title: SSIS Supported Components +toc_min_heading_level: 2 +toc_max_heading_level: 3 +--- + +# SSIS Supported Components + +This page lists all SSIS components and their Databricks equivalents. For usage and conversion examples, see [SSIS Conversion Examples](/docs/transpile/source_systems/ssis/examples). + +--- + +## Control Flow (Orchestration) - Supported + +| SSIS Component | Microsoft Name | Spark Equivalent | Notes | +|---|---|---|---| +| **Data Flow Task** | Microsoft.DataFlowTask | Spark SQL temp views | Core data transformations | +| **Execute SQL Task** | Microsoft.ExecuteSQLTask | Spark SQL statements | SQL execution with parameter mapping | +| **Execute Package Task** | Microsoft.ExecutePackageTask | `dbutils.notebook.run()` | Nested notebook execution | +| **File System Task** | Microsoft.FileSystemTask | `dbutils.fs` commands | File operations (copy, move, delete, rename) | +| **Script Task** | Microsoft.ScriptTask | Python code with SQL | C#/VB.NET scripts converted to Python/SQL | +| **For Loop Container** | STOCK:FORLOOP | SQL iteration pattern | Iteration with counter | +| **Foreach Loop Container** | STOCK:FOREACHLOOP | SQL wildcard file reading | File/folder iteration | +| **Sequence Container** | STOCK:SEQUENCE | Function or notebook section | Grouping tasks | +| **Execute Process Task** | Microsoft.ExecuteProcess | `subprocess.run()` | External process execution | +| **Extensible File Task** | ExtensibleFileTask | `dbutils.fs` commands | Extended file operations | + +--- + +## Control Flow (Orchestration) - Unsupported + +The following components are **not supported** and require manual conversion: + +| SSIS Component | Microsoft Name | Reason | +|---|---|---| +| **Analysis Services Execute DDL** | Microsoft.AnalysisServicesExecuteDDLTask | SSAS-specific; migrate to Delta/SQL manually | +| **Analysis Services Processing** | Microsoft.AnalysisServicesProcessingTask | SSAS-specific; migrate to Delta/SQL manually | +| **Bulk Insert Task** | Microsoft.BulkInsertTask | Use Data Flow Task with JDBC/Delta | +| **Data Profiling Task** | Microsoft.DataProfilingTask | Use Databricks Data Profile UI or custom profiling | +| **FTP Task** | Microsoft.FTPTask | Implement using Python `ftplib` or `dbutils.fs` | +| **Message Queue Task** | Microsoft.MessageQueueTask | Requires custom Kafka/Event Hub integration | +| **Send Mail Task** | Microsoft.SendMailTask | Use Databricks job notifications or Python `smtplib` | +| **Web Service Task** | Microsoft.WebServiceTask | Implement using Python `requests` | +| **WMI Data Reader Task** | Microsoft.WMIDataReaderTask | Windows-specific; requires alternative monitoring | +| **XML Task** | Microsoft.XMLTask | Implement using Python `xml.etree` or PySpark XML | + +:::warning Script Task Conversion +**Script Task** contains C# or VB.NET code bodies that cannot be automatically converted. The converter preserves the logic structure but the implementation must be rewritten in Python manually. +::: + +--- + +## Data Flow (Transformation) - Supported + +### Sources + +| SSIS Component | Microsoft Name | Spark Equivalent | Notes | +|---|---|---|---| +| **OLE DB Source** | Microsoft.OLEDBSource | `spark.read.format("jdbc")` | Database reads via JDBC | +| **Flat File Source** | Microsoft.FlatFileSource | `spark.read.csv()` or SQL `csv.\`path\`` | CSV, delimited, fixed-width files | +| **Excel Source** | Microsoft.ExcelSource | `spark.read.format("excel")` | Excel file reads | +| **Raw File Source** | Microsoft.RawSource | `spark.read.format("parquet")` | SSIS raw files converted to Parquet | + +### Transformations + +| SSIS Component | Microsoft Name | Spark Equivalent | Notes | +|---|---|---|---| +| **Aggregate** | Microsoft.Aggregate | `GROUP BY` with aggregation | Sum, count, avg, min, max | +| **Audit** | Microsoft.Audit | Add columns in SELECT | Add audit columns (timestamp, user, etc.) | +| **Cache Transform** | Microsoft.Cache | Temp views or CTEs | Cache data for lookups | +| **Character Map** | Microsoft.CharacterMap | String functions | upper, lower, etc. | +| **Conditional Split** | Microsoft.ConditionalSplit | Multiple `WHERE` clauses | Route rows by conditions | +| **Copy Column** | Microsoft.CopyColumn | Column in SELECT | Duplicate columns | +| **Data Conversion** | Microsoft.DataConvert | `CAST()` | Type conversions | +| **Derived Column** | Microsoft.DerivedColumn | Calculated columns in SELECT | Column transformations | +| **Lookup** | Microsoft.Lookup | `LEFT JOIN` | Reference data lookup | +| **Merge** | Microsoft.Merge | `UNION` | Merge sorted datasets | +| **Merge Join** | Microsoft.MergeJoin | `JOIN` | Sorted input joins | +| **Multicast** | Microsoft.Multicast | Temp view reuse | Send data to multiple outputs | +| **OLE DB Command** | Microsoft.OLEDBCommand | Row-by-row SQL execution | Row-level SQL | +| **Percentage Sampling** | Microsoft.PercentageSampling | `TABLESAMPLE` | Statistical sampling | +| **Pivot** | Microsoft.Pivot | `PIVOT` clause | Pivot operations | +| **Row Count** | Microsoft.RowCount | `COUNT(*)` | Count rows into variable | +| **Script Component** | Microsoft.SqlServer.Dts.Pipeline.ScriptComponentHost | SQL UDFs or CASE statements | Custom transformations | +| **Slowly Changing Dimension** | Microsoft.SCD | `MERGE` with Delta Lake | SCD Type 1, 2, 3 patterns | +| **Sort** | Microsoft.Sort | `ORDER BY` | Sorting | +| **Union All** | Microsoft.UnionAll | `UNION ALL` | Combine datasets | +| **Unpivot** | Microsoft.UnPivot | `UNPIVOT` or `STACK()` | Unpivot | + +### Destinations + +| SSIS Component | Microsoft Name | Spark Equivalent | Notes | +|---|---|---|---| +| **OLE DB Destination** | Microsoft.OLEDBDestination | `INSERT INTO` SQL or Delta Lake | Database writes | +| **Flat File Destination** | Microsoft.FlatFileDestination | `df.write.csv()` or SQL INSERT | CSV/delimited writes | +| **Excel Destination** | Microsoft.ExcelDestination | `df.write.format('excel')` | Excel writes | +| **Raw File Destination** | Microsoft.RawDestination | `df.write.format('parquet')` | Parquet writes | + +--- + +## Data Flow (Transformation) - Unsupported + +| SSIS Component | Microsoft Name | Reason | +|---|---|---| +| **Export Column** | Microsoft.ExportColumn | Implement using custom UDF with file writing | +| **Import Column** | Microsoft.ImportColumn | Implement using custom UDF with file reading | + +:::warning Script Component Conversion +The **Script Component** contains C# or VB.NET row-by-row processing logic that cannot be automatically converted. The converter identifies the component but the actual C#/VB code must be rewritten as SQL UDFs or CASE statements, then tested thoroughly since row-by-row logic often needs redesign for set-based SQL. +::: diff --git a/docs/lakebridge/docusaurus.config.ts b/docs/lakebridge/docusaurus.config.ts index be247da3b5..0c80bee2d4 100644 --- a/docs/lakebridge/docusaurus.config.ts +++ b/docs/lakebridge/docusaurus.config.ts @@ -196,7 +196,7 @@ const config: Config = { label: "Overview" }, { - to: '/docs/installation/', + to: '/docs/getting_started/', position: 'left', label: "Get Started" }, diff --git a/docs/lakebridge/package.json b/docs/lakebridge/package.json index 0eef92b10e..baa9118e50 100644 --- a/docs/lakebridge/package.json +++ b/docs/lakebridge/package.json @@ -59,7 +59,7 @@ ] }, "engines": { - "node": ">=25.0 <26" + "node": ">=26.0 <27" }, "engineStrict": true, "resolutions": { diff --git a/docs/lakebridge/src/pages/index.tsx b/docs/lakebridge/src/pages/index.tsx index 023c69064a..2a87fffa76 100644 --- a/docs/lakebridge/src/pages/index.tsx +++ b/docs/lakebridge/src/pages/index.tsx @@ -36,7 +36,7 @@ const Hero = () => { />