DC-307 feat: Initial Integration of ingestion logic to CI/CD flow thr…

.github/workflows/databricks-asset-bundle-deploy.yml

@@ -0,0 +1,55 @@
+name: Deploy Migration Accelerator through Databricks Asset Bundles
+
+on:
+  push:
+    branches: [ main, develop ]
+  pull_request:
+    branches: [ main, develop ]
+    types: [ opened, synchronize, reopened ]
+  workflow_dispatch:
+
+env:
+  DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }}
+  DATABRICKS_CLIENT_ID: ${{ secrets.DATABRICKS_CLIENT_ID }}
+  DATABRICKS_CLIENT_SECRET: ${{ secrets.DATABRICKS_CLIENT_SECRET }}
+  DATABRICKS_AUTH_TYPE: oauth-m2m
+  POETRY_VERSION: "2.2.1"
+  PYTHON_VERSION: "3.12"
+
+jobs:
+  deploy-bundle:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ env.PYTHON_VERSION }}
+
+    - name: Install pipx and Poetry
+      run: |
+        python -m pip install --upgrade pip pipx
+        pipx install poetry==${{ env.POETRY_VERSION }}
+        poetry --version
+
+    - name: Install project dependencies
+      run: |
+        poetry install --no-interaction --no-ansi
+
+    - name: Install Databricks CLI
+      uses: databricks/setup-cli@main
+
+    - name: Validate bundle
+      run: |
+        databricks bundle validate
+
+    - name: Deploy bundle
+      run: |
+        databricks bundle deploy -t dev
+        databricks bundle summary
+        databricks bundle resources

.gitignore

@@ -82,6 +82,12 @@ target/
 profile_default/
 ipython_config.py
 
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+
 # pyenv
 #   For a library or package, you might want to ignore these files since the code is
 #   intended to run in multiple environments; otherwise, check them in:
@@ -201,6 +207,18 @@ cython_debug/
 .cursorignore
 .cursorindexingignore
 
+# Databricks
+.databricks/
+*.log
+
+# Logs
+logs/
+*.log
+
+# Build artifacts
+dist/
+build/
+
 # Marimo
 marimo/_static/
 marimo/_lsp/

databricks.yml

@@ -0,0 +1,9 @@
+bundle:
+  name: migration-accelerator
+
+include:
+  - resources/*.yml   
+
+targets:
+  dev:
+    default: true

pyproject.toml

@@ -0,0 +1,25 @@
+[tool.poetry]
+name = "migration_accelerator_package"
+version = "0.0.1"
+description = "Reusable python logic for Snowflake - Databricks Migration."
+authors = ["Samuel Solarte samuel.solarte@qubika.com"]
+packages = [{ include = "migration_accelerator_package", from = "src" }]
+
+
+[tool.poetry.dependencies]
+python = ">=3.12,<3.14"
+requests = ">=2.31.0"
+snowflake-connector-python = ">=3.0.0"
+snowflake-snowpark-python = ">=1.9.0"
+python-dotenv = ">=1.0.0"
+databricks-sdk = "*"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry.scripts]
+migration-accelerator = "migration_accelerator_package.main:main"
+snowpark-reader = "migration_accelerator_package.snowpark:main"
+

resources/artifacts.yml

@@ -0,0 +1,9 @@
+artifacts:
+  migration_accelerator_package:
+    type: whl
+    build: |
+      cd "$PROJECT_ROOT"
+      poetry build
+    path: ../
+
+

resources/jobs.yml

@@ -0,0 +1,26 @@
+resources:
+  jobs:
+    snowflake_ingestion_job:
+      name: "snowflake_ingestion_job"
+
+      permissions: 
+        - level: CAN_MANAGE_RUN
+          group_name: migration-accelerator-devs
+
+
+      tasks:
+        - task_key: snowflake_ingestion_task
+          python_wheel_task:
+            entry_point: snowpark-reader
+            package_name: migration_accelerator_package
+          environment_key: "serverless_wheel"
+
+      environments:
+        - environment_key: "serverless_wheel"
+          spec:
+            environment_version: "4"
+            dependencies:
+              - "../dist/*.whl"
+
+      timeout_seconds: 3600
+      max_concurrent_runs: 1

resources/unity_catalog.yml

@@ -0,0 +1,25 @@
+variables:
+  catalog_name: 
+    description: "The Unity Catalog in which the project will be contained."
+    default: "qubika_partner_solutions"
+
+resources:
+  schemas:
+    migration_accelerator:
+      name: migration_accelerator
+      catalog_name: ${var.catalog_name}
+      comment: Contains the Unity Catalog Resources pertaining to the Databricks Governance Migration Agent Accelerator
+      grants: 
+        - principal: migration-accelerator-devs
+          privileges: 
+            - ALL_PRIVILEGES
+
+  volumes:
+    snowflake_artifacts_raw:
+      name: snowflake_artifacts_raw
+      schema_name: migration_accelerator
+      catalog_name: ${var.catalog_name}
+      comment: Contains the metadata of the extracted Snowflake artifacts in a raw JSON format. 
+
+
+

src/migration_accelerator_package/constants.py

@@ -0,0 +1,12 @@
+from enum import Enum
+
+class SnowflakeConfig(Enum):
+    SNOWFLAKE_ROLE = "SYSADMIN"
+    SNOWFLAKE_WAREHOUSE = "COMPUTE_WH"
+    SNOWFLAKE_DATABASE = "DATA_MIGRATION_DB"
+    SNOWFLAKE_SCHEMA = "DATA_MIGRATION_SCHEMA"
+
+class UnityCatalogConfig(Enum):
+    CATALOG = "qubika_partner_solutions"
+    SCHEMA = "migration_accelerator"
+    RAW_VOLUME = "snowflake_artifacts_raw"

src/migration_accelerator_package/main.py

@@ -0,0 +1,8 @@
+# src/migration_accelerator_package/main.py
+
+def main():
+    """Main entry point for the Snowflake to Databricks Migration Accelerator package CLI."""
+    print("Migration Accelerator package is running.")
+
+if __name__ == "__main__":
+    main()