Merge pull request #32 from Ritik574-coder/dbt_branch #68
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: dbt CI | |
| on: | |
| pull_request: | |
| branches: [main] | |
| push: | |
| branches: [main] | |
| concurrency: | |
| group: dbt-ci-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| DBT_PROFILES_DIR: .github/profiles | |
| DBT_SQLSERVER_HOST: localhost | |
| DBT_SQLSERVER_PORT: 1433 | |
| DBT_SQLSERVER_DATABASE: RetailDB | |
| DBT_SQLSERVER_SCHEMA: dbt_ci | |
| DBT_SQLSERVER_USER: sa | |
| DBT_SQLSERVER_PASSWORD: TestPassword123! | |
| PYTHON_VERSION: "3.11" | |
| jobs: | |
| lint-and-parse: | |
| name: Lint & parse | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| cache: pip | |
| cache-dependency-path: requirements.txt | |
| - name: Install ODBC Driver 18 | |
| run: | | |
| curl -fsSL https://packages.microsoft.com/keys/microsoft.asc \ | |
| | sudo gpg --dearmor --batch --yes \ | |
| -o /usr/share/keyrings/microsoft-prod.gpg | |
| echo "deb [arch=amd64 signed-by=/usr/share/keyrings/microsoft-prod.gpg] https://packages.microsoft.com/ubuntu/22.04/prod jammy main" \ | |
| | sudo tee /etc/apt/sources.list.d/microsoft-prod.list | |
| sudo apt-get update | |
| sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 mssql-tools18 unixodbc-dev | |
| echo "/opt/mssql-tools18/bin" >> "$GITHUB_PATH" | |
| - name: Install dependencies | |
| run: pip install -r requirements.txt | |
| - name: Validate YAML files | |
| run: | | |
| python - <<'PY' | |
| import yaml | |
| from pathlib import Path | |
| for path in Path(".").rglob("*.yml"): | |
| if ".git" in path.parts or "target" in path.parts: | |
| continue | |
| with path.open() as f: | |
| yaml.safe_load(f) | |
| print(f"OK: {path}") | |
| PY | |
| - name: Install dbt packages | |
| run: dbt deps | |
| - name: Parse dbt project | |
| run: dbt parse | |
| - name: Lint SQL (when models exist) | |
| run: | | |
| if find models -name '*.sql' -print -quit | grep -q .; then | |
| sqlfluff lint models --ignore=templating | |
| else | |
| echo "No SQL models found — skipping sqlfluff." | |
| fi | |
| dbt-integration: | |
| name: dbt integration | |
| runs-on: ubuntu-latest | |
| needs: lint-and-parse | |
| services: | |
| sqlserver: | |
| image: mcr.microsoft.com/mssql/server:2022-latest | |
| env: | |
| ACCEPT_EULA: Y | |
| MSSQL_SA_PASSWORD: TestPassword123! | |
| ports: | |
| - 1433:1433 | |
| options: >- | |
| --health-cmd "/opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P TestPassword123! -C -Q 'SELECT 1' || exit 1" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 12 | |
| --health-start-period 30s | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| cache: pip | |
| cache-dependency-path: requirements.txt | |
| - name: Install ODBC Driver 18 | |
| run: | | |
| curl -fsSL https://packages.microsoft.com/keys/microsoft.asc \ | |
| | sudo gpg --dearmor --batch --yes \ | |
| -o /usr/share/keyrings/microsoft-prod.gpg | |
| echo "deb [arch=amd64 signed-by=/usr/share/keyrings/microsoft-prod.gpg] https://packages.microsoft.com/ubuntu/22.04/prod jammy main" \ | |
| | sudo tee /etc/apt/sources.list.d/microsoft-prod.list | |
| sudo apt-get update | |
| sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 mssql-tools18 unixodbc-dev | |
| echo "/opt/mssql-tools18/bin" >> "$GITHUB_PATH" | |
| - name: Install dependencies | |
| run: pip install -r requirements.txt | |
| - name: Wait for SQL Server | |
| run: bash scripts/wait_for_sqlserver.sh | |
| - name: Create Bronze Tables | |
| run: | | |
| sqlcmd -S localhost,1433 -U sa -P "$DBT_SQLSERVER_PASSWORD" -C -i scripts/ci-init-db.sql | |
| - name: Create Bronze Tables | |
| run: | | |
| sqlcmd -S localhost,1433 -U sa -P "$DBT_SQLSERVER_PASSWORD" -C -i scripts/loading/ddl_bronze.sql | |
| - name: Load Bronze Data | |
| run: python scripts/loading/python_load.py | |
| - name: Install dbt packages | |
| run: dbt deps | |
| - name: dbt debug | |
| run: dbt debug | |
| - name: dbt compile | |
| run: dbt compile | |
| - name: dbt run | |
| run: dbt run | |
| - name: dbt test | |
| run: dbt test |