Skip to content

Commit d928936

Browse files
Ritik574-coderRitik574-coder
authored
Merge pull request #5 from Ritik574-coder/dbt_branch
chore: improve gitignore and exclude generated artifacts
2 parents 00b23be + 32a07d5 commit d928936

22 files changed

Lines changed: 988 additions & 229 deletions

‎.env.example‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# Copy to .env and fill in values for local development.
2+
# .env is gitignored — never commit real secrets.
3+
4+
DBT_SQLSERVER_HOST=localhost
5+
DBT_SQLSERVER_PORT=1433
6+
DBT_SQLSERVER_DATABASE=retail_analytics
7+
DBT_SQLSERVER_SCHEMA=dbt_dev
8+
DBT_SQLSERVER_USER=sa
9+
DBT_SQLSERVER_PASSWORD=YourStrong!Passw0rd
10+
11+
# Docker Compose (used by docker-compose.yml)
12+
MSSQL_SA_PASSWORD=YourStrong!Passw0rd

‎.github/profiles/profiles.yml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# CI/CD profile — credentials supplied via GitHub Actions env vars or secrets.
2+
dbt_analytics_engineering:
3+
target: ci
4+
outputs:
5+
ci:
6+
type: sqlserver
7+
driver: "ODBC Driver 18 for SQL Server"
8+
server: "{{ env_var('DBT_SQLSERVER_HOST', 'localhost') }}"
9+
port: "{{ env_var('DBT_SQLSERVER_PORT', '1433') | int }}"
10+
database: "{{ env_var('DBT_SQLSERVER_DATABASE', 'master') }}"
11+
schema: "{{ env_var('DBT_SQLSERVER_SCHEMA', 'dbt_ci') }}"
12+
user: "{{ env_var('DBT_SQLSERVER_USER', 'sa') }}"
13+
password: "{{ env_var('DBT_SQLSERVER_PASSWORD') }}"
14+
trust_cert: true
15+
encrypt: true

‎.github/workflows/dbt-cd.yml‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
name: dbt CD
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
target:
7+
description: dbt target (dev or prod)
8+
required: true
9+
default: prod
10+
type: choice
11+
options:
12+
- dev
13+
- prod
14+
push:
15+
branches: [main]
16+
paths:
17+
- "models/**"
18+
- "macros/**"
19+
- "seeds/**"
20+
- "snapshots/**"
21+
- "dbt_project.yml"
22+
- "packages.yml"
23+
24+
concurrency:
25+
group: dbt-cd-${{ github.ref }}
26+
cancel-in-progress: false
27+
28+
env:
29+
PYTHON_VERSION: "3.11"
30+
31+
jobs:
32+
deploy:
33+
name: Deploy (${{ github.event_name == 'workflow_dispatch' && inputs.target || 'prod' }})
34+
runs-on: ubuntu-latest
35+
environment: production
36+
steps:
37+
- name: Checkout
38+
uses: actions/checkout@v4
39+
40+
- name: Set up Python
41+
uses: actions/setup-python@v5
42+
with:
43+
python-version: ${{ env.PYTHON_VERSION }}
44+
cache: pip
45+
cache-dependency-path: requirements.txt
46+
47+
- name: Install ODBC Driver 18
48+
run: |
49+
curl https://packages.microsoft.com/keys/microsoft.asc | sudo gpg --dearmor -o /usr/share/keyrings/microsoft-prod.gpg
50+
echo "deb [arch=amd64 signed-by=/usr/share/keyrings/microsoft-prod.gpg] https://packages.microsoft.com/ubuntu/22.04/prod jammy main" | sudo tee /etc/apt/sources.list.d/microsoft-prod.list
51+
sudo apt-get update
52+
sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 unixodbc-dev
53+
54+
- name: Install dependencies
55+
run: pip install -r requirements.txt
56+
57+
- name: Configure dbt profile
58+
env:
59+
DBT_PROFILES_DIR: ${{ runner.temp }}/dbt-profiles
60+
DBT_SQLSERVER_HOST: ${{ secrets.DBT_SQLSERVER_HOST }}
61+
DBT_SQLSERVER_PORT: ${{ secrets.DBT_SQLSERVER_PORT }}
62+
DBT_SQLSERVER_DATABASE: ${{ secrets.DBT_SQLSERVER_DATABASE }}
63+
DBT_SQLSERVER_SCHEMA: ${{ secrets.DBT_SQLSERVER_SCHEMA }}
64+
DBT_SQLSERVER_USER: ${{ secrets.DBT_SQLSERVER_USER }}
65+
DBT_SQLSERVER_PASSWORD: ${{ secrets.DBT_SQLSERVER_PASSWORD }}
66+
run: |
67+
mkdir -p "$DBT_PROFILES_DIR"
68+
cp profiles.example.yml "$DBT_PROFILES_DIR/profiles.yml"
69+
echo "DBT_PROFILES_DIR=$DBT_PROFILES_DIR" >> "$GITHUB_ENV"
70+
71+
- name: Resolve dbt target
72+
run: |
73+
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
74+
echo "DBT_TARGET=${{ inputs.target }}" >> "$GITHUB_ENV"
75+
else
76+
echo "DBT_TARGET=prod" >> "$GITHUB_ENV"
77+
fi
78+
79+
- name: dbt deps
80+
env:
81+
DBT_SQLSERVER_HOST: ${{ secrets.DBT_SQLSERVER_HOST }}
82+
DBT_SQLSERVER_PORT: ${{ secrets.DBT_SQLSERVER_PORT }}
83+
DBT_SQLSERVER_DATABASE: ${{ secrets.DBT_SQLSERVER_DATABASE }}
84+
DBT_SQLSERVER_SCHEMA: ${{ secrets.DBT_SQLSERVER_SCHEMA }}
85+
DBT_SQLSERVER_USER: ${{ secrets.DBT_SQLSERVER_USER }}
86+
DBT_SQLSERVER_PASSWORD: ${{ secrets.DBT_SQLSERVER_PASSWORD }}
87+
run: dbt deps
88+
89+
- name: dbt debug
90+
env:
91+
DBT_SQLSERVER_HOST: ${{ secrets.DBT_SQLSERVER_HOST }}
92+
DBT_SQLSERVER_PORT: ${{ secrets.DBT_SQLSERVER_PORT }}
93+
DBT_SQLSERVER_DATABASE: ${{ secrets.DBT_SQLSERVER_DATABASE }}
94+
DBT_SQLSERVER_SCHEMA: ${{ secrets.DBT_SQLSERVER_SCHEMA }}
95+
DBT_SQLSERVER_USER: ${{ secrets.DBT_SQLSERVER_USER }}
96+
DBT_SQLSERVER_PASSWORD: ${{ secrets.DBT_SQLSERVER_PASSWORD }}
97+
run: dbt debug --target "$DBT_TARGET"
98+
99+
- name: dbt run
100+
env:
101+
DBT_SQLSERVER_HOST: ${{ secrets.DBT_SQLSERVER_HOST }}
102+
DBT_SQLSERVER_PORT: ${{ secrets.DBT_SQLSERVER_PORT }}
103+
DBT_SQLSERVER_DATABASE: ${{ secrets.DBT_SQLSERVER_DATABASE }}
104+
DBT_SQLSERVER_SCHEMA: ${{ secrets.DBT_SQLSERVER_SCHEMA }}
105+
DBT_SQLSERVER_USER: ${{ secrets.DBT_SQLSERVER_USER }}
106+
DBT_SQLSERVER_PASSWORD: ${{ secrets.DBT_SQLSERVER_PASSWORD }}
107+
run: dbt run --target "$DBT_TARGET"
108+
109+
- name: dbt test
110+
env:
111+
DBT_SQLSERVER_HOST: ${{ secrets.DBT_SQLSERVER_HOST }}
112+
DBT_SQLSERVER_PORT: ${{ secrets.DBT_SQLSERVER_PORT }}
113+
DBT_SQLSERVER_DATABASE: ${{ secrets.DBT_SQLSERVER_DATABASE }}
114+
DBT_SQLSERVER_SCHEMA: ${{ secrets.DBT_SQLSERVER_SCHEMA }}
115+
DBT_SQLSERVER_USER: ${{ secrets.DBT_SQLSERVER_USER }}
116+
DBT_SQLSERVER_PASSWORD: ${{ secrets.DBT_SQLSERVER_PASSWORD }}
117+
run: dbt test --target "$DBT_TARGET"

‎.github/workflows/dbt-ci.yml‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
name: dbt CI
2+
3+
on:
4+
pull_request:
5+
branches: [main]
6+
push:
7+
branches: [main]
8+
9+
concurrency:
10+
group: dbt-ci-${{ github.ref }}
11+
cancel-in-progress: true
12+
13+
env:
14+
DBT_PROFILES_DIR: .github/profiles
15+
DBT_SQLSERVER_HOST: localhost
16+
DBT_SQLSERVER_PORT: 1433
17+
DBT_SQLSERVER_DATABASE: retail_analytics
18+
DBT_SQLSERVER_SCHEMA: dbt_ci
19+
DBT_SQLSERVER_USER: sa
20+
DBT_SQLSERVER_PASSWORD: TestPassword123!
21+
PYTHON_VERSION: "3.11"
22+
23+
jobs:
24+
lint-and-parse:
25+
name: Lint & parse
26+
runs-on: ubuntu-latest
27+
steps:
28+
- name: Checkout
29+
uses: actions/checkout@v4
30+
31+
- name: Set up Python
32+
uses: actions/setup-python@v5
33+
with:
34+
python-version: ${{ env.PYTHON_VERSION }}
35+
cache: pip
36+
cache-dependency-path: requirements.txt
37+
38+
- name: Install dependencies
39+
run: pip install -r requirements.txt
40+
41+
- name: Validate YAML files
42+
run: |
43+
python - <<'PY'
44+
import yaml
45+
from pathlib import Path
46+
47+
for path in Path(".").rglob("*.yml"):
48+
if ".git" in path.parts or "target" in path.parts:
49+
continue
50+
with path.open() as f:
51+
yaml.safe_load(f)
52+
print(f"OK: {path}")
53+
PY
54+
55+
- name: Install dbt packages
56+
run: dbt deps
57+
58+
- name: Parse dbt project
59+
run: dbt parse
60+
61+
- name: Lint SQL (when models exist)
62+
run: |
63+
if find models -name '*.sql' -print -quit | grep -q .; then
64+
sqlfluff lint models
65+
else
66+
echo "No SQL models found — skipping sqlfluff."
67+
fi
68+
69+
dbt-integration:
70+
name: dbt integration
71+
runs-on: ubuntu-latest
72+
needs: lint-and-parse
73+
services:
74+
sqlserver:
75+
image: mcr.microsoft.com/mssql/server:2022-latest
76+
env:
77+
ACCEPT_EULA: Y
78+
MSSQL_SA_PASSWORD: TestPassword123!
79+
ports:
80+
- 1433:1433
81+
options: >-
82+
--health-cmd "/opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P TestPassword123! -C -Q 'SELECT 1' || exit 1"
83+
--health-interval 10s
84+
--health-timeout 5s
85+
--health-retries 12
86+
--health-start-period 30s
87+
88+
steps:
89+
- name: Checkout
90+
uses: actions/checkout@v4
91+
92+
- name: Set up Python
93+
uses: actions/setup-python@v5
94+
with:
95+
python-version: ${{ env.PYTHON_VERSION }}
96+
cache: pip
97+
cache-dependency-path: requirements.txt
98+
99+
- name: Install ODBC Driver 18
100+
run: |
101+
curl -fsSL https://packages.microsoft.com/keys/microsoft.asc \
102+
| sudo gpg --dearmor --batch --yes \
103+
-o /usr/share/keyrings/microsoft-prod.gpg
104+
105+
echo "deb [arch=amd64 signed-by=/usr/share/keyrings/microsoft-prod.gpg] https://packages.microsoft.com/ubuntu/22.04/prod jammy main" \
106+
| sudo tee /etc/apt/sources.list.d/microsoft-prod.list
107+
108+
sudo apt-get update
109+
sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 mssql-tools18 unixodbc-dev
110+
111+
echo "/opt/mssql-tools18/bin" >> "$GITHUB_PATH"
112+
113+
- name: Install dependencies
114+
run: pip install -r requirements.txt
115+
116+
- name: Wait for SQL Server
117+
run: bash scripts/wait_for_sqlserver.sh
118+
119+
- name: Initialize CI database
120+
run: |
121+
sqlcmd -S localhost,1433 -U sa -P "$DBT_SQLSERVER_PASSWORD" -C -i scripts/ci-init-db.sql
122+
123+
- name: Install dbt packages
124+
run: dbt deps
125+
126+
- name: dbt debug
127+
run: dbt debug
128+
129+
- name: dbt compile
130+
run: dbt compile
131+
132+
- name: dbt run
133+
run: dbt run
134+
135+
- name: dbt test
136+
run: dbt test

‎.github/workflows/dbt-docs.yml‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
name: dbt Docs
2+
3+
on:
4+
pull_request:
5+
branches: [main]
6+
push:
7+
branches: [main]
8+
workflow_dispatch:
9+
10+
concurrency:
11+
group: dbt-docs-${{ github.ref }}
12+
cancel-in-progress: true
13+
14+
env:
15+
DBT_PROFILES_DIR: .github/profiles
16+
DBT_SQLSERVER_HOST: localhost
17+
DBT_SQLSERVER_PORT: 1433
18+
DBT_SQLSERVER_DATABASE: retail_analytics
19+
DBT_SQLSERVER_SCHEMA: dbt_ci
20+
DBT_SQLSERVER_USER: sa
21+
DBT_SQLSERVER_PASSWORD: TestPassword123!
22+
PYTHON_VERSION: "3.11"
23+
24+
jobs:
25+
generate-docs:
26+
name: Generate documentation
27+
runs-on: ubuntu-latest
28+
permissions:
29+
contents: read
30+
pages: write
31+
id-token: write
32+
services:
33+
sqlserver:
34+
image: mcr.microsoft.com/mssql/server:2022-latest
35+
env:
36+
ACCEPT_EULA: Y
37+
MSSQL_SA_PASSWORD: TestPassword123!
38+
ports:
39+
- 1433:1433
40+
options: >-
41+
--health-cmd "/opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P TestPassword123! -C -Q 'SELECT 1' || exit 1"
42+
--health-interval 10s
43+
--health-timeout 5s
44+
--health-retries 12
45+
--health-start-period 30s
46+
47+
steps:
48+
- name: Checkout
49+
uses: actions/checkout@v4
50+
51+
- name: Set up Python
52+
uses: actions/setup-python@v5
53+
with:
54+
python-version: ${{ env.PYTHON_VERSION }}
55+
cache: pip
56+
cache-dependency-path: requirements.txt
57+
58+
- name: Install ODBC Driver 18
59+
run: |
60+
curl -fsSL https://packages.microsoft.com/keys/microsoft.asc \
61+
| sudo gpg --dearmor --batch --yes \
62+
-o /usr/share/keyrings/microsoft-prod.gpg
63+
64+
echo "deb [arch=amd64 signed-by=/usr/share/keyrings/microsoft-prod.gpg] https://packages.microsoft.com/ubuntu/22.04/prod jammy main" \
65+
| sudo tee /etc/apt/sources.list.d/microsoft-prod.list
66+
67+
sudo apt-get update
68+
sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 mssql-tools18 unixodbc-dev
69+
70+
echo "/opt/mssql-tools18/bin" >> "$GITHUB_PATH"
71+
72+
- name: Install dependencies
73+
run: pip install -r requirements.txt
74+
75+
- name: Wait for SQL Server
76+
run: bash scripts/wait_for_sqlserver.sh
77+
78+
- name: Initialize CI database
79+
run: |
80+
sqlcmd -S localhost,1433 -U sa -P "$DBT_SQLSERVER_PASSWORD" -C -i scripts/ci-init-db.sql
81+
82+
- name: Generate dbt docs
83+
run: |
84+
dbt deps
85+
dbt docs generate
86+
87+
- name: Upload docs artifact
88+
uses: actions/upload-artifact@v4
89+
with:
90+
name: dbt-docs
91+
path: target/
92+
retention-days: 14
93+
94+
- name: Deploy docs to GitHub Pages
95+
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
96+
uses: actions/upload-pages-artifact@v3
97+
with:
98+
path: target/
99+
100+
deploy-pages:
101+
name: Publish GitHub Pages
102+
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
103+
needs: generate-docs
104+
runs-on: ubuntu-latest
105+
environment:
106+
name: github-pages
107+
url: ${{ steps.deployment.outputs.page_url }}
108+
permissions:
109+
pages: write
110+
id-token: write
111+
steps:
112+
- name: Deploy to GitHub Pages
113+
id: deployment
114+
uses: actions/deploy-pages@v4

0 commit comments

Comments
 (0)