|
1 | | -#!/usr/bin/env bash |
2 | | -set -euo pipefail |
3 | 1 |
|
4 | | -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" |
5 | | -cd "$ROOT_DIR" |
6 | | - |
7 | | -load_env_file() { |
8 | | - local env_file="$1" |
9 | | - while IFS= read -r line; do |
10 | | - [[ -z "$line" || "$line" =~ ^[[:space:]]*# ]] && continue |
11 | | - [[ "$line" != *=* ]] && continue |
12 | | - |
13 | | - local key="${line%%=*}" |
14 | | - local value="${line#*=}" |
15 | | - key="${key//[[:space:]]/}" |
16 | | - |
17 | | - if [[ -z "$key" ]]; then |
18 | | - continue |
19 | | - fi |
20 | | - |
21 | | - # Keep externally provided values (useful for Docker/CI overrides). |
22 | | - if [[ -z "${!key+x}" ]]; then |
23 | | - export "$key=$value" |
24 | | - fi |
25 | | - done < "$env_file" |
26 | | -} |
27 | | - |
28 | | -if [[ -f ".env" ]]; then |
29 | | - load_env_file ".env" |
30 | | -else |
31 | | - echo "No .env found. Using environment variables only." |
32 | | -fi |
33 | | - |
34 | | -if [[ -z "${DATABASE_URL:-}" ]]; then |
35 | | - if [[ -n "${DB_HOST:-}" && -n "${DB_PORT:-}" && -n "${DB_NAME:-}" && -n "${DB_USER:-}" && -n "${DB_PASSWORD:-}" ]]; then |
36 | | - export DATABASE_URL="postgresql://${DB_USER}:${DB_PASSWORD}@${DB_HOST}:${DB_PORT}/${DB_NAME}" |
37 | | - else |
38 | | - echo "DATABASE_URL is not set. Provide it directly or via DB_* variables." |
39 | | - exit 1 |
40 | | - fi |
41 | | -fi |
42 | | - |
43 | | -export PGPASSWORD="${DB_PASSWORD:-${PGPASSWORD:-}}" |
44 | | - |
45 | | -ADMIN_URL="postgresql://${DB_USER:-postgres}:${DB_PASSWORD:-postgres}@${DB_HOST:-localhost}:${DB_PORT:-5432}/postgres" |
46 | | -TARGET_DB="${DB_NAME:-sql_data_engineering}" |
47 | | - |
48 | | -echo "Checking target database: $TARGET_DB" |
49 | | -DB_EXISTS="$(psql "$ADMIN_URL" -tAc "SELECT 1 FROM pg_database WHERE datname='${TARGET_DB}'" || true)" |
50 | | -if [[ "$DB_EXISTS" != "1" ]]; then |
51 | | - echo "Creating database: $TARGET_DB" |
52 | | - psql "$ADMIN_URL" -v ON_ERROR_STOP=1 -c "CREATE DATABASE ${TARGET_DB};" |
53 | | -fi |
54 | | - |
55 | | -run_sql() { |
56 | | - local file="$1" |
57 | | - echo "Running $file" |
58 | | - psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -f "$file" |
59 | | -} |
60 | | - |
61 | | -run_sql "database/schema.sql" |
62 | | -run_sql "database/tables.sql" |
63 | | -run_sql "database/constraints.sql" |
64 | | -run_sql "database/indexes.sql" |
65 | | - |
66 | | -run_sql "etl/extract.sql" |
67 | | -run_sql "etl/transform.sql" |
68 | | -run_sql "etl/load.sql" |
69 | | -run_sql "etl/incremental_load.sql" |
70 | | - |
71 | | -run_sql "warehouse/star_schema.sql" |
72 | | - |
73 | | -run_sql "data_quality/null_checks.sql" |
74 | | -run_sql "data_quality/duplicate_checks.sql" |
75 | | -run_sql "data_quality/referential_integrity.sql" |
76 | | -run_sql "data_quality/validation_queries.sql" |
77 | | - |
78 | | -run_sql "monitoring/row_count_checks.sql" |
79 | | -run_sql "monitoring/anomaly_detection.sql" |
80 | | - |
81 | | -run_sql "analytics/revenue_analysis.sql" |
82 | | -run_sql "analytics/customer_segmentation.sql" |
83 | | -run_sql "analytics/fraud_detection.sql" |
84 | | -run_sql "analytics/retention_analysis.sql" |
85 | | -run_sql "analytics/window_functions.sql" |
86 | | - |
87 | | -run_sql "performance/explain_queries.sql" |
88 | | -run_sql "performance/indexing_tests.sql" |
89 | | -run_sql "performance/optimization_examples.sql" |
90 | | - |
91 | | -run_sql "tests/test_data_load.sql" |
92 | | -run_sql "tests/test_scd_logic.sql" |
93 | | -run_sql "tests/test_quality_checks.sql" |
94 | | -run_sql "tests/test_customer_segmentation.sql" |
95 | | - |
96 | | -echo "Pipeline completed successfully." |
0 commit comments