From 1b3ca0220df920bad030e736cfc7ffb7217745b2 Mon Sep 17 00:00:00 2001 From: David Huang Date: Sat, 6 Jun 2026 17:27:08 -0700 Subject: [PATCH] Add synthetic demo data notebooks and README links --- README.md | 4 + notebooks/demo-data/README.md | 50 + notebooks/demo-data/generate_banking_data.py | 2249 +++++++++++++++ .../generate_hospital_readmission_data.py | 805 ++++++ .../demo-data/generate_retail_apparel_data.py | 816 ++++++ .../demo-data/generate_saas_churn_data.py | 860 ++++++ .../generate_talent_advisory_data.py | 2410 +++++++++++++++++ .../generate_wind_turbine_maintenance_data.py | 812 ++++++ 8 files changed, 8006 insertions(+) create mode 100644 notebooks/demo-data/README.md create mode 100644 notebooks/demo-data/generate_banking_data.py create mode 100644 notebooks/demo-data/generate_hospital_readmission_data.py create mode 100644 notebooks/demo-data/generate_retail_apparel_data.py create mode 100644 notebooks/demo-data/generate_saas_churn_data.py create mode 100644 notebooks/demo-data/generate_talent_advisory_data.py create mode 100644 notebooks/demo-data/generate_wind_turbine_maintenance_data.py diff --git a/README.md b/README.md index 54430adce..b75e5ee5d 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,10 @@ For subsequent local terminal updates: Do not run `databricks bundle init`; this project already has its bundle configuration. +## Demo Data + +If you do not have a dataset ready, `notebooks/demo-data/` includes standalone Databricks notebooks that generate synthetic Unity Catalog datasets for banking, healthcare, retail, SaaS churn, talent advisory, and wind turbine maintenance demos. See [notebooks/demo-data/README.md](notebooks/demo-data/README.md) for the available schemas, required widgets, permissions, and metric view notes. + ## Prerequisites Notebook installer: diff --git a/notebooks/demo-data/README.md b/notebooks/demo-data/README.md new file mode 100644 index 000000000..4f2356719 --- /dev/null +++ b/notebooks/demo-data/README.md @@ -0,0 +1,50 @@ +# Demo Data Notebooks + +These Databricks notebooks generate fictional Unity Catalog datasets for Genie Workbench demos and evaluation. They are useful when users want to try Create Agent, IQ Scan, Quick Fix, or Auto-Optimize but do not have a ready dataset. + +Run them from a Databricks workspace. They are Databricks notebook source files and are not intended for local execution. + +## Required Widgets + +Each notebook defines the same setup widgets: + +| Widget | Required | Default | Description | +|---|---:|---|---| +| `catalog` | Yes | empty | Unity Catalog where the demo schema will be created. There is no safe default. | +| `schema` | No | dataset-specific | Schema/database name for the generated tables and metric views. | +| `overwrite_existing` | No | `false` | When `false`, writes fail if a table already exists. Set to `true` to recreate existing demo tables. | + +The notebooks create the schema if it does not exist, then write Delta tables, add table and column comments, register primary/foreign key constraints, and create metric views. + +## Permissions + +The user running a notebook needs: + +- `USE CATALOG` on the selected catalog +- `CREATE SCHEMA` on the selected catalog +- Permission to create Delta tables, constraints, comments, and views in the generated schema +- A Databricks Runtime or SQL warehouse/runtime combination that supports Unity Catalog metric views + +Metric views use Databricks `CREATE VIEW ... WITH METRICS LANGUAGE YAML`. See the Databricks docs for [CREATE VIEW](https://docs.databricks.com/aws/en/sql/language-manual/sql-ref-syntax-ddl-create-view) and the [metric view YAML reference](https://docs.databricks.com/aws/en/business-semantics/metric-views/yaml-reference). + +## Available Datasets + +| Notebook | Default schema | Tables | Metric views | Story | +|---|---|---|---|---| +| `generate_banking_data.py` | `horizon_bank` | `products`, `branches`, `customers`, `accounts`, `transactions`, `service_requests` | `mv_banking_transactions`, `mv_customer_health`, `mv_service_quality` | Bank customers shift toward digital channels while a service outage drives complaint spikes and lower satisfaction. | +| `generate_hospital_readmission_data.py` | `hospital_readmission` | `patients`, `hospitals`, `encounters`, `care_transitions`, `claims`, `readmissions` | `mv_readmission_quality`, `mv_claims_cost`, `mv_care_transitions` | Discharge follow-up, diagnosis mix, and weekend discharges affect 30-day readmission risk and claims cost. | +| `generate_retail_apparel_data.py` | `retail_apparel` | `products`, `stores`, `customers`, `inventory_snapshots`, `sales`, `returns` | `mv_retail_sales`, `mv_inventory_health`, `mv_retail_returns` | Holiday demand, online growth, clearance activity, stockouts, and returns shape retail revenue and margin. | +| `generate_saas_churn_data.py` | `saas_churn` | `accounts`, `subscriptions`, `product_usage`, `support_tickets`, `invoices`, `churn_events` | `mv_subscription_revenue`, `mv_product_usage`, `mv_churn_risk` | Product adoption, support severity, billing behavior, and utilization explain SaaS churn and ARR loss. | +| `generate_talent_advisory_data.py` | `talent_advisory` | Workforce source tables plus curated marts for planning, hiring, retention, mobility, compensation, and succession | `mv_workforce_planning`, `mv_hiring_funnel`, `mv_retention_engagement`, `mv_internal_mobility`, `mv_comp_performance`, `mv_succession_planning` | Workforce planning connects engagement, mobility, compensation, hiring, and succession coverage across business units. | +| `generate_wind_turbine_maintenance_data.py` | `wind_turbine_maintenance` | `wind_farms`, `turbines`, `components`, `sensor_readings`, `maintenance_events`, `failure_events` | `mv_turbine_performance`, `mv_maintenance_reliability`, `mv_failure_events` | Sensor anomalies, icing, preventive maintenance, and model differences drive turbine failures and downtime. | + +## Recommended Flow + +1. Open one notebook in a Databricks workspace. +2. Set `catalog` to a Unity Catalog where you can create schemas and tables. +3. Optionally change `schema`. +4. Leave `overwrite_existing` as `false` for the first run. +5. Run the notebook from the top. +6. In Genie Workbench, create or optimize a Genie space that references the generated tables or metric views. + +To refresh a demo dataset, rerun the notebook with `overwrite_existing` set to `true`. diff --git a/notebooks/demo-data/generate_banking_data.py b/notebooks/demo-data/generate_banking_data.py new file mode 100644 index 000000000..7debbd94e --- /dev/null +++ b/notebooks/demo-data/generate_banking_data.py @@ -0,0 +1,2249 @@ +# Databricks notebook source + +# MAGIC %md +# MAGIC # Horizon Bank — Synthetic Dataset Generator +# MAGIC +# MAGIC Generates **6 Delta tables** for the Horizon Bank Genie demo space. +# MAGIC +# MAGIC | Table | Rows | Description | +# MAGIC |---|---|---| +# MAGIC | `products` | 20 | Product catalog (static) | +# MAGIC | `branches` | 25 | Branch dimension | +# MAGIC | `customers` | 1,000 | Customer dimension | +# MAGIC | `accounts` | ~2,500 | Account dimension (FK → customers, products) | +# MAGIC | `transactions` | 10,000 | Primary fact table (2023–2025) | +# MAGIC | `service_requests` | 3,000 | Secondary fact table (2023–2025) | +# MAGIC +# MAGIC **Setup:** Set the `catalog` and `schema` widgets (or edit defaults below), then **Run All**. +# MAGIC +# MAGIC Seed: `42` — fully reproducible. + +# COMMAND ---------- + +# MAGIC %pip install faker==40.19.1 --quiet +# MAGIC %restart_python + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 1. Configuration + +# COMMAND ---------- + +# ============================================================================= +# CONFIGURATION — Edit only this section before running +# ============================================================================= +DEFAULT_CATALOG = "" # Unity Catalog name; required +DEFAULT_SCHEMA = "horizon_bank" # Schema / database name + +# Widgets let the bulk runner pass values while preserving standalone defaults. +try: + dbutils.widgets.text("catalog", DEFAULT_CATALOG, "Unity Catalog name") +except Exception: + pass + +try: + dbutils.widgets.text("schema", DEFAULT_SCHEMA, "Schema / database name") +except Exception: + pass + +try: + dbutils.widgets.dropdown("overwrite_existing", "false", ["false", "true"], "Overwrite existing tables") +except Exception: + pass + +try: + CATALOG = dbutils.widgets.get("catalog").strip() or DEFAULT_CATALOG +except Exception: + CATALOG = DEFAULT_CATALOG + +try: + SCHEMA = dbutils.widgets.get("schema").strip() or DEFAULT_SCHEMA +except Exception: + SCHEMA = DEFAULT_SCHEMA + +try: + OVERWRITE_EXISTING = dbutils.widgets.get("overwrite_existing").strip().lower() == "true" +except Exception: + OVERWRITE_EXISTING = False + +if not CATALOG: + raise ValueError("Set the catalog widget to a Unity Catalog name before running this notebook.") + +WRITE_MODE = "overwrite" if OVERWRITE_EXISTING else "errorifexists" + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 2. Imports & Global Setup + +# COMMAND ---------- + +import math + +# ============================================================================= +# IMPORTS & GLOBAL SETUP +# ============================================================================= +import random +from calendar import monthrange +from datetime import date, timedelta + +import numpy as np +import pandas as pd +from faker import Faker + +SEED = 42 +random.seed(SEED) +np.random.seed(SEED) +fake = Faker("en_US") +Faker.seed(SEED) + +# Date range for fact tables +START_DATE = date(2023, 1, 1) +END_DATE = date(2025, 12, 31) + +# Ordered list of (year, month) tuples covering the 3-year window +MONTH_LIST = [(y, m) for y in [2023, 2024, 2025] for m in range(1, 13)] + +# State → Region lookup +STATE_REGION = { + "NY": "Northeast", + "MA": "Northeast", + "NJ": "Northeast", + "CT": "Northeast", + "PA": "Northeast", + "FL": "Southeast", + "GA": "Southeast", + "NC": "Southeast", + "SC": "Southeast", + "VA": "Southeast", + "TN": "Southeast", + "IL": "Midwest", + "OH": "Midwest", + "MI": "Midwest", + "IN": "Midwest", + "WI": "Midwest", + "MN": "Midwest", + "CA": "West", + "WA": "West", + "OR": "West", + "TX": "Southwest", + "AZ": "Southwest", + "NV": "Southwest", + "NM": "Southwest", + "CO": "Southwest", +} + +# State pool for customer assignment (weighted toward NY, CA, TX, FL, IL) +CUSTOMER_STATES = [ + "NY", + "CA", + "TX", + "FL", + "IL", + "NJ", + "MA", + "GA", + "NC", + "WA", + "OH", + "MI", + "PA", + "VA", + "AZ", + "CO", + "SC", + "IN", + "WI", + "OR", +] +STATE_WEIGHTS = [13, 12, 10, 10, 7, 6, 5, 4, 4, 4, 4, 3, 3, 3, 3, 2, 2, 2, 2, 1] + +print("Setup complete.") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 3. Table 1: Products + +# COMMAND ---------- + +# ============================================================================= +# TABLE 1: PRODUCTS (20 rows) +# ============================================================================= + +products_data = [ + # ── DEPOSIT ────────────────────────────────────────────────────────────── + { + "product_id": "PROD-001", + "product_name": "Horizon Basic Checking", + "product_category": "Deposit", + "product_type": "Checking", + "annual_fee_usd": 0.0, + "base_interest_rate_pct": 0.00, + "reward_program": "None", + "min_balance_usd": 0.0, + "is_active": True, + "launched_date": date(2010, 1, 1), + }, + { + "product_id": "PROD-002", + "product_name": "Horizon Plus Checking", + "product_category": "Deposit", + "product_type": "Checking", + "annual_fee_usd": 0.0, + "base_interest_rate_pct": 0.01, + "reward_program": "Cash Back", + "min_balance_usd": 1500.0, + "is_active": True, + "launched_date": date(2012, 6, 1), + }, + { + "product_id": "PROD-003", + "product_name": "Horizon Premier Checking", + "product_category": "Deposit", + "product_type": "Checking", + "annual_fee_usd": 25.0, + "base_interest_rate_pct": 0.05, + "reward_program": "Points", + "min_balance_usd": 5000.0, + "is_active": True, + "launched_date": date(2015, 3, 1), + }, + { + "product_id": "PROD-004", + "product_name": "Horizon Business Checking", + "product_category": "Deposit", + "product_type": "Checking", + "annual_fee_usd": 15.0, + "base_interest_rate_pct": 0.00, + "reward_program": "None", + "min_balance_usd": 0.0, + "is_active": True, + "launched_date": date(2013, 1, 1), + }, + { + "product_id": "PROD-005", + "product_name": "Horizon Basic Savings", + "product_category": "Deposit", + "product_type": "Savings", + "annual_fee_usd": 0.0, + "base_interest_rate_pct": 0.50, + "reward_program": "None", + "min_balance_usd": 100.0, + "is_active": True, + "launched_date": date(2010, 1, 1), + }, + { + "product_id": "PROD-006", + "product_name": "Horizon High-Yield Savings", + "product_category": "Deposit", + "product_type": "Savings", + "annual_fee_usd": 0.0, + "base_interest_rate_pct": 4.50, + "reward_program": "None", + "min_balance_usd": 1000.0, + "is_active": True, + "launched_date": date(2019, 7, 15), + }, + { + "product_id": "PROD-007", + "product_name": "Horizon Money Market", + "product_category": "Deposit", + "product_type": "Savings", + "annual_fee_usd": 0.0, + "base_interest_rate_pct": 3.80, + "reward_program": "None", + "min_balance_usd": 2500.0, + "is_active": True, + "launched_date": date(2015, 1, 1), + }, + { + "product_id": "PROD-008", + "product_name": "Horizon 12-Month CD", + "product_category": "Deposit", + "product_type": "Savings", + "annual_fee_usd": 0.0, + "base_interest_rate_pct": 5.10, + "reward_program": "None", + "min_balance_usd": 500.0, + "is_active": True, + "launched_date": date(2020, 1, 1), + }, + { + "product_id": "PROD-009", + "product_name": "Horizon Student Savings", + "product_category": "Deposit", + "product_type": "Savings", + "annual_fee_usd": 0.0, + "base_interest_rate_pct": 1.00, + "reward_program": "None", + "min_balance_usd": 0.0, + "is_active": True, + "launched_date": date(2018, 9, 1), + }, + # ── CREDIT ─────────────────────────────────────────────────────────────── + { + "product_id": "PROD-010", + "product_name": "Horizon Rewards Visa", + "product_category": "Credit", + "product_type": "Credit Card", + "annual_fee_usd": 0.0, + "base_interest_rate_pct": 19.99, + "reward_program": "Cash Back", + "min_balance_usd": 0.0, + "is_active": True, + "launched_date": date(2011, 4, 1), + }, + { + "product_id": "PROD-011", + "product_name": "Horizon Platinum Card", + "product_category": "Credit", + "product_type": "Credit Card", + "annual_fee_usd": 95.0, + "base_interest_rate_pct": 17.99, + "reward_program": "Points", + "min_balance_usd": 0.0, + "is_active": True, + "launched_date": date(2013, 1, 1), + }, + { + "product_id": "PROD-012", + "product_name": "Horizon Travel Rewards Card", + "product_category": "Credit", + "product_type": "Credit Card", + "annual_fee_usd": 195.0, + "base_interest_rate_pct": 20.99, + "reward_program": "Miles", + "min_balance_usd": 0.0, + "is_active": True, + "launched_date": date(2016, 6, 15), + }, + { + "product_id": "PROD-013", + "product_name": "Horizon Business Credit Card", + "product_category": "Credit", + "product_type": "Credit Card", + "annual_fee_usd": 0.0, + "base_interest_rate_pct": 21.99, + "reward_program": "Cash Back", + "min_balance_usd": 0.0, + "is_active": True, + "launched_date": date(2014, 3, 1), + }, + # ── LENDING ────────────────────────────────────────────────────────────── + { + "product_id": "PROD-014", + "product_name": "Horizon 30yr Fixed Mortgage", + "product_category": "Lending", + "product_type": "Mortgage", + "annual_fee_usd": 0.0, + "base_interest_rate_pct": 6.75, + "reward_program": "None", + "min_balance_usd": 0.0, + "is_active": True, + "launched_date": date(2010, 1, 1), + }, + { + "product_id": "PROD-015", + "product_name": "Horizon 15yr Fixed Mortgage", + "product_category": "Lending", + "product_type": "Mortgage", + "annual_fee_usd": 0.0, + "base_interest_rate_pct": 6.00, + "reward_program": "None", + "min_balance_usd": 0.0, + "is_active": True, + "launched_date": date(2010, 1, 1), + }, + { + "product_id": "PROD-016", + "product_name": "Horizon 5/1 ARM Mortgage", + "product_category": "Lending", + "product_type": "Mortgage", + "annual_fee_usd": 0.0, + "base_interest_rate_pct": 5.50, + "reward_program": "None", + "min_balance_usd": 0.0, + "is_active": True, + "launched_date": date(2012, 1, 1), + }, + { + "product_id": "PROD-017", + "product_name": "Horizon Auto Loan 48-Month", + "product_category": "Lending", + "product_type": "Auto Loan", + "annual_fee_usd": 0.0, + "base_interest_rate_pct": 5.25, + "reward_program": "None", + "min_balance_usd": 0.0, + "is_active": True, + "launched_date": date(2010, 1, 1), + }, + { + "product_id": "PROD-018", + "product_name": "Horizon Auto Loan 60-Month", + "product_category": "Lending", + "product_type": "Auto Loan", + "annual_fee_usd": 0.0, + "base_interest_rate_pct": 5.75, + "reward_program": "None", + "min_balance_usd": 0.0, + "is_active": True, + "launched_date": date(2010, 1, 1), + }, + { + "product_id": "PROD-019", + "product_name": "Horizon Home Equity Line", + "product_category": "Lending", + "product_type": "HELOC", + "annual_fee_usd": 50.0, + "base_interest_rate_pct": 7.50, + "reward_program": "None", + "min_balance_usd": 0.0, + "is_active": True, + "launched_date": date(2011, 1, 1), + }, + { + "product_id": "PROD-020", + "product_name": "Horizon Home Equity Loan", + "product_category": "Lending", + "product_type": "HELOC", + "annual_fee_usd": 0.0, + "base_interest_rate_pct": 7.00, + "reward_program": "None", + "min_balance_usd": 0.0, + "is_active": True, + "launched_date": date(2011, 6, 1), + }, +] + +# Product lookup by type (for account → product assignment) +PRODUCTS_BY_TYPE = {} +for p in products_data: + pt = p["product_type"] + PRODUCTS_BY_TYPE.setdefault(pt, []).append(p["product_id"]) + +# Tier-biased product selection: +# Standard → first product of type, Preferred → middle, Private Client → last +TIER_PRODUCT_IDX = {"Standard": 0, "Preferred": 1, "Private Client": -1} + + +def pick_product(account_type, tier): + """Return a product_id for the given account_type and customer tier.""" + # Map account_type to product_type + atype_to_ptype = { + "Checking": "Checking", + "Savings": "Savings", + "Credit Card": "Credit Card", + "Mortgage": "Mortgage", + "Auto Loan": "Auto Loan", + "Home Equity": "HELOC", + } + ptype = atype_to_ptype.get(account_type, "Checking") + candidates = PRODUCTS_BY_TYPE.get(ptype, ["PROD-001"]) + idx = TIER_PRODUCT_IDX.get(tier, 0) + # Add small random variation so not all same-tier customers get identical products + jitter = random.choice([0, 0, 0, 1]) if len(candidates) > 1 else 0 + return candidates[min(idx + jitter, len(candidates) - 1)] + + +df_products = pd.DataFrame(products_data) +print(f"products: {len(df_products)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 4. Table 2: Branches + +# COMMAND ---------- + +# ============================================================================= +# TABLE 2: BRANCHES (25 rows) +# ============================================================================= + + +def _branch(bid, name, btype, region, state, city, opened, cost, hc): + return { + "branch_id": bid, + "branch_name": name, + "branch_type": btype, + "region": region, + "state": state, + "city": city, + "opened_date": date(*opened), + "monthly_operating_cost_usd": float(cost), + "headcount": hc, + "is_active": True, + } + + +branches_data = [ + # ── NORTHEAST (6) ──────────────────────────────────────────────────────── + _branch( + "BRNCH-001", + "Manhattan Financial District Branch", + "Full Service", + "Northeast", + "NY", + "New York", + (2005, 3, 1), + 145000, + 32, + ), + _branch( + "BRNCH-002", + "Brooklyn Heights Branch", + "Full Service", + "Northeast", + "NY", + "Brooklyn", + (2008, 7, 1), + 118000, + 25, + ), + _branch( + "BRNCH-003", + "Long Island Branch", + "Limited Service", + "Northeast", + "NY", + "Garden City", + (2011, 4, 1), + 72000, + 13, + ), + _branch( + "BRNCH-004", + "Newark Main Branch", + "Full Service", + "Northeast", + "NJ", + "Newark", + (2006, 9, 1), + 105000, + 22, + ), + _branch( + "BRNCH-005", + "Princeton Branch", + "Limited Service", + "Northeast", + "NJ", + "Princeton", + (2014, 2, 1), + 65000, + 11, + ), + _branch( + "BRNCH-006", + "Boston Financial Branch", + "Full Service", + "Northeast", + "MA", + "Boston", + (2007, 5, 1), + 132000, + 28, + ), + # ── SOUTHEAST (6) — injected: 20% higher avg txn value ────────────────── + _branch( + "BRNCH-007", + "Miami South Branch", + "Full Service", + "Southeast", + "FL", + "Miami", + (2006, 1, 1), + 138000, + 30, + ), + _branch( + "BRNCH-008", + "Orlando Central Branch", + "Full Service", + "Southeast", + "FL", + "Orlando", + (2009, 3, 1), + 112000, + 24, + ), + _branch( + "BRNCH-009", + "Tampa Bay Branch", + "Limited Service", + "Southeast", + "FL", + "Tampa", + (2013, 6, 1), + 68000, + 12, + ), + _branch( + "BRNCH-010", + "Atlanta Midtown Branch", + "Full Service", + "Southeast", + "GA", + "Atlanta", + (2007, 11, 1), + 125000, + 27, + ), + _branch( + "BRNCH-011", + "Savannah Branch", + "Drive-Through", + "Southeast", + "GA", + "Savannah", + (2015, 4, 1), + 32000, + 6, + ), + _branch( + "BRNCH-012", + "Charlotte Branch", + "Full Service", + "Southeast", + "NC", + "Charlotte", + (2010, 8, 1), + 108000, + 23, + ), + # ── MIDWEST (5) ────────────────────────────────────────────────────────── + _branch( + "BRNCH-013", + "Chicago Loop Branch", + "Full Service", + "Midwest", + "IL", + "Chicago", + (2004, 6, 1), + 150000, + 35, + ), + _branch( + "BRNCH-014", + "Chicago North Shore Branch", + "Limited Service", + "Midwest", + "IL", + "Evanston", + (2010, 1, 1), + 74000, + 14, + ), + _branch( + "BRNCH-015", + "Chicago South Branch", + "Drive-Through", + "Midwest", + "IL", + "Oak Park", + (2012, 9, 1), + 38000, + 7, + ), + _branch( + "BRNCH-016", + "Columbus Branch", + "Full Service", + "Midwest", + "OH", + "Columbus", + (2009, 4, 1), + 98000, + 20, + ), + _branch( + "BRNCH-017", + "Detroit Branch", + "Full Service", + "Midwest", + "MI", + "Detroit", + (2011, 2, 1), + 95000, + 19, + ), + # ── WEST (4) ───────────────────────────────────────────────────────────── + _branch( + "BRNCH-018", + "Los Angeles Downtown Branch", + "Full Service", + "West", + "CA", + "Los Angeles", + (2005, 9, 1), + 148000, + 33, + ), + _branch( + "BRNCH-019", + "San Francisco Financial Branch", + "Full Service", + "West", + "CA", + "San Francisco", + (2006, 3, 1), + 152000, + 34, + ), + _branch( + "BRNCH-020", + "San Diego Branch", + "Limited Service", + "West", + "CA", + "San Diego", + (2012, 5, 1), + 79000, + 15, + ), + _branch( + "BRNCH-021", + "Seattle Branch", + "Full Service", + "West", + "WA", + "Seattle", + (2010, 7, 1), + 115000, + 24, + ), + # ── SOUTHWEST (4) ──────────────────────────────────────────────────────── + _branch( + "BRNCH-022", + "Houston Main Branch", + "Full Service", + "Southwest", + "TX", + "Houston", + (2006, 2, 1), + 128000, + 27, + ), + _branch( + "BRNCH-023", + "Dallas Branch", + "Full Service", + "Southwest", + "TX", + "Dallas", + (2008, 10, 1), + 122000, + 26, + ), + _branch( + "BRNCH-024", + "Austin Branch", + "Limited Service", + "Southwest", + "TX", + "Austin", + (2014, 6, 1), + 71000, + 13, + ), + _branch( + "BRNCH-025", + "Phoenix Branch", + "Full Service", + "Southwest", + "AZ", + "Phoenix", + (2009, 1, 1), + 103000, + 21, + ), +] + +df_branches = pd.DataFrame(branches_data) + +# Branch lookup helpers +BRANCH_IDS = [b["branch_id"] for b in branches_data] +BRANCHES_BY_REGION = {} +for b in branches_data: + BRANCHES_BY_REGION.setdefault(b["region"], []).append(b["branch_id"]) +BRANCH_REGION = {b["branch_id"]: b["region"] for b in branches_data} + +print(f"branches: {len(df_branches)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 5. Table 3: Customers + +# COMMAND ---------- + +# ============================================================================= +# TABLE 3: CUSTOMERS (1,000 rows) +# ============================================================================= + +# Tier → credit_score_band weights +CREDIT_SCORE_BANDS = ["Poor", "Fair", "Good", "Very Good", "Exceptional"] +CREDIT_WEIGHTS = { + "Standard": [5, 20, 35, 30, 10], + "Preferred": [1, 8, 28, 42, 21], + "Private Client": [0, 2, 12, 40, 46], +} + +# Tier → has_* flag probabilities +HAS_FLAGS = { + "Standard": { + "checking": 0.70, + "savings": 0.62, + "credit_card": 0.38, + "mortgage": 0.18, + }, + "Preferred": { + "checking": 0.88, + "savings": 0.78, + "credit_card": 0.62, + "mortgage": 0.34, + }, + "Private Client": { + "checking": 0.97, + "savings": 0.90, + "credit_card": 0.80, + "mortgage": 0.60, + }, +} + +customers_data = [] +for i in range(1, 1001): + cid = f"CUST-{i:04d}" + tier = random.choices( + ["Standard", "Preferred", "Private Client"], weights=[60, 30, 10] + )[0] + + # Segment correlated with tier + if tier == "Private Client": + segment = random.choices( + ["Retail", "Small Business", "Wealth Management"], weights=[15, 30, 55] + )[0] + elif tier == "Preferred": + segment = random.choices( + ["Retail", "Small Business", "Wealth Management"], weights=[55, 35, 10] + )[0] + else: + segment = random.choices( + ["Retail", "Small Business", "Wealth Management"], weights=[75, 22, 3] + )[0] + + state = random.choices(CUSTOMER_STATES, weights=STATE_WEIGHTS)[0] + region = STATE_REGION[state] + + # Pick branch in same region + branch_id = random.choice(BRANCHES_BY_REGION[region]) + + flags = HAS_FLAGS[tier] + has_checking = random.random() < flags["checking"] + has_savings = random.random() < flags["savings"] + has_credit_card = random.random() < flags["credit_card"] + has_mortgage = random.random() < flags["mortgage"] + + # Ensure at least one deposit account + if not has_checking and not has_savings: + has_checking = True + + # Age band correlated with segment/tier + age_weights = [8, 20, 22, 20, 17, 13] + age_band = random.choices( + ["18-24", "25-34", "35-44", "45-54", "55-64", "65+"], weights=age_weights + )[0] + + # Income band correlated with tier + if tier == "Private Client": + income_band = random.choices( + ["<$50K", "$50K-$100K", "$100K-$200K", "$200K+"], weights=[2, 10, 35, 53] + )[0] + elif tier == "Preferred": + income_band = random.choices( + ["<$50K", "$50K-$100K", "$100K-$200K", "$200K+"], weights=[8, 28, 45, 19] + )[0] + else: + income_band = random.choices( + ["<$50K", "$50K-$100K", "$100K-$200K", "$200K+"], weights=[28, 38, 26, 8] + )[0] + + credit_score_band = random.choices( + CREDIT_SCORE_BANDS, weights=CREDIT_WEIGHTS[tier] + )[0] + + # customer_since_date: 2018–2024 + since_days = random.randint(0, (date(2024, 12, 31) - date(2018, 1, 1)).days) + since_date = date(2018, 1, 1) + timedelta(days=since_days) + + customers_data.append( + { + "customer_id": cid, + "customer_name": fake.name(), + "age_band": age_band, + "income_band": income_band, + "customer_segment": segment, + "relationship_tier": tier, + "state": state, + "region": region, + "primary_branch_id": branch_id, + "acquisition_channel": random.choices( + ["Branch", "Online", "Mobile", "Referral"], weights=[30, 35, 25, 10] + )[0], + "customer_since_date": since_date, + "is_active": random.random() < 0.94, + "has_checking": has_checking, + "has_savings": has_savings, + "has_credit_card": has_credit_card, + "has_mortgage": has_mortgage, + "credit_score_band": credit_score_band, + } + ) + +df_customers = pd.DataFrame(customers_data) +print(f"customers: {len(df_customers)} rows") +print(df_customers["relationship_tier"].value_counts().to_dict()) + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 6. Table 4: Accounts + +# COMMAND ---------- + +# ============================================================================= +# TABLE 4: ACCOUNTS (~2,500 rows) +# ============================================================================= +# Injected pattern: Private Client customers have 3x the avg balance. +# Achieved via tier-scaled lognormal distributions for deposit accounts. +# +# Balance parameters (lognormal: mean = exp(mu + sigma^2/2)): +# Standard deposit: mu=7.8, sigma=0.6 → mean ≈ $2,919 +# Preferred deposit: mu=8.4, sigma=0.6 → mean ≈ $5,297 +# Private Client deposit:mu=8.9, sigma=0.6 → mean ≈ $8,789 (≈3x Standard) + +BALANCE_PARAMS = { + # (mu, sigma) for lognormal draws — deposit accounts + "Standard": {"deposit": (7.8, 0.6)}, + "Preferred": {"deposit": (8.4, 0.6)}, + "Private Client": {"deposit": (8.9, 0.6)}, +} + + +def deposit_balance(tier): + mu, sigma = BALANCE_PARAMS[tier]["deposit"] + return round(float(np.random.lognormal(mu, sigma)), 2) + + +def credit_card_params(tier): + """Returns (credit_limit, current_balance).""" + if tier == "Standard": + limit = round(random.uniform(3000, 12000), 2) + elif tier == "Preferred": + limit = round(random.uniform(10000, 30000), 2) + else: + limit = round(random.uniform(25000, 100000), 2) + utilization = random.uniform(0.05, 0.70) + balance = round(limit * utilization, 2) + return limit, balance + + +def lending_balance(account_type, tier): + ranges = { + "Mortgage": { + "Standard": (100000, 400000), + "Preferred": (200000, 700000), + "Private Client": (400000, 2000000), + }, + "Auto Loan": { + "Standard": (8000, 30000), + "Preferred": (20000, 55000), + "Private Client": (40000, 120000), + }, + "Home Equity": { + "Standard": (20000, 80000), + "Preferred": (50000, 150000), + "Private Client": (100000, 500000), + }, + } + lo, hi = ranges[account_type][tier] + return round(random.uniform(lo, hi), 2) + + +accounts_data = [] +account_counter = 1 + +# Track which customers are "high value" (Private Client = top 10%) +# Used later for transaction concentration pattern +HIGH_VALUE_CUSTOMERS = set( + c["customer_id"] + for c in customers_data + if c["relationship_tier"] == "Private Client" +) + +for cust in customers_data: + cid = cust["customer_id"] + tier = cust["relationship_tier"] + since = cust["customer_since_date"] + + # Collect account types this customer should have + account_types = [] + if cust["has_checking"]: + account_types.append("Checking") + if cust["has_savings"]: + account_types.append("Savings") + if cust["has_credit_card"]: + account_types.append("Credit Card") + if cust["has_mortgage"]: + account_types.append("Mortgage") + + # Extra accounts (Auto Loan, Home Equity) by tier + auto_prob = {"Standard": 0.15, "Preferred": 0.25, "Private Client": 0.40} + heloc_prob = {"Standard": 0.10, "Preferred": 0.20, "Private Client": 0.35} + if random.random() < auto_prob[tier]: + account_types.append("Auto Loan") + if random.random() < heloc_prob[tier]: + account_types.append("Home Equity") + + primary_set = False + for idx, atype in enumerate(account_types): + aid = f"ACCT-{account_counter:05d}" + account_counter += 1 + + is_primary = (not primary_set) and (atype == "Checking") + if atype == "Checking" and not primary_set: + primary_set = True + # If no checking, make first account primary + if idx == 0 and not primary_set: + is_primary = True + primary_set = True + + product_id = pick_product(atype, tier) + + # Derive product base_interest_rate + prod = next(p for p in products_data if p["product_id"] == product_id) + base_rate = prod["base_interest_rate_pct"] + rate_noise = round(random.uniform(-0.25, 0.25), 4) + interest_rate = max(0.0, round(base_rate + rate_noise, 4)) + + # Open date: within 6 months after customer_since_date, up to 2024 + max_open = date(2024, 12, 31) + open_offset = random.randint(0, 180) + open_date = min(since + timedelta(days=open_offset), max_open) + + # Status: Active(78%), Dormant(10%), Closed(8%), Delinquent(4%) + status = random.choices( + ["Active", "Dormant", "Closed", "Delinquent"], weights=[78, 10, 8, 4] + )[0] + + close_date = None + if status == "Closed": + max_offset = max(1, (date(2024, 12, 31) - open_date).days) + close_offset = random.randint(min(180, max_offset), max_offset) + close_date = open_date + timedelta(days=close_offset) + + # Balance by account type + credit_limit = None + if atype in ("Checking", "Savings"): + balance = deposit_balance(tier) + elif atype == "Credit Card": + credit_limit, balance = credit_card_params(tier) + else: # Mortgage, Auto Loan, Home Equity + balance = lending_balance(atype, tier) + + accounts_data.append( + { + "account_id": aid, + "customer_id": cid, + "product_id": product_id, + "account_type": atype, + "open_date": open_date, + "close_date": close_date, + "status": status, + "current_balance_usd": balance, + "credit_limit_usd": credit_limit, + "interest_rate_pct": interest_rate, + "is_primary_account": is_primary, + } + ) + +df_accounts = pd.DataFrame(accounts_data) +print(f"accounts: {len(df_accounts)} rows") +print(df_accounts["account_type"].value_counts().to_dict()) + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 7. Table 5: Transactions + +# COMMAND ---------- + +# ============================================================================= +# TABLE 5: TRANSACTIONS (10,000 rows) +# ============================================================================= +# Injected patterns: +# 1. Nov/Dec volume spike: +45% transactions vs monthly average +# 2. Q2 2024 deposit dip: deposit amounts -15% in Apr/May/Jun 2024 +# 3. Mobile channel growth: 25% share (Jan 2023) → 48% share (Dec 2025) +# 4. Fee revenue growth: wire + overdraft fees +20% YoY +# 5. Top 10% customers (Private Client) = ~35% of deposit volume +# Achieved by giving them 35% of deposit transaction count. + +# ── Month weights (Nov/Dec = 1.45x) ────────────────────────────────────────── +month_weights = [] +for y, m in MONTH_LIST: + month_weights.append(1.45 if m in (11, 12) else 1.0) +total_w = sum(month_weights) # 38.7 +txns_per_month_raw = [w / total_w * 10000 for w in month_weights] +txns_per_month = [int(round(x)) for x in txns_per_month_raw] +diff = 10000 - sum(txns_per_month) +txns_per_month[-1] += diff # absorb rounding into Dec 2025 + +# ── Account pools for sampling ──────────────────────────────────────────────── +all_account_ids = [a["account_id"] for a in accounts_data] +account_cust_map = {a["account_id"]: a["customer_id"] for a in accounts_data} +account_type_map = {a["account_id"]: a["account_type"] for a in accounts_data} +account_region_map = {} +cust_region_map = {c["customer_id"]: c["region"] for c in customers_data} +cust_branch_map = {c["customer_id"]: c["primary_branch_id"] for c in customers_data} +for a in accounts_data: + account_region_map[a["account_id"]] = cust_region_map[a["customer_id"]] + +# Separate account pools +hv_accounts = [ + a["account_id"] + for a in accounts_data + if account_cust_map[a["account_id"]] in HIGH_VALUE_CUSTOMERS +] +reg_accounts = [ + a["account_id"] + for a in accounts_data + if account_cust_map[a["account_id"]] not in HIGH_VALUE_CUSTOMERS +] + +# ── Transaction type weights by account type ────────────────────────────────── +TXN_TYPES = { + "Checking": ( + ["Deposit", "Withdrawal", "Transfer", "Payment", "Fee", "Interest"], + [25, 25, 20, 15, 8, 7], + ), + "Savings": ( + ["Deposit", "Withdrawal", "Transfer", "Interest", "Fee"], + [35, 15, 20, 25, 5], + ), + "Credit Card": (["Purchase", "Payment", "Fee", "Interest"], [55, 30, 10, 5]), + "Mortgage": (["Payment", "Fee", "Interest"], [80, 10, 10]), + "Auto Loan": (["Payment", "Fee", "Interest"], [80, 10, 10]), + "Home Equity": (["Payment", "Withdrawal", "Fee", "Interest"], [65, 20, 10, 5]), +} + +MERCHANT_CATS = [ + "Groceries", + "Travel", + "Dining", + "Gas", + "Retail", + "Healthcare", + "Entertainment", +] +MERCHANT_WEIGHTS = [20, 12, 18, 10, 17, 13, 10] + + +def channel_shares(month_idx): + """Linear interpolation: mobile grows 25%→48%, branch/online/ATM shrink.""" + t = month_idx / 35.0 + mobile = 0.25 + 0.23 * t + branch = 0.30 - 0.10 * t + atm = 0.20 - 0.05 * t + online = 0.20 - 0.08 * t + wire = 0.05 + return [mobile, branch, atm, online, wire] + + +CHANNELS = ["Mobile", "Branch", "ATM", "Online", "Wire"] + + +def fee_amount(txn_type, channel, year): + """Generate fee_usd with 20% YoY growth starting from 2023 base.""" + year_mult = 1.0 + 0.20 * (year - 2023) # 1.0 / 1.2 / 1.44 + if channel == "Wire": + return round(25.0 * year_mult, 2) + if txn_type in ("Withdrawal", "Transfer") and random.random() < 0.04: + # Occasional overdraft/service fee + return round(35.0 * year_mult, 2) + if txn_type == "Fee": + # Explicit fee transaction + base = random.choice([10.0, 15.0, 25.0, 35.0]) + return round(base * year_mult, 2) + return 0.0 + + +def amount_for(txn_type, account_type, is_high_value, year, month): + """Generate amount_usd (always positive).""" + if account_type == "Mortgage": + # Monthly payment ~$1,500–$5,000 + return round(random.uniform(1200, 5000), 2) + if account_type == "Auto Loan": + return round(random.uniform(300, 800), 2) + if account_type == "Home Equity": + return round(random.uniform(500, 3000), 2) + if account_type == "Credit Card": + if txn_type == "Purchase": + return round(random.uniform(10, 500), 2) + if txn_type == "Payment": + return round(random.uniform(50, 2000), 2) + if txn_type == "Interest": + return round(random.uniform(20, 200), 2) + return round(random.uniform(10, 100), 2) + + # Checking / Savings + if txn_type == "Deposit": + if is_high_value: + amt = round(float(np.random.lognormal(8.3, 0.7)), 2) # mean ≈ $6,000 + else: + amt = round(float(np.random.lognormal(6.8, 0.7)), 2) # mean ≈ $1,200 + # Q2 2024 deposit dip: -15% + if year == 2024 and month in (4, 5, 6): + amt *= 0.85 + return round(amt, 2) + if txn_type == "Withdrawal": + return round(float(np.random.lognormal(6.2, 0.8)), 2) + if txn_type == "Transfer": + return round(float(np.random.lognormal(7.0, 0.7)), 2) + if txn_type == "Payment": + return round(random.uniform(50, 1500), 2) + if txn_type == "Interest": + return round(random.uniform(1, 80), 2) + if txn_type == "Fee": + return 0.0 # fee captured in fee_usd column + return round(random.uniform(5, 200), 2) + + +# ── Running balance tracker per account ─────────────────────────────────────── +# Initialize each account with a starting balance slightly above current_balance_usd +account_running_balance = {} +for a in accounts_data: + account_running_balance[a["account_id"]] = a[ + "current_balance_usd" + ] * random.uniform(0.8, 1.2) + +# ── Generate transactions ───────────────────────────────────────────────────── +transactions_data = [] +txn_counter = 1 + +# Pre-compute high-value deposit allocation: +# Top 10% (Private Client) accounts for ~35% of deposit transactions. +# We track this by biasing account selection for Deposit transactions. + +for month_idx, (year, month) in enumerate(MONTH_LIST): + n_txns = txns_per_month[month_idx] + shares = channel_shares(month_idx) + max_day = monthrange(year, month)[1] + + for _ in range(n_txns): + # Pick transaction date within month + txn_date = date(year, month, random.randint(1, max_day)) + + # Select account — for deposits: 35% chance of drawing from high-value pool + # For other types: draw proportionally from all accounts + if hv_accounts and random.random() < 0.35: + acct_id = random.choice(hv_accounts) + else: + acct_id = random.choice(reg_accounts if reg_accounts else all_account_ids) + + cust_id = account_cust_map[acct_id] + atype = account_type_map[acct_id] + region = account_region_map[acct_id] + is_hv = cust_id in HIGH_VALUE_CUSTOMERS + + # Transaction type + types_list, type_weights = TXN_TYPES.get(atype, TXN_TYPES["Checking"]) + txn_type = random.choices(types_list, weights=type_weights)[0] + + # Channel — weighted by time, restricted by transaction type + if txn_type in ("Interest", "Fee") and atype in ("Mortgage", "Auto Loan"): + channel = "Online" + else: + channel = random.choices(CHANNELS, weights=shares)[0] + + # Branch ID — NULL for digital channels + branch_id = None + if channel == "Branch": + branch_id = cust_branch_map.get(cust_id) + + # Amount + amt = amount_for(txn_type, atype, is_hv, year, month) + if amt < 0: + amt = 0.01 + + # Southeast 20% transaction value premium (branch transactions only) + if region == "Southeast" and branch_id is not None: + amt = round(amt * 1.20, 2) + + # Fee + fee = fee_amount(txn_type, channel, year) + + # Running balance + prev_bal = account_running_balance[acct_id] + if txn_type in ("Deposit", "Interest"): + new_bal = prev_bal + amt + elif txn_type == "Fee": + new_bal = prev_bal - fee + else: + new_bal = prev_bal - amt - fee + account_running_balance[acct_id] = new_bal + bal_after = round(new_bal, 2) + + # Merchant category (credit card purchases only) + merchant_cat = None + if atype == "Credit Card" and txn_type == "Purchase": + merchant_cat = random.choices(MERCHANT_CATS, weights=MERCHANT_WEIGHTS)[0] + + # International: rare, higher for travel + intl_prob = 0.08 if merchant_cat == "Travel" else 0.015 + is_intl = random.random() < intl_prob + + # Status & flag + status = random.choices(["Posted", "Pending", "Reversed"], weights=[95, 3, 2])[ + 0 + ] + is_flagged = random.random() < 0.005 + + transactions_data.append( + { + "transaction_id": f"TXN-{txn_counter:07d}", + "transaction_date": txn_date, + "transaction_year": year, + "transaction_month": month, + "transaction_quarter": (month - 1) // 3 + 1, + "account_id": acct_id, + "customer_id": cust_id, + "branch_id": branch_id, + "transaction_type": txn_type, + "channel": channel, + "amount_usd": max(0.0, amt), + "fee_usd": max(0.0, fee), + "balance_after_usd": bal_after, + "merchant_category": merchant_cat, + "is_international": is_intl, + "status": status, + "is_flagged": is_flagged, + } + ) + txn_counter += 1 + +# Update accounts.current_balance_usd to reflect final running balance +final_balance_map = dict(account_running_balance) +for a in accounts_data: + a["current_balance_usd"] = round(final_balance_map[a["account_id"]], 2) +df_accounts = pd.DataFrame(accounts_data) # rebuild with updated balances + +df_transactions = pd.DataFrame(transactions_data) +print(f"transactions: {len(df_transactions)} rows") +print(df_transactions["transaction_type"].value_counts().to_dict()) +print(df_transactions["channel"].value_counts().to_dict()) + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 8. Table 6: Service Requests + +# COMMAND ---------- + +# ============================================================================= +# TABLE 6: SERVICE REQUESTS (3,000 rows) +# ============================================================================= +# Injected pattern: Complaint category spikes in Jan 2024 (+80% overall requests) +# to simulate a system outage narrative. + +# Month weights: Jan 2024 = 1.80x, all others = 1.0x +sr_month_weights = [1.80 if (y == 2024 and m == 1) else 1.0 for (y, m) in MONTH_LIST] +total_sr_w = sum(sr_month_weights) # 36.8 +sr_per_month_raw = [w / total_sr_w * 3000 for w in sr_month_weights] +sr_per_month = [int(round(x)) for x in sr_per_month_raw] +sr_diff = 3000 - sum(sr_per_month) +sr_per_month[-1] += sr_diff + +SR_CHANNELS = ["Phone", "Chat", "Branch", "App"] +SR_CATEGORIES = [ + "Account Inquiry", + "Dispute", + "Complaint", + "Product Inquiry", + "Technical Issue", +] +SR_STATUSES = ["Resolved", "Open", "Escalated"] + +all_customer_ids = [c["customer_id"] for c in customers_data] + +service_requests_data = [] +sr_counter = 1 + +for month_idx, (year, month) in enumerate(MONTH_LIST): + n_srs = sr_per_month[month_idx] + is_jan2024 = year == 2024 and month == 1 + max_day = monthrange(year, month)[1] + + for _ in range(n_srs): + req_date = date(year, month, random.randint(1, max_day)) + cust_id = random.choice(all_customer_ids) + + # Jan 2024 complaint spike: complaints elevated to 55% vs normal 15% + if is_jan2024: + cat_weights = [10, 12, 55, 13, 10] + else: + cat_weights = [35, 15, 15, 20, 15] + + category = random.choices(SR_CATEGORIES, weights=cat_weights)[0] + channel = random.choices(SR_CHANNELS, weights=[35, 30, 15, 20])[0] + + # Branch for branch-channel requests + branch_id = None + if channel == "Branch": + region = cust_region_map.get(cust_id, "Northeast") + branch_id = random.choice(BRANCHES_BY_REGION[region]) + + status = random.choices(SR_STATUSES, weights=[72, 18, 10])[0] + + # Resolution time: NULL if open/escalated + res_days = None + if status == "Resolved": + if category == "Complaint": + res_days = random.randint(3, 21) + elif category == "Dispute": + res_days = random.randint(5, 30) + else: + res_days = random.randint(0, 5) + elif status == "Escalated": + res_days = random.randint(10, 45) + + # Satisfaction score: NULL if not resolved; lower for complaints + sat_score = None + if status == "Resolved": + if category == "Complaint": + sat_score = random.choices( + [1, 2, 3, 4, 5], weights=[20, 30, 25, 15, 10] + )[0] + else: + sat_score = random.choices( + [1, 2, 3, 4, 5], weights=[5, 10, 20, 35, 30] + )[0] + elif status == "Escalated" and random.random() < 0.3: + sat_score = random.choices([1, 2, 3], weights=[50, 35, 15])[0] + + service_requests_data.append( + { + "request_id": f"SR-{sr_counter:06d}", + "request_date": req_date, + "request_year": year, + "request_month": month, + "customer_id": cust_id, + "branch_id": branch_id, + "channel": channel, + "category": category, + "status": status, + "resolution_time_days": res_days, + "satisfaction_score": sat_score, + "is_resolved": status == "Resolved", + } + ) + sr_counter += 1 + +df_service_requests = pd.DataFrame(service_requests_data) +print(f"service_requests: {len(df_service_requests)} rows") +print(df_service_requests["category"].value_counts().to_dict()) + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 9. Create Schema & Write Delta Tables + +# COMMAND ---------- + +# ============================================================================= +# CREATE SCHEMA & WRITE DELTA TABLES +# ============================================================================= +spark.sql(f"CREATE SCHEMA IF NOT EXISTS `{CATALOG}`.`{SCHEMA}`") + + +def write_table(df_pd, table_name, mode=WRITE_MODE): + """Convert pandas → Spark and write as Delta table.""" + df_spark = spark.createDataFrame(df_pd) + ( + df_spark.write.format("delta") + .mode(mode) + .option("overwriteSchema", "true") + .saveAsTable(f"`{CATALOG}`.`{SCHEMA}`.`{table_name}`") + ) + print(f" ✓ {table_name}: {df_spark.count()} rows written") + + +print(f"Writing tables to `{CATALOG}`.`{SCHEMA}` with mode={WRITE_MODE}...") +write_table(df_products, "products") +write_table(df_branches, "branches") +write_table(df_customers, "customers") +write_table(df_accounts, "accounts") +write_table(df_transactions, "transactions") +write_table(df_service_requests, "service_requests") +print("All tables written.") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 10. Verification + +# COMMAND ---------- + +# ============================================================================= +# VERIFICATION +# ============================================================================= +C = f"`{CATALOG}`.`{SCHEMA}`" + +print("=" * 60) +print("ROW COUNTS") +print("=" * 60) +for tbl in [ + "products", + "branches", + "customers", + "accounts", + "transactions", + "service_requests", +]: + n = spark.sql(f"SELECT COUNT(*) AS n FROM {C}.`{tbl}`").collect()[0]["n"] + print(f" {tbl:<20} {n:>6,}") + +print() +print("=" * 60) +print("PATTERN VALIDATION") +print("=" * 60) + +# 1. Nov/Dec spike +q1 = spark.sql( + f""" + SELECT + CASE WHEN transaction_month IN (11,12) THEN 'Nov/Dec' ELSE 'Other' END AS period, + COUNT(*) AS txn_count, + ROUND(COUNT(*) / SUM(COUNT(*)) OVER (), 4) AS share + FROM {C}.`transactions` + GROUP BY 1 +""" +) +print("\n1. Nov/Dec volume share (target: ~23% of all transactions):") +q1.show() + +# 2. Q2 2024 deposit dip +q2 = spark.sql( + f""" + SELECT transaction_year AS yr, transaction_quarter AS qtr, + ROUND(AVG(amount_usd),2) AS avg_deposit_amt, + COUNT(*) AS cnt + FROM {C}.`transactions` + WHERE transaction_type = 'Deposit' + AND transaction_year IN (2023,2024) + GROUP BY 1,2 + ORDER BY 1,2 +""" +) +print( + "\n2. Avg deposit amount by year/quarter (Q2 2024 should be ~15% below Q1/Q3 2024):" +) +q2.show() + +# 3. Mobile channel growth +q3 = spark.sql( + f""" + SELECT transaction_year AS yr, + ROUND(SUM(CASE WHEN channel='Mobile' THEN 1 ELSE 0 END)*100.0/COUNT(*),1) + AS mobile_pct + FROM {C}.`transactions` + GROUP BY 1 + ORDER BY 1 +""" +) +print("\n3. Mobile share by year (target: 25%→~35%→~48% trend):") +q3.show() + +# 4. Top 10% customer deposit concentration +q4 = spark.sql( + f""" + WITH cust_deposits AS ( + SELECT customer_id, + SUM(amount_usd) AS total_deposits + FROM {C}.`transactions` + WHERE transaction_type = 'Deposit' + GROUP BY customer_id + ), + ranked AS ( + SELECT *, + NTILE(10) OVER (ORDER BY total_deposits DESC) AS decile + FROM cust_deposits + ) + SELECT + CASE WHEN decile = 1 THEN 'Top 10%' ELSE 'Other 90%' END AS segment, + ROUND(SUM(total_deposits),2) AS deposit_volume, + ROUND(SUM(total_deposits)*100.0/SUM(SUM(total_deposits)) OVER (),1) AS pct + FROM ranked + GROUP BY 1 +""" +) +print("\n4. Deposit concentration (target: top 10% ≈ 35% of volume):") +q4.show() + +# 5. Jan 2024 complaint spike +q5 = spark.sql( + f""" + SELECT request_year AS yr, request_month AS mo, + SUM(CASE WHEN category='Complaint' THEN 1 ELSE 0 END) AS complaints, + COUNT(*) AS total_requests + FROM {C}.`service_requests` + WHERE (request_year = 2024 AND request_month BETWEEN 1 AND 3) + OR (request_year = 2023 AND request_month = 12) + GROUP BY 1,2 + ORDER BY 1,2 +""" +) +print("\n5. Jan 2024 complaint spike (should be markedly higher than adjacent months):") +q5.show() + +# 6. FK integrity check +fk_checks = [ + ( + "transactions → accounts", + f"SELECT COUNT(*) FROM {C}.`transactions` t " + f"LEFT JOIN {C}.`accounts` a USING(account_id) WHERE a.account_id IS NULL", + ), + ( + "transactions → customers", + f"SELECT COUNT(*) FROM {C}.`transactions` t " + f"LEFT JOIN {C}.`customers` c USING(customer_id) WHERE c.customer_id IS NULL", + ), + ( + "accounts → customers", + f"SELECT COUNT(*) FROM {C}.`accounts` a " + f"LEFT JOIN {C}.`customers` c USING(customer_id) WHERE c.customer_id IS NULL", + ), + ( + "accounts → products", + f"SELECT COUNT(*) FROM {C}.`accounts` a " + f"LEFT JOIN {C}.`products` p USING(product_id) WHERE p.product_id IS NULL", + ), + ( + "customers → branches", + f"SELECT COUNT(*) FROM {C}.`customers` c " + f"LEFT JOIN {C}.`branches` b ON c.primary_branch_id = b.branch_id WHERE b.branch_id IS NULL", + ), +] +print("\n6. FK integrity (all should be 0):") +for label, q in fk_checks: + bad = spark.sql(q).collect()[0][0] + status = "✓" if bad == 0 else "✗ VIOLATION" + print(f" {status} {label}: {bad} orphans") + +print("\nData generation complete.") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 11. Create Metric Views + +# COMMAND ---------- + +# ============================================================================= +# CREATE METRIC VIEWS +# ============================================================================= +print("Creating metric views...") + +metric_views = { + "mv_banking_transactions": f""" +version: "0.1" +source: {CATALOG}.{SCHEMA}.transactions + +joins: + - name: customer + source: {CATALOG}.{SCHEMA}.customers + on: source.customer_id = customer.customer_id + - name: branch + source: {CATALOG}.{SCHEMA}.branches + on: source.branch_id = branch.branch_id + - name: account + source: {CATALOG}.{SCHEMA}.accounts + on: source.account_id = account.account_id + +dimensions: + - name: Transaction Month + expr: DATE_TRUNC('MONTH', transaction_date) + - name: Transaction Year + expr: transaction_year + - name: Channel + expr: channel + - name: Transaction Type + expr: transaction_type + - name: Channel Type + expr: CASE WHEN channel IN ('Mobile', 'Online') THEN 'Digital' ELSE 'In-Person' END + - name: Relationship Tier + expr: customer.relationship_tier + - name: Customer Segment + expr: customer.customer_segment + - name: Region + expr: customer.region + - name: State + expr: customer.state + - name: Account Type + expr: account.account_type + +measures: + - name: Transaction Count + expr: COUNT(1) + - name: Total Transaction Amount + expr: SUM(amount_usd) + - name: Deposit Volume + expr: SUM(amount_usd) FILTER (WHERE transaction_type = 'Deposit') + - name: Withdrawal Volume + expr: SUM(amount_usd) FILTER (WHERE transaction_type = 'Withdrawal') + - name: Net Flow + expr: SUM(amount_usd) FILTER (WHERE transaction_type = 'Deposit') - SUM(amount_usd) FILTER (WHERE transaction_type = 'Withdrawal') + - name: Fee Revenue + expr: SUM(fee_usd) + - name: Unique Customers + expr: COUNT(DISTINCT source.customer_id) + - name: Average Transaction Amount + expr: AVG(amount_usd) + - name: Digital Transaction Count + expr: COUNT(1) FILTER (WHERE channel IN ('Mobile', 'Online')) + - name: Digital Share Pct + expr: COUNT(1) FILTER (WHERE channel IN ('Mobile', 'Online')) * 100.0 / COUNT(1) + - name: Flagged Transaction Count + expr: COUNT(1) FILTER (WHERE is_flagged = TRUE) + - name: Fee Revenue per Customer + expr: SUM(fee_usd) / COUNT(DISTINCT source.customer_id) + - name: Previous Month Deposit Volume + expr: SUM(amount_usd) FILTER (WHERE transaction_type = 'Deposit') + window: + - order: Transaction Month + range: trailing 1 month + semiadditive: last + - name: YTD Deposit Volume + expr: SUM(amount_usd) FILTER (WHERE transaction_type = 'Deposit') + window: + - order: Transaction Month + range: cumulative + semiadditive: last + - order: Transaction Year + range: current + semiadditive: last + - name: Trailing 3-Month Fee Revenue + expr: SUM(fee_usd) + window: + - order: Transaction Month + range: trailing 3 month + semiadditive: last + - name: Current Month Deposit Volume + expr: SUM(amount_usd) FILTER (WHERE transaction_type = 'Deposit') + window: + - order: Transaction Month + range: current + semiadditive: last + - name: Month-over-Month Deposit Growth Pct + expr: try_divide(MEASURE(`Current Month Deposit Volume`) - MEASURE(`Previous Month Deposit Volume`), MEASURE(`Previous Month Deposit Volume`)) * 100 +""", + "mv_customer_health": f""" +version: "1.1" +source: {CATALOG}.{SCHEMA}.accounts + +joins: + - name: customer + source: {CATALOG}.{SCHEMA}.customers + on: source.customer_id = customer.customer_id + - name: product + source: {CATALOG}.{SCHEMA}.products + on: source.product_id = product.product_id + +dimensions: + - name: Relationship Tier + expr: customer.relationship_tier + - name: Customer Segment + expr: customer.customer_segment + - name: Account Type + expr: account_type + - name: Account Status + expr: status + - name: Product Category + expr: product.product_category + - name: Product Name + expr: product.product_name + - name: Region + expr: customer.region + - name: State + expr: customer.state + - name: Acquisition Channel + expr: customer.acquisition_channel + - name: Income Band + expr: customer.income_band + - name: Age Band + expr: customer.age_band + +measures: + - name: Account Count + expr: COUNT(1) + - name: Active Account Count + expr: COUNT(1) FILTER (WHERE status = 'Active') + - name: Delinquent Account Count + expr: COUNT(1) FILTER (WHERE status = 'Delinquent') + - name: Dormant Account Count + expr: COUNT(1) FILTER (WHERE status = 'Dormant') + - name: Total Balance + expr: SUM(current_balance_usd) + - name: Average Balance + expr: AVG(current_balance_usd) + - name: Total Deposit Balance + expr: SUM(current_balance_usd) FILTER (WHERE account_type IN ('Checking', 'Savings')) + - name: Total Loan Balance + expr: SUM(current_balance_usd) FILTER (WHERE account_type IN ('Mortgage', 'Auto Loan', 'Home Equity')) + - name: Average Credit Limit + expr: AVG(credit_limit_usd) FILTER (WHERE credit_limit_usd > 0) + - name: Unique Customers + expr: COUNT(DISTINCT source.customer_id) + - name: Average Balance per Customer + expr: SUM(current_balance_usd) / COUNT(DISTINCT source.customer_id) + - name: Delinquency Rate Pct + expr: COUNT(1) FILTER (WHERE status = 'Delinquent') * 100.0 / COUNT(1) + - name: Dormancy Rate Pct + expr: COUNT(1) FILTER (WHERE status = 'Dormant') * 100.0 / COUNT(1) + - name: Mortgage Penetration Rate Pct + expr: COUNT(DISTINCT source.customer_id) FILTER (WHERE account_type = 'Mortgage') * 100.0 / COUNT(DISTINCT source.customer_id) +""", + "mv_service_quality": f""" +version: "0.1" +source: {CATALOG}.{SCHEMA}.service_requests + +joins: + - name: customer + source: {CATALOG}.{SCHEMA}.customers + on: source.customer_id = customer.customer_id + - name: branch + source: {CATALOG}.{SCHEMA}.branches + on: source.branch_id = branch.branch_id + +dimensions: + - name: Request Month + expr: DATE_TRUNC('MONTH', request_date) + - name: Request Year + expr: request_year + - name: Channel + expr: channel + - name: Category + expr: category + - name: Status + expr: status + - name: Relationship Tier + expr: customer.relationship_tier + - name: Customer Segment + expr: customer.customer_segment + - name: Region + expr: customer.region + - name: Branch Region + expr: branch.region + +measures: + - name: Request Count + expr: COUNT(1) + - name: Complaint Count + expr: COUNT(1) FILTER (WHERE category = 'Complaint') + - name: Escalated Count + expr: COUNT(1) FILTER (WHERE status = 'Escalated') + - name: Resolved Count + expr: COUNT(1) FILTER (WHERE status = 'Resolved') + - name: Open Count + expr: COUNT(1) FILTER (WHERE status = 'Open') + - name: Unique Customers + expr: COUNT(DISTINCT source.customer_id) + - name: Average Resolution Time Days + expr: AVG(resolution_time_days) FILTER (WHERE status = 'Resolved') + - name: Average Satisfaction Score + expr: AVG(satisfaction_score) + - name: Resolution Rate Pct + expr: COUNT(1) FILTER (WHERE status = 'Resolved') * 100.0 / COUNT(1) + - name: Escalation Rate Pct + expr: COUNT(1) FILTER (WHERE status = 'Escalated') * 100.0 / COUNT(1) + - name: Complaint Rate Pct + expr: COUNT(1) FILTER (WHERE category = 'Complaint') * 100.0 / COUNT(1) + - name: Previous Month Complaint Count + expr: COUNT(1) FILTER (WHERE category = 'Complaint') + window: + - order: Request Month + range: trailing 1 month + semiadditive: last + - name: Current Month Complaint Count + expr: COUNT(1) FILTER (WHERE category = 'Complaint') + window: + - order: Request Month + range: current + semiadditive: last + - name: Month-over-Month Complaint Change Pct + expr: try_divide(MEASURE(`Current Month Complaint Count`) - MEASURE(`Previous Month Complaint Count`), MEASURE(`Previous Month Complaint Count`)) * 100 + - name: Trailing 3-Month Resolution Rate Pct + expr: COUNT(1) FILTER (WHERE status = 'Resolved') * 100.0 / COUNT(1) + window: + - order: Request Month + range: trailing 3 month + semiadditive: last +""", +} + +for mv_name, yaml_body in metric_views.items(): + ddl = ( + f"CREATE OR REPLACE VIEW `{CATALOG}`.`{SCHEMA}`.`{mv_name}`\n" + "WITH METRICS\n" + "LANGUAGE YAML\n" + "AS $$" + yaml_body + "$$" + ) + spark.sql(ddl) + print(f" ✓ {mv_name}") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 12. Register Constraints & Column Comments + +# COMMAND ---------- + +# ============================================================================= +# REGISTER CONSTRAINTS & COLUMN COMMENTS +# ============================================================================= +print("Registering constraints and column comments...") + +C = f"`{CATALOG}`.`{SCHEMA}`" # shorthand for fully-qualified table refs + +# --- Set PK columns NOT NULL (required before adding primary key constraints) --- +spark.sql(f"ALTER TABLE {C}.products ALTER COLUMN product_id SET NOT NULL") +spark.sql(f"ALTER TABLE {C}.branches ALTER COLUMN branch_id SET NOT NULL") +spark.sql(f"ALTER TABLE {C}.customers ALTER COLUMN customer_id SET NOT NULL") +spark.sql(f"ALTER TABLE {C}.accounts ALTER COLUMN account_id SET NOT NULL") +spark.sql(f"ALTER TABLE {C}.transactions ALTER COLUMN transaction_id SET NOT NULL") +spark.sql(f"ALTER TABLE {C}.service_requests ALTER COLUMN request_id SET NOT NULL") + +# --- Primary keys --- +spark.sql( + f"ALTER TABLE {C}.products ADD CONSTRAINT pk_products PRIMARY KEY (product_id)" +) +spark.sql( + f"ALTER TABLE {C}.branches ADD CONSTRAINT pk_branches PRIMARY KEY (branch_id)" +) +spark.sql( + f"ALTER TABLE {C}.customers ADD CONSTRAINT pk_customers PRIMARY KEY (customer_id)" +) +spark.sql( + f"ALTER TABLE {C}.accounts ADD CONSTRAINT pk_accounts PRIMARY KEY (account_id)" +) +spark.sql( + f"ALTER TABLE {C}.transactions ADD CONSTRAINT pk_transactions PRIMARY KEY (transaction_id)" +) +spark.sql( + f"ALTER TABLE {C}.service_requests ADD CONSTRAINT pk_service_requests PRIMARY KEY (request_id)" +) + +# --- Foreign keys --- +spark.sql( + f"ALTER TABLE {C}.customers ADD CONSTRAINT fk_customers_branch FOREIGN KEY (primary_branch_id) REFERENCES {C}.branches (branch_id)" +) +spark.sql( + f"ALTER TABLE {C}.accounts ADD CONSTRAINT fk_accounts_customer FOREIGN KEY (customer_id) REFERENCES {C}.customers (customer_id)" +) +spark.sql( + f"ALTER TABLE {C}.accounts ADD CONSTRAINT fk_accounts_product FOREIGN KEY (product_id) REFERENCES {C}.products (product_id)" +) +spark.sql( + f"ALTER TABLE {C}.transactions ADD CONSTRAINT fk_transactions_account FOREIGN KEY (account_id) REFERENCES {C}.accounts (account_id)" +) +spark.sql( + f"ALTER TABLE {C}.transactions ADD CONSTRAINT fk_transactions_customer FOREIGN KEY (customer_id) REFERENCES {C}.customers (customer_id)" +) +spark.sql( + f"ALTER TABLE {C}.transactions ADD CONSTRAINT fk_transactions_branch FOREIGN KEY (branch_id) REFERENCES {C}.branches (branch_id)" +) +spark.sql( + f"ALTER TABLE {C}.service_requests ADD CONSTRAINT fk_sr_customer FOREIGN KEY (customer_id) REFERENCES {C}.customers (customer_id)" +) +spark.sql( + f"ALTER TABLE {C}.service_requests ADD CONSTRAINT fk_sr_branch FOREIGN KEY (branch_id) REFERENCES {C}.branches (branch_id)" +) + +# --- Column comments: products --- +spark.sql( + f"ALTER TABLE {C}.products ALTER COLUMN product_id COMMENT 'Primary key. Format: PROD-NNN'" +) +spark.sql( + f"ALTER TABLE {C}.products ALTER COLUMN product_name COMMENT 'Full commercial name of the product (e.g., Horizon Rewards Visa)'" +) +spark.sql( + f"ALTER TABLE {C}.products ALTER COLUMN product_category COMMENT 'High-level category: Deposit, Credit, or Lending'" +) +spark.sql( + f"ALTER TABLE {C}.products ALTER COLUMN product_type COMMENT 'Specific type: Checking, Savings, Credit Card, Mortgage, Auto Loan, HELOC'" +) +spark.sql( + f"ALTER TABLE {C}.products ALTER COLUMN annual_fee_usd COMMENT 'Annual fee charged to the customer in USD. 0 for fee-free products'" +) +spark.sql( + f"ALTER TABLE {C}.products ALTER COLUMN base_interest_rate_pct COMMENT 'Base APR or APY in percent (e.g., 19.99 means 19.99%). Deposit rates are APY; lending rates are APR'" +) +spark.sql( + f"ALTER TABLE {C}.products ALTER COLUMN reward_program COMMENT 'Loyalty/reward program: None, Cash Back, Points, or Miles'" +) +spark.sql( + f"ALTER TABLE {C}.products ALTER COLUMN min_balance_usd COMMENT 'Minimum balance required to avoid fees or qualify for rate. 0 if no minimum'" +) +spark.sql( + f"ALTER TABLE {C}.products ALTER COLUMN is_active COMMENT 'True if the product is currently offered to new customers'" +) +spark.sql( + f"ALTER TABLE {C}.products ALTER COLUMN launched_date COMMENT 'Date the product was first made available'" +) + +# --- Column comments: branches --- +spark.sql( + f"ALTER TABLE {C}.branches ALTER COLUMN branch_id COMMENT 'Primary key. Format: BRNCH-NNN'" +) +spark.sql( + f"ALTER TABLE {C}.branches ALTER COLUMN branch_name COMMENT 'Descriptive name of the branch location'" +) +spark.sql( + f"ALTER TABLE {C}.branches ALTER COLUMN branch_type COMMENT 'Service level: Full Service, Limited Service, or Drive-Through'" +) +spark.sql( + f"ALTER TABLE {C}.branches ALTER COLUMN region COMMENT 'Geographic region: Northeast, Southeast, Midwest, West, Southwest'" +) +spark.sql( + f"ALTER TABLE {C}.branches ALTER COLUMN state COMMENT 'US 2-letter state code (e.g., NY, CA)'" +) +spark.sql( + f"ALTER TABLE {C}.branches ALTER COLUMN city COMMENT 'City where the branch is located'" +) +spark.sql( + f"ALTER TABLE {C}.branches ALTER COLUMN opened_date COMMENT 'Date the branch first opened for business'" +) +spark.sql( + f"ALTER TABLE {C}.branches ALTER COLUMN monthly_operating_cost_usd COMMENT 'Fixed monthly cost to operate the branch in USD (rent, salaries, utilities)'" +) +spark.sql( + f"ALTER TABLE {C}.branches ALTER COLUMN headcount COMMENT 'Number of full-time employees at the branch'" +) +spark.sql( + f"ALTER TABLE {C}.branches ALTER COLUMN is_active COMMENT 'True if the branch is currently open'" +) + +# --- Column comments: customers --- +spark.sql( + f"ALTER TABLE {C}.customers ALTER COLUMN customer_id COMMENT 'Primary key. Format: CUST-NNNN'" +) +spark.sql( + f"ALTER TABLE {C}.customers ALTER COLUMN customer_name COMMENT 'Full name of the customer'" +) +spark.sql( + f"ALTER TABLE {C}.customers ALTER COLUMN age_band COMMENT 'Age range bracket: 18-24, 25-34, 35-44, 45-54, 55-64, 65+'" +) +spark.sql( + f"ALTER TABLE {C}.customers ALTER COLUMN income_band COMMENT 'Annual household income bracket: <$50K, $50K-$100K, $100K-$200K, $200K+'" +) +spark.sql( + f"ALTER TABLE {C}.customers ALTER COLUMN customer_segment COMMENT 'Business segment: Retail, Small Business, or Wealth Management'" +) +spark.sql( + f"ALTER TABLE {C}.customers ALTER COLUMN relationship_tier COMMENT 'Relationship depth tier: Standard, Preferred, or Private Client. Private Client customers hold 3x higher average balances'" +) +spark.sql( + f"ALTER TABLE {C}.customers ALTER COLUMN state COMMENT 'US 2-letter state code for customer primary residence'" +) +spark.sql( + f"ALTER TABLE {C}.customers ALTER COLUMN region COMMENT 'Geographic region derived from state: Northeast, Southeast, Midwest, West, Southwest'" +) +spark.sql( + f"ALTER TABLE {C}.customers ALTER COLUMN primary_branch_id COMMENT 'FK → branches. The branch this customer is primarily associated with'" +) +spark.sql( + f"ALTER TABLE {C}.customers ALTER COLUMN acquisition_channel COMMENT 'How the customer was acquired: Branch, Online, Mobile, or Referral'" +) +spark.sql( + f"ALTER TABLE {C}.customers ALTER COLUMN customer_since_date COMMENT 'Date the customer first opened a relationship with Horizon Bank'" +) +spark.sql( + f"ALTER TABLE {C}.customers ALTER COLUMN is_active COMMENT 'True if the customer relationship is active (~94% of customers)'" +) +spark.sql( + f"ALTER TABLE {C}.customers ALTER COLUMN has_checking COMMENT 'True if the customer holds at least one checking account'" +) +spark.sql( + f"ALTER TABLE {C}.customers ALTER COLUMN has_savings COMMENT 'True if the customer holds at least one savings account'" +) +spark.sql( + f"ALTER TABLE {C}.customers ALTER COLUMN has_credit_card COMMENT 'True if the customer holds at least one credit card'" +) +spark.sql( + f"ALTER TABLE {C}.customers ALTER COLUMN has_mortgage COMMENT 'True if the customer holds at least one mortgage. Private Client: ~60% penetration'" +) +spark.sql( + f"ALTER TABLE {C}.customers ALTER COLUMN credit_score_band COMMENT 'Credit score bracket: Poor (<580), Fair (580-669), Good (670-739), Very Good (740-799), Exceptional (800+)'" +) + +# --- Column comments: accounts --- +spark.sql( + f"ALTER TABLE {C}.accounts ALTER COLUMN account_id COMMENT 'Primary key. Format: ACCT-NNNNN'" +) +spark.sql( + f"ALTER TABLE {C}.accounts ALTER COLUMN customer_id COMMENT 'FK → customers. Owner of this account'" +) +spark.sql( + f"ALTER TABLE {C}.accounts ALTER COLUMN product_id COMMENT 'FK → products. The specific product this account is based on'" +) +spark.sql( + f"ALTER TABLE {C}.accounts ALTER COLUMN account_type COMMENT 'Type of account: Checking, Savings, Credit Card, Mortgage, Auto Loan, Home Equity'" +) +spark.sql( + f"ALTER TABLE {C}.accounts ALTER COLUMN open_date COMMENT 'Date the account was opened'" +) +spark.sql( + f"ALTER TABLE {C}.accounts ALTER COLUMN close_date COMMENT 'Date the account was closed. NULL if account is still open'" +) +spark.sql( + f"ALTER TABLE {C}.accounts ALTER COLUMN status COMMENT 'Current account status: Active, Dormant, Closed, or Delinquent'" +) +spark.sql( + f"ALTER TABLE {C}.accounts ALTER COLUMN current_balance_usd COMMENT 'Current balance in USD. For credit accounts: outstanding balance owed. For deposit accounts: available balance'" +) +spark.sql( + f"ALTER TABLE {C}.accounts ALTER COLUMN credit_limit_usd COMMENT 'Credit limit in USD. Only populated for Credit Card and HELOC accounts; NULL for deposit and other lending accounts'" +) +spark.sql( + f"ALTER TABLE {C}.accounts ALTER COLUMN interest_rate_pct COMMENT 'Account-specific interest rate in percent. May vary from product base_interest_rate_pct due to promotional or relationship pricing'" +) +spark.sql( + f"ALTER TABLE {C}.accounts ALTER COLUMN is_primary_account COMMENT 'True if this is the customer''s primary account (typically the first checking account)'" +) + +# --- Column comments: transactions --- +spark.sql( + f"ALTER TABLE {C}.transactions ALTER COLUMN transaction_id COMMENT 'Primary key. Format: TXN-NNNNNNN'" +) +spark.sql( + f"ALTER TABLE {C}.transactions ALTER COLUMN transaction_date COMMENT 'Date the transaction occurred. Range: 2023-01-01 to 2025-12-31'" +) +spark.sql( + f"ALTER TABLE {C}.transactions ALTER COLUMN transaction_year COMMENT 'Calendar year of the transaction (2023, 2024, or 2025). Redundant column for easy filtering without date functions'" +) +spark.sql( + f"ALTER TABLE {C}.transactions ALTER COLUMN transaction_month COMMENT 'Calendar month of the transaction (1–12). Redundant column for easy filtering'" +) +spark.sql( + f"ALTER TABLE {C}.transactions ALTER COLUMN transaction_quarter COMMENT 'Calendar quarter of the transaction (1–4). Redundant column for easy filtering'" +) +spark.sql( + f"ALTER TABLE {C}.transactions ALTER COLUMN account_id COMMENT 'FK → accounts. The account on which this transaction was recorded'" +) +spark.sql( + f"ALTER TABLE {C}.transactions ALTER COLUMN customer_id COMMENT 'FK → customers. Denormalized from accounts for query convenience'" +) +spark.sql( + f"ALTER TABLE {C}.transactions ALTER COLUMN branch_id COMMENT 'FK → branches. NULL for digital-channel transactions (Online, Mobile, ATM, Wire)'" +) +spark.sql( + f"ALTER TABLE {C}.transactions ALTER COLUMN transaction_type COMMENT 'Type: Deposit (in), Withdrawal (out), Transfer, Payment, Purchase (credit card), Fee, or Interest. Amounts are always positive — direction is determined by transaction_type'" +) +spark.sql( + f"ALTER TABLE {C}.transactions ALTER COLUMN channel COMMENT 'Channel used: Branch, ATM, Online, Mobile, or Wire. Mobile share grows from 25% (Jan 2023) to 48% (Dec 2025)'" +) +spark.sql( + f"ALTER TABLE {C}.transactions ALTER COLUMN amount_usd COMMENT 'Transaction amount in USD. Always stored as a positive value. Use transaction_type to determine direction of money flow'" +) +spark.sql( + f"ALTER TABLE {C}.transactions ALTER COLUMN fee_usd COMMENT 'Explicit fee in USD. Separate from amount_usd. 0 for most transactions — non-zero for wire transfers and overdraft events. Grows ~20% YoY'" +) +spark.sql( + f"ALTER TABLE {C}.transactions ALTER COLUMN balance_after_usd COMMENT 'Account balance after this transaction was applied'" +) +spark.sql( + f"ALTER TABLE {C}.transactions ALTER COLUMN merchant_category COMMENT 'Merchant category for credit card purchases only: Groceries, Travel, Dining, Gas, Retail, Healthcare, Entertainment. NULL for all other transaction types'" +) +spark.sql( + f"ALTER TABLE {C}.transactions ALTER COLUMN is_international COMMENT 'True if the transaction involved a foreign counterparty or cross-border transfer'" +) +spark.sql( + f"ALTER TABLE {C}.transactions ALTER COLUMN status COMMENT 'Settlement status: Posted (cleared), Pending, or Reversed'" +) +spark.sql( + f"ALTER TABLE {C}.transactions ALTER COLUMN is_flagged COMMENT 'True if the transaction was flagged by fraud or AML monitoring (~0.5% of transactions)'" +) + +# --- Column comments: service_requests --- +spark.sql( + f"ALTER TABLE {C}.service_requests ALTER COLUMN request_id COMMENT 'Primary key. Format: SR-NNNNNN'" +) +spark.sql( + f"ALTER TABLE {C}.service_requests ALTER COLUMN request_date COMMENT 'Date the service request was submitted. Range: 2023-01-01 to 2025-12-31'" +) +spark.sql( + f"ALTER TABLE {C}.service_requests ALTER COLUMN request_year COMMENT 'Calendar year of the request. Redundant for easy filtering'" +) +spark.sql( + f"ALTER TABLE {C}.service_requests ALTER COLUMN request_month COMMENT 'Calendar month of the request (1–12). Redundant for easy filtering'" +) +spark.sql( + f"ALTER TABLE {C}.service_requests ALTER COLUMN customer_id COMMENT 'FK → customers. The customer who submitted this request'" +) +spark.sql( + f"ALTER TABLE {C}.service_requests ALTER COLUMN branch_id COMMENT 'FK → branches. NULL for non-branch channels (Phone, Chat, App)'" +) +spark.sql( + f"ALTER TABLE {C}.service_requests ALTER COLUMN channel COMMENT 'Channel used to submit the request: Phone, Chat, Branch, or App'" +) +spark.sql( + f"ALTER TABLE {C}.service_requests ALTER COLUMN category COMMENT 'Category: Account Inquiry, Dispute, Complaint, Product Inquiry, or Technical Issue. Complaint requests spike +80% in Jan 2024 (system outage narrative)'" +) +spark.sql( + f"ALTER TABLE {C}.service_requests ALTER COLUMN status COMMENT 'Resolution status: Resolved, Open, or Escalated'" +) +spark.sql( + f"ALTER TABLE {C}.service_requests ALTER COLUMN resolution_time_days COMMENT 'Days from submission to resolution. NULL if still Open. Escalated requests may have a partial value'" +) +spark.sql( + f"ALTER TABLE {C}.service_requests ALTER COLUMN satisfaction_score COMMENT 'Customer satisfaction rating 1–5 (5 = most satisfied). NULL if not yet rated. Lower scores correlate with Complaint category'" +) +spark.sql( + f"ALTER TABLE {C}.service_requests ALTER COLUMN is_resolved COMMENT 'True if status = Resolved. Convenience boolean for filtering'" +) + +# --- Table comments --- +spark.sql( + f"COMMENT ON TABLE {C}.products IS 'Product catalog for Horizon Bank. 20 rows covering Deposit, Credit, and Lending product types. Static reference dimension.'" +) +spark.sql( + f"COMMENT ON TABLE {C}.branches IS 'Branch dimension for Horizon Bank. 25 branches across 5 US regions. Southeast branches have ~20% higher average transaction values vs. national average.'" +) +spark.sql( + f"COMMENT ON TABLE {C}.customers IS 'Customer dimension. 1,000 synthetic customers. Private Client tier (top 10%) holds 3x the average account balance and has 60% mortgage penetration.'" +) +spark.sql( + f"COMMENT ON TABLE {C}.accounts IS 'Account dimension. ~2,500 accounts across all product types. FK to customers and products. current_balance_usd reflects the balance after all transactions in the 2023–2025 window.'" +) +spark.sql( + f"COMMENT ON TABLE {C}.transactions IS 'Primary fact table. 10,000 transactions from 2023-01-01 to 2025-12-31. Key patterns: (1) Nov/Dec volume +45%, (2) Q2 2024 deposit dip -15%, (3) Mobile channel grows 25% to 48%, (4) Fee revenue +20% YoY, (5) Top 10% customers = ~35% of deposit volume. Amounts are always positive — use transaction_type for direction.'" +) +spark.sql( + f"COMMENT ON TABLE {C}.service_requests IS 'Secondary fact table. 3,000 service requests from 2023–2025. Complaint category spikes +80% in Jan 2024, correlating with a simulated system outage narrative.'" +) + +print(" ✓ PK/FK constraints and column comments registered") + +print() +print("=" * 60) +print("SETUP COMPLETE") +print("=" * 60) +print(f" Catalog : {CATALOG}") +print(f" Schema : {SCHEMA}") +print( + " Tables : products, branches, customers, accounts,transactions, service_requests" +) +print(" Metric Views: mv_banking_transactions, mv_customer_health, mv_service_quality") +print(" Next : Create a Genie space from these tables, or see notebooks/demo-data/README.md") diff --git a/notebooks/demo-data/generate_hospital_readmission_data.py b/notebooks/demo-data/generate_hospital_readmission_data.py new file mode 100644 index 000000000..35331bace --- /dev/null +++ b/notebooks/demo-data/generate_hospital_readmission_data.py @@ -0,0 +1,805 @@ +# Databricks notebook source + +# MAGIC %md +# MAGIC # Hospital Readmission - Synthetic Dataset Generator +# MAGIC +# MAGIC Generates a fictional hospital readmission dataset for Databricks AI/BI Genie demos. +# MAGIC +# MAGIC Patient records use synthetic IDs only. The notebook does not generate patient names, DOBs, +# MAGIC addresses, emails, phone numbers, or real PHI-like identifiers. +# MAGIC +# MAGIC | Table | Rows | Description | +# MAGIC |---|---:|---| +# MAGIC | `patients` | 2,000 | Synthetic patient risk dimension | +# MAGIC | `hospitals` | 12 | Hospital dimension | +# MAGIC | `encounters` | 7,000 | Index admission encounters | +# MAGIC | `care_transitions` | 7,000 | Discharge follow-up and transition actions | +# MAGIC | `claims` | 7,000 | Encounter-level claim facts | +# MAGIC | `readmissions` | variable | 30-day readmission facts | +# MAGIC +# MAGIC **Setup:** Set the `catalog` and `schema` widgets (or edit defaults below), then **Run All**. +# MAGIC +# MAGIC Seed: `42` - fully reproducible. + +# COMMAND ---------- + +# MAGIC %pip install faker==40.19.1 --quiet +# MAGIC %restart_python + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 1. Configuration + +# COMMAND ---------- + +# ============================================================================= +# CONFIGURATION - Edit only this section before running +# ============================================================================= +DEFAULT_CATALOG = "" # Unity Catalog name; required +DEFAULT_SCHEMA = "hospital_readmission" # Schema / database name + +# Widgets let the bulk runner pass values while preserving standalone defaults. +try: + dbutils.widgets.text("catalog", DEFAULT_CATALOG, "Unity Catalog name") +except Exception: + pass + +try: + dbutils.widgets.text("schema", DEFAULT_SCHEMA, "Schema / database name") +except Exception: + pass + +try: + dbutils.widgets.dropdown("overwrite_existing", "false", ["false", "true"], "Overwrite existing tables") +except Exception: + pass + +try: + CATALOG = dbutils.widgets.get("catalog").strip() or DEFAULT_CATALOG +except Exception: + CATALOG = DEFAULT_CATALOG + +try: + SCHEMA = dbutils.widgets.get("schema").strip() or DEFAULT_SCHEMA +except Exception: + SCHEMA = DEFAULT_SCHEMA + +try: + OVERWRITE_EXISTING = dbutils.widgets.get("overwrite_existing").strip().lower() == "true" +except Exception: + OVERWRITE_EXISTING = False + +if not CATALOG: + raise ValueError("Set the catalog widget to a Unity Catalog name before running this notebook.") + +WRITE_MODE = "overwrite" if OVERWRITE_EXISTING else "errorifexists" + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 2. Imports & Global Setup + +# COMMAND ---------- + +# ============================================================================= +# IMPORTS & GLOBAL SETUP +# ============================================================================= +import random +from calendar import monthrange +from datetime import date, timedelta + +import numpy as np +import pandas as pd +from faker import Faker + +SEED = 42 +random.seed(SEED) +np.random.seed(SEED) +fake = Faker("en_US") +Faker.seed(SEED) + +START_DATE = date(2023, 1, 1) +END_DATE = date(2025, 12, 31) +MONTH_LIST = [(y, m) for y in [2023, 2024, 2025] for m in range(1, 13)] + +SERVICE_LINES = ["Medicine", "Cardiology", "Pulmonology", "Surgery", "Orthopedics", "Neurology"] +DIAGNOSIS_GROUPS = ["CHF", "COPD", "Pneumonia", "Sepsis", "Joint Replacement", "Stroke", "Diabetes", "GI"] +PAYER_TYPES = ["Medicare", "Medicaid", "Commercial", "Self Pay"] + +print("Setup complete.") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 3. Hospitals + +# COMMAND ---------- + +# ============================================================================= +# TABLE 1: HOSPITALS (12 rows) +# ============================================================================= +HOSPITAL_SEEDS = [ + ("HOSP-001", "North Valley Medical Center", "Northeast", "NY", 420, "Academic"), + ("HOSP-002", "Harborview Regional", "Northeast", "MA", 310, "Community"), + ("HOSP-003", "Piedmont County Hospital", "Southeast", "GA", 260, "Community"), + ("HOSP-004", "Gulf Coast Medical", "Southeast", "FL", 380, "Teaching"), + ("HOSP-005", "Lake Plains Hospital", "Midwest", "IL", 340, "Teaching"), + ("HOSP-006", "Riverbend General", "Midwest", "OH", 220, "Community"), + ("HOSP-007", "Front Range Health", "West", "CO", 295, "Community"), + ("HOSP-008", "Pacific Crest Hospital", "West", "CA", 510, "Academic"), + ("HOSP-009", "Cascade Valley Medical", "West", "WA", 275, "Community"), + ("HOSP-010", "Desert Springs Hospital", "Southwest", "AZ", 240, "Community"), + ("HOSP-011", "Lone Star Medical Center", "Southwest", "TX", 460, "Teaching"), + ("HOSP-012", "Capital Care Hospital", "Northeast", "PA", 330, "Community"), +] + +hospitals_data = [] +for idx, (hid, name, region, state, beds, hospital_type) in enumerate(HOSPITAL_SEEDS, start=1): + has_program = idx in (1, 4, 5, 8, 11) + hospitals_data.append( + { + "hospital_id": hid, + "hospital_name": name, + "region": region, + "state": state, + "bed_count": beds, + "hospital_type": hospital_type, + "transition_program_start_date": date(2024, 7, 1) if has_program else None, + "has_transition_program": bool(has_program), + "is_active": True, + } + ) + +df_hospitals = pd.DataFrame(hospitals_data) +HOSPITAL_BY_ID = {h["hospital_id"]: h for h in hospitals_data} +HOSPITAL_IDS = [h["hospital_id"] for h in hospitals_data] +print(f"hospitals: {len(df_hospitals)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 4. Patients + +# COMMAND ---------- + +# ============================================================================= +# TABLE 2: PATIENTS (2,000 rows) +# ============================================================================= +# No patient names, DOBs, addresses, emails, or phone numbers are generated. +patients_data = [] +for i in range(1, 2001): + age_band = random.choices( + ["18-44", "45-64", "65-74", "75-84", "85+"], + weights=[14, 27, 25, 22, 12], + )[0] + risk_band = random.choices(["Low", "Medium", "High", "Very High"], weights=[28, 38, 24, 10])[0] + chronic_base = {"Low": 0, "Medium": 1, "High": 2, "Very High": 4}[risk_band] + patients_data.append( + { + "patient_id": f"PT-{i:06d}", + "age_band": age_band, + "clinical_risk_band": risk_band, + "chronic_condition_count": int(chronic_base + random.choice([0, 0, 1, 1, 2])), + "primary_region": random.choice(["Northeast", "Southeast", "Midwest", "West", "Southwest"]), + "payer_type": random.choices(PAYER_TYPES, weights=[42, 18, 34, 6])[0], + "prior_admission_count": random.choices([0, 1, 2, 3, 4], weights=[45, 25, 17, 9, 4])[0], + "social_risk_index": round(random.uniform(0.05, 0.95), 2), + "is_active": True, + } + ) + +df_patients = pd.DataFrame(patients_data) +PATIENT_BY_ID = {p["patient_id"]: p for p in patients_data} +PATIENT_IDS = [p["patient_id"] for p in patients_data] +print(f"patients: {len(df_patients)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 5. Encounters + +# COMMAND ---------- + +# ============================================================================= +# TABLE 3: ENCOUNTERS (7,000 rows) +# ============================================================================= +# Injected patterns: +# 1. CHF and COPD have higher readmission risk. +# 2. Respiratory admissions rise during flu season. +# 3. Weekend discharges have higher readmission risk. +# 4. Transition program hospitals improve after mid-2024. + +month_weights = [] +for year, month in MONTH_LIST: + if month in (1, 2, 12): + month_weights.append(1.30) + elif month in (7, 8): + month_weights.append(0.90) + else: + month_weights.append(1.0) +encounters_per_month_raw = [w / sum(month_weights) * 7000 for w in month_weights] +encounters_per_month = [int(round(x)) for x in encounters_per_month_raw] +encounters_per_month[-1] += 7000 - sum(encounters_per_month) + + +def diagnosis_for_month(month): + if month in (1, 2, 12): + return random.choices(DIAGNOSIS_GROUPS, weights=[16, 18, 26, 11, 6, 7, 9, 7])[0] + return random.choices(DIAGNOSIS_GROUPS, weights=[15, 14, 12, 14, 14, 9, 12, 10])[0] + + +def service_for_diagnosis(diagnosis): + mapping = { + "CHF": "Cardiology", + "COPD": "Pulmonology", + "Pneumonia": "Pulmonology", + "Sepsis": "Medicine", + "Joint Replacement": "Orthopedics", + "Stroke": "Neurology", + "Diabetes": "Medicine", + "GI": "Surgery", + } + return mapping[diagnosis] + + +encounters_data = [] +encounter_counter = 1 + +for month_idx, (year, month) in enumerate(MONTH_LIST): + n_encounters = encounters_per_month[month_idx] + max_day = monthrange(year, month)[1] + for _ in range(n_encounters): + patient_id = random.choice(PATIENT_IDS) + hospital_id = random.choice(HOSPITAL_IDS) + diagnosis = diagnosis_for_month(month) + service_line = service_for_diagnosis(diagnosis) + admit_date = date(year, month, random.randint(1, max_day)) + los = random.choices([1, 2, 3, 4, 5, 6, 7, 8, 10, 12], weights=[8, 13, 18, 17, 14, 10, 8, 5, 4, 3])[0] + if diagnosis in ("CHF", "COPD", "Sepsis", "Stroke"): + los += random.choice([0, 1, 1, 2, 3]) + discharge_date = min(END_DATE, admit_date + timedelta(days=los)) + weekend_discharge = discharge_date.weekday() >= 5 + encounters_data.append( + { + "encounter_id": f"ENC-{encounter_counter:07d}", + "patient_id": patient_id, + "hospital_id": hospital_id, + "admit_date": admit_date, + "discharge_date": discharge_date, + "encounter_year": year, + "encounter_month": month, + "service_line": service_line, + "primary_diagnosis_group": diagnosis, + "acuity_level": random.choices(["Low", "Medium", "High", "Critical"], weights=[15, 46, 30, 9])[0], + "length_of_stay_days": int(los), + "discharge_disposition": random.choices( + ["Home", "Home Health", "Skilled Nursing", "Rehab", "Expired"], + weights=[58, 21, 12, 8, 1], + )[0], + "weekend_discharge": bool(weekend_discharge), + } + ) + encounter_counter += 1 + +df_encounters = pd.DataFrame(encounters_data) +ENCOUNTER_BY_ID = {e["encounter_id"]: e for e in encounters_data} +print(f"encounters: {len(df_encounters)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 6. Care Transitions + +# COMMAND ---------- + +# ============================================================================= +# TABLE 4: CARE TRANSITIONS (one row per encounter) +# ============================================================================= +care_transitions_data = [] +transition_counter = 1 + +for encounter in encounters_data: + hospital = HOSPITAL_BY_ID[encounter["hospital_id"]] + patient = PATIENT_BY_ID[encounter["patient_id"]] + program_active = ( + hospital["has_transition_program"] + and encounter["discharge_date"] >= hospital["transition_program_start_date"] + ) + followup_prob = 0.48 + (0.25 if program_active else 0.0) + if patient["clinical_risk_band"] in ("High", "Very High"): + followup_prob += 0.08 + follow_up = random.random() < min(0.90, followup_prob) + med_rec = random.random() < (0.82 if program_active else 0.68) + home_health = encounter["discharge_disposition"] in ("Home Health", "Skilled Nursing") or random.random() < 0.14 + care_transitions_data.append( + { + "transition_id": f"TRN-{transition_counter:07d}", + "encounter_id": encounter["encounter_id"], + "patient_id": encounter["patient_id"], + "hospital_id": encounter["hospital_id"], + "discharge_date": encounter["discharge_date"], + "follow_up_within_7_days": bool(follow_up), + "medication_reconciliation_completed": bool(med_rec), + "home_health_referral": bool(home_health), + "transition_program_enrolled": bool(program_active and random.random() < 0.78), + "outreach_attempts": int(random.choices([0, 1, 2, 3, 4], weights=[12, 38, 29, 15, 6])[0]), + } + ) + transition_counter += 1 + +df_care_transitions = pd.DataFrame(care_transitions_data) +TRANSITION_BY_ENCOUNTER = {t["encounter_id"]: t for t in care_transitions_data} +print(f"care_transitions: {len(df_care_transitions)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 7. Claims + +# COMMAND ---------- + +# ============================================================================= +# TABLE 5: CLAIMS (one row per encounter) +# ============================================================================= +BASE_COST = { + "Medicine": 9200, + "Cardiology": 18500, + "Pulmonology": 12800, + "Surgery": 23500, + "Orthopedics": 27000, + "Neurology": 21200, +} +PAYER_MULT = {"Medicare": 1.00, "Medicaid": 0.72, "Commercial": 1.28, "Self Pay": 0.58} + +claims_data = [] +claim_counter = 1 +for encounter in encounters_data: + patient = PATIENT_BY_ID[encounter["patient_id"]] + payer = patient["payer_type"] + base = BASE_COST[encounter["service_line"]] + severity_mult = {"Low": 0.76, "Medium": 1.0, "High": 1.42, "Critical": 2.1}[encounter["acuity_level"]] + allowed = base * severity_mult * PAYER_MULT[payer] * random.uniform(0.78, 1.35) + patient_resp = allowed * {"Medicare": 0.08, "Medicaid": 0.02, "Commercial": 0.12, "Self Pay": 0.45}[payer] + claim_status = random.choices(["Paid", "Denied", "Pending"], weights=[91, 4, 5])[0] + paid_date = None + if claim_status == "Paid": + paid_date = min(END_DATE, encounter["discharge_date"] + timedelta(days=random.randint(20, 90))) + claims_data.append( + { + "claim_id": f"CLM-{claim_counter:07d}", + "encounter_id": encounter["encounter_id"], + "patient_id": encounter["patient_id"], + "hospital_id": encounter["hospital_id"], + "payer_type": payer, + "service_line": encounter["service_line"], + "allowed_amount_usd": round(allowed, 2), + "patient_responsibility_usd": round(patient_resp, 2), + "claim_status": claim_status, + "paid_date": paid_date, + } + ) + claim_counter += 1 + +df_claims = pd.DataFrame(claims_data) +print(f"claims: {len(df_claims)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 8. Readmissions + +# COMMAND ---------- + +# ============================================================================= +# TABLE 6: READMISSIONS +# ============================================================================= +readmissions_data = [] +readmission_counter = 1 + +for encounter in encounters_data: + if encounter["discharge_date"] > END_DATE - timedelta(days=30): + continue + patient = PATIENT_BY_ID[encounter["patient_id"]] + transition = TRANSITION_BY_ENCOUNTER[encounter["encounter_id"]] + hospital = HOSPITAL_BY_ID[encounter["hospital_id"]] + diagnosis = encounter["primary_diagnosis_group"] + prob = 0.07 + if diagnosis in ("CHF", "COPD"): + prob += 0.085 + if diagnosis in ("Pneumonia", "Sepsis"): + prob += 0.035 + if patient["clinical_risk_band"] == "High": + prob += 0.035 + if patient["clinical_risk_band"] == "Very High": + prob += 0.070 + if patient["age_band"] in ("75-84", "85+"): + prob += 0.035 + if encounter["weekend_discharge"]: + prob += 0.030 + if not transition["follow_up_within_7_days"]: + prob += 0.060 + if hospital["has_transition_program"] and encounter["discharge_date"] >= hospital["transition_program_start_date"]: + prob -= 0.045 + prob = max(0.015, min(0.38, prob)) + if random.random() < prob: + days_to_readmit = random.randint(2, 30) + readmit_date = encounter["discharge_date"] + timedelta(days=days_to_readmit) + readmit_diag = diagnosis if random.random() < 0.66 else diagnosis_for_month(readmit_date.month) + readmissions_data.append( + { + "readmission_id": f"READM-{readmission_counter:07d}", + "index_encounter_id": encounter["encounter_id"], + "patient_id": encounter["patient_id"], + "hospital_id": encounter["hospital_id"], + "readmission_date": readmit_date, + "readmission_year": readmit_date.year, + "readmission_month": readmit_date.month, + "days_to_readmission": int(days_to_readmit), + "readmission_diagnosis_group": readmit_diag, + "planned_readmission": bool(random.random() < 0.08), + "readmission_cost_usd": round(BASE_COST[service_for_diagnosis(readmit_diag)] * random.uniform(0.75, 1.55), 2), + } + ) + readmission_counter += 1 + +df_readmissions = pd.DataFrame(readmissions_data) +print(f"readmissions: {len(df_readmissions)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 9. Create Schema & Write Delta Tables + +# COMMAND ---------- + +# ============================================================================= +# CREATE SCHEMA & WRITE DELTA TABLES +# ============================================================================= +spark.sql(f"CREATE SCHEMA IF NOT EXISTS `{CATALOG}`.`{SCHEMA}`") + + +def write_table(df_pd, table_name, mode=WRITE_MODE): + df_spark = spark.createDataFrame(df_pd) + ( + df_spark.write.format("delta") + .mode(mode) + .option("overwriteSchema", "true") + .saveAsTable(f"`{CATALOG}`.`{SCHEMA}`.`{table_name}`") + ) + print(f" OK {table_name}: {df_spark.count()} rows written") + + +TABLES = { + "patients": df_patients, + "hospitals": df_hospitals, + "encounters": df_encounters, + "care_transitions": df_care_transitions, + "claims": df_claims, + "readmissions": df_readmissions, +} + +print(f"Writing tables to `{CATALOG}`.`{SCHEMA}` with mode={WRITE_MODE}...") +for _table_name, _df in TABLES.items(): + write_table(_df, _table_name) +print("All tables written.") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 10. Verification + +# COMMAND ---------- + +# ============================================================================= +# VERIFICATION +# ============================================================================= +C = f"`{CATALOG}`.`{SCHEMA}`" + +print("=" * 60) +print("ROW COUNTS") +print("=" * 60) +for tbl in TABLES: + n = spark.sql(f"SELECT COUNT(*) AS n FROM {C}.`{tbl}`").collect()[0]["n"] + print(f" {tbl:<24} {n:>8,}") + +print() +print("=" * 60) +print("PATTERN VALIDATION") +print("=" * 60) + +print("\n1. CHF/COPD should have higher readmission rates.") +spark.sql( + f""" + SELECT CASE WHEN e.primary_diagnosis_group IN ('CHF', 'COPD') THEN 'CHF/COPD' ELSE 'Other' END AS diagnosis_group, + COUNT(DISTINCT e.encounter_id) AS encounters, + COUNT(DISTINCT r.readmission_id) AS readmissions, + ROUND(COUNT(DISTINCT r.readmission_id) * 100.0 / COUNT(DISTINCT e.encounter_id), 1) AS readmission_rate_pct + FROM {C}.`encounters` e + LEFT JOIN {C}.`readmissions` r ON e.encounter_id = r.index_encounter_id + GROUP BY 1 + """ +).show() + +print("\n2. 7-day follow-up should lower readmission rate.") +spark.sql( + f""" + SELECT ct.follow_up_within_7_days, + COUNT(DISTINCT e.encounter_id) AS encounters, + COUNT(DISTINCT r.readmission_id) AS readmissions, + ROUND(COUNT(DISTINCT r.readmission_id) * 100.0 / COUNT(DISTINCT e.encounter_id), 1) AS readmission_rate_pct + FROM {C}.`encounters` e + JOIN {C}.`care_transitions` ct ON e.encounter_id = ct.encounter_id + LEFT JOIN {C}.`readmissions` r ON e.encounter_id = r.index_encounter_id + GROUP BY ct.follow_up_within_7_days + """ +).show() + +print("\n3. Flu season should show more respiratory admissions.") +spark.sql( + f""" + SELECT CASE WHEN encounter_month IN (1, 2, 12) THEN 'Flu Season' ELSE 'Other' END AS period, + COUNT(*) FILTER (WHERE primary_diagnosis_group IN ('COPD', 'Pneumonia')) AS respiratory_encounters, + COUNT(*) AS total_encounters + FROM {C}.`encounters` + GROUP BY 1 + """ +).show() + +print("\n4. Weekend discharges should have higher readmission risk.") +spark.sql( + f""" + SELECT e.weekend_discharge, + COUNT(DISTINCT e.encounter_id) AS encounters, + COUNT(DISTINCT r.readmission_id) AS readmissions, + ROUND(COUNT(DISTINCT r.readmission_id) * 100.0 / COUNT(DISTINCT e.encounter_id), 1) AS readmission_rate_pct + FROM {C}.`encounters` e + LEFT JOIN {C}.`readmissions` r ON e.encounter_id = r.index_encounter_id + GROUP BY e.weekend_discharge + """ +).show() + +print("\n5. Program hospitals should improve after transition program launch.") +spark.sql( + f""" + SELECT CASE WHEN e.discharge_date >= h.transition_program_start_date THEN 'After Launch' ELSE 'Before Launch' END AS period, + COUNT(DISTINCT e.encounter_id) AS encounters, + COUNT(DISTINCT r.readmission_id) AS readmissions, + ROUND(COUNT(DISTINCT r.readmission_id) * 100.0 / COUNT(DISTINCT e.encounter_id), 1) AS readmission_rate_pct + FROM {C}.`encounters` e + JOIN {C}.`hospitals` h ON e.hospital_id = h.hospital_id + LEFT JOIN {C}.`readmissions` r ON e.encounter_id = r.index_encounter_id + WHERE h.has_transition_program = TRUE + GROUP BY 1 + """ +).show() + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 11. Create Metric Views + +# COMMAND ---------- + +# ============================================================================= +# CREATE METRIC VIEWS +# ============================================================================= +print("Creating metric views...") + +metric_views = { + "mv_readmission_quality": f""" +version: "0.1" +source: {CATALOG}.{SCHEMA}.encounters + +joins: + - name: hospital + source: {CATALOG}.{SCHEMA}.hospitals + on: source.hospital_id = hospital.hospital_id + - name: patient + source: {CATALOG}.{SCHEMA}.patients + on: source.patient_id = patient.patient_id + - name: transition + source: {CATALOG}.{SCHEMA}.care_transitions + on: source.encounter_id = transition.encounter_id + - name: readmission + source: {CATALOG}.{SCHEMA}.readmissions + on: source.encounter_id = readmission.index_encounter_id + +dimensions: + - name: Discharge Month + expr: DATE_TRUNC('MONTH', discharge_date) + - name: Encounter Year + expr: encounter_year + - name: Hospital Region + expr: hospital.region + - name: Hospital Type + expr: hospital.hospital_type + - name: Service Line + expr: service_line + - name: Diagnosis Group + expr: primary_diagnosis_group + - name: Follow Up Within 7 Days + expr: transition.follow_up_within_7_days + - name: Weekend Discharge + expr: weekend_discharge + - name: Clinical Risk Band + expr: patient.clinical_risk_band + +measures: + - name: Encounter Count + expr: COUNT(DISTINCT source.encounter_id) + - name: Readmission Count + expr: COUNT(DISTINCT readmission.readmission_id) + - name: Readmission Rate Pct + expr: COUNT(DISTINCT readmission.readmission_id) * 100.0 / COUNT(DISTINCT source.encounter_id) + - name: Average Length of Stay + expr: AVG(length_of_stay_days) + - name: Weekend Discharge Count + expr: COUNT(1) FILTER (WHERE weekend_discharge = TRUE) + - name: Follow Up Rate Pct + expr: COUNT(1) FILTER (WHERE transition.follow_up_within_7_days = TRUE) * 100.0 / COUNT(1) +""", + "mv_claims_cost": f""" +version: "0.1" +source: {CATALOG}.{SCHEMA}.claims + +joins: + - name: hospital + source: {CATALOG}.{SCHEMA}.hospitals + on: source.hospital_id = hospital.hospital_id + - name: encounter + source: {CATALOG}.{SCHEMA}.encounters + on: source.encounter_id = encounter.encounter_id + +dimensions: + - name: Paid Month + expr: DATE_TRUNC('MONTH', paid_date) + - name: Payer Type + expr: payer_type + - name: Service Line + expr: service_line + - name: Claim Status + expr: claim_status + - name: Hospital Region + expr: hospital.region + - name: Diagnosis Group + expr: encounter.primary_diagnosis_group + +measures: + - name: Claim Count + expr: COUNT(1) + - name: Allowed Amount + expr: SUM(allowed_amount_usd) + - name: Average Allowed Amount + expr: AVG(allowed_amount_usd) + - name: Patient Responsibility + expr: SUM(patient_responsibility_usd) + - name: Denied Claim Count + expr: COUNT(1) FILTER (WHERE claim_status = 'Denied') +""", + "mv_care_transitions": f""" +version: "0.1" +source: {CATALOG}.{SCHEMA}.care_transitions + +joins: + - name: hospital + source: {CATALOG}.{SCHEMA}.hospitals + on: source.hospital_id = hospital.hospital_id + +dimensions: + - name: Discharge Month + expr: DATE_TRUNC('MONTH', discharge_date) + - name: Hospital Region + expr: hospital.region + - name: Transition Program Enrolled + expr: transition_program_enrolled + - name: Follow Up Within 7 Days + expr: follow_up_within_7_days + +measures: + - name: Transition Count + expr: COUNT(1) + - name: Follow Up Rate Pct + expr: COUNT(1) FILTER (WHERE follow_up_within_7_days = TRUE) * 100.0 / COUNT(1) + - name: Medication Reconciliation Rate Pct + expr: COUNT(1) FILTER (WHERE medication_reconciliation_completed = TRUE) * 100.0 / COUNT(1) + - name: Home Health Referral Count + expr: COUNT(1) FILTER (WHERE home_health_referral = TRUE) + - name: Average Outreach Attempts + expr: AVG(outreach_attempts) +""", +} + +for mv_name, yaml_body in metric_views.items(): + ddl = ( + f"CREATE OR REPLACE VIEW `{CATALOG}`.`{SCHEMA}`.`{mv_name}`\n" + "WITH METRICS\n" + "LANGUAGE YAML\n" + "AS $$" + + yaml_body + + "$$" + ) + spark.sql(ddl) + print(f" OK {mv_name}") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 12. Register Constraints & Comments + +# COMMAND ---------- + +# ============================================================================= +# REGISTER CONSTRAINTS AND COMMENTS +# ============================================================================= +print("Registering constraints and comments...") + +PRIMARY_KEYS = { + "patients": "patient_id", + "hospitals": "hospital_id", + "encounters": "encounter_id", + "care_transitions": "transition_id", + "claims": "claim_id", + "readmissions": "readmission_id", +} + +FOREIGN_KEYS = [ + ("encounters", "patient_id", "patients", "patient_id"), + ("encounters", "hospital_id", "hospitals", "hospital_id"), + ("care_transitions", "encounter_id", "encounters", "encounter_id"), + ("care_transitions", "patient_id", "patients", "patient_id"), + ("care_transitions", "hospital_id", "hospitals", "hospital_id"), + ("claims", "encounter_id", "encounters", "encounter_id"), + ("claims", "patient_id", "patients", "patient_id"), + ("claims", "hospital_id", "hospitals", "hospital_id"), + ("readmissions", "index_encounter_id", "encounters", "encounter_id"), + ("readmissions", "patient_id", "patients", "patient_id"), + ("readmissions", "hospital_id", "hospitals", "hospital_id"), +] + +for table, pk in PRIMARY_KEYS.items(): + spark.sql(f"ALTER TABLE {C}.`{table}` ALTER COLUMN `{pk}` SET NOT NULL") + spark.sql(f"ALTER TABLE {C}.`{table}` ADD CONSTRAINT pk_{table} PRIMARY KEY (`{pk}`)") + +for table, col, ref_table, ref_col in FOREIGN_KEYS: + spark.sql( + f"ALTER TABLE {C}.`{table}` ADD CONSTRAINT fk_{table}_{col} " + f"FOREIGN KEY (`{col}`) REFERENCES {C}.`{ref_table}` (`{ref_col}`)" + ) + + +def sql_text(value): + return value.replace("'", "''") + + +TABLE_COMMENTS = { + "patients": "Synthetic patient risk dimension using IDs only. Contains no patient names, DOBs, addresses, emails, or phone numbers.", + "hospitals": "Fictional hospital dimension with region, bed count, hospital type, and transition program flag.", + "encounters": "Index admission encounters from 2023-01-01 to 2025-12-31 with diagnosis, service line, discharge, and acuity attributes.", + "care_transitions": "One discharge transition row per encounter, including 7-day follow-up, medication reconciliation, and outreach actions.", + "claims": "Encounter-level claim facts with payer type, allowed amount, patient responsibility, and claim status.", + "readmissions": "30-day readmission facts linked to index encounters. CHF, COPD, missed follow-up, and weekend discharge increase risk.", +} + +for table, comment in TABLE_COMMENTS.items(): + spark.sql(f"COMMENT ON TABLE {C}.`{table}` IS '{sql_text(comment)}'") + +for table, df_pd in TABLES.items(): + for column in df_pd.columns: + readable = column.replace("_", " ") + spark.sql( + f"ALTER TABLE {C}.`{table}` ALTER COLUMN `{column}` " + f"COMMENT '{sql_text(readable)} for the synthetic hospital readmission dataset'" + ) + +print(" OK constraints and comments registered") + +print() +print("=" * 60) +print("SETUP COMPLETE") +print("=" * 60) +print(f" Catalog : {CATALOG}") +print(f" Schema : {SCHEMA}") +print(" Tables : patients, hospitals, encounters, care_transitions, claims, readmissions") +print(" Metric Views: mv_readmission_quality, mv_claims_cost, mv_care_transitions") +print(" Next : Create a Genie space from these tables, or see notebooks/demo-data/README.md") diff --git a/notebooks/demo-data/generate_retail_apparel_data.py b/notebooks/demo-data/generate_retail_apparel_data.py new file mode 100644 index 000000000..d5618f88f --- /dev/null +++ b/notebooks/demo-data/generate_retail_apparel_data.py @@ -0,0 +1,816 @@ +# Databricks notebook source + +# MAGIC %md +# MAGIC # Retail Apparel - Synthetic Dataset Generator +# MAGIC +# MAGIC Generates a fictional retail apparel dataset for Databricks AI/BI Genie demos. +# MAGIC +# MAGIC | Table | Rows | Description | +# MAGIC |---|---:|---| +# MAGIC | `products` | 80 | Apparel product catalog | +# MAGIC | `stores` | 12 | Store and ecommerce locations | +# MAGIC | `customers` | 1,500 | Loyalty customer dimension | +# MAGIC | `inventory_snapshots` | 34,560 | Product/store/month inventory health | +# MAGIC | `sales` | 15,000 | Item-level sales facts from 2023-2025 | +# MAGIC | `returns` | variable | Return facts linked to sales | +# MAGIC +# MAGIC **Setup:** Set the `catalog` and `schema` widgets (or edit defaults below), then **Run All**. +# MAGIC +# MAGIC Seed: `42` - fully reproducible. + +# COMMAND ---------- + +# MAGIC %pip install faker==40.19.1 --quiet +# MAGIC %restart_python + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 1. Configuration + +# COMMAND ---------- + +# ============================================================================= +# CONFIGURATION - Edit only this section before running +# ============================================================================= +DEFAULT_CATALOG = "" # Unity Catalog name; required +DEFAULT_SCHEMA = "retail_apparel" # Schema / database name + +# Widgets let the bulk runner pass values while preserving standalone defaults. +try: + dbutils.widgets.text("catalog", DEFAULT_CATALOG, "Unity Catalog name") +except Exception: + pass + +try: + dbutils.widgets.text("schema", DEFAULT_SCHEMA, "Schema / database name") +except Exception: + pass + +try: + dbutils.widgets.dropdown("overwrite_existing", "false", ["false", "true"], "Overwrite existing tables") +except Exception: + pass + +try: + CATALOG = dbutils.widgets.get("catalog").strip() or DEFAULT_CATALOG +except Exception: + CATALOG = DEFAULT_CATALOG + +try: + SCHEMA = dbutils.widgets.get("schema").strip() or DEFAULT_SCHEMA +except Exception: + SCHEMA = DEFAULT_SCHEMA + +try: + OVERWRITE_EXISTING = dbutils.widgets.get("overwrite_existing").strip().lower() == "true" +except Exception: + OVERWRITE_EXISTING = False + +if not CATALOG: + raise ValueError("Set the catalog widget to a Unity Catalog name before running this notebook.") + +WRITE_MODE = "overwrite" if OVERWRITE_EXISTING else "errorifexists" + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 2. Imports & Global Setup + +# COMMAND ---------- + +# ============================================================================= +# IMPORTS & GLOBAL SETUP +# ============================================================================= +import random +from calendar import monthrange +from datetime import date, timedelta + +import numpy as np +import pandas as pd +from faker import Faker + +SEED = 42 +random.seed(SEED) +np.random.seed(SEED) +fake = Faker("en_US") +Faker.seed(SEED) + +START_DATE = date(2023, 1, 1) +END_DATE = date(2025, 12, 31) +MONTH_LIST = [(y, m) for y in [2023, 2024, 2025] for m in range(1, 13)] + +REGIONS = ["Northeast", "Southeast", "Midwest", "West"] +REGION_STATES = { + "Northeast": ["NY", "MA", "NJ", "PA"], + "Southeast": ["FL", "GA", "NC", "TN"], + "Midwest": ["IL", "OH", "MI", "MN"], + "West": ["CA", "WA", "OR", "CO"], +} + +print("Setup complete.") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 3. Products + +# COMMAND ---------- + +# ============================================================================= +# TABLE 1: PRODUCTS (80 rows) +# ============================================================================= +CATEGORY_CONFIG = { + "Tops": {"base_price": (24, 78), "return_weight": 0.09, "season": "All Season"}, + "Bottoms": {"base_price": (38, 110), "return_weight": 0.11, "season": "All Season"}, + "Dresses": {"base_price": (58, 180), "return_weight": 0.14, "season": "Spring"}, + "Outerwear": {"base_price": (88, 260), "return_weight": 0.08, "season": "Winter"}, + "Footwear": {"base_price": (55, 210), "return_weight": 0.16, "season": "All Season"}, + "Accessories": {"base_price": (12, 85), "return_weight": 0.05, "season": "All Season"}, + "Activewear": {"base_price": (28, 120), "return_weight": 0.10, "season": "Summer"}, +} +CATEGORY_WEIGHTS = { + "Tops": 18, + "Bottoms": 14, + "Dresses": 10, + "Outerwear": 9, + "Footwear": 12, + "Accessories": 20, + "Activewear": 17, +} +BRAND_LINES = ["Aster", "Summit", "Kindred", "Northline", "Vista", "Everyday"] +CATEGORY_LABELS = { + "Tops": "Top", + "Bottoms": "Bottom", + "Dresses": "Dress", + "Outerwear": "Outerwear", + "Footwear": "Footwear", + "Accessories": "Accessory", + "Activewear": "Activewear", +} + +products_data = [] +categories = list(CATEGORY_CONFIG.keys()) +for i in range(1, 81): + category = random.choices(categories, weights=[CATEGORY_WEIGHTS[c] for c in categories])[0] + cfg = CATEGORY_CONFIG[category] + list_price = round(random.uniform(*cfg["base_price"]), 2) + unit_cost = round(list_price * random.uniform(0.34, 0.58), 2) + brand_line = random.choice(BRAND_LINES) + products_data.append( + { + "product_id": f"PROD-{i:04d}", + "sku": f"SKU-{100000 + i}", + "product_name": f"{brand_line} {CATEGORY_LABELS[category]} {i:02d}", + "category": category, + "brand_line": brand_line, + "season": cfg["season"], + "list_price_usd": list_price, + "unit_cost_usd": unit_cost, + "target_margin_pct": round((list_price - unit_cost) * 100.0 / list_price, 2), + "launch_date": START_DATE + timedelta(days=random.randint(0, 650)), + "is_active": random.random() < 0.92, + } + ) + +df_products = pd.DataFrame(products_data) +PRODUCT_LOOKUP = {p["product_id"]: p for p in products_data} +print(f"products: {len(df_products)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 4. Stores + +# COMMAND ---------- + +# ============================================================================= +# TABLE 2: STORES (12 rows) +# ============================================================================= +STORE_SEEDS = [ + ("STOR-001", "Manhattan Flagship", "Northeast", "NY", "New York", "Flagship"), + ("STOR-002", "Boston Back Bay", "Northeast", "MA", "Boston", "Mall"), + ("STOR-003", "Philadelphia Center", "Northeast", "PA", "Philadelphia", "Outlet"), + ("STOR-004", "Miami Lincoln", "Southeast", "FL", "Miami", "Flagship"), + ("STOR-005", "Atlanta Midtown", "Southeast", "GA", "Atlanta", "Mall"), + ("STOR-006", "Charlotte SouthPark", "Southeast", "NC", "Charlotte", "Mall"), + ("STOR-007", "Chicago Loop", "Midwest", "IL", "Chicago", "Flagship"), + ("STOR-008", "Columbus Easton", "Midwest", "OH", "Columbus", "Mall"), + ("STOR-009", "Detroit Somerset", "Midwest", "MI", "Troy", "Outlet"), + ("STOR-010", "Los Angeles Grove", "West", "CA", "Los Angeles", "Flagship"), + ("STOR-011", "Seattle Bellevue", "West", "WA", "Bellevue", "Mall"), + ("STOR-012", "Online Flagship", "West", "CA", "San Francisco", "Ecommerce"), +] + +stores_data = [] +for sid, name, region, state, city, store_type in STORE_SEEDS: + stores_data.append( + { + "store_id": sid, + "store_name": name, + "store_type": store_type, + "region": region, + "state": state, + "city": city, + "opened_date": date(2015 + random.randint(0, 8), random.randint(1, 12), 1), + "selling_sqft": 0 if store_type == "Ecommerce" else random.choice([3800, 5200, 7400, 9800]), + "monthly_rent_usd": 0.0 if store_type == "Ecommerce" else float(random.randint(22000, 115000)), + "is_active": True, + } + ) + +df_stores = pd.DataFrame(stores_data) +STORE_IDS = [s["store_id"] for s in stores_data] +PHYSICAL_STORE_IDS = [s["store_id"] for s in stores_data if s["store_type"] != "Ecommerce"] +ONLINE_STORE_ID = "STOR-012" +print(f"stores: {len(df_stores)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 5. Customers + +# COMMAND ---------- + +# ============================================================================= +# TABLE 3: CUSTOMERS (1,500 rows) +# ============================================================================= +LOYALTY_TIERS = ["Member", "Silver", "Gold", "VIP"] +LOYALTY_WEIGHTS = [55, 25, 15, 5] +LOYALTY_DISCOUNT = {"Member": 0.00, "Silver": 0.03, "Gold": 0.06, "VIP": 0.10} + +customers_data = [] +for i in range(1, 1501): + tier = random.choices(LOYALTY_TIERS, weights=LOYALTY_WEIGHTS)[0] + region = random.choice(REGIONS) + state = random.choice(REGION_STATES[region]) + customers_data.append( + { + "customer_id": f"CUST-{i:05d}", + "customer_name": fake.name(), + "loyalty_tier": tier, + "age_band": random.choices( + ["18-24", "25-34", "35-44", "45-54", "55-64", "65+"], + weights=[14, 28, 24, 18, 11, 5], + )[0], + "home_region": region, + "home_state": state, + "acquisition_channel": random.choices( + ["Store", "Paid Search", "Social", "Referral", "Email"], + weights=[35, 20, 18, 12, 15], + )[0], + "customer_since_date": START_DATE - timedelta(days=random.randint(0, 1500)), + "is_active": random.random() < 0.95, + } + ) + +df_customers = pd.DataFrame(customers_data) +CUSTOMER_IDS = [c["customer_id"] for c in customers_data] +CUSTOMER_TIER = {c["customer_id"]: c["loyalty_tier"] for c in customers_data} +print(f"customers: {len(df_customers)} rows") +print(df_customers["loyalty_tier"].value_counts().to_dict()) + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 6. Inventory Snapshots + +# COMMAND ---------- + +# ============================================================================= +# TABLE 4: INVENTORY SNAPSHOTS (product x store x month) +# ============================================================================= +inventory_data = [] +stockout_lookup = {} +snapshot_counter = 1 + +for year, month in MONTH_LIST: + for store in stores_data: + for product in products_data: + category = product["category"] + is_q4 = month in (10, 11, 12) + base_on_hand = random.randint(8, 85) + if store["store_type"] == "Ecommerce": + base_on_hand = int(base_on_hand * 2.3) + if is_q4 and category in ("Outerwear", "Footwear"): + stockout_days = random.choices([0, 1, 2, 4, 7, 10, 14], weights=[12, 15, 20, 20, 15, 12, 6])[0] + else: + stockout_days = random.choices([0, 1, 2, 4, 7, 10], weights=[30, 28, 20, 12, 7, 3])[0] + if stockout_days >= 7: + on_hand_units = random.randint(0, 8) + else: + on_hand_units = base_on_hand + lost_sales = max(0, stockout_days - 2) * random.uniform(1.5, 5.0) + stockout_lookup[(product["product_id"], store["store_id"], year, month)] = stockout_days + inventory_data.append( + { + "snapshot_id": f"INV-{snapshot_counter:07d}", + "snapshot_month": date(year, month, 1), + "snapshot_year": year, + "snapshot_month_num": month, + "product_id": product["product_id"], + "store_id": store["store_id"], + "on_hand_units": int(on_hand_units), + "reorder_point_units": int(random.randint(10, 40)), + "stockout_days": int(stockout_days), + "lost_sales_estimate_units": round(lost_sales, 2), + "inventory_value_usd": round(on_hand_units * product["unit_cost_usd"], 2), + } + ) + snapshot_counter += 1 + +df_inventory_snapshots = pd.DataFrame(inventory_data) +print(f"inventory_snapshots: {len(df_inventory_snapshots)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 7. Sales + +# COMMAND ---------- + +# ============================================================================= +# TABLE 5: SALES (15,000 rows) +# ============================================================================= +# Injected patterns: +# 1. Holiday Nov/Dec sales spike. +# 2. Online channel share grows from 28% to 52%. +# 3. January and July clearance periods have deeper discounts. +# 4. High-stockout product/store/month combinations have lower quantities. +# 5. VIP and Gold loyalty tiers are a small customer share but large revenue share. + +month_weights = [1.65 if m in (11, 12) else 1.20 if m in (1, 7) else 1.0 for _, m in MONTH_LIST] +sales_per_month_raw = [w / sum(month_weights) * 15000 for w in month_weights] +sales_per_month = [int(round(x)) for x in sales_per_month_raw] +sales_per_month[-1] += 15000 - sum(sales_per_month) + + +def online_share(month_idx): + return 0.28 + (0.52 - 0.28) * (month_idx / 35.0) + + +def pick_customer_for_sale(): + # Revenue concentration: Gold/VIP customers are more likely to purchase. + tier = random.choices(LOYALTY_TIERS, weights=[35, 28, 25, 12])[0] + candidates = [c["customer_id"] for c in customers_data if c["loyalty_tier"] == tier] + return random.choice(candidates) + + +sales_data = [] +sale_counter = 1 +product_weights = [CATEGORY_WEIGHTS[p["category"]] for p in products_data] + +for month_idx, (year, month) in enumerate(MONTH_LIST): + n_sales = sales_per_month[month_idx] + max_day = monthrange(year, month)[1] + o_share = online_share(month_idx) + is_clearance_month = month in (1, 7) + + for _ in range(n_sales): + sale_date = date(year, month, random.randint(1, max_day)) + customer_id = pick_customer_for_sale() + loyalty_tier = CUSTOMER_TIER[customer_id] + channel = random.choices(["Online", "Store"], weights=[o_share, 1.0 - o_share])[0] + store_id = ONLINE_STORE_ID if channel == "Online" else random.choice(PHYSICAL_STORE_IDS) + product = random.choices(products_data, weights=product_weights)[0] + stockout_days = stockout_lookup[(product["product_id"], store_id, year, month)] + quantity = random.choices([1, 2, 3, 4], weights=[68, 22, 7, 3])[0] + if stockout_days >= 7: + quantity = 1 + + base_discount = LOYALTY_DISCOUNT[loyalty_tier] + promo_discount = random.choice([0.00, 0.05, 0.10, 0.15]) + clearance_discount = random.uniform(0.20, 0.45) if is_clearance_month else 0.0 + discount_pct = min(0.65, base_discount + promo_discount + clearance_discount) + gross_sales = product["list_price_usd"] * quantity + discount_amount = gross_sales * discount_pct + net_sales = gross_sales - discount_amount + + sales_data.append( + { + "sale_id": f"SALE-{sale_counter:07d}", + "sale_date": sale_date, + "sale_year": year, + "sale_month": month, + "sale_quarter": (month - 1) // 3 + 1, + "customer_id": customer_id, + "product_id": product["product_id"], + "store_id": store_id, + "channel": channel, + "quantity": int(quantity), + "gross_sales_usd": round(gross_sales, 2), + "discount_pct": round(discount_pct * 100, 2), + "discount_amount_usd": round(discount_amount, 2), + "net_sales_usd": round(net_sales, 2), + "unit_cost_usd": product["unit_cost_usd"], + "gross_margin_usd": round(net_sales - product["unit_cost_usd"] * quantity, 2), + "loyalty_tier": loyalty_tier, + "is_clearance": bool(is_clearance_month), + "stockout_days_at_sale": int(stockout_days), + } + ) + sale_counter += 1 + +df_sales = pd.DataFrame(sales_data) +print(f"sales: {len(df_sales)} rows") +print(df_sales["channel"].value_counts().to_dict()) + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 8. Returns + +# COMMAND ---------- + +# ============================================================================= +# TABLE 6: RETURNS +# ============================================================================= +RETURN_REASONS = ["Fit", "Changed Mind", "Damaged", "Late Delivery", "Wrong Item"] +returns_data = [] +return_counter = 1 + +for sale in sales_data: + product = PRODUCT_LOOKUP[sale["product_id"]] + base_prob = CATEGORY_CONFIG[product["category"]]["return_weight"] + channel_mult = 1.55 if sale["channel"] == "Online" else 0.82 + clearance_mult = 0.75 if sale["is_clearance"] else 1.0 + return_prob = min(0.35, base_prob * channel_mult * clearance_mult) + if random.random() < return_prob: + reason = random.choices(RETURN_REASONS, weights=[33, 28, 12, 18, 9])[0] + return_date = min(END_DATE, sale["sale_date"] + timedelta(days=random.randint(3, 45))) + return_qty = random.randint(1, sale["quantity"]) + unit_net_price = sale["net_sales_usd"] / sale["quantity"] + returns_data.append( + { + "return_id": f"RTRN-{return_counter:07d}", + "return_date": return_date, + "return_year": return_date.year, + "return_month": return_date.month, + "sale_id": sale["sale_id"], + "customer_id": sale["customer_id"], + "product_id": sale["product_id"], + "store_id": sale["store_id"], + "channel": sale["channel"], + "return_reason": reason, + "return_quantity": return_qty, + "return_amount_usd": round(unit_net_price * return_qty, 2), + "days_to_return": (return_date - sale["sale_date"]).days, + } + ) + return_counter += 1 + +df_returns = pd.DataFrame(returns_data) +print(f"returns: {len(df_returns)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 9. Create Schema & Write Delta Tables + +# COMMAND ---------- + +# ============================================================================= +# CREATE SCHEMA & WRITE DELTA TABLES +# ============================================================================= +spark.sql(f"CREATE SCHEMA IF NOT EXISTS `{CATALOG}`.`{SCHEMA}`") + + +def write_table(df_pd, table_name, mode=WRITE_MODE): + df_spark = spark.createDataFrame(df_pd) + ( + df_spark.write.format("delta") + .mode(mode) + .option("overwriteSchema", "true") + .saveAsTable(f"`{CATALOG}`.`{SCHEMA}`.`{table_name}`") + ) + print(f" OK {table_name}: {df_spark.count()} rows written") + + +TABLES = { + "products": df_products, + "stores": df_stores, + "customers": df_customers, + "inventory_snapshots": df_inventory_snapshots, + "sales": df_sales, + "returns": df_returns, +} + +print(f"Writing tables to `{CATALOG}`.`{SCHEMA}` with mode={WRITE_MODE}...") +for _table_name, _df in TABLES.items(): + write_table(_df, _table_name) +print("All tables written.") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 10. Verification + +# COMMAND ---------- + +# ============================================================================= +# VERIFICATION +# ============================================================================= +C = f"`{CATALOG}`.`{SCHEMA}`" + +print("=" * 60) +print("ROW COUNTS") +print("=" * 60) +for tbl in TABLES: + n = spark.sql(f"SELECT COUNT(*) AS n FROM {C}.`{tbl}`").collect()[0]["n"] + print(f" {tbl:<24} {n:>8,}") + +print() +print("=" * 60) +print("PATTERN VALIDATION") +print("=" * 60) + +print("\n1. Holiday sales spike: Nov/Dec should have higher order counts.") +spark.sql( + f""" + SELECT CASE WHEN sale_month IN (11, 12) THEN 'Nov/Dec' ELSE 'Other' END AS period, + COUNT(*) AS orders, + ROUND(SUM(net_sales_usd), 2) AS revenue + FROM {C}.`sales` + GROUP BY 1 + """ +).show() + +print("\n2. Online channel growth: online share should rise from 2023 to 2025.") +spark.sql( + f""" + SELECT sale_year, + ROUND(SUM(CASE WHEN channel = 'Online' THEN 1 ELSE 0 END) * 100.0 / COUNT(*), 1) AS online_order_pct + FROM {C}.`sales` + GROUP BY sale_year + ORDER BY sale_year + """ +).show() + +print("\n3. Clearance months should have deeper discounts.") +spark.sql( + f""" + SELECT is_clearance, + ROUND(AVG(discount_pct), 1) AS avg_discount_pct, + COUNT(*) AS orders + FROM {C}.`sales` + GROUP BY is_clearance + """ +).show() + +print("\n4. Stockout impact: high stockout sales should have lower units per order.") +spark.sql( + f""" + SELECT CASE WHEN stockout_days_at_sale >= 7 THEN 'High Stockout' ELSE 'Normal' END AS stockout_group, + ROUND(AVG(quantity), 2) AS avg_units_per_order, + COUNT(*) AS orders + FROM {C}.`sales` + GROUP BY 1 + """ +).show() + +print("\n5. Higher online return rate.") +spark.sql( + f""" + SELECT s.channel, + COUNT(DISTINCT r.return_id) AS returns, + COUNT(DISTINCT s.sale_id) AS sales, + ROUND(COUNT(DISTINCT r.return_id) * 100.0 / COUNT(DISTINCT s.sale_id), 1) AS return_rate_pct + FROM {C}.`sales` s + LEFT JOIN {C}.`returns` r ON s.sale_id = r.sale_id + GROUP BY s.channel + """ +).show() + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 11. Create Metric Views + +# COMMAND ---------- + +# ============================================================================= +# CREATE METRIC VIEWS +# ============================================================================= +print("Creating metric views...") + +metric_views = { + "mv_retail_sales": f""" +version: "0.1" +source: {CATALOG}.{SCHEMA}.sales + +joins: + - name: product + source: {CATALOG}.{SCHEMA}.products + on: source.product_id = product.product_id + - name: store + source: {CATALOG}.{SCHEMA}.stores + on: source.store_id = store.store_id + - name: customer + source: {CATALOG}.{SCHEMA}.customers + on: source.customer_id = customer.customer_id + +dimensions: + - name: Sale Month + expr: DATE_TRUNC('MONTH', sale_date) + - name: Sale Year + expr: sale_year + - name: Channel + expr: channel + - name: Product Category + expr: product.category + - name: Brand Line + expr: product.brand_line + - name: Loyalty Tier + expr: source.loyalty_tier + - name: Store Region + expr: store.region + - name: Store Type + expr: store.store_type + - name: Clearance Flag + expr: is_clearance + +measures: + - name: Order Count + expr: COUNT(1) + - name: Units Sold + expr: SUM(quantity) + - name: Net Sales + expr: SUM(net_sales_usd) + - name: Gross Margin + expr: SUM(gross_margin_usd) + - name: Discount Amount + expr: SUM(discount_amount_usd) + - name: Average Order Value + expr: SUM(net_sales_usd) / COUNT(1) + - name: Online Sales + expr: SUM(net_sales_usd) FILTER (WHERE channel = 'Online') + - name: Online Order Share Pct + expr: COUNT(1) FILTER (WHERE channel = 'Online') * 100.0 / COUNT(1) + - name: Clearance Sales + expr: SUM(net_sales_usd) FILTER (WHERE is_clearance = TRUE) +""", + "mv_inventory_health": f""" +version: "0.1" +source: {CATALOG}.{SCHEMA}.inventory_snapshots + +joins: + - name: product + source: {CATALOG}.{SCHEMA}.products + on: source.product_id = product.product_id + - name: store + source: {CATALOG}.{SCHEMA}.stores + on: source.store_id = store.store_id + +dimensions: + - name: Snapshot Month + expr: snapshot_month + - name: Snapshot Year + expr: snapshot_year + - name: Product Category + expr: product.category + - name: Store Region + expr: store.region + - name: Store Type + expr: store.store_type + +measures: + - name: Inventory Value + expr: SUM(inventory_value_usd) + - name: On Hand Units + expr: SUM(on_hand_units) + - name: Stockout Days + expr: SUM(stockout_days) + - name: Lost Sales Estimate Units + expr: SUM(lost_sales_estimate_units) + - name: Stockout Snapshot Rate Pct + expr: COUNT(1) FILTER (WHERE stockout_days >= 7) * 100.0 / COUNT(1) +""", + "mv_retail_returns": f""" +version: "0.1" +source: {CATALOG}.{SCHEMA}.returns + +joins: + - name: product + source: {CATALOG}.{SCHEMA}.products + on: source.product_id = product.product_id + - name: store + source: {CATALOG}.{SCHEMA}.stores + on: source.store_id = store.store_id + +dimensions: + - name: Return Month + expr: DATE_TRUNC('MONTH', return_date) + - name: Return Year + expr: return_year + - name: Channel + expr: channel + - name: Product Category + expr: product.category + - name: Return Reason + expr: return_reason + - name: Store Region + expr: store.region + +measures: + - name: Return Count + expr: COUNT(1) + - name: Returned Units + expr: SUM(return_quantity) + - name: Return Amount + expr: SUM(return_amount_usd) + - name: Average Days to Return + expr: AVG(days_to_return) +""", +} + +for mv_name, yaml_body in metric_views.items(): + ddl = ( + f"CREATE OR REPLACE VIEW `{CATALOG}`.`{SCHEMA}`.`{mv_name}`\n" + "WITH METRICS\n" + "LANGUAGE YAML\n" + "AS $$" + + yaml_body + + "$$" + ) + spark.sql(ddl) + print(f" OK {mv_name}") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 12. Register Constraints & Comments + +# COMMAND ---------- + +# ============================================================================= +# REGISTER CONSTRAINTS AND COMMENTS +# ============================================================================= +print("Registering constraints and comments...") + +PRIMARY_KEYS = { + "products": "product_id", + "stores": "store_id", + "customers": "customer_id", + "inventory_snapshots": "snapshot_id", + "sales": "sale_id", + "returns": "return_id", +} + +FOREIGN_KEYS = [ + ("sales", "customer_id", "customers", "customer_id"), + ("sales", "product_id", "products", "product_id"), + ("sales", "store_id", "stores", "store_id"), + ("inventory_snapshots", "product_id", "products", "product_id"), + ("inventory_snapshots", "store_id", "stores", "store_id"), + ("returns", "sale_id", "sales", "sale_id"), + ("returns", "customer_id", "customers", "customer_id"), + ("returns", "product_id", "products", "product_id"), + ("returns", "store_id", "stores", "store_id"), +] + +for table, pk in PRIMARY_KEYS.items(): + spark.sql(f"ALTER TABLE {C}.`{table}` ALTER COLUMN `{pk}` SET NOT NULL") + spark.sql(f"ALTER TABLE {C}.`{table}` ADD CONSTRAINT pk_{table} PRIMARY KEY (`{pk}`)") + +for table, col, ref_table, ref_col in FOREIGN_KEYS: + spark.sql( + f"ALTER TABLE {C}.`{table}` ADD CONSTRAINT fk_{table}_{col} " + f"FOREIGN KEY (`{col}`) REFERENCES {C}.`{ref_table}` (`{ref_col}`)" + ) + + +def sql_text(value): + return value.replace("'", "''") + + +TABLE_COMMENTS = { + "products": "Product catalog for a fictional apparel retailer. Includes category, brand line, price, cost, season, and active flag.", + "stores": "Store dimension covering physical retail stores and an ecommerce storefront across US regions.", + "customers": "Synthetic loyalty customer dimension. VIP and Gold customers drive a disproportionate share of revenue.", + "inventory_snapshots": "Monthly product-store inventory snapshots with stockout days and estimated lost sales units.", + "sales": "Item-level retail apparel sales from 2023-01-01 to 2025-12-31. Patterns include holiday spikes, online growth, clearance discounts, and stockout effects.", + "returns": "Returned merchandise facts linked to sales. Online sales have a higher return rate than store sales.", +} + +for table, comment in TABLE_COMMENTS.items(): + spark.sql(f"COMMENT ON TABLE {C}.`{table}` IS '{sql_text(comment)}'") + +for table, df_pd in TABLES.items(): + for column in df_pd.columns: + readable = column.replace("_", " ") + spark.sql( + f"ALTER TABLE {C}.`{table}` ALTER COLUMN `{column}` " + f"COMMENT '{sql_text(readable)} for the synthetic retail apparel dataset'" + ) + +print(" OK constraints and comments registered") + +print() +print("=" * 60) +print("SETUP COMPLETE") +print("=" * 60) +print(f" Catalog : {CATALOG}") +print(f" Schema : {SCHEMA}") +print(" Tables : products, stores, customers, inventory_snapshots, sales, returns") +print(" Metric Views: mv_retail_sales, mv_inventory_health, mv_retail_returns") +print(" Next : Create a Genie space from these tables, or see notebooks/demo-data/README.md") diff --git a/notebooks/demo-data/generate_saas_churn_data.py b/notebooks/demo-data/generate_saas_churn_data.py new file mode 100644 index 000000000..701b09153 --- /dev/null +++ b/notebooks/demo-data/generate_saas_churn_data.py @@ -0,0 +1,860 @@ +# Databricks notebook source + +# MAGIC %md +# MAGIC # SaaS Customer Churn - Synthetic Dataset Generator +# MAGIC +# MAGIC Generates a fictional SaaS customer churn dataset for Databricks AI/BI Genie demos. +# MAGIC +# MAGIC | Table | Rows | Description | +# MAGIC |---|---:|---| +# MAGIC | `accounts` | 900 | Customer account dimension | +# MAGIC | `subscriptions` | 900 | Subscription and ARR dimension | +# MAGIC | `product_usage` | variable | Monthly account usage facts | +# MAGIC | `support_tickets` | 2,500 | Customer support ticket facts | +# MAGIC | `invoices` | variable | Monthly invoice and payment facts | +# MAGIC | `churn_events` | variable | Churn facts linked to subscriptions | +# MAGIC +# MAGIC **Setup:** Set the `catalog` and `schema` widgets (or edit defaults below), then **Run All**. +# MAGIC +# MAGIC Seed: `42` - fully reproducible. + +# COMMAND ---------- + +# MAGIC %pip install faker==40.19.1 --quiet +# MAGIC %restart_python + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 1. Configuration + +# COMMAND ---------- + +# ============================================================================= +# CONFIGURATION - Edit only this section before running +# ============================================================================= +DEFAULT_CATALOG = "" # Unity Catalog name; required +DEFAULT_SCHEMA = "saas_churn" # Schema / database name + +# Widgets let the bulk runner pass values while preserving standalone defaults. +try: + dbutils.widgets.text("catalog", DEFAULT_CATALOG, "Unity Catalog name") +except Exception: + pass + +try: + dbutils.widgets.text("schema", DEFAULT_SCHEMA, "Schema / database name") +except Exception: + pass + +try: + dbutils.widgets.dropdown("overwrite_existing", "false", ["false", "true"], "Overwrite existing tables") +except Exception: + pass + +try: + CATALOG = dbutils.widgets.get("catalog").strip() or DEFAULT_CATALOG +except Exception: + CATALOG = DEFAULT_CATALOG + +try: + SCHEMA = dbutils.widgets.get("schema").strip() or DEFAULT_SCHEMA +except Exception: + SCHEMA = DEFAULT_SCHEMA + +try: + OVERWRITE_EXISTING = dbutils.widgets.get("overwrite_existing").strip().lower() == "true" +except Exception: + OVERWRITE_EXISTING = False + +if not CATALOG: + raise ValueError("Set the catalog widget to a Unity Catalog name before running this notebook.") + +WRITE_MODE = "overwrite" if OVERWRITE_EXISTING else "errorifexists" + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 2. Imports & Global Setup + +# COMMAND ---------- + +# ============================================================================= +# IMPORTS & GLOBAL SETUP +# ============================================================================= +import random +from calendar import monthrange +from datetime import date, timedelta + +import numpy as np +import pandas as pd +from faker import Faker + +SEED = 42 +random.seed(SEED) +np.random.seed(SEED) +fake = Faker("en_US") +Faker.seed(SEED) + +START_DATE = date(2023, 1, 1) +END_DATE = date(2025, 12, 31) +MONTH_LIST = [(y, m) for y in [2023, 2024, 2025] for m in range(1, 13)] + +SEGMENTS = ["SMB", "Mid-Market", "Enterprise"] +PLAN_TIERS = ["Starter", "Professional", "Enterprise"] +INDUSTRIES = ["Technology", "Retail", "Financial Services", "Healthcare", "Manufacturing", "Education"] + +print("Setup complete.") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 3. Accounts + +# COMMAND ---------- + +# ============================================================================= +# TABLE 1: ACCOUNTS (900 rows) +# ============================================================================= +accounts_data = [] +for i in range(1, 901): + segment = random.choices(SEGMENTS, weights=[52, 32, 16])[0] + if segment == "Enterprise": + employee_count = random.randint(1500, 45000) + csm_tier = random.choices(["Strategic", "Named", "Pooled"], weights=[70, 25, 5])[0] + elif segment == "Mid-Market": + employee_count = random.randint(250, 2500) + csm_tier = random.choices(["Strategic", "Named", "Pooled"], weights=[10, 65, 25])[0] + else: + employee_count = random.randint(10, 400) + csm_tier = random.choices(["Named", "Pooled", "Tech Touch"], weights=[12, 48, 40])[0] + ai_adopter = random.random() < {"SMB": 0.28, "Mid-Market": 0.42, "Enterprise": 0.58}[segment] + accounts_data.append( + { + "account_id": f"ACCT-{i:05d}", + "account_name": fake.company(), + "segment": segment, + "industry": random.choice(INDUSTRIES), + "region": random.choice(["North America", "EMEA", "APAC", "LATAM"]), + "employee_count": int(employee_count), + "acquisition_channel": random.choices( + ["Sales Outbound", "Partner", "Inbound", "Product-Led", "Event"], + weights=[25, 18, 24, 23, 10], + )[0], + "customer_since_date": START_DATE - timedelta(days=random.randint(0, 1200)), + "csm_tier": csm_tier, + "ai_feature_adopter": bool(ai_adopter), + "is_active": True, + } + ) + +df_accounts = pd.DataFrame(accounts_data) +ACCOUNT_BY_ID = {a["account_id"]: a for a in accounts_data} +ACCOUNT_IDS = [a["account_id"] for a in accounts_data] +print(f"accounts: {len(df_accounts)} rows") +print(df_accounts["segment"].value_counts().to_dict()) + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 4. Subscriptions + +# COMMAND ---------- + +# ============================================================================= +# TABLE 2: SUBSCRIPTIONS (900 rows) +# ============================================================================= +subscriptions_data = [] +SUBSCRIPTION_BY_ID = {} +for i, account in enumerate(accounts_data, start=1): + segment = account["segment"] + if segment == "Enterprise": + plan = random.choices(PLAN_TIERS, weights=[2, 18, 80])[0] + seats = random.randint(300, 6000) + base_mrr_per_seat = random.uniform(58, 115) + elif segment == "Mid-Market": + plan = random.choices(PLAN_TIERS, weights=[12, 68, 20])[0] + seats = random.randint(60, 700) + base_mrr_per_seat = random.uniform(38, 82) + else: + plan = random.choices(PLAN_TIERS, weights=[58, 38, 4])[0] + seats = random.randint(5, 120) + base_mrr_per_seat = random.uniform(18, 55) + billing_term = random.choices(["Monthly", "Annual"], weights=[64, 36] if segment != "Enterprise" else [22, 78])[0] + discount_pct = random.uniform(0.02, 0.15) if billing_term == "Annual" else random.uniform(0.00, 0.08) + mrr = seats * base_mrr_per_seat * (1.0 - discount_pct) + start_date = START_DATE - timedelta(days=random.randint(0, 700)) + row = { + "subscription_id": f"SUB-{i:06d}", + "account_id": account["account_id"], + "plan_tier": plan, + "billing_term": billing_term, + "start_date": start_date, + "end_date": None, + "subscription_status": "Active", + "seats_purchased": int(seats), + "monthly_recurring_revenue_usd": round(mrr, 2), + "annual_recurring_revenue_usd": round(mrr * 12.0, 2), + "discount_pct": round(discount_pct * 100.0, 2), + } + subscriptions_data.append(row) + SUBSCRIPTION_BY_ID[row["subscription_id"]] = row + +df_subscriptions = pd.DataFrame(subscriptions_data) +SUBSCRIPTION_IDS = [s["subscription_id"] for s in subscriptions_data] +SUB_BY_ACCOUNT = {s["account_id"]: s for s in subscriptions_data} +print(f"subscriptions: {len(df_subscriptions)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 5. Product Usage + +# COMMAND ---------- + +# ============================================================================= +# TABLE 3: PRODUCT USAGE (monthly) +# ============================================================================= +# Injected patterns: +# 1. Low seat utilization predicts churn. +# 2. AI feature adoption improves health score and reduces churn risk. +# 3. Enterprise accounts have higher ARR and generally stronger retention. + +usage_data = [] +usage_counter = 1 +USAGE_BY_SUB = {} + +for sub in subscriptions_data: + account = ACCOUNT_BY_ID[sub["account_id"]] + segment = account["segment"] + seats = sub["seats_purchased"] + risk_profile = random.choices(["Healthy", "At Risk"], weights=[73, 27])[0] + base_utilization = {"SMB": 0.54, "Mid-Market": 0.62, "Enterprise": 0.71}[segment] + if risk_profile == "At Risk": + base_utilization -= random.uniform(0.18, 0.32) + if account["ai_feature_adopter"]: + base_utilization += random.uniform(0.04, 0.10) + for year, month in MONTH_LIST: + usage_month = date(year, month, 1) + if usage_month < date(sub["start_date"].year, sub["start_date"].month, 1): + continue + month_idx = (year - 2023) * 12 + month - 1 + trend = 0.002 * month_idx if risk_profile == "Healthy" else -0.003 * month_idx + seasonal = 0.04 if month in (1, 9, 10) else -0.03 if month in (7, 12) else 0.0 + utilization = max(0.04, min(0.98, base_utilization + trend + seasonal + random.gauss(0, 0.07))) + active_users = max(1, int(seats * utilization)) + ai_users = int(active_users * random.uniform(0.22, 0.72)) if account["ai_feature_adopter"] else 0 + queries = int(active_users * random.uniform(18, 85) * (1.25 if account["ai_feature_adopter"] else 1.0)) + workflows = int(active_users * random.uniform(0.4, 2.4)) + health_score = max(1, min(100, 42 + utilization * 55 + (8 if account["ai_feature_adopter"] else 0) + random.gauss(0, 7))) + row = { + "usage_id": f"USG-{usage_counter:08d}", + "usage_month": usage_month, + "usage_year": year, + "usage_month_num": month, + "account_id": sub["account_id"], + "subscription_id": sub["subscription_id"], + "segment": segment, + "plan_tier": sub["plan_tier"], + "seats_purchased": int(seats), + "active_users": int(active_users), + "seat_utilization_pct": round(utilization * 100.0, 2), + "ai_feature_enabled": bool(account["ai_feature_adopter"]), + "ai_feature_active_users": int(ai_users), + "queries_run": int(queries), + "workflows_created": int(workflows), + "data_exports": int(active_users * random.uniform(0.05, 0.8)), + "health_score": round(health_score, 1), + } + usage_data.append(row) + USAGE_BY_SUB.setdefault(sub["subscription_id"], []).append(row) + usage_counter += 1 + +df_product_usage = pd.DataFrame(usage_data) +print(f"product_usage: {len(df_product_usage)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 6. Support Tickets + +# COMMAND ---------- + +# ============================================================================= +# TABLE 4: SUPPORT TICKETS (2,500 rows) +# ============================================================================= +support_data = [] +ticket_counter = 1 + +sub_weights = [] +for sub in subscriptions_data: + account = ACCOUNT_BY_ID[sub["account_id"]] + recent_usage = USAGE_BY_SUB[sub["subscription_id"]][-6:] + avg_health = sum(u["health_score"] for u in recent_usage) / len(recent_usage) + weight = 1.0 + max(0, 70 - avg_health) / 25.0 + if account["segment"] == "Enterprise": + weight *= 1.5 + sub_weights.append(weight) + +for _ in range(2500): + sub = random.choices(subscriptions_data, weights=sub_weights)[0] + account = ACCOUNT_BY_ID[sub["account_id"]] + created_year, created_month = random.choice(MONTH_LIST) + created_date = date(created_year, created_month, random.randint(1, monthrange(created_year, created_month)[1])) + recent_usage = [ + u for u in USAGE_BY_SUB[sub["subscription_id"]] + if u["usage_month"] <= date(created_year, created_month, 1) + ][-3:] + avg_health = sum(u["health_score"] for u in recent_usage) / len(recent_usage) if recent_usage else 70 + severity_weights = [58, 28, 11, 3] + if avg_health < 55: + severity_weights = [38, 32, 21, 9] + severity = random.choices(["Low", "Medium", "High", "Critical"], weights=severity_weights)[0] + status = random.choices(["Resolved", "Open", "Escalated"], weights=[82, 12, 6] if severity in ("Low", "Medium") else [64, 18, 18])[0] + resolution_hours = None + if status == "Resolved": + resolution_hours = random.uniform(1, 18) if severity == "Low" else random.uniform(6, 72) if severity == "Medium" else random.uniform(24, 168) + elif status == "Escalated": + resolution_hours = random.uniform(48, 240) + csat = None + if status == "Resolved": + csat = random.choices([1, 2, 3, 4, 5], weights=[5, 8, 15, 34, 38] if severity in ("Low", "Medium") else [18, 24, 26, 20, 12])[0] + support_data.append( + { + "ticket_id": f"TCKT-{ticket_counter:07d}", + "created_date": created_date, + "ticket_year": created_year, + "ticket_month": created_month, + "account_id": sub["account_id"], + "subscription_id": sub["subscription_id"], + "severity": severity, + "category": random.choice(["Bug", "How To", "Integration", "Billing", "Performance", "Feature Request"]), + "status": status, + "resolution_time_hours": None if resolution_hours is None else round(resolution_hours, 2), + "csat_score": csat, + "is_escalated": status == "Escalated", + } + ) + ticket_counter += 1 + +df_support_tickets = pd.DataFrame(support_data) +TICKETS_BY_SUB = {} +for ticket in support_data: + TICKETS_BY_SUB.setdefault(ticket["subscription_id"], []).append(ticket) +print(f"support_tickets: {len(df_support_tickets)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 7. Invoices + +# COMMAND ---------- + +# ============================================================================= +# TABLE 5: INVOICES +# ============================================================================= +invoice_data = [] +invoice_counter = 1 +INVOICES_BY_SUB = {} + +for sub in subscriptions_data: + account = ACCOUNT_BY_ID[sub["account_id"]] + start_month_anchor = date(sub["start_date"].year, sub["start_date"].month, 1) + for year, month in MONTH_LIST: + invoice_month = date(year, month, 1) + if invoice_month < date(sub["start_date"].year, sub["start_date"].month, 1): + continue + if sub["billing_term"] == "Annual": + months_since_start = (invoice_month.year - start_month_anchor.year) * 12 + ( + invoice_month.month - start_month_anchor.month + ) + if months_since_start % 12 != 0: + continue + amount_due = sub["monthly_recurring_revenue_usd"] * (12.0 if sub["billing_term"] == "Annual" else 1.0) + late_prob = 0.07 + if account["segment"] == "SMB": + late_prob += 0.08 + if sub["billing_term"] == "Monthly": + late_prob += 0.05 + days_late = 0 + payment_status = "Paid" + if random.random() < late_prob: + days_late = random.choices([5, 12, 21, 35, 60], weights=[25, 30, 24, 14, 7])[0] + payment_status = "Late" + if random.random() < 0.025: + payment_status = "Open" + days_late = random.choice([15, 30, 45, 75]) + row = { + "invoice_id": f"INV-{invoice_counter:08d}", + "invoice_month": invoice_month, + "invoice_year": year, + "invoice_month_num": month, + "invoice_date": invoice_month, + "account_id": sub["account_id"], + "subscription_id": sub["subscription_id"], + "billing_term": sub["billing_term"], + "amount_due_usd": round(amount_due, 2), + "amount_paid_usd": round(0.0 if payment_status == "Open" else amount_due, 2), + "payment_status": payment_status, + "days_late": int(days_late), + } + invoice_data.append(row) + INVOICES_BY_SUB.setdefault(sub["subscription_id"], []).append(row) + invoice_counter += 1 + +df_invoices = pd.DataFrame(invoice_data) +print(f"invoices: {len(df_invoices)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 8. Churn Events + +# COMMAND ---------- + +# ============================================================================= +# TABLE 6: CHURN EVENTS +# ============================================================================= +churn_data = [] +churn_counter = 1 + +for sub in subscriptions_data: + account = ACCOUNT_BY_ID[sub["account_id"]] + usage_rows = USAGE_BY_SUB[sub["subscription_id"]] + avg_util = sum(u["seat_utilization_pct"] for u in usage_rows[-6:]) / len(usage_rows[-6:]) + avg_health = sum(u["health_score"] for u in usage_rows[-6:]) / len(usage_rows[-6:]) + tickets = TICKETS_BY_SUB.get(sub["subscription_id"], []) + severe_ticket_count = sum(1 for t in tickets if t["severity"] in ("High", "Critical")) + late_invoice_count = sum(1 for inv in INVOICES_BY_SUB.get(sub["subscription_id"], []) if inv["days_late"] >= 15) + prob = 0.06 + if avg_util < 35: + prob += 0.22 + elif avg_util < 50: + prob += 0.11 + if sub["billing_term"] == "Annual": + prob -= 0.055 + if account["segment"] == "Enterprise": + prob -= 0.045 + if account["ai_feature_adopter"]: + prob -= 0.060 + prob += min(0.14, severe_ticket_count * 0.012) + prob += min(0.12, late_invoice_count * 0.020) + prob = max(0.01, min(0.45, prob)) + if random.random() < prob: + severe_dates = [t["created_date"] for t in tickets if t["severity"] in ("High", "Critical") and t["created_date"] <= date(2025, 10, 1)] + if severe_dates: + anchor = random.choice(severe_dates) + churn_date = min(END_DATE, anchor + timedelta(days=random.randint(21, 90))) + if churn_date < date(2024, 1, 1): + churn_date = date(2024, random.randint(1, 12), random.randint(1, 25)) + else: + churn_date = date(random.choice([2024, 2025]), random.randint(1, 12), random.randint(1, 25)) + reason = random.choices( + ["Low Adoption", "Product Gaps", "Budget", "Support Experience", "Consolidation", "Competitor"], + weights=[34 if avg_util < 50 else 15, 18, 18, 18 if severe_ticket_count else 8, 10, 12], + )[0] + churn_data.append( + { + "churn_event_id": f"CHURN-{churn_counter:07d}", + "churn_date": churn_date, + "churn_year": churn_date.year, + "churn_month": churn_date.month, + "account_id": sub["account_id"], + "subscription_id": sub["subscription_id"], + "segment": account["segment"], + "billing_term": sub["billing_term"], + "plan_tier": sub["plan_tier"], + "churn_reason": reason, + "churned_arr_usd": sub["annual_recurring_revenue_usd"], + "prior_6mo_avg_utilization_pct": round(avg_util, 2), + "prior_6mo_avg_health_score": round(avg_health, 1), + "severe_ticket_count": int(severe_ticket_count), + "late_invoice_count": int(late_invoice_count), + "ai_feature_adopter": bool(account["ai_feature_adopter"]), + } + ) + sub["subscription_status"] = "Churned" + sub["end_date"] = churn_date + account["is_active"] = False + churn_counter += 1 + +df_churn_events = pd.DataFrame(churn_data) +df_subscriptions = pd.DataFrame(subscriptions_data) +df_accounts = pd.DataFrame(accounts_data) +print(f"churn_events: {len(df_churn_events)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 9. Create Schema & Write Delta Tables + +# COMMAND ---------- + +# ============================================================================= +# CREATE SCHEMA & WRITE DELTA TABLES +# ============================================================================= +spark.sql(f"CREATE SCHEMA IF NOT EXISTS `{CATALOG}`.`{SCHEMA}`") + + +def write_table(df_pd, table_name, mode=WRITE_MODE): + df_spark = spark.createDataFrame(df_pd) + ( + df_spark.write.format("delta") + .mode(mode) + .option("overwriteSchema", "true") + .saveAsTable(f"`{CATALOG}`.`{SCHEMA}`.`{table_name}`") + ) + print(f" OK {table_name}: {df_spark.count()} rows written") + + +TABLES = { + "accounts": df_accounts, + "subscriptions": df_subscriptions, + "product_usage": df_product_usage, + "support_tickets": df_support_tickets, + "invoices": df_invoices, + "churn_events": df_churn_events, +} + +print(f"Writing tables to `{CATALOG}`.`{SCHEMA}` with mode={WRITE_MODE}...") +for _table_name, _df in TABLES.items(): + write_table(_df, _table_name) +print("All tables written.") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 10. Verification + +# COMMAND ---------- + +# ============================================================================= +# VERIFICATION +# ============================================================================= +C = f"`{CATALOG}`.`{SCHEMA}`" + +print("=" * 60) +print("ROW COUNTS") +print("=" * 60) +for tbl in TABLES: + n = spark.sql(f"SELECT COUNT(*) AS n FROM {C}.`{tbl}`").collect()[0]["n"] + print(f" {tbl:<24} {n:>8,}") + +print() +print("=" * 60) +print("PATTERN VALIDATION") +print("=" * 60) + +print("\n1. Churned subscriptions should have lower prior utilization.") +spark.sql( + f""" + SELECT s.subscription_status, + ROUND(AVG(u.seat_utilization_pct), 1) AS avg_recent_utilization_pct, + COUNT(DISTINCT s.subscription_id) AS subscriptions + FROM {C}.`subscriptions` s + JOIN {C}.`product_usage` u ON s.subscription_id = u.subscription_id + WHERE u.usage_month >= '2025-07-01' + GROUP BY s.subscription_status + """ +).show() + +print("\n2. Annual billing should have lower churn than monthly billing.") +spark.sql( + f""" + SELECT billing_term, + COUNT(*) AS subscriptions, + SUM(CASE WHEN subscription_status = 'Churned' THEN 1 ELSE 0 END) AS churned, + ROUND(SUM(CASE WHEN subscription_status = 'Churned' THEN 1 ELSE 0 END) * 100.0 / COUNT(*), 1) AS churn_rate_pct + FROM {C}.`subscriptions` + GROUP BY billing_term + """ +).show() + +print("\n3. Enterprise accounts should have higher ARR and lower churn.") +spark.sql( + f""" + SELECT a.segment, + ROUND(AVG(s.annual_recurring_revenue_usd), 2) AS avg_arr, + ROUND(SUM(CASE WHEN s.subscription_status = 'Churned' THEN 1 ELSE 0 END) * 100.0 / COUNT(*), 1) AS churn_rate_pct + FROM {C}.`subscriptions` s + JOIN {C}.`accounts` a ON s.account_id = a.account_id + GROUP BY a.segment + ORDER BY avg_arr DESC + """ +).show() + +print("\n4. Churned accounts should have more severe support tickets.") +spark.sql( + f""" + SELECT s.subscription_status, + ROUND(AVG(CASE WHEN t.severity IN ('High', 'Critical') THEN 1 ELSE 0 END), 3) AS severe_ticket_share + FROM {C}.`subscriptions` s + LEFT JOIN {C}.`support_tickets` t ON s.subscription_id = t.subscription_id + GROUP BY s.subscription_status + """ +).show() + +print("\n5. AI feature adopters should have lower churn.") +spark.sql( + f""" + SELECT a.ai_feature_adopter, + COUNT(*) AS subscriptions, + ROUND(SUM(CASE WHEN s.subscription_status = 'Churned' THEN 1 ELSE 0 END) * 100.0 / COUNT(*), 1) AS churn_rate_pct + FROM {C}.`subscriptions` s + JOIN {C}.`accounts` a ON s.account_id = a.account_id + GROUP BY a.ai_feature_adopter + """ +).show() + +print("\n6. Late payment behavior should correlate with churn.") +spark.sql( + f""" + WITH invoice_lateness AS ( + SELECT subscription_id, + SUM(CASE WHEN days_late >= 15 THEN 1 ELSE 0 END) AS late_invoice_count + FROM {C}.`invoices` + GROUP BY subscription_id + ) + SELECT s.subscription_status, + ROUND(AVG(l.late_invoice_count), 2) AS avg_late_invoice_count + FROM {C}.`subscriptions` s + JOIN invoice_lateness l ON s.subscription_id = l.subscription_id + GROUP BY s.subscription_status + """ +).show() + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 11. Create Metric Views + +# COMMAND ---------- + +# ============================================================================= +# CREATE METRIC VIEWS +# ============================================================================= +print("Creating metric views...") + +metric_views = { + "mv_subscription_revenue": f""" +version: "0.1" +source: {CATALOG}.{SCHEMA}.subscriptions + +joins: + - name: account + source: {CATALOG}.{SCHEMA}.accounts + on: source.account_id = account.account_id + +dimensions: + - name: Segment + expr: account.segment + - name: Industry + expr: account.industry + - name: Region + expr: account.region + - name: Plan Tier + expr: plan_tier + - name: Billing Term + expr: billing_term + - name: Subscription Status + expr: subscription_status + - name: AI Feature Adopter + expr: account.ai_feature_adopter + +measures: + - name: Subscription Count + expr: COUNT(1) + - name: Active Subscription Count + expr: COUNT(1) FILTER (WHERE subscription_status = 'Active') + - name: Churned Subscription Count + expr: COUNT(1) FILTER (WHERE subscription_status = 'Churned') + - name: ARR + expr: SUM(annual_recurring_revenue_usd) + - name: MRR + expr: SUM(monthly_recurring_revenue_usd) + - name: Average ARR + expr: AVG(annual_recurring_revenue_usd) + - name: Seats Purchased + expr: SUM(seats_purchased) + - name: Logo Churn Rate Pct + expr: COUNT(1) FILTER (WHERE subscription_status = 'Churned') * 100.0 / COUNT(1) +""", + "mv_product_usage": f""" +version: "0.1" +source: {CATALOG}.{SCHEMA}.product_usage + +joins: + - name: account + source: {CATALOG}.{SCHEMA}.accounts + on: source.account_id = account.account_id + - name: subscription + source: {CATALOG}.{SCHEMA}.subscriptions + on: source.subscription_id = subscription.subscription_id + +dimensions: + - name: Usage Month + expr: usage_month + - name: Usage Year + expr: usage_year + - name: Segment + expr: account.segment + - name: Plan Tier + expr: subscription.plan_tier + - name: Billing Term + expr: subscription.billing_term + - name: AI Feature Enabled + expr: ai_feature_enabled + +measures: + - name: Usage Row Count + expr: COUNT(1) + - name: Active Users + expr: SUM(active_users) + - name: Average Seat Utilization Pct + expr: AVG(seat_utilization_pct) + - name: Average Health Score + expr: AVG(health_score) + - name: Queries Run + expr: SUM(queries_run) + - name: Workflows Created + expr: SUM(workflows_created) + - name: AI Active Users + expr: SUM(ai_feature_active_users) + - name: AI Adoption Rate Pct + expr: COUNT(1) FILTER (WHERE ai_feature_enabled = TRUE) * 100.0 / COUNT(1) +""", + "mv_churn_risk": f""" +version: "0.1" +source: {CATALOG}.{SCHEMA}.churn_events + +joins: + - name: account + source: {CATALOG}.{SCHEMA}.accounts + on: source.account_id = account.account_id + - name: subscription + source: {CATALOG}.{SCHEMA}.subscriptions + on: source.subscription_id = subscription.subscription_id + +dimensions: + - name: Churn Month + expr: DATE_TRUNC('MONTH', churn_date) + - name: Churn Year + expr: churn_year + - name: Segment + expr: segment + - name: Plan Tier + expr: plan_tier + - name: Billing Term + expr: billing_term + - name: Churn Reason + expr: churn_reason + - name: AI Feature Adopter + expr: ai_feature_adopter + +measures: + - name: Churn Event Count + expr: COUNT(1) + - name: Churned ARR + expr: SUM(churned_arr_usd) + - name: Average Prior Utilization Pct + expr: AVG(prior_6mo_avg_utilization_pct) + - name: Average Prior Health Score + expr: AVG(prior_6mo_avg_health_score) + - name: Severe Tickets Before Churn + expr: SUM(severe_ticket_count) + - name: Late Invoices Before Churn + expr: SUM(late_invoice_count) +""", +} + +for mv_name, yaml_body in metric_views.items(): + ddl = ( + f"CREATE OR REPLACE VIEW `{CATALOG}`.`{SCHEMA}`.`{mv_name}`\n" + "WITH METRICS\n" + "LANGUAGE YAML\n" + "AS $$" + + yaml_body + + "$$" + ) + spark.sql(ddl) + print(f" OK {mv_name}") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 12. Register Constraints & Comments + +# COMMAND ---------- + +# ============================================================================= +# REGISTER CONSTRAINTS AND COMMENTS +# ============================================================================= +print("Registering constraints and comments...") + +PRIMARY_KEYS = { + "accounts": "account_id", + "subscriptions": "subscription_id", + "product_usage": "usage_id", + "support_tickets": "ticket_id", + "invoices": "invoice_id", + "churn_events": "churn_event_id", +} + +FOREIGN_KEYS = [ + ("subscriptions", "account_id", "accounts", "account_id"), + ("product_usage", "account_id", "accounts", "account_id"), + ("product_usage", "subscription_id", "subscriptions", "subscription_id"), + ("support_tickets", "account_id", "accounts", "account_id"), + ("support_tickets", "subscription_id", "subscriptions", "subscription_id"), + ("invoices", "account_id", "accounts", "account_id"), + ("invoices", "subscription_id", "subscriptions", "subscription_id"), + ("churn_events", "account_id", "accounts", "account_id"), + ("churn_events", "subscription_id", "subscriptions", "subscription_id"), +] + +for table, pk in PRIMARY_KEYS.items(): + spark.sql(f"ALTER TABLE {C}.`{table}` ALTER COLUMN `{pk}` SET NOT NULL") + spark.sql(f"ALTER TABLE {C}.`{table}` ADD CONSTRAINT pk_{table} PRIMARY KEY (`{pk}`)") + +for table, col, ref_table, ref_col in FOREIGN_KEYS: + spark.sql( + f"ALTER TABLE {C}.`{table}` ADD CONSTRAINT fk_{table}_{col} " + f"FOREIGN KEY (`{col}`) REFERENCES {C}.`{ref_table}` (`{ref_col}`)" + ) + + +def sql_text(value): + return value.replace("'", "''") + + +TABLE_COMMENTS = { + "accounts": "Fictional SaaS customer account dimension with segment, industry, region, CSM tier, and AI adoption flag.", + "subscriptions": "Subscription dimension with plan, billing term, seats, MRR, ARR, status, and churn end date.", + "product_usage": "Monthly usage facts. Low seat utilization and lower health scores increase synthetic churn risk.", + "support_tickets": "Support ticket facts with severity, category, status, resolution time, CSAT, and escalation flag.", + "invoices": "Invoice and payment facts. Late payment behavior is correlated with churn risk.", + "churn_events": "Churn facts linked to subscriptions with churn reason, churned ARR, prior usage, support, and invoice signals.", +} + +for table, comment in TABLE_COMMENTS.items(): + spark.sql(f"COMMENT ON TABLE {C}.`{table}` IS '{sql_text(comment)}'") + +for table, df_pd in TABLES.items(): + for column in df_pd.columns: + readable = column.replace("_", " ") + spark.sql( + f"ALTER TABLE {C}.`{table}` ALTER COLUMN `{column}` " + f"COMMENT '{sql_text(readable)} for the synthetic SaaS churn dataset'" + ) + +print(" OK constraints and comments registered") + +print() +print("=" * 60) +print("SETUP COMPLETE") +print("=" * 60) +print(f" Catalog : {CATALOG}") +print(f" Schema : {SCHEMA}") +print(" Tables : accounts, subscriptions, product_usage, support_tickets, invoices, churn_events") +print(" Metric Views: mv_subscription_revenue, mv_product_usage, mv_churn_risk") +print(" Next : Create a Genie space from these tables, or see notebooks/demo-data/README.md") diff --git a/notebooks/demo-data/generate_talent_advisory_data.py b/notebooks/demo-data/generate_talent_advisory_data.py new file mode 100644 index 000000000..bb22998e3 --- /dev/null +++ b/notebooks/demo-data/generate_talent_advisory_data.py @@ -0,0 +1,2410 @@ +# Databricks notebook source + +# MAGIC %md +# MAGIC # Talent Advisory - Synthetic Dataset Generator +# MAGIC +# MAGIC Generates a fictional HR talent advisory dataset for Databricks AI/BI Genie demos. +# MAGIC The data is designed for multi-Genie demos where focused spaces answer workforce +# MAGIC planning, hiring, retention, mobility, compensation, and succession questions. +# MAGIC +# MAGIC | Table | Rows | Description | +# MAGIC |---|---:|---| +# MAGIC | `locations` | 10 | Work locations and regions | +# MAGIC | `org_units` | 18 | Organizational hierarchy | +# MAGIC | `job_roles` | 44 | Role catalog with compensation bands and criticality | +# MAGIC | `skills` | 36 | Skill taxonomy | +# MAGIC | `employees` | ~2,850 | Synthetic employee dimension, ~2,500 active | +# MAGIC | `employee_skills` | ~17,000 | Employee skill proficiency and gap facts | +# MAGIC | `learning_activity` | variable | Learning completions and mobility program activity | +# MAGIC | `talent_events` | variable | Hires, exits, promotions, and transfers | +# MAGIC | `performance_reviews` | variable | Annual performance and potential reviews | +# MAGIC | `compensation_snapshots` | variable | Annual salary, bonus, and compa-ratio facts | +# MAGIC | `engagement_pulses` | variable | Quarterly engagement pulse facts | +# MAGIC | `retention_risk_snapshots` | variable | Quarterly retention risk snapshots | +# MAGIC | `requisitions` | ~640 | Hiring requisitions | +# MAGIC | `applications` | variable | Candidate funnel facts | +# MAGIC | `succession_plans` | variable | Critical role successor coverage | +# MAGIC +# MAGIC The notebook also creates six curated mart tables and six metric views for +# MAGIC focused Genie spaces. +# MAGIC +# MAGIC **Setup:** Set the `catalog` and `schema` widgets (or edit defaults below), then **Run All**. +# MAGIC +# MAGIC Seed: `42` - fully reproducible. + +# COMMAND ---------- + +# MAGIC %pip install faker==40.19.1 --quiet +# MAGIC %restart_python + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 1. Configuration + +# COMMAND ---------- + +# ============================================================================= +# CONFIGURATION - Edit only this section before running +# ============================================================================= +DEFAULT_CATALOG = "" # Unity Catalog name; required +DEFAULT_SCHEMA = "talent_advisory" # Schema / database name + +# Widgets let the bulk runner pass values while preserving standalone defaults. +try: + dbutils.widgets.text("catalog", DEFAULT_CATALOG, "Unity Catalog name") +except Exception: + pass + +try: + dbutils.widgets.text("schema", DEFAULT_SCHEMA, "Schema / database name") +except Exception: + pass + +try: + dbutils.widgets.dropdown("overwrite_existing", "false", ["false", "true"], "Overwrite existing tables") +except Exception: + pass + +try: + CATALOG = dbutils.widgets.get("catalog").strip() or DEFAULT_CATALOG +except Exception: + CATALOG = DEFAULT_CATALOG + +try: + SCHEMA = dbutils.widgets.get("schema").strip() or DEFAULT_SCHEMA +except Exception: + SCHEMA = DEFAULT_SCHEMA + +try: + OVERWRITE_EXISTING = dbutils.widgets.get("overwrite_existing").strip().lower() == "true" +except Exception: + OVERWRITE_EXISTING = False + +if not CATALOG: + raise ValueError("Set the catalog widget to a Unity Catalog name before running this notebook.") + +WRITE_MODE = "overwrite" if OVERWRITE_EXISTING else "errorifexists" + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 2. Imports & Global Setup + +# COMMAND ---------- + +# ============================================================================= +# IMPORTS & GLOBAL SETUP +# ============================================================================= +import math +import random +from calendar import monthrange +from datetime import date, timedelta + +import numpy as np +import pandas as pd +from faker import Faker + +SEED = 42 +random.seed(SEED) +np.random.seed(SEED) +fake = Faker("en_US") +Faker.seed(SEED) + +START_DATE = date(2023, 1, 1) +END_DATE = date(2025, 12, 31) +YEARS = [2023, 2024, 2025] +MONTH_LIST = [(y, m) for y in YEARS for m in range(1, 13)] +QUARTER_ENDS = [ + date(2023, 3, 31), + date(2023, 6, 30), + date(2023, 9, 30), + date(2023, 12, 31), + date(2024, 3, 31), + date(2024, 6, 30), + date(2024, 9, 30), + date(2024, 12, 31), + date(2025, 3, 31), + date(2025, 6, 30), + date(2025, 9, 30), + date(2025, 12, 31), +] + + +def clamp(value, low, high): + return max(low, min(high, value)) + + +def date_between(start_date, end_date): + days = (end_date - start_date).days + return start_date + timedelta(days=random.randint(0, max(0, days))) + + +def end_of_month(year, month): + return date(year, month, monthrange(year, month)[1]) + + +def quarter_label(d): + return f"{d.year}-Q{((d.month - 1) // 3) + 1}" + + +def is_employed_on(employee, as_of_date): + term_date = employee.get("termination_date") + return employee["hire_date"] <= as_of_date and (term_date is None or term_date > as_of_date) + + +def active_during(employee, period_start, period_end): + term_date = employee.get("termination_date") + return employee["hire_date"] <= period_end and (term_date is None or term_date >= period_start) + + +print("Setup complete.") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 3. Locations + +# COMMAND ---------- + +# ============================================================================= +# TABLE 1: LOCATIONS (10 rows) +# ============================================================================= +locations_data = [ + { + "location_id": "LOC-001", + "location_name": "San Francisco HQ", + "region": "West", + "country": "United States", + "state": "CA", + "city": "San Francisco", + "time_zone": "America/Los_Angeles", + "work_model": "Hybrid Hub", + "cost_index": 1.28, + "is_active": True, + }, + { + "location_id": "LOC-002", + "location_name": "Seattle Engineering Hub", + "region": "West", + "country": "United States", + "state": "WA", + "city": "Seattle", + "time_zone": "America/Los_Angeles", + "work_model": "Hybrid Hub", + "cost_index": 1.18, + "is_active": True, + }, + { + "location_id": "LOC-003", + "location_name": "Austin GTM Hub", + "region": "Central", + "country": "United States", + "state": "TX", + "city": "Austin", + "time_zone": "America/Chicago", + "work_model": "Hybrid Hub", + "cost_index": 0.98, + "is_active": True, + }, + { + "location_id": "LOC-004", + "location_name": "New York Sales Office", + "region": "East", + "country": "United States", + "state": "NY", + "city": "New York", + "time_zone": "America/New_York", + "work_model": "Hybrid Hub", + "cost_index": 1.24, + "is_active": True, + }, + { + "location_id": "LOC-005", + "location_name": "Atlanta Success Hub", + "region": "East", + "country": "United States", + "state": "GA", + "city": "Atlanta", + "time_zone": "America/New_York", + "work_model": "Hybrid Hub", + "cost_index": 0.94, + "is_active": True, + }, + { + "location_id": "LOC-006", + "location_name": "Chicago Operations Hub", + "region": "Central", + "country": "United States", + "state": "IL", + "city": "Chicago", + "time_zone": "America/Chicago", + "work_model": "Hybrid Hub", + "cost_index": 1.02, + "is_active": True, + }, + { + "location_id": "LOC-007", + "location_name": "London EMEA Hub", + "region": "EMEA", + "country": "United Kingdom", + "state": None, + "city": "London", + "time_zone": "Europe/London", + "work_model": "Hybrid Hub", + "cost_index": 1.12, + "is_active": True, + }, + { + "location_id": "LOC-008", + "location_name": "Dublin Support Center", + "region": "EMEA", + "country": "Ireland", + "state": None, + "city": "Dublin", + "time_zone": "Europe/Dublin", + "work_model": "Office First", + "cost_index": 1.03, + "is_active": True, + }, + { + "location_id": "LOC-009", + "location_name": "Bangalore Product Center", + "region": "APAC", + "country": "India", + "state": None, + "city": "Bangalore", + "time_zone": "Asia/Kolkata", + "work_model": "Office First", + "cost_index": 0.62, + "is_active": True, + }, + { + "location_id": "LOC-010", + "location_name": "Remote - Americas", + "region": "Americas Remote", + "country": "United States", + "state": None, + "city": "Remote", + "time_zone": "Mixed", + "work_model": "Remote", + "cost_index": 0.96, + "is_active": True, + }, +] + +df_locations = pd.DataFrame(locations_data) +LOCATION_BY_ID = {row["location_id"]: row for row in locations_data} +LOCATION_IDS = [row["location_id"] for row in locations_data] +print(f"locations: {len(df_locations)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 4. Org Units + +# COMMAND ---------- + +# ============================================================================= +# TABLE 2: ORG_UNITS (18 rows) +# ============================================================================= +org_units_data = [ + ("ORG-001", "Executive Leadership", None, "Corporate", "Corporate", "Global", 10, False), + ("ORG-002", "Engineering", "ORG-001", "Engineering", "Product & Technology", "Global", 320, True), + ("ORG-003", "Data & AI Platform", "ORG-002", "Engineering", "Product & Technology", "Global", 190, True), + ("ORG-004", "Core Platform", "ORG-002", "Engineering", "Product & Technology", "Global", 230, False), + ("ORG-005", "Product Management", "ORG-001", "Product", "Product & Technology", "Global", 210, True), + ("ORG-006", "Design & Research", "ORG-005", "Product", "Product & Technology", "Global", 90, False), + ("ORG-007", "Sales", "ORG-001", "Sales", "Go To Market", "Global", 260, True), + ("ORG-008", "Sales East", "ORG-007", "Sales", "Go To Market", "East", 230, True), + ("ORG-009", "Sales West", "ORG-007", "Sales", "Go To Market", "West", 210, True), + ("ORG-010", "Solutions Consulting", "ORG-007", "Sales", "Go To Market", "Global", 150, True), + ("ORG-011", "Customer Success", "ORG-001", "Customer Success", "Go To Market", "Global", 260, True), + ("ORG-012", "Implementation Services", "ORG-011", "Customer Success", "Go To Market", "Global", 130, False), + ("ORG-013", "Support Operations", "ORG-011", "Customer Success", "Go To Market", "Global", 160, False), + ("ORG-014", "Marketing", "ORG-001", "Marketing", "Go To Market", "Global", 140, False), + ("ORG-015", "People & Culture", "ORG-001", "People", "Corporate", "Global", 90, False), + ("ORG-016", "Finance", "ORG-001", "Finance", "Corporate", "Global", 90, False), + ("ORG-017", "Legal & Compliance", "ORG-001", "Legal", "Corporate", "Global", 60, False), + ("ORG-018", "Business Operations", "ORG-001", "Operations", "Corporate", "Global", 130, False), +] + +org_units_data = [ + { + "org_unit_id": org_id, + "org_unit_name": name, + "parent_org_unit_id": parent_id, + "function": function, + "business_unit": business_unit, + "region_scope": region_scope, + "target_headcount_weight": int(weight), + "is_critical_org": bool(is_critical), + } + for org_id, name, parent_id, function, business_unit, region_scope, weight, is_critical in org_units_data +] + +df_org_units = pd.DataFrame(org_units_data) +ORG_BY_ID = {row["org_unit_id"]: row for row in org_units_data} +ORG_IDS = [row["org_unit_id"] for row in org_units_data] +ORG_WEIGHTS = [row["target_headcount_weight"] for row in org_units_data] +print(f"org_units: {len(df_org_units)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 5. Job Roles + +# COMMAND ---------- + +# ============================================================================= +# TABLE 3: JOB_ROLES (44 rows) +# ============================================================================= +role_specs = [ + ("Chief Executive Officer", "Corporate", "Management", 7, "Executive", 330000, True, 8, "Run"), + ("VP Engineering", "Engineering", "Management", 7, "Executive", 305000, True, 8, "Scale"), + ("Engineering Director", "Engineering", "Management", 6, "Director", 250000, True, 7, "Scale"), + ("Engineering Manager", "Engineering", "Management", 5, "Manager", 205000, True, 6, "Scale"), + ("Staff Software Engineer", "Engineering", "IC", 5, "Staff", 218000, True, 8, "Scale"), + ("Senior Software Engineer", "Engineering", "IC", 4, "Senior", 178000, False, 6, "Scale"), + ("Software Engineer", "Engineering", "IC", 3, "Professional", 142000, False, 5, "Maintain"), + ("Data Platform Engineer", "Data & AI", "IC", 4, "Senior", 188000, True, 8, "Invest"), + ("ML Engineer", "Data & AI", "IC", 4, "Senior", 196000, True, 9, "Invest"), + ("AI Platform Architect", "Data & AI", "IC", 5, "Staff", 235000, True, 10, "Invest"), + ("Data Scientist", "Data & AI", "IC", 3, "Professional", 158000, False, 8, "Invest"), + ("Analytics Engineer", "Data & AI", "IC", 3, "Professional", 148000, False, 7, "Invest"), + ("Data Science Manager", "Data & AI", "Management", 5, "Manager", 215000, True, 8, "Invest"), + ("VP Product", "Product", "Management", 7, "Executive", 295000, True, 8, "Invest"), + ("Group Product Manager", "Product", "Management", 5, "Manager", 212000, True, 8, "Invest"), + ("Senior Product Manager", "Product", "IC", 4, "Senior", 180000, False, 7, "Invest"), + ("Product Manager", "Product", "IC", 3, "Professional", 145000, False, 6, "Maintain"), + ("Product Operations Manager", "Product", "IC", 3, "Professional", 132000, False, 5, "Maintain"), + ("Product Designer", "Product", "IC", 3, "Professional", 138000, False, 6, "Maintain"), + ("Design Manager", "Product", "Management", 5, "Manager", 188000, False, 6, "Maintain"), + ("VP Sales", "Sales", "Management", 7, "Executive", 285000, True, 8, "Scale"), + ("Regional Sales Director", "Sales", "Management", 6, "Director", 230000, True, 8, "Scale"), + ("Sales Manager", "Sales", "Management", 5, "Manager", 175000, True, 7, "Scale"), + ("Account Executive", "Sales", "IC", 3, "Professional", 128000, False, 6, "Scale"), + ("Senior Account Executive", "Sales", "IC", 4, "Senior", 158000, False, 7, "Scale"), + ("Revenue Operations Manager", "Sales", "IC", 4, "Senior", 155000, True, 7, "Maintain"), + ("Solutions Consultant", "Sales", "IC", 4, "Senior", 168000, True, 8, "Scale"), + ("VP Customer Success", "Customer Success", "Management", 7, "Executive", 260000, True, 7, "Scale"), + ("Customer Success Director", "Customer Success", "Management", 6, "Director", 205000, True, 7, "Scale"), + ("Customer Success Manager", "Customer Success", "IC", 3, "Professional", 118000, False, 5, "Maintain"), + ("Senior Customer Success Manager", "Customer Success", "IC", 4, "Senior", 145000, False, 6, "Maintain"), + ("Implementation Consultant", "Customer Success", "IC", 3, "Professional", 124000, False, 6, "Maintain"), + ("Support Engineer", "Customer Success", "IC", 3, "Professional", 112000, False, 5, "Maintain"), + ("Demand Generation Manager", "Marketing", "IC", 4, "Senior", 142000, False, 5, "Maintain"), + ("Product Marketing Manager", "Marketing", "IC", 4, "Senior", 152000, False, 6, "Maintain"), + ("Marketing Operations Analyst", "Marketing", "IC", 3, "Professional", 108000, False, 4, "Maintain"), + ("People Business Partner", "People", "IC", 4, "Senior", 138000, False, 5, "Maintain"), + ("Talent Acquisition Partner", "People", "IC", 3, "Professional", 115000, False, 5, "Maintain"), + ("Learning Program Manager", "People", "IC", 4, "Senior", 132000, False, 5, "Invest"), + ("Finance Manager", "Finance", "Management", 5, "Manager", 165000, False, 5, "Maintain"), + ("Financial Analyst", "Finance", "IC", 3, "Professional", 105000, False, 4, "Maintain"), + ("Legal Counsel", "Legal", "IC", 4, "Senior", 172000, False, 5, "Maintain"), + ("Compliance Manager", "Legal", "IC", 4, "Senior", 145000, False, 5, "Maintain"), + ("Business Operations Manager", "Operations", "IC", 4, "Senior", 142000, False, 5, "Maintain"), +] + +job_roles_data = [] +for i, spec in enumerate(role_specs, start=1): + title, family, track, level_num, level_name, midpoint, critical, scarcity, growth_priority = spec + premium = 0.12 if family == "Data & AI" else 0.08 if family == "Product" else 0.04 if family == "Engineering" else 0.0 + job_roles_data.append( + { + "role_id": f"ROLE-{i:03d}", + "role_title": title, + "job_family": family, + "job_track": track, + "job_level": level_name, + "job_level_num": int(level_num), + "base_salary_midpoint_usd": int(midpoint), + "salary_range_min_usd": int(midpoint * 0.80), + "salary_range_max_usd": int(midpoint * 1.20), + "market_premium_pct": round(premium * 100.0, 1), + "scarcity_score": int(scarcity), + "is_critical_role": bool(critical), + "growth_priority": growth_priority, + } + ) + +df_job_roles = pd.DataFrame(job_roles_data) +ROLE_BY_ID = {row["role_id"]: row for row in job_roles_data} +ROLES_BY_FAMILY = {} +for row in job_roles_data: + ROLES_BY_FAMILY.setdefault(row["job_family"], []).append(row) +print(f"job_roles: {len(df_job_roles)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 6. Skills + +# COMMAND ---------- + +# ============================================================================= +# TABLE 4: SKILLS (36 rows) +# ============================================================================= +skill_specs = [ + ("Python", "Technical", False, 6), + ("SQL", "Technical", False, 6), + ("Spark", "Technical", False, 7), + ("Cloud Architecture", "Technical", False, 7), + ("API Design", "Technical", False, 5), + ("Data Modeling", "Data & AI", False, 6), + ("Machine Learning", "Data & AI", True, 8), + ("MLOps", "Data & AI", True, 9), + ("Generative AI", "Data & AI", True, 10), + ("Experimentation", "Data & AI", False, 7), + ("Data Governance", "Data & AI", False, 7), + ("Product Strategy", "Product", False, 7), + ("Roadmap Planning", "Product", False, 6), + ("User Research", "Product", False, 6), + ("Design Systems", "Product", False, 5), + ("Pricing Strategy", "GTM", False, 6), + ("Enterprise Selling", "GTM", False, 7), + ("Pipeline Management", "GTM", False, 6), + ("Solution Discovery", "GTM", False, 7), + ("Customer Health", "GTM", False, 6), + ("Renewal Strategy", "GTM", False, 6), + ("Implementation Planning", "GTM", False, 5), + ("Technical Troubleshooting", "GTM", False, 5), + ("Demand Generation", "Marketing", False, 5), + ("Product Messaging", "Marketing", False, 6), + ("Marketing Analytics", "Marketing", False, 5), + ("Workforce Planning", "People", False, 6), + ("Talent Advising", "People", False, 6), + ("Compensation Analysis", "People", False, 7), + ("Financial Planning", "Finance", False, 5), + ("Forecasting", "Finance", False, 5), + ("Contract Review", "Legal", False, 5), + ("Risk Management", "Legal", False, 6), + ("Change Management", "Leadership", False, 6), + ("People Management", "Leadership", False, 7), + ("Executive Communication", "Leadership", False, 6), +] + +skills_data = [ + { + "skill_id": f"SKILL-{i:03d}", + "skill_name": name, + "skill_category": category, + "is_emerging_skill": bool(is_emerging), + "scarcity_score": int(scarcity), + } + for i, (name, category, is_emerging, scarcity) in enumerate(skill_specs, start=1) +] + +df_skills = pd.DataFrame(skills_data) +SKILL_BY_ID = {row["skill_id"]: row for row in skills_data} +SKILL_ID_BY_NAME = {row["skill_name"]: row["skill_id"] for row in skills_data} +print(f"skills: {len(df_skills)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 7. Employees + +# COMMAND ---------- + +# ============================================================================= +# TABLE 5: EMPLOYEES (~2,850 rows, ~2,500 active) +# ============================================================================= +ORG_ROLE_FAMILIES = { + "Engineering": ["Engineering", "Data & AI"], + "Product": ["Product"], + "Sales": ["Sales"], + "Customer Success": ["Customer Success", "Sales"], + "Marketing": ["Marketing"], + "People": ["People"], + "Finance": ["Finance"], + "Legal": ["Legal"], + "Operations": ["Operations", "Finance", "People"], + "Corporate": ["Corporate", "Finance", "People", "Legal"], +} + +FUNCTION_LOCATION_WEIGHTS = { + "Engineering": (["LOC-001", "LOC-002", "LOC-009", "LOC-010"], [24, 30, 28, 18]), + "Product": (["LOC-001", "LOC-002", "LOC-009", "LOC-010"], [36, 18, 28, 18]), + "Sales": (["LOC-003", "LOC-004", "LOC-005", "LOC-007", "LOC-010"], [24, 32, 18, 12, 14]), + "Customer Success": (["LOC-003", "LOC-005", "LOC-008", "LOC-010"], [25, 28, 26, 21]), + "Marketing": (["LOC-001", "LOC-004", "LOC-010"], [35, 30, 35]), + "People": (["LOC-001", "LOC-006", "LOC-010"], [35, 25, 40]), + "Finance": (["LOC-001", "LOC-006", "LOC-010"], [32, 38, 30]), + "Legal": (["LOC-001", "LOC-004", "LOC-010"], [40, 30, 30]), + "Operations": (["LOC-003", "LOC-006", "LOC-010"], [30, 35, 35]), + "Corporate": (["LOC-001", "LOC-004"], [70, 30]), +} + + +def choose_role_for_org(org): + families = ORG_ROLE_FAMILIES[org["function"]] + if org["org_unit_name"] == "Data & AI Platform": + families = ["Data & AI", "Engineering"] + if org["org_unit_name"] == "Solutions Consulting": + families = ["Sales", "Data & AI"] + role_pool = [] + role_weights = [] + for family in families: + for role in ROLES_BY_FAMILY.get(family, []): + role_pool.append(role) + weight = {3: 42, 4: 30, 5: 15, 6: 7, 7: 2}.get(role["job_level_num"], 4) + if role["job_track"] == "Management": + weight *= 0.65 + if org["org_unit_name"] == "Data & AI Platform" and role["job_family"] == "Data & AI": + weight *= 2.6 + if org["function"] == "Corporate" and role["job_level_num"] < 5: + weight *= 0.5 + role_weights.append(weight) + return random.choices(role_pool, weights=role_weights)[0] + + +def choose_hire_date(role_family): + if role_family == "Data & AI": + period = random.choices( + ["before", "2023", "2024", "2025"], weights=[38, 16, 22, 24] + )[0] + elif role_family == "Product": + period = random.choices( + ["before", "2023", "2024", "2025"], weights=[50, 17, 18, 15] + )[0] + else: + period = random.choices( + ["before", "2023", "2024", "2025"], weights=[66, 13, 11, 10] + )[0] + + if period == "before": + return date_between(date(2016, 1, 1), date(2022, 12, 31)) + return date_between(date(int(period), 1, 1), date(int(period), 11, 15)) + + +def choose_location(function): + location_ids, weights = FUNCTION_LOCATION_WEIGHTS[function] + return random.choices(location_ids, weights=weights)[0] + + +employees_data = [] +for i in range(1, 2851): + org = random.choices(org_units_data, weights=ORG_WEIGHTS)[0] + role = choose_role_for_org(org) + hire_date = choose_hire_date(role["job_family"]) + location_id = choose_location(org["function"]) + location = LOCATION_BY_ID[location_id] + performance_profile = random.choices( + ["Developing", "Solid", "Strong", "Exceptional"], + weights=[9, 54, 29, 8], + )[0] + is_high_performer_profile = performance_profile in ("Strong", "Exceptional") and random.random() < ( + 0.78 if performance_profile == "Strong" else 0.96 + ) + compa_ratio_profile = random.gauss(0.98, 0.08) + if is_high_performer_profile and random.random() < 0.28: + compa_ratio_profile = random.uniform(0.82, 0.89) + if role["job_family"] == "Product" and random.random() < 0.18: + compa_ratio_profile -= random.uniform(0.03, 0.08) + compa_ratio_profile = round(clamp(compa_ratio_profile, 0.72, 1.22), 3) + + mobility_program_participant = bool( + random.random() + < ( + 0.24 + if is_high_performer_profile + else 0.10 if role["job_family"] in ("Data & AI", "Product", "Sales") else 0.07 + ) + ) + + exit_prob = 0.070 + if org["org_unit_name"] == "Sales East": + exit_prob += 0.075 + if is_high_performer_profile and compa_ratio_profile < 0.90: + exit_prob += 0.120 + if role["job_family"] == "Data & AI": + exit_prob -= 0.020 + if mobility_program_participant: + exit_prob -= 0.040 + if hire_date > date(2025, 6, 30): + exit_prob *= 0.25 + exit_prob = clamp(exit_prob, 0.015, 0.32) + + termination_date = None + termination_type = None + employment_status = "Active" + if hire_date < date(2025, 6, 30) and random.random() < exit_prob: + min_exit = max(hire_date + timedelta(days=180), date(2024, 1, 1)) + if org["org_unit_name"] == "Sales East": + min_exit = max(min_exit, date(2024, 7, 1)) + if min_exit <= date(2025, 12, 15): + termination_date = date_between(min_exit, date(2025, 12, 15)) + termination_type = random.choices( + ["Voluntary", "Involuntary", "Reduction in Force"], weights=[74, 18, 8] + )[0] + employment_status = "Inactive" + + employee = { + "employee_id": f"EMP-{i:05d}", + "employee_name": fake.name(), + "hire_date": hire_date, + "termination_date": termination_date, + "employment_status": employment_status, + "termination_type": termination_type, + "org_unit_id": org["org_unit_id"], + "location_id": location_id, + "role_id": role["role_id"], + "job_family": role["job_family"], + "job_level": role["job_level"], + "job_level_num": int(role["job_level_num"]), + "worker_type": random.choices(["Full Time", "Part Time"], weights=[96, 4])[0], + "work_model": location["work_model"], + "current_manager_employee_id": None, + "mobility_program_participant": mobility_program_participant, + "performance_profile": performance_profile, + "high_performer_profile": bool(is_high_performer_profile), + "compa_ratio_profile": compa_ratio_profile, + "is_active": employment_status == "Active", + } + employees_data.append(employee) + +# Assign managers after all employees exist. +EMP_BY_ID = {row["employee_id"]: row for row in employees_data} +employees_by_org = {} +for employee in employees_data: + employees_by_org.setdefault(employee["org_unit_id"], []).append(employee) + +for employee in employees_data: + if employee["job_level_num"] >= 7: + continue + candidates = [ + manager + for manager in employees_by_org.get(employee["org_unit_id"], []) + if manager["employee_id"] != employee["employee_id"] + and manager["job_level_num"] > employee["job_level_num"] + and ROLE_BY_ID[manager["role_id"]]["job_track"] == "Management" + ] + if not candidates: + parent_id = ORG_BY_ID[employee["org_unit_id"]]["parent_org_unit_id"] + candidates = [ + manager + for manager in employees_by_org.get(parent_id, []) + if manager["job_level_num"] >= max(5, employee["job_level_num"] + 1) + ] + if candidates: + employee["current_manager_employee_id"] = random.choice(candidates)["employee_id"] + +df_employees = pd.DataFrame(employees_data) +EMPLOYEE_IDS = [row["employee_id"] for row in employees_data] +ACTIVE_EMPLOYEES = [row for row in employees_data if row["is_active"]] +print(f"employees: {len(df_employees)} rows") +print(f"active employees: {len(ACTIVE_EMPLOYEES)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 8. Employee Skills + +# COMMAND ---------- + +# ============================================================================= +# TABLE 6: EMPLOYEE_SKILLS +# ============================================================================= +FAMILY_SKILL_NAMES = { + "Engineering": ["Python", "SQL", "Spark", "Cloud Architecture", "API Design", "Change Management"], + "Data & AI": ["Python", "SQL", "Spark", "Data Modeling", "Machine Learning", "MLOps", "Generative AI", "Data Governance"], + "Product": ["Product Strategy", "Roadmap Planning", "User Research", "Experimentation", "Executive Communication"], + "Sales": ["Enterprise Selling", "Pipeline Management", "Solution Discovery", "Pricing Strategy", "Executive Communication"], + "Customer Success": ["Customer Health", "Renewal Strategy", "Implementation Planning", "Technical Troubleshooting", "Executive Communication"], + "Marketing": ["Demand Generation", "Product Messaging", "Marketing Analytics", "Executive Communication"], + "People": ["Workforce Planning", "Talent Advising", "Compensation Analysis", "Change Management"], + "Finance": ["Financial Planning", "Forecasting", "SQL", "Executive Communication"], + "Legal": ["Contract Review", "Risk Management", "Executive Communication"], + "Operations": ["Workforce Planning", "Forecasting", "Change Management", "Executive Communication"], + "Corporate": ["Executive Communication", "People Management", "Risk Management", "Forecasting"], +} + +employee_skills_data = [] +skill_counter = 1 +for employee in employees_data: + role = ROLE_BY_ID[employee["role_id"]] + family_skills = FAMILY_SKILL_NAMES.get(role["job_family"], ["Executive Communication"]) + selected_names = list(dict.fromkeys(random.sample(family_skills, min(5, len(family_skills))) + ["Change Management"])) + if role["job_track"] == "Management": + selected_names.append("People Management") + selected_names = list(dict.fromkeys(selected_names))[:7] + + for skill_name in selected_names: + skill = SKILL_BY_ID[SKILL_ID_BY_NAME[skill_name]] + target_level = clamp(role["job_level_num"] - 1 + (1 if skill["scarcity_score"] >= 8 else 0), 2, 5) + if role["job_family"] == "Data & AI" and skill["is_emerging_skill"]: + current_level = random.choices([1, 2, 3, 4], weights=[18, 38, 32, 12])[0] + else: + current_level = random.choices([1, 2, 3, 4, 5], weights=[6, 18, 36, 30, 10])[0] + current_level = int(clamp(current_level + (1 if employee["high_performer_profile"] and random.random() < 0.22 else 0), 1, 5)) + gap = max(0, int(target_level) - current_level) + employee_skills_data.append( + { + "employee_skill_id": f"ESKILL-{skill_counter:08d}", + "employee_id": employee["employee_id"], + "skill_id": skill["skill_id"], + "assessed_date": date(2025, 6, 30), + "current_proficiency_level": int(current_level), + "target_proficiency_level": int(target_level), + "skill_gap": int(gap), + "is_critical_gap": bool(gap >= 2 and skill["scarcity_score"] >= 7), + } + ) + skill_counter += 1 + +df_employee_skills = pd.DataFrame(employee_skills_data) +SKILL_ROWS_BY_EMP = {} +for row in employee_skills_data: + SKILL_ROWS_BY_EMP.setdefault(row["employee_id"], []).append(row) +print(f"employee_skills: {len(df_employee_skills)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 9. Learning Activity + +# COMMAND ---------- + +# ============================================================================= +# TABLE 7: LEARNING_ACTIVITY +# ============================================================================= +learning_activity_data = [] +learning_counter = 1 +PROGRAM_BY_CATEGORY = { + "Data & AI": ["GenAI Builder Academy", "MLOps Practitioner Path", "Data Governance Bootcamp"], + "GTM": ["Enterprise Advisory Selling", "Strategic Renewal Lab", "Solution Discovery Intensive"], + "Product": ["Product Strategy Studio", "Experimentation Lab", "Research Foundations"], + "Leadership": ["Manager Essentials", "Leading Through Change", "Executive Communication Lab"], + "People": ["Talent Advisor Certification", "Workforce Planning Lab", "Compensation Fundamentals"], +} + +for employee in employees_data: + if employee["termination_date"] and employee["termination_date"] < START_DATE: + continue + role = ROLE_BY_ID[employee["role_id"]] + base_activity_count = random.choices([0, 1, 2, 3], weights=[42, 34, 18, 6])[0] + if employee["mobility_program_participant"]: + base_activity_count += random.choices([1, 2], weights=[65, 35])[0] + if role["job_family"] == "Data & AI": + base_activity_count += random.choice([0, 1]) + skill_rows = sorted(SKILL_ROWS_BY_EMP[employee["employee_id"]], key=lambda row: row["skill_gap"], reverse=True) + for activity_idx in range(base_activity_count): + skill_row = skill_rows[min(activity_idx, len(skill_rows) - 1)] + skill = SKILL_BY_ID[skill_row["skill_id"]] + category = skill["skill_category"] + program_names = PROGRAM_BY_CATEGORY.get(category, PROGRAM_BY_CATEGORY.get(role["job_family"], ["Talent Marketplace Accelerator"])) + if employee["mobility_program_participant"] and activity_idx == 0: + program_name = "Talent Marketplace Accelerator" + program_type = "Mobility Program" + activity_date = date_between(date(2025, 1, 15), date(2025, 11, 15)) + else: + program_name = random.choice(program_names) + program_type = random.choice(["Course", "Workshop", "Certification", "Coaching"]) + activity_date = date_between(START_DATE, END_DATE) + if activity_date < employee["hire_date"]: + activity_date = employee["hire_date"] + timedelta(days=random.randint(15, 180)) + if employee["termination_date"] and activity_date > employee["termination_date"]: + continue + completion_status = random.choices(["Completed", "In Progress", "Dropped"], weights=[82, 13, 5])[0] + learning_activity_data.append( + { + "learning_activity_id": f"LRN-{learning_counter:08d}", + "employee_id": employee["employee_id"], + "skill_id": skill["skill_id"], + "activity_date": activity_date, + "activity_year": activity_date.year, + "program_name": program_name, + "program_type": program_type, + "skill_category": category, + "hours_spent": round(random.uniform(2.0, 32.0), 1), + "completion_status": completion_status, + "mobility_program_activity": bool(program_name == "Talent Marketplace Accelerator"), + } + ) + learning_counter += 1 + +df_learning_activity = pd.DataFrame(learning_activity_data) +LEARNING_BY_EMP = {} +for row in learning_activity_data: + LEARNING_BY_EMP.setdefault(row["employee_id"], []).append(row) +print(f"learning_activity: {len(df_learning_activity)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 10. Talent Events + +# COMMAND ---------- + +# ============================================================================= +# TABLE 8: TALENT_EVENTS +# ============================================================================= +talent_events_data = [] +event_counter = 1 + + +def add_talent_event(employee, event_type, event_date, reason, from_org=None, to_org=None, from_role=None, to_role=None): + global event_counter + role = ROLE_BY_ID[employee["role_id"]] + org = ORG_BY_ID[employee["org_unit_id"]] + is_exit = event_type == "Exit" + is_regretted_loss = bool( + is_exit + and employee["termination_type"] == "Voluntary" + and (employee["high_performer_profile"] or role["is_critical_role"]) + ) + talent_events_data.append( + { + "talent_event_id": f"TEVENT-{event_counter:08d}", + "event_date": event_date, + "event_year": event_date.year, + "event_month": event_date.month, + "employee_id": employee["employee_id"], + "event_type": event_type, + "event_reason": reason, + "from_org_unit_id": from_org, + "to_org_unit_id": to_org, + "from_role_id": from_role, + "to_role_id": to_role, + "org_unit_id": employee["org_unit_id"], + "role_id": employee["role_id"], + "job_family": role["job_family"], + "business_unit": org["business_unit"], + "is_internal_move": bool(event_type in ("Promotion", "Transfer")), + "is_regretted_loss": is_regretted_loss, + "mobility_program_related": bool(employee["mobility_program_participant"] and event_date.year == 2025), + } + ) + event_counter += 1 + + +for employee in employees_data: + if START_DATE <= employee["hire_date"] <= END_DATE: + add_talent_event( + employee, + "Hire", + employee["hire_date"], + "Growth hire" if ROLE_BY_ID[employee["role_id"]]["growth_priority"] == "Invest" else "Backfill", + from_org=None, + to_org=employee["org_unit_id"], + from_role=None, + to_role=employee["role_id"], + ) + + move_prob = 0.115 + if employee["mobility_program_participant"]: + move_prob += 0.190 + if employee["high_performer_profile"]: + move_prob += 0.035 + if random.random() < move_prob and employee["hire_date"] < date(2025, 6, 1): + if employee["mobility_program_participant"]: + move_date = date_between(max(employee["hire_date"] + timedelta(days=180), date(2025, 1, 1)), date(2025, 11, 30)) + else: + move_date = date_between(max(employee["hire_date"] + timedelta(days=180), START_DATE), date(2025, 11, 30)) + if employee["termination_date"] and move_date > employee["termination_date"]: + move_date = employee["termination_date"] - timedelta(days=random.randint(30, 120)) + if START_DATE <= move_date <= END_DATE: + role = ROLE_BY_ID[employee["role_id"]] + event_type = random.choices(["Promotion", "Transfer"], weights=[58, 42])[0] + if event_type == "Promotion": + higher_roles = [ + candidate + for candidate in ROLES_BY_FAMILY.get(role["job_family"], []) + if candidate["job_level_num"] == role["job_level_num"] + 1 + ] + to_role = random.choice(higher_roles)["role_id"] if higher_roles else employee["role_id"] + to_org = employee["org_unit_id"] + reason = "Promotion after sustained performance" + else: + possible_orgs = [ + candidate_org + for candidate_org in org_units_data + if candidate_org["org_unit_id"] != employee["org_unit_id"] + and candidate_org["function"] in (ORG_BY_ID[employee["org_unit_id"]]["function"], "Engineering", "Product", "Sales") + ] + to_org = random.choice(possible_orgs)["org_unit_id"] if possible_orgs else employee["org_unit_id"] + to_role = employee["role_id"] + reason = "Internal mobility transfer" + add_talent_event( + employee, + event_type, + move_date, + reason, + from_org=employee["org_unit_id"], + to_org=to_org, + from_role=employee["role_id"], + to_role=to_role, + ) + + if employee["termination_date"] and START_DATE <= employee["termination_date"] <= END_DATE: + add_talent_event( + employee, + "Exit", + employee["termination_date"], + employee["termination_type"], + from_org=employee["org_unit_id"], + to_org=None, + from_role=employee["role_id"], + to_role=None, + ) + +df_talent_events = pd.DataFrame(talent_events_data) +print(f"talent_events: {len(df_talent_events)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 11. Performance Reviews + +# COMMAND ---------- + +# ============================================================================= +# TABLE 9: PERFORMANCE_REVIEWS +# ============================================================================= +performance_reviews_data = [] +review_counter = 1 +PERFORMANCE_BY_EMP_YEAR = {} + +for employee in employees_data: + for review_year in YEARS: + period_start = date(review_year, 1, 1) + period_end = date(review_year, 12, 31) + if not active_during(employee, period_start, period_end): + continue + base_rating = { + "Developing": 2.5, + "Solid": 3.35, + "Strong": 4.05, + "Exceptional": 4.65, + }[employee["performance_profile"]] + rating = clamp(base_rating + random.gauss(0, 0.28), 1.0, 5.0) + if employee["mobility_program_participant"] and review_year == 2025: + rating = clamp(rating + 0.10, 1.0, 5.0) + potential = random.choices([1, 2, 3], weights=[25, 52, 23])[0] + if employee["high_performer_profile"]: + potential = random.choices([2, 3], weights=[38, 62])[0] + goal_attainment = clamp(72 + rating * 8 + random.gauss(0, 8), 45, 145) + row = { + "review_id": f"REV-{review_counter:08d}", + "employee_id": employee["employee_id"], + "review_year": review_year, + "review_date": date(review_year, 12, 15), + "org_unit_id": employee["org_unit_id"], + "role_id": employee["role_id"], + "performance_rating": round(rating, 2), + "performance_bucket": "High" if rating >= 4.1 else "Solid" if rating >= 3.0 else "Developing", + "potential_rating": int(potential), + "goal_attainment_pct": round(goal_attainment, 1), + "is_high_performer": bool(rating >= 4.1), + } + performance_reviews_data.append(row) + PERFORMANCE_BY_EMP_YEAR[(employee["employee_id"], review_year)] = row + review_counter += 1 + +df_performance_reviews = pd.DataFrame(performance_reviews_data) +print(f"performance_reviews: {len(df_performance_reviews)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 12. Compensation Snapshots + +# COMMAND ---------- + +# ============================================================================= +# TABLE 10: COMPENSATION_SNAPSHOTS +# ============================================================================= +compensation_snapshots_data = [] +comp_counter = 1 +COMP_BY_EMP_YEAR = {} + +for employee in employees_data: + role = ROLE_BY_ID[employee["role_id"]] + location = LOCATION_BY_ID[employee["location_id"]] + market_midpoint = role["base_salary_midpoint_usd"] * (1 + role["market_premium_pct"] / 100.0) * location["cost_index"] + for comp_year in YEARS: + snapshot_date = date(comp_year, 12, 31) + if not active_during(employee, date(comp_year, 1, 1), snapshot_date): + continue + merit_progression = 1.0 + 0.018 * (comp_year - 2023) + if employee["high_performer_profile"]: + merit_progression += 0.012 * (comp_year - 2023) + compa_ratio = clamp(employee["compa_ratio_profile"] * merit_progression + random.gauss(0, 0.015), 0.70, 1.25) + salary = market_midpoint * compa_ratio + bonus_target = 0.10 + 0.025 * max(0, role["job_level_num"] - 3) + if role["job_family"] == "Sales": + bonus_target += 0.10 + review = PERFORMANCE_BY_EMP_YEAR.get((employee["employee_id"], comp_year), {}) + perf_rating = review.get("performance_rating", 3.3) + bonus_actual = salary * bonus_target * clamp(0.72 + perf_rating / 5.0 * 0.56 + random.gauss(0, 0.08), 0.35, 1.55) + row = { + "compensation_snapshot_id": f"COMP-{comp_counter:08d}", + "employee_id": employee["employee_id"], + "snapshot_year": comp_year, + "snapshot_date": snapshot_date, + "org_unit_id": employee["org_unit_id"], + "role_id": employee["role_id"], + "job_family": role["job_family"], + "job_level": role["job_level"], + "base_salary_usd": round(salary, 2), + "bonus_target_pct": round(bonus_target * 100.0, 1), + "bonus_actual_usd": round(bonus_actual, 2), + "market_midpoint_usd": round(market_midpoint, 2), + "salary_range_min_usd": round(market_midpoint * 0.80, 2), + "salary_range_max_usd": round(market_midpoint * 1.20, 2), + "compa_ratio": round(compa_ratio, 3), + "range_penetration_pct": round(clamp((salary - market_midpoint * 0.80) / (market_midpoint * 0.40), 0, 1.25) * 100.0, 1), + "pay_band": f"{role['job_level']} {role['job_family']}", + } + compensation_snapshots_data.append(row) + COMP_BY_EMP_YEAR[(employee["employee_id"], comp_year)] = row + comp_counter += 1 + +df_compensation_snapshots = pd.DataFrame(compensation_snapshots_data) +print(f"compensation_snapshots: {len(df_compensation_snapshots)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 13. Engagement Pulses and Retention Risk + +# COMMAND ---------- + +# ============================================================================= +# TABLE 11: ENGAGEMENT_PULSES +# TABLE 12: RETENTION_RISK_SNAPSHOTS +# ============================================================================= +engagement_pulses_data = [] +retention_risk_data = [] +pulse_counter = 1 +risk_counter = 1 + +for employee in employees_data: + role = ROLE_BY_ID[employee["role_id"]] + org = ORG_BY_ID[employee["org_unit_id"]] + base_engagement = random.gauss(3.75, 0.42) + if employee["high_performer_profile"]: + base_engagement += 0.12 + if org["org_unit_name"] == "Sales East": + base_engagement -= 0.08 + for q_end in QUARTER_ENDS: + if not is_employed_on(employee, q_end): + continue + year = q_end.year + quarter = ((q_end.month - 1) // 3) + 1 + engagement = base_engagement + random.gauss(0, 0.22) + manager_effectiveness = random.gauss(3.82, 0.38) + workload = random.gauss(3.18, 0.45) + if org["org_unit_name"] == "Sales East" and year == 2024: + engagement -= 0.72 + manager_effectiveness -= 0.38 + workload += 0.62 + if employee["mobility_program_participant"] and year == 2025: + engagement += 0.28 + manager_effectiveness += 0.10 + workload -= 0.12 + if role["job_family"] == "Data & AI": + workload += 0.18 + engagement = round(clamp(engagement, 1.0, 5.0), 2) + manager_effectiveness = round(clamp(manager_effectiveness, 1.0, 5.0), 2) + workload = round(clamp(workload, 1.0, 5.0), 2) + burnout_risk = round(clamp(1.2 + workload * 0.72 + random.gauss(0, 0.28), 1.0, 5.0), 2) + row = { + "engagement_pulse_id": f"PULSE-{pulse_counter:08d}", + "pulse_date": q_end, + "pulse_year": year, + "pulse_quarter": quarter, + "pulse_quarter_label": quarter_label(q_end), + "employee_id": employee["employee_id"], + "org_unit_id": employee["org_unit_id"], + "role_id": employee["role_id"], + "job_family": role["job_family"], + "engagement_score": engagement, + "manager_effectiveness_score": manager_effectiveness, + "workload_score": workload, + "growth_opportunity_score": round(clamp(random.gauss(3.55, 0.48) + (0.25 if employee["mobility_program_participant"] and year == 2025 else 0), 1.0, 5.0), 2), + "burnout_risk_score": burnout_risk, + } + engagement_pulses_data.append(row) + pulse_counter += 1 + + comp = COMP_BY_EMP_YEAR.get((employee["employee_id"], year), {}) + review = PERFORMANCE_BY_EMP_YEAR.get((employee["employee_id"], year), {}) + compa_ratio = comp.get("compa_ratio", employee["compa_ratio_profile"]) + is_high_performer = bool(review.get("is_high_performer", employee["high_performer_profile"])) + risk_score = 22 + (5.0 - engagement) * 9.5 + burnout_risk * 5.0 + risk_driver = "Engagement" + if is_high_performer and compa_ratio < 0.90: + risk_score += 23 + risk_driver = "Below-market compensation for high performer" + if org["org_unit_name"] == "Sales East" and year in (2024, 2025): + risk_score += 14 + risk_driver = "Sales East engagement dip" + if role["job_family"] == "Data & AI" and role["scarcity_score"] >= 8: + risk_score += 7 + risk_driver = "Scarce external market" + if employee["mobility_program_participant"] and year == 2025: + risk_score -= 16 + if risk_driver == "Engagement": + risk_driver = "Mobility program mitigated risk" + learning_rows = [ + learning + for learning in LEARNING_BY_EMP.get(employee["employee_id"], []) + if learning["activity_year"] == year and learning["completion_status"] == "Completed" + ] + if learning_rows: + risk_score -= 4 + risk_score = round(clamp(risk_score + random.gauss(0, 4), 1, 99), 1) + risk_band = "High" if risk_score >= 70 else "Medium" if risk_score >= 45 else "Low" + retention_risk_data.append( + { + "retention_risk_snapshot_id": f"RISK-{risk_counter:08d}", + "snapshot_date": q_end, + "snapshot_year": year, + "snapshot_quarter": quarter, + "snapshot_quarter_label": quarter_label(q_end), + "employee_id": employee["employee_id"], + "org_unit_id": employee["org_unit_id"], + "role_id": employee["role_id"], + "job_family": role["job_family"], + "risk_score": risk_score, + "risk_band": risk_band, + "risk_driver": risk_driver, + "compa_ratio": round(compa_ratio, 3), + "is_high_performer": is_high_performer, + "mobility_program_participant": bool(employee["mobility_program_participant"]), + "recommended_action": "Retention conversation" if risk_band == "High" else "Monitor" if risk_band == "Medium" else "No immediate action", + } + ) + risk_counter += 1 + +df_engagement_pulses = pd.DataFrame(engagement_pulses_data) +df_retention_risk_snapshots = pd.DataFrame(retention_risk_data) +print(f"engagement_pulses: {len(df_engagement_pulses)} rows") +print(f"retention_risk_snapshots: {len(df_retention_risk_snapshots)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 14. Requisitions + +# COMMAND ---------- + +# ============================================================================= +# TABLE 13: REQUISITIONS +# ============================================================================= +requisitions_data = [] +req_counter = 1 + + +def choose_org_for_family(job_family): + candidates = [] + weights = [] + for org in org_units_data: + families = ORG_ROLE_FAMILIES[org["function"]] + if org["org_unit_name"] == "Data & AI Platform": + families = ["Data & AI", "Engineering"] + if org["org_unit_name"] == "Solutions Consulting": + families = ["Sales", "Data & AI"] + if job_family in families: + candidates.append(org) + weights.append(org["target_headcount_weight"]) + return random.choices(candidates, weights=weights)[0] + + +for year, req_count in [(2023, 170), (2024, 210), (2025, 260)]: + family_weights = { + "Engineering": 20, + "Data & AI": 8 + (year - 2023) * 8, + "Product": 12 + (year - 2023) * 2, + "Sales": 24, + "Customer Success": 18, + "Marketing": 7, + "People": 4, + "Finance": 3, + "Legal": 2, + "Operations": 4, + } + families = list(family_weights.keys()) + weights = list(family_weights.values()) + for _ in range(req_count): + job_family = random.choices(families, weights=weights)[0] + role = random.choice(ROLES_BY_FAMILY[job_family]) + org = choose_org_for_family(job_family) + location_id = choose_location(org["function"]) + open_date = date_between(date(year, 1, 1), date(year, 12, 10)) + status = random.choices(["Filled", "Open", "Cancelled"], weights=[78, 15, 7])[0] + base_ttf = random.randint(28, 72) + if job_family == "Data & AI": + base_ttf += random.randint(20, 38) + if job_family == "Product": + base_ttf += random.randint(8, 24) + if role["is_critical_role"]: + base_ttf += random.randint(8, 22) + close_date = None + time_to_fill_days = None + if status == "Filled": + time_to_fill_days = int(base_ttf) + close_date = min(END_DATE, open_date + timedelta(days=time_to_fill_days)) + elif status == "Cancelled": + close_date = min(END_DATE, open_date + timedelta(days=random.randint(20, 100))) + internal_fill_prob = 0.16 if year < 2025 else 0.34 + if job_family in ("Product", "Data & AI"): + internal_fill_prob += 0.06 + is_internal_fill = bool(status == "Filled" and random.random() < internal_fill_prob) + eligible_internal = [ + emp + for emp in ACTIVE_EMPLOYEES + if emp["job_family"] == job_family + and emp["employee_id"] not in (None, "") + and (emp["mobility_program_participant"] or random.random() < 0.18) + ] + filled_by_employee_id = random.choice(eligible_internal)["employee_id"] if is_internal_fill and eligible_internal else None + hiring_managers = [ + emp + for emp in ACTIVE_EMPLOYEES + if emp["org_unit_id"] == org["org_unit_id"] and ROLE_BY_ID[emp["role_id"]]["job_track"] == "Management" + ] + requisitions_data.append( + { + "requisition_id": f"REQ-{req_counter:06d}", + "opened_date": open_date, + "opened_year": year, + "opened_month": open_date.month, + "closed_date": close_date, + "status": status, + "org_unit_id": org["org_unit_id"], + "location_id": location_id, + "role_id": role["role_id"], + "job_family": job_family, + "hiring_manager_employee_id": random.choice(hiring_managers)["employee_id"] if hiring_managers else None, + "requisition_reason": "Growth" if job_family == "Data & AI" or random.random() < 0.58 else "Backfill", + "priority": "Critical" if role["is_critical_role"] else random.choices(["High", "Standard"], weights=[35, 65])[0], + "approved_headcount": 1, + "time_to_fill_days": time_to_fill_days, + "is_internal_fill": is_internal_fill, + "filled_by_employee_id": filled_by_employee_id, + "candidate_market": "Scarce" if role["scarcity_score"] >= 8 else "Competitive" if role["scarcity_score"] >= 6 else "Balanced", + } + ) + req_counter += 1 + +df_requisitions = pd.DataFrame(requisitions_data) +REQ_BY_ID = {row["requisition_id"]: row for row in requisitions_data} +print(f"requisitions: {len(df_requisitions)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 15. Applications + +# COMMAND ---------- + +# ============================================================================= +# TABLE 14: APPLICATIONS +# ============================================================================= +applications_data = [] +app_counter = 1 + +for req in requisitions_data: + role = ROLE_BY_ID[req["role_id"]] + app_count = random.randint(9, 24) + if role["job_family"] == "Data & AI": + app_count = random.randint(5, 15) + if req["candidate_market"] == "Scarce": + app_count = max(4, app_count - random.randint(2, 6)) + accepted_created = False + for app_idx in range(app_count): + source = random.choices( + ["Inbound", "Referral", "Sourcing", "Agency", "Internal"], + weights=[34, 22, 24, 8, 12 if req["opened_year"] < 2025 else 22], + )[0] + candidate_employee_id = None + if source == "Internal": + internal_candidates = [ + emp + for emp in ACTIVE_EMPLOYEES + if emp["job_family"] == role["job_family"] and emp["employee_id"] != req.get("filled_by_employee_id") + ] + if internal_candidates: + candidate_employee_id = random.choice(internal_candidates)["employee_id"] + stage = random.choices( + ["Applied", "Recruiter Screen", "Interview", "Final Interview", "Offer", "Rejected"], + weights=[38, 20, 22, 8, 5, 7], + )[0] + offer_salary = None + offer_date = None + offer_accepted = None + current_status = "In Process" if req["status"] == "Open" and stage != "Rejected" else "Rejected" + interview_count = 0 + decline_reason = None + if stage in ("Interview", "Final Interview", "Offer"): + interview_count = random.randint(1, 4) + if stage == "Offer": + offer_date = req["opened_date"] + timedelta(days=random.randint(18, max(24, req["time_to_fill_days"] or 90))) + market_midpoint = role["base_salary_midpoint_usd"] * (1 + role["market_premium_pct"] / 100.0) + offer_salary = market_midpoint * random.uniform(0.86, 1.10) + below_midpoint = offer_salary < market_midpoint + accept_prob = 0.58 + if role["job_family"] == "Product" and below_midpoint: + accept_prob = 0.27 + elif below_midpoint: + accept_prob = 0.43 + if source == "Internal": + accept_prob += 0.16 + offer_accepted = bool(req["status"] == "Filled" and random.random() < accept_prob) + current_status = "Offer Accepted" if offer_accepted else "Offer Declined" + if offer_accepted: + accepted_created = True + elif below_midpoint: + decline_reason = "Compensation below market midpoint" + else: + decline_reason = random.choice(["Accepted competing offer", "Role scope mismatch", "Timing"]) + applications_data.append( + { + "application_id": f"APP-{app_counter:08d}", + "requisition_id": req["requisition_id"], + "application_date": max(req["opened_date"], req["opened_date"] + timedelta(days=random.randint(0, 45))), + "candidate_source": source, + "candidate_employee_id": candidate_employee_id, + "candidate_stage": stage, + "current_status": current_status, + "interview_count": int(interview_count), + "offer_date": offer_date, + "offer_salary_usd": None if offer_salary is None else round(offer_salary, 2), + "offer_to_market_midpoint_pct": None if offer_salary is None else round(offer_salary * 100.0 / (role["base_salary_midpoint_usd"] * (1 + role["market_premium_pct"] / 100.0)), 1), + "offer_accepted": offer_accepted, + "decline_reason": decline_reason, + "candidate_quality_score": round(clamp(random.gauss(72, 13), 25, 99), 1), + } + ) + app_counter += 1 + + if req["status"] == "Filled" and not accepted_created: + source = "Internal" if req["is_internal_fill"] else random.choice(["Referral", "Sourcing", "Inbound"]) + role_mid = role["base_salary_midpoint_usd"] * (1 + role["market_premium_pct"] / 100.0) + offer_salary = role_mid * random.uniform(0.95, 1.12) + applications_data.append( + { + "application_id": f"APP-{app_counter:08d}", + "requisition_id": req["requisition_id"], + "application_date": req["opened_date"] + timedelta(days=random.randint(2, 30)), + "candidate_source": source, + "candidate_employee_id": req["filled_by_employee_id"] if source == "Internal" else None, + "candidate_stage": "Offer", + "current_status": "Offer Accepted", + "interview_count": random.randint(2, 5), + "offer_date": req["closed_date"] or req["opened_date"] + timedelta(days=req["time_to_fill_days"] or 60), + "offer_salary_usd": round(offer_salary, 2), + "offer_to_market_midpoint_pct": round(offer_salary * 100.0 / role_mid, 1), + "offer_accepted": True, + "decline_reason": None, + "candidate_quality_score": round(clamp(random.gauss(84, 7), 55, 99), 1), + } + ) + app_counter += 1 + +df_applications = pd.DataFrame(applications_data) +print(f"applications: {len(df_applications)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 16. Succession Plans + +# COMMAND ---------- + +# ============================================================================= +# TABLE 15: SUCCESSION_PLANS +# ============================================================================= +succession_plans_data = [] +succession_counter = 1 + +critical_incumbents = [ + employee + for employee in ACTIVE_EMPLOYEES + if ROLE_BY_ID[employee["role_id"]]["is_critical_role"] and employee["job_level_num"] >= 4 +] +random.shuffle(critical_incumbents) + +for employee in critical_incumbents[:520]: + role = ROLE_BY_ID[employee["role_id"]] + org = ORG_BY_ID[employee["org_unit_id"]] + weak_coverage_org = org["business_unit"] == "Go To Market" or org["org_unit_name"] == "Data & AI Platform" + coverage_prob = 0.54 if weak_coverage_org else 0.78 + has_successor = random.random() < coverage_prob + successor_rows = [] + if has_successor: + successor_pool = [ + candidate + for candidate in ACTIVE_EMPLOYEES + if candidate["employee_id"] != employee["employee_id"] + and candidate["job_family"] == role["job_family"] + and candidate["job_level_num"] >= max(3, role["job_level_num"] - 2) + and candidate["job_level_num"] <= role["job_level_num"] + ] + successor_count = random.choices([1, 2], weights=[72, 28])[0] + successor_rows = random.sample(successor_pool, min(successor_count, len(successor_pool))) if successor_pool else [] + if not successor_rows: + succession_plans_data.append( + { + "succession_plan_id": f"SUCC-{succession_counter:07d}", + "plan_review_date": date(2025, 9, 30), + "org_unit_id": employee["org_unit_id"], + "role_id": employee["role_id"], + "incumbent_employee_id": employee["employee_id"], + "successor_employee_id": None, + "criticality_tier": "Tier 1" if role["scarcity_score"] >= 8 else "Tier 2", + "readiness_level": "No Named Successor", + "readiness_months": None, + "coverage_status": "No Ready Successor", + "successor_retention_risk_band": None, + "weak_coverage_flag": True, + } + ) + succession_counter += 1 + else: + for successor in successor_rows: + readiness = random.choices( + ["Ready Now", "Ready <12 Months", "Ready 12-24 Months", "Ready 24+ Months"], + weights=[18, 34, 34, 14] if not weak_coverage_org else [8, 24, 44, 24], + )[0] + readiness_months = { + "Ready Now": 0, + "Ready <12 Months": random.randint(3, 11), + "Ready 12-24 Months": random.randint(12, 24), + "Ready 24+ Months": random.randint(25, 36), + }[readiness] + succession_plans_data.append( + { + "succession_plan_id": f"SUCC-{succession_counter:07d}", + "plan_review_date": date(2025, 9, 30), + "org_unit_id": employee["org_unit_id"], + "role_id": employee["role_id"], + "incumbent_employee_id": employee["employee_id"], + "successor_employee_id": successor["employee_id"], + "criticality_tier": "Tier 1" if role["scarcity_score"] >= 8 else "Tier 2", + "readiness_level": readiness, + "readiness_months": readiness_months, + "coverage_status": "Covered" if readiness in ("Ready Now", "Ready <12 Months") else "Developing", + "successor_retention_risk_band": random.choices(["Low", "Medium", "High"], weights=[58, 32, 10])[0], + "weak_coverage_flag": bool(weak_coverage_org and readiness not in ("Ready Now", "Ready <12 Months")), + } + ) + succession_counter += 1 + +df_succession_plans = pd.DataFrame(succession_plans_data) +print(f"succession_plans: {len(df_succession_plans)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 17. Curated Topic Marts + +# COMMAND ---------- + +# ============================================================================= +# CURATED TOPIC MARTS +# ============================================================================= +df_emp = df_employees.copy() +df_emp["hire_month"] = df_emp["hire_date"].apply(lambda d: date(d.year, d.month, 1)) +df_emp["termination_month"] = df_emp["termination_date"].apply( + lambda d: None if pd.isna(d) or d is None else date(d.year, d.month, 1) +) + +# Workforce planning mart: monthly by org, job family, and location. +workforce_rows = [] +workforce_counter = 1 +employee_combos = df_emp[["org_unit_id", "job_family", "location_id"]].drop_duplicates().to_dict("records") +for year, month in MONTH_LIST: + month_start = date(year, month, 1) + month_end = end_of_month(year, month) + for combo in employee_combos: + org_id = combo["org_unit_id"] + job_family = combo["job_family"] + location_id = combo["location_id"] + emp_mask = ( + (df_emp["org_unit_id"] == org_id) + & (df_emp["job_family"] == job_family) + & (df_emp["location_id"] == location_id) + ) + active_mask = emp_mask & (df_emp["hire_date"] <= month_end) & ( + df_emp["termination_date"].isna() | (df_emp["termination_date"] > month_end) + ) + start_headcount_mask = emp_mask & (df_emp["hire_date"] < month_start) & ( + df_emp["termination_date"].isna() | (df_emp["termination_date"] >= month_start) + ) + hires = int((emp_mask & (df_emp["hire_month"] == month_start)).sum()) + exits = int((emp_mask & (df_emp["termination_month"] == month_start)).sum()) + active_headcount = int(active_mask.sum()) + start_headcount = int(start_headcount_mask.sum()) + req_mask = ( + (df_requisitions["org_unit_id"] == org_id) + & (df_requisitions["job_family"] == job_family) + & (df_requisitions["location_id"] == location_id) + ) + open_reqs = int( + ( + req_mask + & (df_requisitions["opened_date"] <= month_end) + & ( + df_requisitions["closed_date"].isna() + | (df_requisitions["closed_date"] > month_end) + ) + & (df_requisitions["status"] == "Open") + ).sum() + ) + filled_reqs = int( + ( + req_mask + & (df_requisitions["status"] == "Filled") + & df_requisitions["closed_date"].notna() + & (df_requisitions["closed_date"] >= month_start) + & (df_requisitions["closed_date"] <= month_end) + ).sum() + ) + if active_headcount + hires + exits + open_reqs + filled_reqs == 0: + continue + org = ORG_BY_ID[org_id] + workforce_rows.append( + { + "mart_workforce_planning_id": f"MWP-{workforce_counter:08d}", + "snapshot_month": month_start, + "snapshot_year": year, + "snapshot_month_num": month, + "org_unit_id": org_id, + "org_unit_name": org["org_unit_name"], + "business_unit": org["business_unit"], + "function": org["function"], + "job_family": job_family, + "location_id": location_id, + "region": LOCATION_BY_ID[location_id]["region"], + "active_headcount": active_headcount, + "start_headcount": start_headcount, + "hires": hires, + "exits": exits, + "open_requisitions": open_reqs, + "filled_requisitions": filled_reqs, + "attrition_rate_pct": round(exits * 100.0 / start_headcount, 2) if start_headcount else 0.0, + "vacancy_rate_pct": round(open_reqs * 100.0 / (active_headcount + open_reqs), 2) if active_headcount + open_reqs else 0.0, + } + ) + workforce_counter += 1 + +df_mart_workforce_planning = pd.DataFrame(workforce_rows) + +# Hiring funnel mart: one row per requisition. +app_group = df_applications.groupby("requisition_id") +hiring_rows = [] +for req in requisitions_data: + role = ROLE_BY_ID[req["role_id"]] + org = ORG_BY_ID[req["org_unit_id"]] + apps = app_group.get_group(req["requisition_id"]) if req["requisition_id"] in app_group.groups else pd.DataFrame() + offers = apps[apps["candidate_stage"] == "Offer"] if len(apps) else pd.DataFrame() + accepted = apps[apps["current_status"] == "Offer Accepted"] if len(apps) else pd.DataFrame() + declined = apps[apps["current_status"] == "Offer Declined"] if len(apps) else pd.DataFrame() + hiring_rows.append( + { + "mart_hiring_funnel_id": f"MHF-{len(hiring_rows) + 1:08d}", + "requisition_id": req["requisition_id"], + "opened_date": req["opened_date"], + "opened_year": req["opened_year"], + "status": req["status"], + "org_unit_id": req["org_unit_id"], + "org_unit_name": org["org_unit_name"], + "business_unit": org["business_unit"], + "role_id": req["role_id"], + "role_title": role["role_title"], + "job_family": role["job_family"], + "priority": req["priority"], + "candidate_market": req["candidate_market"], + "requisition_reason": req["requisition_reason"], + "applications_count": int(len(apps)), + "internal_applications": int((apps["candidate_source"] == "Internal").sum()) if len(apps) else 0, + "interview_count": int((apps["interview_count"] > 0).sum()) if len(apps) else 0, + "offer_count": int(len(offers)), + "accepted_offer_count": int(len(accepted)), + "declined_offer_count": int(len(declined)), + "acceptance_rate_pct": round(len(accepted) * 100.0 / len(offers), 2) if len(offers) else 0.0, + "avg_offer_to_market_midpoint_pct": round(float(offers["offer_to_market_midpoint_pct"].mean()), 2) if len(offers) else None, + "time_to_fill_days": req["time_to_fill_days"], + "is_internal_fill": bool(req["is_internal_fill"]), + } + ) + +df_mart_hiring_funnel = pd.DataFrame(hiring_rows) + +# Retention and engagement mart: quarterly by org and job family. +risk_enriched = df_retention_risk_snapshots.merge( + df_org_units[["org_unit_id", "org_unit_name", "business_unit", "function"]], + on="org_unit_id", + how="left", +) +retention_rows = [] +for keys, group in risk_enriched.groupby(["snapshot_year", "snapshot_quarter", "snapshot_quarter_label", "org_unit_id", "org_unit_name", "business_unit", "function", "job_family"]): + snapshot_year, snapshot_quarter, snapshot_quarter_label, org_id, org_name, business_unit, function, job_family = keys + exits = df_talent_events[ + (df_talent_events["event_type"] == "Exit") + & (df_talent_events["event_year"] == snapshot_year) + & (((df_talent_events["event_month"] - 1) // 3 + 1) == snapshot_quarter) + & (df_talent_events["org_unit_id"] == org_id) + & (df_talent_events["job_family"] == job_family) + ] + pulses = df_engagement_pulses[ + (df_engagement_pulses["pulse_year"] == snapshot_year) + & (df_engagement_pulses["pulse_quarter"] == snapshot_quarter) + & (df_engagement_pulses["org_unit_id"] == org_id) + & (df_engagement_pulses["job_family"] == job_family) + ] + retention_rows.append( + { + "mart_retention_engagement_id": f"MRE-{len(retention_rows) + 1:08d}", + "snapshot_year": int(snapshot_year), + "snapshot_quarter": int(snapshot_quarter), + "snapshot_quarter_label": snapshot_quarter_label, + "org_unit_id": org_id, + "org_unit_name": org_name, + "business_unit": business_unit, + "function": function, + "job_family": job_family, + "employee_count": int(group["employee_id"].nunique()), + "avg_engagement_score": round(float(pulses["engagement_score"].mean()), 2) if len(pulses) else None, + "avg_manager_effectiveness_score": round(float(pulses["manager_effectiveness_score"].mean()), 2) if len(pulses) else None, + "avg_burnout_risk_score": round(float(pulses["burnout_risk_score"].mean()), 2) if len(pulses) else None, + "avg_retention_risk_score": round(float(group["risk_score"].mean()), 2), + "high_risk_employee_count": int((group["risk_band"] == "High").sum()), + "regretted_exit_count": int(exits["is_regretted_loss"].sum()) if len(exits) else 0, + "total_exit_count": int(len(exits)), + "mobility_program_participants": int(group[group["mobility_program_participant"]]["employee_id"].nunique()), + } + ) + +df_mart_retention_engagement = pd.DataFrame(retention_rows) + +# Internal mobility mart: annual by org and job family. +mobility_rows = [] +skill_df = df_employee_skills.merge(df_employees[["employee_id", "org_unit_id", "job_family", "mobility_program_participant"]], on="employee_id") +learning_df = df_learning_activity.merge(df_employees[["employee_id", "org_unit_id", "job_family"]], on="employee_id") +for year in YEARS: + active_year = df_emp[ + (df_emp["hire_date"] <= date(year, 12, 31)) + & (df_emp["termination_date"].isna() | (df_emp["termination_date"] >= date(year, 1, 1))) + ] + for keys, group in active_year.groupby(["org_unit_id", "job_family"]): + org_id, job_family = keys + org = ORG_BY_ID[org_id] + events = df_talent_events[ + (df_talent_events["event_year"] == year) + & (df_talent_events["org_unit_id"] == org_id) + & (df_talent_events["job_family"] == job_family) + ] + learning = learning_df[ + (learning_df["activity_year"] == year) + & (learning_df["org_unit_id"] == org_id) + & (learning_df["job_family"] == job_family) + ] + skills = skill_df[ + (skill_df["org_unit_id"] == org_id) + & (skill_df["job_family"] == job_family) + ] + reqs = df_requisitions[ + (df_requisitions["opened_year"] == year) + & (df_requisitions["org_unit_id"] == org_id) + & (df_requisitions["job_family"] == job_family) + ] + mobility_rows.append( + { + "mart_internal_mobility_id": f"MIM-{len(mobility_rows) + 1:08d}", + "snapshot_year": year, + "org_unit_id": org_id, + "org_unit_name": org["org_unit_name"], + "business_unit": org["business_unit"], + "function": org["function"], + "job_family": job_family, + "employee_count": int(group["employee_id"].nunique()), + "mobility_program_participants": int(group[group["mobility_program_participant"]]["employee_id"].nunique()), + "promotion_count": int((events["event_type"] == "Promotion").sum()) if len(events) else 0, + "transfer_count": int((events["event_type"] == "Transfer").sum()) if len(events) else 0, + "internal_fill_count": int(reqs["is_internal_fill"].sum()) if len(reqs) else 0, + "learning_completion_count": int((learning["completion_status"] == "Completed").sum()) if len(learning) else 0, + "avg_skill_gap": round(float(skills["skill_gap"].mean()), 2) if len(skills) else None, + "critical_skill_gap_count": int(skills["is_critical_gap"].sum()) if len(skills) else 0, + } + ) + +df_mart_internal_mobility = pd.DataFrame(mobility_rows) + +# Compensation and performance mart: annual by org, family, and level. +comp_perf_df = df_compensation_snapshots.merge( + df_performance_reviews[ + ["employee_id", "review_year", "performance_rating", "is_high_performer", "potential_rating"] + ], + left_on=["employee_id", "snapshot_year"], + right_on=["employee_id", "review_year"], + how="left", +) +comp_perf_df = comp_perf_df.merge( + df_org_units[["org_unit_id", "org_unit_name", "business_unit", "function"]], + on="org_unit_id", + how="left", +) +latest_high_risk = ( + df_retention_risk_snapshots.groupby(["employee_id", "snapshot_year"])["risk_score"] + .max() + .reset_index() + .rename(columns={"risk_score": "max_retention_risk_score"}) +) +comp_perf_df = comp_perf_df.merge( + latest_high_risk, + left_on=["employee_id", "snapshot_year"], + right_on=["employee_id", "snapshot_year"], + how="left", +) +comp_rows = [] +for keys, group in comp_perf_df.groupby(["snapshot_year", "org_unit_id", "org_unit_name", "business_unit", "function", "job_family", "job_level"]): + snapshot_year, org_id, org_name, business_unit, function, job_family, job_level = keys + high_perf_low_comp = group[(group["is_high_performer"] == True) & (group["compa_ratio"] < 0.90)] + comp_rows.append( + { + "mart_comp_performance_id": f"MCP-{len(comp_rows) + 1:08d}", + "snapshot_year": int(snapshot_year), + "org_unit_id": org_id, + "org_unit_name": org_name, + "business_unit": business_unit, + "function": function, + "job_family": job_family, + "job_level": job_level, + "employee_count": int(group["employee_id"].nunique()), + "avg_base_salary_usd": round(float(group["base_salary_usd"].mean()), 2), + "avg_bonus_actual_usd": round(float(group["bonus_actual_usd"].mean()), 2), + "avg_compa_ratio": round(float(group["compa_ratio"].mean()), 3), + "avg_range_penetration_pct": round(float(group["range_penetration_pct"].mean()), 1), + "avg_performance_rating": round(float(group["performance_rating"].mean()), 2), + "high_performer_count": int((group["is_high_performer"] == True).sum()), + "high_performer_below_90_compa_count": int(len(high_perf_low_comp)), + "high_risk_employee_count": int((group["max_retention_risk_score"] >= 70).sum()), + } + ) + +df_mart_comp_performance = pd.DataFrame(comp_rows) + +# Succession planning mart: by org, family, and criticality. +succession_df = df_succession_plans.merge(df_job_roles[["role_id", "job_family", "role_title"]], on="role_id", how="left") +succession_df = succession_df.merge(df_org_units[["org_unit_id", "org_unit_name", "business_unit", "function"]], on="org_unit_id", how="left") +succession_rows = [] +for keys, group in succession_df.groupby(["org_unit_id", "org_unit_name", "business_unit", "function", "job_family", "criticality_tier"]): + org_id, org_name, business_unit, function, job_family, criticality_tier = keys + critical_positions = int(group["incumbent_employee_id"].nunique()) + ready_successors = int(group[group["readiness_level"].isin(["Ready Now", "Ready <12 Months"])]["successor_employee_id"].nunique()) + covered_positions = int(group[group["coverage_status"] == "Covered"]["incumbent_employee_id"].nunique()) + succession_rows.append( + { + "mart_succession_planning_id": f"MSP-{len(succession_rows) + 1:08d}", + "plan_review_date": date(2025, 9, 30), + "org_unit_id": org_id, + "org_unit_name": org_name, + "business_unit": business_unit, + "function": function, + "job_family": job_family, + "criticality_tier": criticality_tier, + "critical_position_count": critical_positions, + "ready_successor_count": ready_successors, + "covered_position_count": covered_positions, + "weak_coverage_position_count": int(group[group["weak_coverage_flag"]]["incumbent_employee_id"].nunique()), + "coverage_rate_pct": round(covered_positions * 100.0 / critical_positions, 2) if critical_positions else 0.0, + "avg_readiness_months": round(float(group["readiness_months"].dropna().mean()), 1) if group["readiness_months"].notna().any() else None, + } + ) + +df_mart_succession_planning = pd.DataFrame(succession_rows) + +print(f"mart_workforce_planning: {len(df_mart_workforce_planning)} rows") +print(f"mart_hiring_funnel: {len(df_mart_hiring_funnel)} rows") +print(f"mart_retention_engagement: {len(df_mart_retention_engagement)} rows") +print(f"mart_internal_mobility: {len(df_mart_internal_mobility)} rows") +print(f"mart_comp_performance: {len(df_mart_comp_performance)} rows") +print(f"mart_succession_planning: {len(df_mart_succession_planning)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 18. Create Schema & Write Delta Tables + +# COMMAND ---------- + +# ============================================================================= +# CREATE SCHEMA & WRITE DELTA TABLES +# ============================================================================= +spark.sql(f"CREATE SCHEMA IF NOT EXISTS `{CATALOG}`.`{SCHEMA}`") + + +def write_table(df_pd, table_name, mode=WRITE_MODE): + df_spark = spark.createDataFrame(df_pd) + ( + df_spark.write.format("delta") + .mode(mode) + .option("overwriteSchema", "true") + .saveAsTable(f"`{CATALOG}`.`{SCHEMA}`.`{table_name}`") + ) + print(f" OK {table_name}: {df_spark.count()} rows written") + + +TABLES = { + "locations": df_locations, + "org_units": df_org_units, + "job_roles": df_job_roles, + "skills": df_skills, + "employees": df_employees, + "employee_skills": df_employee_skills, + "learning_activity": df_learning_activity, + "talent_events": df_talent_events, + "performance_reviews": df_performance_reviews, + "compensation_snapshots": df_compensation_snapshots, + "engagement_pulses": df_engagement_pulses, + "retention_risk_snapshots": df_retention_risk_snapshots, + "requisitions": df_requisitions, + "applications": df_applications, + "succession_plans": df_succession_plans, + "mart_workforce_planning": df_mart_workforce_planning, + "mart_hiring_funnel": df_mart_hiring_funnel, + "mart_retention_engagement": df_mart_retention_engagement, + "mart_internal_mobility": df_mart_internal_mobility, + "mart_comp_performance": df_mart_comp_performance, + "mart_succession_planning": df_mart_succession_planning, +} + +print(f"Writing tables to `{CATALOG}`.`{SCHEMA}` with mode={WRITE_MODE}...") +for _table_name, _df in TABLES.items(): + write_table(_df, _table_name) +print("All tables written.") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 19. Verification + +# COMMAND ---------- + +# ============================================================================= +# VERIFICATION +# ============================================================================= +C = f"`{CATALOG}`.`{SCHEMA}`" + +print("=" * 60) +print("ROW COUNTS") +print("=" * 60) +for tbl in TABLES: + n = spark.sql(f"SELECT COUNT(*) AS n FROM {C}.`{tbl}`").collect()[0]["n"] + print(f" {tbl:<32} {n:>8,}") + +print() +print("=" * 60) +print("PATTERN VALIDATION") +print("=" * 60) + +print("\n1. 2025 mobility program participants should have lower average retention risk.") +spark.sql( + f""" + SELECT mobility_program_participant, + ROUND(AVG(risk_score), 1) AS avg_risk_score, + COUNT(DISTINCT employee_id) AS employees + FROM {C}.`retention_risk_snapshots` + WHERE snapshot_year = 2025 + GROUP BY mobility_program_participant + ORDER BY mobility_program_participant DESC + """ +).show() + +print("\n2. Sales East engagement should dip in 2024 and show higher regretted exits.") +spark.sql( + f""" + WITH engagement AS ( + SELECT e.pulse_year, + ROUND(AVG(e.engagement_score), 2) AS avg_engagement, + COUNT(DISTINCT e.employee_id) AS employees + FROM {C}.`engagement_pulses` e + JOIN {C}.`org_units` o ON e.org_unit_id = o.org_unit_id + WHERE o.org_unit_name = 'Sales East' + GROUP BY e.pulse_year + ), + exits AS ( + SELECT t.event_year, + SUM(CASE WHEN t.is_regretted_loss THEN 1 ELSE 0 END) AS regretted_exits + FROM {C}.`talent_events` t + JOIN {C}.`org_units` o ON t.org_unit_id = o.org_unit_id + WHERE o.org_unit_name = 'Sales East' + AND t.event_type = 'Exit' + GROUP BY t.event_year + ) + SELECT e.pulse_year, + e.avg_engagement, + e.employees, + COALESCE(x.regretted_exits, 0) AS regretted_exits + FROM engagement e + LEFT JOIN exits x ON e.pulse_year = x.event_year + ORDER BY e.pulse_year + """ +).show() + +print("\n3. Data & AI roles should show faster growth and longer time to fill.") +spark.sql( + f""" + SELECT job_family, + COUNT(*) AS requisitions, + ROUND(AVG(time_to_fill_days), 1) AS avg_time_to_fill_days, + SUM(CASE WHEN requisition_reason = 'Growth' THEN 1 ELSE 0 END) AS growth_reqs + FROM {C}.`requisitions` + WHERE status = 'Filled' + GROUP BY job_family + ORDER BY avg_time_to_fill_days DESC + """ +).show() + +print("\n4. High performers below 0.90 compa-ratio should have higher retention risk.") +spark.sql( + f""" + WITH annual_risk AS ( + SELECT employee_id, snapshot_year, MAX(risk_score) AS max_risk_score + FROM {C}.`retention_risk_snapshots` + GROUP BY employee_id, snapshot_year + ) + SELECT CASE + WHEN p.is_high_performer AND c.compa_ratio < 0.90 THEN 'High performer below 0.90 compa' + WHEN p.is_high_performer THEN 'High performer at/above 0.90 compa' + ELSE 'Other' + END AS segment, + COUNT(*) AS employee_years, + ROUND(AVG(r.max_risk_score), 1) AS avg_max_risk_score + FROM {C}.`compensation_snapshots` c + JOIN {C}.`performance_reviews` p + ON c.employee_id = p.employee_id AND c.snapshot_year = p.review_year + JOIN annual_risk r + ON c.employee_id = r.employee_id AND c.snapshot_year = r.snapshot_year + GROUP BY segment + ORDER BY avg_max_risk_score DESC + """ +).show(truncate=False) + +print("\n5. Product offers below market midpoint should have lower acceptance.") +spark.sql( + f""" + SELECT CASE WHEN a.offer_to_market_midpoint_pct < 100 THEN 'Below midpoint' ELSE 'At/above midpoint' END AS offer_position, + COUNT(*) AS offers, + ROUND(SUM(CASE WHEN a.offer_accepted THEN 1 ELSE 0 END) * 100.0 / COUNT(*), 1) AS acceptance_rate_pct + FROM {C}.`applications` a + JOIN {C}.`requisitions` r ON a.requisition_id = r.requisition_id + WHERE r.job_family = 'Product' + AND a.offer_salary_usd IS NOT NULL + GROUP BY offer_position + ORDER BY offer_position + """ +).show() + +print("\n6. GTM and Data & AI Platform critical roles should have weaker succession coverage.") +spark.sql( + f""" + SELECT CASE + WHEN org_unit_name = 'Data & AI Platform' THEN 'Data & AI Platform' + WHEN business_unit = 'Go To Market' THEN 'GTM' + ELSE 'Other' + END AS org_group, + SUM(critical_position_count) AS critical_positions, + SUM(covered_position_count) AS covered_positions, + ROUND(SUM(covered_position_count) * 100.0 / SUM(critical_position_count), 1) AS coverage_rate_pct + FROM {C}.`mart_succession_planning` + GROUP BY org_group + ORDER BY coverage_rate_pct + """ +).show() + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 20. Create Metric Views + +# COMMAND ---------- + +# ============================================================================= +# CREATE METRIC VIEWS +# ============================================================================= +print("Creating metric views...") + +metric_views = { + "mv_workforce_planning": f""" +version: "0.1" +source: {CATALOG}.{SCHEMA}.mart_workforce_planning + +dimensions: + - name: Snapshot Month + expr: snapshot_month + - name: Snapshot Year + expr: snapshot_year + - name: Business Unit + expr: business_unit + - name: Function + expr: function + - name: Org Unit + expr: org_unit_name + - name: Job Family + expr: job_family + - name: Region + expr: region + +measures: + - name: Headcount + expr: SUM(active_headcount) + - name: Starting Headcount + expr: SUM(start_headcount) + - name: Hires + expr: SUM(hires) + - name: Exits + expr: SUM(exits) + - name: Open Requisitions + expr: SUM(open_requisitions) + - name: Filled Requisitions + expr: SUM(filled_requisitions) + - name: Attrition Rate Pct + expr: CASE WHEN SUM(start_headcount) = 0 THEN 0 ELSE SUM(exits) * 100.0 / SUM(start_headcount) END + - name: Vacancy Rate Pct + expr: CASE WHEN SUM(active_headcount + open_requisitions) = 0 THEN 0 ELSE SUM(open_requisitions) * 100.0 / SUM(active_headcount + open_requisitions) END +""", + "mv_hiring_funnel": f""" +version: "0.1" +source: {CATALOG}.{SCHEMA}.mart_hiring_funnel + +dimensions: + - name: Opened Year + expr: opened_year + - name: Status + expr: status + - name: Business Unit + expr: business_unit + - name: Org Unit + expr: org_unit_name + - name: Job Family + expr: job_family + - name: Role Title + expr: role_title + - name: Priority + expr: priority + - name: Candidate Market + expr: candidate_market + - name: Requisition Reason + expr: requisition_reason + +measures: + - name: Requisition Count + expr: COUNT(1) + - name: Applications + expr: SUM(applications_count) + - name: Internal Applications + expr: SUM(internal_applications) + - name: Interviews + expr: SUM(interview_count) + - name: Offers + expr: SUM(offer_count) + - name: Accepted Offers + expr: SUM(accepted_offer_count) + - name: Declined Offers + expr: SUM(declined_offer_count) + - name: Offer Acceptance Rate Pct + expr: CASE WHEN SUM(offer_count) = 0 THEN 0 ELSE SUM(accepted_offer_count) * 100.0 / SUM(offer_count) END + - name: Internal Fill Rate Pct + expr: SUM(CASE WHEN is_internal_fill THEN 1 ELSE 0 END) * 100.0 / COUNT(1) + - name: Average Time To Fill Days + expr: AVG(time_to_fill_days) +""", + "mv_retention_engagement": f""" +version: "0.1" +source: {CATALOG}.{SCHEMA}.mart_retention_engagement + +dimensions: + - name: Snapshot Year + expr: snapshot_year + - name: Snapshot Quarter + expr: snapshot_quarter_label + - name: Business Unit + expr: business_unit + - name: Function + expr: function + - name: Org Unit + expr: org_unit_name + - name: Job Family + expr: job_family + +measures: + - name: Employees Surveyed + expr: SUM(employee_count) + - name: Average Engagement Score + expr: AVG(avg_engagement_score) + - name: Average Manager Effectiveness Score + expr: AVG(avg_manager_effectiveness_score) + - name: Average Burnout Risk Score + expr: AVG(avg_burnout_risk_score) + - name: Average Retention Risk Score + expr: AVG(avg_retention_risk_score) + - name: High Risk Employees + expr: SUM(high_risk_employee_count) + - name: Regretted Exits + expr: SUM(regretted_exit_count) + - name: Total Exits + expr: SUM(total_exit_count) + - name: High Risk Rate Pct + expr: CASE WHEN SUM(employee_count) = 0 THEN 0 ELSE SUM(high_risk_employee_count) * 100.0 / SUM(employee_count) END +""", + "mv_internal_mobility": f""" +version: "0.1" +source: {CATALOG}.{SCHEMA}.mart_internal_mobility + +dimensions: + - name: Snapshot Year + expr: snapshot_year + - name: Business Unit + expr: business_unit + - name: Function + expr: function + - name: Org Unit + expr: org_unit_name + - name: Job Family + expr: job_family + +measures: + - name: Employee Count + expr: SUM(employee_count) + - name: Mobility Program Participants + expr: SUM(mobility_program_participants) + - name: Promotions + expr: SUM(promotion_count) + - name: Transfers + expr: SUM(transfer_count) + - name: Internal Fills + expr: SUM(internal_fill_count) + - name: Learning Completions + expr: SUM(learning_completion_count) + - name: Critical Skill Gaps + expr: SUM(critical_skill_gap_count) + - name: Average Skill Gap + expr: AVG(avg_skill_gap) + - name: Internal Mobility Events + expr: SUM(promotion_count + transfer_count) +""", + "mv_comp_performance": f""" +version: "0.1" +source: {CATALOG}.{SCHEMA}.mart_comp_performance + +dimensions: + - name: Snapshot Year + expr: snapshot_year + - name: Business Unit + expr: business_unit + - name: Function + expr: function + - name: Org Unit + expr: org_unit_name + - name: Job Family + expr: job_family + - name: Job Level + expr: job_level + +measures: + - name: Employee Count + expr: SUM(employee_count) + - name: Average Base Salary + expr: AVG(avg_base_salary_usd) + - name: Average Bonus + expr: AVG(avg_bonus_actual_usd) + - name: Average Compa Ratio + expr: AVG(avg_compa_ratio) + - name: Average Range Penetration Pct + expr: AVG(avg_range_penetration_pct) + - name: Average Performance Rating + expr: AVG(avg_performance_rating) + - name: High Performers + expr: SUM(high_performer_count) + - name: High Performers Below 90 Compa + expr: SUM(high_performer_below_90_compa_count) + - name: High Risk Employees + expr: SUM(high_risk_employee_count) +""", + "mv_succession_planning": f""" +version: "0.1" +source: {CATALOG}.{SCHEMA}.mart_succession_planning + +dimensions: + - name: Business Unit + expr: business_unit + - name: Function + expr: function + - name: Org Unit + expr: org_unit_name + - name: Job Family + expr: job_family + - name: Criticality Tier + expr: criticality_tier + +measures: + - name: Critical Positions + expr: SUM(critical_position_count) + - name: Ready Successors + expr: SUM(ready_successor_count) + - name: Covered Positions + expr: SUM(covered_position_count) + - name: Weak Coverage Positions + expr: SUM(weak_coverage_position_count) + - name: Coverage Rate Pct + expr: CASE WHEN SUM(critical_position_count) = 0 THEN 0 ELSE SUM(covered_position_count) * 100.0 / SUM(critical_position_count) END + - name: Average Readiness Months + expr: AVG(avg_readiness_months) +""", +} + +for mv_name, yaml_body in metric_views.items(): + ddl = ( + f"CREATE OR REPLACE VIEW `{CATALOG}`.`{SCHEMA}`.`{mv_name}`\n" + "WITH METRICS\n" + "LANGUAGE YAML\n" + "AS $$" + + yaml_body + + "$$" + ) + spark.sql(ddl) + print(f" OK {mv_name}") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 21. Register Constraints & Comments + +# COMMAND ---------- + +# ============================================================================= +# REGISTER CONSTRAINTS AND COMMENTS +# ============================================================================= +print("Registering constraints and comments...") + +PRIMARY_KEYS = { + "locations": "location_id", + "org_units": "org_unit_id", + "job_roles": "role_id", + "skills": "skill_id", + "employees": "employee_id", + "employee_skills": "employee_skill_id", + "learning_activity": "learning_activity_id", + "talent_events": "talent_event_id", + "performance_reviews": "review_id", + "compensation_snapshots": "compensation_snapshot_id", + "engagement_pulses": "engagement_pulse_id", + "retention_risk_snapshots": "retention_risk_snapshot_id", + "requisitions": "requisition_id", + "applications": "application_id", + "succession_plans": "succession_plan_id", + "mart_workforce_planning": "mart_workforce_planning_id", + "mart_hiring_funnel": "mart_hiring_funnel_id", + "mart_retention_engagement": "mart_retention_engagement_id", + "mart_internal_mobility": "mart_internal_mobility_id", + "mart_comp_performance": "mart_comp_performance_id", + "mart_succession_planning": "mart_succession_planning_id", +} + +FOREIGN_KEYS = [ + ("org_units", "parent_org_unit_id", "org_units", "org_unit_id"), + ("employees", "org_unit_id", "org_units", "org_unit_id"), + ("employees", "location_id", "locations", "location_id"), + ("employees", "role_id", "job_roles", "role_id"), + ("employees", "current_manager_employee_id", "employees", "employee_id"), + ("employee_skills", "employee_id", "employees", "employee_id"), + ("employee_skills", "skill_id", "skills", "skill_id"), + ("learning_activity", "employee_id", "employees", "employee_id"), + ("learning_activity", "skill_id", "skills", "skill_id"), + ("talent_events", "employee_id", "employees", "employee_id"), + ("talent_events", "org_unit_id", "org_units", "org_unit_id"), + ("talent_events", "role_id", "job_roles", "role_id"), + ("performance_reviews", "employee_id", "employees", "employee_id"), + ("performance_reviews", "org_unit_id", "org_units", "org_unit_id"), + ("performance_reviews", "role_id", "job_roles", "role_id"), + ("compensation_snapshots", "employee_id", "employees", "employee_id"), + ("compensation_snapshots", "org_unit_id", "org_units", "org_unit_id"), + ("compensation_snapshots", "role_id", "job_roles", "role_id"), + ("engagement_pulses", "employee_id", "employees", "employee_id"), + ("engagement_pulses", "org_unit_id", "org_units", "org_unit_id"), + ("engagement_pulses", "role_id", "job_roles", "role_id"), + ("retention_risk_snapshots", "employee_id", "employees", "employee_id"), + ("retention_risk_snapshots", "org_unit_id", "org_units", "org_unit_id"), + ("retention_risk_snapshots", "role_id", "job_roles", "role_id"), + ("requisitions", "org_unit_id", "org_units", "org_unit_id"), + ("requisitions", "location_id", "locations", "location_id"), + ("requisitions", "role_id", "job_roles", "role_id"), + ("applications", "requisition_id", "requisitions", "requisition_id"), + ("succession_plans", "org_unit_id", "org_units", "org_unit_id"), + ("succession_plans", "role_id", "job_roles", "role_id"), + ("succession_plans", "incumbent_employee_id", "employees", "employee_id"), +] + +for table, pk in PRIMARY_KEYS.items(): + spark.sql(f"ALTER TABLE {C}.`{table}` ALTER COLUMN `{pk}` SET NOT NULL") + spark.sql(f"ALTER TABLE {C}.`{table}` ADD CONSTRAINT pk_{table} PRIMARY KEY (`{pk}`)") + +for table, col, ref_table, ref_col in FOREIGN_KEYS: + spark.sql( + f"ALTER TABLE {C}.`{table}` ADD CONSTRAINT fk_{table}_{col} " + f"FOREIGN KEY (`{col}`) REFERENCES {C}.`{ref_table}` (`{ref_col}`)" + ) + + +def sql_text(value): + return value.replace("'", "''") + + +TABLE_COMMENTS = { + "locations": "Work location dimension for the synthetic talent advisory dataset. No employee home addresses are included.", + "org_units": "Organizational hierarchy with functions, business units, and critical org flags.", + "job_roles": "Role catalog with job family, level, salary bands, market premiums, scarcity, and critical-role flags.", + "skills": "Skill taxonomy used for skills-gap and learning analysis.", + "employees": "Synthetic employee dimension with names, org, role, location, hire and termination dates. No real PII or protected-class attributes are included.", + "employee_skills": "Employee skill proficiency facts with target levels and critical gap flags.", + "learning_activity": "Learning activity facts, including the 2025 Talent Marketplace Accelerator mobility program.", + "talent_events": "Hires, exits, promotions, and transfers used for workforce, mobility, and attrition analytics.", + "performance_reviews": "Annual performance and potential ratings for compensation, performance, and retention demos.", + "compensation_snapshots": "Annual synthetic compensation snapshots with salary, bonus, market midpoint, compa-ratio, and range penetration.", + "engagement_pulses": "Quarterly engagement pulse facts. Sales East has a 2024 engagement dip.", + "retention_risk_snapshots": "Quarterly retention risk snapshots. High performers below 0.90 compa-ratio have elevated risk; 2025 mobility participants have lower risk.", + "requisitions": "Hiring requisitions with time-to-fill, internal fill, candidate-market, priority, and growth/backfill signals.", + "applications": "Candidate funnel facts without external candidate names. Product offers below market midpoint have lower synthetic acceptance.", + "succession_plans": "Critical role succession coverage. GTM and Data & AI Platform have weaker ready-successor coverage.", + "mart_workforce_planning": "Curated monthly workforce planning mart for headcount, hires, exits, open requisitions, and vacancy metrics.", + "mart_hiring_funnel": "Curated requisition-level hiring funnel mart for application, interview, offer, acceptance, and time-to-fill metrics.", + "mart_retention_engagement": "Curated quarterly retention and engagement mart for risk, engagement, burnout, and regretted exit metrics.", + "mart_internal_mobility": "Curated annual mobility and skills mart for promotions, transfers, internal fills, learning, and skill gaps.", + "mart_comp_performance": "Curated annual compensation and performance mart for salary, bonus, compa-ratio, performance, and high-risk groups.", + "mart_succession_planning": "Curated succession planning mart for critical position coverage and readiness metrics.", +} + +for table, comment in TABLE_COMMENTS.items(): + spark.sql(f"COMMENT ON TABLE {C}.`{table}` IS '{sql_text(comment)}'") + +for table, df_pd in TABLES.items(): + for column in df_pd.columns: + readable = column.replace("_", " ") + spark.sql( + f"ALTER TABLE {C}.`{table}` ALTER COLUMN `{column}` " + f"COMMENT '{sql_text(readable)} for the synthetic talent advisory dataset'" + ) + +print(" OK constraints and comments registered") + +print() +print("=" * 60) +print("SETUP COMPLETE") +print("=" * 60) +print(f" Catalog : {CATALOG}") +print(f" Schema : {SCHEMA}") +print(" Source Tables: locations, org_units, job_roles, skills, employees, employee_skills,") +print(" learning_activity, talent_events, performance_reviews,") +print(" compensation_snapshots, engagement_pulses, retention_risk_snapshots,") +print(" requisitions, applications, succession_plans") +print(" Topic Marts : mart_workforce_planning, mart_hiring_funnel, mart_retention_engagement,") +print(" mart_internal_mobility, mart_comp_performance, mart_succession_planning") +print(" Metric Views: mv_workforce_planning, mv_hiring_funnel, mv_retention_engagement,") +print(" mv_internal_mobility, mv_comp_performance, mv_succession_planning") +print(" Next : Create a Genie space from these tables, or see notebooks/demo-data/README.md") diff --git a/notebooks/demo-data/generate_wind_turbine_maintenance_data.py b/notebooks/demo-data/generate_wind_turbine_maintenance_data.py new file mode 100644 index 000000000..0454ed631 --- /dev/null +++ b/notebooks/demo-data/generate_wind_turbine_maintenance_data.py @@ -0,0 +1,812 @@ +# Databricks notebook source + +# MAGIC %md +# MAGIC # Wind Turbine Predictive Maintenance - Synthetic Dataset Generator +# MAGIC +# MAGIC Generates a fictional wind turbine operations dataset for Databricks AI/BI Genie demos. +# MAGIC +# MAGIC | Table | Rows | Description | +# MAGIC |---|---:|---| +# MAGIC | `wind_farms` | 8 | Wind farm dimension | +# MAGIC | `turbines` | 96 | Turbine asset dimension | +# MAGIC | `components` | 384 | Major component dimension | +# MAGIC | `sensor_readings` | 3,456 | Monthly turbine telemetry from 2023-2025 | +# MAGIC | `maintenance_events` | variable | Preventive, inspection, and corrective work orders | +# MAGIC | `failure_events` | variable | Component failure facts | +# MAGIC +# MAGIC **Setup:** Set the `catalog` and `schema` widgets (or edit defaults below), then **Run All**. +# MAGIC +# MAGIC Seed: `42` - fully reproducible. + +# COMMAND ---------- + +# MAGIC %pip install faker==40.19.1 --quiet +# MAGIC %restart_python + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 1. Configuration + +# COMMAND ---------- + +# ============================================================================= +# CONFIGURATION - Edit only this section before running +# ============================================================================= +DEFAULT_CATALOG = "" # Unity Catalog name; required +DEFAULT_SCHEMA = "wind_turbine_maintenance" # Schema / database name + +# Widgets let the bulk runner pass values while preserving standalone defaults. +try: + dbutils.widgets.text("catalog", DEFAULT_CATALOG, "Unity Catalog name") +except Exception: + pass + +try: + dbutils.widgets.text("schema", DEFAULT_SCHEMA, "Schema / database name") +except Exception: + pass + +try: + dbutils.widgets.dropdown("overwrite_existing", "false", ["false", "true"], "Overwrite existing tables") +except Exception: + pass + +try: + CATALOG = dbutils.widgets.get("catalog").strip() or DEFAULT_CATALOG +except Exception: + CATALOG = DEFAULT_CATALOG + +try: + SCHEMA = dbutils.widgets.get("schema").strip() or DEFAULT_SCHEMA +except Exception: + SCHEMA = DEFAULT_SCHEMA + +try: + OVERWRITE_EXISTING = dbutils.widgets.get("overwrite_existing").strip().lower() == "true" +except Exception: + OVERWRITE_EXISTING = False + +if not CATALOG: + raise ValueError("Set the catalog widget to a Unity Catalog name before running this notebook.") + +WRITE_MODE = "overwrite" if OVERWRITE_EXISTING else "errorifexists" + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 2. Imports & Global Setup + +# COMMAND ---------- + +# ============================================================================= +# IMPORTS & GLOBAL SETUP +# ============================================================================= +import random +from calendar import monthrange +from datetime import date, timedelta + +import numpy as np +import pandas as pd +from faker import Faker + +SEED = 42 +random.seed(SEED) +np.random.seed(SEED) +fake = Faker("en_US") +Faker.seed(SEED) + +START_DATE = date(2023, 1, 1) +END_DATE = date(2025, 12, 31) +MONTH_LIST = [(y, m) for y in [2023, 2024, 2025] for m in range(1, 13)] + +MODEL_CONFIG = { + "WT-2.5A": {"capacity_mw": 2.5, "failure_mult": 0.85, "hub_height_m": 90}, + "WT-3.0B": {"capacity_mw": 3.0, "failure_mult": 1.00, "hub_height_m": 105}, + "WT-4.2C": {"capacity_mw": 4.2, "failure_mult": 1.45, "hub_height_m": 120}, +} +COMPONENT_TYPES = ["Gearbox", "Generator", "Blade Set", "Yaw System"] + +print("Setup complete.") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 3. Wind Farms + +# COMMAND ---------- + +# ============================================================================= +# TABLE 1: WIND FARMS (8 rows) +# ============================================================================= +FARM_SEEDS = [ + ("FARM-001", "Prairie Ridge", "Midwest", "IA", 8.7, "Plains"), + ("FARM-002", "Lone Mesa", "Southwest", "TX", 9.1, "Mesa"), + ("FARM-003", "Columbia Gorge", "West", "WA", 10.2, "Ridge"), + ("FARM-004", "High Desert", "West", "CA", 8.4, "Desert"), + ("FARM-005", "Great Lakes Shore", "Midwest", "MI", 8.9, "Coastal"), + ("FARM-006", "Dakota Plains", "Midwest", "ND", 9.5, "Plains"), + ("FARM-007", "Appalachian Ridge", "Northeast", "PA", 7.8, "Ridge"), + ("FARM-008", "Gulf Coast Bend", "Southeast", "LA", 8.2, "Coastal"), +] + +wind_farms_data = [] +for farm_id, name, region, state, wind_speed, terrain in FARM_SEEDS: + wind_farms_data.append( + { + "farm_id": farm_id, + "farm_name": name, + "region": region, + "state": state, + "terrain_type": terrain, + "mean_wind_speed_mps": float(wind_speed), + "grid_interconnect": random.choice(["North", "South", "East", "West"]), + "commissioned_date": date(2013 + random.randint(0, 7), random.randint(1, 12), 1), + "is_active": True, + } + ) + +df_wind_farms = pd.DataFrame(wind_farms_data) +FARM_BY_ID = {f["farm_id"]: f for f in wind_farms_data} +print(f"wind_farms: {len(df_wind_farms)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 4. Turbines + +# COMMAND ---------- + +# ============================================================================= +# TABLE 2: TURBINES (96 rows) +# ============================================================================= +turbines_data = [] +turbine_counter = 1 + +for farm in wind_farms_data: + for _ in range(12): + model = random.choices(list(MODEL_CONFIG.keys()), weights=[36, 42, 22])[0] + cfg = MODEL_CONFIG[model] + install_date = farm["commissioned_date"] + timedelta(days=random.randint(0, 540)) + turbines_data.append( + { + "turbine_id": f"TURB-{turbine_counter:04d}", + "farm_id": farm["farm_id"], + "turbine_model": model, + "rated_capacity_mw": cfg["capacity_mw"], + "hub_height_m": cfg["hub_height_m"], + "rotor_diameter_m": random.choice([100, 112, 126, 136]), + "install_date": install_date, + "commissioning_year": install_date.year, + "status": random.choices(["Operating", "Derated", "Offline"], weights=[92, 6, 2])[0], + } + ) + turbine_counter += 1 + +df_turbines = pd.DataFrame(turbines_data) +TURBINE_BY_ID = {t["turbine_id"]: t for t in turbines_data} +TURBINE_IDS = [t["turbine_id"] for t in turbines_data] +print(f"turbines: {len(df_turbines)} rows") +print(df_turbines["turbine_model"].value_counts().to_dict()) + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 5. Components + +# COMMAND ---------- + +# ============================================================================= +# TABLE 3: COMPONENTS (384 rows) +# ============================================================================= +components_data = [] +component_counter = 1 +COMPONENTS_BY_TURBINE = {} + +for turbine in turbines_data: + COMPONENTS_BY_TURBINE[turbine["turbine_id"]] = [] + for component_type in COMPONENT_TYPES: + component_id = f"COMP-{component_counter:06d}" + component_counter += 1 + expected_life = { + "Gearbox": 12, + "Generator": 15, + "Blade Set": 20, + "Yaw System": 10, + }[component_type] + row = { + "component_id": component_id, + "turbine_id": turbine["turbine_id"], + "component_type": component_type, + "manufacturer": random.choice(["Apex Rotor", "Northwind Systems", "HelioWorks", "Vector Drive"]), + "install_date": turbine["install_date"], + "expected_life_years": expected_life, + "status": random.choices(["In Service", "Watchlist", "Replaced"], weights=[88, 10, 2])[0], + } + components_data.append(row) + COMPONENTS_BY_TURBINE[turbine["turbine_id"]].append(row) + +df_components = pd.DataFrame(components_data) +COMPONENT_BY_ID = {c["component_id"]: c for c in components_data} +print(f"components: {len(df_components)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 6. Preventive Maintenance + +# COMMAND ---------- + +# ============================================================================= +# TABLE 4: MAINTENANCE EVENTS (base preventive and inspection events) +# ============================================================================= +maintenance_data = [] +maintenance_counter = 1 + +for turbine in turbines_data: + for year in [2023, 2024, 2025]: + for quarter_start in [1, 4, 7, 10]: + if random.random() < 0.74: + event_date = date(year, quarter_start, 1) + timedelta(days=random.randint(0, 70)) + component = random.choice(COMPONENTS_BY_TURBINE[turbine["turbine_id"]]) + event_type = random.choices(["Preventive", "Inspection"], weights=[72, 28])[0] + duration = random.uniform(2.0, 8.0) if event_type == "Inspection" else random.uniform(6.0, 18.0) + maintenance_data.append( + { + "maintenance_id": f"MAINT-{maintenance_counter:07d}", + "maintenance_date": event_date, + "maintenance_year": event_date.year, + "maintenance_month": event_date.month, + "farm_id": turbine["farm_id"], + "turbine_id": turbine["turbine_id"], + "component_id": component["component_id"], + "maintenance_type": event_type, + "work_order_priority": random.choices(["Low", "Medium", "High"], weights=[45, 45, 10])[0], + "duration_hours": round(duration, 2), + "cost_usd": round(duration * random.uniform(180, 420), 2), + "technician_team": random.choice(["Team A", "Team B", "Team C", "Vendor"]), + } + ) + maintenance_counter += 1 + + +def had_recent_preventive(turbine_id, event_date, days=90): + lower = event_date - timedelta(days=days) + for event in maintenance_data: + if ( + event["turbine_id"] == turbine_id + and event["maintenance_type"] == "Preventive" + and lower <= event["maintenance_date"] <= event_date + ): + return True + return False + + +print(f"base maintenance_events: {len(maintenance_data)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 7. Failure Events + +# COMMAND ---------- + +# ============================================================================= +# TABLE 5: FAILURE EVENTS +# ============================================================================= +# Injected patterns: +# 1. WT-4.2C turbines fail more often than other models. +# 2. Gearbox temperature and vibration rise before failures. +# 3. Winter icing increases blade-related failures and reduces capacity factor. +# 4. Recent preventive maintenance reduces downtime. + +failure_data = [] +failure_counter = 1 + +for turbine in turbines_data: + model_mult = MODEL_CONFIG[turbine["turbine_model"]]["failure_mult"] + farm = FARM_BY_ID[turbine["farm_id"]] + base_failures = random.choices([0, 1, 2, 3], weights=[50, 34, 12, 4])[0] + if random.random() < (model_mult - 1.0) * 0.22: + base_failures += 1 + for _ in range(base_failures): + year = random.choice([2023, 2024, 2025]) + month = random.choices(list(range(1, 13)), weights=[13, 12, 10, 8, 7, 7, 11, 12, 8, 6, 3, 3])[0] + max_day = monthrange(year, month)[1] + failure_date = date(year, month, random.randint(1, max_day)) + if month in (1, 2, 12): + component_type = random.choices(COMPONENT_TYPES, weights=[24, 18, 44, 14])[0] + failure_type = random.choice(["Icing Damage", "Pitch Fault", "Bearing Wear"]) + elif month in (6, 7, 8): + component_type = random.choices(COMPONENT_TYPES, weights=[46, 23, 18, 13])[0] + failure_type = random.choice(["Thermal Alarm", "Gearbox Wear", "Oil Pressure"]) + else: + component_type = random.choices(COMPONENT_TYPES, weights=[34, 26, 24, 16])[0] + failure_type = random.choice(["Bearing Wear", "Electrical Fault", "Pitch Fault", "Sensor Fault"]) + component = random.choice([c for c in COMPONENTS_BY_TURBINE[turbine["turbine_id"]] if c["component_type"] == component_type]) + recent_pm = had_recent_preventive(turbine["turbine_id"], failure_date) + severity = random.choices(["Minor", "Major", "Critical"], weights=[42, 43, 15])[0] + severity_mult = {"Minor": 0.7, "Major": 1.0, "Critical": 1.8}[severity] + downtime = random.uniform(10, 72) * severity_mult + if recent_pm: + downtime *= 0.62 + repair_cost = downtime * random.uniform(800, 1800) + failure_data.append( + { + "failure_id": f"FAIL-{failure_counter:07d}", + "failure_date": failure_date, + "failure_year": failure_date.year, + "failure_month": failure_date.month, + "farm_id": turbine["farm_id"], + "turbine_id": turbine["turbine_id"], + "component_id": component["component_id"], + "component_type": component_type, + "failure_type": failure_type, + "severity": severity, + "had_recent_preventive": recent_pm, + "downtime_hours": round(downtime, 2), + "repair_cost_usd": round(repair_cost, 2), + } + ) + # Corrective maintenance work order tied to the failure. + corrective_date = min(END_DATE, failure_date + timedelta(days=random.randint(0, 4))) + maintenance_data.append( + { + "maintenance_id": f"MAINT-{maintenance_counter:07d}", + "maintenance_date": corrective_date, + "maintenance_year": corrective_date.year, + "maintenance_month": corrective_date.month, + "farm_id": turbine["farm_id"], + "turbine_id": turbine["turbine_id"], + "component_id": component["component_id"], + "maintenance_type": "Corrective", + "work_order_priority": "High" if severity != "Minor" else "Medium", + "duration_hours": round(downtime, 2), + "cost_usd": round(repair_cost, 2), + "technician_team": random.choice(["Team A", "Team B", "Team C", "Vendor"]), + } + ) + maintenance_counter += 1 + failure_counter += 1 + +df_failure_events = pd.DataFrame(failure_data) +df_maintenance_events = pd.DataFrame(maintenance_data) +FAILURES_BY_TURBINE = {} +for failure in failure_data: + FAILURES_BY_TURBINE.setdefault(failure["turbine_id"], []).append(failure) +print(f"failure_events: {len(df_failure_events)} rows") +print(f"maintenance_events: {len(df_maintenance_events)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 8. Sensor Readings + +# COMMAND ---------- + +# ============================================================================= +# TABLE 6: SENSOR READINGS (monthly turbine telemetry) +# ============================================================================= + + +def days_to_next_failure(turbine_id, reading_date): + candidates = [ + (f["failure_date"] - reading_date).days + for f in FAILURES_BY_TURBINE.get(turbine_id, []) + if f["failure_date"] >= reading_date + ] + return min(candidates) if candidates else None + + +sensor_data = [] +reading_counter = 1 + +for turbine in turbines_data: + farm = FARM_BY_ID[turbine["farm_id"]] + farm_wind = farm["mean_wind_speed_mps"] + model = turbine["turbine_model"] + capacity_kw = turbine["rated_capacity_mw"] * 1000.0 + farm_cf_bias = (farm_wind - 8.5) * 2.2 + for year, month in MONTH_LIST: + reading_date = date(year, month, min(15, monthrange(year, month)[1])) + ambient_temp = random.uniform(-8, 8) if month in (1, 2, 12) else random.uniform(18, 35) if month in (6, 7, 8) else random.uniform(5, 24) + icing_flag = month in (1, 2, 12) and random.random() < 0.34 + wind_speed = max(2.0, random.gauss(farm_wind, 1.6)) + capacity_factor = max(8.0, min(58.0, 31.0 + farm_cf_bias + (wind_speed - farm_wind) * 3.0 + random.gauss(0, 4.2))) + if icing_flag: + capacity_factor *= random.uniform(0.68, 0.86) + gearbox_temp = random.gauss(66, 7) + if month in (6, 7, 8): + gearbox_temp += random.uniform(6, 13) + vibration = max(0.5, random.gauss(2.4, 0.55)) + days_next = days_to_next_failure(turbine["turbine_id"], reading_date) + anomaly_score = max(0.0, random.gauss(18, 8)) + if days_next is not None and days_next <= 60: + anomaly_score += 45 - (days_next / 2.0) + vibration += random.uniform(1.4, 3.2) + gearbox_temp += random.uniform(8, 18) + elif days_next is not None and days_next <= 120: + anomaly_score += 16 + vibration += random.uniform(0.4, 1.2) + power_output = capacity_kw * capacity_factor / 100.0 + sensor_data.append( + { + "reading_id": f"SENS-{reading_counter:07d}", + "reading_date": reading_date, + "reading_year": year, + "reading_month": month, + "farm_id": turbine["farm_id"], + "turbine_id": turbine["turbine_id"], + "turbine_model": model, + "wind_speed_mps": round(wind_speed, 2), + "ambient_temp_c": round(ambient_temp, 2), + "power_output_kw": round(power_output, 2), + "capacity_factor_pct": round(capacity_factor, 2), + "gearbox_temp_c": round(gearbox_temp, 2), + "vibration_mm_s": round(vibration, 2), + "blade_pitch_deg": round(random.uniform(2, 18), 2), + "icing_flag": bool(icing_flag), + "anomaly_score": round(min(100.0, anomaly_score), 2), + "days_to_next_failure": days_next, + } + ) + reading_counter += 1 + +df_sensor_readings = pd.DataFrame(sensor_data) +print(f"sensor_readings: {len(df_sensor_readings)} rows") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 9. Create Schema & Write Delta Tables + +# COMMAND ---------- + +# ============================================================================= +# CREATE SCHEMA & WRITE DELTA TABLES +# ============================================================================= +spark.sql(f"CREATE SCHEMA IF NOT EXISTS `{CATALOG}`.`{SCHEMA}`") + + +def write_table(df_pd, table_name, mode=WRITE_MODE): + df_spark = spark.createDataFrame(df_pd) + ( + df_spark.write.format("delta") + .mode(mode) + .option("overwriteSchema", "true") + .saveAsTable(f"`{CATALOG}`.`{SCHEMA}`.`{table_name}`") + ) + print(f" OK {table_name}: {df_spark.count()} rows written") + + +TABLES = { + "wind_farms": df_wind_farms, + "turbines": df_turbines, + "components": df_components, + "sensor_readings": df_sensor_readings, + "maintenance_events": df_maintenance_events, + "failure_events": df_failure_events, +} + +print(f"Writing tables to `{CATALOG}`.`{SCHEMA}` with mode={WRITE_MODE}...") +for _table_name, _df in TABLES.items(): + write_table(_df, _table_name) +print("All tables written.") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 10. Verification + +# COMMAND ---------- + +# ============================================================================= +# VERIFICATION +# ============================================================================= +C = f"`{CATALOG}`.`{SCHEMA}`" + +print("=" * 60) +print("ROW COUNTS") +print("=" * 60) +for tbl in TABLES: + n = spark.sql(f"SELECT COUNT(*) AS n FROM {C}.`{tbl}`").collect()[0]["n"] + print(f" {tbl:<24} {n:>8,}") + +print() +print("=" * 60) +print("PATTERN VALIDATION") +print("=" * 60) + +print("\n1. Anomaly scores should rise within 60 days of failure.") +spark.sql( + f""" + SELECT CASE WHEN days_to_next_failure BETWEEN 0 AND 60 THEN 'Within 60 Days' + WHEN days_to_next_failure BETWEEN 61 AND 120 THEN '61-120 Days' + ELSE 'No Near Failure' END AS failure_window, + ROUND(AVG(anomaly_score), 1) AS avg_anomaly_score, + ROUND(AVG(vibration_mm_s), 2) AS avg_vibration, + COUNT(*) AS readings + FROM {C}.`sensor_readings` + GROUP BY 1 + ORDER BY avg_anomaly_score DESC + """ +).show() + +print("\n2. Winter icing should reduce capacity factor.") +spark.sql( + f""" + SELECT icing_flag, + ROUND(AVG(capacity_factor_pct), 1) AS avg_capacity_factor_pct, + COUNT(*) AS readings + FROM {C}.`sensor_readings` + GROUP BY icing_flag + """ +).show() + +print("\n3. Summer months should have higher gearbox temperatures.") +spark.sql( + f""" + SELECT CASE WHEN reading_month IN (6, 7, 8) THEN 'Summer' ELSE 'Other' END AS period, + ROUND(AVG(gearbox_temp_c), 1) AS avg_gearbox_temp_c + FROM {C}.`sensor_readings` + GROUP BY 1 + """ +).show() + +print("\n4. WT-4.2C should show a higher failure rate.") +spark.sql( + f""" + SELECT t.turbine_model, + COUNT(DISTINCT f.failure_id) AS failures, + COUNT(DISTINCT t.turbine_id) AS turbines, + ROUND(COUNT(DISTINCT f.failure_id) * 1.0 / COUNT(DISTINCT t.turbine_id), 2) AS failures_per_turbine + FROM {C}.`turbines` t + LEFT JOIN {C}.`failure_events` f ON t.turbine_id = f.turbine_id + GROUP BY t.turbine_model + ORDER BY failures_per_turbine DESC + """ +).show() + +print("\n5. Recent preventive maintenance should reduce failure downtime.") +spark.sql( + f""" + SELECT had_recent_preventive, + ROUND(AVG(downtime_hours), 1) AS avg_downtime_hours, + COUNT(*) AS failures + FROM {C}.`failure_events` + GROUP BY had_recent_preventive + """ +).show() + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 11. Create Metric Views + +# COMMAND ---------- + +# ============================================================================= +# CREATE METRIC VIEWS +# ============================================================================= +print("Creating metric views...") + +metric_views = { + "mv_turbine_performance": f""" +version: "0.1" +source: {CATALOG}.{SCHEMA}.sensor_readings + +joins: + - name: turbine + source: {CATALOG}.{SCHEMA}.turbines + on: source.turbine_id = turbine.turbine_id + - name: farm + source: {CATALOG}.{SCHEMA}.wind_farms + on: source.farm_id = farm.farm_id + +dimensions: + - name: Reading Month + expr: DATE_TRUNC('MONTH', reading_date) + - name: Reading Year + expr: reading_year + - name: Farm Name + expr: farm.farm_name + - name: Region + expr: farm.region + - name: Turbine Model + expr: turbine.turbine_model + - name: Icing Flag + expr: icing_flag + +measures: + - name: Reading Count + expr: COUNT(1) + - name: Average Capacity Factor Pct + expr: AVG(capacity_factor_pct) + - name: Average Power Output KW + expr: AVG(power_output_kw) + - name: Average Anomaly Score + expr: AVG(anomaly_score) + - name: High Anomaly Count + expr: COUNT(1) FILTER (WHERE anomaly_score >= 70) + - name: Average Gearbox Temp C + expr: AVG(gearbox_temp_c) + - name: Average Vibration + expr: AVG(vibration_mm_s) +""", + "mv_maintenance_reliability": f""" +version: "0.1" +source: {CATALOG}.{SCHEMA}.maintenance_events + +joins: + - name: turbine + source: {CATALOG}.{SCHEMA}.turbines + on: source.turbine_id = turbine.turbine_id + - name: farm + source: {CATALOG}.{SCHEMA}.wind_farms + on: source.farm_id = farm.farm_id + - name: component + source: {CATALOG}.{SCHEMA}.components + on: source.component_id = component.component_id + +dimensions: + - name: Maintenance Month + expr: DATE_TRUNC('MONTH', maintenance_date) + - name: Maintenance Year + expr: maintenance_year + - name: Maintenance Type + expr: maintenance_type + - name: Component Type + expr: component.component_type + - name: Farm Name + expr: farm.farm_name + - name: Turbine Model + expr: turbine.turbine_model + +measures: + - name: Maintenance Count + expr: COUNT(1) + - name: Preventive Count + expr: COUNT(1) FILTER (WHERE maintenance_type = 'Preventive') + - name: Corrective Count + expr: COUNT(1) FILTER (WHERE maintenance_type = 'Corrective') + - name: Total Maintenance Cost + expr: SUM(cost_usd) + - name: Average Duration Hours + expr: AVG(duration_hours) +""", + "mv_failure_events": f""" +version: "0.1" +source: {CATALOG}.{SCHEMA}.failure_events + +joins: + - name: turbine + source: {CATALOG}.{SCHEMA}.turbines + on: source.turbine_id = turbine.turbine_id + - name: farm + source: {CATALOG}.{SCHEMA}.wind_farms + on: source.farm_id = farm.farm_id + +dimensions: + - name: Failure Month + expr: DATE_TRUNC('MONTH', failure_date) + - name: Failure Year + expr: failure_year + - name: Component Type + expr: component_type + - name: Failure Type + expr: failure_type + - name: Severity + expr: severity + - name: Turbine Model + expr: turbine.turbine_model + - name: Farm Name + expr: farm.farm_name + - name: Recent Preventive Flag + expr: had_recent_preventive + +measures: + - name: Failure Count + expr: COUNT(1) + - name: Total Downtime Hours + expr: SUM(downtime_hours) + - name: Average Downtime Hours + expr: AVG(downtime_hours) + - name: Total Repair Cost + expr: SUM(repair_cost_usd) + - name: Recent Preventive Failure Count + expr: COUNT(1) FILTER (WHERE had_recent_preventive = TRUE) +""", +} + +for mv_name, yaml_body in metric_views.items(): + ddl = ( + f"CREATE OR REPLACE VIEW `{CATALOG}`.`{SCHEMA}`.`{mv_name}`\n" + "WITH METRICS\n" + "LANGUAGE YAML\n" + "AS $$" + + yaml_body + + "$$" + ) + spark.sql(ddl) + print(f" OK {mv_name}") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 12. Register Constraints & Comments + +# COMMAND ---------- + +# ============================================================================= +# REGISTER CONSTRAINTS AND COMMENTS +# ============================================================================= +print("Registering constraints and comments...") + +PRIMARY_KEYS = { + "wind_farms": "farm_id", + "turbines": "turbine_id", + "components": "component_id", + "sensor_readings": "reading_id", + "maintenance_events": "maintenance_id", + "failure_events": "failure_id", +} + +FOREIGN_KEYS = [ + ("turbines", "farm_id", "wind_farms", "farm_id"), + ("components", "turbine_id", "turbines", "turbine_id"), + ("sensor_readings", "farm_id", "wind_farms", "farm_id"), + ("sensor_readings", "turbine_id", "turbines", "turbine_id"), + ("maintenance_events", "farm_id", "wind_farms", "farm_id"), + ("maintenance_events", "turbine_id", "turbines", "turbine_id"), + ("maintenance_events", "component_id", "components", "component_id"), + ("failure_events", "farm_id", "wind_farms", "farm_id"), + ("failure_events", "turbine_id", "turbines", "turbine_id"), + ("failure_events", "component_id", "components", "component_id"), +] + +for table, pk in PRIMARY_KEYS.items(): + spark.sql(f"ALTER TABLE {C}.`{table}` ALTER COLUMN `{pk}` SET NOT NULL") + spark.sql(f"ALTER TABLE {C}.`{table}` ADD CONSTRAINT pk_{table} PRIMARY KEY (`{pk}`)") + +for table, col, ref_table, ref_col in FOREIGN_KEYS: + spark.sql( + f"ALTER TABLE {C}.`{table}` ADD CONSTRAINT fk_{table}_{col} " + f"FOREIGN KEY (`{col}`) REFERENCES {C}.`{ref_table}` (`{ref_col}`)" + ) + + +def sql_text(value): + return value.replace("'", "''") + + +TABLE_COMMENTS = { + "wind_farms": "Wind farm dimension with region, state, terrain, and wind resource attributes.", + "turbines": "Turbine asset dimension. WT-4.2C turbines have an intentionally higher synthetic failure rate.", + "components": "Major turbine components tracked for maintenance and failure analysis.", + "sensor_readings": "Monthly turbine telemetry. Anomaly score, vibration, and gearbox temperature rise before failures.", + "maintenance_events": "Preventive, inspection, and corrective maintenance work orders with duration and cost.", + "failure_events": "Component failure facts with severity, downtime, repair cost, and recent preventive maintenance flag.", +} + +for table, comment in TABLE_COMMENTS.items(): + spark.sql(f"COMMENT ON TABLE {C}.`{table}` IS '{sql_text(comment)}'") + +for table, df_pd in TABLES.items(): + for column in df_pd.columns: + readable = column.replace("_", " ") + spark.sql( + f"ALTER TABLE {C}.`{table}` ALTER COLUMN `{column}` " + f"COMMENT '{sql_text(readable)} for the synthetic wind turbine maintenance dataset'" + ) + +print(" OK constraints and comments registered") + +print() +print("=" * 60) +print("SETUP COMPLETE") +print("=" * 60) +print(f" Catalog : {CATALOG}") +print(f" Schema : {SCHEMA}") +print(" Tables : wind_farms, turbines, components, sensor_readings, maintenance_events, failure_events") +print(" Metric Views: mv_turbine_performance, mv_maintenance_reliability, mv_failure_events") +print(" Next : Create a Genie space from these tables, or see notebooks/demo-data/README.md")