Skip to content

Commit 840f973

Browse files
BLMgithubBLMgithub
committed
feat: codify bigquery datasets and external tables; update ci pipeline and iac docs
1 parent d89ca84 commit 840f973

9 files changed

Lines changed: 114 additions & 23 deletions

File tree

‎.gcp/terraforms/bigquery.tf‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
# ------------------------------------------------------------
2+
# OPS EXTERNALIZED TABLES (For metadata caching)
3+
# ------------------------------------------------------------
4+
5+
resource "google_bigquery_connection" "biglake_connection" {
6+
connection_id = "ops_biglake_connection"
7+
location = var.region
8+
friendly_name = "BigLake Connection for GCS Parquet Scanning"
9+
cloud_resource {}
10+
}
11+
12+
# Enable connection service to access pipeline bucket
13+
resource "google_storage_bucket_iam_member" "biglake_storage_viewer" {
14+
bucket = google_storage_bucket.ops_pipeline_bucket.name
15+
role = "roles/storage.objectViewer"
16+
member = "serviceAccount:${google_bigquery_connection.biglake_connection.cloud_resource[0].service_account_id}"
17+
}
18+
19+
resource "google_bigquery_dataset" "silver_dataset" {
20+
dataset_id = var.bq_dataset_id
21+
location = var.region
22+
23+
delete_contents_on_destroy = false
24+
}
25+
26+
locals {
27+
external_tables = [
28+
"df_orders",
29+
"df_customers",
30+
"df_order_items",
31+
"df_products",
32+
"df_payments"
33+
]
34+
}
35+
36+
resource "google_bigquery_table" "external_tables" {
37+
for_each = toset(local.external_tables)
38+
dataset_id = google_bigquery_dataset.silver_dataset.dataset_id
39+
table_id = each.key
40+
41+
# Might throw error if contracted/ is empty
42+
external_data_configuration {
43+
autodetect = true
44+
source_format = "PARQUET"
45+
connection_id = google_bigquery_connection.biglake_connection.name
46+
source_uris = ["gs://${google_storage_bucket.ops_pipeline_bucket.name}/contracted/${each.key}_*.parquet"]
47+
48+
# Triggered manually by pipeline
49+
metadata_cache_mode = "MANUAL"
50+
}
51+
lifecycle {
52+
prevent_destroy = true
53+
}
54+
}
55+
56+
57+
# ------------------------------------------------------------
58+
# BIGQUERY SEMANTTIC DATASETS (For table versionining)
59+
# ------------------------------------------------------------
60+
61+
locals {
62+
# Expiration for versioned tables
63+
one_month_ms = 2678400000
64+
65+
semantic_datasets = [
66+
"seller_semantic",
67+
"customer_semantic",
68+
"product_semantic"
69+
]
70+
}
71+
72+
resource "google_bigquery_dataset" "semantic_datasets" {
73+
for_each = toset(local.semantic_datasets)
74+
dataset_id = each.key
75+
location = var.region
76+
77+
delete_contents_on_destroy = false
78+
default_table_expiration_ms = local.one_month_ms
79+
80+
description = "Semantic layer for ${each.key}. Tables expire after 1 month."
81+
82+
labels = {
83+
env = var.environment
84+
layer = "semantic"
85+
}
86+
87+
lifecycle {
88+
prevent_destroy = true
89+
}
90+
}

‎.gcp/terraforms/iam_bindings.tf‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,8 @@ locals {
2525
"roles/logging.configWriter", # Required for log-based alert policies
2626
"roles/iam.serviceAccountAdmin", # Manage Alert policies in monitoring.tf
2727
"roles/iam.admin", # Manage Iam roles
28-
"roles/bigquery.admin" # Manage BigQuery datasets and views
28+
"roles/bigquery.admin", # Manage BigQuery datasets and views
29+
"roles/serviceusage.serviceUsageAdmin", # Manage APIs
2930
]
3031
}
3132

‎.gcp/terraforms/jobs.tf‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,18 +16,26 @@ resource "google_cloud_run_v2_job" "pipeline" {
1616

1717
resources {
1818
limits = {
19-
cpu = "2"
19+
cpu = "4"
2020
memory = "8Gi"
2121
}
2222
}
2323
env {
2424
name = "POLARS_MAX_THREADS"
25-
value = "2"
25+
value = "4"
2626
}
2727
env {
2828
name = "GCP_REGION"
2929
value = var.region
3030
}
31+
env {
32+
name = "BQ_DATASET_ID"
33+
value = var.bq_dataset_id
34+
}
35+
env {
36+
name = "GCP_PROJECT"
37+
value = var.project_id
38+
}
3139

3240
volume_mounts {
3341
name = "ephemeral-disk-1"

‎.gcp/terraforms/main.tf‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ locals {
3232
"iamcredentials.googleapis.com",
3333
"drive.googleapis.com",
3434
"bigquery.googleapis.com",
35+
"bigqueryconnection.googleapis.com",
3536
]
3637
}
3738

‎.gcp/terraforms/storage.tf‎

Lines changed: 0 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -47,22 +47,3 @@ resource "google_storage_bucket" "ops_pipeline_bucket" {
4747
}
4848
}
4949

50-
# BigQuery Semantic Datasets
51-
locals {
52-
semantic_datasets = [
53-
"seller_semantic",
54-
"customer_semantic",
55-
"product_semantic"
56-
]
57-
}
58-
59-
resource "google_bigquery_dataset" "semantic_datasets" {
60-
for_each = toset(local.semantic_datasets)
61-
dataset_id = each.key
62-
location = var.region
63-
delete_contents_on_destroy = false
64-
65-
lifecycle {
66-
prevent_destroy = true
67-
}
68-
}

‎.gcp/terraforms/variables.tf‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ variable "project_id" {
44
}
55

66
variable "region" {
7-
description = "The Default GCP region"
7+
description = "The Project GCP region"
88
type = string
99
default = "us-east1"
1010
}
@@ -24,3 +24,8 @@ variable "alert_email_map" {
2424
description = "List of emails to receive pipeline alerts"
2525
sensitive = true
2626
}
27+
28+
variable "bq_dataset_id" {
29+
description = "BigQuery dataset containing externalized GCS tables"
30+
type = string
31+
}

‎.github/workflows/ci-infra.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,4 +55,5 @@ jobs:
5555
TF_VAR_region: ${{ env.REGION }}
5656
TF_VAR_github_repo: ${{ env.GITHUB_REPO }}
5757
TF_VAR_alert_email_map: ${{ secrets.ALERT_EMAIL_MAP }}
58+
TF_VAR_bq_dataset_id: ${{secrets.BQ_DATASET_ID}}
5859
run: terraform apply -auto-approve

‎dev-requirements.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ tzdata
1010
# pipeline
1111
polars==1.39.0
1212
pyarrow==19.0.0
13+
# duckdb==1.5.2
1314
google-cloud-storage
1415
google-cloud-bigquery>=3.0.0
1516
google-cloud-bigquery-storage>=2.36.0

‎docs/terraform/gcp-iac.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,10 +95,13 @@ This project implements **Zero Trust** via Workload Identity Federation and gran
9595
| Name | Type | Sensitive | Description |
9696
| :--- | :--- | :--- | :--- |
9797
| `project_id` | `string` | No | Target Google Cloud Project ID. |
98+
| `region` | `string` | No | The Project GCP region. |
9899
| `environment` | `string` | No | Deployment environment (dev, prod). |
99100
| `github_repo` | `string` | No | Format: `owner/repository`. |
101+
| `bq_dataset_id` | `string` | No | BigQuery dataset containing externalized GCS tables. |
100102
| `alert_email_map` | `map` | **Yes** | Monitoring notification recipients. |
101103

104+
102105
## State Management
103106
State is managed remotely in GCS to ensure consistency and locking.
104107
```hcl

0 commit comments

Comments
 (0)