Skip to content

Commit 319ee60

Browse files
BLMgithubBLMgithub
committed
feat: add terraform IaC for pipeline orchestration and ci/cd auth
1 parent b77abca commit 319ee60

9 files changed

Lines changed: 333 additions & 0 deletions

File tree

‎.gcp/.gitignore‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
**/.terraform/*
2+
3+
*.tfstate
4+
*.tfstate.*
5+
6+
crash.log
7+
crash.*.log
8+
9+
*.tfvars
10+
*.tfvars.json
11+
12+
override.tf
13+
override.tf.json
14+
*_override.tf
15+
*_override.tf.json
16+
17+
18+
*tfplan*

‎.gcp/terraforms/.terraform.lock.hcl‎

Lines changed: 21 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎.gcp/terraforms/iam.tf‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Create required SAs
2+
locals {
3+
service_accounts = [
4+
"job-invoker-sa",
5+
"eventarc-invoker-sa",
6+
"ops-pipeline-sa",
7+
"drive-extractor-sa",
8+
"github-actions-deployer"
9+
]
10+
}
11+
12+
resource "google_service_account" "platform_accounts" {
13+
for_each = toset(local.service_accounts)
14+
account_id = each.key
15+
display_name = "Managed by Terraform: ${each.key}"
16+
project = var.project_id
17+
18+
depends_on = [google_project_service.enabled_APIs]
19+
}

‎.gcp/terraforms/iam_bindings.tf‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
# ------------------------------------------------------------
2+
# PROJECT-LEVEL BINDINGS
3+
# ------------------------------------------------------------
4+
5+
# SA to enable repository deployment
6+
resource "google_service_account_iam_member" "github_deployer_sa" {
7+
service_account_id = google_service_account.platform_accounts["github-actions-deployer"].name
8+
role = "roles/iam.workloadIdentityUser"
9+
member = "principalSet://iam.googleapis.com/${google_iam_workload_identity_pool.github_pool.name}/attribute.repository/${var.github_repo}"
10+
}
11+
12+
# Roles for github-actions-deployer
13+
locals {
14+
deployer_roles = [
15+
"roles/run.developer",
16+
"roles/workflows.editor",
17+
"roles/cloudscheduler.admin",
18+
"roles/iam.serviceAccountUser"
19+
]
20+
}
21+
22+
# Enables to:
23+
# Deploy Cloud Run containers
24+
# Deploy pipeline-dispatcher.yml
25+
# Update the cron job
26+
# Attach SAs to Cloud Run
27+
resource "google_project_iam_member" "github_deployer_permissions" {
28+
for_each = toset(local.deployer_roles)
29+
project = var.project_id
30+
role = each.key
31+
member = "serviceAccount:${google_service_account.platform_accounts["github-actions-deployer"].email}"
32+
}
33+
34+
# Enable to receive events
35+
resource "google_project_iam_member" "eventarc_event_receiver" {
36+
project = var.project_id
37+
role = "roles/eventarc.eventReceiver"
38+
member = "serviceAccount:${google_service_account.platform_accounts["eventarc-invoker-sa"].email}"
39+
}
40+
41+
# Enable Eventarc to trigger Workflows
42+
resource "google_project_iam_member" "eventarc_workflows_invoker" {
43+
project = var.project_id
44+
role = "roles/workflows.invoker"
45+
member = "serviceAccount:${google_service_account.platform_accounts["eventarc-invoker-sa"].email}"
46+
}
47+
48+
49+
# ------------------------------------------------------------
50+
# RESOURCE-LEVEL BINDINGS (Storage & Compute)
51+
# ------------------------------------------------------------
52+
53+
# Extractor Bucket Access
54+
resource "google_storage_bucket_iam_member" "extractor_pipeline_bucket_access" {
55+
bucket = google_storage_bucket.ops_pipeline_bucket.name
56+
role = "roles/storage.objectAdmin"
57+
member = "serviceAccount:${google_service_account.platform_accounts["drive-extractor-sa"].email}"
58+
}
59+
60+
# Pipeline Runner Bucket Access
61+
resource "google_storage_bucket_iam_member" "pipeline_runner_archival_access" {
62+
bucket = google_storage_bucket.ops_archival_bucket.name
63+
role = "roles/storage.objectAdmin"
64+
member = "serviceAccount:${google_service_account.platform_accounts["ops-pipeline-sa"].email}"
65+
}
66+
67+
resource "google_storage_bucket_iam_member" "pipeline_runner_pipeline_access" {
68+
bucket = google_storage_bucket.ops_pipeline_bucket.name
69+
role = "roles/storage.objectAdmin"
70+
member = "serviceAccount:${google_service_account.platform_accounts["ops-pipeline-sa"].email}"
71+
}
72+
73+
74+
# ------------------------------------------------------------
75+
# GOOGLE SERVICE AGENTS (Pub/Sub)
76+
# ------------------------------------------------------------
77+
78+
# GCS Service Agent email
79+
data "google_storage_project_service_account" "gcs_account" {
80+
project = var.project_id
81+
}
82+
83+
# Enable permssion to publishing to agent
84+
resource "google_project_iam_member" "gcs_pubsub_publishing" {
85+
project = var.project_id
86+
role = "roles/pubsub.publisher"
87+
member = "serviceAccount:${data.google_storage_project_service_account.gcs_account.email_address}"
88+
}

‎.gcp/terraforms/main.tf‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
provider "google" {
2+
project = var.project_id
3+
region = var.region
4+
}
5+
6+
# Enable needed GCP APIs
7+
locals {
8+
services = [
9+
"iam.googleapis.com",
10+
"cloudresourcemanager.googleapis.com",
11+
"run.googleapis.com",
12+
"workflows.googleapis.com",
13+
"eventarc.googleapis.com",
14+
"cloudscheduler.googleapis.com",
15+
"iamcredentials.googleapis.com"
16+
]
17+
}
18+
19+
resource "google_project_service" "enabled_APIs" {
20+
for_each = toset(local.services)
21+
project = var.project_id
22+
service = each.key
23+
disable_on_destroy = false
24+
}

‎.gcp/terraforms/orchestration.tf‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
# Google Workflows
2+
resource "google_workflows_workflow" "pipeline_dispatcher" {
3+
name = "pipeline-trigger-flow-${var.environment}"
4+
region = var.region
5+
description = "Evaluates .success files and triggers pipeline"
6+
service_account = google_service_account.platform_accounts["eventarc-invoker-sa"].email
7+
8+
# Workflow yml
9+
source_contents = file("${path.module}/../workflows/pipeline-dispatcher.yml")
10+
11+
depends_on = [google_project_service.enabled_APIs]
12+
}
13+
14+
# Pipeline Trigger: Eventarc
15+
resource "google_eventarc_trigger" "archival_success_trigger" {
16+
name = "archival-success-trigger-${var.environment}"
17+
location = var.region
18+
19+
# Monitor Archival Bucket
20+
matching_criteria {
21+
attribute = "type"
22+
value = "google.cloud.storage.object.v1.finalized"
23+
}
24+
matching_criteria {
25+
attribute = "bucket"
26+
value = google_storage_bucket.ops_archival_bucket.name
27+
}
28+
29+
# Route event to Workflow
30+
destination {
31+
workflow = google_workflows_workflow.pipeline_dispatcher.id
32+
}
33+
34+
service_account = google_service_account.platform_accounts["eventarc-invoker-sa"].email
35+
36+
# Waits on these SAs
37+
depends_on = [
38+
google_project_iam_member.eventarc_event_receiver,
39+
google_project_iam_member.eventarc_workflows_invoker
40+
]
41+
}
42+
43+
# Drive Extractor Trigger: Cloud Scheduler
44+
resource "google_cloud_scheduler_job" "extractor_trigger" {
45+
name = "midnight-extractor-trigger-${var.environment}"
46+
description = "Execute drive-extractor daily 12AM (PHT)"
47+
schedule = "0 0 * * *"
48+
time_zone = "Asia/Manila"
49+
region = var.region
50+
51+
http_target {
52+
http_method = "POST"
53+
# Points to the Deployed Cloud Run job (data extractor)
54+
uri = "https://${var.region}-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/${var.project_id}/jobs/drive-extractor:run"
55+
56+
oauth_token {
57+
service_account_email = google_service_account.platform_accounts["job-invoker-sa"].email
58+
}
59+
}
60+
}

‎.gcp/terraforms/storage.tf‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
# Archival Bucket
2+
resource "google_storage_bucket" "ops_archival_bucket" {
3+
name = "ops-archival-bucket-${var.environment}"
4+
location = var.region
5+
force_destroy = false
6+
uniform_bucket_level_access = true
7+
8+
# To Coldline after 400 days
9+
lifecycle_rule {
10+
condition {
11+
age = 400
12+
matches_storage_class = ["STANDARD"]
13+
}
14+
action {
15+
type = "SetStorageClass"
16+
storage_class = "COLDLINE"
17+
}
18+
}
19+
20+
# Delete after 3 years
21+
lifecycle_rule {
22+
condition {
23+
age = 1095
24+
}
25+
action {
26+
type = "Delete"
27+
}
28+
}
29+
}
30+
31+
# Pipeline Bucket
32+
resource "google_storage_bucket" "ops_pipeline_bucket" {
33+
name = "ops-pipeline-bucket-${var.environment}"
34+
location = var.region
35+
force_destroy = false
36+
uniform_bucket_level_access = true
37+
38+
# Raw source, delete after 1 week
39+
lifecycle_rule {
40+
condition {
41+
age = 7
42+
matches_prefix = ["raw/"]
43+
}
44+
action {
45+
type = "Delete"
46+
}
47+
}
48+
}

‎.gcp/terraforms/variables.tf‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
variable "project_id" {
2+
description = "The GCP Project ID"
3+
type = string
4+
}
5+
6+
variable "region" {
7+
description = "The Default GCP region"
8+
type = string
9+
default = "us-east1"
10+
}
11+
12+
variable "environment" {
13+
description = "The environment (e.g. Dev, Staging, Production)"
14+
type = string
15+
}
16+
17+
variable "github_repo" {
18+
description = "GitHub Repository (Format: owner/repository)"
19+
type = string
20+
}

‎.gcp/terraforms/wif.tf‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# WIF Pool
2+
resource "google_iam_workload_identity_pool" "github_pool" {
3+
project = var.project_id
4+
workload_identity_pool_id = "github-actions-pool-${var.environment}"
5+
display_name = "GitHub Pool (${var.environment})"
6+
7+
depends_on = [google_project_service.enabled_APIs]
8+
}
9+
10+
# Access/auth provider for Github
11+
resource "google_iam_workload_identity_pool_provider" "github_provider" {
12+
project = var.project_id
13+
workload_identity_pool_id = google_iam_workload_identity_pool.github_pool.workload_identity_pool_id
14+
workload_identity_pool_provider_id = "github-provider"
15+
display_name = "GitHub Provider"
16+
17+
attribute_mapping = {
18+
"google.subject" = "assertion.sub"
19+
"attribute.actor" = "assertion.actor"
20+
"attribute.repository" = "assertion.repository"
21+
}
22+
23+
# Set allowed repo
24+
attribute_condition = "attribute.repository == \"${var.github_repo}\""
25+
26+
oidc {
27+
issuer_uri = "https://token.actions.githubusercontent.com"
28+
}
29+
}
30+
31+
# CLI output
32+
output "GITHUB_WIF_PROVIDER_NAME" {
33+
value = google_iam_workload_identity_pool_provider.github_provider.name
34+
description = "GitHub Repository Secret: WIF_PROVIDER"
35+
}

0 commit comments

Comments
 (0)