Skip to content

Commit a7b0d2c

Browse files
BLMgithubBLMgithub
authored
Event-driven orchestration and automated storage lifecycle
2 parents d16bc29 + 319ee60 commit a7b0d2c

13 files changed

Lines changed: 439 additions & 0 deletions

‎.dockerignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
.gitignore
33
.gitattributes
44
.github/
5+
.gcp/
56

67
.venv/
78
venv/

‎.gcp/.gitignore‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
**/.terraform/*
2+
3+
*.tfstate
4+
*.tfstate.*
5+
6+
crash.log
7+
crash.*.log
8+
9+
*.tfvars
10+
*.tfvars.json
11+
12+
override.tf
13+
override.tf.json
14+
*_override.tf
15+
*_override.tf.json
16+
17+
18+
*tfplan*

‎.gcp/lifecycle/archival-lifecycle.json‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
{
2+
"rule": [
3+
{
4+
"action": {
5+
"type": "SetStorageClass",
6+
"storageClass": "COLDLINE"
7+
},
8+
"condition": {
9+
"age": 400,
10+
"matchesStorageClass": [
11+
"STANDARD"
12+
]
13+
}
14+
},
15+
{
16+
"action": {
17+
"type": "Delete"
18+
},
19+
"condition": {
20+
"age": 1095
21+
}
22+
}
23+
]
24+
}

‎.gcp/lifecycle/staging-lifecycle.json‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
{
2+
"rule": [
3+
{
4+
"action": {
5+
"type": "Delete"
6+
},
7+
"condition": {
8+
"age": 7,
9+
"matchesPrefix": [
10+
"raw/"
11+
]
12+
}
13+
}
14+
]
15+
}

‎.gcp/terraforms/.terraform.lock.hcl‎

Lines changed: 21 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎.gcp/terraforms/iam.tf‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Create required SAs
2+
locals {
3+
service_accounts = [
4+
"job-invoker-sa",
5+
"eventarc-invoker-sa",
6+
"ops-pipeline-sa",
7+
"drive-extractor-sa",
8+
"github-actions-deployer"
9+
]
10+
}
11+
12+
resource "google_service_account" "platform_accounts" {
13+
for_each = toset(local.service_accounts)
14+
account_id = each.key
15+
display_name = "Managed by Terraform: ${each.key}"
16+
project = var.project_id
17+
18+
depends_on = [google_project_service.enabled_APIs]
19+
}

‎.gcp/terraforms/iam_bindings.tf‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
# ------------------------------------------------------------
2+
# PROJECT-LEVEL BINDINGS
3+
# ------------------------------------------------------------
4+
5+
# SA to enable repository deployment
6+
resource "google_service_account_iam_member" "github_deployer_sa" {
7+
service_account_id = google_service_account.platform_accounts["github-actions-deployer"].name
8+
role = "roles/iam.workloadIdentityUser"
9+
member = "principalSet://iam.googleapis.com/${google_iam_workload_identity_pool.github_pool.name}/attribute.repository/${var.github_repo}"
10+
}
11+
12+
# Roles for github-actions-deployer
13+
locals {
14+
deployer_roles = [
15+
"roles/run.developer",
16+
"roles/workflows.editor",
17+
"roles/cloudscheduler.admin",
18+
"roles/iam.serviceAccountUser"
19+
]
20+
}
21+
22+
# Enables to:
23+
# Deploy Cloud Run containers
24+
# Deploy pipeline-dispatcher.yml
25+
# Update the cron job
26+
# Attach SAs to Cloud Run
27+
resource "google_project_iam_member" "github_deployer_permissions" {
28+
for_each = toset(local.deployer_roles)
29+
project = var.project_id
30+
role = each.key
31+
member = "serviceAccount:${google_service_account.platform_accounts["github-actions-deployer"].email}"
32+
}
33+
34+
# Enable to receive events
35+
resource "google_project_iam_member" "eventarc_event_receiver" {
36+
project = var.project_id
37+
role = "roles/eventarc.eventReceiver"
38+
member = "serviceAccount:${google_service_account.platform_accounts["eventarc-invoker-sa"].email}"
39+
}
40+
41+
# Enable Eventarc to trigger Workflows
42+
resource "google_project_iam_member" "eventarc_workflows_invoker" {
43+
project = var.project_id
44+
role = "roles/workflows.invoker"
45+
member = "serviceAccount:${google_service_account.platform_accounts["eventarc-invoker-sa"].email}"
46+
}
47+
48+
49+
# ------------------------------------------------------------
50+
# RESOURCE-LEVEL BINDINGS (Storage & Compute)
51+
# ------------------------------------------------------------
52+
53+
# Extractor Bucket Access
54+
resource "google_storage_bucket_iam_member" "extractor_pipeline_bucket_access" {
55+
bucket = google_storage_bucket.ops_pipeline_bucket.name
56+
role = "roles/storage.objectAdmin"
57+
member = "serviceAccount:${google_service_account.platform_accounts["drive-extractor-sa"].email}"
58+
}
59+
60+
# Pipeline Runner Bucket Access
61+
resource "google_storage_bucket_iam_member" "pipeline_runner_archival_access" {
62+
bucket = google_storage_bucket.ops_archival_bucket.name
63+
role = "roles/storage.objectAdmin"
64+
member = "serviceAccount:${google_service_account.platform_accounts["ops-pipeline-sa"].email}"
65+
}
66+
67+
resource "google_storage_bucket_iam_member" "pipeline_runner_pipeline_access" {
68+
bucket = google_storage_bucket.ops_pipeline_bucket.name
69+
role = "roles/storage.objectAdmin"
70+
member = "serviceAccount:${google_service_account.platform_accounts["ops-pipeline-sa"].email}"
71+
}
72+
73+
74+
# ------------------------------------------------------------
75+
# GOOGLE SERVICE AGENTS (Pub/Sub)
76+
# ------------------------------------------------------------
77+
78+
# GCS Service Agent email
79+
data "google_storage_project_service_account" "gcs_account" {
80+
project = var.project_id
81+
}
82+
83+
# Enable permssion to publishing to agent
84+
resource "google_project_iam_member" "gcs_pubsub_publishing" {
85+
project = var.project_id
86+
role = "roles/pubsub.publisher"
87+
member = "serviceAccount:${data.google_storage_project_service_account.gcs_account.email_address}"
88+
}

‎.gcp/terraforms/main.tf‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
provider "google" {
2+
project = var.project_id
3+
region = var.region
4+
}
5+
6+
# Enable needed GCP APIs
7+
locals {
8+
services = [
9+
"iam.googleapis.com",
10+
"cloudresourcemanager.googleapis.com",
11+
"run.googleapis.com",
12+
"workflows.googleapis.com",
13+
"eventarc.googleapis.com",
14+
"cloudscheduler.googleapis.com",
15+
"iamcredentials.googleapis.com"
16+
]
17+
}
18+
19+
resource "google_project_service" "enabled_APIs" {
20+
for_each = toset(local.services)
21+
project = var.project_id
22+
service = each.key
23+
disable_on_destroy = false
24+
}

‎.gcp/terraforms/orchestration.tf‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
# Google Workflows
2+
resource "google_workflows_workflow" "pipeline_dispatcher" {
3+
name = "pipeline-trigger-flow-${var.environment}"
4+
region = var.region
5+
description = "Evaluates .success files and triggers pipeline"
6+
service_account = google_service_account.platform_accounts["eventarc-invoker-sa"].email
7+
8+
# Workflow yml
9+
source_contents = file("${path.module}/../workflows/pipeline-dispatcher.yml")
10+
11+
depends_on = [google_project_service.enabled_APIs]
12+
}
13+
14+
# Pipeline Trigger: Eventarc
15+
resource "google_eventarc_trigger" "archival_success_trigger" {
16+
name = "archival-success-trigger-${var.environment}"
17+
location = var.region
18+
19+
# Monitor Archival Bucket
20+
matching_criteria {
21+
attribute = "type"
22+
value = "google.cloud.storage.object.v1.finalized"
23+
}
24+
matching_criteria {
25+
attribute = "bucket"
26+
value = google_storage_bucket.ops_archival_bucket.name
27+
}
28+
29+
# Route event to Workflow
30+
destination {
31+
workflow = google_workflows_workflow.pipeline_dispatcher.id
32+
}
33+
34+
service_account = google_service_account.platform_accounts["eventarc-invoker-sa"].email
35+
36+
# Waits on these SAs
37+
depends_on = [
38+
google_project_iam_member.eventarc_event_receiver,
39+
google_project_iam_member.eventarc_workflows_invoker
40+
]
41+
}
42+
43+
# Drive Extractor Trigger: Cloud Scheduler
44+
resource "google_cloud_scheduler_job" "extractor_trigger" {
45+
name = "midnight-extractor-trigger-${var.environment}"
46+
description = "Execute drive-extractor daily 12AM (PHT)"
47+
schedule = "0 0 * * *"
48+
time_zone = "Asia/Manila"
49+
region = var.region
50+
51+
http_target {
52+
http_method = "POST"
53+
# Points to the Deployed Cloud Run job (data extractor)
54+
uri = "https://${var.region}-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/${var.project_id}/jobs/drive-extractor:run"
55+
56+
oauth_token {
57+
service_account_email = google_service_account.platform_accounts["job-invoker-sa"].email
58+
}
59+
}
60+
}

‎.gcp/terraforms/storage.tf‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
# Archival Bucket
2+
resource "google_storage_bucket" "ops_archival_bucket" {
3+
name = "ops-archival-bucket-${var.environment}"
4+
location = var.region
5+
force_destroy = false
6+
uniform_bucket_level_access = true
7+
8+
# To Coldline after 400 days
9+
lifecycle_rule {
10+
condition {
11+
age = 400
12+
matches_storage_class = ["STANDARD"]
13+
}
14+
action {
15+
type = "SetStorageClass"
16+
storage_class = "COLDLINE"
17+
}
18+
}
19+
20+
# Delete after 3 years
21+
lifecycle_rule {
22+
condition {
23+
age = 1095
24+
}
25+
action {
26+
type = "Delete"
27+
}
28+
}
29+
}
30+
31+
# Pipeline Bucket
32+
resource "google_storage_bucket" "ops_pipeline_bucket" {
33+
name = "ops-pipeline-bucket-${var.environment}"
34+
location = var.region
35+
force_destroy = false
36+
uniform_bucket_level_access = true
37+
38+
# Raw source, delete after 1 week
39+
lifecycle_rule {
40+
condition {
41+
age = 7
42+
matches_prefix = ["raw/"]
43+
}
44+
action {
45+
type = "Delete"
46+
}
47+
}
48+
}

0 commit comments

Comments
 (0)