Implement memory optimization and decoupling stage executor and logic

Author: BLMgithub

.coveragerc

@@ -2,4 +2,6 @@
 omit =
     data_pipeline/prototypes/*
     data_pipeline/shared/table_configs.py
+    data_pipeline/shared/modeling_configs.py
+    data_pipeline/shared/storage_adapter.py
     */__init__.py

.gcp/terraforms/iam_bindings.tf

@@ -51,19 +51,19 @@ resource "google_project_iam_member" "eventarc_workflows_invoker" {
 # ------------------------------------------------------------
 
 # Extractor Bucket Access
-resource "google_storage_bucket_iam_member" "extractor_pipeline_bucket_access" {
-  bucket = google_storage_bucket.ops_pipeline_bucket.name
+resource "google_storage_bucket_iam_member" "drive_extractor_archival_access" {
+  bucket = google_storage_bucket.ops_archival_bucket.name
   role   = "roles/storage.objectAdmin"
   member = "serviceAccount:${google_service_account.platform_accounts["drive-extractor-sa"].email}"
 }
 
-# Pipeline Runner Bucket Access
-resource "google_storage_bucket_iam_member" "pipeline_runner_archival_access" {
-  bucket = google_storage_bucket.ops_archival_bucket.name
+resource "google_storage_bucket_iam_member" "drive_extractor_pipeline_access" {
+  bucket = google_storage_bucket.ops_pipeline_bucket.name
   role   = "roles/storage.objectAdmin"
-  member = "serviceAccount:${google_service_account.platform_accounts["ops-pipeline-sa"].email}"
+  member = "serviceAccount:${google_service_account.platform_accounts["drive-extractor-sa"].email}"
 }
 
+# Pipeline Runner Bucket Access
 resource "google_storage_bucket_iam_member" "pipeline_runner_pipeline_access" {
   bucket = google_storage_bucket.ops_pipeline_bucket.name
   role   = "roles/storage.objectAdmin"

.gitignore

@@ -7,6 +7,7 @@ __pycache__/
 runtime/
 data/published/
 data/run_artifact
+data/contracted/
 
 # local editor configs
 pyrightconfig.json