fix: constrain polars threads to match allocated cpu to prevent memory spikes and SIGKILL

Author: BLMgithub

.gcp/lifecycle/archival-lifecycle.json

@@ -17,9 +17,13 @@ resource "google_cloud_run_v2_job" "pipeline" {
         resources {
           limits = {
             cpu    = "2"
-            memory = "4Gi"
+            memory = "8Gi"
           }
         }
+        env {
+          name  = "POLARS_MAX_THREADS"
+          value = "2"
+        }
       }
     }
   }

.gcp/lifecycle/staging-lifecycle.json

@@ -5,13 +5,15 @@ __pycache__/
 
 # Runtime Directory
 runtime/
+/data/raw
 data/published/
 data/run_artifact
 data/contracted/
 assets/benchmarks/benchmark.py
 
 # local editor configs
 pyrightconfig.json
+troubleshoot/
 
 # C extensions
 *.so

.gcp/terraforms/GCP Orchestration Diagram.png

@@ -9,12 +9,11 @@ services:
     volumes:
       - ./data:/app/data
 
-    deploy:
-      resources:
-        limits:
-          # Provision Specs Test
-          memory: 4G
-          cpus: '2.0'
-          # Simulate 2 virtual cpus
+    # Provision Specs: Memory 8 Gb/ 2cpu/2 threads
+    # Memory tax = Sandbox Tax +  Import Tax + I/O Buffer Tax
+    # Max Data Size = (Total RAM- 8GB) - (Memory Tax - 1.5GB) = 6.5GB
+    mem_limit: 6.5G
+    memswap_limit: 6.5G
+    cpus: '2.0'
     environment:
       - POLARS_MAX_THREADS=2