chore(k8s): restructure manifests, improve deployment stability, add GCP Simple Run

deployment/k8s/kustomizations/helm-values_hatchet.yaml → deployment/k8s/kustomizations/base/helm-values_hatchet.yaml

@@ -22,7 +22,7 @@ api:
   replicaCount: 2
   image:
     repository: "ghcr.io/hatchet-dev/hatchet/hatchet-api"
-    tag: "v0.54.7"
+    tag: "v0.72.8"
     pullPolicy: "Always"
   migrationJob:
     image:
@@ -50,6 +50,15 @@ api:
           port: 8080
         periodSeconds: 5
         initialDelaySeconds: 20
+  extraVolumes:
+    - name: hatchet-keys
+      secret:
+        secretName: hatchet-keyset
+        defaultMode: 0400
+  extraVolumeMounts:
+    - name: hatchet-keys
+      mountPath: /etc/hatchet/keys
+      readOnly: true
 
 grpc:
   enabled: true
@@ -58,7 +67,7 @@ grpc:
   replicaCount: 1
   image:
     repository: "ghcr.io/hatchet-dev/hatchet/hatchet-engine"
-    tag: "v0.54.7"
+    tag: "v0.72.8"
     pullPolicy: "Always"
   setupJob:
     enabled: false
@@ -93,6 +102,15 @@ grpc:
           port: 8733
         periodSeconds: 5
         initialDelaySeconds: 20
+  extraVolumes:
+    - name: hatchet-keys
+      secret:
+        secretName: hatchet-keyset
+        defaultMode: 0400
+  extraVolumeMounts:
+    - name: hatchet-keys
+      mountPath: /etc/hatchet/keys
+      readOnly: true
 
 controllers:
   enabled: true
@@ -101,7 +119,7 @@ controllers:
   replicaCount: 1
   image:
     repository: "ghcr.io/hatchet-dev/hatchet/hatchet-engine"
-    tag: "v0.54.7"
+    tag: "v0.72.8"
     pullPolicy: "Always"
   setupJob:
     enabled: false
@@ -136,6 +154,15 @@ controllers:
           port: 8733
         periodSeconds: 5
         initialDelaySeconds: 20
+  extraVolumes:
+    - name: hatchet-keys
+      secret:
+        secretName: hatchet-keyset
+        defaultMode: 0400
+  extraVolumeMounts:
+    - name: hatchet-keys
+      mountPath: /etc/hatchet/keys
+      readOnly: true
 
 scheduler:
   enabled: true
@@ -144,7 +171,7 @@ scheduler:
   replicaCount: 1
   image:
     repository: "ghcr.io/hatchet-dev/hatchet/hatchet-engine"
-    tag: "v0.54.7"
+    tag: "v0.72.8"
     pullPolicy: "Always"
   setupJob:
     enabled: false
@@ -179,12 +206,21 @@ scheduler:
           port: 8733
         periodSeconds: 5
         initialDelaySeconds: 20
+  extraVolumes:
+    - name: hatchet-keys
+      secret:
+        secretName: hatchet-keyset
+        defaultMode: 0400
+  extraVolumeMounts:
+    - name: hatchet-keys
+      mountPath: /etc/hatchet/keys
+      readOnly: true
 
 frontend:
   enabled: true
   image:
     repository: "ghcr.io/hatchet-dev/hatchet/hatchet-frontend"
-    tag: "v0.54.7"
+    tag: "v0.72.8"
     pullPolicy: "Always"
   service:
     externalPort: 8080
@@ -208,6 +244,8 @@ postgres:
 rabbitmq:
   enabled: true
   auth:
+    tls:
+      enabled: false
 #    username: ""
 #    password: ""
   service:

deployment/k8s/kustomizations/base/helm-values_postgresql.yaml

@@ -0,0 +1,30 @@
+auth:
+  existingSecret: r2r-hatchet-secrets
+  secretKeys:
+    adminPasswordKey: HATCHET_DATABASE_POSTGRES_POSTGRES_PASSWORD
+    userPasswordKey: HATCHET_DATABASE_POSTGRES_PASSWORD
+    replicationPasswordKey: HATCHET_DATABASE_POSTGRES_REPLICA_PASSWORD
+
+#creates hatchet database
+global:
+  storageClass: csi-sc
+  postgresql:
+    auth:
+      database: hatchet
+
+primary:
+  resources:
+    requests:
+      cpu: 200m
+      memory: 1Gi
+    limits:
+      cpu: 1
+      memory: 1Gi
+  extraEnvVars:
+    - name: POSTGRESQL_MAX_CONNECTIONS
+      value: "500"
+  initdb:
+    scripts:
+      01-create-extra-db.sql: |
+        CREATE DATABASE r2r;
+        GRANT ALL PRIVILEGES ON DATABASE r2r TO postgres;

deployment/k8s/kustomizations/include/cm-init-scripts-hatchet.yaml → deployment/k8s/kustomizations/base/include/cm-init-scripts-hatchet.yaml

@@ -7,18 +7,49 @@ metadata:
 data:
   create-db.sh: |
     #!/bin/sh
-    set -e
-    echo 'Waiting for PostgreSQL to be ready...'
+    set -eu
+
     DATABASE_POSTGRES_HOST=${DATABASE_POSTGRES_HOST:-hatchet-postgres}
-    while ! pg_isready -h ${DATABASE_POSTGRES_HOST} -p ${DATABASE_POSTGRES_PORT} -U ${DATABASE_POSTGRES_USERNAME:-hatchet_user}; do
-      sleep 1
+    DATABASE_POSTGRES_PORT=${DATABASE_POSTGRES_PORT:-5432}
+    DATABASE_POSTGRES_USERNAME=${DATABASE_POSTGRES_USERNAME:-hatchet_user}
+    DATABASE_POSTGRES_PASSWORD=${DATABASE_POSTGRES_PASSWORD:-hatchet_password}
+    DATABASE_POSTGRES_DB_NAME=${DATABASE_POSTGRES_DB_NAME:-hatchet}
+
+    echo "Waiting for PostgreSQL ($DATABASE_POSTGRES_HOST:$DATABASE_POSTGRES_PORT) to be stable..."
+
+    successes=0
+    while [ $successes -lt 5 ]; do
+      if PGPASSWORD="$DATABASE_POSTGRES_PASSWORD" \
+         pg_isready -h "$DATABASE_POSTGRES_HOST" -p "$DATABASE_POSTGRES_PORT" -U "$DATABASE_POSTGRES_USERNAME" >/dev/null 2>&1; then
+        successes=$((successes+1))
+        echo "pg_isready success $successes/5"
+      else
+        successes=0
+        echo "pg_isready failed, retrying..."
+      fi
+      sleep 2
+    done
+
+    echo "Running probe query..."
+    i=0
+    until PGPASSWORD="$DATABASE_POSTGRES_PASSWORD" \
+          psql -h "$DATABASE_POSTGRES_HOST" -p "$DATABASE_POSTGRES_PORT" -U "$DATABASE_POSTGRES_USERNAME" -d postgres -tAc "SELECT 1" | grep -q 1; do
+      i=$((i+1))
+      [ $i -ge 60 ] && { echo "Timeout waiting for Postgres query"; exit 1; }
+      echo "Probe query failed, retrying..."
+      sleep 2
     done
-    echo 'PostgreSQL is ready, checking if database exists...'
-    if ! PGPASSWORD=${DATABASE_POSTGRES_PASSWORD:-hatchet_password} psql -h ${DATABASE_POSTGRES_HOST} -p ${DATABASE_POSTGRES_PORT} -U ${DATABASE_POSTGRES_USERNAME:-hatchet_user} -lqt | grep -qw ${DATABASE_POSTGRES_DB_NAME:-hatchet}; then
-      echo 'Database does not exist, creating it...'
-      PGPASSWORD=${DATABASE_POSTGRES_PASSWORD:-hatchet_password} createdb -h ${DATABASE_POSTGRES_HOST} -p ${DATABASE_POSTGRES_PORT} -U ${DATABASE_POSTGRES_USERNAME:-hatchet_user} -w ${DATABASE_POSTGRES_DB_NAME:-hatchet}
+
+    echo "PostgreSQL is ready, checking if database \"$DATABASE_POSTGRES_DB_NAME\" exists..."
+
+    if ! PGPASSWORD="$DATABASE_POSTGRES_PASSWORD" \
+         psql -h "$DATABASE_POSTGRES_HOST" -p "$DATABASE_POSTGRES_PORT" -U "$DATABASE_POSTGRES_USERNAME" -d postgres -tAc \
+         "SELECT 1 FROM pg_database WHERE datname='${DATABASE_POSTGRES_DB_NAME}'" | grep -q 1; then
+      echo "Database \"$DATABASE_POSTGRES_DB_NAME\" does not exist, creating it..."
+      PGPASSWORD="$DATABASE_POSTGRES_PASSWORD" \
+        createdb -h "$DATABASE_POSTGRES_HOST" -p "$DATABASE_POSTGRES_PORT" -U "$DATABASE_POSTGRES_USERNAME" -w "$DATABASE_POSTGRES_DB_NAME"
     else
-      echo 'Database already exists, skipping creation.'
+      echo "Database \"$DATABASE_POSTGRES_DB_NAME\" already exists, skipping creation."
     fi
 
   setup-config.sh: |

deployment/k8s/kustomizations/include/cm-r2r.yaml → deployment/k8s/kustomizations/base/include/cm-r2r.yaml

@@ -7,7 +7,7 @@ metadata:
     argocd.argoproj.io/sync-wave: "-2"
 data:
 #  POSTGRES_HOST: "postgres"
-  R2R_POSTGRES_HOST: "r2r-documentdb"
+  R2R_POSTGRES_HOST: "postgresql"
   R2R_POSTGRES_PORT: "5432"
 #  POSTGRES_PORT: "5432"
   R2R_POSTGRES_DBNAME: "r2r"
@@ -17,8 +17,8 @@ data:
   R2R_LOG_LEVEL: INFO
 
   PYTHONUNBUFFERED: "1"
-  R2R_CONFIG_NAME: "full"
-#  R2R_CONFIG_PATH: "/app/r2r.toml"
+  R2R_CONFIG_NAME: ""
+  R2R_CONFIG_PATH: "/app/r2r.toml"
 #  R2R_CONFIG_TOML: "/app/r2r.toml"
   TELEMETRY_ENABLED: "false"
   R2R_POSTGRES_PROJECT_NAME: "r2r_default"

deployment/k8s/kustomizations/base/include/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: ai-system

deployment/k8s/kustomizations/include/r2r-dashboard-indep.yaml → deployment/k8s/kustomizations/base/include/r2r-dashboard-indep.yaml

@@ -15,7 +15,7 @@ spec:
     spec:
       containers:
       - name: r2r-dashboard
-        image: emrgntcmplxty/r2r-dashboard:1.0.1
+        image: sciphiai/r2r-dashboard:1.0.3
         ports:
         - containerPort: 3000
         env:

deployment/k8s/kustomizations/include/r2r-initc.yaml → deployment/k8s/kustomizations/base/include/r2r-initc.yaml

@@ -60,7 +60,7 @@ spec:
           readOnly: true
       containers:
       - name: r2r
-        image: "ragtoriches/prod:3.3.32"
+        image: "sciphiai/r2r:3.6.6"
         command:
           - sh
           - -c

deployment/k8s/kustomizations/kustomization.yaml → deployment/k8s/kustomizations/base/kustomization.yaml

@@ -18,13 +18,10 @@ images:
   - name: nginx
     newTag: 1.27.3-alpine3.20-slim
 
-    #https://github.com/SciPhi-AI/R2R-Dashboard/blob/main/Dockerfile
-    #https://hub.docker.com/r/emrgntcmplxty/r2r-dashboard/tags
-  - name: emrgntcmplxty/r2r-dashboard
-    newTag: 1.0.0
-    #https://hub.docker.com/r/ragtoriches/prod/tags?name=3.
-  - name: ragtoriches/prod
-    newTag: 3.4.0
+  - name: sciphiai/r2r-dashboard
+    newTag: 1.0.3
+  - name: sciphiai/r2r
+    newTag: 3.6.6
     #https://hub.docker.com/r/ragtoriches/cluster-prod/tags
   - name: ragtoriches/cluster-prod
     newTag: latest
@@ -35,22 +32,22 @@ images:
 
     #ghcr.io/hatchet-dev/hatchet/hatchet-dashboard
   - name: ghcr.io/hatchet-dev/hatchet/hatchet-dashboard
-    newTag: v0.54.7
+    newTag: v0.72.8
     #ghcr.io/hatchet-dev/hatchet/hatchet-engine
   - name: ghcr.io/hatchet-dev/hatchet/hatchet-engine
-    newTag: v0.54.7
+    newTag: v0.72.8
     #ghcr.io/hatchet-dev/hatchet/hatchet-admin
   - name: ghcr.io/hatchet-dev/hatchet/hatchet-admin
-    newTag: v0.54.7
+    newTag: v0.72.8
     #ghcr.io/hatchet-dev/hatchet/hatchet-migrate
   - name: ghcr.io/hatchet-dev/hatchet/hatchet-migrate
-    newTag: v0.54.7
+    newTag: v0.72.8
     #ghcr.io/hatchet-dev/hatchet/hatchet-api
   - name: ghcr.io/hatchet-dev/hatchet/hatchet-api
-    newTag: v0.54.7
+    newTag: v0.72.8
     #ghcr.io/hatchet-dev/hatchet/hatchet-frontend
   - name: ghcr.io/hatchet-dev/hatchet/hatchet-frontend
-    newTag: v0.54.7
+    newTag: v0.72.8
 
     #https://hub.docker.com/r/bitnami/rabbitmq/tags?name=3.
   - name: docker.io/bitnami/rabbitmq
@@ -65,6 +62,7 @@ images:
 #    newTag: 0.8.0-pg17
 
 resources:
+  - include/namespace.yaml
   - include/cm-hatchet.yaml
   - include/cm-r2r.yaml
   - include/cm-unstructured.yaml
@@ -109,25 +107,33 @@ helmCharts:
     namespace: ai-system
 
 patches:
-- path: patches/service.yaml
-  target:
-    kind: Service
-
-- path: patches/hatchet-rabbitmq-sts.yaml
-  target:
-    kind: StatefulSet
-    name: hatchet-rabbitmq
-
-# Remove secrets generated by Helm chart
-- path: patches/rm-secret-hatchet-rabbitmq-config.yaml
-  target:
-    kind: Secret
-    name: hatchet-rabbitmq-config
-- path: patches/rm-secret-hatchet-rabbitmq.yaml
-  target:
-    kind: Secret
-    name: hatchet-rabbitmq
-- path: patches/rm-secret-hatchet-shared-config.yaml
-  target:
-    kind: Secret
-    name: hatchet-shared-config
+  - path: patches/service.yaml
+    target:
+      kind: Service
+
+  - path: patches/hatchet-rabbitmq-sts.yaml
+    target:
+      kind: StatefulSet
+      name: hatchet-rabbitmq
+
+  # Mount volume with hatchet keys
+  - path: patches/hatchet-worker-token-job.yaml
+    target:
+      kind: Job
+      name: '^hatchet-[a-z0-9]{10}-worker-token$'
+
+  # Remove secrets generated by Helm chart
+  - path: patches/rm-secret-hatchet-rabbitmq.yaml
+    target:
+      kind: Secret
+      name: hatchet-rabbitmq
+
+  - path: patches/rm-secret-hatchet-rabbitmq-config.yaml
+    target:
+      kind: Secret
+      name: hatchet-rabbitmq-config
+
+  - path: patches/rm-secret-hatchet-shared-config.yaml
+    target:
+      kind: Secret
+      name: hatchet-shared-config

deployment/k8s/kustomizations/patches/hatchet-rabbitmq-sts.yaml → deployment/k8s/kustomizations/base/patches/hatchet-rabbitmq-sts.yaml

@@ -4,8 +4,8 @@ metadata:
   name: hatchet-rabbitmq
 spec:
   volumeClaimTemplates:
-    - kind: PersistentVolumeClaim
-      apiVersion: v1
+    - apiVersion: v1
+      kind: PersistentVolumeClaim
       metadata:
         name: data
       spec:

deployment/k8s/kustomizations/base/patches/hatchet-worker-token-job.yaml

@@ -0,0 +1,18 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: dummy-not-used
+spec:
+  template:
+    spec:
+      volumes:
+        - name: hatchet-keys
+          secret:
+            defaultMode: 256
+            secretName: hatchet-keyset
+      containers:
+        - name: setup-worker-token
+          volumeMounts:
+            - mountPath: /etc/hatchet/keys
+              name: hatchet-keys
+              readOnly: true