workflow-computing-unit-manager-deployment.yaml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

apiVersion: apps/v1
kind: Deployment
metadata:
  name: {{ .Release.Name }}-{{ .Values.workflowComputingUnitManager.name }}
  namespace: {{ .Release.Namespace }}
  labels:
    app: {{ .Release.Name }}-{{ .Values.workflowComputingUnitManager.name }}
spec:
  replicas: {{ .Values.workflowComputingUnitManager.numOfPods }}
  selector:
    matchLabels:
      app: {{ .Release.Name }}-{{ .Values.workflowComputingUnitManager.name }}
  template:
    metadata:
      labels:
        app: {{ .Release.Name }}-{{ .Values.workflowComputingUnitManager.name }}
    spec:
      serviceAccountName: {{ .Values.workflowComputingUnitManager.serviceAccountName }}
      initContainers:
        - name: wait-lakekeeper
          image: curlimages/curl:latest
          command:
            - /bin/sh
            - -c
            - |
              set -e
              LAKEKEEPER_BASE_URI="http://{{ .Release.Name }}-lakekeeper:{{ .Values.lakekeeper.catalog.service.externalPort }}"
              WAREHOUSE_NAME="{{ .Values.lakekeeperInit.warehouse.name }}"
              echo "Waiting for Lakekeeper to become healthy..."
              until curl -s -f "${LAKEKEEPER_BASE_URI}/health" > /dev/null 2>&1; do
                sleep 1
              done
              echo "Waiting for warehouse '${WAREHOUSE_NAME}' to exist..."
              until curl -s "${LAKEKEEPER_BASE_URI}/management/v1/warehouse" | grep -q "\"name\"[[:space:]]*:[[:space:]]*\"${WAREHOUSE_NAME}\""; do
                sleep 1
              done
              echo "Lakekeeper warehouse is ready."
      containers:
        - name: {{ .Values.workflowComputingUnitManager.name }}
          image: {{ .Values.texera.imageRegistry }}/{{ .Values.workflowComputingUnitManager.imageName }}:{{ .Values.texera.imageTag }}
          imagePullPolicy: {{ .Values.texeraImages.pullPolicy }}
          ports:
            - containerPort: {{ .Values.workflowComputingUnitManager.service.port }}
          env:
            # Kubernetes related variables
            - name: KUBERNETES_COMPUTE_UNIT_POOL_NAMESPACE
              value: {{ .Values.workflowComputingUnitPool.namespace }}
            - name: KUBERNETES_COMPUTE_UNIT_SERVICE_NAME
              value: {{ .Values.workflowComputingUnitPool.name }}-svc
            - name: KUBERNETES_IMAGE_NAME
              value: {{ .Values.texera.imageRegistry }}/{{ .Values.workflowComputingUnitPool.imageName }}:{{ .Values.texera.imageTag }}
            # TexeraDB Access
            - name: STORAGE_JDBC_URL
              value: jdbc:postgresql://{{ .Release.Name }}-postgresql:5432/texera_db?currentSchema=texera_db,public
            - name: STORAGE_JDBC_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: {{ .Release.Name }}-postgresql
                  key: postgres-password
            # FileService Access
            - name: FILE_SERVICE_GET_PRESIGNED_URL_ENDPOINT
              value: http://{{ .Values.fileService.name }}-svc:9092/api/dataset/presign-download
            - name: FILE_SERVICE_UPLOAD_ONE_FILE_TO_DATASET_ENDPOINT
              value: http://{{ .Values.fileService.name }}-svc:9092/api/dataset/did/upload
            # S3 Access (for R UDF large binary support)
            - name: STORAGE_S3_ENDPOINT
              value: http://{{ .Release.Name }}-minio:9000
            - name: STORAGE_S3_AUTH_USERNAME
              valueFrom:
                secretKeyRef:
                  name: {{ .Release.Name }}-minio
                  key: root-user
            - name: STORAGE_S3_AUTH_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: {{ .Release.Name }}-minio
                  key: root-password
            # LakeFS Access (should be removed in production environment)
            - name: STORAGE_LAKEFS_ENDPOINT
              value: http://{{ .Release.Name }}-lakefs.{{ .Release.Namespace }}:8000/api/v1
            - name: STORAGE_LAKEFS_AUTH_USERNAME
              valueFrom:
                secretKeyRef:
                  name: {{ .Release.Name }}-lakefs-secret
                  key: access_key
            - name: STORAGE_LAKEFS_AUTH_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: {{ .Release.Name }}-lakefs-secret
                  key: secret_key
            # Workflow Result
            - name: STORAGE_ICEBERG_CATALOG_TYPE
              value: rest
            - name: STORAGE_ICEBERG_CATALOG_REST_URI
              value: http://{{ .Release.Name }}-lakekeeper:{{ .Values.lakekeeper.catalog.service.externalPort }}/catalog
            - name: STORAGE_ICEBERG_CATALOG_REST_WAREHOUSE_NAME
              value: {{ .Values.lakekeeperInit.warehouse.name | quote }}
            - name: STORAGE_ICEBERG_CATALOG_REST_REGION
              value: {{ .Values.lakekeeperInit.warehouse.region | quote }}
            - name: STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET
              value: {{ .Values.lakekeeperInit.warehouse.s3Bucket | quote }}
            {{- range .Values.texeraEnvVars }}
            - name: {{ .name }}
              value: "{{ .value }}"
            {{- end }}