values.yaml

# Default values for datamate.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.

global:
  namespace: datamate
  image:
    repository: "ghcr.io/modelengine-group/"
    pullPolicy: "IfNotPresent"
    backend:
      name: "datamate-backend"
      tag: "latest"
    backendPython:
      name: "datamate-backend-python"
      tag: "latest"
    gateway:
      name: "datamate-gateway"
      tag: "latest"
    frontend:
      name: "datamate-frontend"
      tag: "latest"
    runtime:
      name: "datamate-runtime"
      tag: "latest"
    database:
      name: "datamate-database"
      tag: "latest"

public:
  persistentVolumeClaim:
    storageClass: ""
    storagePath: ""
    storageNode: ""
    size:
      dataset: 10Gi
      flow: 1Gi
      log: 1Gi
      database: 1Gi
      operator: 1Gi
  secrets:
    data:
      DB_PASSWORD: "password"
      CERT_PASS: ""
      DOMAIN: ""
      HOME_PAGE_URL: ""

datasetVolume: &datasetVolume
  name: dataset-volume
  persistentVolumeClaim:
    claimName: datamate-dataset-pvc

flowVolume: &flowVolume
  name: flow-volume
  persistentVolumeClaim:
    claimName: datamate-flow-pvc

logVolume: &logVolume
  name: log-volume
  persistentVolumeClaim:
    claimName: datamate-log-pvc

dataVolume: &dataVolume
  name: data-volume
  persistentVolumeClaim:
    claimName: datamate-database-pvc

operatorVolume: &operatorVolume
  name: operator-volume
  persistentVolumeClaim:
    claimName: datamate-operator-pvc

database:
  initContainers:
    - name: init-chown
      imagePullPolicy: IfNotPresent
      command:
        - sh
        - -c
        - "chown -R 999:999 /var/log/datamate/database && chmod 750 /var/log/datamate/database"
      securityContext:
        runAsUser: 0
      volumeMounts:
        - name: log-volume
          mountPath: /var/log/datamate/database
          subPath: database
  env:
    - name: POSTGRES_USER
      value: postgres
    - name: POSTGRES_PASSWORD
      valueFrom:
        secretKeyRef:
          name: datamate-conf
          key: DB_PASSWORD
    - name: HOME_PAGE_URL
      valueFrom:
        secretKeyRef:
          name: datamate-conf
          key: HOME_PAGE_URL
  volumes:
    - *dataVolume
    - *logVolume
  volumeMounts:
    - name: data-volume
      mountPath: /var/lib/postgresql/data
      subPath: postgres
    - name: log-volume
      mountPath: /var/log/datamate/database
      subPath: database

backend:
  securityContext:
    capabilities:
      add:
        - SYS_ADMIN
  env:
    - name: DB_PASSWORD
      valueFrom:
        secretKeyRef:
          name: datamate-conf
          key: DB_PASSWORD
    - name: datamate.rag.milvus-uri
      value: "http://milvus:19530"
    - name: datamate.jwt.enable
      value: &DATAMATE_JWT_ENABLE 'false'
  volumes:
    - *datasetVolume
    - *flowVolume
    - *logVolume
    - *operatorVolume
  volumeMounts:
    - name: dataset-volume
      mountPath: /dataset
    - name: flow-volume
      mountPath: /flow
    - name: log-volume
      mountPath: /var/log/datamate
    - name: operator-volume
      mountPath: /operators

backend-python:
  env:
    - name: pgsql_password
      valueFrom:
        secretKeyRef:
          name: datamate-conf
          key: DB_PASSWORD
    - name: datamate_jwt_enable
      value: *DATAMATE_JWT_ENABLE
    - name: milvus_uri
      value: "http://milvus:19530"
  volumes:
    - *datasetVolume
    - *flowVolume
    - *logVolume
    - *operatorVolume
  volumeMounts:
    - name: dataset-volume
      mountPath: /dataset
    - name: flow-volume
      mountPath: /flow
    - name: log-volume
      mountPath: /var/log/datamate
    - name: operator-volume
      mountPath: /operators

gateway:
  env:
    - name: JWT_SECRET
      value: "default-insecure-key-change-in-production"
    - name: datamate.jwt.enable
      value: *DATAMATE_JWT_ENABLE
    - name: OMS_AUTH_ENABLED
      value: "false"
  volumes:
    - *logVolume
  volumeMounts:
    - name: log-volume
      mountPath: /var/log/datamate/gateway
      subPath: gateway

frontend:
  service:
    type: NodePort
    nodePort: 30000
  env:
    - name: CERT_PASS
      valueFrom:
        secretKeyRef:
          name: datamate-conf
          key: CERT_PASS
    - name: DOMAIN
      valueFrom:
        secretKeyRef:
          name: datamate-conf
          key: DOMAIN
    - name: REAL_IP_MODE
      value: "off"
  volumes:
    - *logVolume
    - name: cert-volume
      configMap:
        name: cube-security-nginx-cert
        optional: true
  volumeMounts:
    - name: log-volume
      mountPath: /var/log/datamate/frontend
      subPath: frontend
    - name: cert-volume
      mountPath: /cert

runtime:
  enabled: false
  args: &runtimeArgs
    - python
    - /opt/runtime/datamate/operator_runtime.py
    - --port
    - "8081"
  env: &runtimeEnv
    - name: PG_HOST
      value: "datamate-database"
    - name: PG_PORT
      value: "5432"
    - name: PG_USER
      value: "postgres"
    - name: PG_PASSWORD
      valueFrom:
        secretKeyRef:
          name: datamate-conf
          key: DB_PASSWORD
    - name: PG_DATABASE
      value: "datamate"
  volumes:
    - *datasetVolume
    - *flowVolume
    - *logVolume
    - *operatorVolume
  volumeMounts: &runtimeVolumeMounts
    - mountPath: /tmp/ray
      name: log-volume
      subPath: ray/head
    - mountPath: /var/log/datamate
      name: log-volume
    - mountPath: /dataset
      name: dataset-volume
    - mountPath: /flow
      name: flow-volume
    - mountPath: /opt/runtime/datamate/ops/user
      name: operator-volume
      subPath: extract
    - mountPath: /usr/local/lib/ops/site-packages
      name: operator-volume
      subPath: site-packages

ray-cluster:
  enabled: true
  head:
    enableInTreeAutoscaling: true
    autoscalerOptions:
      upscalingMode: Default
      idleTimeoutSeconds: 60
      imagePullPolicy: IfNotPresent
      resources:
        limits:
          cpu: "500m"
          memory: "512Mi"
        requests:
          cpu: "500m"
          memory: "512Mi"
    rayStartParams:
      num-cpus: "0"
    containerEnv:
      - name: RAY_DEDUP_LOGS
        value: "0"
      - name: RAY_TQDM_PATCH_PRINT
        value: "0"
      - name: PG_HOST
        value: "datamate-database"
      - name: PG_PORT
        value: "5432"
      - name: PG_USER
        value: "postgres"
      - name: PG_PASSWORD
        valueFrom:
          secretKeyRef:
            name: datamate-conf
            key: DB_PASSWORD
      - name: PG_DATABASE
        value: "datamate"
      - name: RAY_enable_autoscaler_v2
        value: "1"
    resources:
      limits:
        cpu: "4"
        memory: "16G"
      requests:
        cpu: "1"
        memory: "2G"
    volumes:
      - *datasetVolume
      - *flowVolume
      - *logVolume
      - *operatorVolume
    volumeMounts:
      - mountPath: /tmp/ray
        name: log-volume
        subPath: ray/head
      - mountPath: /dataset
        name: dataset-volume
      - mountPath: /flow
        name: flow-volume
      - mountPath: /opt/runtime/datamate/ops/user
        name: operator-volume
        subPath: extract
      - mountPath: /usr/local/lib/ops/site-packages
        name: operator-volume
        subPath: site-packages
    sidecarContainers:
      - name: runtime
        imagePullPolicy: IfNotPresent
        args: *runtimeArgs
        env: *runtimeEnv
        ports:
          - containerPort: 8081
        volumeMounts: *runtimeVolumeMounts
  worker:
    containerEnv:
      - name: RAY_DEDUP_LOGS
        value: "0"
      - name: RAY_TQDM_PATCH_PRINT
        value: "0"
      - name: PG_HOST
        value: "datamate-database"
      - name: PG_PORT
        value: "5432"
      - name: PG_USER
        value: "postgres"
      - name: PG_PASSWORD
        valueFrom:
          secretKeyRef:
            name: datamate-conf
            key: DB_PASSWORD
      - name: PG_DATABASE
        value: "datamate"
    resources:
      limits:
        cpu: "8"
        memory: "64G"
      requests:
        cpu: "1"
        memory: "2G"
    volumes:
      - *datasetVolume
      - *flowVolume
      - *logVolume
      - *operatorVolume
    volumeMounts:
      - mountPath: /tmp/ray
        name: log-volume
        subPath: ray/worker
      - mountPath: /dataset
        name: dataset-volume
      - mountPath: /flow
        name: flow-volume
      - mountPath: /opt/runtime/datamate/ops/user
        name: operator-volume
        subPath: extract
      - mountPath: /usr/local/lib/ops/site-packages
        name: operator-volume
        subPath: site-packages
  additionalWorkerGroups:
    npuGroup:
      disabled: false
      replicas: 0
      minReplicas: 0
      maxReplicas: 8
      rayStartParams:
        resources: '"{\"npu\": 1}"'
      containerEnv:
        - name: RAY_DEDUP_LOGS
          value: "0"
        - name: RAY_TQDM_PATCH_PRINT
          value: "0"
        - name: PG_HOST
          value: "datamate-database"
        - name: PG_PORT
          value: "5432"
        - name: PG_USER
          value: "postgres"
        - name: PG_PASSWORD
          valueFrom:
            secretKeyRef:
              name: datamate-conf
              key: DB_PASSWORD
        - name: PG_DATABASE
          value: "datamate"
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
      resources:
        limits:
          cpu: "8"
          memory: "64G"
          huawei.com/Ascend910: 1
        requests:
          cpu: "1"
          memory: "2G"
          huawei.com/Ascend910: 1
      volumes:
        - *datasetVolume
        - *flowVolume
        - *logVolume
        - *operatorVolume
        - name: ascend
          hostPath:
            path: /usr/local/Ascend
      volumeMounts:
        - mountPath: /tmp/ray
          name: log-volume
          subPathExpr: ray/$(POD_NAME)
        - mountPath: /dataset
          name: dataset-volume
        - mountPath: /flow
          name: flow-volume
        - mountPath: /opt/runtime/datamate/ops/user
          name: operator-volume
          subPath: extract
        - mountPath: /usr/local/lib/ops/site-packages
          name: operator-volume
          subPath: site-packages
        - mountPath: /usr/local/Ascend
          name: ascend
    gpuGroup:
      disabled: false
      replicas: 0
      minReplicas: 0
      maxReplicas: 8
      rayStartParams:
        resources: '"{\"gpu\": 1}"'
      containerEnv:
        - name: RAY_DEDUP_LOGS
          value: "0"
        - name: RAY_TQDM_PATCH_PRINT
          value: "0"
        - name: PG_HOST
          value: "datamate-database"
        - name: PG_PORT
          value: "5432"
        - name: PG_USER
          value: "postgres"
        - name: PG_PASSWORD
          valueFrom:
            secretKeyRef:
              name: datamate-conf
              key: DB_PASSWORD
        - name: PG_DATABASE
          value: "datamate"
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
      resources:
        limits:
          cpu: "8"
          memory: "64G"
          nvidia.com/gpu: 1
        requests:
          cpu: "1"
          memory: "2G"
          nvidia.com/gpu: 1
      volumes:
        - *datasetVolume
        - *flowVolume
        - *logVolume
        - *operatorVolume
      volumeMounts:
        - mountPath: /tmp/ray
          name: log-volume
          subPathExpr: ray/$(POD_NAME)
        - mountPath: /dataset
          name: dataset-volume
        - mountPath: /flow
          name: flow-volume
        - mountPath: /opt/runtime/datamate/ops/user
          name: operator-volume
          subPath: extract
        - mountPath: /usr/local/lib/ops/site-packages
          name: operator-volume
          subPath: site-packages