Kuberay extractor working infra

Author: Vismayak

deployments/kubernetes/charts/clowder2/clowder-kuberay.yaml

@@ -0,0 +1,39 @@
+image:
+  repository: vismayak/clowder-ray
+  tag: latest
+  pullPolicy: Always
+
+head:
+  resources:
+    limits:
+      cpu: "2"
+      memory: "4G"
+    requests:
+      cpu: "2"
+      memory: "4G"
+  volumes:
+      - name: minio-dataset
+        persistentVolumeClaim:
+          claimName: minio-dataset
+  volumeMounts:
+    - name: minio-dataset
+      mountPath: /clowderfs
+
+worker:
+  replicas: 2
+  minReplicas: 2
+  maxReplicas: 2
+  resources:
+    limits:
+      cpu: "2"
+      memory: "8G"
+    requests:
+      cpu: "2"
+      memory: "8G"
+  volumes:
+      - name: minio-dataset
+        persistentVolumeClaim:
+          claimName: minio-dataset
+  volumeMounts:
+    - name: minio-dataset
+      mountPath: /clowderfs

deployments/kubernetes/charts/clowder2/ibm-hpc.yaml

@@ -86,28 +86,45 @@ heartbeat:
     tag: release-v2.0-beta-3
 
 extractors:
-  wordcount:
+  # wordcount:
+  #   enabled: true
+  #   image: clowder/extractors-wordcount:latest
+  # rcnn-iwp-inference:
+  #   enabled: true
+  #   image: vismayak/rcnn_iwp_inference_extractor_k8:latest
+  #   env:
+  #     - name: MINIO_MOUNTED_PATH
+  #       value: /clowderfs
+  #   volumes:
+  #     - name: minio-dataset
+  #       persistentVolumeClaim:
+  #         claimName: minio-dataset
+  #   volumeMounts:
+  #     - name: minio-dataset
+  #       mountPath: /clowderfs
+  # rcnn-iwp-finetuning:
+  #   enabled: true
+  #   image: vismayak/rcnn_iwp_finetuning_extractor_k8:latest
+  #   env:
+  #     - name: MINIO_MOUNTED_PATH
+  #       value: /clowderfs
+  #   volumes:
+  #     - name: minio-dataset
+  #       persistentVolumeClaim:
+  #         claimName: minio-dataset
+  #   volumeMounts:
+  #     - name: minio-dataset
+  #       mountPath: /clowderfs
+  rcnn-iwp-inference-kuberay:
     enabled: true
-    image: clowder/extractors-wordcount:latest
-  rcnn-iwp-inference:
-    enabled: true
-    image: vismayak/rcnn_iwp_inference_extractor_k8:latest
-    env:
-      - name: MINIO_MOUNTED_PATH
-        value: /clowderfs
-    volumes:
-      - name: minio-dataset
-        persistentVolumeClaim:
-          claimName: minio-dataset
-    volumeMounts:
-      - name: minio-dataset
-        mountPath: /clowderfs
-  rcnn-iwp-finetuning:
-    enabled: true
-    image: vismayak/rcnn_iwp_finetuning_extractor_k8:latest
+    image: vismayak/rcnn_iwp_inference_kuberay:latest
     env:
       - name: MINIO_MOUNTED_PATH
         value: /clowderfs
+      - name: CLOWDER_VERSION
+        value: "2"
+      - name: RAY_CLUSTER_URL
+        value: "http://clowder-raycluster-kuberay-head-svc.ibm-hpc.svc.cluster.local:8265"
     volumes:
       - name: minio-dataset
         persistentVolumeClaim:

deployments/kubernetes/charts/clowder2/minio-credentials.yaml

@@ -4,5 +4,10 @@ metadata:
   name: minio-credentials
   namespace: ibm-hpc
 stringData:
-  accessKeyID: accessKeyID
-  secretAccessKey: secretAccessKey
+# MINIO_ENDPOINT: http://clowder2-minio:9000
+  # MINIO_ACCESS_KEY: clowder
+  # MINIO_SECRET_KEY: ilikecats
+  # MINIO_MOUNT_POINT: /var/clowderfs
+  # CLOWDER_VERSION: "2"
+  accessKeyID: clowder
+  secretAccessKey: ilikecats

deployments/kubernetes/charts/clowder2/minio-dataset.yaml

@@ -5,8 +5,8 @@ metadata:
   namespace: ibm-hpc
 spec:
   local:
-    readonly: "true"
+    readonly: "false"
     bucket: clowder
     endpoint: http://clowder2-minio.ibm-hpc.svc.cluster.local:9000/
     secret-name: minio-credentials
-    type: COS
+    type: COS