ZOO-Project-DRU Helm chart and Application version bump

gfenoy · gfenoy · commit 52ec0fce71c4 · 2026-04-20T14:09:56.000+02:00
Add dask-gateway as optional subchart dependency (dask.enabled)

Handle CloudEvents following current pubsub spec relying on the configured sink (k_sink) if set

Add stageout-file.yaml CWL wrapper to stage files

Use the ZOO-Project-DRU image tagged dru-b567b1dff96b9f720215aaa4da69dbd3a7ffe031 (using custom versions of Python modules)

Upgrade calrissian Docker image from terradue/calrissian:0.12.0 to upstream calrissian/calrissian:latest
diff --git a/skaffold.yaml b/skaffold.yaml
@@ -165,6 +165,28 @@ profiles:
         port: 9001
         localPort: 9001
 
+  # Profile: dask
+  # Activates dask-gateway deployment as a subchart of ZOO-Project-DRU.
+  # The dask-gateway Traefik proxy will be available at:
+  #   http://traefik-zoo-project-dru-dask-gateway.zoo.svc.cluster.local:80
+  # Combine with other profiles, e.g.: skaffold dev -p dask
+  - name: dask
+    patches:
+      - op: add
+        path: /deploy/helm/releases/0/setValues
+        value:
+          dask.enabled: "true"
+          dask-gateway.traefik.service.type: "ClusterIP"
+      # Use setValueTemplates to allow Skaffold image tag substitution for a custom worker image.
+      # Override gateway.backend.image.name and gateway.backend.image.tag here if needed.
+      # Example for eoap custom worker:
+      #   - op: add
+      #     path: /deploy/helm/releases/0/setValueTemplates
+      #     value:
+      #       dask-gateway.gateway.backend.image.name: ghcr.io/eoap/dev-platform-eoap/dask-gateway-worker:0.1.0
+      #       dask-gateway.gateway.backend.image.tag: "{{.IMAGE_TAG_ghcr_io_eoap_dev_platform_eoap_dask_gateway_worker}}"
+      #       dask-gateway.gateway.backend.imagePullSecrets[0].name: kaniko-secret
+
 portForward:
 
 # ZOO-Project services
diff --git a/zoo-project-dru/Chart.yaml b/zoo-project-dru/Chart.yaml
@@ -23,13 +23,13 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.9.2
+version: 0.9.3
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "0.2.40"
+appVersion: "0.2.41"
 
 keywords:
   - workflows
@@ -70,13 +70,23 @@ dependencies:
     repository: https://prometheus-community.github.io/helm-charts
     condition: monitoring.enabled
 
+  - name: dask-gateway
+    version: "2024.1.0"
+    repository: https://helm.dask.org/
+    condition: dask.enabled
+
 annotations:
   artifacthub.io/changes: |
-    - kind: fixed
-      description: "Fix issue with the WES support using zoo-wes-runner < 0.2.1"
-      links:
-        - name: GitHub Commit
-          url: https://github.com/ZOO-Project/zoo-wes-runner/commit/c3298222a85e184d953e25b51e2c9e9a633e920d
+    - kind: added
+      description: "Add dask-gateway as optional subchart dependency (dask.enabled)"
+    - kind: added
+      description: "Handle CloudEvents following current pubsub spec relying on the configured sink (k_sink) if set"
+    - kind: added
+      description: "Add stageout-file.yaml CWL wrapper to stage files"
+    - kind: changed
+      description: "Use the ZOO-Project-DRU image tagged dru-b567b1dff96b9f720215aaa4da69dbd3a7ffe031 (using custom versions of Python modules)"
+    - kind: added
+      description: "Upgrade calrissian Docker image from terradue/calrissian:0.12.0 to upstream calrissian/calrissian:latest"
 
   artifacthub.io/license: Apache-2.0
 
diff --git a/zoo-project-dru/files/bin/publish.py b/zoo-project-dru/files/bin/publish.py
@@ -20,35 +20,36 @@
 #print(data)
 
 from urllib import parse
+params=parse.parse_qs(os.environ["QUERY_STRING"])
 
-# Ensure that only the zoofpm pod is allowed to invoke the publish CGI script
-config = configparser.ConfigParser()
-config.read("/usr/lib/cgi-bin/main.cfg")
-driver = ogr.GetDriverByName("PostgreSQL")
-connection_string=""
-for key in config["database"]:
-    if key != "type" and key != "schema":
-        connection_string += f"{key}={config['database'][key]} "
-ds=driver.Open(f"PG:{connection_string}")
-layer=ds.ExecuteSQL("SELECT host from servers")
-if layer is not None and layer.GetFeatureCount() > 0:
-    hasAddress = False
-    for i in range(layer.GetFeatureCount()):
-        host = layer.GetFeature(i).GetFieldAsString("host")
-        if os.environ["REMOTE_ADDR"] == host:
-            print("You are allowed to publish this job", file=sys.stderr)
-            hasAddress = True
-            break
-    if not(hasAddress):
+if "jobid" in params:
+    # Ensure that only the zoofpm pod is allowed to invoke the publish CGI script
+    config = configparser.ConfigParser()
+    config.read("/usr/lib/cgi-bin/main.cfg")
+    driver = ogr.GetDriverByName("PostgreSQL")
+    connection_string=""
+    for key in config["database"]:
+        if key != "type" and key != "schema":
+            connection_string += f"{key}={config['database'][key]} "
+    ds=driver.Open(f"PG:{connection_string}")
+    layer=ds.ExecuteSQL("SELECT host from servers")
+    if layer is not None and layer.GetFeatureCount() > 0:
+        hasAddress = False
+        for i in range(layer.GetFeatureCount()):
+            host = layer.GetFeature(i).GetFieldAsString("host")
+            if os.environ["REMOTE_ADDR"] == host:
+                print("You are allowed to publish this job", file=sys.stderr)
+                hasAddress = True
+                break
+        if not(hasAddress):
+            print("You are not allowed to publish this job", file=sys.stderr)
+            sys.exit(0)
+    else:
         print("You are not allowed to publish this job", file=sys.stderr)
         sys.exit(0)
-else:
-    print("You are not allowed to publish this job", file=sys.stderr)
-    sys.exit(0)
 
 {{- if .Values.redis.enabled }}
 try:
-    params=parse.parse_qs(os.environ["QUERY_STRING"])
     r=None
     if "ZOO_REDIS_HOST" in os.environ:
         r = redis.Redis(host=os.environ["ZOO_REDIS_HOST"], port=6379, db=0)
@@ -63,23 +64,41 @@
 {{- end }}
 
 {{- if .Values.notifications.enabled }}
+ce_name_prefix="org.ogc.api.process."
+ce_name_job_prefix=ce_name_prefix + "job."
 k_sink = os.environ.get("K_SINK", None)
 k_ce_overrides_str = os.environ.get("K_CE_OVERRIDES", None)
 configp = configparser.ConfigParser()
 configp.read("/usr/lib/cgi-bin/oas.cfg")
 try:
     if k_sink is not None:
-        params=parse.parse_qs(os.environ["QUERY_STRING"])
         #print(params,file=sys.stderr)
         #print(f"k_sink: {k_sink}",file=sys.stderr)
         # TODO: detect the userid and add it to the URLs
         data = json.loads(data)
         host_name = configp["openapi"]["rootUrl"]
         root_path = configp["openapi"]["rootPath"]
+
+        if "operation" in params:
+            operation = params["operation"][0]
+            attributes = {
+                "type": ce_name_job_prefix + operation,
+                "source": f"{host_name}/{root_path}/job/{job_id}",
+            }
+            if operation != "undeploy":
+                attributes["dataschema"] = "https://schemas.opengis.net/ogcapi/processes/part1/1.0/openapi/schemas/link.yaml"
+                attributes["datacontenttype"] = "application/json"
+            else:
+                attributes["datacontenttype"] = "text/plain"
+            event = CloudEvent(attributes, data)
+            headers, body = to_binary(event)
+            requests.post(k_sink, data=body, headers=headers)
+            sys.exit(0)
+
         try:
             job_id = data["id"]
             attributes = {
-                "type": "org.eoepca.ogc-api-processes-notification.job.update",
+                "type": ce_name_job_prefix + "update",
                 "source": f"{host_name}/{root_path}/job/{job_id}",
                 "datacontenttype": "application/json",
                 "dataschema": "https://schemas.opengis.net/ogcapi/processes/part1/1.0/openapi/schemas/statusInfo.yaml",
@@ -89,7 +108,7 @@
             data = data["stac"]["value"]
             job_id = data["id"]
             attributes = {
-                "type": "org.eoepca.ogc-api-processes-notification.job.result",
+                "type": ce_name_job_prefix + "result",
                 "source": f"{host_name}/{root_path}/job/{job_id}/results",
                 "datacontenttype": "application/json",
                 #"dataschema": "https://schemas.opengis.net/ogcapi/processes/part1/1.0/openapi/schemas/results.yaml",
diff --git a/zoo-project-dru/files/cwlwrapper-assets/stageout-file.yaml b/zoo-project-dru/files/cwlwrapper-assets/stageout-file.yaml
@@ -0,0 +1,160 @@
+cwlVersion: v1.2
+
+class: CommandLineTool
+id: stageout-file
+
+doc: "Stage-out a file to S3 storage"
+
+inputs:
+  file:
+    doc: "The file to stage out"
+    label: "File to stage out"
+    type: File
+  
+  STAGEOUT_OUTPUT:
+    doc: "The S3 output path (e.g., s3://bucket-name)"
+    label: "S3 output path"
+    type: string
+  
+  process:
+    doc: "The process ID"
+    label: "Process ID"
+    type: string
+  
+  STAGEOUT_AWS_ACCESS_KEY_ID:
+    doc: "AWS access key ID"
+    label: "AWS access key ID"
+    type: string
+  
+  STAGEOUT_AWS_SECRET_ACCESS_KEY:
+    doc: "AWS secret access key"
+    label: "AWS secret access key"
+    type: string
+  
+  STAGEOUT_AWS_REGION:
+    doc: "AWS region"
+    label: "AWS region"
+    type: string
+  
+  STAGEOUT_AWS_SERVICEURL:
+    doc: "AWS service URL (for S3-compatible storage)"
+    label: "AWS service URL"
+    type: string
+
+outputs:
+  s3_file_output:
+    doc: "The S3 URI of the staged-out file"
+    label: "S3 file URI"
+    type: https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI
+    outputBinding:
+      glob: file-uri.txt
+      loadContents: true
+      outputEval: |
+        ${
+          return { "value": self[0].contents.trim(), "type": "https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml#URI" };
+        }
+
+stdout: file-uri.txt
+
+baseCommand:
+  - python
+  - stage.py
+
+arguments:
+  - $(inputs.file.path)
+  - $(inputs.STAGEOUT_OUTPUT)
+  - ${
+      var firstPart = (Math.random() * 46656) | 0;
+      var secondPart = (Math.random() * 46656) | 0;
+      firstPart = ("000" + firstPart.toString(36)).slice(-3);
+      secondPart = ("000" + secondPart.toString(36)).slice(-3);
+      return inputs.process + "-" + firstPart + secondPart;
+    }
+
+requirements:
+  NetworkAccess:
+    networkAccess: true
+  
+  SchemaDefRequirement:
+    types:
+      - $import: https://raw.githubusercontent.com/eoap/schemas/main/string_format.yaml
+  
+  DockerRequirement:
+    dockerPull: ghcr.io/eoap/mastering-app-package/stage:1.0.0
+  
+  InlineJavascriptRequirement: {}
+  
+  EnvVarRequirement:
+    envDef:
+      STAGEOUT_AWS_ACCESS_KEY_ID: $(inputs.STAGEOUT_AWS_ACCESS_KEY_ID)
+      STAGEOUT_AWS_SECRET_ACCESS_KEY: $(inputs.STAGEOUT_AWS_SECRET_ACCESS_KEY)
+      STAGEOUT_AWS_REGION: $(inputs.STAGEOUT_AWS_REGION)
+      STAGEOUT_AWS_SERVICEURL: $(inputs.STAGEOUT_AWS_SERVICEURL)
+  
+  ResourceRequirement: {}
+  
+  InitialWorkDirRequirement:
+    listing:
+      - entryname: stage.py
+        entry: |-
+          import os
+          import sys
+          import boto3
+          import botocore
+          from urllib.parse import urlparse
+
+          file_path = sys.argv[1]
+          bucket_uri = sys.argv[2]
+          subfolder = sys.argv[3]
+          if not bucket_uri.startswith("s3://"):
+              bucket_uri = "s3://" + bucket_uri
+
+          print(f"file_path: {file_path}", file=sys.stderr)
+          print(f"bucket_uri: {bucket_uri}", file=sys.stderr)
+          print(f"subfolder: {subfolder}", file=sys.stderr)
+
+          aws_access_key_id = os.environ["STAGEOUT_AWS_ACCESS_KEY_ID"]
+          aws_secret_access_key = os.environ["STAGEOUT_AWS_SECRET_ACCESS_KEY"]
+          region_name = os.environ["STAGEOUT_AWS_REGION"]
+          endpoint_url = os.environ["STAGEOUT_AWS_SERVICEURL"]
+
+          # Parse bucket URI (e.g., s3://bucket-name or s3://bucket-name/path)
+          parsed = urlparse(bucket_uri)
+          bucket = parsed.netloc
+          base_path = parsed.path.lstrip("/") if parsed.path else ""
+
+          # Create S3 client
+          client = boto3.client(
+              "s3",
+              aws_access_key_id=aws_access_key_id,
+              aws_secret_access_key=aws_secret_access_key,
+              endpoint_url=endpoint_url,
+              region_name=region_name,
+          )
+
+          # Get file name from path
+          file_name = os.path.basename(file_path)
+
+          print(f"Base path {base_path}", file=sys.stderr)
+
+          # Construct S3 key
+          if base_path:
+              s3_key = os.path.normpath(f"{base_path}/{subfolder}/{file_name}")
+          else:
+              s3_key = os.path.normpath(f"{subfolder}/{file_name}")
+
+          # Remove leading slashes from s3_key
+          s3_key = s3_key.lstrip("/")
+          bucket = bucket.lstrip("/")
+          print(f"Target s3://{bucket}/", file=sys.stderr)
+          print(f"Target s3://{bucket}/{s3_key}", file=sys.stderr)
+          
+
+          # Upload file to S3
+          print(f"Uploading {file_path} to s3://{bucket}/{s3_key}", file=sys.stderr)
+          client.upload_file(file_path, bucket, s3_key)
+
+          # Construct and return the S3 URI
+          s3_uri = f"s3://{bucket}/{s3_key}"
+          print(f"File staged out to: {s3_uri}", file=sys.stderr)
+          print(s3_uri)
diff --git a/zoo-project-dru/templates/dp-zoofpm.yaml b/zoo-project-dru/templates/dp-zoofpm.yaml
@@ -201,6 +201,11 @@ spec:
               value: {{ $v }}
             {{- end }}
 
+            {{- if .Values.dask.enabled }}
+            - name: DASK_GATEWAY_URL
+              value: {{ if .Values.dask.gatewayUrl }}{{ .Values.dask.gatewayUrl }}{{ else }}{{ printf "http://traefik-%s-dask-gateway.%s.svc.cluster.local:80" .Release.Name .Release.Namespace }}{{ end }}
+            {{- end }}
+
           volumeMounts:
             - name: zoo-project-config
               mountPath: /usr/lib/cgi-bin/main.cfg
@@ -250,6 +255,10 @@ spec:
             - name: cwlwrapper-config
               mountPath: /assets/stagein-file.yaml
               subPath: stagein-file.yaml
+
+            - name: cwlwrapper-config
+              mountPath: /assets/stageout-file.yaml
+              subPath: stageout-file.yaml
             {{- end }}
 
             - name: ades-processing-services
diff --git a/zoo-project-dru/values.schema.json b/zoo-project-dru/values.schema.json
diff --git a/zoo-project-dru/values.yaml b/zoo-project-dru/values.yaml
