Merge pull request #1303 from oceanprotocol/feat/adapt-node-envs

switch envs, resources are shared on multiple envs and are exclusive

Author: dnsi0

.env.example

@@ -65,8 +65,13 @@ export P2P_BOOTSTRAP_NODES=
 export P2P_FILTER_ANNOUNCED_ADDRESSES=
 
 ## compute
-# Example with cross-resource constraints (constraints are optional and backwards-compatible):
-# export DOCKER_COMPUTE_ENVIRONMENTS='[{"socketPath":"/var/run/docker.sock","storageExpiry":604800,"maxJobDuration":3600,"minJobDuration":60,"fees":{"1":[{"feeToken":"0x123","prices":[{"id":"cpu","price":1},{"id":"ram","price":0.1},{"id":"disk","price":0.01}]}]},"resources":[{"id":"cpu","total":8,"max":8,"min":1,"constraints":[{"id":"ram","min":1,"max":3},{"id":"disk","min":10,"max":100}]},{"id":"ram","total":32,"max":32,"min":1},{"id":"disk","total":500,"max":500,"min":10},{"id":"gpu","total":4,"max":4,"min":0,"constraints":[{"id":"ram","min":8,"max":32},{"id":"cpu","min":2,"max":4}]}]}]'
+# Each environment defines its own resources (CPU, RAM, disk, GPUs) with full configuration.
+# CPU, RAM, and disk are per-env exclusive: inUse tracked only within the environment where the job runs.
+# A global check ensures the aggregate usage across all environments does not exceed physical capacity.
+# GPUs are shared-exclusive: if a job on envA uses gpu0, it shows as in-use on envB too.
+# CPU cores are automatically partitioned across environments based on each env's cpu.total.
+# CPU and RAM defaults are auto-detected from the system when not configured.
+# export DOCKER_COMPUTE_ENVIRONMENTS='[{"socketPath":"/var/run/docker.sock","environments":[{"id":"envA","storageExpiry":604800,"maxJobDuration":3600,"minJobDuration":60,"resources":[{"id":"cpu","total":4,"max":4,"min":1,"type":"cpu"},{"id":"ram","total":16,"max":16,"min":1,"type":"ram"},{"id":"disk","total":500,"max":500,"min":10,"type":"disk"},{"id":"gpu0","total":1,"max":1,"min":0,"type":"gpu","init":{"deviceRequests":{"Driver":"nvidia","DeviceIDs":["0"],"Capabilities":[["gpu"]]}}}],"fees":{"1":[{"feeToken":"0x123","prices":[{"id":"cpu","price":1},{"id":"ram","price":0.1},{"id":"disk","price":0.01},{"id":"gpu0","price":5}]}]}}]}]'
 export DOCKER_COMPUTE_ENVIRONMENTS=
 
 

.github/workflows/ci.yml

@@ -290,7 +290,7 @@ jobs:
           DB_PASSWORD: 'changeme'
           MAX_REQ_PER_MINUTE: 320
           MAX_CONNECTIONS_PER_MINUTE: 320
-          DOCKER_COMPUTE_ENVIRONMENTS: '[{"socketPath":"/var/run/docker.sock","resources":[{"id":"disk","total":10}],"storageExpiry":604800,"maxJobDuration":3600,"minJobDuration": 60,"fees":{"8996":[{"prices":[{"id":"cpu","price":1}]}]},"free":{"maxJobDuration":60,"minJobDuration": 10,"maxJobs":3,"resources":[{"id":"cpu","max":1},{"id":"ram","max":1},{"id":"disk","max":1}]}}]'
+          DOCKER_COMPUTE_ENVIRONMENTS: '[{"socketPath":"/var/run/docker.sock","environments":[{"storageExpiry":604800,"maxJobDuration":3600,"minJobDuration":60,"resources":[{"id":"cpu","total":4,"max":4,"min":1,"type":"cpu"},{"id":"ram","total":10,"max":10,"min":1,"type":"ram"},{"id":"disk","total":10,"max":10,"min":0,"type":"disk"}],"fees":{"8996":[{"prices":[{"id":"cpu","price":1}]}]},"free":{"maxJobDuration":60,"maxJobs":3,"resources":[{"id":"cpu","max":1},{"id":"ram","max":1},{"id":"disk","max":1}]}}]}]'
           DOCKER_REGISTRY_AUTHS: ${{ env.DOCKER_REGISTRY_AUTHS }}
       - name: Check Ocean Node is running
         run: |

config.json

@@ -93,57 +93,57 @@
   "claimDurationTimeout": 3600,
   "validateUnsignedDDO": true,
   "jwtSecret": "ocean-node-secret",
+  "enableBenchmark": false,
   "dockerComputeEnvironments": [
     {
       "socketPath": "/var/run/docker.sock",
-      "resources": [
+      "environments": [
         {
-          "id": "disk",
-          "total": 1
-        }
-      ],
-      "storageExpiry": 604800,
-      "maxJobDuration": 3600,
-      "minJobDuration": 60,
-      "access": {
-        "addresses": [],
-        "accessLists": []
-      },
-      "fees": {
-        "8996": [
-          {
-            "prices": [
+          "storageExpiry": 604800,
+          "maxJobDuration": 3600,
+          "minJobDuration": 60,
+          "resources": [
+            {
+              "id": "disk",
+              "total": 1
+            }
+          ],
+          "access": {
+            "addresses": [],
+            "accessLists": []
+          },
+          "fees": {
+            "8996": [
               {
-                "id": "cpu",
-                "price": 1
+                "prices": [
+                  {
+                    "id": "cpu",
+                    "price": 1
+                  }
+                ]
               }
             ]
-          }
-        ]
-      },
-      "free": {
-        "maxJobDuration": 3600,
-        "minJobDuration": 60,
-        "maxJobs": 3,
-        "access": {
-          "addresses": [],
-          "accessLists": []
-        },
-        "resources": [
-          {
-            "id": "cpu",
-            "max": 1
-          },
-          {
-            "id": "ram",
-            "max": 1
           },
-          {
-            "id": "disk",
-            "max": 1
+          "free": {
+            "maxJobDuration": 3600,
+            "maxJobs": 3,
+            "resources": [
+              {
+                "id": "cpu",
+                "max": 1
+              },
+              {
+                "id": "ram",
+                "max": 1
+              },
+              {
+                "id": "disk",
+                "max": 1
+              }
+            ]
           }
-        ]
-      }
+        }
+      ]
     }
   ]
 }

docs/env.md

@@ -203,6 +203,7 @@ The `DOCKER_COMPUTE_ENVIRONMENTS` environment variable should be a JSON array of
 - **imageRetentionDays** - how long docker images are kept, in days. Default: 7
 - **imageCleanupInterval** - how often to run cleanup for docker images, in seconds. Min: 3600 (1hour), Default: 86400 (24 hours)
 - **paymentClaimInterval** - how often to run payment claiming, in seconds. Default: 3600 (1 hour)
+- **enableBenchmark** - when set to `true`, the node will auto-create a benchmark compute environment at startup using the system's available resources (CPU, RAM, disk, GPUs). Default: `false`
 - **storageExpiry**: Amount of seconds for storage expiry.(Mandatory)
 - **maxJobDuration**: Maximum duration in seconds for a job.(Mandatory)
 - **minJobDuration**: Minimum duration in seconds for a job.(Mandatory)

scripts/ocean-node-quickstart.sh

@@ -239,47 +239,51 @@ if [ -z "$DOCKER_COMPUTE_ENVIRONMENTS" ]; then
   export DOCKER_COMPUTE_ENVIRONMENTS='[
     {
       "socketPath": "/var/run/docker.sock",
-      "resources": [
+      "environments": [
         {
-          "id": "disk",
-          "total": 10
-        }
-      ],
-      "storageExpiry": 604800,
-      "maxJobDuration": 36000,
-      "minJobDuration": 60,
-      "fees": {
-        "1": [
-          {
-            "feeToken": "0x123",
-            "prices": [
+          "storageExpiry": 604800,
+          "maxJobDuration": 36000,
+          "minJobDuration": 60,
+          "resources": [
+            {
+              "id": "disk",
+              "total": 10
+            }
+          ],
+          "fees": {
+            "1": [
               {
-                "id": "cpu",
-                "price": 1
+                "feeToken": "0x123",
+                "prices": [
+                  {
+                    "id": "cpu",
+                    "price": 1
+                  }
+                ]
               }
             ]
-          }
-        ]
-      },
-      "free": {
-        "maxJobDuration": 360000,
-        "minJobDuration": 60,
-        "maxJobs": 3,
-        "resources": [
-          {
-            "id": "cpu",
-            "max": 1
-          },
-          {
-            "id": "ram",
-            "max": 1
           },
-          {
-            "id": "disk",
-            "max": 1
+          "free": {
+            "maxJobDuration": 360000,
+            "minJobDuration": 60,
+            "maxJobs": 3,
+            "resources": [
+              {
+                "id": "cpu",
+                "max": 1
+              },
+              {
+                "id": "ram",
+                "max": 1
+              },
+              {
+                "id": "disk",
+                "max": 1
+              }
+            ]
           }
-        ]
-      }
+        }
+      ]
     }
   ]'
 fi
@@ -628,7 +632,7 @@ if command -v jq &> /dev/null; then
 
   if [ "$GPU_COUNT" -gt 0 ]; then
     echo "Detected $GPU_COUNT GPU type(s). Updating configuration..."
-    DOCKER_COMPUTE_ENVIRONMENTS=$(echo "$DOCKER_COMPUTE_ENVIRONMENTS" | jq --argjson gpus "$DETECTED_GPUS" '.[0].resources += $gpus')
+    DOCKER_COMPUTE_ENVIRONMENTS=$(echo "$DOCKER_COMPUTE_ENVIRONMENTS" | jq --argjson gpus "$DETECTED_GPUS" '.[0].environments[0].resources += $gpus')
     echo "GPUs added to Compute Environment resources."
   else
     echo "No GPUs detected."

src/@types/C2D/C2D.ts

@@ -141,6 +141,19 @@ export interface ComputeEnvironment extends ComputeEnvironmentBaseConfig {
   runMaxWaitTimeFree: number
 }
 
+export interface C2DEnvironmentConfig {
+  id?: string
+  description?: string
+  storageExpiry?: number
+  minJobDuration?: number
+  maxJobDuration?: number
+  maxJobs?: number
+  fees?: ComputeEnvFeesStructure
+  access?: ComputeAccessList
+  free?: ComputeEnvironmentFreeOptions
+  resources?: ComputeResource[]
+}
+
 export interface C2DDockerConfig {
   socketPath: string
   protocol: string
@@ -149,19 +162,12 @@ export interface C2DDockerConfig {
   caPath: string
   certPath: string
   keyPath: string
-  storageExpiry?: number
-  maxJobDuration?: number
-  minJobDuration?: number
-  maxJobs?: number
-  fees: ComputeEnvFeesStructure
-  resources?: ComputeResource[] // optional, owner can overwrite
-  free?: ComputeEnvironmentFreeOptions
-  access: ComputeAccessList
   imageRetentionDays?: number // Default: 7 days
   imageCleanupInterval?: number // Default: 86400 seconds (24 hours)
   paymentClaimInterval?: number // Default: 3600 seconds (1 hours)
   scanImages?: boolean
   scanImageDBUpdateInterval?: number // Default: 12 hours
+  environments: C2DEnvironmentConfig[]
   enableNetwork?: boolean // whether network is enabled for algorithm containers
 }
 

src/@types/OceanNode.ts

@@ -140,6 +140,7 @@ export interface OceanNodeConfig {
   jwtSecret?: string
   httpCertPath?: string
   httpKeyPath?: string
+  enableBenchmark?: boolean
   persistentStorage?: PersistentStorageConfig
 }