From 48474eb0933e34bd5c571d3e8d7d3cea320f8d7c Mon Sep 17 00:00:00 2001 From: mariacarmina Date: Thu, 8 May 2025 00:09:11 +0300 Subject: [PATCH 01/34] Integrate credentials checks for service and asset levels in initializeCompute and startCompute handlers. Added tests. --- src/@types/commands.ts | 2 + src/components/core/compute/initialize.ts | 77 +++++++- src/components/core/compute/startCompute.ts | 77 +++++++- src/components/policyServer/index.ts | 26 +++ src/test/data/assets.ts | 142 +++++++++++++++ src/test/integration/credentials.test.ts | 188 +++++++++++++++++++- 6 files changed, 506 insertions(+), 6 deletions(-) diff --git a/src/@types/commands.ts b/src/@types/commands.ts index f7c92dd95..4373b5ccd 100644 --- a/src/@types/commands.ts +++ b/src/@types/commands.ts @@ -183,6 +183,7 @@ export interface ComputeInitializeCommand extends Command { consumerAddress: string signature?: string maxJobDuration: number + policyServer?: any // object to pass to policy server } export interface FreeComputeStartCommand extends Command { @@ -198,6 +199,7 @@ export interface FreeComputeStartCommand extends Command { } export interface PaidComputeStartCommand extends FreeComputeStartCommand { payment: ComputePayment + policyServer?: any // object to pass to policy server } export interface ComputeStopCommand extends Command { diff --git a/src/components/core/compute/initialize.ts b/src/components/core/compute/initialize.ts index 7f5b0b529..c4c2e0c99 100644 --- a/src/components/core/compute/initialize.ts +++ b/src/components/core/compute/initialize.ts @@ -24,13 +24,15 @@ import { validateCommandParameters } from '../../httpRoutes/validateCommands.js' import { isAddress } from 'ethers' -import { getConfiguration } from '../../../utils/index.js' +import { getConfiguration, isPolicyServerConfigured } from '../../../utils/index.js' import { sanitizeServiceFiles } from '../../../utils/util.js' import { FindDdoHandler } from '../handler/ddoHandler.js' import { isOrderingAllowedForAsset } from '../handler/downloadHandler.js' import { getNonceAsNumber } from '../utils/nonceHandler.js' import { C2DEngineDocker, getAlgorithmImage } from '../../c2d/compute_engine_docker.js' import { DDOManager } from '@oceanprotocol/ddo-js' +import { areKnownCredentialTypes, checkCredentials } from '../../../utils/credentials.js' +import { PolicyServer } from '../../policyServer/index.js' export class ComputeInitializeHandler extends CommandHandler { validate(command: ComputeInitializeCommand): ValidateParams { @@ -205,6 +207,39 @@ export class ComputeInitializeHandler extends CommandHandler { } } } + // check credentials (DDO level) + let accessGrantedDDOLevel: boolean + if (ddo.credentials) { + // if POLICY_SERVER_URL exists, then ocean-node will NOT perform any checks. + // It will just use the existing code and let PolicyServer decide. + if (isPolicyServerConfigured() && task.policyServer) { + accessGrantedDDOLevel = await ( + await new PolicyServer().checkStartCompute( + ddo.id, + ddo, + elem.serviceId, + 0, + elem.transferTxId, + task.consumerAddress, + task.policyServer + ) + ).success + } else { + accessGrantedDDOLevel = areKnownCredentialTypes(ddo.credentials) + ? checkCredentials(ddo.credentials, task.consumerAddress) + : true + } + if (!accessGrantedDDOLevel) { + CORE_LOGGER.logMessage(`Error: Access to asset ${ddo.id} was denied`, true) + return { + stream: null, + status: { + httpStatus: 403, + error: `Error: Access to asset ${ddo.id} was denied` + } + } + } + } const service = AssetUtils.getServiceById(ddo, elem.serviceId) if (!service) { const error = `Cannot find service ${elem.serviceId} in DDO ${elem.documentId}` @@ -216,6 +251,46 @@ export class ComputeInitializeHandler extends CommandHandler { } } } + // check credentials on service level + // if using a policy server and we are here it means that access was granted (they are merged/assessed together) + if (service.credentials) { + let accessGrantedServiceLevel: boolean + if (isPolicyServerConfigured() && task.policyServer) { + // we use the previous check or we do it again + // (in case there is no DDO level credentials and we only have Service level ones) + accessGrantedServiceLevel = + accessGrantedDDOLevel || + (await ( + await new PolicyServer().checkStartCompute( + ddo.id, + ddo, + elem.serviceId, + 0, + elem.transferTxId, + task.consumerAddress, + task.policyServer + ) + ).success) + } else { + accessGrantedServiceLevel = areKnownCredentialTypes(service.credentials) + ? checkCredentials(service.credentials, task.consumerAddress) + : true + } + + if (!accessGrantedServiceLevel) { + CORE_LOGGER.logMessage( + `Error: Access to service with id ${service.id} was denied`, + true + ) + return { + stream: null, + status: { + httpStatus: 403, + error: `Error: Access to service with id ${service.id} was denied` + } + } + } + } const ddoInstance = DDOManager.getDDOClass(ddo) const { chainId: ddoChainId, nftAddress } = ddoInstance.getDDOFields() diff --git a/src/components/core/compute/startCompute.ts b/src/components/core/compute/startCompute.ts index 633cb353d..c2f6fc7c9 100644 --- a/src/components/core/compute/startCompute.ts +++ b/src/components/core/compute/startCompute.ts @@ -26,7 +26,7 @@ import { decrypt } from '../../../utils/crypt.js' // import { verifyProviderFees } from '../utils/feesHandler.js' import { Blockchain } from '../../../utils/blockchain.js' import { validateOrderTransaction } from '../utils/validateOrders.js' -import { getConfiguration } from '../../../utils/index.js' +import { getConfiguration, isPolicyServerConfigured } from '../../../utils/index.js' import { sanitizeServiceFiles } from '../../../utils/util.js' import { FindDdoHandler } from '../handler/ddoHandler.js' // import { ProviderFeeValidation } from '../../../@types/Fees.js' @@ -34,6 +34,8 @@ import { isOrderingAllowedForAsset } from '../handler/downloadHandler.js' import { DDOManager } from '@oceanprotocol/ddo-js' import { getNonceAsNumber, checkNonce, NonceResponse } from '../utils/nonceHandler.js' import { createHash } from 'crypto' +import { PolicyServer } from '../../policyServer/index.js' +import { areKnownCredentialTypes, checkCredentials } from '../../../utils/credentials.js' export class PaidComputeStartHandler extends CommandHandler { validate(command: PaidComputeStartCommand): ValidateParams { @@ -159,6 +161,39 @@ export class PaidComputeStartHandler extends CommandHandler { } } } + // check credentials (DDO level) + let accessGrantedDDOLevel: boolean + if (ddo.credentials) { + // if POLICY_SERVER_URL exists, then ocean-node will NOT perform any checks. + // It will just use the existing code and let PolicyServer decide. + if (isPolicyServerConfigured() && task.policyServer) { + accessGrantedDDOLevel = await ( + await new PolicyServer().checkStartCompute( + ddo.id, + ddo, + elem.serviceId, + 0, + elem.transferTxId, + task.consumerAddress, + task.policyServer + ) + ).success + } else { + accessGrantedDDOLevel = areKnownCredentialTypes(ddo.credentials) + ? checkCredentials(ddo.credentials, task.consumerAddress) + : true + } + if (!accessGrantedDDOLevel) { + CORE_LOGGER.logMessage(`Error: Access to asset ${ddo.id} was denied`, true) + return { + stream: null, + status: { + httpStatus: 403, + error: `Error: Access to asset ${ddo.id} was denied` + } + } + } + } const service = AssetUtils.getServiceById(ddo, elem.serviceId) if (!service) { const error = `Cannot find service ${elem.serviceId} in DDO ${elem.documentId}` @@ -170,6 +205,46 @@ export class PaidComputeStartHandler extends CommandHandler { } } } + // check credentials on service level + // if using a policy server and we are here it means that access was granted (they are merged/assessed together) + if (service.credentials) { + let accessGrantedServiceLevel: boolean + if (isPolicyServerConfigured() && task.policyServer) { + // we use the previous check or we do it again + // (in case there is no DDO level credentials and we only have Service level ones) + accessGrantedServiceLevel = + accessGrantedDDOLevel || + (await ( + await new PolicyServer().checkStartCompute( + ddo.id, + ddo, + elem.serviceId, + 0, + elem.transferTxId, + task.consumerAddress, + task.policyServer + ) + ).success) + } else { + accessGrantedServiceLevel = areKnownCredentialTypes(service.credentials) + ? checkCredentials(service.credentials, task.consumerAddress) + : true + } + + if (!accessGrantedServiceLevel) { + CORE_LOGGER.logMessage( + `Error: Access to service with id ${service.id} was denied`, + true + ) + return { + stream: null, + status: { + httpStatus: 403, + error: `Error: Access to service with id ${service.id} was denied` + } + } + } + } const config = await getConfiguration() const ddoInstance = DDOManager.getDDOClass(ddo) diff --git a/src/components/policyServer/index.ts b/src/components/policyServer/index.ts index e5d6425aa..45bf1c7c0 100644 --- a/src/components/policyServer/index.ts +++ b/src/components/policyServer/index.ts @@ -109,6 +109,32 @@ export class PolicyServer { return await this.askServer(command) } + // TODO: when commands for initializeCompute and startCompute will be + // implemented in policy server, we'll update these functions + async checkInitializeCompute( + documentId: string, + ddo: DDO, + serviceId: string, + fileIndex: number, + transferTxId: string, + consumerAddress: string, + policyServer: any + ): Promise { + throw new Error('Not implemented yet in policy server') + } + + async checkStartCompute( + documentId: string, + ddo: DDO, + serviceId: string, + fileIndex: number, + transferTxId: string, + consumerAddress: string, + policyServer: any + ): Promise { + throw new Error('Not implemented yet in policy server') + } + async passThrough(request: any): Promise { return await this.askServer(request) } diff --git a/src/test/data/assets.ts b/src/test/data/assets.ts index b59f69bf6..1f995fe26 100644 --- a/src/test/data/assets.ts +++ b/src/test/data/assets.ts @@ -150,6 +150,148 @@ export const downloadAssetWithCredentials = { } } +export const computeAssetWithCredentials = { + '@context': ['https://w3id.org/did/v1'], + id: '', + nftAddress: '', + version: '4.1.0', + chainId: 8996, + metadata: { + created: '2021-12-20T14:35:20Z', + updated: '2021-12-20T14:35:20Z', + type: 'dataset', + name: 'cli fixed asset', + description: 'asset published using ocean.js cli tool', + tags: ['test'], + author: 'oceanprotocol', + license: 'https://market.oceanprotocol.com/terms', + additionalInformation: { + termsAndConditions: true + } + }, + credentials: nftLevelCredentials, + services: [ + { + id: 'ccb398c50d6abd5b456e8d7242bd856a1767a890b537c2f8c10ba8b8a10e6025', + type: 'compute', + files: { + files: [ + { + type: 'url', + url: 'https://raw.githubusercontent.com/oceanprotocol/testdatasets/main/shs_dataset_test.txt', + method: 'GET' + } + ] + }, + credentials: serviceLevelCredentials, + datatokenAddress: '', + serviceEndpoint: 'https://v4.provider.oceanprotocol.com', + timeout: 86400, + compute: { + allowRawAlgorithm: false, + allowNetworkAccess: true, + publisherTrustedAlgorithmPublishers: [] as any, + publisherTrustedAlgorithms: [] as any + } + } + ], + event: {}, + nft: { + address: '', + name: 'Ocean Data NFT', + symbol: 'OCEAN-NFT', + state: 5, + tokenURI: '', + owner: '', + created: '' + }, + purgatory: { + state: false + }, + datatokens: [] as any, + stats: { + allocated: 0, + orders: 0, + price: { + value: '0' + } + } +} + +export const algoAssetWithCredentials = { + '@context': ['https://w3id.org/did/v1'], + id: '', + nftAddress: '', + version: '4.1.0', + chainId: 8996, + metadata: { + created: '2023-08-01T17:09:39Z', + updated: '2023-08-01T17:09:39Z', + type: 'algorithm', + name: 'CLi Algo', + description: 'Cli algo', + author: 'OPF', + license: 'https://market.oceanprotocol.com/terms', + additionalInformation: { + termsAndConditions: true + }, + algorithm: { + language: '', + version: '0.1', + container: { + entrypoint: 'node $ALGO', + image: 'node', + tag: 'latest', + checksum: + 'sha256:1155995dda741e93afe4b1c6ced2d01734a6ec69865cc0997daf1f4db7259a36' + } + } + }, + credentials: nftLevelCredentials, + services: [ + { + id: 'db164c1b981e4d2974e90e61bda121512e6909c1035c908d68933ae4cfaba6b0', + type: 'compute', + files: { + files: [ + { + type: 'url', + method: 'GET', + url: 'https://raw.githubusercontent.com/oceanprotocol/test-algorithm/master/javascript/algo.js', + contentType: 'text/js', + encoding: 'UTF-8' + } + ] + }, + credentials: serviceLevelCredentials, + timeout: 86400, + serviceEndpoint: 'https://v4.provider.oceanprotocol.com', + compute: { + allowRawAlgorithm: false, + allowNetworkAccess: true, + publisherTrustedAlgorithmPublishers: [] as any, + publisherTrustedAlgorithms: [] as any + } + } + ], + stats: { + allocated: 0, + orders: 0, + price: { + value: '0' + } + }, + nft: { + address: '', + name: 'Ocean Data NFT', + symbol: 'OCEAN-NFT', + state: 5, + tokenURI: '', + owner: '', + created: '' + } +} + export const computeAsset = { '@context': ['https://w3id.org/did/v1'], id: '', diff --git a/src/test/integration/credentials.test.ts b/src/test/integration/credentials.test.ts index c04ce2da8..1bb78d028 100644 --- a/src/test/integration/credentials.test.ts +++ b/src/test/integration/credentials.test.ts @@ -51,29 +51,45 @@ import { getOceanArtifactsAdressesByChainId } from '../../utils/address.js' import { publishAsset, orderAsset } from '../utils/assets.js' -import { downloadAssetWithCredentials } from '../data/assets.js' +import { + algoAsset, + computeAssetWithCredentials, + downloadAssetWithCredentials +} from '../data/assets.js' import { ganachePrivateKeys } from '../utils/addresses.js' import { homedir } from 'os' import AccessListFactory from '@oceanprotocol/contracts/artifacts/contracts/accesslists/AccessListFactory.sol/AccessListFactory.json' assert { type: 'json' } import AccessList from '@oceanprotocol/contracts/artifacts/contracts/accesslists/AccessList.sol/AccessList.json' assert { type: 'json' } import { deployAccessListContract, getContract } from '../utils/contracts.js' +import { ComputeInitializeHandler } from '../../components/core/compute/initialize.js' +import { ComputeAlgorithm, ComputeAsset } from '../../@types/index.js' +import { ComputeGetEnvironmentsHandler } from '../../components/core/compute/environments.js' +import { ComputeInitializeCommand } from '../../@types/commands.js' describe('Should run a complete node flow.', () => { let config: OceanNodeConfig let oceanNode: OceanNode let provider: JsonRpcProvider + let computeEnvironments: any + let firstEnv: any let publisherAccount: Signer let consumerAccounts: Signer[] let consumerAddresses: string[] let ddo: any + let computeDdo: any + let algoDdo: any let did: string + let computeDid: string + let algoDid: string const orderTxIds: string[] = [] const mockSupportedNetworks: RPCS = getMockSupportedNetworks() let previousConfiguration: OverrideEnvConfig[] + let artifactsAddresses: any + let paymentToken: string let blockchain: Blockchain let contractAcessList: Contract @@ -82,6 +98,8 @@ describe('Should run a complete node flow.', () => { before(async () => { provider = new JsonRpcProvider('http://127.0.0.1:8545') publisherAccount = (await provider.getSigner(0)) as Signer + artifactsAddresses = getOceanArtifactsAdresses() + paymentToken = artifactsAddresses.development.Ocean // override and save configuration (always before calling getConfig()) previousConfiguration = await setupEnvironment( @@ -94,7 +112,8 @@ describe('Should run a complete node flow.', () => { ENVIRONMENT_VARIABLES.AUTHORIZED_DECRYPTERS, ENVIRONMENT_VARIABLES.ALLOWED_ADMINS, ENVIRONMENT_VARIABLES.AUTHORIZED_PUBLISHERS, - ENVIRONMENT_VARIABLES.ADDRESS_FILE + ENVIRONMENT_VARIABLES.ADDRESS_FILE, + ENVIRONMENT_VARIABLES.DOCKER_COMPUTE_ENVIRONMENTS ], [ JSON.stringify(mockSupportedNetworks), @@ -105,7 +124,12 @@ describe('Should run a complete node flow.', () => { JSON.stringify([ await publisherAccount.getAddress() // signer 0 ]), - `${homedir}/.ocean/ocean-contracts/artifacts/address.json` + `${homedir}/.ocean/ocean-contracts/artifacts/address.json`, + '[{"socketPath":"/var/run/docker.sock","resources":[{"id":"disk","total":1000000000}],"storageExpiry":604800,"maxJobDuration":3600,"fees":{"' + + DEVELOPMENT_CHAIN_ID + + '":[{"feeToken":"' + + paymentToken + + '","prices":[{"id":"cpu","price":1}]}]},"free":{"maxJobDuration":60,"maxJobs":3,"resources":[{"id":"cpu","max":1},{"id":"ram","max":1000000000},{"id":"disk","max":1000000000}]}}]' ] ) ) @@ -185,6 +209,13 @@ describe('Should run a complete node flow.', () => { publisherAccount ) + const publishedComputeDataset = await publishAsset( + computeAssetWithCredentials, + publisherAccount + ) + + const publishedAlgo = await publishAsset(algoAsset, publisherAccount) + did = publishedDataset.ddo.id const { ddo, wasTimeout } = await waitToIndex( did, @@ -194,6 +225,30 @@ describe('Should run a complete node flow.', () => { if (!ddo) { assert(wasTimeout === true, 'published failed due to timeout!') } + + computeDid = publishedComputeDataset.ddo.id + const resolvedComputeDdo = await waitToIndex( + computeDid, + EVENTS.METADATA_CREATED, + DEFAULT_TEST_TIMEOUT * 3 + ) + const ddoCompute = resolvedComputeDdo.ddo + const timeoutCompute = resolvedComputeDdo.wasTimeout + if (!ddoCompute) { + assert(timeoutCompute === true, 'published failed due to timeout!') + } + + algoDid = publishedAlgo.ddo.id + const resolvedAlgo = await waitToIndex( + algoDid, + EVENTS.METADATA_CREATED, + DEFAULT_TEST_TIMEOUT * 3 + ) + const algo = resolvedAlgo.ddo + const timeoutAlgo = resolvedAlgo.wasTimeout + if (!algo) { + assert(timeoutAlgo === true, 'published failed due to timeout!') + } }) it('should fetch the published ddo', async () => { @@ -201,9 +256,25 @@ describe('Should run a complete node flow.', () => { command: PROTOCOL_COMMANDS.GET_DDO, id: did } - const response = await new GetDdoHandler(oceanNode).handle(getDDOTask) + let response = await new GetDdoHandler(oceanNode).handle(getDDOTask) ddo = await streamToObject(response.stream as Readable) assert(ddo.id === did, 'DDO id not matching') + + const getComputeDDOTask = { + command: PROTOCOL_COMMANDS.GET_DDO, + id: computeDid + } + response = await new GetDdoHandler(oceanNode).handle(getComputeDDOTask) + computeDdo = await streamToObject(response.stream as Readable) + assert(computeDdo.id === computeDid, 'computeDdo id not matching') + + const getAlgoDDOTask = { + command: PROTOCOL_COMMANDS.GET_DDO, + id: algoDid + } + response = await new GetDdoHandler(oceanNode).handle(getAlgoDDOTask) + algoDdo = await streamToObject(response.stream as Readable) + assert(algoDdo.id === algoDid, 'computeDdo id not matching') }) it('should start an order for all consumers', async function () { @@ -267,6 +338,115 @@ describe('Should run a complete node flow.', () => { await doCheck() }) + it('should initializeCompute work for first consumer', async function () { + this.timeout(DEFAULT_TEST_TIMEOUT * 3) + + const consumerAddress = consumerAddresses[0] + + const dataset: ComputeAsset = { + documentId: computeDid, + serviceId: computeDdo.services[0].id + } + const algorithm: ComputeAlgorithm = { + documentId: algoDid, + serviceId: algoDdo.services[0].id + } + const getEnvironmentsTask = { + command: PROTOCOL_COMMANDS.COMPUTE_GET_ENVIRONMENTS + } + const resp = await new ComputeGetEnvironmentsHandler(oceanNode).handle( + getEnvironmentsTask + ) + computeEnvironments = await streamToObject(resp.stream as Readable) + firstEnv = computeEnvironments[0] + + const initializeComputeTask: ComputeInitializeCommand = { + datasets: [dataset], + algorithm, + environment: firstEnv.id, + payment: { + chainId: DEVELOPMENT_CHAIN_ID, + token: paymentToken + }, + maxJobDuration: 2 * 60, + consumerAddress: consumerAddress, + command: PROTOCOL_COMMANDS.COMPUTE_INITIALIZE + } + const response = await new ComputeInitializeHandler(oceanNode).handle( + initializeComputeTask + ) + assert(response) + assert(response.stream, 'stream not present') + assert(response.status.httpStatus === 200, 'http status not 200') + expect(response.stream).to.be.instanceOf(Readable) + }) + + it('should NOT initializeCompute for second consumer - service level credentials', async function () { + this.timeout(DEFAULT_TEST_TIMEOUT * 3) + + const consumerAddress = consumerAddresses[1] + + const dataset: ComputeAsset = { + documentId: computeDid, + serviceId: computeDdo.services[0].id + } + const algorithm: ComputeAlgorithm = { + documentId: algoDid, + serviceId: algoDdo.services[0].id + } + const initializeComputeTask: ComputeInitializeCommand = { + datasets: [dataset], + algorithm, + environment: firstEnv.id, + payment: { + chainId: DEVELOPMENT_CHAIN_ID, + token: paymentToken + }, + maxJobDuration: 2 * 60, + consumerAddress: consumerAddress, + command: PROTOCOL_COMMANDS.COMPUTE_INITIALIZE + } + const response = await new ComputeInitializeHandler(oceanNode).handle( + initializeComputeTask + ) + assert(response) + assert(response.stream === null, 'stream is present') + assert(response.status.httpStatus === 403, 'http status not 403') + }) + + it('should NOT initializeCompute for third consumer - asset level credentials', async function () { + this.timeout(DEFAULT_TEST_TIMEOUT * 3) + + const consumerAddress = consumerAddresses[2] + + const dataset: ComputeAsset = { + documentId: computeDid, + serviceId: computeDdo.services[0].id + } + const algorithm: ComputeAlgorithm = { + documentId: algoDid, + serviceId: algoDdo.services[0].id + } + const initializeComputeTask: ComputeInitializeCommand = { + datasets: [dataset], + algorithm, + environment: firstEnv.id, + payment: { + chainId: DEVELOPMENT_CHAIN_ID, + token: paymentToken + }, + maxJobDuration: 2 * 60, + consumerAddress: consumerAddress, + command: PROTOCOL_COMMANDS.COMPUTE_INITIALIZE + } + const response = await new ComputeInitializeHandler(oceanNode).handle( + initializeComputeTask + ) + assert(response) + assert(response.stream === null, 'stream is present') + assert(response.status.httpStatus === 403, 'http status not 403') + }) + it('should not allow to download the asset for second consumer - service level credentials', async function () { this.timeout(DEFAULT_TEST_TIMEOUT * 3) From dba3198ddb9459b87759a4fbecabd0c8a38bde62 Mon Sep 17 00:00:00 2001 From: mariacarmina Date: Thu, 8 May 2025 01:07:39 +0300 Subject: [PATCH 02/34] skip lint. --- src/components/policyServer/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/components/policyServer/index.ts b/src/components/policyServer/index.ts index 45bf1c7c0..7893c2d0f 100644 --- a/src/components/policyServer/index.ts +++ b/src/components/policyServer/index.ts @@ -111,6 +111,7 @@ export class PolicyServer { // TODO: when commands for initializeCompute and startCompute will be // implemented in policy server, we'll update these functions + // eslint-disable-next-line require-await async checkInitializeCompute( documentId: string, ddo: DDO, @@ -122,7 +123,7 @@ export class PolicyServer { ): Promise { throw new Error('Not implemented yet in policy server') } - + // eslint-disable-next-line require-await async checkStartCompute( documentId: string, ddo: DDO, From dcd9e43ed2670000a4cd8931e8375aab54b2040d Mon Sep 17 00:00:00 2001 From: mariacarmina Date: Thu, 8 May 2025 01:09:59 +0300 Subject: [PATCH 03/34] Update ci docker logs. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f68fa16aa..1aa9e5438 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -231,7 +231,7 @@ jobs: done - name: docker logs - run: docker logs ocean-ocean-contracts-1 && docker logs ocean-kindcluster-1 && docker logs ocean-computetodata-1 && docker logs ocean-typesense-1 + run: docker logs ocean-contracts-1 && docker logs ocean-typesense-1 if: ${{ failure() }} - name: Checkout Ocean Node From fba187aeae4c709d5348bb05f8def54b5ee061cc Mon Sep 17 00:00:00 2001 From: mariacarmina Date: Thu, 8 May 2025 02:49:59 +0300 Subject: [PATCH 04/34] Fix lint. --- src/components/policyServer/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/components/policyServer/index.ts b/src/components/policyServer/index.ts index 7893c2d0f..5148b5cf7 100644 --- a/src/components/policyServer/index.ts +++ b/src/components/policyServer/index.ts @@ -123,6 +123,7 @@ export class PolicyServer { ): Promise { throw new Error('Not implemented yet in policy server') } + // eslint-disable-next-line require-await async checkStartCompute( documentId: string, From 5e8b77e7d3f19bb3f4461207d5d2cf5608e4d6eb Mon Sep 17 00:00:00 2001 From: mariacarmina Date: Thu, 8 May 2025 13:45:27 +0300 Subject: [PATCH 05/34] Added credentials check for free start compute. Use download endpoint from policy server. --- src/@types/commands.ts | 2 +- src/components/core/compute/startCompute.ts | 99 +++++++++++++++++++++ src/components/core/compute/utils.ts | 15 ++++ src/components/policyServer/index.ts | 26 ++++-- 4 files changed, 135 insertions(+), 7 deletions(-) diff --git a/src/@types/commands.ts b/src/@types/commands.ts index 4373b5ccd..b094c8de5 100644 --- a/src/@types/commands.ts +++ b/src/@types/commands.ts @@ -196,10 +196,10 @@ export interface FreeComputeStartCommand extends Command { output?: ComputeOutput resources?: ComputeResourceRequest[] maxJobDuration?: number + policyServer?: any // object to pass to policy server } export interface PaidComputeStartCommand extends FreeComputeStartCommand { payment: ComputePayment - policyServer?: any // object to pass to policy server } export interface ComputeStopCommand extends Command { diff --git a/src/components/core/compute/startCompute.ts b/src/components/core/compute/startCompute.ts index c2f6fc7c9..95a6d50ac 100644 --- a/src/components/core/compute/startCompute.ts +++ b/src/components/core/compute/startCompute.ts @@ -596,6 +596,105 @@ export class FreeComputeStartHandler extends CommandHandler { } } } + for (const elem of [...[task.algorithm], ...task.datasets]) { + const ddo = await new FindDdoHandler(this.getOceanNode()).findAndFormatDdo( + elem.documentId + ) + if (!ddo) { + const error = `DDO ${elem.documentId} not found` + return { + stream: null, + status: { + httpStatus: 500, + error + } + } + } + // check credentials (DDO level) + let accessGrantedDDOLevel: boolean + if (ddo.credentials) { + // if POLICY_SERVER_URL exists, then ocean-node will NOT perform any checks. + // It will just use the existing code and let PolicyServer decide. + if (isPolicyServerConfigured() && task.policyServer) { + accessGrantedDDOLevel = await ( + await new PolicyServer().checkStartCompute( + ddo.id, + ddo, + elem.serviceId, + 0, + elem.transferTxId, + task.consumerAddress, + task.policyServer + ) + ).success + } else { + accessGrantedDDOLevel = areKnownCredentialTypes(ddo.credentials) + ? checkCredentials(ddo.credentials, task.consumerAddress) + : true + } + if (!accessGrantedDDOLevel) { + CORE_LOGGER.logMessage(`Error: Access to asset ${ddo.id} was denied`, true) + return { + stream: null, + status: { + httpStatus: 403, + error: `Error: Access to asset ${ddo.id} was denied` + } + } + } + } + const service = AssetUtils.getServiceById(ddo, elem.serviceId) + if (!service) { + const error = `Cannot find service ${elem.serviceId} in DDO ${elem.documentId}` + return { + stream: null, + status: { + httpStatus: 500, + error + } + } + } + // check credentials on service level + // if using a policy server and we are here it means that access was granted (they are merged/assessed together) + if (service.credentials) { + let accessGrantedServiceLevel: boolean + if (isPolicyServerConfigured() && task.policyServer) { + // we use the previous check or we do it again + // (in case there is no DDO level credentials and we only have Service level ones) + accessGrantedServiceLevel = + accessGrantedDDOLevel || + (await ( + await new PolicyServer().checkStartCompute( + ddo.id, + ddo, + elem.serviceId, + 0, + elem.transferTxId, + task.consumerAddress, + task.policyServer + ) + ).success) + } else { + accessGrantedServiceLevel = areKnownCredentialTypes(service.credentials) + ? checkCredentials(service.credentials, task.consumerAddress) + : true + } + + if (!accessGrantedServiceLevel) { + CORE_LOGGER.logMessage( + `Error: Access to service with id ${service.id} was denied`, + true + ) + return { + stream: null, + status: { + httpStatus: 403, + error: `Error: Access to service with id ${service.id} was denied` + } + } + } + } + } try { const env = await engine.getComputeEnvironment(null, task.environment) if (!env) { diff --git a/src/components/core/compute/utils.ts b/src/components/core/compute/utils.ts index 816a1f7e2..84a41948e 100644 --- a/src/components/core/compute/utils.ts +++ b/src/components/core/compute/utils.ts @@ -76,6 +76,7 @@ export async function validateAlgoForDataset( const datasetService = services.find( (service: any) => service.id === datasetServiceId ) + CORE_LOGGER.logMessage(`datasetService: ${JSON.stringify(datasetService)}`) if (!datasetService) { throw new Error('Dataset service not found') } @@ -85,6 +86,19 @@ export async function validateAlgoForDataset( } if (algoDID) { + CORE_LOGGER.logMessage(`has algoDid: ${algoDID}`) + CORE_LOGGER.logMessage( + `is array1: ${Array.isArray(compute.publisherTrustedAlgorithms)}` + ) + CORE_LOGGER.logMessage( + `is array2: ${Array.isArray(compute.publisherTrustedAlgorithmPublishers)}` + ) + CORE_LOGGER.logMessage( + `check length 1: ${compute.publisherTrustedAlgorithms.length}` + ) + CORE_LOGGER.logMessage( + `check length 2: ${compute.publisherTrustedAlgorithmPublishers.length}` + ) if ( // if not set allow them all (!Array.isArray(compute.publisherTrustedAlgorithms) || @@ -92,6 +106,7 @@ export async function validateAlgoForDataset( (!Array.isArray(compute.publisherTrustedAlgorithmPublishers) || compute.publisherTrustedAlgorithmPublishers.length === 0) ) { + CORE_LOGGER.logMessage(`has algoDid: ${algoDID}`) return true } // if is set only allow if match diff --git a/src/components/policyServer/index.ts b/src/components/policyServer/index.ts index 5148b5cf7..ae0e24de1 100644 --- a/src/components/policyServer/index.ts +++ b/src/components/policyServer/index.ts @@ -109,9 +109,8 @@ export class PolicyServer { return await this.askServer(command) } - // TODO: when commands for initializeCompute and startCompute will be - // implemented in policy server, we'll update these functions - // eslint-disable-next-line require-await + // use checkDownload functionality for initializeCompute and startCompute, + // it will do the same credentials checks async checkInitializeCompute( documentId: string, ddo: DDO, @@ -121,10 +120,17 @@ export class PolicyServer { consumerAddress: string, policyServer: any ): Promise { - throw new Error('Not implemented yet in policy server') + return await this.checkDownload( + documentId, + ddo, + serviceId, + fileIndex, + transferTxId, + consumerAddress, + policyServer + ) } - // eslint-disable-next-line require-await async checkStartCompute( documentId: string, ddo: DDO, @@ -134,7 +140,15 @@ export class PolicyServer { consumerAddress: string, policyServer: any ): Promise { - throw new Error('Not implemented yet in policy server') + return await this.checkDownload( + documentId, + ddo, + serviceId, + fileIndex, + transferTxId, + consumerAddress, + policyServer + ) } async passThrough(request: any): Promise { From b6212cb0c98a57863aec2b3035b4e3ecb91fd3af Mon Sep 17 00:00:00 2001 From: mariacarmina Date: Thu, 8 May 2025 15:00:22 +0300 Subject: [PATCH 06/34] log error. --- src/test/integration/compute.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/integration/compute.test.ts b/src/test/integration/compute.test.ts index 4dd35103c..ad7d3ae60 100644 --- a/src/test/integration/compute.test.ts +++ b/src/test/integration/compute.test.ts @@ -972,6 +972,7 @@ describe('Compute', () => { const response = await handler.handle(command) assert(response.status.httpStatus === 500, 'Failed to get 500 response') assert(response.stream === null, 'Should not get stream') + console.log(`response.status.error: ${response.status.error}`) assert( response.status.error.includes( freeComputeStartPayload.algorithm.meta.container.image From 62873e028ddb0fbb840f1db2db4643d0dc0b03aa Mon Sep 17 00:00:00 2001 From: mariacarmina Date: Thu, 8 May 2025 15:35:30 +0300 Subject: [PATCH 07/34] Check if did is provided on free start compute. --- src/components/core/compute/startCompute.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/components/core/compute/startCompute.ts b/src/components/core/compute/startCompute.ts index 95a6d50ac..4dc76ab8a 100644 --- a/src/components/core/compute/startCompute.ts +++ b/src/components/core/compute/startCompute.ts @@ -597,6 +597,9 @@ export class FreeComputeStartHandler extends CommandHandler { } } for (const elem of [...[task.algorithm], ...task.datasets]) { + if (!('documentId' in elem)) { + continue + } const ddo = await new FindDdoHandler(this.getOceanNode()).findAndFormatDdo( elem.documentId ) From 8a510b14da081c0698be9151d93f47863b004449 Mon Sep 17 00:00:00 2001 From: mariacarmina Date: Thu, 8 May 2025 21:42:03 +0300 Subject: [PATCH 08/34] cleanup logs. --- src/test/integration/compute.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/integration/compute.test.ts b/src/test/integration/compute.test.ts index ad7d3ae60..4dd35103c 100644 --- a/src/test/integration/compute.test.ts +++ b/src/test/integration/compute.test.ts @@ -972,7 +972,6 @@ describe('Compute', () => { const response = await handler.handle(command) assert(response.status.httpStatus === 500, 'Failed to get 500 response') assert(response.stream === null, 'Should not get stream') - console.log(`response.status.error: ${response.status.error}`) assert( response.status.error.includes( freeComputeStartPayload.algorithm.meta.container.image From 9d1a39c283ad4d152fda1b301b2248e43f2fc9c1 Mon Sep 17 00:00:00 2001 From: mariacarmina Date: Mon, 12 May 2025 09:49:34 -0400 Subject: [PATCH 09/34] Integrate commands for compute from policy server. --- src/components/policyServer/index.ts | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/components/policyServer/index.ts b/src/components/policyServer/index.ts index ae0e24de1..552f91bfe 100644 --- a/src/components/policyServer/index.ts +++ b/src/components/policyServer/index.ts @@ -109,23 +109,17 @@ export class PolicyServer { return await this.askServer(command) } - // use checkDownload functionality for initializeCompute and startCompute, - // it will do the same credentials checks async checkInitializeCompute( documentId: string, ddo: DDO, serviceId: string, - fileIndex: number, - transferTxId: string, consumerAddress: string, policyServer: any ): Promise { - return await this.checkDownload( + return await this.checkInitialize( documentId, ddo, serviceId, - fileIndex, - transferTxId, consumerAddress, policyServer ) @@ -140,7 +134,8 @@ export class PolicyServer { consumerAddress: string, policyServer: any ): Promise { - return await this.checkDownload( + const command = { + action: 'startCompute', documentId, ddo, serviceId, @@ -148,7 +143,8 @@ export class PolicyServer { transferTxId, consumerAddress, policyServer - ) + } + return await this.askServer(command) } async passThrough(request: any): Promise { From fd0e7ac7daaf2ebaa62551cc4ed23c46589acd1c Mon Sep 17 00:00:00 2001 From: mariacarmina Date: Mon, 12 May 2025 10:16:51 -0400 Subject: [PATCH 10/34] Fix condition in the test. --- src/test/integration/compute.test.ts | 32 ++++++++++++++++------------ 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/src/test/integration/compute.test.ts b/src/test/integration/compute.test.ts index 4dd35103c..a435bcdc7 100644 --- a/src/test/integration/compute.test.ts +++ b/src/test/integration/compute.test.ts @@ -1109,20 +1109,23 @@ describe('Compute', () => { datasetDDO.services[0].compute = { allowRawAlgorithm: false, allowNetworkAccess: true, - publisherTrustedAlgorithmPublishers: setTrustedAlgosEmpty - ? [] - : [publisherAddress], - publisherTrustedAlgorithms: setTrustedAlgosEmpty - ? [] - : [ - { - did: algoDDO.id, - filesChecksum: - 'f6a7b95e4a2e3028957f69fdd2dac27bd5103986b2171bc8bfee68b52f874dcd', - containerSectionChecksum: - 'ba8885fcc7d366f058d6c3bb0b7bfe191c5f85cb6a4ee3858895342436c23504' - } - ] + publisherTrustedAlgorithmPublishers: [], + publisherTrustedAlgorithms: [] + } + + if (!setTrustedAlgosEmpty) { + datasetDDO.services[0].compute.publisherTrustedAlgorithmPublishers = [ + publisherAddress + ] + datasetDDO.services[0].compute.publisherTrustedAlgorithms = [ + { + did: algoDDO.id, + filesChecksum: + 'f6a7b95e4a2e3028957f69fdd2dac27bd5103986b2171bc8bfee68b52f874dcd', + containerSectionChecksum: + 'ba8885fcc7d366f058d6c3bb0b7bfe191c5f85cb6a4ee3858895342436c23504' + } + ] } const metadata = hexlify(Buffer.from(JSON.stringify(datasetDDO))) @@ -1197,6 +1200,7 @@ describe('Compute', () => { datasetDDOTest.services[0].id, oceanNode ) + console.log(`setTrustedAlgosEmpty: ${setTrustedAlgosEmpty}`) expect(result).to.equal(!setTrustedAlgosEmpty) } else expect(expectedTimeoutFailure(this.test.title)).to.be.equal(wasTimeout) } else expect(expectedTimeoutFailure(this.test.title)).to.be.equal(wasTimeout) From 8a81c81c5063557d415f7385bfcd36fc2c12efed Mon Sep 17 00:00:00 2001 From: mariacarmina Date: Mon, 12 May 2025 10:41:23 -0400 Subject: [PATCH 11/34] Debug result. --- src/test/integration/compute.test.ts | 32 +++++++++++++--------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/src/test/integration/compute.test.ts b/src/test/integration/compute.test.ts index a435bcdc7..d7655bc6b 100644 --- a/src/test/integration/compute.test.ts +++ b/src/test/integration/compute.test.ts @@ -1109,23 +1109,20 @@ describe('Compute', () => { datasetDDO.services[0].compute = { allowRawAlgorithm: false, allowNetworkAccess: true, - publisherTrustedAlgorithmPublishers: [], - publisherTrustedAlgorithms: [] - } - - if (!setTrustedAlgosEmpty) { - datasetDDO.services[0].compute.publisherTrustedAlgorithmPublishers = [ - publisherAddress - ] - datasetDDO.services[0].compute.publisherTrustedAlgorithms = [ - { - did: algoDDO.id, - filesChecksum: - 'f6a7b95e4a2e3028957f69fdd2dac27bd5103986b2171bc8bfee68b52f874dcd', - containerSectionChecksum: - 'ba8885fcc7d366f058d6c3bb0b7bfe191c5f85cb6a4ee3858895342436c23504' - } - ] + publisherTrustedAlgorithmPublishers: setTrustedAlgosEmpty + ? [] + : [publisherAddress], + publisherTrustedAlgorithms: setTrustedAlgosEmpty + ? [] + : [ + { + did: algoDDO.id, + filesChecksum: + 'f6a7b95e4a2e3028957f69fdd2dac27bd5103986b2171bc8bfee68b52f874dcd', + containerSectionChecksum: + 'ba8885fcc7d366f058d6c3bb0b7bfe191c5f85cb6a4ee3858895342436c23504' + } + ] } const metadata = hexlify(Buffer.from(JSON.stringify(datasetDDO))) @@ -1200,6 +1197,7 @@ describe('Compute', () => { datasetDDOTest.services[0].id, oceanNode ) + console.log(`result: ${result}`) console.log(`setTrustedAlgosEmpty: ${setTrustedAlgosEmpty}`) expect(result).to.equal(!setTrustedAlgosEmpty) } else expect(expectedTimeoutFailure(this.test.title)).to.be.equal(wasTimeout) From 3fa443b244559f21c338a519baf4dbc9911aa489 Mon Sep 17 00:00:00 2001 From: mariacarmina Date: Mon, 12 May 2025 11:48:28 -0400 Subject: [PATCH 12/34] cleanup. --- src/test/integration/compute.test.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/integration/compute.test.ts b/src/test/integration/compute.test.ts index d7655bc6b..4dd35103c 100644 --- a/src/test/integration/compute.test.ts +++ b/src/test/integration/compute.test.ts @@ -1197,8 +1197,6 @@ describe('Compute', () => { datasetDDOTest.services[0].id, oceanNode ) - console.log(`result: ${result}`) - console.log(`setTrustedAlgosEmpty: ${setTrustedAlgosEmpty}`) expect(result).to.equal(!setTrustedAlgosEmpty) } else expect(expectedTimeoutFailure(this.test.title)).to.be.equal(wasTimeout) } else expect(expectedTimeoutFailure(this.test.title)).to.be.equal(wasTimeout) From e5f7638193e0c7a21c28fb871eead0bce1fe523b Mon Sep 17 00:00:00 2001 From: mariacarmina Date: Mon, 12 May 2025 12:46:00 -0400 Subject: [PATCH 13/34] print ddo test. --- src/test/integration/compute.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/integration/compute.test.ts b/src/test/integration/compute.test.ts index 4dd35103c..a7db97c75 100644 --- a/src/test/integration/compute.test.ts +++ b/src/test/integration/compute.test.ts @@ -1190,6 +1190,7 @@ describe('Compute', () => { const datasetDDOTest = ddo const datasetInstance = DDOManager.getDDOClass(datasetDDO) if (datasetDDOTest) { + console.log(`datasetDDOTest: ${JSON.stringify(datasetDDOTest)}`) const result = await validateAlgoForDataset( algoDDOTest.id, algoChecksums, From ace914c38bf0a458e865c763aa9d8085b04a3667 Mon Sep 17 00:00:00 2001 From: mariacarmina Date: Mon, 12 May 2025 13:06:08 -0400 Subject: [PATCH 14/34] Updated test. --- src/test/integration/compute.test.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/test/integration/compute.test.ts b/src/test/integration/compute.test.ts index a7db97c75..5a4fa5ca0 100644 --- a/src/test/integration/compute.test.ts +++ b/src/test/integration/compute.test.ts @@ -1190,7 +1190,6 @@ describe('Compute', () => { const datasetDDOTest = ddo const datasetInstance = DDOManager.getDDOClass(datasetDDO) if (datasetDDOTest) { - console.log(`datasetDDOTest: ${JSON.stringify(datasetDDOTest)}`) const result = await validateAlgoForDataset( algoDDOTest.id, algoChecksums, @@ -1198,7 +1197,11 @@ describe('Compute', () => { datasetDDOTest.services[0].id, oceanNode ) - expect(result).to.equal(!setTrustedAlgosEmpty) + // datasetDDOTest does not have set + // publisherTrustedAlgorithms, nor + // publisherTrustedAlgorithmPublishers + // expect the result to be true + expect(result).to.equal(true) } else expect(expectedTimeoutFailure(this.test.title)).to.be.equal(wasTimeout) } else expect(expectedTimeoutFailure(this.test.title)).to.be.equal(wasTimeout) }) From 1547a98d1ee7a8264d113d03e18eacdc6b0c1cfa Mon Sep 17 00:00:00 2001 From: mariacarmina Date: Mon, 12 May 2025 13:14:38 -0400 Subject: [PATCH 15/34] cleanup function. --- src/components/core/compute/utils.ts | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/components/core/compute/utils.ts b/src/components/core/compute/utils.ts index 84a41948e..816a1f7e2 100644 --- a/src/components/core/compute/utils.ts +++ b/src/components/core/compute/utils.ts @@ -76,7 +76,6 @@ export async function validateAlgoForDataset( const datasetService = services.find( (service: any) => service.id === datasetServiceId ) - CORE_LOGGER.logMessage(`datasetService: ${JSON.stringify(datasetService)}`) if (!datasetService) { throw new Error('Dataset service not found') } @@ -86,19 +85,6 @@ export async function validateAlgoForDataset( } if (algoDID) { - CORE_LOGGER.logMessage(`has algoDid: ${algoDID}`) - CORE_LOGGER.logMessage( - `is array1: ${Array.isArray(compute.publisherTrustedAlgorithms)}` - ) - CORE_LOGGER.logMessage( - `is array2: ${Array.isArray(compute.publisherTrustedAlgorithmPublishers)}` - ) - CORE_LOGGER.logMessage( - `check length 1: ${compute.publisherTrustedAlgorithms.length}` - ) - CORE_LOGGER.logMessage( - `check length 2: ${compute.publisherTrustedAlgorithmPublishers.length}` - ) if ( // if not set allow them all (!Array.isArray(compute.publisherTrustedAlgorithms) || @@ -106,7 +92,6 @@ export async function validateAlgoForDataset( (!Array.isArray(compute.publisherTrustedAlgorithmPublishers) || compute.publisherTrustedAlgorithmPublishers.length === 0) ) { - CORE_LOGGER.logMessage(`has algoDid: ${algoDID}`) return true } // if is set only allow if match From bc2ee4dfbee5694939a0e0574f112f476581a388 Mon Sep 17 00:00:00 2001 From: mariacarmina Date: Wed, 21 May 2025 14:33:17 +0300 Subject: [PATCH 16/34] remove initialize compute command. Update initializeCompute handler. --- src/components/core/compute/initialize.ts | 8 ++------ src/components/policyServer/index.ts | 16 ---------------- 2 files changed, 2 insertions(+), 22 deletions(-) diff --git a/src/components/core/compute/initialize.ts b/src/components/core/compute/initialize.ts index c4c2e0c99..4aed812f1 100644 --- a/src/components/core/compute/initialize.ts +++ b/src/components/core/compute/initialize.ts @@ -214,12 +214,10 @@ export class ComputeInitializeHandler extends CommandHandler { // It will just use the existing code and let PolicyServer decide. if (isPolicyServerConfigured() && task.policyServer) { accessGrantedDDOLevel = await ( - await new PolicyServer().checkStartCompute( + await new PolicyServer().checkInitialize( ddo.id, ddo, elem.serviceId, - 0, - elem.transferTxId, task.consumerAddress, task.policyServer ) @@ -261,12 +259,10 @@ export class ComputeInitializeHandler extends CommandHandler { accessGrantedServiceLevel = accessGrantedDDOLevel || (await ( - await new PolicyServer().checkStartCompute( + await new PolicyServer().checkInitialize( ddo.id, ddo, elem.serviceId, - 0, - elem.transferTxId, task.consumerAddress, task.policyServer ) diff --git a/src/components/policyServer/index.ts b/src/components/policyServer/index.ts index 552f91bfe..41926ee66 100644 --- a/src/components/policyServer/index.ts +++ b/src/components/policyServer/index.ts @@ -109,22 +109,6 @@ export class PolicyServer { return await this.askServer(command) } - async checkInitializeCompute( - documentId: string, - ddo: DDO, - serviceId: string, - consumerAddress: string, - policyServer: any - ): Promise { - return await this.checkInitialize( - documentId, - ddo, - serviceId, - consumerAddress, - policyServer - ) - } - async checkStartCompute( documentId: string, ddo: DDO, From cefed63caaae4d8410a0ded8ac77990cf0d76c3a Mon Sep 17 00:00:00 2001 From: mariacarmina Date: Wed, 21 May 2025 14:41:04 +0300 Subject: [PATCH 17/34] Enhance code. --- src/components/core/compute/initialize.ts | 37 ++++++++--------- src/components/core/compute/startCompute.ts | 45 ++++++++++----------- 2 files changed, 38 insertions(+), 44 deletions(-) diff --git a/src/components/core/compute/initialize.ts b/src/components/core/compute/initialize.ts index 4aed812f1..ced256ebc 100644 --- a/src/components/core/compute/initialize.ts +++ b/src/components/core/compute/initialize.ts @@ -209,19 +209,19 @@ export class ComputeInitializeHandler extends CommandHandler { } // check credentials (DDO level) let accessGrantedDDOLevel: boolean + const policyServer = new PolicyServer() if (ddo.credentials) { // if POLICY_SERVER_URL exists, then ocean-node will NOT perform any checks. // It will just use the existing code and let PolicyServer decide. if (isPolicyServerConfigured() && task.policyServer) { - accessGrantedDDOLevel = await ( - await new PolicyServer().checkInitialize( - ddo.id, - ddo, - elem.serviceId, - task.consumerAddress, - task.policyServer - ) - ).success + const response = await policyServer.checkInitialize( + ddo.id, + ddo, + elem.serviceId, + task.consumerAddress, + task.policyServer + ) + accessGrantedDDOLevel = response.success } else { accessGrantedDDOLevel = areKnownCredentialTypes(ddo.credentials) ? checkCredentials(ddo.credentials, task.consumerAddress) @@ -256,17 +256,14 @@ export class ComputeInitializeHandler extends CommandHandler { if (isPolicyServerConfigured() && task.policyServer) { // we use the previous check or we do it again // (in case there is no DDO level credentials and we only have Service level ones) - accessGrantedServiceLevel = - accessGrantedDDOLevel || - (await ( - await new PolicyServer().checkInitialize( - ddo.id, - ddo, - elem.serviceId, - task.consumerAddress, - task.policyServer - ) - ).success) + const response = await policyServer.checkInitialize( + ddo.id, + ddo, + elem.serviceId, + task.consumerAddress, + task.policyServer + ) + accessGrantedServiceLevel = accessGrantedDDOLevel || response.success } else { accessGrantedServiceLevel = areKnownCredentialTypes(service.credentials) ? checkCredentials(service.credentials, task.consumerAddress) diff --git a/src/components/core/compute/startCompute.ts b/src/components/core/compute/startCompute.ts index 4dc76ab8a..dc8827ae6 100644 --- a/src/components/core/compute/startCompute.ts +++ b/src/components/core/compute/startCompute.ts @@ -615,21 +615,21 @@ export class FreeComputeStartHandler extends CommandHandler { } // check credentials (DDO level) let accessGrantedDDOLevel: boolean + const policyServer = new PolicyServer() if (ddo.credentials) { // if POLICY_SERVER_URL exists, then ocean-node will NOT perform any checks. // It will just use the existing code and let PolicyServer decide. if (isPolicyServerConfigured() && task.policyServer) { - accessGrantedDDOLevel = await ( - await new PolicyServer().checkStartCompute( - ddo.id, - ddo, - elem.serviceId, - 0, - elem.transferTxId, - task.consumerAddress, - task.policyServer - ) - ).success + const response = await policyServer.checkStartCompute( + ddo.id, + ddo, + elem.serviceId, + 0, + elem.transferTxId, + task.consumerAddress, + task.policyServer + ) + accessGrantedDDOLevel = response.success } else { accessGrantedDDOLevel = areKnownCredentialTypes(ddo.credentials) ? checkCredentials(ddo.credentials, task.consumerAddress) @@ -664,19 +664,16 @@ export class FreeComputeStartHandler extends CommandHandler { if (isPolicyServerConfigured() && task.policyServer) { // we use the previous check or we do it again // (in case there is no DDO level credentials and we only have Service level ones) - accessGrantedServiceLevel = - accessGrantedDDOLevel || - (await ( - await new PolicyServer().checkStartCompute( - ddo.id, - ddo, - elem.serviceId, - 0, - elem.transferTxId, - task.consumerAddress, - task.policyServer - ) - ).success) + const response = await policyServer.checkStartCompute( + ddo.id, + ddo, + service.id, + 0, + elem.transferTxId, + task.consumerAddress, + task.policyServer + ) + accessGrantedServiceLevel = accessGrantedDDOLevel || response.success } else { accessGrantedServiceLevel = areKnownCredentialTypes(service.credentials) ? checkCredentials(service.credentials, task.consumerAddress) From 656b69bf517eb2c79913563f882e5de16fb15237 Mon Sep 17 00:00:00 2001 From: mariacarmina Date: Thu, 22 May 2025 18:11:15 +0300 Subject: [PATCH 18/34] Define dedicated type for policy server when it comes on the handler task. --- src/@types/commands.ts | 9 +++++---- src/@types/policyServer.ts | 8 ++++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/@types/commands.ts b/src/@types/commands.ts index b094c8de5..724c30b4b 100644 --- a/src/@types/commands.ts +++ b/src/@types/commands.ts @@ -15,6 +15,7 @@ import { UrlFileObject, BaseFileObject } from './fileObject' +import { PolicyServerTask } from './policyServer.js' export interface Command { command: string // command name @@ -56,7 +57,7 @@ export interface DownloadCommand extends Command { consumerAddress: string signature: string aes_encrypted_key?: string // if not present it means download without encryption - policyServer?: any // object to pass to policy server + policyServer?: PolicyServerTask // object to pass to policy server } export interface FileInfoCommand extends Command { @@ -132,7 +133,7 @@ export interface GetFeesCommand extends Command { serviceId: string consumerAddress?: string validUntil?: number // this allows a user to request a fee that is valid only for a limited period of time, less than service.timeout - policyServer?: any // object to pass to policyServer + policyServer?: PolicyServerTask // object to pass to policyServer } // admin commands export interface AdminStopNodeCommand extends AdminCommand {} @@ -183,7 +184,7 @@ export interface ComputeInitializeCommand extends Command { consumerAddress: string signature?: string maxJobDuration: number - policyServer?: any // object to pass to policy server + policyServer?: PolicyServerTask // object to pass to policy server } export interface FreeComputeStartCommand extends Command { @@ -196,7 +197,7 @@ export interface FreeComputeStartCommand extends Command { output?: ComputeOutput resources?: ComputeResourceRequest[] maxJobDuration?: number - policyServer?: any // object to pass to policy server + policyServer?: PolicyServerTask // object to pass to policy server } export interface PaidComputeStartCommand extends FreeComputeStartCommand { payment: ComputePayment diff --git a/src/@types/policyServer.ts b/src/@types/policyServer.ts index b0b42537c..51704e41e 100644 --- a/src/@types/policyServer.ts +++ b/src/@types/policyServer.ts @@ -3,3 +3,11 @@ export interface PolicyServerResult { message?: string // error message, if any httpStatus?: number // status returned by server } + +export interface PolicyServerTask { + sessionId?: string + successRedirectUri?: string + errorRedirectUri?: string + responseRedirectUri?: string + presentationDefinitionUri?: string +} From 04ae7e7219f18b2b81052397f50751cfe7b1183d Mon Sep 17 00:00:00 2001 From: mariacarmina Date: Thu, 22 May 2025 18:26:42 +0300 Subject: [PATCH 19/34] Fix type of policy server on http requests. --- src/components/httpRoutes/compute.ts | 7 +++++-- src/components/httpRoutes/provider.ts | 5 +++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/components/httpRoutes/compute.ts b/src/components/httpRoutes/compute.ts index 33598729f..34f43febb 100644 --- a/src/components/httpRoutes/compute.ts +++ b/src/components/httpRoutes/compute.ts @@ -30,6 +30,7 @@ import { PROTOCOL_COMMANDS, SERVICES_API_BASE_PATH } from '../../utils/constants import { Readable } from 'stream' import { HTTP_LOGGER } from '../../utils/logging/common.js' import { LOG_LEVELS_STR } from '../../utils/logging/Logger.js' +import { PolicyServerTask } from '../../@types/policyServer.js' export const computeRoutes = express.Router() @@ -76,7 +77,8 @@ computeRoutes.post(`${SERVICES_API_BASE_PATH}/compute`, async (req, res) => { algorithm: (req.body.algorithm as ComputeAlgorithm) || null, datasets: (req.body.datasets as unknown as ComputeAsset[]) || null, payment: (req.body.payment as unknown as ComputePayment) || null, - resources: (req.body.resources as unknown as ComputeResourceRequest[]) || null + resources: (req.body.resources as unknown as ComputeResourceRequest[]) || null, + policyServer: (req.query.policyServer as PolicyServerTask) || null } if (req.body.output) { startComputeTask.output = req.body.output as ComputeOutput @@ -118,7 +120,8 @@ computeRoutes.post(`${SERVICES_API_BASE_PATH}/freeCompute`, async (req, res) => algorithm: (req.body.algorithm as ComputeAlgorithm) || null, datasets: (req.body.datasets as unknown as ComputeAsset[]) || null, resources: (req.body.resources as unknown as ComputeResourceRequest[]) || null, - maxJobDuration: req.body.maxJobDuration || null + maxJobDuration: req.body.maxJobDuration || null, + policyServer: (req.query.policyServer as PolicyServerTask) || null } if (req.body.output) { startComputeTask.output = req.body.output as ComputeOutput diff --git a/src/components/httpRoutes/provider.ts b/src/components/httpRoutes/provider.ts index b17bd9033..cc0787fb1 100644 --- a/src/components/httpRoutes/provider.ts +++ b/src/components/httpRoutes/provider.ts @@ -13,6 +13,7 @@ import { FeesHandler } from '../core/handler/feesHandler.js' import { BaseFileObject, EncryptMethod } from '../../@types/fileObject.js' import { P2PCommandResponse } from '../../@types/OceanNode.js' import { getEncryptMethodFromString } from '../../utils/crypt.js' +import { PolicyServerTask } from '../../@types/policyServer.js' export const providerRoutes = express.Router() @@ -153,7 +154,7 @@ providerRoutes.get(`${SERVICES_API_BASE_PATH}/initialize`, async (req, res) => { serviceId: (req.query.serviceId as string) || null, consumerAddress: (req.query.consumerAddress as string) || null, validUntil: parseInt(req.query.validUntil as string) || null, - policyServer: req.query.policyServer || null + policyServer: (req.query.policyServer as PolicyServerTask) || null }) if (result.stream) { const initializeREsponse = await streamToObject(result.stream as Readable) @@ -220,7 +221,7 @@ providerRoutes.get( consumerAddress: consumerAddress as string, signature: signature as string, command: PROTOCOL_COMMANDS.DOWNLOAD, - policyServer: req.query.policyServer || null + policyServer: (req.query.policyServer as PolicyServerTask) || null } const response = await new DownloadHandler(req.oceanNode).handle(downloadTask) From a2a9793922ba8c929dde5f3da15c679b6ee7c6ca Mon Sep 17 00:00:00 2001 From: mariacarmina Date: Tue, 24 Jun 2025 15:16:58 +0300 Subject: [PATCH 20/34] Fix commands usage for policy server. --- src/components/core/compute/initialize.ts | 4 ++-- src/components/core/compute/startCompute.ts | 8 ------- src/components/policyServer/index.ts | 26 --------------------- 3 files changed, 2 insertions(+), 36 deletions(-) diff --git a/src/components/core/compute/initialize.ts b/src/components/core/compute/initialize.ts index ced256ebc..76d267d88 100644 --- a/src/components/core/compute/initialize.ts +++ b/src/components/core/compute/initialize.ts @@ -214,7 +214,7 @@ export class ComputeInitializeHandler extends CommandHandler { // if POLICY_SERVER_URL exists, then ocean-node will NOT perform any checks. // It will just use the existing code and let PolicyServer decide. if (isPolicyServerConfigured() && task.policyServer) { - const response = await policyServer.checkInitialize( + const response = await policyServer.checkStartCompute( ddo.id, ddo, elem.serviceId, @@ -256,7 +256,7 @@ export class ComputeInitializeHandler extends CommandHandler { if (isPolicyServerConfigured() && task.policyServer) { // we use the previous check or we do it again // (in case there is no DDO level credentials and we only have Service level ones) - const response = await policyServer.checkInitialize( + const response = await policyServer.checkStartCompute( ddo.id, ddo, elem.serviceId, diff --git a/src/components/core/compute/startCompute.ts b/src/components/core/compute/startCompute.ts index 309c85d1b..0158b6cd9 100644 --- a/src/components/core/compute/startCompute.ts +++ b/src/components/core/compute/startCompute.ts @@ -178,8 +178,6 @@ export class PaidComputeStartHandler extends CommandHandler { ddo.id, ddo, elem.serviceId, - 0, - elem.transferTxId, task.consumerAddress, task.policyServer ) @@ -225,8 +223,6 @@ export class PaidComputeStartHandler extends CommandHandler { ddo.id, ddo, elem.serviceId, - 0, - elem.transferTxId, task.consumerAddress, task.policyServer ) @@ -626,8 +622,6 @@ export class FreeComputeStartHandler extends CommandHandler { ddo.id, ddo, elem.serviceId, - 0, - elem.transferTxId, task.consumerAddress, task.policyServer ) @@ -670,8 +664,6 @@ export class FreeComputeStartHandler extends CommandHandler { ddo.id, ddo, service.id, - 0, - elem.transferTxId, task.consumerAddress, task.policyServer ) diff --git a/src/components/policyServer/index.ts b/src/components/policyServer/index.ts index 41926ee66..24bb0de6c 100644 --- a/src/components/policyServer/index.ts +++ b/src/components/policyServer/index.ts @@ -69,30 +69,10 @@ export class PolicyServer { return await this.askServer(command) } - async checkInitialize( - documentId: string, - ddo: DDO, - serviceId: string, - consumerAddress: string, - policyServer: any - ): Promise { - const command = { - action: 'initialize', - documentId, - ddo, - serviceId, - consumerAddress, - policyServer - } - return await this.askServer(command) - } - async checkDownload( documentId: string, ddo: DDO, serviceId: string, - fileIndex: number, - transferTxId: string, consumerAddress: string, policyServer: any ): Promise { @@ -101,8 +81,6 @@ export class PolicyServer { documentId, ddo, serviceId, - fileIndex, - transferTxId, consumerAddress, policyServer } @@ -113,8 +91,6 @@ export class PolicyServer { documentId: string, ddo: DDO, serviceId: string, - fileIndex: number, - transferTxId: string, consumerAddress: string, policyServer: any ): Promise { @@ -123,8 +99,6 @@ export class PolicyServer { documentId, ddo, serviceId, - fileIndex, - transferTxId, consumerAddress, policyServer } From 198f0e330a1dca1eae172fcca203e4c96d3c8154 Mon Sep 17 00:00:00 2001 From: mariacarmina Date: Thu, 26 Jun 2025 08:22:43 +0300 Subject: [PATCH 21/34] Fix arguments. --- src/components/core/handler/downloadHandler.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/components/core/handler/downloadHandler.ts b/src/components/core/handler/downloadHandler.ts index 22adf86fc..178374537 100644 --- a/src/components/core/handler/downloadHandler.ts +++ b/src/components/core/handler/downloadHandler.ts @@ -290,8 +290,6 @@ export class DownloadHandler extends CommandHandler { ddo.id, ddo, task.serviceId, - task.fileIndex, - task.transferTxId, task.consumerAddress, task.policyServer ) @@ -372,8 +370,6 @@ export class DownloadHandler extends CommandHandler { ddo.id, ddo, task.serviceId, - task.fileIndex, - task.transferTxId, task.consumerAddress, task.policyServer ) @@ -477,8 +473,6 @@ export class DownloadHandler extends CommandHandler { ddo.id, ddo, service.id, - task.fileIndex, - task.transferTxId, task.consumerAddress, task.policyServer ) From 5ab79f91fe5b99ac0664021a128e4cabdce42ee6 Mon Sep 17 00:00:00 2001 From: mariacarmina Date: Wed, 9 Jul 2025 14:15:16 +0300 Subject: [PATCH 22/34] Fix review. --- src/components/core/compute/initialize.ts | 6 +- src/components/core/compute/startCompute.ts | 8 +-- .../core/handler/downloadHandler.ts | 55 ++++++------------- 3 files changed, 24 insertions(+), 45 deletions(-) diff --git a/src/components/core/compute/initialize.ts b/src/components/core/compute/initialize.ts index 76d267d88..7ff6d1af1 100644 --- a/src/components/core/compute/initialize.ts +++ b/src/components/core/compute/initialize.ts @@ -184,7 +184,7 @@ export class ComputeInitializeHandler extends CommandHandler { const result: any = { validOrder: false } if ('documentId' in elem && elem.documentId) { result.did = elem.documentId - result.serviceId = elem.documentId + result.serviceId = elem.serviceId const ddo = await new FindDdoHandler(node).findAndFormatDdo(elem.documentId) if (!ddo) { const error = `DDO ${elem.documentId} not found` @@ -213,7 +213,7 @@ export class ComputeInitializeHandler extends CommandHandler { if (ddo.credentials) { // if POLICY_SERVER_URL exists, then ocean-node will NOT perform any checks. // It will just use the existing code and let PolicyServer decide. - if (isPolicyServerConfigured() && task.policyServer) { + if (isPolicyServerConfigured()) { const response = await policyServer.checkStartCompute( ddo.id, ddo, @@ -253,7 +253,7 @@ export class ComputeInitializeHandler extends CommandHandler { // if using a policy server and we are here it means that access was granted (they are merged/assessed together) if (service.credentials) { let accessGrantedServiceLevel: boolean - if (isPolicyServerConfigured() && task.policyServer) { + if (isPolicyServerConfigured()) { // we use the previous check or we do it again // (in case there is no DDO level credentials and we only have Service level ones) const response = await policyServer.checkStartCompute( diff --git a/src/components/core/compute/startCompute.ts b/src/components/core/compute/startCompute.ts index a715283b9..39d0c41de 100644 --- a/src/components/core/compute/startCompute.ts +++ b/src/components/core/compute/startCompute.ts @@ -154,7 +154,7 @@ export class PaidComputeStartHandler extends CommandHandler { const result: any = { validOrder: false } if ('documentId' in elem && elem.documentId) { result.did = elem.documentId - result.serviceId = elem.documentId + result.serviceId = elem.serviceId const ddo = await new FindDdoHandler(node).findAndFormatDdo(elem.documentId) if (!ddo) { const error = `DDO ${elem.documentId} not found` @@ -223,7 +223,7 @@ export class PaidComputeStartHandler extends CommandHandler { // if using a policy server and we are here it means that access was granted (they are merged/assessed together) if (service.credentials) { let accessGrantedServiceLevel: boolean - if (isPolicyServerConfigured() && task.policyServer) { + if (isPolicyServerConfigured()) { // we use the previous check or we do it again // (in case there is no DDO level credentials and we only have Service level ones) accessGrantedServiceLevel = @@ -615,7 +615,7 @@ export class FreeComputeStartHandler extends CommandHandler { if (ddo.credentials) { // if POLICY_SERVER_URL exists, then ocean-node will NOT perform any checks. // It will just use the existing code and let PolicyServer decide. - if (isPolicyServerConfigured() && task.policyServer) { + if (isPolicyServerConfigured()) { const response = await policyServer.checkStartCompute( ddo.id, ddo, @@ -655,7 +655,7 @@ export class FreeComputeStartHandler extends CommandHandler { // if using a policy server and we are here it means that access was granted (they are merged/assessed together) if (service.credentials) { let accessGrantedServiceLevel: boolean - if (isPolicyServerConfigured() && task.policyServer) { + if (isPolicyServerConfigured()) { // we use the previous check or we do it again // (in case there is no DDO level credentials and we only have Service level ones) const response = await policyServer.checkStartCompute( diff --git a/src/components/core/handler/downloadHandler.ts b/src/components/core/handler/downloadHandler.ts index 178374537..44083ce9c 100644 --- a/src/components/core/handler/downloadHandler.ts +++ b/src/components/core/handler/downloadHandler.ts @@ -281,19 +281,19 @@ export class DownloadHandler extends CommandHandler { // check credentials (DDO level) let accessGrantedDDOLevel: boolean + const policyServer = new PolicyServer() if (ddo.credentials) { // if POLICY_SERVER_URL exists, then ocean-node will NOT perform any checks. // It will just use the existing code and let PolicyServer decide. if (isPolicyServerConfigured()) { - accessGrantedDDOLevel = await ( - await new PolicyServer().checkDownload( - ddo.id, - ddo, - task.serviceId, - task.consumerAddress, - task.policyServer - ) - ).success + const response = await policyServer.checkDownload( + ddo.id, + ddo, + task.serviceId, + task.consumerAddress, + task.policyServer + ) + accessGrantedDDOLevel = response.success } else { accessGrantedDDOLevel = areKnownCredentialTypes(ddo.credentials) ? checkCredentials(ddo.credentials, task.consumerAddress) @@ -363,17 +363,14 @@ export class DownloadHandler extends CommandHandler { if (isPolicyServerConfigured()) { // we use the previous check or we do it again // (in case there is no DDO level credentials and we only have Service level ones) - accessGrantedServiceLevel = - accessGrantedDDOLevel || - (await ( - await new PolicyServer().checkDownload( - ddo.id, - ddo, - task.serviceId, - task.consumerAddress, - task.policyServer - ) - ).success) + const response = await policyServer.checkDownload( + ddo.id, + ddo, + service.id, + task.consumerAddress, + task.policyServer + ) + accessGrantedServiceLevel = accessGrantedDDOLevel || response.success } else { accessGrantedServiceLevel = areKnownCredentialTypes(service.credentials) ? checkCredentials(service.credentials, task.consumerAddress) @@ -467,24 +464,6 @@ export class DownloadHandler extends CommandHandler { } } } - // policyServer check - const policyServer = new PolicyServer() - const policyStatus = await policyServer.checkDownload( - ddo.id, - ddo, - service.id, - task.consumerAddress, - task.policyServer - ) - if (!policyStatus.success) { - return { - stream: null, - status: { - httpStatus: 405, - error: policyStatus.message - } - } - } try { // 7. Decrypt the url From 1d14e9c73e7d8fc362592f19b1819039ab507f33 Mon Sep 17 00:00:00 2001 From: mariacarmina Date: Thu, 10 Jul 2025 12:17:30 +0300 Subject: [PATCH 23/34] Use ddo.js for DDO fields manipulation. --- src/components/core/compute/initialize.ts | 21 ++++--- src/components/core/compute/startCompute.ts | 62 +++++++++---------- .../core/handler/downloadHandler.ts | 17 ++--- 3 files changed, 52 insertions(+), 48 deletions(-) diff --git a/src/components/core/compute/initialize.ts b/src/components/core/compute/initialize.ts index 7ff6d1af1..de070e7d5 100644 --- a/src/components/core/compute/initialize.ts +++ b/src/components/core/compute/initialize.ts @@ -30,7 +30,7 @@ import { FindDdoHandler } from '../handler/ddoHandler.js' import { isOrderingAllowedForAsset } from '../handler/downloadHandler.js' import { getNonceAsNumber } from '../utils/nonceHandler.js' import { C2DEngineDocker, getAlgorithmImage } from '../../c2d/compute_engine_docker.js' -import { DDOManager } from '@oceanprotocol/ddo-js' +import { Credentials, DDOManager } from '@oceanprotocol/ddo-js' import { areKnownCredentialTypes, checkCredentials } from '../../../utils/credentials.js' import { PolicyServer } from '../../policyServer/index.js' @@ -180,6 +180,7 @@ export class ComputeInitializeHandler extends CommandHandler { // check algo let index = 0 + const policyServer = new PolicyServer() for (const elem of [...[task.algorithm], ...task.datasets]) { const result: any = { validOrder: false } if ('documentId' in elem && elem.documentId) { @@ -196,6 +197,12 @@ export class ComputeInitializeHandler extends CommandHandler { } } } + const ddoInstance = DDOManager.getDDOClass(ddo) + const { + chainId: ddoChainId, + nftAddress, + credentials + } = ddoInstance.getDDOFields() const isOrdable = isOrderingAllowedForAsset(ddo) if (!isOrdable.isOrdable) { CORE_LOGGER.error(isOrdable.reason) @@ -209,13 +216,12 @@ export class ComputeInitializeHandler extends CommandHandler { } // check credentials (DDO level) let accessGrantedDDOLevel: boolean - const policyServer = new PolicyServer() - if (ddo.credentials) { + if (credentials) { // if POLICY_SERVER_URL exists, then ocean-node will NOT perform any checks. // It will just use the existing code and let PolicyServer decide. if (isPolicyServerConfigured()) { const response = await policyServer.checkStartCompute( - ddo.id, + ddoInstance.getDid(), ddo, elem.serviceId, task.consumerAddress, @@ -223,8 +229,8 @@ export class ComputeInitializeHandler extends CommandHandler { ) accessGrantedDDOLevel = response.success } else { - accessGrantedDDOLevel = areKnownCredentialTypes(ddo.credentials) - ? checkCredentials(ddo.credentials, task.consumerAddress) + accessGrantedDDOLevel = areKnownCredentialTypes(credentials as Credentials) + ? checkCredentials(credentials as Credentials, task.consumerAddress) : true } if (!accessGrantedDDOLevel) { @@ -284,9 +290,6 @@ export class ComputeInitializeHandler extends CommandHandler { } } } - - const ddoInstance = DDOManager.getDDOClass(ddo) - const { chainId: ddoChainId, nftAddress } = ddoInstance.getDDOFields() const config = await getConfiguration() const { rpc, network, chainId, fallbackRPCs } = config.supportedNetworks[ddoChainId] diff --git a/src/components/core/compute/startCompute.ts b/src/components/core/compute/startCompute.ts index 39d0c41de..d8b7df473 100644 --- a/src/components/core/compute/startCompute.ts +++ b/src/components/core/compute/startCompute.ts @@ -31,7 +31,7 @@ import { sanitizeServiceFiles } from '../../../utils/util.js' import { FindDdoHandler } from '../handler/ddoHandler.js' // import { ProviderFeeValidation } from '../../../@types/Fees.js' import { isOrderingAllowedForAsset } from '../handler/downloadHandler.js' -import { DDOManager } from '@oceanprotocol/ddo-js' +import { Credentials, DDOManager } from '@oceanprotocol/ddo-js' import { getNonceAsNumber } from '../utils/nonceHandler.js' import { PolicyServer } from '../../policyServer/index.js' import { areKnownCredentialTypes, checkCredentials } from '../../../utils/credentials.js' @@ -148,6 +148,7 @@ export class PaidComputeStartHandler extends CommandHandler { } } } + const policyServer = new PolicyServer() // check algo for (const elem of [...[task.algorithm], ...task.datasets]) { console.log(elem) @@ -166,6 +167,14 @@ export class PaidComputeStartHandler extends CommandHandler { } } } + const ddoInstance = DDOManager.getDDOClass(ddo) + const { + chainId: ddoChainId, + services, + metadata, + nftAddress, + credentials + } = ddoInstance.getDDOFields() const isOrdable = isOrderingAllowedForAsset(ddo) if (!isOrdable.isOrdable) { CORE_LOGGER.error(isOrdable.reason) @@ -179,22 +188,21 @@ export class PaidComputeStartHandler extends CommandHandler { } // check credentials (DDO level) let accessGrantedDDOLevel: boolean - if (ddo.credentials) { + if (credentials) { // if POLICY_SERVER_URL exists, then ocean-node will NOT perform any checks. // It will just use the existing code and let PolicyServer decide. - if (isPolicyServerConfigured() && task.policyServer) { - accessGrantedDDOLevel = await ( - await new PolicyServer().checkStartCompute( - ddo.id, - ddo, - elem.serviceId, - task.consumerAddress, - task.policyServer - ) - ).success + if (isPolicyServerConfigured()) { + const response = await policyServer.checkStartCompute( + ddo.id, + ddo, + elem.serviceId, + task.consumerAddress, + task.policyServer + ) + accessGrantedDDOLevel = response.success } else { - accessGrantedDDOLevel = areKnownCredentialTypes(ddo.credentials) - ? checkCredentials(ddo.credentials, task.consumerAddress) + accessGrantedDDOLevel = areKnownCredentialTypes(credentials as Credentials) + ? checkCredentials(credentials as Credentials, task.consumerAddress) : true } if (!accessGrantedDDOLevel) { @@ -226,17 +234,14 @@ export class PaidComputeStartHandler extends CommandHandler { if (isPolicyServerConfigured()) { // we use the previous check or we do it again // (in case there is no DDO level credentials and we only have Service level ones) - accessGrantedServiceLevel = - accessGrantedDDOLevel || - (await ( - await new PolicyServer().checkStartCompute( - ddo.id, - ddo, - elem.serviceId, - task.consumerAddress, - task.policyServer - ) - ).success) + const response = await policyServer.checkStartCompute( + ddoInstance.getDid(), + ddo, + elem.serviceId, + task.consumerAddress, + task.policyServer + ) + accessGrantedServiceLevel = accessGrantedDDOLevel || response.success } else { accessGrantedServiceLevel = areKnownCredentialTypes(service.credentials) ? checkCredentials(service.credentials, task.consumerAddress) @@ -259,13 +264,6 @@ export class PaidComputeStartHandler extends CommandHandler { } const config = await getConfiguration() - const ddoInstance = DDOManager.getDDOClass(ddo) - const { - chainId: ddoChainId, - services, - metadata, - nftAddress - } = ddoInstance.getDDOFields() const { rpc, network, chainId, fallbackRPCs } = config.supportedNetworks[ddoChainId] const blockchain = new Blockchain(rpc, network, chainId, fallbackRPCs) diff --git a/src/components/core/handler/downloadHandler.ts b/src/components/core/handler/downloadHandler.ts index 44083ce9c..4ca3e4d21 100644 --- a/src/components/core/handler/downloadHandler.ts +++ b/src/components/core/handler/downloadHandler.ts @@ -39,7 +39,7 @@ import { import { sanitizeServiceFiles } from '../../../utils/util.js' import { OrdableAssetResponse } from '../../../@types/Asset.js' import { PolicyServer } from '../../policyServer/index.js' -import { Asset, DDO, Service } from '@oceanprotocol/ddo-js' +import { Asset, Credentials, DDO, DDOManager, Service } from '@oceanprotocol/ddo-js' export const FILE_ENCRYPTION_ALGORITHM = 'aes-256-cbc' export function isOrderingAllowedForAsset(asset: Asset): OrdableAssetResponse { @@ -254,6 +254,8 @@ export class DownloadHandler extends CommandHandler { } } } + const ddoInstance = DDOManager.getDDOClass(ddo) + const policyServer = new PolicyServer() const isOrdable = isOrderingAllowedForAsset(ddo) if (!isOrdable.isOrdable) { @@ -281,13 +283,14 @@ export class DownloadHandler extends CommandHandler { // check credentials (DDO level) let accessGrantedDDOLevel: boolean - const policyServer = new PolicyServer() - if (ddo.credentials) { + + const { credentials } = ddoInstance.getDDOFields() + if (credentials) { // if POLICY_SERVER_URL exists, then ocean-node will NOT perform any checks. // It will just use the existing code and let PolicyServer decide. if (isPolicyServerConfigured()) { const response = await policyServer.checkDownload( - ddo.id, + ddoInstance.getDid(), ddo, task.serviceId, task.consumerAddress, @@ -295,8 +298,8 @@ export class DownloadHandler extends CommandHandler { ) accessGrantedDDOLevel = response.success } else { - accessGrantedDDOLevel = areKnownCredentialTypes(ddo.credentials) - ? checkCredentials(ddo.credentials, task.consumerAddress) + accessGrantedDDOLevel = areKnownCredentialTypes(credentials as Credentials) + ? checkCredentials(credentials as Credentials, task.consumerAddress) : true } if (!accessGrantedDDOLevel) { @@ -364,7 +367,7 @@ export class DownloadHandler extends CommandHandler { // we use the previous check or we do it again // (in case there is no DDO level credentials and we only have Service level ones) const response = await policyServer.checkDownload( - ddo.id, + ddoInstance.getDid(), ddo, service.id, task.consumerAddress, From b142f894bc9f83e64c6e02bf8ce362747d9ed0c0 Mon Sep 17 00:00:00 2001 From: mariacarmina Date: Thu, 10 Jul 2025 12:36:57 +0300 Subject: [PATCH 24/34] Update for free start compute. --- src/components/core/compute/initialize.ts | 7 +++-- src/components/core/compute/startCompute.ts | 26 ++++++++++++------- .../core/handler/downloadHandler.ts | 19 +++++++++----- 3 files changed, 35 insertions(+), 17 deletions(-) diff --git a/src/components/core/compute/initialize.ts b/src/components/core/compute/initialize.ts index de070e7d5..ca468fe20 100644 --- a/src/components/core/compute/initialize.ts +++ b/src/components/core/compute/initialize.ts @@ -234,12 +234,15 @@ export class ComputeInitializeHandler extends CommandHandler { : true } if (!accessGrantedDDOLevel) { - CORE_LOGGER.logMessage(`Error: Access to asset ${ddo.id} was denied`, true) + CORE_LOGGER.logMessage( + `Error: Access to asset ${ddoInstance.getDid()} was denied`, + true + ) return { stream: null, status: { httpStatus: 403, - error: `Error: Access to asset ${ddo.id} was denied` + error: `Error: Access to asset ${ddoInstance.getDid()} was denied` } } } diff --git a/src/components/core/compute/startCompute.ts b/src/components/core/compute/startCompute.ts index d8b7df473..629d7880f 100644 --- a/src/components/core/compute/startCompute.ts +++ b/src/components/core/compute/startCompute.ts @@ -206,12 +206,15 @@ export class PaidComputeStartHandler extends CommandHandler { : true } if (!accessGrantedDDOLevel) { - CORE_LOGGER.logMessage(`Error: Access to asset ${ddo.id} was denied`, true) + CORE_LOGGER.logMessage( + `Error: Access to asset ${ddoInstance.getDid()} was denied`, + true + ) return { stream: null, status: { httpStatus: 403, - error: `Error: Access to asset ${ddo.id} was denied` + error: `Error: Access to asset ${ddoInstance.getDid()} was denied` } } } @@ -590,6 +593,7 @@ export class FreeComputeStartHandler extends CommandHandler { } } } + const policyServer = new PolicyServer() for (const elem of [...[task.algorithm], ...task.datasets]) { if (!('documentId' in elem)) { continue @@ -607,15 +611,16 @@ export class FreeComputeStartHandler extends CommandHandler { } } } + const ddoInstance = DDOManager.getDDOClass(ddo) + const { credentials } = ddoInstance.getDDOFields() // check credentials (DDO level) let accessGrantedDDOLevel: boolean - const policyServer = new PolicyServer() - if (ddo.credentials) { + if (credentials) { // if POLICY_SERVER_URL exists, then ocean-node will NOT perform any checks. // It will just use the existing code and let PolicyServer decide. if (isPolicyServerConfigured()) { const response = await policyServer.checkStartCompute( - ddo.id, + ddoInstance.getDid(), ddo, elem.serviceId, task.consumerAddress, @@ -623,17 +628,20 @@ export class FreeComputeStartHandler extends CommandHandler { ) accessGrantedDDOLevel = response.success } else { - accessGrantedDDOLevel = areKnownCredentialTypes(ddo.credentials) - ? checkCredentials(ddo.credentials, task.consumerAddress) + accessGrantedDDOLevel = areKnownCredentialTypes(credentials as Credentials) + ? checkCredentials(credentials as Credentials, task.consumerAddress) : true } if (!accessGrantedDDOLevel) { - CORE_LOGGER.logMessage(`Error: Access to asset ${ddo.id} was denied`, true) + CORE_LOGGER.logMessage( + `Error: Access to asset ${ddoInstance.getDid()} was denied`, + true + ) return { stream: null, status: { httpStatus: 403, - error: `Error: Access to asset ${ddo.id} was denied` + error: `Error: Access to asset ${ddoInstance.getDid()} was denied` } } } diff --git a/src/components/core/handler/downloadHandler.ts b/src/components/core/handler/downloadHandler.ts index 4ca3e4d21..1e5bdb4c1 100644 --- a/src/components/core/handler/downloadHandler.ts +++ b/src/components/core/handler/downloadHandler.ts @@ -255,6 +255,12 @@ export class DownloadHandler extends CommandHandler { } } const ddoInstance = DDOManager.getDDOClass(ddo) + const { + chainId: ddoChainId, + nftAddress, + metadata, + credentials + } = ddoInstance.getDDOFields() const policyServer = new PolicyServer() const isOrdable = isOrderingAllowedForAsset(ddo) @@ -270,7 +276,7 @@ export class DownloadHandler extends CommandHandler { } // 2. Validate ddo and credentials - if (!ddo.chainId || !ddo.nftAddress || !ddo.metadata) { + if (!ddoChainId || !nftAddress || !metadata) { CORE_LOGGER.logMessage('Error: DDO malformed or disabled', true) return { stream: null, @@ -283,8 +289,6 @@ export class DownloadHandler extends CommandHandler { // check credentials (DDO level) let accessGrantedDDOLevel: boolean - - const { credentials } = ddoInstance.getDDOFields() if (credentials) { // if POLICY_SERVER_URL exists, then ocean-node will NOT perform any checks. // It will just use the existing code and let PolicyServer decide. @@ -303,12 +307,15 @@ export class DownloadHandler extends CommandHandler { : true } if (!accessGrantedDDOLevel) { - CORE_LOGGER.logMessage(`Error: Access to asset ${ddo.id} was denied`, true) + CORE_LOGGER.logMessage( + `Error: Access to asset ${ddoInstance.getDid()} was denied`, + true + ) return { stream: null, status: { httpStatus: 403, - error: `Error: Access to asset ${ddo.id} was denied` + error: `Error: Access to asset ${ddoInstance.getDid()} was denied` } } } @@ -316,7 +323,7 @@ export class DownloadHandler extends CommandHandler { // from now on, we need blockchain checks const config = await getConfiguration() - const { rpc, network, chainId, fallbackRPCs } = config.supportedNetworks[ddo.chainId] + const { rpc, network, chainId, fallbackRPCs } = config.supportedNetworks[ddoChainId] let provider let blockchain try { From ce8806772ce2d9cbb108516202557e65d9c2db72 Mon Sep 17 00:00:00 2001 From: mariacarmina Date: Thu, 10 Jul 2025 12:53:28 +0300 Subject: [PATCH 25/34] Update datasets samples. --- src/test/data/assets.ts | 44 ++++++++++++++++++------ src/test/integration/credentials.test.ts | 8 ++--- src/test/integration/download.test.ts | 2 +- 3 files changed, 39 insertions(+), 15 deletions(-) diff --git a/src/test/data/assets.ts b/src/test/data/assets.ts index 0f70a77bc..7fa8e462a 100644 --- a/src/test/data/assets.ts +++ b/src/test/data/assets.ts @@ -22,7 +22,7 @@ export const downloadAsset = { services: [ { id: 'ccb398c50d6abd5b456e8d7242bd856a1767a890b537c2f8c10ba8b8a10e6025', - type: 'download', + type: 'access', files: { files: [ { @@ -111,7 +111,7 @@ export const downloadAssetWithCredentials = { services: [ { id: 'ccb398c50d6abd5b456e8d7242bd856a1767a890b537c2f8c10ba8b8a10e6025', - type: 'download', + type: 'access', files: { files: [ { @@ -190,8 +190,14 @@ export const computeAssetWithCredentials = { compute: { allowRawAlgorithm: false, allowNetworkAccess: true, - publisherTrustedAlgorithmPublishers: [] as any, - publisherTrustedAlgorithms: [] as any + publisherTrustedAlgorithmPublishers: ['*'], + publisherTrustedAlgorithms: [ + { + did: '*', + filesChecksum: '*', + containerSectionChecksum: '*' + } + ] } } ], @@ -269,8 +275,14 @@ export const algoAssetWithCredentials = { compute: { allowRawAlgorithm: false, allowNetworkAccess: true, - publisherTrustedAlgorithmPublishers: [] as any, - publisherTrustedAlgorithms: [] as any + publisherTrustedAlgorithmPublishers: ['*'], + publisherTrustedAlgorithms: [ + { + did: '*', + filesChecksum: '*', + containerSectionChecksum: '*' + } + ] } } ], @@ -330,8 +342,14 @@ export const computeAsset = { compute: { allowRawAlgorithm: false, allowNetworkAccess: true, - publisherTrustedAlgorithmPublishers: [] as any, - publisherTrustedAlgorithms: [] as any + publisherTrustedAlgorithmPublishers: ['*'], + publisherTrustedAlgorithms: [ + { + did: '*', + filesChecksum: '*', + containerSectionChecksum: '*' + } + ] } } ], @@ -407,8 +425,14 @@ export const algoAsset = { compute: { allowRawAlgorithm: false, allowNetworkAccess: true, - publisherTrustedAlgorithmPublishers: [] as any, - publisherTrustedAlgorithms: [] as any + publisherTrustedAlgorithmPublishers: ['*'], + publisherTrustedAlgorithms: [ + { + did: '*', + filesChecksum: '*', + containerSectionChecksum: '*' + } + ] } } ], diff --git a/src/test/integration/credentials.test.ts b/src/test/integration/credentials.test.ts index 1bb78d028..63a825c6b 100644 --- a/src/test/integration/credentials.test.ts +++ b/src/test/integration/credentials.test.ts @@ -66,7 +66,7 @@ import { ComputeAlgorithm, ComputeAsset } from '../../@types/index.js' import { ComputeGetEnvironmentsHandler } from '../../components/core/compute/environments.js' import { ComputeInitializeCommand } from '../../@types/commands.js' -describe('Should run a complete node flow.', () => { +describe('[Credentials Flow] - Should run a complete node flow.', () => { let config: OceanNodeConfig let oceanNode: OceanNode let provider: JsonRpcProvider @@ -369,7 +369,7 @@ describe('Should run a complete node flow.', () => { token: paymentToken }, maxJobDuration: 2 * 60, - consumerAddress: consumerAddress, + consumerAddress, command: PROTOCOL_COMMANDS.COMPUTE_INITIALIZE } const response = await new ComputeInitializeHandler(oceanNode).handle( @@ -403,7 +403,7 @@ describe('Should run a complete node flow.', () => { token: paymentToken }, maxJobDuration: 2 * 60, - consumerAddress: consumerAddress, + consumerAddress, command: PROTOCOL_COMMANDS.COMPUTE_INITIALIZE } const response = await new ComputeInitializeHandler(oceanNode).handle( @@ -436,7 +436,7 @@ describe('Should run a complete node flow.', () => { token: paymentToken }, maxJobDuration: 2 * 60, - consumerAddress: consumerAddress, + consumerAddress, command: PROTOCOL_COMMANDS.COMPUTE_INITIALIZE } const response = await new ComputeInitializeHandler(oceanNode).handle( diff --git a/src/test/integration/download.test.ts b/src/test/integration/download.test.ts index 64b33c78c..fd629476b 100644 --- a/src/test/integration/download.test.ts +++ b/src/test/integration/download.test.ts @@ -44,7 +44,7 @@ import { downloadAsset } from '../data/assets.js' import { genericDDO } from '../data/ddo.js' import { homedir } from 'os' -describe('Should run a complete node flow.', () => { +describe('[Download Flow] - Should run a complete node flow.', () => { let config: OceanNodeConfig let database: Database let oceanNode: OceanNode From dae8fa7573414a3747c466b39f644bcd52ab1c20 Mon Sep 17 00:00:00 2001 From: mariacarmina Date: Thu, 10 Jul 2025 13:00:42 +0300 Subject: [PATCH 26/34] Update branch for cli. --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a18068739..ef1428996 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -282,7 +282,6 @@ jobs: with: repository: 'oceanprotocol/ocean-cli' path: 'ocean-cli' - ref: 'feature/fix-compute-dataset' - name: Setup Ocean CLI working-directory: ${{ github.workspace }}/ocean-cli run: | From 14e9ebdcbaede1652d3c665e211ffb340ee221bf Mon Sep 17 00:00:00 2001 From: mariacarmina Date: Fri, 11 Jul 2025 11:52:02 +0300 Subject: [PATCH 27/34] Fix datasets samples. --- src/test/data/assets.ts | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/src/test/data/assets.ts b/src/test/data/assets.ts index 7fa8e462a..c77f32349 100644 --- a/src/test/data/assets.ts +++ b/src/test/data/assets.ts @@ -190,14 +190,14 @@ export const computeAssetWithCredentials = { compute: { allowRawAlgorithm: false, allowNetworkAccess: true, - publisherTrustedAlgorithmPublishers: ['*'], + publisherTrustedAlgorithmPublishers: ['*'] as any, publisherTrustedAlgorithms: [ { did: '*', filesChecksum: '*', containerSectionChecksum: '*' } - ] + ] as any } } ], @@ -275,14 +275,14 @@ export const algoAssetWithCredentials = { compute: { allowRawAlgorithm: false, allowNetworkAccess: true, - publisherTrustedAlgorithmPublishers: ['*'], + publisherTrustedAlgorithmPublishers: ['*'] as any, publisherTrustedAlgorithms: [ { did: '*', filesChecksum: '*', containerSectionChecksum: '*' } - ] + ] as any } } ], @@ -342,14 +342,14 @@ export const computeAsset = { compute: { allowRawAlgorithm: false, allowNetworkAccess: true, - publisherTrustedAlgorithmPublishers: ['*'], + publisherTrustedAlgorithmPublishers: ['*'] as any, publisherTrustedAlgorithms: [ { did: '*', filesChecksum: '*', containerSectionChecksum: '*' } - ] + ] as any } } ], @@ -408,7 +408,7 @@ export const algoAsset = { services: [ { id: 'db164c1b981e4d2974e90e61bda121512e6909c1035c908d68933ae4cfaba6b0', - type: 'compute', + type: 'access', files: { files: [ { @@ -421,19 +421,7 @@ export const algoAsset = { ] }, timeout: 86400, - serviceEndpoint: 'https://v4.provider.oceanprotocol.com', - compute: { - allowRawAlgorithm: false, - allowNetworkAccess: true, - publisherTrustedAlgorithmPublishers: ['*'], - publisherTrustedAlgorithms: [ - { - did: '*', - filesChecksum: '*', - containerSectionChecksum: '*' - } - ] - } + serviceEndpoint: 'https://v4.provider.oceanprotocol.com' } ], stats: { From 5477ba2e887b27e17e77dc6fc9ff9b3e53a299cf Mon Sep 17 00:00:00 2001 From: mariacarmina Date: Fri, 11 Jul 2025 12:45:01 +0300 Subject: [PATCH 28/34] Fix samples and ci. --- .github/workflows/ci.yml | 2 +- src/test/data/assets.ts | 34 +++++----------------------------- 2 files changed, 6 insertions(+), 30 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ef1428996..78f0d7a02 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -163,7 +163,7 @@ jobs: ASSET_PURGATORY_URL: 'https://raw.githubusercontent.com/oceanprotocol/list-purgatory/main/list-assets.json' ACCOUNT_PURGATORY_URL: 'https://raw.githubusercontent.com/oceanprotocol/list-purgatory/main/list-accounts.json' - name: docker logs - run: docker logs ocean-ocean-contracts-1 && docker logs ocean-kindcluster-1 && docker logs ocean-computetodata-1 && docker logs ocean-typesense-1 + run: docker logs ocean-ocean-contracts-1 && docker logs ocean-typesense-1 if: ${{ failure() }} - uses: actions/upload-artifact@v4 with: diff --git a/src/test/data/assets.ts b/src/test/data/assets.ts index c77f32349..f99a8e67e 100644 --- a/src/test/data/assets.ts +++ b/src/test/data/assets.ts @@ -190,14 +190,8 @@ export const computeAssetWithCredentials = { compute: { allowRawAlgorithm: false, allowNetworkAccess: true, - publisherTrustedAlgorithmPublishers: ['*'] as any, - publisherTrustedAlgorithms: [ - { - did: '*', - filesChecksum: '*', - containerSectionChecksum: '*' - } - ] as any + publisherTrustedAlgorithmPublishers: [] as any, + publisherTrustedAlgorithms: [] as any } } ], @@ -271,19 +265,7 @@ export const algoAssetWithCredentials = { }, credentials: serviceLevelCredentials, timeout: 86400, - serviceEndpoint: 'https://v4.provider.oceanprotocol.com', - compute: { - allowRawAlgorithm: false, - allowNetworkAccess: true, - publisherTrustedAlgorithmPublishers: ['*'] as any, - publisherTrustedAlgorithms: [ - { - did: '*', - filesChecksum: '*', - containerSectionChecksum: '*' - } - ] as any - } + serviceEndpoint: 'https://v4.provider.oceanprotocol.com' } ], stats: { @@ -342,14 +324,8 @@ export const computeAsset = { compute: { allowRawAlgorithm: false, allowNetworkAccess: true, - publisherTrustedAlgorithmPublishers: ['*'] as any, - publisherTrustedAlgorithms: [ - { - did: '*', - filesChecksum: '*', - containerSectionChecksum: '*' - } - ] as any + publisherTrustedAlgorithmPublishers: [] as any, + publisherTrustedAlgorithms: [] as any } } ], From c9f6910b3cd0db1e26ece94ce1a0f5bcac3433b7 Mon Sep 17 00:00:00 2001 From: mariacarmina Date: Fri, 11 Jul 2025 14:34:40 +0300 Subject: [PATCH 29/34] Update ddo.js. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e232fbbd7..6ee5bc494 100644 --- a/package.json +++ b/package.json @@ -73,7 +73,7 @@ "@libp2p/websockets": "^8.1.1", "@multiformats/multiaddr": "^10.2.0", "@oceanprotocol/contracts": "^2.3.0", - "@oceanprotocol/ddo-js": "^0.1.1", + "@oceanprotocol/ddo-js": "^0.1.2", "@types/lodash.clonedeep": "^4.5.7", "axios": "^1.8.4", "base58-js": "^2.0.0", From faa9da0e37612aaba6808b1327169e94433131e6 Mon Sep 17 00:00:00 2001 From: mariacarmina Date: Fri, 11 Jul 2025 16:22:45 +0300 Subject: [PATCH 30/34] Update lock. --- package-lock.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7ef338266..ce3f44a0a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -34,7 +34,7 @@ "@libp2p/websockets": "^8.1.1", "@multiformats/multiaddr": "^10.2.0", "@oceanprotocol/contracts": "^2.3.0", - "@oceanprotocol/ddo-js": "^0.1.1", + "@oceanprotocol/ddo-js": "^0.1.2", "@types/lodash.clonedeep": "^4.5.7", "axios": "^1.8.4", "base58-js": "^2.0.0", @@ -3564,9 +3564,9 @@ "license": "Apache-2.0" }, "node_modules/@oceanprotocol/ddo-js": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/@oceanprotocol/ddo-js/-/ddo-js-0.1.1.tgz", - "integrity": "sha512-RsDUiWfPjylj/xqk4HtUr3qzYzurlBLd2/YW/oP9vC7Gh0uGgJsCShu+ao7hC0Xid4y0iORhkSgiaCtshVGchQ==", + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/@oceanprotocol/ddo-js/-/ddo-js-0.1.2.tgz", + "integrity": "sha512-cDhZ9yk0rpMDqnhkHL3V8eCp2DHQpSqAijlBKBORI6p1CDyb9Q0nDR1kgJObQzEsP1vCOleFkL5cN5vTYBZI3w==", "license": "Apache-2.0", "dependencies": { "@rdfjs/formats-common": "^3.1.0", From 7e7a3d9c42e0056874212ca442214cfa2f98c5cb Mon Sep 17 00:00:00 2001 From: mariacarmina Date: Sun, 13 Jul 2025 17:55:11 +0300 Subject: [PATCH 31/34] Updated ddo schemas version. --- package.json | 2 +- src/test/data/assets.ts | 8 ++++---- src/test/integration/credentials.test.ts | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 6ee5bc494..1a8bba070 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "mocha-light": "mocha --node-env=test --config .mocharc.json --exclude \"./dist/test/integration/compute.test.js\"", "test": "npm run lint && npm run test:unit:cover && npm run test:integration:cover", "test:unit": "npm run build-tests && npm run mocha \"./dist/test/unit/**/*.test.js\"", - "test:integration": "npm run build-tests && npm run mocha \"./dist/test/integration/**/*.test.js\"", + "test:integration": "npm run build-tests && npm run mocha \"./dist/test/integration/credentials.test.js\"", "test:computeunit": "npm run build-tests && npm run mocha \"./dist/test/unit/compute.test.js\"", "test:computeintegration": "npm run build-tests && npm run mocha \"./dist/test/integration/compute.test.js\"", "test:indexer": "npm run build-tests && npm run mocha \"./dist/test/integration/indexer.test.js\"", diff --git a/src/test/data/assets.ts b/src/test/data/assets.ts index f99a8e67e..b648e651e 100644 --- a/src/test/data/assets.ts +++ b/src/test/data/assets.ts @@ -92,7 +92,7 @@ export const downloadAssetWithCredentials = { '@context': ['https://w3id.org/did/v1'], id: '', nftAddress: '', - version: '4.1.0', + version: '4.7.0', chainId: 80001, metadata: { created: '2021-12-20T14:35:20Z', @@ -154,7 +154,7 @@ export const computeAssetWithCredentials = { '@context': ['https://w3id.org/did/v1'], id: '', nftAddress: '', - version: '4.1.0', + version: '4.7.0', chainId: 8996, metadata: { created: '2021-12-20T14:35:20Z', @@ -222,7 +222,7 @@ export const algoAssetWithCredentials = { '@context': ['https://w3id.org/did/v1'], id: '', nftAddress: '', - version: '4.1.0', + version: '4.7.0', chainId: 8996, metadata: { created: '2023-08-01T17:09:39Z', @@ -356,7 +356,7 @@ export const algoAsset = { '@context': ['https://w3id.org/did/v1'], id: '', nftAddress: '', - version: '4.1.0', + version: '4.7.0', chainId: 137, metadata: { created: '2023-08-01T17:09:39Z', diff --git a/src/test/integration/credentials.test.ts b/src/test/integration/credentials.test.ts index 63a825c6b..30a3585d7 100644 --- a/src/test/integration/credentials.test.ts +++ b/src/test/integration/credentials.test.ts @@ -202,7 +202,7 @@ describe('[Credentials Flow] - Should run a complete node flow.', () => { }) it('should publish download datasets', async function () { - this.timeout(DEFAULT_TEST_TIMEOUT * 3) + this.timeout(DEFAULT_TEST_TIMEOUT * 5) const publishedDataset = await publishAsset( downloadAssetWithCredentials, From 5ea33f64db049aebf2528459796bc96e2f1af84f Mon Sep 17 00:00:00 2001 From: mariacarmina Date: Sun, 13 Jul 2025 18:57:35 +0300 Subject: [PATCH 32/34] Fix samples. --- src/test/data/assets.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/test/data/assets.ts b/src/test/data/assets.ts index b648e651e..2dae5c965 100644 --- a/src/test/data/assets.ts +++ b/src/test/data/assets.ts @@ -92,8 +92,8 @@ export const downloadAssetWithCredentials = { '@context': ['https://w3id.org/did/v1'], id: '', nftAddress: '', - version: '4.7.0', - chainId: 80001, + version: '4.1.0', + chainId: 8996, metadata: { created: '2021-12-20T14:35:20Z', updated: '2021-12-20T14:35:20Z', @@ -154,7 +154,7 @@ export const computeAssetWithCredentials = { '@context': ['https://w3id.org/did/v1'], id: '', nftAddress: '', - version: '4.7.0', + version: '4.1.0', chainId: 8996, metadata: { created: '2021-12-20T14:35:20Z', @@ -222,7 +222,7 @@ export const algoAssetWithCredentials = { '@context': ['https://w3id.org/did/v1'], id: '', nftAddress: '', - version: '4.7.0', + version: '4.1.0', chainId: 8996, metadata: { created: '2023-08-01T17:09:39Z', @@ -291,7 +291,7 @@ export const computeAsset = { id: '', nftAddress: '', version: '4.1.0', - chainId: 80001, + chainId: 8996, metadata: { created: '2021-12-20T14:35:20Z', updated: '2021-12-20T14:35:20Z', @@ -356,8 +356,8 @@ export const algoAsset = { '@context': ['https://w3id.org/did/v1'], id: '', nftAddress: '', - version: '4.7.0', - chainId: 137, + version: '4.1.0', + chainId: 8996, metadata: { created: '2023-08-01T17:09:39Z', updated: '2023-08-01T17:09:39Z', From e5f20683d7be18ab09250e194443c0db024d4297 Mon Sep 17 00:00:00 2001 From: mariacarmina Date: Sun, 13 Jul 2025 19:25:41 +0300 Subject: [PATCH 33/34] Add engines for test. --- src/test/integration/credentials.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/integration/credentials.test.ts b/src/test/integration/credentials.test.ts index 30a3585d7..8f43a3457 100644 --- a/src/test/integration/credentials.test.ts +++ b/src/test/integration/credentials.test.ts @@ -139,6 +139,7 @@ describe('[Credentials Flow] - Should run a complete node flow.', () => { oceanNode = await OceanNode.getInstance(config, database) const indexer = new OceanIndexer(database, config.indexingNetworks) oceanNode.addIndexer(indexer) + await oceanNode.addC2DEngines() const rpcs: RPCS = config.supportedNetworks const chain: SupportedNetwork = rpcs[String(DEVELOPMENT_CHAIN_ID)] From 219e97b0181a6ab4c9b582b2a633fda46c8bb5a1 Mon Sep 17 00:00:00 2001 From: mariacarmina Date: Sun, 13 Jul 2025 19:39:55 +0300 Subject: [PATCH 34/34] Bring all tests back. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1a8bba070..6ee5bc494 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "mocha-light": "mocha --node-env=test --config .mocharc.json --exclude \"./dist/test/integration/compute.test.js\"", "test": "npm run lint && npm run test:unit:cover && npm run test:integration:cover", "test:unit": "npm run build-tests && npm run mocha \"./dist/test/unit/**/*.test.js\"", - "test:integration": "npm run build-tests && npm run mocha \"./dist/test/integration/credentials.test.js\"", + "test:integration": "npm run build-tests && npm run mocha \"./dist/test/integration/**/*.test.js\"", "test:computeunit": "npm run build-tests && npm run mocha \"./dist/test/unit/compute.test.js\"", "test:computeintegration": "npm run build-tests && npm run mocha \"./dist/test/integration/compute.test.js\"", "test:indexer": "npm run build-tests && npm run mocha \"./dist/test/integration/indexer.test.js\"",