From 6c743558da41604436c44b872b2ae2d5068ba497 Mon Sep 17 00:00:00 2001 From: Mohd T Yaqub Date: Fri, 24 Apr 2026 07:49:50 -0700 Subject: [PATCH 1/4] Add CloudHub 2.0 collector with detail-endpoint fan-out (fixes #220) Upstream ch collector calls /cloudhub/api/v2/applications which returns zero deployments on CH2-only orgs. This adds a ch2 collector that calls the AMC Application Manager v2 API (list + per-deployment detail) and reshapes each item into the CH1 object shape via a new adapter DWL so the existing aggregation logic runs unchanged. Fixes #220. --- .../anypoint/platform/collector-cloudhub2.xml | 82 +++++++++++++++++ .../platform/apis/api-call-cloudhub2.xml | 69 ++++++++++++++ .../dw/transform/ch2-to-ch1-adapter.dwl | 92 +++++++++++++++++++ 3 files changed, 243 insertions(+) create mode 100644 src/main/mule/collectors/anypoint/platform/collector-cloudhub2.xml create mode 100644 src/main/mule/sources/anypoint/platform/apis/api-call-cloudhub2.xml create mode 100644 src/main/resources/dw/transform/ch2-to-ch1-adapter.dwl diff --git a/src/main/mule/collectors/anypoint/platform/collector-cloudhub2.xml b/src/main/mule/collectors/anypoint/platform/collector-cloudhub2.xml new file mode 100644 index 00000000..6d4da5b4 --- /dev/null +++ b/src/main/mule/collectors/anypoint/platform/collector-cloudhub2.xml @@ -0,0 +1,82 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + route.payload) }]]> + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/mule/sources/anypoint/platform/apis/api-call-cloudhub2.xml b/src/main/mule/sources/anypoint/platform/apis/api-call-cloudhub2.xml new file mode 100644 index 00000000..9c0b075f --- /dev/null +++ b/src/main/mule/sources/anypoint/platform/apis/api-call-cloudhub2.xml @@ -0,0 +1,69 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/resources/dw/transform/ch2-to-ch1-adapter.dwl b/src/main/resources/dw/transform/ch2-to-ch1-adapter.dwl new file mode 100644 index 00000000..2495be88 --- /dev/null +++ b/src/main/resources/dw/transform/ch2-to-ch1-adapter.dwl @@ -0,0 +1,92 @@ +%dw 2.0 +// ============================================================================ +// CloudHub 2.0 -> CloudHub 1.0 shape adapter +// ---------------------------------------------------------------------------- +// The upstream Catalyst aggregator (build-platform-metrics-aggregation.dwl) +// computes vCore, app count, and runtime-version metrics by reading: +// +// $.status // expected: "STARTED" +// $.workers."type".weight // expected: numeric vCore per worker +// $.workers.amount // expected: worker count +// $.muleVersion.version // expected: "4.x.y" +// +// CH2 Application Manager v2 returns a totally different shape. This adapter +// maps CH2 fields to the CH1 shape so the aggregator runs unchanged. We also +// attach a ch2 block so CH2-native dashboards can use replica / Private Space +// data when we are ready to extend them. +// +// CH2 reference fields: +// payload.items[*].application.status ("RUNNING" | "NOT_RUNNING" | ...) +// payload.items[*].application.vCores (numeric, e.g. 0.1, 0.5, 1.0) +// payload.items[*].target.provider ("MC" = CloudHub 2.0 Managed, +// includes shared + private space) +// payload.items[*].target.targetId (shared space key or private space id) +// payload.items[*].target.replicas +// payload.items[*].target.deploymentSettings.runtimeVersion ("4.9.3:18e-java17") +// payload.items[*].target.deploymentSettings.resources.cpu.reserved ("100m") +// ============================================================================ +output application/java + +// Constants kept consistent with the rest of the toolkit. +var APP_STATUS_STARTED_CH1 = "STARTED" +var APP_STATUS_STOPPED_CH1 = "STOPPED" + +// Strip ":" suffix so "4.9.3:18e-java17" -> "4.9.3" +fun normaliseRuntime(v: String | Null): String = + if (v == null) "" + else (v splitBy ":")[0] default "" + +// CH2 "application.status" -> CH1 "status" +fun mapStatus(ch2Status: String | Null): String = + if (ch2Status == "RUNNING") APP_STATUS_STARTED_CH1 else APP_STATUS_STOPPED_CH1 + +// Resolve vCore-per-replica. Prefer application.vCores (the simplest and +// most reliable field). If not present, fall back to the millicpu value in +// resources.cpu.reserved (e.g. "100m" -> 0.1, "500m" -> 0.5, "1" -> 1.0). +fun resolveVcoresPerReplica(item: Object): Number = + if (item.application.vCores != null) + (item.application.vCores as Number default 0) + else if (item.target.deploymentSettings.resources.cpu.reserved != null) + do { + var raw = item.target.deploymentSettings.resources.cpu.reserved as String + --- + if (raw endsWith "m") + ((raw replace "m" with "") as Number default 0) / 1000 + else (raw as Number default 0) + } + else 0 + +--- +// Filter only CH2-managed deployments. "MC" covers CH2 shared and private +// space. RTF deployments ("RF") are collected by the separate RTF collector. +((payload.items default []) filter ($.target.provider == "MC")) map ((item, idx) -> { + // ---- CH1-compatible fields (the aggregator reads these) ---- + status: mapStatus(item.application.status), + workers: { + "type": { + name: "CH2-" ++ ((item.target.deploymentSettings.instanceType default "standard") as String), + weight: resolveVcoresPerReplica(item), + cpu: (resolveVcoresPerReplica(item) as String) ++ " vCores", + memory: (item.target.deploymentSettings.resources.memory.reserved default "n/a") as String + }, + amount: (item.target.replicas default 0) as Number + }, + muleVersion: { + version: normaliseRuntime(item.target.deploymentSettings.runtimeVersion) + }, + // ---- CH2-native fields (for future dashboards; ignored by aggregator) ---- + ch2: { + deploymentId: item.id, + name: item.name, + targetId: item.target.targetId, + provider: item.target.provider, + replicas: (item.target.replicas default 0) as Number, + vCoresPerReplica: resolveVcoresPerReplica(item), + vCoresTotal: resolveVcoresPerReplica(item) * ((item.target.replicas default 0) as Number), + applicationStatus: item.application.status default "UNKNOWN", + deploymentStatus: item.status default "UNKNOWN", + runtimeVersion: item.target.deploymentSettings.runtimeVersion default "", + updateStrategy: item.target.deploymentSettings.updateStrategy default "rolling", + lastModifiedDate: item.lastModifiedDate default null + } +}) From ca84500b101c567bf4137a40bb849b090c6a83b3 Mon Sep 17 00:00:00 2001 From: Mohd T Yaqub Date: Fri, 24 Apr 2026 07:49:50 -0700 Subject: [PATCH 2/4] Add Private Spaces collector (CH2 networking replacement) The CH1 networking fields (VPCs, VPNs, DLBs, static IPs) do not exist on CH2. This adds a ps collector reading /runtimefabric/api/.../privatespaces and a summariser DWL that emits a CH2-native networking block (privateSpacesTotal, vpnConnectionsTotal, tgwAttachmentsTotal). Trial and free-tier accounts with no Private Spaces receive an empty summary without errors. --- .../platform/collector-private-spaces.xml | 43 +++++++++++++++++ .../platform/apis/api-call-private-spaces.xml | 33 +++++++++++++ .../transform/private-spaces-networking.dwl | 47 +++++++++++++++++++ 3 files changed, 123 insertions(+) create mode 100644 src/main/mule/collectors/anypoint/platform/collector-private-spaces.xml create mode 100644 src/main/mule/sources/anypoint/platform/apis/api-call-private-spaces.xml create mode 100644 src/main/resources/dw/transform/private-spaces-networking.dwl diff --git a/src/main/mule/collectors/anypoint/platform/collector-private-spaces.xml b/src/main/mule/collectors/anypoint/platform/collector-private-spaces.xml new file mode 100644 index 00000000..1c07df45 --- /dev/null +++ b/src/main/mule/collectors/anypoint/platform/collector-private-spaces.xml @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/mule/sources/anypoint/platform/apis/api-call-private-spaces.xml b/src/main/mule/sources/anypoint/platform/apis/api-call-private-spaces.xml new file mode 100644 index 00000000..efecf5ee --- /dev/null +++ b/src/main/mule/sources/anypoint/platform/apis/api-call-private-spaces.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + diff --git a/src/main/resources/dw/transform/private-spaces-networking.dwl b/src/main/resources/dw/transform/private-spaces-networking.dwl new file mode 100644 index 00000000..3009fdde --- /dev/null +++ b/src/main/resources/dw/transform/private-spaces-networking.dwl @@ -0,0 +1,47 @@ +%dw 2.0 +// ============================================================================ +// Private Spaces -> Networking metrics summariser +// ---------------------------------------------------------------------------- +// Replaces the CH1 runtimeManagerMetrics.cloudhub.networking section. +// CH2 constructs used: +// - Private Spaces (replaces VPCs) +// - VPN connections (replaces VPNs) +// - Transit Gateway attachments (replaces peerings) +// - Private Ingress (replaces DLBs) +// Static IPs do not exist as a separate concept in CH2 and are omitted. +// ============================================================================ +output application/json +--- +{ + privateSpacesTotal: sizeOf(payload.content default payload default []), + + privateSpacesByRegion: (payload.content default payload default []) groupBy ($.region default "unknown") + mapObject ((v, k) -> { (k): sizeOf(v) }), + + privateSpacesByEnvType: (payload.content default payload default []) groupBy + (if (isEmpty($.environmentIds default [])) "unassigned" + else ((($.environmentTypes default ["mixed"])[0]) as String default "mixed")) + mapObject ((v, k) -> { (k): sizeOf(v) }), + + privateSpaces: (payload.content default payload default []) map ((ps, idx) -> { + id: ps.id, + name: ps.name, + region: ps.region default "unknown", + status: ps.status default "UNKNOWN", + cidrBlock: ps.network.cidrBlock default "", + reservedCidrs: ps.network.reservedCidrs default [], + associatedEnvironmentCount: sizeOf(ps.environmentIds default []), + connectionCount: sizeOf(ps.connections default []), + vpnConnections: sizeOf((ps.connections default []) filter ($."type" == "vpn" or $.kind == "vpn")), + tgwAttachments: sizeOf((ps.connections default []) filter ($."type" == "tgw" or $.kind == "tgw")), + firewallRules: sizeOf(ps.firewallRules default []) + }), + + // Aggregate roll-ups + vpnConnectionsTotal: sum(((payload.content default payload default []) map + sizeOf(($.connections default []) filter ($."type" == "vpn" or $.kind == "vpn"))) + default [0]), + tgwAttachmentsTotal: sum(((payload.content default payload default []) map + sizeOf(($.connections default []) filter ($."type" == "tgw" or $.kind == "tgw"))) + default [0]) +} From 6c6f0e5edda7f8604c445cd2fa3f52f45ceb21f1 Mon Sep 17 00:00:00 2001 From: Mohd T Yaqub Date: Fri, 24 Apr 2026 07:49:50 -0700 Subject: [PATCH 3/4] Wire ch2 and ps collectors into the aggregator - Replace the ch collector choice with a ch2 choice in the scatter-gather. - Append a ps route at scatter-gather index 10 so existing collector payload indices remain stable. - Rewrite the runtimeManagerMetrics.cloudhub.networking block in the aggregation DWL to read from the ps route instead of CH1 entitlements fields that do not exist on CH2 orgs. CH1 legacy fields retained as null for dashboard schema stability. --- .../aggregator-platform-metrics.xml | 25 +++++++++++++++---- .../build-platform-metrics-aggregation.dwl | 23 ++++++++--------- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/src/main/mule/aggregators/aggregator-platform-metrics.xml b/src/main/mule/aggregators/aggregator-platform-metrics.xml index 6ce9eb08..9bd9ed10 100644 --- a/src/main/mule/aggregators/aggregator-platform-metrics.xml +++ b/src/main/mule/aggregators/aggregator-platform-metrics.xml @@ -52,13 +52,13 @@ http://www.mulesoft.org/schema/mule/ee/core http://www.mulesoft.org/schema/mule/ - - - + + + - - + + @@ -189,6 +189,21 @@ http://www.mulesoft.org/schema/mule/ee/core http://www.mulesoft.org/schema/mule/ + + + + + + + + + + + + + + + diff --git a/src/main/resources/dw/aggregation/build-platform-metrics-aggregation.dwl b/src/main/resources/dw/aggregation/build-platform-metrics-aggregation.dwl index fd5a0bfb..0c0db19a 100644 --- a/src/main/resources/dw/aggregation/build-platform-metrics-aggregation.dwl +++ b/src/main/resources/dw/aggregation/build-platform-metrics-aggregation.dwl @@ -275,18 +275,17 @@ var usableProdVcores = entitlements.vCoresProduction.assigned - entitlements.vCo runtimeManagerMetrics: { cloudhub: { networking: { - vpcsTotal: entitlements.vpcs.assigned, - vpcsAvailable: (entitlements.vpcs.assigned default 0) - (usage.vpcsConsumed default 0), - vpcsUsed: usage.vpcsConsumed, - vpnsTotal: entitlements.vpns.assigned, - vpnsAvailable: (entitlements.vpns.assigned default 0) - (usage.vpnsConsumed default 0), - vpnsUsed: usage.vpnsConsumed, - dlbsTotal: entitlements.loadBalancer.assigned, - dlbsAvailable: (entitlements.loadBalancer.assigned default 0) - (usage.loadBalancersConsumed default 0), - dlbsUsed: usage.loadBalancersConsumed, - staticIPsTotal: entitlements.staticIps.assigned, - staticIPsAvailable: (entitlements.staticIps.assigned default 0) - (usage.staticIpsConsumed default 0), - staticIPsUsed: usage.staticIpsConsumed + // CH2-native metrics (sourced from Private Spaces collector, scatter-gather route index 10) + privateSpacesTotal: (payload[10].payload.privateSpacesTotal default 0), + privateSpacesByRegion: (payload[10].payload.privateSpacesByRegion default {}), + privateSpacesByEnvType: (payload[10].payload.privateSpacesByEnvType default {}), + vpnConnectionsTotal: (payload[10].payload.vpnConnectionsTotal default 0), + tgwAttachmentsTotal: (payload[10].payload.tgwAttachmentsTotal default 0), + // CH1 legacy fields retained for dashboard compatibility, set to null on CH2 + vpcsTotal: null, vpcsAvailable: null, vpcsUsed: null, + vpnsTotal: null, vpnsAvailable: null, vpnsUsed: null, + dlbsTotal: null, dlbsAvailable: null, dlbsUsed: null, + staticIPsTotal: null, staticIPsAvailable: null, staticIPsUsed: null }, applications:{ From faaf7f2bbd69e19b0060edb6e6e25ee7fb5fab61 Mon Sep 17 00:00:00 2001 From: Mohd T Yaqub Date: Fri, 24 Apr 2026 07:49:50 -0700 Subject: [PATCH 4/4] Add CH2/PS API paths, widen collectors regex, document scopes - app-common.yaml: add cloudhub2.* and privatespaces.* API path blocks under anypoint.platform.apis. - collectors.raml: fix a subtle bug in the collectors parameter pattern (the character class [ap|apc|...] matched any letter in the set, not the intended tokens). Rewrite as proper alternation and add ch2, ps, osv2 as valid tokens. - docs/connected-app-scopes.md: new - documents the Connected App scopes required for ch2 and ps collectors, including what changes vs CH1. --- docs/connected-app-scopes.md | 38 +++++++++++++++++++ src/main/resources/api/traits/collectors.raml | 13 ++++--- src/main/resources/properties/app-common.yaml | 20 ++++++++++ 3 files changed, 66 insertions(+), 5 deletions(-) create mode 100644 docs/connected-app-scopes.md diff --git a/docs/connected-app-scopes.md b/docs/connected-app-scopes.md new file mode 100644 index 00000000..771ef228 --- /dev/null +++ b/docs/connected-app-scopes.md @@ -0,0 +1,38 @@ +# Connected App Scopes — CH2 Fork + +The upstream Catalyst README lists scopes for the CH1 APIs. The CH2 and Private Spaces endpoints require an expanded scope set. Use the following when creating or updating the Connected App used by this toolkit. + +## Scopes to ADD (for CH2 deployments + Private Spaces) + +| Scope path | Required by | +|----------------------------------------------------------|-----------------------| +| Runtime Manager / Read Applications (Runtime Fabric + CloudHub 2.0) | AMC App Manager v2 | +| Runtime Manager / Read Servers | AMC App Manager v2 | +| Runtime Manager / Manage Applications (read-only acceptable) | Deployment details | +| Cloudhub 2.0 / Read Private Spaces | Private Spaces list | +| Cloudhub 2.0 / Read Private Space Connections | VPN/TGW counts | + +Assign these scopes at the **Master Org + every Sub-Org + every environment** the poller traverses. On a trial account, that's usually just the root organisation plus the Sandbox environment. + +## Scopes to KEEP (common to both upstream and this fork) + +- Exchange / Exchange Viewer +- Design Center / Design Center Developer +- API Manager / View APIs Configuration, View Contracts, View Policies +- Access Management / Profile, View Environment, View Organization + +## Scopes to REMOVE (CH1-only, no longer needed once `ch` collector is off) + +| Scope path | Originally required by | +|-------------------------------------------|------------------------| +| Runtime Manager / Cloudhub Network Viewer | CH1 VPC/VPN/DLB metrics (no CH2 equivalent) | + +## Notes for trial accounts + +- Trial accounts deploy only to **Shared Space**, not Private Spaces. The `ps` collector still runs but returns an empty summary (zero spaces), which is correct behaviour. +- The `ch2` collector works fine on trial accounts because Shared Space deployments also report `target.provider == "MC"`. +- The Anypoint Platform region is `anypoint.mulesoft.com` for US trials; no property change needed if that's where your trial lives. + +## Notes on upstream Issue #220 + +Upstream Issue #220 (vCores return 0 on CH2 with Connected App) is *caused* by the old `ch` collector calling the CH1 endpoint where a CH2 org has no apps. The `ch2` collector avoids the bug by calling the correct API. diff --git a/src/main/resources/api/traits/collectors.raml b/src/main/resources/api/traits/collectors.raml index c0b90a40..769bf9b5 100644 --- a/src/main/resources/api/traits/collectors.raml +++ b/src/main/resources/api/traits/collectors.raml @@ -15,10 +15,13 @@ queryParameters: Not available for PCE deployment model: - amq: Anypoint MQ - - apma: API Manager Analytics - - ch: Cloudhub - - rtf: Runtime Fabric - default: all + - apma: API Manager Analytics + - ch: Cloudhub (legacy CH1 - use ch2 instead for CloudHub 2.0 orgs) + - ch2: CloudHub 2.0 (fork addition) + - ps: Private Spaces (fork addition; paid orgs only) + - osv2: Object Store V2 + - rtf: Runtime Fabric + default: all required: false type: string - pattern: ^all|([ap|apc|apm|arm|core|dc|ex|amq|apma|ch|rtf]+([,[ap|apc|apm|arm|core|dc|ex|amq|apma|ch|rtf]])*)$ \ No newline at end of file + pattern: ^all|((ap|apc|apm|arm|core|dc|ex|amq|apma|ch|ch2|ps|osv2|rtf)(,(ap|apc|apm|arm|core|dc|ex|amq|apma|ch|ch2|ps|osv2|rtf))*)$ \ No newline at end of file diff --git a/src/main/resources/properties/app-common.yaml b/src/main/resources/properties/app-common.yaml index ed1d0941..afdc2081 100644 --- a/src/main/resources/properties/app-common.yaml +++ b/src/main/resources/properties/app-common.yaml @@ -31,6 +31,26 @@ anypoint: maxConcurrency: "100" apps: path: "/cloudhub/api/v2/applications" + # --- CloudHub 2.0 Application Manager v2 (CH2 fork) ----------------- + cloudhub2: + maxConcurrency: "50" + deployments: + path: "/amc/application-manager/api/v2/organizations/{orgId}/environments/{envId}/deployments" + pageSize: "500" + deployment: + detail: + path: "/amc/application-manager/api/v2/organizations/{orgId}/environments/{envId}/deployments/{deploymentId}" + # --- Private Spaces (Runtime Fabric API surface) -------------------- + # Trial and free-tier accounts do not have Private Spaces. The + # collector handles the absence gracefully and returns an empty summary. + privatespaces: + maxConcurrency: "4" + list: + path: "/runtimefabric/api/organizations/{orgId}/privatespaces" + detail: + path: "/runtimefabric/api/organizations/{orgId}/privatespaces/{spaceId}" + connections: + path: "/runtimefabric/api/organizations/{orgId}/privatespaces/{spaceId}/connections" arm: maxConcurrency: "3" apps: