diff --git a/docs/connected-app-scopes.md b/docs/connected-app-scopes.md new file mode 100644 index 00000000..771ef228 --- /dev/null +++ b/docs/connected-app-scopes.md @@ -0,0 +1,38 @@ +# Connected App Scopes — CH2 Fork + +The upstream Catalyst README lists scopes for the CH1 APIs. The CH2 and Private Spaces endpoints require an expanded scope set. Use the following when creating or updating the Connected App used by this toolkit. + +## Scopes to ADD (for CH2 deployments + Private Spaces) + +| Scope path | Required by | +|----------------------------------------------------------|-----------------------| +| Runtime Manager / Read Applications (Runtime Fabric + CloudHub 2.0) | AMC App Manager v2 | +| Runtime Manager / Read Servers | AMC App Manager v2 | +| Runtime Manager / Manage Applications (read-only acceptable) | Deployment details | +| Cloudhub 2.0 / Read Private Spaces | Private Spaces list | +| Cloudhub 2.0 / Read Private Space Connections | VPN/TGW counts | + +Assign these scopes at the **Master Org + every Sub-Org + every environment** the poller traverses. On a trial account, that's usually just the root organisation plus the Sandbox environment. + +## Scopes to KEEP (common to both upstream and this fork) + +- Exchange / Exchange Viewer +- Design Center / Design Center Developer +- API Manager / View APIs Configuration, View Contracts, View Policies +- Access Management / Profile, View Environment, View Organization + +## Scopes to REMOVE (CH1-only, no longer needed once `ch` collector is off) + +| Scope path | Originally required by | +|-------------------------------------------|------------------------| +| Runtime Manager / Cloudhub Network Viewer | CH1 VPC/VPN/DLB metrics (no CH2 equivalent) | + +## Notes for trial accounts + +- Trial accounts deploy only to **Shared Space**, not Private Spaces. The `ps` collector still runs but returns an empty summary (zero spaces), which is correct behaviour. +- The `ch2` collector works fine on trial accounts because Shared Space deployments also report `target.provider == "MC"`. +- The Anypoint Platform region is `anypoint.mulesoft.com` for US trials; no property change needed if that's where your trial lives. + +## Notes on upstream Issue #220 + +Upstream Issue #220 (vCores return 0 on CH2 with Connected App) is *caused* by the old `ch` collector calling the CH1 endpoint where a CH2 org has no apps. The `ch2` collector avoids the bug by calling the correct API. diff --git a/src/main/mule/aggregators/aggregator-platform-metrics.xml b/src/main/mule/aggregators/aggregator-platform-metrics.xml index 6ce9eb08..9bd9ed10 100644 --- a/src/main/mule/aggregators/aggregator-platform-metrics.xml +++ b/src/main/mule/aggregators/aggregator-platform-metrics.xml @@ -52,13 +52,13 @@ http://www.mulesoft.org/schema/mule/ee/core http://www.mulesoft.org/schema/mule/ - - - + + + - - + + @@ -189,6 +189,21 @@ http://www.mulesoft.org/schema/mule/ee/core http://www.mulesoft.org/schema/mule/ + + + + + + + + + + + + + + + diff --git a/src/main/mule/collectors/anypoint/platform/collector-cloudhub2.xml b/src/main/mule/collectors/anypoint/platform/collector-cloudhub2.xml new file mode 100644 index 00000000..6d4da5b4 --- /dev/null +++ b/src/main/mule/collectors/anypoint/platform/collector-cloudhub2.xml @@ -0,0 +1,82 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + route.payload) }]]> + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/mule/collectors/anypoint/platform/collector-private-spaces.xml b/src/main/mule/collectors/anypoint/platform/collector-private-spaces.xml new file mode 100644 index 00000000..1c07df45 --- /dev/null +++ b/src/main/mule/collectors/anypoint/platform/collector-private-spaces.xml @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/mule/sources/anypoint/platform/apis/api-call-cloudhub2.xml b/src/main/mule/sources/anypoint/platform/apis/api-call-cloudhub2.xml new file mode 100644 index 00000000..9c0b075f --- /dev/null +++ b/src/main/mule/sources/anypoint/platform/apis/api-call-cloudhub2.xml @@ -0,0 +1,69 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/mule/sources/anypoint/platform/apis/api-call-private-spaces.xml b/src/main/mule/sources/anypoint/platform/apis/api-call-private-spaces.xml new file mode 100644 index 00000000..efecf5ee --- /dev/null +++ b/src/main/mule/sources/anypoint/platform/apis/api-call-private-spaces.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + diff --git a/src/main/resources/api/traits/collectors.raml b/src/main/resources/api/traits/collectors.raml index c0b90a40..769bf9b5 100644 --- a/src/main/resources/api/traits/collectors.raml +++ b/src/main/resources/api/traits/collectors.raml @@ -15,10 +15,13 @@ queryParameters: Not available for PCE deployment model: - amq: Anypoint MQ - - apma: API Manager Analytics - - ch: Cloudhub - - rtf: Runtime Fabric - default: all + - apma: API Manager Analytics + - ch: Cloudhub (legacy CH1 - use ch2 instead for CloudHub 2.0 orgs) + - ch2: CloudHub 2.0 (fork addition) + - ps: Private Spaces (fork addition; paid orgs only) + - osv2: Object Store V2 + - rtf: Runtime Fabric + default: all required: false type: string - pattern: ^all|([ap|apc|apm|arm|core|dc|ex|amq|apma|ch|rtf]+([,[ap|apc|apm|arm|core|dc|ex|amq|apma|ch|rtf]])*)$ \ No newline at end of file + pattern: ^all|((ap|apc|apm|arm|core|dc|ex|amq|apma|ch|ch2|ps|osv2|rtf)(,(ap|apc|apm|arm|core|dc|ex|amq|apma|ch|ch2|ps|osv2|rtf))*)$ \ No newline at end of file diff --git a/src/main/resources/dw/aggregation/build-platform-metrics-aggregation.dwl b/src/main/resources/dw/aggregation/build-platform-metrics-aggregation.dwl index fd5a0bfb..0c0db19a 100644 --- a/src/main/resources/dw/aggregation/build-platform-metrics-aggregation.dwl +++ b/src/main/resources/dw/aggregation/build-platform-metrics-aggregation.dwl @@ -275,18 +275,17 @@ var usableProdVcores = entitlements.vCoresProduction.assigned - entitlements.vCo runtimeManagerMetrics: { cloudhub: { networking: { - vpcsTotal: entitlements.vpcs.assigned, - vpcsAvailable: (entitlements.vpcs.assigned default 0) - (usage.vpcsConsumed default 0), - vpcsUsed: usage.vpcsConsumed, - vpnsTotal: entitlements.vpns.assigned, - vpnsAvailable: (entitlements.vpns.assigned default 0) - (usage.vpnsConsumed default 0), - vpnsUsed: usage.vpnsConsumed, - dlbsTotal: entitlements.loadBalancer.assigned, - dlbsAvailable: (entitlements.loadBalancer.assigned default 0) - (usage.loadBalancersConsumed default 0), - dlbsUsed: usage.loadBalancersConsumed, - staticIPsTotal: entitlements.staticIps.assigned, - staticIPsAvailable: (entitlements.staticIps.assigned default 0) - (usage.staticIpsConsumed default 0), - staticIPsUsed: usage.staticIpsConsumed + // CH2-native metrics (sourced from Private Spaces collector, scatter-gather route index 10) + privateSpacesTotal: (payload[10].payload.privateSpacesTotal default 0), + privateSpacesByRegion: (payload[10].payload.privateSpacesByRegion default {}), + privateSpacesByEnvType: (payload[10].payload.privateSpacesByEnvType default {}), + vpnConnectionsTotal: (payload[10].payload.vpnConnectionsTotal default 0), + tgwAttachmentsTotal: (payload[10].payload.tgwAttachmentsTotal default 0), + // CH1 legacy fields retained for dashboard compatibility, set to null on CH2 + vpcsTotal: null, vpcsAvailable: null, vpcsUsed: null, + vpnsTotal: null, vpnsAvailable: null, vpnsUsed: null, + dlbsTotal: null, dlbsAvailable: null, dlbsUsed: null, + staticIPsTotal: null, staticIPsAvailable: null, staticIPsUsed: null }, applications:{ diff --git a/src/main/resources/dw/transform/ch2-to-ch1-adapter.dwl b/src/main/resources/dw/transform/ch2-to-ch1-adapter.dwl new file mode 100644 index 00000000..2495be88 --- /dev/null +++ b/src/main/resources/dw/transform/ch2-to-ch1-adapter.dwl @@ -0,0 +1,92 @@ +%dw 2.0 +// ============================================================================ +// CloudHub 2.0 -> CloudHub 1.0 shape adapter +// ---------------------------------------------------------------------------- +// The upstream Catalyst aggregator (build-platform-metrics-aggregation.dwl) +// computes vCore, app count, and runtime-version metrics by reading: +// +// $.status // expected: "STARTED" +// $.workers."type".weight // expected: numeric vCore per worker +// $.workers.amount // expected: worker count +// $.muleVersion.version // expected: "4.x.y" +// +// CH2 Application Manager v2 returns a totally different shape. This adapter +// maps CH2 fields to the CH1 shape so the aggregator runs unchanged. We also +// attach a ch2 block so CH2-native dashboards can use replica / Private Space +// data when we are ready to extend them. +// +// CH2 reference fields: +// payload.items[*].application.status ("RUNNING" | "NOT_RUNNING" | ...) +// payload.items[*].application.vCores (numeric, e.g. 0.1, 0.5, 1.0) +// payload.items[*].target.provider ("MC" = CloudHub 2.0 Managed, +// includes shared + private space) +// payload.items[*].target.targetId (shared space key or private space id) +// payload.items[*].target.replicas +// payload.items[*].target.deploymentSettings.runtimeVersion ("4.9.3:18e-java17") +// payload.items[*].target.deploymentSettings.resources.cpu.reserved ("100m") +// ============================================================================ +output application/java + +// Constants kept consistent with the rest of the toolkit. +var APP_STATUS_STARTED_CH1 = "STARTED" +var APP_STATUS_STOPPED_CH1 = "STOPPED" + +// Strip ":" suffix so "4.9.3:18e-java17" -> "4.9.3" +fun normaliseRuntime(v: String | Null): String = + if (v == null) "" + else (v splitBy ":")[0] default "" + +// CH2 "application.status" -> CH1 "status" +fun mapStatus(ch2Status: String | Null): String = + if (ch2Status == "RUNNING") APP_STATUS_STARTED_CH1 else APP_STATUS_STOPPED_CH1 + +// Resolve vCore-per-replica. Prefer application.vCores (the simplest and +// most reliable field). If not present, fall back to the millicpu value in +// resources.cpu.reserved (e.g. "100m" -> 0.1, "500m" -> 0.5, "1" -> 1.0). +fun resolveVcoresPerReplica(item: Object): Number = + if (item.application.vCores != null) + (item.application.vCores as Number default 0) + else if (item.target.deploymentSettings.resources.cpu.reserved != null) + do { + var raw = item.target.deploymentSettings.resources.cpu.reserved as String + --- + if (raw endsWith "m") + ((raw replace "m" with "") as Number default 0) / 1000 + else (raw as Number default 0) + } + else 0 + +--- +// Filter only CH2-managed deployments. "MC" covers CH2 shared and private +// space. RTF deployments ("RF") are collected by the separate RTF collector. +((payload.items default []) filter ($.target.provider == "MC")) map ((item, idx) -> { + // ---- CH1-compatible fields (the aggregator reads these) ---- + status: mapStatus(item.application.status), + workers: { + "type": { + name: "CH2-" ++ ((item.target.deploymentSettings.instanceType default "standard") as String), + weight: resolveVcoresPerReplica(item), + cpu: (resolveVcoresPerReplica(item) as String) ++ " vCores", + memory: (item.target.deploymentSettings.resources.memory.reserved default "n/a") as String + }, + amount: (item.target.replicas default 0) as Number + }, + muleVersion: { + version: normaliseRuntime(item.target.deploymentSettings.runtimeVersion) + }, + // ---- CH2-native fields (for future dashboards; ignored by aggregator) ---- + ch2: { + deploymentId: item.id, + name: item.name, + targetId: item.target.targetId, + provider: item.target.provider, + replicas: (item.target.replicas default 0) as Number, + vCoresPerReplica: resolveVcoresPerReplica(item), + vCoresTotal: resolveVcoresPerReplica(item) * ((item.target.replicas default 0) as Number), + applicationStatus: item.application.status default "UNKNOWN", + deploymentStatus: item.status default "UNKNOWN", + runtimeVersion: item.target.deploymentSettings.runtimeVersion default "", + updateStrategy: item.target.deploymentSettings.updateStrategy default "rolling", + lastModifiedDate: item.lastModifiedDate default null + } +}) diff --git a/src/main/resources/dw/transform/private-spaces-networking.dwl b/src/main/resources/dw/transform/private-spaces-networking.dwl new file mode 100644 index 00000000..3009fdde --- /dev/null +++ b/src/main/resources/dw/transform/private-spaces-networking.dwl @@ -0,0 +1,47 @@ +%dw 2.0 +// ============================================================================ +// Private Spaces -> Networking metrics summariser +// ---------------------------------------------------------------------------- +// Replaces the CH1 runtimeManagerMetrics.cloudhub.networking section. +// CH2 constructs used: +// - Private Spaces (replaces VPCs) +// - VPN connections (replaces VPNs) +// - Transit Gateway attachments (replaces peerings) +// - Private Ingress (replaces DLBs) +// Static IPs do not exist as a separate concept in CH2 and are omitted. +// ============================================================================ +output application/json +--- +{ + privateSpacesTotal: sizeOf(payload.content default payload default []), + + privateSpacesByRegion: (payload.content default payload default []) groupBy ($.region default "unknown") + mapObject ((v, k) -> { (k): sizeOf(v) }), + + privateSpacesByEnvType: (payload.content default payload default []) groupBy + (if (isEmpty($.environmentIds default [])) "unassigned" + else ((($.environmentTypes default ["mixed"])[0]) as String default "mixed")) + mapObject ((v, k) -> { (k): sizeOf(v) }), + + privateSpaces: (payload.content default payload default []) map ((ps, idx) -> { + id: ps.id, + name: ps.name, + region: ps.region default "unknown", + status: ps.status default "UNKNOWN", + cidrBlock: ps.network.cidrBlock default "", + reservedCidrs: ps.network.reservedCidrs default [], + associatedEnvironmentCount: sizeOf(ps.environmentIds default []), + connectionCount: sizeOf(ps.connections default []), + vpnConnections: sizeOf((ps.connections default []) filter ($."type" == "vpn" or $.kind == "vpn")), + tgwAttachments: sizeOf((ps.connections default []) filter ($."type" == "tgw" or $.kind == "tgw")), + firewallRules: sizeOf(ps.firewallRules default []) + }), + + // Aggregate roll-ups + vpnConnectionsTotal: sum(((payload.content default payload default []) map + sizeOf(($.connections default []) filter ($."type" == "vpn" or $.kind == "vpn"))) + default [0]), + tgwAttachmentsTotal: sum(((payload.content default payload default []) map + sizeOf(($.connections default []) filter ($."type" == "tgw" or $.kind == "tgw"))) + default [0]) +} diff --git a/src/main/resources/properties/app-common.yaml b/src/main/resources/properties/app-common.yaml index ed1d0941..afdc2081 100644 --- a/src/main/resources/properties/app-common.yaml +++ b/src/main/resources/properties/app-common.yaml @@ -31,6 +31,26 @@ anypoint: maxConcurrency: "100" apps: path: "/cloudhub/api/v2/applications" + # --- CloudHub 2.0 Application Manager v2 (CH2 fork) ----------------- + cloudhub2: + maxConcurrency: "50" + deployments: + path: "/amc/application-manager/api/v2/organizations/{orgId}/environments/{envId}/deployments" + pageSize: "500" + deployment: + detail: + path: "/amc/application-manager/api/v2/organizations/{orgId}/environments/{envId}/deployments/{deploymentId}" + # --- Private Spaces (Runtime Fabric API surface) -------------------- + # Trial and free-tier accounts do not have Private Spaces. The + # collector handles the absence gracefully and returns an empty summary. + privatespaces: + maxConcurrency: "4" + list: + path: "/runtimefabric/api/organizations/{orgId}/privatespaces" + detail: + path: "/runtimefabric/api/organizations/{orgId}/privatespaces/{spaceId}" + connections: + path: "/runtimefabric/api/organizations/{orgId}/privatespaces/{spaceId}/connections" arm: maxConcurrency: "3" apps: