Add CloudHub 2.0 support (fixes #220)

docs/connected-app-scopes.md

@@ -0,0 +1,38 @@
+# Connected App Scopes — CH2 Fork
+
+The upstream Catalyst README lists scopes for the CH1 APIs. The CH2 and Private Spaces endpoints require an expanded scope set. Use the following when creating or updating the Connected App used by this toolkit.
+
+## Scopes to ADD (for CH2 deployments + Private Spaces)
+
+| Scope path                                               | Required by           |
+|----------------------------------------------------------|-----------------------|
+| Runtime Manager / Read Applications (Runtime Fabric + CloudHub 2.0) | AMC App Manager v2    |
+| Runtime Manager / Read Servers                           | AMC App Manager v2    |
+| Runtime Manager / Manage Applications (read-only acceptable) | Deployment details |
+| Cloudhub 2.0 / Read Private Spaces                       | Private Spaces list   |
+| Cloudhub 2.0 / Read Private Space Connections            | VPN/TGW counts        |
+
+Assign these scopes at the **Master Org + every Sub-Org + every environment** the poller traverses. On a trial account, that's usually just the root organisation plus the Sandbox environment.
+
+## Scopes to KEEP (common to both upstream and this fork)
+
+- Exchange / Exchange Viewer
+- Design Center / Design Center Developer
+- API Manager / View APIs Configuration, View Contracts, View Policies
+- Access Management / Profile, View Environment, View Organization
+
+## Scopes to REMOVE (CH1-only, no longer needed once `ch` collector is off)
+
+| Scope path                                | Originally required by |
+|-------------------------------------------|------------------------|
+| Runtime Manager / Cloudhub Network Viewer | CH1 VPC/VPN/DLB metrics (no CH2 equivalent) |
+
+## Notes for trial accounts
+
+- Trial accounts deploy only to **Shared Space**, not Private Spaces. The `ps` collector still runs but returns an empty summary (zero spaces), which is correct behaviour.
+- The `ch2` collector works fine on trial accounts because Shared Space deployments also report `target.provider == "MC"`.
+- The Anypoint Platform region is `anypoint.mulesoft.com` for US trials; no property change needed if that's where your trial lives.
+
+## Notes on upstream Issue #220
+
+Upstream Issue #220 (vCores return 0 on CH2 with Connected App) is *caused* by the old `ch` collector calling the CH1 endpoint where a CH2 org has no apps. The `ch2` collector avoids the bug by calling the correct API.

src/main/mule/aggregators/aggregator-platform-metrics.xml

@@ -52,13 +52,13 @@ http://www.mulesoft.org/schema/mule/ee/core http://www.mulesoft.org/schema/mule/
 			<when expression="#[not isEmpty(vars.environments)]">
 				<scatter-gather doc:name="Scatter-Gather" doc:id="9b9ce41d-173c-4191-ad34-b17eb5f95840">
 					<route>
-						<choice doc:name="If collectors contains ch" doc:id="26ee8f45-60b3-4b0a-a0f8-7c86bd28a82e" >
-							<when expression='#[vars.collectors as Array contains "ch"]'>
-								<flow-ref doc:name="Get CloudHub Apps Metrics " doc:id="4b943a49-d16a-4691-987d-364a82fdcd6f" name="collector-cloudhub-apps-metrics-parallel-flow" />
+						<choice doc:name="If collectors contains ch2" doc:id="ch2-choice-0001" >
+							<when expression='#[vars.collectors as Array contains "ch2"]'>
+								<flow-ref doc:name="Get CloudHub 2.0 Apps Metrics" doc:id="ch2-flowref-0001" name="collector-cloudhub2-apps-metrics-parallel-flow" />
 							</when>
 							<otherwise >
-								<logger level="INFO" doc:name="Log - No CH" doc:id="7a37b092-1ef2-47ec-b50f-a4eee11563ec" message="CloudHub Metrics won't be collected: collector not enabled or not supported for the platform deployment model or authentication mode."/>
-								<ee:transform doc:name="Set Null Response" doc:id="9fe0e67d-09a3-40e9-9ec5-5694b4503671" >
+								<logger level="INFO" doc:name="Log - No CH2" doc:id="ch2-logger-0001" message="CloudHub 2.0 Metrics won't be collected: ch2 collector not enabled."/>
+								<ee:transform doc:name="Set Null Response" doc:id="ch2-setnull-0001" >
 									<ee:message >
 										<ee:set-payload resource="dw/aggregation/set-null-response.dwl" />
 									</ee:message>
@@ -189,6 +189,21 @@ http://www.mulesoft.org/schema/mule/ee/core http://www.mulesoft.org/schema/mule/
 							</otherwise>
 						</choice>
 					</route>
+					<route>
+						<choice doc:name="If collectors contains ps" doc:id="ps-choice-0001" >
+							<when expression='#[vars.collectors as Array contains "ps"]'>
+								<flow-ref doc:name="Get Private Spaces Metrics" doc:id="ps-flowref-0001" name="collector-private-spaces-metrics-flow" />
+							</when>
+							<otherwise >
+								<logger level="INFO" doc:name="Log - No PS" doc:id="ps-logger-0001" message="Private Spaces metrics won't be collected: ps collector not enabled (trial accounts have no Private Spaces)."/>
+								<ee:transform doc:name="Set Null Response" doc:id="ps-setnull-0001" >
+									<ee:message >
+										<ee:set-payload resource="dw/aggregation/set-null-response.dwl" />
+									</ee:message>
+								</ee:transform>
+							</otherwise>
+						</choice>
+					</route>
 				</scatter-gather>
 				<choice doc:name="If rawData is needed" doc:id="3c420ab6-a0bb-4d47-ad1e-fa6812e35830">
 					<when expression="#[vars.rawData]">

src/main/mule/collectors/anypoint/platform/collector-cloudhub2.xml

@@ -0,0 +1,82 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+	CloudHub 2.0 Collector
+	======================
+	Replacement for the upstream CH1 "collector-cloudhub.xml". Collects
+	deployments via the Application Manager v2 API and reshapes each item
+	into the CH1-compatible structure consumed by the existing aggregator
+	(build-platform-metrics-aggregation.dwl) so no downstream DWL changes
+	are required.
+-->
+<mule xmlns:ee="http://www.mulesoft.org/schema/mule/ee/core"
+	xmlns="http://www.mulesoft.org/schema/mule/core"
+	xmlns:doc="http://www.mulesoft.org/schema/mule/documentation"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://www.mulesoft.org/schema/mule/core http://www.mulesoft.org/schema/mule/core/current/mule.xsd
+http://www.mulesoft.org/schema/mule/ee/core http://www.mulesoft.org/schema/mule/ee/core/current/mule-ee.xsd">
+
+	<flow name="collector-cloudhub2-apps-metrics-parallel-flow" doc:id="ch2-collector-parallel-flow-0001">
+		<parallel-foreach doc:name="Parallel For Each - Environments" collection="#[vars.environments]">
+			<flow-ref doc:name="Set Environment Variables" name="collector-commons-set-environment-variables-flow" />
+			<flow-ref doc:name="Get CH2 Deployments" name="api-call-cloudhub2-deployments-flow" />
+
+			<!-- The list endpoint returns a slim summary. Fan out a detail call per
+				 deployment ID and assemble an items[] array of full detail responses
+				 so the adapter DWL can read application.vCores / target.replicas / etc. -->
+			<ee:transform doc:name="Stash list summary items">
+				<ee:variables>
+					<ee:set-variable variableName="deploymentsList"><![CDATA[%dw 2.0
+output application/java
+---
+payload.items default []]]></ee:set-variable>
+				</ee:variables>
+			</ee:transform>
+
+			<!-- parallel-foreach collects per-route results into a payload array -->
+			<parallel-foreach doc:name="Parallel fan-out: deployment detail"
+				collection="#[vars.deploymentsList]">
+				<set-variable variableName="deploymentId" value="#[payload.id]" />
+				<try>
+					<flow-ref doc:name="Get CH2 Deployment Detail"
+						name="api-call-cloudhub2-deployment-detail-flow" />
+					<error-handler>
+						<on-error-continue>
+							<logger level="WARN" message='#["Detail fetch failed for deploymentId=" ++ vars.deploymentId ++ "; keeping list summary."]' />
+							<!-- Fall back to the list summary for this deployment id -->
+							<set-payload value="#[(vars.deploymentsList filter ($.id == vars.deploymentId))[0]]" />
+						</on-error-continue>
+					</error-handler>
+				</try>
+			</parallel-foreach>
+
+			<!-- parallel-foreach wraps each result as { payload: ..., attributes: ... }.
+				 Unwrap and re-shape into { items: [...detail...] } for the adapter. -->
+			<ee:transform doc:name="Reshape items[] with enriched details">
+				<ee:message>
+					<ee:set-payload><![CDATA[%dw 2.0
+output application/java
+---
+{ items: payload map ((route, idx) -> route.payload) }]]></ee:set-payload>
+				</ee:message>
+			</ee:transform>
+
+			<ee:transform doc:name="Adapt CH2 -> CH1 shape">
+				<ee:message>
+					<ee:set-payload resource="dw/transform/ch2-to-ch1-adapter.dwl" />
+				</ee:message>
+			</ee:transform>
+
+			<set-payload
+				value='#[output application/json --- { environment: vars.environmentName, isProduction: vars.isProduction, data: payload }]'
+				doc:name="Aggregate metrics (CH1-shaped)" />
+		</parallel-foreach>
+
+		<error-handler>
+			<on-error-continue enableNotifications="true" logException="true" doc:name="On Error Continue">
+				<logger level="WARN" message="'Unable to get CloudHub 2.0 deployments for BG:' ++ vars.orgId" />
+				<set-variable value="#[vars.errors + error.description]" variableName="errors" />
+			</on-error-continue>
+		</error-handler>
+	</flow>
+
+</mule>

src/main/mule/collectors/anypoint/platform/collector-private-spaces.xml

@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+	Private Spaces Collector
+	========================
+	Org-level collector (not per-env). Replaces the CH1 networking metrics
+	(VPC/VPN/DLB/StaticIP counts) with CH2-native Private Space inventory.
+
+	Note on trial / free-tier accounts: Private Spaces require a paid
+	subscription. On a trial account this API returns an empty list or 403.
+	The on-error-continue handler returns an empty summary so the overall
+	metrics pipeline is not broken by its absence.
+-->
+<mule xmlns:ee="http://www.mulesoft.org/schema/mule/ee/core"
+	xmlns="http://www.mulesoft.org/schema/mule/core"
+	xmlns:doc="http://www.mulesoft.org/schema/mule/documentation"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://www.mulesoft.org/schema/mule/core http://www.mulesoft.org/schema/mule/core/current/mule.xsd
+http://www.mulesoft.org/schema/mule/ee/core http://www.mulesoft.org/schema/mule/ee/core/current/mule-ee.xsd">
+
+	<flow name="collector-private-spaces-metrics-flow" doc:id="ps-collector-0001">
+		<flow-ref doc:name="List Private Spaces" name="api-call-private-spaces-list-flow" />
+		<ee:transform doc:name="Summarise networking">
+			<ee:message>
+				<ee:set-payload resource="dw/transform/private-spaces-networking.dwl" />
+			</ee:message>
+		</ee:transform>
+		<error-handler>
+			<on-error-continue enableNotifications="true" logException="true" doc:name="On Error Continue">
+				<logger level="WARN" message="'Unable to list Private Spaces (possibly not entitled on this tier) for BG:' ++ vars.orgId" />
+				<set-payload value='#[output application/json --- {
+					privateSpacesTotal: 0,
+					privateSpacesByRegion: {},
+					privateSpacesByEnvType: {},
+					vpnConnectionsTotal: 0,
+					tgwAttachmentsTotal: 0,
+					privateSpaces: []
+				}]' />
+				<set-variable value="#[vars.errors + error.description]" variableName="errors" />
+			</on-error-continue>
+		</error-handler>
+	</flow>
+
+</mule>

src/main/mule/sources/anypoint/platform/apis/api-call-cloudhub2.xml

@@ -0,0 +1,69 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+	CloudHub 2.0 Deployments API call
+	=================================
+	Wraps GET /amc/application-manager/api/v2/organizations/{orgId}/environments/{envId}/deployments
+	This replaces the CH1 call to /cloudhub/api/v2/applications. The response shape is completely
+	different and is adapted to the CH1 shape downstream.
+-->
+<mule xmlns:http="http://www.mulesoft.org/schema/mule/http"
+	xmlns="http://www.mulesoft.org/schema/mule/core"
+	xmlns:doc="http://www.mulesoft.org/schema/mule/documentation"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://www.mulesoft.org/schema/mule/core http://www.mulesoft.org/schema/mule/core/current/mule.xsd
+http://www.mulesoft.org/schema/mule/http http://www.mulesoft.org/schema/mule/http/current/mule-http.xsd">
+
+	<flow name="api-call-cloudhub2-deployments-flow" maxConcurrency="${anypoint.platform.apis.cloudhub2.maxConcurrency}">
+		<logger level="DEBUG" message="Calling AMC App Manager v2 - Deployments" />
+		<http:request method="GET" doc:name="Get CH2 Deployments"
+			config-ref="HTTP_Request_configuration"
+			path="${anypoint.platform.apis.cloudhub2.deployments.path}">
+			<http:headers><![CDATA[#[output application/java
+---
+{
+	"Authorization" : "Bearer " ++ vars.token
+}]]]></http:headers>
+			<http:uri-params><![CDATA[#[output application/java
+---
+{
+	"orgId"   : vars.orgId,
+	"envId"   : vars.environmentId
+}]]]></http:uri-params>
+			<http:query-params><![CDATA[#[output application/java
+---
+{
+	"filterBy"    : "all",
+	"limit"       : "${anypoint.platform.apis.cloudhub2.deployments.pageSize}",
+	"offset"      : "0"
+}]]]></http:query-params>
+		</http:request>
+		<logger level="DEBUG" message='#["CH2 Deployments, Response Status Code:" ++ attributes.statusCode]' />
+	</flow>
+
+	<!-- Fetches the full detail of a single deployment. The list endpoint returns a
+		 slim summary (no vCores, no replicas, no deploymentSettings), so we follow
+		 up with this per-deployment detail call and feed the enriched payload to
+		 the adapter DWL.
+		 Inputs (vars): orgId, environmentId, deploymentId, token -->
+	<flow name="api-call-cloudhub2-deployment-detail-flow" maxConcurrency="${anypoint.platform.apis.cloudhub2.maxConcurrency}">
+		<logger level="DEBUG" message='#["Calling AMC App Manager v2 - Deployment Detail: " ++ vars.deploymentId]' />
+		<http:request method="GET" doc:name="Get CH2 Deployment Detail"
+			config-ref="HTTP_Request_configuration"
+			path="${anypoint.platform.apis.cloudhub2.deployment.detail.path}">
+			<http:headers><![CDATA[#[output application/java
+---
+{
+	"Authorization" : "Bearer " ++ vars.token
+}]]]></http:headers>
+			<http:uri-params><![CDATA[#[output application/java
+---
+{
+	"orgId"         : vars.orgId,
+	"envId"         : vars.environmentId,
+	"deploymentId"  : vars.deploymentId
+}]]]></http:uri-params>
+		</http:request>
+		<logger level="DEBUG" message='#["CH2 Deployment Detail, Response Status Code:" ++ attributes.statusCode]' />
+	</flow>
+
+</mule>

src/main/mule/sources/anypoint/platform/apis/api-call-private-spaces.xml

@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+	Private Spaces API calls
+	========================
+	GET /runtimefabric/api/organizations/{orgId}/privatespaces
+-->
+<mule xmlns:http="http://www.mulesoft.org/schema/mule/http"
+	xmlns="http://www.mulesoft.org/schema/mule/core"
+	xmlns:doc="http://www.mulesoft.org/schema/mule/documentation"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://www.mulesoft.org/schema/mule/core http://www.mulesoft.org/schema/mule/core/current/mule.xsd
+http://www.mulesoft.org/schema/mule/http http://www.mulesoft.org/schema/mule/http/current/mule-http.xsd">
+
+	<flow name="api-call-private-spaces-list-flow" maxConcurrency="${anypoint.platform.apis.privatespaces.maxConcurrency}">
+		<logger level="DEBUG" message="Calling Runtime Fabric API - List Private Spaces" />
+		<http:request method="GET" doc:name="List Private Spaces"
+			config-ref="HTTP_Request_configuration"
+			path="${anypoint.platform.apis.privatespaces.list.path}">
+			<http:headers><![CDATA[#[output application/java
+---
+{
+	"Authorization" : "Bearer " ++ vars.token
+}]]]></http:headers>
+			<http:uri-params><![CDATA[#[output application/java
+---
+{
+	"orgId" : vars.orgId
+}]]]></http:uri-params>
+		</http:request>
+		<logger level="DEBUG" message='#["Private Spaces list, Response Status Code:" ++ attributes.statusCode]' />
+	</flow>
+
+</mule>

src/main/resources/api/traits/collectors.raml

@@ -15,10 +15,13 @@ queryParameters:
 
                 Not available for PCE deployment model:
                   - amq: Anypoint MQ
-                  - apma: API Manager Analytics                                          
-                  - ch: Cloudhub   
-                  - rtf: Runtime Fabric 
-    default: all    
+                  - apma: API Manager Analytics
+                  - ch: Cloudhub (legacy CH1 - use ch2 instead for CloudHub 2.0 orgs)
+                  - ch2: CloudHub 2.0 (fork addition)
+                  - ps: Private Spaces (fork addition; paid orgs only)
+                  - osv2: Object Store V2
+                  - rtf: Runtime Fabric
+    default: all
     required: false
     type: string
-    pattern: ^all|([ap|apc|apm|arm|core|dc|ex|amq|apma|ch|rtf]+([,[ap|apc|apm|arm|core|dc|ex|amq|apma|ch|rtf]])*)$  
+    pattern: ^all|((ap|apc|apm|arm|core|dc|ex|amq|apma|ch|ch2|ps|osv2|rtf)(,(ap|apc|apm|arm|core|dc|ex|amq|apma|ch|ch2|ps|osv2|rtf))*)$  

src/main/resources/dw/aggregation/build-platform-metrics-aggregation.dwl

@@ -275,18 +275,17 @@ var usableProdVcores = entitlements.vCoresProduction.assigned - entitlements.vCo
 	runtimeManagerMetrics: {
 		cloudhub: {
 			networking: {
-				vpcsTotal: entitlements.vpcs.assigned,
-				vpcsAvailable: (entitlements.vpcs.assigned default 0) - (usage.vpcsConsumed default 0),
-				vpcsUsed: usage.vpcsConsumed,
-				vpnsTotal: entitlements.vpns.assigned,
-				vpnsAvailable: (entitlements.vpns.assigned default 0) - (usage.vpnsConsumed default 0),
-				vpnsUsed: usage.vpnsConsumed,
-				dlbsTotal: entitlements.loadBalancer.assigned,
-				dlbsAvailable: (entitlements.loadBalancer.assigned default 0) - (usage.loadBalancersConsumed default 0),
-				dlbsUsed: usage.loadBalancersConsumed,
-				staticIPsTotal: entitlements.staticIps.assigned,
-				staticIPsAvailable: (entitlements.staticIps.assigned default 0) - (usage.staticIpsConsumed default 0),
-				staticIPsUsed: usage.staticIpsConsumed
+				// CH2-native metrics (sourced from Private Spaces collector, scatter-gather route index 10)
+				privateSpacesTotal: (payload[10].payload.privateSpacesTotal default 0),
+				privateSpacesByRegion: (payload[10].payload.privateSpacesByRegion default {}),
+				privateSpacesByEnvType: (payload[10].payload.privateSpacesByEnvType default {}),
+				vpnConnectionsTotal: (payload[10].payload.vpnConnectionsTotal default 0),
+				tgwAttachmentsTotal: (payload[10].payload.tgwAttachmentsTotal default 0),
+				// CH1 legacy fields retained for dashboard compatibility, set to null on CH2
+				vpcsTotal: null, vpcsAvailable: null, vpcsUsed: null,
+				vpnsTotal: null, vpnsAvailable: null, vpnsUsed: null,
+				dlbsTotal: null, dlbsAvailable: null, dlbsUsed: null,
+				staticIPsTotal: null, staticIPsAvailable: null, staticIPsUsed: null
 			},
 
 			applications:{