Debugging integration test failures

.github/ops-files/improve-diego-resources.yml

@@ -0,0 +1,49 @@
+---
+# Ops file to improve Diego cell resources for better staging performance
+# This addresses staging failures due to insufficient resources
+
+# Increase Diego cell memory
+- type: replace
+  path: /instance_groups/name=diego-cell/vm_resources/ram
+  value: 16384  # 16GB instead of default 8GB
+
+# Increase Diego cell disk
+- type: replace
+  path: /instance_groups/name=diego-cell/vm_resources/ephemeral_disk_size
+  value: 102400  # 100GB instead of default
+
+# Increase Diego cell CPU
+- type: replace
+  path: /instance_groups/name=diego-cell/vm_resources/cpu
+  value: 4  # 4 CPUs instead of default 2
+
+# Increase executor memory capacity for staging
+- type: replace
+  path: /instance_groups/name=diego-cell/jobs/name=rep/properties/diego/executor/memory_capacity_mb?
+  value: 12288  # 12GB available for containers
+
+# Increase executor disk capacity for staging
+- type: replace
+  path: /instance_groups/name=diego-cell/jobs/name=rep/properties/diego/executor/disk_capacity_mb?
+  value: 81920  # 80GB available for containers
+
+# Increase staging timeout to handle slow buildpack operations
+- type: replace
+  path: /instance_groups/name=diego-cell/jobs/name=rep/properties/diego/executor/max_concurrent_downloads?
+  value: 10  # Allow more concurrent downloads during staging
+
+# Increase garden's default grace time for container shutdown
+- type: replace
+  path: /instance_groups/name=diego-cell/jobs/name=garden/properties/garden/destroy_containers_on_start?
+  value: false  # Don't destroy containers on restart for debugging
+
+# Increase grootfs reserved space to prevent disk pressure
+- type: replace
+  path: /instance_groups/name=diego-cell/jobs/name=garden/properties/garden/graph_cleanup_threshold_in_mb?
+  value: 5120  # 5GB threshold before cleanup
+
+# Increase concurrent operations on Diego cell
+- type: replace
+  path: /instance_groups/name=diego-cell/jobs/name=rep/properties/diego/executor/max_cache_size_in_bytes?
+  value: 10737418240  # 10GB cache for buildpack artifacts
+

.github/workflows/create-bosh-lite.yml

@@ -149,6 +149,7 @@ jobs:
           -o ${GITHUB_WORKSPACE}/cf-deployment/operations/enable-v2-api.yml \
           -o ${GITHUB_WORKSPACE}/cf-deployment/operations/use-internal-lookup-for-route-services.yml \
           -o ${GITHUB_WORKSPACE}/cli/.github/ops-files/diego-cell-instances.yml \
+          -o ${GITHUB_WORKSPACE}/cli/.github/ops-files/improve-diego-resources.yml \
           -o ${GITHUB_WORKSPACE}/cli/.github/ops-files/add-uaa-client-credentials.yml \
           -o ${GITHUB_WORKSPACE}/cli/.github/ops-files/increase-route-registration-interval.yml \
           -o ${GITHUB_WORKSPACE}/cli/.github/ops-files/add-oidc-provider.yml ${additional_args} \

.github/workflows/tests-integration-reusable.yml

@@ -18,7 +18,7 @@ on:
 
 env:
   ENV_NAME: ${{ inputs.env-name }}
-  NODES: 12
+  NODES: 4
   FLAKE_ATTEMPTS: ${{ vars.TEST_FLAKE_ATTEMPTS || '5' }}
   BBL_CLI_VERSION: ${{ vars.BBL_CLI_VERSION }}
   BOSH_CLI_VERSION: ${{ vars.BOSH_CLI_VERSION }}

Makefile

@@ -142,8 +142,12 @@ integration-full-tests: integration-tests-full
 integration-tests-full: build integration-cleanup integration-isolated integration-push integration-experimental integration-plugin integration-global integration-selfcontained ## Run all isolated, push, experimental, plugin, selfcontained, and global integration tests
 
 integration-tests-full-ci: install-test-deps integration-cleanup
-	$(ginkgo_int) -nodes $(NODES)  -flake-attempts $(FLAKE_ATTEMPTS) \
-		integration/shared/isolated integration/v7/isolated integration/shared/plugin integration/shared/experimental integration/v7/experimental integration/v7/push
+	$(ginkgo_int) -nodes $(NODES) -flake-attempts $(FLAKE_ATTEMPTS) \
+		integration/shared/isolated integration/v7/isolated
+	$(ginkgo_int) -nodes $(NODES) -flake-attempts $(FLAKE_ATTEMPTS) \
+		integration/shared/plugin integration/shared/experimental
+	$(ginkgo_int) -nodes $(NODES) -flake-attempts $(FLAKE_ATTEMPTS) \
+		integration/v7/experimental integration/v7/push
 	$(ginkgo_int) -flake-attempts $(FLAKE_ATTEMPTS) integration/shared/global integration/v7/global
 
 lint: format ## Runs all linters and formatters