From f92ff0dc64ed4a134d97b48208ddda4c2f8d9859 Mon Sep 17 00:00:00 2001 From: Jochen Ehret Date: Mon, 2 Mar 2026 15:48:59 +0100 Subject: [PATCH 1/2] Use "storage-cli" implementation in use-s3-blobstore ops file --- operations/README.md | 2 +- operations/use-s3-blobstore.yml | 152 ++++++++++++++++++++++++++++---- 2 files changed, 137 insertions(+), 17 deletions(-) diff --git a/operations/README.md b/operations/README.md index c13868e8e..38c01f858 100644 --- a/operations/README.md +++ b/operations/README.md @@ -16,7 +16,7 @@ This is the README for Ops-files. To learn more about `cf-deployment`, go to the | [`use-gcs-blobstore-access-key.yml`](use-gcs-blobstore-access-key.yml) | Enables access key credentials for Google blobstore. | Requires `use-external-blobstore.yml`. Introduces [new variables](example-vars-files/vars-use-gcs-blobstore-access-key.yml) for access key/secret and bucket names. | **NO** | | **AWS** | | | | | [`aws.yml`](aws.yml) | Overrides the loggregator endpoint port to 4443. | It is required to have a separate port from the standard HTTPS port (443) for loggregator traffic in order to use "classic" AWS ELBs. Newer "Application Load Balancers" and "Network Load Balancers" (as setup by [bbl](https://github.com/cloudfoundry/bosh-bootloader) >= v7.0.0) should not require this port override, so no need to use this ops-file if you're using the newer load balancer. | **YES** | -| [`use-s3-blobstore.yml`](use-s3-blobstore.yml) | Configures external blobstore to use Amazon S3. | Requires `use-external-blobstore.yml`. Introduces [new variables](example-vars-files/vars-use-s3-blobstore.yml) for s3 credentials and bucket names. | **YES** | +| [`use-s3-blobstore.yml`](use-s3-blobstore.yml) | Configures external blobstore to use Amazon S3. Uses the new "storage-cli" implementation instead of the deprecated "fog" library. | Requires `use-external-blobstore.yml`. Introduces [new variables](example-vars-files/vars-use-s3-blobstore.yml) for s3 credentials and bucket names. | **YES** | | **Azure** | | **\* Not validated or supported by the Release Integration team** | | | [`azure.yml`](azure.yml) | Sets gorouter's `frontend_idle_timeout` to value appropriate for Azure load balancers. | Any value below 240 should work. | **NO** | | [`use-azure-storage-blobstore.yml`](use-azure-storage-blobstore.yml) | Configures external blobstore to use Azure Storage. Uses the new "storage-cli" implementation instead of the deprecated "fog" library. | Requires `use-external-blobstore.yml`. Introduces [new variables](example-vars-files/vars-use-azure-storage-blobstore.yml) for Azure credentials and container names. | **NO** | diff --git a/operations/use-s3-blobstore.yml b/operations/use-s3-blobstore.yml index 77db60bae..d2033e139 100644 --- a/operations/use-s3-blobstore.yml +++ b/operations/use-s3-blobstore.yml @@ -1,53 +1,173 @@ --- +# Note: You must apply "use-external-blobstore.yml" before applying this ops file. + +# ========= api ========= - type: replace - path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/buildpacks/fog_connection - error: "Please apply 'use-external-blobstore.yml' before applying 'use-s3-blobstore.yml'." - value: &blobstore-properties - provider: AWS + path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/buildpacks/blobstore_type? + value: storage-cli +- type: replace + path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/buildpacks/blobstore_provider? + value: AWS +- type: replace + path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/buildpacks/connection_config? + value: &buildpack-blobstore-properties + bucket_name: ((buildpack_directory_key)) aws_access_key_id: ((blobstore_access_key_id)) aws_secret_access_key: ((blobstore_secret_access_key)) region: ((aws_region)) +- type: remove + path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/buildpacks/fog_connection - type: replace + path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/droplets/blobstore_type? + value: storage-cli +- type: replace + path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/droplets/blobstore_provider? + value: AWS +- type: replace + path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/droplets/connection_config? + value: &droplet-blobstore-properties + bucket_name: ((droplet_directory_key)) + aws_access_key_id: ((blobstore_access_key_id)) + aws_secret_access_key: ((blobstore_secret_access_key)) + region: ((aws_region)) +- type: remove path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/droplets/fog_connection - value: *blobstore-properties - type: replace + path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/packages/blobstore_type? + value: storage-cli +- type: replace + path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/packages/blobstore_provider? + value: AWS +- type: replace + path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/packages/connection_config? + value: &package-blobstore-properties + bucket_name: ((app_package_directory_key)) + aws_access_key_id: ((blobstore_access_key_id)) + aws_secret_access_key: ((blobstore_secret_access_key)) + region: ((aws_region)) +- type: remove path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/packages/fog_connection - value: *blobstore-properties - type: replace + path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/resource_pool/blobstore_type? + value: storage-cli +- type: replace + path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/resource_pool/blobstore_provider? + value: AWS +- type: replace + path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/resource_pool/connection_config? + value: &resource-pool-blobstore-properties + bucket_name: ((resource_directory_key)) + aws_access_key_id: ((blobstore_access_key_id)) + aws_secret_access_key: ((blobstore_secret_access_key)) + region: ((aws_region)) +- type: remove path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/resource_pool/fog_connection - value: *blobstore-properties +# ========= cc-worker ========= +- type: replace + path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/buildpacks/blobstore_type? + value: storage-cli +- type: replace + path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/buildpacks/blobstore_provider? + value: AWS - type: replace + path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/buildpacks/connection_config? + value: *buildpack-blobstore-properties + +- type: remove path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/buildpacks/fog_connection - value: *blobstore-properties - type: replace + path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/droplets/blobstore_type? + value: storage-cli +- type: replace + path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/droplets/blobstore_provider? + value: AWS +- type: replace + path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/droplets/connection_config? + value: *droplet-blobstore-properties + +- type: remove path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/droplets/fog_connection - value: *blobstore-properties - type: replace + path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/packages/blobstore_type? + value: storage-cli +- type: replace + path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/packages/blobstore_provider? + value: AWS +- type: replace + path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/packages/connection_config? + value: *package-blobstore-properties + +- type: remove path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/packages/fog_connection - value: *blobstore-properties - type: replace + path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/resource_pool/blobstore_type? + value: storage-cli +- type: replace + path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/resource_pool/blobstore_provider? + value: AWS +- type: replace + path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/resource_pool/connection_config? + value: *resource-pool-blobstore-properties + +- type: remove path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/resource_pool/fog_connection - value: *blobstore-properties +# ========= scheduler (clock) ========= +- type: replace + path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/buildpacks/blobstore_type? + value: storage-cli - type: replace + path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/buildpacks/blobstore_provider? + value: AWS +- type: replace + path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/buildpacks/connection_config? + value: *buildpack-blobstore-properties + +- type: remove path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/buildpacks/fog_connection - value: *blobstore-properties - type: replace + path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/droplets/blobstore_type? + value: storage-cli +- type: replace + path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/droplets/blobstore_provider? + value: AWS +- type: replace + path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/droplets/connection_config? + value: *droplet-blobstore-properties + +- type: remove path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/droplets/fog_connection - value: *blobstore-properties - type: replace + path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/packages/blobstore_type? + value: storage-cli +- type: replace + path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/packages/blobstore_provider? + value: AWS +- type: replace + path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/packages/connection_config? + value: *package-blobstore-properties + +- type: remove path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/packages/fog_connection - value: *blobstore-properties - type: replace - path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/resource_pool/fog_connection - value: *blobstore-properties + path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/resource_pool/blobstore_type? + value: storage-cli +- type: replace + path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/resource_pool/blobstore_provider? + value: AWS +- type: replace + path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/resource_pool/connection_config? + value: *resource-pool-blobstore-properties + +- type: remove + path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/resource_pool/fog_connection \ No newline at end of file From e4c16c01307cf311295678769f049c8fe6cd35b1 Mon Sep 17 00:00:00 2001 From: Jochen Ehret Date: Wed, 1 Apr 2026 08:30:26 +0200 Subject: [PATCH 2/2] Use new provider name "s3" insteand of "AWS" for storage-cli --- operations/use-s3-blobstore.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/operations/use-s3-blobstore.yml b/operations/use-s3-blobstore.yml index d2033e139..7592b2ee4 100644 --- a/operations/use-s3-blobstore.yml +++ b/operations/use-s3-blobstore.yml @@ -7,7 +7,7 @@ value: storage-cli - type: replace path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/buildpacks/blobstore_provider? - value: AWS + value: s3 - type: replace path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/buildpacks/connection_config? value: &buildpack-blobstore-properties @@ -23,7 +23,7 @@ value: storage-cli - type: replace path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/droplets/blobstore_provider? - value: AWS + value: s3 - type: replace path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/droplets/connection_config? value: &droplet-blobstore-properties @@ -39,7 +39,7 @@ value: storage-cli - type: replace path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/packages/blobstore_provider? - value: AWS + value: s3 - type: replace path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/packages/connection_config? value: &package-blobstore-properties @@ -55,7 +55,7 @@ value: storage-cli - type: replace path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/resource_pool/blobstore_provider? - value: AWS + value: s3 - type: replace path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/resource_pool/connection_config? value: &resource-pool-blobstore-properties @@ -72,7 +72,7 @@ value: storage-cli - type: replace path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/buildpacks/blobstore_provider? - value: AWS + value: s3 - type: replace path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/buildpacks/connection_config? value: *buildpack-blobstore-properties @@ -85,7 +85,7 @@ value: storage-cli - type: replace path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/droplets/blobstore_provider? - value: AWS + value: s3 - type: replace path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/droplets/connection_config? value: *droplet-blobstore-properties @@ -98,7 +98,7 @@ value: storage-cli - type: replace path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/packages/blobstore_provider? - value: AWS + value: s3 - type: replace path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/packages/connection_config? value: *package-blobstore-properties @@ -111,7 +111,7 @@ value: storage-cli - type: replace path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/resource_pool/blobstore_provider? - value: AWS + value: s3 - type: replace path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/resource_pool/connection_config? value: *resource-pool-blobstore-properties @@ -125,7 +125,7 @@ value: storage-cli - type: replace path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/buildpacks/blobstore_provider? - value: AWS + value: s3 - type: replace path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/buildpacks/connection_config? value: *buildpack-blobstore-properties @@ -138,7 +138,7 @@ value: storage-cli - type: replace path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/droplets/blobstore_provider? - value: AWS + value: s3 - type: replace path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/droplets/connection_config? value: *droplet-blobstore-properties @@ -151,7 +151,7 @@ value: storage-cli - type: replace path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/packages/blobstore_provider? - value: AWS + value: s3 - type: replace path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/packages/connection_config? value: *package-blobstore-properties @@ -164,7 +164,7 @@ value: storage-cli - type: replace path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/resource_pool/blobstore_provider? - value: AWS + value: s3 - type: replace path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/resource_pool/connection_config? value: *resource-pool-blobstore-properties