Merge pull request #1319 from cloudfoundry/use_storage_cli_on_s3

Use "storage-cli" implementation in use-s3-blobstore ops file

Author: jochenehret

operations/README.md

@@ -16,7 +16,7 @@ This is the README for Ops-files. To learn more about `cf-deployment`, go to the
 | [`use-gcs-blobstore-access-key.yml`](use-gcs-blobstore-access-key.yml) | Enables access key credentials for Google blobstore. | Requires `use-external-blobstore.yml`. Introduces [new variables](example-vars-files/vars-use-gcs-blobstore-access-key.yml) for access key/secret and bucket names. | **NO** |
 | **AWS** | | | |
 | [`aws.yml`](aws.yml) | Overrides the loggregator endpoint port to 4443. | It is required to have a separate port from the standard HTTPS port (443) for loggregator traffic in order to use "classic" AWS ELBs. Newer "Application Load Balancers" and "Network Load Balancers" (as setup by [bbl](https://github.com/cloudfoundry/bosh-bootloader) >= v7.0.0) should not require this port override, so no need to use this ops-file if you're using the newer load balancer. | **YES** |
-| [`use-s3-blobstore.yml`](use-s3-blobstore.yml) | Configures external blobstore to use Amazon S3. | Requires `use-external-blobstore.yml`. Introduces [new variables](example-vars-files/vars-use-s3-blobstore.yml) for s3 credentials and bucket names. | **YES** |
+| [`use-s3-blobstore.yml`](use-s3-blobstore.yml) | Configures external blobstore to use Amazon S3. Uses the new "storage-cli" implementation instead of the deprecated "fog" library. | Requires `use-external-blobstore.yml`. Introduces [new variables](example-vars-files/vars-use-s3-blobstore.yml) for s3 credentials and bucket names. | **YES** |
 | **Azure** | | **\* Not validated or supported by the Release Integration team** | |
 | [`azure.yml`](azure.yml) | Sets gorouter's `frontend_idle_timeout` to value appropriate for Azure load balancers. | Any value below 240 should work. | **NO** |
 | [`use-azure-storage-blobstore.yml`](use-azure-storage-blobstore.yml) | Configures external blobstore to use Azure Storage. Uses the new "storage-cli" implementation instead of the deprecated "fog" library. | Requires `use-external-blobstore.yml`. Introduces [new variables](example-vars-files/vars-use-azure-storage-blobstore.yml) for Azure credentials and container names. | **NO** |

operations/use-s3-blobstore.yml

@@ -1,53 +1,173 @@
 ---
+# Note: You must apply "use-external-blobstore.yml" before applying this ops file.
+
+# ========= api =========
 - type: replace
-  path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/buildpacks/fog_connection
-  error: "Please apply 'use-external-blobstore.yml' before applying 'use-s3-blobstore.yml'."
-  value: &blobstore-properties
-    provider: AWS
+  path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/buildpacks/blobstore_type?
+  value: storage-cli
+- type: replace
+  path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/buildpacks/blobstore_provider?
+  value: s3
+- type: replace
+  path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/buildpacks/connection_config?
+  value: &buildpack-blobstore-properties
+    bucket_name: ((buildpack_directory_key))
     aws_access_key_id: ((blobstore_access_key_id))
     aws_secret_access_key: ((blobstore_secret_access_key))
     region: ((aws_region))
+- type: remove
+  path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/buildpacks/fog_connection
 
 - type: replace
+  path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/droplets/blobstore_type?
+  value: storage-cli
+- type: replace
+  path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/droplets/blobstore_provider?
+  value: s3
+- type: replace
+  path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/droplets/connection_config?
+  value: &droplet-blobstore-properties
+    bucket_name: ((droplet_directory_key))
+    aws_access_key_id: ((blobstore_access_key_id))
+    aws_secret_access_key: ((blobstore_secret_access_key))
+    region: ((aws_region))
+- type: remove
   path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/droplets/fog_connection
-  value: *blobstore-properties
 
 - type: replace
+  path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/packages/blobstore_type?
+  value: storage-cli
+- type: replace
+  path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/packages/blobstore_provider?
+  value: s3
+- type: replace
+  path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/packages/connection_config?
+  value: &package-blobstore-properties
+    bucket_name: ((app_package_directory_key))
+    aws_access_key_id: ((blobstore_access_key_id))
+    aws_secret_access_key: ((blobstore_secret_access_key))
+    region: ((aws_region))
+- type: remove
   path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/packages/fog_connection
-  value: *blobstore-properties
 
 - type: replace
+  path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/resource_pool/blobstore_type?
+  value: storage-cli
+- type: replace
+  path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/resource_pool/blobstore_provider?
+  value: s3
+- type: replace
+  path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/resource_pool/connection_config?
+  value: &resource-pool-blobstore-properties
+    bucket_name: ((resource_directory_key))
+    aws_access_key_id: ((blobstore_access_key_id))
+    aws_secret_access_key: ((blobstore_secret_access_key))
+    region: ((aws_region))
+- type: remove
   path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/resource_pool/fog_connection
-  value: *blobstore-properties
 
+# ========= cc-worker =========
+- type: replace
+  path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/buildpacks/blobstore_type?
+  value: storage-cli
+- type: replace
+  path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/buildpacks/blobstore_provider?
+  value: s3
 - type: replace
+  path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/buildpacks/connection_config?
+  value: *buildpack-blobstore-properties
+
+- type: remove
   path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/buildpacks/fog_connection
-  value: *blobstore-properties
 
 - type: replace
+  path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/droplets/blobstore_type?
+  value: storage-cli
+- type: replace
+  path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/droplets/blobstore_provider?
+  value: s3
+- type: replace
+  path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/droplets/connection_config?
+  value: *droplet-blobstore-properties
+
+- type: remove
   path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/droplets/fog_connection
-  value: *blobstore-properties
 
 - type: replace
+  path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/packages/blobstore_type?
+  value: storage-cli
+- type: replace
+  path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/packages/blobstore_provider?
+  value: s3
+- type: replace
+  path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/packages/connection_config?
+  value: *package-blobstore-properties
+
+- type: remove
   path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/packages/fog_connection
-  value: *blobstore-properties
 
 - type: replace
+  path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/resource_pool/blobstore_type?
+  value: storage-cli
+- type: replace
+  path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/resource_pool/blobstore_provider?
+  value: s3
+- type: replace
+  path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/resource_pool/connection_config?
+  value: *resource-pool-blobstore-properties
+
+- type: remove
   path: /instance_groups/name=cc-worker/jobs/name=cloud_controller_worker/properties/cc/resource_pool/fog_connection
-  value: *blobstore-properties
 
+# ========= scheduler (clock) =========
+- type: replace
+  path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/buildpacks/blobstore_type?
+  value: storage-cli
 - type: replace
+  path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/buildpacks/blobstore_provider?
+  value: s3
+- type: replace
+  path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/buildpacks/connection_config?
+  value: *buildpack-blobstore-properties
+
+- type: remove
   path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/buildpacks/fog_connection
-  value: *blobstore-properties
 
 - type: replace
+  path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/droplets/blobstore_type?
+  value: storage-cli
+- type: replace
+  path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/droplets/blobstore_provider?
+  value: s3
+- type: replace
+  path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/droplets/connection_config?
+  value: *droplet-blobstore-properties
+
+- type: remove
   path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/droplets/fog_connection
-  value: *blobstore-properties
 
 - type: replace
+  path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/packages/blobstore_type?
+  value: storage-cli
+- type: replace
+  path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/packages/blobstore_provider?
+  value: s3
+- type: replace
+  path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/packages/connection_config?
+  value: *package-blobstore-properties
+
+- type: remove
   path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/packages/fog_connection
-  value: *blobstore-properties
 
 - type: replace
-  path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/resource_pool/fog_connection
-  value: *blobstore-properties
+  path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/resource_pool/blobstore_type?
+  value: storage-cli
+- type: replace
+  path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/resource_pool/blobstore_provider?
+  value: s3
+- type: replace
+  path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/resource_pool/connection_config?
+  value: *resource-pool-blobstore-properties
+
+- type: remove
+  path: /instance_groups/name=scheduler/jobs/name=cloud_controller_clock/properties/cc/resource_pool/fog_connection