diff --git a/ci/pipelines/builder.yml b/ci/pipelines/builder.yml index 07bdad8285..7a8f981042 100644 --- a/ci/pipelines/builder.yml +++ b/ci/pipelines/builder.yml @@ -1,12 +1,95 @@ -anchors: - ci_bot: - email: &ci_bot_email infra@cloudfoundry.org - name: &ci_bot_name CI Bot - #@ load("@ytt:data", "data") -#@ bats_director_tag = "test-stemcells-" + data.values.stemcell_details.os_short_name #@yaml/text-templated-strings +--- + +#@ def metalink_resource(IAAS, HYPERVISOR, FIPS=""): + name: (@= IAAS @)-(@= HYPERVISOR @)(@= FIPS @) + type: metalink-repository + source: + mirror_files: + - destination: s3://storage.googleapis.com/bosh-core-stemcells-candidate(@= FIPS @)/(@= IAAS @)/{{.Name}} + options: + private_key: ((github_deploy_key_bosh-io-stemcells-core-index.private_key)) + filters: + - repositorypath: "*/(@= IAAS @)-(@= HYPERVISOR @)(@= FIPS @)(@= data.values.stemcell_details.agent_suffix @).meta4" + uri: git+ssh://git@github.com:cloudfoundry/bosh-io-stemcells-core-index.git//dev/(@= data.values.stemcell_details.os_name @)(@= FIPS @)/ + url_handlers: + - include: + - (s3|https)://.* + options: + access_key: ((hmac_accesskey)) + secret_key: ((hmac_secret)) + type: s3 +#@ end + +#@yaml/text-templated-strings +--- + +#@ def build_stemcell(IAAS, HYPERVISOR, FIPS=""): +name: build-(@= IAAS @)-(@= HYPERVISOR @)(@= FIPS @) +serial: true +plan: + - in_parallel: + - get: version + passed: + - build-stemcell + resource: version + trigger: true + - get: bosh-stemcells-ci + - get: os-image-stemcell-builder-registry-image + - get: build-time + passed: + - build-stemcell + trigger: true + - get: bosh-linux-stemcell-builder + passed: + - build-stemcell + resource: bosh-linux-stemcell-builder + - get: stemcells-index + - get: os-image-tarball + passed: + - build-os-image + - task: create-stemcell + file: bosh-stemcells-ci/ci/tasks/build.yml + image: os-image-stemcell-builder-registry-image + params: + HYPERVISOR: #@ HYPERVISOR + IAAS: #@ IAAS + OS_NAME: ubuntu + S3_API_ENDPOINT: storage.googleapis.com + OS_VERSION: (@= data.values.stemcell_details.os_short_name @)(@= FIPS @) + STEMCELL_BUCKET: bosh-core-stemcells-candidate(@= FIPS @) + GIT_USER_EMAIL: (@= data.values.stemcell_details.bot_email @) + GIT_USER_NAME: (@= data.values.stemcell_details.bot_name @) + AGENT_SUFFIX: (@= data.values.stemcell_details.agent_suffix @) + #@ if/end FIPS != "": + UBUNTU_ADVANTAGE_TOKEN: ((ubuntu_advantage_token)) + privileged: true + vars: + image_os_tag: (@= data.values.stemcell_details.os_short_name @) + - in_parallel: + - put: (@= IAAS @)-(@= HYPERVISOR @)(@= FIPS @) + attempts: 3 + params: + files: + - stemcell/*.tgz + rename: "{{.Version}}/(@= IAAS @)-(@= HYPERVISOR @)(@= FIPS @)(@= data.values.stemcell_details.agent_suffix @).meta4" + options: + author_email: (@= data.values.stemcell_details.bot_email @) + author_name: (@= data.values.stemcell_details.bot_name @) + message: 'dev: (@= data.values.stemcell_details.os_name @)' + version: candidate-build-number/number +#@ end + +#@yaml/text-templated-strings +--- + +anchors: + bats_director_tag: &bats-director-tag test-stemcells-(@= data.values.stemcell_details.os_short_name @) + ci_bot: + email: &ci_bot_email (@= data.values.stemcell_details.bot_email @) + name: &ci_bot_name (@= data.values.stemcell_details.bot_name @) groups: - name: build @@ -125,7 +208,7 @@ jobs: GCP_REGION: europe-north2 GCP_NETWORK_NAME: bosh-concourse SUBNET_INT: (@= data.values.stemcell_details.subnet_int @) - DIRECTOR_TAG: (@= bats_director_tag @) + DIRECTOR_TAG: *bats-director-tag - name: process-high-critical-cves serial_groups: [log-cves] @@ -483,62 +566,6 @@ jobs: #! timeout: 15m #! serial: true -#@ def build_stemcell(IAAS, HYPERVISOR, FIPS=""): - name: build-(@= IAAS @)-(@= HYPERVISOR @)(@= FIPS @) - serial: true - plan: - - in_parallel: - - get: version - passed: - - build-stemcell - resource: version - trigger: true - - get: bosh-stemcells-ci - - get: os-image-stemcell-builder-registry-image - - get: build-time - passed: - - build-stemcell - trigger: true - - get: bosh-linux-stemcell-builder - passed: - - build-stemcell - resource: bosh-linux-stemcell-builder - - get: stemcells-index - - get: os-image-tarball - passed: - - build-os-image - - task: create-stemcell - file: bosh-stemcells-ci/ci/tasks/build.yml - image: os-image-stemcell-builder-registry-image - params: - HYPERVISOR: #@ HYPERVISOR - IAAS: #@ IAAS - OS_NAME: ubuntu - S3_API_ENDPOINT: storage.googleapis.com - OS_VERSION: (@= data.values.stemcell_details.os_short_name @)(@= FIPS @) - STEMCELL_BUCKET: bosh-core-stemcells-candidate(@= FIPS @) - GIT_USER_EMAIL: *ci_bot_email - GIT_USER_NAME: *ci_bot_name - AGENT_SUFFIX: (@= data.values.stemcell_details.agent_suffix @) - #@ if/end FIPS != "": - UBUNTU_ADVANTAGE_TOKEN: ((ubuntu_advantage_token)) - privileged: true - vars: - image_os_tag: (@= data.values.stemcell_details.os_short_name @) - - in_parallel: - - put: (@= IAAS @)-(@= HYPERVISOR @)(@= FIPS @) - attempts: 3 - params: - files: - - stemcell/*.tgz - rename: "{{.Version}}/(@= IAAS @)-(@= HYPERVISOR @)(@= FIPS @)(@= data.values.stemcell_details.agent_suffix @).meta4" - options: - author_email: *ci_bot_email - author_name: *ci_bot_name - message: 'dev: (@= data.values.stemcell_details.os_name @)' - version: candidate-build-number/number -#@ end - #@ for iaas in data.values.stemcell_details.include_iaas: - #@ build_stemcell(iaas.iaas, iaas.hypervisor) #@ end @@ -596,7 +623,7 @@ jobs: GCP_PROJECT_ID: ((gcp_project_id)) GCP_ZONE: europe-north2-a GCP_SUBNET_NAME: stemcell-builder-integration-(@= data.values.stemcell_details.subnet_int @) - TAG: (@= bats_director_tag @) + TAG: *bats-director-tag - task: deploy-director file: bosh-stemcells-ci/ci/tasks/gcp/deploy-director.yml image: bosh-integration-image @@ -611,7 +638,7 @@ jobs: INTERNAL_CIDR: 10.100.(@= data.values.stemcell_details.subnet_int @).0/24 INTERNAL_GW: 10.100.(@= data.values.stemcell_details.subnet_int @).1 RESERVED_RANGE: '10.100.(@= data.values.stemcell_details.subnet_int @).2 - 10.100.(@= data.values.stemcell_details.subnet_int @).63, 10.100.(@= data.values.stemcell_details.subnet_int @).126 - 10.100.(@= data.values.stemcell_details.subnet_int @).254' - TAG: (@= bats_director_tag @) + TAG: *bats-director-tag - task: prepare-bats file: bosh-stemcells-ci/ci/tasks/bats/iaas/gcp/prepare-bats-config.yml image: bosh-integration-image @@ -932,11 +959,11 @@ resources: - name: weekly type: time source: + initial_version: true start: 3:00 -0700 stop: 4:30 -0700 days: - Saturday - initial_version: true - name: bosh-agent type: metalink-repository @@ -944,26 +971,6 @@ resources: uri: git+https://github.com/cloudfoundry/bosh-agent-index.git/ version: "*" -#@ def metalink_resource(IAAS, HYPERVISOR, FIPS=""): - name: (@= IAAS @)-(@= HYPERVISOR @)(@= FIPS @) - type: metalink-repository - source: - mirror_files: - - destination: s3://storage.googleapis.com/bosh-core-stemcells-candidate(@= FIPS @)/(@= IAAS @)/{{.Name}} - options: - private_key: ((github_deploy_key_bosh-io-stemcells-core-index.private_key)) - filters: - - repositorypath: "*/(@= IAAS @)-(@= HYPERVISOR @)(@= FIPS @)(@= data.values.stemcell_details.agent_suffix @).meta4" - uri: git+ssh://git@github.com:cloudfoundry/bosh-io-stemcells-core-index.git//dev/(@= data.values.stemcell_details.os_name @)(@= FIPS @)/ - url_handlers: - - include: - - (s3|https)://.* - options: - access_key: ((hmac_accesskey)) - secret_key: ((hmac_secret)) - type: s3 -#@ end - #@ for iaas in data.values.stemcell_details.include_iaas: - #@ metalink_resource(iaas.iaas, iaas.hypervisor) #@ end diff --git a/ci/pipelines/publisher.yml b/ci/pipelines/publisher.yml index 396b61ab68..83c1dca19f 100644 --- a/ci/pipelines/publisher.yml +++ b/ci/pipelines/publisher.yml @@ -1,17 +1,72 @@ #@ load("@ytt:data", "data") -#@ def build_light_aws_stemcell_new(name, stemcell_os, stemcell_version, builder_src, input_stemcell, output_stemcell, prefix, region, bucket_prefix, tag, ami_destinations, efi, ami_excluded_destinations): +#@yaml/text-templated-strings +--- + +#@ def build_light_aws_stemcell_new(stemcell_os, stemcell_version, builder_src, input_stemcell, output_stemcell, prefix, region, bucket_prefix, tag, ami_destinations, efi, ami_excluded_destinations): do: -- #@ get_aws_light_stemcells(input_stemcell, stemcell_os, stemcell_version, builder_src, tag) -- #@ build_light_aws_stemcell(name, builder_src, input_stemcell, output_stemcell, prefix, region, bucket_prefix, tag, ami_destinations, efi, ami_excluded_destinations) + - in_parallel: + - get: (@= input_stemcell @) + params: + include_files: + - bosh-stemcell-*-aws-xen-hvm-(@= stemcell_os @)*.tgz + resource: candidate-(@= stemcell_os @)-stemcell-(@= stemcell_version @) + trigger: true + version: every + #@ if tag != "": + tags: + - (@= tag @) + #@ end + - get: (@= builder_src @) + passed: + - test-aws-unit + - test-aws-integration + - test-aws-drivers + resource: light-aws-builder-src + #@ if tag != "": + tags: + - (@= tag @) + #@ end + - task: build-(@= region @)-stemcell + file: bosh-stemcells-ci/ci/tasks/light-aws/build.yml + image: light-stemcell-builder-registry-image + input_mapping: + builder-src: (@= builder_src @) + input-stemcell: (@= input_stemcell @) + output_mapping: + light-stemcell: (@= output_stemcell @) + params: + AWS_PAGER: + ami_access_key: ((aws_publish_(@= prefix @)_access_key)) + ami_secret_key: ((aws_publish_(@= prefix @)_secret_key)) + ami_region: ((aws_publish_(@= region @)_region)) + ami_bucket_name: ((aws_publish_(@= bucket_prefix @)_bucket)) + ami_description: Light Stemcell Builder Prod AMI + #@ if ami_destinations != "": + ami_destinations: (@= ami_destinations @) + #@ end + #@ if ami_excluded_destinations != "": + ami_excluded_destinations: (@= ami_excluded_destinations @) + #@ end + ami_encrypted: false + ami_kms_key_id: "" + ami_server_side_encryption: "" + ami_virtualization_type: hvm + ami_visibility: public + S3_API_ENDPOINT: storage.googleapis.com + efi: (@= str(efi).lower() @) + #@ if tag != "": + tags: + - (@= tag @) + #@ end #@ end #@yaml/text-templated-strings --- -#@ def cleanup_unpublished_light_stemcells(name, prefix, region): +#@ def cleanup_unpublished_light_stemcells(prefix, region): +task: cleanup-amis-in-(@= prefix @) file: bosh-stemcells-ci/ci/tasks/light-aws/cleanup-ami.yml image: bosh-integration-registry-image -task: cleanup-amis-in-(@= name @) params: AWS_PAGER: ami_access_key: ((aws_publish_(@= prefix @)_access_key)) @@ -21,10 +76,13 @@ params: ami_keep_latest: 5 os_name: (@= data.values.stemcell_details.os_name @) #@ end -#@ def cleanup_old_published_light_stemcells(name, prefix, region): + +#@yaml/text-templated-strings +--- +#@ def cleanup_old_published_light_stemcells(prefix, region): +task: cleanup-amis-in-(@= prefix @) file: bosh-stemcells-ci/ci/tasks/light-aws/cleanup-ami.yml image: bosh-integration-registry-image -task: cleanup-amis-in-(@= name @) params: AWS_PAGER: ami_access_key: ((aws_publish_(@= prefix @)_access_key)) @@ -34,199 +92,6 @@ params: remove_public_images: true #@ end -#@ def build_light_aws_stemcell(name, builder_src, input_stemcell, output_stemcell, prefix, region, bucket_prefix, tag, ami_destinations, efi, ami_excluded_destinations): -file: bosh-stemcells-ci/ci/tasks/light-aws/build.yml -task: #@ name -image: light-stemcell-builder-registry-image -input_mapping: - builder-src: (@= builder_src @) - input-stemcell: (@= input_stemcell @) -output_mapping: - light-stemcell: (@= output_stemcell @) -params: - AWS_PAGER: - ami_access_key: ((aws_publish_(@= prefix @)_access_key)) - ami_secret_key: ((aws_publish_(@= prefix @)_secret_key)) - ami_region: ((aws_publish_(@= region @)_region)) - ami_bucket_name: ((aws_publish_(@= bucket_prefix @)_bucket)) - ami_description: Light Stemcell Builder Prod AMI - #@ if ami_destinations != "": - ami_destinations: (@= ami_destinations @) - #@ end - #@ if ami_excluded_destinations != "": - ami_excluded_destinations: (@= ami_excluded_destinations @) - #@ end - ami_encrypted: false - ami_kms_key_id: "" - ami_server_side_encryption: "" - ami_virtualization_type: hvm - ami_visibility: public - S3_API_ENDPOINT: storage.googleapis.com - efi: #@ efi - #@ if tag != "": - tags: - - #@ tag - #@ end -#@ end - -#@ def get_aws_light_stemcells(input_stemcell, stemcell_os, stemcell_version, builder_src, tag): -in_parallel: -- get: (@= input_stemcell @) - params: - include_files: - - bosh-stemcell-*-aws-xen-hvm-(@= stemcell_os @)*.tgz - resource: candidate-(@= stemcell_os @)-stemcell-(@= stemcell_version @) - trigger: true - version: every - #@ if tag != "": - tags: - - #@ tag - #@ end -- get: (@= builder_src @) - passed: - - test-aws-unit - - test-aws-integration - - test-aws-drivers - resource: light-aws-builder-src - #@ if tag != "": - tags: - - #@ tag - #@ end -#@ end - -#@ def get_google_light_inputs(stemcell_os, stemcell_version): -in_parallel: -- get: stemcell - params: - include_files: - - bosh-stemcell-*-google-kvm-(@= stemcell_os @)*.tgz - resource: candidate-(@= stemcell_os @)-stemcell-(@= str(stemcell_version) @) - trigger: true - version: every -- get: bosh-stemcells-ci -- get: bosh-cpi-release - resource: bosh-google-cpi-release -#@ end - -#@ def deploy_skeletal_google_light_stemcell(terraform_resource, credentials): -do: -- params: - generate_random_name: true - terraform_source: bosh-stemcells-ci/ci/tasks/light-google/terraform/ - put: terraform - resource: (@= terraform_resource @) -- file: bosh-stemcells-ci/ci/tasks/light-google/deploy-skeletal.yml - image: bosh-integration-registry-image - params: - GCE_CREDENTIALS_JSON: (@= credentials @) - SSH_PRIVATE_KEY: ((ssh.private_key)) - task: deploy-skeletal -ensure: - ensure: - get_params: - action: destroy - params: - action: destroy - env_name_file: terraform/name - terraform_source: bosh-stemcells-ci/ci/tasks/light-google/terraform/ - put: terraform - resource: (@= terraform_resource @) - file: bosh-stemcells-ci/ci/tasks/light-google/destroy-skeletal.yml - image: bosh-integration-registry-image - task: destroy-skeletal -on_failure: - image: bosh-integration-registry-image - config: - platform: linux - run: - args: - - | - * remove artifacts of failed jobs: - ```bash - gcloud auth revoke --all # start with clean slate - gcloud auth activate-service-account concourse@((gcp_project_id)).iam.gserviceaccount.com --key-file <( ((gcp_json_key)) ) - gsutil rm gs://bosh-gce-raw-stemcells-new/STEMCELL_TO_BE_DELETED_raw.tar.gz - gsutil rm gs://bosh-gce-raw-stemcells-new/STEMCELL_TO_BE_DELETED_raw.tar.gz.sha1 - gcloud auth revoke --all - ``` - * re-fly pipeline after commenting-out `disable_manual_trigger: true` - * trigger failed jobs - * re-fly pipeline after uncommenting `disable_manual_trigger: true` - path: echo - task: cleanup-failed-run-instructions -#@ end - -#@ def run_stemcell_upload_tests(): -do: -- in_parallel: - - get: bosh-cpi-src - resource: bosh-aws-cpi-release - trigger: false - - get: cpi-release - resource: bosh-aws-cpi-dev-artifacts - trigger: false - - get: bosh-release - trigger: false - - get: bosh-deployment - trigger: false - - get: pipelines - resource: bosh-cpi-certification-concourse-tasks - - get: bosh-cpi-certification-concourse-tasks -- put: environment - resource: light-aws-environment - params: - delete_on_failure: true - generate_random_name: true - terraform_source: bosh-cpi-src/ci/assets/terraform -- file: bosh-cpi-certification-concourse-tasks/shared/tasks/prepare-director.yml - image: bosh-integration-registry-image - input_mapping: - stemcell: light-stemcell - params: - DIRECTOR_VARS_FILE: ((aws_test_director_vars_file)) - INFRASTRUCTURE: aws - OPTIONAL_OPS_FILE: | - -o pipelines/shared/assets/ops/remove-hm.yml - -o bosh-deployment/external-ip-with-registry-not-recommended.yml - -o bosh-deployment/misc/source-releases/bosh.yml - task: prepare-director -- file: bosh-cpi-certification-concourse-tasks/shared/tasks/deploy-director.yml - image: bosh-integration-registry-image - input_mapping: - stemcell: light-stemcell - task: deploy-director -- file: bosh-stemcells-ci/ci/tasks/light-aws/run-upload-test.yml - image: bosh-integration-registry-image - input_mapping: - builder-src: us-builder-src - stemcell: light-stemcell - task: run-stemcell-upload-tests -ensure: - do: - - task: teardown - file: bosh-cpi-certification-concourse-tasks/shared/tasks/teardown.yml - image: bosh-integration-registry-image - - task: ensure-terminated - file: bosh-cpi-src/ci/tasks/ensure-terminated.yml - image: bosh-integration-registry-image - input_mapping: - bosh-aws-cpi-release: bosh-cpi-src - params: - AWS_PAGER: - AWS_ACCESS_KEY_ID: ((aws_publish_us_access_key)) - AWS_SECRET_ACCESS_KEY: ((aws_publish_us_secret_key)) - AWS_DEFAULT_REGION: ((aws_publish_us_region)) - - put: environment - resource: light-aws-environment - get_params: - action: destroy - params: - action: destroy - env_name_file: environment/name - terraform_source: bosh-cpi-src/ci/assets/terraform -#@ end ---- - #@yaml/text-templated-strings --- anchors: @@ -267,8 +132,8 @@ jobs: trigger: true - get: bosh-stemcells-ci - get: bosh-integration-registry-image - - #@ cleanup_old_published_light_stemcells("aws", "us", "us") - - #@ cleanup_old_published_light_stemcells("us-goverment", "us-gov", "us-gov") + - #@ cleanup_old_published_light_stemcells("us", "us") + - #@ cleanup_old_published_light_stemcells("us-gov", "us-gov") - name: cleanup-unpublished-(@= data.values.stemcell_details.os_name @)-aws-light-stemcells serial: true plan: @@ -276,9 +141,9 @@ jobs: trigger: true - get: bosh-stemcells-ci - get: bosh-integration-registry-image - - #@ cleanup_unpublished_light_stemcells("aws", "us", "us") - - #@ cleanup_unpublished_light_stemcells("us-goverment", "us-gov", "us-gov") -#!- #@ cleanup_unpublished_light_stemcells("china", "cn", "cn_north") + - #@ cleanup_unpublished_light_stemcells("us", "us") + - #@ cleanup_unpublished_light_stemcells("us-gov", "us-gov") +#!- #@ cleanup_unpublished_light_stemcells("cn", "cn_north") - name: test-aws-unit plan: @@ -371,15 +236,82 @@ jobs: - get: bosh-integration-registry-image - in_parallel: - do: - - #@ build_light_aws_stemcell_new("build-us-gov-stemcell", data.values.stemcell_details.os_name, str(data.values.stemcell_details.major_version), "us-gov-builder-src", "us-gov-input-stemcell", "us-gov-light-stemcell", "us-gov", "us-gov", "us-gov", "", "", data.values.stemcell_details.use_efi, "") - - #@ build_light_aws_stemcell_new("build-us-stemcell", data.values.stemcell_details.os_name, str(data.values.stemcell_details.major_version), "us-builder-src", "us-input-stemcell", "us-light-stemcell", "us", "us", "us", "", "", data.values.stemcell_details.use_efi, '["me-central-1"]') + - #@ build_light_aws_stemcell_new(data.values.stemcell_details.os_name, str(data.values.stemcell_details.major_version), "us-gov-builder-src", "us-gov-input-stemcell", "us-gov-light-stemcell", "us-gov", "us-gov", "us-gov", "", "", data.values.stemcell_details.use_efi, "") + - #@ build_light_aws_stemcell_new(data.values.stemcell_details.os_name, str(data.values.stemcell_details.major_version), "us-builder-src", "us-input-stemcell", "us-light-stemcell", "us", "us", "us", "", "", data.values.stemcell_details.use_efi, '["me-central-1"]') - file: bosh-stemcells-ci/ci/tasks/light-aws/us-gov-merge-builds.yml image: light-stemcell-builder-registry-image input_mapping: builder-src: us-builder-src task: merge-builds - - #@ run_stemcell_upload_tests() - - params: + - do: + - in_parallel: + - get: bosh-cpi-src + resource: bosh-aws-cpi-release + trigger: false + - get: cpi-release + resource: bosh-aws-cpi-dev-artifacts + trigger: false + - get: bosh-release + trigger: false + - get: bosh-deployment + trigger: false + - get: pipelines + resource: bosh-cpi-certification-concourse-tasks + - get: bosh-cpi-certification-concourse-tasks + - put: environment + resource: light-aws-environment + params: + delete_on_failure: true + generate_random_name: true + terraform_source: bosh-cpi-src/ci/assets/terraform + - task: prepare-director + file: bosh-cpi-certification-concourse-tasks/shared/tasks/prepare-director.yml + image: bosh-integration-registry-image + input_mapping: + stemcell: light-stemcell + params: + DIRECTOR_VARS_FILE: ((aws_test_director_vars_file)) + INFRASTRUCTURE: aws + OPTIONAL_OPS_FILE: | + -o pipelines/shared/assets/ops/remove-hm.yml + -o bosh-deployment/external-ip-with-registry-not-recommended.yml + -o bosh-deployment/misc/source-releases/bosh.yml + - task: deploy-director + file: bosh-cpi-certification-concourse-tasks/shared/tasks/deploy-director.yml + image: bosh-integration-registry-image + input_mapping: + stemcell: light-stemcell + - task: run-stemcell-upload-tests + file: bosh-stemcells-ci/ci/tasks/light-aws/run-upload-test.yml + image: bosh-integration-registry-image + input_mapping: + builder-src: us-builder-src + stemcell: light-stemcell + ensure: + do: + - task: teardown + file: bosh-cpi-certification-concourse-tasks/shared/tasks/teardown.yml + image: bosh-integration-registry-image + - task: ensure-terminated + file: bosh-cpi-src/ci/tasks/ensure-terminated.yml + image: bosh-integration-registry-image + input_mapping: + bosh-aws-cpi-release: bosh-cpi-src + params: + AWS_PAGER: + AWS_ACCESS_KEY_ID: ((aws_publish_us_access_key)) + AWS_SECRET_ACCESS_KEY: ((aws_publish_us_secret_key)) + AWS_DEFAULT_REGION: ((aws_publish_us_region)) + - put: environment + resource: light-aws-environment + get_params: + action: destroy + params: + action: destroy + env_name_file: environment/name + terraform_source: bosh-cpi-src/ci/assets/terraform + - put: candidate-aws-light-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @) + params: files: - light-stemcell/*.tgz options: @@ -387,12 +319,21 @@ jobs: author_name: *ci_bot_name message: 'candidate (light aws): (@= data.values.stemcell_details.os_name @)/(@= str(data.values.stemcell_details.major_version) @).x' version: us-input-stemcell/.resource/version - put: candidate-aws-light-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @) serial: true - name: build-light-google-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @) plan: - - #@ get_google_light_inputs(data.values.stemcell_details.os_name, str(data.values.stemcell_details.major_version)) + - in_parallel: + - get: stemcell + params: + include_files: + - bosh-stemcell-*-google-kvm-(@= data.values.stemcell_details.os_name @)*.tgz + resource: candidate-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @) + trigger: true + version: every + - get: bosh-stemcells-ci + - get: bosh-cpi-release + resource: bosh-google-cpi-release - get: light-stemcell-builder-registry-image - get: gce-cpi-release-registry-image - get: bosh-integration-registry-image @@ -414,8 +355,51 @@ jobs: GCP_SERVICE_ACCOUNT_KEY: ((gcp_json_key)) PROJECT_NAME: ((gcp_project_id)) EFI: #@ data.values.stemcell_details.use_efi - - - #@ deploy_skeletal_google_light_stemcell("light-google-environment-oss", "((gcp_json_key))") + - do: + - put: terraform + resource: light-google-environment-oss + params: + generate_random_name: true + terraform_source: bosh-stemcells-ci/ci/tasks/light-google/terraform/ + - task: deploy-skeletal + file: bosh-stemcells-ci/ci/tasks/light-google/deploy-skeletal.yml + image: bosh-integration-registry-image + params: + GCE_CREDENTIALS_JSON: ((gcp_json_key)) + SSH_PRIVATE_KEY: ((ssh.private_key)) + ensure: + ensure: + get_params: + action: destroy + params: + action: destroy + env_name_file: terraform/name + terraform_source: bosh-stemcells-ci/ci/tasks/light-google/terraform/ + put: terraform + resource: light-google-environment-oss + file: bosh-stemcells-ci/ci/tasks/light-google/destroy-skeletal.yml + image: bosh-integration-registry-image + task: destroy-skeletal + on_failure: + image: bosh-integration-registry-image + config: + platform: linux + run: + args: + - | + * remove artifacts of failed jobs: + ```bash + gcloud auth revoke --all # start with clean slate + gcloud auth activate-service-account concourse@((gcp_project_id)).iam.gserviceaccount.com --key-file <( ((gcp_json_key)) ) + gsutil rm gs://bosh-gce-raw-stemcells-new/STEMCELL_TO_BE_DELETED_raw.tar.gz + gsutil rm gs://bosh-gce-raw-stemcells-new/STEMCELL_TO_BE_DELETED_raw.tar.gz.sha1 + gcloud auth revoke --all + ``` + * re-fly pipeline after commenting-out `disable_manual_trigger: true` + * trigger failed jobs + * re-fly pipeline after uncommenting `disable_manual_trigger: true` + path: echo + task: cleanup-failed-run-instructions - params: files: - light-stemcell/*.tgz diff --git a/ci/pipelines/vars.yml b/ci/pipelines/vars.yml index 66acfd512e..bfa3090a68 100644 --- a/ci/pipelines/vars.yml +++ b/ci/pipelines/vars.yml @@ -9,6 +9,8 @@ stemcell_details: os_short_name: noble subnet_int: "24" #! use last two digits of release year: ex 2010 -> 10 use_efi: true + bot_email: infra@cloudfoundry.org + bot_name: CI Bot include_iaas: [ {iaas: alicloud, hypervisor: kvm}, {iaas: aws, hypervisor: xen-hvm}, diff --git a/image-metalinks/ubuntu-jammy/ubuntu-jammy.meta4 b/image-metalinks/ubuntu-jammy/ubuntu-jammy.meta4 index c8faadd4ab..07f51ea965 100644 --- a/image-metalinks/ubuntu-jammy/ubuntu-jammy.meta4 +++ b/image-metalinks/ubuntu-jammy/ubuntu-jammy.meta4 @@ -1,22 +1,22 @@ - b748ba44907447a03a8e161b370795221f94961f42d59684a5d3f5aef3d48aa523dca132825ce1abf983fca1fc96532c2db9bffbcf4d71714bcff63b5026d62f - a4fff818d92a51015e05337bc3085a06696122f2bff31cdd2053957d99b9ee25 - ab89630e24048ea311a4ee6a2964d3e1998177ed - 25d8f6bba0a5e43838922b5f704e580d - 435207580 + 182693503ed5d20dd8f1e20354eb83f7bf9c98abf50a2dc773cd8e46c3377c9b2f1d5d1538e62c7f2fcaff793377100a6041af5008b960ace8f2b3fd734f9631 + 2d2671e2bc08313f25275db13745b84b49bda65a983480dac5c1f636b98cfe50 + 687ab8e8f8a229c4c354be66d2d6566d38e3ad8b + 9b8ef49de155059b700197509e1f2421 + 435215809 https://storage.googleapis.com/bosh-os-images/ubuntu-jammy/ubuntu-jammy.tgz - 1089.0.0 + 1104.0.0 - 04c6305dd88f1f0fdb8c052c3161682a0320751c96e6e5b6bc0b948bd3c0745d250ff57be25a11e69c1cbc1faf49d6e4dbd9f16163ccdc6ee61f6bdb89e13732 - 52256f7644d5e043f59cd3799675e5b76577fc57b7ccac248349ec4a382b2643 - 17fb83a8835fffe0fa223470bd10451554cf9ca9 - d6447fb301e142d191d4d2ce0a39ae2b - 8289 + be688838ca8686e5c90689bf2ab585cef1137c999b48c70b92f67a5c34dc15697b5d11c982ed6d71be1e1e7f7b4e0733884aa97c3f7a339a8ed03577cf74be09 + 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b + adc83b19e793491b1c6ea0fd8b46cd9f32e592fc + 68b329da9893e34099c7d8ad5cb9c940 + 1 https://storage.googleapis.com/bosh-os-images/ubuntu-jammy/usn-log.json - 1089.0.0 + 1104.0.0 metalink-repository-resource/0.0.0 - 2026-06-12T19:44:25.082634332Z + 2026-06-18T18:21:28.865309981Z diff --git a/stemcell_builder/stages/bosh_alicloud_agent_settings/apply.sh b/stemcell_builder/stages/bosh_alicloud_agent_settings/apply.sh index 206499c44b..50c735abb0 100755 --- a/stemcell_builder/stages/bosh_alicloud_agent_settings/apply.sh +++ b/stemcell_builder/stages/bosh_alicloud_agent_settings/apply.sh @@ -3,4 +3,5 @@ base_dir=$(readlink -nf $(dirname $0)/../..) source $base_dir/lib/prelude_apply.bash -cp $assets_dir/agent.json $chroot/var/vcap/bosh/agent.json +# shellcheck disable=SC2154 +cp "$assets_dir/agent.json" "$chroot/var/vcap/bosh/agent.json" diff --git a/stemcell_builder/stages/bosh_alicloud_agent_settings/assets/agent.json b/stemcell_builder/stages/bosh_alicloud_agent_settings/assets/agent.json index b94e416daa..053668a8da 100644 --- a/stemcell_builder/stages/bosh_alicloud_agent_settings/assets/agent.json +++ b/stemcell_builder/stages/bosh_alicloud_agent_settings/assets/agent.json @@ -24,4 +24,4 @@ "UseRegistry": true } } -} \ No newline at end of file +} diff --git a/stemcell_builder/stages/bosh_aws_agent_settings/apply.sh b/stemcell_builder/stages/bosh_aws_agent_settings/apply.sh index 206499c44b..50c735abb0 100755 --- a/stemcell_builder/stages/bosh_aws_agent_settings/apply.sh +++ b/stemcell_builder/stages/bosh_aws_agent_settings/apply.sh @@ -3,4 +3,5 @@ base_dir=$(readlink -nf $(dirname $0)/../..) source $base_dir/lib/prelude_apply.bash -cp $assets_dir/agent.json $chroot/var/vcap/bosh/agent.json +# shellcheck disable=SC2154 +cp "$assets_dir/agent.json" "$chroot/var/vcap/bosh/agent.json" diff --git a/stemcell_builder/stages/bosh_azure_agent_settings/apply.sh b/stemcell_builder/stages/bosh_azure_agent_settings/apply.sh index 206499c44b..50c735abb0 100755 --- a/stemcell_builder/stages/bosh_azure_agent_settings/apply.sh +++ b/stemcell_builder/stages/bosh_azure_agent_settings/apply.sh @@ -3,4 +3,5 @@ base_dir=$(readlink -nf $(dirname $0)/../..) source $base_dir/lib/prelude_apply.bash -cp $assets_dir/agent.json $chroot/var/vcap/bosh/agent.json +# shellcheck disable=SC2154 +cp "$assets_dir/agent.json" "$chroot/var/vcap/bosh/agent.json" diff --git a/stemcell_builder/stages/bosh_azure_agent_settings/assets/agent.json b/stemcell_builder/stages/bosh_azure_agent_settings/assets/agent.json index 4270848196..2becd50ea6 100644 --- a/stemcell_builder/stages/bosh_azure_agent_settings/assets/agent.json +++ b/stemcell_builder/stages/bosh_azure_agent_settings/assets/agent.json @@ -21,4 +21,4 @@ "UseServerName": true } } -} \ No newline at end of file +} diff --git a/stemcell_builder/stages/bosh_cloudstack_agent_settings/apply.sh b/stemcell_builder/stages/bosh_cloudstack_agent_settings/apply.sh index 206499c44b..50c735abb0 100755 --- a/stemcell_builder/stages/bosh_cloudstack_agent_settings/apply.sh +++ b/stemcell_builder/stages/bosh_cloudstack_agent_settings/apply.sh @@ -3,4 +3,5 @@ base_dir=$(readlink -nf $(dirname $0)/../..) source $base_dir/lib/prelude_apply.bash -cp $assets_dir/agent.json $chroot/var/vcap/bosh/agent.json +# shellcheck disable=SC2154 +cp "$assets_dir/agent.json" "$chroot/var/vcap/bosh/agent.json" diff --git a/stemcell_builder/stages/bosh_cloudstack_agent_settings/assets/agent.json b/stemcell_builder/stages/bosh_cloudstack_agent_settings/assets/agent.json index eb22f17acf..6e6af37de9 100644 --- a/stemcell_builder/stages/bosh_cloudstack_agent_settings/assets/agent.json +++ b/stemcell_builder/stages/bosh_cloudstack_agent_settings/assets/agent.json @@ -22,4 +22,4 @@ "UseRegistry": true } } -} \ No newline at end of file +} diff --git a/stemcell_builder/stages/bosh_google_agent_settings/apply.sh b/stemcell_builder/stages/bosh_google_agent_settings/apply.sh index 206499c44b..50c735abb0 100755 --- a/stemcell_builder/stages/bosh_google_agent_settings/apply.sh +++ b/stemcell_builder/stages/bosh_google_agent_settings/apply.sh @@ -3,4 +3,5 @@ base_dir=$(readlink -nf $(dirname $0)/../..) source $base_dir/lib/prelude_apply.bash -cp $assets_dir/agent.json $chroot/var/vcap/bosh/agent.json +# shellcheck disable=SC2154 +cp "$assets_dir/agent.json" "$chroot/var/vcap/bosh/agent.json" diff --git a/stemcell_builder/stages/bosh_google_agent_settings/assets/agent.json b/stemcell_builder/stages/bosh_google_agent_settings/assets/agent.json index ca27217458..dd5b29e960 100644 --- a/stemcell_builder/stages/bosh_google_agent_settings/assets/agent.json +++ b/stemcell_builder/stages/bosh_google_agent_settings/assets/agent.json @@ -27,4 +27,4 @@ "UseRegistry": false } } -} \ No newline at end of file +} diff --git a/stemcell_builder/stages/bosh_openstack_agent_settings/apply.sh b/stemcell_builder/stages/bosh_openstack_agent_settings/apply.sh index 206499c44b..50c735abb0 100755 --- a/stemcell_builder/stages/bosh_openstack_agent_settings/apply.sh +++ b/stemcell_builder/stages/bosh_openstack_agent_settings/apply.sh @@ -3,4 +3,5 @@ base_dir=$(readlink -nf $(dirname $0)/../..) source $base_dir/lib/prelude_apply.bash -cp $assets_dir/agent.json $chroot/var/vcap/bosh/agent.json +# shellcheck disable=SC2154 +cp "$assets_dir/agent.json" "$chroot/var/vcap/bosh/agent.json" diff --git a/stemcell_builder/stages/bosh_openstack_agent_settings/assets/agent.json b/stemcell_builder/stages/bosh_openstack_agent_settings/assets/agent.json index a6e17ce018..6e1c5a18d7 100644 --- a/stemcell_builder/stages/bosh_openstack_agent_settings/assets/agent.json +++ b/stemcell_builder/stages/bosh_openstack_agent_settings/assets/agent.json @@ -38,4 +38,4 @@ "UseRegistry": true } } -} \ No newline at end of file +} diff --git a/stemcell_builder/stages/bosh_softlayer_agent_settings/assets/agent.json b/stemcell_builder/stages/bosh_softlayer_agent_settings/assets/agent.json new file mode 100644 index 0000000000..b417f23595 --- /dev/null +++ b/stemcell_builder/stages/bosh_softlayer_agent_settings/assets/agent.json @@ -0,0 +1,24 @@ +{ + "Platform": { + "Linux": { + "PartitionerType": "parted", + "CreatePartitionIfNoEphemeralDisk": true, + "ScrubEphemeralDisk": true, + "DevicePathResolutionType": "iscsi", + "UseMonitIptablesFirewall": true + } + }, + "Infrastructure": { + "Settings": { + "Sources": [ + { + "Type": "HTTP", + "URI": "https://api.service.softlayer.com", + "UserDataPath": "/rest/v3.1/SoftLayer_Resource_Metadata/getUserMetadata.json" + } + ], + "UseServerName": true, + "UseRegistry": true + } + } +} diff --git a/stemcell_builder/stages/bosh_vsphere_agent_settings/apply.sh b/stemcell_builder/stages/bosh_vsphere_agent_settings/apply.sh index 206499c44b..50c735abb0 100755 --- a/stemcell_builder/stages/bosh_vsphere_agent_settings/apply.sh +++ b/stemcell_builder/stages/bosh_vsphere_agent_settings/apply.sh @@ -3,4 +3,5 @@ base_dir=$(readlink -nf $(dirname $0)/../..) source $base_dir/lib/prelude_apply.bash -cp $assets_dir/agent.json $chroot/var/vcap/bosh/agent.json +# shellcheck disable=SC2154 +cp "$assets_dir/agent.json" "$chroot/var/vcap/bosh/agent.json" diff --git a/stemcell_builder/stages/bosh_vsphere_agent_settings/assets/agent.json b/stemcell_builder/stages/bosh_vsphere_agent_settings/assets/agent.json index 5c5c32a182..d8288f1fb4 100644 --- a/stemcell_builder/stages/bosh_vsphere_agent_settings/assets/agent.json +++ b/stemcell_builder/stages/bosh_vsphere_agent_settings/assets/agent.json @@ -11,7 +11,8 @@ "Sources": [ { "Type": "VsphereGuestInfo" - }, { + }, + { "Type": "CDROM", "FileName": "env" }