diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
index 4bc21bdb87..7763981d1e 100644
--- a/.github/workflows/go.yml
+++ b/.github/workflows/go.yml
@@ -6,7 +6,7 @@ jobs:
dry-run-acceptance-tests:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v6
+ - uses: actions/checkout@v7
- uses: actions/setup-go@v6
with:
go-version-file: acceptance-tests/go.mod
diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml
index 93003ceb2d..4a2853fb29 100644
--- a/.github/workflows/ruby.yml
+++ b/.github/workflows/ruby.yml
@@ -5,7 +5,7 @@ jobs:
unit_specs:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v6
+ - uses: actions/checkout@v7
with: { lfs: true }
- uses: ruby/setup-ruby@v1
- name: test-bosh-stemcell
diff --git a/README.md b/README.md
index 5e37306ca4..7f2a58b4de 100644
--- a/README.md
+++ b/README.md
@@ -366,7 +366,7 @@ Create a `stemcell-builder-integration-${subnet_int}` subnetworks need by BATs t
Each stemcell line should get its own subnet corresponding to its `subnet_int` equal to
the two digit release year. For example release year 2010 would have `subnet_int="10"`.
-Example per [ci/pipelines/vars.yml](ci/pipelines/vars.yml):
+Example per [ci/pipeline-vars.yml](ci/pipeline-vars.yml):
```yaml
---
diff --git a/ci/configure.sh b/ci/configure.sh
index 12c08ce84c..227ad21a8a 100755
--- a/ci/configure.sh
+++ b/ci/configure.sh
@@ -1,61 +1,35 @@
#!/usr/bin/env bash
set -eu -o pipefail
-STEMCELL_LINE="ubuntu-resolute"
-
-REPO_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
-
if [[ -n "${DEBUG:-}" ]]; then
set -x
fi
-fly="${FLY_CLI:-fly}"
+REPO_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
+
+STEMCELL_LINE="ubuntu-resolute"
+
+pipelines_dir="${REPO_ROOT}/ci"
+pipeline_name="${STEMCELL_LINE}"
+pipeline_template="pipeline-template.yml"
+pipeline_vars="pipeline-vars.yml"
+
concourse_target="${CONCOURSE_TARGET:-stemcell}"
+fly="${FLY_CLI:-fly}"
until "${fly}" -t "${concourse_target}" status; do
"${fly}" -t "${concourse_target}" login
sleep 1
done
-pipelines_dir="${REPO_ROOT}/ci/pipelines"
-vars_file_name="vars.yml"
-
-mapfile -t available_pipelines < \
- <( find "${pipelines_dir}" -maxdepth 1 -type f -name '*.yml' | grep -v "${vars_file_name}" | sort )
-
-if (( ${#available_pipelines[@]} == 0 )); then
- echo "No pipelines found under '${pipelines_dir}'" >&2
- exit 1
-fi
-
-i=1
-echo "Choose a pipeline to configure:"
-for pipeline in "${available_pipelines[@]}"; do
- pipeline_choice_label=$(echo "${pipeline#"${pipelines_dir}/"}" | cut -d/ -f 1)
- printf "%4s. %s\n" "${i}" "${pipeline_choice_label}"
- i=$((i + 1))
-done
-read -rp "pipeline: " pipeline_index
+echo "Rendering..."
+rendered_template="$(ytt -f "${pipelines_dir}/${pipeline_template}" -f "${pipelines_dir}/${pipeline_vars}")"
echo ""
-if ! [[ "${pipeline_index}" =~ ^[0-9]+$ ]] || (( pipeline_index < 1 || pipeline_index > ${#available_pipelines[@]} )); then
- echo "Invalid selection: '${pipeline_index}'" >&2
- exit 1
-fi
-
-pipeline_file=${available_pipelines[(pipeline_index-1)]}
-if [ ! -f "${pipeline_file}" ]; then
- echo "No pipeline found: '${pipeline_file}'" >&2
- exit 1
-fi
-
-pipeline_name=$(basename "${pipeline_file%".yml"}")
-
-echo "Configuring '${pipeline_name}' using '${pipeline_file#"${pipelines_dir}/"}'..."
+echo "Validating..."
+fly validate-pipeline --strict --config <(echo "${rendered_template}")
echo ""
-rendered_template="$(ytt -f "${pipeline_file}" -f "${pipelines_dir}/${vars_file_name}")"
-
-"${fly}" -t "${concourse_target}" set-pipeline \
- -p "${STEMCELL_LINE}-${pipeline_name}" \
+echo "Configuring..."
+"${fly}" -t "${concourse_target}" set-pipeline -p "${pipeline_name}" \
-c <(echo "${rendered_template}")
diff --git a/ci/pipelines/builder.yml b/ci/pipeline-template.yml
similarity index 54%
rename from ci/pipelines/builder.yml
rename to ci/pipeline-template.yml
index aeb4d30875..0769727bac 100644
--- a/ci/pipelines/builder.yml
+++ b/ci/pipeline-template.yml
@@ -85,30 +85,99 @@ plan:
#@yaml/text-templated-strings
---
-anchors:
- bats_director_tag: &bats-director-tag test-stemcells-(@= data.values.stemcell_details.os_short_name @)
- ci_bot:
- email: &ci_bot_email (@= data.values.stemcell_details.bot_email @)
- name: &ci_bot_name (@= data.values.stemcell_details.bot_name @)
+#@ def build_light_aws_stemcell_new(stemcell_os, stemcell_version, prefix, efi, ami_destinations, ami_excluded_destinations):
+do:
+ - in_parallel:
+ - get: (@= prefix @)-input-stemcell
+ params:
+ include_files:
+ - bosh-stemcell-*-aws-xen-hvm-(@= stemcell_os @)*.tgz
+ resource: candidate-(@= stemcell_os @)-stemcell-(@= stemcell_version @)
+ trigger: true
+ version: every
+ - task: build-(@= prefix @)-stemcell
+ file: bosh-stemcells-ci/ci/tasks/light-aws/build.yml
+ image: aws-light-stemcell-builder-registry-image
+ input_mapping:
+ input-stemcell: (@= prefix @)-input-stemcell
+ output_mapping:
+ light-stemcell: (@= prefix @)-light-stemcell
+ params:
+ AWS_PAGER:
+ ami_access_key: ((aws_publish_(@= prefix @)_access_key))
+ ami_secret_key: ((aws_publish_(@= prefix @)_secret_key))
+ ami_region: ((aws_publish_(@= prefix @)_region))
+ ami_bucket_name: ((aws_publish_(@= prefix @)_bucket))
+ ami_description: Light Stemcell Builder Prod AMI
+ #@ if ami_destinations != "":
+ ami_destinations: (@= ami_destinations @)
+ #@ end
+ #@ if ami_excluded_destinations != "":
+ ami_excluded_destinations: (@= ami_excluded_destinations @)
+ #@ end
+ ami_encrypted: false
+ ami_kms_key_id: ""
+ ami_server_side_encryption: ""
+ ami_virtualization_type: hvm
+ ami_visibility: public
+ S3_API_ENDPOINT: storage.googleapis.com
+ efi: (@= str(efi).lower() @)
+#@ end
+#@yaml/text-templated-strings
+---
+#@ def cleanup_unpublished_light_stemcells(prefix):
+task: cleanup-amis-in-(@= prefix @)
+file: bosh-stemcells-ci/ci/tasks/light-aws/cleanup-ami.yml
+image: bosh-integration-registry-image
+params:
+ AWS_PAGER:
+ ami_access_key: ((aws_publish_(@= prefix @)_access_key))
+ ami_secret_key: ((aws_publish_(@= prefix @)_secret_key))
+ ami_region: ((aws_publish_(@= prefix @)_region))
+ ami_older_than_days: 60
+ ami_keep_latest: 5
+ os_name: (@= data.values.stemcell_details.os_name @)
+#@ end
+
+#@yaml/text-templated-strings
+---
+#@ def cleanup_old_published_light_stemcells(prefix):
+task: cleanup-amis-in-(@= prefix @)
+file: bosh-stemcells-ci/ci/tasks/light-aws/cleanup-ami.yml
+image: bosh-integration-registry-image
+params:
+ AWS_PAGER:
+ ami_access_key: ((aws_publish_(@= prefix @)_access_key))
+ ami_secret_key: ((aws_publish_(@= prefix @)_secret_key))
+ ami_region: ((aws_publish_(@= prefix @)_region))
+ ami_older_than_days: 1095
+ remove_public_images: true
+#@ end
+
+#@yaml/text-templated-strings
+---
groups:
- name: build
jobs:
- build-stemcell
- test-unit
- build-os-image
-
#@ for iaas in data.values.stemcell_details.include_iaas:
- build-(@= iaas.iaas @)-(@= iaas.hypervisor @)
#@ end
#@ for iaas in data.values.stemcell_details.include_fips_iaas:
- build-(@= iaas.iaas @)-(@= iaas.hypervisor @)-fips
#@ end
-
- bats
- test-stemcells-ipv4
#! - test-stemcells-ipv6
- aggregate-candidate-stemcells
+- name: publish
+ jobs:
+ - build-light-aws-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @)
+ - build-light-google-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @)
+ - publish-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @)
- name: auto-bumps
jobs:
@@ -125,12 +194,12 @@ groups:
- check-usn-packages-are-available
- log-low-medium-cves
- notify-of-usn
-- name: docker
+- name: infrastructure-tending
jobs:
- build-os-image-stemcell-builder
-- name: infrastructure
- jobs:
- ensure-integration-network
+ - cleanup-published-aws-light-stemcells-older-than-three-years
+ - cleanup-unpublished-(@= data.values.stemcell_details.os_name @)-aws-light-stemcells
#@yaml/text-templated-strings
jobs:
@@ -198,17 +267,17 @@ jobs:
serial: true
plan:
- get: bosh-stemcells-ci
- - get: bosh-integration-image
+ - get: bosh-integration-registry-image
- task: ensure-integration-network
file: bosh-stemcells-ci/ci/tasks/gcp/ensure-integration-network.yml
- image: bosh-integration-image
+ image: bosh-integration-registry-image
params:
GCP_JSON_KEY: ((gcp_json_key))
GCP_PROJECT_ID: ((gcp_project_id))
GCP_REGION: europe-north2
GCP_NETWORK_NAME: bosh-concourse
SUBNET_INT: (@= data.values.stemcell_details.subnet_int @)
- DIRECTOR_TAG: *bats-director-tag
+ DIRECTOR_TAG: &bats-director-tag test-stemcells-(@= data.values.stemcell_details.os_short_name @)
- name: process-high-critical-cves
serial_groups: [log-cves]
@@ -267,7 +336,7 @@ jobs:
- in_parallel:
- get: bosh-stemcells-ci
- get: os-image-stemcell-builder-registry-image
- - get: bosh-integration-image
+ - get: bosh-integration-registry-image
- get: bosh-linux-stemcell-builder
- get: usn-log
passed:
@@ -292,7 +361,7 @@ jobs:
image_os_tag: (@= data.values.stemcell_details.os_short_name @)
- task: write-message
file: bosh-stemcells-ci/ci/tasks/write-bump-message.yml
- image: bosh-integration-image
+ image: bosh-integration-registry-image
params:
MESSAGE_PREFIX: CVE Trigger
- put: stemcell-trigger
@@ -380,8 +449,8 @@ jobs:
- usn-log/usn-log.json
rename: (@= data.values.stemcell_details.branch @)/(@= data.values.stemcell_details.os_name @).meta4
options:
- author_email: *ci_bot_email
- author_name: *ci_bot_name
+ author_email: (@= data.values.stemcell_details.bot_email @)
+ author_name: (@= data.values.stemcell_details.bot_name @)
message: 'Bump os-image tgz'
version: version/version
- put: os-image-version
@@ -391,7 +460,7 @@ jobs:
- name: test-unit
plan:
- get: bosh-stemcells-ci
- - get: bosh-integration-image
+ - get: bosh-integration-registry-image
- get: bosh-linux-stemcell-builder
trigger: true
passed:
@@ -405,7 +474,7 @@ jobs:
- build-os-image
- task: test-unit
file: bosh-stemcells-ci/ci/tasks/test-unit.yml
- image: bosh-integration-image
+ image: bosh-integration-registry-image
privileged: true
serial: true
@@ -446,7 +515,7 @@ jobs:
resource: version
trigger: true
- get: bosh-stemcells-ci
- - get: bosh-integration-image
+ - get: bosh-integration-registry-image
- get: bosh-linux-stemcell-builder
- get: bosh-deployment
- get: syslog-release
@@ -469,7 +538,7 @@ jobs:
- do:
- task: deploy-director
file: bosh-stemcells-ci/ci/tasks/gcp/deploy-director.yml
- image: bosh-integration-image
+ image: bosh-integration-registry-image
params:
GCP_PROJECT_ID: ((gcp_project_id))
GCP_ZONE: europe-north2-a
@@ -486,7 +555,7 @@ jobs:
- task: test-stemcell
attempts: 3
file: bosh-stemcells-ci/ci/tasks/test-stemcell.yml
- image: bosh-integration-image
+ image: bosh-integration-registry-image
params:
BOSH_os_name: (@= data.values.stemcell_details.os_name @)
package: ipv4director
@@ -494,7 +563,7 @@ jobs:
do:
- task: teardown
file: bosh-stemcells-ci/ci/tasks/teardown.yml
- image: bosh-integration-image
+ image: bosh-integration-registry-image
attempts: 3
timeout: 15m
serial: true
@@ -514,7 +583,7 @@ jobs:
#! resource: version
#! trigger: true
#! - get: bosh-stemcells-ci
-#! - get: bosh-integration-image
+#! - get: bosh-integration-registry-image
#! - get: bosh-linux-stemcell-builder
#! - get: bosh-deployment
#! - get: syslog-release
@@ -537,7 +606,7 @@ jobs:
#! - do:
#! - task: deploy-director-ipv6
#! file: bosh-stemcells-ci/ci/tasks/gcp/deploy-director-ipv6.yml
-#! image: bosh-integration-image
+#! image: bosh-integration-registry-image
#! params:
#! GCP_PROJECT_ID: ((gcp_project_id))
#! GCP_ZONE: europe-north2-a
@@ -555,7 +624,7 @@ jobs:
#! - task: test-stemcell-ipv6
#! attempts: 3
#! file: bosh-stemcells-ci/ci/tasks/test-stemcell.yml
-#! image: bosh-integration-image
+#! image: bosh-integration-registry-image
#! params:
#! BOSH_os_name: (@= data.values.stemcell_details.os_name @)
#! package: ipv6director
@@ -563,7 +632,7 @@ jobs:
#! do:
#! - task: teardown
#! file: bosh-stemcells-ci/ci/tasks/teardown.yml
-#! image: bosh-integration-image
+#! image: bosh-integration-registry-image
#! attempts: 3
#! timeout: 15m
#! serial: true
@@ -588,7 +657,7 @@ jobs:
- get: bats
- get: bosh-deployment
- get: bosh-stemcells-ci
- - get: bosh-integration-image
+ - get: bosh-integration-registry-image
- get: bosh-linux-stemcell-builder
passed:
#@ for iaas in data.values.stemcell_details.include_iaas:
@@ -619,7 +688,7 @@ jobs:
- do:
- task: cleanup-bats-vms
file: bosh-stemcells-ci/ci/tasks/gcp/cleanup-bats-vms.yml
- image: bosh-integration-image
+ image: bosh-integration-registry-image
params:
GCP_JSON_KEY: ((gcp_json_key))
GCP_PROJECT_ID: ((gcp_project_id))
@@ -628,7 +697,7 @@ jobs:
TAG: *bats-director-tag
- task: deploy-director
file: bosh-stemcells-ci/ci/tasks/gcp/deploy-director.yml
- image: bosh-integration-image
+ image: bosh-integration-registry-image
params:
GCP_PROJECT_ID: ((gcp_project_id))
GCP_ZONE: europe-north2-a
@@ -643,7 +712,7 @@ jobs:
TAG: *bats-director-tag
- task: prepare-bats
file: bosh-stemcells-ci/ci/tasks/bats/iaas/gcp/prepare-bats-config.yml
- image: bosh-integration-image
+ image: bosh-integration-registry-image
params:
VARS_STEMCELL_NAME: bosh-google-kvm-ubuntu-(@= data.values.stemcell_details.os_short_name @)(@= data.values.stemcell_details.agent_suffix @)
VARS_NETWORK_DEFAULT: bosh-concourse
@@ -660,12 +729,12 @@ jobs:
VARS_GATEWAY_DEFAULT: 10.100.(@= data.values.stemcell_details.subnet_int @).1
- task: run-bats
file: bats/ci/tasks/run-bats.yml
- image: bosh-integration-image
+ image: bosh-integration-registry-image
ensure:
do:
- task: teardown
file: bosh-stemcells-ci/ci/tasks/teardown.yml
- image: bosh-integration-image
+ image: bosh-integration-registry-image
attempts: 3
timeout: 15m
@@ -680,7 +749,7 @@ jobs:
resource: version
trigger: true
- get: bosh-stemcells-ci
- - get: bosh-integration-image
+ - get: bosh-integration-registry-image
- get: bosh-linux-stemcell-builder
passed:
- bats
@@ -696,12 +765,12 @@ jobs:
file: bosh-stemcells-ci/ci/tasks/commit-build-time.yml
image: os-image-stemcell-builder-registry-image
params:
- GIT_USER_EMAIL: *ci_bot_email
- GIT_USER_NAME: *ci_bot_name
+ GIT_USER_EMAIL: (@= data.values.stemcell_details.bot_email @)
+ GIT_USER_NAME: (@= data.values.stemcell_details.bot_name @)
#@ if len(data.values.stemcell_details.include_fips_iaas) > 0:
- task: copy-fips-artifacts
file: bosh-stemcells-ci/ci/tasks/publish.yml
- image: bosh-integration-image
+ image: bosh-integration-registry-image
params:
AWS_ACCESS_KEY_ID: ((hmac_accesskey))
AWS_SECRET_ACCESS_KEY: ((hmac_secret))
@@ -718,12 +787,12 @@ jobs:
TO_INDEX: candidate
AWS_ENDPOINT: "https://storage.googleapis.com"
S3_API_ENDPOINT: storage.googleapis.com
- GIT_USER_EMAIL: *ci_bot_email
- GIT_USER_NAME: *ci_bot_name
+ GIT_USER_EMAIL: (@= data.values.stemcell_details.bot_email @)
+ GIT_USER_NAME: (@= data.values.stemcell_details.bot_name @)
#@ end
- task: copy-artifacts
file: bosh-stemcells-ci/ci/tasks/publish.yml
- image: bosh-integration-image
+ image: bosh-integration-registry-image
params:
AWS_ACCESS_KEY_ID: ((hmac_accesskey))
AWS_SECRET_ACCESS_KEY: ((hmac_secret))
@@ -740,8 +809,8 @@ jobs:
TO_INDEX: candidate
AWS_ENDPOINT: "https://storage.googleapis.com"
S3_API_ENDPOINT: storage.googleapis.com
- GIT_USER_EMAIL: *ci_bot_email
- GIT_USER_NAME: *ci_bot_name
+ GIT_USER_EMAIL: (@= data.values.stemcell_details.bot_email @)
+ GIT_USER_NAME: (@= data.values.stemcell_details.bot_name @)
- in_parallel:
- put: bosh-linux-stemcell-builder-push-tags
no_get: true
@@ -759,13 +828,13 @@ jobs:
- get: (@= data.values.stemcell_details.os_short_name @)-usn
passed:
- process-high-critical-cves
- - get: bosh-integration-image
+ - get: bosh-integration-registry-image
- get: high-critical-cves-processed
passed:
- process-high-critical-cves
trigger: true
- task: build-slack-message
- image: bosh-integration-image
+ image: bosh-integration-registry-image
config:
inputs:
- name: (@= data.values.stemcell_details.os_short_name @)-usn
@@ -813,7 +882,7 @@ jobs:
- get: bosh-ruby-release-registry-image
- get: weekly
trigger: true
- - get: bosh-integration-image
+ - get: bosh-integration-registry-image
- get: os-image-tarball
passed:
- build-os-image
@@ -828,13 +897,13 @@ jobs:
output_mapping:
output-repo: bosh-linux-stemcell-builder
params:
- GIT_USER_EMAIL: *ci_bot_email
- GIT_USER_NAME: *ci_bot_name
+ GIT_USER_EMAIL: (@= data.values.stemcell_details.bot_email @)
+ GIT_USER_NAME: (@= data.values.stemcell_details.bot_name @)
PACKAGE: ruby-((.:ruby_version))
VENDOR: true
- task: test-unit
file: bosh-stemcells-ci/ci/tasks/test-unit.yml
- image: bosh-integration-image
+ image: bosh-integration-registry-image
privileged: true
- put: bosh-linux-stemcell-builder-push
params:
@@ -847,7 +916,7 @@ jobs:
trigger: true
- get: bosh-stemcells-ci
- get: bosh-linux-stemcell-builder
- - get: bosh-integration-image
+ - get: bosh-integration-registry-image
- get: golang-release
- task: bump-deps
file: golang-release/ci/tasks/shared/bump-deps.yml
@@ -856,11 +925,11 @@ jobs:
output_mapping:
output_repo: bosh-linux-stemcell-builder
params:
- GIT_USER_EMAIL: *ci_bot_email
- GIT_USER_NAME: *ci_bot_name
+ GIT_USER_EMAIL: (@= data.values.stemcell_details.bot_email @)
+ GIT_USER_NAME: (@= data.values.stemcell_details.bot_name @)
SOURCE_PATH: acceptance-tests/
- task: acceptance-tests-dry-run
- image: bosh-integration-image
+ image: bosh-integration-registry-image
config:
platform: linux
inputs:
@@ -888,14 +957,14 @@ jobs:
resource: bosh-agent
trigger: true
- get: bosh-stemcells-ci
- - get: bosh-integration-image
+ - get: bosh-integration-registry-image
- get: bosh-linux-stemcell-builder
- task: bump
file: bosh-stemcells-ci/ci/tasks/bump-bosh-agent.yml
- image: bosh-integration-image
+ image: bosh-integration-registry-image
params:
- GIT_USER_EMAIL: *ci_bot_email
- GIT_USER_NAME: *ci_bot_name
+ GIT_USER_EMAIL: (@= data.values.stemcell_details.bot_email @)
+ GIT_USER_NAME: (@= data.values.stemcell_details.bot_name @)
- put: bosh-linux-stemcell-builder-push
params:
rebase: true
@@ -908,15 +977,15 @@ jobs:
resource: bosh-blobstore-(@= blobstore_type @)
trigger: true
- get: bosh-stemcells-ci
- - get: bosh-integration-image
+ - get: bosh-integration-registry-image
- get: bosh-linux-stemcell-builder
- task: bump-bosh-blobstore-cli
file: bosh-stemcells-ci/ci/tasks/bump-bosh-blobstore-cli.yml
- image: bosh-integration-image
+ image: bosh-integration-registry-image
params:
BLOBSTORE_TYPE: (@= blobstore_type @)
- GIT_USER_EMAIL: *ci_bot_email
- GIT_USER_NAME: *ci_bot_name
+ GIT_USER_EMAIL: (@= data.values.stemcell_details.bot_email @)
+ GIT_USER_NAME: (@= data.values.stemcell_details.bot_name @)
- put: bosh-linux-stemcell-builder-push
params:
rebase: true
@@ -924,6 +993,356 @@ jobs:
serial: true
#@ end
+- name: cleanup-published-aws-light-stemcells-older-than-three-years
+ serial: true
+ plan:
+ - get: every-week-on-monday
+ trigger: true
+ - get: bosh-stemcells-ci
+ - get: bosh-integration-registry-image
+ - #@ cleanup_old_published_light_stemcells("us")
+ - #@ cleanup_old_published_light_stemcells("us-gov")
+- name: cleanup-unpublished-(@= data.values.stemcell_details.os_name @)-aws-light-stemcells
+ serial: true
+ plan:
+ - get: every-week-on-monday
+ trigger: true
+ - get: bosh-stemcells-ci
+ - get: bosh-integration-registry-image
+ - #@ cleanup_unpublished_light_stemcells("us")
+ - #@ cleanup_unpublished_light_stemcells("us-gov")
+ #!- #@ cleanup_unpublished_light_stemcells("cn")
+
+- name: build-light-aws-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @)
+ plan:
+ - get: bosh-stemcells-ci
+ - get: aws-light-stemcell-builder-registry-image
+ - get: bosh-integration-registry-image
+ - in_parallel:
+ - do:
+ - #@ build_light_aws_stemcell_new(data.values.stemcell_details.os_name, str(data.values.stemcell_details.major_version), "us-gov", data.values.stemcell_details.use_efi, "", "")
+ - #@ build_light_aws_stemcell_new(data.values.stemcell_details.os_name, str(data.values.stemcell_details.major_version), "us", data.values.stemcell_details.use_efi, "", '["me-central-1"]')
+ - task: merge-builds
+ file: bosh-stemcells-ci/ci/tasks/light-aws/us-gov-merge-builds.yml
+ image: aws-light-stemcell-builder-registry-image
+ - do:
+ - in_parallel:
+ - get: bosh-cpi-src
+ resource: bosh-aws-cpi-release
+ trigger: false
+ - get: cpi-release
+ resource: bosh-aws-cpi-dev-artifacts
+ trigger: false
+ - get: bosh-release
+ trigger: false
+ - get: bosh-deployment
+ trigger: false
+ - get: pipelines
+ resource: bosh-cpi-certification-concourse-tasks
+ - get: bosh-cpi-certification-concourse-tasks
+ - put: environment
+ resource: light-aws-environment
+ params:
+ delete_on_failure: true
+ generate_random_name: true
+ terraform_source: bosh-cpi-src/ci/assets/terraform
+ - task: prepare-director
+ file: bosh-cpi-certification-concourse-tasks/shared/tasks/prepare-director.yml
+ image: bosh-integration-registry-image
+ input_mapping:
+ stemcell: light-stemcell
+ params:
+ DIRECTOR_VARS_FILE: ((aws_test_director_vars_file))
+ INFRASTRUCTURE: aws
+ OPTIONAL_OPS_FILE: |
+ -o pipelines/shared/assets/ops/remove-hm.yml
+ -o bosh-deployment/external-ip-with-registry-not-recommended.yml
+ -o bosh-deployment/misc/source-releases/bosh.yml
+ - task: deploy-director
+ file: bosh-cpi-certification-concourse-tasks/shared/tasks/deploy-director.yml
+ image: bosh-integration-registry-image
+ input_mapping:
+ stemcell: light-stemcell
+ - task: run-stemcell-upload-tests
+ file: bosh-stemcells-ci/ci/tasks/light-aws/run-upload-test.yml
+ image: bosh-integration-registry-image
+ input_mapping:
+ stemcell: light-stemcell
+ ensure:
+ do:
+ - task: teardown
+ file: bosh-cpi-certification-concourse-tasks/shared/tasks/teardown.yml
+ image: bosh-integration-registry-image
+ - task: ensure-terminated
+ file: bosh-cpi-src/ci/tasks/ensure-terminated.yml
+ image: bosh-integration-registry-image
+ input_mapping:
+ bosh-aws-cpi-release: bosh-cpi-src
+ params:
+ AWS_PAGER:
+ AWS_ACCESS_KEY_ID: ((aws_publish_us_access_key))
+ AWS_SECRET_ACCESS_KEY: ((aws_publish_us_secret_key))
+ AWS_DEFAULT_REGION: ((aws_publish_us_region))
+ - put: environment
+ resource: light-aws-environment
+ get_params:
+ action: destroy
+ params:
+ action: destroy
+ env_name_file: environment/name
+ terraform_source: bosh-cpi-src/ci/assets/terraform
+ - put: candidate-aws-light-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
+ params:
+ files:
+ - light-stemcell/*.tgz
+ options:
+ author_email: (@= data.values.stemcell_details.bot_email @)
+ author_name: (@= data.values.stemcell_details.bot_name @)
+ message: 'candidate (light aws): (@= data.values.stemcell_details.os_name @)/(@= str(data.values.stemcell_details.major_version) @).x'
+ version: us-input-stemcell/.resource/version
+ serial: true
+
+- name: build-light-google-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @)
+ plan:
+ - in_parallel:
+ - get: stemcell
+ params:
+ include_files:
+ - bosh-stemcell-*-google-kvm-(@= data.values.stemcell_details.os_name @)*.tgz
+ resource: candidate-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
+ trigger: true
+ version: every
+ - get: bosh-stemcells-ci
+ - get: bosh-cpi-release
+ resource: bosh-google-cpi-release
+ - get: gce-cpi-release-registry-image
+ - get: bosh-integration-registry-image
+ - task: make-raw-from-heavy-stemcell
+ file: bosh-stemcells-ci/ci/tasks/light-google/make-raw-from-heavy-stemcell.yml
+ image: gce-cpi-release-registry-image
+ params:
+ BUCKET_NAME: bosh-gce-raw-stemcells-new
+ STEMCELL_BUCKET_PATH: bosh-gce-light-stemcells
+ S3_API_ENDPOINT: storage.googleapis.com
+ - params:
+ file: raw-stemcell/bosh-stemcell-*-google-kvm-ubuntu-*.tar.gz
+ predefined_acl: publicRead
+ put: base-oss-google-ubuntu-stemcell
+ - task: create-public-image
+ file: bosh-stemcells-ci/ci/tasks/light-google/create-public-image.yml
+ image: gce-cpi-release-registry-image
+ params:
+ GCP_SERVICE_ACCOUNT_KEY: ((gcp_json_key))
+ PROJECT_NAME: ((gcp_project_id))
+ EFI: #@ data.values.stemcell_details.use_efi
+ - do:
+ - put: terraform
+ resource: light-google-environment-oss
+ params:
+ generate_random_name: true
+ terraform_source: bosh-stemcells-ci/ci/tasks/light-google/terraform/
+ - task: deploy-skeletal
+ file: bosh-stemcells-ci/ci/tasks/light-google/deploy-skeletal.yml
+ image: bosh-integration-registry-image
+ params:
+ GCE_CREDENTIALS_JSON: ((gcp_json_key))
+ SSH_PRIVATE_KEY: ((ssh.private_key))
+ ensure:
+ ensure:
+ get_params:
+ action: destroy
+ params:
+ action: destroy
+ env_name_file: terraform/name
+ terraform_source: bosh-stemcells-ci/ci/tasks/light-google/terraform/
+ put: terraform
+ resource: light-google-environment-oss
+ file: bosh-stemcells-ci/ci/tasks/light-google/destroy-skeletal.yml
+ image: bosh-integration-registry-image
+ task: destroy-skeletal
+ on_failure:
+ image: bosh-integration-registry-image
+ config:
+ platform: linux
+ run:
+ args:
+ - |
+ * remove artifacts of failed jobs:
+ ```bash
+ gcloud auth revoke --all # start with clean slate
+ gcloud auth activate-service-account concourse@((gcp_project_id)).iam.gserviceaccount.com --key-file <( ((gcp_json_key)) )
+ gsutil rm gs://bosh-gce-raw-stemcells-new/STEMCELL_TO_BE_DELETED_raw.tar.gz
+ gsutil rm gs://bosh-gce-raw-stemcells-new/STEMCELL_TO_BE_DELETED_raw.tar.gz.sha1
+ gcloud auth revoke --all
+ ```
+ * re-fly pipeline after commenting-out `disable_manual_trigger: true`
+ * trigger failed jobs
+ * re-fly pipeline after uncommenting `disable_manual_trigger: true`
+ path: echo
+ task: cleanup-failed-run-instructions
+ - params:
+ files:
+ - light-stemcell/*.tgz
+ options:
+ author_email: (@= data.values.stemcell_details.bot_email @)
+ author_name: (@= data.values.stemcell_details.bot_name @)
+ message: 'candidate (light google): (@= data.values.stemcell_details.os_name @)/(@= str(data.values.stemcell_details.major_version) @).x'
+ version: stemcell/.resource/version
+ put: candidate-google-light-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
+ serial: true
+
+- name: publish-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @)
+ plan:
+ - in_parallel:
+ - get: bosh-stemcells-ci
+ - get: bosh-linux-stemcell-builder
+ resource: bosh-linux-stemcell-builder-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @).x
+ - get: bosh-integration-registry-image
+ - get: os-image-stemcell-builder-registry-image
+ - get: stemcell-metalink
+ params:
+ skip_download: true
+ passed:
+ - build-light-aws-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @)
+ - build-light-google-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @)
+ resource: candidate-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
+ - get: candidate-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
+ passed:
+ - build-light-aws-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @)
+ - build-light-google-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @)
+ #@ if len(data.values.stemcell_details.include_fips_iaas) > 0:
+ - get: candidate-(@= data.values.stemcell_details.os_name @)-fips-stemcell-(@= str(data.values.stemcell_details.major_version) @)
+ #@ end
+ - get: candidate-google-light-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
+ passed:
+ - build-light-google-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @)
+ - get: candidate-aws-light-stemcell
+ resource: candidate-aws-light-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
+ passed:
+ - build-light-aws-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @)
+ - file: bosh-stemcells-ci/ci/tasks/build-release-metadata.yml
+ image: os-image-stemcell-builder-registry-image
+ task: build-release-metadata
+ params:
+ OS_NAME: ubuntu
+ OS_VERSION: (@= str(data.values.stemcell_details.os_short_name) @)
+ BRANCH: (@= data.values.stemcell_details.branch @)
+ KERNEL_PACKAGE: linux-generic
+ input_mapping:
+ candidate-stemcell: candidate-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
+ - task: extract-warden-image
+ image: bosh-integration-registry-image
+ config:
+ inputs:
+ - name: candidate-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
+ outputs:
+ - name: stemcell-image
+ platform: linux
+ run:
+ dir: stemcell-image
+ path: /bin/bash
+ args:
+ - -ce
+ - |
+ tar xvf ../candidate-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)/bosh-stemcell-*-warden-boshlite-(@= data.values.stemcell_details.os_name @)*.tgz image
+ printf 'FROM scratch\nADD image /\n' > Dockerfile
+ - task: build-stemcell-oci-image
+ privileged: true
+ config:
+ platform: linux
+ image_resource:
+ type: registry-image
+ source:
+ repository: concourse/oci-build-task
+ inputs:
+ - name: stemcell-image
+ outputs:
+ - name: image
+ params:
+ CONTEXT: stemcell-image
+ run:
+ path: build
+ - put: github-container-registry-(@= data.values.stemcell_details.os_name @)-stemcell
+ params:
+ image: image/image.tar
+ additional_tags: candidate-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)/.resource/version
+ - put: published-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
+ params:
+ files:
+ - candidate-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)/*.tgz
+ options:
+ author_email: (@= data.values.stemcell_details.bot_email @)
+ author_name: (@= data.values.stemcell_details.bot_name @)
+ message: 'publish (heavy): (@= data.values.stemcell_details.os_name @)/(@= str(data.values.stemcell_details.major_version) @).x'
+ rename: '{{.Version}}/stemcells.meta4'
+ version: candidate-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)/.resource/version
+ #@ if len(data.values.stemcell_details.include_fips_iaas) > 0 :
+ - put: published-(@= data.values.stemcell_details.os_name @)-fips-stemcell-(@= str(data.values.stemcell_details.major_version) @)
+ params:
+ files:
+ - candidate-(@= data.values.stemcell_details.os_name @)-fips-stemcell-(@= str(data.values.stemcell_details.major_version) @)/*.tgz
+ options:
+ author_email: (@= data.values.stemcell_details.bot_email @)
+ author_name: (@= data.values.stemcell_details.bot_name @)
+ message: 'publish (heavy): (@= data.values.stemcell_details.os_name @)-fips/(@= str(data.values.stemcell_details.major_version) @).x'
+ rename: '{{.Version}}/stemcells.meta4'
+ version: candidate-(@= data.values.stemcell_details.os_name @)-fips-stemcell-(@= str(data.values.stemcell_details.major_version) @)/.resource/version
+ #@ end
+ - params:
+ files:
+ - candidate-aws-light-stemcell/*.tgz
+ options:
+ author_email: (@= data.values.stemcell_details.bot_email @)
+ author_name: (@= data.values.stemcell_details.bot_name @)
+ message: 'publish (light aws): (@= data.values.stemcell_details.os_name @)/(@= str(data.values.stemcell_details.major_version) @).x'
+ rename: '{{.Version}}/stemcells.aws.meta4'
+ version: candidate-aws-light-stemcell/.resource/version
+ put: published-aws-light-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
+
+ #! once we release all regions with the same account, we can unify these again
+ - file: bosh-stemcells-ci/ci/tasks/light-aws/tag-aws-ami-light.yml
+ image: bosh-integration-registry-image
+ task: tag-published-aws-ami-light-stemcells
+ params:
+ AWS_PAGER:
+ AWS_ACCESS_KEY_ID: ((aws_publish_us_access_key))
+ AWS_SECRET_ACCESS_KEY: ((aws_publish_us_secret_key))
+ GREP_PATTERN: grep -v 'gov-\|cn-'
+ - file: bosh-stemcells-ci/ci/tasks/light-aws/tag-aws-ami-light.yml
+ image: bosh-integration-registry-image
+ task: tag-gov-published-aws-ami-light-stemcells
+ params:
+ AWS_PAGER:
+ AWS_ACCESS_KEY_ID: ((aws_publish_us-gov_access_key))
+ AWS_SECRET_ACCESS_KEY: ((aws_publish_us-gov_secret_key))
+ GREP_PATTERN: grep 'gov-'
+ #! - file: bosh-stemcells-ci/ci/tasks/light-aws/tag-aws-ami-light.yml
+ #! task: tag-china-published-aws-ami-light-stemcells
+ #! params:
+ #! AWS_ACCESS_KEY_ID: ((aws_publish_cn_access_key))
+ #! AWS_SECRET_ACCESS_KEY: ((aws_publish_cn_secret_key))
+ #! GREP_PATTERN: grep 'cn-'
+ - params:
+ files:
+ - candidate-google-light-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)/*.tgz
+ options:
+ author_email: (@= data.values.stemcell_details.bot_email @)
+ author_name: (@= data.values.stemcell_details.bot_name @)
+ message: 'publish (light google): (@= data.values.stemcell_details.os_name @)/(@= str(data.values.stemcell_details.major_version) @).x'
+ rename: '{{.Version}}/stemcells.gcp.meta4'
+ version: candidate-google-light-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)/.resource/version
+ put: published-google-light-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
+ - params:
+ body: release-metadata/body
+ name: release-metadata/name
+ tag: release-metadata/tag
+ put: gh-release-oss
+ - params:
+ acl: public-read
+ file: usn-log/usn-log.json
+ put: usn-log-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @).x
+ serial: true
+
resource_types:
- name: ami-resource
type: registry-image
@@ -953,7 +1372,11 @@ resource_types:
type: registry-image
source:
repository: frodenas/gcs-resource
-#@yaml/text-templated-strings
+- name: terraform_type
+ source:
+ repository: ljfranklin/terraform-resource
+ type: registry-image
+
resources:
- name: daily
type: time
@@ -968,6 +1391,246 @@ resources:
stop: 4:30 -0700
days:
- Saturday
+- name: every-week-on-monday
+ type: time
+ source:
+ initial_version: true
+ start: 6:00 -0700
+ stop: 8:30 -0700
+ days:
+ - Monday
+ interval: 168h
+- name: gh-release-oss
+ type: github-release
+ source:
+ access_token: ((github_public_repo_token))
+ drafts: true
+ owner: cloudfoundry
+ repository: bosh-linux-stemcell-builder
+
+- name: bosh-cpi-certification-concourse-tasks
+ type: git
+ source:
+ branch: master
+ uri: https://github.com/cloudfoundry/bosh-cpi-certification
+
+- name: bosh-aws-cpi-release
+ type: git
+ source:
+ branch: master
+ ignore_paths:
+ - .final_builds/**/*.yml
+ - releases/**/*.yml
+ uri: https://github.com/cloudfoundry/bosh-aws-cpi-release.git
+
+- name: bosh-aws-cpi-dev-artifacts
+ type: gcs-resource
+ source:
+ versioned_file: bosh-aws-cpi-dev-release.tgz
+ bucket: bosh-aws-cpi-pipeline
+ json_key: ((gcp_json_key))
+
+- name: light-aws-environment
+ source:
+ backend_type: s3
+ backend_config:
+ access_key: ((aws_publish_us_access_key))
+ secret_key: ((aws_publish_us_secret_key))
+ bucket: bosh-aws-light-terraform
+ key: state/terraform.tfstate
+ region: ((aws_publish_us_region))
+ vars:
+ access_key: ((aws_publish_us_access_key))
+ secret_key: ((aws_publish_us_secret_key))
+ public_key: ((aws_test_public_key_file))
+ region: ((aws_publish_us_region))
+ type: terraform_type
+
+- name: bosh-google-cpi-release
+ source:
+ repository: cloudfoundry/bosh-google-cpi-release
+ type: bosh-io-release
+- name: light-google-environment-oss
+ source:
+ delete_on_failure: true
+ backend_type: gcs
+ backend_config:
+ credentials: ((gcp_json_key))
+ bucket: bosh-gce-light-stemcell-ci-terraform-state
+ prefix: stemcell-ci-terraform/
+ vars:
+ gce_credentials_json: ((gcp_json_key))
+ gce_project_id: ((gcp_project_id))
+ type: terraform_type
+
+- name: base-oss-google-ubuntu-stemcell
+ source:
+ bucket: bosh-gce-raw-stemcells-new
+ json_key: ((gcp_json_key))
+ regexp: bosh-stemcell-([0-9\.]+)-google-kvm-ubuntu-*-raw.tar.gz
+ type: gcs-resource
+
+- name: candidate-aws-light-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
+ source:
+ mirror_files:
+ - destination: s3://storage.googleapis.com/bosh-aws-light-stemcells-candidate/{{.Version}}/{{.Name}}
+ options:
+ private_key: ((github_deploy_key_bosh-io-stemcells-cpi-index.private_key))
+ uri: git+ssh://git@github.com:cloudfoundry/bosh-io-stemcells-cpi-index.git//candidate-aws-light/(@= data.values.stemcell_details.os_name @)
+ url_handlers:
+ - include:
+ - (s3|https)://.*
+ options:
+ access_key: ((hmac_accesskey))
+ secret_key: ((hmac_secret))
+ type: s3
+ version: (@= str(data.values.stemcell_details.major_version) @).x
+ type: metalink-repository
+- name: candidate-google-light-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
+ source:
+ mirror_files:
+ - destination: s3://storage.googleapis.com/bosh-gce-light-stemcells-candidate/{{.Version}}/{{.Name}}
+ options:
+ private_key: ((github_deploy_key_bosh-io-stemcells-cpi-index.private_key))
+ uri: git+ssh://git@github.com:cloudfoundry/bosh-io-stemcells-cpi-index.git//candidate-gcp-light/(@= data.values.stemcell_details.os_name @)
+ url_handlers:
+ - include:
+ - (s3|https)://.*
+ options:
+ access_key: ((hmac_accesskey))
+ secret_key: ((hmac_secret))
+ type: s3
+ version: (@= str(data.values.stemcell_details.major_version) @).x
+ type: metalink-repository
+- name: candidate-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
+ source:
+ filters:
+ - repositorypath: '*/stemcells.meta4'
+ options:
+ private_key: ((github_deploy_key_bosh-io-stemcells-core-index.private_key))
+ uri: git+ssh://git@github.com:cloudfoundry/bosh-io-stemcells-core-index.git//candidate/(@= data.values.stemcell_details.os_name @)
+ url_handlers:
+ - include:
+ - (s3|https)://.*
+ type: s3
+ version: (@= str(data.values.stemcell_details.major_version) @).x
+ type: metalink-repository
+ #@ if len(data.values.stemcell_details.include_fips_iaas) > 0:
+- name: candidate-(@= data.values.stemcell_details.os_name @)-fips-stemcell-(@= str(data.values.stemcell_details.major_version) @)
+ source:
+ filters:
+ - repositorypath: '*/stemcells.meta4'
+ options:
+ private_key: ((github_deploy_key_bosh-io-stemcells-core-index.private_key))
+ uri: git+ssh://git@github.com:cloudfoundry/bosh-io-stemcells-core-index.git//candidate/(@= data.values.stemcell_details.os_name @)-fips
+ url_handlers:
+ - include:
+ - (s3|https)://.*
+ type: s3
+ options:
+ access_key: ((hmac_accesskey))
+ secret_key: ((hmac_secret))
+ version: (@= str(data.values.stemcell_details.major_version) @).x
+ type: metalink-repository
+ #@ end
+- name: bosh-linux-stemcell-builder-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @).x
+ source:
+ branch: (@= data.values.stemcell_details.branch @)
+ private_key: ((bosh_src_key.private_key))
+ uri: git@github.com:cloudfoundry/bosh-linux-stemcell-builder
+ type: git
+
+- name: usn-log-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @).x
+ type: gcs-resource
+ source:
+ bucket: bosh-stemcell-triggers
+ json_key: ((gcp_json_key))
+ versioned_file: (@= data.values.stemcell_details.branch @)/usn-log.json
+
+- name: published-aws-light-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
+ source:
+ filters:
+ - repositorypath: '*/stemcells.aws.meta4'
+ mirror_files:
+ - destination: s3://storage.googleapis.com/bosh-aws-light-stemcells/{{.Version}}/{{.Name}}
+ options:
+ private_key: ((github_deploy_key_bosh-io-stemcells-cpi-index.private_key))
+ uri: git+ssh://git@github.com:cloudfoundry/bosh-io-stemcells-cpi-index.git//published/(@= data.values.stemcell_details.os_name @)
+ url_handlers:
+ - include:
+ - (s3|https)://.*
+ options:
+ access_key: ((hmac_accesskey))
+ secret_key: ((hmac_secret))
+ type: s3
+ version: (@= str(data.values.stemcell_details.major_version) @).x
+ type: metalink-repository
+
+- name: published-google-light-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
+ source:
+ filters:
+ - repositorypath: '*/stemcells.gcp.meta4'
+ mirror_files:
+ - destination: s3://storage.googleapis.com/bosh-gce-light-stemcells/{{.Version}}/{{.Name}}
+ options:
+ private_key: ((github_deploy_key_bosh-io-stemcells-cpi-index.private_key))
+ uri: git+ssh://git@github.com:cloudfoundry/bosh-io-stemcells-cpi-index.git//published/(@= data.values.stemcell_details.os_name @)
+ url_handlers:
+ - include:
+ - (s3|https)://.*
+ options:
+ access_key: ((hmac_accesskey))
+ secret_key: ((hmac_secret))
+ type: s3
+ version: (@= str(data.values.stemcell_details.major_version) @).x
+ type: metalink-repository
+
+- name: published-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
+ source:
+ filters:
+ - repositorypath: '*/stemcells.meta4'
+ mirror_files:
+ - destination: s3://storage.googleapis.com/bosh-core-stemcells/{{.Version}}/{{.Name}}
+ options:
+ private_key: ((github_deploy_key_bosh-io-stemcells-core-index.private_key))
+ uri: git+ssh://git@github.com:cloudfoundry/bosh-io-stemcells-core-index.git//published/(@= data.values.stemcell_details.os_name @)
+ url_handlers:
+ - include:
+ - (s3|https)://.*
+ options:
+ access_key: ((hmac_accesskey))
+ secret_key: ((hmac_secret))
+ type: s3
+ version: (@= str(data.values.stemcell_details.major_version) @).x
+ type: metalink-repository
+ #@ if len(data.values.stemcell_details.include_fips_iaas) > 0:
+- name: published-(@= data.values.stemcell_details.os_name @)-fips-stemcell-(@= str(data.values.stemcell_details.major_version) @)
+ source:
+ filters:
+ - repositorypath: '*/stemcells.meta4'
+ mirror_files:
+ - destination: s3://storage.googleapis.com/bosh-core-stemcells-fips/{{.Version}}/{{.Name}}
+ options:
+ private_key: ((github_deploy_key_bosh-io-stemcells-core-index.private_key))
+ uri: git+ssh://git@github.com:cloudfoundry/bosh-io-stemcells-core-index.git//published/(@= data.values.stemcell_details.os_name @)-fips
+ url_handlers:
+ - include:
+ - (s3|https)://.*
+ options:
+ access_key: ((hmac_accesskey))
+ secret_key: ((hmac_secret))
+ type: s3
+ version: (@= str(data.values.stemcell_details.major_version) @).x
+ type: metalink-repository
+ #@ end
+
+- name: github-container-registry-(@= data.values.stemcell_details.os_name @)-stemcell
+ type: registry-image
+ source:
+ repository: ghcr.io/cloudfoundry/(@= data.values.stemcell_details.os_name @)-stemcell
+ username: ((github_read_write_packages.username))
+ password: ((github_read_write_packages.password))
+ tag: latest
- name: bosh-agent
type: metalink-repository
@@ -1112,7 +1775,7 @@ resources:
branch: master
uri: https://github.com/cloudfoundry/bosh-acceptance-tests.git
-- name: bosh-integration-image
+- name: bosh-integration-registry-image
type: registry-image
source:
repository: ghcr.io/cloudfoundry/bosh/integration
@@ -1127,6 +1790,11 @@ resources:
private_key: ((github_deploy_key_bosh-io-stemcells-core-index.private_key))
uri: git@github.com:cloudfoundry/bosh-io-stemcells-core-index.git
+- name: bosh-release
+ type: bosh-io-release
+ source:
+ repository: cloudfoundry/bosh
+
- name: syslog-release
type: bosh-io-release
source:
@@ -1147,6 +1815,7 @@ resources:
source:
branch: master
uri: https://github.com/cloudfoundry/bosh-deployment
+
- name: (@= data.values.stemcell_details.os_short_name @)-usn-low-medium
type: usn
source:
@@ -1216,6 +1885,18 @@ resources:
bucket: bosh-vmware-ovftool
regexp: (@= data.values.stemcell_details.os_short_name @)/(.*).bundle
+- name: aws-light-stemcell-builder-registry-image
+ type: registry-image
+ source:
+ repository: bosh/light-stemcell-builder
+ username: ((dockerhub_username))
+ password: ((dockerhub_password))
+
+- name: gce-cpi-release-registry-image
+ type: registry-image
+ source:
+ repository: foundationalinfrastructure/gce-cpi-release
+
- name: bosh-blobstore-dav
type: s3
source:
diff --git a/ci/pipelines/vars.yml b/ci/pipeline-vars.yml
similarity index 100%
rename from ci/pipelines/vars.yml
rename to ci/pipeline-vars.yml
diff --git a/ci/pipelines/publisher.yml b/ci/pipelines/publisher.yml
deleted file mode 100644
index 83c1dca19f..0000000000
--- a/ci/pipelines/publisher.yml
+++ /dev/null
@@ -1,881 +0,0 @@
-#@ load("@ytt:data", "data")
-
-#@yaml/text-templated-strings
----
-
-#@ def build_light_aws_stemcell_new(stemcell_os, stemcell_version, builder_src, input_stemcell, output_stemcell, prefix, region, bucket_prefix, tag, ami_destinations, efi, ami_excluded_destinations):
-do:
- - in_parallel:
- - get: (@= input_stemcell @)
- params:
- include_files:
- - bosh-stemcell-*-aws-xen-hvm-(@= stemcell_os @)*.tgz
- resource: candidate-(@= stemcell_os @)-stemcell-(@= stemcell_version @)
- trigger: true
- version: every
- #@ if tag != "":
- tags:
- - (@= tag @)
- #@ end
- - get: (@= builder_src @)
- passed:
- - test-aws-unit
- - test-aws-integration
- - test-aws-drivers
- resource: light-aws-builder-src
- #@ if tag != "":
- tags:
- - (@= tag @)
- #@ end
- - task: build-(@= region @)-stemcell
- file: bosh-stemcells-ci/ci/tasks/light-aws/build.yml
- image: light-stemcell-builder-registry-image
- input_mapping:
- builder-src: (@= builder_src @)
- input-stemcell: (@= input_stemcell @)
- output_mapping:
- light-stemcell: (@= output_stemcell @)
- params:
- AWS_PAGER:
- ami_access_key: ((aws_publish_(@= prefix @)_access_key))
- ami_secret_key: ((aws_publish_(@= prefix @)_secret_key))
- ami_region: ((aws_publish_(@= region @)_region))
- ami_bucket_name: ((aws_publish_(@= bucket_prefix @)_bucket))
- ami_description: Light Stemcell Builder Prod AMI
- #@ if ami_destinations != "":
- ami_destinations: (@= ami_destinations @)
- #@ end
- #@ if ami_excluded_destinations != "":
- ami_excluded_destinations: (@= ami_excluded_destinations @)
- #@ end
- ami_encrypted: false
- ami_kms_key_id: ""
- ami_server_side_encryption: ""
- ami_virtualization_type: hvm
- ami_visibility: public
- S3_API_ENDPOINT: storage.googleapis.com
- efi: (@= str(efi).lower() @)
- #@ if tag != "":
- tags:
- - (@= tag @)
- #@ end
-#@ end
-
-#@yaml/text-templated-strings
----
-#@ def cleanup_unpublished_light_stemcells(prefix, region):
-task: cleanup-amis-in-(@= prefix @)
-file: bosh-stemcells-ci/ci/tasks/light-aws/cleanup-ami.yml
-image: bosh-integration-registry-image
-params:
- AWS_PAGER:
- ami_access_key: ((aws_publish_(@= prefix @)_access_key))
- ami_secret_key: ((aws_publish_(@= prefix @)_secret_key))
- ami_region: ((aws_publish_(@= region @)_region))
- ami_older_than_days: 60
- ami_keep_latest: 5
- os_name: (@= data.values.stemcell_details.os_name @)
-#@ end
-
-#@yaml/text-templated-strings
----
-#@ def cleanup_old_published_light_stemcells(prefix, region):
-task: cleanup-amis-in-(@= prefix @)
-file: bosh-stemcells-ci/ci/tasks/light-aws/cleanup-ami.yml
-image: bosh-integration-registry-image
-params:
- AWS_PAGER:
- ami_access_key: ((aws_publish_(@= prefix @)_access_key))
- ami_secret_key: ((aws_publish_(@= prefix @)_secret_key))
- ami_region: ((aws_publish_(@= region @)_region))
- ami_older_than_days: 1095
- remove_public_images: true
-#@ end
-
-#@yaml/text-templated-strings
----
-anchors:
- ci_bot:
- email: &ci_bot_email bots@cloudfoundry.org
- name: &ci_bot_name CI Bot
-
-groups:
-- name: all
- jobs:
- - test-aws-unit
- - test-aws-drivers
- - test-aws-integration
- - build-light-aws-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @)
- - build-light-google-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @)
- - publish-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @)
-- name: oss
- jobs:
- - build-light-aws-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @)
- - build-light-google-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @)
- - publish-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @)
-- name: (@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @)
- jobs:
- - build-light-aws-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @)
- - build-light-google-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @)
- - publish-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @)
-- name: cleanup-aws-light-stemcells
- jobs:
- - cleanup-light-aws-builder-test-amis
- - cleanup-published-aws-light-stemcells-older-than-three-years
- - cleanup-unpublished-(@= data.values.stemcell_details.os_name @)-aws-light-stemcells
-
-jobs:
-- name: cleanup-published-aws-light-stemcells-older-than-three-years
- serial: true
- plan:
- - get: every-week-on-monday
- trigger: true
- - get: bosh-stemcells-ci
- - get: bosh-integration-registry-image
- - #@ cleanup_old_published_light_stemcells("us", "us")
- - #@ cleanup_old_published_light_stemcells("us-gov", "us-gov")
-- name: cleanup-unpublished-(@= data.values.stemcell_details.os_name @)-aws-light-stemcells
- serial: true
- plan:
- - get: every-week-on-monday
- trigger: true
- - get: bosh-stemcells-ci
- - get: bosh-integration-registry-image
- - #@ cleanup_unpublished_light_stemcells("us", "us")
- - #@ cleanup_unpublished_light_stemcells("us-gov", "us-gov")
-#!- #@ cleanup_unpublished_light_stemcells("cn", "cn_north")
-
-- name: test-aws-unit
- plan:
- - get: bosh-stemcells-ci
- - get: bosh-integration-registry-image
- - get: builder-src
- resource: light-aws-builder-src
- trigger: true
- - file: bosh-stemcells-ci/ci/tasks/light-aws/test-unit.yml
- image: bosh-integration-registry-image
- task: test
- serial: true
-
-- name: test-aws-drivers
- plan:
- - get: bosh-stemcells-ci
- - get: bosh-integration-registry-image
- - get: builder-src
- resource: light-aws-builder-src
- trigger: true
- - file: bosh-stemcells-ci/ci/tasks/light-aws/test-drivers.yml
- image: bosh-integration-registry-image
- params:
- AWS_PAGER:
- aws_account_id: ((aws_test_account_id))
- access_key: ((aws_test_access_key))
- secret_key: ((aws_test_secret_key))
- bucket_name: ((aws_test_bucket_name))
- copy_region: ((aws_test_copy_region))
- region: ((aws_test_region))
- ami_fixture_id: ((aws_test_ami_fixture_id))
- private_ami_fixture_id: ((aws_test_private_ami_fixture_id))
- existing_snapshot_id: ((aws_test_snapshot_fixture_id))
- existing_volume_id: ((aws_test_volume_fixture_id))
- #! kms key id should be the one created in the region of "copy_region"
- kms_key_id: ((aws_test_kms_key_id))
- kms_multi_region_key: ((aws_test_kms_multi_region_key_id))
- kms_multi_region_key_replication_test: ((aws_test_kms_multi_region_replication_test_key_id))
- uploaded_machine_image_url: https://stemcell-test-publish.s3.eu-central-1.amazonaws.com/fixtures/root.img
- task: test
- attempts: 3
- serial: true
-
-- name: test-aws-integration
- plan:
- - get: bosh-stemcells-ci
- - get: bosh-integration-registry-image
- - get: builder-src
- resource: light-aws-builder-src
- trigger: true
- - file: bosh-stemcells-ci/ci/tasks/light-aws/test-integration.yml
- image: bosh-integration-registry-image
- params:
- AWS_PAGER:
- access_key: ((aws_test_access_key))
- secret_key: ((aws_test_secret_key))
- bucket_name: ((aws_test_bucket_name))
- #! cn_access_key: ((test__cn_access_key))
- #! cn_bucket_name: ((test__cn_bucket_name))
- #! cn_region: ((test__cn_region))
- #! cn_secret_key: ((test__cn_secret_key))
- copy_region: ((aws_test_copy_region))
- region: ((aws_test_region))
-
- task: test
- serial: true
-
-- name: cleanup-light-aws-builder-test-amis
- plan:
- - get: every-week-on-monday
- trigger: true
- - get: bosh-stemcells-ci
- - get: bosh-integration-registry-image
- - task: cleanup-aws-test-amis
- file: bosh-stemcells-ci/ci/tasks/light-aws/cleanup-ami.yml
- image: bosh-integration-registry-image
- params:
- AWS_PAGER:
- ami_access_key: ((aws_test_access_key))
- ami_secret_key: ((aws_test_secret_key))
- ami_region: ((aws_test_region))
- ami_older_than_days: 1
- ami_keep_latest: 0
- snapshot_id: ((aws_test_snapshot_fixture_id))
-
-- name: build-light-aws-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @)
- plan:
- - get: bosh-stemcells-ci
- - get: light-stemcell-builder-registry-image
- - get: bosh-integration-registry-image
- - in_parallel:
- - do:
- - #@ build_light_aws_stemcell_new(data.values.stemcell_details.os_name, str(data.values.stemcell_details.major_version), "us-gov-builder-src", "us-gov-input-stemcell", "us-gov-light-stemcell", "us-gov", "us-gov", "us-gov", "", "", data.values.stemcell_details.use_efi, "")
- - #@ build_light_aws_stemcell_new(data.values.stemcell_details.os_name, str(data.values.stemcell_details.major_version), "us-builder-src", "us-input-stemcell", "us-light-stemcell", "us", "us", "us", "", "", data.values.stemcell_details.use_efi, '["me-central-1"]')
- - file: bosh-stemcells-ci/ci/tasks/light-aws/us-gov-merge-builds.yml
- image: light-stemcell-builder-registry-image
- input_mapping:
- builder-src: us-builder-src
- task: merge-builds
- - do:
- - in_parallel:
- - get: bosh-cpi-src
- resource: bosh-aws-cpi-release
- trigger: false
- - get: cpi-release
- resource: bosh-aws-cpi-dev-artifacts
- trigger: false
- - get: bosh-release
- trigger: false
- - get: bosh-deployment
- trigger: false
- - get: pipelines
- resource: bosh-cpi-certification-concourse-tasks
- - get: bosh-cpi-certification-concourse-tasks
- - put: environment
- resource: light-aws-environment
- params:
- delete_on_failure: true
- generate_random_name: true
- terraform_source: bosh-cpi-src/ci/assets/terraform
- - task: prepare-director
- file: bosh-cpi-certification-concourse-tasks/shared/tasks/prepare-director.yml
- image: bosh-integration-registry-image
- input_mapping:
- stemcell: light-stemcell
- params:
- DIRECTOR_VARS_FILE: ((aws_test_director_vars_file))
- INFRASTRUCTURE: aws
- OPTIONAL_OPS_FILE: |
- -o pipelines/shared/assets/ops/remove-hm.yml
- -o bosh-deployment/external-ip-with-registry-not-recommended.yml
- -o bosh-deployment/misc/source-releases/bosh.yml
- - task: deploy-director
- file: bosh-cpi-certification-concourse-tasks/shared/tasks/deploy-director.yml
- image: bosh-integration-registry-image
- input_mapping:
- stemcell: light-stemcell
- - task: run-stemcell-upload-tests
- file: bosh-stemcells-ci/ci/tasks/light-aws/run-upload-test.yml
- image: bosh-integration-registry-image
- input_mapping:
- builder-src: us-builder-src
- stemcell: light-stemcell
- ensure:
- do:
- - task: teardown
- file: bosh-cpi-certification-concourse-tasks/shared/tasks/teardown.yml
- image: bosh-integration-registry-image
- - task: ensure-terminated
- file: bosh-cpi-src/ci/tasks/ensure-terminated.yml
- image: bosh-integration-registry-image
- input_mapping:
- bosh-aws-cpi-release: bosh-cpi-src
- params:
- AWS_PAGER:
- AWS_ACCESS_KEY_ID: ((aws_publish_us_access_key))
- AWS_SECRET_ACCESS_KEY: ((aws_publish_us_secret_key))
- AWS_DEFAULT_REGION: ((aws_publish_us_region))
- - put: environment
- resource: light-aws-environment
- get_params:
- action: destroy
- params:
- action: destroy
- env_name_file: environment/name
- terraform_source: bosh-cpi-src/ci/assets/terraform
- - put: candidate-aws-light-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
- params:
- files:
- - light-stemcell/*.tgz
- options:
- author_email: *ci_bot_email
- author_name: *ci_bot_name
- message: 'candidate (light aws): (@= data.values.stemcell_details.os_name @)/(@= str(data.values.stemcell_details.major_version) @).x'
- version: us-input-stemcell/.resource/version
- serial: true
-
-- name: build-light-google-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @)
- plan:
- - in_parallel:
- - get: stemcell
- params:
- include_files:
- - bosh-stemcell-*-google-kvm-(@= data.values.stemcell_details.os_name @)*.tgz
- resource: candidate-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
- trigger: true
- version: every
- - get: bosh-stemcells-ci
- - get: bosh-cpi-release
- resource: bosh-google-cpi-release
- - get: light-stemcell-builder-registry-image
- - get: gce-cpi-release-registry-image
- - get: bosh-integration-registry-image
- - task: make-raw-from-heavy-stemcell
- file: bosh-stemcells-ci/ci/tasks/light-google/make-raw-from-heavy-stemcell.yml
- image: light-stemcell-builder-registry-image
- params:
- BUCKET_NAME: bosh-gce-raw-stemcells-new
- STEMCELL_BUCKET_PATH: bosh-gce-light-stemcells
- S3_API_ENDPOINT: storage.googleapis.com
- - params:
- file: raw-stemcell/bosh-stemcell-*-google-kvm-ubuntu-*.tar.gz
- predefined_acl: publicRead
- put: base-oss-google-ubuntu-stemcell
- - task: create-public-image
- file: bosh-stemcells-ci/ci/tasks/light-google/create-public-image.yml
- image: gce-cpi-release-registry-image
- params:
- GCP_SERVICE_ACCOUNT_KEY: ((gcp_json_key))
- PROJECT_NAME: ((gcp_project_id))
- EFI: #@ data.values.stemcell_details.use_efi
- - do:
- - put: terraform
- resource: light-google-environment-oss
- params:
- generate_random_name: true
- terraform_source: bosh-stemcells-ci/ci/tasks/light-google/terraform/
- - task: deploy-skeletal
- file: bosh-stemcells-ci/ci/tasks/light-google/deploy-skeletal.yml
- image: bosh-integration-registry-image
- params:
- GCE_CREDENTIALS_JSON: ((gcp_json_key))
- SSH_PRIVATE_KEY: ((ssh.private_key))
- ensure:
- ensure:
- get_params:
- action: destroy
- params:
- action: destroy
- env_name_file: terraform/name
- terraform_source: bosh-stemcells-ci/ci/tasks/light-google/terraform/
- put: terraform
- resource: light-google-environment-oss
- file: bosh-stemcells-ci/ci/tasks/light-google/destroy-skeletal.yml
- image: bosh-integration-registry-image
- task: destroy-skeletal
- on_failure:
- image: bosh-integration-registry-image
- config:
- platform: linux
- run:
- args:
- - |
- * remove artifacts of failed jobs:
- ```bash
- gcloud auth revoke --all # start with clean slate
- gcloud auth activate-service-account concourse@((gcp_project_id)).iam.gserviceaccount.com --key-file <( ((gcp_json_key)) )
- gsutil rm gs://bosh-gce-raw-stemcells-new/STEMCELL_TO_BE_DELETED_raw.tar.gz
- gsutil rm gs://bosh-gce-raw-stemcells-new/STEMCELL_TO_BE_DELETED_raw.tar.gz.sha1
- gcloud auth revoke --all
- ```
- * re-fly pipeline after commenting-out `disable_manual_trigger: true`
- * trigger failed jobs
- * re-fly pipeline after uncommenting `disable_manual_trigger: true`
- path: echo
- task: cleanup-failed-run-instructions
- - params:
- files:
- - light-stemcell/*.tgz
- options:
- author_email: *ci_bot_email
- author_name: *ci_bot_name
- message: 'candidate (light google): (@= data.values.stemcell_details.os_name @)/(@= str(data.values.stemcell_details.major_version) @).x'
- version: stemcell/.resource/version
- put: candidate-google-light-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
- serial: true
-
-- name: publish-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @)
- plan:
- - in_parallel:
- - get: bosh-stemcells-ci
- - get: bosh-linux-stemcell-builder
- resource: bosh-linux-stemcell-builder-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @).x
- - get: bosh-integration-registry-image
- - get: os-image-stemcell-builder-registry-image
- - get: bosh-ecosystem-concourse-registry-image
- - get: stemcell-metalink
- params:
- skip_download: true
- passed:
- - build-light-aws-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @)
- - build-light-google-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @)
- resource: candidate-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
- - get: candidate-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
- passed:
- - build-light-aws-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @)
- - build-light-google-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @)
- #@ if len(data.values.stemcell_details.include_fips_iaas) > 0:
- - get: candidate-(@= data.values.stemcell_details.os_name @)-fips-stemcell-(@= str(data.values.stemcell_details.major_version) @)
- #@ end
- - get: candidate-google-light-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
- passed:
- - build-light-google-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @)
- - get: candidate-aws-light-stemcell
- resource: candidate-aws-light-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
- passed:
- - build-light-aws-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @)
- - file: bosh-stemcells-ci/ci/tasks/build-release-metadata.yml
- image: os-image-stemcell-builder-registry-image
- task: build-release-metadata
- params:
- OS_NAME: ubuntu
- OS_VERSION: (@= str(data.values.stemcell_details.os_short_name) @)
- BRANCH: (@= data.values.stemcell_details.branch @)
- KERNEL_PACKAGE: linux-generic
- input_mapping:
- candidate-stemcell: candidate-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
- - task: extract-warden-image
- image: bosh-integration-registry-image
- config:
- inputs:
- - name: candidate-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
- outputs:
- - name: stemcell-image
- platform: linux
- run:
- dir: stemcell-image
- path: /bin/bash
- args:
- - -ce
- - |
- tar xvf ../candidate-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)/bosh-stemcell-*-warden-boshlite-(@= data.values.stemcell_details.os_name @)*.tgz image
- printf 'FROM scratch\nADD image /\n' > Dockerfile
- - task: build-stemcell-oci-image
- privileged: true
- config:
- platform: linux
- image_resource:
- type: registry-image
- source:
- repository: concourse/oci-build-task
- inputs:
- - name: stemcell-image
- outputs:
- - name: image
- params:
- CONTEXT: stemcell-image
- run:
- path: build
- - put: github-container-registry-(@= data.values.stemcell_details.os_name @)-stemcell
- params:
- image: image/image.tar
- additional_tags: candidate-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)/.resource/version
- - put: published-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
- params:
- files:
- - candidate-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)/*.tgz
- options:
- author_email: *ci_bot_email
- author_name: *ci_bot_name
- message: 'publish (heavy): (@= data.values.stemcell_details.os_name @)/(@= str(data.values.stemcell_details.major_version) @).x'
- rename: '{{.Version}}/stemcells.meta4'
- version: candidate-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)/.resource/version
- #@ if len(data.values.stemcell_details.include_fips_iaas) > 0 :
- - put: published-(@= data.values.stemcell_details.os_name @)-fips-stemcell-(@= str(data.values.stemcell_details.major_version) @)
- params:
- files:
- - candidate-(@= data.values.stemcell_details.os_name @)-fips-stemcell-(@= str(data.values.stemcell_details.major_version) @)/*.tgz
- options:
- author_email: *ci_bot_email
- author_name: *ci_bot_name
- message: 'publish (heavy): (@= data.values.stemcell_details.os_name @)-fips/(@= str(data.values.stemcell_details.major_version) @).x'
- rename: '{{.Version}}/stemcells.meta4'
- version: candidate-(@= data.values.stemcell_details.os_name @)-fips-stemcell-(@= str(data.values.stemcell_details.major_version) @)/.resource/version
- #@ end
- - params:
- files:
- - candidate-aws-light-stemcell/*.tgz
- options:
- author_email: *ci_bot_email
- author_name: *ci_bot_name
- message: 'publish (light aws): (@= data.values.stemcell_details.os_name @)/(@= str(data.values.stemcell_details.major_version) @).x'
- rename: '{{.Version}}/stemcells.aws.meta4'
- version: candidate-aws-light-stemcell/.resource/version
- put: published-aws-light-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
-
- #! once we release all regions with the same account, we can unify these again
- - file: bosh-stemcells-ci/ci/tasks/light-aws/tag-aws-ami-light.yml
- image: bosh-ecosystem-concourse-registry-image
- task: tag-published-aws-ami-light-stemcells
- params:
- AWS_PAGER:
- AWS_ACCESS_KEY_ID: ((aws_publish_us_access_key))
- AWS_SECRET_ACCESS_KEY: ((aws_publish_us_secret_key))
- GREP_PATTERN: grep -v 'gov-\|cn-'
- - file: bosh-stemcells-ci/ci/tasks/light-aws/tag-aws-ami-light.yml
- image: bosh-ecosystem-concourse-registry-image
- task: tag-gov-published-aws-ami-light-stemcells
- params:
- AWS_PAGER:
- AWS_ACCESS_KEY_ID: ((aws_publish_us-gov_access_key))
- AWS_SECRET_ACCESS_KEY: ((aws_publish_us-gov_secret_key))
- GREP_PATTERN: grep 'gov-'
-#! - file: bosh-stemcells-ci/ci/tasks/light-aws/tag-aws-ami-light.yml
-#! task: tag-china-published-aws-ami-light-stemcells
-#! params:
-#! AWS_ACCESS_KEY_ID: ((aws_publish_cn_access_key))
-#! AWS_SECRET_ACCESS_KEY: ((aws_publish_cn_secret_key))
-#! GREP_PATTERN: grep 'cn-'
- - params:
- files:
- - candidate-google-light-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)/*.tgz
- options:
- author_email: *ci_bot_email
- author_name: *ci_bot_name
- message: 'publish (light google): (@= data.values.stemcell_details.os_name @)/(@= str(data.values.stemcell_details.major_version) @).x'
- rename: '{{.Version}}/stemcells.gcp.meta4'
- version: candidate-google-light-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)/.resource/version
- put: published-google-light-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
- - params:
- body: release-metadata/body
- name: release-metadata/name
- tag: release-metadata/tag
- put: gh-release-oss
- - params:
- acl: public-read
- file: usn-log/usn-log.json
- put: usn-log-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @).x
- serial: true
-
-resource_types:
-- name: metalink-repository
- source:
- repository: dpb587/metalink-repository-resource
- tag: latest
- type: registry-image
-- name: terraform_type
- source:
- repository: ljfranklin/terraform-resource
- type: registry-image
-- name: gcs-resource
- source:
- repository: frodenas/gcs-resource
- type: registry-image
-
-resources:
-- name: every-week-on-monday
- type: time
- source:
- days:
- - Monday
- interval: 168h
- location: America/Los_Angeles
- start: "6:00"
- stop: "8:30"
-
-- name: gh-release-oss
- type: github-release
- source:
- access_token: ((github_public_repo_token))
- drafts: true
- owner: cloudfoundry
- repository: bosh-linux-stemcell-builder
-
-- name: bosh-cpi-certification-concourse-tasks
- type: git
- source:
- branch: master
- uri: https://github.com/cloudfoundry/bosh-cpi-certification
-
-- name: bosh-release
- type: bosh-io-release
- source:
- repository: cloudfoundry/bosh
-
-- name: bosh-deployment
- type: git
- source:
- branch: master
- uri: https://github.com/cloudfoundry/bosh-deployment
-
-- name: bosh-stemcells-ci
- type: git
- source:
- branch: (@= data.values.stemcell_details.branch @)
- paths:
- - ci
- uri: https://github.com/cloudfoundry/bosh-linux-stemcell-builder.git
-
-- name: light-aws-builder-src
- type: git
- source:
- branch: master
- uri: https://github.com/cloudfoundry/bosh-aws-light-stemcell-builder
-
-- name: bosh-aws-cpi-release
- type: git
- source:
- branch: master
- ignore_paths:
- - .final_builds/**/*.yml
- - releases/**/*.yml
- uri: https://github.com/cloudfoundry/bosh-aws-cpi-release.git
-
-- name: light-aws-environment
- source:
- backend_type: s3
- backend_config:
- access_key: ((aws_publish_us_access_key))
- secret_key: ((aws_publish_us_secret_key))
- bucket: bosh-aws-light-terraform
- key: state/terraform.tfstate
- region: ((aws_publish_us_region))
- vars:
- access_key: ((aws_publish_us_access_key))
- secret_key: ((aws_publish_us_secret_key))
- public_key: ((aws_test_public_key_file))
- region: ((aws_publish_us_region))
- type: terraform_type
-
-- name: bosh-aws-cpi-dev-artifacts
- type: gcs-resource
- source:
- versioned_file: bosh-aws-cpi-dev-release.tgz
- bucket: bosh-aws-cpi-pipeline
- json_key: ((gcp_json_key))
-
-- name: bosh-google-cpi-release
- source:
- repository: cloudfoundry/bosh-google-cpi-release
- type: bosh-io-release
-- name: light-google-environment-oss
- source:
- delete_on_failure: true
- backend_type: gcs
- backend_config:
- credentials: ((gcp_json_key))
- bucket: bosh-gce-light-stemcell-ci-terraform-state
- prefix: stemcell-ci-terraform/
- vars:
- gce_credentials_json: ((gcp_json_key))
- gce_project_id: ((gcp_project_id))
- type: terraform_type
-- name: base-oss-google-ubuntu-stemcell
- source:
- bucket: bosh-gce-raw-stemcells-new
- json_key: ((gcp_json_key))
- regexp: bosh-stemcell-([0-9\.]+)-google-kvm-ubuntu-*-raw.tar.gz
- type: gcs-resource
-
-- name: candidate-aws-light-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
- source:
- mirror_files:
- - destination: s3://storage.googleapis.com/bosh-aws-light-stemcells-candidate/{{.Version}}/{{.Name}}
- options:
- private_key: ((github_deploy_key_bosh-io-stemcells-cpi-index.private_key))
- uri: git+ssh://git@github.com:cloudfoundry/bosh-io-stemcells-cpi-index.git//candidate-aws-light/(@= data.values.stemcell_details.os_name @)
- url_handlers:
- - include:
- - (s3|https)://.*
- options:
- access_key: ((hmac_accesskey))
- secret_key: ((hmac_secret))
- type: s3
- version: (@= str(data.values.stemcell_details.major_version) @).x
- type: metalink-repository
-- name: candidate-google-light-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
- source:
- mirror_files:
- - destination: s3://storage.googleapis.com/bosh-gce-light-stemcells-candidate/{{.Version}}/{{.Name}}
- options:
- private_key: ((github_deploy_key_bosh-io-stemcells-cpi-index.private_key))
- uri: git+ssh://git@github.com:cloudfoundry/bosh-io-stemcells-cpi-index.git//candidate-gcp-light/(@= data.values.stemcell_details.os_name @)
- url_handlers:
- - include:
- - (s3|https)://.*
- options:
- access_key: ((hmac_accesskey))
- secret_key: ((hmac_secret))
- type: s3
- version: (@= str(data.values.stemcell_details.major_version) @).x
- type: metalink-repository
-- name: candidate-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
- source:
- filters:
- - repositorypath: '*/stemcells.meta4'
- options:
- private_key: ((github_deploy_key_bosh-io-stemcells-core-index.private_key))
- uri: git+ssh://git@github.com:cloudfoundry/bosh-io-stemcells-core-index.git//candidate/(@= data.values.stemcell_details.os_name @)
- url_handlers:
- - include:
- - (s3|https)://.*
- type: s3
- version: (@= str(data.values.stemcell_details.major_version) @).x
- type: metalink-repository
-#@ if len(data.values.stemcell_details.include_fips_iaas) > 0:
-- name: candidate-(@= data.values.stemcell_details.os_name @)-fips-stemcell-(@= str(data.values.stemcell_details.major_version) @)
- source:
- filters:
- - repositorypath: '*/stemcells.meta4'
- options:
- private_key: ((github_deploy_key_bosh-io-stemcells-core-index.private_key))
- uri: git+ssh://git@github.com:cloudfoundry/bosh-io-stemcells-core-index.git//candidate/(@= data.values.stemcell_details.os_name @)-fips
- url_handlers:
- - include:
- - (s3|https)://.*
- type: s3
- options:
- access_key: ((hmac_accesskey))
- secret_key: ((hmac_secret))
- version: (@= str(data.values.stemcell_details.major_version) @).x
- type: metalink-repository
-#@ end
-- name: bosh-linux-stemcell-builder-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @).x
- source:
- branch: (@= data.values.stemcell_details.branch @)
- private_key: ((bosh_src_key.private_key))
- uri: git@github.com:cloudfoundry/bosh-linux-stemcell-builder
- type: git
-
-- name: usn-log-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @).x
- type: gcs-resource
- source:
- bucket: bosh-stemcell-triggers
- json_key: ((gcp_json_key))
- versioned_file: (@= data.values.stemcell_details.branch @)/usn-log.json
-
-- name: published-aws-light-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
- source:
- filters:
- - repositorypath: '*/stemcells.aws.meta4'
- mirror_files:
- - destination: s3://storage.googleapis.com/bosh-aws-light-stemcells/{{.Version}}/{{.Name}}
- options:
- private_key: ((github_deploy_key_bosh-io-stemcells-cpi-index.private_key))
- uri: git+ssh://git@github.com:cloudfoundry/bosh-io-stemcells-cpi-index.git//published/(@= data.values.stemcell_details.os_name @)
- url_handlers:
- - include:
- - (s3|https)://.*
- options:
- access_key: ((hmac_accesskey))
- secret_key: ((hmac_secret))
- type: s3
- version: (@= str(data.values.stemcell_details.major_version) @).x
- type: metalink-repository
-
-- name: published-google-light-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
- source:
- filters:
- - repositorypath: '*/stemcells.gcp.meta4'
- mirror_files:
- - destination: s3://storage.googleapis.com/bosh-gce-light-stemcells/{{.Version}}/{{.Name}}
- options:
- private_key: ((github_deploy_key_bosh-io-stemcells-cpi-index.private_key))
- uri: git+ssh://git@github.com:cloudfoundry/bosh-io-stemcells-cpi-index.git//published/(@= data.values.stemcell_details.os_name @)
- url_handlers:
- - include:
- - (s3|https)://.*
- options:
- access_key: ((hmac_accesskey))
- secret_key: ((hmac_secret))
- type: s3
- version: (@= str(data.values.stemcell_details.major_version) @).x
- type: metalink-repository
-
-- name: published-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
- source:
- filters:
- - repositorypath: '*/stemcells.meta4'
- mirror_files:
- - destination: s3://storage.googleapis.com/bosh-core-stemcells/{{.Version}}/{{.Name}}
- options:
- private_key: ((github_deploy_key_bosh-io-stemcells-core-index.private_key))
- uri: git+ssh://git@github.com:cloudfoundry/bosh-io-stemcells-core-index.git//published/(@= data.values.stemcell_details.os_name @)
- url_handlers:
- - include:
- - (s3|https)://.*
- options:
- access_key: ((hmac_accesskey))
- secret_key: ((hmac_secret))
- type: s3
- version: (@= str(data.values.stemcell_details.major_version) @).x
- type: metalink-repository
- #@ if len(data.values.stemcell_details.include_fips_iaas) > 0:
-- name: published-(@= data.values.stemcell_details.os_name @)-fips-stemcell-(@= str(data.values.stemcell_details.major_version) @)
- source:
- filters:
- - repositorypath: '*/stemcells.meta4'
- mirror_files:
- - destination: s3://storage.googleapis.com/bosh-core-stemcells-fips/{{.Version}}/{{.Name}}
- options:
- private_key: ((github_deploy_key_bosh-io-stemcells-core-index.private_key))
- uri: git+ssh://git@github.com:cloudfoundry/bosh-io-stemcells-core-index.git//published/(@= data.values.stemcell_details.os_name @)-fips
- url_handlers:
- - include:
- - (s3|https)://.*
- options:
- access_key: ((hmac_accesskey))
- secret_key: ((hmac_secret))
- type: s3
- version: (@= str(data.values.stemcell_details.major_version) @).x
- type: metalink-repository
- #@ end
-
-- name: github-container-registry-(@= data.values.stemcell_details.os_name @)-stemcell
- type: registry-image
- source:
- repository: ghcr.io/cloudfoundry/(@= data.values.stemcell_details.os_name @)-stemcell
- username: ((github_read_write_packages.username))
- password: ((github_read_write_packages.password))
- tag: latest
-
-- name: bosh-integration-registry-image
- type: registry-image
- source:
- repository: ghcr.io/cloudfoundry/bosh/integration
- tag: main
- username: ((github_read_write_packages.username))
- password: ((github_read_write_packages.password))
-
-- name: light-stemcell-builder-registry-image
- type: registry-image
- source:
- repository: bosh/light-stemcell-builder
- username: ((dockerhub_username))
- password: ((dockerhub_password))
-
-- name: os-image-stemcell-builder-registry-image
- type: registry-image
- source:
- repository: bosh/os-image-stemcell-builder
- username: ((dockerhub_username))
- password: ((dockerhub_password))
-
-- name: bosh-ecosystem-concourse-registry-image
- type: registry-image
- source:
- repository: bosh/bosh-ecosystem-concourse
- username: ((dockerhub_username))
- password: ((dockerhub_password))
-
-- name: gce-cpi-release-registry-image
- type: registry-image
- source:
- repository: foundationalinfrastructure/gce-cpi-release
diff --git a/ci/tasks/light-aws/build.sh b/ci/tasks/light-aws/build.sh
index aff06ce410..7383e3e06f 100755
--- a/ci/tasks/light-aws/build.sh
+++ b/ci/tasks/light-aws/build.sh
@@ -14,20 +14,20 @@ ami_kms_key_id=${ami_kms_key_id:-}
ami_server_side_encryption=${ami_server_side_encryption:-}
ami_excluded_destinations=${ami_excluded_destinations:-}
-: ${bosh_io_bucket_name:?}
-: ${ami_description:?}
-: ${ami_virtualization_type:?}
-: ${ami_visibility:?}
-: ${ami_region:?}
-: ${ami_access_key:?}
-: ${ami_secret_key:?}
-: ${ami_bucket_name:?}
-: ${ami_encrypted:?}
-: ${efi:?}
-
-export AWS_ACCESS_KEY_ID=$ami_access_key
-export AWS_SECRET_ACCESS_KEY=$ami_secret_key
-export AWS_DEFAULT_REGION=$ami_region
+: "${bosh_io_bucket_name:?}"
+: "${ami_description:?}"
+: "${ami_virtualization_type:?}"
+: "${ami_visibility:?}"
+: "${ami_region:?}"
+: "${ami_access_key:?}"
+: "${ami_secret_key:?}"
+: "${ami_bucket_name:?}"
+: "${ami_encrypted:?}"
+: "${efi:?}"
+
+export AWS_ACCESS_KEY_ID=${ami_access_key}
+export AWS_SECRET_ACCESS_KEY=${ami_secret_key}
+export AWS_DEFAULT_REGION=${ami_region}
saved_ami_destinations="$( aws ec2 describe-regions \
--query "Regions[?RegionName != '${ami_region}'][].RegionName" \
@@ -38,7 +38,7 @@ if [[ -n "${ami_excluded_destinations}" ]]; then
| jq --argjson exclude "$ami_excluded_destinations" '. - $exclude' -c )"
fi
-: ${ami_destinations:=$saved_ami_destinations}
+: "${ami_destinations:=$saved_ami_destinations}"
stemcell_path=$(ls "${REPO_PARENT}"/input-stemcell/*.tgz)
version=$(cat "${REPO_PARENT}/input-stemcell/.resource/version")
@@ -55,14 +55,11 @@ if [ "${ami_virtualization_type}" = "hvm" ]; then
fi
bosh_io_light_stemcell_url="https://$S3_API_ENDPOINT/$bosh_io_bucket_name/$version/$light_stemcell_name"
-set +e
-wget --spider "$bosh_io_light_stemcell_url"
-if [[ "$?" == "0" ]]; then
+if wget --spider "$bosh_io_light_stemcell_url"; then
echo "AWS light stemcell '$light_stemcell_name' already exists!"
echo "You can download here: $bosh_io_light_stemcell_url"
exit 1
fi
-set -e
echo "Building light stemcell..."
echo " Starting region: ${ami_region}"
@@ -70,40 +67,40 @@ echo " Copy regions: ${ami_destinations}"
export CONFIG_PATH="${REPO_PARENT}/config.json"
-cat > $CONFIG_PATH << EOF
+cat > "${CONFIG_PATH}" << EOF
{
"ami_configuration": {
- "description": "$ami_description",
- "virtualization_type": "$ami_virtualization_type",
- "encrypted": $ami_encrypted,
- "kms_key_id": "$ami_kms_key_id",
- "visibility": "$ami_visibility",
+ "description": "${ami_description}",
+ "virtualization_type": "${ami_virtualization_type}",
+ "encrypted": ${ami_encrypted},
+ "kms_key_id": "${ami_kms_key_id}",
+ "visibility": "${ami_visibility}",
"efi": ${efi}
},
"ami_regions": [
{
- "name": "$ami_region",
+ "name": "${ami_region}",
"credentials": {
- "access_key": "$ami_access_key",
- "secret_key": "$ami_secret_key"
+ "access_key": "${ami_access_key}",
+ "secret_key": "${ami_secret_key}"
},
- "bucket_name": "$ami_bucket_name",
- "server_side_encryption": "$ami_server_side_encryption",
- "destinations": $ami_destinations
+ "bucket_name": "${ami_bucket_name}",
+ "server_side_encryption": "${ami_server_side_encryption}",
+ "destinations": ${ami_destinations}
}
]
}
EOF
extracted_stemcell_dir="${REPO_PARENT}/extracted-stemcell"
-mkdir -p ${extracted_stemcell_dir}
-tar -C ${extracted_stemcell_dir} -xf ${stemcell_path}
-tar -xf ${extracted_stemcell_dir}/image
+mkdir -p "${extracted_stemcell_dir}"
+tar -C "${extracted_stemcell_dir}" -xf "${stemcell_path}"
+tar -xf "${extracted_stemcell_dir}"/image
# image format can be raw or stream optimized vmdk
stemcell_image="$(echo "${REPO_PARENT}"/root.*)"
stemcell_manifest=${extracted_stemcell_dir}/stemcell.MF
-manifest_contents="$(cat ${stemcell_manifest})"
+manifest_contents="$(cat "${stemcell_manifest}")"
disk_regex="disk: ([0-9]+)"
format_regex="disk_format: ([a-z]+)"
@@ -120,23 +117,22 @@ disk_size_gb=$(mb_to_gb "${BASH_REMATCH[1]}")
[[ "${manifest_contents}" =~ ${format_regex} ]]
disk_format="${BASH_REMATCH[1]}"
-pushd "${REPO_PARENT}/builder-src" > /dev/null
- # Make sure we've closed the manifest file before writing to it
- go run main.go \
- -c $CONFIG_PATH \
- --image ${stemcell_image} \
- --format ${disk_format} \
- --volume-size ${disk_size_gb} \
- --manifest ${stemcell_manifest} \
- | tee tmp-manifest
+# Make sure we've closed the manifest file before writing to it
+# see https://github.com/cloudfoundry/bosh-aws-light-stemcell-builder/blob/master/ci/docker/Dockerfile#L30
+light-stemcell-builder \
+ -c "${CONFIG_PATH}" \
+ --image "${stemcell_image}" \
+ --format "${disk_format}" \
+ --volume-size "${disk_size_gb}" \
+ --manifest "${stemcell_manifest}" \
+ | tee tmp-manifest
- mv tmp-manifest ${stemcell_manifest}
+mv tmp-manifest "${stemcell_manifest}"
-popd
-
-pushd ${extracted_stemcell_dir}
- > image
+pushd "${extracted_stemcell_dir}"
+ : > image
# the bosh cli sees the stemcell as invalid if tar contents have leading ./
+ # shellcheck disable=SC2035
tar -czf "${REPO_PARENT}/light-stemcell/${light_stemcell_name}" *
popd
diff --git a/ci/tasks/light-aws/build.yml b/ci/tasks/light-aws/build.yml
index 5d553e7a9b..19e321c918 100644
--- a/ci/tasks/light-aws/build.yml
+++ b/ci/tasks/light-aws/build.yml
@@ -2,7 +2,6 @@
platform: linux
inputs:
-- name: builder-src
- name: bosh-stemcells-ci
- name: input-stemcell
diff --git a/ci/tasks/light-aws/cleanup-ami.sh b/ci/tasks/light-aws/cleanup-ami.sh
index c1a24a3d9e..9d21e2f041 100755
--- a/ci/tasks/light-aws/cleanup-ami.sh
+++ b/ci/tasks/light-aws/cleanup-ami.sh
@@ -6,16 +6,17 @@ REPO_PARENT="$( cd "${REPO_ROOT}/.." && pwd )"
if [[ -n "${DEBUG:-}" ]]; then
set -x
- export BOSH_LOG_LEVEL=debug
- export BOSH_LOG_PATH="${BOSH_LOG_PATH:-${REPO_PARENT}/bosh-debug.log}"
fi
-: ${ami_older_than_days:?}
-: ${ami_keep_latest:?}
+: "${ami_older_than_days:?}"
+: "${ami_keep_latest:?}"
+: "${ami_access_key:?}"
+: "${ami_secret_key:?}"
+: "${ami_region:?}"
-export AWS_ACCESS_KEY_ID=${ami_access_key}
-export AWS_SECRET_ACCESS_KEY=${ami_secret_key}
-export AWS_DEFAULT_REGION=${ami_region}
+export AWS_ACCESS_KEY_ID="${ami_access_key}"
+export AWS_SECRET_ACCESS_KEY="${ami_secret_key}"
+export AWS_DEFAULT_REGION="${ami_region}"
if [ -n "${ami_role_arn:-}" ]; then
export AWS_ROLE_ARN=${ami_role_arn}
@@ -29,66 +30,67 @@ if [ -n "${ami_role_arn:-}" ]; then
export AWS_PROFILE=resource_account
fi
-__PASTDUE=$(date --date="$ami_older_than_days days ago" +"%Y-%m-%d")
-
+past_due=$(date --date="${ami_older_than_days} days ago" +"%Y-%m-%d")
+# shellcheck disable=SC2016
+past_due_query='sort_by(Images,&CreationDate)[?CreationDate<`'"${past_due}"'`].{ImageId: ImageId, date:CreationDate, SnapshotId: BlockDeviceMappings[0].Ebs.SnapshotId,Version: Tags[?Key==`name`]|[0].Value}'
ami_destinations="$(aws ec2 describe-regions --output text --query "Regions[?RegionName][].RegionName")"
-for region in $ami_destinations; do
- ami_list="[]"
+for region in ${ami_destinations}; do
+ ami_list="[]"
- if [ "${remove_public_images:-}" == "true" ]; then
- results=$(aws ec2 describe-images \
- --owners self \
- --output json \
- --region ${region} \
- --filters "Name=name,Values=BOSH*" "Name=is-public,Values=true" \
- --query 'sort_by(Images,&CreationDate)[?CreationDate<`'"$__PASTDUE"'`].{ImageId: ImageId, date:CreationDate, SnapshotId: BlockDeviceMappings[0].Ebs.SnapshotId,Version: Tags[?Key==`name`]|[0].Value}')
- ami_list=$(jq -s '.[0] + .[1]' <(echo "${ami_list}") <(echo "${results}"))
- fi
+ if [ "${remove_public_images:-}" == "true" ]; then
+ results=$(aws ec2 describe-images \
+ --owners self \
+ --output json \
+ --region "${region}" \
+ --filters "Name=name,Values=BOSH*" "Name=is-public,Values=true" \
+ --query "${past_due_query}")
+ ami_list=$(jq -s '.[0] + .[1]' <(echo "${ami_list}") <(echo "${results}"))
+ fi
- if [ -n "${os_name:-}" ]; then
- # 'ami_ids' array should be orderered by creation date
- results=$(aws ec2 describe-images \
- --owners self \
- --output json \
- --region ${region} \
- --filters "Name=name,Values=BOSH*" "Name=tag:published,Values=false" "Name=tag:distro,Values=${os_name}" \
- --query 'sort_by(Images,&CreationDate)[?CreationDate<`'"$__PASTDUE"'`].{ImageId: ImageId, date:CreationDate, SnapshotId: BlockDeviceMappings[0].Ebs.SnapshotId,Version: Tags[?Key==`name`]|[0].Value}' | jq 'reverse | del(.[range(env.ami_keep_latest|tonumber)])')
- ami_list=$(jq -s '.[0] + .[1]' <(echo "${ami_list}") <(echo "${results}"))
- fi
+ if [ -n "${os_name:-}" ]; then
+ # 'ami_ids' array should be ordered by creation date
+ results=$(aws ec2 describe-images \
+ --owners self \
+ --output json \
+ --region "${region}" \
+ --filters "Name=name,Values=BOSH*" "Name=tag:published,Values=false" "Name=tag:distro,Values=${os_name}" \
+ --query "${past_due_query}" | jq 'reverse | del(.[range(env.ami_keep_latest|tonumber)])')
+ ami_list=$(jq -s '.[0] + .[1]' <(echo "${ami_list}") <(echo "${results}"))
+ fi
- if [ -n "${snapshot_id:-}" ]; then
- results=$(aws ec2 describe-images \
- --owners self \
- --output json \
- --region ${region} \
- --filters "Name=block-device-mapping.snapshot-id,Values=${snapshot_id}" \
- --query 'sort_by(Images,&CreationDate)[?CreationDate<`'"$__PASTDUE"'`].{ImageId: ImageId, date:CreationDate, SnapshotId: BlockDeviceMappings[0].Ebs.SnapshotId,Version: Tags[?Key==`name`]|[0].Value}' | jq 'reverse | del(.[range(env.ami_keep_latest|tonumber)])')
- ami_list=$(jq -s '.[0] + .[1]' <(echo "${ami_list}") <(echo "${results}"))
- fi
+ if [ -n "${snapshot_id:-}" ]; then
+ results=$(aws ec2 describe-images \
+ --owners self \
+ --output json \
+ --region "${region}" \
+ --filters "Name=block-device-mapping.snapshot-id,Values=${snapshot_id}" \
+ --query "${past_due_query}" | jq 'reverse | del(.[range(env.ami_keep_latest|tonumber)])')
+ ami_list=$(jq -s '.[0] + .[1]' <(echo "${ami_list}") <(echo "${results}"))
+ fi
- # 'ami_list' is a json array of objects, each object is an ami and its snapshot
- for row in $(echo "${ami_list}" | jq -r '.[] | @base64'); do
- _jq() {
- echo ${row} | base64 --decode | jq -r ${1}
- }
- echo "
- ===============================================
- Cleaning up Ami and its snashots in $region
- Ami id: $(_jq '.ImageId')
- Version: $(_jq '.Version')
- Creation data: $(_jq '.date')
- Snapshot id: $(_jq '.SnapshotId')
- "
+ # 'ami_list' is a json array of objects, each object is an ami and its snapshot
+ for row in $(echo "${ami_list}" | jq -r '.[] | @base64'); do
+ _jq() {
+ echo "${row}" | base64 --decode | jq -r "${1}"
+ }
+ echo "
+ ===============================================
+ Cleaning up Ami and its snashots in ${region}
+ Ami id: $(_jq '.ImageId')
+ Version: $(_jq '.Version')
+ Creation data: $(_jq '.date')
+ Snapshot id: $(_jq '.SnapshotId')
+ "
- aws ec2 deregister-image \
- --image-id $(_jq '.ImageId') \
- --region $region
+ aws ec2 deregister-image \
+ --image-id "$(_jq '.ImageId')" \
+ --region "${region}"
- if [ "${snapshot_id:-}" != "$(_jq '.SnapshotId')" ]; then
- aws ec2 delete-snapshot \
- --snapshot-id $(_jq '.SnapshotId') \
- --region $region
- fi
- done
+ if [ "${snapshot_id:-}" != "$(_jq '.SnapshotId')" ]; then
+ aws ec2 delete-snapshot \
+ --snapshot-id "$(_jq '.SnapshotId')" \
+ --region "${region}"
+ fi
+ done
done
diff --git a/ci/tasks/light-aws/cleanup-ami.yml b/ci/tasks/light-aws/cleanup-ami.yml
index feae8d0c4a..e30236167c 100644
--- a/ci/tasks/light-aws/cleanup-ami.yml
+++ b/ci/tasks/light-aws/cleanup-ami.yml
@@ -1,19 +1,18 @@
---
platform: linux
-
inputs:
- name: bosh-stemcells-ci
run:
path: bosh-stemcells-ci/ci/tasks/light-aws/cleanup-ami.sh
params:
- ami_region: "eu-central-1" # AWS default region
- ami_access_key: ""
- ami_secret_key: ""
- ami_role_arn: ""
- ami_older_than_days: "60" # Number of days AMI to keep excluding those currently being running
- ami_keep_latest: "5" # Number of previous AMI to keep excluding those currently being running
- os_name: "" # e.g ubuntu-jammy
- snapshot_id: "" # Snapshot id to delete
- remove_public_images: "false"
+ ami_region: "eu-central-1" # AWS default region
+ ami_access_key: ""
+ ami_secret_key: ""
+ ami_role_arn: ""
+ ami_older_than_days: "60" # Number of days AMI to keep excluding those currently being running
+ ami_keep_latest: "5" # Number of previous AMI to keep excluding those currently being running
+ os_name: "" # e.g ubuntu-jammy
+ snapshot_id: "" # Snapshot id to delete
+ remove_public_images: "false"
diff --git a/ci/tasks/light-aws/run-upload-test.sh b/ci/tasks/light-aws/run-upload-test.sh
index d9c11e1b32..7826e164ca 100755
--- a/ci/tasks/light-aws/run-upload-test.sh
+++ b/ci/tasks/light-aws/run-upload-test.sh
@@ -13,6 +13,6 @@ fi
source "${REPO_PARENT}/director-state/director.env"
pushd "${REPO_PARENT}/stemcell"
- time bosh -n upload-stemcell *.tgz
+ time bosh -n upload-stemcell ./*.tgz
popd
diff --git a/ci/tasks/light-aws/run-upload-test.yml b/ci/tasks/light-aws/run-upload-test.yml
index 2f56f6ec86..978fb5bd88 100644
--- a/ci/tasks/light-aws/run-upload-test.yml
+++ b/ci/tasks/light-aws/run-upload-test.yml
@@ -8,10 +8,9 @@ inputs:
- name: environment
- name: stemcell
- name: director-state
- - name: builder-src
run:
path: bosh-stemcells-ci/ci/tasks/light-aws/run-upload-test.sh
params:
- BOSH_DEBUG_LEVEL: info
+ BOSH_DEBUG_LEVEL: info
diff --git a/ci/tasks/light-aws/test-drivers.sh b/ci/tasks/light-aws/test-drivers.sh
deleted file mode 100755
index 685f092fb3..0000000000
--- a/ci/tasks/light-aws/test-drivers.sh
+++ /dev/null
@@ -1,63 +0,0 @@
-#!/usr/bin/env bash
-set -eu -o pipefail
-
-REPO_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../.." && pwd )"
-REPO_PARENT="$( cd "${REPO_ROOT}/.." && pwd )"
-
-if [[ -n "${DEBUG:-}" ]]; then
- set -x
- export BOSH_LOG_LEVEL=debug
- export BOSH_LOG_PATH="${BOSH_LOG_PATH:-${REPO_PARENT}/bosh-debug.log}"
-fi
-
-tmp_dir="$(mktemp -d /tmp/stemcell_builder.XXXXXXX)"
-trap '{ rm -rf ${tmp_dir}; }' EXIT
-
-: ${aws_account_id:?must be set}
-: ${access_key:?must be set}
-: ${secret_key:?must be set}
-: ${bucket_name:?must be set}
-: ${region:?must be set}
-: ${copy_region:?must be set}
-: ${ami_fixture_id:?must be set}
-: ${private_ami_fixture_id:?must be set}
-: ${existing_volume_id:?must be set}
-: ${existing_snapshot_id:?must be set}
-: ${uploaded_machine_image_url:?must be set}
-: ${kms_key_id:?must be set}
-: ${kms_multi_region_key:?must be set}
-: ${kms_multi_region_key_replication_test:?must be set}
-
-: ${uploaded_machine_image_format:=RAW}
-
-# US Regions
-export AWS_ACCOUNT=$aws_account_id
-export AWS_ACCESS_KEY_ID=$access_key
-export AWS_SECRET_ACCESS_KEY=$secret_key
-export AWS_BUCKET_NAME=$bucket_name
-export AWS_REGION=$region
-export AWS_DESTINATION_REGION=${copy_region}
-export AWS_KMS_KEY_ID=${kms_key_id}
-export MULTI_REGION_KEY=${kms_multi_region_key}
-export MULTI_REGION_KEY_REPLICATION_TEST=${kms_multi_region_key_replication_test}
-
-# Fixtures
-export S3_MACHINE_IMAGE_URL=${uploaded_machine_image_url}
-export S3_MACHINE_IMAGE_FORMAT=${uploaded_machine_image_format}
-export EBS_VOLUME_ID=${existing_volume_id}
-export EBS_SNAPSHOT_ID=${existing_snapshot_id}
-export AMI_FIXTURE_ID=${ami_fixture_id}
-export PRIVATE_AMI_FIXTURE_ID=${private_ami_fixture_id}
-
-echo "Downloading machine image"
-export MACHINE_IMAGE_PATH=${tmp_dir}/image.iso
-export MACHINE_IMAGE_FORMAT="RAW"
-wget -O ${MACHINE_IMAGE_PATH} http://tinycorelinux.net/7.x/x86_64/archive/7.1/TinyCorePure64-7.1.iso
-
-echo "Running driver tests"
-
-pushd "${REPO_PARENT}/builder-src" > /dev/null
- # Run all driver specs in parallel to reduce test time
- spec_count="$(grep "It(" -r driver | wc -l)"
- go run github.com/onsi/ginkgo/v2/ginkgo -nodes ${spec_count} -r driver
-popd
diff --git a/ci/tasks/light-aws/test-drivers.yml b/ci/tasks/light-aws/test-drivers.yml
deleted file mode 100644
index d6ca5d227d..0000000000
--- a/ci/tasks/light-aws/test-drivers.yml
+++ /dev/null
@@ -1,23 +0,0 @@
----
-platform: linux
-
-inputs:
-- name: builder-src
-- name: bosh-stemcells-ci
-run:
- path: bosh-stemcells-ci/ci/tasks/light-aws/test-drivers.sh
-params:
- aws_account_id: ""
- access_key: ""
- secret_key: ""
- bucket_name: ""
- region: ""
- copy_region: ""
- ami_fixture_id: ""
- private_ami_fixture_id: ""
- kms_key_id: ""
- kms_multi_region_key: ""
- kms_multi_region_key_replication_test: ""
- existing_volume_id: ""
- existing_snapshot_id: ""
- uploaded_machine_image_url: ""
diff --git a/ci/tasks/light-aws/test-integration.sh b/ci/tasks/light-aws/test-integration.sh
deleted file mode 100755
index 47224938b3..0000000000
--- a/ci/tasks/light-aws/test-integration.sh
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/usr/bin/env bash
-set -eu -o pipefail
-
-REPO_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../.." && pwd )"
-REPO_PARENT="$( cd "${REPO_ROOT}/.." && pwd )"
-
-if [[ -n "${DEBUG:-}" ]]; then
- set -x
- export BOSH_LOG_LEVEL=debug
- export BOSH_LOG_PATH="${BOSH_LOG_PATH:-${REPO_PARENT}/bosh-debug.log}"
-fi
-
-tmp_dir="$(mktemp -d /tmp/stemcell_builder.XXXXXXX)"
-trap '{ rm -rf ${tmp_dir}; }' EXIT
-
-: ${access_key:?must be set}
-: ${secret_key:?must be set}
-: ${bucket_name:?must be set}
-: ${region:?must be set}
-: ${copy_region:?must be set}
-# : ${cn_access_key:?must be set}
-# : ${cn_secret_key:?must be set}
-# : ${cn_bucket_name:?must be set}
-# : ${cn_region:?must be set}
-
-# US Regions
-export AWS_ACCESS_KEY_ID=$access_key
-export AWS_SECRET_ACCESS_KEY=$secret_key
-export AWS_BUCKET_NAME=$bucket_name
-export AWS_REGION=$region
-export AWS_DESTINATION_REGION=${copy_region}
-
-# # China Region
-# export AWS_CN_ACCESS_KEY_ID=$cn_access_key
-# export AWS_CN_SECRET_ACCESS_KEY=$cn_secret_key
-# export AWS_CN_BUCKET_NAME=$cn_bucket_name
-# export AWS_CN_REGION=$cn_region
-
-echo "Downloading machine image"
-export MACHINE_IMAGE_PATH=${tmp_dir}/image.iso
-export MACHINE_IMAGE_FORMAT="RAW"
-wget -O ${MACHINE_IMAGE_PATH} http://tinycorelinux.net/7.x/x86_64/archive/7.1/TinyCorePure64-7.1.iso
-
-echo "Running integration tests"
-
-pushd "${REPO_PARENT}/builder-src" > /dev/null
- go run github.com/onsi/ginkgo/v2/ginkgo -v -r integration
-popd
diff --git a/ci/tasks/light-aws/test-integration.yml b/ci/tasks/light-aws/test-integration.yml
deleted file mode 100644
index f92f66431b..0000000000
--- a/ci/tasks/light-aws/test-integration.yml
+++ /dev/null
@@ -1,18 +0,0 @@
----
-platform: linux
-
-inputs:
-- name: builder-src
-- name: bosh-stemcells-ci
-run:
- path: bosh-stemcells-ci/ci/tasks/light-aws/test-integration.sh
-params:
- access_key: ""
- secret_key: ""
- bucket_name: ""
- region: ""
- copy_region: ""
- # cn_access_key: ""
- # cn_secret_key: ""
- # cn_bucket_name: ""
- # cn_region: ""
diff --git a/ci/tasks/light-aws/test-unit.sh b/ci/tasks/light-aws/test-unit.sh
deleted file mode 100755
index f8bbd52a90..0000000000
--- a/ci/tasks/light-aws/test-unit.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/usr/bin/env bash
-set -eu -o pipefail
-
-REPO_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../.." && pwd )"
-REPO_PARENT="$( cd "${REPO_ROOT}/.." && pwd )"
-
-if [[ -n "${DEBUG:-}" ]]; then
- set -x
- export BOSH_LOG_LEVEL=debug
- export BOSH_LOG_PATH="${BOSH_LOG_PATH:-${REPO_PARENT}/bosh-debug.log}"
-fi
-
-echo "Running unit tests"
-
-pushd "${REPO_PARENT}/builder-src" > /dev/null
- go run github.com/onsi/ginkgo/v2/ginkgo -p -r --skip-package "driver,integration"
- go run github.com/onsi/ginkgo/v2/ginkgo -p -r driverset # driverset is skipped by previous command
-popd
diff --git a/ci/tasks/light-aws/test-unit.yml b/ci/tasks/light-aws/test-unit.yml
deleted file mode 100644
index 72017042b4..0000000000
--- a/ci/tasks/light-aws/test-unit.yml
+++ /dev/null
@@ -1,8 +0,0 @@
----
-platform: linux
-
-inputs:
-- name: builder-src
-- name: bosh-stemcells-ci
-run:
- path: bosh-stemcells-ci/ci/tasks/light-aws/test-unit.sh
diff --git a/ci/tasks/light-aws/us-gov-merge-builds.yml b/ci/tasks/light-aws/us-gov-merge-builds.yml
index 68b41ba348..4dfcd106e1 100644
--- a/ci/tasks/light-aws/us-gov-merge-builds.yml
+++ b/ci/tasks/light-aws/us-gov-merge-builds.yml
@@ -3,7 +3,6 @@ platform: linux
inputs:
-- name: builder-src
- name: bosh-stemcells-ci
- name: us-light-stemcell
- name: cn-north-light-stemcell
diff --git a/ci/tasks/light-google/create-public-image.sh b/ci/tasks/light-google/create-public-image.sh
index 81dc90058c..453900d305 100755
--- a/ci/tasks/light-google/create-public-image.sh
+++ b/ci/tasks/light-google/create-public-image.sh
@@ -10,8 +10,8 @@ if [[ -n "${DEBUG:-}" ]]; then
export BOSH_LOG_PATH="${BOSH_LOG_PATH:-${REPO_PARENT}/bosh-debug.log}"
fi
-: ${PROJECT_NAME:?}
-: ${GCP_SERVICE_ACCOUNT_KEY:?}
+: "${PROJECT_NAME:?}"
+: "${GCP_SERVICE_ACCOUNT_KEY:?}"
echo "Creating light stemcell..."
@@ -25,10 +25,12 @@ raw_stemcell_filename="$(basename "${raw_stemcell}")"
raw_stemcell_uri="$(cat "${REPO_PARENT}/base-oss-google-ubuntu-stemcell/url")"
-image_name=$(echo "$raw_stemcell_filename" | sed -e 's/[^0-9a-zA-Z]/-/g' -e 's/-tar-gz$//' -e 's/-go-agent-raw//' -e 's/^bosh-//')
+image_name=$(echo "$raw_stemcell_filename" \
+ | sed -e 's/[^0-9a-zA-Z]/-/g' -e 's/-tar-gz$//' -e 's/-go-agent-raw//' -e 's/^bosh-//')
# authenticate with service account
-echo ${GCP_SERVICE_ACCOUNT_KEY} | gcloud auth activate-service-account --key-file - --project ${PROJECT_NAME}
+echo "${GCP_SERVICE_ACCOUNT_KEY}" \
+ | gcloud auth activate-service-account --key-file - --project "${PROJECT_NAME}"
guest_os_features=()
if [[ "${EFI:-false}" == "true" ]]; then
@@ -45,14 +47,14 @@ if (( ${#guest_os_features[@]} > 0 )); then
fi
# create image
+# shellcheck disable=SC2086
gcloud compute images create "${image_name}" \
--project="${PROJECT_NAME}" \
--source-uri="${raw_stemcell_uri}" \
${guest_os_features_flag} \
--storage-location=eu
-
-gcloud compute images add-iam-policy-binding ${image_name} \
+gcloud compute images add-iam-policy-binding "${image_name}" \
--member='allAuthenticatedUsers' \
--role='roles/compute.imageUser'
@@ -61,14 +63,14 @@ pushd "${REPO_PARENT}/working_dir"
# create final light stemcell
tar xvf "${original_stemcell}"
- > image
+ : > image
packaged_image_stemcell_sha1=$(sha1sum image | awk '{print $1}')
cp stemcell.MF /tmp/stemcell.MF.tmp
bosh int \
-o "${REPO_ROOT}/ci/tasks/light-google/assets/public-image-stemcell-ops.yml" \
- -v "packaged_image_stemcell_sha1=$packaged_image_stemcell_sha1" \
+ -v "packaged_image_stemcell_sha1=${packaged_image_stemcell_sha1}" \
-v 'stemcell_formats=["google-light"]' \
-v "image_url=https://www.googleapis.com/compute/v1/projects/${PROJECT_NAME}/global/images/${image_name}" \
/tmp/stemcell.MF.tmp > stemcell.MF
diff --git a/ci/tasks/light-google/deploy-skeletal.sh b/ci/tasks/light-google/deploy-skeletal.sh
index d0bee28e16..a5e57c4df5 100755
--- a/ci/tasks/light-google/deploy-skeletal.sh
+++ b/ci/tasks/light-google/deploy-skeletal.sh
@@ -11,8 +11,8 @@ if [[ -n "${DEBUG:-}" ]]; then
fi
# env
-: ${SSH_PRIVATE_KEY:?}
-: ${GCE_CREDENTIALS_JSON:?}
+: "${SSH_PRIVATE_KEY:?}"
+: "${GCE_CREDENTIALS_JSON:?}"
mkdir -p "${REPO_PARENT}/deployment-state/assets/"
diff --git a/ci/tasks/light-google/make-raw-from-heavy-stemcell.sh b/ci/tasks/light-google/make-raw-from-heavy-stemcell.sh
index ca39e402bb..0603fc587c 100755
--- a/ci/tasks/light-google/make-raw-from-heavy-stemcell.sh
+++ b/ci/tasks/light-google/make-raw-from-heavy-stemcell.sh
@@ -10,18 +10,21 @@ if [[ -n "${DEBUG:-}" ]]; then
export BOSH_LOG_PATH="${BOSH_LOG_PATH:-${REPO_PARENT}/bosh-debug.log}"
fi
-: ${BUCKET_NAME:?}
-: ${STEMCELL_BUCKET_PATH:?} # used to check if current stemcell already exists
+: "${BUCKET_NAME:?}"
+: "${STEMCELL_BUCKET_PATH:?}" # used to check if current stemcell already exists
stemcell_url() {
- resource="/${STEMCELL_BUCKET_PATH}/${light_stemcell_name}"
+ local name
+ name=${1}
+
+ resource="/${STEMCELL_BUCKET_PATH}/${name}"
if [ ! -z "$AWS_ACCESS_KEY_ID" ]; then
expires=$(date +%s)
expires=$((expires + 30))
string_to_sign="HEAD\n\n\n${expires}\n${resource}"
- signature=$(echo -en "$string_to_sign" | openssl sha1 -hmac ${AWS_SECRET_ACCESS_KEY} -binary | base64)
+ signature=$(echo -en "$string_to_sign" | openssl sha1 -hmac "${AWS_SECRET_ACCESS_KEY}" -binary | base64)
signature=$(python -c "import urllib; print urllib.quote_plus('${signature}')")
echo -n "https://${S3_API_ENDPOINT}${resource}?AWSAccessKeyId=${AWS_ACCESS_KEY_ID}&Expires=${expires}&Signature=${signature}"
else
@@ -32,22 +35,20 @@ stemcell_url() {
echo "Creating light stemcell..."
salt=$(date +%s)
-original_stemcell="$(echo ${REPO_PARENT}/stemcell/*.tgz)"
+original_stemcell="$(echo "${REPO_PARENT}"/stemcell/*.tgz)"
original_stemcell_name="$(basename "${original_stemcell}")"
-raw_stemcell_name="$(basename "${original_stemcell}" .tgz)-raw-$salt.tar.gz"
-light_stemcell_name="light-${original_stemcell_name}"
+raw_stemcell_name="$(basename "${original_stemcell}" .tgz)-raw-${salt}.tar.gz"
echo "Using raw stemcell name: ${raw_stemcell_name}"
-light_stemcell_url="$(stemcell_url)"
-set +e
-wget --spider "$light_stemcell_url"
-if [[ "$?" == "0" ]]; then
- echo "Google light stemcell '$light_stemcell_name' already exists!"
- echo "You can download here: $light_stemcell_url"
+light_stemcell_name="light-${original_stemcell_name}"
+light_stemcell_url="$(stemcell_url "${light_stemcell_name}")"
+
+if wget --spider "${light_stemcell_url}"; then
+ echo "Google light stemcell '${light_stemcell_name}' already exists!"
+ echo "You can download here: ${light_stemcell_url}"
exit 1
fi
-set -e
mkdir "${REPO_PARENT}/working_dir"
pushd "${REPO_PARENT}/working_dir"
diff --git a/docs/new_stemcell_line.md b/docs/new_stemcell_line.md
index 74b64e006b..7864944a92 100644
--- a/docs/new_stemcell_line.md
+++ b/docs/new_stemcell_line.md
@@ -8,7 +8,7 @@
git switch -c ubuntu-${short_name} {commit}
```
-2. Update `ci/pipelines/vars.yml` with the appropriate values
+2. Update `ci/pipeline-vars.yml` with the appropriate values
```yaml
#@data/values
diff --git a/image-metalinks/ubuntu-jammy/ubuntu-jammy.meta4 b/image-metalinks/ubuntu-jammy/ubuntu-jammy.meta4
index f04bc3f6da..28fa87f9b0 100644
--- a/image-metalinks/ubuntu-jammy/ubuntu-jammy.meta4
+++ b/image-metalinks/ubuntu-jammy/ubuntu-jammy.meta4
@@ -1,22 +1,22 @@
- 31dea7af0f628a67ab91ebbe4005d0b89873b9aa7560a722b5665dc418a30dc0220241782b826ac0d5c45d369c7e9401e5f90afb6a2cd54d2300e86be1c77812
- 480707f5a3eff94e8668fdffceccc3d135b6999ff9f14e21cd8b5dfd2d7d63c2
- 84a9075825fce098b1fba916c4862556a16588d0
- ab7f1a3bea00723d2522362d6931a003
- 435213458
+ f5aa668b4001073ae7cc4470f02993798756ecf51f5cb2c75efdded48fb1979ac702709e45ed01ca4d0361f2175095458d5fdec59e64792169799eeed3fdc795
+ bb01b17347b77f28de2758794d9b9c6cfcbd1ce8f9fc4bf855ec013e945240b1
+ 7d1b9139bcde2a4cc5bd49d566ef4d00c185fcd4
+ bc82513c38a4d4074ee7d5eb93fb2c31
+ 435226218
https://storage.googleapis.com/bosh-os-images/ubuntu-jammy/ubuntu-jammy.tgz
- 1105.0.0
+ 1111.0.0
- be688838ca8686e5c90689bf2ab585cef1137c999b48c70b92f67a5c34dc15697b5d11c982ed6d71be1e1e7f7b4e0733884aa97c3f7a339a8ed03577cf74be09
- 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b
- adc83b19e793491b1c6ea0fd8b46cd9f32e592fc
- 68b329da9893e34099c7d8ad5cb9c940
- 1
+ 8fb4b6e9b9f2469249a8f20ee65a56dbdd663bf7b3eee14e49fefeaa8dc5826c553a9dc81b2af4cb56a1ff29cc18c5cc8610b2ca670ef647fedd4c9645a6db02
+ 3f6e5e7d27eca945fe908f06f9ae1c9775cb9a4d57f0e3b0aa2ee01735ab5d15
+ 79e5158fb4aa4e628569e5deb6f6f5343b348a4d
+ a1ad3da61ce7f95f9ca6a247f554a59d
+ 2177
https://storage.googleapis.com/bosh-os-images/ubuntu-jammy/usn-log.json
- 1105.0.0
+ 1111.0.0
metalink-repository-resource/0.0.0
- 2026-06-18T22:29:21.024149406Z
+ 2026-06-21T00:16:18.604878649Z
diff --git a/image-metalinks/ubuntu-noble/ubuntu-noble.meta4 b/image-metalinks/ubuntu-noble/ubuntu-noble.meta4
index 97f3dfd68e..3b3723b5ca 100644
--- a/image-metalinks/ubuntu-noble/ubuntu-noble.meta4
+++ b/image-metalinks/ubuntu-noble/ubuntu-noble.meta4
@@ -1,22 +1,22 @@
- 00a885490bd7be17da5ad861b473080660cd471ca16f37fd93342fadf7bccb74756c2e82abc028cef7c857193096a7f98aebc45fe19e5988387cc1cdce7d9ce2
- d99124e0798a66582ec2bf73090aa012fc9577912977eaa28ad9dcfac59864cb
- 2db51e8336075e1b4ca870412c167927abfefd6b
- fd8bb35a87cf83b82f5c17ea473d1207
- 374571238
+ a59cf5f49530bbf3e5c8e3984271e9ca899a9210b0c600af03d47adaf751bad582144edda6fa5f8b37bfe50c61128037bef448c86166937390616445bc50af08
+ 16e5c20f6c73e30f086040088ad47c283c72dcfdd481aaafbfe14330c6ac9169
+ fb2c61e81957e9aa08729f928e8e5a645ed3c8f9
+ adca95d2db5cac2a19ec1e0e66c7a82e
+ 374576198
https://storage.googleapis.com/bosh-os-images/ubuntu-noble/ubuntu-noble.tgz
- 136.0.0
+ 142.0.0
- be253863f3ed36a79daa272c9f23f902422486f46fb84114da8dcb42ce48610048cafa8021e0122c8adf468f9f4c81472a1795db6290f8747a4c8cf5bcdeb18c
- 6964ac8f84ef68d727fa7f98b7da660f45cdf98ee86d430140157b998e0fb40e
- 5d380f950e833f8d143b49a3134a3384cb7354c1
- b1e24e5bb6d70a57b4d282b948136d67
- 8523
+ 8e6ab7e8e7af3485bc01bf86d30e4602ac829387f14edabd340f95659bfb92e5263672a44a7c77a484205da1c30a3736901ecd987176932e14873ceda78be9ee
+ 1c5efc55cf8392396e9fcc76ee2dac6b7479ed7292f3cde0351639d8275b2ae6
+ 250857d7052f120f05ff1a98f115b3ea3925f2dc
+ d1da05acbb3ead37c41ad0dfcc60467e
+ 10702
https://storage.googleapis.com/bosh-os-images/ubuntu-noble/usn-log.json
- 136.0.0
+ 142.0.0
metalink-repository-resource/0.0.0
- 2026-06-18T22:55:58.219185055Z
+ 2026-06-21T00:15:09.227960758Z