Merge pull request #579 from cloudfoundry/merge-jammy

Merge jammy into noble

Author: aramprice

.github/pull_request_template.md

@@ -0,0 +1,9 @@
+NOTE: this repository uses a "Merge Forward" strategy
+
+Changes should be made in the earliest applicable branch, and
+merged forward through subsequent branches.
+1. Create a PR into the oldest branch (`ubuntu-<short_name>`)
+2. After this PR has been merged create a `merge-to-<next_short_name>` branch
+3. Merge `ubuntu-<short_name>` into `merge-to-<next_short_name>`
+4. Create a PR to merge `merge-to-<next_short_name>` into `ubuntu-<next_short_name>`
+5. Repeat as needed for subsequent branches

README.md

@@ -296,19 +296,21 @@ Concourse publishes its artifacts to GCS.
 #### Create Buckets
 
 ```shell
-gsutil mb -l europe-west4  gs://bosh-aws-light-stemcells
-gsutil mb -l europe-west4  gs://bosh-aws-light-stemcells-candidate
+export gcp_region="europe-west4"
 
-gsutil mb -l europe-west4  gs://bosh-gce-light-stemcell-ci-terraform-state
+gsutil mb -l "${gcp_region}"  gs://bosh-aws-light-stemcells
+gsutil mb -l "${gcp_region}"  gs://bosh-aws-light-stemcells-candidate
 
-gsutil mb -l europe-west4  gs://bosh-gce-light-stemcells
-gsutil mb -l europe-west4  gs://bosh-gce-light-stemcells-candidate
-gsutil mb -l europe-west4  gs://bosh-gce-raw-stemcells-new
+gsutil mb -l "${gcp_region}"  gs://bosh-gce-light-stemcell-ci-terraform-state
 
-gsutil mb -l europe-west4  gs://bosh-core-stemcells
-gsutil mb -l europe-west4  gs://bosh-core-stemcells-candidate
-gsutil mb -l europe-west4  gs://bosh-os-images
-gsutil mb -l europe-west4  gs://bosh-stemcell-triggers
+gsutil mb -l "${gcp_region}"  gs://bosh-gce-light-stemcells
+gsutil mb -l "${gcp_region}"  gs://bosh-gce-light-stemcells-candidate
+gsutil mb -l "${gcp_region}"  gs://bosh-gce-raw-stemcells-new
+
+gsutil mb -l "${gcp_region}"  gs://bosh-core-stemcells
+gsutil mb -l "${gcp_region}"  gs://bosh-core-stemcells-candidate
+gsutil mb -l "${gcp_region}"  gs://bosh-os-images
+gsutil mb -l "${gcp_region}"  gs://bosh-stemcell-triggers
 ```
 
 #### Make Buckets Publicly Readable
@@ -343,18 +345,28 @@ gcloud compute firewall-rules update default-allow-internal --source-ranges 10.0
 
 #### Create Integration Networks
 
-Create the bosh-integration networks for tests and BATs tests. Each stemcell
-line should get its own subnet corresponding to its `subnet_int`.
+Create a `stemcell-builder-integration-${subnet_int}` subnetworks need by BATs tests.
+Each stemcell line should get its own subnet corresponding to its `subnet_int` equal to
+the two digit release year. For example release year 2010 would have `subnet_int="10"`.
 
-Example:
+Example per [ci/pipelines/vars.yml](ci/pipelines/vars.yml):
 
-- subnet_id=44
-    - subnet_range=10.100.44.0/24
-    - subnet_name=bosh-integration-44
+```yaml
+---
+stemcell_details:
+  # ... snip
+  subnet_int: "10" #! use last two digits of release year: ex 2010 -> 10
+  # ... snip
+```
+
+Would mean creating the following subnet in GCP:
 
 ```shell
 # branch: ubuntu-${short_name}
-gcloud compute networks subnets create --network default --range 10.100.0.0/24 bosh-integration-0
+export subnet_int="10"
+
+gcloud compute networks subnets create --network default \
+  --range "10.100.${subnet_int}.0/24" "stemcell-builder-integration-${subnet_int}"
 ```
 
 ### AWS

bosh-stemcell/spec/assets/dpkg-list-ubuntu-azure-additions.txt

@@ -4,13 +4,15 @@ linux-cloud-tools-6.8
 linux-cloud-tools-6.8-generic
 linux-cloud-tools-common
 linux-cloud-tools-generic
+netplan-generator
 netplan.io
 python-is-python3
 python3-attr
 python3-certifi
 python3-chardet
 python3-configobj
 python3-debconf
+python3-netplan
 python3-idna
 python3-jinja2
 python3-json-pointer

ci/pipelines/builder.yml

@@ -446,6 +446,7 @@ jobs:
 
 #@ def build_stemcell(IAAS, HYPERVISOR, FIPS=""):
   name: build-(@= IAAS @)-(@= HYPERVISOR @)(@= FIPS @)
+  serial: true
   plan:
   - in_parallel:
     - get: version
@@ -468,7 +469,6 @@ jobs:
       passed:
       - build-os-image
   - task: create-stemcell
-    attempts: 3
     file: bosh-stemcells-ci/ci/tasks/build.yml
     image: os-image-stemcell-builder-registry-image
     params:
@@ -480,7 +480,7 @@ jobs:
       STEMCELL_BUCKET: bosh-core-stemcells-candidate(@= FIPS @)
       GIT_USER_EMAIL: *ci_bot_email
       GIT_USER_NAME: *ci_bot_name
-      #! AGENT_SUFFIX: ""
+      AGENT_SUFFIX: (@= data.values.stemcell_details.agent_suffix @)
       #@ if/end FIPS != "":
       UBUNTU_ADVANTAGE_TOKEN: ((ubuntu_advantage_token))
     privileged: true
@@ -577,7 +577,7 @@ jobs:
         file: bosh-stemcells-ci/ci/tasks/bats/iaas/gcp/prepare-bats-config.yml
         image: bosh-integration-image
         params:
-          VARS_STEMCELL_NAME: bosh-google-kvm-ubuntu-(@= data.values.stemcell_details.os_short_name @)
+          VARS_STEMCELL_NAME: bosh-google-kvm-ubuntu-(@= data.values.stemcell_details.os_short_name @)(@= data.values.stemcell_details.agent_suffix @)
           VARS_NETWORK_DEFAULT: bosh-concourse
           VARS_AVAILABILITY_ZONE: europe-north2-a
           VARS_MACHINE_TYPE: e2-standard-2
@@ -641,7 +641,7 @@ jobs:
       COMMIT_PREFIX: candidate
       COPY_KEYS:
       #@ for iaas in data.values.stemcell_details.include_fips_iaas:
-        - (@= iaas.iaas @)/bosh-stemcell-%s-(@= iaas.iaas @)-(@= iaas.hypervisor @)-(@= data.values.stemcell_details.os_name @)-fips.tgz
+        - (@= iaas.iaas @)/bosh-stemcell-%s-(@= iaas.iaas @)-(@= iaas.hypervisor @)-(@= data.values.stemcell_details.os_name @)-fips(@= data.values.stemcell_details.agent_suffix @).tgz
       #@ end
       FROM_BUCKET_NAME: bosh-core-stemcells-candidate-fips
       FROM_INDEX: dev
@@ -663,7 +663,7 @@ jobs:
       COMMIT_PREFIX: candidate
       COPY_KEYS:
       #@ for iaas in data.values.stemcell_details.include_iaas:
-        - (@= iaas.iaas @)/bosh-stemcell-%s-(@= iaas.iaas @)-(@= iaas.hypervisor @)-(@= data.values.stemcell_details.os_name @).tgz
+        - (@= iaas.iaas @)/bosh-stemcell-%s-(@= iaas.iaas @)-(@= iaas.hypervisor @)-(@= data.values.stemcell_details.os_name @)(@= data.values.stemcell_details.agent_suffix @).tgz
       #@ end
       FROM_BUCKET_NAME: bosh-core-stemcells-candidate
       FROM_INDEX: dev
@@ -916,7 +916,7 @@ resources:
     options:
       private_key: ((github_deploy_key_bosh-io-stemcells-core-index.private_key))
     filters:
-    - repositorypath: "*/(@= IAAS @)-(@= HYPERVISOR @)(@= FIPS @).meta4"
+    - repositorypath: "*/(@= IAAS @)-(@= HYPERVISOR @)(@= FIPS @)(@= data.values.stemcell_details.agent_suffix @).meta4"
     uri: git+ssh://git@github.com:cloudfoundry/bosh-io-stemcells-core-index.git//dev/(@= data.values.stemcell_details.os_name @)(@= FIPS @)/
     url_handlers:
     - include:

ci/pipelines/vars.yml

@@ -1,6 +1,7 @@
 #@data/values
 ---
 stemcell_details:
+  agent_suffix: "" #! empty
   branch: ubuntu-noble
   major_version: 1
   os_version: "24.04"

ci/tasks/build.sh

@@ -23,7 +23,6 @@ check_param IAAS
 check_param HYPERVISOR
 check_param OS_NAME
 check_param OS_VERSION
-AGENT_SUFFIX="${AGENT_SUFFIX--go_agent}"
 
 export CANDIDATE_BUILD_NUMBER=$( cat "${REPO_PARENT}/version/number" | sed 's/\.0$//;s/\.0$//' )
 
@@ -88,20 +87,20 @@ set -e
 cd "${REPO_PARENT}/bosh-linux-stemcell-builder"
 bundle install
 
-if [[ -z "$OS_IMAGE" ]]; then
-  bundle exec rake stemcell:build[$IAAS,$HYPERVISOR,$OS_NAME,$OS_VERSION,$CANDIDATE_BUILD_NUMBER]
+if [[ -z "${OS_IMAGE}" ]]; then
+  bundle exec rake stemcell:build[${IAAS},${HYPERVISOR},${OS_NAME},${OS_VERSION},${CANDIDATE_BUILD_NUMBER}]
   rm -f ./tmp/base_os_image.tgz
 else
-  bundle exec rake stemcell:build_with_local_os_image[$IAAS,$HYPERVISOR,$OS_NAME,$OS_VERSION,$OS_IMAGE,$CANDIDATE_BUILD_NUMBER]
+  bundle exec rake stemcell:build_with_local_os_image[${IAAS},${HYPERVISOR},${OS_NAME},${OS_VERSION},${OS_IMAGE},${CANDIDATE_BUILD_NUMBER}]
 fi
 SUDO
 
 #
 # Output and checksum the stemcell artifacts
 #
 
-stemcell_name="bosh-stemcell-${CANDIDATE_BUILD_NUMBER}-$IAAS-$HYPERVISOR-$OS_NAME-$OS_VERSION${AGENT_SUFFIX}"
-meta4_path="${REPO_PARENT}/stemcells-index-output/dev/$OS_NAME-$OS_VERSION/${CANDIDATE_BUILD_NUMBER}/$IAAS-$HYPERVISOR${AGENT_SUFFIX}.meta4"
+stemcell_name="bosh-stemcell-${CANDIDATE_BUILD_NUMBER}-${IAAS}-${HYPERVISOR}-${OS_NAME}-${OS_VERSION}${AGENT_SUFFIX}"
+meta4_path="${REPO_PARENT}/stemcells-index-output/dev/${OS_NAME}-${OS_VERSION}/${CANDIDATE_BUILD_NUMBER}/${IAAS}-${HYPERVISOR}${AGENT_SUFFIX}.meta4"
 
 echo "${CANDIDATE_BUILD_NUMBER}" > "${REPO_PARENT}/candidate-build-number/number"
 mkdir -p "$( dirname "$meta4_path" )"
@@ -137,4 +136,4 @@ cd "${REPO_PARENT}/stemcells-index-output"
 git add -A
 git config --global user.email "${GIT_USER_EMAIL}"
 git config --global user.name "${GIT_USER_NAME}"
-git commit -m "dev: ${OS_NAME}-${OS_VERSION}/${CANDIDATE_BUILD_NUMBER} ($IAAS-$HYPERVISOR)"
+git commit -m "dev: ${OS_NAME}-${OS_VERSION}/${CANDIDATE_BUILD_NUMBER} (${IAAS}-${HYPERVISOR})"

ci/tasks/build.yml

@@ -27,6 +27,6 @@ params:
   STEMCELL_BUCKET: replace-me
   S3_API_ENDPOINT: s3.amazonaws.com
   ESM_TOKEN:
-  AGENT_SUFFIX: -go_agent
+  AGENT_SUFFIX:
   GIT_USER_EMAIL:
   GIT_USER_NAME:

scripts/stemcell-repack-debugger.sh

@@ -2,6 +2,8 @@
 
 set -e
 
+example_stemcell_tgz_path="path/to/bosh-linux-stemcell-builder/tmp/bosh-stemcell-VERSION-IAAS-HYPERVISOR-OS_NAME.tgz"
+
 show_help() {
     echo "Usage: $0 [options]"
     echo
@@ -22,7 +24,7 @@ show_help() {
     echo "  TMPDIR                    Path with enough disk space to hold all temporary files for repacking *OPTIONAL*"
     echo
     echo "Example:"
-    echo "  export stemcell_tgz=\"/home/username/workspace/bosh/bosh-linux-stemcell-builder/tmp/bosh-stemcell-0.0.8-google-kvm-ubuntu-noble-go_agent.tgz\""
+    echo "  export stemcell_tgz=\"${example_stemcell_tgz_path}\""
     echo "  export AGENT_BINARY=\"/path/to/bosh-agent\""
     echo "  export AGENT_JSON=\"/path/to/agent.json\""
     echo "  export BOSH_DEBUG_PUB_KEY=\"ssh-rsa AAA... user@hostname\""
@@ -74,7 +76,7 @@ fi
 
 
 if [ -z ${stemcell_tgz} ]; then
-echo "stemcell_tgz is not set. export stemcell_tgz="/home/username/workspace/bosh/bosh-linux-stemcell-builder/tmp/bosh-stemcell-0.0.8-google-kvm-ubuntu-noble-go_agent.tgz""
+echo "stemcell_tgz is not set. export stemcell_tgz=\"${example_stemcell_tgz_path}\""
 fi
 
 # stemcell_tgz=/tmp/stemcell.tgz