Skip to content

Commit b916206

Browse files
arampricearamprice
committed
Merge branch 'ubuntu-jammy' into merge-jammy
2 parents 7af7d12 + 18b5978 commit b916206

5 files changed

Lines changed: 285 additions & 90 deletions

File tree

‎ci/pipelines/builder.yml‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,9 @@ groups:
4444
- name: docker
4545
jobs:
4646
- build-os-image-stemcell-builder
47+
- name: infrastructure
48+
jobs:
49+
- ensure-integration-network
4750

4851
#@yaml/text-templated-strings
4952
jobs:
@@ -89,6 +92,25 @@ jobs:
8992
get_params:
9093
skip_download: true
9194

95+
#! Manually triggered job that idempotently ensures the GCP subnetwork and
96+
#! firewall rule consumed by deploy-director / cleanup-bats-vms / prepare-bats
97+
#! in the test-stemcells-ipv4 and bats jobs below exist. GCP is the source of
98+
#! truth — no state file is required.
99+
- name: ensure-integration-network
100+
serial: true
101+
plan:
102+
- get: bosh-stemcells-ci
103+
- get: bosh-integration-image
104+
- task: ensure-integration-network
105+
file: bosh-stemcells-ci/ci/tasks/gcp/ensure-integration-network.yml
106+
image: bosh-integration-image
107+
params:
108+
GCP_JSON_KEY: ((gcp_json_key))
109+
GCP_PROJECT_ID: ((gcp_project_id))
110+
GCP_REGION: europe-north2
111+
GCP_NETWORK_NAME: bosh-concourse
112+
SUBNET_INT: (@= data.values.stemcell_details.subnet_int @)
113+
92114
- name: process-high-critical-cves
93115
serial_groups: [log-cves]
94116
plan:
@@ -885,7 +907,6 @@ resource_types:
885907
type: registry-image
886908
source:
887909
repository: frodenas/gcs-resource
888-
889910
#@yaml/text-templated-strings
890911
resources:
891912
- name: daily

‎ci/pipelines/publisher.yml‎

Lines changed: 96 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -467,81 +467,100 @@ jobs:
467467
KERNEL_PACKAGE: linux-generic
468468
input_mapping:
469469
candidate-stemcell: candidate-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
470-
- task: extract-warden-image
471-
image: bosh-integration-registry-image
472-
config:
473-
inputs:
474-
- name: candidate-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
475-
outputs:
476-
- name: stemcell-image
477-
platform: linux
478-
run:
479-
dir: stemcell-image
480-
path: /bin/bash
481-
args:
482-
- -ce
483-
- tar xvf ../candidate-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)/bosh-stemcell-*-warden-boshlite-(@= data.values.stemcell_details.os_name @)*.tgz image
484-
- put: github-container-registry-(@= data.values.stemcell_details.os_name @)-stemcell
485-
params:
486-
import_file: stemcell-image/image
487-
tag_file: candidate-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)/.resource/version
488-
tag_as_latest: true
489-
- put: published-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
490-
params:
491-
files:
492-
- candidate-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)/*.tgz
493-
options:
494-
author_email: *ci_bot_email
495-
author_name: *ci_bot_name
496-
message: 'publish (heavy): (@= data.values.stemcell_details.os_name @)/(@= str(data.values.stemcell_details.major_version) @).x'
497-
rename: '{{.Version}}/stemcells.meta4'
498-
version: candidate-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)/.resource/version
499-
#@ if len(data.values.stemcell_details.include_fips_iaas) > 0 :
500-
- put: published-(@= data.values.stemcell_details.os_name @)-fips-stemcell-(@= str(data.values.stemcell_details.major_version) @)
501-
params:
502-
files:
503-
- candidate-(@= data.values.stemcell_details.os_name @)-fips-stemcell-(@= str(data.values.stemcell_details.major_version) @)/*.tgz
504-
options:
505-
author_email: *ci_bot_email
506-
author_name: *ci_bot_name
507-
message: 'publish (heavy): (@= data.values.stemcell_details.os_name @)-fips/(@= str(data.values.stemcell_details.major_version) @).x'
508-
rename: '{{.Version}}/stemcells.meta4'
509-
version: candidate-(@= data.values.stemcell_details.os_name @)-fips-stemcell-(@= str(data.values.stemcell_details.major_version) @)/.resource/version
510-
#@ end
511-
- params:
512-
files:
513-
- candidate-aws-light-stemcell/*.tgz
514-
options:
515-
author_email: *ci_bot_email
516-
author_name: *ci_bot_name
517-
message: 'publish (light aws): (@= data.values.stemcell_details.os_name @)/(@= str(data.values.stemcell_details.major_version) @).x'
518-
rename: '{{.Version}}/stemcells.aws.meta4'
519-
version: candidate-aws-light-stemcell/.resource/version
520-
put: published-aws-light-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
521-
522-
#! once we release all regions with the same account, we can unify these again
523-
- file: bosh-stemcells-ci/ci/tasks/light-aws/tag-aws-ami-light.yml
524-
image: bosh-ecosystem-concourse-registry-image
525-
task: tag-published-aws-ami-light-stemcells
526-
params:
527-
AWS_PAGER:
528-
AWS_ACCESS_KEY_ID: ((aws_publish_us_access_key))
529-
AWS_SECRET_ACCESS_KEY: ((aws_publish_us_secret_key))
530-
GREP_PATTERN: grep -v 'gov-\|cn-'
531-
- file: bosh-stemcells-ci/ci/tasks/light-aws/tag-aws-ami-light.yml
532-
image: bosh-ecosystem-concourse-registry-image
533-
task: tag-gov-published-aws-ami-light-stemcells
534-
params:
535-
AWS_PAGER:
536-
AWS_ACCESS_KEY_ID: ((aws_publish_us-gov_access_key))
537-
AWS_SECRET_ACCESS_KEY: ((aws_publish_us-gov_secret_key))
538-
GREP_PATTERN: grep 'gov-'
539-
#! - file: bosh-stemcells-ci/ci/tasks/light-aws/tag-aws-ami-light.yml
540-
#! task: tag-china-published-aws-ami-light-stemcells
541-
#! params:
542-
#! AWS_ACCESS_KEY_ID: ((aws_publish_cn_access_key))
543-
#! AWS_SECRET_ACCESS_KEY: ((aws_publish_cn_secret_key))
544-
#! GREP_PATTERN: grep 'cn-'
470+
- in_parallel:
471+
- do:
472+
- task: extract-warden-image
473+
image: bosh-integration-registry-image
474+
config:
475+
inputs:
476+
- name: candidate-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
477+
outputs:
478+
- name: stemcell-image
479+
platform: linux
480+
run:
481+
dir: stemcell-image
482+
path: /bin/bash
483+
args:
484+
- -ce
485+
- |
486+
tar xvf ../candidate-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)/bosh-stemcell-*-warden-boshlite-(@= data.values.stemcell_details.os_name @)*.tgz image
487+
printf 'FROM scratch\nADD image /\n' > Dockerfile
488+
- task: build-stemcell-oci-image
489+
privileged: true
490+
config:
491+
platform: linux
492+
image_resource:
493+
type: registry-image
494+
source:
495+
repository: concourse/oci-build-task
496+
inputs:
497+
- name: stemcell-image
498+
outputs:
499+
- name: image
500+
params:
501+
CONTEXT: stemcell-image
502+
run:
503+
path: build
504+
- put: github-container-registry-(@= data.values.stemcell_details.os_name @)-stemcell
505+
params:
506+
image: image/image.tar
507+
additional_tags: candidate-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)/.resource/version
508+
- put: published-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
509+
params:
510+
files:
511+
- candidate-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)/*.tgz
512+
options:
513+
author_email: *ci_bot_email
514+
author_name: *ci_bot_name
515+
message: 'publish (heavy): (@= data.values.stemcell_details.os_name @)/(@= str(data.values.stemcell_details.major_version) @).x'
516+
rename: '{{.Version}}/stemcells.meta4'
517+
version: candidate-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)/.resource/version
518+
#@ if len(data.values.stemcell_details.include_fips_iaas) > 0 :
519+
- put: published-(@= data.values.stemcell_details.os_name @)-fips-stemcell-(@= str(data.values.stemcell_details.major_version) @)
520+
params:
521+
files:
522+
- candidate-(@= data.values.stemcell_details.os_name @)-fips-stemcell-(@= str(data.values.stemcell_details.major_version) @)/*.tgz
523+
options:
524+
author_email: *ci_bot_email
525+
author_name: *ci_bot_name
526+
message: 'publish (heavy): (@= data.values.stemcell_details.os_name @)-fips/(@= str(data.values.stemcell_details.major_version) @).x'
527+
rename: '{{.Version}}/stemcells.meta4'
528+
version: candidate-(@= data.values.stemcell_details.os_name @)-fips-stemcell-(@= str(data.values.stemcell_details.major_version) @)/.resource/version
529+
#@ end
530+
- params:
531+
files:
532+
- candidate-aws-light-stemcell/*.tgz
533+
options:
534+
author_email: *ci_bot_email
535+
author_name: *ci_bot_name
536+
message: 'publish (light aws): (@= data.values.stemcell_details.os_name @)/(@= str(data.values.stemcell_details.major_version) @).x'
537+
rename: '{{.Version}}/stemcells.aws.meta4'
538+
version: candidate-aws-light-stemcell/.resource/version
539+
put: published-aws-light-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)
540+
541+
#! once we release all regions with the same account, we can unify these again
542+
- file: bosh-stemcells-ci/ci/tasks/light-aws/tag-aws-ami-light.yml
543+
image: bosh-ecosystem-concourse-registry-image
544+
task: tag-published-aws-ami-light-stemcells
545+
params:
546+
AWS_PAGER:
547+
AWS_ACCESS_KEY_ID: ((aws_publish_us_access_key))
548+
AWS_SECRET_ACCESS_KEY: ((aws_publish_us_secret_key))
549+
GREP_PATTERN: grep -v 'gov-\|cn-'
550+
- file: bosh-stemcells-ci/ci/tasks/light-aws/tag-aws-ami-light.yml
551+
image: bosh-ecosystem-concourse-registry-image
552+
task: tag-gov-published-aws-ami-light-stemcells
553+
params:
554+
AWS_PAGER:
555+
AWS_ACCESS_KEY_ID: ((aws_publish_us-gov_access_key))
556+
AWS_SECRET_ACCESS_KEY: ((aws_publish_us-gov_secret_key))
557+
GREP_PATTERN: grep 'gov-'
558+
#! - file: bosh-stemcells-ci/ci/tasks/light-aws/tag-aws-ami-light.yml
559+
#! task: tag-china-published-aws-ami-light-stemcells
560+
#! params:
561+
#! AWS_ACCESS_KEY_ID: ((aws_publish_cn_access_key))
562+
#! AWS_SECRET_ACCESS_KEY: ((aws_publish_cn_secret_key))
563+
#! GREP_PATTERN: grep 'cn-'
545564
- params:
546565
files:
547566
- candidate-google-light-(@= data.values.stemcell_details.os_name @)-stemcell-(@= str(data.values.stemcell_details.major_version) @)/*.tgz
@@ -838,11 +857,12 @@ resources:
838857
#@ end
839858

840859
- name: github-container-registry-(@= data.values.stemcell_details.os_name @)-stemcell
841-
type: docker-image
860+
type: registry-image
842861
source:
843862
repository: ghcr.io/cloudfoundry/(@= data.values.stemcell_details.os_name @)-stemcell
844863
username: ((github_read_write_packages.username))
845864
password: ((github_read_write_packages.password))
865+
tag: latest
846866

847867
- name: bosh-integration-registry-image
848868
type: registry-image

‎ci/tasks/gcp/ensure-integration-network.sh‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
#!/usr/bin/env bash
2+
set -eu -o pipefail
3+
4+
: "${GCP_JSON_KEY:?}"
5+
: "${GCP_PROJECT_ID:?}"
6+
: "${GCP_REGION:?}"
7+
: "${GCP_NETWORK_NAME:?}"
8+
: "${SUBNET_INT:?}"
9+
10+
echo "${GCP_JSON_KEY}" | gcloud auth activate-service-account --key-file - --project "${GCP_PROJECT_ID}"
11+
12+
SUBNET_NAME="stemcell-builder-integration-${SUBNET_INT}"
13+
SUBNET_CIDR="10.100.${SUBNET_INT}.0/24"
14+
15+
# 'bat' => BATS created VM tag
16+
# 'test-stemcells-bats' => director, and compilation VM tag
17+
FIREWALL_TAGS="bat,test-stemcells-bats"
18+
19+
gcloud_stderr="$(mktemp)"
20+
trap 'rm -f "${gcloud_stderr}"' EXIT
21+
22+
echo "Checking for subnet '${SUBNET_NAME}' in region '${GCP_REGION}'..."
23+
existing_subnet_name="$(gcloud compute networks subnets list \
24+
--regions="${GCP_REGION}" \
25+
--project="${GCP_PROJECT_ID}" \
26+
--filter="name=('${SUBNET_NAME}')" \
27+
--format='value(name)' \
28+
2>"${gcloud_stderr}")" && subnet_lookup_ok=true || subnet_lookup_ok=false
29+
30+
if ${subnet_lookup_ok}; then
31+
if [[ -n "${existing_subnet_name}" ]]; then
32+
current_subnet="$(gcloud compute networks subnets describe "${SUBNET_NAME}" \
33+
--region="${GCP_REGION}" \
34+
--project="${GCP_PROJECT_ID}" \
35+
--format='csv[no-heading](network.basename(),ipCidrRange,privateIpGoogleAccess,stackType)' \
36+
2>"${gcloud_stderr}")"
37+
expected_subnet="${GCP_NETWORK_NAME},${SUBNET_CIDR},True,IPV4_ONLY"
38+
if [[ "${current_subnet}" != "${expected_subnet}" ]]; then
39+
echo "ERROR: Subnet '${SUBNET_NAME}' exists but is misconfigured."
40+
echo " Expected: ${expected_subnet}"
41+
echo " Actual: ${current_subnet}"
42+
exit 1
43+
fi
44+
echo "Subnet '${SUBNET_NAME}' already exists and matches expected configuration."
45+
else
46+
echo "Creating subnet '${SUBNET_NAME}'..."
47+
gcloud compute networks subnets create "${SUBNET_NAME}" \
48+
--network="${GCP_NETWORK_NAME}" \
49+
--region="${GCP_REGION}" \
50+
--range="${SUBNET_CIDR}" \
51+
--enable-private-ip-google-access \
52+
--stack-type=IPV4_ONLY \
53+
--project="${GCP_PROJECT_ID}"
54+
echo "Subnet '${SUBNET_NAME}' created."
55+
fi
56+
else
57+
echo "ERROR: gcloud subnet lookup failed for subnet '${SUBNET_NAME}':"
58+
cat "${gcloud_stderr}" >&2
59+
exit 1
60+
fi
61+
62+
echo "Checking for firewall rule '${SUBNET_NAME}'..."
63+
existing_fw_name="$(gcloud compute firewall-rules list \
64+
--project="${GCP_PROJECT_ID}" \
65+
--filter="name=('${SUBNET_NAME}')" \
66+
--format='value(name)' \
67+
2>"${gcloud_stderr}")" && fw_lookup_ok=true || fw_lookup_ok=false
68+
69+
if ${fw_lookup_ok}; then
70+
if [[ -n "${existing_fw_name}" ]]; then
71+
current_fw_json="$(gcloud compute firewall-rules describe "${SUBNET_NAME}" \
72+
--project="${GCP_PROJECT_ID}" \
73+
--format=json \
74+
2>"${gcloud_stderr}")"
75+
76+
# Validate network, direction, disabled
77+
actual_network="$(echo "${current_fw_json}" | jq -r '.network | split("/") | last')"
78+
actual_direction="$(echo "${current_fw_json}" | jq -r '.direction')"
79+
actual_disabled="$(echo "${current_fw_json}" | jq -r '.disabled')"
80+
81+
if [[ "${actual_network}" != "${GCP_NETWORK_NAME}" ]] || \
82+
[[ "${actual_direction}" != "INGRESS" ]] || \
83+
[[ "${actual_disabled}" != "false" ]]; then
84+
echo "ERROR: Firewall rule '${SUBNET_NAME}' exists but is misconfigured."
85+
echo " Expected network=${GCP_NETWORK_NAME}, direction=INGRESS, disabled=false"
86+
echo " Actual network=${actual_network}, direction=${actual_direction}, disabled=${actual_disabled}"
87+
exit 1
88+
fi
89+
90+
# Validate allowed (should be exactly [{IPProtocol: "all"}])
91+
actual_allowed="$(echo "${current_fw_json}" | jq -c '[.allowed[] | {protocol: .IPProtocol, ports: (.ports // [])}] | sort_by(.protocol)')"
92+
expected_allowed='[{"protocol":"all","ports":[]}]'
93+
if [[ "${actual_allowed}" != "${expected_allowed}" ]]; then
94+
echo "ERROR: Firewall rule '${SUBNET_NAME}' has wrong allowed configuration."
95+
echo " Expected: ${expected_allowed}"
96+
echo " Actual: ${actual_allowed}"
97+
exit 1
98+
fi
99+
100+
# Validate sourceRanges (should be exactly the subnet CIDR)
101+
actual_ranges="$(echo "${current_fw_json}" | jq -c '(.sourceRanges // []) | sort')"
102+
expected_ranges="$(printf '["%s"]' "${SUBNET_CIDR}")"
103+
if [[ "${actual_ranges}" != "${expected_ranges}" ]]; then
104+
echo "ERROR: Firewall rule '${SUBNET_NAME}' has wrong source ranges."
105+
echo " Expected: ${expected_ranges}"
106+
echo " Actual: ${actual_ranges}"
107+
exit 1
108+
fi
109+
110+
# Validate targetTags (order-insensitive)
111+
actual_tags="$(echo "${current_fw_json}" | jq -c '(.targetTags // []) | sort')"
112+
expected_tags="$(printf '%s\n' ${FIREWALL_TAGS//,/ } | jq -R . | jq -sc 'sort')"
113+
if [[ "${actual_tags}" != "${expected_tags}" ]]; then
114+
echo "ERROR: Firewall rule '${SUBNET_NAME}' has wrong target tags."
115+
echo " Expected: ${expected_tags}"
116+
echo " Actual: ${actual_tags}"
117+
exit 1
118+
fi
119+
120+
echo "Firewall rule '${SUBNET_NAME}' already exists and matches expected configuration."
121+
else
122+
echo "Creating firewall rule '${SUBNET_NAME}'..."
123+
gcloud compute firewall-rules create "${SUBNET_NAME}" \
124+
--network="${GCP_NETWORK_NAME}" \
125+
--project="${GCP_PROJECT_ID}" \
126+
--direction=INGRESS \
127+
--priority=1000 \
128+
--allow=all \
129+
--source-ranges="${SUBNET_CIDR}" \
130+
--target-tags="${FIREWALL_TAGS}"
131+
echo "Firewall rule '${SUBNET_NAME}' created."
132+
fi
133+
else
134+
echo "ERROR: gcloud firewall-rules lookup failed for '${SUBNET_NAME}':"
135+
cat "${gcloud_stderr}" >&2
136+
exit 1
137+
fi
138+
139+
echo "Integration network '${SUBNET_NAME}' is ready."

‎ci/tasks/gcp/ensure-integration-network.yml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
---
2+
platform: linux
3+
4+
inputs:
5+
- name: bosh-stemcells-ci
6+
7+
params:
8+
GCP_JSON_KEY:
9+
GCP_PROJECT_ID:
10+
GCP_REGION:
11+
GCP_NETWORK_NAME:
12+
SUBNET_INT:
13+
14+
run:
15+
path: bosh-stemcells-ci/ci/tasks/gcp/ensure-integration-network.sh

0 commit comments

Comments
 (0)