assorted pipeline and test improvements

* add bpm to acceptance test manifests as syslog will depend on it
* remove --all from cleanup - the pipeline already runs `bosh clean-up --all` in the teardown
step. Removing this allows BPM to be uploaded once. It also makes
the tests faster as the releases don't have to be recompiled
for each suite.
* better pam error logging, handle Resolute `sa1` location
* fix libpam-lastlog2 - this seems like a bug in the libpam-lastlog2 package

Author: mkocher

acceptance-tests/ipv4director/smoke/manifest.yml

@@ -4,6 +4,8 @@ name: stemcell-acceptance-tests
 releases:
 - name: syslog
   version: latest
+- name: bpm
+  version: latest
 
 stemcells:
 - alias: default
@@ -25,6 +27,8 @@ instance_groups:
   - {name: default}
   azs: [z1]
   jobs:
+  - name: bpm
+    release: bpm
   - name: syslog_forwarder
     release: syslog
     properties:

acceptance-tests/ipv4director/smoke/smoke_test.go

@@ -83,7 +83,16 @@ var _ = Describe("Stemcell", func() {
 
 		contents, err := io.ReadAll(tempFile)
 		Expect(err).ToNot(HaveOccurred())
-		Expect(contents).ToNot(ContainSubstring("No such file or directory"))
+
+		// Extract only the offending lines so failures are readable (auth.log can be
+		// hundreds of kilobytes and Gomega truncates the full-content diff).
+		var offending []string
+		for _, line := range strings.Split(string(contents), "\n") {
+			if strings.Contains(line, "No such file or directory") {
+				offending = append(offending, line)
+			}
+		}
+		Expect(offending).To(BeEmpty(), "auth.log contained 'No such file or directory':\n%s", strings.Join(offending, "\n"))
 	})
 
 	It("#141987897: disables ipv6 in the kernel", func() {
@@ -97,8 +106,11 @@ var _ = Describe("Stemcell", func() {
 		_, _, exitStatus, err := bosh.Run(
 			"--column=stdout",
 			"ssh", "default/0", "-r", "-c",
-			// sleep to ensure we have multiple samples so average can be verified
-			`sudo /usr/lib/sysstat/sa1 && sudo /usr/lib/sysstat/sa1 1 1 && sleep 2`,
+			// Ubuntu 26.04+ relocated sa1 to /usr/libexec/sysstat/; fall back to the
+			// legacy path for older releases.  sleep ensures multiple samples for the
+			// Average: check.
+			`SA1=$(ls /usr/lib/sysstat/sa1 /usr/libexec/sysstat/sa1 2>/dev/null | head -1) && `+
+				`sudo "$SA1" && sudo "$SA1" 1 1 && sleep 2`,
 		)
 		Expect(err).ToNot(HaveOccurred())
 		Expect(exitStatus).To(Equal(0))

acceptance-tests/ipv4director/syslogrelease/manifest.yml

@@ -4,6 +4,8 @@ name: stemcell-acceptance-tests
 releases:
 - name: syslog
   version: latest
+- name: bpm
+  version: latest
 
 stemcells:
 - alias: default
@@ -40,6 +42,8 @@ instance_groups:
   networks:
   - {name: default}
   jobs:
+  - name: bpm
+    release: bpm
   - name: syslog_forwarder
     release: syslog
     consumes:

acceptance-tests/ipv4director/syslogrelease/smoke_suite_test.go

@@ -22,9 +22,11 @@ var _ = BeforeSuite(func() {
 	bosh = testhelpers.NewBOSH()
 	stemcellPath := testhelpers.RequireEnv("STEMCELL_PATH")
 	syslogReleasePath := testhelpers.RequireEnv("SYSLOG_RELEASE_PATH")
+	bpmReleasePath := testhelpers.RequireEnv("BPM_RELEASE_PATH")
 
 	bosh.UploadStemcell(stemcellPath)
 	bosh.UploadRelease(syslogReleasePath)
+	bosh.UploadRelease(bpmReleasePath)
 	bosh.SafeDeploy()
 })
 

acceptance-tests/testhelpers/bosh.go

@@ -35,7 +35,7 @@ func (b *BOSH) Teardown() {
 	Expect(err).ToNot(HaveOccurred())
 	Expect(exitStatus).To(Equal(0), fmt.Sprintf("stdOut: %s \n stdErr: %s", stdOut, stdErr))
 
-	stdOut, stdErr, exitStatus, err = b.Run("clean-up", "--all")
+	stdOut, stdErr, exitStatus, err = b.Run("clean-up")
 	Expect(err).ToNot(HaveOccurred())
 	Expect(exitStatus).To(Equal(0), fmt.Sprintf("stdOut: %s \n stdErr: %s", stdOut, stdErr))
 }

ci/pipelines/builder.yml

@@ -1098,10 +1098,11 @@ resources:
   type: git
   source:
     branch: (@= data.values.stemcell_details.branch @)
+    uri: https://github.com/cloudfoundry/bosh-linux-stemcell-builder.git
     paths:
       - ci
       - .ruby-version
-    uri: https://github.com/cloudfoundry/bosh-linux-stemcell-builder
+      - acceptance-tests
 
 - name: bats
   type: git
@@ -1133,6 +1134,12 @@ resources:
   type: bosh-io-release
   source:
     repository: cloudfoundry/os-conf-release
+
+- name: bpm-release
+  type: bosh-io-release
+  source:
+    repository: cloudfoundry/bpm-release
+
 - name: bosh-deployment
   type: git
   source:

ci/tasks/test-stemcell.sh

@@ -16,6 +16,7 @@ BOSH_CLIENT_SECRET="$(bosh int "${REPO_PARENT}/director-state/director-creds.yml
 BOSH_ENVIRONMENT="$(bosh int "${REPO_PARENT}/director-state/director-creds.yml" --path /internal_ip)"
 SYSLOG_RELEASE_PATH="$(realpath "${REPO_PARENT}/syslog-release"/*.tgz)"
 OS_CONF_RELEASE_PATH="$(realpath "${REPO_PARENT}/os-conf-release"/*.tgz)"
+BPM_RELEASE_PATH="$(realpath "${REPO_PARENT}/bpm-release"/*.tgz)"
 STEMCELL_PATH="$(realpath "${REPO_PARENT}/stemcell"/*.tgz)"
 # Quote value since the bosh CLI YAML parses it which results in `0.40` becoming `0.4`
 # shellcheck disable=SC2089
@@ -28,6 +29,7 @@ export BOSH_CLIENT_SECRET
 export BOSH_ENVIRONMENT
 export SYSLOG_RELEASE_PATH
 export OS_CONF_RELEASE_PATH
+export BPM_RELEASE_PATH
 export STEMCELL_PATH
 export BOSH_stemcell_version
 

ci/tasks/test-stemcell.yml

@@ -7,6 +7,7 @@ inputs:
 - name: stemcell
 - name: syslog-release
 - name: os-conf-release
+- name: bpm-release
 - name: director-state
 
 params:

stemcell_builder/stages/password_policies/apply.sh

@@ -33,6 +33,16 @@ patch -p1 $chroot/etc/pam.d/common-auth < $assets_dir/ubuntu/common-auth.patch
 strip_trailing_whitespace_from $chroot/etc/pam.d/common-password
 patch -p1 $chroot/etc/pam.d/common-password < $assets_dir/ubuntu/common-password.patch
 
+# libpam-lastlog2 installs pam_lastlog2.so only to the multiarch path
+# (/usr/lib/x86_64-linux-gnu/security/) but PAM's securedir is /usr/lib/security/.
+# Bridge the gap so PAM can load the module referenced above.
+if [ -f "$chroot/usr/lib/x86_64-linux-gnu/security/pam_lastlog2.so" ] && \
+   [ ! -e "$chroot/usr/lib/security/pam_lastlog2.so" ]; then
+  mkdir -p "$chroot/usr/lib/security"
+  ln -sf /usr/lib/x86_64-linux-gnu/security/pam_lastlog2.so \
+      "$chroot/usr/lib/security/pam_lastlog2.so"
+fi
+
 strip_trailing_whitespace_from $chroot/etc/pam.d/login
 patch $chroot/etc/pam.d/login < $assets_dir/ubuntu/login.patch