[Y-Build] Align retention policy for Y-build p2-repositories with drops

Similar to the Y-build drops, retain only the latest nine Y-build
p2-repositories.

Author: HannesWell

JenkinsJobs/Cleanup/cleanupBuilds.jenkinsfile

@@ -28,6 +28,7 @@ pipeline {
 		EP_ROOT = '/home/data/httpd/download.eclipse.org'
 		EP_ECLIPSE_DROPS = "${EP_ROOT}/eclipse/downloads/drops4"
 		EP_EQUINOX_DROPS = "${EP_ROOT}/equinox/drops"
+		EP_ECLIPSE_UPDATES = "${EP_ROOT}/eclipse/updates"
 
 		SSH = 'ssh genie.releng@projects-storage.eclipse.org'
 	}
@@ -58,17 +59,14 @@ pipeline {
 						}
 					}
 				}
-				stage('Eclipse I-builds') {
+				stage('Eclipse') {
 					steps {
 						sshagent (['projects-storage.eclipse.org-bot-ssh']) {
+							// I-builds
 							removeOldBuildDropsOfCurrentStream("${EP_ECLIPSE_DROPS}", 'I', ECLIPSE_I_BUILD_RETENTION_DAYS, ECLIPSE_I_BUILD_RETENTION_COUNT)
-						}
-					}
-				}
-				stage('Eclipse Y-builds') {
-					steps {
-						sshagent (['projects-storage.eclipse.org-bot-ssh']) {
+							// Y-builds
 							removeSurplusBuildDrops("${EP_ECLIPSE_DROPS}", 'Y', ECLIPSE_Y_BUILD_RETENTION_COUNT)
+							removeSurplusDirectories("${EP_ECLIPSE_UPDATES}/${devVersionMajor}.${devVersionMinor}-Y-builds", 'Y*', ECLIPSE_Y_BUILD_RETENTION_COUNT)
 						}
 					}
 				}
@@ -93,10 +91,19 @@ def int devVersionService = null
 
 def removeSurplusBuildDrops(String remoteDirectory, String buildType, int retentionCount) {
 	def drops = utilities.listBuildDropDirectoriesOnRemote(remoteDirectory, "${buildType}*") // sorted in ascending order
-	if (retentionCount < drops.size()) {
-		utilities.removeDropsOnRemote(remoteDirectory, drops.subList(0, drops.size() - retentionCount))
+	removeSurplusLeadingDirectories(drops, remoteDirectory, "${buildType}*", retentionCount)
+}
+
+def removeSurplusDirectories(String remoteDirectory, String dirNamePattern, int retentionCount) {
+	def directories = utilities.listDirectoriesOnRemote(remoteDirectory, dirNamePattern) // sorted in ascending order
+	removeSurplusLeadingDirectories(directories, remoteDirectory, dirNamePattern, retentionCount)
+}
+
+def removeSurplusLeadingDirectories(List<String> directories, String remoteDirectory, String dirNamePattern, int retentionCount) {
+	if (directories.size() > retentionCount) {
+		utilities.removeDropsOnRemote(remoteDirectory, directories.subList(0, directories.size() - retentionCount))
 	} else {
-		echo "Nothing to clean in ${remoteDirectory} with pattern '${buildType}*'. Found only ${drops}, not exceeding the threshold of ${retentionCount}."
+		echo "Nothing to clean in ${remoteDirectory} with pattern '${dirNamePattern}'. Found only ${directories}, not exceeding the threshold of ${retentionCount}."
 	}
 }
 
@@ -105,7 +112,7 @@ def removeOldBuildDropsOfCurrentStream(String remoteDirectory, String buildType,
 	// But keep at least the minimal retention count of builds regardless of their age and all of those younger than the minimal retention time.
 	def retentionThresholdDate = java.time.LocalDate.now().minusDays(retentionDays)
 	def allBuilds = utilities.listBuildDropDirectoriesOnRemote(remoteDirectory, "${buildType}*",
-		devVersionMajor, devVersionMinor, devVersionService).sort() // sort ascending to start with Mondays
+		devVersionMajor, devVersionMinor, devVersionService) // sorted in ascending order -> starts with Mondays
 	echo "Number of ${buildType}-builds before cleaning: ${allBuilds.size()}"
 	def oldBuilds = allBuilds.findAll{ b -> parseDate(b) < retentionThresholdDate} // sorted ascendingly to start with Mondays
 	// Keep the specified count of latest builds

JenkinsJobs/Cleanup/cleanupReleaseArtifacts.jenkinsfile

@@ -77,9 +77,8 @@ pipeline {
 								// Keep Y-build p2-repositories for the latest release
 								removedUpdateSites.add("${versions.version}-Y-builds")
 							}
-							//TODO: maybe only keep the last 10 or so children, if this is the latest release? Regardless of a new Java-release is close or not
 						}
-						def allUpdateRepositories = utilities.listDirectoryContentOnRemote("${EP_ECLIPSE_UPDATES}")
+						def allUpdateRepositories = utilities.listDirectoriesOnRemote("${EP_ECLIPSE_UPDATES}")
 							.findAll{ d -> d ==~ /\d+\.\d+(\.\d+)?/ }
 							.collect{ v -> java.lang.Runtime.Version.parse(v.endsWith(".0") ? v.substring(0, v.length() - 2) : v) }
 							.sort().collect{ v -> v.toString() } // sort in ascending order

JenkinsJobs/shared/utilities.groovy

@@ -223,13 +223,21 @@ def List<String> listBuildDropDirectoriesOnRemote(String remoteDirectory, String
 	def result = sh(script: """ssh genie.releng@projects-storage.eclipse.org "cd ${remoteDirectory} && \
 		find -maxdepth 2 -type f -path './${dropNamePattern}/buildproperties.properties' \
 		${versionFilter} | xargs dirname | sort -u"
-	""", returnStdout: true).trim()
-	return result.isEmpty() ? [] : result.split('\\s+').collect{ d -> d.startsWith('./') ? d.substring(2) : d }
+	""", returnStdout: true)
+	return parsePathList(result)
 }
 
-def List<String> listDirectoryContentOnRemote(String remoteDirectory) {
-	def result = sh(script: "ssh genie.releng@projects-storage.eclipse.org 'ls ${remoteDirectory}'", returnStdout: true).trim()
-	return result.isEmpty() ? [] : result.split('\\s+').collect{ d -> d.endsWith('/') ? d.substring(0, d.length() - 1) : d }
+def List<String> listDirectoriesOnRemote(String remoteDirectory, String dirNamePattern = "*") {
+	def result = sh(script: "ssh genie.releng@projects-storage.eclipse.org 'cd ${remoteDirectory} && ls -d ${dirNamePattern}'", returnStdout: true)
+	return parsePathList(result)
+}
+
+@NonCPS
+def List<String> parsePathList(String list) {
+	list = list.trim()
+	return list.isEmpty() ? [] : list.split('\\s+').collect{ d ->
+		d.substring(d.startsWith('./') ? 2 : 0, d.length() - (d.endsWith('/') ? 1 : 0))
+	}.sort()
 }
 
 private void removeDropsOnRemote(String remoteDirectory, List<String> drops) {