Rework git-log section in build website

Inline the separate git-log page into the main page of a build as a
details section and simplify it:
Instead of listing all commits per changed repository, it just provide
links to the corresponding GitHub repositories, that compare the current
build's tag with the baseline tag and consequently let GH render the
commit list and diffs.
For I/Y-builds, the baseline tag is the previous I/Y-build (as it used
to be previously).
When promoting an I-build to a stable- or release-build, the
baseline-tag is updated to be the previous build of same kind (the list
of changed repositories is updated accordingly). So an S-build
(milestone or RC) is compared to the previous S-build and a release is
compared to the previous release.

Author: HannesWell

JenkinsJobs/Builds/build.jenkinsfile

@@ -134,9 +134,11 @@ pipeline {
 							git describe --tags --match "${BUILD_TYPE}*" --abbrev=0 > "${WORKSPACE}/lastTag"
 						'''
 					}
-					script { // Check for changes and abort if nothing changed since the last build
+					script {
+						assignEnvVariable('GIT_BASELINE_TAG', readFile("${WORKSPACE}/lastTag").trim())
+						// Check for changes and abort if nothing changed since the last build
 						def boolean isScheduledBuild = !currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause').isEmpty()
-						if (isScheduledBuild && sh(script: 'git diff $(cat "${WORKSPACE}/lastTag")..HEAD --name-only', returnStdout: true).trim().isEmpty()) {
+						if (isScheduledBuild && sh(script: 'git diff ${GIT_BASELINE_TAG}..HEAD --name-only', returnStdout: true).trim().isEmpty()) {
 							emailext subject: "${RELEASE_VER} ${BUILD_TYPE}-Build: ${BUILD_ID} - Build skipped", body: """\
 								No changes have been made since the last successful ${BUILD_TYPE}-Build and therefore this scheduled build was skipped:
 								${BUILD_URL}console
@@ -168,43 +170,10 @@ pipeline {
 						'''
 					}
 					// Git log creation
-					sh '''#!/bin/bash -xe
-						reportTimestamp=$(TZ='UTC' date +%Y%m%d-%H%M)
-						export lastTag=$(cat "${WORKSPACE}/lastTag")
-						if [[ -n "$lastTag" ]]; then
-							export gitLogFile="${DROP_DIR}/$BUILD_ID/gitLog.json"
-							mkdir -p $(dirname "${gitLogFile}")
-							
-							echo '{' > $gitLogFile
-							echo "\\"label\\": \\"${BUILD_ID}\\"," >> $gitLogFile
-							echo "\\"logFrom\\": \\"${lastTag}\\"," >> $gitLogFile
-							echo "\\"logTo\\": \\"${BUILD_ID}\\"," >> $gitLogFile
-							echo "\\"timestamp\\": \\"${reportTimestamp}\\"," >> $gitLogFile
-							echo "\\"repositories\\": [" >> $gitLogFile >> $gitLogFile
-							
-							function createGitLog() {
-								gitURL=$(git config remote.origin.url | sed --expression 's,git@github.com:,https://github.com/,' | sed 's/\\.git$//')
-								gitName="${gitURL##*/}"
-								#TODO: handle potential double-quotes in commit message?
-								gitLog=$(git log $lastTag..$BUILD_ID --date=short --format=format:"{ \\"date\\": \\"%cd\\", \\"message\\": \\"%s\\", \\"author\\": \\"%aN\\", \\"url\\": \\"${gitURL}/commit/%H\\"},")
-								if [ -n "$gitLog" ]; then
-									echo "Record changes in ${gitName}"
-									echo "{" >> $gitLogFile
-									echo "\\"name\\": \\"${gitName}\\"," >> $gitLogFile
-									echo "\\"commits\\": [" >> $gitLogFile
-									echo "${gitLog%,}" >> $gitLogFile
-									echo "]}," >> $gitLogFile
-								fi
-							}
-							export -f createGitLog
-							createGitLog
-							git submodule foreach 'createGitLog'
-							
-							echo "{}]}" >> $gitLogFile # Append empty object to keep JSON valid
-						else
-							echo -e "\n\tGit log not generated because a reasonable previous tag could not be found." > $gitLogFile
-						fi
-					'''
+					script {
+						def gitReposChanged = utilities.listChangedGitRepositoryURLs("${GIT_BASELINE_TAG}", "${BUILD_ID}")
+						assignEnvVariable('GIT_REPOS_CHANGED', gitReposChanged.join(','))
+					}
 				}
 			}
 		}
@@ -261,6 +230,7 @@ pipeline {
 					source $CJE_ROOT/buildproperties.shsource
 
 					# Gather maven properties
+					mkdir -p ${DROP_DIR}/$BUILD_ID
 					cp ${AGG_DIR}/eclipse-platform-parent/target/mavenproperties.properties ${DROP_DIR}/$BUILD_ID/mavenproperties.properties
 					comparatorRepo=$(grep '^comparator.repo=' ${DROP_DIR}/${BUILD_ID}/mavenproperties.properties | cut -d'=' -f2-)
 
@@ -439,6 +409,8 @@ pipeline {
 
 									java \
 										-DdropDirectory=${DROP_DIR}/${BUILD_ID} \
+										-DgitBaselineTag=${GIT_BASELINE_TAG} \
+										-DgitReposChanged=${GIT_REPOS_CHANGED} \
 										${SCRIPTS}/releng/BuildDropDataGenerator.java mainEclipse
 									java \
 										-DdropDirectory=${DROP_DIR}/${BUILD_ID} \

JenkinsJobs/Releng/promoteBuild.jenkinsfile

@@ -143,14 +143,29 @@ pipeline {
 
 					echo 'Promote Eclipse'
 					dir("${WORKSPACE}/eclipse") {
-						renameBuildDrop('eclipse/downloads/drops4', "${DROP_ID}", "${BUILD_LABEL}", "${DL_DROP_ID}", "${DL_LABEL}")
+						renameBuildDrop('eclipse/downloads/drops4', "${DROP_ID}", "${BUILD_LABEL}", "${DL_DROP_ID}", "${DL_LABEL}") { buildProperties ->
+							script {
+								def baselineBuilds = utilities.listBuildDropDirectoriesOnRemote("${EP_ROOT}/eclipse/downloads/drops4", "${DL_TYPE}*") // sorted in ascending order
+								if (baselineBuilds.isEmpty() && "${DL_TYPE}" == 'S') { // For M1 usually no previous stable build exists
+									baselineBuilds = utilities.listBuildDropDirectoriesOnRemote("${EP_ROOT}/eclipse/downloads/drops4", 'R*')
+								}
+								buildProperties.gitTag = utilities.stableBuildGitTag("${DL_DROP_ID}")
+								buildProperties.gitBaselineTag = utilities.stableBuildGitTag("${baselineBuilds.last()}")
+								echo "gitTag: ${buildProperties.gitTag}, gitBaselineTag: ${buildProperties.gitBaselineTag}"
+								dir("${WORKSPACE}/repository") {
+									def changedRepositories = utilities.listChangedGitRepositoryURLs(buildProperties.gitBaselineTag, buildProperties.gitTag)
+									echo "Changed repositories: ${changedRepositories}"
+									buildProperties.gitReposChanged = changedRepositories
+								}
+							}
+						}
 					}
 				}
 			}
 		}
 		stage('Tag Build') {
 			environment {
-				TAG = "${DL_TYPE}${BUILD_MAJOR}_${BUILD_MINOR}${(CHECKPOINT || BUILD_SERVICE != '0') ? ('_' + BUILD_SERVICE) : ''}${CHECKPOINT ? ('_' + CHECKPOINT) : ''}"
+				TAG = "${utilities.stableBuildGitTag(DL_DROP_ID)}"
 			}
 			steps {
 				dir("${WORKSPACE}/repository") {
@@ -312,7 +327,7 @@ pipeline {
 	}
 }
 
-def renameBuildDrop(String baseDropPath, String oldDropID, String oldBuildLabel, String newDropID, String newBuildLabel) {
+def renameBuildDrop(String baseDropPath, String oldDropID, String oldBuildLabel, String newDropID, String newBuildLabel, Closure extraModification = null) {
 	def sourcePath="${EP_ROOT}/${baseDropPath}/${oldDropID}"
 	def targetPath="${EP_ROOT}/${env.DRY_RUN_PREFIX ?: ''}${baseDropPath}/${newDropID}"
 
@@ -353,6 +368,9 @@ EOF
 	utilities.modifyJSON('buildproperties.json') { buildProperties ->
 		buildProperties.identifier = newDropID
 		buildProperties.label = newBuildLabel
+		if (extraModification) {
+			extraModification.call(buildProperties)
+		}
 		// Update filenames in lists
 		for (value in buildProperties.values()) {
 			if (value instanceof List) {

JenkinsJobs/shared/utilities.groovy

@@ -31,6 +31,16 @@ def matchBuildIdentifier(String dropID, Closure iBuildHandler, Closure sBuildHan
 	])
 }
 
+@NonCPS
+def String stableBuildGitTag(Map<String, String> id) {
+	def service = id.service ?: '0'
+	return "${id.type}${id.major}_${id.minor}${(id.checkpoint || service != '0') ? ('_' + service) : ''}${id.checkpoint ? ('_' + id.checkpoint) : ''}"
+}
+
+def stableBuildGitTag(CharSequence dropID) {
+	return stableBuildGitTag(matchBuildIdentifier(dropID, null, { }))
+}
+
 // --- local file modifications ---
 
 def replaceAllInFile(String filePath, Map<String,String> replacements) {
@@ -97,6 +107,27 @@ def forEachGitSubmodule(Closure task) {
 	}
 }
 
+def listChangedGitRepositoryURLs(String fromTag, String toTag) {
+	// Diff the relative path's of each sub-module from within the aggregator (not from within each submodule)
+	// to avoid the need to clone each submodule in full depth (just to test if something changed, not necessarily what changed)
+	def gitReposChanged = sh(script: """
+		echoURLOfChangedRepository() {
+			if ! git diff ${fromTag}..${toTag} --quiet \$1; then
+				pushd "\$1" > /dev/null
+				git config remote.origin.url
+				popd > /dev/null
+			fi
+		}
+		echoURLOfChangedRepository
+		for submodulePath in \$(git submodule foreach --quiet 'echo \$sm_path'); do
+			echoURLOfChangedRepository \$submodulePath
+		done
+	""", returnStdout: true).split('\\s+')
+	return gitReposChanged.collect{ url ->
+		return url.endsWith('.git') ? url.substring(0, url.length() - 4) : url
+	}
+}
+
 def gitCommitAllExcludingSubmodules(String commitMessage) {
 	withEnv(["COMMIT_MESSAGE=${commitMessage}"]) {
 		sh '''

scripts/releng/BuildDropDataGenerator.java

@@ -25,6 +25,11 @@
  * Considered JVM properties are:
  * <ul>
  * <li>{@code directory} (required) -- path to the drop directory root.</li>
+ * <li>{@code gitBaselineTag} (required for {@code mainEclipse}) -- the Git
+ * baseline tag.</li>
+ * <li>{@code gitReposChanged} (required for {@code mainEclipse}) -- The comma
+ * separated list of URLs of Git repositories that changed since the
+ * baseline.</li>
  * <li>{@code filesList} (optional) -- path to a file listing all files and
  * their size in the directory.<br>
  * Can be generated using the UNIX command
@@ -54,6 +59,9 @@ void main(String[] args) throws IOException {
 final Pattern COMMA = Pattern.compile(",");
 
 void mainEclipsePageData() throws IOException {
+	String gitBaselineTag = OS.readProperty("gitBaselineTag").trim();
+	List<String> gitReposChanged = List.of(OS.readProperty("gitReposChanged").trim().split(","));
+
 	Map<Path, Long> files = OS.listFileTree(DIRECTORY, FILES_LIST_FILE, null, 1);
 	Map<String, String> properties = OS.loadProperties(DIRECTORY.resolve("buildproperties.properties"));
 	String buildId = properties.get("BUILD_ID");
@@ -69,6 +77,16 @@ void mainEclipsePageData() throws IOException {
 	buildProperties.add("previousReleaseAPILabel", JSON.String.create(previousReleaseAPILabel(properties)));
 	buildProperties.add("timestamp", JSON.String.create(buildDate.toString()));
 
+	// git log
+	buildProperties.add("gitTag", JSON.String.create(buildId));
+	buildProperties.add("gitBaselineTag", JSON.String.create(gitBaselineTag));
+	if (!gitBaselineTag.isEmpty()) {
+		buildProperties.add("gitReposChanged",
+				gitReposChanged.stream().map(JSON.String::create).collect(JSON.Array.toJSONArray()));
+	} else {
+		IO.println("Git log not generated because a reasonable previous tag could not be found.");
+	}
+
 	// tests
 	JSON.Array testConfigurations = COMMA.splitAsStream(properties.get("TEST_CONFIGURATIONS_EXPECTED")).map(c -> {
 		var config = c.split("-");

sites/eclipse/build/gitLog.html

@@ -51,10 +51,16 @@ <h4 id="tests">Test Results Summary</h4>
 			</thead>
 		</table>
 
-		<h4 id="git-log">Git Log</h4>
-		<ul>
-			<li><a href="gitLog.html">Git log</a></li>
-		</ul>
+		<details>
+			<summary id="git-log" class="toc-level-4">Git Log</summary>
+			<div style="padding: 5px;">
+				<p>
+					The following Git repositories changed since <b class="data-ref">${gitBaselineTag}</b> until this build (<b class="data-ref">${gitTag}</b>):
+				<ul id="git-log-repo-list">
+				</ul>
+				</p>
+			</div>
+		</details>
 
 		<h4 id="signatures">Checksums&nbsp;(SHA 512) and GPG&nbsp;Signatures</h4>
 		<ul>
@@ -170,6 +176,19 @@ <h3 id="swt">SWT Binary and Source
 				For details, see <a href="${build.signOffIssue}">${signOffLabel}</a>.`
 			}
 
+			// Inject git log
+			const gitLogRepoList = document.getElementById('git-log-repo-list')
+			for (let repoURL of build.gitReposChanged) {
+				if (repoURL.endsWith('.git')) {
+					repoURL = repoURL.substring(0, repoURL.length - 4)
+				}
+
+				const repoName = repoURL.substring(repoURL.indexOf('github.com') + 10 + 1, repoURL.length)
+				const repoItem = document.createElement('li')
+				repoItem.innerHTML = `<b><a href="${repoURL}/compare/${build.gitBaselineTag}...${build.gitTag}">${repoName}</a></b>`
+				gitLogRepoList.appendChild(repoItem)
+			}
+
 			// Inject resource files
 			const dataTables = Array.from(mainElement.getElementsByClassName('files-table'))
 			for (const table of dataTables) {

sites/eclipse/build/index.html

@@ -2,6 +2,7 @@
 @import "https://fonts.googleapis.com/css2?family=Libre+Franklin:ital,wght@0,100;0,200;0,300;0,400;0,500;0,600;0,700;0,800;0,900;1,100;1,200;1,300;1,400;1,500;1,600;1,700;1,800;1,900&family=Roboto:ital,wght@0,100;0,300;0,400;0,500;0,700;0,900;1,100;1,300;1,400;1,500;1,700;1,900&display=swap";
 
 :root {
+	interpolate-size: allow-keywords;
 	--marker-error: red;
 	--marker-warning: orange;
 	--marker-info: dodgerblue;
@@ -78,6 +79,45 @@ li.toc-item a {
 	text-decoration: none;
 }
 
+/* Details sections */
+
+details {
+	background-color: whitesmoke;
+}
+
+details summary {
+	font-weight: bold;
+	background-color: lightsteelblue;
+	padding: 5px;
+	cursor: pointer;
+	/* Ensure state triangle in closed state */
+	display: list-item;
+	list-style-type: disclosure-closed;
+}
+
+details[open] summary {
+	/* Ensure state triangle in open state */
+	list-style-type: disclosure-open;
+}
+
+details summary:hover {
+	color: white;
+	background-color: #4B78B4;
+}
+
+/* Slide down/up animation when opening/closing details */
+details::details-content {
+	transition: height 0.5s ease, opacity 0.5s ease;
+	height: 0;
+	opacity: 0;
+	overflow: hidden;
+}
+
+details[open]::details-content {
+	height: auto;
+	opacity: 1;
+}
+
 .error-message-box {
 	background-color: #ffdddd;
 	border-left: 6px solid crimson;