Default to Teamscale Compact Coverage in NORMAL mode

CHANGELOG.md

@@ -5,6 +5,7 @@ We use [semantic versioning](http://semver.org/):
 - PATCH version when you make backwards compatible bug fixes.
 
 # Next version
+- [breaking feature] _agent_: Coverage is now reported in the Teamscale Compact Coverage format by default in NORMAL mode, which is significantly smaller and faster to upload. Set `report-format=JACOCO` in the agent options to keep the previous JaCoCo XML behavior. The `convert` CLI now picks the format from the output file extension (`.xml` for JaCoCo XML, `.json` for Teamscale Compact Coverage).
 - [breaking] _report-generator_: `RevisionInfo` is now a sealed class with polymorphic Jackson serialization (`@JsonTypeInfo` / `@JsonSubTypes`). The JSON representation now uses "COMMIT" and "REVISION" as type discriminator values instead of the previous `ERevisionType` enum names.
 - [feature] _agent_: Added official support for Java 26 and experimental support for Java 27 (via JaCoCo 0.8.15)
 

agent/src/main/kotlin/com/teamscale/jacoco/agent/Agent.kt

@@ -10,7 +10,6 @@ import com.teamscale.jacoco.agent.upload.teamscale.TeamscaleUploader
 import com.teamscale.jacoco.agent.util.AgentUtils
 import com.teamscale.report.jacoco.CoverageFile
 import com.teamscale.report.jacoco.EmptyReportException
-import com.teamscale.report.jacoco.JaCoCoXmlReportGenerator
 import com.teamscale.report.jacoco.dump.Dump
 import org.glassfish.jersey.server.ResourceConfig
 import org.glassfish.jersey.server.ServerProperties
@@ -149,22 +148,17 @@ class Agent(options: AgentOptions, instrumentation: Instrumentation?) : AgentBas
 			return
 		}
 
-		val generator = JaCoCoXmlReportGenerator(
-			options.classDirectoriesOrZips,
-			options.locationIncludeFilter,
-			options.duplicateClassFileBehavior,
-			options.ignoreUncoveredClasses,
-			LoggingUtils.wrap(logger)
-		)
+		val format = options.normalModeReportFormat
+		val generator = options.createReportGenerator(LoggingUtils.wrap(logger))
 
 		try {
-			benchmark("Generating the XML report") {
-				val outputFile = options.createNewFileInOutputDirectory("jacoco", "xml")
+			benchmark("Generating the coverage report") {
+				val outputFile = options.createNewFileInOutputDirectory(format.fileNamePrefix, format.fileExtension)
 				val coverageFile = generator.convertSingleDumpToReport(dump, outputFile)
 				uploader.upload(coverageFile)
 			}
 		} catch (e: IOException) {
-			logger.error("Converting binary dump to XML failed", e)
+			logger.error("Converting binary dump to coverage report failed", e)
 		} catch (e: EmptyReportException) {
 			logger.error("No coverage was collected. ${e.message}", e)
 		}

agent/src/main/kotlin/com/teamscale/jacoco/agent/convert/ConvertCommand.kt

@@ -18,8 +18,10 @@ import java.io.IOException
  * Encapsulates all command line options for the convert command for parsing with [JCommander].
  */
 @Parameters(
-	commandNames = ["convert"], commandDescription = "Converts a binary .exec coverage file to XML. " +
-			"Note that the XML report will only contain source file coverage information, but no class coverage."
+	commandNames = ["convert"], commandDescription = "Converts a binary .exec coverage file to a coverage report. " +
+			"The output format is selected by the extension of --out: .xml for JaCoCo XML, .json for Teamscale " +
+			"Compact Coverage. Note that the report will only contain source file coverage information, but no " +
+			"class coverage."
 )
 class ConvertCommand : ICommand {
 	/** The directories and/or zips that contain all class files being profiled.  */
@@ -65,11 +67,13 @@ class ConvertCommand : ICommand {
 	)
 	var inputFiles = mutableListOf<String>()
 
-	/** The directory to write the XML traces to.  */
+	/** The output file. The extension selects the format: .xml = JaCoCo XML, .json = Teamscale Compact Coverage.  */
 	@JvmField
 	@Parameter(
 		names = ["--out", "-o"], required = true, description = (""
-				+ "The file to write the generated XML report to.")
+				+ "The file to write the generated coverage report to. The file extension selects the format: "
+				+ ".xml for JaCoCo XML, .json for Teamscale Compact Coverage. When --testwise-coverage is set, "
+				+ "testwise JSON is produced regardless of the extension.")
 	)
 	var outputFile = ""
 
@@ -86,7 +90,8 @@ class ConvertCommand : ICommand {
 	@Parameter(
 		names = ["--ignore-uncovered-classes"], required = false, arity = 1, description = (""
 				+ "Whether to ignore uncovered classes."
-				+ " These classes will not be part of the XML report at all, making it considerably smaller in some cases. Defaults to false.")
+				+ " These classes will not be part of the JaCoCo XML report at all, making it considerably smaller in some cases. Defaults to false. "
+				+ "Has no effect when the output format is Teamscale Compact Coverage, which never includes uncovered classes.")
 	)
 	var shouldIgnoreUncoveredClasses = false
 

agent/src/main/kotlin/com/teamscale/jacoco/agent/convert/Converter.kt

@@ -4,10 +4,10 @@ import com.teamscale.client.TestDetails
 import com.teamscale.jacoco.agent.benchmark
 import com.teamscale.jacoco.agent.logging.LoggingUtils
 import com.teamscale.jacoco.agent.options.AgentOptionParseException
+import com.teamscale.jacoco.agent.options.EAgentReportFormat
 import com.teamscale.report.ReportUtils
 import com.teamscale.report.ReportUtils.listFiles
 import com.teamscale.report.jacoco.EmptyReportException
-import com.teamscale.report.jacoco.JaCoCoXmlReportGenerator
 import com.teamscale.report.testwise.ETestArtifactFormat
 import com.teamscale.report.testwise.TestwiseCoverageReportWriter
 import com.teamscale.report.testwise.jacoco.JaCoCoTestwiseReportGenerator
@@ -28,11 +28,20 @@ class Converter
 	/** The command line arguments.  */
 	private val arguments: ConvertCommand
 ) {
-	/** Converts one .exec binary coverage file to XML.  */
-	@Throws(IOException::class)
+	/**
+	 * Converts one .exec binary coverage file to a coverage report. The output format is selected based on the output
+	 * file extension: `.xml` produces a JaCoCo XML report, `.json` produces a Teamscale Compact Coverage report.
+	 */
+	@Throws(IOException::class, AgentOptionParseException::class)
 	fun runJaCoCoReportGeneration() {
 		val logger = LoggingUtils.getLogger(this)
-		val generator = JaCoCoXmlReportGenerator(
+		val outputFile = Paths.get(arguments.outputFile).toFile()
+		val format = EAgentReportFormat.fromFileExtension(outputFile.extension)
+			?: throw AgentOptionParseException(
+				"Unsupported output file extension '${outputFile.extension}' for '${arguments.outputFile}'. " +
+						"Use .xml for JaCoCo XML or .json for Teamscale Compact Coverage."
+			)
+		val generator = format.createGenerator(
 			arguments.getClassDirectoriesOrZips(),
 			wildcardIncludeExcludeFilter,
 			arguments.duplicateClassFileBehavior,
@@ -42,8 +51,8 @@ class Converter
 
 		val jacocoExecutionDataList = listFiles(ETestArtifactFormat.JACOCO, arguments.getInputFiles())
 		try {
-			benchmark("Generating the XML report") {
-				generator.convertExecFilesToReport(jacocoExecutionDataList, Paths.get(arguments.outputFile).toFile())
+			benchmark("Generating the coverage report") {
+				generator.convertExecFilesToReport(jacocoExecutionDataList, outputFile)
 			}
 		} catch (e: EmptyReportException) {
 			logger.warn("Converted report was empty.", e)

agent/src/main/kotlin/com/teamscale/jacoco/agent/options/AgentOptions.kt

@@ -34,6 +34,7 @@ import com.teamscale.jacoco.agent.upload.teamscale.TeamscaleUploader
 import com.teamscale.jacoco.agent.util.AgentUtils
 import com.teamscale.jacoco.agent.util.AgentUtils.mainTempDirectory
 import com.teamscale.report.EDuplicateClassFileBehavior
+import com.teamscale.report.jacoco.JaCoCoBasedReportGenerator
 import com.teamscale.report.util.ClasspathWildcardIncludeFilter
 import com.teamscale.report.util.ILogger
 import java.io.File
@@ -111,6 +112,13 @@ open class AgentOptions(private val logger: ILogger) {
 	@JvmField
 	var mode = EMode.NORMAL
 
+	/**
+	 * The report format the agent produces in NORMAL mode.
+	 * Has no effect in TESTWISE mode, which always produces a testwise coverage JSON.
+	 */
+	@JvmField
+	var normalModeReportFormat = EAgentReportFormat.TEAMSCALE_COMPACT_COVERAGE
+
 	/** The interval in minutes for dumping XML data. */
 	@JvmField
 	var dumpIntervalInMinutes = 480
@@ -419,7 +427,8 @@ open class AgentOptions(private val logger: ILogger) {
 		EUploadMethod.ARTIFACTORY -> createArtifactoryUploader(instrumentation)
 		EUploadMethod.AZURE_FILE_STORAGE -> AzureFileStorageUploader(
 			azureFileStorageConfig,
-			additionalMetaDataFiles
+			additionalMetaDataFiles,
+			reportFormat
 		)
 		EUploadMethod.SAP_NWDI_TEAMSCALE -> {
 			logger.info("NWDI configuration detected. The Agent will try and auto-detect commit information by searching all profiled Jar/War/Ear/... files.")
@@ -452,7 +461,7 @@ open class AgentOptions(private val logger: ILogger) {
 
 	private fun createTeamscaleSingleProjectUploader(instrumentation: Instrumentation?): IUploader {
 		if (teamscaleServer.hasCommitOrRevision()) {
-			return TeamscaleUploader(teamscaleServer)
+			return TeamscaleUploader(teamscaleServer, reportFormat)
 		}
 
 		val uploader = createDelayedSingleProjectTeamscaleUploader()
@@ -478,7 +487,7 @@ open class AgentOptions(private val logger: ILogger) {
 	private fun createTeamscaleMultiProjectUploader(
 		instrumentation: Instrumentation?
 	): DelayedTeamscaleMultiProjectUploader {
-		val uploader = DelayedTeamscaleMultiProjectUploader { project, commitInfo ->
+		val uploader = DelayedTeamscaleMultiProjectUploader(reportFormat) { project, commitInfo ->
 			if (commitInfo!!.preferCommitDescriptorOverRevision || isEmpty(commitInfo.revision)) {
 				return@DelayedTeamscaleMultiProjectUploader teamscaleServer.withProjectAndCommit(
 					project!!,
@@ -544,7 +553,7 @@ open class AgentOptions(private val logger: ILogger) {
 			} else {
 				teamscaleServer.revision = projectAndCommit.commitInfo.revision
 			}
-			TeamscaleUploader(teamscaleServer)
+			TeamscaleUploader(teamscaleServer, reportFormat)
 		}
 
 	private fun startMultiGitPropertiesFileSearchInJarFile(
@@ -585,7 +594,8 @@ open class AgentOptions(private val logger: ILogger) {
 	private fun createNwdiTeamscaleUploader(instrumentation: Instrumentation?): IUploader {
 		val uploader = DelayedSapNwdiMultiUploader { commit, application ->
 			TeamscaleUploader(
-				teamscaleServer.withProjectAndCommit(application.teamscaleProject, commit)
+				teamscaleServer.withProjectAndCommit(application.teamscaleProject, commit),
+				reportFormat
 			)
 		}
 		instrumentation?.addTransformer(
@@ -599,7 +609,17 @@ open class AgentOptions(private val logger: ILogger) {
 	private val reportFormat: EReportFormat
 		get() = if (useTestwiseCoverageMode()) {
 			EReportFormat.TESTWISE_COVERAGE
-		} else EReportFormat.JACOCO
+		} else normalModeReportFormat.reportFormat
+
+	/** Creates the report generator used by the agent in NORMAL mode based on [normalModeReportFormat]. */
+	fun createReportGenerator(logger: ILogger): JaCoCoBasedReportGenerator<*> =
+		normalModeReportFormat.createGenerator(
+			classDirectoriesOrZips,
+			locationIncludeFilter,
+			duplicateClassFileBehavior,
+			ignoreUncoveredClasses,
+			logger
+		)
 
 	/**
 	 * Creates a new file with the given prefix, extension and current timestamp and ensures that the parent folder

agent/src/main/kotlin/com/teamscale/jacoco/agent/options/AgentOptionsParser.kt

@@ -311,6 +311,10 @@ class AgentOptionsParser @VisibleForTesting internal constructor(
 				options.mode = parseEnumValue(key, value)
 				return true
 			}
+			"report-format" -> {
+				options.normalModeReportFormat = parseEnumValue(key, value)
+				return true
+			}
 			"includes" -> {
 				options.jacocoIncludes = value.replace(";".toRegex(), ":")
 				return true

agent/src/main/kotlin/com/teamscale/jacoco/agent/options/EAgentReportFormat.kt

@@ -0,0 +1,53 @@
+package com.teamscale.jacoco.agent.options
+
+import com.teamscale.client.EReportFormat
+import com.teamscale.report.EDuplicateClassFileBehavior
+import com.teamscale.report.compact.CompactCoverageReportGenerator
+import com.teamscale.report.jacoco.JaCoCoBasedReportGenerator
+import com.teamscale.report.jacoco.JaCoCoXmlReportGenerator
+import com.teamscale.report.util.ClasspathWildcardIncludeFilter
+import com.teamscale.report.util.ILogger
+import java.io.File
+
+/**
+ * The coverage report format the agent produces in NORMAL mode.
+ *
+ * Each entry bundles the downstream [EReportFormat] used during upload with the on-disk file naming
+ * convention so that filename, format identifier, and zip-entry name cannot drift apart.
+ */
+enum class EAgentReportFormat(
+	/** The [EReportFormat] identifier sent to the upload backend.  */
+	@JvmField val reportFormat: EReportFormat,
+	/** Filename prefix for files written to the output directory.  */
+	@JvmField val fileNamePrefix: String,
+	/** Filename extension (without leading dot) for files written to the output directory.  */
+	@JvmField val fileExtension: String,
+) {
+	/** Teamscale Compact Coverage (JSON).  */
+	TEAMSCALE_COMPACT_COVERAGE(EReportFormat.TEAMSCALE_COMPACT_COVERAGE, "compact-coverage", "json"),
+	/** JaCoCo XML.  */
+	JACOCO(EReportFormat.JACOCO, "jacoco", "xml");
+
+	/** Creates the report generator that produces this format from JaCoCo binary execution data. */
+	fun createGenerator(
+		classDirectoriesOrArchives: List<File>,
+		locationIncludeFilter: ClasspathWildcardIncludeFilter,
+		duplicateClassFileBehavior: EDuplicateClassFileBehavior,
+		ignoreUncoveredClasses: Boolean,
+		logger: ILogger
+	): JaCoCoBasedReportGenerator<*> = when (this) {
+		TEAMSCALE_COMPACT_COVERAGE -> CompactCoverageReportGenerator(
+			classDirectoriesOrArchives, locationIncludeFilter, duplicateClassFileBehavior, logger
+		)
+		JACOCO -> JaCoCoXmlReportGenerator(
+			classDirectoriesOrArchives, locationIncludeFilter, duplicateClassFileBehavior, ignoreUncoveredClasses, logger
+		)
+	}
+
+	companion object {
+		/** Returns the format whose [fileExtension] matches the given extension (case-insensitive), or `null`. */
+		@JvmStatic
+		fun fromFileExtension(extension: String): EAgentReportFormat? =
+			entries.firstOrNull { it.fileExtension.equals(extension, ignoreCase = true) }
+	}
+}

agent/src/main/kotlin/com/teamscale/jacoco/agent/upload/HttpZipUploaderBase.kt

@@ -138,6 +138,6 @@ abstract class HttpZipUploaderBase<T>
 	}
 
 	protected open fun getZipEntryCoverageFileName(coverageFile: CoverageFile): String {
-		return "coverage.xml"
+		return coverageFile.name
 	}
 }

agent/src/main/kotlin/com/teamscale/jacoco/agent/upload/azure/AzureFileStorageUploader.kt

@@ -30,7 +30,8 @@ import java.util.regex.Pattern
 /** Uploads the coverage archive to a provided azure file storage.  */
 class AzureFileStorageUploader(
 	config: AzureFileStorageConfig,
-	additionalMetaDataFiles: List<Path>
+	additionalMetaDataFiles: List<Path>,
+	private val reportFormat: EReportFormat
 ) : HttpZipUploaderBase<IAzureUploadApi>(
 	config.url!!,
 	additionalMetaDataFiles,
@@ -139,7 +140,7 @@ class AzureFileStorageUploader(
 
 	/** Creates a file name for the zip-archive containing the coverage.  */
 	private fun createFileName() =
-		"${EReportFormat.JACOCO.name.lowercase(Locale.getDefault())}-${System.currentTimeMillis()}.zip"
+		"${reportFormat.name.lowercase(Locale.getDefault())}-${System.currentTimeMillis()}.zip"
 
 	/** Checks if the file with the given name exists  */
 	@Throws(IOException::class, UploaderException::class)

agent/src/main/kotlin/com/teamscale/jacoco/agent/upload/delay/DelayedUploader.kt

@@ -1,6 +1,7 @@
 package com.teamscale.jacoco.agent.upload.delay
 
 import com.teamscale.jacoco.agent.logging.LoggingUtils.getLogger
+import com.teamscale.jacoco.agent.options.EAgentReportFormat
 import com.teamscale.jacoco.agent.upload.IUploader
 import com.teamscale.jacoco.agent.util.DaemonThreadFactory
 import com.teamscale.report.jacoco.CoverageFile
@@ -90,17 +91,19 @@ class DelayedUploader<T> internal constructor(
 	private fun uploadCachedXmls() {
 		try {
 			if (!cacheDir.isDirectory()) {
-				// Found data before XML was dumped
+				// Found data before any coverage was dumped
 				return
 			}
-			val xmlFiles = cacheDir.listDirectoryEntries().filter { path ->
+			val coverageFiles = cacheDir.listDirectoryEntries().filter { path ->
 				val fileName = path.fileName.toString()
-				fileName.startsWith("jacoco-") && fileName.endsWith(".xml")
+				EAgentReportFormat.entries.any { format ->
+					fileName.startsWith("${format.fileNamePrefix}-") && fileName.endsWith(".${format.fileExtension}")
+				}
 			}
-			xmlFiles.forEach { path -> wrappedUploader?.upload(CoverageFile(path.toFile())) }
-			logger.debug("Finished upload of cached XMLs to {}", wrappedUploader?.describe())
+			coverageFiles.forEach { path -> wrappedUploader?.upload(CoverageFile(path.toFile())) }
+			logger.debug("Finished upload of cached coverage files to {}", wrappedUploader?.describe())
 		} catch (e: IOException) {
-			logger.error("Failed to list cached coverage XML files in {}", cacheDir.toAbsolutePath(), e)
+			logger.error("Failed to list cached coverage files in {}", cacheDir.toAbsolutePath(), e)
 		}
 	}
 }

agent/src/main/kotlin/com/teamscale/jacoco/agent/upload/teamscale/DelayedTeamscaleMultiProjectUploader.kt

@@ -1,5 +1,6 @@
 package com.teamscale.jacoco.agent.upload.teamscale
 
+import com.teamscale.client.EReportFormat
 import com.teamscale.client.TeamscaleServer
 import com.teamscale.jacoco.agent.commit_resolution.git_properties.CommitInfo
 import com.teamscale.jacoco.agent.options.ProjectAndCommit
@@ -9,6 +10,7 @@ import java.io.File
 
 /** Wrapper for [TeamscaleUploader] that allows to upload the same coverage file to multiple Teamscale projects.  */
 class DelayedTeamscaleMultiProjectUploader(
+	private val reportFormat: EReportFormat,
 	private val teamscaleServerFactory: (String?, CommitInfo?) -> TeamscaleServer
 ) : DelayedMultiUploaderBase(), IUploader {
 	@JvmField
@@ -31,7 +33,7 @@ class DelayedTeamscaleMultiProjectUploader(
 			)
 			return
 		}
-		teamscaleUploaders.add(TeamscaleUploader(teamscaleServer))
+		teamscaleUploaders.add(TeamscaleUploader(teamscaleServer, reportFormat))
 	}
 
 	override val wrappedUploaders: MutableCollection<IUploader>