chore: Add a back-off for Muzzle Version Range (#11433)

chore: Add a back-off for Muzzle Version Range

Co-authored-by: devflow.devflow-routing-intake <devflow.devflow-routing-intake@kubernetes.us1.ddbuild.io>

Author: bric3

buildSrc/src/main/kotlin/datadog/gradle/plugin/muzzle/MuzzleMavenRepoUtils.kt

@@ -9,16 +9,21 @@ import org.eclipse.aether.connector.basic.BasicRepositoryConnectorFactory
 import org.eclipse.aether.repository.LocalRepository
 import org.eclipse.aether.repository.RemoteRepository
 import org.eclipse.aether.resolution.VersionRangeRequest
+import org.eclipse.aether.resolution.VersionRangeResolutionException
 import org.eclipse.aether.resolution.VersionRangeResult
 import org.eclipse.aether.spi.connector.RepositoryConnectorFactory
 import org.eclipse.aether.spi.connector.transport.TransporterFactory
 import org.eclipse.aether.transport.file.FileTransporterFactory
 import org.eclipse.aether.transport.http.HttpTransporterFactory
 import org.eclipse.aether.version.Version
 import org.gradle.api.GradleException
+import org.gradle.api.logging.Logging
 import java.nio.file.Files
 
 internal object MuzzleMavenRepoUtils {
+  private val log = Logging.getLogger(MuzzleMavenRepoUtils::class.java)
+  private val backoffDelaysSeconds = listOf(5L, 10L, 30L)
+
   /**
    * Remote repositories used to query version ranges and fetch dependencies.
    *
@@ -122,12 +127,15 @@ internal object MuzzleMavenRepoUtils {
   /**
    * Resolves the version range for a given MuzzleDirective using the provided RepositorySystem and RepositorySystemSession.
    * Equivalent to the Groovy implementation in MuzzlePlugin.
+   *
+   * @param enableBackoffRetries if true, waits 5s, 10s, and 30s after the first three immediate retries
    */
   fun resolveVersionRange(
     muzzleDirective: MuzzleDirective,
     system: RepositorySystem,
     session: RepositorySystemSession,
-    defaultRepos: List<RemoteRepository> = defaultMuzzleRepos()
+    defaultRepos: List<RemoteRepository> = defaultMuzzleRepos(),
+    enableBackoffRetries: Boolean = true
   ): VersionRangeResult {
     val directiveArtifact: Artifact = DefaultArtifact(
       muzzleDirective.group,
@@ -142,16 +150,56 @@ internal object MuzzleMavenRepoUtils {
     }
 
     // In rare cases, the version resolution range silently failed with the maven proxy,
-    // retries 3 times at most then suggest to restart the job later.
-    var range = system.resolveVersionRange(session, rangeRequest)
-    for (i in 0..3) {
-      if (range.lowestVersion != null && range.highestVersion != null) {
+    // retries 3 times immediately, then backs off before suggesting to restart the job later.
+    var attemptCount = 0
+    var range: VersionRangeResult? = null
+    var failure: VersionRangeResolutionException? = null
+    fun attemptResolve(): VersionRangeResult? {
+      attemptCount++
+      return try {
+        range = system.resolveVersionRange(session, rangeRequest)
+        failure = null
+        range?.takeIf { it.hasBounds() }
+      } catch (e: VersionRangeResolutionException) {
+        failure = e
+        range = e.result ?: range
+        null
+      }
+    }
+
+    repeat(4) {
+      attemptResolve()?.let { range ->
         return range
       }
-      range = system.resolveVersionRange(session, rangeRequest)
     }
 
-    throw IllegalStateException("The version range resolution failed during report, this is not expected. Advised course of action: Restart the job later.")
+    var waitedSeconds = 0L
+    if (enableBackoffRetries) {
+      for (delaySeconds in backoffDelaysSeconds) {
+        sleepBeforeBackoffRetry(delaySeconds, directiveArtifact)
+        waitedSeconds += delaySeconds
+        attemptResolve()?.let { resolvedRange ->
+          log.warn(
+            "Muzzle version range resolution for ${artifactCoordinates(directiveArtifact)} " +
+              "succeeded after waiting ${waitedSeconds}s across $attemptCount attempts"
+          )
+          return resolvedRange
+        }
+      }
+    }
+
+    throw IllegalStateException(
+      versionRangeFailureMessage(
+        directiveArtifact,
+        rangeRequest.repositories,
+        range,
+        failure,
+        attemptCount,
+        waitedSeconds,
+        enableBackoffRetries
+      ),
+      failure
+    )
   }
 
   /**
@@ -205,6 +253,76 @@ internal object MuzzleMavenRepoUtils {
    */
   fun lowest(a: Version, b: Version): Version = if (a < b) a else b
 
+  private fun VersionRangeResult.hasBounds(): Boolean =
+    lowestVersion != null && highestVersion != null
+
+  private fun sleepBeforeBackoffRetry(delaySeconds: Long, artifact: Artifact) {
+    try {
+      Thread.sleep(delaySeconds * 1000L)
+    } catch (e: InterruptedException) {
+      Thread.currentThread().interrupt()
+      throw IllegalStateException(
+        "Interrupted while waiting ${delaySeconds}s before retrying version range resolution for " +
+          artifactCoordinates(artifact),
+        e
+      )
+    }
+  }
+
+  private fun versionRangeFailureMessage(
+    artifact: Artifact,
+    repositories: List<RemoteRepository>,
+    range: VersionRangeResult?,
+    failure: VersionRangeResolutionException?,
+    attemptCount: Int,
+    waitedSeconds: Long,
+    enableBackoffRetries: Boolean
+  ): String {
+    val backoffDetails =
+      if (enableBackoffRetries) {
+        "enabled; waited ${waitedSeconds}s using delays ${backoffDelaysSeconds.joinToString(", ") { "${it}s" }}"
+      } else {
+        "disabled"
+      }
+    return buildString {
+      appendLine("Muzzle version range resolution failed.")
+      appendLine("Artifact:")
+      appendLine("  ${artifactCoordinates(artifact)}")
+      appendLine("Repositories:")
+      repositories.forEach { appendLine("  - ${it.id}: ${it.url}") }
+      appendLine("Attempts:")
+      appendLine("  $attemptCount")
+      appendLine("Backoff:")
+      appendLine("  $backoffDetails")
+      appendLine("Last resolution result:")
+      if (range == null) {
+        appendLine("  <none returned>")
+      } else {
+        appendLine("  lowestVersion=${range.lowestVersion ?: "<missing>"}")
+        appendLine("  highestVersion=${range.highestVersion ?: "<missing>"}")
+        appendLine("  versionCount=${range.versions.size}")
+      }
+      if (failure != null) {
+        appendLine("Last resolution failure:")
+        appendLine("  ${failure.javaClass.name}: ${failure.message ?: "<no message>"}")
+      }
+      appendLine()
+      appendLine("Maven metadata resolution may have returned an incomplete range, especially through a proxy.")
+      appendLine("Restart the job later if the repositories above are reachable.")
+    }.trimEnd()
+  }
+
+  private fun artifactCoordinates(artifact: Artifact): String {
+    val classifier = artifact.classifier?.takeUnless { it.isEmpty() }
+    return listOfNotNull(
+      artifact.groupId,
+      artifact.artifactId,
+      classifier,
+      artifact.extension,
+      artifact.version
+    ).joinToString(":")
+  }
+
   /**
    * Convert a muzzle directive to a set of artifacts for all filtered versions.
    * Throws GradleException if no artifacts are found.

buildSrc/src/test/kotlin/datadog/gradle/plugin/muzzle/MuzzleMavenRepoUtilsTest.kt

@@ -1,9 +1,11 @@
 package datadog.gradle.plugin.muzzle
 
 import datadog.gradle.plugin.MavenRepoFixture
+import org.eclipse.aether.RepositorySystem
 import org.eclipse.aether.artifact.DefaultArtifact
 import org.eclipse.aether.repository.RemoteRepository
 import org.eclipse.aether.resolution.VersionRangeRequest
+import org.eclipse.aether.resolution.VersionRangeResolutionException
 import org.eclipse.aether.resolution.VersionRangeResult
 import org.eclipse.aether.util.version.GenericVersionScheme
 import org.gradle.api.GradleException
@@ -12,6 +14,8 @@ import org.junit.jupiter.api.io.TempDir
 import org.junit.jupiter.params.ParameterizedTest
 import org.junit.jupiter.params.provider.CsvSource
 import java.io.File
+import java.lang.reflect.Proxy
+import java.util.concurrent.atomic.AtomicInteger
 import org.assertj.core.api.Assertions.assertThat
 import org.assertj.core.api.Assertions.assertThatThrownBy
 
@@ -54,6 +58,32 @@ class MuzzleMavenRepoUtilsTest {
     assertThat(resolvedVersions).containsExactly("2.0.0", "3.0.0")
   }
 
+  @Test
+  fun `resolveVersionRange retries thrown resolution failures`() {
+    val directive = MuzzleDirective().apply {
+      group = "com.example"
+      module = "mylib"
+      versions = "[1.0,)"
+    }
+    val attempts = AtomicInteger()
+    val retryingSystem = repositorySystemThrowingThenResolving(
+      failuresBeforeSuccess = 3,
+      result = createVersionRangeResult("1.0.0"),
+      attempts = attempts
+    )
+
+    val result = MuzzleMavenRepoUtils.resolveVersionRange(
+      directive,
+      retryingSystem,
+      newSession(),
+      emptyList(),
+      enableBackoffRetries = false
+    )
+
+    assertThat(result.versions.map { it.toString() }).containsExactly("1.0.0")
+    assertThat(attempts).hasValue(4)
+  }
+
   @Test
   fun `resolveVersionRange throws IllegalStateException when resolution consistently fails`() {
     val emptyRepo = RemoteRepository.Builder("empty", "default", File(tempDir, "empty").apply { mkdirs() }.toURI().toString()).build()
@@ -64,8 +94,49 @@ class MuzzleMavenRepoUtilsTest {
     }
 
     assertThatThrownBy {
-      MuzzleMavenRepoUtils.resolveVersionRange(directive, system, newSession(), listOf(emptyRepo))
+      MuzzleMavenRepoUtils.resolveVersionRange(
+        directive,
+        system,
+        newSession(),
+        listOf(emptyRepo),
+        enableBackoffRetries = false
+      )
     }.isInstanceOf(IllegalStateException::class.java)
+      .hasMessageContaining("Muzzle version range resolution failed")
+      .hasMessageContaining("com.example:nonexistent:jar:[1.0,)")
+      .hasMessageContaining("empty:")
+      .hasMessageContaining("Attempts:\n  4")
+      .hasMessageContaining("Backoff:\n  disabled")
+  }
+
+  @Test
+  fun `resolveVersionRange failure includes thrown resolution failure details`() {
+    val directive = MuzzleDirective().apply {
+      group = "com.example"
+      module = "mylib"
+      versions = "[1.0,)"
+    }
+    val attempts = AtomicInteger()
+    val throwingSystem = repositorySystemThrowingThenResolving(
+      failuresBeforeSuccess = 4,
+      result = createVersionRangeResult("1.0.0"),
+      attempts = attempts
+    )
+
+    assertThatThrownBy {
+      MuzzleMavenRepoUtils.resolveVersionRange(
+        directive,
+        throwingSystem,
+        newSession(),
+        emptyList(),
+        enableBackoffRetries = false
+      )
+    }.isInstanceOf(IllegalStateException::class.java)
+      .hasCauseInstanceOf(VersionRangeResolutionException::class.java)
+      .hasMessageContaining("Attempts:\n  4")
+      .hasMessageContaining("Last resolution failure:")
+      .hasMessageContaining("transient version range failure 4")
+    assertThat(attempts).hasValue(4)
   }
 
   @Test
@@ -224,4 +295,30 @@ class MuzzleMavenRepoUtilsTest {
     // lowestVersion/highestVersion are computed as versions[0] and versions[last]
     return VersionRangeResult(request).apply { this.versions = versions }
   }
+
+  private fun repositorySystemThrowingThenResolving(
+    failuresBeforeSuccess: Int,
+    result: VersionRangeResult,
+    attempts: AtomicInteger
+  ): RepositorySystem =
+    Proxy.newProxyInstance(
+      RepositorySystem::class.java.classLoader,
+      arrayOf(RepositorySystem::class.java)
+    ) { _, method, args ->
+      when (method.name) {
+        "resolveVersionRange" -> {
+          val attempt = attempts.incrementAndGet()
+          if (attempt <= failuresBeforeSuccess) {
+            val request = args?.get(1) as VersionRangeRequest
+            throw VersionRangeResolutionException(
+              VersionRangeResult(request),
+              "transient version range failure $attempt"
+            )
+          }
+          result
+        }
+        "toString" -> "repositorySystemThrowingThenResolving"
+        else -> throw UnsupportedOperationException(method.name)
+      }
+    } as RepositorySystem
 }