Skip to content

Commit 5c92c57

Browse files
GedochaoGedochao
authored
Merge pull request #4272 from Gedochao/fix/3793
Improve dependency update actionable diagnostic handling for slow/unresponsive custom repositories
1 parent 6439993 commit 5c92c57

4 files changed

Lines changed: 344 additions & 10 deletions

File tree

‎modules/build/src/test/scala/scala/build/tests/ActionableDiagnosticTests.scala‎

Lines changed: 272 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,23 @@
11
package scala.build.tests
22

33
import com.eed3si9n.expecty.Expecty.expect
4+
import com.sun.net.httpserver.HttpServer
5+
import coursier.cache.FileCache
6+
import coursier.util.Task
47
import coursier.version.Version
58

9+
import java.net.InetSocketAddress
10+
import java.util.UUID
11+
import java.util.concurrent.atomic.AtomicBoolean
12+
import java.util.concurrent.{ConcurrentLinkedQueue, Executors}
13+
614
import scala.build.Ops.*
715
import scala.build.Position.File
816
import scala.build.actionable.ActionableDiagnostic.*
917
import scala.build.actionable.ActionablePreprocessor
1018
import scala.build.options.{BuildOptions, InternalOptions, SuppressWarningOptions}
1119
import scala.build.{BuildThreads, Directories, LocalRepo}
20+
import scala.jdk.CollectionConverters.*
1221

1322
class ActionableDiagnosticTests extends TestUtil.ScalaCliBuildSuite {
1423

@@ -23,6 +32,50 @@ class ActionableDiagnosticTests extends TestUtil.ScalaCliBuildSuite {
2332

2433
def path2url(p: os.Path): String = p.toIO.toURI.toURL.toString
2534

35+
/** Minimal HTTP Maven repo: records every request path, then optional delay on
36+
* `maven-metadata.xml` when `delayWhen()` is true, then serves a body from `responses` or 404.
37+
*/
38+
def withRecordingMavenRepo(
39+
responses: Map[String, Array[Byte]],
40+
delayOnMetadataMs: Long = 0,
41+
delayWhen: () => Boolean = () => false
42+
)(body: (String, ConcurrentLinkedQueue[String]) => Unit): Unit =
43+
val recorded = new ConcurrentLinkedQueue[String]()
44+
val address = "127.0.0.1"
45+
val server = HttpServer.create(new InetSocketAddress(address, 0), 0)
46+
server.setExecutor(Executors.newCachedThreadPool())
47+
server.createContext(
48+
"/",
49+
ex => {
50+
val path = ex.getRequestURI.getPath
51+
recorded.offer(path)
52+
if delayOnMetadataMs > 0 && delayWhen() && path.endsWith("maven-metadata.xml") then
53+
Thread.sleep(delayOnMetadataMs)
54+
responses.get(path) match
55+
case Some(bytes) =>
56+
ex.getResponseHeaders.set("Content-Type", "application/xml")
57+
ex.sendResponseHeaders(200, bytes.length)
58+
ex.getResponseBody.write(bytes)
59+
ex.getResponseBody.close()
60+
case None =>
61+
ex.sendResponseHeaders(404, -1)
62+
ex.close()
63+
}
64+
)
65+
server.start()
66+
try
67+
val base = s"http://$address:${server.getAddress.getPort}/"
68+
body(base, recorded)
69+
finally server.stop(0)
70+
71+
def buildOptionsWithEmptyCoursierCache(opts: BuildOptions): BuildOptions =
72+
val dir = os.temp.dir(prefix = "scala-cli-actionable-diagnostic-coursier-")
73+
opts.copy(internal =
74+
opts.internal.copy(
75+
cache = Some(FileCache[Task]().withLocation(dir.toString))
76+
)
77+
)
78+
2679
test("using outdated os-lib") {
2780
val dependencyOsLib = "com.lihaoyi::os-lib:0.7.8"
2881
val testInputs = TestInputs(
@@ -263,4 +316,223 @@ class ActionableDiagnosticTests extends TestUtil.ScalaCliBuildSuite {
263316
expect(testLibDiagnosticOpt.isEmpty)
264317
}
265318
}
319+
320+
test("actionable outdated check for toolkit skips user repository metadata") {
321+
val meta =
322+
"""<?xml version="1.0" encoding="UTF-8"?>
323+
|<metadata>
324+
| <groupId>org.scala-lang</groupId>
325+
| <artifactId>toolkit_3</artifactId>
326+
| <versioning>
327+
| <latest>99.0.0</latest>
328+
| <release>99.0.0</release>
329+
| <versions>
330+
| <version>0.3.0</version>
331+
| <version>99.0.0</version>
332+
| </versions>
333+
| </versioning>
334+
|</metadata>
335+
|""".stripMargin.getBytes("UTF-8")
336+
val responses = Map("/org/scala-lang/toolkit_3/maven-metadata.xml" -> meta)
337+
withRecordingMavenRepo(responses)((repoUrl, recorded) =>
338+
val testInputs = TestInputs(
339+
os.rel / "Foo.scala" ->
340+
"""//> using toolkit 0.3.0
341+
|
342+
|object Hello extends App {
343+
| println("Hello")
344+
|}
345+
|""".stripMargin
346+
)
347+
val withRepo = baseOptions.copy(
348+
classPathOptions =
349+
baseOptions.classPathOptions.copy(extraRepositories = Seq(repoUrl))
350+
)
351+
testInputs.withBuild(withRepo, buildThreads, None, actionableDiagnostics = true) {
352+
(_, _, maybeBuild) =>
353+
val build = maybeBuild.orThrow
354+
ActionablePreprocessor
355+
.generateActionableDiagnostics(buildOptionsWithEmptyCoursierCache(build.options))
356+
.orThrow
357+
val paths = recorded.asScala.toSeq
358+
expect(!paths.exists(_.contains("toolkit_3/maven-metadata.xml")))
359+
}
360+
)
361+
}
362+
363+
test("actionable outdated check for org.scala-lang skips user repository metadata") {
364+
val u = UUID.randomUUID().toString.replace("-", "")
365+
val art = s"scala_cli_fake_$u"
366+
val meta =
367+
s"""<?xml version="1.0" encoding="UTF-8"?>
368+
|<metadata>
369+
| <groupId>org.scala-lang</groupId>
370+
| <artifactId>${art}_3</artifactId>
371+
| <versioning>
372+
| <latest>99.0.0</latest>
373+
| <release>99.0.0</release>
374+
| <versions>
375+
| <version>0.1.0</version>
376+
| <version>99.0.0</version>
377+
| </versions>
378+
| </versioning>
379+
|</metadata>
380+
|""".stripMargin.getBytes("UTF-8")
381+
val pom =
382+
s"""<?xml version='1.0' encoding='UTF-8'?>
383+
|<project>
384+
| <groupId>org.scala-lang</groupId>
385+
| <artifactId>${art}_3</artifactId>
386+
| <version>0.1.0</version>
387+
|</project>""".stripMargin.getBytes("UTF-8")
388+
val responses = Map(
389+
s"/org/scala-lang/${art}_3/maven-metadata.xml" -> meta,
390+
s"/org/scala-lang/${art}_3/0.1.0/${art}_3-0.1.0.pom" -> pom
391+
)
392+
withRecordingMavenRepo(responses)((repoUrl, recorded) =>
393+
val testInputs = TestInputs(
394+
os.rel / "Foo.scala" ->
395+
s"""//> using dep org.scala-lang::$art:0.1.0
396+
|
397+
|object Hello extends App {
398+
| println("Hello")
399+
|}
400+
|""".stripMargin
401+
)
402+
val withRepo = baseOptions.copy(
403+
classPathOptions =
404+
baseOptions.classPathOptions.copy(extraRepositories = Seq(repoUrl))
405+
)
406+
testInputs.withBuild(withRepo, buildThreads, None, actionableDiagnostics = true) {
407+
(_, _, maybeBuild) =>
408+
val build = maybeBuild.orThrow
409+
ActionablePreprocessor
410+
.generateActionableDiagnostics(buildOptionsWithEmptyCoursierCache(build.options))
411+
.orThrow
412+
val paths = recorded.asScala.toSeq
413+
expect(!paths.exists(p => p.contains(s"${art}_3/") && p.contains("maven-metadata.xml")))
414+
}
415+
)
416+
}
417+
418+
test("actionable outdated check still consults user repository for other organizations") {
419+
val u = UUID.randomUUID().toString.replace("-", "")
420+
val art = s"scala_cli_fake_$u"
421+
val meta =
422+
s"""<?xml version="1.0" encoding="UTF-8"?>
423+
|<metadata>
424+
| <groupId>test-org</groupId>
425+
| <artifactId>${art}_3</artifactId>
426+
| <versioning>
427+
| <latest>99.0.0</latest>
428+
| <release>99.0.0</release>
429+
| <versions>
430+
| <version>0.1.0</version>
431+
| <version>99.0.0</version>
432+
| </versions>
433+
| </versioning>
434+
|</metadata>
435+
|""".stripMargin.getBytes("UTF-8")
436+
val pom =
437+
s"""<?xml version='1.0' encoding='UTF-8'?>
438+
|<project>
439+
| <groupId>test-org</groupId>
440+
| <artifactId>${art}_3</artifactId>
441+
| <version>0.1.0</version>
442+
|</project>""".stripMargin.getBytes("UTF-8")
443+
val responses = Map(
444+
s"/test-org/${art}_3/maven-metadata.xml" -> meta,
445+
s"/test-org/${art}_3/0.1.0/${art}_3-0.1.0.pom" -> pom
446+
)
447+
withRecordingMavenRepo(responses)((repoUrl, recorded) =>
448+
val testInputs = TestInputs(
449+
os.rel / "Foo.scala" ->
450+
s"""//> using dep test-org::$art:0.1.0
451+
|
452+
|object Hello extends App {
453+
| println("Hello")
454+
|}
455+
|""".stripMargin
456+
)
457+
val withRepo = baseOptions.copy(
458+
classPathOptions =
459+
baseOptions.classPathOptions.copy(extraRepositories = Seq(repoUrl))
460+
)
461+
testInputs.withBuild(withRepo, buildThreads, None, actionableDiagnostics = true) {
462+
(_, _, maybeBuild) =>
463+
val build = maybeBuild.orThrow
464+
val paths = recorded.asScala.toSeq
465+
expect(paths.exists(p => p.contains(s"${art}_3/") && p.contains("maven-metadata.xml")))
466+
val updateDiagnostics =
467+
ActionablePreprocessor.generateActionableDiagnostics(build.options).orThrow
468+
val dOpt = updateDiagnostics.collectFirst {
469+
case diagnostic: ActionableDependencyUpdateDiagnostic => diagnostic
470+
}
471+
expect(dOpt.nonEmpty)
472+
expect(dOpt.get.newVersion == "99.0.0")
473+
}
474+
)
475+
}
476+
477+
test("actionable outdated check times out slow user repository") {
478+
val u = UUID.randomUUID().toString.replace("-", "")
479+
val art = s"scala_cli_fake_$u"
480+
val meta =
481+
s"""<?xml version="1.0" encoding="UTF-8"?>
482+
|<metadata>
483+
| <groupId>test-org</groupId>
484+
| <artifactId>${art}_3</artifactId>
485+
| <versioning>
486+
| <latest>0.2.0</latest>
487+
| <release>0.2.0</release>
488+
| <versions>
489+
| <version>0.1.0</version>
490+
| <version>0.2.0</version>
491+
| </versions>
492+
| </versioning>
493+
|</metadata>
494+
|""".stripMargin.getBytes("UTF-8")
495+
val pom =
496+
s"""<?xml version='1.0' encoding='UTF-8'?>
497+
|<project>
498+
| <groupId>test-org</groupId>
499+
| <artifactId>${art}_3</artifactId>
500+
| <version>0.1.0</version>
501+
|</project>""".stripMargin.getBytes("UTF-8")
502+
val responses = Map(
503+
s"/test-org/${art}_3/maven-metadata.xml" -> meta,
504+
s"/test-org/${art}_3/0.1.0/${art}_3-0.1.0.pom" -> pom
505+
)
506+
val slowAfterClear = new AtomicBoolean(false)
507+
withRecordingMavenRepo(
508+
responses,
509+
delayOnMetadataMs = 30_000L,
510+
delayWhen = () => slowAfterClear.get()
511+
)((repoUrl, _) =>
512+
val testInputs = TestInputs(
513+
os.rel / "Foo.scala" ->
514+
s"""//> using dep test-org::$art:0.1.0
515+
|
516+
|object Hello extends App {
517+
| println("Hello")
518+
|}
519+
|""".stripMargin
520+
)
521+
val withRepo = baseOptions.copy(
522+
classPathOptions =
523+
baseOptions.classPathOptions.copy(extraRepositories = Seq(repoUrl))
524+
)
525+
testInputs.withBuild(withRepo, buildThreads, None, actionableDiagnostics = true) {
526+
(_, _, maybeBuild) =>
527+
val build = maybeBuild.orThrow
528+
slowAfterClear.set(true)
529+
val t0 = System.nanoTime()
530+
ActionablePreprocessor
531+
.generateActionableDiagnostics(buildOptionsWithEmptyCoursierCache(build.options))
532+
.orThrow
533+
val elapsedMs = (System.nanoTime() - t0) / 1_000_000
534+
expect(elapsedMs < 15_000)
535+
}
536+
)
537+
}
266538
}

‎modules/options/src/main/scala/scala/build/actionable/ActionableDependencyHandler.scala‎

Lines changed: 55 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package scala.build.actionable
2+
import coursier.Versions
23
import coursier.cache.FileCache
3-
import coursier.core.Repository
4+
import coursier.core.{Repository, Versions as CoreVersions}
45
import coursier.util.Task
56
import coursier.version.{Latest, Version}
67
import dependency.*
@@ -11,8 +12,10 @@ import scala.build.errors.{BuildException, Severity}
1112
import scala.build.internal.Constants
1213
import scala.build.internal.Util.*
1314
import scala.build.options.BuildOptions
14-
import scala.build.options.ScalaVersionUtil.versions
1515
import scala.build.{Logger, Positioned}
16+
import scala.concurrent.duration.{DurationInt, FiniteDuration}
17+
import scala.concurrent.{Await, ExecutionContext, Future, TimeoutException}
18+
import scala.util.control.NonFatal
1619

1720
case object ActionableDependencyHandler
1821
extends ActionableHandler[ActionableDependencyUpdateDiagnostic] {
@@ -44,7 +47,7 @@ case object ActionableDependencyHandler
4447
if dependency.userParams.exists(_._1 == Constants.toolkitName)
4548
then
4649
val toolkitSuggestion =
47-
if dependency.module.organization == Constants.toolkitOrganization then latestVersion
50+
if dependency.isScalaLangOrganization then latestVersion
4851
else if dependency.module.organization == Constants.typelevelOrganization then
4952
s"typelevel:$latestVersion"
5053
else s"${dependency.module.organization}:$latestVersion"
@@ -68,6 +71,13 @@ case object ActionableDependencyHandler
6871
/** Versions like 'latest.*': 'latest.release', 'latest.integration', 'latest.stable'
6972
*/
7073
private def isLatestSyntaxVersion(version: String): Boolean = Latest(version).nonEmpty
74+
75+
private val perRepoVersionsTimeout: FiniteDuration = 5.seconds
76+
77+
private def mergeCoreVersions(parts: Seq[CoreVersions]): CoreVersions =
78+
val mergedAvailable = parts.flatMap(_.available0).distinctBy(_.asString).toList
79+
CoreVersions.empty.withAvailable0(mergedAvailable)
80+
7181
private def findLatestVersion(
7282
buildOptions: BuildOptions,
7383
setting: Positioned[AnyDependency],
@@ -77,13 +87,50 @@ case object ActionableDependencyHandler
7787
val scalaParams: Option[ScalaParameters] = value(buildOptions.scalaParams)
7888
val cache: FileCache[Task] = buildOptions.finalCache
7989
val csModule: coursier.core.Module = value(dependency.toCs(scalaParams)).module
80-
val repositories: Seq[Repository] = value(buildOptions.finalRepositories)
90+
val includeUserExtraRepositories = !dependency.isScalaLangOrganization
91+
val repositories: Seq[Repository] =
92+
value(buildOptions.finalRepositories(includeUserExtraRepositories))
93+
94+
given ExecutionContext = cache.ec
95+
96+
val perRepoFutures: Seq[Future[CoreVersions]] = repositories.map { repo =>
97+
val label = repo.toString.take(200)
98+
val listing: Future[CoreVersions] =
99+
Versions(cache)
100+
.withModule(csModule)
101+
.addRepositories(repo)
102+
.result()
103+
.future()
104+
.map(_.versions)
105+
listing.recover {
106+
case NonFatal(e) =>
107+
loggerOpt.foreach(_.debug(
108+
s"Failed listing versions for ${dependency.render} from repository $label: ${e.getMessage}"
109+
))
110+
CoreVersions.empty
111+
}
112+
}
113+
114+
val perRepoParts: Seq[CoreVersions] = perRepoFutures.map { f =>
115+
try Await.result(f, perRepoVersionsTimeout)
116+
catch {
117+
case _: TimeoutException =>
118+
loggerOpt.foreach(_.debug(
119+
s"Timeout listing versions for ${dependency.render} (after $perRepoVersionsTimeout)"
120+
))
121+
CoreVersions.empty
122+
case NonFatal(e) =>
123+
loggerOpt.foreach(_.debug(
124+
s"Failed listing versions for ${dependency.render}: ${e.getMessage}"
125+
))
126+
CoreVersions.empty
127+
}
128+
}
81129

82-
val latestVersionOpt = cache.versions(csModule, repositories)
83-
.versions
84-
.latest(Latest.Stable)
130+
val mergedVersions = mergeCoreVersions(perRepoParts)
131+
val latestVersionOpt = mergedVersions.latest(Latest.Stable)
85132

86-
if (latestVersionOpt.isEmpty)
133+
if latestVersionOpt.isEmpty then
87134
loggerOpt.foreach(_.diagnostic(
88135
s"No latest version found for ${dependency.render}",
89136
Severity.Warning,

0 commit comments

Comments
 (0)