Add support for ```java snippets in Markdown inputs (#4284)

Author: Gedochao

modules/build/src/main/scala/scala/build/Sources.scala

@@ -120,7 +120,7 @@ object Sources {
   ): Seq[Preprocessor] =
     Seq(
       ScriptPreprocessor,
-      MarkdownPreprocessor,
+      MarkdownPreprocessor(archiveCache, javaClassNameVersionOpt, javaCommand),
       JavaPreprocessor(archiveCache, javaClassNameVersionOpt, javaCommand),
       ScalaPreprocessor,
       DataPreprocessor,

modules/build/src/main/scala/scala/build/internal/markdown/MarkdownCodeBlock.scala

@@ -22,25 +22,39 @@ case class MarkdownCodeBlock(
   endLine: Int
 ) {
 
+  private def supportedLanguage: Boolean =
+    info.headOption.exists(lang => lang == "scala" || lang == "java")
+
   /** @return
     *   `true` if this snippet should be ignored, `false` otherwise
     */
-  def shouldIgnore: Boolean = info.head != "scala" || info.contains("ignore")
+  def shouldIgnore: Boolean = !supportedLanguage || info.contains("ignore")
+
+  /** @return
+    *   `true` if this snippet is a Scala snippet, `false` otherwise
+    */
+  def isScala: Boolean = info.headOption.contains("scala")
+
+  /** @return
+    *   `true` if this snippet is a Java snippet, `false` otherwise
+    */
+  def isJava: Boolean = info.headOption.contains("java")
 
   /** @return
     *   `true` if this snippet should have its scope reset, `false` otherwise
     */
-  def resetScope: Boolean = info.contains("reset")
+  def resetScope: Boolean = isScala && info.contains("reset")
 
   /** @return
     *   `true` if this snippet is a test snippet, `false` otherwise
     */
   def isTest: Boolean = info.contains("test")
 
   /** @return
-    *   `true` if this snippet is a raw snippet, `false` otherwise
+    *   `true` if this snippet is a raw snippet, `false` otherwise. Only meaningful for Scala
+    *   snippets; Java snippets are always emitted as raw `.java` sources.
     */
-  def isRaw: Boolean = info.contains("raw")
+  def isRaw: Boolean = isScala && info.contains("raw")
 }
 
 object MarkdownCodeBlock {

modules/build/src/main/scala/scala/build/preprocessing/MarkdownCodeBlockProcessor.scala

@@ -14,7 +14,9 @@ object MarkdownCodeBlockProcessor {
     logger: Logger,
     maybeRecoverOnError: BuildException => Option[BuildException]
   ): Either[BuildException, PreprocessedMarkdown] = either {
-    val (rawCodeBlocks, remaining)         = codeBlocks.partition(_.isRaw)
+    val (javaBlocks, scalaBlocks)          = codeBlocks.partition(_.isJava)
+    val (javaTestBlocks, javaMainBlocks)   = javaBlocks.partition(_.isTest)
+    val (rawCodeBlocks, remaining)         = scalaBlocks.partition(_.isRaw)
     val (testCodeBlocks, scriptCodeBlocks) = remaining.partition(_.isTest)
     def preprocessCodeBlocks(cbs: Seq[MarkdownCodeBlock])
       : Either[BuildException, PreprocessedMarkdownCodeBlocks] = either {
@@ -39,7 +41,9 @@ object MarkdownCodeBlockProcessor {
     PreprocessedMarkdown(
       value(preprocessCodeBlocks(scriptCodeBlocks)),
       value(preprocessCodeBlocks(rawCodeBlocks)),
-      value(preprocessCodeBlocks(testCodeBlocks))
+      value(preprocessCodeBlocks(testCodeBlocks)),
+      value(preprocessCodeBlocks(javaMainBlocks)),
+      value(preprocessCodeBlocks(javaTestBlocks))
     )
   }
 }

modules/build/src/main/scala/scala/build/preprocessing/MarkdownPreprocessor.scala

@@ -1,16 +1,34 @@
 package scala.build.preprocessing
 
+import coursier.cache.ArchiveCache
+import coursier.util.Task
+
 import java.nio.charset.StandardCharsets
 
 import scala.build.EitherCps.{either, value}
 import scala.build.Logger
 import scala.build.errors.BuildException
 import scala.build.input.{MarkdownFile, ScalaCliInvokeData, SingleElement, VirtualMarkdownFile}
+import scala.build.internal.JavaParserProxyMaker
 import scala.build.internal.markdown.{MarkdownCodeBlock, MarkdownCodeWrapper}
 import scala.build.options.SuppressWarningOptions
 import scala.build.preprocessing.ScalaPreprocessor.ProcessingOutput
+import scala.build.preprocessing.directives.PreprocessedDirectives
 
-case object MarkdownPreprocessor extends Preprocessor {
+/** Markdown source preprocessor.
+  *
+  * @param archiveCache
+  *   when using a java-class-name external binary to infer a class name (see [[JavaParserProxy]]),
+  *   a cache to download that binary with
+  * @param javaClassNameVersionOpt
+  *   when using a java-class-name external binary to infer a class name (see [[JavaParserProxy]]),
+  *   this forces the java-class-name version to download
+  */
+final case class MarkdownPreprocessor(
+  archiveCache: ArchiveCache[Task],
+  javaClassNameVersionOpt: Option[String],
+  javaCommand: () => String
+) extends Preprocessor {
   def preprocess(
     input: SingleElement,
     logger: Logger,
@@ -23,15 +41,18 @@ case object MarkdownPreprocessor extends Preprocessor {
         val res = either {
           val content      = value(PreprocessingUtil.maybeRead(markdown.path))
           val preprocessed = value {
-            MarkdownPreprocessor.preprocess(
-              Right(markdown.path),
-              content,
-              markdown.subPath,
-              ScopePath.fromPath(markdown.path),
-              logger,
-              maybeRecoverOnError,
-              allowRestrictedFeatures,
-              suppressWarningOptions
+            preprocessContent(
+              archiveCache = archiveCache,
+              javaClassNameVersionOpt = javaClassNameVersionOpt,
+              javaCommand = javaCommand,
+              reportingPath = Right(markdown.path),
+              content = content,
+              subPath = markdown.subPath,
+              scopePath = ScopePath.fromPath(markdown.path),
+              logger = logger,
+              maybeRecoverOnError = maybeRecoverOnError,
+              allowRestrictedFeatures = allowRestrictedFeatures,
+              suppressWarningOptions = suppressWarningOptions
             )
           }
           preprocessed
@@ -41,15 +62,18 @@ case object MarkdownPreprocessor extends Preprocessor {
         val content = new String(markdown.content, StandardCharsets.UTF_8)
         val res     = either {
           val preprocessed = value {
-            MarkdownPreprocessor.preprocess(
-              Left(markdown.source),
-              content,
-              markdown.wrapperPath,
-              markdown.scopePath,
-              logger,
-              maybeRecoverOnError,
-              allowRestrictedFeatures,
-              suppressWarningOptions
+            preprocessContent(
+              archiveCache = archiveCache,
+              javaClassNameVersionOpt = javaClassNameVersionOpt,
+              javaCommand = javaCommand,
+              reportingPath = Left(markdown.source),
+              content = content,
+              subPath = markdown.wrapperPath,
+              scopePath = markdown.scopePath,
+              logger = logger,
+              maybeRecoverOnError = maybeRecoverOnError,
+              allowRestrictedFeatures = allowRestrictedFeatures,
+              suppressWarningOptions = suppressWarningOptions
             )
           }
           preprocessed
@@ -59,7 +83,10 @@ case object MarkdownPreprocessor extends Preprocessor {
         None
     }
 
-  private def preprocess(
+  private def preprocessContent(
+    archiveCache: ArchiveCache[Task],
+    javaClassNameVersionOpt: Option[String],
+    javaCommand: () => String,
     reportingPath: Either[String, os.Path],
     content: String,
     subPath: os.SubPath,
@@ -106,6 +133,38 @@ case object MarkdownPreprocessor extends Preprocessor {
           }
       }
 
+    def emitJavaSnippets(
+      blocks: PreprocessedMarkdownCodeBlocks,
+      isTest: Boolean
+    ): Either[BuildException, List[PreprocessedSource.InMemory]] =
+      either {
+        val javaParser =
+          (new JavaParserProxyMaker)
+            .get(
+              archiveCache,
+              javaClassNameVersionOpt,
+              logger,
+              () => javaCommand()
+            )
+        blocks.codeBlocks.zipWithIndex.map { (block, index) =>
+          value {
+            emitJavaSnippet(
+              block = block,
+              index = index,
+              isTest = isTest,
+              subPath = subPath,
+              scopePath = scopePath,
+              reportingPath = reportingPath,
+              javaParser = javaParser,
+              logger = logger,
+              allowRestrictedFeatures = allowRestrictedFeatures,
+              suppressWarningOptions = suppressWarningOptions,
+              maybeRecoverOnError = maybeRecoverOnError
+            )
+          }
+        }.toList
+      }
+
     val codeBlocks: Seq[MarkdownCodeBlock] =
       value(MarkdownCodeBlock.findCodeBlocks(subPath, content, maybeRecoverOnError))
     val preprocessedMarkdown: PreprocessedMarkdown =
@@ -123,8 +182,60 @@ case object MarkdownPreprocessor extends Preprocessor {
     val maybeMainFile = value(preprocessSnippets(mainScalaCode, ".scala"))
     val maybeRawFile  = value(preprocessSnippets(rawScalaCode, ".raw.scala"))
     val maybeTestFile = value(preprocessSnippets(testScalaCode, ".test.scala"))
+    val javaFiles     = value(emitJavaSnippets(preprocessedMarkdown.javaCodeBlocks, isTest = false))
+    val javaTestFiles =
+      value(emitJavaSnippets(preprocessedMarkdown.javaTestCodeBlocks, isTest = true))
 
-    maybeMainFile.toList ++ maybeTestFile ++ maybeRawFile
+    maybeMainFile.toList ++ maybeTestFile ++ maybeRawFile ++ javaFiles ++ javaTestFiles
   }
 
+  private def emitJavaSnippet(
+    block: MarkdownCodeBlock,
+    index: Int,
+    isTest: Boolean,
+    subPath: os.SubPath,
+    scopePath: ScopePath,
+    reportingPath: Either[String, os.Path],
+    javaParser: scala.build.internal.JavaParserProxy,
+    logger: Logger,
+    allowRestrictedFeatures: Boolean,
+    suppressWarningOptions: SuppressWarningOptions,
+    maybeRecoverOnError: BuildException => Option[BuildException]
+  )(using ScalaCliInvokeData): Either[BuildException, PreprocessedSource.InMemory] = either {
+    val classNameOpt = value {
+      javaParser.className(block.body.getBytes(StandardCharsets.UTF_8))
+    }
+    val mdBaseName   = subPath.last.stripSuffix(".md")
+    val baseName     = classNameOpt.getOrElse(s"${mdBaseName}_md_snippet$index")
+    val javaFileName =
+      if isTest then s"$baseName.test.java"
+      else s"$baseName.java"
+    val generatedRelPath =
+      if isTest then os.rel / (subPath / os.up) / s"$baseName.java"
+      else os.rel / (subPath / os.up) / javaFileName
+    val snippetScopePath = scopePath.copy(subPath = (subPath / os.up) / javaFileName)
+    val preprocessedDirectives: PreprocessedDirectives = value {
+      DirectivesPreprocessor(
+        reportingPath,
+        snippetScopePath,
+        logger,
+        allowRestrictedFeatures,
+        suppressWarningOptions,
+        maybeRecoverOnError
+      ).preprocess(block.body)
+    }
+    PreprocessedSource.InMemory(
+      originalPath = reportingPath.map(subPath -> _),
+      relPath = generatedRelPath,
+      content = block.body.getBytes(StandardCharsets.UTF_8),
+      wrapperParamsOpt = None,
+      options = Some(preprocessedDirectives.globalUsings),
+      optionsWithTargetRequirements = preprocessedDirectives.usingsWithReqs,
+      requirements = Some(preprocessedDirectives.globalReqs),
+      scopedRequirements = preprocessedDirectives.scopedReqs,
+      mainClassOpt = None,
+      scopePath = snippetScopePath,
+      directivesPositions = preprocessedDirectives.directivesPositions
+    )
+  }
 }

modules/build/src/main/scala/scala/build/preprocessing/PreprocessedMarkdown.scala

@@ -15,7 +15,9 @@ object PreprocessedMarkdownCodeBlocks {
 case class PreprocessedMarkdown(
   scriptCodeBlocks: PreprocessedMarkdownCodeBlocks = PreprocessedMarkdownCodeBlocks.empty,
   rawCodeBlocks: PreprocessedMarkdownCodeBlocks = PreprocessedMarkdownCodeBlocks.empty,
-  testCodeBlocks: PreprocessedMarkdownCodeBlocks = PreprocessedMarkdownCodeBlocks.empty
+  testCodeBlocks: PreprocessedMarkdownCodeBlocks = PreprocessedMarkdownCodeBlocks.empty,
+  javaCodeBlocks: PreprocessedMarkdownCodeBlocks = PreprocessedMarkdownCodeBlocks.empty,
+  javaTestCodeBlocks: PreprocessedMarkdownCodeBlocks = PreprocessedMarkdownCodeBlocks.empty
 )
 
 object PreprocessedMarkdown {

modules/build/src/test/scala/scala/build/tests/PreprocessingTests.scala

@@ -1,12 +1,32 @@
 package scala.build.tests
 
 import com.eed3si9n.expecty.Expecty.expect
+import coursier.cache.Cache.Fetch
+import coursier.cache.{ArchiveCache, ArtifactError, Cache}
+import coursier.util.{Artifact, EitherT, Task}
 
+import java.io.File
+
+import scala.build.Sources
 import scala.build.input.{MarkdownFile, ScalaCliInvokeData, Script, SourceScalaFile}
 import scala.build.options.SuppressWarningOptions
 import scala.build.preprocessing.{MarkdownPreprocessor, ScalaPreprocessor, ScriptPreprocessor}
+import scala.concurrent.ExecutionContext
 
 class PreprocessingTests extends TestUtil.ScalaCliBuildSuite {
+  private val markdownPreprocessor: MarkdownPreprocessor =
+    Sources.defaultPreprocessors(
+      ArchiveCache().withCache(
+        new Cache[Task] {
+          def fetch: Fetch[Task] = _ => sys.error("shouldn't be used")
+          def file(artifact: Artifact): EitherT[Task, ArtifactError, File] =
+            sys.error("shouldn't be used")
+          def ec: ExecutionContext = sys.error("shouldn't be used")
+        }
+      ),
+      javaClassNameVersionOpt = None,
+      javaCommand = () => sys.error("shouldn't be used")
+    ).collectFirst { case m: MarkdownPreprocessor => m }.get
   test("Report error if scala file not exists") {
     val logger    = TestLogger()
     val scalaFile = SourceScalaFile(os.temp.dir(), os.SubPath("NotExists.scala"))
@@ -45,7 +65,7 @@ class PreprocessingTests extends TestUtil.ScalaCliBuildSuite {
     val logger       = TestLogger()
     val markdownFile = MarkdownFile(os.temp.dir(), os.SubPath("NotExists.md"))
 
-    val res = MarkdownPreprocessor.preprocess(
+    val res = markdownPreprocessor.preprocess(
       markdownFile,
       logger,
       allowRestrictedFeatures = false,