Merge pull request #4201 from durban/wip/multithreaded-wstp

Add support for Native multithreaded execution

Author: djspiewak

.cirrus.yml

@@ -10,3 +10,16 @@ jvm_highcore_task:
       script: sbt '++ 2.13' testsJVM/test ioAppTestsJVM/test
     - name: JVM high-core-count 3
       script: sbt '++ 3' testsJVM/test ioAppTestsJVM/test
+
+native_highcore_task:
+  only_if: $CIRRUS_TAG != '' || $CIRRUS_PR != ''
+  required_pr_labels: Cirrus Native
+  container:
+    dockerfile: .cirrus/Dockerfile
+    cpu: 8
+    memory: 16G
+  matrix:
+    - name: Native high-core-count 2.13
+      script: sbt '++ 2.13' testsNative/test ioAppTestsNative/test
+    - name: Native high-core-count 3
+      script: sbt '++ 3' testsNative/test ioAppTestsNative/test

.github/workflows/ci.yml

@@ -670,5 +670,5 @@ jobs:
       - name: Submit Dependencies
         uses: scalacenter/sbt-dependency-submission@v2
         with:
-          modules-ignore: cats-effect-benchmarks_3 cats-effect-benchmarks_2.12 cats-effect-benchmarks_2.13 cats-effect_3 cats-effect_2.12 cats-effect_2.13 cats-effect-example_sjs1_3 cats-effect-example_sjs1_2.12 cats-effect-example_sjs1_2.13 rootjs_3 rootjs_2.12 rootjs_2.13 ioapptestsnative_3 ioapptestsnative_2.12 ioapptestsnative_2.13 cats-effect-graalvm-example_3 cats-effect-graalvm-example_2.12 cats-effect-graalvm-example_2.13 cats-effect-tests_sjs1_3 cats-effect-tests_sjs1_2.12 cats-effect-tests_sjs1_2.13 rootjvm_3 rootjvm_2.12 rootjvm_2.13 rootnative_3 rootnative_2.12 rootnative_2.13 cats-effect-example_native0.4_3 cats-effect-example_native0.4_2.12 cats-effect-example_native0.4_2.13 cats-effect-example_3 cats-effect-example_2.12 cats-effect-example_2.13 cats-effect-tests_3 cats-effect-tests_2.12 cats-effect-tests_2.13 ioapptestsjvm_3 ioapptestsjvm_2.12 ioapptestsjvm_2.13 ioapptestsjs_3 ioapptestsjs_2.12 ioapptestsjs_2.13 cats-effect-tests_native0.4_3 cats-effect-tests_native0.4_2.12 cats-effect-tests_native0.4_2.13
+          modules-ignore: cats-effect-benchmarks_3 cats-effect-benchmarks_2.12 cats-effect-benchmarks_2.13 cats-effect_3 cats-effect_2.12 cats-effect_2.13 cats-effect-example_sjs1_3 cats-effect-example_sjs1_2.12 cats-effect-example_sjs1_2.13 rootjs_3 rootjs_2.12 rootjs_2.13 ioapptestsnative_3 ioapptestsnative_2.12 ioapptestsnative_2.13 cats-effect-graalvm-example_3 cats-effect-graalvm-example_2.12 cats-effect-graalvm-example_2.13 cats-effect-tests_sjs1_3 cats-effect-tests_sjs1_2.12 cats-effect-tests_sjs1_2.13 rootjvm_3 rootjvm_2.12 rootjvm_2.13 rootnative_3 rootnative_2.12 rootnative_2.13 cats-effect-example_native0.5_3 cats-effect-example_native0.5_2.12 cats-effect-example_native0.5_2.13 cats-effect-example_3 cats-effect-example_2.12 cats-effect-example_2.13 cats-effect-tests_3 cats-effect-tests_2.12 cats-effect-tests_2.13 ioapptestsjvm_3 ioapptestsjvm_2.12 ioapptestsjvm_2.13 ioapptestsjs_3 ioapptestsjs_2.12 ioapptestsjs_2.13 cats-effect-tests_native0.5_3 cats-effect-tests_native0.5_2.12 cats-effect-tests_native0.5_2.13
           configs-ignore: test scala-tool scala-doc-tool test-internal

build.sbt

@@ -23,11 +23,14 @@ import org.openqa.selenium.firefox.{FirefoxOptions, FirefoxProfile}
 import org.scalajs.jsenv.nodejs.NodeJSEnv
 import org.scalajs.jsenv.selenium.SeleniumJSEnv
 import sbtcrossproject.CrossProject
+import scala.scalanative.build._
 
 import JSEnv._
 
+lazy val inCI = Option(System.getenv("CI")).contains("true")
+
 // sbt-git workarounds
-ThisBuild / useConsoleForROGit := !Option(System.getenv("CI")).contains("true")
+ThisBuild / useConsoleForROGit := !inCI
 
 ThisBuild / git.gitUncommittedChanges := {
   if ((ThisBuild / githubIsWorkflowBuild).value) {
@@ -40,7 +43,7 @@ ThisBuild / git.gitUncommittedChanges := {
   }
 }
 
-ThisBuild / tlBaseVersion := "3.6"
+ThisBuild / tlBaseVersion := "3.7"
 ThisBuild / tlUntaggedAreSnapshots := false
 
 ThisBuild / organization := "org.typelevel"
@@ -322,13 +325,13 @@ ThisBuild / autoAPIMappings := true
 
 ThisBuild / Test / testOptions += Tests.Argument("+l")
 
-val CatsVersion = "2.11.0"
-val CatsMtlVersion = "1.3.1"
-val ScalaCheckVersion = "1.17.1"
-val CoopVersion = "1.2.0"
-val MUnitVersion = "1.0.0-M11"
-val MUnitScalaCheckVersion = "1.0.0-M11"
-val DisciplineMUnitVersion = "2.0.0-M3"
+val CatsVersion = "2.12.0"
+val CatsMtlVersion = "1.5.0"
+val ScalaCheckVersion = "1.18.1"
+val CoopVersion = "1.3.0"
+val MUnitVersion = "1.1.0"
+val MUnitScalaCheckVersion = "1.1.0"
+val DisciplineMUnitVersion = "2.0.0"
 
 val MacrotaskExecutorVersion = "1.1.1"
 
@@ -358,6 +361,17 @@ Global / tlCommandAliases ++= Map(
   )
 )
 
+lazy val nativeTestSettings = Seq(
+  nativeConfig ~= { c => // TODO: remove this when it seems to work
+    c.withSourceLevelDebuggingConfig(_.enableAll) // enable generation of debug information
+      .withOptimize(false) // disable Scala Native optimizer
+      .withMode(Mode.debug) // compile using LLVM without optimizations
+      .withCompileOptions(c.compileOptions ++ Seq("-gdwarf-4"))
+  },
+  envVars ++= { if (inCI) Map("GC_MAXIMUM_HEAP_SIZE" -> "8g") else Map.empty[String, String] },
+  parallelExecution := !inCI
+)
+
 val jsProjects: Seq[ProjectReference] =
   Seq(
     kernel.js,
@@ -441,7 +455,7 @@ lazy val kernel = crossProject(JSPlatform, JVMPlatform, NativePlatform)
     libraryDependencies += "org.scala-js" %%% "scala-js-macrotask-executor" % MacrotaskExecutorVersion % Test
   )
   .nativeSettings(
-    libraryDependencies += "io.github.cquiroz" %%% "scala-java-time" % "2.5.0"
+    libraryDependencies += "io.github.cquiroz" %%% "scala-java-time" % "2.6.0"
   )
 
 /**
@@ -887,7 +901,12 @@ lazy val core = crossProject(JSPlatform, JVMPlatform, NativePlatform)
         ProblemFilters.exclude[MissingClassProblem](
           "cats.effect.metrics.JsCpuStarvationMetrics"),
         ProblemFilters.exclude[MissingClassProblem](
-          "cats.effect.metrics.JsCpuStarvationMetrics$")
+          "cats.effect.metrics.JsCpuStarvationMetrics$"),
+        // all package-private classes; introduced when we made Native multithreaded
+        ProblemFilters.exclude[MissingClassProblem]("cats.effect.unsafe.FiberExecutor"),
+        ProblemFilters.exclude[IncompatibleMethTypeProblem](
+          "cats.effect.unsafe.FiberMonitorImpl.this"),
+        ProblemFilters.exclude[MissingClassProblem]("cats.effect.unsafe.FiberMonitorPlatform")
       )
     },
     mimaBinaryIssueFilters ++= {
@@ -944,6 +963,7 @@ lazy val testkit = crossProject(JSPlatform, JVMPlatform, NativePlatform)
       "org.scalacheck" %%% "scalacheck" % ScalaCheckVersion
     )
   )
+  .nativeSettings(nativeTestSettings)
 
 /**
  * Unit tests for the core project, utilizing the support provided by testkit.
@@ -975,7 +995,8 @@ lazy val tests: CrossProject = crossProject(JSPlatform, JVMPlatform, NativePlatf
     Test / javaOptions += "-Dcats.effect.trackFiberContext=true"
   )
   .nativeSettings(
-    Compile / mainClass := Some("catseffect.examples.NativeRunner")
+    Compile / mainClass := Some("catseffect.examples.NativeRunner"),
+    nativeTestSettings
   )
 
 def configureIOAppTests(p: Project): Project =
@@ -986,7 +1007,7 @@ def configureIOAppTests(p: Project): Project =
       buildInfoPackage := "cats.effect",
       buildInfoKeys ++= Seq(
         "jsRunner" -> (tests.js / Compile / fastOptJS / artifactPath).value,
-        "nativeRunner" -> (tests.native / Compile / nativeLink / artifactPath).value
+        "nativeRunner" -> (tests.native / Compile / crossTarget).value / (tests.native / Compile / moduleName).value
       )
     )
 

core/js/src/main/scala/cats/effect/unsafe/BatchingMacrotaskExecutor.scala

@@ -39,8 +39,7 @@ import scala.scalajs.{js, LinkingInfo}
 private[effect] final class BatchingMacrotaskExecutor(
     batchSize: Int,
     reportFailure0: Throwable => Unit
-) extends ExecutionContextExecutor
-    with FiberExecutor {
+) extends ExecutionContextExecutor {
 
   private[this] val queueMicrotask: js.Function1[js.Function0[Any], Any] =
     if (js.typeOf(js.Dynamic.global.queueMicrotask) == "function")

…la/cats/effect/unsafe/FiberMonitor.scala …la/cats/effect/unsafe/FiberMonitor.scalacore/js-native/src/main/scala/cats/effect/unsafe/FiberMonitor.scala renamed to core/js/src/main/scala/cats/effect/unsafe/FiberMonitor.scala core/js-native/src/main/scala/cats/effect/unsafe/FiberMonitor.scala renamed to core/js/src/main/scala/cats/effect/unsafe/FiberMonitor.scala

@@ -17,6 +17,9 @@
 package cats.effect
 package unsafe
 
+import scala.concurrent.ExecutionContext
+import scala.scalajs.{js, LinkingInfo}
+
 private[effect] sealed abstract class FiberMonitor extends FiberMonitorShared {
 
   /**
@@ -41,8 +44,8 @@ private[effect] sealed abstract class FiberMonitor extends FiberMonitorShared {
 
 private final class FiberMonitorImpl(
     // A reference to the compute pool of the `IORuntime` in which this suspended fiber bag
-    // operates. `null` if the compute pool of the `IORuntime` is not a `FiberExecutor`.
-    private[this] val compute: FiberExecutor
+    // operates. `null` if the compute pool of the `IORuntime` is not a `BatchingMacrotaskExecutor`.
+    private[this] val compute: BatchingMacrotaskExecutor
 ) extends FiberMonitor {
   private[this] val bag = new WeakBag[IOFiber[?]]()
 
@@ -92,4 +95,26 @@ private final class NoOpFiberMonitor extends FiberMonitor {
   def liveFiberSnapshot(print: String => Unit): Unit = ()
 }
 
-private[effect] object FiberMonitor extends FiberMonitorPlatform
+private[effect] object FiberMonitor {
+  def apply(compute: ExecutionContext): FiberMonitor = {
+    if (LinkingInfo.developmentMode && weakRefsAvailable) {
+      if (compute.isInstanceOf[BatchingMacrotaskExecutor]) {
+        val bmec = compute.asInstanceOf[BatchingMacrotaskExecutor]
+        new FiberMonitorImpl(bmec)
+      } else {
+        new FiberMonitorImpl(null)
+      }
+    } else {
+      new NoOpFiberMonitor()
+    }
+  }
+
+  private[this] final val Undefined = "undefined"
+
+  /**
+   * Feature-tests for all the required, well, features :)
+   */
+  private[unsafe] def weakRefsAvailable: Boolean =
+    js.typeOf(js.Dynamic.global.WeakRef) != Undefined &&
+      js.typeOf(js.Dynamic.global.FinalizationRegistry) != Undefined
+}