Incremental Scala.js Linking (#2928)

* Incremental Scala.js Linking

* Cleanup code

* Rename function

* Use released version

* Close streams before destroying process

modules/build/src/main/scala/scala/build/internal/Runner.scala

@@ -30,22 +30,25 @@ object Runner {
       logger,
       allowExecve = true,
       cwd,
-      extraEnv
+      extraEnv,
+      inheritStreams = true
     )
 
   def run(
     command: Seq[String],
     logger: Logger,
     cwd: Option[os.Path] = None,
-    extraEnv: Map[String, String] = Map.empty
+    extraEnv: Map[String, String] = Map.empty,
+    inheritStreams: Boolean = true
   ): Process =
     run0(
       "unused",
       command,
       logger,
       allowExecve = false,
       cwd,
-      extraEnv
+      extraEnv,
+      inheritStreams
     )
 
   def run0(
@@ -54,7 +57,8 @@ object Runner {
     logger: Logger,
     allowExecve: Boolean,
     cwd: Option[os.Path],
-    extraEnv: Map[String, String]
+    extraEnv: Map[String, String],
+    inheritStreams: Boolean
   ): Process = {
 
     import logger.{log, debug}
@@ -81,6 +85,12 @@ object Runner {
     else {
       val b = new ProcessBuilder(command: _*)
         .inheritIO()
+
+      if (!inheritStreams) {
+        b.redirectInput(ProcessBuilder.Redirect.PIPE)
+        b.redirectOutput(ProcessBuilder.Redirect.PIPE)
+      }
+
       if (extraEnv.nonEmpty) {
         val env = b.environment()
         for ((k, v) <- extraEnv)

modules/cli/src/main/scala/scala/cli/commands/package0/Package.scala

@@ -913,6 +913,37 @@ object Package extends ScalaCommand[PackageOptions] with BuildCommandHelpers {
     finally os.remove(jar)
   }
 
+  private object LinkingDir {
+    case class Input(linkJsInput: ScalaJsLinker.LinkJSInput, scratchDirOpt: Option[os.Path])
+    private var currentInput: Option[Input]        = None
+    private var currentLinkingDir: Option[os.Path] = None
+    def getOrCreate(
+      linkJsInput: ScalaJsLinker.LinkJSInput,
+      scratchDirOpt: Option[os.Path]
+    ): os.Path =
+      val input = Input(linkJsInput, scratchDirOpt)
+      currentLinkingDir match {
+        case Some(linkingDir) if currentInput.contains(input) =>
+          linkingDir
+        case _ =>
+          scratchDirOpt.foreach(os.makeDir.all(_))
+
+          currentLinkingDir.foreach(dir => os.remove.all(dir))
+          currentLinkingDir = None
+
+          val linkingDirectory = os.temp.dir(
+            dir = scratchDirOpt.orNull,
+            prefix = "scala-cli-js-linking",
+            deleteOnExit = scratchDirOpt.isEmpty
+          )
+
+          currentInput = Some(input)
+          currentLinkingDir = Some(linkingDirectory)
+
+          linkingDirectory
+      }
+  }
+
   def linkJs(
     build: Build.Successful,
     dest: os.Path,
@@ -926,30 +957,29 @@ object Package extends ScalaCommand[PackageOptions] with BuildCommandHelpers {
   ): Either[BuildException, os.Path] = {
     val mainJar   = Library.libraryJar(build)
     val classPath = mainJar +: build.artifacts.classPath
-    val delete    = scratchDirOpt.isEmpty
-    scratchDirOpt.foreach(os.makeDir.all(_))
-    val linkingDir =
-      os.temp.dir(
-        dir = scratchDirOpt.orNull,
-        prefix = "scala-cli-js-linking",
-        deleteOnExit = delete
-      )
+    val input = ScalaJsLinker.LinkJSInput(
+      options = build.options.notForBloopOptions.scalaJsLinkerOptions,
+      javaCommand =
+        build.options.javaHome().value.javaCommand, // FIXME Allow users to use another JVM here?
+      classPath = classPath,
+      mainClassOrNull = mainClassOpt.orNull,
+      addTestInitializer = addTestInitializer,
+      config = config,
+      fullOpt = fullOpt,
+      noOpt = noOpt,
+      scalaJsVersion = build.options.scalaJsOptions.finalVersion
+    )
+
+    val linkingDir = LinkingDir.getOrCreate(input, scratchDirOpt)
+
     either {
       value {
         ScalaJsLinker.link(
-          build.options.notForBloopOptions.scalaJsLinkerOptions,
-          build.options.javaHome().value.javaCommand, // FIXME Allow users to use another JVM here?
-          classPath,
-          mainClassOpt.orNull,
-          addTestInitializer,
-          config,
+          input,
           linkingDir,
-          fullOpt,
-          noOpt,
           logger,
           build.options.finalCache,
-          build.options.archiveCache,
-          build.options.scalaJsOptions.finalVersion
+          build.options.archiveCache
         )
       }
       val relMainJs      = os.rel / "main.js"
@@ -988,8 +1018,7 @@ object Package extends ScalaCommand[PackageOptions] with BuildCommandHelpers {
             os.copy(sourceMapJs, sourceMapDest, replaceExisting = true)
             logger.message(s"Emitted js source maps to: $sourceMapDest")
           }
-          if (delete)
-            os.remove.all(linkingDir)
+
           dest
         }
       else {

modules/cli/src/main/scala/scala/cli/commands/pgp/PgpExternalCommand.scala

@@ -62,7 +62,8 @@ abstract class PgpExternalCommand extends ExternalCommand {
       logger,
       allowExecve = allowExecve,
       cwd = None,
-      extraEnv = extraEnv
+      extraEnv = extraEnv,
+      inheritStreams = true
     ).waitFor()
   }
 

modules/cli/src/main/scala/scala/cli/internal/ScalaJsLinker.scala

@@ -7,18 +7,31 @@ import coursier.util.Task
 import dependency._
 import org.scalajs.testing.adapter.{TestAdapterInitializer => TAI}
 
-import java.io.File
+import java.io.{File, InputStream, OutputStream}
 
 import scala.build.EitherCps.{either, value}
 import scala.build.errors.{BuildException, ScalaJsLinkingError}
 import scala.build.internal.Util.{DependencyOps, ModuleOps}
 import scala.build.internal.{ExternalBinaryParams, FetchExternalBinary, Runner, ScalaJsLinkerConfig}
 import scala.build.options.scalajs.ScalaJsLinkerOptions
 import scala.build.{Logger, Positioned}
+import scala.io.Source
 import scala.util.Properties
 
 object ScalaJsLinker {
 
+  case class LinkJSInput(
+    options: ScalaJsLinkerOptions,
+    javaCommand: String,
+    classPath: Seq[os.Path],
+    mainClassOrNull: String,
+    addTestInitializer: Boolean,
+    config: ScalaJsLinkerConfig,
+    fullOpt: Boolean,
+    noOpt: Boolean,
+    scalaJsVersion: String
+  )
+
   private def linkerMainClass = "org.scalajs.cli.Scalajsld"
 
   private def linkerCommand(
@@ -98,61 +111,157 @@ object ScalaJsLinker {
     }
   }
 
-  def link(
-    options: ScalaJsLinkerOptions,
-    javaCommand: String,
-    classPath: Seq[os.Path],
-    mainClassOrNull: String,
-    addTestInitializer: Boolean,
-    config: ScalaJsLinkerConfig,
+  private def getCommand(
+    input: LinkJSInput,
     linkingDir: os.Path,
-    fullOpt: Boolean,
-    noOpt: Boolean,
     logger: Logger,
     cache: FileCache[Task],
     archiveCache: ArchiveCache[Task],
-    scalaJsVersion: String
-  ): Either[BuildException, Unit] = either {
-
+    useLongRunning: Boolean
+  ) = either {
     val command = value {
-      linkerCommand(options, javaCommand, logger, cache, archiveCache, scalaJsVersion)
+      linkerCommand(
+        input.options,
+        input.javaCommand,
+        logger,
+        cache,
+        archiveCache,
+        input.scalaJsVersion
+      )
     }
 
     val allArgs = {
-      val outputArgs = Seq("--outputDir", linkingDir.toString)
+      val outputArgs  = Seq("--outputDir", linkingDir.toString)
+      val longRunning = if (useLongRunning) Seq("--longRunning") else Seq.empty[String]
       val mainClassArgs =
-        Option(mainClassOrNull).toSeq.flatMap(mainClass => Seq("--mainMethod", mainClass + ".main"))
+        Option(input.mainClassOrNull).toSeq.flatMap(mainClass =>
+          Seq("--mainMethod", mainClass + ".main")
+        )
       val testInitializerArgs =
-        if (addTestInitializer)
+        if (input.addTestInitializer)
           Seq("--mainMethodWithNoArgs", TAI.ModuleClassName + "." + TAI.MainMethodName)
         else
           Nil
       val optArg =
-        if (noOpt) "--noOpt"
-        else if (fullOpt) "--fullOpt"
+        if (input.noOpt) "--noOpt"
+        else if (input.fullOpt) "--fullOpt"
         else "--fastOpt"
 
       Seq[os.Shellable](
         outputArgs,
         mainClassArgs,
         testInitializerArgs,
         optArg,
-        config.linkerCliArgs,
-        classPath.map(_.toString)
+        input.config.linkerCliArgs,
+        input.classPath.map(_.toString),
+        longRunning
       )
     }
 
-    val cmd     = command ++ allArgs.flatMap(_.value)
-    val res     = Runner.run(cmd, logger)
-    val retCode = res.waitFor()
+    command ++ allArgs.flatMap(_.value)
+  }
+
+  def link(
+    input: LinkJSInput,
+    linkingDir: os.Path,
+    logger: Logger,
+    cache: FileCache[Task],
+    archiveCache: ArchiveCache[Task]
+  ): Either[BuildException, Unit] = either {
+    val useLongRunning = !input.fullOpt
 
-    if (retCode == 0)
-      logger.debug("Scala.js linker ran successfully")
+    if (useLongRunning)
+      longRunningProcess.startOrReuse(input, linkingDir, logger, cache, archiveCache)
     else {
-      logger.debug(s"Scala.js linker exited with return code $retCode")
-      value(Left(new ScalaJsLinkingError))
+      val cmd =
+        value(getCommand(input, linkingDir, logger, cache, archiveCache, useLongRunning = false))
+      val res     = Runner.run(cmd, logger)
+      val retCode = res.waitFor()
+
+      if (retCode == 0)
+        logger.debug("Scala.js linker ran successfully")
+      else {
+        logger.debug(s"Scala.js linker exited with return code $retCode")
+        value(Left(new ScalaJsLinkingError))
+      }
     }
   }
+
+  private object longRunningProcess {
+    case class Proc(process: Process, stdin: OutputStream, stdout: InputStream) {
+      val stdoutLineIterator: Iterator[String] = Source.fromInputStream(stdout).getLines()
+    }
+    case class Input(input: LinkJSInput, linkingDir: os.Path)
+    var currentInput: Option[Input] = None
+    var currentProc: Option[Proc]   = None
+
+    def startOrReuse(
+      linkJsInput: LinkJSInput,
+      linkingDir: os.Path,
+      logger: Logger,
+      cache: FileCache[Task],
+      archiveCache: ArchiveCache[Task]
+    ) = either {
+      val input = Input(linkJsInput, linkingDir)
+
+      def createProcess(): Proc = {
+        val cmd =
+          value(getCommand(
+            linkJsInput,
+            linkingDir,
+            logger,
+            cache,
+            archiveCache,
+            useLongRunning = true
+          ))
+        val process = Runner.run(cmd, logger, inheritStreams = false)
+        val stdin   = process.getOutputStream()
+        val stdout  = process.getInputStream()
+        val proc    = Proc(process, stdin, stdout)
+        currentProc = Some(proc)
+        currentInput = Some(input)
+        proc
+      }
+
+      def loop(proc: Proc): Unit =
+        if (proc.stdoutLineIterator.hasNext) {
+          val line = proc.stdoutLineIterator.next()
+
+          if (line == "SCALA_JS_LINKING_DONE")
+            logger.debug("Scala.js linker ran successfully")
+          else {
+            // inherit other stdout from Scala.js
+            println(line)
+
+            loop(proc)
+          }
+        }
+        else {
+          val retCode = proc.process.waitFor()
+          logger.debug(s"Scala.js linker exited with return code $retCode")
+          value(Left(new ScalaJsLinkingError))
+        }
+
+      val proc = currentProc match {
+        case Some(proc) if currentInput.contains(input) && proc.process.isAlive() =>
+          // trigger new linking
+          proc.stdin.write('\n')
+          proc.stdin.flush()
+
+          proc
+        case Some(proc) =>
+          proc.stdin.close()
+          proc.stdout.close()
+          proc.process.destroy()
+          createProcess()
+        case _ =>
+          createProcess()
+      }
+
+      loop(proc)
+    }
+  }
+
   def updateSourceMappingURL(mainJsPath: os.Path) =
     val content = os.read(mainJsPath)
     content.replace(