From 1577821689ecec76905bb525ab431eafb80b50f7 Mon Sep 17 00:00:00 2001 From: Guillaume Martres Date: Wed, 22 Aug 2018 16:19:28 +0900 Subject: [PATCH 1/4] Upgrade ammonite to 1.1.2-30-53edc31 This is mainly to get https://github.com/lihaoyi/Ammonite/pull/851 which should reduce the amount of unnecessary work done by incremental compilation in the Mill build. This requires some code changes since this means we now depend on a more recent version of coursier, as a side-effect this means that we do not depend on scalaz anymore. Also use the same ammonite version in the Mill build and in ScalaModule#ammoniteReplClasspath. Also remove an incorrect dependency in the caffeine integration test. This was always wrong but did not start failing until this commit, probably due to dependencies appearing in a different order on the classpath. --- build.sc | 3 ++- integration/test/resources/caffeine/build.sc | 7 +++---- main/src/mill/modules/Jvm.scala | 21 ++++++++++++------- scalalib/src/mill/scalalib/ScalaModule.scala | 2 +- scalalib/src/mill/scalalib/Versions.scala | 2 ++ .../metadata/MavenMetadataLoader.scala | 6 ++++-- .../metadata/MetadataLoaderFactoryTests.scala | 4 ++-- 7 files changed, 27 insertions(+), 18 deletions(-) diff --git a/build.sc b/build.sc index 456260d6c95..f750d6e02d6 100755 --- a/build.sc +++ b/build.sc @@ -85,7 +85,8 @@ object main extends MillModule { ) def ivyDeps = Agg( - ivy"com.lihaoyi:::ammonite:1.1.2-6-27842d9", + // Keep synchronized with ammonite in Versions.scala + ivy"com.lihaoyi:::ammonite:1.1.2-30-53edc31", // Necessary so we can share the JNA classes throughout the build process ivy"net.java.dev.jna:jna:4.5.0", ivy"net.java.dev.jna:jna-platform:4.5.0" diff --git a/integration/test/resources/caffeine/build.sc b/integration/test/resources/caffeine/build.sc index aaa66626424..a4967f31ed1 100644 --- a/integration/test/resources/caffeine/build.sc +++ b/integration/test/resources/caffeine/build.sc @@ -8,11 +8,11 @@ import deps.{benchmarkLibraries, benchmarkVersions, libraries, testLibraries, te trait CaffeineModule extends MavenModule{ def repositories = super.repositories ++ Seq( - coursier.ivy.IvyRepository( + coursier.ivy.IvyRepository.parse( "https://dl.bintray.com/sbt/sbt-plugin-releases/" + coursier.ivy.Pattern.default.string, dropInfoAttributes = true - ), + ).toOption.get, MavenRepository("https://jcenter.bintray.com/"), MavenRepository("https://jitpack.io/"), MavenRepository("http://repo.spring.io/plugins-release") @@ -25,7 +25,6 @@ trait CaffeineModule extends MavenModule{ libraries.guava, testLibraries.mockito, testLibraries.hamcrest, - ivy"org.hamcrest:hamcrest-library:1.3", testLibraries.awaitility, ) ++ testLibraries.testng ++ @@ -151,4 +150,4 @@ object simulator extends CaffeineModule { def ivyDeps = super.ivyDeps() ++ testLibraries.testng } -} \ No newline at end of file +} diff --git a/main/src/mill/modules/Jvm.scala b/main/src/mill/modules/Jvm.scala index 3a6dcb9f203..32563f8faaa 100644 --- a/main/src/mill/modules/Jvm.scala +++ b/main/src/mill/modules/Jvm.scala @@ -10,6 +10,7 @@ import java.util.jar.{JarEntry, JarFile, JarOutputStream} import ammonite.ops._ import coursier.{Cache, Dependency, Fetch, Repository, Resolution} +import coursier.util.{Gather, Task} import geny.Generator import mill.main.client.InputPumper import mill.eval.{PathRef, Result} @@ -413,17 +414,19 @@ object Jvm { def load(artifacts: Seq[coursier.Artifact]) = { val logger = None - val loadedArtifacts = scalaz.concurrent.Task.gatherUnordered( + + import scala.concurrent.ExecutionContext.Implicits.global + val loadedArtifacts = Gather[Task].gather( for (a <- artifacts) - yield coursier.Cache.file(a, logger = logger).run + yield coursier.Cache.file[Task](a, logger = logger).run .map(a.isOptional -> _) - ).unsafePerformSync + ).unsafeRun val errors = loadedArtifacts.collect { - case (false, scalaz.-\/(x)) => x - case (true, scalaz.-\/(x)) if !x.notFound => x + case (false, Left(x)) => x + case (true, Left(x)) if !x.notFound => x } - val successes = loadedArtifacts.collect { case (_, scalaz.\/-(x)) => x } + val successes = loadedArtifacts.collect { case (_, Right(x)) => x } (errors, successes) } @@ -459,8 +462,10 @@ object Jvm { mapDependencies = mapDependencies ) - val fetch = Fetch.from(repositories, Cache.fetch()) - val resolution = start.process.run(fetch).unsafePerformSync + val fetch = Fetch.from(repositories, Cache.fetch[Task]()) + + import scala.concurrent.ExecutionContext.Implicits.global + val resolution = start.process.run(fetch).unsafeRun() (deps.toSeq, resolution) } } diff --git a/scalalib/src/mill/scalalib/ScalaModule.scala b/scalalib/src/mill/scalalib/ScalaModule.scala index 1899ee1477c..c56a7b742fa 100644 --- a/scalalib/src/mill/scalalib/ScalaModule.scala +++ b/scalalib/src/mill/scalalib/ScalaModule.scala @@ -204,7 +204,7 @@ trait ScalaModule extends JavaModule { outer => unmanagedClasspath() ++ resolveDeps(T.task{ runIvyDeps() ++ scalaLibraryIvyDeps() ++ transitiveIvyDeps() ++ - Agg(ivy"com.lihaoyi:::ammonite:1.1.2") + Agg(ivy"com.lihaoyi:::ammonite:${Versions.ammonite}") })() } diff --git a/scalalib/src/mill/scalalib/Versions.scala b/scalalib/src/mill/scalalib/Versions.scala index 6c1c385f267..094f2a551be 100644 --- a/scalalib/src/mill/scalalib/Versions.scala +++ b/scalalib/src/mill/scalalib/Versions.scala @@ -1,6 +1,8 @@ package mill.scalalib object Versions { + // Keep synchronized with ammonite dependency in core in build.sc + val ammonite = "1.1.2-30-53edc31" // Keep synchronized with zinc dependency in scalalib.worker in build.sc val zinc = "1.1.7" } diff --git a/scalalib/src/mill/scalalib/dependency/metadata/MavenMetadataLoader.scala b/scalalib/src/mill/scalalib/dependency/metadata/MavenMetadataLoader.scala index 7f25764b3c1..491911bf45c 100644 --- a/scalalib/src/mill/scalalib/dependency/metadata/MavenMetadataLoader.scala +++ b/scalalib/src/mill/scalalib/dependency/metadata/MavenMetadataLoader.scala @@ -2,16 +2,18 @@ package mill.scalalib.dependency.metadata import coursier.Cache import coursier.maven.MavenRepository +import coursier.util.Task import mill.scalalib.dependency.versions.Version private[dependency] final case class MavenMetadataLoader(mavenRepo: MavenRepository) extends MetadataLoader { - private val fetch = Cache.fetch() + private val fetch = Cache.fetch[Task]() override def getVersions(module: coursier.Module): List[Version] = { + import scala.concurrent.ExecutionContext.Implicits.global // TODO fallback to 'versionsFromListing' if 'versions' doesn't work? (needs to be made public in coursier first) - val allVersions = mavenRepo.versions(module, fetch).run.unsafePerformSync + val allVersions = mavenRepo.versions(module, fetch).run.unsafeRun allVersions .map(_.available.map(Version(_))) .getOrElse(List.empty) diff --git a/scalalib/test/src/mill/scalalib/dependency/metadata/MetadataLoaderFactoryTests.scala b/scalalib/test/src/mill/scalalib/dependency/metadata/MetadataLoaderFactoryTests.scala index 8187976c836..4c2206b8d4d 100644 --- a/scalalib/test/src/mill/scalalib/dependency/metadata/MetadataLoaderFactoryTests.scala +++ b/scalalib/test/src/mill/scalalib/dependency/metadata/MetadataLoaderFactoryTests.scala @@ -32,7 +32,7 @@ import coursier.Fetch.Content import coursier.core.{Artifact, Module, Project, Repository} import coursier.ivy.IvyRepository import coursier.maven.MavenRepository -import scalaz.{EitherT, Monad} +import coursier.util.{EitherT, Monad} import utest._ object MetadataLoaderFactoryTests extends TestSuite { @@ -45,7 +45,7 @@ object MetadataLoaderFactoryTests extends TestSuite { } } 'ivyRepository - { - val ivyRepo = IvyRepository( + val Right(ivyRepo) = IvyRepository.parse( "https://dl.bintray.com/sbt/sbt-plugin-releases/" + coursier.ivy.Pattern.default.string, dropInfoAttributes = true) assertMatch(MetadataLoaderFactory(ivyRepo)) { case None => } From ad27d38d69ee033c6b68525e2a0c485049f1cfd9 Mon Sep 17 00:00:00 2001 From: Guillaume Martres Date: Wed, 22 Aug 2018 01:07:54 +0900 Subject: [PATCH 2/4] Upgrade to Zinc 1.2.1 --- build.sc | 2 +- scalalib/src/mill/scalalib/Versions.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sc b/build.sc index f750d6e02d6..b2745404d92 100755 --- a/build.sc +++ b/build.sc @@ -177,7 +177,7 @@ object scalalib extends MillModule { def ivyDeps = Agg( // Keep synchronized with zinc in Versions.scala - ivy"org.scala-sbt::zinc:1.1.7" + ivy"org.scala-sbt::zinc:1.2.1" ) def testArgs = Seq( "-DMILL_SCALA_WORKER=" + runClasspath().map(_.path).mkString(",") diff --git a/scalalib/src/mill/scalalib/Versions.scala b/scalalib/src/mill/scalalib/Versions.scala index 094f2a551be..bfcc5495a75 100644 --- a/scalalib/src/mill/scalalib/Versions.scala +++ b/scalalib/src/mill/scalalib/Versions.scala @@ -4,5 +4,5 @@ object Versions { // Keep synchronized with ammonite dependency in core in build.sc val ammonite = "1.1.2-30-53edc31" // Keep synchronized with zinc dependency in scalalib.worker in build.sc - val zinc = "1.1.7" + val zinc = "1.2.1" } From 4a7a699456d3160965de6c7e4524cdf384c324e1 Mon Sep 17 00:00:00 2001 From: Guillaume Martres Date: Tue, 21 Aug 2018 22:31:01 +0900 Subject: [PATCH 3/4] Rename ScalaWorker to ZincWorker Starting with the next commit, it will be used in Java-only projects too, so the name is misleading. --- docs/VisualizePlan.svg | 30 +++++++++---------- docs/pages/2 - Configuring Mill.md | 6 ++-- docs/pages/5 - Modules.md | 2 +- main/src/mill/modules/Jvm.scala | 2 +- .../src/mill/scalajslib/ScalaJSModule.scala | 2 +- scalalib/src/mill/scalalib/JavaModule.scala | 10 +++---- scalalib/src/mill/scalalib/Lib.scala | 2 +- scalalib/src/mill/scalalib/ScalaModule.scala | 8 ++--- ...alaWorkerApi.scala => ZincWorkerApi.scala} | 12 ++++---- .../scalalib/scalafmt/ScalafmtModule.scala | 2 +- ...aWorkerImpl.scala => ZincWorkerImpl.scala} | 4 +-- .../scalanativelib/ScalaNativeModule.scala | 4 +-- 12 files changed, 42 insertions(+), 42 deletions(-) rename scalalib/src/mill/scalalib/{ScalaWorkerApi.scala => ZincWorkerApi.scala} (87%) rename scalalib/worker/src/mill/scalalib/worker/{ScalaWorkerImpl.scala => ZincWorkerImpl.scala} (97%) diff --git a/docs/VisualizePlan.svg b/docs/VisualizePlan.svg index f6e50b39800..9f13384fae9 100644 --- a/docs/VisualizePlan.svg +++ b/docs/VisualizePlan.svg @@ -3,33 +3,33 @@ example1 - + -mill.scalalib.ScalaWorkerModule.classpath +mill.scalalib.ZincWorkerModule.classpath -mill.scalalib.ScalaWorkerModule.classpath +mill.scalalib.ZincWorkerModule.classpath - + -mill.scalalib.ScalaWorkerModule.compilerInterfaceClasspath +mill.scalalib.ZincWorkerModule.compilerInterfaceClasspath -mill.scalalib.ScalaWorkerModule.compilerInterfaceClasspath +mill.scalalib.ZincWorkerModule.compilerInterfaceClasspath - + -mill.scalalib.ScalaWorkerModule.worker +mill.scalalib.ZincWorkerModule.worker -mill.scalalib.ScalaWorkerModule.worker +mill.scalalib.ZincWorkerModule.worker - + -mill.scalalib.ScalaWorkerModule.worker->mill.scalalib.ScalaWorkerModule.classpath +mill.scalalib.ZincWorkerModule.worker->mill.scalalib.ZincWorkerModule.classpath - + -mill.scalalib.ScalaWorkerModule.worker->mill.scalalib.ScalaWorkerModule.compilerInterfaceClasspath +mill.scalalib.ZincWorkerModule.worker->mill.scalalib.ZincWorkerModule.compilerInterfaceClasspath @@ -297,9 +297,9 @@ moduledefs.compile - + -moduledefs.compile->mill.scalalib.ScalaWorkerModule.worker +moduledefs.compile->mill.scalalib.ZincWorkerModule.worker diff --git a/docs/pages/2 - Configuring Mill.md b/docs/pages/2 - Configuring Mill.md index 7db19833579..989272dd493 100644 --- a/docs/pages/2 - Configuring Mill.md +++ b/docs/pages/2 - Configuring Mill.md @@ -72,20 +72,20 @@ def repositories = super.repositories ++ Seq( ``` To add custom resolvers to the initial bootstrap of the build, you can create a -custom `ScalaWorkerModule`, and override the `scalaWorker` method in your +custom `ZincWorkerModule`, and override the `zincWorker` method in your `ScalaModule` by pointing it to that custom object: ```scala import coursier.maven.MavenRepository -object CustomScalaWorkerModule extends ScalaWorkerModule { +object CustomZincWorkerModule extends ZincWorkerModule { def repositories() = super.repositories ++ Seq( MavenRepository("https://oss.sonatype.org/content/repositories/releases") ) } object YourBuild extends ScalaModule { - def scalaWorker = CustomScalaWorkerModule + def zincWorker = CustomZincWorkerModule // ... rest of your build definitions } ``` diff --git a/docs/pages/5 - Modules.md b/docs/pages/5 - Modules.md index 2c13a3e9530..fa6260f3205 100644 --- a/docs/pages/5 - Modules.md +++ b/docs/pages/5 - Modules.md @@ -152,7 +152,7 @@ mill foo.Bar/qux `ExternalModule`s are useful for someone providing a library for use with Mill that is shared by the entire build: for example, -`mill.scalalib.ScalaWorkerApi/scalaWorker` provides a shared Scala compilation +`mill.scalalib.ZincWorkerApi/zincWorker` provides a shared Scala compilation service & cache that is shared between all `ScalaModule`s, and `mill.scalalib.GenIdea/idea` lets you generate IntelliJ projects without needing to define your own `T.command` in your `build.sc` file diff --git a/main/src/mill/modules/Jvm.scala b/main/src/mill/modules/Jvm.scala index 32563f8faaa..96992d20fce 100644 --- a/main/src/mill/modules/Jvm.scala +++ b/main/src/mill/modules/Jvm.scala @@ -384,7 +384,7 @@ object Jvm { /** * Resolve dependencies using Coursier. * - * We do not bother breaking this out into the separate ScalaWorker classpath, + * We do not bother breaking this out into the separate ZincWorkerApi classpath, * because Coursier is already bundled with mill/Ammonite to support the * `import $ivy` syntax. */ diff --git a/scalajslib/src/mill/scalajslib/ScalaJSModule.scala b/scalajslib/src/mill/scalajslib/ScalaJSModule.scala index efa4dfe0190..e0ee55a529a 100644 --- a/scalajslib/src/mill/scalajslib/ScalaJSModule.scala +++ b/scalajslib/src/mill/scalajslib/ScalaJSModule.scala @@ -15,7 +15,7 @@ trait ScalaJSModule extends scalalib.ScalaModule { outer => def scalaJSVersion: T[String] trait Tests extends TestScalaJSModule { - override def scalaWorker = outer.scalaWorker + override def zincWorker = outer.zincWorker override def scalaOrganization = outer.scalaOrganization() override def scalaVersion = outer.scalaVersion() override def scalaJSVersion = outer.scalaJSVersion() diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index 259e5cd367e..b5e257b8b25 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -17,7 +17,7 @@ import mill.util.Loose.Agg * Core configuration required to compile a single Scala compilation target */ trait JavaModule extends mill.Module with TaskModule { outer => - def scalaWorker: ScalaWorkerModule = mill.scalalib.ScalaWorkerModule + def zincWorker: ZincWorkerModule = mill.scalalib.ZincWorkerModule trait Tests extends TestModule{ override def moduleDeps = Seq(outer) @@ -98,7 +98,7 @@ trait JavaModule extends mill.Module with TaskModule { outer => } - def repositories: Seq[Repository] = scalaWorker.repositories + def repositories: Seq[Repository] = zincWorker.repositories def platformSuffix = T{ "" } @@ -317,7 +317,7 @@ trait JavaModule extends mill.Module with TaskModule { outer => val (procId, procTombstone, token) = backgroundSetup(T.ctx().dest) try Result.Success(Jvm.interactiveSubprocess( "mill.scalalib.backgroundwrapper.BackgroundWrapper", - (runClasspath() ++ scalaWorker.backgroundWrapperClasspath()).map(_.path), + (runClasspath() ++ zincWorker.backgroundWrapperClasspath()).map(_.path), forkArgs(), forkEnv(), Seq(procId.toString, procTombstone.toString, token, finalMainClass()) ++ args, @@ -332,7 +332,7 @@ trait JavaModule extends mill.Module with TaskModule { outer => val (procId, procTombstone, token) = backgroundSetup(T.ctx().dest) try Result.Success(Jvm.interactiveSubprocess( "mill.scalalib.backgroundwrapper.BackgroundWrapper", - (runClasspath() ++ scalaWorker.backgroundWrapperClasspath()).map(_.path), + (runClasspath() ++ zincWorker.backgroundWrapperClasspath()).map(_.path), forkArgs(), forkEnv(), Seq(procId.toString, procTombstone.toString, token, mainClass) ++ args, @@ -384,7 +384,7 @@ trait TestModule extends JavaModule with TaskModule { Jvm.subprocess( mainClass = "mill.scalalib.TestRunner", - classPath = scalaWorker.scalalibClasspath().map(_.path), + classPath = zincWorker.scalalibClasspath().map(_.path), jvmArgs = forkArgs(), envArgs = forkEnv(), mainArgs = diff --git a/scalalib/src/mill/scalalib/Lib.scala b/scalalib/src/mill/scalalib/Lib.scala index e890caaf59d..d1429188a4b 100644 --- a/scalalib/src/mill/scalalib/Lib.scala +++ b/scalalib/src/mill/scalalib/Lib.scala @@ -104,7 +104,7 @@ object Lib{ /** * Resolve dependencies using Coursier. * - * We do not bother breaking this out into the separate ScalaWorker classpath, + * We do not bother breaking this out into the separate ZincWorker classpath, * because Coursier is already bundled with mill/Ammonite to support the * `import $ivy` syntax. */ diff --git a/scalalib/src/mill/scalalib/ScalaModule.scala b/scalalib/src/mill/scalalib/ScalaModule.scala index c56a7b742fa..c6374729d45 100644 --- a/scalalib/src/mill/scalalib/ScalaModule.scala +++ b/scalalib/src/mill/scalalib/ScalaModule.scala @@ -24,7 +24,7 @@ trait ScalaModule extends JavaModule { outer => override def scalacPluginIvyDeps = outer.scalacPluginIvyDeps override def scalacOptions = outer.scalacOptions override def javacOptions = outer.javacOptions - override def scalaWorker = outer.scalaWorker + override def zincWorker = outer.zincWorker override def moduleDeps: Seq[JavaModule] = Seq(outer) } @@ -64,7 +64,7 @@ trait ScalaModule extends JavaModule { outer => mainClass() match{ case Some(m) => Right(m) case None => - scalaWorker.worker().discoverMainClasses(compile())match { + zincWorker.worker().discoverMainClasses(compile())match { case Seq() => Left("No main class specified or found") case Seq(main) => Right(main) case mains => @@ -81,7 +81,7 @@ trait ScalaModule extends JavaModule { outer => def scalacOptions = T{ Seq.empty[String] } - override def repositories: Seq[Repository] = scalaWorker.repositories + override def repositories: Seq[Repository] = zincWorker.repositories private val Milestone213 = raw"""2.13.(\d+)-M(\d+)""".r @@ -142,7 +142,7 @@ trait ScalaModule extends JavaModule { outer => } override def compile: T[CompilationResult] = T.persistent{ - scalaWorker.worker().compileScala( + zincWorker.worker().compileScala( scalaVersion(), allSourceFiles().map(_.path), scalaCompilerBridgeSources(), diff --git a/scalalib/src/mill/scalalib/ScalaWorkerApi.scala b/scalalib/src/mill/scalalib/ZincWorkerApi.scala similarity index 87% rename from scalalib/src/mill/scalalib/ScalaWorkerApi.scala rename to scalalib/src/mill/scalalib/ZincWorkerApi.scala index 547cc5db3c5..f52332309a3 100644 --- a/scalalib/src/mill/scalalib/ScalaWorkerApi.scala +++ b/scalalib/src/mill/scalalib/ZincWorkerApi.scala @@ -11,10 +11,10 @@ import mill.scalalib.Lib.resolveDependencies import mill.util.Loose import mill.util.JsonFormatters._ -object ScalaWorkerModule extends mill.define.ExternalModule with ScalaWorkerModule{ +object ZincWorkerModule extends mill.define.ExternalModule with ZincWorkerModule{ lazy val millDiscover = Discover[this.type] } -trait ScalaWorkerModule extends mill.Module{ +trait ZincWorkerModule extends mill.Module{ def repositories = Seq( Cache.ivy2Local, MavenRepository("https://repo1.maven.org/maven2"), @@ -36,15 +36,15 @@ trait ScalaWorkerModule extends mill.Module{ ) } - def worker: Worker[ScalaWorkerApi] = T.worker{ + def worker: Worker[ZincWorkerApi] = T.worker{ val cl = mill.util.ClassLoader.create( classpath().map(_.path.toNIO.toUri.toURL).toVector, getClass.getClassLoader ) - val cls = cl.loadClass("mill.scalalib.worker.ScalaWorkerImpl") + val cls = cl.loadClass("mill.scalalib.worker.ZincWorkerImpl") val instance = cls.getConstructor(classOf[mill.util.Ctx], classOf[Array[String]]) .newInstance(T.ctx(), compilerInterfaceClasspath().map(_.path.toString).toArray[String]) - instance.asInstanceOf[ScalaWorkerApi] + instance.asInstanceOf[ZincWorkerApi] } def compilerInterfaceClasspath = T{ @@ -57,7 +57,7 @@ trait ScalaWorkerModule extends mill.Module{ } -trait ScalaWorkerApi { +trait ZincWorkerApi { def compileScala(scalaVersion: String, sources: Agg[Path], diff --git a/scalalib/src/mill/scalalib/scalafmt/ScalafmtModule.scala b/scalalib/src/mill/scalalib/scalafmt/ScalafmtModule.scala index c4d64b0e648..74f64e28c23 100644 --- a/scalalib/src/mill/scalalib/scalafmt/ScalafmtModule.scala +++ b/scalalib/src/mill/scalalib/scalafmt/ScalafmtModule.scala @@ -23,7 +23,7 @@ trait ScalafmtModule extends JavaModule { def scalafmtDeps: T[Agg[PathRef]] = T { Lib.resolveDependencies( - scalaWorker.repositories, + zincWorker.repositories, Lib.depToDependency(_, "2.12.4"), Seq(ivy"com.geirsson::scalafmt-cli:${scalafmtVersion()}") ) diff --git a/scalalib/worker/src/mill/scalalib/worker/ScalaWorkerImpl.scala b/scalalib/worker/src/mill/scalalib/worker/ZincWorkerImpl.scala similarity index 97% rename from scalalib/worker/src/mill/scalalib/worker/ScalaWorkerImpl.scala rename to scalalib/worker/src/mill/scalalib/worker/ZincWorkerImpl.scala index ebeb9561295..5fce933cd85 100644 --- a/scalalib/worker/src/mill/scalalib/worker/ScalaWorkerImpl.scala +++ b/scalalib/worker/src/mill/scalalib/worker/ZincWorkerImpl.scala @@ -24,8 +24,8 @@ case class MockedLookup(am: File => Optional[CompileAnalysis]) extends PerClassp Locate.definesClass(classpathEntry) } -class ScalaWorkerImpl(ctx0: mill.util.Ctx, - compilerBridgeClasspath: Array[String]) extends mill.scalalib.ScalaWorkerApi{ +class ZincWorkerImpl(ctx0: mill.util.Ctx, + compilerBridgeClasspath: Array[String]) extends mill.scalalib.ZincWorkerApi{ @volatile var compilersCache = Option.empty[(Long, Compilers)] /** Compile the bridge if it doesn't exist yet and return the output directory. diff --git a/scalanativelib/src/mill/scalanativelib/ScalaNativeModule.scala b/scalanativelib/src/mill/scalanativelib/ScalaNativeModule.scala index 5e4fc4718e9..748afb79ea8 100644 --- a/scalanativelib/src/mill/scalanativelib/ScalaNativeModule.scala +++ b/scalanativelib/src/mill/scalanativelib/ScalaNativeModule.scala @@ -46,7 +46,7 @@ trait ScalaNativeModule extends ScalaModule { outer => override def artifactSuffix: T[String] = s"${platformSuffix()}_${artifactScalaVersion()}" trait Tests extends TestScalaNativeModule { - override def scalaWorker = outer.scalaWorker + override def zincWorker = outer.zincWorker override def scalaOrganization = outer.scalaOrganization() override def scalaVersion = outer.scalaVersion() override def scalaNativeVersion = outer.scalaNativeVersion() @@ -238,7 +238,7 @@ trait TestScalaNativeModule extends ScalaNativeModule with TestModule { testOute // creates a specific binary used for running tests - has a different (generated) main class // which knows the names of all the tests and references to invoke them object testRunnerNative extends ScalaNativeModule { - override def scalaWorker = testOuter.scalaWorker + override def zincWorker = testOuter.zincWorker override def scalaOrganization = testOuter.scalaOrganization() override def scalaVersion = testOuter.scalaVersion() override def scalaNativeVersion = testOuter.scalaNativeVersion() From cc9efca6e8521dcc21bb53da0c4fd4b14d69933f Mon Sep 17 00:00:00 2001 From: Guillaume Martres Date: Wed, 22 Aug 2018 01:07:05 +0900 Subject: [PATCH 4/4] Fix incremental compilation when a Scala project depends on a Java project Before this commit, JavaModule#compile simply called javac unconditionally, thus generating new classfiles every time. But if a Scala project depends on a Java project, this will throw off the incremental compilation algorithm which will unnecessarily recompile files. To avoid this we now use Zinc to compile Java projects too (as a bonus this means that Java compilation becomes incremental). This required some refactoring in ZincWorkerImpl to be able to compile stuff without having to pass Scala-specific options. The issue solved by this commit could be reproduced by running in the Mill repository: $ mill main.compile $ mill -i @ main.compile() and observing that before this commit, the `main.compile()` call ended up recompiling code. --- scalalib/src/mill/scalalib/JavaModule.scala | 24 +++- scalalib/src/mill/scalalib/Lib.scala | 28 ---- scalalib/src/mill/scalalib/ScalaModule.scala | 31 +---- .../src/mill/scalalib/ZincWorkerApi.scala | 20 ++- .../mill/scalalib/worker/ZincWorkerImpl.scala | 129 ++++++++++++------ 5 files changed, 126 insertions(+), 106 deletions(-) diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index b5e257b8b25..ab4f8795a94 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -23,6 +23,7 @@ trait JavaModule extends mill.Module with TaskModule { outer => override def moduleDeps = Seq(outer) override def repositories = outer.repositories override def javacOptions = outer.javacOptions + override def zincWorker = outer.zincWorker } def defaultCommandName() = "run" @@ -38,7 +39,16 @@ trait JavaModule extends mill.Module with TaskModule { outer => def finalMainClassOpt: T[Either[String, String]] = T{ mainClass() match{ case Some(m) => Right(m) - case None => Left("No main class specified or found") + case None => + zincWorker.worker().discoverMainClasses(compile())match { + case Seq() => Left("No main class specified or found") + case Seq(main) => Right(main) + case mains => + Left( + s"Multiple main classes found (${mains.mkString(",")}) " + + "please explicitly specify which one to use by overriding mainClass" + ) + } } } @@ -133,12 +143,12 @@ trait JavaModule extends mill.Module with TaskModule { outer => } yield PathRef(path) } - def compile: T[CompilationResult] = T{ - Lib.compileJava( - allSourceFiles().map(_.path.toIO).toArray, - compileClasspath().map(_.path.toIO).toArray, - javacOptions(), - upstreamCompileOutput() + def compile: T[CompilationResult] = T.persistent{ + zincWorker.worker().compileJava( + upstreamCompileOutput(), + allSourceFiles().map(_.path), + compileClasspath().map(_.path), + javacOptions() ) } diff --git a/scalalib/src/mill/scalalib/Lib.scala b/scalalib/src/mill/scalalib/Lib.scala index d1429188a4b..827c6bf4551 100644 --- a/scalalib/src/mill/scalalib/Lib.scala +++ b/scalalib/src/mill/scalalib/Lib.scala @@ -27,34 +27,6 @@ object CompilationResult { case class CompilationResult(analysisFile: Path, classes: PathRef) object Lib{ - def compileJava(sources: Array[java.io.File], - classpath: Array[java.io.File], - javaOpts: Seq[String], - upstreamCompileOutput: Seq[CompilationResult]) - (implicit ctx: mill.util.Ctx) = { - val javac = ToolProvider.getSystemJavaCompiler() - if (javac == null) { - throw new Exception( - "Your Java installation is not a JDK, so it can't compile Java code;" + - " Please install the JDK version of Java") - } - - rm(ctx.dest / 'classes) - mkdir(ctx.dest / 'classes) - val cpArgs = - if(classpath.isEmpty) Seq() - else Seq("-cp", classpath.mkString(File.pathSeparator)) - - val args = Seq("-d", ctx.dest / 'classes) ++ cpArgs ++ javaOpts ++ sources - - javac.run( - ctx.log.inStream, ctx.log.outputStream, ctx.log.errorStream, - args.map(_.toString):_* - ) - if (ls(ctx.dest / 'classes).isEmpty) mill.eval.Result.Failure("Compilation Failed") - else mill.eval.Result.Success(CompilationResult(ctx.dest / 'zinc, PathRef(ctx.dest / 'classes))) - } - private val ReleaseVersion = raw"""(\d+)\.(\d+)\.(\d+)""".r private val MinorSnapshotVersion = raw"""(\d+)\.(\d+)\.([1-9]\d*)-SNAPSHOT""".r private val DottyVersion = raw"""0\.(\d+)\.(\d+).*""".r diff --git a/scalalib/src/mill/scalalib/ScalaModule.scala b/scalalib/src/mill/scalalib/ScalaModule.scala index c6374729d45..3e90521e80b 100644 --- a/scalalib/src/mill/scalalib/ScalaModule.scala +++ b/scalalib/src/mill/scalalib/ScalaModule.scala @@ -60,29 +60,10 @@ trait ScalaModule extends JavaModule { outer => ) } - override def finalMainClassOpt: T[Either[String, String]] = T{ - mainClass() match{ - case Some(m) => Right(m) - case None => - zincWorker.worker().discoverMainClasses(compile())match { - case Seq() => Left("No main class specified or found") - case Seq(main) => Right(main) - case mains => - Left( - s"Multiple main classes found (${mains.mkString(",")}) " + - "please explicitly specify which one to use by overriding mainClass" - ) - } - } - } - - def scalacPluginIvyDeps = T{ Agg.empty[Dep] } def scalacOptions = T{ Seq.empty[String] } - override def repositories: Seq[Repository] = zincWorker.repositories - private val Milestone213 = raw"""2.13.(\d+)-M(\d+)""".r def scalaCompilerBridgeSources = T { @@ -142,16 +123,16 @@ trait ScalaModule extends JavaModule { outer => } override def compile: T[CompilationResult] = T.persistent{ - zincWorker.worker().compileScala( - scalaVersion(), + zincWorker.worker().compileMixed( + upstreamCompileOutput(), allSourceFiles().map(_.path), - scalaCompilerBridgeSources(), compileClasspath().map(_.path), - scalaCompilerClasspath().map(_.path), + javacOptions(), + scalaVersion(), scalacOptions(), + scalaCompilerBridgeSources(), + scalaCompilerClasspath().map(_.path), scalacPluginClasspath().map(_.path), - javacOptions(), - upstreamCompileOutput() ) } diff --git a/scalalib/src/mill/scalalib/ZincWorkerApi.scala b/scalalib/src/mill/scalalib/ZincWorkerApi.scala index f52332309a3..ebe2f0717a0 100644 --- a/scalalib/src/mill/scalalib/ZincWorkerApi.scala +++ b/scalalib/src/mill/scalalib/ZincWorkerApi.scala @@ -58,19 +58,25 @@ trait ZincWorkerModule extends mill.Module{ } trait ZincWorkerApi { + /** Compile a Java-only project */ + def compileJava(upstreamCompileOutput: Seq[CompilationResult], + sources: Agg[Path], + compileClasspath: Agg[Path], + javacOptions: Seq[String]) + (implicit ctx: mill.util.Ctx): mill.eval.Result[CompilationResult] - def compileScala(scalaVersion: String, + /** Compile a mixed Scala/Java or Scala-only project */ + def compileMixed(upstreamCompileOutput: Seq[CompilationResult], sources: Agg[Path], - compilerBridgeSources: Path, compileClasspath: Agg[Path], - compilerClasspath: Agg[Path], - scalacOptions: Seq[String], - scalacPluginClasspath: Agg[Path], javacOptions: Seq[String], - upstreamCompileOutput: Seq[CompilationResult]) + scalaVersion: String, + scalacOptions: Seq[String], + compilerBridgeSources: Path, + compilerClasspath: Agg[Path], + scalacPluginClasspath: Agg[Path]) (implicit ctx: mill.util.Ctx): mill.eval.Result[CompilationResult] - def discoverMainClasses(compilationResult: CompilationResult) (implicit ctx: mill.util.Ctx): Seq[String] } diff --git a/scalalib/worker/src/mill/scalalib/worker/ZincWorkerImpl.scala b/scalalib/worker/src/mill/scalalib/worker/ZincWorkerImpl.scala index 5fce933cd85..6b21de843be 100644 --- a/scalalib/worker/src/mill/scalalib/worker/ZincWorkerImpl.scala +++ b/scalalib/worker/src/mill/scalalib/worker/ZincWorkerImpl.scala @@ -26,7 +26,29 @@ case class MockedLookup(am: File => Optional[CompileAnalysis]) extends PerClassp class ZincWorkerImpl(ctx0: mill.util.Ctx, compilerBridgeClasspath: Array[String]) extends mill.scalalib.ZincWorkerApi{ - @volatile var compilersCache = Option.empty[(Long, Compilers)] + private val ic = new sbt.internal.inc.IncrementalCompilerImpl() + val javaOnlyCompilers = { + // Keep the classpath as written by the user + val classpathOptions = ClasspathOptions.of(false, false, false, false, false) + + val dummyFile = new java.io.File("") + // Zinc does not have an entry point for Java-only compilation, so we need + // to make up a dummy ScalaCompiler instance. + val scalac = ZincUtil.scalaCompiler( + new ScalaInstance("", null, null, dummyFile, dummyFile, new Array(0), Some("")), null, + classpathOptions // this is used for javac too + ) + + ic.compilers( + instance = null, + classpathOptions, + None, + scalac + ) + } + + @volatile var mixedCompilersCache = Option.empty[(Long, Compilers)] + /** Compile the bridge if it doesn't exist yet and return the output directory. * TODO: Proper invalidation, see #389 @@ -78,27 +100,38 @@ class ZincWorkerImpl(ctx0: mill.util.Ctx, .getOrElse(Seq.empty[String]) } + def compileJava(upstreamCompileOutput: Seq[CompilationResult], + sources: Agg[Path], + compileClasspath: Agg[Path], + javacOptions: Seq[String]) + (implicit ctx: mill.util.Ctx): mill.eval.Result[CompilationResult] = { + compileInternal( + upstreamCompileOutput, + sources, + compileClasspath, + javacOptions, + scalacOptions = Nil, + javaOnlyCompilers + ) + } - def compileScala(scalaVersion: String, + def compileMixed(upstreamCompileOutput: Seq[CompilationResult], sources: Agg[Path], - compilerBridgeSources: Path, compileClasspath: Agg[Path], - compilerClasspath: Agg[Path], - scalacOptions: Seq[String], - scalacPluginClasspath: Agg[Path], javacOptions: Seq[String], - upstreamCompileOutput: Seq[CompilationResult]) + scalaVersion: String, + scalacOptions: Seq[String], + compilerBridgeSources: Path, + compilerClasspath: Agg[Path], + scalacPluginClasspath: Agg[Path]) (implicit ctx: mill.util.Ctx): mill.eval.Result[CompilationResult] = { - val compileClasspathFiles = compileClasspath.map(_.toIO).toArray val compilerJars = compilerClasspath.toArray.map(_.toIO) val compilerBridge = compileZincBridgeIfNeeded(scalaVersion, compilerBridgeSources, compilerJars) - - val ic = new sbt.internal.inc.IncrementalCompilerImpl() - val compilerBridgeSig = compilerBridge.mtime.toMillis + val compilersSig = compilerBridgeSig + compilerClasspath.map(p => p.toString().hashCode + p.mtime.toMillis).sum - val compilers = compilersCache match { + val compilers = mixedCompilersCache match { case Some((k, v)) if k == compilersSig => v case _ => val compilerName = @@ -120,12 +153,28 @@ class ZincWorkerImpl(ctx0: mill.util.Ctx, None, ZincUtil.scalaCompiler(scalaInstance, compilerBridge.toIO) ) - compilersCache = Some((compilersSig, compilers)) + mixedCompilersCache = Some((compilersSig, compilers)) compilers } - mkdir(ctx.dest) + compileInternal( + upstreamCompileOutput, + sources, + compileClasspath, + javacOptions, + scalacOptions = scalacPluginClasspath.map(jar => s"-Xplugin:${jar}").toSeq ++ scalacOptions, + compilers + ) + } + private def compileInternal(upstreamCompileOutput: Seq[CompilationResult], + sources: Agg[Path], + compileClasspath: Agg[Path], + javacOptions: Seq[String], + scalacOptions: Seq[String], + compilers: Compilers) + (implicit ctx: mill.util.Ctx): mill.eval.Result[CompilationResult] = { + mkdir(ctx.dest) val logger = { val consoleAppender = MainAppender.defaultScreen(ConsoleOut.printStreamOut( @@ -158,33 +207,35 @@ class ZincWorkerImpl(ctx0: mill.util.Ctx, val store = FileAnalysisStore.binary(zincIOFile) + val inputs = ic.inputs( + classpath = classesIODir +: compileClasspath.map(_.toIO).toArray, + sources = sources.toArray.map(_.toIO), + classesDirectory = classesIODir, + scalacOptions = scalacOptions.toArray, + javacOptions = javacOptions.toArray, + maxErrors = 10, + sourcePositionMappers = Array(), + order = CompileOrder.Mixed, + compilers = compilers, + setup = ic.setup( + lookup, + skip = false, + zincIOFile, + new FreshCompilerCache, + IncOptions.of(), + new ManagedLoggedReporter(10, logger), + None, + Array() + ), + pr = { + val prev = store.get() + PreviousResult.of(prev.map(_.getAnalysis), prev.map(_.getMiniSetup)) + } + ) + try { val newResult = ic.compile( - ic.inputs( - classpath = classesIODir +: compileClasspathFiles, - sources = sources.toArray.map(_.toIO), - classesDirectory = classesIODir, - scalacOptions = (scalacPluginClasspath.map(jar => s"-Xplugin:${jar}") ++ scalacOptions).toArray, - javacOptions = javacOptions.toArray, - maxErrors = 10, - sourcePositionMappers = Array(), - order = CompileOrder.Mixed, - compilers = compilers, - setup = ic.setup( - lookup, - skip = false, - zincIOFile, - new FreshCompilerCache, - IncOptions.of(), - new ManagedLoggedReporter(10, logger), - None, - Array() - ), - pr = { - val prev = store.get() - PreviousResult.of(prev.map(_.getAnalysis), prev.map(_.getMiniSetup)) - } - ), + in = inputs, logger = logger )