Allow Mill CLI to select the meta-build frame it operates on (#2719)

Add a new CLI option `--meta-level` accepting an `Int`. Default is `0`
and means the root project, `1` is the parent meta-build, if defined, or
the built-in bootstrap module, and so on.

**Example: Find version updates in the meta build**

Here is some example output (applied to the mill repo):

``` 
$ mill --meta-level 1 mill.scalalib.Dependency/showUpdates
[1657/1657] dev.run 
Found 3 dependency update for 
  net.sourceforge.htmlcleaner:htmlcleaner : 2.25 -> 2.26 -> 2.27 -> 2.28 -> 2.29
  com.lihaoyi:mill-contrib-buildinfo_2.13 : 0.11.2-6-261437 -> 0.11.2
  com.github.lolgab:mill-mima_mill0.11_2.13 : 0.0.23 -> 0.0.24
```

**Meta information about the build**

I also added a new external module `mill.runner.MillBuild` to get some
meta-information about the project, for now, the meta-module count or
frame count.

Here on a project with one meta-build:

```
$ mill show mill.runner.MillBuild/levelCount
3
```

Pull request: #2719

build.sc

@@ -4,9 +4,10 @@ import $file.ci.shared
 import $file.ci.upload
 import $ivy.`org.scalaj::scalaj-http:2.4.2`
 import $ivy.`de.tototec::de.tobiasroeser.mill.vcs.version::0.4.0`
-
 import $ivy.`com.github.lolgab::mill-mima::0.0.23`
 import $ivy.`net.sourceforge.htmlcleaner:htmlcleaner:2.25`
+import mill.define.NamedTask
+import mill.main.Tasks
 
 // imports
 import com.github.lolgab.mill.mima.{CheckDirection, ProblemFilter, Mima}
@@ -369,7 +370,7 @@ trait MillStableScalaModule extends MillPublishScalaModule with Mima {
     ProblemFilter.exclude[Problem]("mill.eval.ProfileLogger*"),
     ProblemFilter.exclude[Problem]("mill.eval.GroupEvaluator*"),
     ProblemFilter.exclude[Problem]("mill.eval.Tarjans*"),
-    ProblemFilter.exclude[Problem]("mill.define.Ctx#Impl*"),
+    ProblemFilter.exclude[Problem]("mill.define.Ctx#Impl*")
   )
   def mimaPreviousVersions: T[Seq[String]] = Settings.mimaBaseVersions
 
@@ -486,33 +487,34 @@ object main extends MillStableScalaModule with BuildInfo {
   }
 
   object codesig extends MillPublishScalaModule {
-    override def ivyDeps = Agg(ivy"org.ow2.asm:asm-tree:9.5", Deps.osLib, ivy"com.lihaoyi::pprint:0.8.1")
+    override def ivyDeps =
+      Agg(ivy"org.ow2.asm:asm-tree:9.5", Deps.osLib, ivy"com.lihaoyi::pprint:0.8.1")
     def moduleDeps = Seq(util)
 
-    override lazy val test: CodeSigTests = new CodeSigTests{}
+    override lazy val test: CodeSigTests = new CodeSigTests {}
     trait CodeSigTests extends MillScalaTests {
       val caseKeys = interp.watchValue(
         os.walk(millSourcePath / "cases", maxDepth = 3)
           .map(_.subRelativeTo(millSourcePath / "cases").segments)
-          .collect{case Seq(a, b, c) => s"$a-$b-$c"}
+          .collect { case Seq(a, b, c) => s"$a-$b-$c" }
       )
 
-      def testLogFolder = T{ T.dest }
+      def testLogFolder = T { T.dest }
 
       def caseEnvs[V](f1: CaseModule => Task[V])(s: String, f2: V => String) = {
         T.traverse(caseKeys) { i => f1(cases(i)).map(v => s"MILL_TEST_${s}_$i" -> f2(v)) }
       }
-      def forkEnv = T{
+      def forkEnv = T {
         Map("MILL_TEST_LOGS" -> testLogFolder().toString) ++
-        caseEnvs(_.compile)("CLASSES", _.classes.path.toString)() ++
-        caseEnvs(_.compileClasspath)("CLASSPATH", _.map(_.path).mkString(","))() ++
-        caseEnvs(_.sources)("SOURCES", _.head.path.toString)()
+          caseEnvs(_.compile)("CLASSES", _.classes.path.toString)() ++
+          caseEnvs(_.compileClasspath)("CLASSPATH", _.map(_.path).mkString(","))() ++
+          caseEnvs(_.sources)("SOURCES", _.head.path.toString)()
       }
 
       object cases extends Cross[CaseModule](caseKeys)
-      trait CaseModule extends ScalaModule with Cross.Module[String]{
+      trait CaseModule extends ScalaModule with Cross.Module[String] {
         def caseName = crossValue
-        object external extends ScalaModule{
+        object external extends ScalaModule {
           def scalaVersion = "2.13.10"
         }
 
@@ -521,7 +523,7 @@ object main extends MillStableScalaModule with BuildInfo {
         val Array(prefix, suffix, rest) = caseName.split("-", 3)
         def millSourcePath = super.millSourcePath / prefix / suffix / rest
         def scalaVersion = "2.13.10"
-        def ivyDeps = T{
+        def ivyDeps = T {
           if (!caseName.contains("realistic") && !caseName.contains("sourcecode")) super.ivyDeps()
           else Agg(
             Deps.fastparse,
@@ -531,7 +533,7 @@ object main extends MillStableScalaModule with BuildInfo {
             Deps.mainargs,
             Deps.requests,
             Deps.osLib,
-            Deps.upickle,
+            Deps.upickle
           )
         }
       }
@@ -1454,7 +1456,7 @@ object docs extends Module {
     val pagesWd = T.dest / "modules" / "ROOT" / "pages"
     val partialsWd = T.dest / "modules" / "ROOT" / "partials"
 
-    os.copy(projectReadme().path, partialsWd  / "project-readme.adoc", createFolders = true)
+    os.copy(projectReadme().path, partialsWd / "project-readme.adoc", createFolders = true)
 
     val renderedExamples: Seq[(os.SubPath, PathRef)] =
       T.traverse(example.exampleModules)(m =>
@@ -1763,23 +1765,24 @@ def uploadToGithub(authKey: String) = T.command {
   }
 }
 
-def validate(ev: Evaluator): Command[Unit] = T.command {
-  mill.main.RunScript.evaluateTasksNamed(
-    ev.withFailFast(false),
-    Seq(
-      "__.compile",
-      "+",
-      "__.mimaReportBinaryIssues",
-      "+",
-      "mill.scalalib.scalafmt.ScalafmtModule/checkFormatAll",
-      "__.sources",
-      "+",
-      "docs.localPages"
-    ),
-    selectMode = SelectMode.Separated
-  )
+private def resolveTasks[T](taskNames: String*): Seq[NamedTask[T]] = {
+  mill.resolve.Resolve.Tasks.resolve(
+    build,
+    taskNames,
+    SelectMode.Separated
+  ).map(x => x.asInstanceOf[Seq[mill.define.NamedTask[T]]]).getOrElse(???)
+}
+
+def validate(): Command[Unit] = {
+  val tasks = resolveTasks("__.compile", "__.minaReportBinaryIssues")
+  val sources = resolveTasks("__.sources")
 
-  ()
+  T.command {
+    T.sequence(tasks)()
+    mill.scalalib.scalafmt.ScalafmtModule.checkFormatAll(Tasks(sources))()
+    docs.localPages()
+    ()
+  }
 }
 
 /** Dummy module to let Scala-Steward find and bump dependency versions we use at runtime */

docs/modules/ROOT/pages/Extending_Mill.adoc

@@ -11,6 +11,46 @@ include::example/misc/3-import-file-ivy.adoc[]
 
 == The Mill Meta-Build
 
+The meta-build manages the compilation of the `build.sc`.
+If you don't configure it explicitly, a built-in synthetic meta-build is used.
+
+To customize it, you need to explicitly enable it with `import $meta._`.
+Once enabled, the meta-build lives in the `mill-build/` directory.
+It needs to contain a top-level module of type `MillBuildRootModule`.
+
+Meta-builds are recursive, which means, it can itself have a nested meta-builds, and so on.
+
+To run a task on a meta-build, you specifying the `--meta-level` option to select the meta-build level.
+
+=== Example: Format the `build.sc`
+
+As an example of running a task on the meta-build, you can format the `build.sc` with Scalafmt.
+Everything is already provided by Mill.
+You only need a `.scalafmt.conf` config file which at least needs configure the Scalafmt version.
+
+.Run Scalafmt on the `build.sc` (and potentially included files)
+----
+$ mill --meta-level 1 mill.scalalib.scalafmt.ScalafmtModule/reformatAll sources
+----
+
+* `--meta-level 1` selects the first meta-build. Without any customization, this is the only built-in meta-build.
+* `mill.scalalib.scalafmt.ScalafmtModule/reformatAll` is a generic task to format scala source files with Scalafmt. It requires the targets that refer to the source files as argument
+* `sources` this selects the `sources` targets of the meta-build, which at least contains the `build.sc`.
+
+=== Example: Find plugin updates
+
+Mill plugins are defined as `ivyDeps` in the meta-build.
+Hence, you can easily search for updates with the external `mill.scalalib.Dependency` module.
+
+.Check for Mill Plugin updates
+----
+$ mill --meta-level 1 mill.scalalib.Dependency/showUpdates
+Found 1 dependency update for
+  de.tototec:de.tobiasroeser.mill.vcs.version_mill0.11_2.13 : 0.3.1-> 0.4.0
+----
+
+=== Example: Customizing the Meta-Build
+
 include::example/misc/4-mill-build-folder.adoc[]
 
 == Using ScalaModule.run as a task

docs/modules/ROOT/pages/Intro_to_Mill.adoc

@@ -124,35 +124,48 @@ Run `mill --help` for a complete list of options
 ----
 Mill Build Tool, version {mill-version}
 usage: mill [options] [[target [target-options]] [+ [target ...]]]
-  -D --define <k=v>       Define (or overwrite) a system property.
-  -b --bell               Ring the bell once if the run completes successfully, twice if it fails.
-  --bsp                   Enable BSP server mode.
-  --color <bool>          Enable or disable colored output; by default colors are enabled in both
-                          REPL and scripts mode if the console is interactive, and disabled
-                          otherwise.
-  -d --debug              Show debug output on STDOUT
-  --disable-ticker        Disable ticker log (e.g. short-lived prints of stages and progress bars).
-  --enable-ticker <bool>  Enable ticker log (e.g. short-lived prints of stages and progress bars).
-  -h --home <path>        (internal) The home directory of internally used Ammonite script engine;
-                          where it looks for config and caches.
-  --help                  Print this help message and exit.
-  -i --interactive        Run Mill in interactive mode, suitable for opening REPLs and taking user
-                          input. This implies --no-server and no mill server will be used. Must be
-                          the first argument.
-  --import <str>          Additional ivy dependencies to load into mill, e.g. plugins.
-  -j --jobs <int>         Allow processing N targets in parallel. Use 1 to disable parallel and 0 to
-                          use as much threads as available processors.
-  -k --keep-going         Continue build, even after build failures.
-  --no-server             Run Mill in single-process mode. In this mode, no Mill server will be
-                          started or used. Must be the first argument.
-  --repl                  This flag is no longer supported.
-  -s --silent             Make ivy logs during script import resolution go silent instead of
-                          printing; though failures will still throw exception.
-  -v --version            Show mill version information and exit.
-  -w --watch              Watch and re-run your scripts when they change.
-  target <str>...         The name or a pattern of the target(s) you want to build, followed by any
-                          parameters you wish to pass to those targets. To specify multiple target
-                          names or patterns, use the `+` separator.
+  -D --define <k=v>                 Define (or overwrite) a system property.
+  -b --bell                         Ring the bell once if the run completes successfully, twice if
+                                    it fails.
+  --bsp                             Enable BSP server mode.
+  --color <bool>                    Enable or disable colored output; by default colors are enabled
+                                    in both REPL and scripts mode if the console is interactive, and
+                                    disabled otherwise.
+  -d --debug                        Show debug output on STDOUT
+  --disable-callgraph-invalidation  Disable the fine-grained callgraph-based target invalidation in
+                                    response to code changes, and instead fall back to the previous
+                                    coarse-grained implementation relying on the script `import
+                                    $file` graph
+  --disable-ticker                  Disable ticker log (e.g. short-lived prints of stages and
+                                    progress bars).
+  --enable-ticker <bool>            Enable ticker log (e.g. short-lived prints of stages and
+                                    progress bars).
+  -h --home <path>                  (internal) The home directory of internally used Ammonite script
+                                    engine; where it looks for config and caches.
+  --help                            Print this help message and exit.
+  -i --interactive                  Run Mill in interactive mode, suitable for opening REPLs and
+                                    taking user input. This implies --no-server and no mill server
+                                    will be used. Must be the first argument.
+  --import <str>                    Additional ivy dependencies to load into mill, e.g. plugins.
+  -j --jobs <int>                   Allow processing N targets in parallel. Use 1 to disable
+                                    parallel and 0 to use as much threads as available processors.
+  -k --keep-going                   Continue build, even after build failures.
+  --meta-level <int>                Experimental: Select a meta-build level to run the given
+                                    targets. Level 0 is the normal project, level 1 the first
+                                    meta-build, and so on. The last level is the built-in synthetic
+                                    meta-build which Mill uses to bootstrap the project.
+  --no-server                       Run Mill in single-process mode. In this mode, no Mill server
+                                    will be started or used. Must be the first argument.
+  --repl                            This flag is no longer supported.
+  -s --silent                       Make ivy logs during script import resolution go silent instead
+                                    of printing; though failures will still throw exception.
+  -v --version                      Show mill version information and exit.
+  -w --watch                        Watch and re-run your scripts when they change.
+  target <str>...                   The name or a pattern of the target(s) you want to build,
+                                    followed by any parameters you wish to pass to those targets. To
+                                    specify multiple target names or patterns, use the `+`
+                                    separator.
+
 ----
 
 All _options_ must be given before the first target.

example/misc/4-mill-build-folder/build.sc

@@ -63,3 +63,19 @@ scalatagsVersion: 0.8.2
 
 */
 
+// You can also run tasks on the meta-build by using the `--meta-level`
+// cli option.
+
+/** Usage
+
+> ./mill --meta-level 1 show sources
+[
+.../build.sc",
+.../mill-build/src"
+]
+
+> ./mill --meta-level 2 show sources
+.../mill-build/build.sc"
+
+
+*/

example/misc/5-module-run-task/build.sc

@@ -17,7 +17,7 @@ object bar extends ScalaModule{
   def ivyDeps = Agg(ivy"com.lihaoyi::os-lib:0.9.1")
 }
 
-// This example demonstrates using Mill `ScalaModule`s as build tasks: rather
+// This example demonstrates using Mill ``ScalaModule``s as build tasks: rather
 // than defining the task logic in the `build.sc`, we instead put the build
 // logic within the `bar` module as `bar/src/Bar.scala`. In this example, we use
 // `Bar.scala` as a source-code pre-processor on the `foo` module source code:
@@ -34,7 +34,7 @@ Foo.value: HELLO
 */
 
 // This example does a trivial string-replace of "hello" with "HELLO", but is
-// enough to demonstrate how you can use Mill `ScalaModule`s to implement your
+// enough to demonstrate how you can use Mill ``ScalaModule``s to implement your
 // own arbitrarily complex transformations. This is useful for build logic that
 // may not fit nicely inside a `build.sc` file, whether due to the sheer lines
 // of code or due to dependencies that may conflict with the Mill classpath

runner/src/mill/runner/MillBuild.scala

@@ -0,0 +1,21 @@
+package mill.runner
+
+import mill.T
+import mill.define.{Command, Discover, ExternalModule, Module}
+import mill.eval.Evaluator.AllBootstrapEvaluators
+
+trait MillBuild extends Module {
+
+  /**
+   * Count of the nested build-levels, the main project and all its nested meta-builds.
+   * If you run this on a meta-build, the non-meta-builds are not included.
+   */
+  def levelCount(evaluators: AllBootstrapEvaluators): Command[Int] = T.command {
+    evaluators.value.size
+  }
+
+}
+
+object MillBuild extends ExternalModule with MillBuild {
+  override lazy val millDiscover = Discover[this.type]
+}