Compile compiler bridge from sources, close #363

Motivation:

Scala compiler doesn't provide binary compatibility.
Binary compatibility was broken in 2.13.1 so incremental compiler crashes.

Modifications:

Recompile the compiler-bridge from sources instead of using the binary on maven central.
Cache it in `~/.sbt/1.0/zinc/org.scala-sbt` by default (configurable).

Result:

scala-maven-plugin support compiling in Scala 2.13.1.

src/main/java/sbt_inc/SbtIncrementalCompiler.java

@@ -1,5 +1,11 @@
 package sbt_inc;
 
+import org.apache.maven.artifact.Artifact;
+import sbt.io.AllPassFilter$;
+import sbt.io.IO;
+import sbt.util.Logger;
+import scala.Tuple2;
+import scala.collection.JavaConverters;
 import scala.compat.java8.functionConverterImpls.*;
 
 import org.apache.maven.plugin.logging.Log;
@@ -8,35 +14,51 @@
 import sbt.internal.inc.ScalaInstance;
 import sbt.internal.inc.classpath.ClasspathUtilities;
 import scala.Option;
+import scala_maven.MavenArtifactResolver;
 import scala_maven.VersionNumber;
-import xsbti.Logger;
+import util.FileUtils;
 import xsbti.T2;
 import xsbti.compile.*;
 import xsbti.compile.AnalysisStore;
 import xsbti.compile.CompilerCache;
 
 import java.io.File;
+import java.io.InputStream;
 import java.net.URL;
 import java.net.URLClassLoader;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
+import java.nio.file.Files;
+import java.util.*;
+import java.util.jar.Attributes;
+import java.util.jar.Manifest;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 public class SbtIncrementalCompiler {
 
+    private static final String SBT_GROUP_ID = "org.scala-sbt";
+    private static final String USER_HOME = System.getProperty("user.home");
+    private static final String JAVA_CLASS_VERSION = System.getProperty("java.class.version");
+    private static final File DEFAULT_SECONDARY_CACHE_DIR = new File(
+        USER_HOME + "/.sbt/1.0/zinc/org.scala-sbt".replaceAll("/", File.separator));
+
     private final IncrementalCompiler compiler = ZincUtil.defaultIncrementalCompiler();
     private final CompileOrder compileOrder;
     private final Logger logger;
     private final Compilers compilers;
     private final Setup setup;
     private final AnalysisStore analysisStore;
+    private final MavenArtifactResolver resolver;
+    private final File secondaryCacheDir;
 
     public SbtIncrementalCompiler(File libraryJar, File reflectJar, File compilerJar, VersionNumber scalaVersion,
-        List<File> extraJars, File compilerBridgeJar, Log l, File cacheFile, CompileOrder compileOrder)
-        throws Exception {
+        List<File> extraJars, MavenArtifactResolver resolver, File secondaryCacheDir, Log mavenLogger, File cacheFile,
+        CompileOrder compileOrder) throws Exception {
         this.compileOrder = compileOrder;
-        this.logger = new SbtLogger(l);
-        l.info("Using incremental compilation using " + compileOrder + " compile order");
+        this.logger = new SbtLogger(mavenLogger);
+        mavenLogger.info("Using incremental compilation using " + compileOrder + " compile order");
+        this.resolver = resolver;
+        this.secondaryCacheDir = secondaryCacheDir != null ? secondaryCacheDir : DEFAULT_SECONDARY_CACHE_DIR;
+        this.secondaryCacheDir.mkdirs();
 
         List<File> allJars = new ArrayList<>(extraJars);
         allJars.add(libraryJar);
@@ -54,6 +76,8 @@ public SbtIncrementalCompiler(File libraryJar, File reflectJar, File compilerJar
             Option.apply(scalaVersion.toString()) // explicitActual
         );
 
+        File compilerBridgeJar = getCompiledBridgeJar(scalaInstance, mavenLogger);
+
         ScalaCompiler scalaCompiler = new AnalyzingCompiler( //
             scalaInstance, // scalaInstance
             ZincCompilerUtil.constantBridgeProvider(scalaInstance, compilerBridgeJar), // provider
@@ -117,7 +141,8 @@ public void compile(List<String> classpathElements, List<File> sources, File cla
         CompileOptions options = CompileOptions.of( //
             fullClasspath.toArray(new File[] {}), // classpath
             sources.toArray(new File[] {}), // sources
-            classesDirectory, scalacOptions.toArray(new String[] {}), // scalacOptions
+            classesDirectory, //
+            scalacOptions.toArray(new String[] {}), // scalacOptions
             javacOptions.toArray(new String[] {}), // javacOptions
             100, // maxErrors
             pos -> pos, // sourcePositionMappers
@@ -130,4 +155,105 @@ public void compile(List<String> classpathElements, List<File> sources, File cla
         CompileResult newResult = compiler.compile(inputs, logger);
         analysisStore.set(AnalysisContents.create(newResult.analysis(), newResult.setup()));
     }
+
+    private String compilerBridgeArtifactId(String scalaVersion) {
+        if (scalaVersion.startsWith("2.10.")) {
+            return "compiler-bridge_2.10";
+        } else if (scalaVersion.startsWith("2.11.")) {
+            return "compiler-bridge_2.11";
+        } else if (scalaVersion.startsWith("2.12.") || scalaVersion.equals("2.13.0-M1")) {
+            return "compiler-bridge_2.12";
+        } else {
+            return "compiler-bridge_2.13";
+        }
+    }
+
+    private File getCompiledBridgeJar(ScalaInstance scalaInstance, Log mavenLogger) throws Exception {
+
+        // eg
+        // org.scala-sbt-compiler-bridge_2.12-1.2.4-bin_2.12.10__52.0-1.2.4_20181015T090407.jar
+        String bridgeArtifactId = compilerBridgeArtifactId(scalaInstance.actualVersion());
+
+        // this file is localed in compiler-interface
+        Properties properties = new Properties();
+        try (InputStream is = getClass().getClassLoader()
+            .getResourceAsStream("incrementalcompiler.version.properties")) {
+            properties.load(is);
+        }
+
+        String zincVersion = properties.getProperty("version");
+        String timestamp = properties.getProperty("timestamp");
+
+        String cacheFileName = SBT_GROUP_ID + '-' + bridgeArtifactId + '-' + zincVersion + "-bin_"
+            + scalaInstance.actualVersion() + "__" + JAVA_CLASS_VERSION + '-' + zincVersion + '_' + timestamp + ".jar";
+
+        File cachedCompiledBridgeJar = new File(secondaryCacheDir, cacheFileName);
+
+        if (mavenLogger.isInfoEnabled()) {
+            mavenLogger.info("Compiler bridge file: " + cachedCompiledBridgeJar);
+        }
+
+        if (!cachedCompiledBridgeJar.exists()) {
+            mavenLogger.info("Compiler bridge file is not installed yet");
+            // compile and install
+            RawCompiler rawCompiler = new RawCompiler(scalaInstance, ClasspathOptionsUtil.auto(), logger);
+
+            File bridgeSources = resolver.getJar(SBT_GROUP_ID, bridgeArtifactId, zincVersion, "sources").getFile();
+
+            Set<File> bridgeSourcesDependencies = resolver
+                .getJarAndDependencies(SBT_GROUP_ID, bridgeArtifactId, zincVersion, "sources") //
+                .stream() //
+                .filter(artifact -> artifact.getScope() != null && !artifact.getScope().equals("provided")) //
+                .map(Artifact::getFile) //
+                .collect(Collectors.toSet());
+
+            for (File scalaJars : scalaInstance.allJars()) {
+                bridgeSourcesDependencies.add(scalaJars);
+            }
+
+            File sourcesDir = Files.createTempDirectory("scala-maven-plugin-compiler-bridge-sources").toFile();
+            File classesDir = Files.createTempDirectory("scala-maven-plugin-compiler-bridge-classes").toFile();
+
+            IO.unzip(bridgeSources, sourcesDir, AllPassFilter$.MODULE$, true);
+
+            try {
+                rawCompiler.apply(
+                    JavaConverters.iterableAsScalaIterable(FileUtils.listDirectoryContent(sourcesDir.toPath(),
+                        file -> file.isFile() && file.getName().endsWith(".scala"))).seq().toSeq(), // sources:Seq[File]
+                    JavaConverters.iterableAsScalaIterable(bridgeSourcesDependencies).seq().toSeq(), // classpath:Seq[File],
+                    classesDir, // outputDirectory:File,
+                    JavaConverters.collectionAsScalaIterable(Collections.<String>emptyList()).seq().toSeq() // options:Seq[String]
+                );
+
+                Manifest manifest = new Manifest();
+                Attributes mainAttributes = manifest.getMainAttributes();
+                mainAttributes.putValue(Attributes.Name.MANIFEST_VERSION.toString(), "1.0");
+                mainAttributes.putValue(Attributes.Name.SPECIFICATION_VENDOR.toString(), SBT_GROUP_ID);
+                mainAttributes.putValue(Attributes.Name.SPECIFICATION_TITLE.toString(), "Compiler Bridge");
+                mainAttributes.putValue(Attributes.Name.SPECIFICATION_VERSION.toString(), zincVersion);
+
+                int classesDirPathLength = classesDir.toString().length();
+                Stream<Tuple2<File, String>> stream = FileUtils.listDirectoryContent(classesDir.toPath(), file -> true) //
+                    .stream() //
+                    .map(file -> {
+                        String path = file.toString().substring(classesDirPathLength + 1).replace(File.separator, "/");
+                        if (file.isDirectory()) {
+                            path = path + "/";
+                        }
+                        return new Tuple2(file, path);
+                    });
+                List<Tuple2<File, String>> classes = stream.collect(Collectors.toList());
+
+                IO.jar(JavaConverters.collectionAsScalaIterable(classes), cachedCompiledBridgeJar, new Manifest());
+
+                mavenLogger.info("Compiler bridge installed");
+
+            } finally {
+                FileUtils.deleteDirectory(sourcesDir.toPath());
+                FileUtils.deleteDirectory(classesDir.toPath());
+            }
+        }
+
+        return cachedCompiledBridgeJar;
+    }
 }

src/main/java/scala_maven/MavenArtifactResolver.java

@@ -0,0 +1,48 @@
+package scala_maven;
+
+import org.apache.maven.artifact.Artifact;
+import org.apache.maven.artifact.resolver.ArtifactResolutionRequest;
+import org.apache.maven.execution.MavenSession;
+import org.apache.maven.repository.RepositorySystem;
+
+import java.util.Set;
+
+public class MavenArtifactResolver {
+
+    private final RepositorySystem repositorySystem;
+    private final MavenSession session;
+
+    public MavenArtifactResolver(RepositorySystem repositorySystem, MavenSession session) {
+        this.repositorySystem = repositorySystem;
+        this.session = session;
+    }
+
+    public Artifact getJar(String groupId, String artifactId, String version, String classifier) {
+        Artifact artifact = createJarArtifact(groupId, artifactId, version, classifier);
+        return resolve(artifact, false).iterator().next();
+    }
+
+    public Set<Artifact> getJarAndDependencies(String groupId, String artifactId, String version, String classifier) {
+        Artifact artifact = createJarArtifact(groupId, artifactId, version, classifier);
+        return resolve(artifact, true);
+    }
+
+    private Artifact createJarArtifact(String groupId, String artifactId, String version, String classifier) {
+        return classifier == null ? repositorySystem.createArtifact(groupId, artifactId, version, ScalaMojoSupport.JAR)
+            : repositorySystem.createArtifactWithClassifier(groupId, artifactId, version, ScalaMojoSupport.JAR,
+                classifier);
+    }
+
+    private Set<Artifact> resolve(Artifact artifact, boolean transitively) {
+        ArtifactResolutionRequest request = new ArtifactResolutionRequest() //
+            .setArtifact(artifact) //
+            .setResolveRoot(true) //
+            .setResolveTransitively(transitively) //
+            .setServers(session.getRequest().getServers()) //
+            .setMirrors(session.getRequest().getMirrors()) //
+            .setProxies(session.getRequest().getProxies()) //
+            .setLocalRepository(session.getLocalRepository()) //
+            .setRemoteRepositories(session.getCurrentProject().getRemoteArtifactRepositories());
+        return repositorySystem.resolve(request).getArtifacts();
+    }
+}

src/main/java/scala_maven/ScalaCompilerSupport.java

@@ -58,6 +58,14 @@ public enum RecompileMode {
     @Parameter(property = "compileOrder", defaultValue = "Mixed")
     private CompileOrder compileOrder;
 
+    /**
+     * Location of the incremental compile will install compiled compiler bridge
+     * jars. Default is sbt's "~/.sbt/1.0/zinc/org.scala-sbt".
+     *
+     */
+    @Parameter(property = "secondaryCacheDir")
+    private File secondaryCacheDir;
+
     abstract protected File getOutputDir() throws Exception;
 
     abstract protected List<String> getClasspathElements() throws Exception;
@@ -252,17 +260,6 @@ void setLastSuccessfullTS(long v) throws Exception {
     //
     // Incremental compilation
     //
-    private static final String SBT_GROUP_ID = "org.scala-sbt";
-    private static final String ZINC_ARTIFACT_ID = "zinc_2.12";
-    private static final String COMPILER_BRIDGE_ARTIFACT_ID = "compiler-bridge";
-
-    protected File getCompilerBridgeJar() throws Exception {
-        VersionNumber scalaVersion = findScalaVersion();
-        String zincVersion = findVersionFromPluginArtifacts(SBT_GROUP_ID, ZINC_ARTIFACT_ID);
-        return getArtifactJar(SBT_GROUP_ID,
-            scalaVersion.applyScalaArtifactVersioningScheme(COMPILER_BRIDGE_ARTIFACT_ID), zincVersion);
-    }
-
     private int incrementalCompile(List<String> classpathElements, List<File> sourceRootDirs, File outputDir,
         File cacheFile, boolean compileInLoop) throws Exception {
         List<File> sources = findSourceWithFilters(sourceRootDirs);
@@ -285,7 +282,8 @@ private int incrementalCompile(List<String> classpathElements, List<File> source
                 getCompilerJar(), //
                 findScalaVersion(), //
                 extraJars, //
-                getCompilerBridgeJar(), //
+                new MavenArtifactResolver(factory, session), //
+                secondaryCacheDir, //
                 getLog(), //
                 cacheFile, //
                 compileOrder);

src/main/java/scala_maven/ScalaMojoSupport.java

@@ -41,7 +41,6 @@
 import org.apache.maven.toolchain.ToolchainManager;
 import org.codehaus.plexus.util.StringUtils;
 
-import sbt_inc.SbtIncrementalCompiler;
 import scala_maven_dependency.CheckScalaVersionVisitor;
 import scala_maven_dependency.ScalaDistroArtifactFilter;
 import scala_maven_executions.JavaMainCaller;
@@ -375,9 +374,9 @@ public String getScalaOrganization() {
      *
      * @return a {@link Artifact} for the Scala Compiler.
      */
-    final Artifact scalaCompilerArtifact(final String scalaVersion) {
-        return this.factory.createArtifact(this.getScalaOrganization(), ScalaMojoSupport.SCALA_COMPILER_ARTIFACTID,
-            scalaVersion, "", ScalaMojoSupport.POM);
+    final Artifact scalaCompilerArtifact(String scalaVersion) {
+        return factory.createArtifact(getScalaOrganization(), ScalaMojoSupport.SCALA_COMPILER_ARTIFACTID, scalaVersion,
+            "", ScalaMojoSupport.POM);
     }
 
     /**
@@ -444,7 +443,7 @@ private Set<Artifact> resolveDependencyArtifacts(final Artifact artifact, final
             resolutionFilter, remoteRepositories, localRepository);
 
         // TODO follow the dependenciesManagement and override rules
-        return this.resolver.resolve(arr).getArtifacts();
+        return factory.resolve(arr).getArtifacts();
     }
 
     /**
@@ -498,7 +497,7 @@ private void addToClasspath(String groupId, String artifactId, String version, S
 
     /**
      * added for classifier support.
-     * 
+     *
      * @author Christoph Radig
      * @todo might want to merge with existing "addToClasspath" methods.
      */
@@ -650,7 +649,7 @@ void checkScalaVersion() throws Exception {
 
     /**
      * this method checks to see if there are multiple versions of the scala library
-     * 
+     *
      * @throws Exception
      */
     private void checkCorrectVersionsOfScalaLibrary(String scalaDefVersion) throws Exception {
@@ -974,14 +973,4 @@ private Set<String> getCompilerPlugins() throws Exception {
         }
         return plugins;
     }
-
-    protected String findVersionFromPluginArtifacts(String groupId, String artifactId) {
-        for (Artifact art : pluginArtifacts) {
-            if (groupId.equals(art.getGroupId()) && artifactId.equals(art.getArtifactId())) {
-                return art.getVersion();
-            }
-        }
-        throw new IllegalArgumentException(
-            "Could not locate artifact " + groupId + ":" + artifactId + " in plugin's dependencies");
-    }
 }