From 0df06f9d7c8cb1ed5702a23da66230c99a1fa15b Mon Sep 17 00:00:00 2001 From: Ivan Gavrilovic Date: Thu, 21 Mar 2019 16:55:56 +0000 Subject: [PATCH] Incremental KAPT - analyze classpath changes Use artifact transforms to capture structure and dependencies of classpath entries. In the KAPT task this information is used to compare previous classpath structure with the current one. Once changed classes are detected, all classes that transitively depend on those are identified, and that set is passed to KAPT invocation. In order to avoid unrelated classpath changes, we record an ABI snapshot of the classpath entry. This snapshot ignores all private members, and @Metadata annotation. #KT-23880 --- .../KaptIncrementalWithAggregatingApt.kt | 65 ++++ .../gradle/KaptIncrementalWithIsolatingApt.kt | 25 +- .../kotlin-gradle-plugin/build.gradle.kts | 17 +- .../kapt/Kapt3KotlinGradleSubplugin.kt | 32 +- .../kotlin/gradle/internal/kapt/KaptTask.kt | 70 +++- .../internal/kapt/KaptWithKotlincTask.kt | 11 +- .../internal/kapt/KaptWithoutKotlincTask.kt | 13 +- .../kapt/incremental/ClassAbiExtractor.kt | 52 +++ .../incremental/ClassTypeExtractorVisitor.kt | 177 +++++++++ .../kapt/incremental/ClasspathAnalyzer.kt | 210 +++++++++++ .../kapt/incremental/ClasspathSnapshot.kt | 179 +++++++++ .../kapt/incremental/ClassAbiExtractorTest.kt | 346 ++++++++++++++++++ .../ClassTypeExtractorVisitorTest.kt | 138 +++++++ .../kapt/incremental/ClasspathAnalyzerTest.kt | 106 ++++++ .../kapt/incremental/ClasspathSnapshotTest.kt | 120 ++++++ .../kotlin/gradle/util/bytecodeUtils.kt | 13 + .../kotlin/kapt3/base/KaptOptions.kt | 2 +- 17 files changed, 1546 insertions(+), 30 deletions(-) create mode 100644 libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/incremental/ClassAbiExtractor.kt create mode 100644 libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/incremental/ClassTypeExtractorVisitor.kt create mode 100644 libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/incremental/ClasspathAnalyzer.kt create mode 100644 libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/incremental/ClasspathSnapshot.kt create mode 100644 libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/incremental/ClassAbiExtractorTest.kt create mode 100644 libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/incremental/ClassTypeExtractorVisitorTest.kt create mode 100644 libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/incremental/ClasspathAnalyzerTest.kt create mode 100644 libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/incremental/ClasspathSnapshotTest.kt diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KaptIncrementalWithAggregatingApt.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KaptIncrementalWithAggregatingApt.kt index c77a5691b6ec2..4dfa55e18db33 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KaptIncrementalWithAggregatingApt.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KaptIncrementalWithAggregatingApt.kt @@ -71,4 +71,69 @@ class KaptIncrementalWithAggregatingApt : KaptIncrementalIT() { ) } } + + @Test + fun testClasspathChanges() { + val project = Project( + "incrementalMultiproject", + GradleVersionRequired.None + ).apply { + setupWorkingDir() + val processorPath = generateProcessor("AGGREGATING") + + projectDir.resolve("app/build.gradle").appendText( + """ + + apply plugin: "kotlin-kapt" + dependencies { + implementation "org.jetbrains.kotlin:kotlin-stdlib:${'$'}kotlin_version" + kapt files("${processorPath.invariantSeparatorsPath}") + } + """.trimIndent() + ) + + projectDir.resolve("lib/build.gradle").appendText( + """ + + dependencies { + implementation "org.jetbrains.kotlin:kotlin-stdlib:${'$'}kotlin_version" + } + """.trimIndent() + ) + } + + project.build("clean", ":app:build") { + assertSuccessful() + } + + project.projectFile("A.kt").modify { current -> + val lastBrace = current.lastIndexOf("}") + current.substring(0, lastBrace) + "fun anotherFun() {}\n }" + } + project.build("build") { + assertSuccessful() + + assertEquals( + setOf( + fileInWorkingDir("app/build/tmp/kapt3/stubs/main/foo/AA.java").absolutePath, + fileInWorkingDir("app/build/tmp/kapt3/stubs/main/foo/AAA.java").absolutePath, + fileInWorkingDir("app/build/tmp/kapt3/stubs/main/foo/BB.java").absolutePath, + fileInWorkingDir("app/build/tmp/kapt3/stubs/main/foo/FooUseAKt.java").absolutePath, + fileInWorkingDir("app/build/tmp/kapt3/stubs/main/foo/FooUseBKt.java").absolutePath, + fileInWorkingDir("app/build/tmp/kapt3/stubs/main/foo/FooUseAAKt.java").absolutePath, + fileInWorkingDir("app/build/tmp/kapt3/stubs/main/foo/FooUseBBKt.java").absolutePath + + ), getProcessedSources(output) + ) + } + + project.projectFile("A.kt").modify { current -> + val lastBrace = current.lastIndexOf("}") + current.substring(0, lastBrace) + "private fun privateFunction() {}\n }" + } + project.build("build") { + assertSuccessful() + assertTrue(getProcessedSources(output).isEmpty()) + } + } } \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KaptIncrementalWithIsolatingApt.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KaptIncrementalWithIsolatingApt.kt index 4de5eb4d55b0e..342b0c1cb0471 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KaptIncrementalWithIsolatingApt.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KaptIncrementalWithIsolatingApt.kt @@ -10,6 +10,7 @@ import org.jetbrains.kotlin.gradle.util.modify import org.junit.Assert.assertEquals import org.junit.Assert.assertTrue import org.junit.Test +import java.io.File import java.util.zip.ZipEntry import java.util.zip.ZipOutputStream @@ -70,13 +71,22 @@ private const val patternApt = "Processing java sources with annotation processo fun getProcessedSources(output: String): Set { val logging = output.lines().single { it.contains(patternApt) } val indexOf = logging.indexOf(patternApt) + patternApt.length - return logging.drop(indexOf).split(",").map { it.trim() }.toSet() + return logging.drop(indexOf).split(",").map { it.trim() }.filter { !it.isEmpty() }.toSet() } -fun BaseGradleIT.Project.setupIncrementalAptProject(procType: String) { +fun BaseGradleIT.Project.setupIncrementalAptProject(procType: String, buildFile: File = projectDir.resolve("build.gradle")) { setupWorkingDir() - val buildFile = projectDir.resolve("build.gradle") val content = buildFile.readText() + val processorPath = generateProcessor(procType) + + val updatedContent = content.replace( + Regex("^\\s*kapt\\s\"org\\.jetbrain.*$", RegexOption.MULTILINE), + " kapt files(\"${processorPath.invariantSeparatorsPath}\")" + ) + buildFile.writeText(updatedContent) +} + +fun BaseGradleIT.Project.generateProcessor(procType: String): File { val processorPath = projectDir.resolve("incrementalProcessor.jar") ZipOutputStream(processorPath.outputStream()).use { @@ -92,10 +102,5 @@ fun BaseGradleIT.Project.setupIncrementalAptProject(procType: String) { it.write(IncrementalProcessor::class.java.name.toByteArray()) it.closeEntry() } - - val updatedContent = content.replace( - Regex("^\\s*kapt\\s\"org\\.jetbrain.*$", RegexOption.MULTILINE), - " kapt files(\"${processorPath.invariantSeparatorsPath}\")" - ) - buildFile.writeText(updatedContent) -} + return processorPath +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/build.gradle.kts b/libraries/tools/kotlin-gradle-plugin/build.gradle.kts index 5fe2609571ffc..b9a675d267fee 100644 --- a/libraries/tools/kotlin-gradle-plugin/build.gradle.kts +++ b/libraries/tools/kotlin-gradle-plugin/build.gradle.kts @@ -15,7 +15,7 @@ publish() // todo: make lazy val jar: Jar by tasks -runtimeJar(rewriteDepsToShadedCompiler(jar)) +val jarContents by configurations.creating sourcesJar() javadocJar() @@ -61,6 +61,10 @@ dependencies { runtime(projectRuntimeJar(":kotlin-scripting-compiler-embeddable")) runtime(project(":kotlin-reflect")) + jarContents(compileOnly(intellijDep()) { + includeJars("asm-all", rootProject = rootProject) + }) + // com.android.tools.build:gradle has ~50 unneeded transitive dependencies compileOnly("com.android.tools.build:gradle:3.0.0") { isTransitive = false } compileOnly("com.android.tools.build:gradle-core:3.0.0") { isTransitive = false } @@ -77,6 +81,17 @@ dependencies { testCompileOnly(project(":kotlin-annotation-processing-gradle")) } +runtimeJar(rewriteDepsToShadedCompiler(jar)) { + dependsOn(jarContents) + + from { + jarContents.asFileTree.map { + if (it.endsWith(".jar")) zipTree(it) + else it + } + } +} + tasks { withType { kotlinOptions.jdkHome = rootProject.extra["JDK_18"] as String diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/Kapt3KotlinGradleSubplugin.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/Kapt3KotlinGradleSubplugin.kt index 7d1bf84c6ad42..2eeda0ce68a23 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/Kapt3KotlinGradleSubplugin.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/Kapt3KotlinGradleSubplugin.kt @@ -13,6 +13,7 @@ import org.gradle.api.Project import org.gradle.api.Task import org.gradle.api.artifacts.Configuration import org.gradle.api.artifacts.Dependency +import org.gradle.api.attributes.Attribute import org.gradle.api.tasks.SourceSet import org.gradle.api.tasks.TaskDependency import org.gradle.api.tasks.compile.AbstractCompile @@ -23,6 +24,8 @@ import org.jetbrains.kotlin.gradle.internal.Kapt3GradleSubplugin.Companion.getKa import org.jetbrains.kotlin.gradle.internal.Kapt3GradleSubplugin.Companion.getKaptGeneratedKotlinSourcesDir import org.jetbrains.kotlin.gradle.internal.Kapt3GradleSubplugin.Companion.getKaptGeneratedSourcesDir import org.jetbrains.kotlin.gradle.internal.Kapt3KotlinGradleSubplugin.Companion.KAPT_WORKER_DEPENDENCIES_CONFIGURATION_NAME +import org.jetbrains.kotlin.gradle.internal.kapt.incremental.CLASS_STRUCTURE_ARTIFACT_TYPE +import org.jetbrains.kotlin.gradle.internal.kapt.incremental.StructureArtifactTransform import org.jetbrains.kotlin.gradle.model.builder.KaptModelBuilder import org.jetbrains.kotlin.gradle.plugin.* import org.jetbrains.kotlin.gradle.tasks.CompilerPluginOptions @@ -374,7 +377,8 @@ class Kapt3KotlinGradleSubplugin : KotlinGradleSubplugin { private fun Kapt3SubpluginContext.createKaptKotlinTask(useWorkerApi: Boolean): KaptTask { val taskClass = if (useWorkerApi) KaptWithoutKotlincTask::class.java else KaptWithKotlincTask::class.java - val kaptTask = project.tasks.create(getKaptTaskName("kapt"), taskClass) + val taskName = getKaptTaskName("kapt") + val kaptTask = project.tasks.create(taskName, taskClass) kaptTask.useBuildCache = kaptExtension.useBuildCache @@ -389,6 +393,13 @@ class Kapt3KotlinGradleSubplugin : KotlinGradleSubplugin { kaptTask.isIncremental = project.isIncrementalKapt() if (kaptTask.isIncremental) { kaptTask.incAptCache = getKaptIncrementalAnnotationProcessingCache() + + maybeRegisterTransform(project) + val classStructure = project.configurations.create("_classStructure${taskName}") + project.dependencies.add(classStructure.name, kotlinCompile.classpath) + kaptTask.classpathStructure = classStructure.incoming.artifactView { viewConfig -> + viewConfig.attributes.attribute(artifactType, CLASS_STRUCTURE_ARTIFACT_TYPE) + }.files } kotlinCompilation?.run { @@ -437,6 +448,23 @@ class Kapt3KotlinGradleSubplugin : KotlinGradleSubplugin { return kaptTask } + private fun maybeRegisterTransform(project: Project) { + if (!project.extensions.extraProperties.has("KaptStructureTransformAdded")) { + project.dependencies.registerTransform { variantTransform -> + variantTransform.artifactTransform(StructureArtifactTransform::class.java) + variantTransform.from.attribute(artifactType, "jar") + variantTransform.to.attribute(artifactType, CLASS_STRUCTURE_ARTIFACT_TYPE) + } + project.dependencies.registerTransform { variantTransform -> + variantTransform.artifactTransform(StructureArtifactTransform::class.java) + variantTransform.from.attribute(artifactType, "directory") + variantTransform.to.attribute(artifactType, CLASS_STRUCTURE_ARTIFACT_TYPE) + } + + project.extensions.extraProperties["KaptStructureTransformAdded"] = true + } + } + private fun Kapt3SubpluginContext.createKaptGenerateStubsTask(): KaptGenerateStubsTask { val kaptTask = project.tasks.create( getKaptTaskName("kaptGenerateStubs"), @@ -495,6 +523,8 @@ class Kapt3KotlinGradleSubplugin : KotlinGradleSubplugin { override fun getPluginArtifact(): SubpluginArtifact = JetBrainsSubpluginArtifact(artifactId = KAPT_ARTIFACT_NAME) } +private val artifactType = Attribute.of("artifactType", String::class.java) + internal fun registerGeneratedJavaSource(kaptTask: KaptTask, javaTask: AbstractCompile) { javaTask.source(kaptTask.destinationDir) diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptTask.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptTask.kt index 51b4a67e3e64b..53248d9b0b45a 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptTask.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptTask.kt @@ -6,6 +6,10 @@ import org.gradle.api.file.FileCollection import org.gradle.api.internal.ConventionTask import org.gradle.api.tasks.* import org.gradle.api.tasks.incremental.IncrementalTaskInputs +import org.jetbrains.kotlin.gradle.internal.kapt.incremental.KaptClasspathChanges +import org.jetbrains.kotlin.gradle.internal.kapt.incremental.ClasspathSnapshot +import org.jetbrains.kotlin.gradle.internal.kapt.incremental.KaptIncrementalChanges +import org.jetbrains.kotlin.gradle.internal.kapt.incremental.UnknownSnapshot import org.jetbrains.kotlin.gradle.internal.tasks.TaskWithLocalState import org.jetbrains.kotlin.gradle.tasks.KotlinCompile import org.jetbrains.kotlin.gradle.tasks.cacheOnlyIfEnabledForKotlin @@ -47,6 +51,12 @@ abstract class KaptTask : ConventionTask(), TaskWithLocalState { @get:Internal internal lateinit var kaptClasspathConfigurations: List + + @get:PathSensitive(PathSensitivity.NONE) + @get:Optional + @get:InputFiles + internal var classpathStructure: FileCollection? = null + /** Output directory that contains caches necessary to support incremental annotation processing. */ @get:OutputDirectory @get:Optional @@ -156,23 +166,67 @@ abstract class KaptTask : ConventionTask(), TaskWithLocalState { @Internal protected fun getCompiledSources() = listOfNotNull(kotlinCompileTask.destinationDir, kotlinCompileTask.javaOutputDir) - protected fun getChangedFiles(inputs: IncrementalTaskInputs): List { - if (!isIncremental || !inputs.isIncremental || !getCompiledSources().all { it.exists() }) { + protected fun getIncrementalChanges(inputs: IncrementalTaskInputs): KaptIncrementalChanges { + val changedFiles = getChangedFiles(inputs) + + return if (isIncremental) { + val classpathChanges = if (changedFiles.isEmpty()) { + classpath.files + } else { + classpath.files.let { cp -> + changedFiles.filter { cp.contains(it) } + } + } + val classpathStatus = findClasspathChanges(classpathChanges) + when (classpathStatus) { + is KaptClasspathChanges.Unknown -> KaptIncrementalChanges.Unknown + is KaptClasspathChanges.Known -> KaptIncrementalChanges.Known( + changedFiles.filter { it.extension == "java" }.toSet(), classpathStatus.names + ) + } + } else { + KaptIncrementalChanges.Unknown + } + } + + private fun getChangedFiles(inputs: IncrementalTaskInputs): List { + return if (!isIncremental || !inputs.isIncremental) { clearLocalState() - return emptyList() + emptyList() } else { - val changes = with(mutableSetOf()) { + with(mutableSetOf()) { inputs.outOfDate { this.add(it.file) } inputs.removed { this.add(it.file) } return@with this.toList() } + } + } - return if (changes.all { it.extension == "java" }) { - changes - } else { - emptyList() + private fun findClasspathChanges(changedClasspath: Iterable): KaptClasspathChanges { + incAptCache!!.mkdirs() + + val startTime = System.currentTimeMillis() + + val previousSnapshot = ClasspathSnapshot.ClasspathSnapshotFactory.loadFrom(incAptCache!!) + val currentSnapshot = ClasspathSnapshot.ClasspathSnapshotFactory.createCurrent(incAptCache!!, classpath.files.toList(), classpathStructure!!.files) + + val classpathChanges = currentSnapshot.diff(previousSnapshot, changedClasspath.toSet()) + currentSnapshot.writeToCache() + + if (logger.isInfoEnabled) { + val time = "Took ${System.currentTimeMillis() - startTime}ms." + when { + previousSnapshot == UnknownSnapshot -> + logger.info("Initializing classpath information for KAPT. $time") + classpathChanges == KaptClasspathChanges.Unknown -> + logger.info("Unable to use existing data, re-initializing classpath information for KAPT. $time") + else -> { + classpathChanges as KaptClasspathChanges.Known + logger.info("Full list of impacted classpath names: ${classpathChanges.names}. $time") + } } } + return classpathChanges } private fun hasAnnotationProcessors(file: File): Boolean { diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptWithKotlincTask.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptWithKotlincTask.kt index e7fb8bff4aef9..cbbfefbe4ffb9 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptWithKotlincTask.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptWithKotlincTask.kt @@ -17,6 +17,7 @@ import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments import org.jetbrains.kotlin.compilerRunner.GradleCompilerEnvironment import org.jetbrains.kotlin.compilerRunner.GradleCompilerRunner import org.jetbrains.kotlin.compilerRunner.OutputItemsCollectorImpl +import org.jetbrains.kotlin.gradle.internal.kapt.incremental.KaptIncrementalChanges import org.jetbrains.kotlin.gradle.internal.tasks.allOutputFiles import org.jetbrains.kotlin.gradle.logging.GradleKotlinLogger import org.jetbrains.kotlin.gradle.logging.GradlePrintingMessageCollector @@ -73,11 +74,11 @@ open class KaptWithKotlincTask : KaptTask(), CompilerArgumentAwareWithInput { - changedFiles = incrementalChanges - classpathChanges = emptyList() + val incrementalChanges = getIncrementalChanges(inputs) + when (incrementalChanges) { + is KaptIncrementalChanges.Known -> { + changedFiles = incrementalChanges.changedSources.toList() + classpathChanges = incrementalChanges.changedClasspathJvmNames.toList() processIncrementally = true } else -> { diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptWithoutKotlincTask.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptWithoutKotlincTask.kt index e278296604293..b01048bcb62d2 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptWithoutKotlincTask.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptWithoutKotlincTask.kt @@ -13,6 +13,7 @@ import org.gradle.api.tasks.incremental.IncrementalTaskInputs import org.gradle.workers.IsolationMode import org.gradle.workers.WorkerExecutor import org.jetbrains.kotlin.gradle.internal.Kapt3KotlinGradleSubplugin.Companion.KAPT_WORKER_DEPENDENCIES_CONFIGURATION_NAME +import org.jetbrains.kotlin.gradle.internal.kapt.incremental.KaptIncrementalChanges import org.jetbrains.kotlin.gradle.plugin.KotlinAndroidPluginWrapper import org.jetbrains.kotlin.gradle.tasks.findKotlinStdlibClasspath import org.jetbrains.kotlin.gradle.tasks.findToolsJar @@ -49,7 +50,11 @@ open class KaptWithoutKotlincTask @Inject constructor(private val workerExecutor logger.info("Running kapt annotation processing using the Gradle Worker API") checkAnnotationProcessorClasspath() - val incrementalChanges = getChangedFiles(inputs) + val incrementalChanges = getIncrementalChanges(inputs) + val (changedFiles, classpathChanges) = when (incrementalChanges) { + is KaptIncrementalChanges.Unknown -> Pair(emptyList(), emptyList()) + is KaptIncrementalChanges.Known -> Pair(incrementalChanges.changedSources.toList(), incrementalChanges.changedClasspathJvmNames) + } val compileClasspath = classpath.files.toMutableList() if (project.plugins.none { it is KotlinAndroidPluginWrapper }) { @@ -67,11 +72,11 @@ open class KaptWithoutKotlincTask @Inject constructor(private val workerExecutor compileClasspath, javaSourceRoots.toList(), - incrementalChanges, + changedFiles, getCompiledSources(), incAptCache, - emptyList(), - incrementalChanges.isNotEmpty(), + classpathChanges.toList(), + incrementalChanges is KaptIncrementalChanges.Known, destinationDir, classesDir, diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/incremental/ClassAbiExtractor.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/incremental/ClassAbiExtractor.kt new file mode 100644 index 0000000000000..f118a979ddfd8 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/incremental/ClassAbiExtractor.kt @@ -0,0 +1,52 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.internal.kapt.incremental + +import org.jetbrains.org.objectweb.asm.* + +const val metadataDescriptor: String = "Lkotlin/Metadata;" +class ClassAbiExtractor(private val writer: ClassWriter) : ClassVisitor(Opcodes.API_VERSION, writer) { + + override fun visitMethod( + access: Int, + name: String?, + desc: String?, + signature: String?, + exceptions: Array? + ): MethodVisitor? { + return if (access.isAbi()) { + super.visitMethod(access, name, desc, signature, exceptions) + } else { + null + } + } + + override fun visitAnnotation(desc: String?, visible: Boolean): AnnotationVisitor? { + return if (desc != null && desc != metadataDescriptor) { + super.visitAnnotation(desc, visible) + } else { + null + } + } + + override fun visitField(access: Int, name: String?, desc: String?, signature: String?, value: Any?): FieldVisitor? { + return if (access.isAbi()) { + super.visitField(access, name, desc, signature, value) + } else { + null + } + } + + override fun visitInnerClass(name: String?, outerName: String?, innerName: String?, access: Int) { + if (access.isAbi() && outerName != null && innerName != null) { + super.visitInnerClass(name, outerName, innerName, access) + } + } + + fun getBytes(): ByteArray = writer.toByteArray() + + private fun Int.isAbi() = (this and Opcodes.ACC_PRIVATE) == 0 +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/incremental/ClassTypeExtractorVisitor.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/incremental/ClassTypeExtractorVisitor.kt new file mode 100644 index 0000000000000..60c06d3194917 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/incremental/ClassTypeExtractorVisitor.kt @@ -0,0 +1,177 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.internal.kapt.incremental + +import org.jetbrains.org.objectweb.asm.* + +private val toIgnore = setOf("java/lang/Object", "kotlin/Metadata", "org/jetbrains/annotations/NotNull") + +class ClassTypeExtractorVisitor(visitor: ClassVisitor) : ClassVisitor(Opcodes.API_VERSION, visitor) { + + private val abiTypes = mutableSetOf() + private val privateTypes = mutableSetOf() + + private lateinit var classInternalName: String + + fun getAbiTypes() = abiTypes.filter { !toIgnore.contains(it) && it != classInternalName }.toSet() + fun getPrivateTypes() = privateTypes.filter { !toIgnore.contains(it) && it != classInternalName && !abiTypes.contains(it)}.toSet() + + override fun visit(version: Int, access: Int, name: String?, signature: String?, superName: String?, interfaces: Array?) { + super.visit(version, access, name, signature, superName, interfaces) + classInternalName = name!! + superName?.let { + abiTypes.add(it) + } + interfaces?.let { + abiTypes.addAll(it) + } + } + + override fun visitMethod( + access: Int, + name: String?, + desc: String?, + signature: String?, + exceptions: Array? + ): MethodVisitor? { + val typeCollector = if (access and Opcodes.ACC_PRIVATE != 0) { + privateTypes + } else { + abiTypes + } + + desc?.also { + val type = Type.getType(desc) + + maybeAdd(typeCollector, type.returnType) + type.argumentTypes.forEach { + maybeAdd(typeCollector, it) + } + } + + return MethodTypeExtractorVisitor(typeCollector, super.visitMethod(access, name, desc, signature, exceptions)) + } + + override fun visitField(access: Int, name: String?, desc: String?, signature: String?, value: Any?): FieldVisitor? { + val typeCollector = if (access and Opcodes.ACC_PRIVATE != 0) { + privateTypes + } else { + abiTypes + } + + desc?.also { + val type = Type.getType(desc) + maybeAdd(typeCollector, type) + } + + return FieldTypeExtractorVisitor(typeCollector, super.visitField(access, name, desc, signature, value)) + } + + override fun visitAnnotation(desc: String?, visible: Boolean): AnnotationVisitor? { + desc?.let { + maybeAdd(abiTypes, Type.getType(it)) + } + return AnnotationTypeExtractorVisitor(abiTypes, super.visitAnnotation(desc, visible)) + } + + override fun visitTypeAnnotation(typeRef: Int, typePath: TypePath?, desc: String?, visible: Boolean): AnnotationVisitor? { + desc?.let { + maybeAdd(abiTypes, Type.getType(it)) + } + + return AnnotationTypeExtractorVisitor(abiTypes, super.visitTypeAnnotation(typeRef, typePath, desc, visible)) + } +} + +private class AnnotationTypeExtractorVisitor(private val typeCollector: MutableSet, visitor: AnnotationVisitor?) : + AnnotationVisitor(Opcodes.ASM5, visitor) { + + override fun visit(name: String?, value: Any?) { + if (value is Type) { + typeCollector.add(value.className) + } + super.visit(name, value) + } + + override fun visitAnnotation(name: String?, desc: String?): AnnotationVisitor? { + desc?.let { + maybeAdd(typeCollector, Type.getType(it)) + } + return super.visitAnnotation(name, desc) + } + + override fun visitArray(name: String?): AnnotationVisitor? { + return AnnotationTypeExtractorVisitor(typeCollector, super.visitArray(name)) + } + + override fun visitEnum(name: String?, desc: String?, value: String?) { + desc?.let { + maybeAdd(typeCollector, Type.getType(it)) + } + super.visitEnum(name, desc, value) + } +} + +private class FieldTypeExtractorVisitor(private val typeCollector: MutableSet, visitor: FieldVisitor?) : + FieldVisitor(Opcodes.ASM5, visitor) { + override fun visitAnnotation(desc: String?, visible: Boolean): AnnotationVisitor? { + desc?.let { + maybeAdd(typeCollector, Type.getType(it)) + } + return AnnotationTypeExtractorVisitor(typeCollector, super.visitAnnotation(desc, visible)) + } + + override fun visitTypeAnnotation(typeRef: Int, typePath: TypePath?, desc: String?, visible: Boolean): AnnotationVisitor? { + desc?.let { + maybeAdd(typeCollector, Type.getType(it)) + } + return AnnotationTypeExtractorVisitor(typeCollector, super.visitTypeAnnotation(typeRef, typePath, desc, visible)) + } +} + +private class MethodTypeExtractorVisitor(private val typeCollector: MutableSet, visitor: MethodVisitor?) : + MethodVisitor(Opcodes.ASM5, visitor) { + + override fun visitAnnotationDefault(): AnnotationVisitor { + return AnnotationTypeExtractorVisitor(typeCollector, super.visitAnnotationDefault()) + } + + override fun visitAnnotation(desc: String?, visible: Boolean): AnnotationVisitor? { + desc?.let { + maybeAdd(typeCollector, Type.getType(it)) + } + return AnnotationTypeExtractorVisitor(typeCollector, super.visitAnnotation(desc, visible)) + } + + override fun visitParameterAnnotation(parameter: Int, desc: String?, visible: Boolean): AnnotationVisitor? { + desc?.let { + maybeAdd(typeCollector, Type.getType(it)) + } + return AnnotationTypeExtractorVisitor(typeCollector, super.visitParameterAnnotation(parameter, desc, visible)) + } + + override fun visitTypeAnnotation(typeRef: Int, typePath: TypePath?, desc: String?, visible: Boolean): AnnotationVisitor? { + desc?.let { + maybeAdd(typeCollector, Type.getType(it)) + } + + return AnnotationTypeExtractorVisitor(typeCollector, super.visitTypeAnnotation(typeRef, typePath, desc, visible)) + } +} + +private fun maybeAdd(set: MutableSet, type: Type) { + type.finalInternalName()?.let { set.add(it) } +} + +private fun Type.finalInternalName(): String? { + return when (this.sort) { + Type.VOID, Type.BOOLEAN, Type.CHAR, Type.BYTE, Type.SHORT, Type.INT, Type.FLOAT, Type.DOUBLE -> null + Type.ARRAY -> this.elementType.finalInternalName() + Type.OBJECT -> this.internalName + Type.METHOD -> null + else -> null + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/incremental/ClasspathAnalyzer.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/incremental/ClasspathAnalyzer.kt new file mode 100644 index 0000000000000..d19a360648fdc --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/incremental/ClasspathAnalyzer.kt @@ -0,0 +1,210 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.internal.kapt.incremental + +import com.google.common.hash.Hashing +import org.gradle.api.artifacts.transform.ArtifactTransform +import org.jetbrains.org.objectweb.asm.ClassReader +import org.jetbrains.org.objectweb.asm.ClassWriter +import java.io.* +import java.util.zip.ZipFile + +const val CLASS_STRUCTURE_ARTIFACT_TYPE = "class-structure" + +class StructureArtifactTransform : ArtifactTransform() { + override fun transform(input: File): MutableList { + try { + val data = if (input.isDirectory) { + visitDirectory(input) + } else { + visitJar(input) + } + + val dataFile = outputDirectory.resolve("output.bin") + data.saveTo(dataFile) + + val lazyStructureFile = outputDirectory.resolve("lazy-output.bin") + LazyClasspathEntryData(input, dataFile).saveToFile(lazyStructureFile) + + return mutableListOf(lazyStructureFile) + } catch (e: Throwable) { + throw e + } + } +} + +private fun visitDirectory(directory: File): ClasspathEntryData { + val entryData = ClasspathEntryData() + + directory.walk().filter { + it.extension == "class" && !it.relativeTo(directory).toString().toLowerCase().startsWith("meta-inf") + }.forEach { + val internalName = it.relativeTo(directory).toString().dropLast(".class".length) + analyzeInputStream(it.inputStream(), internalName, entryData) + } + + return entryData +} + +private fun visitJar(jar: File): ClasspathEntryData { + val entryData = ClasspathEntryData() + + ZipFile(jar).use { zipFile -> + val entries = zipFile.entries() + while (entries.hasMoreElements()) { + val entry = entries.nextElement() + + if (entry.name.endsWith("class") && !entry.name.toLowerCase().startsWith("meta-inf")) { + analyzeInputStream(zipFile.getInputStream(entry), entry.name.dropLast(".class".length), entryData) + } + } + } + + return entryData +} + +private fun analyzeInputStream(input: InputStream, internalName: String, entryData: ClasspathEntryData) { + val abiExtractor = ClassAbiExtractor(ClassWriter(0)) + val typeDependenciesExtractor = ClassTypeExtractorVisitor(abiExtractor) + ClassReader(BufferedInputStream(input).readBytes()).accept( + typeDependenciesExtractor, + ClassReader.SKIP_CODE or ClassReader.SKIP_DEBUG or ClassReader.SKIP_FRAMES + ) + + val bytes = abiExtractor.getBytes() + val hashBytes = Hashing.murmur3_128().hashBytes(bytes) + entryData.classAbiHash[internalName] = hashBytes.asBytes() + entryData.classDependencies[internalName] = + ClassDependencies(typeDependenciesExtractor.getAbiTypes(), typeDependenciesExtractor.getPrivateTypes()) +} + +class LazyClasspathEntryData(val classpathEntry: File, private val dataFile: File) : Serializable { + + object LazyClasspathEntrySerializer { + fun loadFromFile(file: File): LazyClasspathEntryData { + ObjectInputStream(BufferedInputStream(file.inputStream())).use { + return it.readObject() as LazyClasspathEntryData + } + } + } + + fun saveToFile(file: File) { + ObjectOutputStream(BufferedOutputStream(file.outputStream())).use { + it.writeObject(this) + } + } + + fun getClasspathEntryData(): ClasspathEntryData = ClasspathEntryData.ClasspathEntrySerializer.loadFrom(dataFile) +} + +class ClasspathEntryData : Serializable { + + object ClasspathEntrySerializer { + fun loadFrom(file: File): ClasspathEntryData { + ObjectInputStream(BufferedInputStream(file.inputStream())).use { + return it.readObject() as ClasspathEntryData + } + } + } + + @Transient + var classAbiHash = mutableMapOf() + @Transient + var classDependencies = mutableMapOf() + + private fun writeObject(output: ObjectOutputStream) { + val names = mutableMapOf() + classAbiHash.keys.forEach { names[it] = names.size } + classDependencies.values.forEach { + it.abiTypes.forEach { + if (!names.containsKey(it)) names[it] = names.size + } + it.privateTypes.forEach { + if (!names.containsKey(it)) names[it] = names.size + } + } + + output.writeInt(names.size) + names.forEach { key, value -> + output.writeInt(value) + output.writeUTF(key) + } + + output.writeInt(classAbiHash.size) + classAbiHash.forEach { + output.writeInt(names[it.key]!!) + output.writeInt(it.value.size) + output.write(it.value) + } + + output.writeInt(classDependencies.size) + classDependencies.forEach { + output.writeInt(names[it.key]!!) + + output.writeInt(it.value.abiTypes.size) + it.value.abiTypes.forEach { + output.writeInt(names[it]!!) + } + + output.writeInt(it.value.privateTypes.size) + it.value.privateTypes.forEach { + output.writeInt(names[it]!!) + } + } + } + + @Suppress("UNCHECKED_CAST") + private fun readObject(input: ObjectInputStream) { + val namesSize = input.readInt() + val names = HashMap(namesSize) + repeat(namesSize) { + val classId = input.readInt() + val classInternalName = input.readUTF() + names[classId] = classInternalName + } + + val abiHashesSize = input.readInt() + classAbiHash = HashMap(abiHashesSize) + repeat(abiHashesSize) { + val internalName = names[input.readInt()]!! + val byteArraySize = input.readInt() + val hash = ByteArray(byteArraySize) + repeat(byteArraySize) { + hash[it] = input.readByte() + } + classAbiHash[internalName] = hash + } + + val dependenciesSize = input.readInt() + classDependencies = HashMap(dependenciesSize) + + repeat(dependenciesSize) { + val internalName = names[input.readInt()]!! + + val abiTypesSize = input.readInt() + val abiTypeNames = HashSet(abiTypesSize) + repeat(abiTypesSize) { + abiTypeNames.add(names[input.readInt()]!!) + } + + val privateTypesSize = input.readInt() + val privateTypeNames = HashSet(privateTypesSize) + repeat(privateTypesSize) { + privateTypeNames.add(names[input.readInt()]!!) + } + + classDependencies[internalName] = ClassDependencies(abiTypeNames, privateTypeNames) + } + } + + fun saveTo(file: File) { + ObjectOutputStream(BufferedOutputStream(file.outputStream())).use { + it.writeObject(this) + } + } +} + +class ClassDependencies(val abiTypes: Collection, val privateTypes: Collection) \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/incremental/ClasspathSnapshot.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/incremental/ClasspathSnapshot.kt new file mode 100644 index 0000000000000..2dfbe94fc3f97 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/incremental/ClasspathSnapshot.kt @@ -0,0 +1,179 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.internal.kapt.incremental + +import java.io.* +import java.util.* + +open class ClasspathSnapshot protected constructor( + private val cacheDir: File, + private val classpath: Iterable, + val dataForFiles: (Set) -> Map +) { + val classpathData: (Set) -> Map = { files -> + val missingFiles = files.filter { !computedClasspathData.keys.contains(it) }.toSet() + + if (!missingFiles.isEmpty()) { + val computedData = dataForFiles(missingFiles) + computedClasspathData.putAll(computedData) + } + computedClasspathData.filter { files.contains(it.key) } + } + private val computedClasspathData: MutableMap = mutableMapOf() + + object ClasspathSnapshotFactory { + fun loadFrom(cacheDir: File): ClasspathSnapshot { + val classpathEntries = cacheDir.resolve("classpath-entries.bin") + val classpathStructureData = cacheDir.resolve("classpath-structure.bin") + if (!classpathEntries.exists() || !classpathStructureData.exists()) { + return UnknownSnapshot + } + + val classpathFiles = ObjectInputStream(BufferedInputStream(classpathEntries.inputStream())).use { + @Suppress("UNCHECKED_CAST") + it.readObject() as Iterable + } + + val classpathData = { _: Set -> + loadPreviousData(classpathStructureData) + } + return ClasspathSnapshot(cacheDir, classpathFiles, classpathData) + } + + fun createCurrent(cacheDir: File, classpath: Iterable, lazyClasspathData: Set): ClasspathSnapshot { + val lazyData = lazyClasspathData.map { LazyClasspathEntryData.LazyClasspathEntrySerializer.loadFromFile(it) } + val data = { files: Set -> + lazyData.filter { files.contains(it.classpathEntry) }.associate { it.classpathEntry to it.getClasspathEntryData() } + } + + return ClasspathSnapshot(cacheDir, classpath, data) + } + + private fun loadPreviousData(file: File): Map { + ObjectInputStream(BufferedInputStream(file.inputStream())).use { + @Suppress("UNCHECKED_CAST") + return it.readObject() as Map + } + } + } + + private fun isCompatible(snapshot: ClasspathSnapshot) = + this != UnknownSnapshot && classpath == snapshot.classpath + + /** Compare this snapshot with the specified one only for the specified files. */ + fun diff(previousSnapshot: ClasspathSnapshot, changedFiles: Set): KaptClasspathChanges { + if (!isCompatible(previousSnapshot)) { + return KaptClasspathChanges.Unknown + } + + val currentData = classpathData(changedFiles) + val previousData = previousSnapshot.classpathData(changedFiles) + + val changedClasses = mutableSetOf() + + for (changed in changedFiles) { + val previous = previousData.getValue(changed) + val current = currentData.getValue(changed) + + for (key in previous.classAbiHash.keys + current.classAbiHash.keys) { + val previousHash = previous.classAbiHash[key] + if (previousHash == null) { + changedClasses.add(key) + continue + } + val currentHash = current.classAbiHash[key] + if (currentHash == null) { + changedClasses.add(key) + continue + } + if (!previousHash.contentEquals(currentHash)) { + changedClasses.add(key) + } + } + } + + // We do not compute structural data for unchanged files of the current snapshot for performance reasons. That is why we + // update the previous snapshot as that one contains all entries. + computedClasspathData.putAll(previousData) + computedClasspathData.putAll(currentData) + + val allImpactedClasses = findAllImpacted(changedClasses) + + return KaptClasspathChanges.Known(allImpactedClasses) + } + + fun writeToCache() { + val classpathEntries = cacheDir.resolve("classpath-entries.bin") + ObjectOutputStream(BufferedOutputStream(classpathEntries.outputStream())).use { + it.writeObject(classpath) + } + + val classpathStructureData = cacheDir.resolve("classpath-structure.bin") + storeCurrentStructure(classpathStructureData, classpathData(classpath.toSet())) + } + + private fun storeCurrentStructure(file: File, structure: Map) { + ObjectOutputStream(BufferedOutputStream(file.outputStream())).use { + it.writeObject(structure) + } + } + + private fun findAllImpacted(changedClasses: Set): Set { + // TODO (gavra): Avoid building all reverse lookups. Most changes are local to the classpath entry, use that. + val transitiveDeps = HashMap>() + val nonTransitiveDeps = HashMap>() + + for (entry in computedClasspathData.values) { + for ((className, classDependency) in entry.classDependencies) { + for (abiType in classDependency.abiTypes) { + (transitiveDeps[abiType] ?: LinkedList()).let { + it.add(className) + transitiveDeps[abiType] = it + } + } + for (privateType in classDependency.privateTypes) { + (nonTransitiveDeps[privateType] ?: LinkedList()).let { + it.add(className) + nonTransitiveDeps[privateType] = it + } + + } + } + } + + val allImpacted = mutableSetOf() + var current = changedClasses + while(current.isNotEmpty()) { + val newRound = mutableSetOf() + for (klass in current) { + if (allImpacted.add(klass)) { + transitiveDeps[klass]?.let { + newRound.addAll(it) + } + + nonTransitiveDeps[klass]?.let { + allImpacted.addAll(it) + } + } + } + current = newRound + } + + return allImpacted + } +} + +object UnknownSnapshot : ClasspathSnapshot(File(""), emptyList(), { emptyMap() }) + +sealed class KaptIncrementalChanges { + object Unknown : KaptIncrementalChanges() + class Known(val changedSources: Set, val changedClasspathJvmNames: Set) : KaptIncrementalChanges() +} + +sealed class KaptClasspathChanges { + object Unknown : KaptClasspathChanges() + class Known(val names: Set) : KaptClasspathChanges() +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/incremental/ClassAbiExtractorTest.kt b/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/incremental/ClassAbiExtractorTest.kt new file mode 100644 index 0000000000000..5ceedbbbc7481 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/incremental/ClassAbiExtractorTest.kt @@ -0,0 +1,346 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.internal.kapt.incremental + +import org.jetbrains.kotlin.gradle.util.compileSources +import org.jetbrains.org.objectweb.asm.ClassReader +import org.jetbrains.org.objectweb.asm.ClassWriter +import org.jetbrains.org.objectweb.asm.Opcodes +import org.junit.Assert +import org.junit.Assert.assertArrayEquals +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import java.util.* + +class ClassAbiExtractorTest { + @Rule + @JvmField + var tmp = TemporaryFolder() + + @Test + fun testDifferentClassName() { + val firstHash = getHash( + """ + public class A { + } + """.trimIndent() + ) + + val secondHash = getHash( + """ + public class B { + } + """.trimIndent(), "B" + ) + + assertArrayNotEquals(firstHash, secondHash) + } + + @Test + fun testAbiMethod() { + val firstHash = getHash( + """ + public class A { + public void run() {} + void doSomething1() {} + } + """.trimIndent() + ) + + val secondHash = getHash( + """ + public class A { + public void run() {} + void doSomething2() {} + } + """.trimIndent() + ) + + assertArrayNotEquals(firstHash, secondHash) + } + + @Test + fun testAbiMethodAnnotations() { + val firstHash = getHash( + """ + public class A { + @Annotation1 + public void run() {} + } + @interface Annotation1 {} + """.trimIndent() + ) + + val secondHash = getHash( + """ + public class A { + @Annotation2 + public void run() {} + } + @interface Annotation2 {} + """.trimIndent() + ) + + assertArrayNotEquals(firstHash, secondHash) + } + + @Test + fun testMethodBodiesIgnored() { + val firstHash = getHash( + """ + public class A { + public void run() { + System.out.println("1"); + } + } + """.trimIndent() + ) + + val secondHash = getHash( + """ + public class A { + public void run() { + System.out.println("2"); + } + } + """.trimIndent() + ) + + assertArrayEquals(firstHash, secondHash) + } + + @Test + fun testPrivateMethodIgnored() { + val firstHash = getHash( + """ + public class A { + public void run() {} + private void doSomething1() {} + } + """.trimIndent() + ) + + val secondHash = getHash( + """ + public class A { + public void run() {} + private void doSomething2() {} + } + """.trimIndent() + ) + + assertArrayEquals(firstHash, secondHash) + } + + @Test + fun testAbiField() { + val firstHash = getHash( + """ + public class A { + protected String value; + public String data1; + } + """.trimIndent() + ) + + val secondHash = getHash( + """ + public class A { + protected String value; + public String data2; + } + """.trimIndent() + ) + + assertArrayNotEquals(firstHash, secondHash) + } + + @Test + fun testFieldAnnotation() { + val firstHash = getHash( + """ + public class A { + @Annotation1 + protected String value; + } + @interface Annotation1 {} + """.trimIndent() + ) + + val secondHash = getHash( + """ + public class A { + @Annotation2 + protected String value; + } + @interface Annotation2 {} + """.trimIndent() + ) + + assertArrayNotEquals(firstHash, secondHash) + } + + @Test + fun testConstants() { + val firstHash = getHash( + """ + public class A { + static final String VALUE = "value_1"; + } + """.trimIndent() + ) + + val secondHash = getHash( + """ + public class A { + static final String VALUE = "value_2"; + } + """.trimIndent() + ) + + assertArrayNotEquals(firstHash, secondHash) + } + + @Test + fun testSameConstants() { + val firstHash = getHash( + """ + public class A { + static final String VALUE = "value_1"; + } + """.trimIndent() + ) + + val secondHash = getHash( + """ + public class A { + static final String VALUE = "value_1"; + } + """.trimIndent() + ) + + assertArrayEquals(firstHash, secondHash) + } + + @Test + fun testPrivateFieldsIgnored() { + val firstHash = getHash( + """ + public class A { + protected String value; + private String data; + } + """.trimIndent() + ) + + val secondHash = getHash( + """ + public class A { + protected String value; + private int data; + } + """.trimIndent() + ) + + assertArrayEquals(firstHash, secondHash) + } + + @Test + fun testAbiInnerClass() { + val firstHash = getHash( + """ + public class A { + class Inner1 {} + } + """.trimIndent() + ) + + val secondHash = getHash( + """ + public class A { + class Inner2 {} + } + """.trimIndent() + ) + + assertArrayNotEquals(firstHash, secondHash) + } + + + @Test + fun testPrivateInnerClassesIgnored() { + val firstHash = getHash( + """ + public class A { + protected String value; + private String data; + + private static class Inner1 {} + } + """.trimIndent() + ) + + val secondHash = getHash( + """ + public class A { + protected String value; + private int data; + private static class Inner2 {} + } + """.trimIndent() + ) + + assertArrayEquals(firstHash, secondHash) + } + + @Test + fun testKotlinMetadataIgnored() { + val firstHash = getHash( + """ + package kotlin; + + @Metadata + public class A { + } + @interface Metadata {} + """.trimIndent() + ) + + val secondHash = getHash( + """ + package kotlin; + public class A { + + } + """.trimIndent() + ) + + assertArrayEquals(firstHash, secondHash) + } + + private fun assertArrayNotEquals(first: ByteArray, second: ByteArray) { + Assert.assertFalse(Arrays.equals(first, second)) + } + + + private fun getHash(source: String, className: String = "A"): ByteArray { + val src = tmp.newFolder().resolve("$className.java") + + src.writeText(source) + + val output = tmp.newFolder() + compileSources(listOf(src), output) + + val classFile = output.walk().filter { it.name == "$className.class" }.single() + + classFile.inputStream().use { + val extractor = ClassAbiExtractor(ClassWriter(0)) + ClassReader(it.readBytes()).accept(extractor, ClassReader.SKIP_CODE or ClassReader.SKIP_FRAMES or ClassReader.SKIP_DEBUG) + return extractor.getBytes() + } + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/incremental/ClassTypeExtractorVisitorTest.kt b/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/incremental/ClassTypeExtractorVisitorTest.kt new file mode 100644 index 0000000000000..900357427a334 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/incremental/ClassTypeExtractorVisitorTest.kt @@ -0,0 +1,138 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.internal.kapt.incremental + +import org.jetbrains.kotlin.gradle.util.compileSources +import org.jetbrains.org.objectweb.asm.ClassReader +import org.jetbrains.org.objectweb.asm.ClassVisitor +import org.jetbrains.org.objectweb.asm.Opcodes +import org.junit.Assert.assertEquals +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +class ClassTypeExtractorVisitorTest { + @Rule + @JvmField + var tmp = TemporaryFolder() + + @Test + fun testSupertypes() { + val src = """ + public class A extends B implements java.lang.Runnable { + public void run() {} + } + + class B {} + """.trimIndent() + val (abiTypes, privateTypes) = extractTypesFor(src) + assertEquals(setOf("B", "java/lang/Runnable"), abiTypes) + assertEquals(emptySet(), privateTypes) + } + + @Test + fun testObjectIgnored() { + val src = """ + public class A {} + """.trimIndent() + val (abiTypes, privateTypes) = extractTypesFor(src) + assertEquals(emptySet(), abiTypes) + assertEquals(emptySet(), privateTypes) + } + + @Test + fun testMethod() { + val src = """ + public class A { + public String process(Cloneable c) { + Runnable ignored = null; + return null; + } + public int ignorePrimitiveTypes(char c, boolean z, float f, double d, short s, long j, long[] jArray) { + return 1; + } + private java.util.HashSet[] getSets(java.util.ArrayList list) { + Runnable ignored = null; + return null; + } + } + """.trimIndent() + val (abiTypes, privateTypes) = extractTypesFor(src) + assertEquals(setOf("java/lang/String", "java/lang/Cloneable"), abiTypes) + assertEquals(setOf("java/util/HashSet", "java/util/ArrayList"), privateTypes) + } + + @Test + fun testField() { + val src = """ + public class A { + public String first; + protected String second; + String third; + private Runnable fourth; + public int ignored; + } + """.trimIndent() + val (abiTypes, privateTypes) = extractTypesFor(src) + assertEquals(setOf("java/lang/String"), abiTypes) + assertEquals(setOf("java/lang/Runnable"), privateTypes) + } + + @Test + fun testClassAnnotations() { + val src = """ + import java.lang.annotation.*; + + @Annotation + public class A extends @TypeAnnotation B { + } + class B {} + @interface Annotation {} + + @Target(value=ElementType.TYPE_USE) + @interface TypeAnnotation {} + """.trimIndent() + val (abiTypes, privateTypes) = extractTypesFor(src) + assertEquals(setOf("B", "Annotation", "TypeAnnotation"), abiTypes) + assertEquals(emptySet(), privateTypes) + } + + @Test + fun testMemberAnnotations() { + val src = """ + import java.lang.annotation.*; + + public class A { + @FieldAnnotation String data; + + @MethodAnnotation + String getName(@ParameterAnnotation String originalName) { + return originalName + "suffix"; + } + } + @interface FieldAnnotation {} + @interface MethodAnnotation {} + @interface ParameterAnnotation {} + """.trimIndent() + val (abiTypes, privateTypes) = extractTypesFor(src) + assertEquals(setOf("FieldAnnotation", "MethodAnnotation", "ParameterAnnotation", "java/lang/String"), abiTypes) + assertEquals(emptySet(), privateTypes) + } + + private fun extractTypesFor(source: String, className: String = "A"): Pair, Set> { + val src = tmp.newFolder().resolve("$className.java") + src.writeText(source) + + val output = tmp.newFolder() + compileSources(listOf(src), output) + val classFile = output.walk().filter { it.name == "$className.class" }.single() + val extractor = ClassTypeExtractorVisitor(object : ClassVisitor(Opcodes.API_VERSION) {}) + + classFile.inputStream().use { + ClassReader(it.readBytes()).accept(extractor, ClassReader.SKIP_CODE or ClassReader.SKIP_DEBUG or ClassReader.SKIP_FRAMES) + return Pair(extractor.getAbiTypes(), extractor.getPrivateTypes()) + } + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/incremental/ClasspathAnalyzerTest.kt b/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/incremental/ClasspathAnalyzerTest.kt new file mode 100644 index 0000000000000..3609c7420fa48 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/incremental/ClasspathAnalyzerTest.kt @@ -0,0 +1,106 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.internal.kapt.incremental + +import org.jetbrains.org.objectweb.asm.ClassWriter +import org.jetbrains.org.objectweb.asm.Opcodes +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + +class ClasspathAnalyzerTest { + @Rule + @JvmField + var tmp = TemporaryFolder() + + @Test + fun testDirectory() { + val classesDir = tmp.newFolder().also { dir -> + dir.resolve("test").mkdirs() + dir.resolve("test/A.class").writeBytes(emptyClass("test/A")) + dir.resolve("test/B.class").writeBytes(emptyClass("test/B")) + dir.resolve("ignore.txt").writeBytes(emptyClass("test/B")) + dir.resolve("META-INF/versions/9/A.class").also { + it.parentFile.mkdirs() + it.writeBytes(emptyClass("A")) + } + } + val transform = StructureArtifactTransform().also { it.outputDirectory = tmp.newFolder() } + val outputs = transform.transform(classesDir) + + val lazyData = LazyClasspathEntryData.LazyClasspathEntrySerializer.loadFromFile(outputs.single()) + assertEquals(classesDir, lazyData.classpathEntry) + + val data = lazyData.getClasspathEntryData() + assertEquals(setOf("test/A", "test/B"), data.classAbiHash.keys) + assertEquals(setOf("test/A", "test/B"), data.classDependencies.keys) + assertEquals(emptySet(), data.classDependencies["test/A"]!!.abiTypes) + assertEquals(emptySet(), data.classDependencies["test/A"]!!.privateTypes) + + assertEquals(emptySet(), data.classDependencies["test/B"]!!.abiTypes) + assertEquals(emptySet(), data.classDependencies["test/B"]!!.privateTypes) + } + + @Test + fun testJar() { + val inputJar = tmp.newFile("input.jar").also { jar -> + ZipOutputStream(jar.outputStream()).use { + it.putNextEntry(ZipEntry("test/A.class")) + it.write(emptyClass("test/A")) + it.closeEntry() + + it.putNextEntry(ZipEntry("test/B.class")) + it.write(emptyClass("test/B")) + it.closeEntry() + + it.putNextEntry(ZipEntry("ignored.txt")) + it.closeEntry() + + it.putNextEntry(ZipEntry("META-INF/versions/9/test/A.class")) + it.write(emptyClass("test/A")) + it.closeEntry() + } + } + val transform = StructureArtifactTransform().also { it.outputDirectory = tmp.newFolder() } + val outputs = transform.transform(inputJar) + + val lazyData = LazyClasspathEntryData.LazyClasspathEntrySerializer.loadFromFile(outputs.single()) + assertEquals(inputJar, lazyData.classpathEntry) + + val data = lazyData.getClasspathEntryData() + assertEquals(setOf("test/A", "test/B"), data.classAbiHash.keys) + assertEquals(setOf("test/A", "test/B"), data.classDependencies.keys) + assertEquals(emptySet(), data.classDependencies["test/A"]!!.abiTypes) + assertEquals(emptySet(), data.classDependencies["test/A"]!!.privateTypes) + + assertEquals(emptySet(), data.classDependencies["test/B"]!!.abiTypes) + assertEquals(emptySet(), data.classDependencies["test/B"]!!.privateTypes) + } + + @Test + fun emptyInput() { + val inputDir = tmp.newFolder("input") + val transform = StructureArtifactTransform().also { it.outputDirectory = tmp.newFolder() } + val outputs = transform.transform(inputDir) + + val lazyData = LazyClasspathEntryData.LazyClasspathEntrySerializer.loadFromFile(outputs.single()) + assertEquals(inputDir, lazyData.classpathEntry) + + val data = lazyData.getClasspathEntryData() + assertTrue(data.classAbiHash.isEmpty()) + assertTrue(data.classDependencies.isEmpty()) + } + + private fun emptyClass(internalName: String): ByteArray { + val writer = ClassWriter(Opcodes.API_VERSION) + writer.visit(Opcodes.V1_8, Opcodes.ACC_PUBLIC, internalName, null, "java/lang/Object", emptyArray()) + return writer.toByteArray() + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/incremental/ClasspathSnapshotTest.kt b/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/incremental/ClasspathSnapshotTest.kt new file mode 100644 index 0000000000000..0bdc81cb57809 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/incremental/ClasspathSnapshotTest.kt @@ -0,0 +1,120 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.internal.kapt.incremental + +import org.junit.Assert.assertEquals +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import java.io.File + +class ClasspathSnapshotTest { + @Rule + @JvmField + var tmp = TemporaryFolder() + + @Test + fun testSerialization() { + val (firstJar, lazyData) = generateLazyData( + ClassData("first/A"), ClassData("first/B") + ) + + val snapshotDir = tmp.newFolder() + val currentSnapshot = ClasspathSnapshot.ClasspathSnapshotFactory.createCurrent(snapshotDir, listOf(firstJar), setOf(lazyData)) + assertEquals(KaptClasspathChanges.Unknown, currentSnapshot.diff(UnknownSnapshot, setOf(firstJar))) + currentSnapshot.writeToCache() + + val loadedSnapshot = ClasspathSnapshot.ClasspathSnapshotFactory.loadFrom(snapshotDir) + + val diff = loadedSnapshot.diff(currentSnapshot, setOf(firstJar)) as KaptClasspathChanges.Known + assertEquals(emptySet(), diff.names) + } + + @Test + fun testIncompatibleClasspaths() { + val firstSnapshot = ClasspathSnapshot.ClasspathSnapshotFactory.createCurrent(File(""), listOf(File("1.jar")), emptySet()) + val secondSnapshot = + ClasspathSnapshot.ClasspathSnapshotFactory.createCurrent(File(""), listOf(File("1.jar"), File("added.jar")), emptySet()) + assertEquals(KaptClasspathChanges.Unknown, firstSnapshot.diff(secondSnapshot, setOf(File("added.jar")))) + } + + @Test + fun testChangedClassesFound() { + val (firstJar, firstLazyData) = generateLazyData( + ClassData("first/A"), + ClassData("first/B").also { it.withAbiDependencies("first/A") } + ) + val firstSnapshot = ClasspathSnapshot.ClasspathSnapshotFactory.createCurrent(File(""), listOf(firstJar), setOf(firstLazyData)) + firstSnapshot.diff(UnknownSnapshot, setOf(firstJar)) + + val (_, changedLazyData) = generateLazyData( + ClassData("first/A", ByteArray(1)), + ClassData("first/B").also { it.withAbiDependencies("first/A") }, + jarInput = firstJar + ) + val changedSnapshot = ClasspathSnapshot.ClasspathSnapshotFactory.createCurrent(File(""), listOf(firstJar), setOf(changedLazyData)) + + val diff = changedSnapshot.diff(firstSnapshot, setOf(firstJar)) as KaptClasspathChanges.Known + assertEquals(setOf("first/A", "first/B"), diff.names) + } + + @Test + fun testChangedClassesAcrossEntries() { + val (firstJar, firstLazyData) = generateLazyData( + ClassData("first/A").also { it.withAbiDependencies("library/C") }, + ClassData("first/B").also { it.withAbiDependencies("first/A") } + ) + + val (libraryJar, libraryLazyData) = generateLazyData(ClassData("library/C")) + + val cacheDir = tmp.newFolder() + val firstSnapshot = + ClasspathSnapshot.ClasspathSnapshotFactory.createCurrent( + cacheDir, + listOf(firstJar, libraryJar), + setOf(firstLazyData, libraryLazyData) + ) + firstSnapshot.diff(UnknownSnapshot, setOf(firstJar, libraryJar)) + firstSnapshot.writeToCache() + + val (_, changedLazyData) = generateLazyData(ClassData("library/C", ByteArray(1)), jarInput = libraryJar) + val changedSnapshot = + ClasspathSnapshot.ClasspathSnapshotFactory.createCurrent( + cacheDir, + listOf(firstJar, libraryJar), + setOf(firstLazyData, changedLazyData) + ) + + val diff = changedSnapshot.diff(firstSnapshot, setOf(libraryJar)) as KaptClasspathChanges.Known + assertEquals(setOf("library/C", "first/A", "first/B"), diff.names) + } + + private fun generateLazyData( + vararg classData: ClassData, + jarInput: File = tmp.newFile() + ): Pair { + val data = ClasspathEntryData() + classData.forEach { + data.classAbiHash[it.internalName] = it.hash + data.classDependencies[it.internalName] = ClassDependencies(it.abiDeps, it.privateDeps) + } + val serialized = tmp.newFile().also { data.saveTo(it) } + val lazyData = tmp.newFile().also { LazyClasspathEntryData(jarInput, serialized).saveToFile(it) } + + return Pair(jarInput, lazyData) + } + + private class ClassData( + val internalName: String, + val hash: ByteArray = ByteArray(0) + ) { + val abiDeps = mutableSetOf() + val privateDeps = mutableSetOf() + fun withAbiDependencies(vararg names: String) { + abiDeps.addAll(names) + } + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/util/bytecodeUtils.kt b/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/util/bytecodeUtils.kt index 126873b1d2d75..6766e59d5fb31 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/util/bytecodeUtils.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/util/bytecodeUtils.kt @@ -5,6 +5,7 @@ import org.jetbrains.org.objectweb.asm.util.TraceClassVisitor import java.io.File import java.io.PrintWriter import java.io.StringWriter +import javax.tools.ToolProvider fun classFileBytecodeString(classFile: File): String { val out = StringWriter() @@ -29,4 +30,16 @@ fun checkBytecodeNotContains(classFile: File, strings: Iterable) { for (string in strings) { assert(!bytecode.contains(string)) { "Bytecode should NOT contain '$string':\n$bytecode" } } +} + +fun compileSources(sources: Collection, outputDir: File) { + val compiler = ToolProvider.getSystemJavaCompiler() + compiler.getStandardFileManager(null, null, null).use { fileManager -> + val compilationTask = + compiler.getTask( + null, fileManager, null, listOf("-d", outputDir.absolutePath), null, fileManager.getJavaFileObjectsFromFiles(sources) + ) + + compilationTask.call() + } } \ No newline at end of file diff --git a/plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/KaptOptions.kt b/plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/KaptOptions.kt index d7427ce36b75a..c93116d03d15f 100644 --- a/plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/KaptOptions.kt +++ b/plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/KaptOptions.kt @@ -171,6 +171,6 @@ fun KaptOptions.logString(additionalInfo: String = "") = buildString { appendln("[incremental apt] Changed files: $changedFiles") appendln("[incremental apt] Compiled sources directories: ${compiledSources.joinToString()}") appendln("[incremental apt] Cache directory for incremental compilation: $incrementalCache") - appendln("[incremental apt] Changes classpath names: ${classpathChanges.joinToString()}") + appendln("[incremental apt] Changed classpath names: ${classpathChanges.joinToString()}") appendln("[incremental apt] If processing incrementally: $processIncrementally") } \ No newline at end of file