Skip to content

Commit

Permalink
Incremental KAPT - analyze classpath changes
Browse files Browse the repository at this point in the history 
Use artifact transforms to capture structure and
dependencies of classpath entries. In the KAPT task
this information is used to compare previous classpath
structure with the current one. Once changed classes are
detected, all classes that transitively depend on those
are identified, and that set is passed to KAPT invocation.

In order to avoid unrelated classpath changes, we record an
ABI snapshot of the classpath entry. This snapshot ignores
all private members, and @metadata annotation.

 #KT-23880
  • Loading branch information
@gavra0 @ilmirus
gavra0 authored and ilmirus committed Apr 23, 2019
1 parent dc08b1d commit 0df06f9
Show file tree
Hide file tree
Showing 17 changed files with 1,546 additions and 30 deletions.
65 changes: 65 additions & 0 deletions ...on-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KaptIncrementalWithAggregatingApt.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,4 +71,69 @@ class KaptIncrementalWithAggregatingApt : KaptIncrementalIT() {
)
}
}

@Test
fun testClasspathChanges() {
val project = Project(
"incrementalMultiproject",
GradleVersionRequired.None
).apply {
setupWorkingDir()
val processorPath = generateProcessor("AGGREGATING")

projectDir.resolve("app/build.gradle").appendText(
"""
apply plugin: "kotlin-kapt"
dependencies {
implementation "org.jetbrains.kotlin:kotlin-stdlib:${'$'}kotlin_version"
kapt files("${processorPath.invariantSeparatorsPath}")
}
""".trimIndent()
)

projectDir.resolve("lib/build.gradle").appendText(
"""
dependencies {
implementation "org.jetbrains.kotlin:kotlin-stdlib:${'$'}kotlin_version"
}
""".trimIndent()
)
}

project.build("clean", ":app:build") {
assertSuccessful()
}

project.projectFile("A.kt").modify { current ->
val lastBrace = current.lastIndexOf("}")
current.substring(0, lastBrace) + "fun anotherFun() {}\n }"
}
project.build("build") {
assertSuccessful()

assertEquals(
setOf(
fileInWorkingDir("app/build/tmp/kapt3/stubs/main/foo/AA.java").absolutePath,
fileInWorkingDir("app/build/tmp/kapt3/stubs/main/foo/AAA.java").absolutePath,
fileInWorkingDir("app/build/tmp/kapt3/stubs/main/foo/BB.java").absolutePath,
fileInWorkingDir("app/build/tmp/kapt3/stubs/main/foo/FooUseAKt.java").absolutePath,
fileInWorkingDir("app/build/tmp/kapt3/stubs/main/foo/FooUseBKt.java").absolutePath,
fileInWorkingDir("app/build/tmp/kapt3/stubs/main/foo/FooUseAAKt.java").absolutePath,
fileInWorkingDir("app/build/tmp/kapt3/stubs/main/foo/FooUseBBKt.java").absolutePath

), getProcessedSources(output)
)
}

project.projectFile("A.kt").modify { current ->
val lastBrace = current.lastIndexOf("}")
current.substring(0, lastBrace) + "private fun privateFunction() {}\n }"
}
project.build("build") {
assertSuccessful()
assertTrue(getProcessedSources(output).isEmpty())
}
}
}
25 changes: 15 additions & 10 deletions ...tion-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KaptIncrementalWithIsolatingApt.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import org.jetbrains.kotlin.gradle.util.modify
import org.junit.Assert.assertEquals
import org.junit.Assert.assertTrue
import org.junit.Test
import java.io.File
import java.util.zip.ZipEntry
import java.util.zip.ZipOutputStream

Expand Down Expand Up @@ -70,13 +71,22 @@ private const val patternApt = "Processing java sources with annotation processo
fun getProcessedSources(output: String): Set<String> {
val logging = output.lines().single { it.contains(patternApt) }
val indexOf = logging.indexOf(patternApt) + patternApt.length
return logging.drop(indexOf).split(",").map { it.trim() }.toSet()
return logging.drop(indexOf).split(",").map { it.trim() }.filter { !it.isEmpty() }.toSet()
}

fun BaseGradleIT.Project.setupIncrementalAptProject(procType: String) {
fun BaseGradleIT.Project.setupIncrementalAptProject(procType: String, buildFile: File = projectDir.resolve("build.gradle")) {
setupWorkingDir()
val buildFile = projectDir.resolve("build.gradle")
val content = buildFile.readText()
val processorPath = generateProcessor(procType)

val updatedContent = content.replace(
Regex("^\\s*kapt\\s\"org\\.jetbrain.*$", RegexOption.MULTILINE),
" kapt files(\"${processorPath.invariantSeparatorsPath}\")"
)
buildFile.writeText(updatedContent)
}

fun BaseGradleIT.Project.generateProcessor(procType: String): File {
val processorPath = projectDir.resolve("incrementalProcessor.jar")

ZipOutputStream(processorPath.outputStream()).use {
Expand All @@ -92,10 +102,5 @@ fun BaseGradleIT.Project.setupIncrementalAptProject(procType: String) {
it.write(IncrementalProcessor::class.java.name.toByteArray())
it.closeEntry()
}

val updatedContent = content.replace(
Regex("^\\s*kapt\\s\"org\\.jetbrain.*$", RegexOption.MULTILINE),
" kapt files(\"${processorPath.invariantSeparatorsPath}\")"
)
buildFile.writeText(updatedContent)
}
return processorPath
}
17 changes: 16 additions & 1 deletion libraries/tools/kotlin-gradle-plugin/build.gradle.kts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ publish()

// todo: make lazy
val jar: Jar by tasks
runtimeJar(rewriteDepsToShadedCompiler(jar))
val jarContents by configurations.creating

sourcesJar()
javadocJar()
Expand Down Expand Up @@ -61,6 +61,10 @@ dependencies {
runtime(projectRuntimeJar(":kotlin-scripting-compiler-embeddable"))
runtime(project(":kotlin-reflect"))

jarContents(compileOnly(intellijDep()) {
includeJars("asm-all", rootProject = rootProject)
})

// com.android.tools.build:gradle has ~50 unneeded transitive dependencies
compileOnly("com.android.tools.build:gradle:3.0.0") { isTransitive = false }
compileOnly("com.android.tools.build:gradle-core:3.0.0") { isTransitive = false }
Expand All @@ -77,6 +81,17 @@ dependencies {
testCompileOnly(project(":kotlin-annotation-processing-gradle"))
}

runtimeJar(rewriteDepsToShadedCompiler(jar)) {
dependsOn(jarContents)

from {
jarContents.asFileTree.map {
if (it.endsWith(".jar")) zipTree(it)
else it
}
}
}

tasks {
withType<KotlinCompile> {
kotlinOptions.jdkHome = rootProject.extra["JDK_18"] as String
Expand Down
32 changes: 31 additions & 1 deletion ...n/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/Kapt3KotlinGradleSubplugin.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.artifacts.Configuration
import org.gradle.api.artifacts.Dependency
import org.gradle.api.attributes.Attribute
import org.gradle.api.tasks.SourceSet
import org.gradle.api.tasks.TaskDependency
import org.gradle.api.tasks.compile.AbstractCompile
Expand All @@ -23,6 +24,8 @@ import org.jetbrains.kotlin.gradle.internal.Kapt3GradleSubplugin.Companion.getKa
import org.jetbrains.kotlin.gradle.internal.Kapt3GradleSubplugin.Companion.getKaptGeneratedKotlinSourcesDir
import org.jetbrains.kotlin.gradle.internal.Kapt3GradleSubplugin.Companion.getKaptGeneratedSourcesDir
import org.jetbrains.kotlin.gradle.internal.Kapt3KotlinGradleSubplugin.Companion.KAPT_WORKER_DEPENDENCIES_CONFIGURATION_NAME
import org.jetbrains.kotlin.gradle.internal.kapt.incremental.CLASS_STRUCTURE_ARTIFACT_TYPE
import org.jetbrains.kotlin.gradle.internal.kapt.incremental.StructureArtifactTransform
import org.jetbrains.kotlin.gradle.model.builder.KaptModelBuilder
import org.jetbrains.kotlin.gradle.plugin.*
import org.jetbrains.kotlin.gradle.tasks.CompilerPluginOptions
Expand Down Expand Up @@ -374,7 +377,8 @@ class Kapt3KotlinGradleSubplugin : KotlinGradleSubplugin<KotlinCompile> {

private fun Kapt3SubpluginContext.createKaptKotlinTask(useWorkerApi: Boolean): KaptTask {
val taskClass = if (useWorkerApi) KaptWithoutKotlincTask::class.java else KaptWithKotlincTask::class.java
val kaptTask = project.tasks.create(getKaptTaskName("kapt"), taskClass)
val taskName = getKaptTaskName("kapt")
val kaptTask = project.tasks.create(taskName, taskClass)

kaptTask.useBuildCache = kaptExtension.useBuildCache

Expand All @@ -389,6 +393,13 @@ class Kapt3KotlinGradleSubplugin : KotlinGradleSubplugin<KotlinCompile> {
kaptTask.isIncremental = project.isIncrementalKapt()
if (kaptTask.isIncremental) {
kaptTask.incAptCache = getKaptIncrementalAnnotationProcessingCache()

maybeRegisterTransform(project)
val classStructure = project.configurations.create("_classStructure${taskName}")
project.dependencies.add(classStructure.name, kotlinCompile.classpath)
kaptTask.classpathStructure = classStructure.incoming.artifactView { viewConfig ->
viewConfig.attributes.attribute(artifactType, CLASS_STRUCTURE_ARTIFACT_TYPE)
}.files
}

kotlinCompilation?.run {
Expand Down Expand Up @@ -437,6 +448,23 @@ class Kapt3KotlinGradleSubplugin : KotlinGradleSubplugin<KotlinCompile> {
return kaptTask
}

private fun maybeRegisterTransform(project: Project) {
if (!project.extensions.extraProperties.has("KaptStructureTransformAdded")) {
project.dependencies.registerTransform { variantTransform ->
variantTransform.artifactTransform(StructureArtifactTransform::class.java)
variantTransform.from.attribute(artifactType, "jar")
variantTransform.to.attribute(artifactType, CLASS_STRUCTURE_ARTIFACT_TYPE)
}
project.dependencies.registerTransform { variantTransform ->
variantTransform.artifactTransform(StructureArtifactTransform::class.java)
variantTransform.from.attribute(artifactType, "directory")
variantTransform.to.attribute(artifactType, CLASS_STRUCTURE_ARTIFACT_TYPE)
}

project.extensions.extraProperties["KaptStructureTransformAdded"] = true
}
}

private fun Kapt3SubpluginContext.createKaptGenerateStubsTask(): KaptGenerateStubsTask {
val kaptTask = project.tasks.create(
getKaptTaskName("kaptGenerateStubs"),
Expand Down Expand Up @@ -495,6 +523,8 @@ class Kapt3KotlinGradleSubplugin : KotlinGradleSubplugin<KotlinCompile> {
override fun getPluginArtifact(): SubpluginArtifact =
JetBrainsSubpluginArtifact(artifactId = KAPT_ARTIFACT_NAME)
}
private val artifactType = Attribute.of("artifactType", String::class.java)


internal fun registerGeneratedJavaSource(kaptTask: KaptTask, javaTask: AbstractCompile) {
javaTask.source(kaptTask.destinationDir)
Expand Down
70 changes: 62 additions & 8 deletions ...otlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptTask.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@ import org.gradle.api.file.FileCollection
import org.gradle.api.internal.ConventionTask
import org.gradle.api.tasks.*
import org.gradle.api.tasks.incremental.IncrementalTaskInputs
import org.jetbrains.kotlin.gradle.internal.kapt.incremental.KaptClasspathChanges
import org.jetbrains.kotlin.gradle.internal.kapt.incremental.ClasspathSnapshot
import org.jetbrains.kotlin.gradle.internal.kapt.incremental.KaptIncrementalChanges
import org.jetbrains.kotlin.gradle.internal.kapt.incremental.UnknownSnapshot
import org.jetbrains.kotlin.gradle.internal.tasks.TaskWithLocalState
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
import org.jetbrains.kotlin.gradle.tasks.cacheOnlyIfEnabledForKotlin
Expand Down Expand Up @@ -47,6 +51,12 @@ abstract class KaptTask : ConventionTask(), TaskWithLocalState {
@get:Internal
internal lateinit var kaptClasspathConfigurations: List<Configuration>


@get:PathSensitive(PathSensitivity.NONE)
@get:Optional
@get:InputFiles
internal var classpathStructure: FileCollection? = null

/** Output directory that contains caches necessary to support incremental annotation processing. */
@get:OutputDirectory
@get:Optional
Expand Down Expand Up @@ -156,23 +166,67 @@ abstract class KaptTask : ConventionTask(), TaskWithLocalState {
@Internal
protected fun getCompiledSources() = listOfNotNull(kotlinCompileTask.destinationDir, kotlinCompileTask.javaOutputDir)

protected fun getChangedFiles(inputs: IncrementalTaskInputs): List<File> {
if (!isIncremental || !inputs.isIncremental || !getCompiledSources().all { it.exists() }) {
protected fun getIncrementalChanges(inputs: IncrementalTaskInputs): KaptIncrementalChanges {
val changedFiles = getChangedFiles(inputs)

return if (isIncremental) {
val classpathChanges = if (changedFiles.isEmpty()) {
classpath.files
} else {
classpath.files.let { cp ->
changedFiles.filter { cp.contains(it) }
}
}
val classpathStatus = findClasspathChanges(classpathChanges)
when (classpathStatus) {
is KaptClasspathChanges.Unknown -> KaptIncrementalChanges.Unknown
is KaptClasspathChanges.Known -> KaptIncrementalChanges.Known(
changedFiles.filter { it.extension == "java" }.toSet(), classpathStatus.names
)
}
} else {
KaptIncrementalChanges.Unknown
}
}

private fun getChangedFiles(inputs: IncrementalTaskInputs): List<File> {
return if (!isIncremental || !inputs.isIncremental) {
clearLocalState()
return emptyList()
emptyList()
} else {
val changes = with(mutableSetOf<File>()) {
with(mutableSetOf<File>()) {
inputs.outOfDate { this.add(it.file) }
inputs.removed { this.add(it.file) }
return@with this.toList()
}
}
}

return if (changes.all { it.extension == "java" }) {
changes
} else {
emptyList()
private fun findClasspathChanges(changedClasspath: Iterable<File>): KaptClasspathChanges {
incAptCache!!.mkdirs()

val startTime = System.currentTimeMillis()

val previousSnapshot = ClasspathSnapshot.ClasspathSnapshotFactory.loadFrom(incAptCache!!)
val currentSnapshot = ClasspathSnapshot.ClasspathSnapshotFactory.createCurrent(incAptCache!!, classpath.files.toList(), classpathStructure!!.files)

val classpathChanges = currentSnapshot.diff(previousSnapshot, changedClasspath.toSet())
currentSnapshot.writeToCache()

if (logger.isInfoEnabled) {
val time = "Took ${System.currentTimeMillis() - startTime}ms."
when {
previousSnapshot == UnknownSnapshot ->
logger.info("Initializing classpath information for KAPT. $time")
classpathChanges == KaptClasspathChanges.Unknown ->
logger.info("Unable to use existing data, re-initializing classpath information for KAPT. $time")
else -> {
classpathChanges as KaptClasspathChanges.Known
logger.info("Full list of impacted classpath names: ${classpathChanges.names}. $time")
}
}
}
return classpathChanges
}

private fun hasAnnotationProcessors(file: File): Boolean {
Expand Down
11 changes: 6 additions & 5 deletions ...e-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptWithKotlincTask.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments
import org.jetbrains.kotlin.compilerRunner.GradleCompilerEnvironment
import org.jetbrains.kotlin.compilerRunner.GradleCompilerRunner
import org.jetbrains.kotlin.compilerRunner.OutputItemsCollectorImpl
import org.jetbrains.kotlin.gradle.internal.kapt.incremental.KaptIncrementalChanges
import org.jetbrains.kotlin.gradle.internal.tasks.allOutputFiles
import org.jetbrains.kotlin.gradle.logging.GradleKotlinLogger
import org.jetbrains.kotlin.gradle.logging.GradlePrintingMessageCollector
Expand Down Expand Up @@ -73,11 +74,11 @@ open class KaptWithKotlincTask : KaptTask(), CompilerArgumentAwareWithInput<K2JV
logger.debug("Running kapt annotation processing using the Kotlin compiler")
checkAnnotationProcessorClasspath()

val incrementalChanges = getChangedFiles(inputs)
when {
incrementalChanges.isNotEmpty() -> {
changedFiles = incrementalChanges
classpathChanges = emptyList()
val incrementalChanges = getIncrementalChanges(inputs)
when (incrementalChanges) {
is KaptIncrementalChanges.Known -> {
changedFiles = incrementalChanges.changedSources.toList()
classpathChanges = incrementalChanges.changedClasspathJvmNames.toList()
processIncrementally = true
}
else -> {
Expand Down
13 changes: 9 additions & 4 deletions ...lugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptWithoutKotlincTask.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import org.gradle.api.tasks.incremental.IncrementalTaskInputs
import org.gradle.workers.IsolationMode
import org.gradle.workers.WorkerExecutor
import org.jetbrains.kotlin.gradle.internal.Kapt3KotlinGradleSubplugin.Companion.KAPT_WORKER_DEPENDENCIES_CONFIGURATION_NAME
import org.jetbrains.kotlin.gradle.internal.kapt.incremental.KaptIncrementalChanges
import org.jetbrains.kotlin.gradle.plugin.KotlinAndroidPluginWrapper
import org.jetbrains.kotlin.gradle.tasks.findKotlinStdlibClasspath
import org.jetbrains.kotlin.gradle.tasks.findToolsJar
Expand Down Expand Up @@ -49,7 +50,11 @@ open class KaptWithoutKotlincTask @Inject constructor(private val workerExecutor
logger.info("Running kapt annotation processing using the Gradle Worker API")
checkAnnotationProcessorClasspath()

val incrementalChanges = getChangedFiles(inputs)
val incrementalChanges = getIncrementalChanges(inputs)
val (changedFiles, classpathChanges) = when (incrementalChanges) {
is KaptIncrementalChanges.Unknown -> Pair(emptyList<File>(), emptyList<String>())
is KaptIncrementalChanges.Known -> Pair(incrementalChanges.changedSources.toList(), incrementalChanges.changedClasspathJvmNames)
}

val compileClasspath = classpath.files.toMutableList()
if (project.plugins.none { it is KotlinAndroidPluginWrapper }) {
Expand All @@ -67,11 +72,11 @@ open class KaptWithoutKotlincTask @Inject constructor(private val workerExecutor
compileClasspath,
javaSourceRoots.toList(),

incrementalChanges,
changedFiles,
getCompiledSources(),
incAptCache,
emptyList(),
incrementalChanges.isNotEmpty(),
classpathChanges.toList(),
incrementalChanges is KaptIncrementalChanges.Known,

destinationDir,
classesDir,
Expand Down
Loading

0 comments on commit 0df06f9

Please sign in to comment.