From 19fc16c9586b3bbce9c1812d6755bf47cfbb6df4 Mon Sep 17 00:00:00 2001 From: Maksim Pelevin Date: Mon, 17 Jul 2023 14:54:35 +0300 Subject: [PATCH] Fix JaCoCo for Contest Estimator (#2373) --- .../kotlin/org/utbot/framework/UtSettings.kt | 5 + .../org/utbot/framework/plugin/api/Api.kt | 6 +- .../org/utbot/fuzzer/FuzzerFunctions.kt | 28 +---- .../src/main/kotlin/org/utbot/fuzzing/Api.kt | 20 +++- .../kotlin/org/utbot/fuzzing/Configuration.kt | 5 + .../kotlin/org/utbot/fuzzing/Providers.kt | 61 ++++++---- .../kotlin/org/utbot/fuzzing/ProvidersTest.kt | 21 ++++ .../kotlin/org/utbot/fuzzing/JavaLanguage.kt | 5 +- .../org/utbot/fuzzing/providers/Objects.kt | 108 +++++++++++++----- utbot-junit-contest/build.gradle | 47 ++++++-- .../main/kotlin/org/utbot/contest/Contest.kt | 6 +- 11 files changed, 211 insertions(+), 101 deletions(-) diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt index b6ea38e511..ea7353a2b0 100644 --- a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt +++ b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt @@ -236,6 +236,11 @@ object UtSettings : AbstractSettings(logger, defaultKeyForSettingsPath, defaultS */ var fuzzingTimeoutInMillis: Long by getLongProperty(3_000L, 0, Long.MAX_VALUE) + /** + * Find implementations of interfaces and abstract classes to fuzz. + */ + var fuzzingImplementationOfAbstractClasses: Boolean by getBooleanProperty(true) + /** * Generate tests that treat possible overflows in arithmetic operations as errors * that throw Arithmetic Exception. diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt index 99d71bc9da..80f0f070d2 100644 --- a/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt +++ b/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt @@ -881,6 +881,8 @@ val Type.classId: ClassId else -> error("Unknown type $this") } +private val logger = KotlinLogging.logger {} + /** * Class id. Contains name, not a full qualified name. * @@ -905,7 +907,9 @@ open class ClassId @JvmOverloads constructor( get() = jClass.modifiers open val canonicalName: String - get() = jClass.canonicalName ?: error("ClassId $name does not have canonical name") + get() = jClass.canonicalName ?: name.also { + logger.error("ClassId $name does not have canonical name") + } open val simpleName: String get() = jClass.simpleName diff --git a/utbot-framework/src/main/kotlin/org/utbot/fuzzer/FuzzerFunctions.kt b/utbot-framework/src/main/kotlin/org/utbot/fuzzer/FuzzerFunctions.kt index a6e82fb5b1..5f7fc57d9e 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/fuzzer/FuzzerFunctions.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/fuzzer/FuzzerFunctions.kt @@ -2,15 +2,7 @@ package org.utbot.fuzzer import mu.KotlinLogging import org.utbot.framework.plugin.api.classId -import org.utbot.framework.plugin.api.util.booleanClassId -import org.utbot.framework.plugin.api.util.byteClassId -import org.utbot.framework.plugin.api.util.charClassId -import org.utbot.framework.plugin.api.util.doubleClassId -import org.utbot.framework.plugin.api.util.floatClassId -import org.utbot.framework.plugin.api.util.intClassId -import org.utbot.framework.plugin.api.util.longClassId -import org.utbot.framework.plugin.api.util.shortClassId -import org.utbot.framework.plugin.api.util.stringClassId +import org.utbot.framework.plugin.api.util.* import org.utbot.framework.util.executableId import soot.BooleanType import soot.ByteType @@ -28,23 +20,7 @@ import soot.jimple.Constant import soot.jimple.IntConstant import soot.jimple.InvokeExpr import soot.jimple.NullConstant -import soot.jimple.internal.AbstractSwitchStmt -import soot.jimple.internal.ImmediateBox -import soot.jimple.internal.JAssignStmt -import soot.jimple.internal.JCastExpr -import soot.jimple.internal.JEqExpr -import soot.jimple.internal.JGeExpr -import soot.jimple.internal.JGtExpr -import soot.jimple.internal.JIfStmt -import soot.jimple.internal.JInvokeStmt -import soot.jimple.internal.JLeExpr -import soot.jimple.internal.JLookupSwitchStmt -import soot.jimple.internal.JLtExpr -import soot.jimple.internal.JNeExpr -import soot.jimple.internal.JSpecialInvokeExpr -import soot.jimple.internal.JStaticInvokeExpr -import soot.jimple.internal.JTableSwitchStmt -import soot.jimple.internal.JVirtualInvokeExpr +import soot.jimple.internal.* import soot.toolkits.graph.ExceptionalUnitGraph private val logger = KotlinLogging.logger {} diff --git a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt index 083e8be67b..6911dcb8dd 100644 --- a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt +++ b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt @@ -7,6 +7,7 @@ import org.utbot.fuzzing.seeds.KnownValue import org.utbot.fuzzing.utils.MissedSeed import org.utbot.fuzzing.utils.chooseOne import org.utbot.fuzzing.utils.flipCoin +import org.utbot.fuzzing.utils.transformIfNotEmpty import kotlin.random.Random private val logger by lazy { KotlinLogging.logger {} } @@ -288,7 +289,14 @@ private object EmptyFeedback : Feedback { class NoSeedValueException internal constructor( // this type cannot be generalized because Java forbids types for [Throwable]. val type: Any? -) : Exception("No seed candidates generated for type: $type") +) : Exception() { + override fun fillInStackTrace(): Throwable { + return this + } + + override val message: String + get() = "No seed candidates generated for type: $type" +} suspend fun , F : Feedback> Fuzzing.fuzz( description: D, @@ -515,11 +523,11 @@ private fun , FEEDBACK : Feedback< } ), modify = task.modify -// .toMutableList() -// .transformIfNotEmpty { -// shuffle(random) -// take(random.nextInt(size + 1)) -// } + .toMutableList() + .transformIfNotEmpty { + shuffle(random) + take(configuration.maxNumberOfRecursiveSeedModifications) + } .mapTo(arrayListOf()) { routine -> fuzz( routine.types, diff --git a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Configuration.kt b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Configuration.kt index 5675d70cf1..a6c19926c9 100644 --- a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Configuration.kt +++ b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Configuration.kt @@ -90,4 +90,9 @@ data class Configuration( * to generate a recursive object, but will use [Seed.Recursive.empty] instead. */ var generateEmptyRecursiveForMissedTypes: Boolean = true, + + /** + * Limits maximum number of recursive seed modifications + */ + var maxNumberOfRecursiveSeedModifications: Int = 10, ) \ No newline at end of file diff --git a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Providers.kt b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Providers.kt index 85ad2bbe5e..f64fb7d996 100644 --- a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Providers.kt +++ b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Providers.kt @@ -96,9 +96,9 @@ fun interface ValueProvider> { */ fun except(filter: (ValueProvider) -> Boolean): ValueProvider { return if (this is Combined) { - Combined(providers.filterNot(filter)) + Combined(providers.map { it.unwrapIfFallback() }.filterNot(filter)) } else { - Combined(if (filter(this)) emptyList() else listOf(this)) + Combined(if (filter(unwrapIfFallback())) emptyList() else listOf(this)) } } @@ -117,26 +117,7 @@ fun interface ValueProvider> { * Uses fallback value provider in case when 'this' one failed to generate any value. */ fun withFallback(fallback: ValueProvider) : ValueProvider { - val thisProvider = this - return object : ValueProvider { - override fun enrich(description: D, type: T, scope: Scope) { - thisProvider.enrich(description, type, scope) - // Enriching scope by fallback value provider in this point is not quite right, - // but it doesn't look as a problem right now. - fallback.enrich(description, type, scope) - } - - override fun generate(description: D, type: T): Sequence> { - val default = if (thisProvider.accept(type)) thisProvider.generate(description, type) else emptySequence() - return if (default.iterator().hasNext()) { - default - } else if (fallback.accept(type)) { - fallback.generate(description, type) - } else { - emptySequence() - } - } - } + return Fallback(this, fallback) } /** @@ -152,6 +133,42 @@ fun interface ValueProvider> { return if (flag) block(this) else this } + /** + * Checks if current provider has fallback and return the initial provider. + * + * If the initial provider is also fallback, then it is unwrapped too. + * + * @return unwrapped provider or this if it is not fallback + */ + fun unwrapIfFallback(): ValueProvider { + return if (this is Fallback) { provider.unwrapIfFallback() } else { this } + } + + private class Fallback>( + val provider: ValueProvider, + val fallback: ValueProvider, + ): ValueProvider { + + override fun enrich(description: D, type: T, scope: Scope) { + provider.enrich(description, type, scope) + // Enriching scope by fallback value provider in this point is not quite right, + // but it doesn't look as a problem right now. + fallback.enrich(description, type, scope) + } + + override fun generate(description: D, type: T): Sequence> { + val default = if (provider.accept(type)) provider.generate(description, type) else emptySequence() + return if (default.iterator().hasNext()) { + default + } else if (fallback.accept(type)) { + fallback.generate(description, type) + } else { + emptySequence() + } + } + + } + /** * Wrapper class that delegates implementation to the [providers]. */ diff --git a/utbot-fuzzing/src/test/kotlin/org/utbot/fuzzing/ProvidersTest.kt b/utbot-fuzzing/src/test/kotlin/org/utbot/fuzzing/ProvidersTest.kt index 9d306b9ad7..74b2d8c204 100644 --- a/utbot-fuzzing/src/test/kotlin/org/utbot/fuzzing/ProvidersTest.kt +++ b/utbot-fuzzing/src/test/kotlin/org/utbot/fuzzing/ProvidersTest.kt @@ -111,6 +111,27 @@ class ProvidersTest { Assertions.assertEquals(3, (seq.drop(1).first() as Seed.Simple).value) } + @Test + fun `test fallback unwrapping from providers`() { + val provider1 = p { 2 } + val provider2 = p { 3 } + val fallback = p { 4 } + val providers1 = ValueProvider.of(listOf( + provider1.withFallback(fallback), + provider2 + )) + val seq1 = providers1.generate(description, Unit).toSet() + Assertions.assertEquals(2, seq1.count()) + Assertions.assertEquals(2, (seq1.first() as Seed.Simple).value) + Assertions.assertEquals(3, (seq1.drop(1).first() as Seed.Simple).value) + + val providers2 = providers1.except(provider1) + + val seq2 = providers2.generate(description, Unit).toSet() + Assertions.assertEquals(1, seq2.count()) + Assertions.assertEquals(3, (seq2.first() as Seed.Simple).value) + } + @Test fun `provider is not called when accept-method returns false`() { val seq = ValueProvider.of(listOf( diff --git a/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/JavaLanguage.kt b/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/JavaLanguage.kt index 78332a0f8b..b55e567c68 100644 --- a/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/JavaLanguage.kt +++ b/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/JavaLanguage.kt @@ -1,6 +1,7 @@ package org.utbot.fuzzing import mu.KotlinLogging +import org.utbot.framework.UtSettings import org.utbot.framework.plugin.api.ClassId import org.utbot.framework.plugin.api.ExecutableId import org.utbot.framework.plugin.api.Instruction @@ -42,7 +43,9 @@ fun defaultValueProviders(idGenerator: IdentityPreservingIdGenerator) = lis FloatValueProvider, StringValueProvider, NumberValueProvider, - ObjectValueProvider(idGenerator), + ObjectValueProvider(idGenerator).letIf(UtSettings.fuzzingImplementationOfAbstractClasses) { ovp -> + ovp.withFallback(AbstractsObjectValueProvider(idGenerator)) + }, ArrayValueProvider(idGenerator), EnumValueProvider(idGenerator), ListSetValueProvider(idGenerator), diff --git a/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/providers/Objects.kt b/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/providers/Objects.kt index b34b92473a..68ab56e678 100644 --- a/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/providers/Objects.kt +++ b/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/providers/Objects.kt @@ -1,28 +1,39 @@ package org.utbot.fuzzing.providers +import mu.KotlinLogging import org.utbot.framework.plugin.api.* import org.utbot.framework.plugin.api.util.* -import org.utbot.fuzzer.FuzzedType -import org.utbot.fuzzer.FuzzedValue -import org.utbot.fuzzer.IdGenerator -import org.utbot.fuzzer.fuzzed +import org.utbot.fuzzer.* import org.utbot.fuzzing.* import org.utbot.fuzzing.utils.hex +import soot.Scene +import soot.SootClass import java.lang.reflect.Field import java.lang.reflect.Member import java.lang.reflect.Method import java.lang.reflect.Modifier +private val logger = KotlinLogging.logger {} + +private fun isKnownTypes(type: ClassId): Boolean { + return type == stringClassId + || type == dateClassId + || type == NumberValueProvider.classId + || type.isCollectionOrMap + || type.isPrimitiveWrapper + || type.isEnum +} + +private fun isIgnored(type: ClassId): Boolean { + return isKnownTypes(type) + || type.isAbstract + || (type.isInner && !type.isStatic) +} + class ObjectValueProvider( val idGenerator: IdGenerator, ) : ValueProvider { - private val unwantedConstructorsClasses = listOf( - stringClassId, - dateClassId, - NumberValueProvider.classId - ) - override fun accept(type: FuzzedType) = !isIgnored(type.classId) override fun generate( @@ -30,10 +41,8 @@ class ObjectValueProvider( type: FuzzedType ) = sequence { val classId = type.classId - val constructors = findTypesOfNonRecursiveConstructor(type, description.description.packageName) - .takeIf { it.isNotEmpty() } - ?.asSequence() - ?: classId.allConstructors.filter { + val constructors = classId.allConstructors + .filter { isAccessible(it.constructor, description.description.packageName) } constructors.forEach { constructorId -> @@ -89,23 +98,6 @@ class ObjectValueProvider( empty = nullRoutine(classId) ) } - - private fun isIgnored(type: ClassId): Boolean { - return unwantedConstructorsClasses.contains(type) - || type.isCollectionOrMap - || type.isPrimitiveWrapper - || type.isEnum - || type.isAbstract - || (type.isInner && !type.isStatic) - } - - private fun findTypesOfNonRecursiveConstructor(type: FuzzedType, packageName: String?): List { - return type.classId.allConstructors - .filter { isAccessible(it.constructor, packageName) } - .filter { c -> - c.parameters.all { it.isPrimitive || it == stringClassId || it.isArray } - }.toList() - } } @Suppress("unused") @@ -134,6 +126,60 @@ object NullValueProvider : ValueProvider, +) : ValueProvider { + + override fun accept(type: FuzzedType) = type.classId.isRefType && !isKnownTypes(type.classId) + + override fun generate(description: FuzzedDescription, type: FuzzedType) = sequence> { + val t = try { + Scene.v().getRefType(type.classId.canonicalName).sootClass + } catch (ignore: NoClassDefFoundError) { + logger.error(ignore) { "Soot may be not initialized" } + return@sequence + } + fun canCreateClass(sc: SootClass): Boolean { + try { + if (!sc.isConcrete) return false + val packageName = sc.packageName + if (packageName != null) { + if (packageName.startsWith("jdk.internal") || + packageName.startsWith("org.utbot") || + packageName.startsWith("sun.")) + return false + } + val isAnonymousClass = sc.name.matches(""".*\$\d+$""".toRegex()) + if (isAnonymousClass) { + return false + } + val jClass = sc.id.jClass + return isAccessible(jClass, description.description.packageName) && + jClass.declaredConstructors.any { isAccessible(it, description.description.packageName) } + } catch (ignore: Throwable) { + return false + } + } + + val implementations = when { + t.isInterface -> Scene.v().fastHierarchy.getAllImplementersOfInterface(t).filter(::canCreateClass) + t.isAbstract -> Scene.v().fastHierarchy.getSubclassesOf(t).filter(::canCreateClass) + else -> emptyList() + } + implementations.shuffled(description.random).take(10).forEach { concrete -> + yield(Seed.Recursive( + construct = Routine.Create(listOf(toFuzzerType(concrete.id.jClass, description.typeCache))) { + it.first() + }, + empty = nullRoutine(type.classId) + )) + } + } +} + internal class PublicSetterGetter( val setter: Method, val getter: Method, diff --git a/utbot-junit-contest/build.gradle b/utbot-junit-contest/build.gradle index b1319b02aa..189d36e5f6 100644 --- a/utbot-junit-contest/build.gradle +++ b/utbot-junit-contest/build.gradle @@ -13,22 +13,30 @@ compileJava { compileTestJava { options.fork = true - options.forkOptions.executable = 'javac' options.compilerArgs << "-XDignore.symbol.file" } +def testProjects = [ + 'build/output/test/antlr', + 'build/output/test/codeforces', + 'build/output/test/fastjson-1.2.50', + 'build/output/test/fescar', + 'build/output/test/guava', + 'build/output/test/guava-26.0', + 'build/output/test/guava-30.0', + 'build/output/test/pdfbox', + 'build/output/test/seata', + 'build/output/test/seata-core-0.5.0', + 'build/output/test/spoon', + 'build/output/test/spoon-core-7.0.0', +] + sourceSets { test { java { - srcDir('build/output/test/antlr') - srcDir('build/output/test/custom') - srcDir('build/output/test/guava') - srcDir('build/output/test/fescar') - srcDir('build/output/test/pdfbox') - srcDir('build/output/test/seata') - srcDir('build/output/test/spoon') - srcDir('build/output/test/samples') - srcDir('build/output/test/utbottest') + testProjects.forEach { + srcDir(it) + } } } } @@ -39,7 +47,26 @@ test { } jacocoTestReport { + afterEvaluate { + def r = testProjects.collect { + fileTree(dir: it) + }.findAll { + it.dir.exists() + } + sourceDirectories.setFrom(r.collect {files(it) }) + classDirectories.setFrom( + r.collect { + fileTree(dir: it.dir.toPath().parent.resolveSibling("unzipped").resolve(it.dir.name)) + }.findAll { + it.dir.exists() + }.collect { + files(it) + } + ) + } + reports { + csv.enabled = true html.enabled = true } } diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt index 8d317ed4f2..e0812048a0 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt @@ -10,9 +10,6 @@ import org.utbot.common.isAbstract import org.utbot.engine.EngineController import org.utbot.framework.TestSelectionStrategyType import org.utbot.framework.UtSettings -import org.utbot.framework.codegen.domain.ForceStaticMocking -import org.utbot.framework.codegen.domain.StaticsMocking -import org.utbot.framework.codegen.domain.junitByVersion import org.utbot.framework.plugin.api.util.UtContext import org.utbot.framework.plugin.api.util.executableId import org.utbot.framework.plugin.api.util.id @@ -51,7 +48,7 @@ import kotlinx.coroutines.newSingleThreadContext import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withTimeoutOrNull import org.utbot.framework.SummariesGenerationType -import org.utbot.framework.codegen.domain.ProjectType +import org.utbot.framework.codegen.domain.* import org.utbot.framework.codegen.generator.CodeGenerator import org.utbot.framework.codegen.services.language.CgLanguageAssistant import org.utbot.framework.minimization.minimizeExecutions @@ -228,6 +225,7 @@ fun runGeneration( forceStaticMocking = forceStaticMocking, generateWarningsForStaticMocking = false, cgLanguageAssistant = CgLanguageAssistant.getByCodegenLanguage(CodegenLanguage.defaultItem), + runtimeExceptionTestsBehaviour = RuntimeExceptionTestsBehaviour.PASS, ) logger.info().measureTime({ "class ${cut.fqn}" }, { statsForClass }) {