From a17647e83bbf298a98a102fc8185d931b974bb27 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 9 Jun 2023 06:37:52 +0000 Subject: [PATCH 1/5] Update CPG packages to v7 --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 002d6767a..b13570e2e 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,6 +1,6 @@ [versions] kotlin = "1.8.22" -cpg = "6.2.2" +cpg = "7.0.0" koin = "3.4.1" detekt = "1.23.0" spotless = "6.19.0" From 461fa923c30cba01ca6735f20d8b49a21d2f2055 Mon Sep 17 00:00:00 2001 From: Selina Lin Date: Fri, 16 Jun 2023 13:10:51 +0200 Subject: [PATCH 2/5] replace AssignmentTarget --- .../backends/cpg/coko/evaluators/OrderEvaluator.kt | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/codyze-backends/cpg/src/main/kotlin/de/fraunhofer/aisec/codyze/backends/cpg/coko/evaluators/OrderEvaluator.kt b/codyze-backends/cpg/src/main/kotlin/de/fraunhofer/aisec/codyze/backends/cpg/coko/evaluators/OrderEvaluator.kt index 4b78aee52..0a7f7a1e7 100644 --- a/codyze-backends/cpg/src/main/kotlin/de/fraunhofer/aisec/codyze/backends/cpg/coko/evaluators/OrderEvaluator.kt +++ b/codyze-backends/cpg/src/main/kotlin/de/fraunhofer/aisec/codyze/backends/cpg/coko/evaluators/OrderEvaluator.kt @@ -28,9 +28,9 @@ import de.fraunhofer.aisec.codyze.specificationLanguages.coko.core.dsl.FunctionO import de.fraunhofer.aisec.codyze.specificationLanguages.coko.core.dsl.Op import de.fraunhofer.aisec.codyze.specificationLanguages.coko.core.dsl.Order import de.fraunhofer.aisec.codyze.specificationLanguages.coko.core.ordering.* -import de.fraunhofer.aisec.cpg.graph.AssignmentTarget -import de.fraunhofer.aisec.cpg.graph.Node -import de.fraunhofer.aisec.cpg.graph.followNextEOG +import de.fraunhofer.aisec.cpg.graph.* +import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration +import de.fraunhofer.aisec.cpg.graph.statements.expressions.DeclaredReferenceExpression import mu.KotlinLogging import kotlin.reflect.full.createType import kotlin.reflect.full.declaredMemberFunctions @@ -162,8 +162,12 @@ class OrderEvaluator(val baseNodes: Collection, val order: Order) : Evalua dfa = dfa, hashToMethod = hashToMethod, nodeToRelevantMethod = nodesToOp, - consideredBases = baseNodes.map { node -> - node.followNextEOG { it.end is AssignmentTarget }!!.last().end + consideredBases = baseNodes.flatMap { node -> + node.followNextDFGEdgesUntilHit { next -> + next is VariableDeclaration || next is DeclaredReferenceExpression + }.fulfilled.mapNotNull { path -> + path.lastOrNull() + } }.toSet(), consideredResetNodes = baseNodes.toSet(), context = context, From bcda3ddf6d415570914aa3c2db215f0102e039bb Mon Sep 17 00:00:00 2001 From: Selina Lin Date: Fri, 16 Jun 2023 13:12:48 +0200 Subject: [PATCH 3/5] change `passes` to store the KClass of the passes instead of the actual instances --- .../codyze/backends/cpg/CPGConfiguration.kt | 3 +- .../codyze/backends/cpg/CPGOptionGroup.kt | 28 ++++++++++++------- .../codyze/backends/cpg/CpgOptionGroupTest.kt | 8 +++--- .../aisec/codyze/backends/cpg/TestUtils.kt | 2 +- .../coko/dsl/CokoCpgIntegrationTest.kt | 15 +++++----- 5 files changed, 33 insertions(+), 23 deletions(-) diff --git a/codyze-backends/cpg/src/main/kotlin/de/fraunhofer/aisec/codyze/backends/cpg/CPGConfiguration.kt b/codyze-backends/cpg/src/main/kotlin/de/fraunhofer/aisec/codyze/backends/cpg/CPGConfiguration.kt index 145fd4944..d87e31f38 100644 --- a/codyze-backends/cpg/src/main/kotlin/de/fraunhofer/aisec/codyze/backends/cpg/CPGConfiguration.kt +++ b/codyze-backends/cpg/src/main/kotlin/de/fraunhofer/aisec/codyze/backends/cpg/CPGConfiguration.kt @@ -19,6 +19,7 @@ import de.fraunhofer.aisec.codyze.core.backend.BackendConfiguration import de.fraunhofer.aisec.cpg.passes.Pass import mu.KotlinLogging import java.nio.file.Path +import kotlin.reflect.KClass private val logger = KotlinLogging.logger {} @@ -44,7 +45,7 @@ data class CPGConfiguration( val defaultPasses: Boolean, val additionalLanguages: Set, val symbols: Map, - val passes: List, + val passes: List>>, val loadIncludes: Boolean, val includePaths: List, val includeAllowlist: List, diff --git a/codyze-backends/cpg/src/main/kotlin/de/fraunhofer/aisec/codyze/backends/cpg/CPGOptionGroup.kt b/codyze-backends/cpg/src/main/kotlin/de/fraunhofer/aisec/codyze/backends/cpg/CPGOptionGroup.kt index 49fcb3067..3a8c9552c 100644 --- a/codyze-backends/cpg/src/main/kotlin/de/fraunhofer/aisec/codyze/backends/cpg/CPGOptionGroup.kt +++ b/codyze-backends/cpg/src/main/kotlin/de/fraunhofer/aisec/codyze/backends/cpg/CPGOptionGroup.kt @@ -24,6 +24,9 @@ import de.fraunhofer.aisec.codyze.core.config.combineSources import de.fraunhofer.aisec.codyze.core.config.resolvePaths import de.fraunhofer.aisec.cpg.passes.Pass import java.nio.file.Path +import kotlin.reflect.KClass +import kotlin.reflect.full.isSubclassOf +import kotlin.reflect.full.isSuperclassOf /** * Holds the common CLI options for all CPG based Codyze backends. @@ -173,10 +176,10 @@ class CPGOptionGroup : BackendOptions(helpName = "CPG Backend Options") { ) } - private val rawPasses: List by option("--passes", help = "Definition of additional symbols.") + private val rawPasses: List>> by option("--passes", help = "Definition of additional symbols.") .convert { convertPass(it) } .multiple() - private val rawPassesAdditions: List by option( + private val rawPassesAdditions: List>> by option( "--passes-additions", help = "See --passes, but appends the values to the ones specified in configuration file." @@ -185,7 +188,9 @@ class CPGOptionGroup : BackendOptions(helpName = "CPG Backend Options") { .multiple() /** Lazy property that combines all symbols from the different options into a single map. */ - val passes: List by lazy { resolvePasses(passes = rawPasses, additionalPasses = rawPassesAdditions) } + val passes: List>> by lazy { + resolvePasses(passes = rawPasses, additionalPasses = rawPassesAdditions) + } val loadIncludes: Boolean by option( "--analyze-includes", @@ -285,17 +290,20 @@ class CPGOptionGroup : BackendOptions(helpName = "CPG Backend Options") { return symbols + additionalSymbols } - private fun resolvePasses(passes: List, additionalPasses: List): List { + private fun resolvePasses( + passes: List>>, + additionalPasses: List>> + ): List>> { return passes + additionalPasses } - @Suppress("SwallowedException", "ThrowsCount") - private fun convertPass(className: String) = + @Suppress("SwallowedException", "ThrowsCount", "UNCHECKED_CAST") + private fun convertPass(className: String): KClass> = try { - val clazz = Class.forName(className) - if (Pass::class.java.isAssignableFrom(clazz)) { - // TODO: use 'isSubtypeOf' ? - clazz.getDeclaredConstructor().newInstance() as Pass + val clazz = Class.forName(className).kotlin + if (clazz.isSubclassOf(Pass::class)) { + if (clazz.isSuperclassOf(Pass::class)) throw ReflectiveOperationException("Cannot register $className") + (clazz as? KClass>) ?: throw ReflectiveOperationException("$className is not a CPG Pass") } else { throw ReflectiveOperationException("$className is not a CPG Pass") } diff --git a/codyze-backends/cpg/src/test/kotlin/de/fraunhofer/aisec/codyze/backends/cpg/CpgOptionGroupTest.kt b/codyze-backends/cpg/src/test/kotlin/de/fraunhofer/aisec/codyze/backends/cpg/CpgOptionGroupTest.kt index f4ce288bf..0bbb815fa 100644 --- a/codyze-backends/cpg/src/test/kotlin/de/fraunhofer/aisec/codyze/backends/cpg/CpgOptionGroupTest.kt +++ b/codyze-backends/cpg/src/test/kotlin/de/fraunhofer/aisec/codyze/backends/cpg/CpgOptionGroupTest.kt @@ -105,10 +105,10 @@ class CpgOptionGroupTest { ) val expectedPassesNames = - listOf(EdgeCachePass(), FilenameMapper(), CallResolver()).map { p -> - p::class.qualifiedName + listOf(EdgeCachePass::class, FilenameMapper::class, CallResolver::class).map { p -> + p.qualifiedName } - val actualPassesNames = cli.cpgOptions.passes.map { p -> p::class.qualifiedName } + val actualPassesNames = cli.cpgOptions.passes.map { p -> p.qualifiedName } logger.info { actualPassesNames.joinToString(",") } @@ -330,7 +330,7 @@ class CpgOptionGroupTest { assertNotNull(translationOptionName) return Stream.of( - Arguments.of(arrayOf("--passes", passName)), + Arguments.of(arrayOf("--source", testDir1.toString(), "--passes", passName)), Arguments.of(arrayOf("--passes", "my.passes.MyPass")), Arguments.of(arrayOf("--passes", translationOptionName)) ) diff --git a/codyze-backends/cpg/src/test/kotlin/de/fraunhofer/aisec/codyze/backends/cpg/TestUtils.kt b/codyze-backends/cpg/src/test/kotlin/de/fraunhofer/aisec/codyze/backends/cpg/TestUtils.kt index acc478177..a3cea8180 100644 --- a/codyze-backends/cpg/src/test/kotlin/de/fraunhofer/aisec/codyze/backends/cpg/TestUtils.kt +++ b/codyze-backends/cpg/src/test/kotlin/de/fraunhofer/aisec/codyze/backends/cpg/TestUtils.kt @@ -40,7 +40,7 @@ fun createCpgConfiguration(vararg sourceFile: Path) = includePaths = listOf(), includeAllowlist = listOf(), loadIncludes = false, - passes = listOf(EdgeCachePass(), UnreachableEOGPass()), + passes = listOf(EdgeCachePass::class, UnreachableEOGPass::class), ) @Rule diff --git a/codyze-specification-languages/coko/coko-dsl/src/test/kotlin/de/fraunhofer/aisec/codyze/specificationLanguages/coko/dsl/CokoCpgIntegrationTest.kt b/codyze-specification-languages/coko/coko-dsl/src/test/kotlin/de/fraunhofer/aisec/codyze/specificationLanguages/coko/dsl/CokoCpgIntegrationTest.kt index 9af415894..6e5dbfd93 100644 --- a/codyze-specification-languages/coko/coko-dsl/src/test/kotlin/de/fraunhofer/aisec/codyze/specificationLanguages/coko/dsl/CokoCpgIntegrationTest.kt +++ b/codyze-specification-languages/coko/coko-dsl/src/test/kotlin/de/fraunhofer/aisec/codyze/specificationLanguages/coko/dsl/CokoCpgIntegrationTest.kt @@ -19,6 +19,7 @@ import de.fraunhofer.aisec.codyze.backends.cpg.CPGConfiguration import de.fraunhofer.aisec.codyze.backends.cpg.coko.CokoCpgBackend import de.fraunhofer.aisec.codyze.specificationLanguages.coko.dsl.host.CokoExecutor import de.fraunhofer.aisec.cpg.passes.EdgeCachePass +import de.fraunhofer.aisec.cpg.passes.Pass import de.fraunhofer.aisec.cpg.passes.UnreachableEOGPass import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Test @@ -60,7 +61,7 @@ class CokoCpgIntegrationTest { includePaths = listOf(), includeAllowlist = listOf(), loadIncludes = false, - passes = listOf(UnreachableEOGPass(), EdgeCachePass()), + passes = listOf(UnreachableEOGPass::class, EdgeCachePass::class), ) /** @@ -89,7 +90,7 @@ class CokoCpgIntegrationTest { val run = executor.evaluate() // assertions for the order rule - assertEquals(run.results?.size, 16) + assertEquals(16, run.results?.size) } /** @@ -119,7 +120,7 @@ class CokoCpgIntegrationTest { val run = executor.evaluate() // assertions for the order rule - assertEquals(run.results?.size, 1) + assertEquals(1, run.results?.size) } /** @@ -148,7 +149,7 @@ class CokoCpgIntegrationTest { val executor = CokoExecutor(cokoConfiguration, backend) val run = executor.evaluate() - assertEquals(run.results?.size, 16) + assertEquals(16, run.results?.size) } /** @@ -177,7 +178,7 @@ class CokoCpgIntegrationTest { val executor = CokoExecutor(cokoConfiguration, backend) val run = executor.evaluate() - assertEquals(run.results?.size, 16) + assertEquals(16, run.results?.size) } /** @@ -187,7 +188,7 @@ class CokoCpgIntegrationTest { * The order of the files in `specFiles` is permuted to verify that the order in which the spec files are evaluated * does not have an impact on the results. */ - @Disabled("Too many permutations (120) of the specFiles order") +// @Disabled("Too many permutations (120) of the specFiles order") @ParameterizedTest(name = "{index} {1}") @MethodSource("fiveFiles") fun `test coko with cpg backend and permutation of five dependent spec files`( @@ -208,7 +209,7 @@ class CokoCpgIntegrationTest { val executor = CokoExecutor(cokoConfiguration, backend) val run = executor.evaluate() - assertEquals(run.results?.size, 16) + assertEquals(16, run.results?.size) } /** From bcff57a58f732e3c2604e546f740fa8c48bd5982 Mon Sep 17 00:00:00 2001 From: Selina Lin Date: Fri, 16 Jun 2023 13:34:54 +0200 Subject: [PATCH 4/5] add java and cxx language as dependency and register them in the cpg --- .../aisec/codyze/backends/cpg/CPGBackend.kt | 10 +++++++++- gradle/libs.versions.toml | 15 ++++++++++----- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/codyze-backends/cpg/src/main/kotlin/de/fraunhofer/aisec/codyze/backends/cpg/CPGBackend.kt b/codyze-backends/cpg/src/main/kotlin/de/fraunhofer/aisec/codyze/backends/cpg/CPGBackend.kt index 119aa3d74..b57f52d64 100644 --- a/codyze-backends/cpg/src/main/kotlin/de/fraunhofer/aisec/codyze/backends/cpg/CPGBackend.kt +++ b/codyze-backends/cpg/src/main/kotlin/de/fraunhofer/aisec/codyze/backends/cpg/CPGBackend.kt @@ -68,7 +68,6 @@ open class CPGBackend(config: BackendConfiguration) : Backend { .failOnError(failOnError) .useParallelFrontends(useParallelFrontends) .typeSystemActiveInFrontend(typeSystemActiveInFrontend) - .defaultLanguages() .sourceLocations(source.map { (it.toFile()) }) .symbols(symbols) .useUnityBuild(useUnityBuild) @@ -86,6 +85,15 @@ open class CPGBackend(config: BackendConfiguration) : Backend { if (defaultPasses) translationConfiguration.defaultPasses() passes.forEach { translationConfiguration.registerPass(it) } + translationConfiguration.optionalLanguage( + "de.fraunhofer.aisec.cpg.frontends.cxx.CLanguage" + ) + translationConfiguration.optionalLanguage( + "de.fraunhofer.aisec.cpg.frontends.cxx.CPPLanguage" + ) + translationConfiguration.optionalLanguage( + "de.fraunhofer.aisec.cpg.frontends.java.JavaLanguage" + ) translationConfiguration.optionalLanguage( "de.fraunhofer.aisec.cpg.frontends.python.PythonLanguage" ) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index b13570e2e..8c158e3c3 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -13,13 +13,18 @@ kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serializa kotlin-reflect = { module = "org.jetbrains.kotlin:kotlin-reflect", version.ref = "kotlin"} # CPG official releases -#cpg-core = { module = "de.fraunhofer.aisec:cpg-core", version.ref = "cpg"} -#cpg-analysis = { module = "de.fraunhofer.aisec:cpg-analysis", version.ref = "cpg"} +cpg-core = { module = "de.fraunhofer.aisec:cpg-core", version.ref = "cpg"} +cpg-analysis = { module = "de.fraunhofer.aisec:cpg-analysis", version.ref = "cpg"} +cpg-language-cxx = { module = "de.fraunhofer.aisec:cpg-language-cxx", version.ref = "cpg"} +cpg-language-java = { module = "de.fraunhofer.aisec:cpg-language-java", version.ref = "cpg"} #cpg-language-go = { module = "de.fraunhofer.aisec:cpg-language-go", version.ref = "cpg"} +#cpg-language-python = { module = "de.fraunhofer.aisec:cpg-language-python", version.ref = "cpg"} +#cpg-language-llvm = { module = "de.fraunhofer.aisec:cpg-language-llvm", version.ref = "cpg"} +#cpg-language-typescript = { module = "de.fraunhofer.aisec:cpg-language-typescript", version.ref = "cpg"} # CPG GitHub builds using JitPack -cpg-core = { module = "de.fraunhofer.aisec:cpg-core", version.ref = "cpg"} -cpg-analysis = { module = "de.fraunhofer.aisec:cpg-analysis", version.ref = "cpg"} +#cpg-core = { module = "com.github.Fraunhofer-AISEC.cpg:cpg-core", version.ref = "cpg"} +#cpg-analysis = { module = "com.github.Fraunhofer-AISEC.cpg:cpg-analysis", version.ref = "cpg"} #cpg-language-go = { module = "com.github.Fraunhofer-AISEC.cpg:cpg-language-go", version.ref = "cpg"} kotlin-logging = { module = "io.github.microutils:kotlin-logging-jvm", version = "3.0.5"} @@ -49,7 +54,7 @@ dokka-base = { module = "org.jetbrains.dokka:dokka-base", version.ref = "dokka" [bundles] -cpg = ["cpg-core", "cpg-analysis"] # without "cpg-language-go" +cpg = ["cpg-core", "cpg-analysis", "cpg-language-cxx", "cpg-language-java"] # without "cpg-language-go" sarif = ["sarif4k", "kotlinx-serialization-json"] [plugins] From 3a9fc98a67c495f18ccfb0e168699c439f31e227 Mon Sep 17 00:00:00 2001 From: Selina Lin Date: Fri, 16 Jun 2023 13:43:37 +0200 Subject: [PATCH 5/5] disable large permutation test again --- .../specificationLanguages/coko/dsl/CokoCpgIntegrationTest.kt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/codyze-specification-languages/coko/coko-dsl/src/test/kotlin/de/fraunhofer/aisec/codyze/specificationLanguages/coko/dsl/CokoCpgIntegrationTest.kt b/codyze-specification-languages/coko/coko-dsl/src/test/kotlin/de/fraunhofer/aisec/codyze/specificationLanguages/coko/dsl/CokoCpgIntegrationTest.kt index 6e5dbfd93..0f710d243 100644 --- a/codyze-specification-languages/coko/coko-dsl/src/test/kotlin/de/fraunhofer/aisec/codyze/specificationLanguages/coko/dsl/CokoCpgIntegrationTest.kt +++ b/codyze-specification-languages/coko/coko-dsl/src/test/kotlin/de/fraunhofer/aisec/codyze/specificationLanguages/coko/dsl/CokoCpgIntegrationTest.kt @@ -19,7 +19,6 @@ import de.fraunhofer.aisec.codyze.backends.cpg.CPGConfiguration import de.fraunhofer.aisec.codyze.backends.cpg.coko.CokoCpgBackend import de.fraunhofer.aisec.codyze.specificationLanguages.coko.dsl.host.CokoExecutor import de.fraunhofer.aisec.cpg.passes.EdgeCachePass -import de.fraunhofer.aisec.cpg.passes.Pass import de.fraunhofer.aisec.cpg.passes.UnreachableEOGPass import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Test @@ -188,7 +187,7 @@ class CokoCpgIntegrationTest { * The order of the files in `specFiles` is permuted to verify that the order in which the spec files are evaluated * does not have an impact on the results. */ -// @Disabled("Too many permutations (120) of the specFiles order") + @Disabled("Too many permutations (120) of the specFiles order") @ParameterizedTest(name = "{index} {1}") @MethodSource("fiveFiles") fun `test coko with cpg backend and permutation of five dependent spec files`(