From b65b017634c67d484c2bca9c6f97e131afab886e Mon Sep 17 00:00:00 2001 From: Ankit Kumar <118803988+ankit-privado@users.noreply.github.com> Date: Mon, 21 Oct 2024 14:31:42 +0530 Subject: [PATCH] Kotlin types, no resolve flag (#118) * add support for no-resolve-types flag * fix test case * fix test case * fix failing test case --------- Co-authored-by: Khemraj Rathore --- .../io/joern/kotlin2cpg/Kotlin2Cpg.scala | 8 +++--- .../main/scala/io/joern/kotlin2cpg/Main.scala | 12 +++++++-- .../types/ContentSourcesPicker.scala | 14 ++++++++--- .../compiler/CompilerAPITests.scala | 25 ++++++++++++++++++- .../types/KotlinScriptFilteringTests.scala | 3 ++- 5 files changed, 50 insertions(+), 12 deletions(-) diff --git a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/Kotlin2Cpg.scala b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/Kotlin2Cpg.scala index 66c483416994..ac03b2ddb71f 100644 --- a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/Kotlin2Cpg.scala +++ b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/Kotlin2Cpg.scala @@ -160,8 +160,8 @@ class Kotlin2Cpg extends X2CpgFrontend[Config] with UsesService { defaultContentRootJars } - private def gatherDirsForSourcesToCompile(sourceDir: String): Seq[String] = { - val dirsForSourcesToCompile = ContentSourcesPicker.dirsForRoot(sourceDir) + private def gatherDirsForSourcesToCompile(sourceDir: String, config: Config): Seq[String] = { + val dirsForSourcesToCompile = ContentSourcesPicker.dirsForRoot(sourceDir, config) if (dirsForSourcesToCompile.isEmpty) { logger.warn("The list of directories to analyze is empty.") } @@ -213,9 +213,9 @@ class Kotlin2Cpg extends X2CpgFrontend[Config] with UsesService { val filesWithJavaExtension = gatherFilesWithJavaExtension(sourceDir, config) val mavenCoordinates = gatherMavenCoordinates(sourceDir, config) val defaultContentRootJars = gatherDefaultContentRootJars(sourceDir, config, filesWithJavaExtension) - val dirsForSourcesToCompile = gatherDirsForSourcesToCompile(sourceDir) + val dirsForSourcesToCompile = gatherDirsForSourcesToCompile(sourceDir, config) val environment = CompilerAPI.makeEnvironment( - Seq(sourceDir), + dirsForSourcesToCompile, filesWithJavaExtension, defaultContentRootJars, new ErrorLoggingMessageCollector diff --git a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/Main.scala b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/Main.scala index 4d20c13c1e90..0cd5a6df3a02 100644 --- a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/Main.scala +++ b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/Main.scala @@ -15,7 +15,8 @@ final case class Config( includeJavaSourceFiles: Boolean = false, generateNodesForDependencies: Boolean = false, downloadDependencies: Boolean = false, - keepTypeArguments: Boolean = false + keepTypeArguments: Boolean = false, + resolveTypes: Boolean = true ) extends X2CpgConfig[Config] with DependencyDownloadConfig[Config] { @@ -54,6 +55,10 @@ final case class Config( def withKeepTypeArguments(value: Boolean): Config = { copy(keepTypeArguments = value).withInheritedFields(this) } + + def withResolveTypes(value: Boolean): Config = { + copy(resolveTypes = value).withInheritedFields(this) + } } private object Frontend { @@ -91,7 +96,10 @@ private object Frontend { opt[Unit]("keep-type-arguments") .hidden() .action((_, c) => c.withKeepTypeArguments(true)) - .text("Type full names of variables keep their type arguments.") + .text("Type full names of variables keep their type arguments."), + opt[Unit]("no-resolve-types") + .action((_, c) => c.withResolveTypes(false)) + .text("Skip generating types") ) } } diff --git a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/types/ContentSourcesPicker.scala b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/types/ContentSourcesPicker.scala index 45e3bc225e60..af8f688e0e2c 100644 --- a/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/types/ContentSourcesPicker.scala +++ b/joern-cli/frontends/kotlin2cpg/src/main/scala/io/joern/kotlin2cpg/types/ContentSourcesPicker.scala @@ -1,7 +1,9 @@ package io.joern.kotlin2cpg.types -import better.files.{File => BFile} +import better.files.File as BFile +import io.joern.kotlin2cpg.Config import io.joern.kotlin2cpg.DefaultContentRootJarPath +import io.joern.x2cpg.SourceFiles.toRelativePath object ContentSourcesPicker { @@ -18,13 +20,13 @@ object ContentSourcesPicker { // The list of paths which are acceptable for the current version of the Kotlin compiler API is: // `Seq("dir1/dir2/dir3")` and nothing else. - def dirsForRoot(rootDir: String): Seq[String] = { + def dirsForRoot(rootDir: String, config: Config): Seq[String] = { val dir = BFile(rootDir) val hasSubDirs = dir.list.exists(_.isDirectory) - if (!hasSubDirs) { + if (!hasSubDirs || !config.resolveTypes) { return Seq(rootDir) } - dir.listRecursively + val dirPaths = dir.listRecursively .filter(_.isDirectory) .flatMap { f => val hasKtsFile = f.listRecursively.exists { f => f.hasExtension && f.pathAsString.endsWith(".kts") } @@ -36,5 +38,9 @@ object ContentSourcesPicker { } .flatten .toSeq + dirPaths.filterNot(path => + val t = toRelativePath(path, config.inputPath) + config.ignoredFilesRegex.matches(toRelativePath(path, config.inputPath)) + ) } } diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/compiler/CompilerAPITests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/compiler/CompilerAPITests.scala index 908ca02254d5..e0a9519dfb07 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/compiler/CompilerAPITests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/compiler/CompilerAPITests.scala @@ -4,6 +4,7 @@ import better.files.File import io.joern.kotlin2cpg.Config import io.joern.kotlin2cpg.DefaultContentRootJarPath import io.joern.kotlin2cpg.Kotlin2Cpg +import io.joern.kotlin2cpg.types.ContentSourcesPicker import io.joern.x2cpg.utils.ExternalCommand import io.joern.x2cpg.Defines import io.shiftleft.semanticcpg.language.* @@ -75,7 +76,7 @@ class CompilerAPITests extends AnyFreeSpec with Matchers { ProjectRoot.relativise("joern-cli/frontends/kotlin2cpg/src/test/resources/code/springboot-kotlin-webgoat") val projectDependenciesPath = Paths.get(projectDirPath, "build", "gatheredDependencies") - "should not contain methods with unresolved types/namespaces" ignore { + "should not contain methods with unresolved types/namespaces" in { val command = if (scala.util.Properties.isWin) "cmd.exe /C gradlew.bat gatherDependencies" else "./gradlew gatherDependencies" ExternalCommand.run(command, projectDirPath) shouldBe Symbol("success") @@ -87,5 +88,27 @@ class CompilerAPITests extends AnyFreeSpec with Matchers { cpg.method.signature(s".*${Defines.UnresolvedNamespace}.*") shouldBe empty } + "should return all the individual folder name" in { + val paths = ContentSourcesPicker.dirsForRoot(projectDirPath, Config().withInputPath(projectDirPath)) + paths.size shouldBe 26 + } + + "should return all the individual folder name excluding paths, mentioned in exclusion regex" in { + import java.io.File + val paths = ContentSourcesPicker.dirsForRoot( + projectDirPath, + Config().withInputPath(projectDirPath).withIgnoredFilesRegex(s".*test.*") + ) + paths.size shouldBe 21 + } + + "should return all the individual folder name excluding paths, when no-resolve-type" in { + val paths = + ContentSourcesPicker.dirsForRoot(projectDirPath, Config().withInputPath(projectDirPath).withResolveTypes(false)) + + paths.size shouldBe 1 + paths.head shouldBe projectDirPath + } + } } diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/types/KotlinScriptFilteringTests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/types/KotlinScriptFilteringTests.scala index 7996bbc123a6..80d9aa9536a9 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/types/KotlinScriptFilteringTests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/types/KotlinScriptFilteringTests.scala @@ -1,5 +1,6 @@ package io.joern.kotlin2cpg.types +import io.joern.kotlin2cpg.Config import io.joern.kotlin2cpg.compiler.{CompilerAPI, ErrorLoggingMessageCollector} import org.jetbrains.kotlin.resolve.BindingContext import org.scalatest.freespec.AnyFreeSpec @@ -22,7 +23,7 @@ class KotlinScriptFilteringTests extends AnyFreeSpec with Matchers { "should not return an empty binding context" in { val sourceDir = "src/test/resources/external_projects/kotlin-dsl" - val dirsForSourcesToCompile = ContentSourcesPicker.dirsForRoot(sourceDir) + val dirsForSourcesToCompile = ContentSourcesPicker.dirsForRoot(sourceDir, config = Config()) val environment = CompilerAPI.makeEnvironment(dirsForSourcesToCompile, Seq(), Seq(), new ErrorLoggingMessageCollector) environment.getSourceFiles should not be List()