From 5839ae7b674fbc5b267e476e5b7cbc08d25ca8b5 Mon Sep 17 00:00:00 2001 From: Michael Pollmeier Date: Wed, 27 Nov 2024 09:08:33 +0100 Subject: [PATCH] fixup initialisation order of cpg generators (#5137) We trigger the post-processing directly if a cpg is loaded on startup (e.g. `./joern cpg.bin`). In that case, the typeRecoveryConfig has not been initialized. There's a reason why in Scala you have to explicitly make an effort to start with `null`, because more often than not you'll be missing out some random edge case... lazy val is the better alternative fixes https://github.com/joernio/joern/issues/4999 (again...) --- .../cpgcreation/JavaSrcCpgGenerator.scala | 10 ++++------ .../console/cpgcreation/JsSrcCpgGenerator.scala | 5 +---- .../console/cpgcreation/PhpCpgGenerator.scala | 17 ++++++++--------- .../cpgcreation/PythonSrcCpgGenerator.scala | 6 ++---- .../cpgcreation/SwiftSrcCpgGenerator.scala | 5 ++--- 5 files changed, 17 insertions(+), 26 deletions(-) diff --git a/console/src/main/scala/io/joern/console/cpgcreation/JavaSrcCpgGenerator.scala b/console/src/main/scala/io/joern/console/cpgcreation/JavaSrcCpgGenerator.scala index 09a5fb003cb7..769a6e15cfd7 100644 --- a/console/src/main/scala/io/joern/console/cpgcreation/JavaSrcCpgGenerator.scala +++ b/console/src/main/scala/io/joern/console/cpgcreation/JavaSrcCpgGenerator.scala @@ -6,22 +6,20 @@ import io.joern.x2cpg.passes.frontend.XTypeRecoveryConfig import io.shiftleft.codepropertygraph.generated.Cpg import java.nio.file.Path -import scala.compiletime.uninitialized import scala.util.Try /** Source-based front-end for Java */ case class JavaSrcCpgGenerator(config: FrontendConfig, rootPath: Path) extends CpgGenerator { private lazy val command: Path = if (isWin) rootPath.resolve("javasrc2cpg.bat") else rootPath.resolve("javasrc2cpg") - private var enableTypeRecovery = false - private var typeRecoveryConfig: XTypeRecoveryConfig = uninitialized + private lazy val cmdLineArgs = config.cmdLineParams.toSeq + private lazy val enableTypeRecovery = cmdLineArgs.exists(_ == s"--${javasrc2cpg.ParameterNames.EnableTypeRecovery}") + private lazy val typeRecoveryConfig = XTypeRecoveryConfig.parse(cmdLineArgs) /** Generate a CPG for the given input path. Returns the output path, or None, if no CPG was generated. */ override def generate(inputPath: String, outputPath: String = "cpg.bin"): Try[String] = { - val arguments = config.cmdLineParams.toSeq ++ Seq(inputPath, "--output", outputPath) - enableTypeRecovery = arguments.exists(_ == s"--${javasrc2cpg.ParameterNames.EnableTypeRecovery}") - if (enableTypeRecovery) typeRecoveryConfig = XTypeRecoveryConfig.parse(arguments) + val arguments = cmdLineArgs ++ Seq(inputPath, "--output", outputPath) runShellCommand(command.toString, arguments).map(_ => outputPath) } diff --git a/console/src/main/scala/io/joern/console/cpgcreation/JsSrcCpgGenerator.scala b/console/src/main/scala/io/joern/console/cpgcreation/JsSrcCpgGenerator.scala index 7d2ce5c5eb93..0ec4347ba392 100644 --- a/console/src/main/scala/io/joern/console/cpgcreation/JsSrcCpgGenerator.scala +++ b/console/src/main/scala/io/joern/console/cpgcreation/JsSrcCpgGenerator.scala @@ -7,18 +7,15 @@ import io.joern.x2cpg.passes.frontend.XTypeRecoveryConfig import io.shiftleft.codepropertygraph.generated.Cpg import java.nio.file.Path -import scala.compiletime.uninitialized import scala.util.Try case class JsSrcCpgGenerator(config: FrontendConfig, rootPath: Path) extends CpgGenerator { private lazy val command: Path = if (isWin) rootPath.resolve("jssrc2cpg.bat") else rootPath.resolve("jssrc2cpg.sh") - private var typeRecoveryConfig: XTypeRecoveryConfig = uninitialized + private lazy val typeRecoveryConfig = XTypeRecoveryConfig.parse(config.cmdLineParams.toSeq) /** Generate a CPG for the given input path. Returns the output path, or None, if no CPG was generated. */ override def generate(inputPath: String, outputPath: String = "cpg.bin.zip"): Try[String] = { - typeRecoveryConfig = XTypeRecoveryConfig.parse(config.cmdLineParams.toSeq) - if (File(inputPath).isDirectory) { invoke(inputPath, outputPath) } else { diff --git a/console/src/main/scala/io/joern/console/cpgcreation/PhpCpgGenerator.scala b/console/src/main/scala/io/joern/console/cpgcreation/PhpCpgGenerator.scala index 765d9a824c29..a570751769e0 100644 --- a/console/src/main/scala/io/joern/console/cpgcreation/PhpCpgGenerator.scala +++ b/console/src/main/scala/io/joern/console/cpgcreation/PhpCpgGenerator.scala @@ -7,24 +7,23 @@ import io.shiftleft.codepropertygraph.generated.Cpg import scopt.OParser import java.nio.file.Path -import scala.compiletime.uninitialized import scala.util.Try case class PhpCpgGenerator(config: FrontendConfig, rootPath: Path) extends CpgGenerator { - private lazy val command: Path = if (isWin) rootPath.resolve("php2cpg.bat") else rootPath.resolve("php2cpg") - private var typeRecoveryConfig: XTypeRecoveryConfig = uninitialized - private var setKnownTypesConfig: XTypeStubsParserConfig = uninitialized - - override def generate(inputPath: String, outputPath: String): Try[String] = { - val cmdLineArgs = config.cmdLineParams.toSeq - typeRecoveryConfig = XTypeRecoveryConfig.parse(cmdLineArgs) - setKnownTypesConfig = OParser + private lazy val command: Path = if (isWin) rootPath.resolve("php2cpg.bat") else rootPath.resolve("php2cpg") + private lazy val cmdLineArgs = config.cmdLineParams.toSeq + private lazy val typeRecoveryConfig = XTypeRecoveryConfig.parse(cmdLineArgs) + private lazy val setKnownTypesConfig: XTypeStubsParserConfig = { + OParser .parse(XTypeStubsParser.parserOptions2, cmdLineArgs, XTypeStubsParserConfig()) .getOrElse( throw new RuntimeException( s"unable to parse XTypeStubsParserConfig from commandline arguments ${cmdLineArgs.mkString(" ")}" ) ) + } + + override def generate(inputPath: String, outputPath: String): Try[String] = { val arguments = List(inputPath) ++ Seq("-o", outputPath) ++ config.cmdLineParams runShellCommand(command.toString, arguments).map(_ => outputPath) } diff --git a/console/src/main/scala/io/joern/console/cpgcreation/PythonSrcCpgGenerator.scala b/console/src/main/scala/io/joern/console/cpgcreation/PythonSrcCpgGenerator.scala index bf73f66be772..e77d8643dd23 100644 --- a/console/src/main/scala/io/joern/console/cpgcreation/PythonSrcCpgGenerator.scala +++ b/console/src/main/scala/io/joern/console/cpgcreation/PythonSrcCpgGenerator.scala @@ -14,13 +14,12 @@ import scala.util.Try import scala.compiletime.uninitialized case class PythonSrcCpgGenerator(config: FrontendConfig, rootPath: Path) extends CpgGenerator { - private lazy val command: Path = if (isWin) rootPath.resolve("pysrc2cpg.bat") else rootPath.resolve("pysrc2cpg") - private var typeRecoveryConfig: XTypeRecoveryConfig = uninitialized + private lazy val command: Path = if (isWin) rootPath.resolve("pysrc2cpg.bat") else rootPath.resolve("pysrc2cpg") + private lazy val typeRecoveryConfig = XTypeRecoveryConfig.parse(config.cmdLineParams.toSeq) /** Generate a CPG for the given input path. Returns the output path, or None, if no CPG was generated. */ override def generate(inputPath: String, outputPath: String = "cpg.bin.zip"): Try[String] = { - typeRecoveryConfig = XTypeRecoveryConfig.parse(config.cmdLineParams.toSeq) val arguments = Seq(inputPath, "-o", outputPath) ++ config.cmdLineParams runShellCommand(command.toString, arguments).map(_ => outputPath) } @@ -30,7 +29,6 @@ case class PythonSrcCpgGenerator(config: FrontendConfig, rootPath: Path) extends override def applyPostProcessingPasses(cpg: Cpg): Cpg = { pysrc2cpg.postProcessingPasses(cpg, typeRecoveryConfig).foreach(_.createAndApply()) - cpg } diff --git a/console/src/main/scala/io/joern/console/cpgcreation/SwiftSrcCpgGenerator.scala b/console/src/main/scala/io/joern/console/cpgcreation/SwiftSrcCpgGenerator.scala index ed0eed6f6f48..504750f4ef62 100644 --- a/console/src/main/scala/io/joern/console/cpgcreation/SwiftSrcCpgGenerator.scala +++ b/console/src/main/scala/io/joern/console/cpgcreation/SwiftSrcCpgGenerator.scala @@ -7,13 +7,13 @@ import io.joern.x2cpg.passes.frontend.XTypeRecoveryConfig import io.shiftleft.codepropertygraph.generated.Cpg import java.nio.file.Path -import scala.compiletime.uninitialized import scala.util.Try case class SwiftSrcCpgGenerator(config: FrontendConfig, rootPath: Path) extends CpgGenerator { private lazy val command: Path = if (isWin) rootPath.resolve("swiftsrc2cpg.bat") else rootPath.resolve("swiftsrc2cpg.sh") - private var typeRecoveryConfig: XTypeRecoveryConfig = uninitialized + + private lazy val typeRecoveryConfig = XTypeRecoveryConfig.parse(config.cmdLineParams.toSeq) /** Generate a CPG for the given input path. Returns the output path, or None, if no CPG was generated. */ @@ -29,7 +29,6 @@ case class SwiftSrcCpgGenerator(config: FrontendConfig, rootPath: Path) extends private def invoke(inputPath: String, outputPath: String): Try[String] = { val arguments = Seq(inputPath, "--output", outputPath) ++ config.cmdLineParams - typeRecoveryConfig = XTypeRecoveryConfig.parse(config.cmdLineParams.toSeq) runShellCommand(command.toString, arguments).map(_ => outputPath) }