diff --git a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/passes/reachingdef/ReachingDefPass.scala b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/passes/reachingdef/ReachingDefPass.scala index e8a23efa6d80..6644015c552b 100755 --- a/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/passes/reachingdef/ReachingDefPass.scala +++ b/dataflowengineoss/src/main/scala/io/joern/dataflowengineoss/passes/reachingdef/ReachingDefPass.scala @@ -22,15 +22,20 @@ class ReachingDefPass(cpg: Cpg, maxNumberOfDefinitions: Int = 4000)(implicit s: override def runOnPart(dstGraph: DiffGraphBuilder, method: Method): Unit = { logger.info("Calculating reaching definitions for: {} in {}", method.fullName, method.filename) - val problem = ReachingDefProblem.create(method) - if (shouldBailOut(method, problem)) { - logger.warn("Skipping.") - return + try { + val problem = ReachingDefProblem.create(method) + if (shouldBailOut(method, problem)) { + logger.warn("Skipping.") + return + } + + val solution = new DataFlowSolver().calculateMopSolutionForwards(problem) + val ddgGenerator = new DdgGenerator(s) + ddgGenerator.addReachingDefEdges(dstGraph, method, problem, solution) + } catch { + case ex: Exception => + logger.warn(s"Error for the METHOD node -> '${method.fullName}' in file '${method.filename}'") } - - val solution = new DataFlowSolver().calculateMopSolutionForwards(problem) - val ddgGenerator = new DdgGenerator(s) - ddgGenerator.addReachingDefEdges(dstGraph, method, problem, solution) } /** Before we start propagating definitions in the graph, which is the bulk of the work, we check how many definitions diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/AstLinkerPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/AstLinkerPass.scala index 5fd666d2fa43..90f1947006f4 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/AstLinkerPass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/AstLinkerPass.scala @@ -3,27 +3,33 @@ package io.joern.x2cpg.passes.base import io.joern.x2cpg.utils.LinkingUtil import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.nodes.StoredNode -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes} +import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes, Properties} import io.shiftleft.passes.CpgPass import io.shiftleft.semanticcpg.language.* +import org.slf4j.{Logger, LoggerFactory} class AstLinkerPass(cpg: Cpg) extends CpgPass(cpg) with LinkingUtil { - + private val logger: Logger = LoggerFactory.getLogger(this.getClass) override def run(dstGraph: DiffGraphBuilder): Unit = { - cpg.method.whereNot(_.astParent).foreach { method => - addAstParent(method, method.fullName, method.astParentType, method.astParentFullName, dstGraph) - } - cpg.typeDecl.whereNot(_.astParent).foreach { typeDecl => - addAstParent(typeDecl, typeDecl.fullName, typeDecl.astParentType, typeDecl.astParentFullName, dstGraph) - } - cpg.member.whereNot(_.astParent).foreach { member => - addAstParent( - member, - s"${member.astParentFullName}.${member.name}", - member.astParentType, - member.astParentFullName, - dstGraph - ) + try { + cpg.method.whereNot(_.astParent).foreach { method => + addAstParent(method, method.fullName, method.astParentType, method.astParentFullName, dstGraph) + } + cpg.typeDecl.whereNot(_.astParent).foreach { typeDecl => + addAstParent(typeDecl, typeDecl.fullName, typeDecl.astParentType, typeDecl.astParentFullName, dstGraph) + } + cpg.member.whereNot(_.astParent).foreach { member => + addAstParent( + member, + s"${member.astParentFullName}.${member.name}", + member.astParentType, + member.astParentFullName, + dstGraph + ) + } + } catch { + case ex: Exception => + logger.warn(s"Error in AstLinkerPass", ex) } } @@ -38,25 +44,33 @@ class AstLinkerPass(cpg: Cpg) extends CpgPass(cpg) with LinkingUtil { astParentFullName: String, dstGraph: DiffGraphBuilder ): Unit = { - val astParentOption: Option[StoredNode] = - astParentType match { - case NodeTypes.METHOD => methodFullNameToNode(cpg, astParentFullName) - case NodeTypes.TYPE_DECL => typeDeclFullNameToNode(cpg, astParentFullName) - case NodeTypes.NAMESPACE_BLOCK => namespaceBlockFullNameToNode(cpg, astParentFullName) - case _ => - logger.warn( - s"Invalid AST_PARENT_TYPE=$astParentFullName;" + - s" astChild LABEL=${astChild.label};" + - s" astChild FULL_NAME=$astChildFullName" - ) - None - } + try { + val astParentOption: Option[StoredNode] = + astParentType match { + case NodeTypes.METHOD => methodFullNameToNode(cpg, astParentFullName) + case NodeTypes.TYPE_DECL => typeDeclFullNameToNode(cpg, astParentFullName) + case NodeTypes.NAMESPACE_BLOCK => namespaceBlockFullNameToNode(cpg, astParentFullName) + case _ => + logger.warn( + s"Invalid AST_PARENT_TYPE=$astParentFullName;" + + s" astChild LABEL=${astChild.label};" + + s" astChild FULL_NAME=$astChildFullName" + ) + None + } - astParentOption match { - case Some(astParent) => - dstGraph.addEdge(astParent, astChild, EdgeTypes.AST) - case None => - logFailedSrcLookup(EdgeTypes.AST, astParentType, astParentFullName, astChild.label, astChild.id.toString) + astParentOption match { + case Some(astParent) => + dstGraph.addEdge(astParent, astChild, EdgeTypes.AST) + case None => + logFailedSrcLookup(EdgeTypes.AST, astParentType, astParentFullName, astChild.label, astChild.id.toString) + } + } catch { + case ex: Exception => + logger.warn( + s"Error in AstLinkerPass for node in file '${astChild.propertyOption(Properties.FILENAME).toString}''", + ex + ) } } } diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/ContainsEdgePass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/ContainsEdgePass.scala index 71c98e2ad5f7..eb84a3896afd 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/ContainsEdgePass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/ContainsEdgePass.scala @@ -1,31 +1,41 @@ package io.joern.x2cpg.passes.base import io.shiftleft.codepropertygraph.Cpg -import io.shiftleft.codepropertygraph.generated.nodes._ -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes} +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes, Properties} import io.shiftleft.passes.ConcurrentWriterCpgPass +import org.slf4j.{Logger, LoggerFactory} import scala.collection.mutable -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.* /** This pass has MethodStubCreator and TypeDeclStubCreator as prerequisite for language frontends which do not provide * method stubs and type decl stubs. */ class ContainsEdgePass(cpg: Cpg) extends ConcurrentWriterCpgPass[AstNode](cpg) { - import ContainsEdgePass._ + import ContainsEdgePass.* + private val logger: Logger = LoggerFactory.getLogger(this.getClass) override def generateParts(): Array[AstNode] = cpg.graph.nodes(sourceTypes*).asScala.map(_.asInstanceOf[AstNode]).toArray override def runOnPart(dstGraph: DiffGraphBuilder, source: AstNode): Unit = { - // AST is assumed to be a tree. If it contains cycles, then this will give a nice endless loop with OOM - val queue = mutable.ArrayDeque[StoredNode](source) - while (queue.nonEmpty) { - val parent = queue.removeHead() - for (nextNode <- parent._astOut) { - if (isDestinationType(nextNode)) dstGraph.addEdge(source, nextNode, EdgeTypes.CONTAINS) - if (!isSourceType(nextNode)) queue.append(nextNode) + try { + // AST is assumed to be a tree. If it contains cycles, then this will give a nice endless loop with OOM + val queue = mutable.ArrayDeque[StoredNode](source) + while (queue.nonEmpty) { + val parent = queue.removeHead() + for (nextNode <- parent._astOut) { + if (isDestinationType(nextNode)) dstGraph.addEdge(source, nextNode, EdgeTypes.CONTAINS) + if (!isSourceType(nextNode)) queue.append(nextNode) + } } + } catch { + case ex: Exception => + logger.warn( + s"Error in ContainsEdgePass for node in file '${source.propertyOption(Properties.FILENAME).toString}''", + ex + ) } } } diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/FileCreationPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/FileCreationPass.scala index 0d860c62bb2a..a6e0c33551ef 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/FileCreationPass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/FileCreationPass.scala @@ -7,6 +7,7 @@ import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes, PropertyN import io.shiftleft.passes.CpgPass import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.types.structure.FileTraversal +import org.slf4j.{Logger, LoggerFactory} import scala.collection.mutable @@ -14,7 +15,8 @@ import scala.collection.mutable * SOURCE_FILE edges. */ class FileCreationPass(cpg: Cpg) extends CpgPass(cpg) with LinkingUtil { - private val srcLabels = List(NodeTypes.NAMESPACE_BLOCK, NodeTypes.TYPE_DECL, NodeTypes.METHOD, NodeTypes.COMMENT) + private val logger: Logger = LoggerFactory.getLogger(this.getClass) + private val srcLabels = List(NodeTypes.NAMESPACE_BLOCK, NodeTypes.TYPE_DECL, NodeTypes.METHOD, NodeTypes.COMMENT) override def run(dstGraph: DiffGraphBuilder): Unit = { val originalFileNameToNode = mutable.Map.empty[String, StoredNode] @@ -38,21 +40,25 @@ class FileCreationPass(cpg: Cpg) extends CpgPass(cpg) with LinkingUtil { dstGraph.addEdge(srcNode, newFile, EdgeTypes.SOURCE_FILE) } } - - // Create SOURCE_FILE edges from nodes of various types to FILE - linkToSingle( - cpg, - srcNodes = cpg.graph.nodes(srcLabels*).toList, - srcLabels = srcLabels, - dstNodeLabel = NodeTypes.FILE, - edgeType = EdgeTypes.SOURCE_FILE, - dstNodeMap = { x => - originalFileNameToNode.get(x) - }, - dstFullNameKey = PropertyNames.FILENAME, - dstGraph, - Some(createFileIfDoesNotExist) - ) + try { + // Create SOURCE_FILE edges from nodes of various types to FILE + linkToSingle( + cpg, + srcNodes = cpg.graph.nodes(srcLabels*).toList, + srcLabels = srcLabels, + dstNodeLabel = NodeTypes.FILE, + edgeType = EdgeTypes.SOURCE_FILE, + dstNodeMap = { x => + originalFileNameToNode.get(x) + }, + dstFullNameKey = PropertyNames.FILENAME, + dstGraph, + Some(createFileIfDoesNotExist) + ) + } catch { + case ex: Exception => + logger.warn(s"Error in FileCreationPass", ex) + } } } diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/MethodDecoratorPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/MethodDecoratorPass.scala index 5ccb872a9a37..faac38be1ee2 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/MethodDecoratorPass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/MethodDecoratorPass.scala @@ -3,7 +3,7 @@ package io.joern.x2cpg.passes.base import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.{EdgeTypes, nodes} import io.shiftleft.passes.CpgPass -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.slf4j.{Logger, LoggerFactory} /** Adds a METHOD_PARAMETER_OUT for each METHOD_PARAMETER_IN to the graph and connects those with a PARAMETER_LINK edge. @@ -18,41 +18,46 @@ class MethodDecoratorPass(cpg: Cpg) extends CpgPass(cpg) { private var loggedMissingTypeFullName = false override def run(dstGraph: DiffGraphBuilder): Unit = { - cpg.parameter.foreach { parameterIn => - if (!parameterIn._parameterLinkOut.hasNext) { - val parameterOut = nodes - .NewMethodParameterOut() - .code(parameterIn.code) - .order(parameterIn.order) - .index(parameterIn.index) - .name(parameterIn.name) - .evaluationStrategy(parameterIn.evaluationStrategy) - .typeFullName(parameterIn.typeFullName) - .isVariadic(parameterIn.isVariadic) - .lineNumber(parameterIn.lineNumber) - .columnNumber(parameterIn.columnNumber) + try { + cpg.parameter.foreach { parameterIn => + if (!parameterIn._parameterLinkOut.hasNext) { + val parameterOut = nodes + .NewMethodParameterOut() + .code(parameterIn.code) + .order(parameterIn.order) + .index(parameterIn.index) + .name(parameterIn.name) + .evaluationStrategy(parameterIn.evaluationStrategy) + .typeFullName(parameterIn.typeFullName) + .isVariadic(parameterIn.isVariadic) + .lineNumber(parameterIn.lineNumber) + .columnNumber(parameterIn.columnNumber) - val method = parameterIn.astIn.headOption - if (method.isEmpty) { - logger.warn("Parameter without method encountered: " + parameterIn.toString) - } else { - if (parameterIn.typeFullName == null) { - val evalType = parameterIn.typ - dstGraph.addEdge(parameterOut, evalType, EdgeTypes.EVAL_TYPE) - if (!loggedMissingTypeFullName) { - logger.warn("Using deprecated CPG format with missing TYPE_FULL_NAME on METHOD_PARAMETER_IN nodes.") - loggedMissingTypeFullName = true + val method = parameterIn.astIn.headOption + if (method.isEmpty) { + logger.warn("Parameter without method encountered: " + parameterIn.toString) + } else { + if (parameterIn.typeFullName == null) { + val evalType = parameterIn.typ + dstGraph.addEdge(parameterOut, evalType, EdgeTypes.EVAL_TYPE) + if (!loggedMissingTypeFullName) { + logger.warn("Using deprecated CPG format with missing TYPE_FULL_NAME on METHOD_PARAMETER_IN nodes.") + loggedMissingTypeFullName = true + } } - } - dstGraph.addNode(parameterOut) - dstGraph.addEdge(method.get, parameterOut, EdgeTypes.AST) - dstGraph.addEdge(parameterIn, parameterOut, EdgeTypes.PARAMETER_LINK) + dstGraph.addNode(parameterOut) + dstGraph.addEdge(method.get, parameterOut, EdgeTypes.AST) + dstGraph.addEdge(parameterIn, parameterOut, EdgeTypes.PARAMETER_LINK) + } + } else if (!loggedDeprecatedWarning) { + logger.warn("Using deprecated CPG format with PARAMETER_LINK edges") + loggedDeprecatedWarning = true } - } else if (!loggedDeprecatedWarning) { - logger.warn("Using deprecated CPG format with PARAMETER_LINK edges") - loggedDeprecatedWarning = true } + } catch { + case ex: Exception => + logger.warn(s"Error in MethodDecoratorPass", ex) } } } diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/MethodStubCreator.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/MethodStubCreator.scala index 4f80594394e3..35b7933eece6 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/MethodStubCreator.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/MethodStubCreator.scala @@ -3,10 +3,11 @@ package io.joern.x2cpg.passes.base import io.joern.x2cpg.Defines import io.joern.x2cpg.passes.base.MethodStubCreator.createMethodStub import io.shiftleft.codepropertygraph.Cpg -import io.shiftleft.codepropertygraph.generated.nodes._ +import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{DispatchTypes, EdgeTypes, EvaluationStrategies, NodeTypes} import io.shiftleft.passes.CpgPass -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* +import org.slf4j.{Logger, LoggerFactory} import overflowdb.BatchedUpdate import overflowdb.BatchedUpdate.DiffGraphBuilder @@ -19,28 +20,34 @@ case class CallSummary(name: String, signature: String, fullName: String, dispat */ class MethodStubCreator(cpg: Cpg) extends CpgPass(cpg) { + private val logger: Logger = LoggerFactory.getLogger(this.getClass) // Since the method fullNames for fuzzyc are not unique, we do not have // a 1to1 relation and may overwrite some values. This is ok for now. private val methodFullNameToNode = mutable.LinkedHashMap[String, Method]() private val methodToParameterCount = mutable.LinkedHashMap[CallSummary, Int]() override def run(dstGraph: BatchedUpdate.DiffGraphBuilder): Unit = { - for (method <- cpg.method) { - methodFullNameToNode.put(method.fullName, method) - } - - for (call <- cpg.call if call.methodFullName != Defines.DynamicCallUnknownFullName) { - methodToParameterCount.put( - CallSummary(call.name, call.signature, call.methodFullName, call.dispatchType), - call.argument.size - ) - } - - for ( - (CallSummary(name, signature, fullName, dispatchType), parameterCount) <- methodToParameterCount - if !methodFullNameToNode.contains(fullName) - ) { - createMethodStub(name, fullName, signature, dispatchType, parameterCount, dstGraph) + try { + for (method <- cpg.method) { + methodFullNameToNode.put(method.fullName, method) + } + + for (call <- cpg.call if call.methodFullName != Defines.DynamicCallUnknownFullName) { + methodToParameterCount.put( + CallSummary(call.name, call.signature, call.methodFullName, call.dispatchType), + call.argument.size + ) + } + + for ( + (CallSummary(name, signature, fullName, dispatchType), parameterCount) <- methodToParameterCount + if !methodFullNameToNode.contains(fullName) + ) { + createMethodStub(name, fullName, signature, dispatchType, parameterCount, dstGraph) + } + } catch { + case ex: Exception => + logger.warn(s"Error in TypeDeclStubCreator", ex) } } diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/NamespaceCreator.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/NamespaceCreator.scala index 8ad95222a9c7..18b028c97954 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/NamespaceCreator.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/NamespaceCreator.scala @@ -4,24 +4,31 @@ import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.EdgeTypes import io.shiftleft.codepropertygraph.generated.nodes.NewNamespace import io.shiftleft.passes.CpgPass -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* +import org.slf4j.{Logger, LoggerFactory} /** Creates NAMESPACE nodes and connects NAMESPACE_BLOCKs to corresponding NAMESPACE nodes. * * This pass has no other pass as prerequisite. */ class NamespaceCreator(cpg: Cpg) extends CpgPass(cpg) { + private val logger: Logger = LoggerFactory.getLogger(this.getClass) /** Creates NAMESPACE nodes and connects NAMESPACE_BLOCKs to corresponding NAMESPACE nodes. */ override def run(dstGraph: DiffGraphBuilder): Unit = { - cpg.namespaceBlock - .groupBy(_.name) - .foreach { case (name: String, blocks) => - val namespace = NewNamespace().name(name) - dstGraph.addNode(namespace) - blocks.foreach(block => dstGraph.addEdge(block, namespace, EdgeTypes.REF)) - } + try { + cpg.namespaceBlock + .groupBy(_.name) + .foreach { case (name: String, blocks) => + val namespace = NewNamespace().name(name) + dstGraph.addNode(namespace) + blocks.foreach(block => dstGraph.addEdge(block, namespace, EdgeTypes.REF)) + } + } catch { + case ex: Exception => + logger.warn(s"Error in NamespaceCreator", ex) + } } } diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/ParameterIndexCompatPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/ParameterIndexCompatPass.scala index 2d088addbe15..ac0aaa6847c9 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/ParameterIndexCompatPass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/ParameterIndexCompatPass.scala @@ -4,18 +4,25 @@ import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.PropertyNames import io.shiftleft.codepropertygraph.generated.nodes.MethodParameterIn.PropertyDefaults import io.shiftleft.passes.CpgPass -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* +import org.slf4j.{Logger, LoggerFactory} /** Old CPGs use the `order` field to indicate the parameter index while newer CPGs use the `parameterIndex` field. This * pass checks whether `parameterIndex` is not set, in which case the value of `order` is copied over. */ class ParameterIndexCompatPass(cpg: Cpg) extends CpgPass(cpg) { + private val logger: Logger = LoggerFactory.getLogger(this.getClass) override def run(diffGraph: DiffGraphBuilder): Unit = { - cpg.parameter.foreach { param => - if (param.index == PropertyDefaults.Index) { - diffGraph.setNodeProperty(param, PropertyNames.INDEX, param.order) + try { + cpg.parameter.foreach { param => + if (param.index == PropertyDefaults.Index) { + diffGraph.setNodeProperty(param, PropertyNames.INDEX, param.order) + } } + } catch { + case ex: Exception => + logger.warn(s"Error in ParameterIndexCompatPass", ex) } } diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeDeclStubCreator.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeDeclStubCreator.scala index 5c7d946efb5f..689cb01648c1 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeDeclStubCreator.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeDeclStubCreator.scala @@ -4,14 +4,15 @@ import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.NodeTypes import io.shiftleft.codepropertygraph.generated.nodes.{NewTypeDecl, TypeDeclBase} import io.shiftleft.passes.CpgPass -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.types.structure.{FileTraversal, NamespaceTraversal} +import org.slf4j.{Logger, LoggerFactory} /** This pass has no other pass as prerequisite. For each `TYPE` node that does not have a corresponding `TYPE_DECL` * node, this pass creates a `TYPE_DECL` node. The `TYPE_DECL` is considered external. */ class TypeDeclStubCreator(cpg: Cpg) extends CpgPass(cpg) { - + private val logger: Logger = LoggerFactory.getLogger(this.getClass) private var typeDeclFullNameToNode = Map[String, TypeDeclBase]() private def privateInit(): Unit = { @@ -22,15 +23,20 @@ class TypeDeclStubCreator(cpg: Cpg) extends CpgPass(cpg) { } override def run(dstGraph: DiffGraphBuilder): Unit = { - privateInit() - - cpg.typ - .filterNot(typ => typeDeclFullNameToNode.isDefinedAt(typ.fullName)) - .foreach { typ => - val newTypeDecl = TypeDeclStubCreator.createTypeDeclStub(typ.name, typ.fullName) - typeDeclFullNameToNode += typ.fullName -> newTypeDecl - dstGraph.addNode(newTypeDecl) - } + try { + privateInit() + + cpg.typ + .filterNot(typ => typeDeclFullNameToNode.isDefinedAt(typ.fullName)) + .foreach { typ => + val newTypeDecl = TypeDeclStubCreator.createTypeDeclStub(typ.name, typ.fullName) + typeDeclFullNameToNode += typ.fullName -> newTypeDecl + dstGraph.addNode(newTypeDecl) + } + } catch { + case ex: Exception => + logger.warn(s"Error in TypeDeclStubCreator", ex) + } } } diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeEvalPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeEvalPass.scala index 43c997914661..185fe4a9f0b6 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeEvalPass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeEvalPass.scala @@ -4,10 +4,13 @@ import io.joern.x2cpg.utils.LinkingUtil import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes, PropertyNames} import io.shiftleft.passes.ForkJoinParallelCpgPass +import org.slf4j.{Logger, LoggerFactory} import overflowdb.Node import overflowdb.traversal.* class TypeEvalPass(cpg: Cpg) extends ForkJoinParallelCpgPass[List[Node]](cpg) with LinkingUtil { + private val logger: Logger = LoggerFactory.getLogger(this.getClass) + val srcLabels = List( NodeTypes.METHOD_PARAMETER_IN, NodeTypes.METHOD_PARAMETER_OUT, @@ -28,16 +31,21 @@ class TypeEvalPass(cpg: Cpg) extends ForkJoinParallelCpgPass[List[Node]](cpg) wi nodes.grouped(getBatchSize(nodes.size)).toArray } def runOnPart(builder: DiffGraphBuilder, part: List[overflowdb.Node]): Unit = { - linkToSingle( - cpg = cpg, - srcNodes = part, - srcLabels = srcLabels, - dstNodeLabel = NodeTypes.TYPE, - edgeType = EdgeTypes.EVAL_TYPE, - dstNodeMap = typeFullNameToNode(cpg, _), - dstFullNameKey = PropertyNames.TYPE_FULL_NAME, - dstGraph = builder, - None - ) + try { + linkToSingle( + cpg = cpg, + srcNodes = part, + srcLabels = srcLabels, + dstNodeLabel = NodeTypes.TYPE, + edgeType = EdgeTypes.EVAL_TYPE, + dstNodeMap = typeFullNameToNode(cpg, _), + dstFullNameKey = PropertyNames.TYPE_FULL_NAME, + dstGraph = builder, + None + ) + } catch { + case ex: Exception => + logger.warn(s"Error in TypeEvalPass", ex) + } } } diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeRefPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeRefPass.scala index 073859f8de3d..6121a9b1e6b9 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeRefPass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/TypeRefPass.scala @@ -4,27 +4,33 @@ import io.joern.x2cpg.utils.LinkingUtil import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes, PropertyNames} import io.shiftleft.passes.ForkJoinParallelCpgPass +import org.slf4j.{Logger, LoggerFactory} import overflowdb.Node import overflowdb.traversal.* class TypeRefPass(cpg: Cpg) extends ForkJoinParallelCpgPass[List[Node]](cpg) with LinkingUtil { - val srcLabels = List(NodeTypes.TYPE) - + val srcLabels = List(NodeTypes.TYPE) + private val logger: Logger = LoggerFactory.getLogger(this.getClass) def generateParts(): Array[List[Node]] = { val nodes = cpg.graph.nodes(srcLabels*).toList nodes.grouped(getBatchSize(nodes.size)).toArray } def runOnPart(builder: DiffGraphBuilder, part: List[overflowdb.Node]): Unit = { - linkToSingle( - cpg = cpg, - srcNodes = part, - srcLabels = srcLabels, - dstNodeLabel = NodeTypes.TYPE_DECL, - edgeType = EdgeTypes.REF, - dstNodeMap = typeDeclFullNameToNode(cpg, _), - dstFullNameKey = PropertyNames.TYPE_DECL_FULL_NAME, - dstGraph = builder, - None - ) + try { + linkToSingle( + cpg = cpg, + srcNodes = part, + srcLabels = srcLabels, + dstNodeLabel = NodeTypes.TYPE_DECL, + edgeType = EdgeTypes.REF, + dstNodeMap = typeDeclFullNameToNode(cpg, _), + dstFullNameKey = PropertyNames.TYPE_DECL_FULL_NAME, + dstGraph = builder, + None + ) + } catch { + case ex: Exception => + logger.warn(s"Error in TypeRefPass", ex) + } } } diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/callgraph/DynamicCallLinker.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/callgraph/DynamicCallLinker.scala index 2778e653fe0b..658307af8f13 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/callgraph/DynamicCallLinker.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/callgraph/DynamicCallLinker.scala @@ -5,12 +5,12 @@ import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.nodes.{Call, Method, TypeDecl} import io.shiftleft.codepropertygraph.generated.{DispatchTypes, EdgeTypes, PropertyNames} import io.shiftleft.passes.CpgPass -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.slf4j.{Logger, LoggerFactory} import overflowdb.{NodeDb, NodeRef} import scala.collection.mutable -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.* /** We compute the set of possible call-targets for each dynamic call, and add them as CALL edges to the graph, based on * call.methodFullName, method.name and method.signature, the inheritance hierarchy and the AST of typedecls and @@ -23,8 +23,7 @@ import scala.jdk.CollectionConverters._ * Virtual Calls from Memory Corruption Attacks. 10.14722/ndss.2014.23287. */ class DynamicCallLinker(cpg: Cpg) extends CpgPass(cpg) { - - import DynamicCallLinker._ + import DynamicCallLinker.* // Used to track potential method candidates for a given method fullname. Since our method full names contain the type // decl we don't need to specify an addition map to wrap this in. LinkedHashSets are used here to preserve order in // the best interest of reproducibility during debugging. @@ -50,30 +49,35 @@ class DynamicCallLinker(cpg: Cpg) extends CpgPass(cpg) { */ override def run(dstGraph: DiffGraphBuilder): Unit = { // Perform early stopping in the case of no virtual calls - if (!cpg.call.exists(_.dispatchType == DispatchTypes.DYNAMIC_DISPATCH)) { - return - } - initMaps() - // ValidM maps class C and method name N to the set of - // func ptrs implementing N for C and its subclasses - for ( - typeDecl <- cpg.typeDecl; - method <- typeDecl._methodViaAstOut - ) { - val methodName = method.fullName - val candidates = allSubclasses(typeDecl.fullName).flatMap { staticLookup(_, method) } - validM.put(methodName, candidates) - } + try { + if (!cpg.call.exists(_.dispatchType == DispatchTypes.DYNAMIC_DISPATCH)) { + return + } + initMaps() + // ValidM maps class C and method name N to the set of + // func ptrs implementing N for C and its subclasses + for ( + typeDecl <- cpg.typeDecl; + method <- typeDecl._methodViaAstOut + ) { + val methodName = method.fullName + val candidates = allSubclasses(typeDecl.fullName).flatMap { staticLookup(_, method) } + validM.put(methodName, candidates) + } - subclassCache.clear() + subclassCache.clear() - cpg.call.filter(_.dispatchType == DispatchTypes.DYNAMIC_DISPATCH).foreach { call => - try { - linkDynamicCall(call, dstGraph) - } catch { - case exception: Exception => - throw new RuntimeException(exception) + cpg.call.filter(_.dispatchType == DispatchTypes.DYNAMIC_DISPATCH).foreach { call => + try { + linkDynamicCall(call, dstGraph) + } catch { + case exception: Exception => + throw new RuntimeException(exception) + } } + } catch { + case ex: Exception => + logger.warn(s"Error in DynamicCallLinker", ex) } } diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/callgraph/MethodRefLinker.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/callgraph/MethodRefLinker.scala index f828d084a59c..a942d51c0223 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/callgraph/MethodRefLinker.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/callgraph/MethodRefLinker.scala @@ -4,28 +4,35 @@ import io.joern.x2cpg.utils.LinkingUtil import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.* import io.shiftleft.passes.CpgPass +import org.slf4j.LoggerFactory import overflowdb.traversal.* /** This pass has MethodStubCreator and TypeDeclStubCreator as prerequisite for language frontends which do not provide * method stubs and type decl stubs. */ class MethodRefLinker(cpg: Cpg) extends CpgPass(cpg) with LinkingUtil { + private val logger = LoggerFactory.getLogger(getClass) private val srcLabels = List(NodeTypes.METHOD_REF) override def run(dstGraph: DiffGraphBuilder): Unit = { - // Create REF edges from METHOD_REFs to METHOD - linkToSingle( - cpg, - srcNodes = cpg.graph.nodes(srcLabels*).toList, - srcLabels = List(NodeTypes.METHOD_REF), - dstNodeLabel = NodeTypes.METHOD, - edgeType = EdgeTypes.REF, - dstNodeMap = methodFullNameToNode(cpg, _), - dstFullNameKey = PropertyNames.METHOD_FULL_NAME, - dstGraph, - None - ) + try { + // Create REF edges from METHOD_REFs to METHOD + linkToSingle( + cpg, + srcNodes = cpg.graph.nodes(srcLabels*).toList, + srcLabels = List(NodeTypes.METHOD_REF), + dstNodeLabel = NodeTypes.METHOD, + edgeType = EdgeTypes.REF, + dstNodeMap = methodFullNameToNode(cpg, _), + dstFullNameKey = PropertyNames.METHOD_FULL_NAME, + dstGraph, + None + ) + } catch { + case ex: Exception => + logger.warn(s"Error in MethodRefLinker", ex) + } } } diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/CfgCreationPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/CfgCreationPass.scala index 2507a470f33c..67afd3fed686 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/CfgCreationPass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/CfgCreationPass.scala @@ -3,8 +3,9 @@ package io.joern.x2cpg.passes.controlflow import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.nodes.Method import io.shiftleft.passes.ConcurrentWriterCpgPass -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import io.joern.x2cpg.passes.controlflow.cfgcreation.CfgCreator +import org.slf4j.LoggerFactory /** A pass that creates control flow graphs from abstract syntax trees. * @@ -15,13 +16,17 @@ import io.joern.x2cpg.passes.controlflow.cfgcreation.CfgCreator * pass only creates edges at the moment. Therefore, we currently do without key pools. */ class CfgCreationPass(cpg: Cpg) extends ConcurrentWriterCpgPass[Method](cpg) { - + private val logger = LoggerFactory.getLogger(getClass) override def generateParts(): Array[Method] = cpg.method.toArray override def runOnPart(diffGraph: DiffGraphBuilder, method: Method): Unit = { val localDiff = new DiffGraphBuilder - new CfgCreator(method, localDiff).run() - diffGraph.absorb(localDiff) + try { + new CfgCreator(method, localDiff).run() + diffGraph.absorb(localDiff) + } catch { + case ex: Exception => + logger.error(s"Error for the METHOD node -> '${method.fullName}' in file '${method.filename}'") + } } - } diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/cfgdominator/CfgDominatorPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/cfgdominator/CfgDominatorPass.scala index b5908077d0e7..c6861e6cf95c 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/cfgdominator/CfgDominatorPass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/cfgdominator/CfgDominatorPass.scala @@ -4,27 +4,34 @@ import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.EdgeTypes import io.shiftleft.codepropertygraph.generated.nodes.{Method, StoredNode} import io.shiftleft.passes.ForkJoinParallelCpgPass -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* +import org.slf4j.LoggerFactory import scala.collection.mutable /** This pass has no prerequisites. */ class CfgDominatorPass(cpg: Cpg) extends ForkJoinParallelCpgPass[Method](cpg) { + private val logger = LoggerFactory.getLogger(getClass) override def generateParts(): Array[Method] = cpg.method.toArray override def runOnPart(dstGraph: DiffGraphBuilder, method: Method): Unit = { - val cfgAdapter = new CpgCfgAdapter() - val dominatorCalculator = new CfgDominator(cfgAdapter) + try { + val cfgAdapter = new CpgCfgAdapter() + val dominatorCalculator = new CfgDominator(cfgAdapter) - val reverseCfgAdapter = new ReverseCpgCfgAdapter() - val postDominatorCalculator = new CfgDominator(reverseCfgAdapter) + val reverseCfgAdapter = new ReverseCpgCfgAdapter() + val postDominatorCalculator = new CfgDominator(reverseCfgAdapter) - val cfgNodeToImmediateDominator = dominatorCalculator.calculate(method) - addDomTreeEdges(dstGraph, cfgNodeToImmediateDominator) + val cfgNodeToImmediateDominator = dominatorCalculator.calculate(method) + addDomTreeEdges(dstGraph, cfgNodeToImmediateDominator) - val cfgNodeToPostImmediateDominator = postDominatorCalculator.calculate(method.methodReturn) - addPostDomTreeEdges(dstGraph, cfgNodeToPostImmediateDominator) + val cfgNodeToPostImmediateDominator = postDominatorCalculator.calculate(method.methodReturn) + addPostDomTreeEdges(dstGraph, cfgNodeToPostImmediateDominator) + } catch { + case ex: Exception => + logger.warn(s"Error for the METHOD node -> '${method.fullName}' in file '${method.filename}'") + } } diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/codepencegraph/CdgPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/codepencegraph/CdgPass.scala index fb42b0e5ffa3..ee0ca054a230 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/codepencegraph/CdgPass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/controlflow/codepencegraph/CdgPass.scala @@ -25,34 +25,38 @@ class CdgPass(cpg: Cpg) extends ForkJoinParallelCpgPass[Method](cpg) { override def generateParts(): Array[Method] = cpg.method.toArray override def runOnPart(dstGraph: DiffGraphBuilder, method: Method): Unit = { + try { + val dominanceFrontier = new CfgDominatorFrontier(new ReverseCpgCfgAdapter, new CpgPostDomTreeAdapter) - val dominanceFrontier = new CfgDominatorFrontier(new ReverseCpgCfgAdapter, new CpgPostDomTreeAdapter) + val cfgNodes = method._containsOut.toList + val postDomFrontiers = dominanceFrontier.calculate(method :: cfgNodes) - val cfgNodes = method._containsOut.toList - val postDomFrontiers = dominanceFrontier.calculate(method :: cfgNodes) - - postDomFrontiers.foreach { case (node, postDomFrontierNodes) => - postDomFrontierNodes.foreach { - case postDomFrontierNode @ (_: Literal | _: Identifier | _: Call | _: MethodRef | _: Unknown | - _: ControlStructure | _: JumpTarget) => - dstGraph.addEdge(postDomFrontierNode, node, EdgeTypes.CDG) - case postDomFrontierNode => - val nodeLabel = postDomFrontierNode.label - val containsIn = postDomFrontierNode._containsIn - if (containsIn == null || !containsIn.hasNext) { - logger.warn(s"Found CDG edge starting at $nodeLabel node. This is most likely caused by an invalid CFG.") - } else { - val method = containsIn.next() - logger.warn( - s"Found CDG edge starting at $nodeLabel node. This is most likely caused by an invalid CFG." + - s" Method: ${method match { - case m: Method => m.fullName; - case other => other.label - }}" + - s" number of outgoing CFG edges from $nodeLabel node: ${postDomFrontierNode._cfgOut.size}" - ) - } + postDomFrontiers.foreach { case (node, postDomFrontierNodes) => + postDomFrontierNodes.foreach { + case postDomFrontierNode @ (_: Literal | _: Identifier | _: Call | _: MethodRef | _: Unknown | + _: ControlStructure | _: JumpTarget) => + dstGraph.addEdge(postDomFrontierNode, node, EdgeTypes.CDG) + case postDomFrontierNode => + val nodeLabel = postDomFrontierNode.label + val containsIn = postDomFrontierNode._containsIn + if (containsIn == null || !containsIn.hasNext) { + logger.warn(s"Found CDG edge starting at $nodeLabel node. This is most likely caused by an invalid CFG.") + } else { + val method = containsIn.next() + logger.warn( + s"Found CDG edge starting at $nodeLabel node. This is most likely caused by an invalid CFG." + + s" Method: ${method match { + case m: Method => m.fullName; + case other => other.label + }}" + + s" number of outgoing CFG edges from $nodeLabel node: ${postDomFrontierNode._cfgOut.size}" + ) + } + } } + } catch { + case ex: Exception => + logger.warn(s"Error for the METHOD node -> '${method.fullName}' in file '${method.filename}'") } } } diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/AliasLinkerPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/AliasLinkerPass.scala index b8f92f48d30a..ebc48afe7be3 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/AliasLinkerPass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/AliasLinkerPass.scala @@ -1,27 +1,33 @@ package io.joern.x2cpg.passes.typerelations +import io.joern.x2cpg.utils.LinkingUtil import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.nodes.TypeDecl import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes, PropertyNames} import io.shiftleft.passes.CpgPass -import io.joern.x2cpg.utils.LinkingUtil +import org.slf4j.{Logger, LoggerFactory} class AliasLinkerPass(cpg: Cpg) extends CpgPass(cpg) with LinkingUtil { - + private val logger: Logger = LoggerFactory.getLogger(this.getClass) override def run(dstGraph: DiffGraphBuilder): Unit = { // Create ALIAS_OF edges from TYPE_DECL nodes to TYPE - linkToMultiple( - cpg, - srcLabels = List(NodeTypes.TYPE_DECL), - dstNodeLabel = NodeTypes.TYPE, - edgeType = EdgeTypes.ALIAS_OF, - dstNodeMap = typeFullNameToNode(cpg, _), - getDstFullNames = (srcNode: TypeDecl) => { - srcNode.aliasTypeFullName - }, - dstFullNameKey = PropertyNames.ALIAS_TYPE_FULL_NAME, - dstGraph - ) + try { + linkToMultiple( + cpg, + srcLabels = List(NodeTypes.TYPE_DECL), + dstNodeLabel = NodeTypes.TYPE, + edgeType = EdgeTypes.ALIAS_OF, + dstNodeMap = typeFullNameToNode(cpg, _), + getDstFullNames = (srcNode: TypeDecl) => { + srcNode.aliasTypeFullName + }, + dstFullNameKey = PropertyNames.ALIAS_TYPE_FULL_NAME, + dstGraph + ) + } catch { + case ex: Exception => + logger.warn(s"Error in AliasLinkerPass", ex) + } } } diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/FieldAccessLinkerPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/FieldAccessLinkerPass.scala index e60479b1be34..c9d6f7e26261 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/FieldAccessLinkerPass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/FieldAccessLinkerPass.scala @@ -2,12 +2,11 @@ package io.joern.x2cpg.passes.typerelations import io.joern.x2cpg.passes.frontend.Dereference import io.joern.x2cpg.utils.LinkingUtil -import io.shiftleft.codepropertygraph.generated.nodes.{Call, Member, StoredNode} import io.shiftleft.codepropertygraph.generated.* +import io.shiftleft.codepropertygraph.generated.nodes.{Call, Member, StoredNode} import io.shiftleft.passes.CpgPass import io.shiftleft.semanticcpg.language.* import io.shiftleft.semanticcpg.language.operatorextension.{OpNodes, allFieldAccessTypes} -import io.shiftleft.semanticcpg.utils.MemberAccess import org.slf4j.LoggerFactory import scala.jdk.CollectionConverters.* @@ -20,16 +19,21 @@ class FieldAccessLinkerPass(cpg: Cpg) extends CpgPass(cpg) with LinkingUtil { private val DOT = "." override def run(dstGraph: DiffGraphBuilder): Unit = { - linkToMultiple( - cpg, - srcLabels = List(NodeTypes.CALL), - dstNodeLabel = NodeTypes.MEMBER, - edgeType = EdgeTypes.REF, - dstNodeMap = typeDeclMemberToNode(cpg, _), - getDstFullNames = (call: Call) => dstMemberFullNames(call), - dstFullNameKey = PropertyNames.NAME, - dstGraph - ) + try { + linkToMultiple( + cpg, + srcLabels = List(NodeTypes.CALL), + dstNodeLabel = NodeTypes.MEMBER, + edgeType = EdgeTypes.REF, + dstNodeMap = typeDeclMemberToNode(cpg, _), + getDstFullNames = (call: Call) => dstMemberFullNames(call), + dstFullNameKey = PropertyNames.NAME, + dstGraph + ) + } catch { + case ex: Exception => + logger.warn(s"Error in FieldAccessLinkerPass", ex) + } } private def dstMemberFullNames(call: Call): Seq[String] = { diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/TypeHierarchyPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/TypeHierarchyPass.scala index 16a0d0f3e95c..c6abd6ca0f94 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/TypeHierarchyPass.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/typerelations/TypeHierarchyPass.scala @@ -1,32 +1,38 @@ package io.joern.x2cpg.passes.typerelations +import io.joern.x2cpg.utils.LinkingUtil import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.nodes.TypeDecl import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes, PropertyNames} import io.shiftleft.passes.CpgPass -import io.joern.x2cpg.utils.LinkingUtil +import org.slf4j.{Logger, LoggerFactory} /** Create INHERITS_FROM edges from `TYPE_DECL` nodes to `TYPE` nodes. */ class TypeHierarchyPass(cpg: Cpg) extends CpgPass(cpg) with LinkingUtil { - + private val logger: Logger = LoggerFactory.getLogger(this.getClass) override def run(dstGraph: DiffGraphBuilder): Unit = { - linkToMultiple( - cpg, - srcLabels = List(NodeTypes.TYPE_DECL), - dstNodeLabel = NodeTypes.TYPE, - edgeType = EdgeTypes.INHERITS_FROM, - dstNodeMap = typeFullNameToNode(cpg, _), - getDstFullNames = (srcNode: TypeDecl) => { - if (srcNode.inheritsFromTypeFullName != null) { - srcNode.inheritsFromTypeFullName - } else { - Seq() - } - }, - dstFullNameKey = PropertyNames.INHERITS_FROM_TYPE_FULL_NAME, - dstGraph - ) + try { + linkToMultiple( + cpg, + srcLabels = List(NodeTypes.TYPE_DECL), + dstNodeLabel = NodeTypes.TYPE, + edgeType = EdgeTypes.INHERITS_FROM, + dstNodeMap = typeFullNameToNode(cpg, _), + getDstFullNames = (srcNode: TypeDecl) => { + if (srcNode.inheritsFromTypeFullName != null) { + srcNode.inheritsFromTypeFullName + } else { + Seq() + } + }, + dstFullNameKey = PropertyNames.INHERITS_FROM_TYPE_FULL_NAME, + dstGraph + ) + } catch { + case ex: Exception => + logger.warn(s"Error in TypeHierarchyPass", ex) + } } } diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/LinkingUtil.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/LinkingUtil.scala index 321b08b4fbc8..57885e871e2b 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/LinkingUtil.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/LinkingUtil.scala @@ -61,46 +61,54 @@ trait LinkingUtil { srcNodes.foreach { srcNode => // If the source node does not have any outgoing edges of this type // This check is just required for backward compatibility - if (srcNode.outE(edgeType).isEmpty) { - val key = new PropertyKey[String](dstFullNameKey) - srcNode - .propertyOption(key) - .filter { dstFullName => - val dereferenceDstFullName = dereference.dereferenceTypeFullName(dstFullName) - srcNode.propertyDefaultValue(dstFullNameKey) != dereferenceDstFullName - } - .ifPresent { dstFullName => - // for `UNKNOWN` this is not always set, so we're using an Option here - val srcStoredNode = srcNode.asInstanceOf[StoredNode] - val dereferenceDstFullName = dereference.dereferenceTypeFullName(dstFullName) - dstNodeMap(dereferenceDstFullName) match { - case Some(dstNode) => - dstGraph.addEdge(srcStoredNode, dstNode, edgeType) - case None if dstNodeMap(dstFullName).isDefined => - dstGraph.addEdge(srcStoredNode, dstNodeMap(dstFullName).get, edgeType) - case None if dstNotExistsHandler.isDefined => - dstNotExistsHandler.get(srcStoredNode, dereferenceDstFullName) - case _ => - logFailedDstLookup(edgeType, srcNode.label, srcNode.id.toString, dstNodeLabel, dereferenceDstFullName) + try { + if (srcNode.outE(edgeType).isEmpty) { + val key = new PropertyKey[String](dstFullNameKey) + srcNode + .propertyOption(key) + .filter { dstFullName => + val dereferenceDstFullName = dereference.dereferenceTypeFullName(dstFullName) + srcNode.propertyDefaultValue(dstFullNameKey) != dereferenceDstFullName + } + .ifPresent { dstFullName => + // for `UNKNOWN` this is not always set, so we're using an Option here + val srcStoredNode = srcNode.asInstanceOf[StoredNode] + val dereferenceDstFullName = dereference.dereferenceTypeFullName(dstFullName) + dstNodeMap(dereferenceDstFullName) match { + case Some(dstNode) => + dstGraph.addEdge(srcStoredNode, dstNode, edgeType) + case None if dstNodeMap(dstFullName).isDefined => + dstGraph.addEdge(srcStoredNode, dstNodeMap(dstFullName).get, edgeType) + case None if dstNotExistsHandler.isDefined => + dstNotExistsHandler.get(srcStoredNode, dereferenceDstFullName) + case _ => + logFailedDstLookup(edgeType, srcNode.label, srcNode.id.toString, dstNodeLabel, dereferenceDstFullName) + } } + } else { + srcNode.out(edgeType).property(Properties.FULL_NAME).nextOption() match { + case Some(dstFullName) => + dstGraph.setNodeProperty( + srcNode.asInstanceOf[StoredNode], + dstFullNameKey, + dereference.dereferenceTypeFullName(dstFullName) + ) + case None => logger.info(s"Missing outgoing edge of type $edgeType from node $srcNode") } - } else { - srcNode.out(edgeType).property(Properties.FULL_NAME).nextOption() match { - case Some(dstFullName) => - dstGraph.setNodeProperty( - srcNode.asInstanceOf[StoredNode], - dstFullNameKey, - dereference.dereferenceTypeFullName(dstFullName) + if (!loggedDeprecationWarning) { + logger.info( + s"Using deprecated CPG format with already existing $edgeType edge between" + + s" a source node of type $srcLabels and a $dstNodeLabel node." ) - case None => logger.info(s"Missing outgoing edge of type $edgeType from node $srcNode") + loggedDeprecationWarning = true + } } - if (!loggedDeprecationWarning) { - logger.info( - s"Using deprecated CPG format with already existing $edgeType edge between" + - s" a source node of type $srcLabels and a $dstNodeLabel node." + } catch { + case ex: Exception => + logger.warn( + s"Error in linkToSingle for node in file '${srcNode.propertyOption(Properties.FILENAME).toString}''", + ex ) - loggedDeprecationWarning = true - } } } } @@ -118,28 +126,36 @@ trait LinkingUtil { var loggedDeprecationWarning = false val dereference = Dereference(cpg) cpg.graph.nodes(srcLabels*).asScala.cast[SRC_NODE_TYPE].foreach { srcNode => - if (!srcNode.outE(edgeType).hasNext) { - getDstFullNames(srcNode).foreach { dstFullName => - val dereferenceDstFullName = dereference.dereferenceTypeFullName(dstFullName) - dstNodeMap(dereferenceDstFullName) match { - case Some(dstNode) => - dstGraph.addEdge(srcNode, dstNode, edgeType) - case None if dstNodeMap(dstFullName).isDefined => - dstGraph.addEdge(srcNode, dstNodeMap(dstFullName).get, edgeType) - case None => - logFailedDstLookup(edgeType, srcNode.label, srcNode.id.toString, dstNodeLabel, dereferenceDstFullName) + try { + if (!srcNode.outE(edgeType).hasNext) { + getDstFullNames(srcNode).foreach { dstFullName => + val dereferenceDstFullName = dereference.dereferenceTypeFullName(dstFullName) + dstNodeMap(dereferenceDstFullName) match { + case Some(dstNode) => + dstGraph.addEdge(srcNode, dstNode, edgeType) + case None if dstNodeMap(dstFullName).isDefined => + dstGraph.addEdge(srcNode, dstNodeMap(dstFullName).get, edgeType) + case None => + logFailedDstLookup(edgeType, srcNode.label, srcNode.id.toString, dstNodeLabel, dereferenceDstFullName) + } + } + } else { + val dstFullNames = srcNode.out(edgeType).property(Properties.FULL_NAME).l + dstGraph.setNodeProperty(srcNode, dstFullNameKey, dstFullNames.map(dereference.dereferenceTypeFullName)) + if (!loggedDeprecationWarning) { + logger.info( + s"Using deprecated CPG format with already existing $edgeType edge between" + + s" a source node of type $srcLabels and a $dstNodeLabel node." + ) + loggedDeprecationWarning = true } } - } else { - val dstFullNames = srcNode.out(edgeType).property(Properties.FULL_NAME).l - dstGraph.setNodeProperty(srcNode, dstFullNameKey, dstFullNames.map(dereference.dereferenceTypeFullName)) - if (!loggedDeprecationWarning) { - logger.info( - s"Using deprecated CPG format with already existing $edgeType edge between" + - s" a source node of type $srcLabels and a $dstNodeLabel node." + } catch { + case ex: Exception => + logger.warn( + s"Error in linkToMultiple for node in file '${srcNode.propertyOption(Properties.FILENAME).toString}''", + ex ) - loggedDeprecationWarning = true - } } } }