Error handling in passes (#142)

* Error handling in passes * Review comment fixes
Privado-Inc · Nov 18, 2024 · 9746a9a · 9746a9a
1 parent 147def9
commit 9746a9a
Show file tree

Hide file tree

Showing 20 changed files with 479 additions and 339 deletions.
diff --git a/...ineoss/src/main/scala/io/joern/dataflowengineoss/passes/reachingdef/ReachingDefPass.scala b/...ineoss/src/main/scala/io/joern/dataflowengineoss/passes/reachingdef/ReachingDefPass.scala
@@ -22,15 +22,20 @@ class ReachingDefPass(cpg: Cpg, maxNumberOfDefinitions: Int = 4000)(implicit s:
 
   override def runOnPart(dstGraph: DiffGraphBuilder, method: Method): Unit = {
     logger.info("Calculating reaching definitions for: {} in {}", method.fullName, method.filename)
-    val problem = ReachingDefProblem.create(method)
-    if (shouldBailOut(method, problem)) {
-      logger.warn("Skipping.")
-      return
+    try {
+      val problem = ReachingDefProblem.create(method)
+      if (shouldBailOut(method, problem)) {
+        logger.warn("Skipping.")
+        return
+      }
+
+      val solution     = new DataFlowSolver().calculateMopSolutionForwards(problem)
+      val ddgGenerator = new DdgGenerator(s)
+      ddgGenerator.addReachingDefEdges(dstGraph, method, problem, solution)
+    } catch {
+      case ex: Exception =>
+        logger.warn(s"Error for the METHOD node -> '${method.fullName}' in file '${method.filename}'")
     }
-
-    val solution     = new DataFlowSolver().calculateMopSolutionForwards(problem)
-    val ddgGenerator = new DdgGenerator(s)
-    ddgGenerator.addReachingDefEdges(dstGraph, method, problem, solution)
   }
 
   /** Before we start propagating definitions in the graph, which is the bulk of the work, we check how many definitions

diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/AstLinkerPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/AstLinkerPass.scala
@@ -3,27 +3,33 @@ package io.joern.x2cpg.passes.base
 import io.joern.x2cpg.utils.LinkingUtil
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.generated.nodes.StoredNode
-import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes}
+import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes, Properties}
 import io.shiftleft.passes.CpgPass
 import io.shiftleft.semanticcpg.language.*
+import org.slf4j.{Logger, LoggerFactory}
 
 class AstLinkerPass(cpg: Cpg) extends CpgPass(cpg) with LinkingUtil {
-
+  private val logger: Logger = LoggerFactory.getLogger(this.getClass)
   override def run(dstGraph: DiffGraphBuilder): Unit = {
-    cpg.method.whereNot(_.astParent).foreach { method =>
-      addAstParent(method, method.fullName, method.astParentType, method.astParentFullName, dstGraph)
-    }
-    cpg.typeDecl.whereNot(_.astParent).foreach { typeDecl =>
-      addAstParent(typeDecl, typeDecl.fullName, typeDecl.astParentType, typeDecl.astParentFullName, dstGraph)
-    }
-    cpg.member.whereNot(_.astParent).foreach { member =>
-      addAstParent(
-        member,
-        s"${member.astParentFullName}.${member.name}",
-        member.astParentType,
-        member.astParentFullName,
-        dstGraph
-      )
+    try {
+      cpg.method.whereNot(_.astParent).foreach { method =>
+        addAstParent(method, method.fullName, method.astParentType, method.astParentFullName, dstGraph)
+      }
+      cpg.typeDecl.whereNot(_.astParent).foreach { typeDecl =>
+        addAstParent(typeDecl, typeDecl.fullName, typeDecl.astParentType, typeDecl.astParentFullName, dstGraph)
+      }
+      cpg.member.whereNot(_.astParent).foreach { member =>
+        addAstParent(
+          member,
+          s"${member.astParentFullName}.${member.name}",
+          member.astParentType,
+          member.astParentFullName,
+          dstGraph
+        )
+      }
+    } catch {
+      case ex: Exception =>
+        logger.warn(s"Error in AstLinkerPass", ex)
     }
   }
 
@@ -38,25 +44,33 @@ class AstLinkerPass(cpg: Cpg) extends CpgPass(cpg) with LinkingUtil {
     astParentFullName: String,
     dstGraph: DiffGraphBuilder
   ): Unit = {
-    val astParentOption: Option[StoredNode] =
-      astParentType match {
-        case NodeTypes.METHOD          => methodFullNameToNode(cpg, astParentFullName)
-        case NodeTypes.TYPE_DECL       => typeDeclFullNameToNode(cpg, astParentFullName)
-        case NodeTypes.NAMESPACE_BLOCK => namespaceBlockFullNameToNode(cpg, astParentFullName)
-        case _ =>
-          logger.warn(
-            s"Invalid AST_PARENT_TYPE=$astParentFullName;" +
-              s" astChild LABEL=${astChild.label};" +
-              s" astChild FULL_NAME=$astChildFullName"
-          )
-          None
-      }
+    try {
+      val astParentOption: Option[StoredNode] =
+        astParentType match {
+          case NodeTypes.METHOD          => methodFullNameToNode(cpg, astParentFullName)
+          case NodeTypes.TYPE_DECL       => typeDeclFullNameToNode(cpg, astParentFullName)
+          case NodeTypes.NAMESPACE_BLOCK => namespaceBlockFullNameToNode(cpg, astParentFullName)
+          case _ =>
+            logger.warn(
+              s"Invalid AST_PARENT_TYPE=$astParentFullName;" +
+                s" astChild LABEL=${astChild.label};" +
+                s" astChild FULL_NAME=$astChildFullName"
+            )
+            None
+        }
 
-    astParentOption match {
-      case Some(astParent) =>
-        dstGraph.addEdge(astParent, astChild, EdgeTypes.AST)
-      case None =>
-        logFailedSrcLookup(EdgeTypes.AST, astParentType, astParentFullName, astChild.label, astChild.id.toString)
+      astParentOption match {
+        case Some(astParent) =>
+          dstGraph.addEdge(astParent, astChild, EdgeTypes.AST)
+        case None =>
+          logFailedSrcLookup(EdgeTypes.AST, astParentType, astParentFullName, astChild.label, astChild.id.toString)
+      }
+    } catch {
+      case ex: Exception =>
+        logger.warn(
+          s"Error in AstLinkerPass for node in file '${astChild.propertyOption(Properties.FILENAME).toString}''",
+          ex
+        )
     }
   }
 }
diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/ContainsEdgePass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/ContainsEdgePass.scala
@@ -1,31 +1,41 @@
 package io.joern.x2cpg.passes.base
 
 import io.shiftleft.codepropertygraph.Cpg
-import io.shiftleft.codepropertygraph.generated.nodes._
-import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes}
+import io.shiftleft.codepropertygraph.generated.nodes.*
+import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes, Properties}
 import io.shiftleft.passes.ConcurrentWriterCpgPass
+import org.slf4j.{Logger, LoggerFactory}
 
 import scala.collection.mutable
-import scala.jdk.CollectionConverters._
+import scala.jdk.CollectionConverters.*
 
 /** This pass has MethodStubCreator and TypeDeclStubCreator as prerequisite for language frontends which do not provide
   * method stubs and type decl stubs.
   */
 class ContainsEdgePass(cpg: Cpg) extends ConcurrentWriterCpgPass[AstNode](cpg) {
-  import ContainsEdgePass._
+  import ContainsEdgePass.*
 
+  private val logger: Logger = LoggerFactory.getLogger(this.getClass)
   override def generateParts(): Array[AstNode] =
     cpg.graph.nodes(sourceTypes*).asScala.map(_.asInstanceOf[AstNode]).toArray
 
   override def runOnPart(dstGraph: DiffGraphBuilder, source: AstNode): Unit = {
-    // AST is assumed to be a tree. If it contains cycles, then this will give a nice endless loop with OOM
-    val queue = mutable.ArrayDeque[StoredNode](source)
-    while (queue.nonEmpty) {
-      val parent = queue.removeHead()
-      for (nextNode <- parent._astOut) {
-        if (isDestinationType(nextNode)) dstGraph.addEdge(source, nextNode, EdgeTypes.CONTAINS)
-        if (!isSourceType(nextNode)) queue.append(nextNode)
+    try {
+      // AST is assumed to be a tree. If it contains cycles, then this will give a nice endless loop with OOM
+      val queue = mutable.ArrayDeque[StoredNode](source)
+      while (queue.nonEmpty) {
+        val parent = queue.removeHead()
+        for (nextNode <- parent._astOut) {
+          if (isDestinationType(nextNode)) dstGraph.addEdge(source, nextNode, EdgeTypes.CONTAINS)
+          if (!isSourceType(nextNode)) queue.append(nextNode)
+        }
       }
+    } catch {
+      case ex: Exception =>
+        logger.warn(
+          s"Error in ContainsEdgePass for node in file '${source.propertyOption(Properties.FILENAME).toString}''",
+          ex
+        )
     }
   }
 }

diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/FileCreationPass.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/FileCreationPass.scala
@@ -7,14 +7,16 @@ import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes, PropertyN
 import io.shiftleft.passes.CpgPass
 import io.shiftleft.semanticcpg.language.*
 import io.shiftleft.semanticcpg.language.types.structure.FileTraversal
+import org.slf4j.{Logger, LoggerFactory}
 
 import scala.collection.mutable
 
 /** For all nodes with FILENAME fields, create corresponding FILE nodes and connect node with FILE node via outgoing
   * SOURCE_FILE edges.
   */
 class FileCreationPass(cpg: Cpg) extends CpgPass(cpg) with LinkingUtil {
-  private val srcLabels = List(NodeTypes.NAMESPACE_BLOCK, NodeTypes.TYPE_DECL, NodeTypes.METHOD, NodeTypes.COMMENT)
+  private val logger: Logger = LoggerFactory.getLogger(this.getClass)
+  private val srcLabels      = List(NodeTypes.NAMESPACE_BLOCK, NodeTypes.TYPE_DECL, NodeTypes.METHOD, NodeTypes.COMMENT)
 
   override def run(dstGraph: DiffGraphBuilder): Unit = {
     val originalFileNameToNode = mutable.Map.empty[String, StoredNode]
@@ -38,21 +40,25 @@ class FileCreationPass(cpg: Cpg) extends CpgPass(cpg) with LinkingUtil {
         dstGraph.addEdge(srcNode, newFile, EdgeTypes.SOURCE_FILE)
       }
     }
-
-    // Create SOURCE_FILE edges from nodes of various types to FILE
-    linkToSingle(
-      cpg,
-      srcNodes = cpg.graph.nodes(srcLabels*).toList,
-      srcLabels = srcLabels,
-      dstNodeLabel = NodeTypes.FILE,
-      edgeType = EdgeTypes.SOURCE_FILE,
-      dstNodeMap = { x =>
-        originalFileNameToNode.get(x)
-      },
-      dstFullNameKey = PropertyNames.FILENAME,
-      dstGraph,
-      Some(createFileIfDoesNotExist)
-    )
+    try {
+      // Create SOURCE_FILE edges from nodes of various types to FILE
+      linkToSingle(
+        cpg,
+        srcNodes = cpg.graph.nodes(srcLabels*).toList,
+        srcLabels = srcLabels,
+        dstNodeLabel = NodeTypes.FILE,
+        edgeType = EdgeTypes.SOURCE_FILE,
+        dstNodeMap = { x =>
+          originalFileNameToNode.get(x)
+        },
+        dstFullNameKey = PropertyNames.FILENAME,
+        dstGraph,
+        Some(createFileIfDoesNotExist)
+      )
+    } catch {
+      case ex: Exception =>
+        logger.warn(s"Error in FileCreationPass", ex)
+    }
   }
 
 }
diff --git a/...n-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/MethodDecoratorPass.scala b/...n-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/MethodDecoratorPass.scala
@@ -3,7 +3,7 @@ package io.joern.x2cpg.passes.base
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.generated.{EdgeTypes, nodes}
 import io.shiftleft.passes.CpgPass
-import io.shiftleft.semanticcpg.language._
+import io.shiftleft.semanticcpg.language.*
 import org.slf4j.{Logger, LoggerFactory}
 
 /** Adds a METHOD_PARAMETER_OUT for each METHOD_PARAMETER_IN to the graph and connects those with a PARAMETER_LINK edge.
@@ -18,41 +18,46 @@ class MethodDecoratorPass(cpg: Cpg) extends CpgPass(cpg) {
   private var loggedMissingTypeFullName = false
 
   override def run(dstGraph: DiffGraphBuilder): Unit = {
-    cpg.parameter.foreach { parameterIn =>
-      if (!parameterIn._parameterLinkOut.hasNext) {
-        val parameterOut = nodes
-          .NewMethodParameterOut()
-          .code(parameterIn.code)
-          .order(parameterIn.order)
-          .index(parameterIn.index)
-          .name(parameterIn.name)
-          .evaluationStrategy(parameterIn.evaluationStrategy)
-          .typeFullName(parameterIn.typeFullName)
-          .isVariadic(parameterIn.isVariadic)
-          .lineNumber(parameterIn.lineNumber)
-          .columnNumber(parameterIn.columnNumber)
+    try {
+      cpg.parameter.foreach { parameterIn =>
+        if (!parameterIn._parameterLinkOut.hasNext) {
+          val parameterOut = nodes
+            .NewMethodParameterOut()
+            .code(parameterIn.code)
+            .order(parameterIn.order)
+            .index(parameterIn.index)
+            .name(parameterIn.name)
+            .evaluationStrategy(parameterIn.evaluationStrategy)
+            .typeFullName(parameterIn.typeFullName)
+            .isVariadic(parameterIn.isVariadic)
+            .lineNumber(parameterIn.lineNumber)
+            .columnNumber(parameterIn.columnNumber)
 
-        val method = parameterIn.astIn.headOption
-        if (method.isEmpty) {
-          logger.warn("Parameter without method encountered: " + parameterIn.toString)
-        } else {
-          if (parameterIn.typeFullName == null) {
-            val evalType = parameterIn.typ
-            dstGraph.addEdge(parameterOut, evalType, EdgeTypes.EVAL_TYPE)
-            if (!loggedMissingTypeFullName) {
-              logger.warn("Using deprecated CPG format with missing TYPE_FULL_NAME on METHOD_PARAMETER_IN nodes.")
-              loggedMissingTypeFullName = true
+          val method = parameterIn.astIn.headOption
+          if (method.isEmpty) {
+            logger.warn("Parameter without method encountered: " + parameterIn.toString)
+          } else {
+            if (parameterIn.typeFullName == null) {
+              val evalType = parameterIn.typ
+              dstGraph.addEdge(parameterOut, evalType, EdgeTypes.EVAL_TYPE)
+              if (!loggedMissingTypeFullName) {
+                logger.warn("Using deprecated CPG format with missing TYPE_FULL_NAME on METHOD_PARAMETER_IN nodes.")
+                loggedMissingTypeFullName = true
+              }
             }
-          }
 
-          dstGraph.addNode(parameterOut)
-          dstGraph.addEdge(method.get, parameterOut, EdgeTypes.AST)
-          dstGraph.addEdge(parameterIn, parameterOut, EdgeTypes.PARAMETER_LINK)
+            dstGraph.addNode(parameterOut)
+            dstGraph.addEdge(method.get, parameterOut, EdgeTypes.AST)
+            dstGraph.addEdge(parameterIn, parameterOut, EdgeTypes.PARAMETER_LINK)
+          }
+        } else if (!loggedDeprecatedWarning) {
+          logger.warn("Using deprecated CPG format with PARAMETER_LINK edges")
+          loggedDeprecatedWarning = true
         }
-      } else if (!loggedDeprecatedWarning) {
-        logger.warn("Using deprecated CPG format with PARAMETER_LINK edges")
-        loggedDeprecatedWarning = true
       }
+    } catch {
+      case ex: Exception =>
+        logger.warn(s"Error in MethodDecoratorPass", ex)
     }
   }
 }

diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/MethodStubCreator.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/base/MethodStubCreator.scala
@@ -3,10 +3,11 @@ package io.joern.x2cpg.passes.base
 import io.joern.x2cpg.Defines
 import io.joern.x2cpg.passes.base.MethodStubCreator.createMethodStub
 import io.shiftleft.codepropertygraph.Cpg
-import io.shiftleft.codepropertygraph.generated.nodes._
+import io.shiftleft.codepropertygraph.generated.nodes.*
 import io.shiftleft.codepropertygraph.generated.{DispatchTypes, EdgeTypes, EvaluationStrategies, NodeTypes}
 import io.shiftleft.passes.CpgPass
-import io.shiftleft.semanticcpg.language._
+import io.shiftleft.semanticcpg.language.*
+import org.slf4j.{Logger, LoggerFactory}
 import overflowdb.BatchedUpdate
 import overflowdb.BatchedUpdate.DiffGraphBuilder
 
@@ -19,28 +20,34 @@ case class CallSummary(name: String, signature: String, fullName: String, dispat
   */
 class MethodStubCreator(cpg: Cpg) extends CpgPass(cpg) {
 
+  private val logger: Logger = LoggerFactory.getLogger(this.getClass)
   // Since the method fullNames for fuzzyc are not unique, we do not have
   // a 1to1 relation and may overwrite some values. This is ok for now.
   private val methodFullNameToNode   = mutable.LinkedHashMap[String, Method]()
   private val methodToParameterCount = mutable.LinkedHashMap[CallSummary, Int]()
 
   override def run(dstGraph: BatchedUpdate.DiffGraphBuilder): Unit = {
-    for (method <- cpg.method) {
-      methodFullNameToNode.put(method.fullName, method)
-    }
-
-    for (call <- cpg.call if call.methodFullName != Defines.DynamicCallUnknownFullName) {
-      methodToParameterCount.put(
-        CallSummary(call.name, call.signature, call.methodFullName, call.dispatchType),
-        call.argument.size
-      )
-    }
-
-    for (
-      (CallSummary(name, signature, fullName, dispatchType), parameterCount) <- methodToParameterCount
-      if !methodFullNameToNode.contains(fullName)
-    ) {
-      createMethodStub(name, fullName, signature, dispatchType, parameterCount, dstGraph)
+    try {
+      for (method <- cpg.method) {
+        methodFullNameToNode.put(method.fullName, method)
+      }
+
+      for (call <- cpg.call if call.methodFullName != Defines.DynamicCallUnknownFullName) {
+        methodToParameterCount.put(
+          CallSummary(call.name, call.signature, call.methodFullName, call.dispatchType),
+          call.argument.size
+        )
+      }
+
+      for (
+        (CallSummary(name, signature, fullName, dispatchType), parameterCount) <- methodToParameterCount
+        if !methodFullNameToNode.contains(fullName)
+      ) {
+        createMethodStub(name, fullName, signature, dispatchType, parameterCount, dstGraph)
+      }
+    } catch {
+      case ex: Exception =>
+        logger.warn(s"Error in TypeDeclStubCreator", ex)
     }
   }