[ruby] Implicit Require Handling (joernio#4484)

[Autoloading in Ruby](https://www.rubyguides.com/2019/08/autoloading-in-ruby/) is fairly common, however it's important for us to represent these import nodes explicitly.

This pass adds a check for modules with no high-level require calls, to check call receivers and constructor allocations for types from other files, and add explicit require calls for them.

joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala

@@ -6,7 +6,13 @@ import io.joern.rubysrc2cpg.datastructures.RubyProgramSummary
 import io.joern.rubysrc2cpg.deprecated.parser.DeprecatedRubyParser
 import io.joern.rubysrc2cpg.deprecated.parser.DeprecatedRubyParser.*
 import io.joern.rubysrc2cpg.parser.RubyParser
-import io.joern.rubysrc2cpg.passes.{AstCreationPass, ConfigFileCreationPass, DependencyPass, ImportsPass}
+import io.joern.rubysrc2cpg.passes.{
+  AstCreationPass,
+  ConfigFileCreationPass,
+  DependencyPass,
+  ImplicitRequirePass,
+  ImportsPass
+}
 import io.joern.rubysrc2cpg.utils.DependencyDownloader
 import io.joern.x2cpg.X2Cpg.withNewEmptyCpg
 import io.joern.x2cpg.passes.base.AstLinkerPass
@@ -65,10 +71,9 @@ class RubySrc2Cpg extends X2CpgFrontend[Config] {
         internalProgramSummary
       }
 
-      val astCreationPass = new AstCreationPass(cpg, astCreators.map(_.withSummary(programSummary)))
-      astCreationPass.createAndApply()
-      val importsPass = new ImportsPass(cpg)
-      importsPass.createAndApply()
+      AstCreationPass(cpg, astCreators.map(_.withSummary(programSummary))).createAndApply()
+      ImplicitRequirePass(cpg, programSummary).createAndApply()
+      ImportsPass(cpg).createAndApply()
       TypeNodePass.withTypesFromCpg(cpg).createAndApply()
     }
   }

joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/ImplicitRequirePass.scala

@@ -0,0 +1,88 @@
+package io.joern.rubysrc2cpg.passes
+
+import io.joern.rubysrc2cpg.datastructures.{RubyProgramSummary, RubyType}
+import io.shiftleft.codepropertygraph.generated.nodes.*
+import io.shiftleft.codepropertygraph.generated.{Cpg, DispatchTypes, EdgeTypes, Operators}
+import io.shiftleft.passes.ForkJoinParallelCpgPass
+import io.shiftleft.semanticcpg.language.*
+
+import scala.collection.mutable
+
+/** In some Ruby frameworks, it is common to have an autoloader library that implicitly loads requirements onto the
+  * stack. This pass makes these imports explicit.
+  */
+class ImplicitRequirePass(cpg: Cpg, programSummary: RubyProgramSummary) extends ForkJoinParallelCpgPass[Method](cpg) {
+
+  private val importCallName: String = "require"
+  private val typeToPath             = mutable.Map.empty[String, String]
+
+  override def init(): Unit = {
+    programSummary.pathToType.foreach { case (path, types) =>
+      types.foreach { typ => typeToPath.put(typ.name, path) }
+    }
+  }
+
+  override def generateParts(): Array[Method] =
+    cpg.method.isModule.whereNot(_.astChildren.isCall.nameExact(importCallName)).toArray
+
+  /** Collects methods within a module.
+    */
+  private def findMethodsViaAstChildren(module: Method): Iterator[Method] = {
+    Iterator(module) ++ module.astChildren.flatMap {
+      case x: TypeDecl => x.method.flatMap(findMethodsViaAstChildren)
+      case x: Method   => Iterator(x) ++ x.astChildren.collectAll[Method].flatMap(findMethodsViaAstChildren)
+      case _           => Iterator.empty
+    }
+  }
+
+  override def runOnPart(builder: DiffGraphBuilder, part: Method): Unit = {
+    findMethodsViaAstChildren(part).ast.isCall
+      .flatMap {
+        case x if x.name == Operators.alloc =>
+          x.argument.isIdentifier
+        case x =>
+          x.receiver.isIdentifier
+      }
+      .map(i => i -> programSummary.matchingTypes(i.name))
+      .distinct
+      .foreach { case (identifier, rubyTypes) =>
+        val requireCalls = rubyTypes.flatMap { rubyType =>
+          typeToPath.get(rubyType.name) match {
+            case Some(path)
+                if identifier.file.name
+                  .map(_.replace("\\", "/"))
+                  .headOption
+                  .exists(x => rubyType.name.startsWith(x)) =>
+              None // do not add an import to a file that defines the type
+            case Some(path) => Option(createRequireCall(builder, rubyType, path))
+            case None       => None
+          }
+        }
+        val startIndex = part.block.astChildren.size
+        requireCalls.zipWithIndex.foreach { case (call, idx) =>
+          call.order(startIndex + idx)
+          builder.addEdge(part.block, call, EdgeTypes.AST)
+        }
+      }
+  }
+
+  private def createRequireCall(builder: DiffGraphBuilder, rubyType: RubyType, path: String): NewCall = {
+    val requireCallNode = NewCall()
+      .name(importCallName)
+      .code(s"$importCallName '$path'")
+      .methodFullName(s"__builtin:$importCallName")
+      .dispatchType(DispatchTypes.DYNAMIC_DISPATCH)
+      .typeFullName(Defines.Any)
+    val receiverIdentifier =
+      NewIdentifier().name(importCallName).code(importCallName).typeFullName(Defines.Any).argumentIndex(0).order(1)
+    val pathLiteralNode = NewLiteral().code(s"'$path'").typeFullName("__builtin.String").argumentIndex(1).order(2)
+    builder.addNode(requireCallNode)
+    builder.addEdge(requireCallNode, receiverIdentifier, EdgeTypes.AST)
+    builder.addEdge(requireCallNode, receiverIdentifier, EdgeTypes.ARGUMENT)
+    builder.addEdge(requireCallNode, receiverIdentifier, EdgeTypes.RECEIVER)
+    builder.addEdge(requireCallNode, pathLiteralNode, EdgeTypes.AST)
+    builder.addEdge(requireCallNode, pathLiteralNode, EdgeTypes.ARGUMENT)
+    requireCallNode
+  }
+
+}

joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/passes/ImportsPass.scala

@@ -1,17 +1,17 @@
 package io.joern.rubysrc2cpg.passes
 
+import io.joern.x2cpg.Imports.createImportNodeAndLink
+import io.joern.x2cpg.X2Cpg.stripQuotes
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.generated.nodes.*
+import io.shiftleft.passes.ForkJoinParallelCpgPass
 import io.shiftleft.semanticcpg.language.*
-import io.shiftleft.passes.ConcurrentWriterCpgPass
-import io.joern.x2cpg.Imports.createImportNodeAndLink
-import io.joern.x2cpg.X2Cpg.stripQuotes
 
-class ImportsPass(cpg: Cpg) extends ConcurrentWriterCpgPass[Call](cpg) {
-  protected val importCallName: String = "require"
-  override def generateParts(): Array[Call] = cpg
-    .call(importCallName)
-    .toArray
+class ImportsPass(cpg: Cpg) extends ForkJoinParallelCpgPass[Call](cpg) {
+
+  private val importCallName: String = "require"
+
+  override def generateParts(): Array[Call] = cpg.call.nameExact(importCallName).toArray
 
   override def runOnPart(diffGraph: DiffGraphBuilder, call: Call): Unit = {
     val importedEntity = stripQuotes(call.argument.isLiteral.code.l match {

joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/ImportTests.scala

@@ -90,4 +90,64 @@ class ImportTests extends RubyCode2CpgFixture with Inspectors {
       methodName should endWith(s"${moduleName}:foo")
     }
   }
+
+  "implicitly imported types (common in frameworks like Ruby on Rails)" should {
+
+    val cpg = code(
+      """
+        |class A
+        |end
+        |""".stripMargin,
+      "A.rb"
+    )
+      .moreCode(
+        """
+          |class B
+          | def bar
+          | end
+          |end
+          |
+          |B::bar
+          |""".stripMargin,
+        "bar/B.rb"
+      )
+      .moreCode(
+        """
+          |a = A.new
+          |""".stripMargin,
+        "Foo.rb"
+      )
+      .moreCode(
+        """
+          |B.bar
+          |""".stripMargin,
+        "Bar.rb"
+      )
+
+    "be explicitly detected and imported for a constructor type" in {
+      inside(cpg.imports.where(_.call.file.name(".*Foo.*")).headOption) {
+        case Some(i) =>
+          i.importedAs shouldBe Some("A")
+          i.importedEntity shouldBe Some("A")
+        case None =>
+          fail("Expected `A` to be explicitly imported into `Foo.rb`")
+      }
+    }
+
+    "be explicitly detected and imported for a call invocation" in {
+      inside(cpg.imports.where(_.call.file.name(".*Bar.*")).headOption) {
+        case Some(i) =>
+          i.importedAs shouldBe Some("bar/B")
+          i.importedEntity shouldBe Some("bar/B")
+        case None =>
+          fail("Expected `B` to be explicitly imported into `Foo.rb`")
+      }
+    }
+
+    "not import a type from the type's defining file" in {
+      cpg.imports.where(_.call.file.name(".*B.rb")).size shouldBe 0
+    }
+
+  }
+
 }