[ruby] Here-Docs Pt 1 (joernio#4322)

* [ruby] Fixed the trivial here-doc impl, just enough so the parser doesn't fall over anymore

* [ruby] Removed import on heredochandling, simplified tests for heredoc

* [ruby] Changed type on heredoc to heredoc, added test for heredoc in assingment

* PR comments

joern-cli/frontends/rubysrc2cpg/src/main/antlr4/io/joern/rubysrc2cpg/parser/RubyLexer.g4

@@ -380,7 +380,7 @@ HERE_DOC_IDENTIFIER
  ;
 
 HERE_DOC
- : '<<' [-~]? [\t]* IDENTIFIER [a-zA-Z_0-9]* NL ( {!heredocEndAhead(getText())}? . )* [a-zA-Z_] [a-zA-Z_0-9]*
+ : '<<' [-~]? [\t]* IDENTIFIER [a-zA-Z_0-9]* NL WS* ( {!heredocEndAhead(getText())}? . )* NL? WS* [a-zA-Z_] [a-zA-Z_0-9]*
  ;
 
 // --------------------------------------------------------

joern-cli/frontends/rubysrc2cpg/src/main/antlr4/io/joern/rubysrc2cpg/parser/RubyParser.g4

@@ -403,6 +403,8 @@ primaryValue
         # logicalOrExpression
     |   primaryValue rangeOperator          NL* primaryValue
         # rangeExpression
+    |   hereDoc
+        # hereDocs
     ;
 
 commandOrPrimaryValue
@@ -668,6 +670,9 @@ symbol
         # doubleQuotedSymbolLiteral
     ;
 
+hereDoc
+    : HERE_DOC
+    ;
 
 // --------------------------------------------------------
 // Commons

joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstForExpressionsCreator.scala

@@ -18,6 +18,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) {
 
   protected def astForExpression(node: RubyNode): Ast = node match
     case node: StaticLiteral            => astForStaticLiteral(node)
+    case node: HereDocNode              => astForHereDoc(node)
     case node: DynamicLiteral           => astForDynamicLiteral(node)
     case node: UnaryExpression          => astForUnary(node)
     case node: BinaryExpression         => astForBinary(node)
@@ -50,6 +51,10 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) {
     Ast(literalNode(node, code(node), node.typeFullName))
   }
 
+  protected def astForHereDoc(node: HereDocNode): Ast = {
+    Ast(literalNode(node, code(node), getBuiltInType("String")))
+  }
+
   // Helper for nil literals to put in empty clauses
   protected def astForNilLiteral: Ast = Ast(NewLiteral().code("nil").typeFullName(getBuiltInType(Defines.NilClass)))
   protected def astForNilBlock: Ast   = blockAst(NewBlock(), List(astForNilLiteral))

joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/RubyIntermediateAst.scala

@@ -350,4 +350,6 @@ object RubyIntermediateAst {
   final case class UnaryExpression(op: String, expression: RubyNode)(span: TextSpan) extends RubyNode(span)
 
   final case class BinaryExpression(lhs: RubyNode, op: String, rhs: RubyNode)(span: TextSpan) extends RubyNode(span)
+
+  final case class HereDocNode(content: String)(span: TextSpan) extends RubyNode(span)
 }

joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/HereDocHandling.scala

@@ -14,22 +14,30 @@ trait HereDocHandling { this: RubyLexerBase =>
       // If the lexer is not at the start of a line, no end-delimiter can be possible
       false
     } else {
+      // Count WS characters to ignore
+      var idxWs   = 0
+      var wsCount = 0
+
+      while (this._input.LA(idxWs + 1).toChar.isWhitespace) {
+        wsCount += 1
+        idxWs += 1
+      }
+
       // Get the delimiter
       HereDocHandling.getHereDocDelimiter(partialHeredoc) match
-        case Some(delimiter) if !delimiter.zipWithIndex.exists { case (c, idx) => this._input.LA(idx + 1) != c } =>
+        case Some(delimiter) if !delimiter.zipWithIndex.exists { case (c, idx) =>
+              this._input.LA(idx + wsCount + 1) != c
+            } =>
           // If we get to this point, we know there is an end delimiter ahead in the char stream, make
           // sure it is followed by a white space (or the EOF). If we don't do this, then "FOOS" would also
           // be considered the end for the delimiter "FOO"
-          val charAfterDelimiter = this._input.LA(delimiter.length + 1)
+          val charAfterDelimiter = this._input.LA(delimiter.length + wsCount + 1)
           charAfterDelimiter == EOF || Character.isWhitespace(charAfterDelimiter)
         case _ => false
     }
-
 }
 
 object HereDocHandling {
-
   def getHereDocDelimiter(hereDoc: String): Option[String] =
     hereDoc.split("\r?\n|\r").headOption.map(_.replaceAll("^<<[~-]\\s*", ""))
-
 }

joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyNodeCreator.scala

@@ -172,6 +172,10 @@ class RubyNodeCreator extends RubyParserBaseVisitor[RubyNode] {
     }
   }
 
+  override def visitHereDocs(ctx: RubyParser.HereDocsContext): RubyNode = {
+    HereDocNode(ctx.hereDoc().getText)(ctx.toTextSpan)
+  }
+
   override def visitPowerExpression(ctx: RubyParser.PowerExpressionContext): RubyNode = {
     BinaryExpression(visit(ctx.primaryValue(0)), ctx.powerOperator.getText, visit(ctx.primaryValue(1)))(ctx.toTextSpan)
   }

joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/HereDocTests.scala

@@ -0,0 +1,67 @@
+package io.joern.rubysrc2cpg.querying
+
+import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture
+import io.shiftleft.codepropertygraph.generated.nodes.{Call, Literal, Local, Method, Return}
+import io.shiftleft.semanticcpg.language.*
+
+class HereDocTests extends RubyCode2CpgFixture {
+
+  "HereDoc simple" should {
+    val cpg = code("""
+        | def f()
+        |   a = 10
+        |   <<-SQL
+        |     this is some sql heredoc code
+        |   SQL
+        |   a
+        | end
+        |""".stripMargin)
+
+    "have a LITERAL node" in {
+      inside(cpg.method.name("f").l) {
+        case func :: Nil =>
+          inside(func.block.astChildren.l) {
+            case (localAst: Local) :: (callAst: Call) :: (literalAst: Literal) :: (returnAst: Return) :: Nil =>
+              localAst.code shouldBe "a"
+              callAst.code shouldBe "a = 10"
+
+              literalAst.typeFullName shouldBe "__builtin.String"
+
+              returnAst.code shouldBe "a"
+            case _ =>
+          }
+        case _ => fail("Expected one method for f")
+      }
+    }
+  }
+
+  "HereDoc as function argument" should {
+    val cpg = code("""
+        |def foo()
+        | a = <<-SQL
+        |   SELECT * FROM TABLE;
+        | SQL
+        |
+        | a
+        |end
+        |""".stripMargin)
+
+    "parse Heredocs" in {
+      inside(cpg.method.name("foo").l) {
+        case fooFunc :: Nil =>
+          inside(fooFunc.block.astChildren.isCall.l) {
+            case assignmentCall :: Nil =>
+              inside(assignmentCall.argument.l) {
+                case lhsArg :: (rhsArg: Literal) :: Nil =>
+                  lhsArg.code shouldBe "a"
+                  rhsArg.typeFullName shouldBe "__builtin.String"
+                case _ => fail("Expected LHS and RHS for assignment")
+              }
+            case _ => fail("Expected call for assignment")
+          }
+        case _ => fail("Expected one method for foo")
+      }
+    }
+  }
+
+}