From e7d2be6fea7b1cf8cb5b3f3148266f18b7dc29f3 Mon Sep 17 00:00:00 2001 From: simerplaha Date: Tue, 14 Jan 2025 14:31:07 +1100 Subject: [PATCH 1/2] `BStringParser` --- .../compiler/parser/soft/BStringParser.scala | 31 ++++ .../access/compiler/parser/soft/Demo.scala | 5 + .../parser/soft/ExpressionParser.scala | 1 + .../compiler/parser/soft/ast/SoftAST.scala | 9 + .../compiler/parser/soft/ast/Token.scala | 1 + .../compiler/parser/soft/BStringSpec.scala | 167 ++++++++++++++++++ .../compiler/parser/soft/TestParser.scala | 3 + .../parser/soft/ast/TestSoftAST.scala | 12 ++ 8 files changed, 229 insertions(+) create mode 100644 compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/BStringParser.scala create mode 100644 compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/BStringSpec.scala diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/BStringParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/BStringParser.scala new file mode 100644 index 000000000..17ff6429b --- /dev/null +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/BStringParser.scala @@ -0,0 +1,31 @@ +package org.alephium.ralph.lsp.access.compiler.parser.soft + +import fastparse._ +import fastparse.NoWhitespace.noWhitespaceImplicit +import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range +import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token} + +private object BStringParser { + + def parseOrFail[Unknown: P]: P[SoftAST.BString] = + P { + Index ~ + TokenParser.parseOrFail(Token.B) ~ + SpaceParser.parseOrFail.? ~ + TokenParser.parseOrFail(Token.Tick) ~ + TextParser.parseOrFail(Token.Tick).? ~ + TokenParser.parse(Token.Tick) ~ + Index + } map { + case (from, b, postBSpace, startTick, text, endTick, to) => + SoftAST.BString( + index = range(from, to), + b = b, + postBSpace = postBSpace, + startTick = startTick, + text = text, + endTick = endTick + ) + } + +} diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/Demo.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/Demo.scala index 86fd7cfe8..c55a8c377 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/Demo.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/Demo.scala @@ -44,6 +44,11 @@ object Demo extends App { | | // mutable binding | let (a, mut b, _) = (1, 2, 3) + | + | let string = + | b`some text 🎸 + | some text in the middle + | some more text 🤙` | } | | 🚀 diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ExpressionParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ExpressionParser.scala index c4cc59d51..5c46c3fa9 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ExpressionParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ExpressionParser.scala @@ -101,6 +101,7 @@ private object ExpressionParser { TupleParser.parseOrFail | NumberParser.parseOrFail | BooleanParser.parseOrFail | + BStringParser.parseOrFail | IdentifierParser.parseOrFail } diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ast/SoftAST.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ast/SoftAST.scala index 84dbe42ee..3d53aa0cc 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ast/SoftAST.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ast/SoftAST.scala @@ -429,6 +429,15 @@ object SoftAST { unit: Option[TokenDocumented[Token.AlphLowercase.type]]) extends ExpressionAST + case class BString( + index: SourceIndex, + b: TokenDocumented[Token.B.type], + postBSpace: Option[Space], + startTick: TokenDocumented[Token.Tick.type], + text: Option[CodeString], + endTick: TokenDocExpectedAST[Token.Tick.type]) + extends ExpressionAST + sealed trait SpaceAST extends SoftAST case class Space( diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ast/Token.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ast/Token.scala index ced264393..b1049dde6 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ast/Token.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ast/Token.scala @@ -96,6 +96,7 @@ object Token { case object At extends Punctuator("@") with Reserved case object Tick extends Punctuator("`") with Reserved case object Quote extends Punctuator("\"") with Reserved + case object B extends Punctuator("b") with Token sealed abstract class Data(override val lexeme: String) extends Token case object Const extends Data("const") with Reserved diff --git a/compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/BStringSpec.scala b/compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/BStringSpec.scala new file mode 100644 index 000000000..c5b8b2e11 --- /dev/null +++ b/compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/BStringSpec.scala @@ -0,0 +1,167 @@ +package org.alephium.ralph.lsp.access.compiler.parser.soft + +import org.alephium.ralph.lsp.access.compiler.parser.soft.TestParser._ +import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token} +import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.TestSoftAST._ +import org.alephium.ralph.lsp.access.util.TestCodeUtil._ +import org.alephium.ralph.lsp.access.util.TestFastParse.assertIsFastParseError +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec +import org.scalatest.OptionValues._ + +class BStringSpec extends AnyWordSpec with Matchers { + + "b alone" should { + "not parser as String literal" in { + // this is because "b" is not a reserved token. + // "b" followed by tick is required for string literal. + assertIsFastParseError { + parseBString("b") + } + } + } + + "b followed by a tick" should { + "parse as string literal" in { + // this is successfully parsed and the ending tick is reported as missing. + val string = + parseBString("b`") // closing tick is missing + + string shouldBe + SoftAST.BString( + index = indexOf(">>b`<<"), + b = B(indexOf(">>b<<`")), + postBSpace = None, + startTick = Tick(indexOf("b>>`<<")), + text = None, + endTick = SoftAST.TokenExpected(indexOf("b`>><<"), Token.Tick) + ) + } + } + + "empty string" in { + val string = + parseBString("b``") + + string shouldBe + SoftAST.BString( + index = indexOf(">>b``<<"), + b = B(indexOf(">>b<<``")), + postBSpace = None, + startTick = Tick(indexOf("b>>`<<`")), + text = None, + endTick = Tick(indexOf("b`>>`<<")) + ) + } + + "spaces only" in { + val string = + parseBString("b ` `") + + string shouldBe + SoftAST.BString( + index = indexOf(">>b ` `<<"), + b = B(indexOf(">>b<< ` `")), + postBSpace = Some(SpaceOne(indexOf("b>> <<` `"))), + startTick = Tick(indexOf("b >>`<< `")), + text = Some(SoftAST.CodeString(indexOf("b `>> <<`"), " ")), + endTick = Tick(indexOf("b ` >>`<<")) + ) + } + + "comment exist before b" in { + val string = + parseBString { + """// this is a byte string + |b ` `""".stripMargin + } + + // Simply check that the comment exists before b. + // No need to assert the comment AST, CommentsSpec should assert this. + val comments = + string.b.documentation.value + + comments.index shouldBe + indexOf { + """>>// this is a byte string + |<>// this is a byte string + |<<` `""".stripMargin + } + + comments.toCode() shouldBe + """// this is a byte string + |""".stripMargin + + } + + "string characters exist" when { + "closing tick is missing" should { + "parse the entire string" in { + val string = + parseBString { + """b` this is + | + |a string value + |""".stripMargin + } + + string.text.value shouldBe + SoftAST.CodeString( + index = indexOf { + """b`>> this is + | + |a string value + |<<""".stripMargin + }, + text = """ this is + | + |a string value + |""".stripMargin + ) + + string.text.value.text shouldBe + """ this is + | + |a string value + |""".stripMargin + + string.endTick shouldBe + SoftAST.TokenExpected( + index = indexOf { + """b` this is + | + |a string value + |>><<""".stripMargin + }, + token = Token.Tick + ) + } + } + } + +} diff --git a/compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TestParser.scala b/compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TestParser.scala index 3e162d76f..bb68d1e76 100644 --- a/compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TestParser.scala +++ b/compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TestParser.scala @@ -78,6 +78,9 @@ object TestParser { def parseNumber(code: String): SoftAST.Number = runSoftParser(NumberParser.parseOrFail(_))(code) + def parseBString(code: String): SoftAST.BString = + runSoftParser(BStringParser.parseOrFail(_))(code) + def findAnnotation(identifier: String)(code: String): Option[SoftAST.Annotation] = findAnnotation( identifier = identifier, diff --git a/compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ast/TestSoftAST.scala b/compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ast/TestSoftAST.scala index c0b4a9589..10e42c1eb 100644 --- a/compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ast/TestSoftAST.scala +++ b/compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ast/TestSoftAST.scala @@ -126,6 +126,18 @@ object TestSoftAST { token = Token.Plus ) + def B(index: SourceIndex): SoftAST.TokenDocumented[Token.B.type] = + TokenDocumented( + index = index, + token = Token.B + ) + + def Tick(index: SourceIndex): SoftAST.TokenDocumented[Token.Tick.type] = + TokenDocumented( + index = index, + token = Token.Tick + ) + def Identifier( index: SourceIndex, text: String): SoftAST.Identifier = From e8d0773ca83272e3b5bd294e26b4d246f22ae537 Mon Sep 17 00:00:00 2001 From: simerplaha Date: Wed, 15 Jan 2025 08:26:35 +1100 Subject: [PATCH 2/2] typo --- .../ralph/lsp/access/compiler/parser/soft/BStringSpec.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/BStringSpec.scala b/compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/BStringSpec.scala index c5b8b2e11..71c8cf2ac 100644 --- a/compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/BStringSpec.scala +++ b/compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/BStringSpec.scala @@ -12,7 +12,7 @@ import org.scalatest.OptionValues._ class BStringSpec extends AnyWordSpec with Matchers { "b alone" should { - "not parser as String literal" in { + "not parse as String literal" in { // this is because "b" is not a reserved token. // "b" followed by tick is required for string literal. assertIsFastParseError {