From d6878df46f9a92f17d9e6dcc058fbc69c619f7d0 Mon Sep 17 00:00:00 2001 From: simerplaha Date: Mon, 13 Jan 2025 12:19:25 +1100 Subject: [PATCH 01/11] Added `Token.PlusPlus` --- .../ralph/lsp/access/compiler/parser/soft/ast/Token.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ast/Token.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ast/Token.scala index b89af88f1..fed73c833 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ast/Token.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ast/Token.scala @@ -52,6 +52,7 @@ object Token { sealed abstract class Operator(override val lexeme: String) extends Token case object Equality extends Operator("==") with Reserved with InfixOperator case object GreaterThanOrEqual extends Operator(">=") with Reserved with InfixOperator + case object PlusPlus extends Operator("++") with Reserved with InfixOperator case object PlusEquals extends Operator("+=") with Reserved with InfixOperator case object MinusEquals extends Operator("-=") with Reserved with InfixOperator case object LessThanOrEqual extends Operator("<=") with Reserved with InfixOperator From 772d58dd627070798aa5003aaabe21f73b93d5f2 Mon Sep 17 00:00:00 2001 From: simerplaha Date: Mon, 13 Jan 2025 12:25:08 +1100 Subject: [PATCH 02/11] format following other parsers --- .../compiler/parser/soft/FunctionParser.scala | 16 ++++++++++++++-- .../ralph/lsp/access/util/ParserUtil.scala | 6 +++--- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/FunctionParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/FunctionParser.scala index dfc7dcf64..53ec110c6 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/FunctionParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/FunctionParser.scala @@ -64,7 +64,15 @@ private object FunctionParser { * such as its name, parameters and return type. */ private def signature[Unknown: P]: P[SoftAST.FunctionSignature] = - P(Index ~ identifier ~ spaceOrFail.? ~ ParameterParser.parse ~ spaceOrFail.? ~ returnSignature ~ Index) map { + P { + Index ~ + identifier ~ + spaceOrFail.? ~ + ParameterParser.parse ~ + spaceOrFail.? ~ + returnSignature ~ + Index + } map { case (from, fnName, headSpace, params, tailSpace, returns, to) => SoftAST.FunctionSignature( index = range(from, to), @@ -83,7 +91,11 @@ private object FunctionParser { * type or an error indicating that the return type is expected. */ private def returnSignature[Unknown: P]: P[SoftAST.FunctionReturnAST] = - P(Index ~ (TokenParser.parse(Token.ForwardArrow) ~ spaceOrFail.? ~ TypeParser.parse).? ~ Index) map { + P { + Index ~ + (TokenParser.parse(Token.ForwardArrow) ~ spaceOrFail.? ~ TypeParser.parse).? ~ + Index + } map { case (from, Some((forwardArrow, space, tpe)), to) => SoftAST.FunctionReturn( index = range(from, to), diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/util/ParserUtil.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/util/ParserUtil.scala index c4b7b174d..84d3c80df 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/util/ParserUtil.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/util/ParserUtil.scala @@ -34,10 +34,10 @@ object ParserUtil { parser = createParser(_: T) ) - @inline private def createParser[Unknown: P, T <: Token](item: T): P[T] = - P(item.lexeme) map { + @inline private def createParser[Unknown: P, T <: Token](token: T): P[T] = + P(token.lexeme) map { _ => - item + token } } From 77f1ad1ecf082154af18cfb58f6e72f4298f4797 Mon Sep 17 00:00:00 2001 From: simerplaha Date: Mon, 13 Jan 2025 12:25:13 +1100 Subject: [PATCH 03/11] use diff instead of filter --- .../ralph/lsp/access/compiler/parser/soft/TokenParser.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TokenParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TokenParser.scala index 9fa173821..90e7da133 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TokenParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TokenParser.scala @@ -66,7 +66,7 @@ private object TokenParser { def InfixOperatorOrFail[Unknown: P]: P[SoftAST.TokenDocumented[Token.InfixOperator]] = { val infixOps = ParserUtil.orCombinator( - items = Token.infix.iterator.filter(_ != Token.ForwardSlash), // remove forward-slash + items = Token.infix.diff(Seq(Token.ForwardSlash)).iterator, // remove forward-slash parser = TokenParser.parseOrFail(_: Token.InfixOperator) ) From c3cc235bd2360583a3f926d257a53b67f1ff1532 Mon Sep 17 00:00:00 2001 From: simerplaha Date: Mon, 13 Jan 2025 12:34:57 +1100 Subject: [PATCH 04/11] replace `CharsWhileNot` with `TokenParser.WhileNotOrFail` --- .../compiler/parser/soft/BlockParser.scala | 16 ++++++++-------- .../compiler/parser/soft/CommentParser.scala | 2 +- .../compiler/parser/soft/CommonParser.scala | 14 ++++---------- .../compiler/parser/soft/TokenParser.scala | 10 ++++++++++ 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/BlockParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/BlockParser.scala index df07260a5..017691b4b 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/BlockParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/BlockParser.scala @@ -29,7 +29,7 @@ private object BlockParser { Index ~ TokenParser.parse(required, Token.OpenCurly) ~ spaceOrFail.? ~ - body(Some(Token.CloseCurly.lexeme)) ~ + body(Some(Token.CloseCurly)) ~ spaceOrFail.? ~ TokenParser.parse(Token.CloseCurly) ~ Index @@ -46,13 +46,13 @@ private object BlockParser { } def body[Unknown: P]: P[SoftAST.BlockBody] = - body(stopChars = None) + body(stop = None) - private def body[Unknown: P](stopChars: Option[String]): P[SoftAST.BlockBody] = + private def body[Unknown: P](stop: Option[Token]): P[SoftAST.BlockBody] = P { Index ~ spaceOrFail.? ~ - bodyPart(stopChars).rep ~ + bodyPart(stop).rep ~ Index } map { case (from, headSpace, parts, to) => @@ -63,10 +63,10 @@ private object BlockParser { ) } - private def bodyPart[Unknown: P](stopChars: Option[String]): P[SoftAST.BlockBodyPart] = + private def bodyPart[Unknown: P](stop: Option[Token]): P[SoftAST.BlockBodyPart] = P { Index ~ - part(stopChars) ~ + part(stop) ~ spaceOrFail.? ~ Index } map { @@ -78,14 +78,14 @@ private object BlockParser { ) } - private def part[Unknown: P](stopChars: Option[String]): P[SoftAST.BodyPartAST] = + private def part[Unknown: P](stop: Option[Token]): P[SoftAST.BodyPartAST] = P { TemplateParser.parseOrFail | DataTemplateParser.parseOrFail | FunctionParser.parseOrFail | ExpressionParser.parseOrFail | CommentParser.parseOrFail | - unresolved(stopChars) + unresolved(stop) } } diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommentParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommentParser.scala index 066785499..a23516696 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommentParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommentParser.scala @@ -65,7 +65,7 @@ private object CommentParser { Index ~ TokenParser.parseOrFailUndocumented(Token.DoubleForwardSlash) ~ spaceOrFail.? ~ - text.? ~ + text(Token.Newline).? ~ spaceOrFail.? ~ Index } map { diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommonParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommonParser.scala index 1141d6373..464c9ad79 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommonParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommonParser.scala @@ -38,8 +38,8 @@ private object CommonParser { SoftAST.Space(text) } - def text[Unknown: P]: P[SoftAST.CodeString] = - P(Index ~ CharsWhileNot(Token.Newline.lexeme).! ~ Index) map { + def text[Unknown: P](stop: Token*): P[SoftAST.CodeString] = + P(Index ~ TokenParser.WhileNotOrFail(stop: _*).! ~ Index) map { case (from, text, to) => SoftAST.CodeString( text = text, @@ -47,10 +47,10 @@ private object CommonParser { ) } - def unresolved[Unknown: P](stopChars: Option[String]): P[SoftAST.Unresolved] = + def unresolved[Unknown: P](stop: Option[Token]): P[SoftAST.Unresolved] = P { Index ~ - toCodeOrFail(CharsWhileNot(Token.Space.lexeme ++ Token.Newline.lexeme ++ stopChars.getOrElse("")).!) ~ + CodeParser.parseOrFail(TokenParser.WhileNotOrFail(Seq(Token.Space, Token.Newline) ++ stop: _*).!) ~ CommentParser.parseOrFail.? ~ Index } map { @@ -116,12 +116,6 @@ private object CommonParser { ) } - private def CharsWhileNot[Unknown: P](chars: String): P[Unit] = - CharsWhile { - char => - !(chars contains char) - } - private def isLetterDigitOrUnderscore[Unknown: P]: P[Unit] = CharsWhile { char => diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TokenParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TokenParser.scala index 90e7da133..0460023ea 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TokenParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TokenParser.scala @@ -78,4 +78,14 @@ private object TokenParser { infixOps | forwardSlashOperator } + /** + * Reads characters until at least one of the input tokens is matched. + * + * If none of the tokens are found, the parser fails. + * + * @param tokens the token to check for. + */ + def WhileNotOrFail[Unknown: P](tokens: Token*): P[Unit] = + P((!ParserUtil.orTokenCombinator(tokens.iterator) ~ AnyChar).rep(1)) + } From b78d5300ab8e4625e9d3c616ca819658af4d0249 Mon Sep 17 00:00:00 2001 From: simerplaha Date: Mon, 13 Jan 2025 12:37:24 +1100 Subject: [PATCH 05/11] move `text` to `TextParser` --- .../compiler/parser/soft/CommentParser.scala | 2 +- .../compiler/parser/soft/CommonParser.scala | 9 --------- .../compiler/parser/soft/TextParser.scala | 19 +++++++++++++++++++ 3 files changed, 20 insertions(+), 10 deletions(-) create mode 100644 compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TextParser.scala diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommentParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommentParser.scala index a23516696..01e7c97c4 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommentParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommentParser.scala @@ -65,7 +65,7 @@ private object CommentParser { Index ~ TokenParser.parseOrFailUndocumented(Token.DoubleForwardSlash) ~ spaceOrFail.? ~ - text(Token.Newline).? ~ + TextParser.parseOrFail(Token.Newline).? ~ spaceOrFail.? ~ Index } map { diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommonParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommonParser.scala index 464c9ad79..a575b412e 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommonParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommonParser.scala @@ -38,15 +38,6 @@ private object CommonParser { SoftAST.Space(text) } - def text[Unknown: P](stop: Token*): P[SoftAST.CodeString] = - P(Index ~ TokenParser.WhileNotOrFail(stop: _*).! ~ Index) map { - case (from, text, to) => - SoftAST.CodeString( - text = text, - index = range(from, to) - ) - } - def unresolved[Unknown: P](stop: Option[Token]): P[SoftAST.Unresolved] = P { Index ~ diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TextParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TextParser.scala new file mode 100644 index 000000000..d9587b278 --- /dev/null +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TextParser.scala @@ -0,0 +1,19 @@ +package org.alephium.ralph.lsp.access.compiler.parser.soft + +import fastparse._ +import fastparse.NoWhitespace.noWhitespaceImplicit +import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range +import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token} + +private object TextParser { + + def parseOrFail[Unknown: P](stop: Token*): P[SoftAST.CodeString] = + P(Index ~ TokenParser.WhileNotOrFail(stop: _*).! ~ Index) map { + case (from, text, to) => + SoftAST.CodeString( + text = text, + index = range(from, to) + ) + } + +} From 900aeec80552c9038262a1537caf170e4a550e60 Mon Sep 17 00:00:00 2001 From: simerplaha Date: Mon, 13 Jan 2025 12:38:52 +1100 Subject: [PATCH 06/11] move `unresolved` to `UnresolvedParser` --- .../compiler/parser/soft/BlockParser.scala | 2 +- .../compiler/parser/soft/CommonParser.scala | 15 ------- .../parser/soft/UnresolvedParser.scala | 41 +++++++++++++++++++ 3 files changed, 42 insertions(+), 16 deletions(-) create mode 100644 compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/UnresolvedParser.scala diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/BlockParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/BlockParser.scala index 017691b4b..8675c3625 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/BlockParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/BlockParser.scala @@ -85,7 +85,7 @@ private object BlockParser { FunctionParser.parseOrFail | ExpressionParser.parseOrFail | CommentParser.parseOrFail | - unresolved(stop) + UnresolvedParser.parseOrFail(stop) } } diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommonParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommonParser.scala index a575b412e..09a681954 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommonParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommonParser.scala @@ -38,21 +38,6 @@ private object CommonParser { SoftAST.Space(text) } - def unresolved[Unknown: P](stop: Option[Token]): P[SoftAST.Unresolved] = - P { - Index ~ - CodeParser.parseOrFail(TokenParser.WhileNotOrFail(Seq(Token.Space, Token.Newline) ++ stop: _*).!) ~ - CommentParser.parseOrFail.? ~ - Index - } map { - case (from, text, tailComment, to) => - SoftAST.Unresolved( - index = range(from, to), - code = text, - documentation = tailComment - ) - } - def identifier[Unknown: P](required: Boolean): P[SoftAST.IdentifierAST] = if (required) identifier diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/UnresolvedParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/UnresolvedParser.scala new file mode 100644 index 000000000..4c54351c5 --- /dev/null +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/UnresolvedParser.scala @@ -0,0 +1,41 @@ +// Copyright 2024 The Alephium Authors +// This file is part of the alephium project. +// +// The library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the library. If not, see http://www.gnu.org/licenses/. + +package org.alephium.ralph.lsp.access.compiler.parser.soft + +import fastparse._ +import fastparse.NoWhitespace.noWhitespaceImplicit +import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range +import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token} + +private object UnresolvedParser { + + def parseOrFail[Unknown: P](stop: Option[Token]): P[SoftAST.Unresolved] = + P { + Index ~ + CodeParser.parseOrFail(TokenParser.WhileNotOrFail(Seq(Token.Space, Token.Newline) ++ stop: _*).!) ~ + CommentParser.parseOrFail.? ~ + Index + } map { + case (from, text, tailComment, to) => + SoftAST.Unresolved( + index = range(from, to), + code = text, + documentation = tailComment + ) + } + +} From 87db0c9bb38b3c537d84b840fed480436ec552a2 Mon Sep 17 00:00:00 2001 From: simerplaha Date: Mon, 13 Jan 2025 12:50:15 +1100 Subject: [PATCH 07/11] resolved `TODO` in `CommonParser` and moved `identifier` to `IdentifierParser` --- .../parser/soft/AnnotationParser.scala | 2 +- .../parser/soft/AssignmentParser.scala | 2 +- .../compiler/parser/soft/CommonParser.scala | 55 +--------------- .../parser/soft/DataTemplateParser.scala | 2 +- .../parser/soft/ExpressionParser.scala | 3 +- .../compiler/parser/soft/FunctionParser.scala | 2 +- .../parser/soft/IdentifierParser.scala | 65 +++++++++++++++++++ .../parser/soft/ReferenceCallParser.scala | 2 +- .../compiler/parser/soft/TemplateParser.scala | 4 +- .../compiler/parser/soft/TokenParser.scala | 4 +- .../parser/soft/TypeAssignmentParser.scala | 2 +- .../compiler/parser/soft/TypeParser.scala | 3 +- .../compiler/parser/soft/TestParser.scala | 10 +-- .../parser/soft/ast/ReservedTokenSpec.scala | 29 ++++++--- 14 files changed, 104 insertions(+), 81 deletions(-) create mode 100644 compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/IdentifierParser.scala diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/AnnotationParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/AnnotationParser.scala index c7e4c71f6..a7fb32e30 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/AnnotationParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/AnnotationParser.scala @@ -41,7 +41,7 @@ private object AnnotationParser { Index ~ TokenParser.parseOrFail(Token.At) ~ spaceOrFail.? ~ - identifier ~ + IdentifierParser.parse ~ spaceOrFail.? ~ TupleParser.parseOrFail.? ~ spaceOrFail.? ~ diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/AssignmentParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/AssignmentParser.scala index fce74b9de..a30768830 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/AssignmentParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/AssignmentParser.scala @@ -12,7 +12,7 @@ private object AssignmentParser { P { Index ~ AssignmentAccessModifierParser.parseOrFail.rep ~ - identifierOrFail ~ + IdentifierParser.parseOrFail ~ spaceOrFail.? ~ TokenParser.parseOrFail(Token.Equal) ~ spaceOrFail.? ~ diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommonParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommonParser.scala index 09a681954..aea8cf783 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommonParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommonParser.scala @@ -18,8 +18,8 @@ package org.alephium.ralph.lsp.access.compiler.parser.soft import fastparse._ import fastparse.NoWhitespace.noWhitespaceImplicit -import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.{point, range} -import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token} +import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range +import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.SoftAST private object CommonParser { @@ -38,51 +38,6 @@ private object CommonParser { SoftAST.Space(text) } - def identifier[Unknown: P](required: Boolean): P[SoftAST.IdentifierAST] = - if (required) - identifier - else - identifierOrFail - - def identifier[Unknown: P]: P[SoftAST.IdentifierAST] = - P(Index ~ identifierOrFail.?) map { - case (_, Some(identifier)) => - identifier - - case (from, None) => - SoftAST.IdentifierExpected(point(from)) - } - - /** - * Parses identifiers as long as they are not reserved tokens [[Token.Reserved]]. - * - * For example, the following code will result in an [[SoftAST.IdentifierExpected]] error - * because `let` is a reserved token [[Token.Let]]: - * - * {{{ - * fn let() -> () - * }}} - * - * TODO: Handle cases such as `fn letter() -> ()` - * - * @return A successfully parsed identifier instance [[SoftAST.Identifier]] or a parser error. - */ - def identifierOrFail[Unknown: P]: P[SoftAST.Identifier] = - P { - Index ~ - CommentParser.parseOrFail.? ~ - !TokenParser.Reserved ~ - toCodeOrFail(isLetterDigitOrUnderscore.!) ~ - Index - } map { - case (from, documentation, code, to) => - SoftAST.Identifier( - index = range(from, to), - code = code, - documentation = documentation - ) - } - def toCodeOrFail[Unknown: P](parser: => P[String]): P[SoftAST.CodeString] = P(Index ~ parser ~ Index) map { case (from, code, to) => @@ -92,10 +47,4 @@ private object CommonParser { ) } - private def isLetterDigitOrUnderscore[Unknown: P]: P[Unit] = - CharsWhile { - char => - char.isLetterOrDigit || Token.Underscore.lexeme.contains(char) - } - } diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/DataTemplateParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/DataTemplateParser.scala index 41af9b487..6800ba6b5 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/DataTemplateParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/DataTemplateParser.scala @@ -13,7 +13,7 @@ private object DataTemplateParser { Index ~ (TokenParser.parseOrFail(Token.Struct) | TokenParser.parseOrFail(Token.Enum) | TokenParser.parseOrFail(Token.Event)) ~ space ~ - identifier ~ + IdentifierParser.parse ~ spaceOrFail.? ~ ParameterParser.parse ~ Index diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ExpressionParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ExpressionParser.scala index 1d156e8b5..de19158ab 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ExpressionParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ExpressionParser.scala @@ -19,7 +19,6 @@ package org.alephium.ralph.lsp.access.compiler.parser.soft import fastparse._ import fastparse.NoWhitespace.noWhitespaceImplicit import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range -import org.alephium.ralph.lsp.access.compiler.parser.soft.CommonParser._ import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.SoftAST private object ExpressionParser { @@ -84,7 +83,7 @@ private object ExpressionParser { ReferenceCallParser.parseOrFail | AnnotationParser.parseOrFail | TupleParser.parseOrFail | - identifierOrFail + IdentifierParser.parseOrFail } } diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/FunctionParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/FunctionParser.scala index 53ec110c6..5197e2aeb 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/FunctionParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/FunctionParser.scala @@ -66,7 +66,7 @@ private object FunctionParser { private def signature[Unknown: P]: P[SoftAST.FunctionSignature] = P { Index ~ - identifier ~ + IdentifierParser.parse ~ spaceOrFail.? ~ ParameterParser.parse ~ spaceOrFail.? ~ diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/IdentifierParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/IdentifierParser.scala new file mode 100644 index 000000000..824fea4a5 --- /dev/null +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/IdentifierParser.scala @@ -0,0 +1,65 @@ +package org.alephium.ralph.lsp.access.compiler.parser.soft + +import fastparse._ +import fastparse.NoWhitespace.noWhitespaceImplicit +import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra._ +import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token} +import CommonParser._ + +object IdentifierParser { + + def parse[Unknown: P](required: Boolean): P[SoftAST.IdentifierAST] = + if (required) + parse + else + parseOrFail + + def parse[Unknown: P]: P[SoftAST.IdentifierAST] = + P(Index ~ parseOrFail.?) map { + case (_, Some(identifier)) => + identifier + + case (from, None) => + SoftAST.IdentifierExpected(point(from)) + } + + /** + * Parses identifiers as long as they are not reserved tokens [[Token.Reserved]]. + * + * For example, the following code will result in an [[SoftAST.IdentifierExpected]] error + * because `let` is a reserved token [[Token.Let]]: + * + * {{{ + * fn let() -> () + * }}} + * + * @return A successfully parsed identifier instance [[SoftAST.Identifier]] or a parser error. + */ + def parseOrFail[Unknown: P]: P[SoftAST.Identifier] = { + @inline def reserved() = + TokenParser.Reserved(Token.Hash) + + P { + Index ~ + CommentParser.parseOrFail.? ~ + !(reserved() ~ spaceOrFail) ~ // disallow reserved names such as `let mut = 1`. + !(reserved() ~ End) ~ // also handle cases where tail is the end of file `let mut`. + CodeParser.parseOrFail(isDevDefinedName.!) ~ + Index + } map { + case (from, documentation, code, to) => + SoftAST.Identifier( + index = range(from, to), + code = code, + documentation = documentation + ) + } + } + + private def isDevDefinedName[Unknown: P]: P[Unit] = + CharsWhile { + char => + char.isLetterOrDigit || Token.Underscore.lexeme.contains(char) + } + +} diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ReferenceCallParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ReferenceCallParser.scala index d2cc4a546..2181a713c 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ReferenceCallParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ReferenceCallParser.scala @@ -26,7 +26,7 @@ private object ReferenceCallParser { private def parse[Unknown: P](required: Boolean): P[SoftAST.ReferenceCall] = P { Index ~ - identifier(required) ~ + IdentifierParser.parse(required) ~ spaceOrFail.? ~ TupleParser.parse(required) ~ Index diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TemplateParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TemplateParser.scala index 308ddceb3..c2b8b6507 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TemplateParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TemplateParser.scala @@ -13,7 +13,7 @@ private object TemplateParser { Index ~ (TokenParser.parseOrFail(Token.Contract) | TokenParser.parseOrFail(Token.TxScript) | TokenParser.parseOrFail(Token.AssetScript)) ~ space ~ - identifier ~ + IdentifierParser.parse ~ spaceOrFail.? ~ ParameterParser.parseOrFail.? ~ spaceOrFail.? ~ @@ -41,7 +41,7 @@ private object TemplateParser { Index ~ (TokenParser.parseOrFail(Token.Implements) | TokenParser.parseOrFail(Token.Extends)) ~ space ~ - (ReferenceCallParser.parseOrFail | identifier) ~ + (ReferenceCallParser.parseOrFail | IdentifierParser.parse) ~ spaceOrFail.? ~ Index } map { diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TokenParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TokenParser.scala index 0460023ea..3fb9496e8 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TokenParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TokenParser.scala @@ -57,8 +57,8 @@ private object TokenParser { /** * Parses all reserved tokens defined in [[Token.reserved]] and returns the first match. */ - def Reserved[Unknown: P]: P[Token.Reserved] = - ParserUtil.orTokenCombinator(Token.reserved.iterator) + def Reserved[Unknown: P](remove: Token.Reserved*): P[Token.Reserved] = + ParserUtil.orTokenCombinator(Token.reserved.diff(remove).iterator) /** * Parses all tokens of type [[Token.InfixOperator]] and also their comments. diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TypeAssignmentParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TypeAssignmentParser.scala index 059353ef8..662c13da6 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TypeAssignmentParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TypeAssignmentParser.scala @@ -12,7 +12,7 @@ private object TypeAssignmentParser { P { Index ~ AssignmentAccessModifierParser.parseOrFail.rep ~ - identifierOrFail ~ + IdentifierParser.parseOrFail ~ spaceOrFail.? ~ TokenParser.parseOrFail(Token.Colon) ~ spaceOrFail.? ~ diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TypeParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TypeParser.scala index 7014af840..0a5f4328b 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TypeParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TypeParser.scala @@ -17,12 +17,11 @@ package org.alephium.ralph.lsp.access.compiler.parser.soft import fastparse._ -import org.alephium.ralph.lsp.access.compiler.parser.soft.CommonParser._ import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.SoftAST private object TypeParser { def parse[Unknown: P]: P[SoftAST.TypeAST] = - P(TupleParser.parseOrFail | identifier) + P(TupleParser.parseOrFail | IdentifierParser.parse) } diff --git a/compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TestParser.scala b/compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TestParser.scala index a1087a13e..495d1ada6 100644 --- a/compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TestParser.scala +++ b/compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TestParser.scala @@ -51,14 +51,14 @@ object TestParser { def parseType(code: String): SoftAST.TypeAST = runSoftParser(TypeParser.parse(_))(code) - def parseReservedToken(code: String): Token.Reserved = - runAnyParser(TokenParser.Reserved(_))(code) + def parseReservedToken(remove: Token.Reserved*)(code: String): Token.Reserved = + runAnyParser(TokenParser.Reserved(remove: _*)(_))(code) def parseInfixOperatorOrFail(code: String): SoftAST.TokenDocumented[Token.InfixOperator] = runAnyParser(TokenParser.InfixOperatorOrFail(_))(code) - def parseReservedTokenOrError(code: String): Either[Parsed.Failure, Token.Reserved] = - runAnyParserOrError(TokenParser.Reserved(_))(code) + def parseReservedTokenOrError(remove: Token.Reserved*)(code: String): Either[Parsed.Failure, Token.Reserved] = + runAnyParserOrError(TokenParser.Reserved(remove: _*)(_))(code) def findAnnotation(identifier: String)(code: String): Option[SoftAST.Annotation] = findAnnotation( @@ -96,7 +96,7 @@ object TestParser { // AST returned by the parser should ALWAYS emit code identical to the inputted code. // Always assert this for each test case. try - code shouldBe astToCode + astToCode shouldBe code catch { case throwable: Throwable => // Print debug info for cases where the emitted code differs from the input code. diff --git a/compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ast/ReservedTokenSpec.scala b/compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ast/ReservedTokenSpec.scala index 8adc7ecdd..cb6af9f71 100644 --- a/compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ast/ReservedTokenSpec.scala +++ b/compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ast/ReservedTokenSpec.scala @@ -10,20 +10,31 @@ import scala.util.Random class ReservedTokenSpec extends AnyWordSpec with Matchers { - "parse reserved tokens" in { - val tokens = - Random.shuffle(Token.reserved.toList) + "succeed" when { + "tokens are reserved" in { + val tokens = + Random.shuffle(Token.reserved.toList) - tokens should not be empty + tokens should not be empty - tokens foreach { - token => - parseReservedToken(token.lexeme) shouldBe token + tokens foreach { + token => + parseReservedToken()(token.lexeme) shouldBe token + } } } - "parse reserved without spaces" in { - parseReservedTokenOrError("blah").left.value shouldBe a[Parsed.Failure] + "fail" when { + "token is not reserved" in { + parseReservedTokenOrError()("blah").left.value shouldBe a[Parsed.Failure] + } + + "a reserved token is removed" in { + // First parse without removing Hash. + parseReservedTokenOrError()(Token.Hash.lexeme).value shouldBe Token.Hash + // Then parse removing Hash. This should error because Hash is no longer a reserved token. + parseReservedTokenOrError(remove = Token.Hash)(Token.Hash.lexeme).left.value shouldBe a[Parsed.Failure] + } } } From b5572b2ff53e0cd2c30f54368d2085495fa3cf74 Mon Sep 17 00:00:00 2001 From: simerplaha Date: Mon, 13 Jan 2025 12:56:32 +1100 Subject: [PATCH 08/11] `SpaceParser` --- .../parser/soft/AccessModifierParser.scala | 7 +++---- .../compiler/parser/soft/AnnotationParser.scala | 7 +++---- .../soft/AssignmentAccessModifierParser.scala | 3 +-- .../compiler/parser/soft/AssignmentParser.scala | 5 ++--- .../compiler/parser/soft/BlockParser.scala | 9 ++++----- .../compiler/parser/soft/CommentParser.scala | 10 +++++----- .../parser/soft/DataTemplateParser.scala | 5 ++--- .../compiler/parser/soft/ForLoopParser.scala | 17 ++++++++--------- .../compiler/parser/soft/FunctionParser.scala | 13 ++++++------- .../compiler/parser/soft/IdentifierParser.scala | 5 ++--- .../compiler/parser/soft/InfixCallParser.scala | 5 ++--- .../compiler/parser/soft/MethodCallParser.scala | 5 ++--- .../parser/soft/ReferenceCallParser.scala | 3 +-- .../parser/soft/ReturnStatementParser.scala | 3 +-- .../{CommonParser.scala => SpaceParser.scala} | 10 +++++----- .../compiler/parser/soft/TemplateParser.scala | 11 +++++------ .../compiler/parser/soft/TupleParser.scala | 9 ++++----- .../parser/soft/TypeAssignmentParser.scala | 5 ++--- .../compiler/parser/soft/WhileLoopParser.scala | 9 ++++----- 19 files changed, 62 insertions(+), 79 deletions(-) rename compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/{CommonParser.scala => SpaceParser.scala} (85%) diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/AccessModifierParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/AccessModifierParser.scala index cef9f5052..571af486b 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/AccessModifierParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/AccessModifierParser.scala @@ -19,7 +19,6 @@ package org.alephium.ralph.lsp.access.compiler.parser.soft import fastparse._ import fastparse.NoWhitespace.noWhitespaceImplicit import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range -import org.alephium.ralph.lsp.access.compiler.parser.soft.CommonParser._ import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token} private object AccessModifierParser { @@ -28,13 +27,13 @@ private object AccessModifierParser { P { Index ~ TokenParser.parseOrFail(Token.Pub) ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ Index } map { - case (from, doubleForwardSlash, space, to) => + case (from, pub, space, to) => SoftAST.AccessModifier( index = range(from, to), - doubleForwardSlash, + pub = pub, postTokenSpace = space ) } diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/AnnotationParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/AnnotationParser.scala index a7fb32e30..4b21b5a33 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/AnnotationParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/AnnotationParser.scala @@ -19,7 +19,6 @@ package org.alephium.ralph.lsp.access.compiler.parser.soft import fastparse._ import fastparse.NoWhitespace.noWhitespaceImplicit import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range -import org.alephium.ralph.lsp.access.compiler.parser.soft.CommonParser._ import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token} private object AnnotationParser { @@ -40,11 +39,11 @@ private object AnnotationParser { P { Index ~ TokenParser.parseOrFail(Token.At) ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ IdentifierParser.parse ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ TupleParser.parseOrFail.? ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ Index } map { case (from, at, preIdentifierSpace, identifier, postIdentifierSpace, tuple, postTupleSpace, to) => diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/AssignmentAccessModifierParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/AssignmentAccessModifierParser.scala index 6ddf2dbcf..7833e9f60 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/AssignmentAccessModifierParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/AssignmentAccessModifierParser.scala @@ -19,7 +19,6 @@ package org.alephium.ralph.lsp.access.compiler.parser.soft import fastparse._ import fastparse.NoWhitespace.noWhitespaceImplicit import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range -import org.alephium.ralph.lsp.access.compiler.parser.soft.CommonParser._ import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token} private object AssignmentAccessModifierParser { @@ -28,7 +27,7 @@ private object AssignmentAccessModifierParser { P { Index ~ (TokenParser.parseOrFail(Token.Let) | TokenParser.parseOrFail(Token.Mut)) ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ Index } map { case (from, dataAssignment, space, to) => diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/AssignmentParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/AssignmentParser.scala index a30768830..f863dcb12 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/AssignmentParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/AssignmentParser.scala @@ -3,7 +3,6 @@ package org.alephium.ralph.lsp.access.compiler.parser.soft import fastparse._ import fastparse.NoWhitespace.noWhitespaceImplicit import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range -import org.alephium.ralph.lsp.access.compiler.parser.soft.CommonParser._ import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token} private object AssignmentParser { @@ -13,9 +12,9 @@ private object AssignmentParser { Index ~ AssignmentAccessModifierParser.parseOrFail.rep ~ IdentifierParser.parseOrFail ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ TokenParser.parseOrFail(Token.Equal) ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ ExpressionParser.parse ~ Index } map { diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/BlockParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/BlockParser.scala index 8675c3625..3ee3128bc 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/BlockParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/BlockParser.scala @@ -19,7 +19,6 @@ package org.alephium.ralph.lsp.access.compiler.parser.soft import fastparse._ import fastparse.NoWhitespace.noWhitespaceImplicit import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range -import org.alephium.ralph.lsp.access.compiler.parser.soft.CommonParser._ import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token} private object BlockParser { @@ -28,9 +27,9 @@ private object BlockParser { P { Index ~ TokenParser.parse(required, Token.OpenCurly) ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ body(Some(Token.CloseCurly)) ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ TokenParser.parse(Token.CloseCurly) ~ Index } map { @@ -51,7 +50,7 @@ private object BlockParser { private def body[Unknown: P](stop: Option[Token]): P[SoftAST.BlockBody] = P { Index ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ bodyPart(stop).rep ~ Index } map { @@ -67,7 +66,7 @@ private object BlockParser { P { Index ~ part(stop) ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ Index } map { case (from, ast, space, to) => diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommentParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommentParser.scala index 01e7c97c4..7c03ea37b 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommentParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommentParser.scala @@ -19,7 +19,7 @@ package org.alephium.ralph.lsp.access.compiler.parser.soft import fastparse._ import fastparse.NoWhitespace.noWhitespaceImplicit import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range -import org.alephium.ralph.lsp.access.compiler.parser.soft.CommonParser._ +import org.alephium.ralph.lsp.access.compiler.parser.soft.SpaceParser import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token} private object CommentParser { @@ -39,9 +39,9 @@ private object CommentParser { def parseOrFail[Unknown: P]: P[SoftAST.Comments] = P { Index ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ one.rep(1) ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ Index } map { case (from, preSpace, comments, postSpace, to) => @@ -64,9 +64,9 @@ private object CommentParser { P { Index ~ TokenParser.parseOrFailUndocumented(Token.DoubleForwardSlash) ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ TextParser.parseOrFail(Token.Newline).? ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ Index } map { case (from, doubleForwardSlash, preTextSpace, text, postTextSpace, to) => diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/DataTemplateParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/DataTemplateParser.scala index 6800ba6b5..98f161e58 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/DataTemplateParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/DataTemplateParser.scala @@ -3,7 +3,6 @@ package org.alephium.ralph.lsp.access.compiler.parser.soft import fastparse._ import fastparse.NoWhitespace.noWhitespaceImplicit import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range -import org.alephium.ralph.lsp.access.compiler.parser.soft.CommonParser._ import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token} private object DataTemplateParser { @@ -12,9 +11,9 @@ private object DataTemplateParser { P { Index ~ (TokenParser.parseOrFail(Token.Struct) | TokenParser.parseOrFail(Token.Enum) | TokenParser.parseOrFail(Token.Event)) ~ - space ~ + SpaceParser.parse ~ IdentifierParser.parse ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ ParameterParser.parse ~ Index } map { diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ForLoopParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ForLoopParser.scala index e27fa3f46..2ca74a1dc 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ForLoopParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ForLoopParser.scala @@ -3,7 +3,6 @@ package org.alephium.ralph.lsp.access.compiler.parser.soft import fastparse._ import fastparse.NoWhitespace.noWhitespaceImplicit import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range -import org.alephium.ralph.lsp.access.compiler.parser.soft.CommonParser._ import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token} private object ForLoopParser { @@ -12,21 +11,21 @@ private object ForLoopParser { P { Index ~ TokenParser.parseOrFail(Token.For) ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ TokenParser.parse(Token.OpenParen) ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ ExpressionParser.parse ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ TokenParser.parse(Token.Semicolon) ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ ExpressionParser.parse ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ TokenParser.parse(Token.Semicolon) ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ ExpressionParser.parse ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ TokenParser.parse(Token.CloseParen) ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ BlockParser.clause(required = true) ~ Index } map { diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/FunctionParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/FunctionParser.scala index 5197e2aeb..848c13f92 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/FunctionParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/FunctionParser.scala @@ -19,7 +19,6 @@ package org.alephium.ralph.lsp.access.compiler.parser.soft import fastparse._ import fastparse.NoWhitespace.noWhitespaceImplicit import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range -import org.alephium.ralph.lsp.access.compiler.parser.soft.CommonParser._ import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token} private object FunctionParser { @@ -34,12 +33,12 @@ private object FunctionParser { P { Index ~ AnnotationParser.parseOrFail.rep ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ AccessModifierParser.parseOrFail.? ~ TokenParser.parseOrFail(Token.Fn) ~ - space ~ + SpaceParser.parse ~ signature ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ BlockParser.clause(required = false).? ~ Index } map { @@ -67,9 +66,9 @@ private object FunctionParser { P { Index ~ IdentifierParser.parse ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ ParameterParser.parse ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ returnSignature ~ Index } map { @@ -93,7 +92,7 @@ private object FunctionParser { private def returnSignature[Unknown: P]: P[SoftAST.FunctionReturnAST] = P { Index ~ - (TokenParser.parse(Token.ForwardArrow) ~ spaceOrFail.? ~ TypeParser.parse).? ~ + (TokenParser.parse(Token.ForwardArrow) ~ SpaceParser.parseOrFail.? ~ TypeParser.parse).? ~ Index } map { case (from, Some((forwardArrow, space, tpe)), to) => diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/IdentifierParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/IdentifierParser.scala index 824fea4a5..99def2be8 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/IdentifierParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/IdentifierParser.scala @@ -4,7 +4,6 @@ import fastparse._ import fastparse.NoWhitespace.noWhitespaceImplicit import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra._ import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token} -import CommonParser._ object IdentifierParser { @@ -42,8 +41,8 @@ object IdentifierParser { P { Index ~ CommentParser.parseOrFail.? ~ - !(reserved() ~ spaceOrFail) ~ // disallow reserved names such as `let mut = 1`. - !(reserved() ~ End) ~ // also handle cases where tail is the end of file `let mut`. + !(reserved() ~ SpaceParser.parseOrFail) ~ // disallow reserved names such as `let mut = 1`. + !(reserved() ~ End) ~ // also handle cases where tail is the end of file `let mut`. CodeParser.parseOrFail(isDevDefinedName.!) ~ Index } map { diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/InfixCallParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/InfixCallParser.scala index 782641381..8942fef75 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/InfixCallParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/InfixCallParser.scala @@ -3,7 +3,6 @@ package org.alephium.ralph.lsp.access.compiler.parser.soft import fastparse._ import fastparse.NoWhitespace.noWhitespaceImplicit import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range -import org.alephium.ralph.lsp.access.compiler.parser.soft.CommonParser._ import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.SoftAST case object InfixCallParser { @@ -12,9 +11,9 @@ case object InfixCallParser { P { Index ~ ExpressionParser.parseOrFailSelective(parseInfix = false, parseMethodCall = true) ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ TokenParser.InfixOperatorOrFail ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ ExpressionParser.parse ~ Index } map { diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/MethodCallParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/MethodCallParser.scala index 77619f7e4..0b43fbcfe 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/MethodCallParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/MethodCallParser.scala @@ -3,7 +3,6 @@ package org.alephium.ralph.lsp.access.compiler.parser.soft import fastparse._ import fastparse.NoWhitespace.noWhitespaceImplicit import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range -import org.alephium.ralph.lsp.access.compiler.parser.soft.CommonParser._ import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token} case object MethodCallParser { @@ -12,7 +11,7 @@ case object MethodCallParser { P { Index ~ ExpressionParser.parseOrFailSelective(parseInfix = false, parseMethodCall = false) ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ dotCall.rep(1) ~ Index } map { @@ -29,7 +28,7 @@ case object MethodCallParser { P { Index ~ TokenParser.parseOrFail(Token.Dot) ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ ReferenceCallParser.parse ~ Index } map { diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ReferenceCallParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ReferenceCallParser.scala index 2181a713c..6bc9f1d90 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ReferenceCallParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ReferenceCallParser.scala @@ -3,7 +3,6 @@ package org.alephium.ralph.lsp.access.compiler.parser.soft import fastparse._ import fastparse.NoWhitespace.noWhitespaceImplicit import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range -import org.alephium.ralph.lsp.access.compiler.parser.soft.CommonParser._ import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.SoftAST private object ReferenceCallParser { @@ -27,7 +26,7 @@ private object ReferenceCallParser { P { Index ~ IdentifierParser.parse(required) ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ TupleParser.parse(required) ~ Index } map { diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ReturnStatementParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ReturnStatementParser.scala index bd6c20a1a..d7ee11aca 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ReturnStatementParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ReturnStatementParser.scala @@ -3,7 +3,6 @@ package org.alephium.ralph.lsp.access.compiler.parser.soft import fastparse._ import fastparse.NoWhitespace.noWhitespaceImplicit import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range -import org.alephium.ralph.lsp.access.compiler.parser.soft.CommonParser._ import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token} private object ReturnStatementParser { @@ -12,7 +11,7 @@ private object ReturnStatementParser { P { Index ~ TokenParser.parseOrFail(Token.Return) ~ - space ~ + SpaceParser.parse ~ ExpressionParser.parse ~ Index } map { diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommonParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/SpaceParser.scala similarity index 85% rename from compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommonParser.scala rename to compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/SpaceParser.scala index aea8cf783..796e2f05c 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommonParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/SpaceParser.scala @@ -21,10 +21,10 @@ import fastparse.NoWhitespace.noWhitespaceImplicit import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.SoftAST -private object CommonParser { +object SpaceParser { - def space[Unknown: P]: P[SoftAST.SpaceAST] = - P(Index ~ spaceOrFail.?) map { + def parse[Unknown: P]: P[SoftAST.SpaceAST] = + P(Index ~ parseOrFail.?) map { case (_, Some(space)) => space @@ -32,13 +32,13 @@ private object CommonParser { SoftAST.SpaceExpected(range(from, from)) } - def spaceOrFail[Unknown: P]: P[SoftAST.Space] = + def parseOrFail[Unknown: P]: P[SoftAST.Space] = P(toCodeOrFail(CharsWhileIn(" \t\r\n").!)) map { text => SoftAST.Space(text) } - def toCodeOrFail[Unknown: P](parser: => P[String]): P[SoftAST.CodeString] = + private def toCodeOrFail[Unknown: P](parser: => P[String]): P[SoftAST.CodeString] = P(Index ~ parser ~ Index) map { case (from, code, to) => SoftAST.CodeString( diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TemplateParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TemplateParser.scala index c2b8b6507..34584e6fc 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TemplateParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TemplateParser.scala @@ -3,7 +3,6 @@ package org.alephium.ralph.lsp.access.compiler.parser.soft import fastparse._ import fastparse.NoWhitespace.noWhitespaceImplicit import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range -import org.alephium.ralph.lsp.access.compiler.parser.soft.CommonParser._ import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token} private object TemplateParser { @@ -12,11 +11,11 @@ private object TemplateParser { P { Index ~ (TokenParser.parseOrFail(Token.Contract) | TokenParser.parseOrFail(Token.TxScript) | TokenParser.parseOrFail(Token.AssetScript)) ~ - space ~ + SpaceParser.parse ~ IdentifierParser.parse ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ ParameterParser.parseOrFail.? ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ inheritance.rep ~ BlockParser.clause(required = true) ~ Index @@ -40,9 +39,9 @@ private object TemplateParser { P { Index ~ (TokenParser.parseOrFail(Token.Implements) | TokenParser.parseOrFail(Token.Extends)) ~ - space ~ + SpaceParser.parse ~ (ReferenceCallParser.parseOrFail | IdentifierParser.parse) ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ Index } map { case (from, token, preConstructorCallSpace, constructorCall, postConstructorCallSpace, to) => diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TupleParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TupleParser.scala index cb340f2f8..23c108c98 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TupleParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TupleParser.scala @@ -19,7 +19,6 @@ package org.alephium.ralph.lsp.access.compiler.parser.soft import fastparse._ import fastparse.NoWhitespace.noWhitespaceImplicit import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.{point, range} -import org.alephium.ralph.lsp.access.compiler.parser.soft.CommonParser._ import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token} private object TupleParser { @@ -45,10 +44,10 @@ private object TupleParser { P { Index ~ TokenParser.parse(required, Token.OpenParen) ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ Index ~ ExpressionParser.parseOrFail.? ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ tailParams.rep ~ TokenParser.parse(Token.CloseParen) ~ Index @@ -85,9 +84,9 @@ private object TupleParser { P { Index ~ TokenParser.parseOrFail(Token.Comma) ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ ExpressionParser.parse ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ Index } map { case (from, comma, preParamNameSpace, argumentName, postParamNameSpace, to) => diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TypeAssignmentParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TypeAssignmentParser.scala index 662c13da6..c1bd02039 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TypeAssignmentParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TypeAssignmentParser.scala @@ -3,7 +3,6 @@ package org.alephium.ralph.lsp.access.compiler.parser.soft import fastparse._ import fastparse.NoWhitespace.noWhitespaceImplicit import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range -import org.alephium.ralph.lsp.access.compiler.parser.soft.CommonParser._ import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token} private object TypeAssignmentParser { @@ -13,9 +12,9 @@ private object TypeAssignmentParser { Index ~ AssignmentAccessModifierParser.parseOrFail.rep ~ IdentifierParser.parseOrFail ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ TokenParser.parseOrFail(Token.Colon) ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ TypeParser.parse ~ Index } map { diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/WhileLoopParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/WhileLoopParser.scala index 89d786364..e000326e5 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/WhileLoopParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/WhileLoopParser.scala @@ -3,7 +3,6 @@ package org.alephium.ralph.lsp.access.compiler.parser.soft import fastparse._ import fastparse.NoWhitespace.noWhitespaceImplicit import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range -import org.alephium.ralph.lsp.access.compiler.parser.soft.CommonParser._ import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token} private object WhileLoopParser { @@ -12,13 +11,13 @@ private object WhileLoopParser { P { Index ~ TokenParser.parseOrFail(Token.While) ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ TokenParser.parse(Token.OpenParen) ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ ExpressionParser.parse ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ TokenParser.parse(Token.CloseParen) ~ - spaceOrFail.? ~ + SpaceParser.parseOrFail.? ~ BlockParser.clause(required = true) ~ Index } map { From 24c8d57ab600d7961225c2748e2dccada8217fd3 Mon Sep 17 00:00:00 2001 From: simerplaha Date: Mon, 13 Jan 2025 13:04:04 +1100 Subject: [PATCH 09/11] `TokenParser.WhileInOrFail` --- .../access/compiler/parser/soft/SpaceParser.scala | 13 ++----------- .../access/compiler/parser/soft/TokenParser.scala | 3 +++ .../lsp/access/compiler/parser/soft/ast/Token.scala | 1 + 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/SpaceParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/SpaceParser.scala index 796e2f05c..54f2a8057 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/SpaceParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/SpaceParser.scala @@ -19,7 +19,7 @@ package org.alephium.ralph.lsp.access.compiler.parser.soft import fastparse._ import fastparse.NoWhitespace.noWhitespaceImplicit import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range -import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.SoftAST +import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token} object SpaceParser { @@ -33,18 +33,9 @@ object SpaceParser { } def parseOrFail[Unknown: P]: P[SoftAST.Space] = - P(toCodeOrFail(CharsWhileIn(" \t\r\n").!)) map { + P(CodeParser.parseOrFail(TokenParser.WhileInOrFail(Token.Space, Token.Tab, Token.Newline).!)) map { text => SoftAST.Space(text) } - private def toCodeOrFail[Unknown: P](parser: => P[String]): P[SoftAST.CodeString] = - P(Index ~ parser ~ Index) map { - case (from, code, to) => - SoftAST.CodeString( - text = code, - index = range(from, to) - ) - } - } diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TokenParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TokenParser.scala index 3fb9496e8..6a50b4167 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TokenParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TokenParser.scala @@ -88,4 +88,7 @@ private object TokenParser { def WhileNotOrFail[Unknown: P](tokens: Token*): P[Unit] = P((!ParserUtil.orTokenCombinator(tokens.iterator) ~ AnyChar).rep(1)) + def WhileInOrFail[Unknown: P](tokens: Token*): P[Unit] = + P(ParserUtil.orTokenCombinator(tokens.iterator).rep(1)) + } diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ast/Token.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ast/Token.scala index fed73c833..db370aaf2 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ast/Token.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/ast/Token.scala @@ -86,6 +86,7 @@ object Token { case object Semicolon extends Delimiter(";") with Reserved case object Newline extends Delimiter(System.lineSeparator()) case object Space extends Delimiter(" ") + case object Tab extends Delimiter("\t") case object DoubleForwardSlash extends Delimiter("//") with Reserved sealed abstract class Punctuator(override val lexeme: String) extends Token From b3a499b0db94a7774422bbcade93210c01ce38d7 Mon Sep 17 00:00:00 2001 From: simerplaha Date: Mon, 13 Jan 2025 13:10:48 +1100 Subject: [PATCH 10/11] allow a sequence of `Token`s instead of `Option` --- .../access/compiler/parser/soft/BlockParser.scala | 12 ++++++------ .../access/compiler/parser/soft/CommentParser.scala | 1 - .../compiler/parser/soft/UnresolvedParser.scala | 4 ++-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/BlockParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/BlockParser.scala index 3ee3128bc..6e2b3b3db 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/BlockParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/BlockParser.scala @@ -28,7 +28,7 @@ private object BlockParser { Index ~ TokenParser.parse(required, Token.OpenCurly) ~ SpaceParser.parseOrFail.? ~ - body(Some(Token.CloseCurly)) ~ + body(Token.CloseCurly) ~ SpaceParser.parseOrFail.? ~ TokenParser.parse(Token.CloseCurly) ~ Index @@ -45,9 +45,9 @@ private object BlockParser { } def body[Unknown: P]: P[SoftAST.BlockBody] = - body(stop = None) + body() - private def body[Unknown: P](stop: Option[Token]): P[SoftAST.BlockBody] = + private def body[Unknown: P](stop: Token*): P[SoftAST.BlockBody] = P { Index ~ SpaceParser.parseOrFail.? ~ @@ -62,7 +62,7 @@ private object BlockParser { ) } - private def bodyPart[Unknown: P](stop: Option[Token]): P[SoftAST.BlockBodyPart] = + private def bodyPart[Unknown: P](stop: Seq[Token]): P[SoftAST.BlockBodyPart] = P { Index ~ part(stop) ~ @@ -77,14 +77,14 @@ private object BlockParser { ) } - private def part[Unknown: P](stop: Option[Token]): P[SoftAST.BodyPartAST] = + private def part[Unknown: P](stop: Seq[Token]): P[SoftAST.BodyPartAST] = P { TemplateParser.parseOrFail | DataTemplateParser.parseOrFail | FunctionParser.parseOrFail | ExpressionParser.parseOrFail | CommentParser.parseOrFail | - UnresolvedParser.parseOrFail(stop) + UnresolvedParser.parseOrFail(stop: _*) } } diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommentParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommentParser.scala index 7c03ea37b..ff5600d16 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommentParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/CommentParser.scala @@ -19,7 +19,6 @@ package org.alephium.ralph.lsp.access.compiler.parser.soft import fastparse._ import fastparse.NoWhitespace.noWhitespaceImplicit import org.alephium.ralph.lsp.access.compiler.message.SourceIndexExtra.range -import org.alephium.ralph.lsp.access.compiler.parser.soft.SpaceParser import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token} private object CommentParser { diff --git a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/UnresolvedParser.scala b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/UnresolvedParser.scala index 4c54351c5..8021c4135 100644 --- a/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/UnresolvedParser.scala +++ b/compiler-access/src/main/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/UnresolvedParser.scala @@ -23,10 +23,10 @@ import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.{SoftAST, Token} private object UnresolvedParser { - def parseOrFail[Unknown: P](stop: Option[Token]): P[SoftAST.Unresolved] = + def parseOrFail[Unknown: P](stop: Token*): P[SoftAST.Unresolved] = P { Index ~ - CodeParser.parseOrFail(TokenParser.WhileNotOrFail(Seq(Token.Space, Token.Newline) ++ stop: _*).!) ~ + CodeParser.parseOrFail(TokenParser.WhileNotOrFail(stop :+ Token.Space :+ Token.Newline: _*).!) ~ CommentParser.parseOrFail.? ~ Index } map { From 2635af6ea7e23dbe3aef4d170c242d506bc63032 Mon Sep 17 00:00:00 2001 From: simerplaha Date: Mon, 13 Jan 2025 13:33:17 +1100 Subject: [PATCH 11/11] `IdentifierSpec` --- .../compiler/parser/soft/IdentifierSpec.scala | 34 +++++++++++++++++++ .../compiler/parser/soft/TestParser.scala | 6 +++- .../ralph/lsp/access/util/TestFastParse.scala | 18 ++++++++++ 3 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/IdentifierSpec.scala create mode 100644 compiler-access/src/test/scala/org/alephium/ralph/lsp/access/util/TestFastParse.scala diff --git a/compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/IdentifierSpec.scala b/compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/IdentifierSpec.scala new file mode 100644 index 000000000..3eebf0fa3 --- /dev/null +++ b/compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/IdentifierSpec.scala @@ -0,0 +1,34 @@ +package org.alephium.ralph.lsp.access.compiler.parser.soft + +import org.alephium.ralph.lsp.access.compiler.parser.soft.ast.Token +import org.alephium.ralph.lsp.access.compiler.parser.soft.TestParser._ +import org.alephium.ralph.lsp.access.util.TestFastParse._ +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +class IdentifierSpec extends AnyWordSpec with Matchers { + + "disallow reserved tokens to be used as identifier" when { + val reserved = + Token.reserved.diff(Seq(Token.Hash)) // Remove hash because `let hash = #` is valid + + "tail has space" in { + reserved foreach { + reserved => + assertIsFastParseError { + parseIdentifier(s"${reserved.lexeme} ") + } + } + } + + "tail is end-of-file" in { + reserved foreach { + reserved => + assertIsFastParseError { + parseIdentifier(reserved.lexeme) + } + } + } + } + +} diff --git a/compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TestParser.scala b/compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TestParser.scala index 495d1ada6..dd00a30fa 100644 --- a/compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TestParser.scala +++ b/compiler-access/src/test/scala/org/alephium/ralph/lsp/access/compiler/parser/soft/TestParser.scala @@ -60,6 +60,9 @@ object TestParser { def parseReservedTokenOrError(remove: Token.Reserved*)(code: String): Either[Parsed.Failure, Token.Reserved] = runAnyParserOrError(TokenParser.Reserved(remove: _*)(_))(code) + def parseIdentifier(code: String): SoftAST.IdentifierAST = + runSoftParser(IdentifierParser.parseOrFail(_))(code) + def findAnnotation(identifier: String)(code: String): Option[SoftAST.Annotation] = findAnnotation( identifier = identifier, @@ -118,7 +121,8 @@ object TestParser { result match { case Left(error) => // Print a formatted error so it's easier to debug. - fail(CompilerError.FastParseError(error).toFormatter().format(Some(Console.RED))) + val throwable = CompilerError.FastParseError(error) + fail(throwable.toFormatter().format(Some(Console.RED)), throwable) case Right(ast) => ast diff --git a/compiler-access/src/test/scala/org/alephium/ralph/lsp/access/util/TestFastParse.scala b/compiler-access/src/test/scala/org/alephium/ralph/lsp/access/util/TestFastParse.scala new file mode 100644 index 000000000..36873e62f --- /dev/null +++ b/compiler-access/src/test/scala/org/alephium/ralph/lsp/access/util/TestFastParse.scala @@ -0,0 +1,18 @@ +package org.alephium.ralph.lsp.access.util + +import org.alephium.ralph.error.CompilerError +import org.scalatest.TryValues._ +import org.scalatest.matchers.should.Matchers._ +import org.scalatest.Assertion + +import scala.util.Try + +object TestFastParse { + + def assertIsFastParseError[A](f: => A): Assertion = + Try(f) + .failure + .exception + .getCause shouldBe a[CompilerError.FastParseError] + +}