✨ feat: Add backquote, colon, dollarlbrace to tokens

caoccao · Mar 22, 2024 · d2ab60c · d2ab60c
1 parent 80d601e
commit d2ab60c
Show file tree

Hide file tree

Showing 4 changed files with 79 additions and 55 deletions.
diff --git a/rust/src/ast_utils.rs b/rust/src/ast_utils.rs
@@ -497,6 +497,15 @@ pub fn token_and_spans_to_java_list<'local>(
             Token::Comma => {
               java_ast_token_factory.create_generic_operator(env, AstTokenType::Comma, index_range, line_break_ahead)
             }
+            Token::BackQuote => {
+              java_ast_token_factory.create_generic_operator(env, AstTokenType::BackQuote, index_range, line_break_ahead)
+            }
+            Token::Colon => {
+              java_ast_token_factory.create_generic_operator(env, AstTokenType::Colon, index_range, line_break_ahead)
+            }
+            Token::DollarLBrace => {
+              java_ast_token_factory.create_generic_operator(env, AstTokenType::DollarLBrace, index_range, line_break_ahead)
+            }
             _ => java_ast_token_factory.create_unknown(env, &text, index_range, line_break_ahead),
           };
           java_array_list.add(env, &list, &ast_token);

diff --git a/rust/src/enums.rs b/rust/src/enums.rs
@@ -32,61 +32,64 @@ pub trait IdentifiableEnum<T> {
 
 #[derive(Debug, Copy, Clone)]
 pub enum AstTokenType {
-  Unknown,    // 0
-  Await,      // 1
-  Break,      // 2
-  Case,       // 3
-  Catch,      // 4
-  Class,      // 5
-  Const,      // 6
-  Continue,   // 7
-  Debugger,   // 8
-  Default_,   // 9
-  Delete,     // 10
-  Do,         // 11
-  Else,       // 12
-  Export,     // 13
-  Extends,    // 14
-  Finally,    // 15
-  For,        // 16
-  Function,   // 17
-  If,         // 18
-  Import,     // 19
-  In,         // 20
-  InstanceOf, // 21
-  Let,        // 22
-  New,        // 23
-  Return,     // 24
-  Super,      // 25
-  Switch,     // 26
-  This,       // 27
-  Throw,      // 28
-  Try,        // 29
-  TypeOf,     // 30
-  Var,        // 31
-  Void,       // 32
-  While,      // 33
-  With,       // 34
-  Yield,      // 35
-  Null,       // 36
-  True,       // 37
-  False,      // 38
-  IdentKnown, // 39
-  IdentOther, // 40
-  Arrow,      // 41
-  Hash,       // 42
-  At,         // 43
-  Dot,        // 44
-  DotDotDot,  // 45
-  Bang,       // 46
-  LParen,     // 47
-  RParen,     // 48
-  LBracket,   // 49
-  RBracket,   // 50
-  LBrace,     // 51
-  RBrace,     // 52
-  Semi,       // 53
-  Comma,      // 54
+  Unknown,      // 0
+  Await,        // 1
+  Break,        // 2
+  Case,         // 3
+  Catch,        // 4
+  Class,        // 5
+  Const,        // 6
+  Continue,     // 7
+  Debugger,     // 8
+  Default_,     // 9
+  Delete,       // 10
+  Do,           // 11
+  Else,         // 12
+  Export,       // 13
+  Extends,      // 14
+  Finally,      // 15
+  For,          // 16
+  Function,     // 17
+  If,           // 18
+  Import,       // 19
+  In,           // 20
+  InstanceOf,   // 21
+  Let,          // 22
+  New,          // 23
+  Return,       // 24
+  Super,        // 25
+  Switch,       // 26
+  This,         // 27
+  Throw,        // 28
+  Try,          // 29
+  TypeOf,       // 30
+  Var,          // 31
+  Void,         // 32
+  While,        // 33
+  With,         // 34
+  Yield,        // 35
+  Null,         // 36
+  True,         // 37
+  False,        // 38
+  IdentKnown,   // 39
+  IdentOther,   // 40
+  Arrow,        // 41
+  Hash,         // 42
+  At,           // 43
+  Dot,          // 44
+  DotDotDot,    // 45
+  Bang,         // 46
+  LParen,       // 47
+  RParen,       // 48
+  LBracket,     // 49
+  RBracket,     // 50
+  LBrace,       // 51
+  RBrace,       // 52
+  Semi,         // 53
+  Comma,        // 54
+  BackQuote,    // 55
+  Colon,        // 56
+  DollarLBrace, // 57
 }
 
 impl IdentifiableEnum<AstTokenType> for AstTokenType {
@@ -146,6 +149,9 @@ impl IdentifiableEnum<AstTokenType> for AstTokenType {
       AstTokenType::RBrace => 52,
       AstTokenType::Semi => 53,
       AstTokenType::Comma => 54,
+      AstTokenType::BackQuote => 55,
+      AstTokenType::Colon => 56,
+      AstTokenType::DollarLBrace => 57,
       _ => 0,
     }
   }
@@ -205,6 +211,9 @@ impl IdentifiableEnum<AstTokenType> for AstTokenType {
       52 => AstTokenType::RBrace,
       53 => AstTokenType::Semi,
       54 => AstTokenType::Comma,
+      55 => AstTokenType::BackQuote,
+      56 => AstTokenType::Colon,
+      57 => AstTokenType::DollarLBrace,
       _ => AstTokenType::Unknown,
     }
   }

diff --git a/src/main/java/com/caoccao/javet/swc4j/enums/Swc4jAstTokenType.java b/src/main/java/com/caoccao/javet/swc4j/enums/Swc4jAstTokenType.java
@@ -77,6 +77,9 @@ public enum Swc4jAstTokenType {
     RBrace(52, "}", false, true),
     Semi(53, ";", false, true),
     Comma(54, ",", false, true),
+    BackQuote(55, "`", false, true),
+    Colon(56, ":", false, true),
+    DollarLBrace(57, "${", false, true),
     ;
 
     private static final int LENGTH = values().length;

diff --git a/src/test/java/com/caoccao/javet/swc4j/TestSwc4j.java b/src/test/java/com/caoccao/javet/swc4j/TestSwc4j.java
@@ -142,6 +142,9 @@ public void testParseTypeScriptWithCaptureTokens() throws Swc4jCoreException {
         parseAndAssert("a={}", options, Swc4jAstTokenType.RBrace, "}", 3, 4, 3, 4);
         parseAndAssert(";", options, Swc4jAstTokenType.Semi, ";", 0, 1);
         parseAndAssert("let a, b;", options, Swc4jAstTokenType.Comma, ",", 5, 6, 2, 5);
+        parseAndAssert("``", options, Swc4jAstTokenType.BackQuote, "`", 0, 1, 0, 2);
+        parseAndAssert("a={b:c}", options, Swc4jAstTokenType.Colon, ":", 4, 5, 4, 7);
+        parseAndAssert("`${a}`", options, Swc4jAstTokenType.DollarLBrace, "${", 1, 3, 1, 5);
     }
 
     @Test