✨ feat: Add questionmark, plusplus, minusminus, tilde to tokens

caoccao · Mar 22, 2024 · 2bbdbcb · 2bbdbcb
1 parent d2ab60c
commit 2bbdbcb
Show file tree

Hide file tree

Showing 4 changed files with 72 additions and 37 deletions.
diff --git a/rust/src/ast_utils.rs b/rust/src/ast_utils.rs
@@ -506,6 +506,18 @@ pub fn token_and_spans_to_java_list<'local>(
             Token::DollarLBrace => {
               java_ast_token_factory.create_generic_operator(env, AstTokenType::DollarLBrace, index_range, line_break_ahead)
             }
+            Token::QuestionMark => {
+              java_ast_token_factory.create_generic_operator(env, AstTokenType::QuestionMark, index_range, line_break_ahead)
+            }
+            Token::PlusPlus => {
+              java_ast_token_factory.create_generic_operator(env, AstTokenType::PlusPlus, index_range, line_break_ahead)
+            }
+            Token::MinusMinus => {
+              java_ast_token_factory.create_generic_operator(env, AstTokenType::MinusMinus, index_range, line_break_ahead)
+            }
+            Token::Tilde => {
+              java_ast_token_factory.create_generic_operator(env, AstTokenType::Tilde, index_range, line_break_ahead)
+            }
             _ => java_ast_token_factory.create_unknown(env, &text, index_range, line_break_ahead),
           };
           java_array_list.add(env, &list, &ast_token);

diff --git a/rust/src/enums.rs b/rust/src/enums.rs
@@ -32,47 +32,50 @@ pub trait IdentifiableEnum<T> {
 
 #[derive(Debug, Copy, Clone)]
 pub enum AstTokenType {
-  Unknown,      // 0
-  Await,        // 1
-  Break,        // 2
-  Case,         // 3
-  Catch,        // 4
-  Class,        // 5
-  Const,        // 6
-  Continue,     // 7
-  Debugger,     // 8
-  Default_,     // 9
-  Delete,       // 10
-  Do,           // 11
-  Else,         // 12
-  Export,       // 13
-  Extends,      // 14
-  Finally,      // 15
-  For,          // 16
-  Function,     // 17
-  If,           // 18
-  Import,       // 19
-  In,           // 20
-  InstanceOf,   // 21
-  Let,          // 22
-  New,          // 23
-  Return,       // 24
-  Super,        // 25
-  Switch,       // 26
-  This,         // 27
-  Throw,        // 28
-  Try,          // 29
-  TypeOf,       // 30
-  Var,          // 31
-  Void,         // 32
-  While,        // 33
-  With,         // 34
-  Yield,        // 35
+  Unknown, // 0
+  // Keyword
+  Await,      // 1
+  Break,      // 2
+  Case,       // 3
+  Catch,      // 4
+  Class,      // 5
+  Const,      // 6
+  Continue,   // 7
+  Debugger,   // 8
+  Default_,   // 9
+  Delete,     // 10
+  Do,         // 11
+  Else,       // 12
+  Export,     // 13
+  Extends,    // 14
+  Finally,    // 15
+  For,        // 16
+  Function,   // 17
+  If,         // 18
+  Import,     // 19
+  In,         // 20
+  InstanceOf, // 21
+  Let,        // 22
+  New,        // 23
+  Return,     // 24
+  Super,      // 25
+  Switch,     // 26
+  This,       // 27
+  Throw,      // 28
+  Try,        // 29
+  TypeOf,     // 30
+  Var,        // 31
+  Void,       // 32
+  While,      // 33
+  With,       // 34
+  Yield,      // 35
+  // Word
   Null,         // 36
   True,         // 37
   False,        // 38
   IdentKnown,   // 39
   IdentOther,   // 40
+  // Operator - Generic
   Arrow,        // 41
   Hash,         // 42
   At,           // 43
@@ -90,6 +93,10 @@ pub enum AstTokenType {
   BackQuote,    // 55
   Colon,        // 56
   DollarLBrace, // 57
+  QuestionMark, // 58
+  PlusPlus,     // 59
+  MinusMinus,   // 60
+  Tilde,        // 61
 }
 
 impl IdentifiableEnum<AstTokenType> for AstTokenType {
@@ -152,6 +159,10 @@ impl IdentifiableEnum<AstTokenType> for AstTokenType {
       AstTokenType::BackQuote => 55,
       AstTokenType::Colon => 56,
       AstTokenType::DollarLBrace => 57,
+      AstTokenType::QuestionMark => 58,
+      AstTokenType::PlusPlus => 59,
+      AstTokenType::MinusMinus => 60,
+      AstTokenType::Tilde => 61,
       _ => 0,
     }
   }
@@ -214,6 +225,10 @@ impl IdentifiableEnum<AstTokenType> for AstTokenType {
       55 => AstTokenType::BackQuote,
       56 => AstTokenType::Colon,
       57 => AstTokenType::DollarLBrace,
+      58 => AstTokenType::QuestionMark,
+      59 => AstTokenType::PlusPlus,
+      60 => AstTokenType::MinusMinus,
+      61 => AstTokenType::Tilde,
       _ => AstTokenType::Unknown,
     }
   }

diff --git a/src/main/java/com/caoccao/javet/swc4j/enums/Swc4jAstTokenType.java b/src/main/java/com/caoccao/javet/swc4j/enums/Swc4jAstTokenType.java
@@ -62,7 +62,7 @@ public enum Swc4jAstTokenType {
     False(38, "false", false),
     IdentKnown(39, "$IdentKnown", false),
     IdentOther(40, "$IdentOther", false),
-    // Operator
+    // Operator - Generic
     Arrow(41, "=>", false, true),
     Hash(42, "#", false, true),
     At(43, "@", false, true),
@@ -80,6 +80,10 @@ public enum Swc4jAstTokenType {
     BackQuote(55, "`", false, true),
     Colon(56, ":", false, true),
     DollarLBrace(57, "${", false, true),
+    QuestionMark(58, "?", false, true),
+    PlusPlus(59, "++", false, true),
+    MinusMinus(60, "--", false, true),
+    Tilde(61, "~", false, true),
     ;
 
     private static final int LENGTH = values().length;

diff --git a/src/test/java/com/caoccao/javet/swc4j/TestSwc4j.java b/src/test/java/com/caoccao/javet/swc4j/TestSwc4j.java
@@ -145,6 +145,10 @@ public void testParseTypeScriptWithCaptureTokens() throws Swc4jCoreException {
         parseAndAssert("``", options, Swc4jAstTokenType.BackQuote, "`", 0, 1, 0, 2);
         parseAndAssert("a={b:c}", options, Swc4jAstTokenType.Colon, ":", 4, 5, 4, 7);
         parseAndAssert("`${a}`", options, Swc4jAstTokenType.DollarLBrace, "${", 1, 3, 1, 5);
+        parseAndAssert("a?.b", options, Swc4jAstTokenType.QuestionMark, "?", 1, 2, 1, 4);
+        parseAndAssert("a++", options, Swc4jAstTokenType.PlusPlus, "++", 1, 3, 1, 2);
+        parseAndAssert("a--", options, Swc4jAstTokenType.MinusMinus, "--", 1, 3, 1, 2);
+        parseAndAssert("~true", options, Swc4jAstTokenType.Tilde, "~", 0, 1, 0, 2);
     }
 
     @Test