✨ feat: Add regex to tokens

caoccao · Mar 23, 2024 · 3d7cea6 · 3d7cea6
1 parent a73cd50
commit 3d7cea6
Show file tree

Hide file tree

Showing 7 changed files with 161 additions and 0 deletions.
diff --git a/rust/src/ast_utils.rs b/rust/src/ast_utils.rs
@@ -47,6 +47,7 @@ pub struct JavaAstTokenFactory {
   method_create_keyword: JStaticMethodID,
   method_create_null: JStaticMethodID,
   method_create_number: JStaticMethodID,
+  method_create_regex: JStaticMethodID,
   method_create_string: JStaticMethodID,
   method_create_true: JStaticMethodID,
   method_create_unknown: JStaticMethodID,
@@ -132,6 +133,13 @@ impl JavaAstTokenFactory {
         "(Ljava/lang/String;DIIZ)Lcom/caoccao/javet/swc4j/ast/atom/bi/Swc4jAstTokenNumber;",
       )
       .expect("Couldn't find method Swc4jAstTokenFactory.createNumber");
+    let method_create_regex = env
+      .get_static_method_id(
+        &class,
+        "createRegex",
+        "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;IIZ)Lcom/caoccao/javet/swc4j/ast/atom/tri/Swc4jAstTokenRegex;",
+      )
+      .expect("Couldn't find method Swc4jAstTokenFactory.createRegex");
     let method_create_string = env
       .get_static_method_id(
         &class,
@@ -165,6 +173,7 @@ impl JavaAstTokenFactory {
       method_create_keyword,
       method_create_null,
       method_create_number,
+      method_create_regex,
       method_create_string,
       method_create_true,
       method_create_unknown,
@@ -521,6 +530,59 @@ impl JavaAstTokenFactory {
     token
   }
 
+  pub fn create_regex<'local, 'a>(
+    &self,
+    env: &mut JNIEnv<'local>,
+    text: &str,
+    value: &str,
+    flags: &str,
+    range: Range<usize>,
+    line_break_ahead: bool,
+  ) -> JObject<'a>
+  where
+    'local: 'a,
+  {
+    let java_string_text = converter::string_to_jstring(env, &text);
+    let java_string_value = converter::string_to_jstring(env, &value);
+    let java_string_flags = converter::string_to_jstring(env, &flags);
+    let text = jvalue {
+      l: java_string_text.as_raw(),
+    };
+    let value = jvalue {
+      l: java_string_value.as_raw(),
+    };
+    let flags = jvalue {
+      l: java_string_flags.as_raw(),
+    };
+    let start_position = jvalue { i: range.start as i32 };
+    let end_position = jvalue { i: range.end as i32 };
+    let line_break_ahead = jvalue {
+      z: line_break_ahead as u8,
+    };
+    let token = unsafe {
+      env
+        .call_static_method_unchecked(
+          &self.class,
+          self.method_create_regex,
+          ReturnType::Object,
+          &[text, value, flags, start_position, end_position, line_break_ahead],
+        )
+        .expect("Couldn't create Swc4jAstTokenRegex")
+        .l()
+        .expect("Couldn't convert Swc4jAstTokenRegex")
+    };
+    env
+      .delete_local_ref(java_string_text)
+      .expect("Couldn't delete local text");
+    env
+      .delete_local_ref(java_string_value)
+      .expect("Couldn't delete local value");
+    env
+      .delete_local_ref(java_string_flags)
+      .expect("Couldn't delete local flags");
+    token
+  }
+
   pub fn create_string<'local, 'a>(
     &self,
     env: &mut JNIEnv<'local>,
@@ -734,6 +796,9 @@ pub fn token_and_spans_to_java_list<'local>(
             Token::BigInt { value: _, raw } => {
               java_ast_token_factory.create_bigint(env, &raw, index_range, line_break_ahead)
             }
+            Token::Regex (value, flags) => {
+              java_ast_token_factory.create_regex(env, &text, &value, &flags, index_range, line_break_ahead)
+            }
             token => match &AstTokenType::parse_by_generic_operator(token) {
               AstTokenType::Unknown => {
                 eprintln!("Unknown {:?}", token);

diff --git a/rust/src/enums.rs b/rust/src/enums.rs
@@ -143,6 +143,7 @@ pub enum AstTokenType {
   Str, // 101
   Num, // 102
   BigInt, // 103
+  Regex, // 104
 }
 
 impl IdentifiableEnum<AstTokenType> for AstTokenType {
@@ -251,6 +252,7 @@ impl IdentifiableEnum<AstTokenType> for AstTokenType {
       AstTokenType::Str => 101,
       AstTokenType::Num => 102,
       AstTokenType::BigInt => 103,
+      AstTokenType::Regex => 104,
       _ => 0,
     }
   }
@@ -359,6 +361,7 @@ impl IdentifiableEnum<AstTokenType> for AstTokenType {
       101 => AstTokenType::Str,
       102 => AstTokenType::Num,
       103 => AstTokenType::BigInt,
+      104 => AstTokenType::Regex,
       _ => AstTokenType::Unknown,
     }
   }

diff --git a/src/main/java/com/caoccao/javet/swc4j/ast/Swc4jAstTokenFactory.java b/src/main/java/com/caoccao/javet/swc4j/ast/Swc4jAstTokenFactory.java
@@ -18,6 +18,7 @@
 
 import com.caoccao.javet.swc4j.ast.atom.bi.Swc4jAstTokenBigInt;
 import com.caoccao.javet.swc4j.ast.atom.bi.Swc4jAstTokenNumber;
+import com.caoccao.javet.swc4j.ast.atom.tri.Swc4jAstTokenRegex;
 import com.caoccao.javet.swc4j.ast.atom.bi.Swc4jAstTokenString;
 import com.caoccao.javet.swc4j.ast.atom.uni.Swc4jAstTokenUnknown;
 import com.caoccao.javet.swc4j.ast.operators.Swc4jAstTokenAssignOperator;
@@ -184,6 +185,23 @@ public static Swc4jAstTokenNumber createNumber(
         return new Swc4jAstTokenNumber(text, value, startPosition, endPosition, lineBreakAhead);
     }
 
+    /**
+     * Create ast token regex.
+     *
+     * @param text           the text
+     * @param value          the value
+     * @param flags          the flags
+     * @param startPosition  the start position
+     * @param endPosition    the end position
+     * @param lineBreakAhead the line break ahead
+     * @return the ast token regex
+     * @since 0.2.0
+     */
+    public static Swc4jAstTokenRegex createRegex(
+            String text, String value, String flags, int startPosition, int endPosition, boolean lineBreakAhead) {
+        return new Swc4jAstTokenRegex(text, value, flags, startPosition, endPosition, lineBreakAhead);
+    }
+
     /**
      * Create ast token string.
      *

diff --git a/src/main/java/com/caoccao/javet/swc4j/ast/atom/tri/Swc4jAstTokenRegex.java b/src/main/java/com/caoccao/javet/swc4j/ast/atom/tri/Swc4jAstTokenRegex.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2024-2024. caoccao.com Sam Cao
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.caoccao.javet.swc4j.ast.atom.tri;
+
+import com.caoccao.javet.swc4j.ast.atom.bi.BaseSwc4jAstTokenBiAtom;
+import com.caoccao.javet.swc4j.enums.Swc4jAstTokenType;
+
+/**
+ * The type Swc4j ast token regex.
+ *
+ * @since 0.2.0
+ */
+public class Swc4jAstTokenRegex extends BaseSwc4jAstTokenBiAtom<String> {
+    /**
+     * The Flags.
+     */
+    protected final String flags;
+
+    /**
+     * Instantiates a new Swc4j ast token regex.
+     *
+     * @param text           the text
+     * @param value          the value
+     * @param flags          the flags
+     * @param startPosition  the start position
+     * @param endPosition    the end position
+     * @param lineBreakAhead the line break ahead
+     * @since 0.2.0
+     */
+    public Swc4jAstTokenRegex(
+            String text, String value, String flags, int startPosition, int endPosition, boolean lineBreakAhead) {
+        super(text, value, startPosition, endPosition, lineBreakAhead);
+        this.flags = flags;
+    }
+
+    /**
+     * Gets flags.
+     *
+     * @return the flags
+     * @since 0.2.0
+     */
+    public String getFlags() {
+        return flags;
+    }
+
+    @Override
+    public Swc4jAstTokenType getType() {
+        return Swc4jAstTokenType.Regex;
+    }
+}
diff --git a/src/main/java/com/caoccao/javet/swc4j/enums/Swc4jAstTokenSubType.java b/src/main/java/com/caoccao/javet/swc4j/enums/Swc4jAstTokenSubType.java
@@ -23,6 +23,7 @@ public enum Swc4jAstTokenSubType {
     GenericOperator,
     Keyword,
     ReservedWord,
+    TriAtom,
     UniAtom,
     Unknown,
     ;
@@ -55,6 +56,10 @@ public boolean isReservedWord() {
         return this == ReservedWord;
     }
 
+    public boolean isTriAtom() {
+        return this == TriAtom;
+    }
+
     public boolean isUniAtom() {
         return this == UniAtom;
     }

diff --git a/src/main/java/com/caoccao/javet/swc4j/enums/Swc4jAstTokenType.java b/src/main/java/com/caoccao/javet/swc4j/enums/Swc4jAstTokenType.java
@@ -129,6 +129,8 @@ public enum Swc4jAstTokenType {
     Str(101, "$Str", Swc4jAstTokenSubType.BiAtom),
     Num(102, "$Num", Swc4jAstTokenSubType.BiAtom),
     BigInt(103, "$BigInt", Swc4jAstTokenSubType.BiAtom),
+    // Atom - Tri
+    Regex(104, "$Regex", Swc4jAstTokenSubType.TriAtom),
     ;
 
     private static final int LENGTH = values().length;

diff --git a/src/test/java/com/caoccao/javet/swc4j/TestSwc4j.java b/src/test/java/com/caoccao/javet/swc4j/TestSwc4j.java
@@ -18,6 +18,7 @@
 
 import com.caoccao.javet.swc4j.ast.BaseSwc4jAstToken;
 import com.caoccao.javet.swc4j.ast.atom.bi.BaseSwc4jAstTokenBiAtom;
+import com.caoccao.javet.swc4j.ast.atom.tri.Swc4jAstTokenRegex;
 import com.caoccao.javet.swc4j.enums.Swc4jAstTokenType;
 import com.caoccao.javet.swc4j.enums.Swc4jMediaType;
 import com.caoccao.javet.swc4j.enums.Swc4jParseMode;
@@ -244,6 +245,9 @@ public void testParseTypeScriptWithCaptureTokens() throws Swc4jCoreException {
         assertTokenValue(BigInteger.valueOf(1), parseAndAssert("a = 1n;", options, Swc4jAstTokenType.BigInt, "1n", 4, 6, 2, 4));
         assertTokenValue(BigInteger.valueOf(1), parseAndAssert("a = -1n;", options, Swc4jAstTokenType.BigInt, "1n", 5, 7, 3, 5));
         assertTokenValue(new BigInteger("1234567890123456789012345678901234567890"), parseAndAssert("a = 1234567890123456789012345678901234567890n;", options, Swc4jAstTokenType.BigInt, "1234567890123456789012345678901234567890n", 4, 45, 2, 4));
+        Swc4jAstTokenRegex astTokenRegex = (Swc4jAstTokenRegex) parseAndAssert("a = /x/ig;", options, Swc4jAstTokenType.Regex, "x/ig", 5, 9, 3, 5);
+        assertEquals("x", astTokenRegex.getValue());
+        assertEquals("ig", astTokenRegex.getFlags());
     }
 
     @Test