Change lexing arrangement (#191)

Lots of changes to how lexing is done. Should fix some issues, make the parsing simpler and faster

Author: kaleidawave

.github/workflows/performance-and-size.yml

@@ -206,24 +206,19 @@ jobs:
         shell: bash
         continue-on-error: true
         run: |
-          strings=(
-            "https://esm.sh/v128/react-dom@18.2.0/es2022/react-dom.mjs"
-            "https://esm.sh/v135/typescript@5.3.3/es2022/typescript.mjs"
-          )
-
-          for url in "${strings[@]}"; do
-            # TODO copy expression
-            curl -sS $url > input.js
+          CORPUS_URL="https://gist.githubusercontent.com/kaleidawave/6708f604bc403021b56cb54ea50cac62/raw/javascript_files.txt"
 
+          curl -s "$CORPUS_URL" | while IFS= read -r URL; do
+            FILENAME="${URL##*/}"
+            curl -s "$URL" > $FILENAME
             echo "::group::Comparison"
-            hyperfine -i \
-              -L compiler ${{ steps.compilers.outputs.BINARIES }} \
-              '{compiler} ast-explorer full input.js --timings'
+            hyperfine -i -L compiler ${{ steps.compilers.outputs.BINARIES }} \
+              '{compiler} ast-explorer full $FILENAME --check'
             echo "::endgroup::"
           done
 
       - name: Upload checker
-        if: github.ref == 'main'
+        if: ${{ github.ref_name == 'main' }}
         uses: actions/upload-artifact@v4
         with:
           name: latest-checker

.github/workflows/rust.yml

@@ -84,13 +84,20 @@ jobs:
               - 'checker/**'
 
       - name: Run parser tests
+        if: steps.changes.outputs.parser == 'true' || github.ref_name == 'main'
+        run: cargo test --no-fail-fast
+        working-directory: parser
+
+      - name: Run parser examples
         if: steps.changes.outputs.parser == 'true' || github.ref_name == 'main'
         run: |
-          cargo test
-          
-          # TODO more
-          curl https://esm.sh/v128/react-dom@18.2.0/es2022/react-dom.mjs > react.js
-          cargo run -p ezno-parser --example parse react.js
+          CORPUS_URL="https://gist.githubusercontent.com/kaleidawave/6708f604bc403021b56cb54ea50cac62/raw/javascript_files.txt"
+
+          curl -s "$CORPUS_URL" | while IFS= read -r URL; do
+              FILENAME="${URL##*/}"
+              curl -s "$URL" > $FILENAME
+              cargo run -p ezno-parser --example parse $FILENAME
+          done
         working-directory: parser
 
       - name: Run checker specification

Cargo.lock

@@ -9,7 +9,6 @@ members = [
   # "lsp/server",
 ]
 
-
 [package]
 name = "ezno"
 description = "A fast and correct TypeScript type checker and compiler with additional experiments. For use as a library or through the CLI"
@@ -100,5 +99,5 @@ result_unit_err = "allow"
 thread_local_initializer_can_be_made_const = "allow"
 implicit_hasher = "allow"
 
-[profile.dev]
-debug = false
+# [profile.dev]
+# debug = false

Cargo.toml

@@ -80,8 +80,8 @@ impl DynamicBoundaryKind {
 }
 
 impl ContextType for Syntax<'_> {
-	fn as_general_context(et: &Context<Self>) -> GeneralContext<'_> {
-		GeneralContext::Syntax(et)
+	fn as_general_context(context: &Context<Self>) -> GeneralContext<'_> {
+		GeneralContext::Syntax(context)
 	}
 
 	fn get_parent(&self) -> Option<&GeneralContext<'_>> {

checker/src/context/environment.rs

@@ -678,9 +678,9 @@ pub(crate) fn call_constant_function(
 						return Err(ConstantFunctionError::CannotComputeConstant);
 					};
 
-					Some(flags.clone())
+					flags.clone()
 				}
-				None => None,
+				None => String::new(),
 			};
 
 			let regexp = types.new_regexp(&pattern.clone(), &flags, &call_site.without_source());

checker/src/features/constant_functions.rs

@@ -21,36 +21,31 @@ pub struct RegExp {
 }
 
 impl RegExp {
-	pub fn new(pattern: &str, flag_options: Option<&str>) -> Result<Self, String> {
-		let source = if let Some(flag_options) = flag_options {
-			format!("/{pattern}/{flag_options}")
-		} else {
-			format!("/{pattern}/")
-		};
-
+	pub fn new(pattern: &str, flag_options: &str) -> Result<Self, String> {
+		let source = format!("/{pattern}/{flag_options}");
 		let mut flags = Flags::default();
 		let mut flags_unsupported = false;
 
-		if let Some(flag_options) = flag_options {
-			for flag in flag_options.chars() {
-				#[allow(clippy::match_same_arms)]
-				match flag {
-					'd' => flags_unsupported = true, // indices for substring matches are not supported
-					'g' => flags_unsupported = true, // stateful regex is not supported
-					'i' => flags.icase = true,
-					'm' => flags.multiline = true,
-					's' => flags.dot_all = true,
-					'u' => flags.unicode = true,
-					'v' => flags.unicode_sets = true,
-					'y' => flags_unsupported = true, // sticky search is not supported
-					_ => panic!("Unknown flag: {flag:?}"),
-				}
+		for flag in flag_options.chars() {
+			#[allow(clippy::match_same_arms)]
+			match flag {
+				'd' => flags_unsupported = true, // indices for substring matches are not supported
+				'g' => flags_unsupported = true, // stateful regex is not supported
+				'i' => flags.icase = true,
+				'm' => flags.multiline = true,
+				's' => flags.dot_all = true,
+				'u' => flags.unicode = true,
+				'v' => flags.unicode_sets = true,
+				'y' => flags_unsupported = true, // sticky search is not supported
+				// Should be caught by parser errors
+				_ => panic!("Unknown flag: {flag:?}"),
 			}
 		}
 
 		let compiled_regex = {
 			let mut ire = backends::try_parse(pattern.chars().map(u32::from), flags)
 				.map_err(|err| err.text)?;
+
 			if !flags.no_opt {
 				backends::optimize(&mut ire);
 			}
@@ -314,10 +309,7 @@ impl BinarySerializable for RegExp {
 
 	fn deserialize<I: Iterator<Item = u8>>(iter: &mut I, source_id: SourceId) -> Self {
 		let source = String::deserialize(iter, source_id);
-
 		let (pattern, flags) = source[1..].rsplit_once('/').unwrap();
-		let flags = if flags.is_empty() { None } else { Some(flags) };
-
 		Self::new(pattern, flags).unwrap()
 	}
 }

checker/src/features/regexp.rs

@@ -162,13 +162,13 @@ fn synthesise_array_to_reference<
 						default_value.clone(),
 					)
 				}
-				parser::ArrayDestructuringField::Comment { content, is_multiline, position } => {
-					AssignableArrayDestructuringField::Comment {
-						content: content.clone(),
-						is_multiline: *is_multiline,
-						position: position.with_source(environment.get_source()),
-					}
-				}
+				// parser::ArrayDestructuringField::Comment { content, is_multiline, position } => {
+				// 	AssignableArrayDestructuringField::Comment {
+				// 		content: content.clone(),
+				// 		is_multiline: *is_multiline,
+				// 		position: position.with_source(environment.get_source()),
+				// 	}
+				// }
 				parser::ArrayDestructuringField::None => AssignableArrayDestructuringField::None,
 			})
 			.collect(),
@@ -269,5 +269,9 @@ pub(crate) fn synthesise_access_to_reference<T: crate::ReadFromFS>(
 				publicity: crate::types::properties::Publicity::Public,
 			}
 		}
+		VariableOrPropertyAccess::NonNullAssertion(on, _) => {
+			// TODO
+			synthesise_access_to_reference(on, environment, checking_data)
+		}
 	}
 }

checker/src/synthesis/assignments.rs

@@ -93,11 +93,15 @@ pub(crate) fn synthesise_declaration<T: crate::ReadFromFS>(
 							}
 						}
 					}
+					parser::declarations::export::Exportable::VarStatement(_) => {
+						todo!()
+					}
 					parser::declarations::export::Exportable::ImportAll { .. }
 					| parser::declarations::export::Exportable::ImportParts { .. }
 					| parser::declarations::export::Exportable::Function(_)
 					| parser::declarations::export::Exportable::EnumDeclaration(_)
 					| parser::declarations::export::Exportable::Interface(_)
+					| parser::declarations::export::Exportable::Namespace(_)
 					| parser::declarations::export::Exportable::TypeAlias(_) => {}
 				}
 			}
@@ -124,7 +128,10 @@ pub(crate) fn synthesise_declaration<T: crate::ReadFromFS>(
 					);
 				}
 			}
-			parser::declarations::ExportDeclaration::DefaultFunction { position, .. } => {
+			parser::declarations::ExportDeclaration::TSDefaultFunctionDeclaration {
+				position,
+				..
+			} => {
 				checking_data.diagnostics_container.add_error(
 					TypeCheckError::FunctionWithoutBodyNotAllowedHere {
 						position: position.with_source(environment.get_source()),
@@ -147,28 +154,33 @@ pub(crate) fn synthesise_declaration<T: crate::ReadFromFS>(
 
 			// TODO remove enumerate, add add function and more
 			for (idx, member) in r#enum.on.members.iter().enumerate() {
-				match member {
-					parser::ast::EnumMember::Variant { name, value, position } => {
-						if let Some(ref _value) = value {
-							checking_data.raise_unimplemented_error(
-								"enum with value",
-								position.with_source(environment.get_source()),
-							);
-						}
-
-						let value = checking_data
-							.types
-							.new_constant_type(Constant::Number((idx as u8).into()));
-
-						basis.append(
-							Publicity::Public,
-							PropertyKey::from(name.clone()),
-							PropertyValue::Value(value),
-							member.get_position().with_source(environment.get_source()),
-							&mut environment.info,
+				let parser::ast::EnumMember { name, value, position } = member;
+				match value {
+					parser::ast::EnumMemberValue::ClassMembers(_) => {
+						checking_data.raise_unimplemented_error(
+							"class enums",
+							position.with_source(environment.get_source()),
 						);
 					}
+					parser::ast::EnumMemberValue::Value(_) => {
+						checking_data.raise_unimplemented_error(
+							"enum with value",
+							position.with_source(environment.get_source()),
+						);
+					}
+					parser::ast::EnumMemberValue::None => {}
 				}
+
+				let value =
+					checking_data.types.new_constant_type(Constant::Number((idx as u8).into()));
+
+				basis.append(
+					Publicity::Public,
+					PropertyKey::from(name.clone()),
+					PropertyValue::Value(value),
+					member.get_position().with_source(environment.get_source()),
+					&mut environment.info,
+				);
 			}
 
 			let variable = crate::VariableId(environment.get_source(), r#enum.get_position().start);

checker/src/synthesis/declarations.rs

@@ -969,13 +969,15 @@ pub(super) fn synthesise_expression<T: crate::ReadFromFS>(
 			let location = function.header.get_location().map(|location| match location {
 				parser::functions::FunctionLocationModifier::Server => "server".to_owned(),
 				parser::functions::FunctionLocationModifier::Worker => "worker".to_owned(),
+				parser::functions::FunctionLocationModifier::Test => "test".to_owned(),
 			});
+			let name = function.name.as_option_str().map(ToOwned::to_owned);
 			Instance::RValue(register_expression_function(
 				expecting,
 				is_async,
 				is_generator,
 				location,
-				function.name.as_option_str().map(ToOwned::to_owned),
+				name,
 				function,
 				environment,
 				checking_data,
@@ -989,7 +991,7 @@ pub(super) fn synthesise_expression<T: crate::ReadFromFS>(
 		Expression::Comment { on, .. } => {
 			return synthesise_expression(on, environment, checking_data, expecting);
 		}
-		Expression::ParenthesizedExpression(inner_expression, _) => Instance::RValue(
+		Expression::Parenthesised(inner_expression, _) => Instance::RValue(
 			synthesise_multiple_expression(inner_expression, environment, checking_data, expecting),
 		),
 		Expression::ClassExpression(class) => Instance::RValue(synthesise_class_declaration(

checker/src/synthesis/expressions.rs

@@ -105,7 +105,7 @@ pub(crate) fn synthesise_jsx_element<T: crate::ReadFromFS>(
 		// 							attribute_name: match attribute {
 		// 								JSXAttribute::Static(name, _, _)
 		// 								| JSXAttribute::Dynamic(name, _, _)
-		// 								| JSXAttribute::BooleanAttribute(name, _) => name.clone(),
+		// 								| JSXAttribute::Boolean(name, _) => name.clone(),
 		// 								JSXAttribute::Spread(_, _) => todo!(),
 		// 								JSXAttribute::Shorthand(_) => todo!(),
 		// 							},
@@ -327,7 +327,7 @@ pub(crate) fn synthesise_jsx_element<T: crate::ReadFromFS>(
 	// 			// 						attribute_name: match attribute {
 	// 			// 							JSXAttribute::Static(name, _, _)
 	// 			// 							| JSXAttribute::Dynamic(name, _, _)
-	// 			// 							| JSXAttribute::BooleanAttribute(name, _) => name.clone(),
+	// 			// 							| JSXAttribute::Boolean(name, _) => name.clone(),
 	// 			// 							JSXAttribute::Spread(_, _) => todo!(),
 	// 			// 							JSXAttribute::Shorthand(_) => todo!(),
 	// 			// 						},
@@ -497,7 +497,7 @@ fn synthesise_attribute<T: crate::ReadFromFS>(
 			// TODO expecting
 			(name, synthesise_expression(expression, environment, checking_data, TypeId::ANY_TYPE))
 		}
-		JSXAttribute::BooleanAttribute(name, _) => (name, TypeId::TRUE),
+		JSXAttribute::Boolean(name, _) => (name, TypeId::TRUE),
 		JSXAttribute::Spread(_, pos) => {
 			checking_data.raise_unimplemented_error(
 				"spread JSX attribute",

checker/src/synthesis/extensions/jsx.rs

@@ -513,8 +513,7 @@ pub(super) fn variable_field_to_string(param: &VariableField) -> String {
 					parser::ArrayDestructuringField::Name(name, ..) => {
 						buf.push_str(&variable_field_to_string(name));
 					}
-					parser::ArrayDestructuringField::Comment { .. }
-					| parser::ArrayDestructuringField::None => {}
+					parser::ArrayDestructuringField::None => {}
 				}
 				if not_at_end {
 					buf.push_str(", ");