feat: self hosted tokenizer

alexjercan · May 24, 2024 · 76ac755 · 76ac755
1 parent 367d9fd
commit 76ac755
Show file tree

Hide file tree

Showing 4 changed files with 85 additions and 50 deletions.
diff --git a/Makefile b/Makefile
@@ -44,7 +44,7 @@ examples: all
 	./coolc examples/coin/client.cl examples/coin/message.cl --module net --module threading --module raylib -o build/coin-client
 
 compiler: all
-	./coolc examples/compiler.cl --module prelude --module data -o build/compiler
+	./coolc examples/compiler/lexer.cl examples/compiler/compiler.cl --module prelude --module data -o build/cool-lexer
 
 dist: clean all
 	rm -rf coolc.tar.gz

diff --git a/checker.sh b/checker.sh
@@ -5,6 +5,7 @@ BUILD_DIR=build
 TESTS_DIR=tests
 TOTAL_TESTS=0
 PASSED_TESTS=0
+COOLC_SELF=$BUILD_DIR/cool
 
 analyzer() {
     if [ "$#" -ne 2 ]; then
@@ -138,10 +139,45 @@ librunner() {
     PASSED_TESTS=$((PASSED_TESTS + passed))
 }
 
+self_hosted_lexical_analyzer() {
+    echo "Testing the self hosted lexical analyzer"
+    if [ "$#" -ne 1 ]; then
+        echo "Usage: $0 <tests_dir>"
+        exit 1
+    fi
+
+    tests_dir=$TESTS_DIR/$1
+
+    echo "Running tests for $1"
+
+    passed=0
+    for file_path in $(ls $tests_dir/*.cl); do
+        ref_path=$tests_dir/$(basename $file_path .cl).ref
+
+        file_name=$(basename $file_path)
+        echo -en "Testing $file_name ... "
+
+        cat $file_path | ./$COOLC_SELF-lexer 2>&1 | diff - $ref_path > /dev/null 2>&1
+
+        if [ $? -eq 0 ]; then
+            echo -e "\e[32mPASSED\e[0m"
+            passed=$((passed + 1))
+        else
+            echo -e "\e[31mFAILED\e[0m"
+        fi
+    done
+
+    total=$(ls $tests_dir/*.cl | wc -l)
+    echo "Passed $passed/$total tests"
+
+    TOTAL_TESTS=$((TOTAL_TESTS + total))
+    PASSED_TESTS=$((PASSED_TESTS + passed))
+}
 
 lexical_analyzer() {
     echo "Testing the lexical analyzer"
     analyzer lexer --lex
+    self_hosted_lexical_analyzer lexer
 }
 
 syntax_analyzer() {
@@ -170,7 +206,7 @@ lib_tests() {
     librunner lib
 }
 
-make clean && make
+make clean && make compiler
 
 ARG1=$1
 if [ "$ARG1" == "--lex" ]; then

diff --git a/examples/compiler.cl → examples/compiler/compiler.cl b/examples/compiler.cl → examples/compiler/compiler.cl
@@ -504,51 +504,3 @@ class Tokenizer {
             }
     };
 };
-
-class Main {
-    main(): Object {
-        let text: String <- new IO.in_string(),
-            tokens: List (* Token *) <- new Tokenizer.init(text).tokenize(),
-            iter: List <- tokens
-        in
-            while not isvoid iter loop
-                {
-                    case iter.value() of
-                        t: Token =>
-                            {
-                                new IO.out_string(t.to_string());
-                                case t of
-                                    t: TokenIllegal => {
-                                        new IO.out_string("(").out_string(t.error().to_string());
-                                        case t.value() of
-                                            v: None => "";
-                                            v: Some => case v.value() of
-                                                v: String => {
-                                                    new IO.out_string(" ").out_string(v);
-                                                };
-                                            esac;
-                                        esac;
-                                        new IO.out_string(")");
-                                    };
-                                    t: Token => {
-                                        case t.value() of
-                                            v: None => "";
-                                            v: Some => case v.value() of
-                                                v: String => {
-                                                    new IO.out_string("(").out_string(v).out_string(")");
-                                                };
-                                            esac;
-                                        esac;
-                                    };
-                                esac;
-
-                                new IO.out_string("\n");
-                            };
-                        o: Object => abort();
-                    esac;
-
-                    iter <- iter.next();
-                }
-            pool
-    };
-};
diff --git a/examples/compiler/lexer.cl b/examples/compiler/lexer.cl
@@ -0,0 +1,47 @@
+class Main {
+    main(): Object {
+        let text: String <- new IO.in_string(),
+            tokens: List (* Token *) <- new Tokenizer.init(text).tokenize(),
+            iter: List <- tokens
+        in
+            while not isvoid iter loop
+                {
+                    case iter.value() of
+                        t: Token =>
+                            {
+                                new IO.out_string(t.to_string());
+                                case t of
+                                    t: TokenIllegal => {
+                                        new IO.out_string("(").out_string(t.error().to_string());
+                                        case t.value() of
+                                            v: None => "";
+                                            v: Some => case v.value() of
+                                                v: String => {
+                                                    new IO.out_string(" ").out_string(v);
+                                                };
+                                            esac;
+                                        esac;
+                                        new IO.out_string(")");
+                                    };
+                                    t: Token => {
+                                        case t.value() of
+                                            v: None => "";
+                                            v: Some => case v.value() of
+                                                v: String => {
+                                                    new IO.out_string("(").out_string(v).out_string(")");
+                                                };
+                                            esac;
+                                        esac;
+                                    };
+                                esac;
+
+                                new IO.out_string("\n");
+                            };
+                        o: Object => abort();
+                    esac;
+
+                    iter <- iter.next();
+                }
+            pool
+    };
+};