Added lexer base

src/ast/location.sn

@@ -1,26 +1,37 @@
 
-public class SourceLocation {
+import pkg::utils::path_holder;
+import std::fs::{Path};
+
+public class SourceLocation extends path_holder::PathHolder {
   let mut line: u32;
   let mut column: u32;
   let mut width: u32;
-  let mut filename: String;
 
   public: SourceLocation() :
+    super(new Path("<unknown>")),
     line(0),
     column(0),
-    width(0),
-    filename("")
+    width(0)
   {}
 
-  SourceLocation(line: u32, column: u32, width: u32, filename: String) :
+  SourceLocation(line: u32, column: u32, width: u32, filename: Path) :
+    super(filename),
     line(line),
     column(column),
-    width(width),
-    filename(filename)
+    width(width)
   {}
 
   @inline func get_line() u32 { return self.line; }
   @inline func get_column() u32 { return self.column; }
   @inline func get_width() u32 { return self.width; }
-  @inline func get_filename() String { return self.filename; }
+}
+
+public class LocationHolder {
+  let mut location: SourceLocation;
+
+  public: LocationHolder() : location(new SourceLocation()) {}
+  public: LocationHolder(location: SourceLocation) : location(location) {}
+
+  @inline func get_location() SourceLocation { return self.location; }
+  @inline func set_location(location: SourceLocation) { self.location = location; }
 }

src/ast/nodes.sn

@@ -78,6 +78,7 @@ public enum AST {
   Block(Vector<Node>),
   Assign(Node, Node),
   Call(Node, Vector<Node>),
+  Cast(Node, AstType),
   BinaryOp(BinaryOp, Node, Node, /* is_unary */ bool),
   FuncDef(Option<Node>, Vector<Node>, Vector<Node>, Vector<GenericDecl>),
   VarDef(Option<Node>, Node),

src/compiler/lib.sn

@@ -0,0 +1,19 @@
+
+import std::fs::{Path};
+import pkg::utils::path_holder;
+
+import pkg::lexer as lexicon;
+import std::io;
+
+public class Compiler extends path_holder::PathHolder {
+  public: Compiler(path: Path) : super(path) {}
+
+  func run() i32 {
+    // TODO: once implemented file walker, self.path should point to a directory
+    // and we should walk through all files in that directory and compile them
+    let lexer = new lexicon::Lexer(self.path.clone());
+    lexer.lex();
+
+    io::println(lexer.get_tokens());
+  }
+}

src/compiler/sn.toml

@@ -0,0 +1,3 @@
+
+[package]
+main = "lib.sn"

src/lexer/lib.sn

@@ -0,0 +1,17 @@
+
+import pkg::utils::path_holder;
+import std::fs::{Path};
+import pkg::utils::files_load::{file_loader};
+import pkg::lexer::token::{Token};
+
+public class Lexer extends path_holder::PathHolder {
+  let mut tokens: Vector<Token> = new Vector<Token>();
+
+  public: Lexer(path: Path) : super(path) {}
+
+  func lex() {
+    let file = file_loader(self.path);
+  }
+
+  @inline func get_tokens() Vector<Token> { return self.tokens; }
+}

src/lexer/sn.toml

@@ -0,0 +1,3 @@
+
+[package]
+main = "lib.sn"

src/lexer/token.sn

@@ -0,0 +1,160 @@
+
+import pkg::ast::location::{LocationHolder, SourceLocation};
+
+public enum TokenType {
+  Identifier(String),
+  Integer(i64),
+  Float(f64),
+  String(String),
+  Char(u8),
+  Continue,
+  Break,
+  Return,
+  If,
+  Else,
+  While,
+  For,
+  Let,
+  Mut,
+  Struct,
+  Enum,
+  Class,
+  Interface,
+  Public,
+  Private,
+  Import,
+  Const,
+  Static,
+  True,
+  False,
+  Fn,
+  New,
+  Super,
+  OpenParen,
+  CloseParen,
+  OpenBrace,
+  CloseBrace,
+  OpenBracket,
+  CloseBracket,
+  Semicolon,
+  Colon,
+  Comma,
+  Dot,
+  Plus,
+  Minus,
+  Star,
+  Slash,
+  Percent,
+  Ampersand,
+  Pipe,
+  At,
+  Arrow,
+  DoubleColon,
+  PlusEqual,
+  MinusEqual,
+  StarEqual,
+  SlashEqual,
+  PercentEqual,
+  AmpersandEqual,
+  PipeEqual,
+  Equal,
+  DoubleEqual,
+  NotEqual,
+  LessThan,
+  LessThanEqual,
+  GreaterThan,
+  GreaterThanEqual,
+  DoublePlus,
+  DoubleMinus,
+  DoubleAmpersand,
+  DoublePipe,
+  DoubleLessThan,
+  DoubleGreaterThan,
+  Question,
+  Exclamation
+}
+
+class Token extends LocationHolder implements ToString {
+    let token_type: TokenType;
+
+  public:
+    Token(token_type: TokenType, location: SourceLocation) :
+      super(location),
+      token_type(token_type)
+    {}
+
+    func to_string() String {
+      case self.token_type {
+        Identifier(value) => return value,
+        Integer(value) => return value.to_string(),
+        Float(value) => return value.to_string(),
+        String(value) => return "\"" + value + "\"",
+        Char(value) => return "'" + value + "'",
+        Continue => return "continue",
+        Break => return "break",
+        Return => return "return",
+        If => return "if",
+        Else => return "else",
+        While => return "while",
+        For => return "for",
+        Let => return "let",
+        Mut => return "mut",
+        Struct => return "struct",
+        Enum => return "enum",
+        Class => return "class",
+        Interface => return "interface",
+        Public => return "public",
+        Private => return "private",
+        Import => return "import",
+        Const => return "const",
+        Static => return "static",
+        True => return "true",
+        False => return "false",
+        Fn => return "fn",
+        New => return "new",
+        Super => return "super",
+        OpenParen => return "(",
+        CloseParen => return ")",
+        OpenBrace => return "{",
+        CloseBrace => return "}",
+        OpenBracket => return "[",
+        CloseBracket => return "]",
+        Semicolon => return ";",
+        Colon => return ":",
+        Comma => return ",",
+        Dot => return ".",
+        Plus => return "+",
+        Minus => return "-",
+        Star => return "*",
+        Slash => return "/",
+        Percent => return "%",
+        Ampersand => return "&",
+        Pipe => return "|",
+        At => return "@",
+        Arrow => return "=>",
+        DoubleColon => return "::",
+        PlusEqual => return "+=",
+        MinusEqual => return "-=",
+        StarEqual => return "*=",
+        SlashEqual => return "/=",
+        PercentEqual => return "%=",
+        AmpersandEqual => return "&=",
+        PipeEqual => return "|=",
+        Equal => return "=",
+        DoubleEqual => return "==",
+        NotEqual => return "!=",
+        LessThan => return "<",
+        LessThanEqual => return "<=",
+        GreaterThan => return ">",
+        GreaterThanEqual => return ">=",
+        DoublePlus => return "++",
+        DoubleMinus => return "--",
+        DoubleAmpersand => return "&&",
+        DoublePipe => return "||",
+        DoubleLessThan => return "<<",
+        DoubleGreaterThan => return ">>",
+        Question => return "?",
+        Exclamation => return "!"
+      }
+    }
+}

src/main.sn

@@ -1,4 +1,8 @@
 
+import pkg::compiler::{Compiler};
+import std::fs::{Path};
+
 public func main() i32 {
-  return 0;
+  let main_path = new Path("tests/_rewrite_test.sn");
+  return new Compiler(main_path).run();
 }

src/utils/files_load.sn

@@ -0,0 +1,10 @@
+
+import std::fs::{Path, File};
+
+public let mut file_loader: Function<func (Path) => File> = default_loader();
+
+func default_loader() Function<func (Path) => File>{
+  return func (path: Path) File {
+    return new File(path, "r");
+  }
+}

src/utils/path_holder.sn

@@ -0,0 +1,10 @@
+
+import std::fs::{Path};
+
+public class PathHolder {
+    let path: Path;
+
+    public: PathHolder(path: Path) : path(path) {}
+
+    @inline func get_path() Path { return self.path; }
+}

stdlib/fs/file.sn

@@ -78,7 +78,7 @@ public class File implements ToString, Debug {
     unsafe {
       let cfile = clib::files::fopen(self.path.to_string().c_str(), mode.c_str());
       if cfile.is_null() {
-        throw new FileOpenError("Failed to open or create file (" + self.path.to_string() + "): " + env::posix_get_error_msg(clib::errno()));
+        throw new FileOpenError("Failed to open or create file: " + env::posix_get_error_msg(clib::errno()));
       }
       self.open = true;
       self.file = cfile;

stdlib/fs/path.sn

@@ -19,7 +19,7 @@ public class PathError extends Exception {}
  * This is a wrapper around a string that provides some useful
  * operations for manipulating paths.
  */
-public class Path implements ToString, Iterable<String> {
+public class Path implements ToString, Iterable<String>, Clone<Self> {
     /// The path as a string.
     let path: Vector<String>;
   public:
@@ -201,4 +201,10 @@ public class Path implements ToString, Iterable<String> {
         return err == 0;
       }
     }
+    /**
+     * @brief Clone the path.
+     * @return The cloned path.
+     */
+    @inline
+    func clone() Path { return new Path(self.path.clone()); }
 }

stdlib/std.sn

@@ -381,7 +381,7 @@ public class Range<N: Numeric = i32> implements Iterable<N>, ToString {
  *       management.
  */
 public class Vector<T: Sized, Allocator: Sized = ptr::Allocator<T>> 
- implements Iterable<T>, ToString {
+ implements Iterable<T>, ToString, Clone<Self> {
   public:
     /**
      * @brief Default constructor.
@@ -581,6 +581,17 @@ public class Vector<T: Sized, Allocator: Sized = ptr::Allocator<T>>
       }
       return -1;
     }
+    /**
+     * @brief It clones the vector.
+     * @return A clone of the vector.
+     */
+    @inline
+    func clone() Self {
+      let mut vec = Self::with_capacity(self.capacity);
+      vec.length = self.length;
+      ptr::copy_nonoverlapping(self.buffer.ptr(), vec.buffer.ptr(), self.length);
+      return vec;
+    }
 
   private:
     /** The capacity of the vector. */