remove unistring package and introduce scope contexts

ast/ident.go

@@ -0,0 +1,20 @@
+package ast
+
+type (
+	ScopeContext int
+
+	Id struct {
+		Name         string
+		ScopeContext ScopeContext
+	}
+
+	Identifier struct {
+		Idx          Idx
+		Name         string
+		ScopeContext ScopeContext
+	}
+)
+
+func (i *Identifier) ToId() Id {
+	return Id{Name: i.Name, ScopeContext: i.ScopeContext}
+}

ast/node.go

@@ -2,7 +2,6 @@ package ast
 
 import (
 	"github.com/t14raptor/go-fast/token"
-	"github.com/t14raptor/go-fast/unistring"
 )
 
 type PropertyKind string
@@ -188,11 +187,6 @@ type (
 		Async         bool
 	}
 
-	Identifier struct {
-		Idx  Idx
-		Name unistring.String
-	}
-
 	PrivateIdentifier struct {
 		*Identifier
 	}
@@ -270,13 +264,13 @@ type (
 	StringLiteral struct {
 		Idx     Idx
 		Literal string
-		Value   unistring.String
+		Value   string
 	}
 
 	TemplateElement struct {
 		Idx     Idx
 		Literal string
-		Parsed  unistring.String
+		Parsed  string
 		Valid   bool
 	}
 

generator/generator.go

@@ -217,9 +217,9 @@ func (g *GenVisitor) VisitMemberExpression(n *ast.MemberExpression) {
 	default:
 		g.gen(n.Object.Expr)
 	}
-	if st, ok := n.Property.Expr.(*ast.StringLiteral); ok && valid(st.Value.String()) {
+	if st, ok := n.Property.Expr.(*ast.StringLiteral); ok && valid(st.Value) {
 		g.out.WriteString(".")
-		g.out.WriteString(st.Value.String())
+		g.out.WriteString(st.Value)
 	} else {
 		g.out.WriteString("[")
 		g.gen(n.Property.Expr)
@@ -325,7 +325,7 @@ func (g *GenVisitor) VisitFunctionLiteral(n *ast.FunctionLiteral) {
 
 func (g *GenVisitor) VisitIdentifier(n *ast.Identifier) {
 	if n != nil {
-		g.out.WriteString(n.Name.String())
+		g.out.WriteString(n.Name)
 	}
 }
 
@@ -596,7 +596,7 @@ func (g *GenVisitor) VisitForLoopInitializerLexicalDecl(n *ast.ForLoopInitialize
 func (g *GenVisitor) VisitTemplateLiteral(n *ast.TemplateLiteral) {
 	g.out.WriteString("`")
 	for i, e := range n.Elements {
-		g.out.WriteString(e.Parsed.String())
+		g.out.WriteString(e.Parsed)
 		if i < len(n.Expressions) {
 			g.out.WriteString("${")
 			g.gen(n.Expressions[i].Expr)

parser/expression.go

@@ -5,7 +5,6 @@ import (
 
 	"github.com/t14raptor/go-fast/ast"
 	"github.com/t14raptor/go-fast/token"
-	"github.com/t14raptor/go-fast/unistring"
 )
 
 func (p *parser) parseIdentifier() *ast.Identifier {
@@ -321,7 +320,7 @@ func (p *parser) parseVariableDeclarationList() (declarationList []*ast.Variable
 	return
 }
 
-func (p *parser) parseObjectPropertyKey() (string, unistring.String, ast.Expr, token.Token) {
+func (p *parser) parseObjectPropertyKey() (string, string, ast.Expr, token.Token) {
 	if p.token == token.LeftBracket {
 		p.next()
 		expr := p.parseAssignmentExpression()
@@ -362,7 +361,7 @@ func (p *parser) parseObjectPropertyKey() (string, unistring.String, ast.Expr, t
 			value = &ast.StringLiteral{
 				Idx:     idx,
 				Literal: literal,
-				Value:   unistring.String(literal),
+				Value:   string(literal),
 			}
 		} else {
 			p.errorUnexpectedToken(tkn)
@@ -672,7 +671,7 @@ func (p *parser) parseNewExpression() ast.Expr {
 		if p.literal == "target" {
 			return &ast.MetaProperty{
 				Meta: &ast.Identifier{
-					Name: unistring.String(token.New.String()),
+					Name: string(token.New.String()),
 					Idx:  idx,
 				},
 				Property: p.parseIdentifier(),

parser/lexer.go

@@ -11,7 +11,6 @@ import (
 
 	"github.com/t14raptor/go-fast/ast"
 	"github.com/t14raptor/go-fast/token"
-	"github.com/t14raptor/go-fast/unistring"
 	"golang.org/x/text/unicode/rangetable"
 )
 
@@ -62,7 +61,7 @@ func isIdentifierPart(chr rune) bool {
 		chr >= utf8.RuneSelf && isIdPartUnicode(chr)
 }
 
-func (p *parser) scanIdentifier() (string, unistring.String, bool, string) {
+func (p *parser) scanIdentifier() (string, string, bool, string) {
 	offset := p.chrOffset
 	hasEscape := false
 	isUnicode := false
@@ -132,7 +131,7 @@ func (p *parser) scanIdentifier() (string, unistring.String, bool, string) {
 	}
 
 	literal := p.str[offset:p.chrOffset]
-	var parsed unistring.String
+	var parsed string
 	if hasEscape || isUnicode {
 		var err string
 		// TODO strict
@@ -141,7 +140,7 @@ func (p *parser) scanIdentifier() (string, unistring.String, bool, string) {
 			return "", "", false, err
 		}
 	} else {
-		parsed = unistring.String(literal)
+		parsed = string(literal)
 	}
 
 	return literal, parsed, hasEscape, ""
@@ -173,7 +172,7 @@ type parserState struct {
 	idx                                ast.Idx
 	tok                                token.Token
 	literal                            string
-	parsedLiteral                      unistring.String
+	parsedLiteral                      string
 	implicitSemicolon, insertSemicolon bool
 	chr                                rune
 	chrOffset, offset                  int
@@ -204,7 +203,7 @@ func (p *parser) peek() token.Token {
 	return tok
 }
 
-func (p *parser) scan() (tkn token.Token, literal string, parsedLiteral unistring.String, idx ast.Idx) {
+func (p *parser) scan() (tkn token.Token, literal string, parsedLiteral string, idx ast.Idx) {
 	p.implicitSemicolon = false
 
 	for {
@@ -691,7 +690,7 @@ func (p *parser) scanEscape(quote rune) (int, bool) {
 	return 1, false
 }
 
-func (p *parser) scanString(offset int, parse bool) (literal string, parsed unistring.String, err string) {
+func (p *parser) scanString(offset int, parse bool) (literal string, parsed string, err string) {
 	// " ' /
 	quote := rune(p.str[offset])
 	length := 0
@@ -768,7 +767,7 @@ func (p *parser) scanNewline() {
 	p.read()
 }
 
-func (p *parser) parseTemplateCharacters() (literal string, parsed unistring.String, finished bool, parseErr, err string) {
+func (p *parser) parseTemplateCharacters() (literal string, parsed string, finished bool, parseErr, err string) {
 	offset := p.chrOffset
 	var end int
 	length := 0
@@ -908,12 +907,13 @@ error:
 	return nil, errors.New("Illegal numeric literal")
 }
 
-func parseStringLiteral(literal string, length int, unicode, strict bool) (unistring.String, string) {
+func parseStringLiteral(literal string, length int, unicode, strict bool) (string, string) {
 	var sb strings.Builder
 	var chars []uint16
 	if unicode {
 		chars = make([]uint16, 1, length+1)
-		chars[0] = unistring.BOM
+		// BOM
+		chars[0] = 0xFEFF
 	} else {
 		sb.Grow(length)
 	}
@@ -1085,12 +1085,12 @@ func parseStringLiteral(literal string, length int, unicode, strict bool) (unist
 		if len(chars) != length+1 {
 			panic(fmt.Errorf("unexpected unicode length while parsing '%s'", literal))
 		}
-		return unistring.FromUtf16(chars), ""
+		return string(utf16.Decode(chars)), ""
 	}
 	if sb.Len() != length {
 		panic(fmt.Errorf("unexpected length while parsing '%s'", literal))
 	}
-	return unistring.String(sb.String()), ""
+	return sb.String(), ""
 }
 
 func (p *parser) scanNumericLiteral(decimalPoint bool) (token.Token, string) {

parser/parser.go

@@ -3,7 +3,6 @@ package parser
 import (
 	"github.com/t14raptor/go-fast/ast"
 	"github.com/t14raptor/go-fast/token"
-	"github.com/t14raptor/go-fast/unistring"
 )
 
 // parser ...
@@ -18,7 +17,7 @@ type parser struct {
 	idx           ast.Idx     // The index of token
 	token         token.Token // The token
 	literal       string      // The literal of the token, if any
-	parsedLiteral unistring.String
+	parsedLiteral string
 
 	scope             *scope
 	insertSemicolon   bool // If we see a newline, then insert an implicit semicolon

parser/scope.go

@@ -1,9 +1,5 @@
 package parser
 
-import (
-	"github.com/t14raptor/go-fast/unistring"
-)
-
 type scope struct {
 	outer        *scope
 	allowIn      bool
@@ -16,7 +12,7 @@ type scope struct {
 	allowAwait   bool
 	allowYield   bool
 
-	labels []unistring.String
+	labels []string
 }
 
 func (p *parser) openScope() {
@@ -30,7 +26,7 @@ func (p *parser) closeScope() {
 	p.scope = p.scope.outer
 }
 
-func (s *scope) hasLabel(name unistring.String) bool {
+func (s *scope) hasLabel(name string) bool {
 	for _, label := range s.labels {
 		if label == name {
 			return true

parser/statement.go

@@ -98,7 +98,7 @@ func (p *parser) parseStatement() ast.Stmt {
 		label := identifier.Name
 		for _, value := range p.scope.labels {
 			if label == value {
-				p.error(label.String())
+				p.error(label)
 			}
 		}
 		p.scope.labels = append(p.scope.labels, label) // Push the label
@@ -887,7 +887,7 @@ func (p *parser) parseBreakStatement() ast.Stmt {
 	if p.token == token.Identifier {
 		identifier := p.parseIdentifier()
 		if !p.scope.hasLabel(identifier.Name) {
-			p.error(identifier.Name.String())
+			p.error(identifier.Name)
 			return &ast.BadStatement{From: idx, To: identifier.Idx1()}
 		}
 		p.semicolon()
@@ -929,7 +929,7 @@ func (p *parser) parseContinueStatement() ast.Stmt {
 	if p.token == token.Identifier {
 		identifier := p.parseIdentifier()
 		if !p.scope.hasLabel(identifier.Name) {
-			p.error(identifier.Name.String())
+			p.error(identifier.Name)
 			return &ast.BadStatement{From: idx, To: identifier.Idx1()}
 		}
 		if !p.scope.inIteration {