Added key type modifiers and wrapped all items as tokens

Shannarra · Jul 24, 2024 · 26116fa · 26116fa
1 parent 12b0e6e
commit 26116fa
Show file tree

Hide file tree

Showing 8 changed files with 97 additions and 44 deletions.
diff --git a/.rubocop.yml b/.rubocop.yml
@@ -17,6 +17,8 @@ Metrics/AbcSize:
   Max: 100
 Metrics/CyclomaticComplexity:
   Max: 100
+Metrics/ClassLength:
+  Max: 110
 Metrics/MethodLength:
   Max: 50
 Metrics/BlockLength:
@@ -25,3 +27,4 @@ Metrics/PerceivedComplexity:
   Max: 50
 Layout/LineLength:
   Max: 140
+
diff --git a/config.json b/config.json
@@ -12,5 +12,6 @@
         "TRUE": "да",
         "FALSE": "не"
     },
-    "NULL": "fuck"
+    "NULL": "fuck",
+		"KEYTYPE": "symbol",
 }
diff --git a/spec/lexer_spec.rb b/spec/lexer_spec.rb
@@ -10,10 +10,10 @@
 multiline string"'
         result = Lexer.lex!(text)
         tokens = [
-          Token.new(TokenType::String, '"hello world"'),
-          ',',
+          Token.new(TokenType::String, '"hello world"', 0, 0),
+          Token.new(TokenType::Symbol, ',', 0, 0),
           Token.new(TokenType::String, '"I am a
-multiline string"')
+multiline string"', 0, 0)
         ]
 
         expect(result.length).to eq 3
@@ -28,17 +28,17 @@
         result = Lexer.lex!(text)
 
         tokens = [
-          Token.new(TokenType::Number, 1),
-          ',',
-          Token.new(TokenType::Number, 6.9),
-          ',',
-          Token.new(TokenType::Number, -2),
-          ',',
-          Token.new(TokenType::Number, -420.0),
-          ',',
-          Token.new(TokenType::Number, 3e6),
-          ',',
-          Token.new(TokenType::Number, -2e6)
+          Token.new(TokenType::Number, 1, 0, 0),
+          Token.new(TokenType::Symbol, ',', 0, 0),
+          Token.new(TokenType::Number, 6.9, 0, 0),
+          Token.new(TokenType::Symbol, ',', 0, 0),
+          Token.new(TokenType::Number, -2, 0, 0),
+          Token.new(TokenType::Symbol, ',', 0, 0),
+          Token.new(TokenType::Number, -420.0, 0, 0),
+          Token.new(TokenType::Symbol, ',', 0, 0),
+          Token.new(TokenType::Number, 3e6, 0, 0),
+          Token.new(TokenType::Symbol, ',', 0, 0),
+          Token.new(TokenType::Number, -2e6, 0, 0)
         ]
 
         expect(result.length).to eq 11
@@ -53,9 +53,9 @@
         result = Lexer.lex!(text)
 
         tokens = [
-          Token.new(TokenType::Boolean, true),
-          ',',
-          Token.new(TokenType::Boolean, false)
+          Token.new(TokenType::Boolean, true, 0, 0),
+          Token.new(TokenType::Symbol, ',', 0, 0),
+          Token.new(TokenType::Boolean, false, 0, 0)
         ]
         expect(result.length).to eq 3
 
@@ -69,7 +69,7 @@
         result = Lexer.lex!(text)
 
         tokens = [
-          Token.new(TokenType::Null, nil)
+          Token.new(TokenType::Null, nil, 0, 0)
         ]
         expect(result.length).to eq 1
 

diff --git a/src/config.rb b/src/config.rb
@@ -17,6 +17,8 @@ def config_from_file!(file)
 
     check_booleans!
 
+    check_keytype!
+
     check_config_values! @__res.values
 
     # All checks done, merge with default for non-required keys if they don't exist
@@ -40,7 +42,7 @@ def default_config
         FALSE: 'false'
       },
       NULL: 'null',
-      KEYTYPE: 'symbol'
+      KEYTYPE: 'string'
     }.freeze
   end
 
@@ -74,6 +76,18 @@ def check_booleans!
     required_object_check!(:BOOLEAN)
   end
 
+  def check_keytype!
+    type = @__res[:KEYTYPE]
+
+    return unless type
+
+    if %w[symbol string].include?(type)
+      @__res[:KEYTYPE] = type
+    else
+      error! "Value for option \"KEYTYPE\" must be either \"symbol\" or \"string\", got a \"#{type}\"."
+    end
+  end
+
   def check_config_values!(values)
     values.each do |value|
       if value.is_a?(Hash)

diff --git a/src/lexer.rb b/src/lexer.rb
@@ -5,13 +5,16 @@
 class Lexer
   include Config
 
-  attr_reader :text, :tokens, :ip
+  attr_reader :text, :tokens, :ip, :line, :col
 
   def initialize(text)
     @@config ||= default_config
     @text = text
     @tokens = []
     @ip = 0
+
+    @line = 0
+    @col = 0
   end
 
   def self.lex!(text)
@@ -39,14 +42,19 @@ def lex
       tokens << null unless null.nil?
 
       if @@config[:WHITESPACE].include?(current)
+        if current == "\n"
+          @line += 1
+          @col = 0
+        end
+
         advance
       elsif @@config[:SYMBOLS].values.include?(current)
-        tokens << current
+        tokens << Token.new(TokenType::Symbol, current, line, col)
         advance
       else
         break unless current
 
-        error! "Unknown token \"#{current}\" encountered at #{ip}"
+        error! "Unknown token \"#{current}\" encountered at #{line}:#{col}"
       end
     end
 
@@ -60,6 +68,7 @@ def current
   end
 
   def advance
+    @col += 1
     @ip += 1
   end
 
@@ -68,7 +77,7 @@ def lex_str
 
     str = current
     advance
-    return Token.new(TokenType::String, (str += current)) if current == @@config[:SYMBOLS][:QUOTE]
+    return Token.new(TokenType::String, (str += current), line, col) if current == @@config[:SYMBOLS][:QUOTE]
 
     loop do
       if current
@@ -78,7 +87,7 @@ def lex_str
       end
 
       advance
-      return Token.new(TokenType::String, (str += current)) if current == @@config[:SYMBOLS][:QUOTE]
+      return Token.new(TokenType::String, (str += current), line, col) if current == @@config[:SYMBOLS][:QUOTE]
     end
   end
 
@@ -96,9 +105,9 @@ def lex_num
 
     return nil if num.empty?
 
-    return Token.new(TokenType::Number, num.to_f) if num.include?('.') || num.include?('e')
+    return Token.new(TokenType::Number, num.to_f, line, col) if num.include?('.') || num.include?('e')
 
-    Token.new(TokenType::Number, num.to_i)
+    Token.new(TokenType::Number, num.to_i, line, col)
   end
 
   def lex_bool
@@ -116,9 +125,9 @@ def lex_bool
 
     case jbool
     when bool_vals[:TRUE]
-      Token.new(TokenType::Boolean, true)
+      Token.new(TokenType::Boolean, true, line, col)
     when bool_vals[:FALSE]
-      Token.new(TokenType::Boolean, false)
+      Token.new(TokenType::Boolean, false, line, col)
     end
   end
 
@@ -132,6 +141,6 @@ def lex_null
       advance
     end
 
-    Token.new(TokenType::Null, nil) if null == @@config[:NULL]
+    Token.new(TokenType::Null, nil, line, col) if null == @@config[:NULL]
   end
 end
diff --git a/src/parser.rb b/src/parser.rb
@@ -21,10 +21,10 @@ def parse(is_root: false, offset: 0)
     @ip = offset
     token = current
 
-    error! 'Expected root to be an object!' if is_root && token != @@config[:SYMBOLS][:LEFTBRACE]
+    error! 'Expected root to be an object!' if is_root && token.value != @@config[:SYMBOLS][:LEFTBRACE]
 
     advance
-    case token
+    case token.value
     when @@config[:SYMBOLS][:LEFTBRACKET] then parse_array
     when @@config[:SYMBOLS][:LEFTBRACE] then parse_object
     else
@@ -42,26 +42,31 @@ def advance
     @ip += 1
   end
 
+  def prev
+    @tokens[@ip - 1]
+  end
+
   def parse_object
     object = {}
 
     if current == @@config[:SYMBOLS][:RIGHTBRACE]
       advance
-      return object
+      return with_matching_key_type(object)
     end
 
     while current
       key = current
 
       unless key.is_a?(Token) && key.string_token?
-        return object if key == @@config[:SYMBOLS][:RIGHTBRACE]
+        return with_matching_key_type(object) if key == @@config[:SYMBOLS][:RIGHTBRACE]
 
         error! "Expected a string key in object, got \"#{key}\" at #{ip}"
       end
 
       advance
-
-      error! "Expected a colon separator character after key in object, got \"#{current}\"" unless current == @@config[:SYMBOLS][:COLON]
+      unless current == @@config[:SYMBOLS][:COLON]
+        error! "Expected a colon separator character after key in object, got #{current.value_with_position}"
+      end
 
       advance
       value = parse(offset: @ip)
@@ -70,11 +75,11 @@ def parse_object
 
       if current == @@config[:SYMBOLS][:RIGHTBRACE]
         advance
-        return object
+        return with_matching_key_type(object)
       elsif current != @@config[:SYMBOLS][:COMMA]
-        return object unless current
+        return with_matching_key_type(object) unless current
 
-        next if current.is_a?(Token) && current.string_token?
+        next if current.string_token? && prev.symbol_token?
 
         error! "Expected a comma after a key-value pair in object, got an \"#{unwrap! current}\""
       end
@@ -97,9 +102,9 @@ def parse_array
       if current == @@config[:SYMBOLS][:RIGHTBRACKET]
         return array
       elsif current == @@config[:SYMBOLS][:RIGHTBRACE]
-        error! 'Improperly closed array in object'
+        error! "Improperly closed array in object, got #{current.value_with_position}"
       elsif current != @@config[:SYMBOLS][:COMMA]
-        error! "Expected a '#{@@config[:SYMBOLS][:COMMA]}' , got #{current}"
+        error! "Expected a '#{@@config[:SYMBOLS][:COMMA]}' , got #{current.value_with_position}"
       else
         advance
       end
@@ -124,4 +129,11 @@ def unwrap!(value)
 
     value
   end
+
+  def with_matching_key_type(obj)
+    case @@config[:KEYTYPE]
+    when 'symbol' then obj.symbolize_keys
+    when 'string' then obj.stringify_keys
+    end
+  end
 end
diff --git a/src/token.rb b/src/token.rb
@@ -7,15 +7,25 @@
 ]
 
 class Token
-  attr_reader :type, :value
+  attr_reader :type, :value, :line, :col
 
-  def initialize(type, value)
+  def initialize(type, value, line, col)
     @type = type
     @value = value
+    @line = line
+    @col = col
+  end
+
+  def position
+    "#{line}:#{col}"
+  end
+
+  def value_with_position
+    "#{value || '"null"'} at #{position}"
   end
 
   def to_s
-    "Token<[#{type}] = #{value}>"
+    "Token<[#{type}] = #{value}> #{position}"
   end
 
   def ==(other)

diff --git a/src/util.rb b/src/util.rb
@@ -37,4 +37,8 @@ class Hash
   def symbolize_keys
     each_with_object({}) { |(k, v), h| h[k.to_sym] = v.is_a?(Hash) ? v.symbolize_keys : v }
   end
+
+  def stringify_keys
+    each_with_object({}) { |(k, v), h| h[k.to_s] = v.is_a?(Hash) ? v.stringify_keys : v }
+  end
 end