ruby_parser - rubygems Package Compare versions

Comparing version 3.8.2 to 3.8.3

History.txt

@@ -0,1 +1,13 @@
+=== 3.8.3 / 2016-10-09
+
+* 1 minor enhancement:
+
+  * Support Ruby 2.1 number literals. (soutaro)
+
+* 3 bug fixes:
+
+  * Fixed line numbers for strs with backslash-newlines. (maxjacobson)
+  * Improved compatibility on tokenizing number. (soutaro)
+  * Refactored and fixed multiline array line numbers. (ptoomey3, with changes)
+
 === 3.8.2 / 2016-05-05
@@ -2,0 +14,0 @@

lib/ruby_lexer.rb

@@ -261,3 +261,14 @@ # encoding: UTF-8
     rb_compile_error "Invalid numeric format" if matched =~ /__/
-    return result(:expr_end, :tINTEGER, matched.to_i(base))
+
+    text = matched
+    case
+    when text.end_with?('ri')
+      return result(:expr_end, :tIMAGINARY, Complex(0, Rational(text.chop.chop.to_i(base))))
+    when text.end_with?('r')
+      return result(:expr_end, :tRATIONAL, Rational(text.chop.to_i(base)))
+    when text.end_with?('i')
+      return result(:expr_end, :tIMAGINARY, Complex(0, text.chop.to_i(base)))
+    else
+      return result(:expr_end, :tINTEGER, text.to_i(base))
+    end
   end
@@ -410,3 +421,13 @@
     rb_compile_error "Invalid numeric format" if text =~ /__/
-    return result(:expr_end, :tFLOAT, text.to_f)
+
+    case
+    when text.end_with?('ri')
+      return result(:expr_end, :tIMAGINARY, Complex(0, Rational(text.chop.chop)))
+    when text.end_with?('r')
+      return result(:expr_end, :tRATIONAL, Rational(text.chop))
+    when text.end_with?('i')
+      return result(:expr_end, :tIMAGINARY, Complex(0, text.chop.to_f))
+    else
+      return result(:expr_end, :tFLOAT, text.to_f)
+    end
   end
@@ -908,2 +929,13 @@
 
+  def eat_whitespace
+    r = scan(/\s+/)
+    self.extra_lineno += r.count("\n") if r
+    r
+  end
+
+  def fixup_lineno extra = 0
+    self.lineno += self.extra_lineno + extra
+    self.extra_lineno = 0
+  end
+
   def scanner_class # TODO: design this out of oedipus_lex. or something.
@@ -1049,3 +1081,4 @@ RPStringScanner
 
-    self.extra_lineno -= 1 if r && s == "n"
+    self.extra_lineno += 1 if s == "\n"     # eg backslash newline strings
+    self.extra_lineno -= 1 if r && s == "n" # literal \n, not newline
 
@@ -1138,6 +1171,6 @@ return r if r
                               when 'W' then
-                                scan(/\s*/)
+                                eat_whitespace
                                 [:tWORDS_BEG,    STR_DQUOTE | STR_FUNC_QWORDS]
                               when 'w' then
-                                scan(/\s*/)
+                                eat_whitespace
                                 [:tQWORDS_BEG,   STR_SQUOTE | STR_FUNC_QWORDS]
@@ -1152,6 +1185,6 @@ when 'x' then
                               when 'I' then
-                                scan(/\s*/)
+                                eat_whitespace
                                 [:tSYMBOLS_BEG, STR_DQUOTE | STR_FUNC_QWORDS]
                               when 'i' then
-                                scan(/\s*/)
+                                eat_whitespace
                                 [:tQSYMBOLS_BEG, STR_SQUOTE | STR_FUNC_QWORDS]
@@ -1186,3 +1219,3 @@ end
 
-    space = true if qwords and scan(/\s+/)
+    space = true if qwords and eat_whitespace
 
@@ -1189,0 +1222,0 @@ if self.string_nest == 0 && scan(/#{term_re}/) then

lib/ruby_lexer.rex.rb

@@ -18,8 +18,8 @@ # encoding: UTF-8
   SSTRING       = /(\\.|[^\'])*/
-  INT_DEC       = /[+]?(?:(?:[1-9][\d_]*|0)(?!\.\d)\b|0d[0-9_]+)/i
-  INT_HEX       = /[+]?0x[a-f0-9_]+/i
-  INT_BIN       = /[+]?0b[01_]+/i
-  INT_OCT       = /[+]?0o?[0-7_]+|0o/i
-  FLOAT         = /[+]?\d[\d_]*\.[\d_]+(e[+-]?[\d_]+)?\b|[+]?[\d_]+e[+-]?[\d_]+\b/i
-  INT_DEC2      = /[+]?\d[0-9_]*(?![e])/i
+  INT_DEC       = /[+]?(?:(?:[1-9][\d_]*|0)(?!\.\d)(ri|r|i)?\b|0d[0-9_]+)(ri|r|i)?/i
+  INT_HEX       = /[+]?0x[a-f0-9_]+(ri|r|i)?/i
+  INT_BIN       = /[+]?0b[01_]+(ri|r|i)?/i
+  INT_OCT       = /[+]?0o?[0-7_]+(ri|r|i)?|0o(ri|r|i)?/i
+  FLOAT         = /[+]?\d[\d_]*\.[\d_]+(e[+-]?[\d_]+)?(?:(ri|r|i)\b)?|[+]?[\d_]+e[+-]?[\d_]+(?:(ri|r|i)\b)?/i
+  INT_DEC2      = /[+]?\d[0-9_]*(?![e])((ri|r|i)\b)?/i
   NUM_BAD       = /[+]?0[xbd]\b/i
@@ -29,3 +29,4 @@ INT_OCT_BAD   = /[+]?0o?[0-7_]*[89]/i
 
-  class ScanError < StandardError ; end
+  class LexerError < StandardError ; end
+  class ScanError < LexerError ; end
 
@@ -48,3 +49,2 @@ attr_accessor :filename
 
-
   def scanner_class
@@ -68,2 +68,8 @@ StringScanner
 
+  def location
+    [
+      (filename || "<input>"),
+    ].compact.join(":")
+  end
+
   def next_token
@@ -304,6 +310,6 @@ return process_string if lex_strterm
             text = ss.string[ss.pos .. -1]
-            raise ScanError, "can not match (#{state.inspect}): '#{text}'"
+            raise ScanError, "can not match (#{state.inspect}) at #{location}: '#{text}'"
           end
         else
-          raise ScanError, "undefined state: '#{state}'"
+          raise ScanError, "undefined state at #{location}: '#{state}'"
         end # token = case state
@@ -314,3 +320,3 @@
 
-    raise "bad lexical result: #{token.inspect}" unless
+    raise LexerError, "bad lexical result at #{location}: #{token.inspect}" unless
       token.nil? || (Array === token && token.size >= 2)
@@ -317,0 +323,0 @@

lib/ruby_parser_extras.rb

@@ -94,3 +94,3 @@ # encoding: ASCII-8BIT
 module RubyParserStuff
-  VERSION = "3.8.2" unless constants.include? "VERSION" # SIGH
+  VERSION = "3.8.3" unless constants.include? "VERSION" # SIGH
 
@@ -879,7 +879,69 @@ attr_accessor :lexer, :in_def, :in_single, :file
     result = s(:str, str)
-    self.lexer.lineno += str.count("\n") + self.lexer.extra_lineno
-    self.lexer.extra_lineno = 0
+    self.lexer.fixup_lineno str.count("\n")
     result
   end
 
+  def new_qword_list_entry val
+    str = val[1]
+    str.force_encoding("ASCII-8BIT") unless str.valid_encoding? unless RUBY_VERSION < "1.9"
+    result = s(:str, str)
+    self.lexer.fixup_lineno
+    result
+  end
+
+  def new_qword_list
+    result = s(:array)
+    self.lexer.fixup_lineno
+    result
+  end
+
+  def new_word_list
+    result = s(:array)
+    self.lexer.fixup_lineno
+    result
+  end
+
+  def new_word_list_entry val
+    result = val[1][0] == :evstr ? s(:dstr, "", val[1]) : val[1]
+    self.lexer.fixup_lineno
+    result
+  end
+
+  def new_qsym_list
+    result = s(:array)
+    self.lexer.fixup_lineno
+    result
+  end
+
+  def new_qsym_list_entry val
+    result = s(:lit, val[1].to_sym)
+    self.lexer.fixup_lineno
+    result
+  end
+
+  def new_symbol_list
+    result = s(:array)
+    self.lexer.fixup_lineno
+    result
+  end
+
+  def new_symbol_list_entry val
+    _list, sym, _nil = val # TODO: use _list
+    result = val[1]
+
+    result ||= s(:str, "")
+
+    case sym[0]
+    when :dstr then
+      sym[0] = :dsym
+    when :str then
+      sym = s(:lit, sym.last.to_sym)
+    else
+      debug20 24
+      sym = s(:dsym, "", sym || s(:str, ""))
+    end
+    self.lexer.fixup_lineno
+    sym
+  end
+
   def new_super args
@@ -886,0 +948,0 @@ if args && args.node_type == :block_pass then

test/test_ruby_parser_extras.rb

@@ -185,3 +185,3 @@ # encoding: US-ASCII
     @env.extend
-    assert_equal nil, @env[:blah]
+    assert_nil @env[:blah]
     @env.unextend
@@ -188,0 +188,0 @@ assert_equal 42, @env[:blah]

No alert changes

No metric changes