ruby_parser - rubygems Package Compare versions

Comparing version 3.0.1 to 3.0.2

History.txt

@@ -0,1 +1,22 @@
+=== 3.0.2 / 2012-11-21
+
+52 down makes 99.9767% or 3.7σ. 130 files failed to parse out of 558k.
+
+* 4 minor enhancements:
+
+  * Added RP_TIMEOUT env var to override default timeout of 10 seconds.
+  * Minor optimization to RubyLexer#parse_number
+  * Only output parseerror output to stderr if $DEBUG.
+  * ruby_parse_extract_error modified to include 'it' blocks in its search.
+
+* 7 bug fixes:
+
+  * 1.9: Fixed args in dot-call forms (eg f.(...)).
+  * 1.9: Fixed lexing stabby lambda w/ do/end
+  * Deal better with DOS files. Ugh.
+  * Fix line number of production after heredoc.
+  * Fixed RubyParser#process to reuse parser instances across calls.
+  * Fixed line numbers for several productions.
+  * new_call sets line number to smallest line number of members.
+
 === 3.0.1 / 2012-11-02
@@ -2,0 +23,0 @@

lib/ruby_lexer.rb

@@ -282,2 +282,4 @@ # encoding: US-ASCII
       rb_compile_error "Invalid numeric format"
+    when src.scan(/[+-]?(?:(?:[1-9][\d_]*|0)(?!\.\d)\b|0[Dd][0-9_]+)/) then
+      int_with_base(10)
     when src.scan(/[+-]?0x[a-f0-9_]+/i) then
@@ -287,4 +289,2 @@ int_with_base(16)
       int_with_base(2)
-    when src.scan(/[+-]?0[Dd][0-9_]+/) then
-      int_with_base(10)
     when src.scan(/[+-]?0[Oo]?[0-7_]*[89]/) then
@@ -303,6 +303,4 @@ rb_compile_error "Illegal octal digit."
       :tFLOAT
-    when src.scan(/[+-]?0\b/) then
+    when src.scan(/[+-]?[0-9_]+(?![e])/) then
       int_with_base(10)
-    when src.scan(/[+-]?[\d_]+\b/) then
-      int_with_base(10)
     else
@@ -1239,3 +1237,3 @@ rb_compile_error "Bad number format"
         elsif src.check(/\_/) then
-          if src.beginning_of_line? && src.scan(/\__END__(\n|\Z)/) then
+          if src.beginning_of_line? && src.scan(/\__END__(\r?\n|\Z)/) then
             self.lineno = nil
@@ -1387,5 +1385,3 @@ return RubyLexer::EOF
             self.command_start = true
-            return :kDO_COND  if cond.is_in_state
-            return :kDO_BLOCK if cmdarg.is_in_state && state != :expr_cmdarg
-            return :kDO_BLOCK if state == :expr_endarg
+
             if defined?(@hack_expects_lambda) && @hack_expects_lambda
@@ -1395,2 +1391,7 @@ @hack_expects_lambda = false
             end
+
+            return :kDO_COND  if cond.is_in_state
+            return :kDO_BLOCK if cmdarg.is_in_state && state != :expr_cmdarg
+            return :kDO_BLOCK if state == :expr_endarg
+
             return :kDO
@@ -1397,0 +1398,0 @@ end

lib/ruby_parser_extras.rb

@@ -81,3 +81,3 @@ # encoding: ASCII-8BIT
     warn({:unread_many => caller[0]}.inspect) if ENV['TALLY']
-    self.extra_lines_added += str.count("\n")
+    self.extra_lines_added += str.count("\n") - 1
     begin
@@ -112,3 +112,3 @@ string[charpos, 0] = str
 module RubyParserStuff
-  VERSION = '3.0.1' unless constants.include? "VERSION" # SIGH
+  VERSION = '3.0.2' unless constants.include? "VERSION" # SIGH
 
@@ -360,2 +360,4 @@ attr_accessor :lexer, :in_def, :in_single, :file
 
+    result.line(result.line - 1) if result.line and lexer.src.bol?
+
     raise "identifier #{id.inspect} is not valid" unless result
@@ -523,3 +525,2 @@
     result = s(:call, recv, meth)
-    result.line = recv.line if recv
 
@@ -536,2 +537,5 @@ # TODO: need a test with f(&b) to produce block_pass
 
+    line = result.grep(Sexp).map(&:line).compact.min
+    result.line = line if line
+
     result
@@ -1054,3 +1058,3 @@ end
   def yyerror msg
-    warn msg
+    warn msg if $DEBUG
     super()
@@ -1063,4 +1067,4 @@ end
     # I don't like how the exception obscures the error message
-    msg = "# ERROR: %s:%p :: %s" % [self.file, lexer.lineno, e.message.strip]
-    warn msg
+    e.message.replace "%s:%p :: %s" % [self.file, lexer.lineno, e.message.strip]
+    warn e.message if $DEBUG
     raise
@@ -1296,5 +1300,5 @@ end
   def process(s, f = "(string)") # parens for emacs *sigh*
-    Ruby19Parser.new.process s, f
+    @p19.process s, f
   rescue Racc::ParseError
-    Ruby18Parser.new.process s, f
+    @p18.process s, f
   end
@@ -1301,0 +1305,0 @@

Rakefile

@@ -147,2 +147,3 @@ # -*- ruby -*-
   require 'ruby_parser'
+  require 'pp'
 
@@ -155,2 +156,4 @@ parser = if ENV["V"] == "18" then
 
+  time = (ENV["RP_TIMEOUT"] || 10).to_i
+
   file = ENV["F"] || ENV["FILE"]
@@ -166,3 +169,3 @@
   begin
-    p parser.process(ruby, file)
+    pp parser.process(ruby, file, time)
   rescue Racc::ParseError => e
@@ -191,2 +194,6 @@ p e
 
+task :bugs do
+  sh "for f in bug*.rb ; do rake debug F=$f && rm $f ; done"
+end
+
 # vim: syntax=Ruby

test/test_ruby_parser.rb

@@ -15,2 +15,14 @@ #!/usr/local/bin/ruby
 
+class Sexp
+  alias oldeq2 ==
+  def ==(obj) # :nodoc:
+    if obj.class == self.class then
+      super and
+        (self.line.nil? or obj.line.nil? or self.line == obj.line)
+    else
+      false
+    end
+  end
+end
+
 class RubyParserTestCase < ParseTreeTestCase
@@ -45,3 +57,3 @@ attr_accessor :result, :processor
 
-    assert_equal emsg, e.message.strip # TODO: why strip?
+    assert_equal emsg, e.message
   end
@@ -51,3 +63,3 @@
     e = nil
-    out, err = capture_io do
+    assert_silent do
       e = assert_raises Racc::ParseError do
@@ -58,6 +70,3 @@ processor.parse rb
 
-    assert_equal "", out
-    assert_match(/parse error on value/, err)
-
-    assert_equal emsg, e.message.strip # TODO: why strip?
+    assert_equal emsg, e.message
   end
@@ -552,17 +561,4 @@
   STARTING_LINE = {
-    "case_nested_inner_no_expr"          => 2,
-    "case_no_expr"                       => 2,
-    "case_splat"                         => 2,
-    "dstr_heredoc_expand"                => 1,
-    "dstr_heredoc_windoze_sucks"         => 1,
-    "dstr_heredoc_yet_again"             => 1,
-    "str_heredoc"                        => 1,
-    "str_heredoc_call"                   => 1,
-    "str_heredoc_empty"                  => 1,
-    "str_heredoc_indent"                 => 1,
+    "case_no_expr"                       => 2, # TODO this should be 1
     "structure_unused_literal_wwtt"      => 3, # yes, 3... odd test
-    "undef_block_1"                      => 2,
-    "undef_block_2"                      => 2,
-    "undef_block_3"                      => 2,
-    "undef_block_wtf"                    => 2,
   }
@@ -671,5 +667,8 @@
 
-    result = processor.parse rb
-    assert_equal 1, result.lasgn.line
-    assert_equal 4, result.call.line
+    pt = s(:block,
+           s(:lasgn, :string,
+             s(:str, "        very long string\n").line(1)).line(1),
+           s(:call, nil, :puts, s(:lvar, :string).line(4)).line(4)).line(1)
+
+    assert_parse rb, pt
   end
@@ -888,2 +887,64 @@
   end
+
+  def test_i_fucking_hate_line_numbers
+    rb = <<-EOM.gsub(/^ {6}/, '')
+      def a
+        p 1
+        a.b 2
+        c.d 3, 4
+        e.f 5
+        g.h 6, 7
+        p(1)
+        a.b(2)
+        c.d(3, 4)
+        e.f(5)
+        g.h(6, 7)
+      end
+    EOM
+
+    pt = s(:defn, :a, s(:args).line(2),
+           s(:call, nil, :p, s(:lit, 1).line(2)).line(2),
+           s(:call, s(:call, nil, :a).line(3), :b,
+             s(:lit, 2).line(3)).line(3),
+           s(:call, s(:call, nil, :c).line(4), :d,
+             s(:lit, 3).line(4), s(:lit, 4).line(4)).line(4),
+           s(:call, s(:call, nil, :e).line(5), :f,
+             s(:lit, 5).line(5)).line(5),
+           s(:call, s(:call, nil, :g).line(6), :h,
+             s(:lit, 6).line(6), s(:lit, 7).line(6)).line(6),
+           s(:call, nil, :p, s(:lit, 1).line(7)).line(7),
+           s(:call, s(:call, nil, :a).line(8), :b,
+             s(:lit, 2).line(8)).line(8),
+           s(:call, s(:call, nil, :c).line(9), :d,
+             s(:lit, 3).line(9), s(:lit, 4).line(9)).line(9),
+           s(:call, s(:call, nil, :e).line(10), :f,
+             s(:lit, 5).line(10)).line(10),
+           s(:call, s(:call, nil, :g).line(11), :h,
+             s(:lit, 6).line(11), s(:lit, 7).line(11)).line(11)
+           ).line(1)
+
+    assert_parse rb, pt
+  end
+
+  def test_i_fucking_hate_line_numbers2
+    rb = <<-EOM.gsub(/^ {6}/, '')
+      def a
+          p('a')
+          b = 1
+          p b
+          c =1
+      end
+      a
+    EOM
+
+      pt = s(:block,
+             s(:defn, :a, s(:args).line(2),
+               s(:call, nil, :p, s(:str, "a").line(2)).line(2),
+               s(:lasgn, :b, s(:lit, 1).line(3)).line(3),
+               s(:call, nil, :p, s(:lvar, :b).line(4)).line(4),
+               s(:lasgn, :c, s(:lit, 1).line(5)).line(5)).line(1),
+             s(:call, nil, :a).line(7)).line(1)
+
+    assert_parse rb, pt
+  end
 end
@@ -899,9 +960,6 @@
 
-    out, err = capture_io do
+    assert_silent do
       assert_equal pt, processor.parse(rb)
     end
 
-    assert_empty out
-    assert_match(/parse error on value .:/, err)
-
     # 1.9 only syntax
@@ -920,3 +978,3 @@ rb = "a.()"
 
-    msg = "parse error on value \"(\" (tLPAREN2)"
+    msg = "(string):1 :: parse error on value \"(\" (tLPAREN2)"
     assert_equal msg, e.message.strip
@@ -1198,3 +1256,3 @@ end
 
-    assert_parse_error rb, "parse error on value \":\" (tCOLON)"
+    assert_parse_error rb, "(string):1 :: parse error on value \":\" (tCOLON)"
   end
@@ -1205,3 +1263,3 @@
 
-    assert_parse_error rb, "parse error on value \",\" (tCOMMA)"
+    assert_parse_error rb, "(string):1 :: parse error on value \",\" (tCOMMA)"
   end
@@ -1217,3 +1275,3 @@
 
-    assert_parse_error rb, "parse error on value \":\" (tCOLON)"
+    assert_parse_error rb, "(string):2 :: parse error on value \":\" (tCOLON)"
   end
@@ -1224,3 +1282,3 @@
 
-    assert_parse_error rb, 'parse error on value "=" (tEQL)'
+    assert_parse_error rb, '(string):1 :: parse error on value "=" (tEQL)'
   end
@@ -1231,3 +1289,3 @@
 
-    assert_parse_error rb, 'parse error on value "=" (tEQL)'
+    assert_parse_error rb, '(string):1 :: parse error on value "=" (tEQL)'
   end
@@ -1428,3 +1486,3 @@
 
-    assert_parse_error rb, 'parse error on value ".." (tDOT2)'
+    assert_parse_error rb, '(string):2 :: parse error on value ".." (tDOT2)'
   end
@@ -1674,2 +1732,22 @@
   end
+
+  def test_lambda_do_vs_brace
+    pt = s(:call, nil, :f, s(:iter, s(:call, nil, :lambda), 0))
+
+    rb = "f ->() {}"
+    assert_parse rb, pt
+
+    rb = "f ->() do end"
+    assert_parse rb, pt
+  end
+
+  def test_thingy
+    pt = s(:call, s(:call, nil, :f), :call, s(:lit, 42))
+
+    rb = "f.(42)"
+    assert_parse rb, pt
+
+    rb = "f::(42)"
+    assert_parse rb, pt
+  end
 end

No alert changes

No metric changes