ruby_parser - rubygems Package Compare versions

Comparing version 3.0.0.a2 to 3.0.0.a3

History.txt

@@ -0,1 +1,34 @@
+=== 3.0.0.a3 / 2012-07-03
+
+* 1 major enhancement:
+
+  * OMG! Implemented all the 1.9 arg crap and I went from 500 lines of
+    structural diff to 32!
+
+* 17 minor enhancements:
+
+  * 1.9: !(...) is now a method call. ugh
+  * 1.9: Added __ENCODING__ keyword (returns Unsupported atm).
+  * 1.9: Added support for "a.()" thingy. whatever you call it. Have I ever mentioned that ruby has too much syntax? I mean really...
+  * 1.9: Moved kRETURN/kBREAK/kNEXT from command_call to command
+  * 1.9: Removed when_args and refactored into args production.
+  * 1.9: Support for not/! being a call to #!. why matz? why?!?
+  * 1.9: Support for trailing comma support in opt call args.
+  * 1.9: Switched keyword support to deal with 1.8 vs 1.9 differences.
+  * 1.9: refactored and created command_asgn
+  * 1.9: removed case/else from primary
+  * 1.9: reworked mlhs and family. Now horrible things like 'a, *, c = f' work
+  * 1.9: reworked top level rules from program to expr. Lots of lame refactorings in mri.
+  * Moved 'a[]' from primary down to method_call
+  * Removed 'args trailer' production from assoc_list (eg {1, 2}).
+  * Removed 1.9 support for : as a terminator in do/then
+  * RubyParser no longer subclasses the 18 parser, so I had to change some consts around.
+  * Undeprecated RubyParser - but now it is a compound parser that tries 1.9 first.
+
+* 3 bug fixes:
+
+  * 1.9: fixed bang method name.
+  * Fixed case/when/splat handling on both 1.8 and 1.9.
+  * Removed lambda_body production from lambda
+
 === 3.0.0.a2 / 2012-06-19
@@ -2,0 +35,0 @@

lib/ruby_lexer.rb

@@ -38,2 +38,4 @@ class RubyLexer
 
+  attr_accessor :space_seen
+
   EOF = :eof_haha!
@@ -225,5 +227,5 @@
     self.version = v
-    self.cond = RubyParser::StackState.new(:cond)
-    self.cmdarg = RubyParser::StackState.new(:cmdarg)
-    self.tern = RubyParser::StackState.new(:tern)
+    self.cond = RubyParserStuff::StackState.new(:cond)
+    self.cmdarg = RubyParserStuff::StackState.new(:cmdarg)
+    self.tern = RubyParserStuff::StackState.new(:tern)
     self.nest = 0
@@ -242,2 +244,3 @@ @comments = []
   def lex_state= o
+    # warn "wtf lex_state = #{o.inspect}"
     raise "wtf\?" unless Symbol === o
@@ -646,3 +649,3 @@ @lex_state = o
     c = ''
-    space_seen = false
+    self.space_seen = false
     command_state = false
@@ -663,3 +666,3 @@ src = self.src
       if src.scan(/[\ \t\r\f\v]/) then # \s - \n + \v
-        space_seen = true
+        self.space_seen = true
         next
@@ -719,5 +722,5 @@ elsif src.check(/[^a-zA-Z]/) then
           result = if ruby18 then
-                     yylex_paren18 space_seen
+                     yylex_paren18
                    else
-                     yylex_paren19 space_seen
+                     yylex_paren19
                    end
@@ -764,6 +767,3 @@
         elsif src.scan(/\:\:/) then
-          if (lex_state == :expr_beg ||
-              lex_state == :expr_mid ||
-              lex_state == :expr_class ||
-              (lex_state.is_argument && space_seen)) then
+          if is_beg? || lex_state == :expr_class || is_space_arg? then
             self.lex_state = :expr_beg
@@ -1149,3 +1149,3 @@ self.yacc_value = "::"
             self.lineno = nil
-            space_seen = true
+            self.space_seen = true
             next
@@ -1244,3 +1244,3 @@ end
 
-  def yylex_paren18 space_seen
+  def yylex_paren18
     self.command_start = true
@@ -1265,8 +1265,27 @@ result = :tLPAREN2
 
-  def yylex_paren19 space_seen
-    if (lex_state == :expr_beg || lex_state == :expr_mid ||
-        lex_state == :expr_value || lex_state == :expr_class) then
+  def is_end?
+    (lex_state == :expr_end    ||
+     lex_state == :expr_endarg ||
+     lex_state == :expr_endfn)
+  end
+
+  def is_arg?
+    lex_state == :expr_arg || lex_state == :expr_cmdarg
+  end
+
+  def is_beg?
+    (lex_state == :expr_beg   ||
+     lex_state == :expr_mid   ||
+     lex_state == :expr_value ||
+     lex_state == :expr_class)
+  end
+
+  def is_space_arg? c = "x"
+    is_arg? and space_seen and c !~ /\s/
+  end
+
+  def yylex_paren19
+    if is_beg? then
       result = :tLPAREN
-    elsif ((lex_state == :expr_arg || lex_state == :expr_cmdarg) and
-           space_seen) then
+    elsif is_space_arg? then
       result = :tLPAREN_ARG
@@ -1277,2 +1296,5 @@ else
     end
+
+    # p :wtf_paren => [lex_state, space_seen, result]
+
     # HACK paren_nest++;
@@ -1348,3 +1370,7 @@
         # See if it is a reserved word.
-        keyword = RubyParser::Keyword.keyword token
+        keyword = if ruby18 then # REFACTOR need 18/19 lexer subclasses
+                    RubyParserStuff::Keyword.keyword18 token
+                  else
+                    RubyParserStuff::Keyword.keyword19 token
+                  end
 
@@ -1351,0 +1377,0 @@ if keyword then

lib/ruby_parser_extras.rb

@@ -5,2 +5,3 @@ require 'stringio'
 require 'strscan'
+require 'ruby_lexer'
 
@@ -119,3 +120,3 @@ def d o
 module RubyParserStuff
-  VERSION = '3.0.0.a2' unless constants.include? "VERSION" # SIGH
+  VERSION = '3.0.0.a3' unless constants.include? "VERSION" # SIGH
 
@@ -228,2 +229,53 @@ attr_accessor :lexer, :in_def, :in_single, :file
 
+  def block_args19 val, id
+    # HACK OMG THIS CODE IS SOOO UGLY! CLEAN ME
+    untested = %w[1 2 3 4 7 9 10 11 12 14]
+    raise "no block_args19 #{id}" if untested.include? id
+
+    r = s(:array)
+
+    val.compact.each do |v|
+      next if %w[,].include? v
+      case v
+      when Sexp then
+        case v.first
+        when :args then
+          r.concat v[1..-1].map { |s| s(:lasgn, s) }
+        when :block_arg then
+          r << s(:lasgn, :"&#{v.last}")
+        else
+          raise "block_args19 #{id} unhandled sexp type:: #{v.inspect}"
+        end
+      when Symbol
+        case v.to_s
+        when /^\*(.+)/ then
+          r << s(:splat, s(:lasgn, $1.to_sym))
+        when /^\*/ then
+          r << s(:splat)
+        else
+          raise "block_args19 #{id} unhandled symbol type:: #{v.inspect}"
+        end
+      else
+        raise "block_args19 #{id} unhandled type:: #{v.inspect}"
+      end
+    end
+
+    if r.size > 2 then
+      r = s(:masgn, r)
+    elsif r.size == 2 then
+      case r.last.first
+      when :splat then
+        r = s(:masgn, r)
+      when :lasgn then
+        r = r.last
+      else
+        raise "oh noes!: #{r.inspect}"
+      end
+    else
+      raise "fuck no #{r.inspect}"
+    end
+
+    r
+  end
+
   def aryset receiver, index
@@ -395,3 +447,3 @@ s(:attrasgn, receiver, :"[]=", *index[1..-1])
     self.lexer.parser = self
-    @env = Environment.new
+    @env = RubyParserStuff::Environment.new
     @comments = []
@@ -590,3 +642,3 @@
   def new_compstmt val
-    result = void_stmts(val[0])
+    result = void_stmts(val.grep(Sexp)[0])
     result = remove_begin(result) if result
@@ -1019,2 +1071,3 @@ result
                 ["alias",    [:kALIAS,    :kALIAS      ], :expr_fname ],
+                ["__ENCODING__", [:k__ENCODING__, :k__ENCODING__], :expr_end],
                ].map { |args| KWtable.new(*args) }
@@ -1024,8 +1077,21 @@
 
-    WORDLIST = Hash[*wordlist.map { |o| [o.name, o] }.flatten] unless
-      defined? WORDLIST
+    WORDLIST18 = Hash[*wordlist.map { |o| [o.name, o] }.flatten]
+    WORDLIST19 = Hash[*wordlist.map { |o| [o.name, o] }.flatten]
 
-    def self.keyword str
-      WORDLIST[str]
+    %w[and case elsif for if in module or unless until when while].each do |k|
+      WORDLIST19[k] = WORDLIST19[k].dup
+      WORDLIST19[k].state = :expr_value
     end
+    %w[not].each do |k|
+      WORDLIST19[k] = WORDLIST19[k].dup
+      WORDLIST19[k].state = :expr_arg
+    end
+
+    def self.keyword18 str # REFACTOR
+      WORDLIST18[str]
+    end
+
+    def self.keyword19 str
+      WORDLIST19[str]
+    end
   end
@@ -1148,8 +1214,18 @@
 
-class RubyParser < Ruby18Parser
+##
+# RubyParser is a compound parser that first attempts to parse using
+# the 1.9 syntax parser and falls back to the 1.8 syntax parser on a
+# parse error.
+
+class RubyParser
   def initialize
-    super
-    warn "WA\RNING: Deprecated: RubyParser. Use Ruby18Parser or Ruby19Parser"
-    warn "  from #{caller.first}"
+    @p18 = Ruby18Parser.new
+    @p19 = Ruby19Parser.new
   end
+
+  def parse s
+    @p19.parse s
+  rescue Racc::ParseError
+    @p18.parse s
+  end
 end
@@ -1183,7 +1259,7 @@
   def add x
-    raise "no" # TODO: need a test to trigger this
+    concat x
   end
 
   def add_all x
-    raise "no" # TODO: need a test to trigger this
+    raise "no: #{self.inspect}.add_all #{x.inspect}" # TODO: need a test to trigger this
   end
@@ -1190,0 +1266,0 @@

Rakefile

@@ -24,3 +24,3 @@ # -*- ruby -*-
 
-  self.racc_flags << " -g" if plugin?(:racc) && ENV["DEBUG"]
+  self.racc_flags << " -t" if plugin?(:racc) && ENV["DEBUG"]
 end
@@ -27,0 +27,0 @@

test/test_ruby_parser_extras.rb

@@ -12,3 +12,3 @@ require 'rubygems'
   def setup
-    @s = RubyParser::StackState.new :test
+    @s = RubyParserStuff::StackState.new :test
   end
@@ -64,3 +64,3 @@
   def setup
-    @env = RubyParser::Environment.new
+    @env = RubyParserStuff::Environment.new
     @env[:blah] = 42
@@ -67,0 +67,0 @@ assert_equal 42, @env[:blah]

test/test_ruby_parser.rb

 #!/usr/local/bin/ruby
 
-ENV['VERBOSE'] = "1"
+# ENV['VERBOSE'] = "1"
 
@@ -10,18 +10,6 @@ require 'rubygems'
 
-$: << File.expand_path('~/Work/p4/zss/src/ParseTree/dev/test')
+$: << File.expand_path('~/Work/p4/zss/src/sexp_processor/dev/lib')
 
 require 'pt_testcase'
 
-class Ruby18Parser # FIX
-  def process input
-    parse input
-  end
-end
-
-class Ruby19Parser
-  def process input
-    parse input
-  end
-end
-
 class RubyParserTestCase < ParseTreeTestCase
@@ -48,2 +36,10 @@ attr_accessor :result, :processor
 
+  def assert_parse_error rb, emsg
+    e = assert_raises Racc::ParseError do
+      processor.parse rb
+    end
+
+    assert_equal emsg, e.message.strip # TODO: why strip?
+  end
+
   def assert_parse_line rb, pt, line
@@ -55,3 +51,3 @@ assert_parse rb, pt
 
-module TestRubyParser
+module TestRubyParserShared
   def test_attrasgn_array_lhs
@@ -623,3 +619,100 @@ rb = '[1, 2, 3, 4][from .. to] = ["a", "b", "c"]'
   end
+end
 
+class TestRubyParser < MiniTest::Unit::TestCase
+  def test_parse
+    processor = RubyParser.new
+
+    # 1.8 only syntax
+    rb = "while false : 42 end"
+    pt = s(:while, s(:false), s(:lit, 42), true)
+
+    assert_equal pt, processor.parse(rb)
+
+    # 1.9 only syntax
+    rb = "a.()"
+    pt = s(:call, s(:call, nil, :a), :call)
+
+    assert_equal pt, processor.parse(rb)
+
+    # bad syntax
+    e = assert_raises Racc::ParseError do
+      processor.parse "a.("
+    end
+
+    msg = "parse error on value \"(\" (tLPAREN2)"
+    assert_equal msg, e.message.strip
+  end
+end
+
+class TestRuby18Parser < RubyParserTestCase
+  include TestRubyParserShared
+
+  def setup
+    super
+
+    self.processor = Ruby18Parser.new
+  end
+
+  def test_flip2_env_lvar
+    rb = "if a..b then end"
+    pt = s(:if, s(:flip2, s(:call, nil, :a), s(:call, nil, :b)), nil, nil)
+
+    assert_parse rb, pt
+
+    top_env = processor.env.env.first
+
+    assert_kind_of Hash, top_env
+
+    flip = top_env.find { |k,v| k =~ /^flip/ }
+
+    assert flip
+    assert_equal :lvar, flip.last
+  end
+
+  def test_assoc_list_18
+    rb = "{1, 2, 3, 4}"
+    pt = s(:hash, s(:lit, 1), s(:lit, 2), s(:lit, 3), s(:lit, 4))
+
+    assert_parse rb, pt
+  end
+
+  def test_case_then_colon_18
+    rb = "case x; when Fixnum: 42; end"
+    pt = s(:case,
+           s(:call, nil, :x),
+           s(:when, s(:array, s(:const, :Fixnum)), s(:lit, 42)),
+           nil)
+
+    assert_parse rb, pt
+  end
+
+  def test_do_colon_18
+    rb = "while false : 42 end"
+    pt = s(:while, s(:false), s(:lit, 42), true)
+
+    assert_parse rb, pt
+  end
+
+  def test_parse_until_not_canonical
+    rb = "until not var.nil?\n  'foo'\nend"
+
+    pt = s(:while,
+           s(:call, s(:call, nil, :var), :nil?),
+           s(:str, "foo"), true)
+
+    assert_parse rb, pt
+  end
+
+  def test_parse_until_not_noncanonical
+    rb = "until not var.nil?\n  'foo'\nend"
+    pt = s(:until,
+           s(:not, s(:call, s(:call, nil, :var), :nil?)),
+           s(:str, "foo"), true)
+
+    processor.canonicalize_conditions = false
+
+    assert_parse rb, pt
+  end
+
   def test_parse_if_not_canonical
@@ -638,4 +731,3 @@ rb = "if not var.nil? then 'foo' else 'bar'\nend"
     pt = s(:if,
-           s(:not,
-             s(:call, s(:call, nil, :var), :nil?)),
+           s(:not, s(:call, s(:call, nil, :var), :nil?)),
            s(:str, "foo"),
@@ -661,4 +753,3 @@ s(:str, "bar"))
     pt = s(:while,
-           s(:not,
-             s(:call, s(:call, nil, :var), :nil?)),
+           s(:not, s(:call, s(:call, nil, :var), :nil?)),
            s(:str, "foo"), true)
@@ -670,8 +761,140 @@
   end
+end
 
+class TestRuby19Parser < RubyParserTestCase
+  include TestRubyParserShared
+
+  def setup
+    super
+
+    self.processor = Ruby19Parser.new
+  end
+
+  def test_mlhs_back_splat
+    rb = "a, b, c, *s = f"
+    pt = s(:masgn,
+           s(:array,
+             s(:lasgn, :a), s(:lasgn, :b), s(:lasgn, :c),
+             s(:splat, s(:lasgn, :s))),
+           s(:to_ary, s(:call, nil, :f)))
+
+    assert_parse rb, pt
+  end
+
+  def test_mlhs_back_anonsplat
+    rb = "a, b, c, * = f"
+    pt = s(:masgn,
+           s(:array,
+             s(:lasgn, :a), s(:lasgn, :b), s(:lasgn, :c),
+             s(:splat)),
+           s(:to_ary, s(:call, nil, :f)))
+
+    assert_parse rb, pt
+  end
+
+  def test_mlhs_mid_splat
+    rb = "a, b, c, *s, x, y, z = f"
+    pt = s(:masgn,
+           s(:array,
+             s(:lasgn, :a), s(:lasgn, :b), s(:lasgn, :c),
+             s(:splat, s(:lasgn, :s)),
+             s(:lasgn, :x), s(:lasgn, :y), s(:lasgn, :z)),
+           s(:to_ary, s(:call, nil, :f)))
+
+    assert_parse rb, pt
+  end
+
+  def test_mlhs_mid_anonsplat
+    rb = "a, b, c, *, x, y, z = f"
+    pt = s(:masgn,
+           s(:array, s(:lasgn, :a), s(:splat), s(:lasgn, :z)),
+           s(:to_ary, s(:call, nil, :f)))
+    pt = s(:masgn,
+           s(:array,
+             s(:lasgn, :a), s(:lasgn, :b), s(:lasgn, :c),
+             s(:splat),
+             s(:lasgn, :x), s(:lasgn, :y), s(:lasgn, :z)),
+           s(:to_ary, s(:call, nil, :f)))
+
+    assert_parse rb, pt
+  end
+
+  def test_mlhs_front_splat
+    rb = "*s, x, y, z = f"
+    pt = s(:masgn,
+           s(:array, s(:splat, s(:lasgn, :s)), s(:lasgn, :z)),
+           s(:to_ary, s(:call, nil, :f)))
+    pt = s(:masgn,
+           s(:array,
+             s(:splat, s(:lasgn, :s)),
+             s(:lasgn, :x), s(:lasgn, :y), s(:lasgn, :z)),
+           s(:to_ary, s(:call, nil, :f)))
+
+    assert_parse rb, pt
+  end
+
+  def test_mlhs_front_anonsplat
+    rb = "*, x, y, z = f"
+    pt = s(:masgn,
+           s(:array,
+             s(:splat),
+             s(:lasgn, :x), s(:lasgn, :y), s(:lasgn, :z)),
+           s(:to_ary, s(:call, nil, :f)))
+
+    assert_parse rb, pt
+  end
+
+  def test_expr_not_bang
+    rb = "! a b"
+    pt = s(:call, s(:call, nil, :a, s(:call, nil, :b)), :"!")
+
+    assert_parse rb, pt
+  end
+
+  def test_encoding
+    rb = '__ENCODING__'
+    pt = s(:str, "Unsupported!")
+
+    assert_parse rb, pt
+  end
+
+  def test_do_colon_19
+    rb = "while false : 42 end"
+
+    assert_parse_error rb, "parse error on value \":\" (tCOLON)"
+  end
+
+  def test_assoc_list_19
+    rb = "{1, 2, 3, 4}"
+
+    assert_parse_error rb, "parse error on value \",\" (tCOMMA)"
+  end
+
+  def test_case_then_colon_19
+    rb = <<-EOM
+      case x
+      when Fixnum : # need the space to not hit new hash arg syntax
+        42
+      end
+    EOM
+
+    assert_parse_error rb, "parse error on value \":\" (tCOLON)"
+  end
+
+  def test_parse_def_xxx1
+    rb = 'def f(a, *b, c = nil) end'
+
+    assert_parse_error rb, 'parse error on value "=" (tEQL)'
+  end
+
+  def test_parse_def_xxx2
+    rb = 'def f(a = nil, *b, c = nil) end'
+
+    assert_parse_error rb, 'parse error on value "=" (tEQL)'
+  end
+
   def test_parse_until_not_canonical
     rb = "until not var.nil?\n  'foo'\nend"
-
-    pt = s(:while,
-           s(:call, s(:call, nil, :var), :nil?),
+    pt = s(:until,
+           s(:call, s(:call, s(:call, nil, :var), :nil?), :"!"),
            s(:str, "foo"), true)
@@ -685,4 +908,3 @@
     pt = s(:until,
-           s(:not,
-             s(:call, s(:call, nil, :var), :nil?)),
+           s(:call, s(:call, s(:call, nil, :var), :nil?), :"!"),
            s(:str, "foo"), true)
@@ -694,37 +916,50 @@
   end
-end
 
-class TestRuby18Parser < RubyParserTestCase
-  include TestRubyParser
+  def test_parse_if_not_canonical
+    rb = "if not var.nil? then 'foo' else 'bar'\nend"
+    pt = s(:if,
+           s(:call, s(:call, s(:call, nil, :var), :nil?), :"!"),
+           s(:str, "foo"),
+           s(:str, "bar"))
 
-  def setup
-    super
+    assert_parse rb, pt
+  end
 
-    self.processor = Ruby18Parser.new
+  def test_parse_if_not_noncanonical
+    rb = "if not var.nil? then 'foo' else 'bar'\nend"
+    pt = s(:if,
+           s(:call, s(:call, s(:call, nil, :var), :nil?), :"!"),
+           s(:str, "foo"),
+           s(:str, "bar"))
+
+    processor.canonicalize_conditions = false
+
+    assert_parse rb, pt
   end
 
-  def test_flip2_env_lvar
-    rb = "if a..b then end"
-    pt = s(:if, s(:flip2, s(:call, nil, :a), s(:call, nil, :b)), nil, nil)
+  def test_parse_while_not_canonical
+    rb = "while not var.nil?\n  'foo'\nend"
+    pt = s(:while,
+           s(:call, s(:call, s(:call, nil, :var), :nil?), :"!"),
+           s(:str, "foo"), true)
 
     assert_parse rb, pt
+  end
 
-    top_env = processor.env.env.first
+  def test_parse_while_not_noncanonical
+    rb = "while not var.nil?\n  'foo'\nend"
+    pt = s(:while,
+           s(:call, s(:call, s(:call, nil, :var), :nil?), :"!"),
+           s(:str, "foo"), true)
 
-    assert_kind_of Hash, top_env
+    processor.canonicalize_conditions = false
 
-    flip = top_env.find { |k,v| k =~ /^flip/ }
-
-    assert flip
-    assert_equal :lvar, flip.last
+    assert_parse rb, pt
   end
-end
 
-class TestRuby19Parser < RubyParserTestCase
-  include TestRubyParser
+  def test_parse_opt_call_args_assocs_comma
+    rb = "1[2=>3,]"
+    pt = s(:call, s(:lit, 1), :[], s(:lit, 2), s(:lit, 3))
 
-  def setup
-    super
-
-    self.processor = Ruby19Parser.new
+    assert_parse rb, pt
   end
@@ -731,0 +966,0 @@

No alert changes

No metric changes