ruby_parser - rubygems Package Compare versions

Comparing version 3.0.0.a5 to 3.0.0.a6

History.txt

@@ -0,1 +1,19 @@
+=== 3.0.0.a6 / 2012-08-20
+
+* 2 minor enhancements:
+
+  * 1.8: Added basic encoding support to 1.8 parser. Assumes -Ku.
+  * 1.9: Added encoding magic comment support to 1.9 parser.
+
+* 8 bug fixes:
+
+  * 1.9: Fixed lexing of -1 w/in conditionals. yeah... I dunno.
+  * 1.9: Fixed parsing of a do | | end.
+  * 1.9: Fixed parsing of not(x).
+  * 1.9: Fixed parsing of op_asgn + rescue: 'a ||= b rescue nil'
+  * 1.9: added \r to the EOL backslash handler. dos files blow
+  * 1.9: hacked in a workaround for 1.9 specific regexps running in 1.8.
+  * Added #reset to RubyParser proxy class
+  * Fixed lexing of conditional w/ %() tokens
+
 === 3.0.0.a5 / 2012-07-31
@@ -2,0 +20,0 @@

lib/ruby_lexer.rb

@@ -0,2 +1,16 @@
+# encoding: US-ASCII
+
 class RubyLexer
+
+  IDENT_CHAR_RE = case RUBY_VERSION
+                  when /^1\.8/ then
+                    /[\w\x80-\xFF]/
+                  when /^(1\.9|2\.0)/ then # HACK - matching 2.0 for now
+                    /[\w\u0080-\uFFFF]/u
+                  else
+                    raise "bork"
+                  end
+
+  IDENT_RE = /^#{IDENT_CHAR_RE}+/
+
   attr_accessor :command_start
@@ -776,3 +790,3 @@ attr_accessor :cmdarg
           return :tCOLON2
-        elsif ! is_end? && src.scan(/:([a-zA-Z_]\w*(?:[?!]|=(?!>))?)/) then
+        elsif ! is_end? && src.scan(/:([a-zA-Z_]#{IDENT_CHAR_RE}*(?:[?!]|=(?!>))?)/) then
           # scanning shortcut to symbols
@@ -902,3 +916,3 @@ self.yacc_value = src[1]
 
-          if (lex_state == :expr_beg || lex_state == :expr_mid ||
+          if (is_beg? ||
               (lex_state.is_argument && space_seen && !src.check(/\s/))) then
@@ -1149,3 +1163,3 @@ if lex_state.is_argument then
         elsif src.scan(/\\/) then
-          if src.scan(/\n/) then
+          if src.scan(/\r?\n/) then
             self.lineno = nil
@@ -1157,3 +1171,3 @@ self.space_seen = true
         elsif src.scan(/\%/) then
-          if lex_state == :expr_beg || lex_state == :expr_mid then
+          if is_beg? then
             return parse_quote
@@ -1236,3 +1250,3 @@ end
       else # alpha check
-        if src.scan(/\W/) then
+        unless src.check IDENT_RE then
           rb_compile_error "Invalid char #{src.matched.inspect} in expression"
@@ -1242,3 +1256,3 @@ end
 
-      self.token = src.matched if self.src.scan(/\w+/)
+      self.token = src.matched if self.src.scan IDENT_RE
 
@@ -1320,3 +1334,3 @@ return process_token(command_state)
 
-    token << src.matched if token =~ /^\w/ && src.scan(/[\!\?](?!=)/)
+    token << src.matched if token =~ IDENT_RE && src.scan(/[\!\?](?!=)/)
 
@@ -1323,0 +1337,0 @@ result = nil

lib/ruby_parser_extras.rb

@@ -81,3 +81,3 @@ require 'stringio'
 module RubyParserStuff
-  VERSION = '3.0.0.a5' unless constants.include? "VERSION" # SIGH
+  VERSION = '3.0.0.a6' unless constants.include? "VERSION" # SIGH
 
@@ -753,3 +753,3 @@ attr_accessor :lexer, :in_def, :in_single, :file
                   Regexp.new(node[1], o)
-                end
+                end rescue node[1] # HACK - drops options
     when :dstr then
@@ -907,4 +907,17 @@ if options =~ /o/ then
 
+    str.lines.first(2).find { |s| s[/^# encoding: (.+)/, 1] }
+    encoding = $1
+
+    str = str.dup
+
+    if encoding then
+      if defined?(Encoding) then
+        str.force_encoding(encoding).encode! "utf-8"
+      else
+        warn "Skipping magic encoding comment"
+      end
+    end
+
     self.file = file
-    self.lexer.src = str.dup
+    self.lexer.src = str
 
@@ -1205,3 +1218,3 @@ @yydebug = ENV.has_key? 'DEBUG'
 
-  def process s, f = "(string)"
+  def process(s, f = "(string)") # parens for emacs *sigh*
     Ruby19Parser.new.process s, f
@@ -1213,2 +1226,7 @@ rescue Racc::ParseError
   alias :parse :process
+
+  def reset
+    @p18.reset
+    @p19.reset
+  end
 end
@@ -1215,0 +1233,0 @@

test/test_ruby_parser.rb

 #!/usr/local/bin/ruby
+# encoding: utf-8
 
@@ -649,2 +650,9 @@ # ENV['VERBOSE'] = "1"
 
+  def test_bug_cond_pct
+    rb = "case; when %r%blahblah%; end"
+    pt = s(:case, nil, s(:when, s(:array, s(:lit, /blahblah/)), nil), nil)
+
+    assert_parse rb, pt
+  end
+
   # according to 2.3.1 parser:
@@ -661,3 +669,3 @@ # rp.process("f { |(a,b),c| }") == rp.process("f { |((a,b),c)| }")
   #              s(:lasgn, :c))))
-  # 
+  #
   #   assert_parse rb, pt.dup
@@ -695,7 +703,14 @@ # end
 
+  def ruby18
+    Ruby18Parser === self.processor
+  end
+
+  def ruby19
+    Ruby19Parser === self.processor
+  end
+
   def test_bug_comma
-    val = case self.processor
-          when Ruby18Parser then
+    val = if ruby18 then
             s(:lit, 100)
-          when Ruby19Parser then
+          elsif ruby19 then
             s(:str, "d")
@@ -751,2 +766,64 @@ else
   end
+
+
+  def test_bug_not_parens
+    rb = "not(a)"
+    pt = if ruby18 then
+           s(:not, s(:call, nil, :a))
+         elsif ruby19 then
+           s(:call, s(:call, nil, :a), :"!")
+         else
+           raise "wtf"
+         end
+
+    assert_parse rb, pt
+  end
+
+  def test_pipe_space
+    rb = "a.b do | | end"
+    pt = s(:iter, s(:call, s(:call, nil, :a), :b), 0)
+
+    assert_parse rb, pt
+  end
+
+  def test_cond_unary_minus
+    rb = "if -1; end"
+    pt = s(:if, s(:lit, -1), nil, nil)
+
+    assert_parse rb, pt
+  end
+
+  def test_bug_op_asgn_rescue
+    rb = "a ||= b rescue nil"
+    pt = s(:rescue,
+           s(:op_asgn_or, s(:lvar, :a), s(:lasgn, :a, s(:call, nil, :b))),
+           s(:resbody, s(:array), s(:nil)))
+
+    assert_parse rb, pt
+  end
+
+  def test_magic_encoding_comment
+    rb = <<-EOM.gsub(/^      /, '')
+      # encoding: utf-8
+      class ExampleUTF8ClassNameVarietà
+        def self.è
+          così = :però
+        end
+      end
+    EOM
+
+    # TODO: class vars
+    # TODO: odd-ternary: a ?bb : c
+    # TODO: globals
+
+    pt = s(:class, :"ExampleUTF8ClassNameVariet\303\240", nil,
+           s(:defs, s(:self), :"\303\250", s(:args),
+             s(:lasgn, :"cos\303\254", s(:lit, :"per\303\262"))))
+
+    err = RUBY_VERSION =~ /^1\.8/ ? "Skipping magic encoding comment\n" : ""
+
+    assert_output "", err do
+      assert_parse rb, pt
+    end
+  end
 end
@@ -1119,2 +1196,9 @@
 
+  # def test_pipe_semicolon # HACK
+  #   rb = "a.b do | ; c | end"
+  #   pt = s(:iter, s(:call, s(:call, nil, :a), :b), 0)
+  # 
+  #   assert_parse rb, pt
+  # end
+
   # HACK: need to figure out the desired structure and get this working
@@ -1121,0 +1205,0 @@ # def test_wtf

No alert changes

No metric changes