ruby_parser - rubygems Package Compare versions

Comparing version 2.0.3 to 2.0.4

History.txt

@@ -0,1 +1,14 @@
+=== 2.0.4 / 2009-08-18
+
+* 1 minor enhancement:
+
+  * Changed requires around to be more accurate.
+
+* 4 bug fixes:
+
+  * Fixed .autotest for minitest
+  * Fixed emacs escape lexing bug: "\C-\\" (maglev/gemstone)
+  * Fixed octal lexing edgecases. (maglev/gemstone)
+  * Fixed regexp lexing edgecases. (maglev/gemstone)
+
 === 2.0.3 / 2009-06-23
@@ -2,0 +15,0 @@

lib/ruby_lexer.rb

@@ -1,5 +0,1 @@
-$: << File.expand_path("~/Work/p4/zss/src/ParseTree/dev/lib") # for me, not you.
-require 'sexp'
-require 'ruby_parser_extras'
-
 class RubyLexer
@@ -11,3 +7,3 @@ attr_accessor :command_start
 
-  ESC_RE = /\\([0-7]{1,3}|x[0-9a-fA-F]{1,2}|M-.|(C-|c)\?|(C-|c).|[^0-7xMCc])/
+  ESC_RE = /\\([0-7]{1,3}|x[0-9a-fA-F]{1,2}|M-[^\\]|(C-|c)[^\\]|[^0-7xMCc])/
 
@@ -252,5 +248,5 @@ # Additional context surrounding tokens that both the lexer and
       int_with_base(10)
-    when src.scan(/[+-]?0o?[0-7_]*[89]/) then
+    when src.scan(/[+-]?0[Oo]?[0-7_]*[89]/) then
       rb_compile_error "Illegal octal digit."
-    when src.scan(/[+-]?0o?[0-7_]+|0o/) then
+    when src.scan(/[+-]?0[Oo]?[0-7_]+|0[Oo]/) then
       int_with_base(8)
@@ -416,3 +412,4 @@ when src.scan(/[+-]?[\d_]+_(e|\.)/) then
       src[1].to_i(16).chr
-    when src.scan(/M-\\/) then
+    when src.check(/M-\\[\\MCc]/) then
+      src.scan(/M-\\/) # eat it
       c = self.read_escape
@@ -425,3 +422,4 @@ c[0] = (c[0].ord | 0x80).chr
       c
-    when src.scan(/C-\\|c\\/) then
+    when src.check(/(C-|c)\\[\\MCc]/) then
+      src.scan(/(C-|c)\\/) # eat it
       c = self.read_escape
@@ -431,3 +429,3 @@ c[0] = (c[0].ord & 0x9f).chr
     when src.scan(/C-\?|c\?/) then
-      0177.chr
+      127.chr
     when src.scan(/(C-|c)(.)/) then
@@ -594,4 +592,4 @@ c = src[2]
       "\n"   => "",
-      "C-\?" => 0177.chr,
-      "c\?"  => 0177.chr,
+      "C-\?" => 127.chr,
+      "c\?"  => 127.chr,
     }[s]
@@ -598,0 +596,0 @@

lib/ruby_parser_extras.rb

@@ -118,3 +118,3 @@ require 'stringio'
 class RubyParser < Racc::Parser
-  VERSION = '2.0.3' unless constants.include? "VERSION" # SIGH
+  VERSION = '2.0.4' unless constants.include? "VERSION" # SIGH
 
@@ -595,3 +595,3 @@ attr_accessor :lexer, :in_def, :in_single, :file
     o, k = 0, nil
-    options.split(//).each do |c| # FIX: this has a better home
+    options.split(//).uniq.each do |c| # FIX: this has a better home
       v = {
@@ -598,0 +598,0 @@ 'x' => Regexp::EXTENDED,

Rakefile

@@ -17,4 +17,4 @@ # -*- ruby -*-
 
-  extra_dev_deps << 'ParseTree'
-  extra_deps     << ['sexp_processor', '>= 3.0.1']
+  extra_dev_deps << ['ParseTree',      '~> 3.0']
+  extra_deps     << ['sexp_processor', '~> 3.0']
 end
@@ -21,0 +21,0 @@

test/test_ruby_lexer.rb

 #!/usr/local/bin/ruby
 
-require "minitest/autorun"
-require "ruby_lexer"
+require 'rubygems'
+require 'minitest/autorun'
+require 'ruby_lexer'
+require 'ruby_parser'
 
@@ -833,2 +835,18 @@ class TestRubyLexer < MiniTest::Unit::TestCase
 
+  def test_yylex_integer_oct_O
+    util_lex_token "0O52", :tINTEGER, 42
+  end
+
+  def test_yylex_integer_oct_O_bad_range
+    util_bad_token "0O8"
+  end
+
+  def test_yylex_integer_oct_O_bad_underscores
+    util_bad_token "0O1__23"
+  end
+
+  def test_yylex_integer_oct_O_not_bad_none
+    util_lex_token "0O ", :tINTEGER, 0
+  end
+
   def test_yylex_integer_oct_o
@@ -1284,2 +1302,6 @@ util_lex_token "0o52", :tINTEGER, 42
 
+  def test_yylex_regexp_escape_hex_bad
+    util_bad_token '/regex\\xzxp/', :tREGEXP_BEG, "/"
+  end
+
   def test_yylex_regexp_escape_hex_one
@@ -1292,6 +1314,2 @@ util_lex_token('/^[\\xd\\xa]{2}/on',
 
-  def test_yylex_regexp_escape_hex_bad
-    util_bad_token '/regex\\xzxp/', :tREGEXP_BEG, "/"
-  end
-
   def test_yylex_regexp_escape_oct1
@@ -1427,22 +1445,45 @@ util_lex_token('/regex\\0xp/',
 
-  def test_yylex_string_double_escape_M
-    util_lex_token('"\\M-g"',
-                   :tSTRING, "\347")
+  def test_yylex_string_double_escape_C
+    util_lex_token('"\\C-a"',
+                   :tSTRING, "\001")
   end
 
-  def test_yylex_string_escape_x_single
-    util_lex_token('"\\x0"',
-                   :tSTRING, "\000")
+  def test_yylex_string_double_escape_C_backslash
+    util_lex_token('"\\C-\\\\"',
+                   :tSTRING_BEG, "\"",
+                   :tSTRING_CONTENT, "\034",
+                   :tSTRING_END, "\"")
   end
 
-  def test_yylex_string_double_escape_chars
-    util_lex_token('"s\\tri\\ng"',
-                   :tSTRING, "s\tri\ng")
+  def test_yylex_string_double_escape_C_escape
+    util_lex_token('"\\C-\\M-a"',
+                   :tSTRING_BEG, "\"",
+                   :tSTRING_CONTENT, "\201",
+                   :tSTRING_END, "\"")
   end
 
-  def test_yylex_string_double_escape_hex
-    util_lex_token('"n = \\x61\\x62\\x63"',
-                   :tSTRING, "n = abc")
+  def test_yylex_string_double_escape_C_question
+    util_lex_token('"\\C-?"',
+                   :tSTRING, "\177")
   end
 
+  def test_yylex_string_double_escape_M
+    util_lex_token('"\\M-a"',
+                   :tSTRING, "\341")
+  end
+
+  def test_yylex_string_double_escape_M_backslash
+    util_lex_token('"\\M-\\\\"',
+                   :tSTRING_BEG, "\"",
+                   :tSTRING_CONTENT, "\334",
+                   :tSTRING_END, "\"")
+  end
+
+  def test_yylex_string_double_escape_M_escape
+    util_lex_token('"\\M-\\C-a"',
+                   :tSTRING_BEG, "\"",
+                   :tSTRING_CONTENT, "\201",
+                   :tSTRING_END, "\"")
+  end
+
   def test_yylex_string_double_escape_bs1
@@ -1458,2 +1499,36 @@ util_lex_token('"a\\a\\a"',
 
+  def test_yylex_string_double_escape_c
+    util_lex_token('"\\ca"',
+                   :tSTRING, "\001")
+  end
+
+  def test_yylex_string_double_escape_c_backslash
+    util_lex_token('"\\c\\"',
+                   :tSTRING_BEG, "\"",
+                   :tSTRING_CONTENT, "\034",
+                   :tSTRING_END, "\"")
+  end
+
+  def test_yylex_string_double_escape_c_escape
+    util_lex_token('"\\c\\M-a"',
+                   :tSTRING_BEG, "\"",
+                   :tSTRING_CONTENT, "\201",
+                   :tSTRING_END, "\"")
+  end
+
+  def test_yylex_string_double_escape_c_question
+    util_lex_token('"\\c?"',
+                   :tSTRING, "\177")
+  end
+
+  def test_yylex_string_double_escape_chars
+    util_lex_token('"s\\tri\\ng"',
+                   :tSTRING, "s\tri\ng")
+  end
+
+  def test_yylex_string_double_escape_hex
+    util_lex_token('"n = \\x61\\x62\\x63"',
+                   :tSTRING, "n = abc")
+  end
+
   def test_yylex_string_double_escape_octal
@@ -1492,2 +1567,7 @@ util_lex_token('"n = \\101\\102\\103"',
 
+  def test_yylex_string_escape_x_single
+    util_lex_token('"\\x0"',
+                   :tSTRING, "\000")
+  end
+
   def test_yylex_string_pct_Q
@@ -1565,2 +1645,10 @@ util_lex_token("%Q[s1 s2]",
 
+  def test_yylex_string_pct_w_tab
+    util_lex_token("%w[abc\tdef]",
+                   :tAWORDS_BEG,      "%w[",
+                   :tSTRING_CONTENT, "abc\tdef",
+                   :tSPACE,              nil,
+                   :tSTRING_END,     nil)
+  end
+
   def test_yylex_string_single
@@ -1567,0 +1655,0 @@ util_lex_token("'string'",

test/test_ruby_parser_extras.rb

@@ -0,1 +1,2 @@
+require 'rubygems'
 require 'minitest/autorun'
@@ -2,0 +3,0 @@ require 'ruby_parser_extras'

test/test_ruby_parser.rb

 #!/usr/local/bin/ruby
 
+require 'rubygems'
 require 'minitest/autorun'
@@ -367,2 +368,18 @@ require 'ruby_parser'
 
+  def test_regexp
+    regexps = {
+      "/wtf/" => /wtf/,
+      "/wtf/n" => /wtf/n,
+      "/wtf/m" => /wtf/m,
+      "/wtf/nm" => /wtf/nm,
+      "/wtf/nmnmnmnm" => /wtf/nm,
+    }
+
+    regexps.each do |rb, lit|
+      assert_equal s(:lit, lit), @processor.parse(rb)
+    end
+
+    # TODO: add more including interpolation etc
+  end
+
   def test_str_pct_Q_nested
@@ -369,0 +386,0 @@ rb = "%Q[before [#\{nest}] after]"

No alert changes

No metric changes