ruby_parser - rubygems Package Compare versions

Comparing version 1.0.0 to 2.0.0

lib/ruby_parser_extras.rb

+require 'stringio'
+require 'racc/parser'
+require 'sexp'
+require 'strscan'
+
+# WHY do I have to do this?!?
+class Regexp
+  ONCE     = 0 # 16 # ?
+  ENC_NONE = /x/n.options
+  ENC_EUC  = /x/e.options
+  ENC_SJIS = /x/s.options
+  ENC_UTF8 = /x/u.options
+end
+
+# I hate ruby 1.9 string changes
+class Fixnum
+  def ord
+    self
+  end
+end unless "a"[0] == "a"
+
+class RPStringScanner < StringScanner
+#   if ENV['TALLY'] then
+#     alias :old_getch :getch
+#     def getch
+#       warn({:getch => caller[0]}.inspect)
+#       old_getch
+#     end
+#   end
+
+  def current_line # HAHA fuck you (HACK)
+    string[0..pos][/\A.*__LINE__/m].split(/\n/).size
+  end
+
+  def lineno
+    string[0...pos].count("\n") + 1
+  end
+
+  # TODO: once we get rid of these, we can make things like
+  # TODO: current_line and lineno much more accurate and easy to do
+  def unread c # TODO: remove this entirely - we should not need it
+    return if c.nil? # UGH
+    warn({:unread => caller[0]}.inspect) if ENV['TALLY']
+    string[pos, 0] = c
+  end
+
+  def unread_many str # TODO: remove this entirely - we should not need it
+    warn({:unread_many => caller[0]}.inspect) if ENV['TALLY']
+    string[pos, 0] = str
+  end
+
+  def begin_of_line?
+    pos == 0 or string[pos-1] == ?\n
+  end
+
+  def was_begin_of_line # TODO: kill me
+    pos <= 2 or string[pos-2] == ?\n
+  end
+
+  if ENV['DEBUG'] then
+    alias :old_getch :getch
+    def getch
+      c = self.old_getch
+      p :getch => [c, caller.first]
+      c
+    end
+
+    alias :old_scan :scan
+    def scan re
+      s = old_scan re
+      p :scan => [s, caller.first] if s
+      s
+    end
+  end
+
+  # TODO:
+  # def last_line(src)
+  #   if n = src.rindex("\n")
+  #     src[(n+1) .. -1]
+  #   else
+  #     src
+  #   end
+  # end
+  # private :last_line
+
+  # def next_words_on_error
+  #   if n = @src.rest.index("\n")
+  #     @src.rest[0 .. (n-1)]
+  #   else
+  #     @src.rest
+  #   end
+  # end
+
+  # def prev_words_on_error(ev)
+  #   pre = @pre
+  #   if ev and /#{Regexp.quote(ev)}$/ =~ pre
+  #     pre = $`
+  #   end
+  #   last_line(pre)
+  # end
+
+  # def on_error(et, ev, values)
+  #   lines_of_rest = @src.rest.to_a.length
+  #   prev_words = prev_words_on_error(ev)
+  #   at = 4 + prev_words.length
+  #   message = <<-MSG
+  # RD syntax error: line #{@blockp.line_index - lines_of_rest}:
+  # ...#{prev_words} #{(ev||'')} #{next_words_on_error()} ...
+  #   MSG
+  #   message << " " * at + "^" * (ev ? ev.length : 0) + "\n"
+  #   raise ParseError, message
+  # end
+end
+
+class RubyParser < Racc::Parser
+  VERSION = '2.0.0'
+
+  attr_accessor :lexer, :in_def, :in_single, :file
+  attr_reader :env, :comments
+
+  def append_to_block head, tail # FIX: wtf is this?!? switch to block_append
+    return head if tail.nil?
+    return tail if head.nil?
+
+    head = s(:block, head) unless head.node_type == :block
+    head << tail
+    head
+  end
+
+  def arg_add(node1, node2) # TODO: nuke
+    return s(:arglist, node2) unless node1
+
+    node1[0] = :arglist if node1[0] == :array
+    return node1 << node2 if node1[0] == :arglist
+
+    return s(:arglist, node1, node2)
+  end
+
+  def arg_blk_pass node1, node2 # TODO: nuke
+    node1 = s(:arglist, node1) unless [:arglist, :array].include? node1.first
+    node1 << node2 if node2
+    node1
+  end
+
+  def arg_concat node1, node2 # TODO: nuke
+    raise "huh" unless node2
+    node1 << s(:splat, node2).compact
+    node1
+  end
+
+  def args arg, optarg, rest_arg, block_arg
+    arg ||= s(:args)
+
+    result = arg
+    if optarg then
+      optarg[1..-1].each do |lasgn| # FIX clean sexp iter
+        raise "wtf? #{lasgn.inspect}" unless lasgn[0] == :lasgn
+        result << lasgn[1]
+      end
+    end
+
+    result << rest_arg  if rest_arg
+    result << :"&#{block_arg.last}" if block_arg
+    result << optarg    if optarg # TODO? huh - processed above as well
+
+    result
+  end
+
+  def aryset receiver, index
+    index[0] = :arglist if index[0] == :array
+    s(:attrasgn, receiver, :"[]=", index)
+  end
+
+  def assignable(lhs, value = nil)
+    id = lhs.to_sym
+    id = id.to_sym if Sexp === id
+
+    raise SyntaxError, "Can't change the value of #{id}" if
+      id.to_s =~ /^(?:self|nil|true|false|__LINE__|__FILE__)$/
+
+    result = case id.to_s
+             when /^@@/ then
+               asgn = in_def || in_single > 0
+               s((asgn ? :cvasgn : :cvdecl), id)
+             when /^@/ then
+               s(:iasgn, id)
+             when /^\$/ then
+               s(:gasgn, id)
+             when /^[A-Z]/ then
+               s(:cdecl, id)
+             else
+               case self.env[id]
+               when :lvar then
+                 s(:lasgn, id)
+               when :dvar, nil then
+                 if self.env.current[id] == :dvar then
+                   s(:lasgn, id)
+                 elsif self.env[id] == :dvar then
+                   self.env.use(id)
+                   s(:lasgn, id)
+                 elsif ! self.env.dynamic? then
+                   s(:lasgn, id)
+                 else
+                   s(:lasgn, id)
+                 end
+               else
+                 raise "wtf? unknown type: #{self.env[id]}"
+               end
+             end
+
+    self.env[id] ||= :lvar
+
+    result << value if value
+
+    return result
+  end
+
+  def block_append(head, tail, strip_tail_block=false)
+    return head unless tail
+    return tail unless head
+
+    case head[0]
+    when :lit, :str then
+      return tail
+    end
+
+    line = [head.line, tail.line].compact.min
+
+    head = remove_begin(head)
+    head = s(:block, head) unless head[0] == :block
+
+    if strip_tail_block and Sexp === tail and tail[0] == :block then
+      head.push(*tail.values)
+    else
+      head << tail
+    end
+
+    head.line = line
+    head
+  end
+
+  def cond node
+    return nil if node.nil?
+    node = value_expr node
+
+    case node.first
+    when :dregex then
+      return s(:match2, node, s(:gvar, "$_".to_sym))
+    when :regex then
+      return s(:match, node)
+    when :lit then
+      if Regexp === node.last then
+        return s(:match, node)
+      else
+        return node
+      end
+    when :and then
+      return s(:and, cond(node[1]), cond(node[2]))
+    when :or then
+      return s(:or,  cond(node[1]), cond(node[2]))
+    when :dot2 then
+      label = "flip#{node.hash}"
+      env[label] = :lvar
+      return s(:flip2, node[1], node[2])
+    when :dot3 then
+      label = "flip#{node.hash}"
+      env[label] = :lvar
+      return s(:flip3, node[1], node[2])
+    else
+      return node
+    end
+  end
+
+  ##
+  # for pure ruby systems only
+
+  def do_parse
+    _racc_do_parse_rb(_racc_setup, false)
+  end if ENV['PURE_RUBY']
+
+  def get_match_node lhs, rhs # TODO: rename to new_match
+    if lhs then
+      case lhs[0]
+      when :dregx, :dregx_once then
+        return s(:match2, lhs, rhs).line(lhs.line)
+      when :lit then
+        return s(:match2, lhs, rhs).line(lhs.line) if Regexp === lhs.last
+      end
+    end
+
+    if rhs then
+      case rhs[0]
+      when :dregx, :dregx_once then
+        return s(:match3, rhs, lhs).line(lhs.line)
+      when :lit then
+        return s(:match3, rhs, lhs).line(lhs.line) if Regexp === rhs.last
+      end
+    end
+
+    return s(:call, lhs, :"=~", s(:arglist, rhs)).line(lhs.line)
+  end
+
+  def gettable(id)
+    raise "no: #{id.inspect}" if Sexp === id
+    id = id.to_sym if Sexp   === id # HACK
+    id = id.to_sym if String === id # HACK
+
+    return s(:self)                  if id == :self
+    return s(:nil)                   if id == :nil
+    return s(:true)                  if id == :true
+    return s(:false)                 if id == :false
+    return s(:str, self.file)        if id == :"__FILE__"
+    return s(:lit, lexer.src.current_line) if id == :"__LINE__"
+
+    result = case id.to_s
+             when /^@@/ then
+               s(:cvar, id)
+             when /^@/ then
+               s(:ivar, id)
+             when /^\$/ then
+               s(:gvar, id)
+             when /^[A-Z]/ then
+               s(:const, id)
+             else
+               type = env[id]
+               if type then
+                 s(type, id)
+               elsif env.dynamic? and :dvar == env[id] then
+                 s(:lvar, id)
+               else
+                 s(:call, nil, id, s(:arglist))
+               end
+             end
+
+    return result if result
+
+    raise "identifier #{id.inspect} is not valid"
+  end
+
+  def initialize
+    super
+    self.lexer = RubyLexer.new
+    self.lexer.parser = self
+    @env = Environment.new
+    @comments = []
+
+    self.reset
+  end
+
+  def list_append list, item # TODO: nuke me *sigh*
+    return s(:array, item) unless list
+    list = s(:array, list) unless Sexp === list && list.first == :array
+    list << item
+  end
+
+  def list_prepend item, list # TODO: nuke me *sigh*
+    list = s(:array, list) unless Sexp === list && list[0] == :array
+    list.insert 1, item
+    list
+  end
+
+  def literal_concat head, tail
+    return tail unless head
+    return head unless tail
+
+    htype, ttype = head[0], tail[0]
+
+    head = s(:dstr, '', head) if htype == :evstr
+
+    case ttype
+    when :str then
+      if htype == :str
+        head[-1] << tail[-1]
+      elsif htype == :dstr and head.size == 2 then
+        head[-1] << tail[-1]
+      else
+        head << tail
+      end
+    when :dstr then
+      if htype == :str then
+        tail[1] = head[-1] + tail[1]
+        head = tail
+      else
+        tail[0] = :array
+        tail[1] = s(:str, tail[1])
+        tail.delete_at 1 if tail[1] == s(:str, '')
+
+        head.push(*tail[1..-1])
+      end
+    when :evstr then
+      head[0] = :dstr if htype == :str
+      if head.size == 2 and tail.size > 1 and tail[1][0] == :str then
+        head[-1] << tail[1][-1]
+        head[0] = :str if head.size == 2 # HACK ?
+      else
+        head.push(tail)
+      end
+    else
+      x = [head, tail]
+      raise "unknown type: #{x.inspect}"
+    end
+
+    return head
+  end
+
+  def logop(type, left, right) # TODO: rename logical_op
+    left = value_expr left
+
+    if left and left[0] == type and not left.paren then
+      node, second = left, nil
+
+      while (second = node[2]) && second[0] == type and not second.paren do
+        node = second
+      end
+
+      node[2] = s(type, second, right)
+
+      return left
+    end
+
+    return s(type, left, right)
+  end
+
+  def new_aref val
+    val[2] ||= s(:arglist)
+    val[2][0] = :arglist if val[2][0] == :array # REFACTOR
+    if val[0].node_type == :self then
+      result = new_call nil, :"[]", val[2]
+    else
+      result = new_call val[0], :"[]", val[2]
+    end
+    result
+  end
+
+  def new_body val
+    result = val[0]
+
+    if val[1] then
+      result = s(:rescue)
+      result << val[0] if val[0]
+
+      resbody = val[1]
+
+      while resbody do
+        result << resbody
+        resbody = resbody.resbody(true)
+      end
+
+      result << val[2] if val[2]
+
+      result.line = (val[0] || val[1]).line
+    elsif not val[2].nil? then
+      warning("else without rescue is useless")
+      result = block_append(result, val[2])
+    end
+
+    result = s(:ensure, result, val[3]).compact if val[3]
+    return result
+  end
+
+  def new_call recv, meth, args = nil
+    result = s(:call, recv, meth)
+    result.line = recv.line if recv
+
+    args ||= s(:arglist)
+    args[0] = :arglist if args.first == :array
+    args = s(:arglist, args) unless args.first == :arglist
+    result << args
+    result
+  end
+
+  def new_case expr, body
+    result = s(:case, expr)
+    line = (expr || body).line
+
+    while body and body.node_type == :when
+      result << body
+      body = body.delete_at 3
+    end
+
+    # else
+    body = nil if body == s(:block)
+    result << body
+
+    result.line = line
+    result
+  end
+
+  def new_class val
+    line, path, superclass, body = val[1], val[2], val[3], val[5]
+    scope = s(:scope, body).compact
+    result = s(:class, path, superclass, scope)
+    result.line = line
+    result.comments = self.comments.pop
+    result
+  end
+
+  def new_compstmt val
+    result = void_stmts(val[0])
+    result = remove_begin(result) if result
+    result
+  end
+
+  def new_defn val
+    line, name, args, body = val[2], val[1], val[3], val[4]
+    body ||= s(:nil)
+
+    body ||= s(:block)
+    body = s(:block, body) unless body.first == :block
+
+    result = s(:defn, name.to_sym, args, s(:scope, body))
+    result.line = line
+    result.comments = self.comments.pop
+    result
+  end
+
+  def new_defs val
+    recv, name, args, body = val[1], val[4], val[6], val[7]
+
+    body ||= s(:block)
+    body = s(:block, body) unless body.first == :block
+
+    result = s(:defs, recv, name.to_sym, args, s(:scope, body))
+    result.line = recv.line
+    result.comments = self.comments.pop
+    result
+  end
+
+  def new_for expr, var, body
+    result = s(:for, expr, var).line(var.line)
+    result << body if body
+    result
+  end
+
+  def new_if c, t, f
+    l = [c.line, t && t.line, f && f.line].compact.min
+    c = cond c
+    c, t, f = c.last, f, t if c[0] == :not
+    s(:if, c, t, f).line(l)
+  end
+
+  def new_iter call, args, body
+    result = s(:iter)
+    result << call if call
+    result << args
+    result << body if body
+    result
+  end
+
+  def new_masgn lhs, rhs, wrap = false
+    rhs = value_expr(rhs)
+    rhs = lhs[1] ? s(:to_ary, rhs) : s(:array, rhs) if wrap
+
+    lhs.delete_at 1 if lhs[1].nil?
+    lhs << rhs
+
+    lhs
+  end
+
+  def new_module val
+    line, path, body = val[1], val[2], val[4]
+    body = s(:scope, body).compact
+    result = s(:module, path, body)
+    result.line = line
+    result.comments = self.comments.pop
+    result
+  end
+
+  def new_op_asgn val
+    lhs, asgn_op, arg = val[0], val[1].to_sym, val[2]
+    name = lhs.value
+    arg = remove_begin(arg)
+    result = case asgn_op # REFACTOR
+             when :"||" then
+               lhs << arg
+               s(:op_asgn_or, self.gettable(name), lhs)
+             when :"&&" then
+               lhs << arg
+               s(:op_asgn_and, self.gettable(name), lhs)
+             else
+               # TODO: why [2] ?
+               lhs[2] = new_call(self.gettable(name), asgn_op,
+                                 s(:arglist, arg))
+               lhs
+             end
+    result.line = lhs.line
+    result
+  end
+
+  def new_regexp val
+    node = val[1] || s(:str, '')
+    options = val[2]
+
+    o, k = 0, nil
+    options.split(//).each do |c| # FIX: this has a better home
+      v = {
+        'x' => Regexp::EXTENDED,
+        'i' => Regexp::IGNORECASE,
+        'm' => Regexp::MULTILINE,
+        'o' => Regexp::ONCE,
+        'n' => Regexp::ENC_NONE,
+        'e' => Regexp::ENC_EUC,
+        's' => Regexp::ENC_SJIS,
+        'u' => Regexp::ENC_UTF8,
+      }[c]
+      raise "unknown regexp option: #{c}" unless v
+      o += v
+      k = c if c =~ /[esu]/
+    end
+
+    case node[0]
+    when :str then
+      node[0] = :lit
+      node[1] = if k then
+                  Regexp.new(node[1], o, k)
+                else
+                  Regexp.new(node[1], o)
+                end
+    when :dstr then
+      if options =~ /o/ then
+        node[0] = :dregx_once
+      else
+        node[0] = :dregx
+      end
+      node << o if o and o != 0
+    else
+      node = s(:dregx, '', node);
+      node[0] = :dregx_once if options =~ /o/
+      node << o if o and o != 0
+    end
+
+    node
+  end
+
+  def new_sclass val
+    recv, in_def, in_single, body = val[3], val[4], val[6], val[7]
+    scope = s(:scope, body).compact
+    result = s(:sclass, recv, scope)
+    result.line = val[2]
+    self.in_def = in_def
+    self.in_single = in_single
+    result
+  end
+
+  def new_super args
+    if args && args.node_type == :block_pass then
+      s(:super, args)
+    else
+      args ||= s(:arglist)
+      s(:super, *args[1..-1])
+    end
+  end
+
+  def new_undef n, m = nil
+    if m then
+      block_append(n, s(:undef, m))
+    else
+      s(:undef, n)
+    end
+  end
+
+  def new_until block, expr, pre
+    expr = (expr.first == :not ? expr.last : s(:not, expr)).line(expr.line)
+    new_while block, expr, pre
+  end
+
+  def new_while block, expr, pre
+    line = [block && block.line, expr.line].compact.min
+    block, pre = block.last, false if block && block[0] == :begin
+
+    expr = cond expr
+    result = if expr.first == :not then
+               s(:until, expr.last, block, pre)
+             else
+               s(:while, expr, block, pre)
+             end
+
+    result.line = line
+    result
+  end
+
+  def new_xstring str
+    if str then
+      case str[0]
+      when :str
+        str[0] = :xstr
+      when :dstr
+        str[0] = :dxstr
+      else
+        str = s(:dxstr, '', str)
+      end
+      str
+    else
+      s(:xstr, '')
+    end
+  end
+
+  def new_yield args = nil
+    raise SyntaxError, "Block argument should not be given." if
+      args && args.node_type == :block_pass
+
+    args ||= s(:arglist)
+    args = s(:arglist, args) unless [:arglist, :array].include? args.first
+
+    return s(:yield, *args[1..-1])
+  end
+
+  def next_token
+    if self.lexer.advance then
+      return self.lexer.token, self.lexer.yacc_value
+    else
+      return [false, '$end']
+    end
+  end
+
+  def node_assign(lhs, rhs) # TODO: rename new_assign
+    return nil unless lhs
+
+    rhs = value_expr rhs
+
+    case lhs[0]
+    when :gasgn, :iasgn, :lasgn, :dasgn, :dasgn_curr,
+      :masgn, :cdecl, :cvdecl, :cvasgn then
+      lhs << rhs
+    when :attrasgn, :call then
+      args = lhs.pop unless Symbol === lhs.last
+      lhs << arg_add(args, rhs)
+    when :const then
+      lhs[0] = :cdecl
+      lhs << rhs
+    else
+      raise "unknown lhs #{lhs.inspect}"
+    end
+
+    lhs
+  end
+
+  def process(str, file = "(string)")
+    raise "bad val: #{str.inspect}" unless String === str
+
+    self.file = file
+    self.lexer.src = str
+
+    @yydebug = ENV.has_key? 'DEBUG'
+
+    do_parse
+  end
+  alias :parse :process
+
+  def remove_begin node
+    oldnode = node
+    if node and :begin == node[0] and node.size == 2 then
+      node = node[-1]
+      node.line = oldnode.line
+    end
+    node
+  end
+
+  def reset
+    lexer.reset
+    self.in_def = false
+    self.in_single = 0
+    self.env.reset
+    self.comments.clear
+  end
+
+  def ret_args node
+    if node then
+      raise SyntaxError, "block argument should not be given" if
+        node[0] == :block_pass
+
+      node = node.last if node[0] == :array && node.size == 2
+      # HACK matz wraps ONE of the FOUR splats in a newline to
+      # distinguish. I use paren for now. ugh
+      node = s(:svalue, node) if node[0] == :splat and not node.paren
+    end
+
+    node
+  end
+
+  def s(*args)
+    result = Sexp.new(*args)
+    result.line ||= lexer.lineno if lexer.src          # otherwise...
+    result.file = self.file
+    result
+  end
+
+  def value_expr oldnode # HACK
+    node = remove_begin oldnode
+    node.line = oldnode.line if oldnode
+    node[2] = value_expr(node[2]) if node and node[0] == :if
+    node
+  end
+
+  def void_stmts node
+    return nil unless node
+    return node unless node[0] == :block
+
+    node[1..-1] = node[1..-1].map { |n| remove_begin(n) }
+    node
+  end
+
+  def warning s
+    # do nothing for now
+  end
+
+  alias :old_yyerror :yyerror
+  def yyerror msg
+    # for now do nothing with the msg
+    old_yyerror
+  end
+end
+
+class Keyword
+  class KWtable
+    attr_accessor :name, :state, :id0, :id1
+    def initialize(name, id=[], state=nil)
+      @name  = name
+      @id0, @id1 = id
+      @state = state
+    end
+  end
+
+  ##
+  # :expr_beg    = ignore newline, +/- is a sign.
+  # :expr_end    = newline significant, +/- is a operator.
+  # :expr_arg    = newline significant, +/- is a operator.
+  # :expr_cmdarg = newline significant, +/- is a operator.
+  # :expr_endarg = newline significant, +/- is a operator.
+  # :expr_mid    = newline significant, +/- is a operator.
+  # :expr_fname  = ignore newline, no reserved words.
+  # :expr_dot    = right after . or ::, no reserved words.
+  # :expr_class  = immediate after class, no here document.
+
+  wordlist = [
+              ["end",      [:kEND,      :kEND        ], :expr_end   ],
+              ["else",     [:kELSE,     :kELSE       ], :expr_beg   ],
+              ["case",     [:kCASE,     :kCASE       ], :expr_beg   ],
+              ["ensure",   [:kENSURE,   :kENSURE     ], :expr_beg   ],
+              ["module",   [:kMODULE,   :kMODULE     ], :expr_beg   ],
+              ["elsif",    [:kELSIF,    :kELSIF      ], :expr_beg   ],
+              ["def",      [:kDEF,      :kDEF        ], :expr_fname ],
+              ["rescue",   [:kRESCUE,   :kRESCUE_MOD ], :expr_mid   ],
+              ["not",      [:kNOT,      :kNOT        ], :expr_beg   ],
+              ["then",     [:kTHEN,     :kTHEN       ], :expr_beg   ],
+              ["yield",    [:kYIELD,    :kYIELD      ], :expr_arg   ],
+              ["for",      [:kFOR,      :kFOR        ], :expr_beg   ],
+              ["self",     [:kSELF,     :kSELF       ], :expr_end   ],
+              ["false",    [:kFALSE,    :kFALSE      ], :expr_end   ],
+              ["retry",    [:kRETRY,    :kRETRY      ], :expr_end   ],
+              ["return",   [:kRETURN,   :kRETURN     ], :expr_mid   ],
+              ["true",     [:kTRUE,     :kTRUE       ], :expr_end   ],
+              ["if",       [:kIF,       :kIF_MOD     ], :expr_beg   ],
+              ["defined?", [:kDEFINED,  :kDEFINED    ], :expr_arg   ],
+              ["super",    [:kSUPER,    :kSUPER      ], :expr_arg   ],
+              ["undef",    [:kUNDEF,    :kUNDEF      ], :expr_fname ],
+              ["break",    [:kBREAK,    :kBREAK      ], :expr_mid   ],
+              ["in",       [:kIN,       :kIN         ], :expr_beg   ],
+              ["do",       [:kDO,       :kDO         ], :expr_beg   ],
+              ["nil",      [:kNIL,      :kNIL        ], :expr_end   ],
+              ["until",    [:kUNTIL,    :kUNTIL_MOD  ], :expr_beg   ],
+              ["unless",   [:kUNLESS,   :kUNLESS_MOD ], :expr_beg   ],
+              ["or",       [:kOR,       :kOR         ], :expr_beg   ],
+              ["next",     [:kNEXT,     :kNEXT       ], :expr_mid   ],
+              ["when",     [:kWHEN,     :kWHEN       ], :expr_beg   ],
+              ["redo",     [:kREDO,     :kREDO       ], :expr_end   ],
+              ["and",      [:kAND,      :kAND        ], :expr_beg   ],
+              ["begin",    [:kBEGIN,    :kBEGIN      ], :expr_beg   ],
+              ["__LINE__", [:k__LINE__, :k__LINE__   ], :expr_end   ],
+              ["class",    [:kCLASS,    :kCLASS      ], :expr_class ],
+              ["__FILE__", [:k__FILE__, :k__FILE__   ], :expr_end   ],
+              ["END",      [:klEND,     :klEND       ], :expr_end   ],
+              ["BEGIN",    [:klBEGIN,   :klBEGIN     ], :expr_end   ],
+              ["while",    [:kWHILE,    :kWHILE_MOD  ], :expr_beg   ],
+              ["alias",    [:kALIAS,    :kALIAS      ], :expr_fname ],
+             ].map { |args| KWtable.new(*args) }
+
+  WORDLIST = Hash[*wordlist.map { |o| [o.name, o] }.flatten]
+
+  def self.keyword str
+    WORDLIST[str]
+  end
+end
+
+class Environment
+  attr_reader :env, :dyn
+
+  def [] k
+    self.all[k]
+  end
+
+  def []= k, v
+    raise "no" if v == true
+    self.current[k] = v
+  end
+
+  def all
+    idx = @dyn.index false
+    @env[0..idx].reverse.inject { |env, scope| env.merge scope }
+  end
+
+  def current
+    @env.first
+  end
+
+  def dynamic
+    idx = @dyn.index false
+    @env[0...idx].reverse.inject { |env, scope| env.merge scope } || {}
+  end
+
+  def dynamic?
+    @dyn[0] != false
+  end
+
+  def extend dyn = false
+    @dyn.unshift dyn
+    @env.unshift({})
+    @use.unshift({})
+   end
+
+  def initialize dyn = false
+    @dyn = []
+    @env = []
+    @use = []
+    self.reset
+  end
+
+  def reset
+    @dyn.clear
+    @env.clear
+    @use.clear
+    self.extend
+  end
+
+  def unextend
+    @dyn.shift
+    @env.shift
+    @use.shift
+    raise "You went too far unextending env" if @env.empty?
+  end
+
+  def use id
+    @env.each_with_index do |env, i|
+      if env[id] then
+        @use[i][id] = true
+      end
+    end
+  end
+
+  def used? id
+    idx = @dyn.index false # REFACTOR
+    u = @use[0...idx].reverse.inject { |env, scope| env.merge scope } || {}
+    u[id]
+  end
+end
+
+class StackState
+  attr_reader :stack
+
+  def initialize(name)
+    @name = name
+    @stack = [false]
+  end
+
+  def inspect
+    "StackState(#{@name}, #{@stack.inspect})"
+  end
+
+  def is_in_state
+    @stack.last
+  end
+
+  def lexpop
+    raise if @stack.size == 0
+    a = @stack.pop
+    b = @stack.pop
+    @stack.push(a || b)
+  end
+
+  def pop
+    r = @stack.pop
+    @stack.push false if @stack.size == 0
+    r
+  end
+
+  def push val
+    @stack.push val
+  end
+end
+
+############################################################
+# HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK
+
+class Symbol
+  def is_argument # TODO: phase this out
+    return self == :expr_arg || self == :expr_cmdarg
+  end
+end
+
+class Sexp
+  attr_writer :paren
+  attr_accessor :comments
+  attr_accessor :file
+
+  def line(n=nil)
+    if n then
+      @line = n
+      self
+    else
+      @line ||= nil
+    end
+  end
+
+  def line= n
+    @line = n
+  end
+
+  def node_type
+    first
+  end
+
+  def paren
+    @paren ||= false
+  end
+
+  def value
+    raise "multi item sexp" if size > 2
+    last
+  end
+
+  def to_sym
+    self.value.to_sym
+  end
+
+  def values
+    self[1..-1]
+  end
+
+  alias :real_inspect :inspect
+  def inspect # :nodoc:
+    sexp_str = self.map {|x|x.inspect}.join(', ')
+    if line && ENV['VERBOSE'] then
+      "s(#{sexp_str}).line(#{line})"
+    else
+      "s(#{sexp_str})"
+    end
+  end
+end
+
+# END HACK
+############################################################

test/test_ruby_parser_extras.rb

+require 'test/unit'
+require 'ruby_parser_extras'
+
+class TestStackState < Test::Unit::TestCase
+  def test_stack_state
+    s = StackState.new :test
+    s.push true
+    s.push false
+    s.lexpop
+    assert_equal [false, true], s.stack
+  end
+
+  def test_is_in_state
+    s = StackState.new :test
+    assert_equal false, s.is_in_state
+    s.push false
+    assert_equal false, s.is_in_state
+    s.push true
+    assert_equal true, s.is_in_state
+    s.push false
+    assert_equal false, s.is_in_state
+  end
+
+  def test_lexpop
+    s = StackState.new :test
+    assert_equal [false], s.stack
+    s.push true
+    s.push false
+    assert_equal [false, true, false], s.stack
+    s.lexpop
+    assert_equal [false, true], s.stack
+  end
+
+  def test_pop
+    s = StackState.new :test
+    assert_equal [false], s.stack
+    s.push true
+    assert_equal [false, true], s.stack
+    assert_equal true, s.pop
+    assert_equal [false], s.stack
+  end
+
+  def test_push
+    s = StackState.new :test
+    assert_equal [false], s.stack
+    s.push true
+    s.push false
+    assert_equal [false, true, false], s.stack
+  end
+end
+
+class TestEnvironment < Test::Unit::TestCase
+  def deny t
+    assert ! t
+  end
+
+  def setup
+    @env = Environment.new
+    @env[:blah] = 42
+    assert_equal 42, @env[:blah]
+  end
+
+  def test_use
+    @env.use :blah
+    expected = [{ :blah => true }]
+    assert_equal expected, @env.instance_variable_get(:"@use")
+  end
+
+  def test_use_scoped
+    @env.use :blah
+    @env.extend
+    expected = [{}, { :blah => true }]
+    assert_equal expected, @env.instance_variable_get(:"@use")
+  end
+
+  def test_used_eh
+    @env.extend :dynamic
+    @env[:x] = :dvar
+    @env.use :x
+    assert_equal true, @env.used?(:x)
+  end
+
+  def test_used_eh_none
+    assert_equal nil, @env.used?(:x)
+  end
+
+  def test_used_eh_scoped
+    self.test_used_eh
+    @env.extend :dynamic
+    assert_equal true, @env.used?(:x)
+  end
+
+  def test_var_scope_dynamic
+    @env.extend :dynamic
+    assert_equal 42, @env[:blah]
+    @env.unextend
+    assert_equal 42, @env[:blah]
+  end
+
+  def test_var_scope_static
+    @env.extend
+    assert_equal nil, @env[:blah]
+    @env.unextend
+    assert_equal 42, @env[:blah]
+  end
+
+  def test_dynamic
+    expected1 = {}
+    expected2 = { :x => 42 }
+
+    assert_equal expected1, @env.dynamic
+    begin
+      @env.extend :dynamic
+      assert_equal expected1, @env.dynamic
+
+      @env[:x] = 42
+      assert_equal expected2, @env.dynamic
+
+      begin
+        @env.extend :dynamic
+        assert_equal expected2, @env.dynamic
+        @env.unextend
+      end
+
+      assert_equal expected2, @env.dynamic
+      @env.unextend
+    end
+    assert_equal expected1, @env.dynamic
+  end
+
+  def test_all_dynamic
+    expected = { :blah => 42 }
+
+    @env.extend :dynamic
+    assert_equal expected, @env.all
+    @env.unextend
+    assert_equal expected, @env.all
+  end
+
+  def test_all_static
+    @env.extend
+    expected = { }
+    assert_equal expected, @env.all
+
+    @env.unextend
+    expected = { :blah => 42 }
+    assert_equal expected, @env.all
+  end
+
+  def test_dynamic_eh
+    assert_equal false, @env.dynamic?
+    @env.extend :dynamic
+    assert_equal true, @env.dynamic?
+    @env.extend
+    assert_equal false, @env.dynamic?
+  end
+
+  def test_all_static_deeper
+    expected0 = { :blah => 42 }
+    expected1 = { :blah => 42, :blah2 => 24 }
+    expected2 = { :blah => 27 }
+
+    @env.extend :dynamic
+    @env[:blah2] = 24
+    assert_equal expected1, @env.all
+
+    @env.extend 
+    @env[:blah] = 27
+    assert_equal expected2, @env.all
+
+    @env.unextend
+    assert_equal expected1, @env.all
+
+    @env.unextend
+    assert_equal expected0, @env.all
+  end
+end

History.txt

@@ -0,1 +1,109 @@
+== 2.0.0 / 2008-10-22
+
+* 1 major enhancement
+
+  * Brought on the AWESOME! 4x faster! no known lexing/parsing bugs!
+
+* 71 minor enhancements
+
+  * 1.9: Added Fixnum#ord.
+  * 1.9: Added missing Regexp constants and did it so it'd work on 1.9.
+  * Added #store_comment and #comments
+  * Added StringScanner #begin_of_line?
+  * Added a bunch of tests for regexp escape chars, #parse_string, #read_escape, ? numbers, ? whitespace.
+  * Added a hack for rubinius' r2l eval bug.
+  * Added a new token type tSTRING that bypasses tSTRING_BEG/END entirely. Only does non-interpolated strings and then falls back to the old way. MUCH cleaner tho.
+  * Added bin/ruby_parse
+  * Added compare rule to Rakefile.
+  * Added coverage files/dirs to clean rule.
+  * Added file and line numbers to all sexp nodes. Column/ranges to come.
+  * Added lex_state change for lvars at the end of yylex.
+  * Added lexed comments to defn/defs/class/module nodes.
+  * Added stats gathering for yylex. Reordered yylex for avg data
+  * Added tSYMBOL token type and parser rule to speed up symbol lexing.
+  * Added tally output for getch, unread, and unread_many.
+  * Added tests for ambigous uminus/uplus, backtick in cmdarg, square and curly brackets, numeric gvars, eos edge cases, string quoting %<> and %%%.
+  * All cases throughout yylex now return directly if they match, no passthroughs.
+  * All lexer cases now slurp entire token in one swoop.
+  * All zarrays are now just empty arrays.
+  * Changed s(:block_arg, :blah) to :"&blah" in args sexp.
+  * Cleaned up lexer error handling. Now just raises all over.
+  * Cleaned up read_escape and regx_options
+  * Cleaned up tokadd_string (for some definition of cleaned).
+  * Converted single quoted strings to new tSTRING token type.
+  * Coverage is currently 94.4% on lexer.
+  * Done what I can to clean up heredoc lexing... still sucks.
+  * Flattened resbodies in rescue node. Fixed .autotest file.
+  * Folded lex_keywords back in now that it screams.
+  * Found very last instanceof ILiteralNode in the code. haha!
+  * Got the tests subclassing PTTC and cleaned up a lot. YAY
+  * Handle yield(*ary) properly
+  * MASSIVELY cleaned out =begin/=end comment processor.
+  * Massive overhaul on Keyword class. All hail the mighty Hash!
+  * Massively cleaned up ident= edge cases and fixed a stupid bug from jruby.
+  * Merged @/@@ scanner together, going to try to do the same everywhere.
+  * Refactored fix_arg_lex_state, common across the lexer.
+  * Refactored new_fcall into new_call.
+  * Refactored some code to get better profile numbers.
+  * Refactored some more #fix_arg_lex_state.
+  * Refactored tail of yylex into its own method.
+  * Removed Module#kill
+  * Removed Token, replaced with Sexp.
+  * Removed all parse_number and parse_quote tests.
+  * Removed argspush, argscat. YAY!
+  * Removed as many token_buffer.split(//)'s as possible. 1 to go.
+  * Removed begins from compstmts
+  * Removed buffer arg for tokadd_string.
+  * Removed crufty (?) solo '@' token... wtf was that anyhow?
+  * Removed most jruby/stringio cruft from StringScanner.
+  * Removed one unread_many... 2 to go. They're harder.
+  * Removed store_comment, now done directly.
+  * Removed token_buffer. Now I just use token ivar.
+  * Removed use of s() from lexer. Changed the way line numbers are gathered.
+  * Renamed *qwords to *awords.
+  * Renamed StringScanner to RPStringScanner (a subclass) to fix namespace trashing.
+  * Renamed parse to process and aliased to parse.
+  * Renamed token_buffer to string_buffer since that arcane shit still needs it.
+  * Resolved the rest of the lexing issues I brought up w/ ruby-core.
+  * Revamped tokadd_escape.
+  * Rewrote Keyword and KWtable.
+  * Rewrote RubyLexer using StringScanner.
+  * Rewrote tokadd_escape. 79 lines down to 21.
+  * Split out lib/ruby_parser_extras.rb so lexer is standalone.
+  * Started to clean up the parser and make it as skinny as possible
+  * Stripped out as much code as possible.
+  * Stripped yylex of some dead code.
+  * Switched from StringIO to StringScanner.
+  * Updated rakefile for new hoe.
+  * Uses pure ruby racc if ENV['PURE_RUBY'], otherwise use c.
+  * Wrote a ton of lexer tests. Coverage is as close to 100% as possible.
+  * Wrote args to clean up the big nasty args processing grammar section.
+  * lex_strterm is now a plain array, removed RubyLexer#s(...).
+  * yield and super now flatten args.
+
+* 21+ bug fixes:
+
+  * I'm sure this list is missing a lot:
+  * Fixed 2 bugs both involving attrasgn (and ilk) esp when lhs is an array.
+  * Fixed a bug in the lexer for strings with single digit hex escapes.
+  * Fixed a bug parsing: a (args) { expr }... the space caused a different route to be followed and all hell broke loose.
+  * Fixed a bug with x\n=beginvar not putting begin back.
+  * Fixed attrasgn to have arglists, not arrays.
+  * Fixed bug in defn/defs with block fixing.
+  * Fixed class/module's name slot if colon2/3.
+  * Fixed dstr with empty interpolation body.
+  * Fixed for 1.9 string/char changes.
+  * Fixed lexer BS wrt determining token type of words.
+  * Fixed lexer BS wrt pass through values and lexing words. SO STUPID.
+  * Fixed lexing of floats.
+  * Fixed lexing of identifiers followed by equals. I hope.
+  * Fixed masgn with splat on lhs
+  * Fixed new_super to deal with block_pass correctly.
+  * Fixed parser's treatment of :colon2 and :colon3.
+  * Fixed regexp scanning of escaped numbers, ANY number is valid, not just octs.
+  * Fixed string scanning of escaped octs, allowing 1-3 chars.
+  * Fixed unescape for \n
+  * Fixed: omg this is stupid. '()' was returning bare nil
+  * Fixed: remove_begin now goes to the end, not sure why it didn't before.
+
 == 1.0.0 / 2007-12-20
@@ -2,0 +110,0 @@

Manifest.txt

@@ -6,5 +6,8 @@ .autotest
 Rakefile
+bin/ruby_parse
 lib/ruby_lexer.rb
 lib/ruby_parser.y
+lib/ruby_parser_extras.rb
 test/test_ruby_lexer.rb
 test/test_ruby_parser.rb
+test/test_ruby_parser_extras.rb

Rakefile

@@ -5,13 +5,16 @@ # -*- ruby -*-
 require 'hoe'
-require './lib/ruby_lexer.rb'
 
-hoe = Hoe.new('ruby_parser', RubyParser::VERSION) do |p|
-  p.rubyforge_name = 'parsetree'
-  p.author = 'Ryan Davis'
-  p.email = '[e-mail]'
-  p.summary = p.paragraphs_of('README.txt', 2).join("\n\n")
-  p.description = p.paragraphs_of('README.txt', 2..6).join("\n\n")
-  p.url = p.paragraphs_of('README.txt', 0).first.split(/\n/)[-1]
-  p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
-  p.extra_deps << 'ParseTree'
+Hoe.add_include_dirs("../../ParseTree/dev/lib",
+                     "../../ParseTree/dev/test",
+                     "../../RubyInline/dev/lib",
+                     "../../sexp_processor/dev/lib")
+
+require './lib/ruby_parser_extras.rb'
+
+hoe = Hoe.new('ruby_parser', RubyParser::VERSION) do |parser|
+  parser.rubyforge_name = 'parsetree'
+  parser.developer('Ryan Davis', '[e-mail]')
+  parser.extra_deps << 'ParseTree'
+
+  parser.extra_deps << ['sexp_processor', '>= 3.0.0']
 end
@@ -21,13 +24,6 @@
 
-module Rake::TaskManager
-  def all_tasks
-    @tasks
-  end
+[:default, :multi, :test].each do |t|
+  task t => :parser
 end
 
-Rake.application.all_tasks["default"].prerequisites.clear
-
-task :default => :parser
-task :test => :parser
-
 path = "pkg/ruby_parser-#{RubyParser::VERSION}"
@@ -40,20 +36,122 @@ task path => :parser do
 
+desc "build the parser"
 task :parser => ["lib/ruby_parser.rb"]
 
 rule '.rb' => '.y' do |t|
-  sh "racc -g -o #{t.name} #{t.source}"
+  # -v = verbose
+  # -t = debugging parser ~4% reduction in speed -- keep for now
+  # -l = no-line-convert
+  sh "racc -v -t -l -o #{t.name} #{t.source}"
 end
 
 task :clean do
-  rm_f(Dir["**/*~"] +
-       Dir["**/*.diff"] +
-       Dir["lib/ruby_parser.rb"] +
-       Dir["lib/*.output"])
+  rm_rf(Dir["**/*~"] +
+        Dir["**/*.diff"] +
+        Dir["coverage.info"] +
+        Dir["coverage"] +
+        Dir["lib/ruby_parser.rb"] +
+        Dir["lib/*.output"])
 end
 
-# require 'rcov/rcovtask'
-# Rcov::RcovTask.new do |t|
-#   t.test_files = FileList['test/test_ruby_lexer.rb']
-# end
+def next_num(glob)
+  num = Dir[glob].max[/\d+/].to_i + 1
+end
 
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |t|
+    pattern = ENV['PATTERN'] || 'test/test_ruby_*.rb'
+
+    t.test_files = FileList[pattern]
+    t.verbose = true
+    t.rcov_opts << "--threshold 80"
+    t.rcov_opts << "--no-color"
+  end
+rescue LoadError
+  # skip
+end
+
+desc "Compares PT to RP and deletes all files that match"
+task :compare do
+  files = Dir["unit/**/*.rb"]
+  puts "Parsing #{files.size} files"
+  files.each do |file|
+    puts file
+    system "./cmp.rb -q #{file} && rm #{file}"
+  end
+  system 'find -d unit -type d -empty -exec rmdir {} \;'
+end
+
+desc "Compares PT to RP and stops on first failure"
+task :find_bug do
+  files = Dir["unit/**/*.rb"]
+  puts "Parsing #{files.size} files"
+  files.each do |file|
+    puts file
+    sh "./cmp.rb -q #{file}"
+  end
+end
+
+task :sort do
+  sh 'grepsort "^ +def" lib/ruby_lexer.rb'
+  sh 'grepsort "^ +def (test|util)" test/test_ruby_lexer.rb'
+end
+
+task :rcov_info => :parser do
+  pattern = ENV['PATTERN'] || "test/test_*.rb"
+  ruby "-Ilib -S rcov --text-report --save coverage.info #{pattern}"
+end
+
+task :rcov_overlay do
+  rcov, eol = Marshal.load(File.read("coverage.info")).last[ENV["FILE"]], 1
+  puts rcov[:lines].zip(rcov[:coverage]).map { |line, coverage|
+    bol, eol = eol, eol + line.length
+    [bol, eol, "#ffcccc"] unless coverage
+  }.compact.inspect
+end
+
+task :loc do
+  loc1  = `wc -l ../1.0.0/lib/ruby_lexer.rb`[/\d+/]
+  flog1 = `flog -s ../1.0.0/lib/ruby_lexer.rb`[/\d+\.\d+/]
+  loc2  = `cat lib/ruby_lexer.rb lib/ruby_parser_extras.rb | wc -l`[/\d+/]
+  flog2 = `flog -s lib/ruby_lexer.rb lib/ruby_parser_extras.rb`[/\d+\.\d+/]
+
+  loc1, loc2, flog1, flog2 = loc1.to_i, loc2.to_i, flog1.to_f, flog2.to_f
+
+  puts "1.0.0: loc = #{loc1} flog = #{flog1}"
+  puts "dev  : loc = #{loc2} flog = #{flog2}"
+  puts "delta: loc = #{loc2-loc1} flog = #{flog2-flog1}"
+end
+
+desc "Validate against all normal files in unit dir"
+task :validate do
+  sh "./cmp.rb unit/*.rb"
+end
+
+def run_and_log cmd, prefix
+  files = ENV['FILES'] || 'unit/*.rb'
+  p, x = prefix, "txt"
+  n = Dir["#{p}.*.#{x}"].map { |s| s[/\d+/].to_i }.max + 1 rescue 1
+  f = "#{p}.#{n}.#{x}"
+
+  sh "#{cmd} #{Hoe::RUBY_FLAGS} bin/ruby_parse -q -g #{files} &> #{f}"
+
+  puts File.read(f)
+end
+
+desc "Benchmark against all normal files in unit dir"
+task :benchmark do
+  run_and_log "ruby", "benchmark"
+end
+
+desc "Profile against all normal files in unit dir"
+task :profile do
+  run_and_log "zenprofile", "profile"
+end
+
+desc "what was that command again?"
+task :huh? do
+  puts "ruby #{Hoe::RUBY_FLAGS} bin/ruby_parse -q -g ..."
+end
+
 # vim: syntax=Ruby

README.txt

@@ -15,2 +15,3 @@ ruby_parser
 * Pure ruby, no compiles.
+* Includes preceding comment data for defn/defs/class/module nodes!
 * Incredibly simple interface.
@@ -24,3 +25,2 @@ * Output is 100% equivalent to ParseTree.
 * Known Issue: dasgn_curr decls can be out of order from ParseTree's.
-* TODO: Add comment nodes.
 
@@ -27,0 +27,0 @@ == SYNOPSIS:

test/test_ruby_lexer.rb

 #!/usr/local/bin/ruby
 
-require 'test/unit'
-require 'ruby_lexer'
+require "test/unit"
+require "ruby_lexer"
 
@@ -12,5 +12,6 @@ class TestRubyLexer < Test::Unit::TestCase
   def setup
-    @lex = RubyLexer.new
-    @lex.src = StringIO.new("blah blah")
-    @lex.lex_state = :expr_beg # HACK ? I have no idea actually
+    p = RubyParser.new
+    @lex = p.lexer
+    @lex.src = "blah blah"
+    @lex.lex_state = :expr_beg
   end
@@ -24,60 +25,243 @@
 
-  def test_is_next_identchar
-    assert @lex.is_next_identchar
-    @lex.src = StringIO.new(" ")
-    deny @lex.is_next_identchar
-    @lex.src = StringIO.new("-")
-    deny @lex.is_next_identchar
+  def test_read_escape
+    util_escape "\\",   "\\"
+    util_escape "\n",   "n"
+    util_escape "\t",   "t"
+    util_escape "\r",   "r"
+    util_escape "\f",   "f"
+    util_escape "\13",  "v"
+    util_escape "\0",   "0"
+    util_escape "\07",  "a"
+    util_escape "\007", "a"
+    util_escape "\033", "e"
+    util_escape "\377", "377"
+    util_escape "\377", "xff"
+    util_escape "\010", "b"
+    util_escape " ",    "s"
+    util_escape "q",    "q" # plain vanilla escape
   end
 
-  def test_is_next_no_case # TODO: worst name evah
-    @lex.src = StringIO.new("123 456")
-    assert @lex.is_next_no_case("123")
-    pos = @lex.src.pos
-    deny @lex.is_next_no_case("begin")
-    assert_equal " 456", @lex.src.read_all, "must put back contents"
+  def test_read_escape_c
+    util_escape "\030", "C-x"
+    util_escape "\030", "cx"
+    util_escape "\230", 'C-\M-x'
+    util_escape "\230", 'c\M-x'
+
+    util_escape "\177", "C-?"
+    util_escape "\177", "c?"
   end
 
-  def test_number_token
-    node = @lex.number_token("42", false, "\0")
-    assert_equal :tINTEGER, node
-    assert_equal 42, @lex.yacc_value
+  def test_read_escape_errors
+    util_escape_bad ""
+
+    util_escape_bad "M"
+    util_escape_bad "M-"
+    util_escape_bad "Mx"
+
+    util_escape_bad "Cx"
+    util_escape_bad "C"
+    util_escape_bad "C-"
+
+    util_escape_bad "c"
   end
 
-  def test_parse_number
-    @lex.src = StringIO.new '42'
-    node = @lex.parse_number('1')
-    assert_equal :tINTEGER, node
-    assert_equal 142, @lex.yacc_value
+  def test_read_escape_m
+    util_escape "\370", "M-x"
+    util_escape "\230", 'M-\C-x'
+    util_escape "\230", 'M-\cx'
   end
 
-  def test_parse_quote
-    @lex.src = StringIO.new 'blah)'
-    node = @lex.parse_quote('(')
-    assert_equal :tSTRING_BEG, node
-    assert_equal s(:strterm, RubyLexer::STR_DQUOTE, ")", "("), @lex.lex_strterm
-    assert_equal ["%)"], @lex.yacc_value.args # FIX double check this
+  def test_yylex_ambiguous_uminus
+    util_lex_token("m -3",
+                   :tIDENTIFIER, "m",
+                   :tUMINUS_NUM, "-",
+                   :tINTEGER, 3)
+    # TODO: verify warning
   end
 
-  def test_yylex_integer
-    util_lex_token "42", :tINTEGER, 42
+  def test_yylex_ambiguous_uplus
+    util_lex_token("m +3",
+                   :tIDENTIFIER, "m",
+                   :tINTEGER, 3)
+    # TODO: verify warning
   end
 
-  def test_yylex_integer_eh_a
-    util_lex_token('?a',
-                   :tINTEGER,     97)
+  def test_yylex_and
+    util_lex_token "&", :tAMPER, "&"
   end
 
-  def test_yylex_integer_eh_escape_M_escape_C
-    util_lex_token('?\M-\C-a',
-                   :tINTEGER,     129)
+  def test_yylex_and2
+    util_lex_token "&&", :tANDOP, "&&"
   end
 
-  def test_yylex_float
-    util_lex_token "1.0", :tFLOAT, 1.0
+  def test_yylex_and2_equals
+    util_lex_token "&&=", :tOP_ASGN, "&&"
   end
 
+  def test_yylex_and_arg
+    @lex.lex_state = :expr_arg
+
+    util_lex_token(" &y",
+                   :tAMPER, "&",
+                   :tIDENTIFIER, "y")
+  end
+
+  def test_yylex_and_equals
+    util_lex_token "&=", :tOP_ASGN, "&"
+  end
+
+  def test_yylex_and_expr
+    @lex.lex_state = :expr_arg
+
+    util_lex_token("x & y",
+                   :tIDENTIFIER, "x",
+                   :tAMPER2, "&",
+                   :tIDENTIFIER, "y")
+  end
+
+  def test_yylex_and_meth
+    util_lex_fname "&", :tAMPER2
+  end
+
+  def test_yylex_assoc
+    util_lex_token "=>", :tASSOC, "=>"
+  end
+
+  def test_yylex_back_ref
+    util_lex_token("[$&, $`, $', $+]",
+                   :tLBRACK,   "[",
+                   :tBACK_REF, :"&", :tCOMMA, ",",
+                   :tBACK_REF, :"`", :tCOMMA, ",",
+                   :tBACK_REF, :"'", :tCOMMA, ",",
+                   :tBACK_REF, :"+",
+                   :tRBRACK,   "]")
+  end
+
+  def test_yylex_backslash
+    util_lex_token("1 \\\n+ 2",
+                   :tINTEGER, 1,
+                   :tPLUS, "+",
+                   :tINTEGER, 2)
+  end
+
+  def test_yylex_backslash_bad
+    util_bad_token("1 \\ + 2",
+                   :tINTEGER, 1)
+  end
+
+  def test_yylex_backtick
+    util_lex_token("`ls`",
+                   :tXSTRING_BEG, "`",
+                   :tSTRING_CONTENT, "ls",
+                   :tSTRING_END, "`")
+  end
+
+  def test_yylex_backtick_cmdarg
+    @lex.lex_state = :expr_dot
+    util_lex_token("\n`", :tBACK_REF2, "`") # \n ensures expr_cmd
+
+    assert_equal :expr_cmdarg, @lex.lex_state
+  end
+
+  def test_yylex_backtick_dot
+    @lex.lex_state = :expr_dot
+    util_lex_token("a.`(3)",
+                   :tIDENTIFIER, "a",
+                   :tDOT, ".",
+                   :tBACK_REF2, "`",
+                   :tLPAREN2, "(",
+                   :tINTEGER, 3,
+                   :tRPAREN, ")")
+  end
+
+  def test_yylex_backtick_method
+    @lex.lex_state = :expr_fname
+    util_lex_token("`", :tBACK_REF2, "`")
+    assert_equal :expr_end, @lex.lex_state
+  end
+
+  def test_yylex_bad_char
+    util_bad_token(" \010 ")
+  end
+
+  def test_yylex_bang
+    util_lex_token "!", :tBANG, "!"
+  end
+
+  def test_yylex_bang_equals
+    util_lex_token "!=", :tNEQ, "!="
+  end
+
+  def test_yylex_bang_tilde
+    util_lex_token "!~", :tNMATCH, "!~"
+  end
+
+  def test_yylex_carat
+    util_lex_token "^", :tCARET, "^"
+  end
+
+  def test_yylex_carat_equals
+    util_lex_token "^=", :tOP_ASGN, "^"
+  end
+
+  def test_yylex_colon2
+    util_lex_token("A::B",
+                   :tCONSTANT, "A",
+                   :tCOLON2,   "::",
+                   :tCONSTANT, "B")
+  end
+
+  def test_yylex_colon3
+    util_lex_token("::Array",
+                   :tCOLON3, "::",
+                   :tCONSTANT, "Array")
+  end
+
+  def test_yylex_comma
+    util_lex_token ",", :tCOMMA, ","
+  end
+
+  def test_yylex_comment
+    util_lex_token("1 # one\n# two\n2",
+                   :tINTEGER, 1,
+                   :tNL, nil,
+                   :tINTEGER, 2)
+    assert_equal "# one\n# two\n", @lex.comments
+  end
+
+  def test_yylex_comment_begin
+    util_lex_token("=begin\nblah\nblah\n=end\n42",
+                   :tINTEGER, 42)
+    assert_equal "=begin\nblah\nblah\n=end\n", @lex.comments
+  end
+
+  def test_yylex_comment_begin_bad
+    util_bad_token("=begin\nblah\nblah\n")
+    assert_equal "", @lex.comments
+  end
+
+  def test_yylex_comment_begin_not_comment
+    util_lex_token("beginfoo = 5\np x \\\n=beginfoo",
+                   :tIDENTIFIER, "beginfoo",
+                   :tEQL,          "=",
+                   :tINTEGER,    5,
+                   :tNL,         nil,
+                   :tIDENTIFIER, "p",
+                   :tIDENTIFIER, "x",
+                   :tEQL,          "=",
+                   :tIDENTIFIER, "beginfoo")
+  end
+
+  def test_yylex_comment_begin_space
+    util_lex_token("=begin blah\nblah\n=end\n")
+    assert_equal "=begin blah\nblah\n=end\n", @lex.comments
+  end
+
+  def test_yylex_comment_eos
+    util_lex_token("# comment")
+  end
+
   def test_yylex_constant
     util_lex_token("ArgumentError",
-                   :tCONSTANT, t("ArgumentError"))
+                   :tCONSTANT, "ArgumentError")
   end
@@ -87,315 +271,1471 @@
     util_lex_token("ArgumentError;",
-                   :tCONSTANT, t("ArgumentError"),
-                   ";", t(";"))
+                   :tCONSTANT, "ArgumentError",
+                   :tSEMI, ";")
   end
 
+  def test_yylex_cvar
+    util_lex_token "@@blah", :tCVAR, "@@blah"
+  end
+
+  def test_yylex_cvar_bad
+    assert_raises SyntaxError do
+      util_lex_token "@@1"
+    end
+  end
+
+  def test_yylex_def_bad_name
+    @lex.lex_state = :expr_fname
+    util_bad_token("def [ ", :kDEF, "def")
+  end
+
+  def test_yylex_div
+    util_lex_token("a / 2",
+                   :tIDENTIFIER, "a",
+                   :tDIVIDE, "/",
+                   :tINTEGER, 2)
+  end
+
+  def test_yylex_div_equals
+    util_lex_token("a /= 2",
+                   :tIDENTIFIER, "a",
+                   :tOP_ASGN, "/",
+                   :tINTEGER, 2)
+  end
+
+  def test_yylex_do
+    util_lex_token("x do 42 end",
+                   :tIDENTIFIER, "x",
+                   :kDO, "do",
+                   :tINTEGER, 42,
+                   :kEND, "end")
+  end
+
+  def test_yylex_do_block
+    @lex.lex_state = :expr_endarg
+    @lex.cmdarg.push true
+
+    util_lex_token("x.y do 42 end",
+                   :tIDENTIFIER, "x",
+                   :tDOT, ".",
+                   :tIDENTIFIER, "y",
+                   :kDO_BLOCK, "do",
+                   :tINTEGER, 42,
+                   :kEND, "end")
+  end
+
+  def test_yylex_do_block2
+    @lex.lex_state = :expr_endarg
+
+    util_lex_token("do 42 end",
+                   :kDO_BLOCK, "do",
+                   :tINTEGER, 42,
+                   :kEND, "end")
+  end
+
+  def test_yylex_do_cond
+    @lex.cond.push true
+
+    util_lex_token("x do 42 end",
+                   :tIDENTIFIER, "x",
+                   :kDO_COND, "do",
+                   :tINTEGER, 42,
+                   :kEND, "end")
+  end
+
+  def test_yylex_dollar
+    util_lex_token("$", "$", "$") # FIX: wtf is this?!?
+  end
+
+  def test_yylex_dot # HINT message sends
+    util_lex_token ".", :tDOT, "."
+  end
+
+  def test_yylex_dot2
+    util_lex_token "..", :tDOT2, ".."
+  end
+
+  def test_yylex_dot3
+    util_lex_token "...", :tDOT3, "..."
+  end
+
+  def test_yylex_equals
+    util_lex_token "=", :tEQL, "=" # FIX: this sucks
+  end
+
+  def test_yylex_equals2
+    util_lex_token "==", :tEQ, "=="
+  end
+
+  def test_yylex_equals3
+    util_lex_token "===", :tEQQ, "==="
+  end
+
+  def test_yylex_equals_tilde
+    util_lex_token "=~", :tMATCH, "=~"
+  end
+
+  def test_yylex_float
+    util_lex_token "1.0", :tFLOAT, 1.0
+  end
+
+  def test_yylex_float_bad_no_underscores
+    util_bad_token "1__0.0"
+  end
+
+  def test_yylex_float_bad_no_zero_leading
+    util_bad_token ".0"
+  end
+
+  def test_yylex_float_bad_trailing_underscore
+    util_bad_token "123_.0"
+  end
+
+  def test_yylex_float_call
+    util_lex_token("1.0.to_s",
+                   :tFLOAT, 1.0,
+                   :tDOT, ".",
+                   :tIDENTIFIER, "to_s")
+  end
+
+  def test_yylex_float_dot_E
+    util_lex_token "1.0E10", :tFLOAT, 1.0e10
+  end
+
+  def test_yylex_float_dot_E_neg
+    util_lex_token("-1.0E10",
+                   :tUMINUS_NUM, "-",
+                   :tFLOAT, 1.0e10)
+  end
+
+  def test_yylex_float_dot_e
+    util_lex_token "1.0e10", :tFLOAT, 1.0e10
+  end
+
+  def test_yylex_float_dot_e_neg
+    util_lex_token("-1.0e10",
+                   :tUMINUS_NUM, "-",
+                   :tFLOAT, 1.0e10)
+  end
+
+  def test_yylex_float_e
+    util_lex_token "1e10", :tFLOAT, 1e10
+  end
+
+  def test_yylex_float_e_bad_double_e
+    util_bad_token "1e2e3"
+  end
+
+  def test_yylex_float_e_bad_trailing_underscore
+    util_bad_token "123_e10"
+  end
+
+  def test_yylex_float_e_minus
+    util_lex_token "1e-10", :tFLOAT, 1e-10
+  end
+
+  def test_yylex_float_e_neg
+    util_lex_token("-1e10",
+                   :tUMINUS_NUM, "-",
+                   :tFLOAT, 1e10)
+  end
+
+  def test_yylex_float_e_neg_minus
+    util_lex_token("-1e-10",
+                   :tUMINUS_NUM, "-",
+                   :tFLOAT, 1e-10)
+  end
+
+  def test_yylex_float_e_neg_plus
+    util_lex_token("-1e+10",
+                   :tUMINUS_NUM, "-",
+                   :tFLOAT, 1e10)
+  end
+
+  def test_yylex_float_e_plus
+    util_lex_token "1e+10", :tFLOAT, 1e10
+  end
+
+  def test_yylex_float_e_zero
+    util_lex_token "0e0", :tFLOAT, 0e0
+  end
+
+  def test_yylex_float_neg
+    util_lex_token("-1.0",
+                   :tUMINUS_NUM, "-",
+                   :tFLOAT, 1.0)
+  end
+
+  def test_yylex_ge
+    util_lex_token("a >= 2",
+                   :tIDENTIFIER, "a",
+                   :tGEQ, ">=",
+                   :tINTEGER, 2)
+  end
+
+  def test_yylex_global
+    util_lex_token("$blah", :tGVAR, "$blah")
+  end
+
+  def test_yylex_global_backref
+    @lex.lex_state = :expr_fname
+    util_lex_token("$`", :tGVAR, "$`")
+  end
+
+  def test_yylex_global_dash_nothing
+    util_lex_token("$- ", :tGVAR, "$-")
+  end
+
+  def test_yylex_global_dash_something
+    util_lex_token("$-x", :tGVAR, "$-x")
+  end
+
+  def test_yylex_global_number
+    @lex.lex_state = :expr_fname
+    util_lex_token("$1", :tGVAR, "$1")
+  end
+
+  def test_yylex_global_number_big
+    @lex.lex_state = :expr_fname
+    util_lex_token("$1234", :tGVAR, "$1234")
+  end
+
+  def test_yylex_global_other
+    util_lex_token("[$~, $*, $$, $?, $!, $@, $/, $\\, $;, $,, $., $=, $:, $<, $>, $\"]",
+                   :tLBRACK, "[",
+                   :tGVAR,   "$~",  :tCOMMA, ",",
+                   :tGVAR,   "$*",  :tCOMMA, ",",
+                   :tGVAR,   "$$",  :tCOMMA, ",",
+                   :tGVAR,   "$\?",  :tCOMMA, ",",
+                   :tGVAR,   "$!",  :tCOMMA, ",",
+                   :tGVAR,   "$@",  :tCOMMA, ",",
+                   :tGVAR,   "$/",  :tCOMMA, ",",
+                   :tGVAR,   "$\\", :tCOMMA, ",",
+                   :tGVAR,   "$;",  :tCOMMA, ",",
+                   :tGVAR,   "$,",  :tCOMMA, ",",
+                   :tGVAR,   "$.",  :tCOMMA, ",",
+                   :tGVAR,   "$=",  :tCOMMA, ",",
+                   :tGVAR,   "$:",  :tCOMMA, ",",
+                   :tGVAR,   "$<",  :tCOMMA, ",",
+                   :tGVAR,   "$>",  :tCOMMA, ",",
+                   :tGVAR,   "$\"",
+                   :tRBRACK, "]")
+  end
+
+  def test_yylex_global_underscore
+    util_lex_token("$_",
+                   :tGVAR,     "$_")
+  end
+
+  def test_yylex_global_wierd
+    util_lex_token("$__blah",
+                   :tGVAR,     "$__blah")
+  end
+
+  def test_yylex_global_zero
+    util_lex_token("$0", :tGVAR, "$0")
+  end
+
+  def test_yylex_gt
+    util_lex_token("a > 2",
+                   :tIDENTIFIER, "a",
+                   :tGT, ">",
+                   :tINTEGER, 2)
+  end
+
+  def test_yylex_heredoc_backtick
+    util_lex_token("a = <<`EOF`\n  blah blah\nEOF\n",
+                   :tIDENTIFIER,     "a",
+                   :tEQL,              "=",
+                   :tXSTRING_BEG,    "`",
+                   :tSTRING_CONTENT, "  blah blah\n",
+                   :tSTRING_END,     "EOF",
+                   :tNL,             nil)
+  end
+
+  def test_yylex_heredoc_double
+    util_lex_token("a = <<\"EOF\"\n  blah blah\nEOF\n",
+                   :tIDENTIFIER,     "a",
+                   :tEQL,              "=",
+                   :tSTRING_BEG,     "\"",
+                   :tSTRING_CONTENT, "  blah blah\n",
+                   :tSTRING_END,     "EOF",
+                   :tNL,             nil)
+  end
+
+  def test_yylex_heredoc_double_dash
+    util_lex_token("a = <<-\"EOF\"\n  blah blah\n  EOF\n",
+                   :tIDENTIFIER,     "a",
+                   :tEQL,              "=",
+                   :tSTRING_BEG,     "\"",
+                   :tSTRING_CONTENT, "  blah blah\n",
+                   :tSTRING_END,     "EOF",
+                   :tNL,             nil)
+  end
+
+  def test_yylex_heredoc_double_eos
+    util_bad_token("a = <<\"EOF\"\nblah",
+                   :tIDENTIFIER,     "a",
+                   :tEQL,              "=",
+                   :tSTRING_BEG,     "\"")
+  end
+
+  def test_yylex_heredoc_double_eos_nl
+    util_bad_token("a = <<\"EOF\"\nblah\n",
+                   :tIDENTIFIER,     "a",
+                   :tEQL,              "=",
+                   :tSTRING_BEG,     "\"")
+  end
+
+  def test_yylex_heredoc_double_interp
+    util_lex_token("a = <<\"EOF\"\n#x a \#@a b \#$b c \#{3} \nEOF\n",
+                   :tIDENTIFIER,     "a",
+                   :tEQL,              "=",
+                   :tSTRING_BEG,     "\"",
+                   :tSTRING_CONTENT, "#x a ",
+                   :tSTRING_DVAR,    "\#@",
+                   :tSTRING_CONTENT, "@a b ", # HUH?
+                   :tSTRING_DVAR,    "\#$",
+                   :tSTRING_CONTENT, "$b c ", # HUH?
+                   :tSTRING_DBEG,    "\#{",
+                   :tSTRING_CONTENT, "3} \n", # HUH?
+                   :tSTRING_END,     "EOF",
+                   :tNL,             nil)
+  end
+
+  def test_yylex_heredoc_none
+    util_lex_token("a = <<EOF\nblah\nblah\nEOF",
+                   :tIDENTIFIER,     "a",
+                   :tEQL,              "=",
+                   :tSTRING_BEG,     "\"",
+                   :tSTRING_CONTENT, "blah\nblah\n",
+                   :tSTRING_CONTENT, "",
+                   :tSTRING_END,     "EOF",
+                   :tNL,             nil)
+  end
+
+  def test_yylex_heredoc_none_bad_eos
+    util_bad_token("a = <<EOF",
+                   :tIDENTIFIER,     "a",
+                   :tEQL,              "=",
+                   :tSTRING_BEG,     "\"")
+  end
+
+  def test_yylex_heredoc_none_dash
+    util_lex_token("a = <<-EOF\nblah\nblah\n  EOF",
+                   :tIDENTIFIER,     "a",
+                   :tEQL,              "=",
+                   :tSTRING_BEG,     "\"",
+                   :tSTRING_CONTENT, "blah\nblah\n",
+                   :tSTRING_CONTENT, "",
+                   :tSTRING_END,     "EOF",
+                   :tNL,             nil)
+  end
+
+  def test_yylex_heredoc_single
+    util_lex_token("a = <<'EOF'\n  blah blah\nEOF\n",
+                   :tIDENTIFIER,     "a",
+                   :tEQL,              "=",
+                   :tSTRING_BEG,     "\"",
+                   :tSTRING_CONTENT, "  blah blah\n",
+                   :tSTRING_END,     "EOF",
+                   :tNL,             nil)
+  end
+
+  def test_yylex_heredoc_single_bad_eos_body
+    util_bad_token("a = <<'EOF'\nblah",
+                   :tIDENTIFIER,     "a",
+                   :tEQL,              "=",
+                   :tSTRING_BEG,     "\"")
+  end
+
+  def test_yylex_heredoc_single_bad_eos_empty
+    util_bad_token("a = <<''\n",
+                   :tIDENTIFIER,     "a",
+                   :tEQL,              "=",
+                   :tSTRING_BEG,     "\"")
+  end
+
+  def test_yylex_heredoc_single_bad_eos_term
+    util_bad_token("a = <<'EOF",
+                   :tIDENTIFIER,     "a",
+                   :tEQL,              "=",
+                   :tSTRING_BEG,     "\"")
+  end
+
+  def test_yylex_heredoc_single_bad_eos_term_nl
+    util_bad_token("a = <<'EOF\ns = 'blah blah'",
+                   :tIDENTIFIER,     "a",
+                   :tEQL,              "=",
+                   :tSTRING_BEG,     "\"")
+  end
+
+  def test_yylex_heredoc_single_dash
+    util_lex_token("a = <<-'EOF'\n  blah blah\n  EOF\n",
+                   :tIDENTIFIER,     "a",
+                   :tEQL,              "=",
+                   :tSTRING_BEG,     "\"",
+                   :tSTRING_CONTENT, "  blah blah\n",
+                   :tSTRING_END,     "EOF",
+                   :tNL,             nil)
+  end
+
   def test_yylex_identifier
-    util_lex_token("identifier",
-                   :tIDENTIFIER, t("identifier"))
+    util_lex_token("identifier", :tIDENTIFIER, "identifier")
   end
 
+  def test_yylex_identifier_bang
+    util_lex_token("identifier!", :tFID, "identifier!")
+  end
+
+  def test_yylex_identifier_cmp
+    util_lex_fname "<=>", :tCMP
+  end
+
+  def test_yylex_identifier_def
+    util_lex_fname "identifier", :tIDENTIFIER, :expr_end
+  end
+
+  def test_yylex_identifier_eh
+    util_lex_token("identifier?", :tFID, "identifier?")
+  end
+
+  def test_yylex_identifier_equals_arrow
+    @lex.lex_state = :expr_fname
+    util_lex_token(":blah==>",
+                   :tSYMBOL, "blah=",
+                   :tASSOC, "=>")
+  end
+
+  def test_yylex_identifier_equals_caret
+    util_lex_fname "^", :tCARET
+  end
+
+  def test_yylex_identifier_equals_def
+    util_lex_fname "identifier=", :tIDENTIFIER, :expr_end
+  end
+
+  def test_yylex_identifier_equals_def2
+    util_lex_fname "==", :tEQ
+  end
+
+  def test_yylex_identifier_equals_expr
+    @lex.lex_state = :expr_dot
+    util_lex_token("y = arg",
+                   :tIDENTIFIER, "y",
+                   :tEQL, "=",
+                   :tIDENTIFIER, "arg")
+
+    assert_equal :expr_arg, @lex.lex_state
+  end
+
+  def test_yylex_identifier_equals_or
+    util_lex_fname "|", :tPIPE
+  end
+
+  def test_yylex_identifier_equals_slash
+    util_lex_fname "/", :tDIVIDE
+  end
+
+  def test_yylex_identifier_equals_tilde
+    @lex.lex_state = :expr_fname # can only set via parser's defs
+    util_lex_token("identifier=~",
+                   :tIDENTIFIER, "identifier",
+                   :tMATCH, "=~")
+  end
+
+  def test_yylex_identifier_gt
+    util_lex_fname ">", :tGT
+  end
+
+  def test_yylex_identifier_le
+    util_lex_fname "<=", :tLEQ
+  end
+
+  def test_yylex_identifier_lt
+    util_lex_fname "<", :tLT
+  end
+
+  def test_yylex_identifier_tilde
+    util_lex_fname "~", :tTILDE
+  end
+
+  def test_yylex_index
+    util_lex_fname "[]", :tAREF
+  end
+
+  def test_yylex_index_equals
+    util_lex_fname "[]=", :tASET
+  end
+
+  def test_yylex_integer
+    util_lex_token "42", :tINTEGER, 42
+  end
+
+  def test_yylex_integer_bin
+    util_lex_token "0b101010", :tINTEGER, 42
+  end
+
+  def test_yylex_integer_bin_bad_none
+    util_bad_token "0b "
+  end
+
+  def test_yylex_integer_bin_bad_underscores
+    util_bad_token "0b10__01"
+  end
+
+  def test_yylex_integer_dec
+    util_lex_token "42", :tINTEGER, 42
+  end
+
+  def test_yylex_integer_dec_bad_underscores
+    util_bad_token "42__24"
+  end
+
+  def test_yylex_integer_dec_d
+    util_lex_token "0d42", :tINTEGER, 42
+  end
+
+  def test_yylex_integer_dec_d_bad_none
+    util_bad_token "0d"
+  end
+
+  def test_yylex_integer_dec_d_bad_underscores
+    util_bad_token "0d42__24"
+  end
+
+  def test_yylex_integer_eh_a
+    util_lex_token '?a', :tINTEGER, 97
+  end
+
+  def test_yylex_integer_eh_escape_M_escape_C
+    util_lex_token '?\M-\C-a', :tINTEGER, 129
+  end
+
+  def test_yylex_integer_hex
+    util_lex_token "0x2a", :tINTEGER, 42
+  end
+
+  def test_yylex_integer_hex_bad_none
+    util_bad_token "0x "
+  end
+
+  def test_yylex_integer_hex_bad_underscores
+    util_bad_token "0xab__cd"
+  end
+
+  def test_yylex_integer_oct
+    util_lex_token "052", :tINTEGER, 42
+  end
+
+  def test_yylex_integer_oct_bad_range
+    util_bad_token "08"
+  end
+
+  def test_yylex_integer_oct_bad_underscores
+    util_bad_token "01__23"
+  end
+
+  def test_yylex_integer_oct_o
+    util_lex_token "0o52", :tINTEGER, 42
+  end
+
+  def test_yylex_integer_oct_o_bad_range
+    util_bad_token "0o8"
+  end
+
+  def test_yylex_integer_oct_o_bad_underscores
+    util_bad_token "0o1__23"
+  end
+
+  def test_yylex_integer_oct_o_not_bad_none
+    util_lex_token "0o ", :tINTEGER, 0
+  end
+
+  def test_yylex_integer_trailing
+    util_lex_token("1.to_s",
+                   :tINTEGER, 1,
+                   :tDOT, '.',
+                   :tIDENTIFIER, 'to_s')
+  end
+
+  def test_yylex_integer_underscore
+    util_lex_token "4_2", :tINTEGER, 42
+  end
+
+  def test_yylex_integer_underscore_bad
+    util_bad_token "4__2"
+  end
+
+  def test_yylex_integer_zero
+    util_lex_token "0", :tINTEGER, 0
+  end
+
+  def test_yylex_ivar
+    util_lex_token "@blah", :tIVAR, "@blah"
+  end
+
+  def test_yylex_ivar_bad
+    util_bad_token "@1"
+  end
+
+  def test_yylex_keyword_expr
+    @lex.lex_state = :expr_endarg
+
+    util_lex_token("if", :kIF_MOD, "if")
+
+    assert_equal :expr_beg, @lex.lex_state
+  end
+
+  def test_yylex_lt
+    util_lex_token "<", :tLT, "<"
+  end
+
+  def test_yylex_lt2
+    util_lex_token("a <\< b",
+                   :tIDENTIFIER, "a",
+                   :tLSHFT, "<\<",
+                   :tIDENTIFIER, "b")
+
+  end
+
+  def test_yylex_lt2_equals
+    util_lex_token("a <\<= b",
+                   :tIDENTIFIER, "a",
+                   :tOP_ASGN, "<\<",
+                   :tIDENTIFIER, "b")
+  end
+
+  def test_yylex_lt_equals
+    util_lex_token "<=", :tLEQ, "<="
+  end
+
+  def test_yylex_minus
+    util_lex_token("1 - 2",
+                   :tINTEGER, 1,
+                   :tMINUS, "-",
+                   :tINTEGER, 2)
+  end
+
+  def test_yylex_minus_equals
+    util_lex_token "-=", :tOP_ASGN, "-"
+  end
+
+  def test_yylex_minus_method
+    @lex.lex_state = :expr_fname
+    util_lex_token "-", :tMINUS, "-"
+  end
+
+  def test_yylex_minus_unary_method
+    @lex.lex_state = :expr_fname
+    util_lex_token "-@", :tUMINUS, "-@"
+  end
+
+  def test_yylex_minus_unary_number
+    util_lex_token("-42",
+                   :tUMINUS_NUM, "-",
+                   :tINTEGER, 42)
+  end
+
+  def test_yylex_nth_ref
+    util_lex_token('[$1, $2, $3, $4, $5, $6, $7, $8, $9]',
+                   :tLBRACK,  "[",
+                   :tNTH_REF, 1, :tCOMMA, ",",
+                   :tNTH_REF, 2, :tCOMMA, ",",
+                   :tNTH_REF, 3, :tCOMMA, ",",
+                   :tNTH_REF, 4, :tCOMMA, ",",
+                   :tNTH_REF, 5, :tCOMMA, ",",
+                   :tNTH_REF, 6, :tCOMMA, ",",
+                   :tNTH_REF, 7, :tCOMMA, ",",
+                   :tNTH_REF, 8, :tCOMMA, ",",
+                   :tNTH_REF, 9,
+                   :tRBRACK,  "]")
+  end
+
+  def test_yylex_open_bracket
+    util_lex_token("(", :tLPAREN, "(")
+  end
+
+  def test_yylex_open_bracket_cmdarg
+    @lex.lex_state = :expr_cmdarg
+    util_lex_token(" (", :tLPAREN_ARG, "(")
+  end
+
+  def test_yylex_open_bracket_exprarg
+    @lex.lex_state = :expr_arg
+    util_lex_token(" (", :tLPAREN2, "(")
+  end
+
+  def test_yylex_open_curly_bracket
+    util_lex_token("{",
+                   :tLBRACE, "{")
+  end
+
+  def test_yylex_open_curly_bracket_arg
+    @lex.lex_state = :expr_arg
+    util_lex_token("m { 3 }",
+                   :tIDENTIFIER, "m",
+                   :tLCURLY, "{",
+                   :tINTEGER, 3,
+                   :tRCURLY, "}")
+  end
+
+  def test_yylex_open_curly_bracket_block
+    @lex.lex_state = :expr_endarg # seen m(3)
+    util_lex_token("{ 4 }",
+                   :tLBRACE_ARG, "{",
+                   :tINTEGER, 4,
+                   :tRCURLY, "}")
+  end
+
+  def test_yylex_open_square_bracket_arg
+    @lex.lex_state = :expr_arg
+    util_lex_token("m [ 3 ]",
+                   :tIDENTIFIER, "m",
+                   :tLBRACK, "[",
+                   :tINTEGER, 3,
+                   :tRBRACK, "]")
+  end
+
+  def test_yylex_open_square_bracket_ary
+    util_lex_token("[1, 2, 3]",
+                   :tLBRACK, "[",
+                   :tINTEGER, 1,
+                   :tCOMMA, ",",
+                   :tINTEGER, 2,
+                   :tCOMMA, ",",
+                   :tINTEGER, 3,
+                   :tRBRACK, "]")
+  end
+
+  def test_yylex_open_square_bracket_meth
+    util_lex_token("m[3]",
+                   :tIDENTIFIER, "m",
+                   "[", "[",
+                   :tINTEGER, 3,
+                   :tRBRACK, "]")
+  end
+
+  def test_yylex_or
+    util_lex_token "|", :tPIPE, "|"
+  end
+
+  def test_yylex_or2
+    util_lex_token "||", :tOROP, "||"
+  end
+
+  def test_yylex_or2_equals
+    util_lex_token "||=", :tOP_ASGN, "||"
+  end
+
+  def test_yylex_or_equals
+    util_lex_token "|=", :tOP_ASGN, "|"
+  end
+
+  def test_yylex_percent
+    util_lex_token("a % 2",
+                   :tIDENTIFIER, "a",
+                   :tPERCENT, "%",
+                   :tINTEGER, 2)
+  end
+
+  def test_yylex_percent_equals
+    util_lex_token("a %= 2",
+                   :tIDENTIFIER, "a",
+                   :tOP_ASGN, "%",
+                   :tINTEGER, 2)
+  end
+
+  def test_yylex_plus
+    util_lex_token("1 + 1", # TODO lex_state?
+                   :tINTEGER, 1,
+                   :tPLUS, "+",
+                   :tINTEGER, 1)
+  end
+
+  def test_yylex_plus_equals
+    util_lex_token "+=", :tOP_ASGN, "+"
+  end
+
+  def test_yylex_plus_method
+    @lex.lex_state = :expr_fname
+    util_lex_token "+", :tPLUS, "+"
+  end
+
+  def test_yylex_plus_unary_method
+    @lex.lex_state = :expr_fname
+    util_lex_token "+@", :tUPLUS, "+@"
+  end
+
+  def test_yylex_plus_unary_number
+    util_lex_token("+42",
+                   :tINTEGER, 42)
+  end
+
+  def test_yylex_question
+    util_lex_token "?*", :tINTEGER, 42
+  end
+
+  def test_yylex_question_bad_eos
+    util_bad_token "?"
+  end
+
+  def test_yylex_question_ws
+    util_lex_token "? ",  :tEH, "?"
+    util_lex_token "?\n", :tEH, "?"
+    util_lex_token "?\t", :tEH, "?"
+    util_lex_token "?\v", :tEH, "?"
+    util_lex_token "?\r", :tEH, "?"
+    util_lex_token "?\f", :tEH, "?"
+  end
+
+  def test_yylex_question_ws_backslashed
+    @lex.lex_state = :expr_beg
+    util_lex_token "?\\ ", :tINTEGER, 32
+    @lex.lex_state = :expr_beg
+    util_lex_token "?\\n", :tINTEGER, 10
+    @lex.lex_state = :expr_beg
+    util_lex_token "?\\t", :tINTEGER, 9
+    @lex.lex_state = :expr_beg
+    util_lex_token "?\\v", :tINTEGER, 11
+    @lex.lex_state = :expr_beg
+    util_lex_token "?\\r", :tINTEGER, 13
+    @lex.lex_state = :expr_beg
+    util_lex_token "?\\f", :tINTEGER, 12
+  end
+
+  def test_yylex_rbracket
+    util_lex_token "]", :tRBRACK, "]"
+  end
+
+  def test_yylex_rcurly
+    util_lex_token "}", :tRCURLY, "}"
+  end
+
   def test_yylex_regexp
     util_lex_token("/regexp/",
-                   :tREGEXP_BEG,     t("/"),
-                   :tSTRING_CONTENT, s(:str, "regexp"),
+                   :tREGEXP_BEG,     "/",
+                   :tSTRING_CONTENT, "regexp",
                    :tREGEXP_END,     "")
   end
 
-  def test_yylex_regexp_nm
-    util_lex_token("/.*/nm",
-                   :tREGEXP_BEG,     t("/"),
-                   :tSTRING_CONTENT, s(:str, ".*"),
-                   :tREGEXP_END,     "nm")
+  def test_yylex_regexp_ambiguous
+    util_lex_token("method /regexp/",
+                   :tIDENTIFIER,     "method",
+                   :tREGEXP_BEG,     "/",
+                   :tSTRING_CONTENT, "regexp",
+                   :tREGEXP_END,     "")
   end
 
-  def test_yylex_regexp_escapes
-    util_lex_token('/re\tge\nxp/',
-                   :tREGEXP_BEG,     t("/"),
-                   :tSTRING_CONTENT, s(:str, "re\\tge\\nxp"),
+  def test_yylex_regexp_bad
+    util_bad_token("/.*/xyz",
+                   :tREGEXP_BEG, "/",
+                   :tSTRING_CONTENT, ".*")
+  end
+
+  def test_yylex_regexp_escape_C
+    util_lex_token('/regex\\C-x/',
+                   :tREGEXP_BEG,     "/",
+                   :tSTRING_CONTENT, "regex\\C-x",
                    :tREGEXP_END,     "")
   end
 
-  def test_yylex_regexp_escape_oct
-    util_lex_token('/re\tge\101xp/',
-                   :tREGEXP_BEG,     t("/"),
-                   :tSTRING_CONTENT, s(:str, "re\\tge\\101xp"),
+  def test_yylex_regexp_escape_C_M
+    util_lex_token('/regex\\C-\\M-x/',
+                   :tREGEXP_BEG,     "/",
+                   :tSTRING_CONTENT, "regex\\C-\\M-x",
                    :tREGEXP_END,     "")
   end
 
+  def test_yylex_regexp_escape_C_M_craaaazy
+    util_lex_token("/regex\\C-\\\n\\M-x/",
+                   :tREGEXP_BEG,     "/",
+                   :tSTRING_CONTENT, "regex\\C-\\M-x",
+                   :tREGEXP_END,     "")
+  end
+
+  def test_yylex_regexp_escape_C_bad_dash
+    util_bad_token '/regex\\Cx/', :tREGEXP_BEG, "/"
+  end
+
+  def test_yylex_regexp_escape_C_bad_dash_eos
+    util_bad_token '/regex\\C-/', :tREGEXP_BEG, "/"
+  end
+
+  def test_yylex_regexp_escape_C_bad_dash_eos2
+    util_bad_token '/regex\\C-', :tREGEXP_BEG, "/"
+  end
+
+  def test_yylex_regexp_escape_C_bad_eos
+    util_bad_token '/regex\\C/', :tREGEXP_BEG, "/"
+  end
+
+  def test_yylex_regexp_escape_C_bad_eos2
+    util_bad_token '/regex\\c', :tREGEXP_BEG, "/"
+  end
+
+  def test_yylex_regexp_escape_M
+    util_lex_token('/regex\\M-x/',
+                   :tREGEXP_BEG,     "/",
+                   :tSTRING_CONTENT, "regex\\M-x",
+                   :tREGEXP_END,     "")
+  end
+
+  def test_yylex_regexp_escape_M_C
+    util_lex_token('/regex\\M-\\C-x/',
+                   :tREGEXP_BEG,     "/",
+                   :tSTRING_CONTENT, "regex\\M-\\C-x",
+                   :tREGEXP_END,     "")
+  end
+
+  def test_yylex_regexp_escape_M_bad_dash
+    util_bad_token '/regex\\Mx/', :tREGEXP_BEG, "/"
+  end
+
+  def test_yylex_regexp_escape_M_bad_dash_eos
+    util_bad_token '/regex\\M-/', :tREGEXP_BEG, "/"
+  end
+
+  def test_yylex_regexp_escape_M_bad_dash_eos2
+    util_bad_token '/regex\\M-', :tREGEXP_BEG, "/"
+  end
+
+  def test_yylex_regexp_escape_M_bad_eos
+    util_bad_token '/regex\\M/', :tREGEXP_BEG, "/"
+  end
+
+  def test_yylex_regexp_escape_backslash_slash
+    util_lex_token('/\\//',
+                   :tREGEXP_BEG,     "/",
+                   :tSTRING_CONTENT, '\\/',
+                   :tREGEXP_END,     "")
+  end
+
+  def test_yylex_regexp_escape_backslash_terminator
+    util_lex_token('%r%blah\\%blah%',
+                   :tREGEXP_BEG,     "%r\000", # FIX ?!?
+                   :tSTRING_CONTENT, "blah\\%blah",
+                   :tREGEXP_END,     "")
+  end
+
+  def test_yylex_regexp_escape_backslash_terminator_meta1
+    util_lex_token('%r{blah\\}blah}',
+                   :tREGEXP_BEG,     "%r{", # FIX ?!?
+                   :tSTRING_CONTENT, "blah\\}blah",
+                   :tREGEXP_END,     "")
+  end
+
+  def test_yylex_regexp_escape_backslash_terminator_meta2
+    util_lex_token('%r/blah\\/blah/',
+                   :tREGEXP_BEG,     "%r\000", # FIX ?!?
+                   :tSTRING_CONTENT, "blah\\/blah",
+                   :tREGEXP_END,     "")
+  end
+
+  def test_yylex_regexp_escape_backslash_terminator_meta3
+    util_lex_token('%r/blah\\%blah/',
+                   :tREGEXP_BEG,     "%r\000", # FIX ?!?
+                   :tSTRING_CONTENT, "blah\\%blah",
+                   :tREGEXP_END,     "")
+  end
+
+  def test_yylex_regexp_escape_bad_eos
+    util_bad_token '/regex\\', :tREGEXP_BEG, "/"
+  end
+
+  def test_yylex_regexp_escape_bs
+    util_lex_token('/regex\\\\regex/',
+                   :tREGEXP_BEG,     "/",
+                   :tSTRING_CONTENT, "regex\\\\regex",
+                   :tREGEXP_END,     "")
+  end
+
+  def test_yylex_regexp_escape_c
+    util_lex_token('/regex\\cxxx/',
+                   :tREGEXP_BEG,     "/",
+                   :tSTRING_CONTENT, "regex\\cxxx",
+                   :tREGEXP_END,     "")
+  end
+
+  def test_yylex_regexp_escape_c_backslash
+    util_lex_token('/regex\\c\\n/',
+                   :tREGEXP_BEG,     "/",
+                   :tSTRING_CONTENT, "regex\\c\\n",
+                   :tREGEXP_END,     "")
+  end
+
+  def test_yylex_regexp_escape_chars
+    util_lex_token('/re\\tge\\nxp/',
+                   :tREGEXP_BEG,     "/",
+                   :tSTRING_CONTENT, "re\\tge\\nxp",
+                   :tREGEXP_END,     "")
+  end
+
+  def test_yylex_regexp_escape_double_backslash
+    regexp = '/[\\/\\\\]$/'
+    util_lex_token(regexp,
+                   :tREGEXP_BEG,     "/",
+                   :tSTRING_CONTENT, regexp[1..-2],
+                   :tREGEXP_END,     "")
+  end
+
   def test_yylex_regexp_escape_hex
-    util_lex_token('/re\tge\x61xp/',
-                   :tREGEXP_BEG,     t("/"),
-                   :tSTRING_CONTENT, s(:str, "re\\tge\\x61xp"),
+    util_lex_token('/regex\\x61xp/',
+                   :tREGEXP_BEG,     "/",
+                   :tSTRING_CONTENT, "regex\\x61xp",
                    :tREGEXP_END,     "")
   end
 
+  def test_yylex_regexp_escape_hex_one
+    util_lex_token('/^[\\xd\\xa]{2}/on',
+                   :tREGEXP_BEG,     '/',
+                   :tSTRING_CONTENT, '^[\\xd\\xa]{2}',
+                   :tREGEXP_END,     'on')
+  end
+
+  def test_yylex_regexp_escape_hex_bad
+    util_bad_token '/regex\\xzxp/', :tREGEXP_BEG, "/"
+  end
+
+  def test_yylex_regexp_escape_oct1
+    util_lex_token('/regex\\0xp/',
+                   :tREGEXP_BEG,     "/",
+                   :tSTRING_CONTENT, "regex\\0xp",
+                   :tREGEXP_END,     "")
+  end
+
+  def test_yylex_regexp_escape_oct2
+    util_lex_token('/regex\\07xp/',
+                   :tREGEXP_BEG,     "/",
+                   :tSTRING_CONTENT, "regex\\07xp",
+                   :tREGEXP_END,     "")
+  end
+
+  def test_yylex_regexp_escape_oct3
+    util_lex_token('/regex\\10142/',
+                   :tREGEXP_BEG,     "/",
+                   :tSTRING_CONTENT, "regex\\10142",
+                   :tREGEXP_END,     "")
+  end
+
+  def test_yylex_regexp_escape_return
+    util_lex_token("/regex\\\nregex/",
+                   :tREGEXP_BEG,     "/",
+                   :tSTRING_CONTENT, "regexregex",
+                   :tREGEXP_END,     "")
+  end
+
+  def test_yylex_regexp_nm
+    util_lex_token("/.*/nm",
+                   :tREGEXP_BEG,     "/",
+                   :tSTRING_CONTENT, ".*",
+                   :tREGEXP_END,     "nm")
+  end
+
+  def test_yylex_rparen
+    util_lex_token ")", :tRPAREN, ")"
+  end
+
+  def test_yylex_rshft
+    util_lex_token("a >> 2",
+                   :tIDENTIFIER, "a",
+                   :tRSHFT, ">>",
+                   :tINTEGER, 2)
+  end
+
+  def test_yylex_rshft_equals
+    util_lex_token("a >>= 2",
+                   :tIDENTIFIER, "a",
+                   :tOP_ASGN, ">>",
+                   :tINTEGER, 2)
+  end
+
+  def test_yylex_star
+    util_lex_token("a * ",
+                   :tIDENTIFIER, "a",
+                   :tSTAR2, "*")
+
+    assert_equal :expr_beg, @lex.lex_state
+  end
+
+  def test_yylex_star2
+    util_lex_token("a ** ",
+                   :tIDENTIFIER, "a",
+                   :tPOW, "**")
+
+    assert_equal :expr_beg, @lex.lex_state
+  end
+
+  def test_yylex_star2_equals
+    util_lex_token("a **= ",
+                   :tIDENTIFIER, "a",
+                   :tOP_ASGN, "**")
+
+    assert_equal :expr_beg, @lex.lex_state
+  end
+
+  def test_yylex_star_arg
+    @lex.lex_state = :expr_arg
+
+    util_lex_token(" *a",
+                   :tSTAR, "*",
+                   :tIDENTIFIER, "a")
+
+    assert_equal :expr_arg, @lex.lex_state
+  end
+
+  def test_yylex_star_arg_beg
+    @lex.lex_state = :expr_beg
+
+    util_lex_token("*a",
+                   :tSTAR, "*",
+                   :tIDENTIFIER, "a")
+
+    assert_equal :expr_arg, @lex.lex_state
+  end
+
+  def test_yylex_star_arg_beg_fname
+    @lex.lex_state = :expr_fname
+
+    util_lex_token("*a",
+                   :tSTAR2, "*",
+                   :tIDENTIFIER, "a")
+
+    assert_equal :expr_arg, @lex.lex_state
+  end
+
+  def test_yylex_star_equals
+    util_lex_token("a *= ",
+                   :tIDENTIFIER, "a",
+                   :tOP_ASGN, "*")
+
+    assert_equal :expr_beg, @lex.lex_state
+  end
+
+  def test_yylex_string_bad_eos
+    util_bad_token('%',
+                   :tSTRING_BEG,     '%')
+  end
+
+  def test_yylex_string_bad_eos_quote
+    util_bad_token('%{nest',
+                   :tSTRING_BEG,     '%}')
+  end
+
   def test_yylex_string_double
     util_lex_token('"string"',
-                   :tSTRING_BEG,     t('"'),
-                   :tSTRING_CONTENT, s(:str, "string"),
-                   :tSTRING_END,     t('"'))
+                   :tSTRING, "string")
   end
 
-  def test_yylex_string_double_escapes
-    util_lex_token('"s\tri\ng"',
-                   :tSTRING_BEG,     t('"'),
-                   :tSTRING_CONTENT, s(:str, "s\tri\ng"),
-                   :tSTRING_END,     t('"'))
+  def test_yylex_string_double_escape_M
+    util_lex_token('"\\M-g"',
+                   :tSTRING, "\347")
   end
 
-  def test_yylex_string_double_escape_M
-    util_lex_token('"\M-g"',
-                   :tSTRING_BEG,     t('"'),
-                   :tSTRING_CONTENT, s(:str, "\347"),
-                   :tSTRING_END,     t('"'))
+  def test_yylex_string_escape_x_single
+    util_lex_token('"\\x0"',
+                   :tSTRING, "\000")
   end
 
-  def test_yylex_string_double_escape_octal
-    util_lex_token('"n = \101\102\103"',
-                   :tSTRING_BEG,     t('"'),
-                   :tSTRING_CONTENT, s(:str, "n = ABC"),
-                   :tSTRING_END,     t('"'))
+  def test_yylex_string_double_escape_chars
+    util_lex_token('"s\\tri\\ng"',
+                   :tSTRING, "s\tri\ng")
   end
 
   def test_yylex_string_double_escape_hex
-    util_lex_token('"n = \x61\x62\x63"',
-                   :tSTRING_BEG,     t('"'),
-                   :tSTRING_CONTENT, s(:str, "n = abc"),
-                   :tSTRING_END,     t('"'))
+    util_lex_token('"n = \\x61\\x62\\x63"',
+                   :tSTRING, "n = abc")
   end
 
-  def test_yylex_string_single
-    util_lex_token("'string'",
-                   :tSTRING_BEG,     t("'"),
-                   :tSTRING_CONTENT, s(:str, "string"),
-                   :tSTRING_END,     t("'"))
+  def test_yylex_string_double_escape_bs1
+    util_lex_token('"a\\a\\a"',
+                   :tSTRING, "a\a\a")
   end
 
-  def test_yylex_string_pct_Q
-    util_lex_token("%Q[string]",
-                   :tSTRING_BEG,     t("%Q["),
-                   :tSTRING_CONTENT, s(:str, "string"),
-                   :tSTRING_END,     t("]"))
+  def test_yylex_string_double_escape_bs2
+    util_lex_token('"a\\\\a"',
+                   :tSTRING, "a\\a")
   end
 
-  def test_yylex_string_single_escapes
-    util_lex_token("'s\\tri\\ng'",
-                   :tSTRING_BEG,     t("'"),
-                   :tSTRING_CONTENT, s(:str, "s\\tri\\ng"),
-                   :tSTRING_END,     t("'"))
+  def test_yylex_string_double_escape_octal
+    util_lex_token('"n = \\101\\102\\103"',
+                   :tSTRING, "n = ABC")
   end
 
-  def test_yylex_global
-    util_lex_token("$blah",
-                   :tGVAR,     t("$blah"))
+  def test_yylex_string_double_interp
+    util_lex_token("\"blah #x a \#@a b \#$b c \#{3} # \"",
+                   :tSTRING_BEG,     "\"",
+                   :tSTRING_CONTENT, "blah #x a ",
+                   :tSTRING_DVAR,    nil,
+                   :tSTRING_CONTENT, "@a b ",
+                   :tSTRING_DVAR,    nil,
+                   :tSTRING_CONTENT, "$b c ",
+                   :tSTRING_DBEG,    nil,
+                   :tSTRING_CONTENT, "3} # ",
+                   :tSTRING_END,     "\"")
   end
 
-  def test_yylex_global_wierd
-    util_lex_token("$__blah",
-                   :tGVAR,     t("$__blah"))
+  def test_yylex_string_double_nested_curlies
+    util_lex_token('%{nest{one{two}one}nest}',
+                   :tSTRING_BEG,     '%}',
+                   :tSTRING_CONTENT, "nest{one{two}one}nest",
+                   :tSTRING_END,     '}')
   end
 
-  def test_yylex_global_dollar_underscore
-    util_lex_token("$_",
-                   :tGVAR,     t("$_"))
+  def test_yylex_string_double_no_interp
+    util_lex_token("\"# blah\"",                                # pound first
+                   :tSTRING, "# blah")
+
+    util_lex_token("\"blah # blah\"",                           # pound not first
+                   :tSTRING, "blah # blah")
   end
 
-  def test_yylex_symbol
-    util_lex_token(":symbol",
-                   :tSYMBEG, t(":"),
-                   :tIDENTIFIER, t("symbol"))
+  def test_yylex_string_pct_Q
+    util_lex_token("%Q[s1 s2]",
+                   :tSTRING_BEG,     "%Q[",
+                   :tSTRING_CONTENT, "s1 s2",
+                   :tSTRING_END,     "]")
   end
 
-  def test_yylex_comment_begin
-    util_lex_token("=begin\nblah\nblah\n=end\n42",
-                   :tINTEGER, 42)
+  def test_yylex_string_pct_W
+    util_lex_token("%W[s1 s2\ns3]", # TODO: add interpolation to these
+                   :tWORDS_BEG,      "%W[",
+                   :tSTRING_CONTENT, "s1",
+                   :tSPACE,              nil,
+                   :tSTRING_CONTENT, "s2",
+                   :tSPACE,              nil,
+                   :tSTRING_CONTENT, "s3",
+                   :tSPACE,              nil,
+                   :tSTRING_END,     nil)
   end
 
-  def util_lex_token input, *args
-    @lex.src = StringIO.new input
+  def test_yylex_string_pct_W_bs_nl
+    util_lex_token("%W[s1 \\\ns2]", # TODO: add interpolation to these
+                   :tWORDS_BEG,      "%W[",
+                   :tSTRING_CONTENT, "s1",
+                   :tSPACE,              nil,
+                   :tSTRING_CONTENT, "\ns2",
+                   :tSPACE,              nil,
+                   :tSTRING_END,     nil)
+  end
 
-    until args.empty? do
-      token = args.shift
-      value = args.shift
-      assert @lex.advance, "no more tokens"
-      assert_equal [token, value], [@lex.token, @lex.yacc_value]
-    end
+  def test_yylex_string_pct_angle
+    util_lex_token("%<blah>",
+                   :tSTRING_BEG, "%>",
+                   :tSTRING_CONTENT, "blah",
+                   :tSTRING_END, ">")
+  end
 
-    deny @lex.advance, "must be empty, but had #{[@lex.token, @lex.yacc_value].inspect}"
+  def test_yylex_string_pct_other
+    util_lex_token("%%blah%",
+                   :tSTRING_BEG, "%%",
+                   :tSTRING_CONTENT, "blah",
+                   :tSTRING_END, "%")
   end
-end
 
-class TestStackState < Test::Unit::TestCase
-  def test_stack_state
-    s = StackState.new :test
-    s.push true
-    s.push false
-    s.lexpop
-    assert_equal [false, true], s.stack
+  def test_yylex_string_pct_w
+    util_bad_token("%w[s1 s2 ",
+                   :tAWORDS_BEG,     "%w[",
+                   :tSTRING_CONTENT, "s1",
+                   :tSPACE,              nil,
+                   :tSTRING_CONTENT, "s2",
+                   :tSPACE,              nil)
   end
 
-  def test_is_in_state
-    s = StackState.new :test
-    assert_equal false, s.is_in_state
-    s.push false
-    assert_equal false, s.is_in_state
-    s.push true
-    assert_equal true, s.is_in_state
-    s.push false
-    assert_equal false, s.is_in_state
+  def test_yylex_string_pct_w_bs_nl
+    util_lex_token("%w[s1 \\\ns2]",
+                   :tAWORDS_BEG,     "%w[",
+                   :tSTRING_CONTENT, "s1",
+                   :tSPACE,              nil,
+                   :tSTRING_CONTENT, "\ns2",
+                   :tSPACE,              nil,
+                   :tSTRING_END,     nil)
   end
 
-  def test_lexpop
-    s = StackState.new :test
-    assert_equal [false], s.stack
-    s.push true
-    s.push false
-    assert_equal [false, true, false], s.stack
-    s.lexpop
-    assert_equal [false, true], s.stack
+  def test_yylex_string_pct_w_bs_sp
+    util_lex_token("%w[s\\ 1 s\\ 2]",
+                   :tAWORDS_BEG,     "%w[",
+                   :tSTRING_CONTENT, "s 1",
+                   :tSPACE,              nil,
+                   :tSTRING_CONTENT, "s 2",
+                   :tSPACE,              nil,
+                   :tSTRING_END,     nil)
   end
 
-  def test_pop
-    s = StackState.new :test
-    assert_equal [false], s.stack
-    s.push true
-    assert_equal [false, true], s.stack
-    assert_equal true, s.pop
-    assert_equal [false], s.stack
+  def test_yylex_string_single
+    util_lex_token("'string'",
+                   :tSTRING, "string")
   end
 
-  def test_push
-    s = StackState.new :test
-    assert_equal [false], s.stack
-    s.push true
-    s.push false
-    assert_equal [false, true, false], s.stack
+  def test_yylex_string_single_escape_chars
+    util_lex_token("'s\\tri\\ng'",
+                   :tSTRING, "s\\tri\\ng")
   end
-end
 
-class TestEnvironment < Test::Unit::TestCase
-  def deny t
-    assert ! t
+  def test_yylex_string_single_nl
+    util_lex_token("'blah\\\nblah'",
+                   :tSTRING, "blah\\\nblah")
   end
 
-  def setup
-    @env = Environment.new
-    @env[:blah] = 42
-    assert_equal 42, @env[:blah]
+  def test_yylex_symbol
+    util_lex_token(":symbol",
+                   :tSYMBOL, "symbol")
   end
 
-  def test_use
-    @env.use :blah
-    expected = [{ :blah => true }]
-    assert_equal expected, @env.instance_variable_get(:"@use")
+  def test_yylex_symbol_bad_zero
+    util_bad_token(":\"blah\0\"",
+                   :tSYMBEG, ":")
   end
 
-  def test_use_scoped
-    @env.use :blah
-    @env.extend
-    expected = [{}, { :blah => true }]
-    assert_equal expected, @env.instance_variable_get(:"@use")
+  def test_yylex_symbol_double
+    util_lex_token(":\"symbol\"",
+                   :tSYMBEG, ":",
+                   :tSTRING_CONTENT, "symbol",
+                   :tSTRING_END, '"')
   end
 
-  def test_used_eh
-    @env.extend :dynamic
-    @env[:x] = :dvar
-    @env.use :x
-    assert_equal true, @env.used?(:x)
+  def test_yylex_symbol_single
+    util_lex_token(":'symbol'",
+                   :tSYMBEG, ":",
+                   :tSTRING_CONTENT, "symbol",
+                   :tSTRING_END, "'")
   end
 
-  def test_used_eh_none
-    assert_equal nil, @env.used?(:x)
+  def test_yylex_ternary
+    util_lex_token("a ? b : c",
+                   :tIDENTIFIER, "a",
+                   :tEH,         "?",
+                   :tIDENTIFIER, "b",
+                   :tCOLON,      ":",
+                   :tIDENTIFIER, "c")
+
+    util_lex_token("a ?bb : c", # GAH! MATZ!!!
+                   :tIDENTIFIER, "a",
+                   :tEH,         "?",
+                   :tIDENTIFIER, "bb",
+                   :tCOLON,      ":",
+                   :tIDENTIFIER, "c")
+
+    util_lex_token("42 ?", # 42 forces expr_end
+                   :tINTEGER, 42,
+                   :tEH,          "?")
   end
 
-  def test_used_eh_scoped
-    self.test_used_eh
-    @env.extend :dynamic
-    assert_equal true, @env.used?(:x)
+  def test_yylex_tilde
+    util_lex_token "~", :tTILDE, "~"
   end
 
-  def test_var_scope_dynamic
-    @env.extend :dynamic
-    assert_equal 42, @env[:blah]
-    @env.unextend
-    assert_equal 42, @env[:blah]
+  def test_yylex_tilde_unary
+    @lex.lex_state = :expr_fname
+    util_lex_token "~@", :tTILDE, "~"
   end
 
-  def test_var_scope_static
-    @env.extend
-    assert_equal nil, @env[:blah]
-    @env.unextend
-    assert_equal 42, @env[:blah]
+  def test_yylex_uminus
+    util_lex_token("-blah",
+                   :tUMINUS, "-",
+                   :tIDENTIFIER, "blah")
   end
 
-  def test_dynamic
-    expected1 = {}
-    expected2 = { :x => 42 }
+  def test_yylex_underscore
+    util_lex_token("_var", :tIDENTIFIER, "_var")
+  end
 
-    assert_equal expected1, @env.dynamic
-    begin
-      @env.extend :dynamic
-      assert_equal expected1, @env.dynamic
+  def test_yylex_underscore_end
+    @lex.src = "__END__\n"
+    deny @lex.advance
+  end
 
-      @env[:x] = 42
-      assert_equal expected2, @env.dynamic
+  def test_yylex_uplus
+    util_lex_token("+blah",
+                   :tUPLUS, "+",
+                   :tIDENTIFIER, "blah")
+  end
 
-      begin
-        @env.extend :dynamic
-        assert_equal expected2, @env.dynamic
-        @env.unextend
-      end
+  def test_zbug_float_in_decl
+    util_lex_token("def initialize(u = ",
+                   :kDEF, "def",
+                   :tIDENTIFIER, "initialize",
+                   :tLPAREN2, "(",
+                   :tIDENTIFIER, "u",
+                   :tEQL, "=")
 
-      assert_equal expected2, @env.dynamic
-      @env.unextend
-    end
-    assert_equal expected1, @env.dynamic
+    assert_equal :expr_beg, @lex.lex_state
+
+    util_lex_token("0.0, s = 0.0",
+                   :tFLOAT, 0.0,
+                   :tCOMMA, ',',
+                   :tIDENTIFIER, "s",
+                   :tEQL, "=",
+                   :tFLOAT, 0.0)
   end
 
-  def test_all_dynamic
-    expected = { :blah => 42 }
+  def test_zbug_id_equals
+    util_lex_token("a =",
+                   :tIDENTIFIER, "a",
+                   :tEQL, "=")
 
-    @env.extend :dynamic
-    assert_equal expected, @env.all
-    @env.unextend
-    assert_equal expected, @env.all
+    assert_equal :expr_beg, @lex.lex_state
+
+    util_lex_token("0.0",
+                   :tFLOAT, 0.0)
   end
 
-  def test_all_static
-    @env.extend
-    expected = { }
-    assert_equal expected, @env.all
+  def test_zbug_no_spaces_in_decl
+    util_lex_token("def initialize(u=",
+                   :kDEF, "def",
+                   :tIDENTIFIER, "initialize",
+                   :tLPAREN2, "(",
+                   :tIDENTIFIER, "u",
+                   :tEQL, "=")
 
-    @env.unextend
-    expected = { :blah => 42 }
-    assert_equal expected, @env.all
+    assert_equal :expr_beg, @lex.lex_state
+
+    util_lex_token("0.0,s=0.0",
+                   :tFLOAT, 0.0,
+                   :tCOMMA, ",",
+                   :tIDENTIFIER, "s",
+                   :tEQL, "=",
+                   :tFLOAT, 0.0)
   end
 
-  def test_dynamic_eh
-    assert_equal false, @env.dynamic?
-    @env.extend :dynamic
-    assert_equal true, @env.dynamic?
-    @env.extend
-    assert_equal false, @env.dynamic?
+  ############################################################
+
+  def util_bad_token s, *args
+    assert_raises SyntaxError do
+      util_lex_token s, *args
+    end
   end
 
-  def test_all_static_deeper
-    expected0 = { :blah => 42 }
-    expected1 = { :blah => 42, :blah2 => 24 }
-    expected2 = { :blah => 27 }
+  def util_escape expected, input
+    @lex.src = input
+    assert_equal expected, @lex.read_escape
+  end
 
-    @env.extend :dynamic
-    @env[:blah2] = 24
-    assert_equal expected1, @env.all
+  def util_escape_bad input
+    @lex.src = input
+    assert_raises SyntaxError do
+      @lex.read_escape
+    end
+  end
 
-    @env.extend 
-    @env[:blah] = 27
-    assert_equal expected2, @env.all
+  def util_lex_fname name, type, end_state = :expr_arg
+    @lex.lex_state = :expr_fname # can only set via parser's defs
 
-    @env.unextend
-    assert_equal expected1, @env.all
+    util_lex_token("def #{name} ", :kDEF, "def", type, name)
 
-    @env.unextend
-    assert_equal expected0, @env.all
+    assert_equal end_state, @lex.lex_state
   end
+
+  def util_lex_token input, *args
+    @lex.src = input
+
+    until args.empty? do
+      token = args.shift
+      value = args.shift
+      assert @lex.advance, "no more tokens"
+      assert_equal [token, value], [@lex.token, @lex.yacc_value]
+    end
+
+    deny @lex.advance, "must be empty, but had #{[@lex.token, @lex.yacc_value].inspect}"
+  end
 end
+

test/test_ruby_parser.rb

@@ -11,56 +11,26 @@ #!/usr/local/bin/ruby
 
-class TestRubyParser < Test::Unit::TestCase # ParseTreeTestCase
+class RubyParser
+  def process input
+    parse input
+  end
+end
 
-  # Regular ParseTreeTestCase tests
-  eval ParseTreeTestCase.testcases.map { |node, data|
-    next if node.to_s =~ /bmethod|dmethod/
-    next if Array === data['Ruby'] # runtime only crap
-    "def test_#{node}
-       rb = #{data['Ruby'].inspect}
-       pt = #{data['ParseTree'].inspect}
+class RubyParserTestCase < ParseTreeTestCase
+  def self.previous key
+    "Ruby"
+  end
 
-       assert_not_nil rb, \"Ruby for #{node} undefined\"
-       assert_not_nil pt, \"ParseTree for #{node} undefined\"
+  def self.generate_test klass, node, data, input_name, output_name
+    return if node.to_s =~ /bmethod|dmethod/
+    return if Array === data['Ruby']
 
-       assert_equal Sexp.from_array(pt), @processor.parse(rb)
-     end"
-  }.compact.join("\n")
+    output_name = "ParseTree"
 
-  # Scour the world and compare against ParseTree
-  if ENV['ZOMGPONIES'] or File.exist? 'zomgponies' then
-    require 'parse_tree'
+    super
+  end
+end
 
-    base = "/usr/lib/ruby"
-    base = "unit"
+class TestRubyParser < RubyParserTestCase
+  alias :refute_nil :assert_not_nil unless defined? Mini
 
-    files = Dir[File.join(base, "**/*.rb")]
-
-    # these files/patterns cause parse_tree_show to bus error (or just suck):
-    files.reject! { |f| f =~ /environments.environment|rss.maker.base|rails_generator|ferret.browser|rubinius.spec.core.module.(constants|name|remove_const)_spec|tkextlib.tcllib.tablelist/ }
-
-    # these are rejected for dasgn_curr ordering failures... I'll fix them later.
-    # (or mri parse errors--I should have separated them out)
-    files.reject! { |f| f =~ /lib.flog|lib.autotest.notify|lib.analyzer.tools.rails.stat|flog.lib.flog|rakelib.struct.generator|rubinius.kernel.core.array|lib.rbosa.rb|src.rbosa.rb|spec.spec.mocks.mock.spec.rb|dsl.shared.behaviour.spec.rb|spec.spec.dsl.behaviour.spec|lib.hpricot.parse.rb|resolve.rb|parsers.parse.f95|rubinius.shotgun.lib.primitives.ltm|rubinius.lib.bin.compile|rubinius.kernel.core.object|rubinius.kernel.core.file|rubinius.compiler2.garnet.bindingagent|ruby_to_c_test_r2ctestcase|lib.more.like.this|resolv\.rb|test.r2ctestcase/ }
-
-    warn "Generating #{files.size} tests from #{base}"
-
-    eval files.map { |file|
-      name = file[base.size..-1].gsub(/\W+/, '_')
-
-      loc = `wc -l #{file}`.strip.to_i
-
-      "def test#{name}_#{loc}
-         file = #{file.inspect}
-         rb = File.read(file)
-
-         pt = ParseTree.new.parse_tree_for_string rb
-         assert_not_nil pt, \"ParseTree for #{name} undefined\"
-
-         rp = @processor.parse rb
-         assert_equal Sexp.from_array(pt).first, rp, \"RP different from PT\"
-         File.unlink #{file.inspect}
-       end"
-    }.compact.join("\n")
-  end
-
   def setup
@@ -74,2 +44,18 @@ super
 
+  def test_attrasgn_array_lhs
+    rb = '[1, 2, 3, 4][from .. to] = ["a", "b", "c"]'
+    pt = s(:attrasgn,
+           s(:array, s(:lit, 1), s(:lit, 2), s(:lit, 3), s(:lit, 4)),
+           :[]=,
+           s(:arglist,
+             s(:dot2,
+               s(:call, nil, :from, s(:arglist)),
+               s(:call, nil, :to, s(:arglist))),
+             s(:array, s(:str, "a"), s(:str, "b"), s(:str, "c"))))
+
+    result = @processor.parse(rb)
+
+    assert_equal pt, result
+  end
+
   def test_block_append
@@ -82,2 +68,9 @@ head = s(:args)
 
+  def test_block_append_begin_begin
+    head = s(:begin, s(:args))
+    tail = s(:begin, s(:args))
+    expected = s(:block, s(:args), s(:begin, s(:args)))
+    assert_equal expected, @processor.block_append(head, tail)
+  end
+
   def test_block_append_block
@@ -90,18 +83,2 @@ head = s(:block, s(:args))
 
-  def test_block_append_tail_block
-    head = s(:vcall, :f1)
-    tail = s(:block, s(:undef, s(:lit, :x)), s(:undef, s(:lit, :y)))
-    expected = s(:block,
-                 s(:vcall, :f1),
-                 s(:block, s(:undef, s(:lit, :x)), s(:undef, s(:lit, :y))))
-    assert_equal expected, @processor.block_append(head, tail)
-  end
-
-  def test_block_append_begin_begin
-    head = s(:begin, s(:args))
-    tail = s(:begin, s(:args))
-    expected = s(:block, s(:args), s(:begin, s(:args)))
-    assert_equal expected, @processor.block_append(head, tail)
-  end
-
   def test_block_append_nil_head
@@ -121,5 +98,14 @@ head = nil
 
+  def test_block_append_tail_block
+    head = s(:call, nil, :f1, s(:arglist))
+    tail = s(:block, s(:undef, s(:lit, :x)), s(:undef, s(:lit, :y)))
+    expected = s(:block,
+                 s(:call, nil, :f1, s(:arglist)),
+                 s(:block, s(:undef, s(:lit, :x)), s(:undef, s(:lit, :y))))
+    assert_equal expected, @processor.block_append(head, tail)
+  end
+
   def test_call_env
     @processor.env[:a] = :lvar
-    expected = s(:call, s(:lvar, :a), :happy)
+    expected = s(:call, s(:lvar, :a), :happy, s(:arglist))
 
@@ -129,7 +115,68 @@ assert_equal expected, @processor.parse('a.happy')
 
+  def test_dasgn_icky2
+    rb = "a do\n  v = nil\n  begin\n    yield\n  rescue Exception => v\n    break\n  end\nend"
+    pt = s(:iter,
+           s(:call, nil, :a, s(:arglist)),
+           nil,
+           s(:block,
+             s(:lasgn, :v, s(:nil)),
+             s(:rescue,
+               s(:yield),
+               s(:resbody,
+                 s(:array, s(:const, :Exception), s(:lasgn, :v, s(:gvar, :$!))),
+                 s(:break)))))
+
+    assert_equal pt, @processor.parse(rb)
+  end
+
+  def test_class_comments
+    rb = "# blah 1\n# blah 2\n\nclass X\n  # blah 3\n  def blah\n    # blah 4\n  end\nend"
+    pt = s(:class, :X, nil,
+           s(:scope,
+             s(:defn, :blah, s(:args), s(:scope, s(:block, s(:nil))))))
+
+    actual = @processor.parse(rb)
+    assert_equal pt, actual
+
+    assert_equal "# blah 1\n# blah 2\n\n", actual.comments
+    assert_equal "# blah 3\n", actual.scope.defn.comments
+  end
+
+  def test_module_comments
+    rb = "# blah 1\n  \n  # blah 2\n\nmodule X\n  # blah 3\n  def blah\n    # blah 4\n  end\nend"
+    pt = s(:module, :X,
+           s(:scope,
+             s(:defn, :blah, s(:args), s(:scope, s(:block, s(:nil))))))
+
+    actual = @processor.parse(rb)
+    assert_equal pt, actual
+    assert_equal "# blah 1\n\n# blah 2\n\n", actual.comments
+    assert_equal "# blah 3\n", actual.scope.defn.comments
+  end
+
+  def test_defn_comments
+    rb = "# blah 1\n# blah 2\n\ndef blah\nend"
+    pt = s(:defn, :blah, s(:args), s(:scope, s(:block, s(:nil))))
+
+    actual = @processor.parse(rb)
+    assert_equal pt, actual
+    assert_equal "# blah 1\n# blah 2\n\n", actual.comments
+  end
+
+  def test_defs_comments
+    rb = "# blah 1\n# blah 2\n\ndef self.blah\nend"
+    pt = s(:defs, s(:self), :blah, s(:args), s(:scope, s(:block)))
+
+    actual = @processor.parse(rb)
+    assert_equal pt, actual
+    assert_equal "# blah 1\n# blah 2\n\n", actual.comments
+  end
+
   def test_do_bug # TODO: rename
     rb = "a 1\na.b do |c|\n  # do nothing\nend"
     pt = s(:block,
-           s(:fcall, :a, s(:array, s(:lit, 1))),
-           s(:iter, s(:call, s(:vcall, :a), :b), s(:dasgn_curr, :c)))
+           s(:call, nil, :a, s(:arglist, s(:lit, 1))),
+           s(:iter,
+             s(:call, s(:call, nil, :a, s(:arglist)), :b, s(:arglist)),
+             s(:lasgn, :c)))
 
@@ -139,2 +186,37 @@ assert_equal pt, @processor.parse(rb)
 
+  def test_dstr_evstr
+    rb = "\"#\{'a'}#\{b}\""
+    pt = s(:dstr, "a", s(:evstr, s(:call, nil, :b, s(:arglist))))
+
+    assert_equal pt, @processor.parse(rb)
+  end
+
+  def test_dstr_str
+    rb = "\"#\{'a'} b\""
+    pt = s(:str, "a b")
+
+    assert_equal pt, @processor.parse(rb)
+  end
+
+  def test_empty
+    rb = ""
+    pt = nil
+
+    assert_equal pt, @processor.parse(rb)
+  end
+
+  def test_evstr_evstr
+    rb = "\"#\{a}#\{b}\""
+    pt = s(:dstr, "", s(:evstr, s(:call, nil, :a, s(:arglist))), s(:evstr, s(:call, nil, :b, s(:arglist))))
+
+    assert_equal pt, @processor.parse(rb)
+  end
+
+  def test_evstr_str
+    rb = "\"#\{a} b\""
+    pt = s(:dstr, "", s(:evstr, s(:call, nil, :a, s(:arglist))), s(:str, " b"))
+
+    assert_equal pt, @processor.parse(rb)
+  end
+
   def test_lasgn_env
@@ -144,3 +226,3 @@ rb = 'a = 42'
     expected_env = { :a => :lvar }
-    
+
     assert_equal pt, @processor.parse(rb)
@@ -150,2 +232,82 @@ assert_equal expected_env, @processor.env.all
 
+  def test_list_append
+    a = s(:lit, 1)
+    b = s(:lit, 2)
+    c = s(:lit, 3)
+
+    result = @processor.list_append(s(:array, b.dup), c.dup)
+
+    assert_equal s(:array, b, c), result
+
+    result = @processor.list_append(b.dup, c.dup)
+
+    assert_equal s(:array, b, c), result
+
+    result = @processor.list_append(result, a.dup)
+
+    assert_equal s(:array, b, c, a), result
+
+    lhs, rhs = s(:array, s(:lit, :iter)), s(:when, s(:const, :BRANCHING), nil)
+    expected = s(:array, s(:lit, :iter), s(:when, s(:const, :BRANCHING), nil))
+
+    assert_equal expected, @processor.list_append(lhs, rhs)
+  end
+
+  def test_list_prepend
+    a = s(:lit, 1)
+    b = s(:lit, 2)
+    c = s(:lit, 3)
+
+    result = @processor.list_prepend(b.dup, s(:array, c.dup))
+
+    assert_equal s(:array, b, c), result
+
+    result = @processor.list_prepend(b.dup, c.dup)
+
+    assert_equal s(:array, b, c), result
+
+    result = @processor.list_prepend(a.dup, result)
+
+    assert_equal s(:array, a, b, c), result
+  end
+
+  def test_literal_concat_dstr_dstr
+    lhs      = s(:dstr, "Failed to download spec ",
+                 s(:evstr, s(:call, nil, :spec_name, s(:arglist))),
+                 s(:str, " from "),
+                 s(:evstr, s(:call, nil, :source_uri, s(:arglist))),
+                 s(:str, ":\n"))
+    rhs      = s(:dstr, "\t",
+                 s(:evstr, s(:call, s(:ivar, :@fetch_error), :message)))
+    expected = s(:dstr, "Failed to download spec ",
+                 s(:evstr, s(:call, nil, :spec_name, s(:arglist))),
+                 s(:str, " from "),
+                 s(:evstr, s(:call, nil, :source_uri, s(:arglist))),
+                 s(:str, ":\n"),
+                 s(:str, "\t"),
+                 s(:evstr, s(:call, s(:ivar, :@fetch_error), :message)))
+
+    assert_equal expected, @processor.literal_concat(lhs, rhs)
+  end
+
+  def test_literal_concat_dstr_evstr
+    lhs, rhs = s(:dstr, "a"), s(:evstr, s(:call, nil, :b, s(:arglist)))
+    expected = s(:dstr, "a", s(:evstr, s(:call, nil, :b, s(:arglist))))
+
+    assert_equal expected, @processor.literal_concat(lhs, rhs)
+  end
+
+  def test_literal_concat_evstr_evstr
+    lhs, rhs = s(:evstr, s(:lit, 1)), s(:evstr, s(:lit, 2))
+    expected = s(:dstr, "", s(:evstr, s(:lit, 1)), s(:evstr, s(:lit, 2)))
+
+    assert_equal expected, @processor.literal_concat(lhs, rhs)
+  end
+
+  def test_literal_concat_str_evstr
+    lhs, rhs = s(:str, ""), s(:evstr, s(:str, "blah"))
+
+    assert_equal s(:str, "blah"), @processor.literal_concat(lhs, rhs)
+  end
+
   def test_logop_12
@@ -159,6 +321,14 @@ lhs = s(:lit, 1)
 
-  def test_logop_12_3
-    lhs = s(:and, s(:lit, 1), s(:lit, 2))
-    rhs = s(:lit, 3)
-    exp = s(:and, s(:lit, 1), s(:and, s(:lit, 2), s(:lit, 3)))
+  def test_logop_1234_5
+    lhs = s(:and, s(:lit, 1), s(:and, s(:lit, 2), s(:and, s(:lit, 3), s(:lit, 4))))
+    rhs = s(:lit, 5)
+    exp = s(:and,
+            s(:lit, 1),
+            s(:and,
+              s(:lit, 2),
+              s(:and,
+                s(:lit, 3),
+                s(:and,
+                  s(:lit, 4),
+                  s(:lit, 5)))))
 
@@ -182,14 +352,6 @@ assert_equal exp, @processor.logop(:and, lhs, rhs)
 
-  def test_logop_1234_5
-    lhs = s(:and, s(:lit, 1), s(:and, s(:lit, 2), s(:and, s(:lit, 3), s(:lit, 4))))
-    rhs = s(:lit, 5)
-    exp = s(:and,
-            s(:lit, 1),
-            s(:and,
-              s(:lit, 2),
-              s(:and,
-                s(:lit, 3),
-                s(:and,
-                  s(:lit, 4),
-                  s(:lit, 5)))))
+  def test_logop_12_3
+    lhs = s(:and, s(:lit, 1), s(:lit, 2))
+    rhs = s(:lit, 3)
+    exp = s(:and, s(:lit, 1), s(:and, s(:lit, 2), s(:lit, 3)))
 
@@ -200,7 +362,7 @@ assert_equal exp, @processor.logop(:and, lhs, rhs)
   def test_logop_nested_mix
-    lhs = s(:or, s(:vcall, :a), s(:vcall, :b))
-    rhs = s(:and, s(:vcall, :c), s(:vcall, :d))
+    lhs = s(:or, s(:call, nil, :a, s(:arglist)), s(:call, nil, :b, s(:arglist)))
+    rhs = s(:and, s(:call, nil, :c, s(:arglist)), s(:call, nil, :d, s(:arglist)))
     exp = s(:or,
-            s(:or, s(:vcall, :a), s(:vcall, :b)),
-            s(:and, s(:vcall, :c), s(:vcall, :d)))
+            s(:or, s(:call, nil, :a, s(:arglist)), s(:call, nil, :b, s(:arglist))),
+            s(:and, s(:call, nil, :c, s(:arglist)), s(:call, nil, :d, s(:arglist))))
 
@@ -213,44 +375,12 @@ lhs.paren = true
 
-  def test_literal_concat_str_evstr
-    lhs, rhs = s(:str, ""), s(:evstr, s(:str, "blah"))
+  def test_str_evstr
+    rb = "\"a #\{b}\""
+    pt = s(:dstr, "a ", s(:evstr, s(:call, nil, :b, s(:arglist))))
 
-    assert_equal s(:str, "blah"), @processor.literal_concat(lhs, rhs)
+    assert_equal pt, @processor.parse(rb)
   end
 
-  def test_literal_concat_evstr_evstr
-    lhs, rhs = s(:evstr, s(:lit, 1)), s(:evstr, s(:lit, 2))
-    expected = s(:dstr, "", s(:evstr, s(:lit, 1)), s(:evstr, s(:lit, 2)))
-
-    assert_equal expected, @processor.literal_concat(lhs, rhs)
-  end
-
-  def test_literal_concat_dstr_evstr
-    lhs, rhs = s(:dstr, "a"), s(:evstr, s(:vcall, :b))
-    expected = s(:dstr, "a", s(:evstr, s(:vcall, :b)))
-
-    assert_equal expected, @processor.literal_concat(lhs, rhs)
-  end
-
-  def test_literal_concat_dstr_dstr
-    lhs      = s(:dstr, "Failed to download spec ",
-                 s(:evstr, s(:vcall, :spec_name)),
-                 s(:str, " from "),
-                 s(:evstr, s(:vcall, :source_uri)),
-                 s(:str, ":\n"))
-    rhs      = s(:dstr, "\t",
-                 s(:evstr, s(:call, s(:ivar, :@fetch_error), :message)))
-    expected = s(:dstr, "Failed to download spec ",
-                 s(:evstr, s(:vcall, :spec_name)),
-                 s(:str, " from "),
-                 s(:evstr, s(:vcall, :source_uri)),
-                 s(:str, ":\n"),
-                 s(:str, "\t"),
-                 s(:evstr, s(:call, s(:ivar, :@fetch_error), :message)))
-
-    assert_equal expected, @processor.literal_concat(lhs, rhs)
-  end
-
   def test_str_pct_Q_nested
     rb = "%Q[before [#\{nest}] after]"
-    pt = s(:dstr, "before [", s(:evstr, s(:vcall, :nest)), s(:str, "] after"))
+    pt = s(:dstr, "before [", s(:evstr, s(:call, nil, :nest, s(:arglist))), s(:str, "] after"))
 
@@ -274,67 +404,79 @@ assert_equal pt, @processor.parse(rb)
 
-  def test_dstr_str
-    rb = "\"#\{'a'} b\""
-    pt = s(:str, "a b")
+  STARTING_LINE = {
+    "begin_def"                           => 2,
+    "block_stmt_after"                    => 2,
+    "block_stmt_after_mri_verbose_flag"   => 2,
+    "block_stmt_before"                   => 2,
+    "block_stmt_before_mri_verbose_flag"  => 2,
+    "block_stmt_both"                     => 2,
+    "block_stmt_both_mri_verbose_flag"    => 2,
+    "case_nested_inner_no_expr"           => 2,
+    "case_no_expr"                        => 2,
+    "case_splat"                          => 2,
+    "cvasgn"                              => 2,
+    "defn_args_none"                      => 2,
+    "defn_zarray"                         => 2,
+    "structure_unused_literal_wwtt"       => 3, # yes, 3... odd test
+    "super_0"                             => 2,
+    "super_1"                             => 2,
+    "super_1_array"                       => 2,
+    "super_n"                             => 2,
+    "super_multi"                         => 2,
+    "undef_block_1"                       => 2,
+    "undef_block_2"                       => 2,
+    "undef_block_3"                       => 2,
+    "undef_block_wtf"                     => 2,
+    "zsuper"                              => 2,
+  }
 
-    assert_equal pt, @processor.parse(rb)
+  def after_process_hook klass, node, data, input_name, output_name
+    expected = STARTING_LINE[node] || 1
+    assert_equal expected, @result.line, "should have proper line number"
   end
 
-  def test_str_evstr
-    rb = "\"a #\{b}\""
-    pt = s(:dstr, "a ", s(:evstr, s(:vcall, :b)))
+  def test_position_info
+    rb = "a = 42\np a"
+    pt = s(:block,
+           s(:lasgn, :a, s(:lit, 42)),
+           s(:call, nil, :p, s(:arglist, s(:lvar, :a))))
 
-    assert_equal pt, @processor.parse(rb)
-  end
+    result = @processor.parse(rb, "blah.rb")
 
-  def test_dstr_evstr
-    rb = "\"#\{'a'}#\{b}\""
-    pt = s(:dstr, "a", s(:evstr, s(:vcall, :b)))
+    assert_equal pt, result
 
-    assert_equal pt, @processor.parse(rb)
-  end
+    assert_equal 1, result.line,       "block should have line number"
+    assert_equal 1, result.lasgn.line, "lasgn should have line number"
+    assert_equal 2, result.call.line,  "call should have line number"
 
-  def test_evstr_str
-    rb = "\"#\{a} b\""
-    pt = s(:dstr, "", s(:evstr, s(:vcall, :a)), s(:str, " b"))
+    expected = "blah.rb"
 
-    assert_equal pt, @processor.parse(rb)
-  end
+    assert_equal expected, result.file
+    assert_equal expected, result.lasgn.file
+    assert_equal expected, result.call.file
 
-  def test_evstr_evstr
-    rb = "\"#\{a}#\{b}\""
-    pt = s(:dstr, "", s(:evstr, s(:vcall, :a)), s(:evstr, s(:vcall, :b)))
-
-    assert_equal pt, @processor.parse(rb)
+    assert_same result.file, result.lasgn.file
+    assert_same result.file, result.call.file
   end
 
+  def test_position_info2
+    rb = "def x(y)\n  p(y)\n  y *= 2\n  return y;\nend" # TODO: remove () & ;
+    pt = s(:defn, :x, s(:args, :y),
+           s(:scope,
+             s(:block,
+               s(:call, nil, :p, s(:arglist, s(:lvar, :y))),
+               s(:lasgn, :y,
+                 s(:call, s(:lvar, :y), :*, s(:arglist, s(:lit, 2)))),
+               s(:return, s(:lvar, :y)))))
 
-  def test_dasgn_icky2
-    rb = "a do\n  v = nil\n  begin\n    yield\n  rescue Exception => v\n    break\n  end\nend"
-    pt = s(:iter,
-           s(:fcall, :a),
-           nil,
-           s(:block,
-             s(:dasgn_curr, :v, s(:nil)),
-             s(:begin,
-               s(:rescue,
-                 s(:yield),
-                 s(:resbody,
-                   s(:array, s(:const, :Exception)),
-                   s(:block, s(:dasgn_curr, :v, s(:gvar, :$!)), s(:break)))))))
+    result = @processor.parse(rb)
 
-    assert_equal pt, @processor.parse(rb)
-  end
+    assert_equal pt, result
 
-  def test_list_append
-    lhs, rhs = s(:array, s(:lit, :iter)), s(:when, s(:const, :BRANCHING), nil)
-    expected = s(:array, s(:lit, :iter), s(:when, s(:const, :BRANCHING), nil))
+    body = result.scope.block
 
-    assert_equal expected, @processor.list_append(lhs, rhs)
+    assert_equal 1, result.line,      "defn should have line number"
+    assert_equal 2, body.call.line,   "call should have line number"
+    assert_equal 3, body.lasgn.line,  "lasgn should have line number"
+    assert_equal 4, body.return.line, "return should have line number"
   end
 end
-
-__END__
-
-# blah18.rb
-
-assert_equal("sub", $_)

No alert changes

No metric changes