ruby_parser - rubygems Package Compare versions

Comparing version 3.9.0 to 3.10.0

History.rdoc

@@ -0,1 +1,18 @@
+=== 3.10.0 / 2017-07-17
+
+* 4 minor enhancements:
+
+  * Added support for 'squiggly heredocs' (indented content). (jjoos)
+  * Bumped oedipus_lex to ~> 2.5.
+  * Bumped sexp_processor to ~> 4.9.
+  * Made STRICT_SEXP=1 safe: converted indexed sexps to semantic calls where needed.
+
+* 5 bug fixes:
+
+  * Clear and restore cmdarg stack around defs args and body. (presidentbeef)
+  * Conditionalize use of sexp_body= to make it compatible w/ sexp_processor 4.9.0
+  * Fix up line numbers in strings with newlines and `#`. (presidentbeef)
+  * Fixed line numbers of resbodies.
+  * Fixed some tracking of lexical state. Fixes bug #249.
+
 === 3.9.0 / 2017-04-13
@@ -2,0 +19,0 @@

lib/rp_extensions.rb

@@ -41,2 +41,6 @@ # :stopdoc:
   attr_accessor :lineno
+
+  def clean_caller
+    self.sub(File.dirname(__FILE__), ".").sub(/:in.*/, "")
+  end if $DEBUG
 end
@@ -70,3 +74,3 @@
   def block_pass?
-    any? { |s| Sexp === s && s[0] == :block_pass }
+    any? { |s| Sexp === s && s.sexp_type == :block_pass }
   end
@@ -73,0 +77,0 @@

lib/ruby_lexer.rb

@@ -25,2 +25,3 @@ # encoding: UTF-8
   STR_FUNC_INDENT = 0x20 # <<-HEREDOC
+  STR_FUNC_ICNTNT = 0x40 # <<~HEREDOC
 
@@ -66,2 +67,4 @@ STR_SQUOTE = STR_FUNC_BORING
 
+  TAB_WIDTH = 8
+
   @@regexp_cache = Hash.new { |h,k| h[k] = Regexp.new(Regexp.escape(k)) }
@@ -94,2 +97,13 @@ @@regexp_cache[nil] = nil
 
+  if $DEBUG then
+    alias lex_state= lex_state=
+    def lex_state=o
+      return if @lex_state == o
+      c = caller.first
+      c = caller[1] if c =~ /\bresult\b/
+      warn "lex_state: %p -> %p from %s" % [@lex_state, o, c.clean_caller]
+      @lex_state = o
+    end
+  end
+
   # Last token read via next_token.
@@ -108,3 +122,7 @@ attr_accessor :token
     self.version = v
+    @lex_state = :expr_none
 
+    self.cmdarg = RubyParserStuff::StackState.new(:cmdarg, $DEBUG)
+    self.cond   = RubyParserStuff::StackState.new(:cond, $DEBUG)
+
     reset
@@ -153,6 +171,7 @@ end
 
-    indent  = (func & STR_FUNC_INDENT) != 0 ? "[ \t]*" : nil
-    expand  = (func & STR_FUNC_EXPAND) != 0
-    eos_re  = /#{indent}#{Regexp.escape eos}(\r*\n|\z)/
-    err_msg = "can't match #{eos_re.inspect} anywhere in "
+    indent         = (func & STR_FUNC_INDENT) != 0 ? "[ \t]*" : nil
+    content_indent = (func & STR_FUNC_ICNTNT) != 0
+    expand         = (func & STR_FUNC_EXPAND) != 0
+    eos_re         = /#{indent}#{Regexp.escape eos}(\r*\n|\z)/
+    err_msg        = "can't match #{eos_re.inspect} anywhere in "
 
@@ -203,5 +222,48 @@ rb_compile_error err_msg if end_of_stream?
 
-    return :tSTRING_CONTENT, string_buffer.join.delete("\r")
+    string_content = string_buffer.join.delete("\r")
+
+    string_content = heredoc_dedent(string_content) if content_indent && ruby23?
+
+    return :tSTRING_CONTENT, string_content
   end
 
+  def heredoc_dedent(string_content)
+    width = string_content.scan(/^[ \t]*(?=\S)/).map do |whitespace|
+      heredoc_whitespace_indent_size whitespace
+    end.min || 0
+
+    string_content.split("\n", -1).map do |line|
+      dedent_string line, width
+    end.join "\n"
+  end
+
+  def dedent_string(string, width)
+    characters_skipped = 0
+    indentation_skipped = 0
+
+    string.chars.each do |char|
+      break if indentation_skipped >= width
+      if char == ' '
+        characters_skipped += 1
+        indentation_skipped += 1
+      elsif char == "\t"
+        proposed = TAB_WIDTH * (indentation_skipped / TAB_WIDTH + 1)
+        break if (proposed > width)
+        characters_skipped += 1
+        indentation_skipped = proposed
+      end
+    end
+    string[characters_skipped..-1]
+  end
+
+  def heredoc_whitespace_indent_size(whitespace)
+    whitespace.chars.inject 0 do |size, char|
+      if char == "\t"
+        size + TAB_WIDTH
+      else
+        size + 1
+      end
+    end
+  end
+
   def heredoc_identifier # TODO: remove / rewrite
@@ -211,6 +273,10 @@ term, func = nil, STR_FUNC_BORING
 
+    heredoc_indent_mods = '-'
+    heredoc_indent_mods += '\~' if ruby23?
+
     case
-    when scan(/(-?)([\'\"\`])(.*?)\2/) then
+    when scan(/([#{heredoc_indent_mods}]?)([\'\"\`])(.*?)\2/) then
       term = ss[2]
       func |= STR_FUNC_INDENT unless ss[1].empty?
+      func |= STR_FUNC_ICNTNT if ss[1] == '~'
       func |= case term
@@ -225,5 +291,5 @@ when "\'" then
       string_buffer << ss[3]
-    when scan(/-?([\'\"\`])(?!\1*\Z)/) then
+    when scan(/[#{heredoc_indent_mods}]?([\'\"\`])(?!\1*\Z)/) then
       rb_compile_error "unterminated here document identifier"
-    when scan(/(-?)(#{IDENT_CHAR}+)/) then
+    when scan(/([#{heredoc_indent_mods}]?)(#{IDENT_CHAR}+)/) then
       term = '"'
@@ -233,2 +299,3 @@ func |= STR_DQUOTE
         func |= STR_FUNC_INDENT
+        func |= STR_FUNC_ICNTNT if ss[1] == '~'
       end
@@ -519,2 +586,3 @@ string_buffer << ss[2]
 
+    # TODO: add :expr_label to :expr_beg (set in expr_result below)
     return expr_result(token, "(")
@@ -830,2 +898,3 @@ end
     when scan(/n/) then                   # newline
+      self.extra_lineno -= 1
       "\n"
@@ -902,3 +971,3 @@ when scan(/t/) then                   # horizontal tab
     self.comments      = []
-    self.lex_state     = nil
+    self.lex_state     = :expr_none
     self.lex_strterm   = nil
@@ -913,4 +982,4 @@ self.lineno        = 1
 
-    self.cmdarg = RubyParserStuff::StackState.new(:cmdarg)
-    self.cond   = RubyParserStuff::StackState.new(:cond)
+    self.cmdarg.reset
+    self.cond.reset
   end
@@ -1122,2 +1191,6 @@
 
+  def ruby23?
+    Ruby23Parser === parser
+  end
+
   def process_string # TODO: rewrite / remove
@@ -1124,0 +1197,0 @@ token = if lex_strterm[0] == :heredoc then

lib/ruby_parser_extras.rb

@@ -10,3 +10,3 @@ # encoding: ASCII-8BIT
 module RubyParserStuff
-  VERSION = "3.9.0"
+  VERSION = "3.10.0"
 
@@ -56,3 +56,3 @@ attr_accessor :lexer, :in_def, :in_single, :file
   def arg_blk_pass node1, node2 # TODO: nuke
-    node1 = s(:arglist, node1) unless [:arglist, :call_args, :array, :args].include? node1.first
+    node1 = s(:arglist, node1) unless [:arglist, :call_args, :array, :args].include? node1.sexp_type 
     node1 << node2 if node2
@@ -72,3 +72,3 @@ node1
       if sexp.size == 2 and sexp[1].sexp_type == :array then
-        s(:masgn, *sexp[1][1..-1].map { |sub| clean_mlhs sub })
+        s(:masgn, *sexp[1].sexp_body.map { |sub| clean_mlhs sub })
       else
@@ -92,3 +92,3 @@ debug20 5
     result = self.args args
-    result[0] = :masgn
+    result.sexp_type = :masgn
     result
@@ -108,3 +108,3 @@ end
     if ary.length > 2 or ary.splat then # HACK
-      s(:masgn, *ary[1..-1])
+      s(:masgn, *ary.sexp_body)
     else
@@ -120,3 +120,3 @@ ary.last
     else
-      s(:hash, *array[1..-1])
+      s(:hash, *array.sexp_body)
     end
@@ -133,3 +133,3 @@ end
         when :array, :args, :call_args then # HACK? remove array at some point
-          result.concat arg[1..-1]
+          result.concat arg.sexp_body
         else
@@ -158,3 +158,3 @@ result << arg
         when :args, :block, :array, :call_args then # HACK call_args mismatch
-          result.concat arg[1..-1]
+          result.concat arg.sexp_body
         when :block_arg then
@@ -188,4 +188,4 @@ result << :"&#{arg.last}"
   def aryset receiver, index
-    index ||= []
-    s(:attrasgn, receiver, :"[]=", *index[1..-1]).compact # [][1..-1] => nil
+    index ||= s()
+    s(:attrasgn, receiver, :"[]=", *index.sexp_body).compact # [].sexp_body => nil
   end
@@ -245,3 +245,3 @@
 
-    case node.first
+    case node.sexp_type
     when :lit then
@@ -260,7 +260,9 @@ if Regexp === node.last then
       env[label] = :lvar
-      return s(:flip2, node[1], node[2])
+      _, lhs, rhs = node
+      return s(:flip2, lhs, rhs)
     when :dot3 then
       label = "flip#{node.hash}"
       env[label] = :lvar
-      return s(:flip3, node[1], node[2])
+      _, lhs, rhs = node
+      return s(:flip3, lhs, rhs)
     else
@@ -280,3 +282,3 @@ return node
     if lhs then
-      case lhs[0]
+      case lhs.sexp_type
       when :dregx, :dregx_once then
@@ -290,3 +292,3 @@ return s(:match2, lhs, rhs).line(lhs.line)
     if rhs then
-      case rhs[0]
+      case rhs.sexp_type
       when :dregx, :dregx_once then
@@ -363,3 +365,3 @@ return s(:match3, rhs, lhs).line(lhs.line)
     return s(:array, item) unless list
-    list = s(:array, list) unless Sexp === list && list.first == :array
+    list = s(:array, list) unless Sexp === list && list.sexp_type == :array
     list << item
@@ -369,3 +371,3 @@ end
   def list_prepend item, list # TODO: nuke me *sigh*
-    list = s(:array, list) unless Sexp === list && list[0] == :array
+    list = s(:array, list) unless Sexp === list && list.sexp_type == :array
     list.insert 1, item
@@ -379,3 +381,3 @@ list
 
-    htype, ttype = head[0], tail[0]
+    htype, ttype = head.sexp_type, tail.sexp_type
 
@@ -387,5 +389,5 @@ head = s(:dstr, '', head) if htype == :evstr
       if htype == :str
-        head[-1] << tail[-1]
+        head.last << tail.last
       elsif htype == :dstr and head.size == 2 then
-        head[-1] << tail[-1]
+        head.last << tail.last
       else
@@ -397,17 +399,23 @@ head << tail
         lineno = head.line
-        tail[1] = head[-1] + tail[1]
+        tail[1] = head.last + tail[1]
         head = tail
         head.line = lineno
       else
-        tail[0] = :array
+        tail.sexp_type = :array
         tail[1] = s(:str, tail[1])
         tail.delete_at 1 if tail[1] == s(:str, '')
 
-        head.push(*tail[1..-1])
+        head.push(*tail.sexp_body)
       end
     when :evstr then
-      head[0] = :dstr if htype == :str
-      if head.size == 2 and tail.size > 1 and tail[1][0] == :str then
-        head[-1] << tail[1][-1]
-        head[0] = :str if head.size == 2 # HACK ?
+      if htype == :str then
+        f, l = head.file, head.line
+        head = s(:dstr, *head.sexp_body)
+        head.file = f
+        head.line = l
+      end
+
+      if head.size == 2 and tail.size > 1 and tail[1].sexp_type == :str then
+        head.last << tail[1].last
+        head.sexp_type = :str if head.size == 2 # HACK ?
       else
@@ -427,10 +435,12 @@ head.push(tail)
 
-    if left and left[0] == type and not left.paren then
-      node, second = left, nil
+    if left and left.sexp_type == type and not left.paren then
+      node, rhs = left, nil
 
-      while (second = node[2]) && second[0] == type and not second.paren do
-        node = second
+      loop do
+        _, _lhs, rhs = node
+        break unless rhs && rhs.sexp_type == type and not rhs.paren
+        node = rhs
       end
 
-      node[2] = s(type, second, right)
+      node[2] = s(type, rhs, right)
 
@@ -448,3 +458,3 @@ return left
     val[2] ||= s(:arglist)
-    val[2][0] = :arglist if val[2][0] == :array # REFACTOR
+    val[2].sexp_type = :arglist if val[2].sexp_type == :array # REFACTOR
     if val[0].node_type == :self then
@@ -491,3 +501,3 @@ result = new_call nil, :"[]", val[2]
   def argl x
-    x = s(:arglist, x) if x and x[0] == :array
+    x = s(:arglist, x) if x and x.sexp_type == :array
     x
@@ -498,3 +508,3 @@ end
     # TODO: need a test for this... obviously
-    case ref.first
+    case ref.sexp_type
     when :nth_ref then
@@ -525,3 +535,3 @@ raise "write a test 2"
     if args
-      if [:arglist, :args, :array, :call_args].include? args.first
+      if [:arglist, :args, :array, :call_args].include? args.sexp_type
         result.concat args.sexp_body
@@ -565,3 +575,3 @@ else
       block = node.block(:delete)
-      node.concat block[1..-1] if block
+      node.concat block.sexp_body if block
     end
@@ -583,4 +593,4 @@
     if body then
-      if body.first == :block then
-        result.push(*body[1..-1])
+      if body.sexp_type == :block then
+        result.push(*body.sexp_body)
       else
@@ -609,4 +619,4 @@ result.push body
     if body then
-      if body.first == :block then
-        result.push(*body[1..-1])
+      if body.sexp_type == :block then
+        result.push(*body.sexp_body)
       else
@@ -631,4 +641,4 @@ result.push body
     if body then
-      if body.first == :block then
-        result.push(*body[1..-1])
+      if body.sexp_type == :block then
+        result.push(*body.sexp_body)
       else
@@ -657,3 +667,3 @@ result.push body
     c = cond c
-    c, t, f = c.last, f, t if c[0] == :not and canonicalize_conditions
+    c, t, f = c.last, f, t if c.sexp_type == :not and canonicalize_conditions
     s(:if, c, t, f).line(l)
@@ -673,3 +683,3 @@ end
 
-    args[0] = :args unless args == 0
+    args.sexp_type = :args unless args == 0
 
@@ -686,6 +696,8 @@ result
   def new_masgn lhs, rhs, wrap = false
+    _, ary = lhs
+
     rhs = value_expr(rhs)
-    rhs = lhs[1] ? s(:to_ary, rhs) : s(:array, rhs) if wrap
+    rhs = ary ? s(:to_ary, rhs) : s(:array, rhs) if wrap
 
-    lhs.delete_at 1 if lhs[1].nil?
+    lhs.delete_at 1 if ary.nil?
     lhs << rhs
@@ -702,4 +714,4 @@
     if body then # REFACTOR?
-      if body.first == :block then
-        result.push(*body[1..-1])
+      if body.sexp_type == :block then
+        result.push(*body.sexp_body)
       else
@@ -775,5 +787,5 @@ result.push body
 
-    case node[0]
+    case node.sexp_type
     when :str then
-      node[0] = :lit
+      node.sexp_type = :lit
       node[1] = if k then
@@ -797,5 +809,5 @@ Regexp.new(node[1], o, k)
       if options =~ /o/ then
-        node[0] = :dregx_once
+        node.sexp_type = :dregx_once
       else
-        node[0] = :dregx
+        node.sexp_type = :dregx
       end
@@ -805,3 +817,3 @@ node << o if o and o != 0
       node = s(:dregx, '', node);
-      node[0] = :dregx_once if options =~ /o/
+      node.sexp_type = :dregx_once if options =~ /o/
       node << o if o and o != 0
@@ -814,3 +826,3 @@ end
   def new_resbody cond, body
-    if body && body.first == :block then
+    if body && body.sexp_type == :block then
       body.shift # remove block and splat it in directly
@@ -820,3 +832,3 @@ else
     end
-    s(:resbody, cond, *body)
+    s(:resbody, cond, *body).line cond.line
   end
@@ -830,4 +842,4 @@
     if body then
-      if body.first == :block then
-        result.push(*body[1..-1])
+      if body.sexp_type == :block then
+        result.push(*body.sexp_body)
       else
@@ -873,3 +885,3 @@ result.push body
   def new_word_list_entry val
-    result = val[1][0] == :evstr ? s(:dstr, "", val[1]) : val[1]
+    result = val[1].sexp_type == :evstr ? s(:dstr, "", val[1]) : val[1]
     self.lexer.fixup_lineno
@@ -903,5 +915,5 @@ result
 
-    case sym[0]
+    case sym.sexp_type
     when :dstr then
-      sym[0] = :dsym
+      sym.sexp_type = :dsym
     when :str then
@@ -922,3 +934,3 @@ sym = s(:lit, sym.last.to_sym)
       args ||= s(:arglist)
-      s(:super, *args[1..-1])
+      s(:super, *args.sexp_body)
     end
@@ -942,7 +954,7 @@ end
     line = [block && block.line, expr.line].compact.min
-    block, pre = block.last, false if block && block[0] == :begin
+    block, pre = block.last, false if block && block.sexp_type == :begin
 
     expr = cond expr
 
-    result = unless expr.first == :not and canonicalize_conditions then
+    result = unless expr.sexp_type == :not and canonicalize_conditions then
                s(type,  expr,      block, pre)
@@ -967,7 +979,7 @@ else
     if str then
-      case str[0]
+      case str.sexp_type
       when :str
-        str[0] = :xstr
+        str.sexp_type = :xstr
       when :dstr
-        str[0] = :dxstr
+        str.sexp_type = :dxstr
       else
@@ -990,6 +1002,6 @@ str = s(:dxstr, '', str)
 
-    args[0] = :arglist if [:call_args, :array].include?(args[0])
-    args = s(:arglist, args) unless args.first == :arglist
+    args.sexp_type = :arglist if [:call_args, :array].include? args.sexp_type
+    args = s(:arglist, args) unless args.sexp_type == :arglist
 
-    return s(:yield, *args[1..-1])
+    return s(:yield, *args.sexp_body)
   end
@@ -1012,7 +1024,7 @@
 
-    case lhs[0]
+    case lhs.sexp_type
     when :lasgn, :iasgn, :cdecl, :cvdecl, :gasgn, :cvasgn, :attrasgn, :safe_attrasgn then
       lhs << rhs
     when :const then
-      lhs[0] = :cdecl
+      lhs.sexp_type = :cdecl
       lhs << rhs
@@ -1127,4 +1139,4 @@ else
     oldnode = node
-    if node and :begin == node[0] and node.size == 2 then
-      node = node[-1]
+    if node and node.sexp_type == :begin and node.size == 2 then
+      node = node.last
       node.line = oldnode.line
@@ -1162,14 +1174,14 @@ end
     if node then
-      raise "write a test 5" if node[0] == :block_pass
+      raise "write a test 5" if node.sexp_type == :block_pass
 
       raise SyntaxError, "block argument should not be given" if
-        node[0] == :block_pass
+        node.sexp_type == :block_pass
 
-      node[0] = :array if node[0] == :call_args
-      node = node.last if node[0] == :array && node.size == 2
+      node.sexp_type = :array if node.sexp_type == :call_args
+      node = node.last if node.sexp_type == :array && node.size == 2
 
       # HACK matz wraps ONE of the FOUR splats in a newline to
       # distinguish. I use paren for now. ugh
-      node = s(:svalue, node) if node[0] == :splat and not node.paren
-      node[0] = :svalue if node[0] == :arglist && node[1][0] == :splat
+      node = s(:svalue, node) if node.sexp_type == :splat and not node.paren
+      node.sexp_type = :svalue if node.sexp_type == :arglist && node[1].sexp_type == :splat
     end
@@ -1190,3 +1202,3 @@
     node.line = oldnode.line if oldnode
-    node[2] = value_expr(node[2]) if node and node[0] == :if
+    node[2] = value_expr node[2] if node and node.sexp_type == :if
     node
@@ -1197,5 +1209,10 @@ end
     return nil unless node
-    return node unless node[0] == :block
+    return node unless node.sexp_type == :block
 
-    node[1..-1] = node[1..-1].map { |n| remove_begin(n) }
+    if node.respond_to? :sexp_body= then
+      node.sexp_body = node.sexp_body.map { |n| remove_begin n }
+    else
+      node[1..-1] = node[1..-1].map { |n| remove_begin(n) }
+    end
+
     node
@@ -1361,8 +1378,13 @@ end
 
-    def initialize(name)
+    def initialize name, debug=false
       @name = name
       @stack = [false]
-      @debug = false
+      @debug = debug
     end
 
+    def reset
+      @stack = [false]
+      warn "#{name}_stack(set): 0" if debug
+    end
+
     def inspect
@@ -1373,3 +1395,2 @@ "StackState(#{@name}, #{@stack.inspect})"
     def is_in_state
-      p :stack_is_in_state => [name, @stack.last, caller.first] if debug
       @stack.last
@@ -1379,3 +1400,3 @@ end
     def lexpop
-      p :stack_lexpop => caller.first if debug
+      warn "#{name}_stack.lexpop" if debug
       raise if @stack.size == 0
@@ -1389,3 +1410,3 @@ a = @stack.pop
       r = @stack.pop
-      p :stack_pop => [name, r, @stack, caller.first] if debug
+      warn "#{name}_stack.pop" if debug
       @stack.push false if @stack.size == 0
@@ -1397,3 +1418,5 @@ r
       @stack.push val
-      p :stack_push => [name, @stack, caller.first] if debug
+      c = caller.first
+      c = caller[1] if c =~ /expr_result/
+      warn "#{name}_stack(push): #{val} at line #{c.clean_caller}" if debug
       nil
@@ -1400,0 +1423,0 @@ end

Rakefile

@@ -26,5 +26,5 @@ # -*- ruby -*-
 
-  dependency "sexp_processor", "~> 4.1"
+  dependency "sexp_processor", "~> 4.9"
   dependency "rake", "< 11", :developer
-  dependency "oedipus_lex", "~> 2.1", :developer
+  dependency "oedipus_lex", "~> 2.5", :developer
 
@@ -31,0 +31,0 @@ if plugin? :perforce then     # generated files

No alert changes

No metric changes