🚀 Socket Launch Week Day 5:Introducing Repository Access Permissions and Custom Roles.Learn more →
Sign In

ruby_parser

Package Overview
Dependencies
Maintainers
1
Versions
80
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

ruby_parser - rubygems Package Compare versions

Comparing version
3.3.0
 to
3.4.0
lib/.document

Sorry, the diff of this file is not supported yet

lib/ruby_lexer.rex

Sorry, the diff of this file is not supported yet

+263
lib/ruby_lexer.rex.rb
#--
# This file is automatically generated. Do not modify it.
# Generated by: oedipus_lex version 2.1.0.
# Source: lib/ruby_lexer.rex
#++
# encoding: UTF-8
# TODO: this needs to be put on the first line
#
# new_ruby_parser.rex
# lexical scanner definition for ruby
class RubyLexer
require 'strscan'
IDENT = /^#{IDENT_CHAR}+/o
ESC = /\\((?>[0-7]{1,3}|x[0-9a-fA-F]{1,2}|M-[^\\]|(C-|c)[^\\]|u[0-9a-fA-F]+|u\{[0-9a-fA-F]+\}|[^0-7xMCc]))/
SIMPLE_STRING = /(#{ESC}|\#(#{ESC}|[^\{\#\@\$\"\\])|[^\"\\\#])*/o
SSTRING = /(\\.|[^\'])*/
INT_DEC = /[+]?(?:(?:[1-9][\d_]*|0)(?!\.\d)\b|0d[0-9_]+)/i
INT_HEX = /[+]?0x[a-f0-9_]+/i
INT_BIN = /[+]?0b[01_]+/i
INT_OCT = /[+]?0o?[0-7_]+|0o/i
FLOAT = /[+]?\d[\d_]*\.[\d_]+(e[+-]?[\d_]+)?\b|[+]?[\d_]+e[+-]?[\d_]+\b/i
INT_DEC2 = /[+]?\d[0-9_]*(?![e])/i
NUM_BAD = /[+]?0[xbd]\b/i
INT_OCT_BAD = /[+]?0o?[0-7_]*[89]/i
FLOAT_BAD = /[+]?\d[\d_]*_(e|\.)/i
class ScanError < StandardError ; end
attr_accessor :lineno
attr_accessor :filename
attr_accessor :ss
attr_accessor :state
alias :match :ss
def matches
m = (1..9).map { |i| ss[i] }
m.pop until m[-1] or m.empty?
m
end
def action
yield
end
def scanner_class
StringScanner
end unless instance_methods(false).map(&:to_s).include?("scanner_class")
def parse str
self.ss = scanner_class.new str
self.lineno = 1
self.state ||= nil
do_parse
end
def parse_file path
self.filename = path
open path do |f|
parse f.read
end
end
def next_token
return process_string if lex_strterm
self.command_state = self.command_start
self.command_start = false
self.space_seen = false
self.last_state = lex_state
token = nil
until ss.eos? or token do
token =
case state
when nil then
case
when text = ss.scan(/[\ \t\r\f\v]/) then
action { self.space_seen = true; next }
when text = ss.scan(/\n|\#/) then
process_newline_or_comment text
when text = ss.scan(/[\]\)\}]/) then
process_bracing text
when text = ss.scan(/\!/) then
process_bang text
when text = ss.scan(/\.\.\.?|,|![=~]?/) then
action { result :expr_beg, TOKENS[text], text }
when text = ss.scan(/\.\d/) then
action { rb_compile_error "no .<digit> floating literal anymore put 0 before dot" }
when text = ss.scan(/\./) then
action { result :expr_dot, :tDOT, "." }
when text = ss.scan(/\(/) then
process_paren text
when text = ss.scan(/\=\=\=|\=\=|\=~|\=>|\=(?!begin\b)/) then
action { result arg_state, TOKENS[text], text }
when bol? && (text = ss.scan(/\=begin(?=\s)/)) then
process_begin text
when text = ss.scan(/\=(?=begin\b)/) then
action { result arg_state, TOKENS[text], text }
when text = ss.scan(/\"(#{SIMPLE_STRING})\"/o) then
action { result :expr_end, :tSTRING, text[1..-2].gsub(ESC) { unescape $1 } }
when text = ss.scan(/\"/) then
action { string STR_DQUOTE; result nil, :tSTRING_BEG, text }
when text = ss.scan(/\@\@?\d/) then
action { rb_compile_error "`#{text}` is not allowed as a variable name" }
when text = ss.scan(/\@\@?#{IDENT_CHAR}+/o) then
process_ivar text
when not_end? && (text = ss.scan(/:([a-zA-Z_]#{IDENT_CHAR}*(?:[?]|[!](?!=)|=(?==>)|=(?![=>]))?)/o)) then
process_symbol text
when not_end? && (text = ss.scan(/\:\"(#{SIMPLE_STRING})\"/o)) then
process_symbol text
when not_end? && (text = ss.scan(/\:\'(#{SSTRING})\'/o)) then
process_symbol text
when text = ss.scan(/\:\:/) then
process_colon2 text
when text = ss.scan(/\:/) then
process_colon1 text
when text = ss.scan(/->/) then
action { result :expr_endfn, :tLAMBDA, nil }
when text = ss.scan(/[+-]/) then
process_plus_minus text
when text = ss.scan(/#{NUM_BAD}/o) then
action { rb_compile_error "Invalid numeric format" }
when text = ss.scan(/#{INT_DEC}/o) then
action { int_with_base 10 }
when text = ss.scan(/#{INT_HEX}/o) then
action { int_with_base 16 }
when text = ss.scan(/#{INT_BIN}/o) then
action { int_with_base 2 }
when text = ss.scan(/#{INT_OCT_BAD}/o) then
action { rb_compile_error "Illegal octal digit." }
when text = ss.scan(/#{INT_OCT}/o) then
action { int_with_base 8 }
when text = ss.scan(/#{FLOAT_BAD}/o) then
action { rb_compile_error "Trailing '_' in number." }
when text = ss.scan(/#{FLOAT}/o) then
process_float text
when text = ss.scan(/#{INT_DEC2}/o) then
action { int_with_base 10 }
when text = ss.scan(/[0-9]/) then
action { rb_compile_error "Bad number format" }
when text = ss.scan(/\[/) then
process_square_bracket text
when text = ss.scan(/\'#{SSTRING}\'/o) then
action { result :expr_end, :tSTRING, matched[1..-2].gsub(/\\\\/, "\\").gsub(/\\'/, "'") } # " stupid emacs
when text = ss.scan(/\|\|\=/) then
action { result :expr_beg, :tOP_ASGN, "||" }
when text = ss.scan(/\|\|/) then
action { result :expr_beg, :tOROP, "||" }
when text = ss.scan(/\|\=/) then
action { result :expr_beg, :tOP_ASGN, "|" }
when text = ss.scan(/\|/) then
action { result :arg_state, :tPIPE, "|" }
when text = ss.scan(/\{/) then
process_curly_brace text
when text = ss.scan(/\*\*=/) then
action { result :expr_beg, :tOP_ASGN, "**" }
when text = ss.scan(/\*\*/) then
action { result(:arg_state, space_vs_beginning(:tDSTAR, :tDSTAR, :tPOW), "**") }
when text = ss.scan(/\*\=/) then
action { result(:expr_beg, :tOP_ASGN, "*") }
when text = ss.scan(/\*/) then
action { result(:arg_state, space_vs_beginning(:tSTAR, :tSTAR, :tSTAR2), "*") }
when text = ss.scan(/\<\=\>/) then
action { result :arg_state, :tCMP, "<=>" }
when text = ss.scan(/\<\=/) then
action { result :arg_state, :tLEQ, "<=" }
when text = ss.scan(/\<\<\=/) then
action { result :arg_state, :tOP_ASGN, "<<" }
when text = ss.scan(/\<\</) then
process_lchevron text
when text = ss.scan(/\</) then
action { result :arg_state, :tLT, "<" }
when text = ss.scan(/\>\=/) then
action { result :arg_state, :tGEQ, ">=" }
when text = ss.scan(/\>\>=/) then
action { result :arg_state, :tOP_ASGN, ">>" }
when text = ss.scan(/\>\>/) then
action { result :arg_state, :tRSHFT, ">>" }
when text = ss.scan(/\>/) then
action { result :arg_state, :tGT, ">" }
when text = ss.scan(/\`/) then
process_backtick text
when text = ss.scan(/\?/) then
process_questionmark text
when text = ss.scan(/\&\&\=/) then
action { result(:expr_beg, :tOP_ASGN, "&&") }
when text = ss.scan(/\&\&/) then
action { result(:expr_beg, :tANDOP, "&&") }
when text = ss.scan(/\&\=/) then
action { result(:expr_beg, :tOP_ASGN, "&" ) }
when text = ss.scan(/\&/) then
process_amper text
when text = ss.scan(/\//) then
process_slash text
when text = ss.scan(/\^=/) then
action { result(:expr_beg, :tOP_ASGN, "^") }
when text = ss.scan(/\^/) then
action { result(:arg_state, :tCARET, "^") }
when text = ss.scan(/\;/) then
action { self.command_start = true; result(:expr_beg, :tSEMI, ";") }
when in_arg_state? && (text = ss.scan(/\~@/)) then
action { result(:arg_state, :tTILDE, "~") }
when text = ss.scan(/\~/) then
action { result(:arg_state, :tTILDE, "~") }
when text = ss.scan(/\\\r?\n/) then
action { self.lineno += 1; self.space_seen = true; next }
when text = ss.scan(/\\/) then
action { rb_compile_error "bare backslash only allowed before newline" }
when text = ss.scan(/\%/) then
process_percent text
when text = ss.scan(/\$_\w+/) then
process_gvar text
when text = ss.scan(/\$_/) then
process_gvar text
when text = ss.scan(/\$[~*$?!@\/\\;,.=:<>\"]|\$-\w?/) then
process_gvar text
when in_fname? && (text = ss.scan(/\$([\&\`\'\+])/)) then
process_gvar text
when text = ss.scan(/\$([\&\`\'\+])/) then
process_backref text
when in_fname? && (text = ss.scan(/\$([1-9]\d*)/)) then
process_gvar text
when text = ss.scan(/\$([1-9]\d*)/) then
process_nthref text
when text = ss.scan(/\$0/) then
process_gvar text
when text = ss.scan(/\$\W|\$\z/) then
process_gvar_oddity text
when text = ss.scan(/\$\w+/) then
process_gvar text
when text = ss.scan(/\_/) then
process_underscore text
when text = ss.scan(/#{IDENT}/o) then
process_token text
when text = ss.scan(/\004|\032|\000|\Z/) then
action { [RubyLexer::EOF, RubyLexer::EOF] }
when text = ss.scan(/./) then
action { rb_compile_error "Invalid char #{text.inspect} in expression" }
else
text = ss.string[ss.pos .. -1]
raise ScanError, "can not match (#{state.inspect}): '#{text}'"
end
else
raise ScanError, "undefined state: '#{state}'"
end # token = case state
next unless token # allow functions to trigger redo w/ nil
end # while
raise "bad lexical result: #{token.inspect}" unless
token.nil? || (Array === token && token.size >= 2)
# auto-switch state
self.state = token.last if token && token.first == :state
token
end # def _next_token
end # class
+50
-0
History.txt

@@ -0,1 +1,51 @@

=== 3.4.0 / 2014-02-04
* 1 major enhancement:
* Replaced hand-written/optimized f'd-up lexer with an oedipus_lex
generated lexer. This makes it roughly 40-50% faster.
* 30 minor enhancements:
* 2.0: Added support for a.b c() do d end.e do |f| g end
* 2.0: Added support for a.b c() do d end.e f do |g| h end
* Added -s flag to ruby_parse_extract_error to output timings.
* Added RubyLexer #command_state and #last_state to deal with oedipus_lex differences.
* Added String#lineno and #lineno= because I'm a bad bad person.
* Added a bunch of RubyLexer scanning methods: beginning_of_line?, check, scan, etc.
* Added a bunch of process_* methods extracted from old yylex. process_amper, etc.
* Added lib/.document to save my laptop's battery from pain and suffering
* Adjust lineno when we lex a bunch of blank lines.
* Attach lineno to tIDENTIFIER values (strings, ugh)
* Cleaned up and re-ordered node_assign to be faster (ordered by actual occurrance).
* Extend RubyParserStuff#gettable to set the lineno if it comes in with the id.
* Extended RubyParserStuff#new_case to take line number.
* Finally dropped RPStringScanner's BS #current_line.
* Finally dropped RPStringScanner's BS line number calculation (lineno).
* Implemented Sexp#add_all since we now have a test case for it.
* Removed :call case of node_assign. I don't think it is possible.
* Removed RubyLexer #extra_lines_added. No longer used. Complex heredoc lineno's possible screwed up.
* Removed RubyLexer#parse_number. Handled by oedipus_lex.
* Removed RubyLexer#yacc_value now that next_token returns pairs.
* Removed RubyLexer's @src. Now taken care of by oedipus_lex.
* Removed RubyParser#advance. RubyParser#next_token takes care of everything now.
* Removed RubyParserExtras#arg_add. (presidentbeef! YAY!)
* Removed lib/gauntlet_rubyparser.rb. I just don't use it anymore. Too slow.
* RubyLexer#is_label_possible? doesn't need an arg
* RubyLexer#process_token is now a normal oedipal lexer method.
* RubyParser#next_token now expects RubyLexer#next_token to return a pair (type, val).
* TRYING a new scheme to figure out encodings... but I'm about to throw in the towel. I hate this stuff so much.
* Turned off oedipus_lex's automatic line counting. (pushing to oedipus_lex soon).
* Updated to oedipus_lex 2.1+.
* 7 bug fixes:
* 1.8: Properly parse `a (:b, :c, :d => :e)`. (presidentbeef)
* Fixed lexing symbol!= vs symbol!. Please use your spacebar. Think of the children.
* Fixed line for dstr spanning multiple lines via backslash. (presidentbeef)
* Fixed line numbers for odd cases with trailing whitespace. (presidentbeef)
* Fixed line numbers on ambiguous calls w/ gvar/ivar args. (presidentbeef)
* Max out unicode hex values to 2-4 or 2-6 chars or pack will overflow and puke.
* Removed ESC_RE from RubyLexer. Must have slipped through.
=== 3.3.0 / 2014-01-14

@@ -2,0 +52,0 @@ 

+501
-663
lib/ruby_lexer.rb

@@ -14,7 +14,2 @@ # encoding: UTF-8

IDENT = /^#{IDENT_CHAR}+/o
ESC = /\\((?>[0-7]{1,3}|x[0-9a-fA-F]{1,2}|M-[^\\]|(C-|c)[^\\]|u[0-9a-fA-F]+|u\{[0-9a-fA-F]+\}|[^0-7xMCc]))/u
SIMPLE_STRING = /(#{ESC}|#(#{ESC}|[^\{\#\@\$\"\\])|[^\"\\\#])*/o
SIMPLE_SSTRING = /(\\.|[^\'])*/
EOF = :eof_haha!

@@ -79,2 +74,4 @@

attr_accessor :command_start
attr_accessor :command_state
attr_accessor :last_state
attr_accessor :cond

@@ -96,7 +93,3 @@

# Stream of data that yylex examines.
attr_reader :src
alias :ss :src
# Last token read via yylex.
# Last token read via next_token.
attr_accessor :token

@@ -110,7 +103,2 @@

# Value of last token which had a value associated with it.
attr_accessor :yacc_value
attr_writer :lineno # reader is lazy initalizer
attr_writer :comments

@@ -124,16 +112,2 @@

##
# How the parser advances to the next token.
#
# @return true if not at end of file (EOF).
def advance
r = yylex
self.token = r
raise "yylex returned nil, near #{ss.rest[0,10].inspect}" unless r
return RubyLexer::EOF != r
end
def arg_ambiguous

@@ -150,2 +124,3 @@ self.warning("Ambiguous first argument. make sure.")

end
alias :bol? :beginning_of_line? # to make .rex file more readable

@@ -183,5 +158,5 @@ def check re

if beginning_of_line? && scan(eos_re) then
self.lineno += 1
ss.unread_many last_line # TODO: figure out how to remove this
self.yacc_value = eos
return :tSTRING_END
return :tSTRING_END, eos
end

@@ -195,7 +170,5 @@

ss.pos -= 1 # FIX omg stupid
self.yacc_value = matched
return :tSTRING_DVAR
return :tSTRING_DVAR, matched
when scan(/#[{]/) then
self.yacc_value = matched
return :tSTRING_DBEG
return :tSTRING_DBEG, matched
when scan(/#/) then

@@ -212,5 +185,5 @@ string_buffer << '#'

if c != "\n" then
self.yacc_value = string_buffer.join.delete("\r")
return :tSTRING_CONTENT
return :tSTRING_CONTENT, string_buffer.join.delete("\r")
else
self.lineno += 1
string_buffer << scan(/\n/)

@@ -230,4 +203,3 @@ end

self.yacc_value = string_buffer.join.delete("\r")
return :tSTRING_CONTENT
return :tSTRING_CONTENT, string_buffer.join.delete("\r")
end

@@ -268,3 +240,2 @@

line = matched
ss.extra_lines_added += 1 # FIX: ugh
else

@@ -277,10 +248,12 @@ line = nil

if term == '`' then
self.yacc_value = "`"
return :tXSTRING_BEG
result nil, :tXSTRING_BEG, "`"
else
self.yacc_value = "\""
return :tSTRING_BEG
result nil, :tSTRING_BEG, "\""
end
end
def in_fname?
in_lex_state? :expr_fname
end
def in_arg_state? # TODO: rename is_after_operator?

@@ -296,5 +269,3 @@ in_lex_state? :expr_fname, :expr_dot

rb_compile_error "Invalid numeric format" if matched =~ /__/
self.yacc_value = matched.to_i(base)
return :tINTEGER
return result(:expr_end, :tINTEGER, matched.to_i(base))
end

@@ -314,3 +285,3 @@

def is_label_possible? command_state
def is_label_possible?
(in_lex_state?(:expr_beg, :expr_endfn) && !command_state) || is_arg?

@@ -323,6 +294,2 @@ end

def lineno
@lineno ||= ss.lineno
end
def matched

@@ -332,158 +299,388 @@ ss.matched

##
# Parse a number from the input stream.
#
# @param c The first character of the number.
# @return A int constant wich represents a token.
def not_end?
not is_end?
end
def parse_number
self.lex_state = :expr_end
def process_amper text
token = if is_arg? && space_seen && !check(/\s/) then
warning("`&' interpreted as argument prefix")
:tAMPER
elsif in_lex_state? :expr_beg, :expr_mid then
:tAMPER
else
:tAMPER2
end
return result(:arg_state, token, "&")
end
def process_backref text
token = ss[1].to_sym
# TODO: can't do lineno hack w/ symbol
result :expr_end, :tBACK_REF, token
end
def process_backtick text
case lex_state
when :expr_fname then
result :expr_end, :tBACK_REF2, "`"
when :expr_dot then
result((command_state ? :expr_cmdarg : :expr_arg), :tBACK_REF2, "`")
else
string STR_XQUOTE
result nil, :tXSTRING_BEG, "`"
end
end
def process_bang text
if in_arg_state? then
return result(:expr_arg, :tUBANG, "!@") if scan(/@/)
end
text = scan(/[=~]/) ? "!#{matched}" : "!"
return result(arg_state, TOKENS[text], text)
end
def process_begin text
@comments << matched
unless scan(/.*?\n=end( |\t|\f)*[^\n]*(\n|\z)/m) then
@comments.clear
rb_compile_error("embedded document meets end of file")
end
@comments << matched
nil # TODO
end
def process_bracing text
cond.lexpop
cmdarg.lexpop
case matched
when "}" then
self.brace_nest -= 1
self.lex_state = :expr_endarg
return :tRCURLY, matched
when "]" then
self.paren_nest -= 1
self.lex_state = :expr_endarg
return :tRBRACK, matched
when ")" then
self.paren_nest -= 1
self.lex_state = :expr_endfn
return :tRPAREN, matched
else
raise "Unknown bracing: #{matched.inspect}"
end
end
def process_colon1 text
# ?: / then / when
if is_end? || check(/\s/) then
return result :expr_beg, :tCOLON, text
end
case
when scan(/[+-]?0[xXbBdD]\b/) then
rb_compile_error "Invalid numeric format"
when scan(/[+-]?(?:(?:[1-9][\d_]*|0)(?!\.\d)\b|0[Dd][0-9_]+)/) then
int_with_base(10)
when scan(/[+-]?0x[a-f0-9_]+/i) then
int_with_base(16)
when scan(/[+-]?0[Bb][01_]+/) then
int_with_base(2)
when scan(/[+-]?0[Oo]?[0-7_]*[89]/) then
rb_compile_error "Illegal octal digit."
when scan(/[+-]?0[Oo]?[0-7_]+|0[Oo]/) then
int_with_base(8)
when scan(/[+-]?[\d_]+_(e|\.)/) then
rb_compile_error "Trailing '_' in number."
when scan(/[+-]?[\d_]+\.[\d_]+(e[+-]?[\d_]+)?\b|[+-]?[\d_]+e[+-]?[\d_]+\b/i) then
number = matched
if number =~ /__/ then
rb_compile_error "Invalid numeric format"
end
self.yacc_value = number.to_f
:tFLOAT
when scan(/[+-]?[0-9_]+(?![e])/) then
int_with_base(10)
when scan(/\'/) then
string STR_SSYM
when scan(/\"/) then
string STR_DSYM
end
result :expr_fname, :tSYMBEG, text
end
def process_colon2 text
if is_beg? || in_lex_state?(:expr_class) || is_space_arg? then
result :expr_beg, :tCOLON3, text
else
rb_compile_error "Bad number format"
result :expr_dot, :tCOLON2, text
end
end
def parse_quote # TODO: remove / rewrite
beg, nnd, short_hand, c = nil, nil, false, nil
def process_curly_brace text
self.brace_nest += 1
if lpar_beg && lpar_beg == paren_nest then
self.lpar_beg = nil
self.paren_nest -= 1
if scan(/[a-z0-9]{1,2}/i) then # Long-hand (e.g. %Q{}).
rb_compile_error "unknown type of %string" if ss.matched_size == 2
c, beg, short_hand = matched, ss.getch, false
else # Short-hand (e.g. %{, %., %!, etc)
c, beg, short_hand = 'Q', ss.getch, true
return expr_result(:tLAMBEG, "{")
end
if end_of_stream? or c == RubyLexer::EOF or beg == RubyLexer::EOF then
rb_compile_error "unterminated quoted string meets end of file"
token = if is_arg? || in_lex_state?(:expr_end, :expr_endfn) then
:tLCURLY # block (primary)
elsif in_lex_state?(:expr_endarg) then
:tLBRACE_ARG # block (expr)
else
:tLBRACE # hash
end
self.command_start = true unless token == :tLBRACE
return expr_result(token, "{")
end
def process_float text
rb_compile_error "Invalid numeric format" if text =~ /__/
return result(:expr_end, :tFLOAT, text.to_f)
end
def process_gvar text
text.lineno = self.lineno
result(:expr_end, :tGVAR, text)
end
def process_gvar_oddity text
result :expr_end, "$", "$" # TODO: wtf is this?
end
def process_ivar text
tok_id = text =~ /^@@/ ? :tCVAR : :tIVAR
text.lineno = self.lineno
return result(:expr_end, tok_id, text)
end
def process_lchevron text
if (!in_lex_state?(:expr_dot, :expr_class) &&
!is_end? &&
(!is_arg? || space_seen)) then
tok = self.heredoc_identifier
return tok if tok
end
# Figure nnd-char. "\0" is special to indicate beg=nnd and that no nesting?
nnd = { "(" => ")", "[" => "]", "{" => "}", "<" => ">" }[beg]
nnd, beg = beg, "\0" if nnd.nil?
return result(:arg_state, :tLSHFT, "\<\<")
end
token_type, text = nil, "%#{c}#{beg}"
token_type, string_type = case c
when 'Q' then
ch = short_hand ? nnd : c + beg
text = "%#{ch}"
[:tSTRING_BEG, STR_DQUOTE]
when 'q' then
[:tSTRING_BEG, STR_SQUOTE]
when 'W' then
scan(/\s*/)
[:tWORDS_BEG, STR_DQUOTE | STR_FUNC_QWORDS]
when 'w' then
scan(/\s*/)
[:tQWORDS_BEG, STR_SQUOTE | STR_FUNC_QWORDS]
when 'x' then
[:tXSTRING_BEG, STR_XQUOTE]
when 'r' then
[:tREGEXP_BEG, STR_REGEXP]
when 's' then
self.lex_state = :expr_fname
[:tSYMBEG, STR_SSYM]
when 'I' then
src.scan(/\s*/)
[:tSYMBOLS_BEG, STR_DQUOTE | STR_FUNC_QWORDS]
when 'i' then
src.scan(/\s*/)
[:tQSYMBOLS_BEG, STR_SQUOTE | STR_FUNC_QWORDS]
end
def process_newline_or_comment text
c = matched
hit = false
rb_compile_error "Bad %string type. Expected [QqWwIixrs], found '#{c}'." if
token_type.nil?
if c == '#' then
ss.pos -= 1
raise "huh" unless string_type
while scan(/\s*\#.*(\n+|\z)/) do
hit = true
self.lineno += matched.lines.to_a.size
@comments << matched.gsub(/^ +#/, '#').gsub(/^ +$/, '')
end
string string_type, nnd, beg
return nil if end_of_stream?
end
self.yacc_value = text
return token_type
self.lineno += 1 unless hit
# Replace a string of newlines with a single one
self.lineno += matched.lines.to_a.size if scan(/\n+/)
return if in_lex_state?(:expr_beg, :expr_value, :expr_class,
:expr_fname, :expr_dot)
if scan(/([\ \t\r\f\v]*)\./) then
self.space_seen = true unless ss[1].empty?
ss.pos -= 1
return unless check(/\.\./)
end
self.command_start = true
return result(:expr_beg, :tNL, nil)
end
def parse_string quote # TODO: rewrite / remove
_, string_type, term, open = quote
def process_nthref text
# TODO: can't do lineno hack w/ number
result :expr_end, :tNTH_REF, ss[1].to_i
end
space = false # FIX: remove these
func = string_type
paren = open
term_re = @@regexp_cache[term]
def process_paren text
token = if ruby18 then
process_paren18
else
process_paren19
end
qwords = (func & STR_FUNC_QWORDS) != 0
regexp = (func & STR_FUNC_REGEXP) != 0
expand = (func & STR_FUNC_EXPAND) != 0
self.paren_nest += 1
unless func then # nil'ed from qwords below. *sigh*
self.lineno = nil
return :tSTRING_END
return expr_result(token, "(")
end
def process_paren18
self.command_start = true
token = :tLPAREN2
if in_lex_state? :expr_beg, :expr_mid then
token = :tLPAREN
elsif space_seen then
if in_lex_state? :expr_cmdarg then
token = :tLPAREN_ARG
elsif in_lex_state? :expr_arg then
warning "don't put space before argument parentheses"
end
else
# not a ternary -- do nothing?
end
space = true if qwords and scan(/\s+/)
token
end
if self.string_nest == 0 && scan(/#{term_re}/) then
if qwords then
quote[1] = nil
return :tSPACE
elsif regexp then
self.lineno = nil
self.yacc_value = self.regx_options
return :tREGEXP_END
def process_paren19
if is_beg? then
:tLPAREN
elsif is_space_arg? then
:tLPAREN_ARG
else
:tLPAREN2 # plain '(' in parse.y
end
end
def process_percent text
return parse_quote if is_beg?
return result(:expr_beg, :tOP_ASGN, "%") if scan(/\=/)
return parse_quote if is_arg? && space_seen && ! check(/\s/)
return result(:arg_state, :tPERCENT, "%")
end
def process_plus_minus text
sign = matched
utype, type = if sign == "+" then
[:tUPLUS, :tPLUS]
else
[:tUMINUS, :tMINUS]
end
if in_arg_state? then
if scan(/@/) then
return result(:expr_arg, utype, "#{sign}@")
else
self.lineno = nil
self.yacc_value = term
return :tSTRING_END
return result(:expr_arg, type, sign)
end
end
return :tSPACE if space
return result(:expr_beg, :tOP_ASGN, sign) if scan(/\=/)
self.string_buffer = []
if (is_beg? || (is_arg? && space_seen && !check(/\s/))) then
arg_ambiguous if is_arg?
if expand
case
when scan(/#(?=[$@])/) then
return :tSTRING_DVAR
when scan(/#[{]/) then
return :tSTRING_DBEG
when scan(/#/) then
string_buffer << '#'
if check(/\d/) then
return nil if utype == :tUPLUS
return result(:expr_beg, :tUMINUS_NUM, sign)
end
return result(:expr_beg, utype, sign)
end
if tokadd_string(func, term, paren) == RubyLexer::EOF then
rb_compile_error "unterminated string meets end of file"
return result(:expr_beg, type, sign)
end
def process_questionmark text
if is_end? then
state = ruby18 ? :expr_beg : :expr_value # HACK?
return result(state, :tEH, "?")
end
self.yacc_value = string_buffer.join
if end_of_stream? then
rb_compile_error "incomplete character syntax: parsed #{text.inspect}"
end
return :tSTRING_CONTENT
if check(/\s|\v/) then
unless is_arg? then
c2 = { " " => 's',
"\n" => 'n',
"\t" => 't',
"\v" => 'v',
"\r" => 'r',
"\f" => 'f' }[matched]
if c2 then
warning("invalid character syntax; use ?\\" + c2)
end
end
# ternary
state = ruby18 ? :expr_beg : :expr_value # HACK?
return result(state, :tEH, "?")
elsif check(/\w(?=\w)/) then # ternary, also
return result(:expr_beg, :tEH, "?")
end
c = if scan(/\\/) then
self.read_escape
else
ss.getch
end
if version == 18 then
return result(:expr_end, :tINTEGER, c[0].ord & 0xff)
else
return result(:expr_end, :tSTRING, c)
end
end
def process_token command_state, last_state
token = self.token
def process_slash text
if is_beg? then
string STR_REGEXP
return result(nil, :tREGEXP_BEG, "/")
end
if scan(/\=/) then
return result(:expr_beg, :tOP_ASGN, "/")
end
if is_arg? && space_seen then
unless scan(/\s/) then
arg_ambiguous
string STR_REGEXP, "/"
return result(nil, :tREGEXP_BEG, "/")
end
end
return result(:arg_state, :tDIVIDE, "/")
end
def process_square_bracket text
self.paren_nest += 1
token = nil
if in_arg_state? then
case
when scan(/\]\=/) then
self.paren_nest -= 1 # HACK? I dunno, or bug in MRI
return result(:expr_arg, :tASET, "[]=")
when scan(/\]/) then
self.paren_nest -= 1 # HACK? I dunno, or bug in MRI
return result(:expr_arg, :tAREF, "[]")
else
rb_compile_error "unexpected '['"
end
elsif is_beg? then
token = :tLBRACK
elsif is_arg? && space_seen then
token = :tLBRACK
else
token = :tLBRACK2
end
return expr_result(token, "[")
end
def process_symbol text
symbol = match[1].gsub(ESC) { unescape $1 }
rb_compile_error "symbol cannot contain '\\0'" if
ruby18 && symbol =~ /\0/
return result(:expr_end, :tSYMBOL, symbol)
end
def process_token text
# TODO: make this always return [token, lineno]
token = self.token = text
token << matched if scan(/[\!\?](?!=)/)

@@ -506,4 +703,4 @@

if !ruby18 and is_label_possible?(command_state) and scan(/:(?!:)/) then
return result(:expr_beg, :tLABEL, [token, ss.lineno]) # HACK: array? TODO: self.lineno
if !ruby18 and is_label_possible? and scan(/:(?!:)/) then
return result(:expr_beg, :tLABEL, [token, self.lineno])
end

@@ -538,2 +735,4 @@

token.lineno = self.lineno # yes, on a string. I know... I know...
return result(state, tok_id, token)

@@ -544,4 +743,5 @@ end

state = keyword.state
value = [token, ss.lineno] # TODO: use self.lineno ?
value = [token, self.lineno]
self.command_start = true if state == :expr_beg and lex_state != :expr_fname

@@ -576,2 +776,12 @@

def process_underscore text
ss.unscan # put back "_"
if beginning_of_line? && scan(/\__END__(\r?\n|\Z)/) then
return [RubyLexer::EOF, RubyLexer::EOF]
elsif scan(/\_\w*/) then
return process_token matched
end
end
def rb_compile_error msg

@@ -630,3 +840,3 @@ msg += ". near line #{self.lineno}: #{ss.rest[/^.*/].inspect}"

matched
when scan(/u([0-9a-fA-F]+|\{[0-9a-fA-F]+\})/) then
when scan(/u([0-9a-fA-F]{2,4}|\{[0-9a-fA-F]{2,6}\})/) then
[ss[1].delete("{}").to_i(16)].pack("U")

@@ -667,8 +877,5 @@ when scan(/[McCx0-9]/) || end_of_stream? then

self.token = nil
self.yacc_value = nil
self.cmdarg = RubyParserStuff::StackState.new(:cmdarg)
self.cond = RubyParserStuff::StackState.new(:cond)
@src = nil
end

@@ -679,4 +886,3 @@

self.lex_state = lex_state if lex_state
self.yacc_value = text
token
[token, text]
end

@@ -696,2 +902,6 @@

def scanner_class # TODO: design this out of oedipus_lex. or something.
RPStringScanner
end
def space_vs_beginning space_type, beg_type, fallback

@@ -713,6 +923,7 @@ if is_space_arg? check(/./m) then

def src= src
raise "bad src: #{src.inspect}" unless String === src
@src = RPStringScanner.new(src)
end
# TODO: consider
# def src= src
# raise "bad src: #{src.inspect}" unless String === src
# @src = RPStringScanner.new(src)
# end

@@ -848,3 +1059,3 @@ def tokadd_escape term # TODO: rewrite / remove

rb_compile_error("Invalid escape character syntax")
when /u([0-9a-fA-F]+|\{[0-9a-fA-F]+\})/ then
when /u([0-9a-fA-F]{2,4}|\{[0-9a-fA-F]{2,6}\})/ then
[$1.delete("{}").to_i(16)].pack("U")

@@ -862,517 +1073,144 @@ else

##
# Returns the next token. Also sets yy_val is needed.
#
# @return Description of the Returned Value
def yylex # 461 lines
c = ''
self.space_seen = false
command_state = false
ss = self.src
self.token = nil
self.yacc_value = nil
return yylex_string if lex_strterm
command_state = self.command_start
self.command_start = false
last_state = lex_state
loop do # START OF CASE
if scan(/[\ \t\r\f\v]/) then # \s - \n + \v
self.space_seen = true
next
elsif check(/[^a-zA-Z]/) then
if scan(/\n|\#/) then
self.lineno = nil
c = matched
if c == '#' then
ss.pos -= 1
while scan(/\s*#.*(\n+|\z)/) do
# TODO: self.lineno += matched.lines.to_a.size
@comments << matched.gsub(/^ +#/, '#').gsub(/^ +$/, '')
end
return RubyLexer::EOF if end_of_stream?
end
# Replace a string of newlines with a single one
scan(/\n+/)
next if in_lex_state?(:expr_beg, :expr_value, :expr_class,
:expr_fname, :expr_dot)
if scan(/([\ \t\r\f\v]*)\./) then
self.space_seen = true unless ss[1].empty?
ss.pos -= 1
next unless check(/\.\./)
end
self.command_start = true
return result(:expr_beg, :tNL, nil)
elsif scan(/[\]\)\}]/) then
if matched == "}" then
self.brace_nest -= 1
else
self.paren_nest -= 1
end
cond.lexpop
cmdarg.lexpop
text = matched
state = text == ")" ? :expr_endfn : :expr_endarg
token = {
")" => :tRPAREN,
"]" => :tRBRACK,
"}" => :tRCURLY
}[text]
return result(state, token, text)
elsif scan(/\!/) then
if in_arg_state? then
return result(:expr_arg, :tUBANG, "!@") if scan(/@/)
end
text = scan(/[=~]/) ? "!#{matched}" : "!"
return result(arg_state, TOKENS[text], text)
elsif scan(/\.\.\.?|,|![=~]?/) then
return result(:expr_beg, TOKENS[matched], matched)
elsif check(/\./) then
if scan(/\.\d/) then
rb_compile_error "no .<digit> floating literal anymore put 0 before dot"
elsif scan(/\./) then
return result(:expr_dot, :tDOT, ".")
end
elsif scan(/\(/) then
token = if ruby18 then
yylex_paren18
else
yylex_paren19
end
self.paren_nest += 1
return expr_result(token, "(")
elsif check(/\=/) then
if scan(/\=\=\=|\=\=|\=~|\=>|\=(?!begin\b)/) then
tok = matched
return result(:arg_state, TOKENS[tok], tok)
elsif beginning_of_line? and scan(/\=begin(?=\s)/) then
@comments << matched
unless scan(/.*?\n=end( |\t|\f)*[^\n]*(\n|\z)/m) then
@comments.clear
rb_compile_error("embedded document meets end of file")
end
@comments << matched
next
elsif scan(/\=(?=begin\b)/) then # h[k]=begin ... end
tok = matched
return result(:arg_state, TOKENS[tok], tok)
else
raise "you shouldn't be able to get here"
end
elsif scan(/\"(#{SIMPLE_STRING})\"/o) then
string = matched[1..-2].gsub(ESC) { unescape $1 }
return result(:expr_end, :tSTRING, string)
elsif scan(/\"/) then # FALLBACK
string STR_DQUOTE, '"' # TODO: question this
return result(nil, :tSTRING_BEG, '"')
elsif scan(/\@\@?#{IDENT_CHAR}+/o) then
self.token = matched
rb_compile_error "`#{self.token}` is not allowed as a variable name" if
self.token =~ /\@\d/
tok_id = matched =~ /^@@/ ? :tCVAR : :tIVAR
return result(:expr_end, tok_id, self.token)
elsif scan(/\:\:/) then
if is_beg? || in_lex_state?(:expr_class) || is_space_arg? then
return result(:expr_beg, :tCOLON3, "::")
end
return result(:expr_dot, :tCOLON2, "::")
elsif ! is_end? && scan(/:([a-zA-Z_]#{IDENT_CHAR}*(?:[?!]|=(?==>)|=(?![=>]))?)/) then
# scanning shortcut to symbols
return result(:expr_end, :tSYMBOL, ss[1])
elsif ! is_end? && (scan(/\:\"(#{SIMPLE_STRING})\"/) ||
scan(/\:\'(#{SIMPLE_SSTRING})\'/)) then
symbol = ss[1].gsub(ESC) { unescape $1 }
rb_compile_error "symbol cannot contain '\\0'" if
ruby18 && symbol =~ /\0/
return result(:expr_end, :tSYMBOL, symbol)
elsif scan(/\:/) then
# ?: / then / when
if is_end? || check(/\s/) then
# TODO warn_balanced(":", "symbol literal");
return result(:expr_beg, :tCOLON, ":")
end
case
when scan(/\'/) then
string STR_SSYM, matched
when scan(/\"/) then
string STR_DSYM, matched
end
return result(:expr_fname, :tSYMBEG, ":")
elsif check(/[0-9]/) then
return parse_number
elsif scan(/\[/) then
self.paren_nest += 1
token = nil
if in_lex_state? :expr_fname, :expr_dot then
case
when scan(/\]\=/) then
self.paren_nest -= 1 # HACK? I dunno, or bug in MRI
return result(:expr_arg, :tASET, "[]=")
when scan(/\]/) then
self.paren_nest -= 1 # HACK? I dunno, or bug in MRI
return result(:expr_arg, :tAREF, "[]")
def process_string # TODO: rewrite / remove
token = if lex_strterm[0] == :heredoc then
self.heredoc lex_strterm
else
rb_compile_error "unexpected '['"
self.parse_string lex_strterm
end
elsif is_beg? then
token = :tLBRACK
elsif is_arg? && space_seen then
token = :tLBRACK
else
token = :tLBRACK2
end
return expr_result(token, "[")
elsif scan(/\'#{SIMPLE_SSTRING}\'/) then
text = matched[1..-2].gsub(/\\\\/, "\\").gsub(/\\'/, "'") # "
return result(:expr_end, :tSTRING, text)
elsif check(/\|/) then
if scan(/\|\|\=/) then
return result(:expr_beg, :tOP_ASGN, "||")
elsif scan(/\|\|/) then
return result(:expr_beg, :tOROP, "||")
elsif scan(/\|\=/) then
return result(:expr_beg, :tOP_ASGN, "|")
elsif scan(/\|/) then
return result(:arg_state, :tPIPE, "|")
end
elsif scan(/\{/) then
self.brace_nest += 1
if lpar_beg && lpar_beg == paren_nest then
self.lpar_beg = nil
self.paren_nest -= 1
token_type, _ = token
return expr_result(:tLAMBEG, "{")
end
if token_type == :tSTRING_END || token_type == :tREGEXP_END then
self.lex_strterm = nil
self.lex_state = :expr_end
end
token = if is_arg? || in_lex_state?(:expr_end, :expr_endfn) then
:tLCURLY # block (primary)
elsif in_lex_state?(:expr_endarg) then
:tLBRACE_ARG # block (expr)
else
:tLBRACE # hash
end
return token
end
self.command_start = true unless token == :tLBRACE
def parse_quote # TODO: remove / rewrite
beg, nnd, short_hand, c = nil, nil, false, nil
return expr_result(token, "{")
elsif scan(/->/) then
return result(:expr_endfn, :tLAMBDA, nil)
elsif scan(/[+-]/) then
sign = matched
utype, type = if sign == "+" then
[:tUPLUS, :tPLUS]
else
[:tUMINUS, :tMINUS]
end
if scan(/[a-z0-9]{1,2}/i) then # Long-hand (e.g. %Q{}).
rb_compile_error "unknown type of %string" if ss.matched_size == 2
c, beg, short_hand = matched, ss.getch, false
else # Short-hand (e.g. %{, %., %!, etc)
c, beg, short_hand = 'Q', ss.getch, true
end
if in_arg_state? then
if scan(/@/) then
return result(:expr_arg, utype, "#{sign}@")
else
return result(:expr_arg, type, sign)
end
end
if end_of_stream? or c == RubyLexer::EOF or beg == RubyLexer::EOF then
rb_compile_error "unterminated quoted string meets end of file"
end
return result(:expr_beg, :tOP_ASGN, sign) if scan(/\=/)
# Figure nnd-char. "\0" is special to indicate beg=nnd and that no nesting?
nnd = { "(" => ")", "[" => "]", "{" => "}", "<" => ">" }[beg]
nnd, beg = beg, "\0" if nnd.nil?
if (is_beg? || (is_arg? && space_seen && !check(/\s/))) then
arg_ambiguous if is_arg?
token_type, text = nil, "%#{c}#{beg}"
token_type, string_type = case c
when 'Q' then
ch = short_hand ? nnd : c + beg
text = "%#{ch}"
[:tSTRING_BEG, STR_DQUOTE]
when 'q' then
[:tSTRING_BEG, STR_SQUOTE]
when 'W' then
scan(/\s*/)
[:tWORDS_BEG, STR_DQUOTE | STR_FUNC_QWORDS]
when 'w' then
scan(/\s*/)
[:tQWORDS_BEG, STR_SQUOTE | STR_FUNC_QWORDS]
when 'x' then
[:tXSTRING_BEG, STR_XQUOTE]
when 'r' then
[:tREGEXP_BEG, STR_REGEXP]
when 's' then
self.lex_state = :expr_fname
[:tSYMBEG, STR_SSYM]
when 'I' then
scan(/\s*/)
[:tSYMBOLS_BEG, STR_DQUOTE | STR_FUNC_QWORDS]
when 'i' then
scan(/\s*/)
[:tQSYMBOLS_BEG, STR_SQUOTE | STR_FUNC_QWORDS]
end
if check(/\d/) then
return self.parse_number if utype == :tUPLUS
return result(:expr_beg, :tUMINUS_NUM, sign)
end
rb_compile_error "Bad %string type. Expected [QqWwIixrs], found '#{c}'." if
token_type.nil?
return result(:expr_beg, utype, sign)
end
raise "huh" unless string_type
return result(:expr_beg, type, sign)
elsif check(/\*/) then
if scan(/\*\*=/) then
return result(:expr_beg, :tOP_ASGN, "**")
elsif scan(/\*\*/) then
token = space_vs_beginning :tDSTAR, :tDSTAR, :tPOW
string string_type, nnd, beg
return result(:arg_state, token, "**")
elsif scan(/\*\=/) then
return result(:expr_beg, :tOP_ASGN, "*")
elsif scan(/\*/) then
token = space_vs_beginning :tSTAR, :tSTAR, :tSTAR2
return token_type, text
end
return result(:arg_state, token, "*")
end
elsif check(/\</) then
if scan(/\<\=\>/) then
return result(:arg_state, :tCMP, "<=>")
elsif scan(/\<\=/) then
return result(:arg_state, :tLEQ, "<=")
elsif scan(/\<\<\=/) then
return result(:arg_state, :tOP_ASGN, "<<")
elsif scan(/\<\</) then
if (!in_lex_state?(:expr_dot, :expr_class) &&
!is_end? &&
(!is_arg? || space_seen)) then
tok = self.heredoc_identifier
return tok if tok
end
def parse_string quote # TODO: rewrite / remove
_, string_type, term, open = quote
return result(:arg_state, :tLSHFT, "\<\<")
elsif scan(/\</) then
return result(:arg_state, :tLT, "<")
end
elsif check(/\>/) then
if scan(/\>\=/) then
return result(:arg_state, :tGEQ, ">=")
elsif scan(/\>\>=/) then
return result(:arg_state, :tOP_ASGN, ">>")
elsif scan(/\>\>/) then
return result(:arg_state, :tRSHFT, ">>")
elsif scan(/\>/) then
return result(:arg_state, :tGT, ">")
end
elsif scan(/\`/) then
case lex_state
when :expr_fname then
return result(:expr_end, :tBACK_REF2, "`")
when :expr_dot then
state = command_state ? :expr_cmdarg : :expr_arg
return result(state, :tBACK_REF2, "`")
else
string STR_XQUOTE, '`'
return result(nil, :tXSTRING_BEG, "`")
end
elsif scan(/\?/) then
if is_end? then
state = ruby18 ? :expr_beg : :expr_value # HACK?
return result(state, :tEH, "?")
end
space = false # FIX: remove these
func = string_type
paren = open
term_re = @@regexp_cache[term]
if end_of_stream? then
rb_compile_error "incomplete character syntax"
end
qwords = (func & STR_FUNC_QWORDS) != 0
regexp = (func & STR_FUNC_REGEXP) != 0
expand = (func & STR_FUNC_EXPAND) != 0
if check(/\s|\v/) then
unless is_arg? then
c2 = { " " => 's',
"\n" => 'n',
"\t" => 't',
"\v" => 'v',
"\r" => 'r',
"\f" => 'f' }[matched]
unless func then # nil'ed from qwords below. *sigh*
return :tSTRING_END, nil
end
if c2 then
warning("invalid character syntax; use ?\\" + c2)
end
end
space = true if qwords and scan(/\s+/)
# ternary
state = ruby18 ? :expr_beg : :expr_value # HACK?
return result(state, :tEH, "?")
elsif check(/\w(?=\w)/) then # ternary, also
return result(:expr_beg, :tEH, "?")
end
if self.string_nest == 0 && scan(/#{term_re}/) then
if qwords then
quote[1] = nil
return :tSPACE, nil
elsif regexp then
return :tREGEXP_END, self.regx_options
else
return :tSTRING_END, term
end
end
c = if scan(/\\/) then
self.read_escape
else
ss.getch
end
return :tSPACE, nil if space
if version == 18 then
return result(:expr_end, :tINTEGER, c[0].ord & 0xff)
else
return result(:expr_end, :tSTRING, c)
end
elsif check(/\&/) then
if scan(/\&\&\=/) then
return result(:expr_beg, :tOP_ASGN, "&&")
elsif scan(/\&\&/) then
return result(:expr_beg, :tANDOP, "&&")
elsif scan(/\&\=/) then
return result(:expr_beg, :tOP_ASGN, "&")
elsif scan(/&/) then
token = if is_arg? && space_seen && !check(/\s/) then
warning("`&' interpreted as argument prefix")
:tAMPER
elsif in_lex_state? :expr_beg, :expr_mid then
:tAMPER
else
:tAMPER2
end
self.string_buffer = []
return result(:arg_state, token, "&")
end
elsif scan(/\//) then
if is_beg? then
string STR_REGEXP, '/'
return result(nil, :tREGEXP_BEG, "/")
end
if scan(/\=/) then
return result(:expr_beg, :tOP_ASGN, "/")
end
if is_arg? && space_seen then
unless scan(/\s/) then
arg_ambiguous
string STR_REGEXP, '/'
return result(nil, :tREGEXP_BEG, "/")
end
end
return result(:arg_state, :tDIVIDE, "/")
elsif scan(/\^=/) then
return result(:expr_beg, :tOP_ASGN, "^")
elsif scan(/\^/) then
return result(:arg_state, :tCARET, "^")
elsif scan(/\;/) then
self.command_start = true
return result(:expr_beg, :tSEMI, ";")
elsif scan(/\~/) then
scan(/@/) if in_lex_state? :expr_fname, :expr_dot
return result(:arg_state, :tTILDE, "~")
elsif scan(/\\/) then
if scan(/\r?\n/) then
self.lineno = nil
self.space_seen = true
next
end
rb_compile_error "bare backslash only allowed before newline"
elsif scan(/\%/) then
return parse_quote if is_beg?
return result(:expr_beg, :tOP_ASGN, "%") if scan(/\=/)
return parse_quote if is_arg? && space_seen && ! check(/\s/)
return result(:arg_state, :tPERCENT, "%")
elsif check(/\$/) then
if scan(/(\$_)(\w+)/) then
self.token = matched
return result(:expr_end, :tGVAR, matched)
elsif scan(/\$_/) then
return result(:expr_end, :tGVAR, matched)
elsif scan(/\$[~*$?!@\/\\;,.=:<>\"]|\$-\w?/) then
return result(:expr_end, :tGVAR, matched)
elsif scan(/\$([\&\`\'\+])/) then
# Explicit reference to these vars as symbols...
if lex_state == :expr_fname then
return result(:expr_end, :tGVAR, matched)
else
return result(:expr_end, :tBACK_REF, ss[1].to_sym)
end
elsif scan(/\$([1-9]\d*)/) then
if lex_state == :expr_fname then
return result(:expr_end, :tGVAR, matched)
else
return result(:expr_end, :tNTH_REF, ss[1].to_i)
end
elsif scan(/\$0/) then
return result(:expr_end, :tGVAR, matched)
elsif scan(/\$\W|\$\z/) then # TODO: remove?
return result(:expr_end, "$", "$") # FIX: "$"??
elsif scan(/\$\w+/)
return result(:expr_end, :tGVAR, matched)
end
elsif check(/\_/) then
if beginning_of_line? && scan(/\__END__(\r?\n|\Z)/) then
self.lineno = nil
return RubyLexer::EOF
elsif scan(/\_\w*/) then
self.token = matched
return process_token command_state, last_state
end
end
end # END OF CASE
if scan(/\004|\032|\000/) || end_of_stream? then # ^D, ^Z, EOF
return RubyLexer::EOF
else # alpha check
rb_compile_error "Invalid char #{ss.rest[0].chr} in expression" unless
check IDENT
if expand
case
when scan(/#(?=[$@])/) then
return :tSTRING_DVAR, nil
when scan(/#[{]/) then
return :tSTRING_DBEG, nil
when scan(/#/) then
string_buffer << '#'
end
self.token = matched if self.scan IDENT
return process_token command_state, last_state
end
end
def yylex_paren18
self.command_start = true
token = :tLPAREN2
if in_lex_state? :expr_beg, :expr_mid then
token = :tLPAREN
elsif space_seen then
if in_lex_state? :expr_cmdarg then
token = :tLPAREN_ARG
elsif in_lex_state? :expr_arg then
warning "don't put space before argument parentheses"
end
else
# not a ternary -- do nothing?
if tokadd_string(func, term, paren) == RubyLexer::EOF then
rb_compile_error "unterminated string meets end of file"
end
token
return :tSTRING_CONTENT, string_buffer.join
end
end
def yylex_paren19
if is_beg? then
:tLPAREN
elsif is_space_arg? then
:tLPAREN_ARG
else
:tLPAREN2 # plain '(' in parse.y
end
end
require "ruby_lexer.rex"
def yylex_string # TODO: rewrite / remove
token = if lex_strterm[0] == :heredoc then
self.heredoc lex_strterm
else
self.parse_string lex_strterm
end
if ENV["DEBUG"] then
class RubyLexer
alias :old_lineno= :lineno=
if token == :tSTRING_END || token == :tREGEXP_END then
self.lineno = nil
self.lex_strterm = nil
self.lex_state = :expr_end
def d o
$stderr.puts o.inspect
end
return token
def lineno= n
self.old_lineno= n
where = caller.first.split(/:/).first(2).join(":")
d :lineno => [n, where, ss && ss.rest[0,40]]
end
end
end
+45
-58
lib/ruby_parser_extras.rb

@@ -62,24 +62,4 @@ # encoding: ASCII-8BIT

def current_line # HAHA fuck you (HACK)
string_to_pos[/\A.*__LINE__/m].split(/\n/).size
end
def extra_lines_added
@extra_lines_added ||= 0
end
def extra_lines_added= val
@extra_lines_added = val
end
def lineno
string[0...charpos].count("\n") + 1 - extra_lines_added
end
# TODO: once we get rid of these, we can make things like
# TODO: current_line and lineno much more accurate and easy to do
def unread_many str # TODO: remove this entirely - we should not need it
warn({:unread_many => caller[0]}.inspect) if ENV['TALLY']
self.extra_lines_added += str.count("\n") - 1
begin

@@ -115,3 +95,3 @@ string[charpos, 0] = str

module RubyParserStuff
VERSION = "3.3.0" unless constants.include? "VERSION" # SIGH
VERSION = "3.4.0" unless constants.include? "VERSION" # SIGH

@@ -153,11 +133,2 @@ attr_accessor :lexer, :in_def, :in_single, :file

def arg_add(node1, node2) # TODO: nuke
return s(:arglist, node2) unless node1
node1[0] = :arglist if node1[0] == :array
return node1 << node2 if node1[0] == :arglist
return s(:arglist, node1, node2)
end
def arg_blk_pass node1, node2 # TODO: nuke

@@ -397,2 +368,3 @@ node1 = s(:arglist, node1) unless [:arglist, :call_args, :array, :args].include? node1.first

def gettable(id)
lineno = id.lineno if id.respond_to? :lineno
id = id.to_sym if String === id

@@ -418,3 +390,3 @@

result.line(result.line - 1) if result.line and lexer.src.bol?
result.line lineno if lineno

@@ -465,3 +437,3 @@ raise "identifier #{id.inspect} is not valid" unless result

def literal_concat head, tail
def literal_concat head, tail # TODO: ugh. rewrite
return tail unless head

@@ -485,4 +457,6 @@ return head unless tail

if htype == :str then
lineno = head.line
tail[1] = head[-1] + tail[1]
head = tail
head.line = lineno
else

@@ -573,3 +547,3 @@ tail[0] = :array

def argl x
x = s(:arglist, x) if x and x[0] != :arglist
x = s(:arglist, x) if x and x[0] == :array
x

@@ -598,9 +572,10 @@ end

args ||= s(:arglist)
args[0] = :arglist if [:args, :array, :call_args].include? args.first
args = s(:arglist, args) unless args.first == :arglist
if args
if [:arglist, :args, :array, :call_args].include? args.first
result.concat args.sexp_body
else
result << args
end
end
# HACK quick hack to make this work quickly... easy to clean up above
result.concat args[1..-1]
line = result.grep(Sexp).map(&:line).compact.min

@@ -612,5 +587,4 @@ result.line = line if line

def new_case expr, body
def new_case expr, body, line
result = s(:case, expr)
line = (expr || body).line

@@ -807,4 +781,10 @@ while body and body.node_type == :when

rescue RegexpError => e
warn "Ignoring: #{e.message}"
Regexp.new(node[1], Regexp::ENC_NONE)
warn "WA\RNING: #{e.message} for #{node[1].inspect} #{options.inspect}"
begin
warn "WA\RNING: trying to recover with ENC_UTF8"
Regexp.new(node[1], Regexp::ENC_UTF8)
rescue RegexpError => e
warn "WA\RNING: trying to recover with ENC_NONE"
Regexp.new(node[1], Regexp::ENC_NONE)
end
end

@@ -933,4 +913,6 @@ end

def next_token
if self.lexer.advance then
return self.lexer.token, self.lexer.yacc_value
token = self.lexer.next_token
if token and token.first != RubyLexer::EOF then
return token
else

@@ -947,9 +929,4 @@ return [false, '$end']

case lhs[0]
when :gasgn, :iasgn, :lasgn, :masgn, :cdecl, :cvdecl, :cvasgn then
when :lasgn, :iasgn, :cdecl, :cvdecl, :gasgn, :cvasgn, :attrasgn then
lhs << rhs
when :attrasgn then
lhs << rhs
when :call then
args = lhs.pop unless Symbol === lhs.last
lhs.concat arg_add(args, rhs)[1..-1]
when :const then

@@ -959,3 +936,3 @@ lhs[0] = :cdecl

else
raise "unknown lhs #{lhs.inspect}"
raise "unknown lhs #{lhs.inspect} w/ #{rhs.inspect}"
end

@@ -1050,6 +1027,8 @@

self.file = file.dup
self.lexer.src = str
@yydebug = ENV.has_key? 'DEBUG'
# HACK -- need to get tests passing more than have graceful code
self.lexer.ss = RPStringScanner.new str
do_parse

@@ -1104,3 +1083,3 @@ end

result = Sexp.new(*args)
result.line ||= lexer.lineno if lexer.src # otherwise...
result.line ||= lexer.lineno if lexer.ss # otherwise...
result.file = self.file

@@ -1388,2 +1367,12 @@ result

class String
##
# This is a hack used by the lexer to sneak in line numbers at the
# identifier level. This should be MUCH smaller than making
# process_token return [value, lineno] and modifying EVERYTHING that
# reduces tIDENTIFIER.
attr_accessor :lineno
end
class Sexp

@@ -1406,8 +1395,6 @@ attr_writer :paren

def add x
concat x
end
alias :add :<<
def add_all x
raise "no: #{self.inspect}.add_all #{x.inspect}" # TODO: need a test to trigger this
self.concat x.sexp_body
end

@@ -1414,0 +1401,0 @@ 

+3
-1
Manifest.txt

@@ -8,3 +8,3 @@ .autotest

bin/ruby_parse_extract_error
lib/gauntlet_rubyparser.rb
lib/.document
lib/ruby18_parser.rb

@@ -17,2 +17,4 @@ lib/ruby18_parser.y

lib/ruby_lexer.rb
lib/ruby_lexer.rex
lib/ruby_lexer.rex.rb
lib/ruby_parser.rb

@@ -19,0 +21,0 @@ lib/ruby_parser_extras.rb

+20
-14
Rakefile
# -*- ruby -*-
require 'rubygems'
require 'hoe'
$:.unshift "../../hoe/dev/lib"
require "rubygems"
require "hoe"
Hoe.plugin :seattlerb

@@ -13,9 +15,10 @@ Hoe.plugin :racc

Hoe.spec 'ruby_parser' do
developer 'Ryan Davis', 'ryand-ruby@zenspider.com'
Hoe.spec "ruby_parser" do
developer "Ryan Davis", "ryand-ruby@zenspider.com"
license "MIT"
dependency 'sexp_processor', '~> 4.1'
dependency 'rake', '< 11', :developer
dependency "sexp_processor", "~> 4.1"
dependency "rake", "< 11", :developer
dependency "oedipus_lex", "~> 2.1", :developer

@@ -26,2 +29,3 @@ if plugin? :perforce then

self.perforce_ignore << "lib/ruby20_parser.rb"
self.perforce_ignore << "lib/ruby_lexer.rex.rb"
end

@@ -35,6 +39,7 @@

file "lib/ruby20_parser.rb" => "lib/ruby20_parser.y"
file "lib/ruby_lexer.rex.rb" => "lib/ruby_lexer.rex"
task :clean do
rm_rf(Dir["**/*~"] +
Dir["**/*.diff"] +
Dir["diff.diff"] + # not all diffs. bit me too many times
Dir["coverage.info"] +

@@ -57,8 +62,8 @@ Dir["coverage"] +

end
system 'find -d unit -type d -empty -exec rmdir {} \;'
system "find -d unit -type d -empty -exec rmdir {} \;"
end
task :sort do
sh 'grepsort "^ +def" lib/ruby_lexer.rb'
sh 'grepsort "^ +def (test|util)" test/test_ruby_lexer.rb'
sh "grepsort '^ +def' lib/ruby_lexer.rb"
sh "grepsort '^ +def (test|util)' test/test_ruby_lexer.rb"
end

@@ -85,3 +90,3 @@

def run_and_log cmd, prefix
files = ENV['FILES'] || 'unit/*.rb'
files = ENV["FILES"] || "unit/*.rb"
p, x = prefix, "txt"

@@ -148,6 +153,7 @@ n = Dir["#{p}.*.#{x}"].map { |s| s[/\d+/].to_i }.max + 1 rescue 1

Rake.application[:parser].invoke # this way we can have DEBUG set
Rake.application[:lexer].invoke # this way we can have DEBUG set
$: << "lib"
require 'ruby_parser'
require 'pp'
require "ruby_parser"
require "pp"

@@ -178,3 +184,3 @@ parser = case ENV["V"]

p e
ss = parser.lexer.src
ss = parser.lexer.ss
src = ss.string

@@ -181,0 +187,0 @@ lines = src[0..ss.pos].split(/\n/)

-117
lib/gauntlet_rubyparser.rb
#!/usr/bin/ruby -ws
$f ||= false
$:.unshift "../../ruby_parser/dev/lib"
$:.unshift "../../ruby2ruby/dev/lib"
require 'rubygems'
require 'ruby2ruby'
require 'ruby_parser'
require 'gauntlet'
class RubyParserGauntlet < Gauntlet
def initialize
super
self.data = Hash.new { |h,k| h[k] = {} }
old_data = load_yaml data_file
self.data.merge! old_data
end
def should_skip? name
if $f then
if Hash === data[name] then
! data[name].empty?
else
data[name]
end
else
data[name] == true # yes, == true on purpose
end
end
def diff_pp o1, o2
require 'pp'
Tempfile.new('ruby_parser_a') do |file_a|
PP.pp o1, file_a
Tempfile.new('ruby_parser_b') do |file_b|
PP.pp o2, file_b
`diff -u #{file_a.path} #{file_b.path}`
end
end
end
def broke name, file, msg
warn "bad"
self.data[name][file] = msg
self.dirty = true
end
def process path, name
begin
$stderr.print " #{path}: "
rp = RubyParser.new
r2r = Ruby2Ruby.new
old_ruby = File.read(path)
begin
old_sexp = rp.process old_ruby
rescue Racc::ParseError => e
self.data[name][path] = :unparsable
self.dirty = true
return
end
new_ruby = r2r.process old_sexp.deep_clone
begin
new_sexp = rp.process new_ruby
rescue Racc::ParseError => e
broke name, path, "couldn't parse new_ruby: #{e.message.strip}"
return
end
if old_sexp != new_sexp then
broke name, path, diff_pp(old_sexp, new_sexp)
return
end
self.data[name][path] = true
self.dirty = true
warn "good"
rescue Interrupt
puts "User cancelled"
exit 1
rescue Exception => e
broke name, path, " UNKNOWN ERROR: #{e}: #{e.message.strip}"
end
end
def run name
warn name
Dir["**/*.rb"].sort.each do |path|
next if path =~ /gemspec.rb/ # HACK
next if data[name][path] == true
process path, name
end
if self.data[name].values.all? { |v| v == true } then
warn " ALL GOOD!"
self.data[name] = true
self.dirty = true
end
end
end
filter = ARGV.shift
filter = Regexp.new filter if filter
gauntlet = RubyParserGauntlet.new
gauntlet.run_the_gauntlet filter
.autotest

Sorry, the diff of this file is not supported yet

bin/ruby_parse_extract_error

Sorry, the diff of this file is not supported yet

lib/ruby18_parser.rb

Sorry, the diff of this file is too big to display

lib/ruby18_parser.y

Sorry, the diff of this file is not supported yet

lib/ruby19_parser.rb

Sorry, the diff of this file is too big to display

lib/ruby19_parser.y

Sorry, the diff of this file is not supported yet

lib/ruby20_parser.rb

Sorry, the diff of this file is too big to display

lib/ruby20_parser.y

Sorry, the diff of this file is not supported yet

test/test_ruby_lexer.rb

Sorry, the diff of this file is too big to display

test/test_ruby_parser.rb

Sorry, the diff of this file is too big to display