🚀 Socket Launch Week Day 5:Introducing Repository Access Permissions and Custom Roles.Learn more
Sign In

ruby_parser

Package Overview
Dependencies
Maintainers
1
Versions
80
Alerts
File Explorer

Advanced tools

Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

ruby_parser - rubygems Package Compare versions

Comparing version
3.4.1
to
3.5.0
lib/ruby21_parser.rb

Sorry, the diff of this file is too big to display

Sorry, the diff of this file is not supported yet

+20
-0

@@ -0,3 +1,21 @@

=== 3.5.0 / 2014-03-24
* 1 major enhancement:
* Added initial support for ruby 2.1 syntax. Doesn't have numeric extensions yet.
* 2 minor enhancements:
* Converted ruby_lexer.rex to use new grouping mechanism. ~15% improvement in speed.
* Various lexer cleanup.
* 2 bug fixes:
* 2.0/2.1: Fixed block kw args.
* Fixed env for kwargs and friends.
=== 3.4.1 / 2014-02-14
21 files failed to parse out of ~834k files makes this 99.9975% or 4.21σ.
* 1 minor enhancement:

@@ -13,2 +31,4 @@

31 files failed to parse out of ~834k files makes this 99.9963% or 4.12σ.
* 1 major enhancement:

@@ -15,0 +35,0 @@

+8
-22

@@ -135,2 +135,10 @@ # encoding: UTF-8

def expr_dot?
lex_state == :expr_dot
end
def expr_fname?
lex_state == :expr_fname
end
def expr_result token, text

@@ -308,24 +316,2 @@ cond.push false

def process_backtick text
case lex_state
when :expr_fname then
result :expr_end, :tBACK_REF2, "`"
when :expr_dot then
result((command_state ? :expr_cmdarg : :expr_arg), :tBACK_REF2, "`")
else
string STR_XQUOTE
result nil, :tXSTRING_BEG, "`"
end
end
def process_bang text
if in_arg_state? then
return result(:expr_arg, :tUBANG, "!@") if scan(/@/)
end
text = scan(/[=~]/) ? "!#{matched}" : "!"
return result(arg_state, TOKENS[text], text)
end
def process_begin text

@@ -332,0 +318,0 @@ @comments << matched

#--
# This file is automatically generated. Do not modify it.
# Generated by: oedipus_lex version 2.1.0.
# Generated by: oedipus_lex version 2.2.0.
# Source: lib/ruby_lexer.rex

@@ -88,18 +88,31 @@ #++

process_bracing text
when text = ss.scan(/\!/) then
process_bang text
when text = ss.scan(/\.\.\.?|,|![=~]?/) then
action { result :expr_beg, TOKENS[text], text }
when text = ss.scan(/\.\d/) then
action { rb_compile_error "no .<digit> floating literal anymore put 0 before dot" }
when text = ss.scan(/\./) then
action { result :expr_dot, :tDOT, "." }
when ss.check(/\!/) then
case
when in_arg_state? && (text = ss.scan(/\!\@/)) then
action { result :expr_arg, :tUBANG, "!@" }
when text = ss.scan(/\![=~]?/) then
action { result :arg_state, TOKENS[text], text }
end # group /\!/
when ss.check(/\./) then
case
when text = ss.scan(/\.\.\.?/) then
action { result :expr_beg, TOKENS[text], text }
when text = ss.scan(/\.\d/) then
action { rb_compile_error "no .<digit> floating literal anymore put 0 before dot" }
when text = ss.scan(/\./) then
action { result :expr_dot, :tDOT, "." }
end # group /\./
when text = ss.scan(/\(/) then
process_paren text
when text = ss.scan(/\=\=\=|\=\=|\=~|\=>|\=(?!begin\b)/) then
action { result arg_state, TOKENS[text], text }
when bol? && (text = ss.scan(/\=begin(?=\s)/)) then
process_begin text
when text = ss.scan(/\=(?=begin\b)/) then
action { result arg_state, TOKENS[text], text }
when text = ss.scan(/\,/) then
action { result :expr_beg, TOKENS[text], text }
when ss.check(/=/) then
case
when text = ss.scan(/\=\=\=|\=\=|\=~|\=>|\=(?!begin\b)/) then
action { result arg_state, TOKENS[text], text }
when bol? && (text = ss.scan(/\=begin(?=\s)/)) then
process_begin text
when text = ss.scan(/\=(?=begin\b)/) then
action { result arg_state, TOKENS[text], text }
end # group /=/
when text = ss.scan(/\"(#{SIMPLE_STRING})\"/o) then

@@ -113,12 +126,15 @@ action { result :expr_end, :tSTRING, text[1..-2].gsub(ESC) { unescape $1 } }

process_ivar text
when not_end? && (text = ss.scan(/:([a-zA-Z_]#{IDENT_CHAR}*(?:[?]|[!](?!=)|=(?==>)|=(?![=>]))?)/o)) then
process_symbol text
when not_end? && (text = ss.scan(/\:\"(#{SIMPLE_STRING})\"/o)) then
process_symbol text
when not_end? && (text = ss.scan(/\:\'(#{SSTRING})\'/o)) then
process_symbol text
when text = ss.scan(/\:\:/) then
process_colon2 text
when text = ss.scan(/\:/) then
process_colon1 text
when ss.check(/:/) then
case
when not_end? && (text = ss.scan(/:([a-zA-Z_]#{IDENT_CHAR}*(?:[?]|[!](?!=)|=(?==>)|=(?![=>]))?)/o)) then
process_symbol text
when not_end? && (text = ss.scan(/\:\"(#{SIMPLE_STRING})\"/o)) then
process_symbol text
when not_end? && (text = ss.scan(/\:\'(#{SSTRING})\'/o)) then
process_symbol text
when text = ss.scan(/\:\:/) then
process_colon2 text
when text = ss.scan(/\:/) then
process_colon1 text
end # group /:/
when text = ss.scan(/->/) then

@@ -128,22 +144,25 @@ action { result :expr_endfn, :tLAMBDA, nil }

process_plus_minus text
when text = ss.scan(/#{NUM_BAD}/o) then
action { rb_compile_error "Invalid numeric format" }
when text = ss.scan(/#{INT_DEC}/o) then
action { int_with_base 10 }
when text = ss.scan(/#{INT_HEX}/o) then
action { int_with_base 16 }
when text = ss.scan(/#{INT_BIN}/o) then
action { int_with_base 2 }
when text = ss.scan(/#{INT_OCT_BAD}/o) then
action { rb_compile_error "Illegal octal digit." }
when text = ss.scan(/#{INT_OCT}/o) then
action { int_with_base 8 }
when text = ss.scan(/#{FLOAT_BAD}/o) then
action { rb_compile_error "Trailing '_' in number." }
when text = ss.scan(/#{FLOAT}/o) then
process_float text
when text = ss.scan(/#{INT_DEC2}/o) then
action { int_with_base 10 }
when text = ss.scan(/[0-9]/) then
action { rb_compile_error "Bad number format" }
when ss.check(/[+\d]/) then
case
when text = ss.scan(/#{NUM_BAD}/o) then
action { rb_compile_error "Invalid numeric format" }
when text = ss.scan(/#{INT_DEC}/o) then
action { int_with_base 10 }
when text = ss.scan(/#{INT_HEX}/o) then
action { int_with_base 16 }
when text = ss.scan(/#{INT_BIN}/o) then
action { int_with_base 2 }
when text = ss.scan(/#{INT_OCT_BAD}/o) then
action { rb_compile_error "Illegal octal digit." }
when text = ss.scan(/#{INT_OCT}/o) then
action { int_with_base 8 }
when text = ss.scan(/#{FLOAT_BAD}/o) then
action { rb_compile_error "Trailing '_' in number." }
when text = ss.scan(/#{FLOAT}/o) then
process_float text
when text = ss.scan(/#{INT_DEC2}/o) then
action { int_with_base 10 }
when text = ss.scan(/[0-9]/) then
action { rb_compile_error "Bad number format" }
end # group /[+\d]/
when text = ss.scan(/\[/) then

@@ -153,88 +172,122 @@ process_square_bracket text

action { result :expr_end, :tSTRING, matched[1..-2].gsub(/\\\\/, "\\").gsub(/\\'/, "'") } # " stupid emacs
when text = ss.scan(/\|\|\=/) then
action { result :expr_beg, :tOP_ASGN, "||" }
when text = ss.scan(/\|\|/) then
action { result :expr_beg, :tOROP, "||" }
when text = ss.scan(/\|\=/) then
action { result :expr_beg, :tOP_ASGN, "|" }
when text = ss.scan(/\|/) then
action { result :arg_state, :tPIPE, "|" }
when ss.check(/\|/) then
case
when text = ss.scan(/\|\|\=/) then
action { result :expr_beg, :tOP_ASGN, "||" }
when text = ss.scan(/\|\|/) then
action { result :expr_beg, :tOROP, "||" }
when text = ss.scan(/\|\=/) then
action { result :expr_beg, :tOP_ASGN, "|" }
when text = ss.scan(/\|/) then
action { result :arg_state, :tPIPE, "|" }
end # group /\|/
when text = ss.scan(/\{/) then
process_curly_brace text
when text = ss.scan(/\*\*=/) then
action { result :expr_beg, :tOP_ASGN, "**" }
when text = ss.scan(/\*\*/) then
action { result(:arg_state, space_vs_beginning(:tDSTAR, :tDSTAR, :tPOW), "**") }
when text = ss.scan(/\*\=/) then
action { result(:expr_beg, :tOP_ASGN, "*") }
when text = ss.scan(/\*/) then
action { result(:arg_state, space_vs_beginning(:tSTAR, :tSTAR, :tSTAR2), "*") }
when text = ss.scan(/\<\=\>/) then
action { result :arg_state, :tCMP, "<=>" }
when text = ss.scan(/\<\=/) then
action { result :arg_state, :tLEQ, "<=" }
when text = ss.scan(/\<\<\=/) then
action { result :arg_state, :tOP_ASGN, "<<" }
when text = ss.scan(/\<\</) then
process_lchevron text
when text = ss.scan(/\</) then
action { result :arg_state, :tLT, "<" }
when text = ss.scan(/\>\=/) then
action { result :arg_state, :tGEQ, ">=" }
when text = ss.scan(/\>\>=/) then
action { result :arg_state, :tOP_ASGN, ">>" }
when text = ss.scan(/\>\>/) then
action { result :arg_state, :tRSHFT, ">>" }
when text = ss.scan(/\>/) then
action { result :arg_state, :tGT, ">" }
when text = ss.scan(/\`/) then
process_backtick text
when ss.check(/\*/) then
case
when text = ss.scan(/\*\*=/) then
action { result :expr_beg, :tOP_ASGN, "**" }
when text = ss.scan(/\*\*/) then
action { result(:arg_state, space_vs_beginning(:tDSTAR, :tDSTAR, :tPOW), "**") }
when text = ss.scan(/\*\=/) then
action { result(:expr_beg, :tOP_ASGN, "*") }
when text = ss.scan(/\*/) then
action { result(:arg_state, space_vs_beginning(:tSTAR, :tSTAR, :tSTAR2), "*") }
end # group /\*/
when ss.check(/</) then
case
when text = ss.scan(/\<\=\>/) then
action { result :arg_state, :tCMP, "<=>" }
when text = ss.scan(/\<\=/) then
action { result :arg_state, :tLEQ, "<=" }
when text = ss.scan(/\<\<\=/) then
action { result :arg_state, :tOP_ASGN, "<<" }
when text = ss.scan(/\<\</) then
process_lchevron text
when text = ss.scan(/\</) then
action { result :arg_state, :tLT, "<" }
end # group /</
when ss.check(/>/) then
case
when text = ss.scan(/\>\=/) then
action { result :arg_state, :tGEQ, ">=" }
when text = ss.scan(/\>\>=/) then
action { result :arg_state, :tOP_ASGN, ">>" }
when text = ss.scan(/\>\>/) then
action { result :arg_state, :tRSHFT, ">>" }
when text = ss.scan(/\>/) then
action { result :arg_state, :tGT, ">" }
end # group />/
when ss.check(/`/) then
case
when expr_fname? && (text = ss.scan(/\`/)) then
action { result(:expr_end, :tBACK_REF2, "`") }
when expr_dot? && (text = ss.scan(/\`/)) then
action { result((command_state ? :expr_cmdarg : :expr_arg), :tBACK_REF2, "`") }
when text = ss.scan(/\`/) then
action { string STR_XQUOTE, '`'; result(nil, :tXSTRING_BEG, "`") }
end # group /`/
when text = ss.scan(/\?/) then
process_questionmark text
when text = ss.scan(/\&\&\=/) then
action { result(:expr_beg, :tOP_ASGN, "&&") }
when text = ss.scan(/\&\&/) then
action { result(:expr_beg, :tANDOP, "&&") }
when text = ss.scan(/\&\=/) then
action { result(:expr_beg, :tOP_ASGN, "&" ) }
when text = ss.scan(/\&/) then
process_amper text
when ss.check(/&/) then
case
when text = ss.scan(/\&\&\=/) then
action { result(:expr_beg, :tOP_ASGN, "&&") }
when text = ss.scan(/\&\&/) then
action { result(:expr_beg, :tANDOP, "&&") }
when text = ss.scan(/\&\=/) then
action { result(:expr_beg, :tOP_ASGN, "&" ) }
when text = ss.scan(/\&/) then
process_amper text
end # group /&/
when text = ss.scan(/\//) then
process_slash text
when text = ss.scan(/\^=/) then
action { result(:expr_beg, :tOP_ASGN, "^") }
when text = ss.scan(/\^/) then
action { result(:arg_state, :tCARET, "^") }
when ss.check(/\^/) then
case
when text = ss.scan(/\^=/) then
action { result(:expr_beg, :tOP_ASGN, "^") }
when text = ss.scan(/\^/) then
action { result(:arg_state, :tCARET, "^") }
end # group /\^/
when text = ss.scan(/\;/) then
action { self.command_start = true; result(:expr_beg, :tSEMI, ";") }
when in_arg_state? && (text = ss.scan(/\~@/)) then
action { result(:arg_state, :tTILDE, "~") }
when text = ss.scan(/\~/) then
action { result(:arg_state, :tTILDE, "~") }
when text = ss.scan(/\\\r?\n/) then
action { self.lineno += 1; self.space_seen = true; next }
when text = ss.scan(/\\/) then
action { rb_compile_error "bare backslash only allowed before newline" }
when ss.check(/~/) then
case
when in_arg_state? && (text = ss.scan(/\~@/)) then
action { result(:arg_state, :tTILDE, "~") }
when text = ss.scan(/\~/) then
action { result(:arg_state, :tTILDE, "~") }
end # group /~/
when ss.check(/\\/) then
case
when text = ss.scan(/\\\r?\n/) then
action { self.lineno += 1; self.space_seen = true; next }
when text = ss.scan(/\\/) then
action { rb_compile_error "bare backslash only allowed before newline" }
end # group /\\/
when text = ss.scan(/\%/) then
process_percent text
when text = ss.scan(/\$_\w+/) then
process_gvar text
when text = ss.scan(/\$_/) then
process_gvar text
when text = ss.scan(/\$[~*$?!@\/\\;,.=:<>\"]|\$-\w?/) then
process_gvar text
when in_fname? && (text = ss.scan(/\$([\&\`\'\+])/)) then
process_gvar text
when text = ss.scan(/\$([\&\`\'\+])/) then
process_backref text
when in_fname? && (text = ss.scan(/\$([1-9]\d*)/)) then
process_gvar text
when text = ss.scan(/\$([1-9]\d*)/) then
process_nthref text
when text = ss.scan(/\$0/) then
process_gvar text
when text = ss.scan(/\$\W|\$\z/) then
process_gvar_oddity text
when text = ss.scan(/\$\w+/) then
process_gvar text
when ss.check(/\$/) then
case
when text = ss.scan(/\$_\w+/) then
process_gvar text
when text = ss.scan(/\$_/) then
process_gvar text
when text = ss.scan(/\$[~*$?!@\/\\;,.=:<>\"]|\$-\w?/) then
process_gvar text
when in_fname? && (text = ss.scan(/\$([\&\`\'\+])/)) then
process_gvar text
when text = ss.scan(/\$([\&\`\'\+])/) then
process_backref text
when in_fname? && (text = ss.scan(/\$([1-9]\d*)/)) then
process_gvar text
when text = ss.scan(/\$([1-9]\d*)/) then
process_nthref text
when text = ss.scan(/\$0/) then
process_gvar text
when text = ss.scan(/\$\W|\$\z/) then
process_gvar_oddity text
when text = ss.scan(/\$\w+/) then
process_gvar text
end # group /\$/
when text = ss.scan(/\_/) then

@@ -241,0 +294,0 @@ process_underscore text

@@ -94,3 +94,3 @@ # encoding: ASCII-8BIT

module RubyParserStuff
VERSION = "3.4.1" unless constants.include? "VERSION" # SIGH
VERSION = "3.5.0" unless constants.include? "VERSION" # SIGH

@@ -244,2 +244,4 @@ attr_accessor :lexer, :in_def, :in_single, :file

when Symbol then
name = arg.to_s.delete("&*")
self.env[name.to_sym] = :lvar unless name.empty?
result << arg

@@ -409,3 +411,3 @@ when ",", "|", ";", "(", ")", nil then

v = self.class.name[/1[89]|20/]
v = self.class.name[/1[89]|2[01]/]

@@ -694,2 +696,8 @@ self.lexer = RubyLexer.new v && v.to_i

def new_masgn_arg rhs, wrap = false
rhs = value_expr(rhs)
rhs = s(:to_ary, rhs) if wrap # HACK: could be array if lhs isn't right
rhs
end
def new_masgn lhs, rhs, wrap = false

@@ -1297,2 +1305,6 @@ rhs = value_expr(rhs)

class Ruby21Parser < Racc::Parser
include RubyParserStuff
end
class Ruby20Parser < Racc::Parser

@@ -1322,12 +1334,15 @@ include RubyParserStuff

@p20 = Ruby20Parser.new
@p21 = Ruby21Parser.new
end
def process(s, f = "(string)", t = 10) # parens for emacs *sigh*
@p20.process s, f, t
rescue Racc::ParseError, RubyParser::SyntaxError
begin
@p19.process s, f, t
rescue Racc::ParseError, RubyParser::SyntaxError
@p18.process s, f, t
def process s, f = "(string)", t = 10
e = nil
[@p21, @p20, @p19, @p18].each do |parser|
begin
return parser.process s, f, t
rescue Racc::ParseError, RubyParser::SyntaxError => exc
e = exc
end
end
raise e
end

@@ -1340,2 +1355,4 @@

@p19.reset
@p20.reset
@p21.reset
end

@@ -1351,2 +1368,4 @@

Ruby20Parser.new
when /^2.1/ then
Ruby21Parser.new
else

@@ -1353,0 +1372,0 @@ raise "unrecognized RUBY_VERSION #{RUBY_VERSION}"

require 'ruby18_parser'
require 'ruby19_parser'
require 'ruby20_parser'
require 'ruby21_parser'
require 'ruby_parser_extras'

@@ -15,2 +15,4 @@ .autotest

lib/ruby20_parser.y
lib/ruby21_parser.rb
lib/ruby21_parser.y
lib/ruby_lexer.rb

@@ -17,0 +19,0 @@ lib/ruby_lexer.rex

+23
-80

@@ -14,2 +14,3 @@ # -*- ruby -*-

Hoe.add_include_dirs "../../minitest/dev/lib"
Hoe.add_include_dirs "../../oedipus_lex/dev/lib"

@@ -29,2 +30,3 @@ Hoe.spec "ruby_parser" do

self.perforce_ignore << "lib/ruby20_parser.rb"
self.perforce_ignore << "lib/ruby21_parser.rb"
self.perforce_ignore << "lib/ruby_lexer.rex.rb"

@@ -39,2 +41,3 @@ end

file "lib/ruby20_parser.rb" => "lib/ruby20_parser.y"
file "lib/ruby21_parser.rb" => "lib/ruby21_parser.y"
file "lib/ruby_lexer.rex.rb" => "lib/ruby_lexer.rex"

@@ -50,17 +53,2 @@

def next_num(glob)
num = Dir[glob].max[/\d+/].to_i + 1
end
desc "Compares PT to RP and deletes all files that match"
task :compare do
files = Dir["unit/**/*.rb"]
puts "Parsing #{files.size} files"
files.each do |file|
puts file
system "./cmp.rb -q #{file} && rm #{file}"
end
system "find -d unit -type d -empty -exec rmdir {} \;"
end
task :sort do

@@ -71,41 +59,2 @@ sh "grepsort '^ +def' lib/ruby_lexer.rb"

task :loc do
loc1 = `wc -l ../1.0.0/lib/ruby_lexer.rb`[/\d+/]
flog1 = `flog -s ../1.0.0/lib/ruby_lexer.rb`[/\d+\.\d+/]
loc2 = `cat lib/ruby_lexer.rb lib/ruby_parser_extras.rb | wc -l`[/\d+/]
flog2 = `flog -s lib/ruby_lexer.rb lib/ruby_parser_extras.rb`[/\d+\.\d+/]
loc1, loc2, flog1, flog2 = loc1.to_i, loc2.to_i, flog1.to_f, flog2.to_f
puts "1.0.0: loc = #{loc1} flog = #{flog1}"
puts "dev : loc = #{loc2} flog = #{flog2}"
puts "delta: loc = #{loc2-loc1} flog = #{flog2-flog1}"
end
desc "Validate against all normal files in unit dir"
task :validate do
sh "./cmp.rb unit/*.rb"
end
def run_and_log cmd, prefix
files = ENV["FILES"] || "unit/*.rb"
p, x = prefix, "txt"
n = Dir["#{p}.*.#{x}"].map { |s| s[/\d+/].to_i }.max + 1 rescue 1
f = "#{p}.#{n}.#{x}"
sh "#{cmd} #{Hoe::RUBY_FLAGS} bin/ruby_parse -q -g #{files} &> #{f}"
puts File.read(f)
end
desc "Benchmark against all normal files in unit dir"
task :benchmark do
run_and_log "ruby", "benchmark"
end
desc "Profile against all normal files in unit dir"
task :profile do
run_and_log "zenprofile", "profile"
end
desc "what was that command again?"

@@ -126,26 +75,20 @@ task :huh? do

task :compare18 do
sh "./yack.rb lib/ruby18_parser.output > racc18.txt"
sh "./yack.rb parse18.output > yacc18.txt"
sh "diff -du racc18.txt yacc18.txt || true"
puts
sh "diff -du racc18.txt yacc18.txt | wc -l"
end
# to create parseXX.output:
#
# 1) check out the XX version of ruby
# 2) Edit uncommon.mk, find the ".y.c" rule and remove the RM lines
# 3) run `rm -f parse.c; make parse.c`
# 4) run `bison -r all parse.tmp.y`
# 5) mv parse.tmp.output parseXX.output
task :compare19 do
sh "./yack.rb lib/ruby19_parser.output > racc19.txt"
sh "./yack.rb parse19.output > yacc19.txt"
sh "diff -du racc19.txt yacc19.txt || true"
puts
sh "diff -du racc19.txt yacc19.txt | wc -l"
%w[18 19 20 21].each do |v|
task "compare#{v}" do
sh "./yack.rb lib/ruby#{v}_parser.output > racc#{v}.txt"
sh "./yack.rb parse#{v}.output > yacc#{v}.txt"
sh "diff -du racc#{v}.txt yacc#{v}.txt || true"
puts
sh "diff -du racc#{v}.txt yacc#{v}.txt | wc -l"
end
end
task :compare20 do
sh "./yack.rb lib/ruby20_parser.output > racc20.txt"
sh "./yack.rb parse20.output > yacc20.txt"
sh "diff -du racc20.txt yacc20.txt || true"
puts
sh "diff -du racc20.txt yacc20.txt | wc -l"
end
task :debug => :isolate do

@@ -165,4 +108,8 @@ ENV["V"] ||= "20"

Ruby19Parser.new
when "20" then
Ruby20Parser.new
when "21" then
Ruby21Parser.new
else
Ruby20Parser.new
raise "Unsupported version #{ENV["V"]}"
end

@@ -192,6 +139,2 @@

def ruby20
"/Users/ryan/.multiruby/install/2.0.0-p195/bin/ruby"
end
task :debug_ruby do

@@ -198,0 +141,0 @@ file = ENV["F"] || ENV["FILE"]

Sorry, the diff of this file is not supported yet

Sorry, the diff of this file is not supported yet

Sorry, the diff of this file is too big to display

Sorry, the diff of this file is not supported yet

Sorry, the diff of this file is too big to display

Sorry, the diff of this file is too big to display