Feature #13686 ยป add-state-to-ripper-for-trunk.patch
| ext/ripper/lib/ripper/filter.rb (working copy) | ||
|---|---|---|
|
@__lexer = Lexer.new(src, filename, lineno)
|
||
|
@__line = nil
|
||
|
@__col = nil
|
||
|
@__state = nil
|
||
|
end
|
||
|
# The file name of the input.
|
||
| ... | ... | |
|
@__col
|
||
|
end
|
||
|
# The scanner's state of the current token.
|
||
|
# This value is the bitwise OR of zero or more of the +Ripper::EXPR_*+ constants.
|
||
|
def state
|
||
|
@__state
|
||
|
end
|
||
|
# Starts the parser.
|
||
|
# +init+ is a data accumulator and is passed to the next event handler (as
|
||
|
# of Enumerable#inject).
|
||
|
def parse(init = nil)
|
||
|
data = init
|
||
|
@__lexer.lex.each do |pos, event, tok|
|
||
|
@__lexer.lex.each do |pos, event, tok, state|
|
||
|
@__line, @__col = *pos
|
||
|
@__state = state
|
||
|
data = if respond_to?(event, true)
|
||
|
then __send__(event, tok, data)
|
||
|
else on_default(event, tok, data)
|
||
| ext/ripper/lib/ripper/lexer.rb (working copy) | ||
|---|---|---|
|
end
|
||
|
# Tokenizes the Ruby program and returns an array of an array,
|
||
|
# which is formatted like <code>[[lineno, column], type, token]</code>.
|
||
|
# which is formatted like
|
||
|
# <code>[[lineno, column], type, token, state]</code>.
|
||
|
#
|
||
|
# require 'ripper'
|
||
|
# require 'pp'
|
||
|
#
|
||
|
# pp Ripper.lex("def m(a) nil end")
|
||
|
# #=> [[[1, 0], :on_kw, "def"],
|
||
|
# [[1, 3], :on_sp, " " ],
|
||
|
# [[1, 4], :on_ident, "m" ],
|
||
|
# [[1, 5], :on_lparen, "(" ],
|
||
|
# [[1, 6], :on_ident, "a" ],
|
||
|
# [[1, 7], :on_rparen, ")" ],
|
||
|
# [[1, 8], :on_sp, " " ],
|
||
|
# [[1, 9], :on_kw, "nil"],
|
||
|
# [[1, 12], :on_sp, " " ],
|
||
|
# [[1, 13], :on_kw, "end"]]
|
||
|
# #=> [[[1, 0], :on_kw, "def", Ripper::EXPR_FNAME ],
|
||
|
# [[1, 3], :on_sp, " ", Ripper::EXPR_FNAME ],
|
||
|
# [[1, 4], :on_ident, "m", Ripper::EXPR_ENDFN ],
|
||
|
# [[1, 5], :on_lparen, "(", Ripper::EXPR_LABEL | Ripper::EXPR_BEG],
|
||
|
# [[1, 6], :on_ident, "a", Ripper::EXPR_ARG ],
|
||
|
# [[1, 7], :on_rparen, ")", Ripper::EXPR_ENDFN ],
|
||
|
# [[1, 8], :on_sp, " ", Ripper::EXPR_BEG ],
|
||
|
# [[1, 9], :on_kw, "nil", Ripper::EXPR_END ],
|
||
|
# [[1, 12], :on_sp, " ", Ripper::EXPR_END ],
|
||
|
# [[1, 13], :on_kw, "end", Ripper::EXPR_END ]]
|
||
|
#
|
||
|
def Ripper.lex(src, filename = '-', lineno = 1)
|
||
|
Lexer.new(src, filename, lineno).lex
|
||
|
end
|
||
|
class Lexer < ::Ripper #:nodoc: internal use only
|
||
|
Elem = Struct.new(:pos, :event, :tok)
|
||
|
Elem = Struct.new(:pos, :event, :tok, :state)
|
||
|
def tokenize
|
||
|
parse().sort_by(&:pos).map(&:tok)
|
||
| ... | ... | |
|
if Elem === e and e.event == :on_tstring_content
|
||
|
tok = e.tok.dup if w > 0 and /\A\s/ =~ e.tok
|
||
|
if (n = dedent_string(e.tok, w)) > 0
|
||
|
ignored_sp << [i, Elem.new(e.pos.dup, :on_ignored_sp, tok[0, n])]
|
||
|
ignored_sp << [i, Elem.new(e.pos.dup, :on_ignored_sp, tok[0, n], e.state)]
|
||
|
e.pos[1] += n
|
||
|
end
|
||
|
end
|
||
| ... | ... | |
|
buf = []
|
||
|
@buf << buf
|
||
|
@buf = buf
|
||
|
@buf.push Elem.new([lineno(), column()], __callee__, tok)
|
||
|
@buf.push Elem.new([lineno(), column()], __callee__, tok, state())
|
||
|
end
|
||
|
def on_heredoc_end(tok)
|
||
|
@buf.push Elem.new([lineno(), column()], __callee__, tok)
|
||
|
@buf.push Elem.new([lineno(), column()], __callee__, tok, state())
|
||
|
@buf = @stack.pop
|
||
|
end
|
||
|
def _push_token(tok)
|
||
|
@buf.push Elem.new([lineno(), column()], __callee__, tok)
|
||
|
@buf.push Elem.new([lineno(), column()], __callee__, tok, state())
|
||
|
end
|
||
|
(SCANNER_EVENTS.map {|event|:"on_#{event}"} - private_instance_methods(false)).each do |event|
|
||
| parse.y (working copy) | ||
|---|---|---|
|
const char *token;
|
||
|
int linenum;
|
||
|
int column;
|
||
|
enum lex_state_e state;
|
||
|
int nonspc;
|
||
|
struct token_info *next;
|
||
|
} token_info;
|
||
| ... | ... | |
|
ptinfo->token = token;
|
||
|
ptinfo->linenum = ruby_sourceline;
|
||
|
ptinfo->column = token_info_get_column(parser, t);
|
||
|
ptinfo->state = lex_state;
|
||
|
ptinfo->nonspc = token_info_has_nonspaces(parser, t);
|
||
|
ptinfo->next = parser->token_info;
|
||
| ... | ... | |
|
return INT2NUM(ruby_sourceline);
|
||
|
}
|
||
|
/*
|
||
|
* call-seq:
|
||
|
* ripper#state -> Integer
|
||
|
*
|
||
|
* Return scanner state of current token.
|
||
|
*/
|
||
|
static VALUE
|
||
|
ripper_state(VALUE self)
|
||
|
{
|
||
|
struct parser_params *parser;
|
||
|
TypedData_Get_Struct(self, struct parser_params, &parser_data_type, parser);
|
||
|
if (!ripper_initialized_p(parser)) {
|
||
|
rb_raise(rb_eArgError, "method called for uninitialized object");
|
||
|
}
|
||
|
if (NIL_P(parser->parsing_thread)) return Qnil;
|
||
|
return INT2NUM(lex_state);
|
||
|
}
|
||
|
#ifdef RIPPER_DEBUG
|
||
|
/* :nodoc: */
|
||
|
static VALUE
|
||
| ... | ... | |
|
rb_define_method(Ripper, "column", ripper_column, 0);
|
||
|
rb_define_method(Ripper, "filename", ripper_filename, 0);
|
||
|
rb_define_method(Ripper, "lineno", ripper_lineno, 0);
|
||
|
rb_define_method(Ripper, "state", ripper_state, 0);
|
||
|
rb_define_method(Ripper, "end_seen?", rb_parser_end_seen_p, 0);
|
||
|
rb_define_method(Ripper, "encoding", rb_parser_encoding, 0);
|
||
|
rb_define_method(Ripper, "yydebug", rb_parser_get_yydebug, 0);
|
||
| ... | ... | |
|
rb_define_singleton_method(Ripper, "dedent_string", parser_dedent_string, 2);
|
||
|
rb_define_private_method(Ripper, "dedent_string", parser_dedent_string, 2);
|
||
|
/* ignore newline, +/- is a sign. */
|
||
|
rb_define_const(Ripper, "EXPR_BEG", INT2NUM(EXPR_BEG));
|
||
|
/* newline significant, +/- is an operator. */
|
||
|
rb_define_const(Ripper, "EXPR_END", INT2NUM(EXPR_END));
|
||
|
/* ditto, and unbound braces. */
|
||
|
rb_define_const(Ripper, "EXPR_ENDARG", INT2NUM(EXPR_ENDARG));
|
||
|
/* ditto, and unbound braces. */
|
||
|
rb_define_const(Ripper, "EXPR_ENDFN", INT2NUM(EXPR_ENDFN));
|
||
|
/* newline significant, +/- is an operator. */
|
||
|
rb_define_const(Ripper, "EXPR_ARG", INT2NUM(EXPR_ARG));
|
||
|
/* newline significant, +/- is an operator. */
|
||
|
rb_define_const(Ripper, "EXPR_CMDARG", INT2NUM(EXPR_CMDARG));
|
||
|
/* newline significant, +/- is an operator. */
|
||
|
rb_define_const(Ripper, "EXPR_MID", INT2NUM(EXPR_MID));
|
||
|
/* ignore newline, no reserved words. */
|
||
|
rb_define_const(Ripper, "EXPR_FNAME", INT2NUM(EXPR_FNAME));
|
||
|
/* right after `.' or `::', no reserved words. */
|
||
|
rb_define_const(Ripper, "EXPR_DOT", INT2NUM(EXPR_DOT));
|
||
|
/* immediate after `class', no here document. */
|
||
|
rb_define_const(Ripper, "EXPR_CLASS", INT2NUM(EXPR_CLASS));
|
||
|
/* flag bit, label is allowed. */
|
||
|
rb_define_const(Ripper, "EXPR_LABEL", INT2NUM(EXPR_LABEL));
|
||
|
/* flag bit, just after a label. */
|
||
|
rb_define_const(Ripper, "EXPR_LABELED", INT2NUM(EXPR_LABELED));
|
||
|
/* symbol literal as FNAME. */
|
||
|
rb_define_const(Ripper, "EXPR_FITEM", INT2NUM(EXPR_FITEM));
|
||
|
/* equals to +EXPR_BEG+ */
|
||
|
rb_define_const(Ripper, "EXPR_VALUE", INT2NUM(EXPR_VALUE));
|
||
|
/* equals to <tt>(EXPR_BEG | EXPR_MID | EXPR_CLASS)</tt> */
|
||
|
rb_define_const(Ripper, "EXPR_BEG_ANY", INT2NUM(EXPR_BEG_ANY));
|
||
|
/* equals to <tt>(EXPR_ARG | EXPR_CMDARG)</tt> */
|
||
|
rb_define_const(Ripper, "EXPR_ARG_ANY", INT2NUM(EXPR_ARG_ANY));
|
||
|
/* equals to <tt>(EXPR_END | EXPR_ENDARG | EXPR_ENDFN)</tt> */
|
||
|
rb_define_const(Ripper, "EXPR_END_ANY", INT2NUM(EXPR_END_ANY));
|
||
|
ripper_init_eventids1_table(Ripper);
|
||
|
ripper_init_eventids2_table(Ripper);
|
||
| test/ripper/test_filter.rb (working copy) | ||
|---|---|---|
|
data[:filename] = filename rescue nil
|
||
|
data[:lineno] = lineno
|
||
|
data[:column] = column
|
||
|
data[:state] = state
|
||
|
data[:token] = token
|
||
|
end
|
||
|
data
|
||
| ... | ... | |
|
assert_equal(last_columns, filter.column)
|
||
|
end
|
||
|
def test_filter_state
|
||
|
data = {}
|
||
|
src = File.read(filename)
|
||
|
filter = Filter.new(src)
|
||
|
assert_equal(nil, filter.state)
|
||
|
filter.parse(data)
|
||
|
assert_not_nil(data[:state])
|
||
|
assert_not_nil(filter.state)
|
||
|
end
|
||
|
def test_filter_token
|
||
|
data = {}
|
||
|
filter = Filter.new("begin; puts 1; end")
|
||
| test/ripper/test_ripper.rb (working copy) | ||
|---|---|---|
|
assert_nil @ripper.column
|
||
|
end
|
||
|
def test_state
|
||
|
assert_nil @ripper.state
|
||
|
end
|
||
|
def test_encoding
|
||
|
assert_equal Encoding::UTF_8, @ripper.encoding
|
||
|
ripper = Ripper.new('# coding: iso-8859-15')
|
||
| test/ripper/test_scanner_events.rb (working copy) | ||
|---|---|---|
|
def test_lex
|
||
|
assert_equal [],
|
||
|
Ripper.lex('')
|
||
|
assert_equal [[[1,0], :on_ident, "a"]],
|
||
|
assert_equal [[[1,0], :on_ident, "a", Ripper::EXPR_CMDARG]],
|
||
|
Ripper.lex('a')
|
||
|
assert_equal [[[1, 0], :on_kw, "nil"]],
|
||
|
assert_equal [[[1, 0], :on_kw, "nil", Ripper::EXPR_END]],
|
||
|
Ripper.lex("nil")
|
||
|
assert_equal [[[1, 0], :on_kw, "def"],
|
||
|
[[1, 3], :on_sp, " "],
|
||
|
[[1, 4], :on_ident, "m"],
|
||
|
[[1, 5], :on_lparen, "("],
|
||
|
[[1, 6], :on_ident, "a"],
|
||
|
[[1, 7], :on_rparen, ")"],
|
||
|
[[1, 8], :on_kw, "end"]],
|
||
|
assert_equal [[[1, 0], :on_kw, "def", Ripper::EXPR_FNAME],
|
||
|
[[1, 3], :on_sp, " ", Ripper::EXPR_FNAME],
|
||
|
[[1, 4], :on_ident, "m", Ripper::EXPR_ENDFN],
|
||
|
[[1, 5], :on_lparen, "(", Ripper::EXPR_BEG | Ripper::EXPR_LABEL],
|
||
|
[[1, 6], :on_ident, "a", Ripper::EXPR_ARG],
|
||
|
[[1, 7], :on_rparen, ")", Ripper::EXPR_ENDFN],
|
||
|
[[1, 8], :on_kw, "end", Ripper::EXPR_END]],
|
||
|
Ripper.lex("def m(a)end")
|
||
|
assert_equal [[[1, 0], :on_int, "1"],
|
||
|
[[1, 1], :on_nl, "\n"],
|
||
|
[[2, 0], :on_int, "2"],
|
||
|
[[2, 1], :on_nl, "\n"],
|
||
|
[[3, 0], :on_int, "3"]],
|
||
|
assert_equal [[[1, 0], :on_int, "1", Ripper::EXPR_END | Ripper::EXPR_ENDARG],
|
||
|
[[1, 1], :on_nl, "\n", Ripper::EXPR_BEG],
|
||
|
[[2, 0], :on_int, "2", Ripper::EXPR_END | Ripper::EXPR_ENDARG],
|
||
|
[[2, 1], :on_nl, "\n", Ripper::EXPR_BEG],
|
||
|
[[3, 0], :on_int, "3", Ripper::EXPR_END | Ripper::EXPR_ENDARG]],
|
||
|
Ripper.lex("1\n2\n3")
|
||
|
assert_equal [[[1, 0], :on_heredoc_beg, "<<""EOS"],
|
||
|
[[1, 5], :on_nl, "\n"],
|
||
|
[[2, 0], :on_tstring_content, "heredoc\n"],
|
||
|
[[3, 0], :on_heredoc_end, "EOS"]],
|
||
|
assert_equal [[[1, 0], :on_heredoc_beg, "<<""EOS", Ripper::EXPR_BEG],
|
||
|
[[1, 5], :on_nl, "\n", Ripper::EXPR_BEG],
|
||
|
[[2, 0], :on_tstring_content, "heredoc\n", Ripper::EXPR_BEG],
|
||
|
[[3, 0], :on_heredoc_end, "EOS", Ripper::EXPR_BEG]],
|
||
|
Ripper.lex("<<""EOS\nheredoc\nEOS")
|
||
|
assert_equal [[[1, 0], :on_heredoc_beg, "<<""EOS"],
|
||
|
[[1, 5], :on_nl, "\n"],
|
||
|
[[2, 0], :on_heredoc_end, "EOS"]],
|
||
|
assert_equal [[[1, 0], :on_heredoc_beg, "<<""EOS", Ripper::EXPR_BEG],
|
||
|
[[1, 5], :on_nl, "\n", Ripper::EXPR_BEG],
|
||
|
[[2, 0], :on_heredoc_end, "EOS", Ripper::EXPR_BEG]],
|
||
|
Ripper.lex("<<""EOS\nEOS"),
|
||
|
"bug#4543"
|
||
|
assert_equal [[[1, 0], :on_regexp_beg, "/"],
|
||
|
[[1, 1], :on_tstring_content, "foo\nbar"],
|
||
|
[[2, 3], :on_regexp_end, "/"]],
|
||
|
assert_equal [[[1, 0], :on_regexp_beg, "/", Ripper::EXPR_BEG],
|
||
|
[[1, 1], :on_tstring_content, "foo\nbar", Ripper::EXPR_BEG],
|
||
|
[[2, 3], :on_regexp_end, "/", Ripper::EXPR_BEG]],
|
||
|
Ripper.lex("/foo\nbar/")
|
||
|
assert_equal [[[1, 0], :on_regexp_beg, "/"],
|
||
|
[[1, 1], :on_tstring_content, "foo\n\u3020"],
|
||
|
[[2, 3], :on_regexp_end, "/"]],
|
||
|
assert_equal [[[1, 0], :on_regexp_beg, "/", Ripper::EXPR_BEG],
|
||
|
[[1, 1], :on_tstring_content, "foo\n\u3020", Ripper::EXPR_BEG],
|
||
|
[[2, 3], :on_regexp_end, "/", Ripper::EXPR_BEG]],
|
||
|
Ripper.lex("/foo\n\u3020/")
|
||
|
assert_equal [[[1, 0], :on_tstring_beg, "'"],
|
||
|
[[1, 1], :on_tstring_content, "foo\n\xe3\x80\xa0"],
|
||
|
[[2, 3], :on_tstring_end, "'"]],
|
||
|
assert_equal [[[1, 0], :on_tstring_beg, "'", Ripper::EXPR_BEG],
|
||
|
[[1, 1], :on_tstring_content, "foo\n\xe3\x80\xa0", Ripper::EXPR_BEG],
|
||
|
[[2, 3], :on_tstring_end, "'", Ripper::EXPR_END | Ripper::EXPR_ENDARG]],
|
||
|
Ripper.lex("'foo\n\xe3\x80\xa0'")
|
||
|
assert_equal [[[1, 0], :on_tstring_beg, "'"],
|
||
|
[[1, 1], :on_tstring_content, "\u3042\n\u3044"],
|
||
|
[[2, 3], :on_tstring_end, "'"]],
|
||
|
assert_equal [[[1, 0], :on_tstring_beg, "'", Ripper::EXPR_BEG],
|
||
|
[[1, 1], :on_tstring_content, "\u3042\n\u3044", Ripper::EXPR_BEG],
|
||
|
[[2, 3], :on_tstring_end, "'", Ripper::EXPR_END | Ripper::EXPR_ENDARG]],
|
||
|
Ripper.lex("'\u3042\n\u3044'")
|
||
|
assert_equal [[[1, 0], :on_rational, "1r"],
|
||
|
[[1, 2], :on_nl, "\n"],
|
||
|
[[2, 0], :on_imaginary, "2i"],
|
||
|
[[2, 2], :on_nl, "\n"],
|
||
|
[[3, 0], :on_imaginary, "3ri"],
|
||
|
[[3, 3], :on_nl, "\n"],
|
||
|
[[4, 0], :on_rational, "4.2r"],
|
||
|
[[4, 4], :on_nl, "\n"],
|
||
|
[[5, 0], :on_imaginary, "5.6ri"],
|
||
|
assert_equal [[[1, 0], :on_rational, "1r", Ripper::EXPR_END | Ripper::EXPR_ENDARG],
|
||
|
[[1, 2], :on_nl, "\n", Ripper::EXPR_BEG],
|
||
|
[[2, 0], :on_imaginary, "2i", Ripper::EXPR_END | Ripper::EXPR_ENDARG],
|
||
|
[[2, 2], :on_nl, "\n", Ripper::EXPR_BEG],
|
||
|
[[3, 0], :on_imaginary, "3ri", Ripper::EXPR_END | Ripper::EXPR_ENDARG],
|
||
|
[[3, 3], :on_nl, "\n", Ripper::EXPR_BEG],
|
||
|
[[4, 0], :on_rational, "4.2r", Ripper::EXPR_END | Ripper::EXPR_ENDARG],
|
||
|
[[4, 4], :on_nl, "\n", Ripper::EXPR_BEG],
|
||
|
[[5, 0], :on_imaginary, "5.6ri", Ripper::EXPR_END | Ripper::EXPR_ENDARG],
|
||
|
],
|
||
|
Ripper.lex("1r\n2i\n3ri\n4.2r\n5.6ri")
|
||
|
assert_equal [[[1, 0], :on_heredoc_beg, "<<~EOS"],
|
||
|
[[1, 6], :on_nl, "\n"],
|
||
|
[[2, 0], :on_ignored_sp, " "],
|
||
|
[[2, 2], :on_tstring_content, "heredoc\n"],
|
||
|
[[3, 0], :on_heredoc_end, "EOS"]
|
||
|
assert_equal [[[1, 0], :on_heredoc_beg, "<<~EOS", Ripper::EXPR_BEG],
|
||
|
[[1, 6], :on_nl, "\n", Ripper::EXPR_BEG],
|
||
|
[[2, 0], :on_ignored_sp, " ", Ripper::EXPR_BEG],
|
||
|
[[2, 2], :on_tstring_content, "heredoc\n", Ripper::EXPR_BEG],
|
||
|
[[3, 0], :on_heredoc_end, "EOS", Ripper::EXPR_BEG]
|
||
|
],
|
||
|
Ripper.lex("<<~EOS\n heredoc\nEOS")
|
||
|
assert_equal [[[1, 0], :on_tstring_beg, "'"],
|
||
|
[[1, 1], :on_tstring_content, "foo"]],
|
||
|
assert_equal [[[1, 0], :on_tstring_beg, "'", Ripper::EXPR_BEG],
|
||
|
[[1, 1], :on_tstring_content, "foo", Ripper::EXPR_BEG]],
|
||
|
Ripper.lex("'foo")
|
||
|
end
|
||