Project

General

Profile

Feature #14683 » irb.patch

aycabta (aycabta .), 04/12/2018 03:06 PM

View differences:

doc/irb/irb.rd.ja
70 70
  --back-trace-limit n
71 71
		    バックトレース表示をバックトレースの頭から n, 後ろ
72 72
		    からnだけ行なう. デフォルトは16
73
  --irb_debug n	    irbのデバッグデバッグレベルをnに設定する(利用しな
74
		    い方が無難でしょう).
75 73
  -v, --version	    irbのバージョンを表示する
76 74

  
77 75
= コンフィギュレーション
......
97 95
  IRB.conf[:IGNORE_EOF] = false
98 96
  IRB.conf[:PROMPT_MODE] = :DEFAULT
99 97
  IRB.conf[:PROMPT] = {...}
100
  IRB.conf[:DEBUG_LEVEL]=0
101 98
  IRB.conf[:VERBOSE]=true
102 99

  
103 100
== プロンプトの設定
......
183 180
    バックトレース表示をバックトレースの頭からn, 後ろからnだけ行なう.
184 181
    デフォルトは16
185 182

  
186
--- conf.debug_level = N
187
    irb用のデバッグレベルの設定
188

  
189 183
--- conf.ignore_eof = true/false
190 184
    ^Dが入力された時の動作を設定する. trueの時は^Dを無視する, falseの
191 185
    時はirbを終了する.
lib/irb.rb
71 71
#     --back-trace-limit n
72 72
#                       Display backtrace top n and tail n. The default
73 73
#                       value is 16.
74
#     --irb_debug n     Set internal debug level to n (not for popular use)
75 74
#     -v, --version     Print the version of irb
76 75
#
77 76
# == Configuration
......
99 98
#     IRB.conf[:IGNORE_EOF] = false
100 99
#     IRB.conf[:PROMPT_MODE] = :DEFAULT
101 100
#     IRB.conf[:PROMPT] = {...}
102
#     IRB.conf[:DEBUG_LEVEL]=0
103 101
#
104 102
# === Auto indentation
105 103
#
......
410 408
      @context = Context.new(self, workspace, input_method, output_method)
411 409
      @context.main.extend ExtendCommandBundle
412 410
      @signal_status = :IN_IRB
413

  
414 411
      @scanner = RubyLex.new
415
      @scanner.exception_on_syntax_error = false
416 412
    end
417 413

  
418 414
    def run(conf = IRB.conf)
lib/irb/context.rb
100 100
      if @echo.nil?
101 101
        @echo = true
102 102
      end
103
      self.debug_level = IRB.conf[:DEBUG_LEVEL]
104 103
    end
105 104

  
106 105
    # The top-level workspace, see WorkSpace#main
......
210 209
    #
211 210
    # A copy of the default <code>IRB.conf[:VERBOSE]</code>
212 211
    attr_accessor :verbose
213
    # The debug level of irb
214
    #
215
    # See #debug_level= for more information.
216
    attr_reader :debug_level
217 212

  
218 213
    # The limit of backtrace lines displayed as top +n+ and tail +n+.
219 214
    #
......
360 355
      print "Do nothing."
361 356
    end
362 357

  
363
    # Sets the debug level of irb
364
    #
365
    # Can also be set using the +--irb_debug+ command line option.
366
    #
367
    # See IRB@Command+line+options for more command line options.
368
    def debug_level=(value)
369
      @debug_level = value
370
      RubyLex.debug_level = value
371
    end
372

  
373
    # Whether or not debug mode is enabled, see #debug_level=.
374
    def debug?
375
      @debug_level > 0
376
    end
377

  
378 358
    def evaluate(line, line_no) # :nodoc:
379 359
      @line_no = line_no
380 360
      set_last_value(@workspace.evaluate(self, line, irb_path, line_no))
lib/irb/init.rb
112 112
    @CONF[:LC_MESSAGES] = Locale.new
113 113

  
114 114
    @CONF[:AT_EXIT] = []
115

  
116
    @CONF[:DEBUG_LEVEL] = 0
117 115
  end
118 116

  
119 117
  def IRB.init_error
......
191 189
        @CONF[:CONTEXT_MODE] = ($1 || argv.shift).to_i
192 190
      when "--single-irb"
193 191
        @CONF[:SINGLE_IRB] = true
194
      when /^--irb_debug(?:=(.+))?/
195
        @CONF[:DEBUG_LEVEL] = ($1 || argv.shift).to_i
196 192
      when "-v", "--version"
197 193
        print IRB.version, "\n"
198 194
        exit 0
lib/irb/lc/help-message
39 39
  --back-trace-limit n
40 40
		    Display backtrace top n and tail n. The default
41 41
		    value is 16.
42
  --irb_debug n	    Set internal debug level to n (not for popular use)
43 42
  --verbose         Show details
44 43
  --noverbose       Don't show details
45 44
  -v, --version	    Print the version of irb
lib/irb/lc/ja/help-message
41 41
		    バックトレース表示をバックトレースの頭から n, 後ろ
42 42
		    からnだけ行なう. デフォルトは16
43 43

  
44
  --irb_debug n	    irbのデバッグレベルをnに設定する(非推奨).
45

  
46 44
  --verbose	    詳細なメッセージを出力する.
47 45
  --noverbose	    詳細なメッセージを出力しない(デフォルト).
48 46
  -v, --version	    irbのバージョンを表示する.
lib/irb/ruby-lex.rb
11 11
#
12 12

  
13 13
require "e2mmap"
14
require "irb/slex"
15
require "irb/ruby-token"
14
require "ripper"
16 15

  
17 16
# :stopdoc:
18 17
class RubyLex
19 18

  
20 19
  extend Exception2MessageMapper
21
  def_exception(:AlreadyDefinedToken, "Already defined token(%s)")
22
  def_exception(:TkReading2TokenNoKey, "key nothing(key='%s')")
23
  def_exception(:TkSymbol2TokenNoKey, "key nothing(key='%s')")
24
  def_exception(:TkReading2TokenDuplicateError,
25
                "key duplicate(token_n='%s', key='%s')")
26
  def_exception(:SyntaxError, "%s")
27

  
28 20
  def_exception(:TerminateLineInput, "Terminate Line Input")
29 21

  
30
  include RubyToken
31

  
32
  class << self
33
    attr_accessor :debug_level
34
    def debug?
35
      @debug_level > 0
36
    end
37
  end
38
  @debug_level = 0
39

  
40 22
  def initialize
41
    lex_init
42
    set_input(STDIN)
43

  
44
    @seek = 0
45 23
    @exp_line_no = @line_no = 1
46
    @base_char_no = 0
47
    @char_no = 0
48
    @rests = []
49
    @readed = []
50
    @here_readed = []
51

  
52 24
    @indent = 0
53
    @indent_stack = []
54
    @lex_state = EXPR_BEG
55
    @space_seen = false
56
    @here_header = false
57
    @post_symbeg = false
58

  
59 25
    @continue = false
60 26
    @line = ""
61

  
62
    @skip_space = false
63
    @readed_auto_clean_up = false
64
    @exception_on_syntax_error = true
65

  
66 27
    @prompt = nil
67 28
  end
68 29

  
69
  attr_accessor :skip_space
70
  attr_accessor :readed_auto_clean_up
71
  attr_accessor :exception_on_syntax_error
72

  
73
  attr_reader :seek
74
  attr_reader :char_no
75
  attr_reader :line_no
76
  attr_reader :indent
77

  
78 30
  # io functions
79 31
  def set_input(io, p = nil, &block)
80 32
    @io = io
......
87 39
    end
88 40
  end
89 41

  
90
  def get_readed
91
    if idx = @readed.rindex("\n")
92
      @base_char_no = @readed.size - (idx + 1)
93
    else
94
      @base_char_no += @readed.size
95
    end
96

  
97
    readed = @readed.join("")
98
    @readed = []
99
    readed
100
  end
101

  
102
  def getc
103
    while @rests.empty?
104
      @rests.push nil unless buf_input
105
    end
106
    c = @rests.shift
107
    if @here_header
108
      @here_readed.push c
109
    else
110
      @readed.push c
111
    end
112
    @seek += 1
113
    if c == "\n"
114
      @line_no += 1
115
      @char_no = 0
116
    else
117
      @char_no += 1
118
    end
119
    c
120
  end
121

  
122
  def gets
123
    l = ""
124
    while c = getc
125
      l.concat(c)
126
      break if c == "\n"
127
    end
128
    return nil if l == "" and c.nil?
129
    l
130
  end
131

  
132
  def eof?
133
    @io.eof?
134
  end
135

  
136
  def getc_of_rests
137
    if @rests.empty?
138
      nil
139
    else
140
      getc
141
    end
142
  end
143

  
144
  def ungetc(c = nil)
145
    if @here_readed.empty?
146
      c2 = @readed.pop
147
    else
148
      c2 = @here_readed.pop
149
    end
150
    c = c2 unless c
151
    @rests.unshift c #c =
152
    @seek -= 1
153
    if c == "\n"
154
      @line_no -= 1
155
      if idx = @readed.rindex("\n")
156
        @char_no = idx + 1
157
      else
158
        @char_no = @base_char_no + @readed.size
159
      end
160
    else
161
      @char_no -= 1
162
    end
163
  end
164

  
165
  def peek_equal?(str)
166
    chrs = str.split(//)
167
    until @rests.size >= chrs.size
168
      return false unless buf_input
169
    end
170
    @rests[0, chrs.size] == chrs
171
  end
172

  
173
  def peek_match?(regexp)
174
    while @rests.empty?
175
      return false unless buf_input
176
    end
177
    regexp =~ @rests.join("")
178
  end
179

  
180
  def peek(i = 0)
181
    while @rests.size <= i
182
      return nil unless buf_input
183
    end
184
    @rests[i]
185
  end
186

  
187
  def buf_input
188
    prompt
189
    line = @input.call
190
    return nil unless line
191
    @rests.concat line.chars.to_a
192
    true
193
  end
194
  private :buf_input
195

  
196 42
  def set_prompt(p = nil, &block)
197 43
    p = block if block_given?
198 44
    if p.respond_to?(:call)
......
210 56

  
211 57
  def initialize_input
212 58
    @ltype = nil
213
    @quoted = nil
214 59
    @indent = 0
215
    @indent_stack = []
216
    @lex_state = EXPR_BEG
217
    @space_seen = false
218
    @here_header = false
219

  
220 60
    @continue = false
221
    @post_symbeg = false
222

  
223
    prompt
224

  
225 61
    @line = ""
226 62
    @exp_line_no = @line_no
63
    @code_block_open = false
227 64
  end
228 65

  
229 66
  def each_top_level_statement
......
231 68
    catch(:TERM_INPUT) do
232 69
      loop do
233 70
        begin
234
          @continue = false
235 71
          prompt
236 72
          unless l = lex
237 73
            throw :TERM_INPUT if @line == ''
238 74
          else
75
            @line_no += 1
76
            next if l == "\n"
239 77
            @line.concat l
240
            if @ltype or @continue or @indent > 0
78
            if @code_block_open or @ltype or @continue or @indent > 0
241 79
              next
242 80
            end
243 81
          end
......
250 88
          @exp_line_no = @line_no
251 89

  
252 90
          @indent = 0
253
          @indent_stack = []
254
          prompt
255 91
        rescue TerminateLineInput
256 92
          initialize_input
257 93
          prompt
258
          get_readed
259 94
        end
260 95
      end
261 96
    end
262 97
  end
263 98

  
264 99
  def lex
265
    continue = @continue
266
    while tk = token
267
      case tk
268
      when TkNL, TkEND_OF_SCRIPT
269
        @continue = continue unless continue.nil?
270
        break unless @continue
271
      when TkSPACE, TkCOMMENT
272
      when TkSEMICOLON, TkBEGIN, TkELSE
273
        @continue = continue = false
274
      else
275
        continue = nil
276
      end
277
    end
278
    line = get_readed
279
    if line == "" and tk.kind_of?(TkEND_OF_SCRIPT) || tk.nil?
100
    line = @input.call
101
    code = @line + (line.nil? ? '' : line)
102
    @tokens = Ripper.lex(code)
103
    @continue = process_continue
104
    @code_block_open = check_code_block(code)
105
    @indent = process_nesting_level
106
    @ltype = process_literal_type
107
    line
108
  end
109

  
110
  def process_continue
111
    continued_bits = Ripper::EXPR_BEG | Ripper::EXPR_FNAME | Ripper::EXPR_DOT
112
    # last token is always newline
113
    if @tokens.size >= 2 and @tokens[-2][1] == :on_regexp_end
114
      # end of regexp literal
115
      return false
116
    elsif @tokens.size >= 2 and @tokens[-2][1] == :on_semicolon
117
      return false
118
    elsif @tokens.size >= 2 and @tokens[-2][1] == :on_kw and (@tokens[-2][2] == 'begin' or @tokens[-2][2] == 'else')
119
      return false
120
    elsif !@tokens.empty? and @tokens.last[2] == "\\\n"
121
      return true
122
    elsif @tokens.size >= 2 and @tokens[-2][3].anybits?(continued_bits)
123
      # end of literal except for regexp
124
      return true
125
    end
126
    false
127
  end
128

  
129
  def check_code_block(code)
130
    return true if @tokens.empty?
131
    if @tokens.last[1] == :on_heredoc_beg
132
      return true
133
    end
134

  
135
    begin # check if parser error are available
136
      RubyVM::InstructionSequence.compile(code)
137
    rescue SyntaxError => e
138
      case e.message
139
      when /unterminated (?:string|regexp) meets end of file/
140
        # "unterminated regexp meets end of file"
141
        #
142
        #   example:
143
        #     /
144
        #
145
        # "unterminated string meets end of file"
146
        #
147
        #   example:
148
        #     '
149
        return true
150
      when /syntax error, unexpected end-of-input/
151
        # "syntax error, unexpected end-of-input, expecting keyword_end"
152
        #
153
        #   example:
154
        #     if ture
155
        #       hoge
156
        #       if false
157
        #         fuga
158
        #       end
159
        return true
160
      when /syntax error, unexpected keyword_end/
161
        # "syntax error, unexpected keyword_end"
162
        #
163
        #   example:
164
        #     if (
165
        #     end
166
        #
167
        #   example:
168
        #     end
169
        return false
170
      when /unexpected tREGEXP_BEG/
171
        # "syntax error, unexpected tREGEXP_BEG, expecting keyword_do or '{' or '('"
172
        #
173
        #   example:
174
        #     method / f /
175
        return false
176
      end
177
    end
178

  
179
    last_lex_state = @tokens.last[3]
180
    if last_lex_state.allbits?(Ripper::EXPR_BEG)
181
      return false
182
    elsif last_lex_state.allbits?(Ripper::EXPR_DOT)
183
      return true
184
    elsif last_lex_state.allbits?(Ripper::EXPR_CLASS)
185
      return true
186
    elsif last_lex_state.allbits?(Ripper::EXPR_FNAME)
187
      return true
188
    elsif last_lex_state.allbits?(Ripper::EXPR_VALUE)
189
      return true
190
    elsif last_lex_state.allbits?(Ripper::EXPR_ARG)
191
      return false
192
    end
193

  
194
    false
195
  end
196

  
197
  def process_nesting_level
198
    @tokens.inject(0) { |indent, t|
199
      case t[1]
200
      when :on_lbracket, :on_lbrace, :on_lparen
201
        indent += 1
202
      when :on_rbracket, :on_rbrace, :on_rparen
203
        indent -= 1
204
      when :on_kw
205
        case t[2]
206
        when 'def', 'do', 'case', 'for', 'begin', 'class', 'module'
207
          indent += 1
208
        when 'if', 'unless', 'while', 'until', 'rescue'
209
          # postfix if/unless/while/until/rescue must be Ripper::EXPR_LABEL
210
          indent += 1 unless t[3].allbits?(Ripper::EXPR_LABEL)
211
        when 'end'
212
          indent -= 1
213
        end
214
      end
215
      # percent literals are not indented
216
      indent
217
    }
218
  end
219

  
220
  def check_string_literal
221
    i = 0
222
    start_token = []
223
    end_type = []
224
    while i < @tokens.size
225
      t = @tokens[i]
226
      case t[1]
227
      when :on_tstring_beg
228
        start_token << t
229
        end_type << :on_tstring_end
230
      when :on_regexp_beg
231
        start_token << t
232
        end_type << :on_regexp_end
233
      when :on_symbeg
234
        if (i + 1) < @tokens.size and @tokens[i + 1][1] != :on_ident
235
          start_token << t
236
          end_type << :on_tstring_end
237
        end
238
      when :on_backtick
239
        start_token << t
240
        end_type << :on_tstring_end
241
      when :on_qwords_beg, :on_words_beg, :on_qsymbols_beg, :on_symbols_beg
242
        start_token << t
243
        end_type << :on_tstring_end
244
      when :on_heredoc_beg
245
        start_token << t
246
        end_type << :on_heredoc_end
247
      when end_type.last
248
        start_token.pop
249
        end_type.pop
250
      end
251
      i += 1
252
    end
253
    start_token.last.nil? ? '' : start_token.last
254
  end
255

  
256
  def process_literal_type
257
    start_token = check_string_literal
258
    case start_token[1]
259
    when :on_tstring_beg
260
      case start_token[2]
261
      when ?"      then ?"
262
      when /^%.$/  then ?"
263
      when /^%Q.$/ then ?"
264
      when ?'      then ?'
265
      when /^%q.$/ then ?'
266
      end
267
    when :on_regexp_beg   then ?/
268
    when :on_symbeg       then ?:
269
    when :on_backtick     then ?`
270
    when :on_qwords_beg   then ?]
271
    when :on_words_beg    then ?]
272
    when :on_qsymbols_beg then ?]
273
    when :on_symbols_beg  then ?]
274
    when :on_heredoc_beg
275
      start_token[2] =~ /<<[-~]?(['"`])[_a-zA-Z0-9]+\1/
276
      case $1
277
      when ?" then ?"
278
      when ?' then ?'
279
      when ?` then ?`
280
      else         ?"
281
      end
282
    else
280 283
      nil
281
    else
282
      line
283
    end
284
  end
285

  
286
  def token
287
    @prev_seek = @seek
288
    @prev_line_no = @line_no
289
    @prev_char_no = @char_no
290
    begin
291
      begin
292
        tk = @OP.match(self)
293
        @space_seen = tk.kind_of?(TkSPACE)
294
        @lex_state = EXPR_END if @post_symbeg && tk.kind_of?(TkOp)
295
        @post_symbeg = tk.kind_of?(TkSYMBEG)
296
      rescue SyntaxError
297
        raise if @exception_on_syntax_error
298
        tk = TkError.new(@seek, @line_no, @char_no)
299
      end
300
    end while @skip_space and tk.kind_of?(TkSPACE)
301
    if @readed_auto_clean_up
302
      get_readed
303
    end
304
    tk
305
  end
306

  
307
  ENINDENT_CLAUSE = [
308
    "case", "class", "def", "do", "for", "if",
309
    "module", "unless", "until", "while", "begin"
310
  ]
311
  DEINDENT_CLAUSE = ["end"
312
  ]
313

  
314
  PERCENT_LTYPE = {
315
    "q" => "\'",
316
    "Q" => "\"",
317
    "x" => "\`",
318
    "r" => "/",
319
    "w" => "]",
320
    "W" => "]",
321
    "i" => "]",
322
    "I" => "]",
323
    "s" => ":"
324
  }
325

  
326
  PERCENT_PAREN = {
327
    "{" => "}",
328
    "[" => "]",
329
    "<" => ">",
330
    "(" => ")"
331
  }
332

  
333
  Ltype2Token = {
334
    "\'" => TkSTRING,
335
    "\"" => TkSTRING,
336
    "\`" => TkXSTRING,
337
    "/" => TkREGEXP,
338
    "]" => TkDSTRING,
339
    ":" => TkSYMBOL
340
  }
341
  DLtype2Token = {
342
    "\"" => TkDSTRING,
343
    "\`" => TkDXSTRING,
344
    "/" => TkDREGEXP,
345
  }
346

  
347
  def lex_init()
348
    @OP = IRB::SLex.new
349
    @OP.def_rules("\0", "\004", "\032") do |op, io|
350
      Token(TkEND_OF_SCRIPT)
351
    end
352

  
353
    @OP.def_rules(" ", "\t", "\f", "\r", "\13") do |op, io|
354
      @space_seen = true
355
      while getc =~ /[ \t\f\r\13]/; end
356
      ungetc
357
      Token(TkSPACE)
358
    end
359

  
360
    @OP.def_rule("#") do |op, io|
361
      identify_comment
362
    end
363

  
364
    @OP.def_rule("=begin",
365
                 proc{|op, io| @prev_char_no == 0 && peek(0) =~ /\s/}) do
366
      |op, io|
367
      @ltype = "="
368
      until getc == "\n"; end
369
      until peek_equal?("=end") && peek(4) =~ /\s/
370
        until getc == "\n"; end
371
      end
372
      gets
373
      @ltype = nil
374
      Token(TkRD_COMMENT)
375
    end
376

  
377
    @OP.def_rule("\n") do |op, io|
378
      print "\\n\n" if RubyLex.debug?
379
      case @lex_state
380
      when EXPR_BEG, EXPR_FNAME, EXPR_DOT
381
        @continue = true
382
      else
383
        @continue = false
384
        @lex_state = EXPR_BEG
385
        until (@indent_stack.empty? ||
386
            [TkLPAREN, TkLBRACK, TkLBRACE,
387
             TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last))
388
          @indent_stack.pop
389
        end
390
      end
391
      @here_header = false
392
      @here_readed = []
393
      Token(TkNL)
394
    end
395

  
396
    @OP.def_rules("*", "**",
397
                  "=", "==", "===",
398
                  "=~", "<=>",
399
                  "<", "<=",
400
                  ">", ">=", ">>",
401
                  "!", "!=", "!~") do
402
      |op, io|
403
      case @lex_state
404
      when EXPR_FNAME, EXPR_DOT
405
        @lex_state = EXPR_ARG
406
      else
407
        @lex_state = EXPR_BEG
408
      end
409
      Token(op)
410
    end
411

  
412
    @OP.def_rules("<<") do
413
      |op, io|
414
      tk = nil
415
      if @lex_state != EXPR_END && @lex_state != EXPR_CLASS &&
416
          (@lex_state != EXPR_ARG || @space_seen)
417
        c = peek(0)
418
        if /[-~"'`\w]/ =~ c
419
          tk = identify_here_document
420
        end
421
      end
422
      unless tk
423
        tk = Token(op)
424
        case @lex_state
425
        when EXPR_FNAME, EXPR_DOT
426
          @lex_state = EXPR_ARG
427
        else
428
          @lex_state = EXPR_BEG
429
        end
430
      end
431
      tk
432
    end
433

  
434
    @OP.def_rules("'", '"') do
435
      |op, io|
436
      identify_string(op)
437
    end
438

  
439
    @OP.def_rules("`") do
440
      |op, io|
441
      if @lex_state == EXPR_FNAME
442
        @lex_state = EXPR_END
443
        Token(op)
444
      else
445
        identify_string(op)
446
      end
447
    end
448

  
449
    @OP.def_rules('?') do
450
      |op, io|
451
      if @lex_state == EXPR_END
452
        @lex_state = EXPR_BEG
453
        Token(TkQUESTION)
454
      else
455
        ch = getc
456
        if @lex_state == EXPR_ARG && ch =~ /\s/
457
          ungetc
458
          @lex_state = EXPR_BEG;
459
          Token(TkQUESTION)
460
        else
461
          if (ch == '\\')
462
            read_escape
463
          end
464
          @lex_state = EXPR_END
465
          Token(TkINTEGER)
466
        end
467
      end
468
    end
469

  
470
    @OP.def_rules("&", "&&", "|", "||") do
471
      |op, io|
472
      @lex_state = EXPR_BEG
473
      Token(op)
474
    end
475

  
476
    @OP.def_rules("+=", "-=", "*=", "**=",
477
                  "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do
478
      |op, io|
479
      @lex_state = EXPR_BEG
480
      op =~ /^(.*)=$/
481
      Token(TkOPASGN, $1)
482
    end
483

  
484
    @OP.def_rule("+@", proc{|op, io| @lex_state == EXPR_FNAME}) do
485
      |op, io|
486
      @lex_state = EXPR_ARG
487
      Token(op)
488
    end
489

  
490
    @OP.def_rule("-@", proc{|op, io| @lex_state == EXPR_FNAME}) do
491
      |op, io|
492
      @lex_state = EXPR_ARG
493
      Token(op)
494
    end
495

  
496
    @OP.def_rules("+", "-") do
497
      |op, io|
498
      catch(:RET) do
499
        if @lex_state == EXPR_ARG
500
          if @space_seen and peek(0) =~ /[0-9]/
501
            throw :RET, identify_number
502
          else
503
            @lex_state = EXPR_BEG
504
          end
505
        elsif @lex_state != EXPR_END and peek(0) =~ /[0-9]/
506
          throw :RET, identify_number
507
        else
508
          @lex_state = EXPR_BEG
509
        end
510
        Token(op)
511
      end
512
    end
513

  
514
    @OP.def_rule(".") do
515
      |op, io|
516
      @lex_state = EXPR_BEG
517
      if peek(0) =~ /[0-9]/
518
        ungetc
519
        identify_number
520
      else
521
        # for "obj.if" etc.
522
        @lex_state = EXPR_DOT
523
        Token(TkDOT)
524
      end
525
    end
526

  
527
    @OP.def_rules("..", "...") do
528
      |op, io|
529
      @lex_state = EXPR_BEG
530
      Token(op)
531
    end
532

  
533
    lex_int2
534
  end
535

  
536
  def lex_int2
537
    @OP.def_rules("]", "}", ")") do
538
      |op, io|
539
      @lex_state = EXPR_END
540
      @indent -= 1
541
      @indent_stack.pop
542
      Token(op)
543
    end
544

  
545
    @OP.def_rule(":") do
546
      |op, io|
547
      if @lex_state == EXPR_END || peek(0) =~ /\s/
548
        @lex_state = EXPR_BEG
549
        Token(TkCOLON)
550
      else
551
        @lex_state = EXPR_FNAME
552
        Token(TkSYMBEG)
553
      end
554
    end
555

  
556
    @OP.def_rule("::") do
557
       |op, io|
558
      if @lex_state == EXPR_BEG or @lex_state == EXPR_ARG && @space_seen
559
        @lex_state = EXPR_BEG
560
        Token(TkCOLON3)
561
      else
562
        @lex_state = EXPR_DOT
563
        Token(TkCOLON2)
564
      end
565
    end
566

  
567
    @OP.def_rule("/") do
568
      |op, io|
569
      if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
570
        identify_string(op)
571
      elsif peek(0) == '='
572
        getc
573
        @lex_state = EXPR_BEG
574
        Token(TkOPASGN, "/") #/)
575
      elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
576
        identify_string(op)
577
      else
578
        @lex_state = EXPR_BEG
579
        Token("/") #/)
580
      end
581
    end
582

  
583
    @OP.def_rules("^") do
584
      |op, io|
585
      @lex_state = EXPR_BEG
586
      Token("^")
587
    end
588

  
589
    @OP.def_rules(",") do
590
      |op, io|
591
      @lex_state = EXPR_BEG
592
      Token(op)
593
    end
594

  
595
    @OP.def_rules(";") do
596
      |op, io|
597
      @lex_state = EXPR_BEG
598
      until (@indent_stack.empty? ||
599
          [TkLPAREN, TkLBRACK, TkLBRACE,
600
           TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last))
601
        @indent_stack.pop
602
      end
603
      Token(op)
604
    end
605

  
606
    @OP.def_rule("~") do
607
      |op, io|
608
      @lex_state = EXPR_BEG
609
      Token("~")
610
    end
611

  
612
    @OP.def_rule("~@", proc{|op, io| @lex_state == EXPR_FNAME}) do
613
      |op, io|
614
      @lex_state = EXPR_BEG
615
      Token("~")
616
    end
617

  
618
    @OP.def_rule("(") do
619
      |op, io|
620
      @indent += 1
621
      if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
622
        @lex_state = EXPR_BEG
623
        tk_c = TkfLPAREN
624
      else
625
        @lex_state = EXPR_BEG
626
        tk_c = TkLPAREN
627
      end
628
      @indent_stack.push tk_c
629
      Token(tk_c)
630
    end
631

  
632
    @OP.def_rule("[]", proc{|op, io| @lex_state == EXPR_FNAME}) do
633
      |op, io|
634
      @lex_state = EXPR_ARG
635
      Token("[]")
636
    end
637

  
638
    @OP.def_rule("[]=", proc{|op, io| @lex_state == EXPR_FNAME}) do
639
      |op, io|
640
      @lex_state = EXPR_ARG
641
      Token("[]=")
642
    end
643

  
644
    @OP.def_rule("[") do
645
      |op, io|
646
      @indent += 1
647
      if @lex_state == EXPR_FNAME
648
        tk_c = TkfLBRACK
649
      else
650
        if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
651
          tk_c = TkLBRACK
652
        elsif @lex_state == EXPR_ARG && @space_seen
653
          tk_c = TkLBRACK
654
        else
655
          tk_c = TkfLBRACK
656
        end
657
        @lex_state = EXPR_BEG
658
      end
659
      @indent_stack.push tk_c
660
      Token(tk_c)
661
    end
662

  
663
    @OP.def_rule("{") do
664
      |op, io|
665
      @indent += 1
666
      if @lex_state != EXPR_END && @lex_state != EXPR_ARG
667
        tk_c = TkLBRACE
668
      else
669
        tk_c = TkfLBRACE
670
      end
671
      @lex_state = EXPR_BEG
672
      @indent_stack.push tk_c
673
      Token(tk_c)
674
    end
675

  
676
    @OP.def_rule('\\') do
677
      |op, io|
678
      if getc == "\n"
679
        @space_seen = true
680
        @continue = true
681
        Token(TkSPACE)
682
      else
683
        read_escape
684
        Token("\\")
685
      end
686
    end
687

  
688
    @OP.def_rule('%') do
689
      |op, io|
690
      if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
691
        identify_quotation
692
      elsif peek(0) == '='
693
        getc
694
        Token(TkOPASGN, :%)
695
      elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
696
        identify_quotation
697
      else
698
        @lex_state = EXPR_BEG
699
        Token("%") #))
700
      end
701
    end
702

  
703
    @OP.def_rule('$') do
704
      |op, io|
705
      identify_gvar
706
    end
707

  
708
    @OP.def_rule('@') do
709
      |op, io|
710
      if peek(0) =~ /[\w@]/
711
        ungetc
712
        identify_identifier
713
      else
714
        Token("@")
715
      end
716
    end
717

  
718
    @OP.def_rule("") do
719
      |op, io|
720
      printf "MATCH: start %s: %s\n", op, io.inspect if RubyLex.debug?
721
      if peek(0) =~ /[0-9]/
722
        t = identify_number
723
      elsif peek(0) =~ /[^\x00-\/:-@\[-^`{-\x7F]/
724
        t = identify_identifier
725
      end
726
      printf "MATCH: end %s: %s\n", op, io.inspect if RubyLex.debug?
727
      t
728
    end
729

  
730
    p @OP if RubyLex.debug?
731
  end
732

  
733
  def identify_gvar
734
    @lex_state = EXPR_END
735

  
736
    case ch = getc
737
    when /[~_*$?!@\/\\;,=:<>".]/   #"
738
      Token(TkGVAR, "$" + ch)
739
    when "-"
740
      Token(TkGVAR, "$-" + getc)
741
    when "&", "`", "'", "+"
742
      Token(TkBACK_REF, "$"+ch)
743
    when /[1-9]/
744
      while getc =~ /[0-9]/; end
745
      ungetc
746
      Token(TkNTH_REF)
747
    when /\w/
748
      ungetc
749
      ungetc
750
      identify_identifier
751
    else
752
      ungetc
753
      Token("$")
754
    end
755
  end
756

  
757
  def identify_identifier
758
    token = ""
759
    if peek(0) =~ /[$@]/
760
      token.concat(c = getc)
761
      if c == "@" and peek(0) == "@"
762
        token.concat getc
763
      end
764
    end
765

  
766
    while (ch = getc) =~ /[^\x00-\/:-@\[-^`{-\x7F]/
767
      print ":", ch, ":" if RubyLex.debug?
768
      token.concat ch
769
    end
770
    ungetc
771

  
772
    if (ch == "!" || ch == "?") && token[0,1] =~ /\w/ && peek(0) != "="
773
      token.concat getc
774
    end
775

  
776
    # almost fix token
777

  
778
    case token
779
    when /^\$/
780
      return Token(TkGVAR, token)
781
    when /^\@\@/
782
      @lex_state = EXPR_END
783
      # p Token(TkCVAR, token)
784
      return Token(TkCVAR, token)
785
    when /^\@/
786
      @lex_state = EXPR_END
787
      return Token(TkIVAR, token)
788
    end
789

  
790
    if @lex_state != EXPR_DOT
791
      print token, "\n" if RubyLex.debug?
792

  
793
      token_c, *trans = TkReading2Token[token]
794
      if token_c
795
        # reserved word?
796

  
797
        if (@lex_state != EXPR_BEG &&
798
            @lex_state != EXPR_FNAME &&
799
            trans[1])
800
          # modifiers
801
          token_c = TkSymbol2Token[trans[1]]
802
          @lex_state = trans[0]
803
        else
804
          if @lex_state != EXPR_FNAME and peek(0) != ':'
805
            if ENINDENT_CLAUSE.include?(token)
806
              # check for ``class = val'' etc.
807
              valid = true
808
              case token
809
              when "class"
810
                valid = false unless peek_match?(/^\s*(<<|\w|::)/)
811
              when "def"
812
                valid = false if peek_match?(/^\s*(([+\-\/*&\|^]|<<|>>|\|\||\&\&)=|\&\&|\|\|)/)
813
              when "do"
814
                valid = false if peek_match?(/^\s*([+\-\/*]?=|\*|<|>|\&)/)
815
              when *ENINDENT_CLAUSE
816
                valid = false if peek_match?(/^\s*([+\-\/*]?=|\*|<|>|\&|\|)/)
817
              else
818
                # no nothing
819
              end
820
              if valid
821
                if token == "do"
822
                  if ![TkFOR, TkWHILE, TkUNTIL].include?(@indent_stack.last)
823
                    @indent += 1
824
                    @indent_stack.push token_c
825
                  end
826
                else
827
                  @indent += 1
828
                  @indent_stack.push token_c
829
                end
830
              end
831

  
832
            elsif DEINDENT_CLAUSE.include?(token)
833
              @indent -= 1
834
              @indent_stack.pop
835
            end
836
            @lex_state = trans[0]
837
          else
838
            @lex_state = EXPR_END
839
          end
840
        end
841
        return Token(token_c, token)
842
      end
843
    end
844

  
845
    if @lex_state == EXPR_FNAME
846
      @lex_state = EXPR_END
847
      if peek(0) == '='
848
        token.concat getc
849
      end
850
    elsif @lex_state == EXPR_BEG || @lex_state == EXPR_DOT
851
      @lex_state = EXPR_ARG
852
    else
853
      @lex_state = EXPR_END
854
    end
855

  
856
    if token[0, 1] =~ /[A-Z]/
857
      return Token(TkCONSTANT, token)
858
    elsif token[token.size - 1, 1] =~ /[!?]/
859
      return Token(TkFID, token)
860
    else
861
      return Token(TkIDENTIFIER, token)
862
    end
863
  end
864

  
865
  def identify_here_document
866
    ch = getc
867
    if ch == "-" || ch == "~"
868
      ch = getc
869
      indent = true
870
    end
871
    if /['"`]/ =~ ch
872
      lt = ch
873
      quoted = ""
874
      while (c = getc) && c != lt
875
        quoted.concat c
876
      end
877
    else
878
      lt = '"'
879
      quoted = ch.dup
880
      while (c = getc) && c =~ /\w/
881
        quoted.concat c
882
      end
883
      ungetc
884
    end
885

  
886
    ltback, @ltype = @ltype, lt
887
    reserve = []
888
    while ch = getc
889
      reserve.push ch
890
      if ch == "\\"
891
        reserve.push ch = getc
892
      elsif ch == "\n"
893
        break
894
      end
895
    end
896

  
897
    @here_header = false
898

  
899
    line = ""
900
    while ch = getc
901
      if ch == "\n"
902
        if line == quoted
903
          break
904
        end
905
        line = ""
906
      else
907
        line.concat ch unless indent && line == "" && /\s/ =~ ch
908
        if @ltype != "'" && ch == "#" && peek(0) == "{"
909
          identify_string_dvar
910
        end
911
      end
912
    end
913

  
914
    @here_header = true
915
    @here_readed.concat reserve
916
    while ch = reserve.pop
917
      ungetc ch
918
    end
919

  
920
    @ltype = ltback
921
    @lex_state = EXPR_END
922
    Token(Ltype2Token[lt])
923
  end
924

  
925
  def identify_quotation
926
    ch = getc
927
    if lt = PERCENT_LTYPE[ch]
928
      ch = getc
929
    elsif ch =~ /\W/
930
      lt = "\""
931
    else
932
      RubyLex.fail SyntaxError, "unknown type of %string"
933
    end
934
    @quoted = ch unless @quoted = PERCENT_PAREN[ch]
935
    identify_string(lt, @quoted)
936
  end
937

  
938
  def identify_number
939
    @lex_state = EXPR_END
940

  
941
    if peek(0) == "0" && peek(1) !~ /[.eE]/
942
      getc
943
      case peek(0)
944
      when /[xX]/
945
        ch = getc
946
        match = /[0-9a-fA-F_]/
947
      when /[bB]/
948
        ch = getc
949
        match = /[01_]/
950
      when /[oO]/
951
        ch = getc
952
        match = /[0-7_]/
953
      when /[dD]/
954
        ch = getc
955
        match = /[0-9_]/
956
      when /[0-7]/
957
        match = /[0-7_]/
958
      when /[89]/
959
        RubyLex.fail SyntaxError, "Invalid octal digit"
960
      else
961
        return Token(TkINTEGER)
962
      end
963

  
964
      len0 = true
965
      non_digit = false
966
      while ch = getc
967
        if match =~ ch
968
          if ch == "_"
969
            if non_digit
970
              RubyLex.fail SyntaxError, "trailing `#{ch}' in number"
971
            else
972
              non_digit = ch
973
            end
974
          else
975
            non_digit = false
976
            len0 = false
977
          end
978
        else
979
          ungetc
980
          if len0
981
            RubyLex.fail SyntaxError, "numeric literal without digits"
982
          end
983
          if non_digit
984
            RubyLex.fail SyntaxError, "trailing `#{non_digit}' in number"
985
          end
986
          break
987
        end
988
      end
989
      return Token(TkINTEGER)
990
    end
991

  
992
    type = TkINTEGER
993
    allow_point = true
994
    allow_e = true
995
    non_digit = false
996
    while ch = getc
997
      case ch
998
      when /[0-9]/
999
        non_digit = false
1000
      when "_"
1001
        non_digit = ch
1002
      when allow_point && "."
1003
        if non_digit
1004
          RubyLex.fail SyntaxError, "trailing `#{non_digit}' in number"
1005
        end
1006
        type = TkFLOAT
1007
        if peek(0) !~ /[0-9]/
1008
          type = TkINTEGER
1009
          ungetc
1010
          break
1011
        end
1012
        allow_point = false
1013
      when allow_e && "e", allow_e && "E"
1014
        if non_digit
1015
          RubyLex.fail SyntaxError, "trailing `#{non_digit}' in number"
1016
        end
1017
        type = TkFLOAT
1018
        if peek(0) =~ /[+-]/
1019
          getc
1020
        end
1021
        allow_e = false
1022
        allow_point = false
1023
        non_digit = ch
1024
      else
1025
        if non_digit
1026
          RubyLex.fail SyntaxError, "trailing `#{non_digit}' in number"
1027
        end
1028
        ungetc
1029
        break
1030
      end
1031
    end
1032
    Token(type)
1033
  end
1034

  
1035
  def identify_string(ltype, quoted = ltype)
1036
    @ltype = ltype
1037
    @quoted = quoted
1038
    subtype = nil
1039
    begin
1040
      nest = 0
1041
      while ch = getc
1042
        if @quoted == ch and nest == 0
1043
          break
1044
        elsif @ltype != "'" && ch == "#" && peek(0) == "{"
1045
          identify_string_dvar
1046
        elsif @ltype != "'" && @ltype != "]" && @ltype != ":" and ch == "#"
1047
          subtype = true
1048
        elsif ch == '\\' and @ltype == "'" #'
1049
          case ch = getc
1050
          when "\\", "\n", "'"
1051
          else
1052
            ungetc
1053
          end
1054
        elsif ch == '\\' #'
1055
          read_escape
1056
        end
1057
        if PERCENT_PAREN.values.include?(@quoted)
1058
          if PERCENT_PAREN[ch] == @quoted
1059
            nest += 1
1060
          elsif ch == @quoted
1061
            nest -= 1
1062
          end
1063
        end
1064
      end
1065
      if @ltype == "/"
1066
        while /[imxoesun]/ =~ peek(0)
1067
          getc
1068
        end
1069
      end
1070
      if subtype
1071
        Token(DLtype2Token[ltype])
1072
      else
1073
        Token(Ltype2Token[ltype])
1074
      end
1075
    ensure
1076
      @ltype = nil
1077
      @quoted = nil
1078
      @lex_state = EXPR_END
1079
    end
1080
  end
1081

  
1082
  def identify_string_dvar
1083
    begin
1084
      getc
1085

  
1086
      reserve_continue = @continue
1087
      reserve_ltype = @ltype
1088
      reserve_indent = @indent
1089
      reserve_indent_stack = @indent_stack
1090
      reserve_state = @lex_state
1091
      reserve_quoted = @quoted
1092

  
1093
      @ltype = nil
1094
      @quoted = nil
1095
      @indent = 0
1096
      @indent_stack = []
1097
      @lex_state = EXPR_BEG
1098

  
1099
      loop do
1100
        @continue = false
1101
        prompt
1102
        tk = token
1103
        if @ltype or @continue or @indent >= 0
1104
          next
1105
        end
1106
        break if tk.kind_of?(TkRBRACE)
1107
      end
1108
    ensure
1109
      @continue = reserve_continue
1110
      @ltype = reserve_ltype
1111
      @indent = reserve_indent
1112
      @indent_stack = reserve_indent_stack
1113
      @lex_state = reserve_state
1114
      @quoted = reserve_quoted
1115
    end
1116
  end
1117

  
1118
  def identify_comment
1119
    @ltype = "#"
1120

  
1121
    while ch = getc
1122
      if ch == "\n"
1123
        @ltype = nil
1124
        ungetc
1125
        break
1126
      end
1127
    end
1128
    return Token(TkCOMMENT)
1129
  end
1130

  
1131
  def read_escape
1132
    case ch = getc
1133
    when "\n", "\r", "\f"
1134
    when "\\", "n", "t", "r", "f", "v", "a", "e", "b", "s" #"
1135
    when /[0-7]/
1136
      ungetc ch
1137
      3.times do
1138
        case ch = getc
1139
        when /[0-7]/
1140
        when nil
1141
          break
1142
        else
1143
          ungetc
1144
          break
1145
        end
1146
      end
1147

  
1148
    when "x"
1149
      2.times do
1150
        case ch = getc
1151
        when /[0-9a-fA-F]/
1152
        when nil
1153
          break
1154
        else
1155
          ungetc
1156
          break
1157
        end
1158
      end
1159

  
1160
    when "M"
1161
      if (ch = getc) != '-'
1162
        ungetc
1163
      else
1164
        if (ch = getc) == "\\" #"
1165
          read_escape
1166
        end
1167
      end
1168

  
1169
    when "C", "c" #, "^"
1170
      if ch == "C" and (ch = getc) != "-"
1171
        ungetc
1172
      elsif (ch = getc) == "\\" #"
1173
        read_escape
1174
      end
1175
    else
1176
      # other characters
1177 284
    end
1178 285
  end
1179 286
end
... This diff was truncated because it exceeds the maximum size that can be displayed.