- _filename = fname
- _text = new Buffer
- _stream = stream
- _stream_pos = -1
- _stream_buf = new Buffer
- build_goto_table
- build_accept_table
- end
-
- # Give the next token (but do not consume it)
- fun peek: Token
- do
- while _token == null do
- _token = get_token
- end
- return _token.as(not null)
- end
-
- # Give and consume the next token
- fun next: Token
- do
- var result = _token
- while result == null do
- result = get_token
- end
- _token = null
- return result.as(not null)
- end
-
- # Get a token, or null if it is discarded
- private fun get_token: nullable Token
- do
- var dfa_state = 0
-
- var start_pos = _pos
- var start_line = _line
-
- var accept_state = -1
- var accept_token = -1
- var accept_length = -1
- var accept_pos = -1
- var accept_line = -1
-
- var goto_table = _goto_table[_state]
- var accept = _accept_table[_state]
- var text = _text
- text.clear
-
- while true do
- var c = get_char
-
- if c != -1 then
- var cr = _cr
- var line = _line
- var pos = _pos
- if c == 10 then
- if cr then
- cr = false
- else
- line = line + 1
- pos = 0
- end
- else if c == 13 then
- line = line + 1
- pos = 0
- cr = true
- else
- pos = pos + 1
- cr = false
- end
-
- text.add(c.ascii)
-
- var first_loop = true # aka until
- while dfa_state < -1 or first_loop do
- var old_state = dfa_state
- if dfa_state < -1 then
- old_state = -2 - dfa_state
- end
-
- dfa_state = -1
-
- var tmp0 = goto_table[old_state]
- var low = 0
- var high = tmp0.length - 1
-
- if high >= 0 then
- var tmp1 = tmp0.intern_items
- while low <= high do
- var middle = (low + high) / 2
- var tmp2 = tmp1[middle].intern_items
-
- if c < tmp2[0] then
- high = middle - 1
- else if c > tmp2[1] then
- low = middle + 1
- else
- dfa_state = tmp2[2]
- low = high + 1 # aka break
- end
- end
- end
- first_loop = false # aka until
- end
-
- _cr = cr
- _line = line
- _pos = pos
- else
- dfa_state = -1
- end
-
- if dfa_state >= 0 then
- if accept[dfa_state] != -1 then
- accept_state = dfa_state
- accept_token = accept[dfa_state]
- accept_length = text.length
- accept_pos = _pos
- accept_line = _line
- end
- else
- if accept_state != -1 then