- while _token == null do
- _token = get_token
- end
- return _token.as(not null)
- end
-
- # Give and consume the next token
- fun next: Token
- do
- var result = _token
- while result == null do
- result = get_token
- end
- _token = null
- return result.as(not null)
- end
-
- # Get a token, or null if it is discarded
- private fun get_token: nullable Token
- do
- var dfa_state = 0
-
- var start_pos = _pos
- var start_line = _line
-
- var accept_state = -1
- var accept_token = -1
- var accept_length = -1
- var accept_pos = -1
- var accept_line = -1
-
- var goto_table = _goto_table[_state]
- var accept = _accept_table[_state]
- _text.clear
-
- while true do
- var c = get_char
-
- if c != -1 then
- if c == 10 then
- if _cr then
- _cr = false
- else
- _line = _line + 1
- _pos = 0
- end
- else if c == 13 then
- _line = _line + 1
- _pos = 0
- _cr = true
- else
- _pos = _pos + 1
- _cr = false
- end
-
- _text.add(c.ascii)
-
- var first_loop = true # aka until
- while dfa_state < -1 or first_loop do
- var old_state = dfa_state
- if dfa_state < -1 then
- old_state = -2 - dfa_state
- end
-
- dfa_state = -1
-
- var tmp1 = goto_table[old_state]
- var low = 0
- var high = tmp1.length - 1
-
- while low <= high do
- var middle = (low + high) / 2
- var tmp2 = tmp1[middle]
-
- if c < tmp2[0] then
- high = middle - 1
- else if c > tmp2[1] then
- low = middle + 1
- else
- dfa_state = tmp2[2]
- low = high + 1 # aka break
- end
- end
- first_loop = false # aka until
- end
- else
- dfa_state = -1
- end
-
- if dfa_state >= 0 then
- if accept[dfa_state] != -1 then
- accept_state = dfa_state
- accept_token = accept[dfa_state]
- accept_length = _text.length
- accept_pos = _pos
- accept_line = _line
- end
- else
- if accept_state != -1 then