- _file = file
- end
-
- # Give the next token (but do not consume it)
- fun peek: Token
- do
- while _token == null do
- _token = get_token
- end
- return _token.as(not null)
- end
-
- # Give and consume the next token
- fun next: Token
- do
- var result = _token
- while result == null do
- result = get_token
- end
- _token = null
- return result
- end
-
- # Get a token, or null if it is discarded
- private fun get_token: nullable Token
- do
- var dfa_state = 0
-
- var sp = _stream_pos
- var start_stream_pos = sp
- var start_pos = _pos
- var start_line = _line
- var string = _file.string
- var string_len = string.length
-
- var accept_state = -1
- var accept_token = -1
- var accept_length = -1
- var accept_pos = -1
- var accept_line = -1
-
- loop
- if sp >= string_len then
- dfa_state = -1
- else
- var c = string[sp].ascii
- sp += 1
-
- var cr = _cr
- var line = _line
- var pos = _pos
- if c == 10 then
- if cr then
- cr = false
- _file.line_starts[line] = sp
- else
- line = line + 1
- pos = 0
- _file.line_starts[line] = sp
- end
- else if c == 13 then
- line = line + 1
- pos = 0
- cr = true
- _file.line_starts[line] = sp
- else
- pos = pos + 1
- cr = false
- end
-
- loop
- var old_state = dfa_state
- if dfa_state < -1 then
- old_state = -2 - dfa_state
- end
-
- dfa_state = -1
-
- var low = 0
- var high = lexer_goto(old_state, 0) - 1
-
- if high >= 0 then
- while low <= high do
- var middle = (low + high) / 2
- var offset = middle * 3 + 1 # +1 because length is at 0
-
- if c < lexer_goto(old_state, offset) then
- high = middle - 1
- else if c > lexer_goto(old_state, offset+1) then
- low = middle + 1
- else
- dfa_state = lexer_goto(old_state, offset+2)
- break
- end
- end
- end
- if dfa_state > -2 then break
- end
-
- _cr = cr
- _line = line
- _pos = pos
- end
-
- if dfa_state >= 0 then
- var tok = lexer_accept(dfa_state)
- if tok != -1 then
- accept_state = dfa_state
- accept_token = tok
- accept_length = sp - start_stream_pos
- accept_pos = _pos
- accept_line = _line
- end
- else
- if accept_state != -1 then
- var location = new Location(_file, start_line + 1, accept_line + 1, start_pos + 1, accept_pos)
- _pos = accept_pos
- _line = accept_line
- _stream_pos = start_stream_pos + accept_length