nitcc_runtime :: Lexer :: defaultinit
# A abstract lexer engine generated by nitcc
abstract class Lexer
# The input stream of characters
var stream: String
# The stating state for lexing
# Used by generated parsers
protected fun start_state: DFAState is abstract
# Lexize a stream of characters and return a sequence of tokens
fun lex: CircularArray[NToken]
do
var res = new CircularArray[NToken]
loop
var t = next_token
if t != null then res.add t
if t isa NEof or t isa NError then break
end
return res
end
# Cursor current position (in chars, starting from 0)
var pos_start = 0
# Cursor current line (starting from 1)
var line_start = 1
# Cursor current column (in chars, starting from 1)
var col_start = 1
# Move the cursor and return the next token.
#
# Returns a `NEof` and the end.
# Returns `null` if the token is ignored.
fun next_token: nullable NToken
do
var state = start_state
var pos = pos_start
var pos_end = pos_start - 1
var line = line_start
var line_end = line_start - 1
var col = col_start
var col_end = col_start - 1
var last_state: nullable DFAState = null
var text = stream
var length = text.length
loop
if state.is_accept then
pos_end = pos - 1
line_end = line
col_end = col
last_state = state
end
var c
var next
if pos >= length then
c = '\0'
next = null
else
c = text.chars[pos]
next = state.trans(c)
end
if next == null then
var token
if pos_start < length then
if last_state == null then
token = new NLexerError
var position = new Position(pos_start, pos, line_start, line, col_start, col)
token.position = position
token.text = text.substring(pos_start, pos-pos_start+1)
else if not last_state.is_ignored then
var position = new Position(pos_start, pos_end, line_start, line_end, col_start, col_end)
token = last_state.make_token(position, text)
else
token = null
end
else
token = new NEof
var position = new Position(pos, pos, line, line, col, col)
token.position = position
token.text = ""
end
pos_start = pos_end + 1
line_start = line_end
col_start = col_end
return token
end
state = next
pos += 1
col += 1
if c == '\n' then
line += 1
col = 1
end
end
end
end
lib/nitcc_runtime/nitcc_runtime.nit:157,1--254,3