# A abstract lexer engine generated by nitcc
abstract class Lexer
	# The input stream of characters
	var stream: String

	# The stating state for lexing
	# Used by generated parsers
	protected fun start_state: DFAState is abstract

	# Lexize a stream of characters and return a sequence of tokens
	fun lex: CircularArray[NToken]
	do
		var res = new CircularArray[NToken]
		chars = stream.chars
		loop
			var t = next_token
			if t != null then res.add t
			if t isa NEof or t isa NError then break
		end
		return res
	end

	# Cursor current position (in chars, starting from 0)
	var pos_start = 0

	# Cursor current line (starting from 1)
	var line_start = 1

	# Cursor current column (in chars, starting from 1)
	var col_start = 1

	# Current
	var chars: SequenceRead[Char] is noinit

	# Move the cursor and return the next token.
	#
	# Returns a `NEof` and the end.
	# Returns `null` if the token is ignored.
	fun next_token: nullable NToken
	do
		var state = start_state
		var pos = pos_start
		var pos_end = pos_start - 1
		var line = line_start
		var line_end = line_start - 1
		var col = col_start
		var col_end = col_start - 1
		var last_state: nullable DFAState = null
		var text = stream
		var chars = self.chars
		var length = text.length
		loop
			if state.is_accept then
				pos_end = pos - 1
				line_end = line
				col_end = col
				last_state = state
			end
			var c
			var next
			if pos >= length then
				c = '\0'
				next = null
			else
				c = chars[pos]
				next = state.trans(c)
			end
			if next == null then
				var token
				if pos_start < length then
					if last_state == null then
						token = new NLexerError
						var position = new Position(pos_start, pos, line_start, line, col_start, col)
						token.position = position
						token.text = text.substring(pos_start, pos-pos_start+1)
					else if not last_state.is_ignored then
						var position = new Position(pos_start, pos_end, line_start, line_end, col_start, col_end)
						token = last_state.make_token(position, text)
					else
						token = null
					end
				else
					token = new NEof
					var position = new Position(pos, pos, line, line, col, col)
					token.position = position
					token.text = ""
				end
				pos_start = pos_end + 1
				line_start = line_end
				col_start = col_end

				return token
			end
			state = next
			pos += 1
			col += 1
			if c == '\n' then
				line += 1
				col = 1
			end
		end
	end
end