# Lexize a stream of characters and return a sequence of tokens
	fun lex: CircularArray[NToken]
	do
		var res = new CircularArray[NToken]
		loop
			var t = next_token
			if t != null then res.add t
			if t isa NEof or t isa NError then break
		end
		return res
	end