# Ancestor of all tokens
# A token is a node that has a `text` but no children.
abstract class Token
	super ANode

	# The raw content on the token
	fun text: String is abstract

	# The raw content on the token
	fun text=(text: String) is abstract

	# The previous token in the Lexer.
	# May have disappeared in the AST
	var prev_token: nullable Token = null

	# The next token in the Lexer.
	# May have disappeared in the AST
	var next_token: nullable Token = null

	# Is `self` a token discarded from the AST?
	#
	# Loose tokens are not present in the AST.
	# It means they were identified by the lexer but were discarded by the parser.
	# It also means that they are not visited or manipulated by AST-related functions.
	#
	# Each loose token is attached to the non-loose token that precedes or follows it.
	# The rules are the following:
	#
	# * tokens that follow a non-loose token on a same line are attached to it.
	#   See `next_looses`.
	# * other tokens, thus that precede a non-loose token on the same line or the next one,
	# are attached to this one. See `prev_looses`.
	#
	# Loose tokens are mostly end of lines (`TEol`) and comments (`TComment`).
	# Whitespace are ignored by the lexer, so they are not even considered as loose tokens.
	# See `blank_before` to get the whitespace that separate tokens.
	var is_loose = false

	redef fun is_structural do return true

	redef fun dump_info(v) do return " {text.escape_to_c}"

	# Loose tokens that precede `self`.
	#
	# These tokens start the line or belong to a line with only loose tokens.
	var prev_looses = new Array[Token] is lazy

	# Loose tokens that follow `self`
	#
	# These tokens are on the same line than `self`.
	var next_looses = new Array[Token] is lazy

	# The verbatim blank text between `prev_token` and `self`
	fun blank_before: String
	do
		if prev_token == null then return ""
		var from = prev_token.location.pend+1
		var to = location.pstart
		return location.file.string.substring(from,to-from)
	end

	redef fun to_s: String do
		return "'{text}'"
	end

	redef fun visit_all(v: Visitor) do end
	redef fun replace_child(old_child: ANode, new_child: nullable ANode) do end
end