Ancestor of all tokens

# Ancestor of all tokens
# A token is a node that has a `text` but no children.
abstract class Token
	super ANode

	# The raw content on the token
	fun text: String is abstract

	# The raw content on the token
	fun text=(text: String) is abstract

	# The previous token in the Lexer.
	# May have disappeared in the AST
	var prev_token: nullable Token = null

	# The next token in the Lexer.
	# May have disappeared in the AST
	var next_token: nullable Token = null

	# Is `self` a token discarded from the AST?
	#
	# Loose tokens are not present in the AST.
	# It means they were identified by the lexer but were discarded by the parser.
	# It also means that they are not visited or manipulated by AST-related functions.
	#
	# Each loose token is attached to the non-loose token that precedes or follows it.
	# The rules are the following:
	#
	# * tokens that follow a non-loose token on a same line are attached to it.
	#   See `next_looses`.
	# * other tokens, thus that precede a non-loose token on the same line or the next one,
	# are attached to this one. See `prev_looses`.
	#
	# Loose tokens are mostly end of lines (`TEol`) and comments (`TComment`).
	# Whitespace are ignored by the lexer, so they are not even considered as loose tokens.
	# See `blank_before` to get the whitespace that separate tokens.
	var is_loose = false

	redef fun is_structural do return true

	redef fun dump_info(v) do return " {text.escape_to_c}"

	# Loose tokens that precede `self`.
	#
	# These tokens start the line or belong to a line with only loose tokens.
	var prev_looses = new Array[Token] is lazy

	# Loose tokens that follow `self`
	#
	# These tokens are on the same line than `self`.
	var next_looses = new Array[Token] is lazy

	# The verbatim blank text between `prev_token` and `self`
	fun blank_before: String
	do
		if prev_token == null then return ""
		var from = prev_token.location.pend+1
		var to = location.pstart
		return location.file.string.substring(from,to-from)
	end

	redef fun to_s: String do
		return "'{text}'"
	end

	redef fun visit_all(v: Visitor) do end
	redef fun replace_child(old_child: ANode, new_child: nullable ANode) do end
end

src/parser/parser_nodes.nit:345,1--412,3

nitc :: lexer_work $ Token

redef class Token
    private var cached_text: nullable String

    redef fun text
    do
        var res = _cached_text
        if res != null then return res
        res = location.text
	_cached_text = res
	return res
    end

    redef fun text=(text)
    do
        _cached_text = text
    end

    fun parser_index: Int is abstract
end

src/parser/lexer_work.nit:21,1--39,3

nitc :: astutil $ Token

redef class Token
	# Is self the first AST token on its line in the source
	# Computed by `parentize_tokens`
	#
	# Note, some tokens detached from the AST
	# may precede `self` even if `is_starting_line` is true
	# One can use `first_real_token_in_line` to get the real starting token
	var is_starting_line = false

	# Is self the last AST token on its line in the source
	# Computed by `parentize_tokens`
	#
	# Note, some tokens detached from the AST (like comments)
	# may follow `self` even if `is_ending_line` is true.
	# One can use `last_real_token_in_line` to get the real ending token
	var is_ending_line = false

	# The first real token that starts the line of `self`
	#
	# This could return a token that is detached from the AST.
	# See `first_token_in_line` if a AST token is required.
	fun first_real_token_in_line: Token
	do
		var line = location.line_start
		var t = self
		loop
			var p = t.prev_token
			if p == null or p.location.line_start != line then
				return t
			end
			t = p
		end
	end

	# The first AST token that starts the line of `self`.
	# May be null is the line contains only detached tokens (only comment)
	#
	# Computed by `parentize_tokens`
	#
	# ENSURE `result != null implies result.is_starting_line`
	fun first_token_in_line: nullable Token
	do
		return first_real_token_in_line.first_ast_token
	end

	# The first AST token.
	# This only work on the `first_real_token_in_line`
	private var first_ast_token: nullable Token

	# The last read token that ends the line of `self`
	#
	# This usually return a detached token lake a TEol or a comment.
	# See `last_token_in_line` if a AST token is required.
	fun last_real_token_in_line: Token
	do
		var line = location.line_start
		var t = self
		loop
			var p = t.next_token
			if p == null or p.location.line_start != line then
				return t
			end
			t = p
		end
	end

	# The last AST token that starts the line of `self`
	# May be null is the line contains only detached tokens (only comment)
	#
	# Computed by `parentize_tokens`
	#
	# ENSURE `result.is_ending_line`
	fun last_token_in_line: nullable Token
	do
		return last_real_token_in_line.last_ast_token
	end

	# The last AST token.
	# This only work on the `last_real_token_in_line`
	private var last_ast_token: nullable Token

	# The productions that starts with `self`, if any
	# Productions goes from the most general to the most specific
	#
	# Computed by `parentize_tokens`
	var starting_prods: nullable Array[Prod]

	# The productions that ends with `self`, if any
	# Productions goes from the most specific to the most general
	#
	# Computed by `parentize_tokens`
	var ending_prods: nullable Array[Prod]
end

src/astutil.nit:109,1--201,3

nitc :: simple_misc_analysis $ Token

redef class Token
	private fun accept_simple_misc_token(v: SimpleMiscVisitor)
	do
	end
end

src/frontend/simple_misc_analysis.nit:90,1--94,3

nitc :: pretty $ Token

redef class Token
	redef fun accept_pretty_printer(v) do
		v.add text.trim
		v.current_token = next_token
	end

	redef fun collect_length do return text.length
	redef fun is_inlinable do return true
	redef fun was_inline do return true
end

src/pretty.nit:348,1--357,3

nitc :: highlight $ Token

redef class Token
	# Return the ANSI colored text
	fun ansi_colored: String do return text
end

src/highlight.nit:138,1--141,3

nitc :: htmlight $ Token

redef class Token
	# Produce an HTMLTag with the correct contents and CSS classes
	# Subclasses can redefine it to decorate the tag
	redef fun make_tag(v): HTMLTag
	do
		var res = new HTMLTag("span")
		res.text(text)
		return res
	end
end

src/htmlight.nit:807,1--816,3

abstract class Token

Summary

Ancestor of all tokens

Introduced properties

_cached_text

_ending_prods

_first_ast_token

_is_ending_line

_is_loose

_is_starting_line

_last_ast_token

_next_looses

_next_token

_prev_looses

_prev_token

_starting_prods

accept_simple_misc_token

ansi_colored

blank_before

cached_text

cached_text=

defaultinit

ending_prods

ending_prods=

first_ast_token

first_ast_token=

first_real_token_in_line

first_token_in_line

is_ending_line

is_ending_line=

is_loose

is_loose=

is_starting_line

is_starting_line=

last_ast_token

last_ast_token=

last_real_token_in_line

last_token_in_line

next_looses

next_looses=

next_token

next_token=

parser_index

prev_looses

prev_looses=

prev_token

prev_token=

starting_prods

starting_prods=

text

text=

Redefined properties

SELF

accept_pretty_printer

collect_length

dump_info

is_inlinable

is_structural

make_tag

replace_child

text

text=

to_s

visit_all

was_inline

Ancestor of all tokens

Introduced properties

private var _cached_text: nullable String

private var _ending_prods: nullable Array[Prod]

private var _first_ast_token: nullable Token

private var _is_ending_line: Bool

private var _is_loose: Bool

private var _is_starting_line: Bool

private var _last_ast_token: nullable Token

private var _next_looses: Array[Token]

private var _next_token: nullable Token

private var _prev_looses: Array[Token]

private var _prev_token: nullable Token

private var _starting_prods: nullable Array[Prod]

private fun accept_simple_misc_token(v: SimpleMiscVisitor)