private class PTokenVisitor
	super Visitor

	var last_token: nullable Token = null

	# productions that need a fisrt token
	var stack = new Array[Prod]

	fun work(n: ANode)
	do
		enter_visit(n)
		# process remaining detashed tokens
		var c = last_token
		if c != null then
			c.is_ending_line = true
			c.last_real_token_in_line.last_ast_token = c
			c = c.next_token
		end
		var r = n.root
		while c != null and c.parent == null do
			c.parent = r
			c = c.next_token
		end
	end

	redef fun visit(n)
	do
		if not n isa Token then
			assert n isa Prod
			stack.push(n)
			n.visit_all(self)
			if n.first_token == null then
				# epsilon production, just discard
				assert stack.pop == n
			else
				var t = last_token
				if t != null then
					# last token ends the production
					n.last_token = t
					if t.ending_prods == null then t.ending_prods = new Array[Prod]
					t.ending_prods.add n
				end
			end
			return
		end

		# We have a token, give it to prods that need one
		if not stack.is_empty then
			n.starting_prods = new Array[Prod]
			for p in stack do
				p.first_token = n
				n.starting_prods.add p
			end
			stack.clear
		end

		var last_token = last_token

		# n starts a new line
		if last_token == null or last_token.location.line_start != n.location.line_start then
			n.is_starting_line = true
			n.first_real_token_in_line.first_ast_token = n

		end
		# last_token ended a line
		if last_token != null and last_token.location.line_start != n.location.line_start then
			last_token.is_ending_line = true
			last_token.last_real_token_in_line.last_ast_token = last_token
		end

		# Get the common parent
		var p
		if last_token == null then
			p = n.root
		else
			p = last_token.common_parent(n)
		end
		if p isa Prod then
			# And apply it to detached tokens between `last_token` and `n`
			var c = n.prev_token
			while c != null and c.parent == null do
				c.parent = p
				c = c.prev_token
			end
		end

		self.last_token = n
	end
end