From: Jean Privat <jean@pryen.org>
Date: Wed, 21 Aug 2013 19:29:04 +0000 (-0400)
Subject: lexer: chain tokens together when we peek then
X-Git-Tag: v0.6.1~15^2~7
X-Git-Url: http://nitlanguage.org

lexer: chain tokens together when we peek then

Signed-off-by: Jean Privat <jean@pryen.org>
---

diff --git a/src/parser/lexer.nit b/src/parser/lexer.nit
index 4fc4969..96d3ea4 100644
--- a/src/parser/lexer.nit
+++ b/src/parser/lexer.nit
@@ -1219,22 +1219,33 @@ class Lexer
 		_file = file
 	end
 
+	# The last peeked token to chain them
+	private var last_token: nullable Token = null
+
 	# Give the next token (but do not consume it)
 	fun peek: Token
 	do
-		while _token == null do
-			_token = get_token
+		var t = _token
+		if t != null then return t
+
+		t = get_token
+		while t == null do t = get_token
+
+		var l = last_token
+		if l != null then
+			l.next_token = t
+			t.prev_token = l
 		end
-		return _token.as(not null)
+
+		last_token = t
+		_token = t
+		return t
 	end
 
 	# Give and consume the next token
 	fun next: Token
 	do
-		var result = _token
-		while result == null do
-			result = get_token
-		end
+		var result = peek
 		_token = null
 		return result
 	end
diff --git a/src/parser/parser_nodes.nit b/src/parser/parser_nodes.nit
index 3571858..7b98fc7 100644
--- a/src/parser/parser_nodes.nit
+++ b/src/parser/parser_nodes.nit
@@ -143,6 +143,14 @@ abstract class Token
 	fun text: String is abstract
 	fun text=(text: String) is abstract
 
+	# The previous token in the Lexer.
+	# May have disapeared in the AST
+	var prev_token: nullable Token
+
+	# The next token in the Lexer.
+	# May have disapeared in the AST
+	var next_token: nullable Token
+
 	redef fun to_s: String do
 		return "'{text}'"
 	end
diff --git a/src/parser/xss/lexer.xss b/src/parser/xss/lexer.xss
index 3fe41f0..6a4d1d0 100644
--- a/src/parser/xss/lexer.xss
+++ b/src/parser/xss/lexer.xss
@@ -53,13 +53,27 @@ $ end foreach
 		_file = file
 	end
 
+	# The last peeked token to chain them
+	private var last_token: nullable Token = null
+
 	# Give the next token (but do not consume it)
 	fun peek: Token
 	do
-		while _token == null do
-			_token = get_token
+		var t = _token
+		if t != null then return t
+
+		t = get_token
+		while t == null do t = get_token
+
+		var l = last_token
+		if l != null then
+			l.next_token = t
+			t.prev_token = l
 		end
-		return _token.as(not null)
+
+		last_token = t
+		_token = t
+		return t
 	end
 
 	# Give and consume the next token