lexer: chain tokens together when we peek then

Signed-off-by: Jean Privat <jean@pryen.org>

diff --git a/src/parser/lexer.nit b/src/parser/lexer.nit
index 4fc4969..96d3ea4 100644 (file)
--- a/src/parser/lexer.nit
+++ b/src/parser/lexer.nit
@@ -1219,22 +1219,33 @@ class Lexer
                _file = file
        end
 
+       # The last peeked token to chain them
+       private var last_token: nullable Token = null
+
        # Give the next token (but do not consume it)
        fun peek: Token
        do
-               while _token == null do
-                       _token = get_token
+               var t = _token
+               if t != null then return t
+
+               t = get_token
+               while t == null do t = get_token
+
+               var l = last_token
+               if l != null then
+                       l.next_token = t
+                       t.prev_token = l
                end
-               return _token.as(not null)
+
+               last_token = t
+               _token = t
+               return t
        end
 
        # Give and consume the next token
        fun next: Token
        do
-               var result = _token
-               while result == null do
-                       result = get_token
-               end
+               var result = peek
                _token = null
                return result
        end

diff --git a/src/parser/parser_nodes.nit b/src/parser/parser_nodes.nit
index 3571858..7b98fc7 100644 (file)
--- a/src/parser/parser_nodes.nit
+++ b/src/parser/parser_nodes.nit
@@ -143,6 +143,14 @@ abstract class Token
        fun text: String is abstract
        fun text=(text: String) is abstract
 
+       # The previous token in the Lexer.
+       # May have disapeared in the AST
+       var prev_token: nullable Token
+
+       # The next token in the Lexer.
+       # May have disapeared in the AST
+       var next_token: nullable Token
+
        redef fun to_s: String do
                return "'{text}'"
        end

diff --git a/src/parser/xss/lexer.xss b/src/parser/xss/lexer.xss
index 3fe41f0..6a4d1d0 100644 (file)
--- a/src/parser/xss/lexer.xss
+++ b/src/parser/xss/lexer.xss
@@ -53,13 +53,27 @@ $ end foreach
                _file = file
        end
 
+       # The last peeked token to chain them
+       private var last_token: nullable Token = null
+
        # Give the next token (but do not consume it)
        fun peek: Token
        do
-               while _token == null do
-                       _token = get_token
+               var t = _token
+               if t != null then return t
+
+               t = get_token
+               while t == null do t = get_token
+
+               var l = last_token
+               if l != null then
+                       l.next_token = t
+                       t.prev_token = l
                end
-               return _token.as(not null)
+
+               last_token = t
+               _token = t
+               return t
        end
 
        # Give and consume the next token