lexer: implements next with peek

[nit.git] / src / parser / xss / lexer.xss

diff --git a/src/parser/xss/lexer.xss b/src/parser/xss/lexer.xss
index 1e2dda9..3fe41f0 100644 (file)
--- a/src/parser/xss/lexer.xss
+++ b/src/parser/xss/lexer.xss
@@ -65,16 +65,14 @@ $ end foreach
        # Give and consume the next token
        fun next: Token
        do
-               var result = _token
-               while result == null do
-                       result = get_token
-               end
+               var result = peek
                _token = null
                return result
        end
 
-       # Get a token, or null if it is discarded
-       private fun get_token: nullable Token
+       # Primitive method to return a token, or return null if it is discarded
+       # Is used to implement `peek` and `next`
+       protected fun get_token: nullable Token
        do
                var dfa_state = 0
 
@@ -180,12 +178,7 @@ $        foreach transition in {transition[@from!=@to]}
 $        end
 $    end if
 $    if {@parser_index}
-$        if {not(@text)}
-                                               var token_text = string.substring(start_stream_pos, accept_length)
-                                               return new @ename.init_tk(token_text, location)
-$        else
                                                return new @ename.init_tk(location)
-$        end
 $    else
                                                return null
 $    end
@@ -196,10 +189,10 @@ $ end foreach
                                        var location = new Location(_file, start_line + 1, start_line + 1, start_pos + 1, start_pos + 1)
                                        if sp > start_stream_pos then
                                                var text = string.substring(start_stream_pos, sp-start_stream_pos)
-                                               var token = new PError.init_error("Syntax error: unknown token {text}.", location)
+                                               var token = new PLexerError.init_lexer_error("Syntax error: unknown token {text}.", location, text)
                                                return token
                                        else
-                                               var token = new EOF(location)
+                                               var token = new EOF.init_tk(location)
                                                return token
                                        end
                                end