parser: do not crash on tokens without location

Do not insert them in the linked lis of tokens
Do not use them to compute locations on productions

Signed-off-by: Jean Privat <jean@pryen.org>

diff --git a/src/parser/lexer_work.nit b/src/parser/lexer_work.nit
index 4a0820e..0b2258b 100644 (file)
--- a/src/parser/lexer_work.nit
+++ b/src/parser/lexer_work.nit
@@ -127,15 +127,17 @@ class Lexer
                t = get_token
                while t == null do t = get_token
 
-               var l = last_token
-               if l != null then
-                       l.next_token = t
-                       t.prev_token = l
-               else
-                       _file.first_token = t
+               if t._location != null then
+                       var l = last_token
+                       if l != null then
+                               l.next_token = t
+                               t.prev_token = l
+                       else
+                               _file.first_token = t
+                       end
+                       last_token = t
                end
 
-               last_token = t
                _token = t
                return t
        end

diff --git a/src/parser/parser_work.nit b/src/parser/parser_work.nit
index 03d82b1..5652cf5 100644 (file)
--- a/src/parser/parser_work.nit
+++ b/src/parser/parser_work.nit
@@ -153,6 +153,8 @@ class Parser
                                (new ComputeProdLocationVisitor).enter_visit(node)
                                return node
                        else if action_type == 3 then # ERROR
+                               # skip injected tokens
+                               while token._location == null do token = lexer.next
                                var node2 = new AParserError.init_parser_error("Syntax error: unexpected {token}.", token.location, token)
                                var node = new Start(null, node2)
                                return node
@@ -186,7 +188,8 @@ private class ComputeProdLocationVisitor
        redef fun visit(n: ANode)
        do
                if n isa Token then
-                       var loc = n.location
+                       var loc = n._location
+                       if loc == null then return
                        _last_location = loc
 
                        # Add a first token to productions that need one