Merge branch 'annotations'
[nit.git] / src / parser / lexer.nit
index 5917517..7e07395 100644 (file)
@@ -6,7 +6,21 @@ intrude import parser_nodes
 private import tables
 
 redef class Token
-    redef readable var _text: String
+    var _text: nullable String
+
+    redef fun text
+    do
+        var res = _text
+        if res != null then return res
+        res = location.text
+       _text = res
+       return res
+    end
+
+    redef fun text=(text)
+    do
+        _text = text
+    end
 
     fun parser_index: Int is abstract
 end
@@ -17,9 +31,8 @@ redef class TEol
        return 0
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
                _location = loc
     end
 end
@@ -30,9 +43,8 @@ redef class TComment
        return 1
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
                _location = loc
     end
 end
@@ -43,9 +55,8 @@ redef class TKwmodule
        return 2
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
                _location = loc
     end
 end
@@ -58,7 +69,6 @@ redef class TKwimport
 
     init init_tk(loc: Location)
     do
-        _text = once "import"
                _location = loc
     end
 end
@@ -71,7 +81,6 @@ redef class TKwclass
 
     init init_tk(loc: Location)
     do
-        _text = once "class"
                _location = loc
     end
 end
@@ -84,7 +93,6 @@ redef class TKwabstract
 
     init init_tk(loc: Location)
     do
-        _text = once "abstract"
                _location = loc
     end
 end
@@ -97,7 +105,6 @@ redef class TKwinterface
 
     init init_tk(loc: Location)
     do
-        _text = once "interface"
                _location = loc
     end
 end
@@ -108,14 +115,13 @@ redef class TKwenum
        return 7
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
                _location = loc
     end
 end
 
-redef class TKwspecial
+redef class TKwend
     redef fun parser_index: Int
     do
        return 8
@@ -123,12 +129,11 @@ redef class TKwspecial
 
     init init_tk(loc: Location)
     do
-        _text = once "special"
                _location = loc
     end
 end
 
-redef class TKwend
+redef class TKwmeth
     redef fun parser_index: Int
     do
        return 9
@@ -136,12 +141,11 @@ redef class TKwend
 
     init init_tk(loc: Location)
     do
-        _text = once "end"
                _location = loc
     end
 end
 
-redef class TKwmeth
+redef class TKwtype
     redef fun parser_index: Int
     do
        return 10
@@ -149,12 +153,11 @@ redef class TKwmeth
 
     init init_tk(loc: Location)
     do
-        _text = once "fun"
                _location = loc
     end
 end
 
-redef class TKwtype
+redef class TKwinit
     redef fun parser_index: Int
     do
        return 11
@@ -162,12 +165,11 @@ redef class TKwtype
 
     init init_tk(loc: Location)
     do
-        _text = once "type"
                _location = loc
     end
 end
 
-redef class TKwinit
+redef class TKwredef
     redef fun parser_index: Int
     do
        return 12
@@ -175,12 +177,11 @@ redef class TKwinit
 
     init init_tk(loc: Location)
     do
-        _text = once "init"
                _location = loc
     end
 end
 
-redef class TKwredef
+redef class TKwis
     redef fun parser_index: Int
     do
        return 13
@@ -188,12 +189,11 @@ redef class TKwredef
 
     init init_tk(loc: Location)
     do
-        _text = once "redef"
                _location = loc
     end
 end
 
-redef class TKwis
+redef class TKwdo
     redef fun parser_index: Int
     do
        return 14
@@ -201,12 +201,11 @@ redef class TKwis
 
     init init_tk(loc: Location)
     do
-        _text = once "is"
                _location = loc
     end
 end
 
-redef class TKwdo
+redef class TKwreadable
     redef fun parser_index: Int
     do
        return 15
@@ -214,12 +213,11 @@ redef class TKwdo
 
     init init_tk(loc: Location)
     do
-        _text = once "do"
                _location = loc
     end
 end
 
-redef class TKwreadable
+redef class TKwwritable
     redef fun parser_index: Int
     do
        return 16
@@ -227,12 +225,11 @@ redef class TKwreadable
 
     init init_tk(loc: Location)
     do
-        _text = once "readable"
                _location = loc
     end
 end
 
-redef class TKwwritable
+redef class TKwvar
     redef fun parser_index: Int
     do
        return 17
@@ -240,12 +237,11 @@ redef class TKwwritable
 
     init init_tk(loc: Location)
     do
-        _text = once "writable"
                _location = loc
     end
 end
 
-redef class TKwvar
+redef class TKwintern
     redef fun parser_index: Int
     do
        return 18
@@ -253,12 +249,11 @@ redef class TKwvar
 
     init init_tk(loc: Location)
     do
-        _text = once "var"
                _location = loc
     end
 end
 
-redef class TKwintern
+redef class TKwextern
     redef fun parser_index: Int
     do
        return 19
@@ -266,12 +261,11 @@ redef class TKwintern
 
     init init_tk(loc: Location)
     do
-        _text = once "intern"
                _location = loc
     end
 end
 
-redef class TKwextern
+redef class TKwprotected
     redef fun parser_index: Int
     do
        return 20
@@ -279,12 +273,11 @@ redef class TKwextern
 
     init init_tk(loc: Location)
     do
-        _text = once "extern"
                _location = loc
     end
 end
 
-redef class TKwprotected
+redef class TKwprivate
     redef fun parser_index: Int
     do
        return 21
@@ -292,12 +285,11 @@ redef class TKwprotected
 
     init init_tk(loc: Location)
     do
-        _text = once "protected"
                _location = loc
     end
 end
 
-redef class TKwprivate
+redef class TKwintrude
     redef fun parser_index: Int
     do
        return 22
@@ -305,12 +297,11 @@ redef class TKwprivate
 
     init init_tk(loc: Location)
     do
-        _text = once "private"
                _location = loc
     end
 end
 
-redef class TKwintrude
+redef class TKwif
     redef fun parser_index: Int
     do
        return 23
@@ -318,12 +309,11 @@ redef class TKwintrude
 
     init init_tk(loc: Location)
     do
-        _text = once "intrude"
                _location = loc
     end
 end
 
-redef class TKwif
+redef class TKwthen
     redef fun parser_index: Int
     do
        return 24
@@ -331,12 +321,11 @@ redef class TKwif
 
     init init_tk(loc: Location)
     do
-        _text = once "if"
                _location = loc
     end
 end
 
-redef class TKwthen
+redef class TKwelse
     redef fun parser_index: Int
     do
        return 25
@@ -344,12 +333,11 @@ redef class TKwthen
 
     init init_tk(loc: Location)
     do
-        _text = once "then"
                _location = loc
     end
 end
 
-redef class TKwelse
+redef class TKwwhile
     redef fun parser_index: Int
     do
        return 26
@@ -357,12 +345,11 @@ redef class TKwelse
 
     init init_tk(loc: Location)
     do
-        _text = once "else"
                _location = loc
     end
 end
 
-redef class TKwwhile
+redef class TKwloop
     redef fun parser_index: Int
     do
        return 27
@@ -370,12 +357,11 @@ redef class TKwwhile
 
     init init_tk(loc: Location)
     do
-        _text = once "while"
                _location = loc
     end
 end
 
-redef class TKwloop
+redef class TKwfor
     redef fun parser_index: Int
     do
        return 28
@@ -383,12 +369,11 @@ redef class TKwloop
 
     init init_tk(loc: Location)
     do
-        _text = once "loop"
                _location = loc
     end
 end
 
-redef class TKwfor
+redef class TKwin
     redef fun parser_index: Int
     do
        return 29
@@ -396,12 +381,11 @@ redef class TKwfor
 
     init init_tk(loc: Location)
     do
-        _text = once "for"
                _location = loc
     end
 end
 
-redef class TKwin
+redef class TKwand
     redef fun parser_index: Int
     do
        return 30
@@ -409,12 +393,11 @@ redef class TKwin
 
     init init_tk(loc: Location)
     do
-        _text = once "in"
                _location = loc
     end
 end
 
-redef class TKwand
+redef class TKwor
     redef fun parser_index: Int
     do
        return 31
@@ -422,12 +405,11 @@ redef class TKwand
 
     init init_tk(loc: Location)
     do
-        _text = once "and"
                _location = loc
     end
 end
 
-redef class TKwor
+redef class TKwnot
     redef fun parser_index: Int
     do
        return 32
@@ -435,12 +417,11 @@ redef class TKwor
 
     init init_tk(loc: Location)
     do
-        _text = once "or"
                _location = loc
     end
 end
 
-redef class TKwnot
+redef class TKwreturn
     redef fun parser_index: Int
     do
        return 33
@@ -448,12 +429,11 @@ redef class TKwnot
 
     init init_tk(loc: Location)
     do
-        _text = once "not"
                _location = loc
     end
 end
 
-redef class TKwreturn
+redef class TKwcontinue
     redef fun parser_index: Int
     do
        return 34
@@ -461,12 +441,11 @@ redef class TKwreturn
 
     init init_tk(loc: Location)
     do
-        _text = once "return"
                _location = loc
     end
 end
 
-redef class TKwcontinue
+redef class TKwbreak
     redef fun parser_index: Int
     do
        return 35
@@ -474,12 +453,11 @@ redef class TKwcontinue
 
     init init_tk(loc: Location)
     do
-        _text = once "continue"
                _location = loc
     end
 end
 
-redef class TKwbreak
+redef class TKwabort
     redef fun parser_index: Int
     do
        return 36
@@ -487,12 +465,11 @@ redef class TKwbreak
 
     init init_tk(loc: Location)
     do
-        _text = once "break"
                _location = loc
     end
 end
 
-redef class TKwabort
+redef class TKwassert
     redef fun parser_index: Int
     do
        return 37
@@ -500,12 +477,11 @@ redef class TKwabort
 
     init init_tk(loc: Location)
     do
-        _text = once "abort"
                _location = loc
     end
 end
 
-redef class TKwassert
+redef class TKwnew
     redef fun parser_index: Int
     do
        return 38
@@ -513,12 +489,11 @@ redef class TKwassert
 
     init init_tk(loc: Location)
     do
-        _text = once "assert"
                _location = loc
     end
 end
 
-redef class TKwnew
+redef class TKwisa
     redef fun parser_index: Int
     do
        return 39
@@ -526,12 +501,11 @@ redef class TKwnew
 
     init init_tk(loc: Location)
     do
-        _text = once "new"
                _location = loc
     end
 end
 
-redef class TKwisa
+redef class TKwonce
     redef fun parser_index: Int
     do
        return 40
@@ -539,12 +513,11 @@ redef class TKwisa
 
     init init_tk(loc: Location)
     do
-        _text = once "isa"
                _location = loc
     end
 end
 
-redef class TKwonce
+redef class TKwsuper
     redef fun parser_index: Int
     do
        return 41
@@ -552,12 +525,11 @@ redef class TKwonce
 
     init init_tk(loc: Location)
     do
-        _text = once "once"
                _location = loc
     end
 end
 
-redef class TKwsuper
+redef class TKwself
     redef fun parser_index: Int
     do
        return 42
@@ -565,12 +537,11 @@ redef class TKwsuper
 
     init init_tk(loc: Location)
     do
-        _text = once "super"
                _location = loc
     end
 end
 
-redef class TKwself
+redef class TKwtrue
     redef fun parser_index: Int
     do
        return 43
@@ -578,12 +549,11 @@ redef class TKwself
 
     init init_tk(loc: Location)
     do
-        _text = once "self"
                _location = loc
     end
 end
 
-redef class TKwtrue
+redef class TKwfalse
     redef fun parser_index: Int
     do
        return 44
@@ -591,12 +561,11 @@ redef class TKwtrue
 
     init init_tk(loc: Location)
     do
-        _text = once "true"
                _location = loc
     end
 end
 
-redef class TKwfalse
+redef class TKwnull
     redef fun parser_index: Int
     do
        return 45
@@ -604,12 +573,11 @@ redef class TKwfalse
 
     init init_tk(loc: Location)
     do
-        _text = once "false"
                _location = loc
     end
 end
 
-redef class TKwnull
+redef class TKwas
     redef fun parser_index: Int
     do
        return 46
@@ -617,12 +585,11 @@ redef class TKwnull
 
     init init_tk(loc: Location)
     do
-        _text = once "null"
                _location = loc
     end
 end
 
-redef class TKwas
+redef class TKwnullable
     redef fun parser_index: Int
     do
        return 47
@@ -630,12 +597,11 @@ redef class TKwas
 
     init init_tk(loc: Location)
     do
-        _text = once "as"
                _location = loc
     end
 end
 
-redef class TKwnullable
+redef class TKwisset
     redef fun parser_index: Int
     do
        return 48
@@ -643,12 +609,11 @@ redef class TKwnullable
 
     init init_tk(loc: Location)
     do
-        _text = once "nullable"
                _location = loc
     end
 end
 
-redef class TKwisset
+redef class TKwlabel
     redef fun parser_index: Int
     do
        return 49
@@ -656,12 +621,11 @@ redef class TKwisset
 
     init init_tk(loc: Location)
     do
-        _text = once "isset"
                _location = loc
     end
 end
 
-redef class TKwlabel
+redef class TKwdebug
     redef fun parser_index: Int
     do
        return 50
@@ -669,12 +633,11 @@ redef class TKwlabel
 
     init init_tk(loc: Location)
     do
-        _text = once "label"
                _location = loc
     end
 end
 
-redef class TKwdebug
+redef class TOpar
     redef fun parser_index: Int
     do
        return 51
@@ -682,12 +645,11 @@ redef class TKwdebug
 
     init init_tk(loc: Location)
     do
-        _text = once "__debug__"
                _location = loc
     end
 end
 
-redef class TOpar
+redef class TCpar
     redef fun parser_index: Int
     do
        return 52
@@ -695,12 +657,11 @@ redef class TOpar
 
     init init_tk(loc: Location)
     do
-        _text = once "("
                _location = loc
     end
 end
 
-redef class TCpar
+redef class TObra
     redef fun parser_index: Int
     do
        return 53
@@ -708,12 +669,11 @@ redef class TCpar
 
     init init_tk(loc: Location)
     do
-        _text = once ")"
                _location = loc
     end
 end
 
-redef class TObra
+redef class TCbra
     redef fun parser_index: Int
     do
        return 54
@@ -721,12 +681,11 @@ redef class TObra
 
     init init_tk(loc: Location)
     do
-        _text = once "["
                _location = loc
     end
 end
 
-redef class TCbra
+redef class TComma
     redef fun parser_index: Int
     do
        return 55
@@ -734,12 +693,11 @@ redef class TCbra
 
     init init_tk(loc: Location)
     do
-        _text = once "]"
                _location = loc
     end
 end
 
-redef class TComma
+redef class TColumn
     redef fun parser_index: Int
     do
        return 56
@@ -747,12 +705,11 @@ redef class TComma
 
     init init_tk(loc: Location)
     do
-        _text = once ","
                _location = loc
     end
 end
 
-redef class TColumn
+redef class TQuad
     redef fun parser_index: Int
     do
        return 57
@@ -760,12 +717,11 @@ redef class TColumn
 
     init init_tk(loc: Location)
     do
-        _text = once ":"
                _location = loc
     end
 end
 
-redef class TQuad
+redef class TAssign
     redef fun parser_index: Int
     do
        return 58
@@ -773,12 +729,11 @@ redef class TQuad
 
     init init_tk(loc: Location)
     do
-        _text = once "::"
                _location = loc
     end
 end
 
-redef class TAssign
+redef class TPluseq
     redef fun parser_index: Int
     do
        return 59
@@ -786,12 +741,11 @@ redef class TAssign
 
     init init_tk(loc: Location)
     do
-        _text = once "="
                _location = loc
     end
 end
 
-redef class TPluseq
+redef class TMinuseq
     redef fun parser_index: Int
     do
        return 60
@@ -799,12 +753,11 @@ redef class TPluseq
 
     init init_tk(loc: Location)
     do
-        _text = once "+="
                _location = loc
     end
 end
 
-redef class TMinuseq
+redef class TDotdotdot
     redef fun parser_index: Int
     do
        return 61
@@ -812,12 +765,11 @@ redef class TMinuseq
 
     init init_tk(loc: Location)
     do
-        _text = once "-="
                _location = loc
     end
 end
 
-redef class TDotdotdot
+redef class TDotdot
     redef fun parser_index: Int
     do
        return 62
@@ -825,12 +777,11 @@ redef class TDotdotdot
 
     init init_tk(loc: Location)
     do
-        _text = once "..."
                _location = loc
     end
 end
 
-redef class TDotdot
+redef class TDot
     redef fun parser_index: Int
     do
        return 63
@@ -838,12 +789,11 @@ redef class TDotdot
 
     init init_tk(loc: Location)
     do
-        _text = once ".."
                _location = loc
     end
 end
 
-redef class TDot
+redef class TPlus
     redef fun parser_index: Int
     do
        return 64
@@ -851,12 +801,11 @@ redef class TDot
 
     init init_tk(loc: Location)
     do
-        _text = once "."
                _location = loc
     end
 end
 
-redef class TPlus
+redef class TMinus
     redef fun parser_index: Int
     do
        return 65
@@ -864,12 +813,11 @@ redef class TPlus
 
     init init_tk(loc: Location)
     do
-        _text = once "+"
                _location = loc
     end
 end
 
-redef class TMinus
+redef class TStar
     redef fun parser_index: Int
     do
        return 66
@@ -877,12 +825,11 @@ redef class TMinus
 
     init init_tk(loc: Location)
     do
-        _text = once "-"
                _location = loc
     end
 end
 
-redef class TStar
+redef class TSlash
     redef fun parser_index: Int
     do
        return 67
@@ -890,12 +837,11 @@ redef class TStar
 
     init init_tk(loc: Location)
     do
-        _text = once "*"
                _location = loc
     end
 end
 
-redef class TSlash
+redef class TPercent
     redef fun parser_index: Int
     do
        return 68
@@ -903,12 +849,11 @@ redef class TSlash
 
     init init_tk(loc: Location)
     do
-        _text = once "/"
                _location = loc
     end
 end
 
-redef class TPercent
+redef class TEq
     redef fun parser_index: Int
     do
        return 69
@@ -916,12 +861,11 @@ redef class TPercent
 
     init init_tk(loc: Location)
     do
-        _text = once "%"
                _location = loc
     end
 end
 
-redef class TEq
+redef class TNe
     redef fun parser_index: Int
     do
        return 70
@@ -929,12 +873,11 @@ redef class TEq
 
     init init_tk(loc: Location)
     do
-        _text = once "=="
                _location = loc
     end
 end
 
-redef class TNe
+redef class TLt
     redef fun parser_index: Int
     do
        return 71
@@ -942,12 +885,11 @@ redef class TNe
 
     init init_tk(loc: Location)
     do
-        _text = once "!="
                _location = loc
     end
 end
 
-redef class TLt
+redef class TLe
     redef fun parser_index: Int
     do
        return 72
@@ -955,12 +897,11 @@ redef class TLt
 
     init init_tk(loc: Location)
     do
-        _text = once "<"
                _location = loc
     end
 end
 
-redef class TLe
+redef class TLl
     redef fun parser_index: Int
     do
        return 73
@@ -968,12 +909,11 @@ redef class TLe
 
     init init_tk(loc: Location)
     do
-        _text = once "<="
                _location = loc
     end
 end
 
-redef class TLl
+redef class TGt
     redef fun parser_index: Int
     do
        return 74
@@ -981,12 +921,11 @@ redef class TLl
 
     init init_tk(loc: Location)
     do
-        _text = once "<<"
                _location = loc
     end
 end
 
-redef class TGt
+redef class TGe
     redef fun parser_index: Int
     do
        return 75
@@ -994,12 +933,11 @@ redef class TGt
 
     init init_tk(loc: Location)
     do
-        _text = once ">"
                _location = loc
     end
 end
 
-redef class TGe
+redef class TGg
     redef fun parser_index: Int
     do
        return 76
@@ -1007,12 +945,11 @@ redef class TGe
 
     init init_tk(loc: Location)
     do
-        _text = once ">="
                _location = loc
     end
 end
 
-redef class TGg
+redef class TStarship
     redef fun parser_index: Int
     do
        return 77
@@ -1020,12 +957,11 @@ redef class TGg
 
     init init_tk(loc: Location)
     do
-        _text = once ">>"
                _location = loc
     end
 end
 
-redef class TStarship
+redef class TBang
     redef fun parser_index: Int
     do
        return 78
@@ -1033,12 +969,11 @@ redef class TStarship
 
     init init_tk(loc: Location)
     do
-        _text = once "<=>"
                _location = loc
     end
 end
 
-redef class TBang
+redef class TAt
     redef fun parser_index: Int
     do
        return 79
@@ -1046,7 +981,6 @@ redef class TBang
 
     init init_tk(loc: Location)
     do
-        _text = once "!"
                _location = loc
     end
 end
@@ -1057,9 +991,8 @@ redef class TClassid
        return 80
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
                _location = loc
     end
 end
@@ -1070,9 +1003,8 @@ redef class TId
        return 81
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
                _location = loc
     end
 end
@@ -1083,9 +1015,8 @@ redef class TAttrid
        return 82
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
                _location = loc
     end
 end
@@ -1096,9 +1027,8 @@ redef class TNumber
        return 83
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
                _location = loc
     end
 end
@@ -1109,100 +1039,104 @@ redef class TFloat
        return 84
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
                _location = loc
     end
 end
 
-redef class TChar
+redef class TString
     redef fun parser_index: Int
     do
        return 85
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
                _location = loc
     end
 end
 
-redef class TString
+redef class TStartString
     redef fun parser_index: Int
     do
        return 86
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
                _location = loc
     end
 end
 
-redef class TStartString
+redef class TMidString
     redef fun parser_index: Int
     do
        return 87
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
                _location = loc
     end
 end
 
-redef class TMidString
+redef class TEndString
     redef fun parser_index: Int
     do
        return 88
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
                _location = loc
     end
 end
 
-redef class TEndString
+redef class TChar
     redef fun parser_index: Int
     do
        return 89
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
                _location = loc
     end
 end
 
-redef class TBadChar
+redef class TBadString
     redef fun parser_index: Int
     do
        return 90
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
                _location = loc
     end
 end
 
-redef class TBadString
+redef class TBadChar
     redef fun parser_index: Int
     do
        return 91
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
+    do
+               _location = loc
+    end
+end
+
+redef class TExternCodeSegment
+    redef fun parser_index: Int
+    do
+       return 92
+    end
+
+    init init_tk(loc: Location)
     do
-        _text = text
                _location = loc
     end
 end
@@ -1211,10 +1145,10 @@ end
 redef class EOF
     redef fun parser_index: Int
     do
-       return 92
+       return 93
     end
 
-    init(loc: Location)
+    init init_tk(loc: Location)
     do
         _text = ""
                _location = loc
@@ -1226,11 +1160,30 @@ redef class AError
 
     init init_error(message: String, loc: Location)
     do
-               init(loc)
+               init_tk(loc)
                _message = message
     end
 end
 
+redef class ALexerError
+    readable var _string: String
+
+    init init_lexer_error(message: String, loc: Location, string: String)
+    do
+               init_error(message, loc)
+               _string = string
+    end
+end
+
+redef class AParserError
+    readable var _token: Token
+
+    init init_parser_error(message: String, loc: Location, token: Token)
+    do
+               init_error(message, loc)
+               _token = token
+    end
+end
 
 # The lexer extract NIT tokens from an input stream.
 # It is better user with the Parser
@@ -1386,16 +1339,13 @@ class Lexer
                                                return null
                                        end
                                        if accept_token == 1 then
-                                               var token_text = string.substring(start_stream_pos, accept_length)
-                                               return new TEol.init_tk(token_text, location)
+                                               return new TEol.init_tk(location)
                                        end
                                        if accept_token == 2 then
-                                               var token_text = string.substring(start_stream_pos, accept_length)
-                                               return new TComment.init_tk(token_text, location)
+                                               return new TComment.init_tk(location)
                                        end
                                        if accept_token == 3 then
-                                               var token_text = string.substring(start_stream_pos, accept_length)
-                                               return new TKwmodule.init_tk(token_text, location)
+                                               return new TKwmodule.init_tk(location)
                                        end
                                        if accept_token == 4 then
                                                return new TKwimport.init_tk(location)
@@ -1410,282 +1360,272 @@ class Lexer
                                                return new TKwinterface.init_tk(location)
                                        end
                                        if accept_token == 8 then
-                                               var token_text = string.substring(start_stream_pos, accept_length)
-                                               return new TKwenum.init_tk(token_text, location)
+                                               return new TKwenum.init_tk(location)
                                        end
                                        if accept_token == 9 then
-                                               return new TKwspecial.init_tk(location)
+                                               return new TKwend.init_tk(location)
                                        end
                                        if accept_token == 10 then
-                                               return new TKwend.init_tk(location)
+                                               return new TKwmeth.init_tk(location)
                                        end
                                        if accept_token == 11 then
-                                               return new TKwmeth.init_tk(location)
+                                               return new TKwtype.init_tk(location)
                                        end
                                        if accept_token == 12 then
-                                               return new TKwtype.init_tk(location)
+                                               return new TKwinit.init_tk(location)
                                        end
                                        if accept_token == 13 then
-                                               return new TKwinit.init_tk(location)
+                                               return new TKwredef.init_tk(location)
                                        end
                                        if accept_token == 14 then
-                                               return new TKwredef.init_tk(location)
+                                               return new TKwis.init_tk(location)
                                        end
                                        if accept_token == 15 then
-                                               return new TKwis.init_tk(location)
+                                               return new TKwdo.init_tk(location)
                                        end
                                        if accept_token == 16 then
-                                               return new TKwdo.init_tk(location)
+                                               return new TKwreadable.init_tk(location)
                                        end
                                        if accept_token == 17 then
-                                               return new TKwreadable.init_tk(location)
+                                               return new TKwwritable.init_tk(location)
                                        end
                                        if accept_token == 18 then
-                                               return new TKwwritable.init_tk(location)
+                                               return new TKwvar.init_tk(location)
                                        end
                                        if accept_token == 19 then
-                                               return new TKwvar.init_tk(location)
+                                               return new TKwintern.init_tk(location)
                                        end
                                        if accept_token == 20 then
-                                               return new TKwintern.init_tk(location)
+                                               return new TKwextern.init_tk(location)
                                        end
                                        if accept_token == 21 then
-                                               return new TKwextern.init_tk(location)
+                                               return new TKwprotected.init_tk(location)
                                        end
                                        if accept_token == 22 then
-                                               return new TKwprotected.init_tk(location)
+                                               return new TKwprivate.init_tk(location)
                                        end
                                        if accept_token == 23 then
-                                               return new TKwprivate.init_tk(location)
+                                               return new TKwintrude.init_tk(location)
                                        end
                                        if accept_token == 24 then
-                                               return new TKwintrude.init_tk(location)
+                                               return new TKwif.init_tk(location)
                                        end
                                        if accept_token == 25 then
-                                               return new TKwif.init_tk(location)
+                                               return new TKwthen.init_tk(location)
                                        end
                                        if accept_token == 26 then
-                                               return new TKwthen.init_tk(location)
+                                               return new TKwelse.init_tk(location)
                                        end
                                        if accept_token == 27 then
-                                               return new TKwelse.init_tk(location)
+                                               return new TKwwhile.init_tk(location)
                                        end
                                        if accept_token == 28 then
-                                               return new TKwwhile.init_tk(location)
+                                               return new TKwloop.init_tk(location)
                                        end
                                        if accept_token == 29 then
-                                               return new TKwloop.init_tk(location)
+                                               return new TKwfor.init_tk(location)
                                        end
                                        if accept_token == 30 then
-                                               return new TKwfor.init_tk(location)
+                                               return new TKwin.init_tk(location)
                                        end
                                        if accept_token == 31 then
-                                               return new TKwin.init_tk(location)
+                                               return new TKwand.init_tk(location)
                                        end
                                        if accept_token == 32 then
-                                               return new TKwand.init_tk(location)
+                                               return new TKwor.init_tk(location)
                                        end
                                        if accept_token == 33 then
-                                               return new TKwor.init_tk(location)
+                                               return new TKwnot.init_tk(location)
                                        end
                                        if accept_token == 34 then
-                                               return new TKwnot.init_tk(location)
+                                               return new TKwreturn.init_tk(location)
                                        end
                                        if accept_token == 35 then
-                                               return new TKwreturn.init_tk(location)
+                                               return new TKwcontinue.init_tk(location)
                                        end
                                        if accept_token == 36 then
-                                               return new TKwcontinue.init_tk(location)
+                                               return new TKwbreak.init_tk(location)
                                        end
                                        if accept_token == 37 then
-                                               return new TKwbreak.init_tk(location)
+                                               return new TKwabort.init_tk(location)
                                        end
                                        if accept_token == 38 then
-                                               return new TKwabort.init_tk(location)
+                                               return new TKwassert.init_tk(location)
                                        end
                                        if accept_token == 39 then
-                                               return new TKwassert.init_tk(location)
+                                               return new TKwnew.init_tk(location)
                                        end
                                        if accept_token == 40 then
-                                               return new TKwnew.init_tk(location)
+                                               return new TKwisa.init_tk(location)
                                        end
                                        if accept_token == 41 then
-                                               return new TKwisa.init_tk(location)
+                                               return new TKwonce.init_tk(location)
                                        end
                                        if accept_token == 42 then
-                                               return new TKwonce.init_tk(location)
+                                               return new TKwsuper.init_tk(location)
                                        end
                                        if accept_token == 43 then
-                                               return new TKwsuper.init_tk(location)
+                                               return new TKwself.init_tk(location)
                                        end
                                        if accept_token == 44 then
-                                               return new TKwself.init_tk(location)
+                                               return new TKwtrue.init_tk(location)
                                        end
                                        if accept_token == 45 then
-                                               return new TKwtrue.init_tk(location)
+                                               return new TKwfalse.init_tk(location)
                                        end
                                        if accept_token == 46 then
-                                               return new TKwfalse.init_tk(location)
+                                               return new TKwnull.init_tk(location)
                                        end
                                        if accept_token == 47 then
-                                               return new TKwnull.init_tk(location)
+                                               return new TKwas.init_tk(location)
                                        end
                                        if accept_token == 48 then
-                                               return new TKwas.init_tk(location)
+                                               return new TKwnullable.init_tk(location)
                                        end
                                        if accept_token == 49 then
-                                               return new TKwnullable.init_tk(location)
+                                               return new TKwisset.init_tk(location)
                                        end
                                        if accept_token == 50 then
-                                               return new TKwisset.init_tk(location)
+                                               return new TKwlabel.init_tk(location)
                                        end
                                        if accept_token == 51 then
-                                               return new TKwlabel.init_tk(location)
+                                               return new TKwdebug.init_tk(location)
                                        end
                                        if accept_token == 52 then
-                                               return new TKwdebug.init_tk(location)
+                                               return new TOpar.init_tk(location)
                                        end
                                        if accept_token == 53 then
-                                               return new TOpar.init_tk(location)
+                                               return new TCpar.init_tk(location)
                                        end
                                        if accept_token == 54 then
-                                               return new TCpar.init_tk(location)
+                                               return new TObra.init_tk(location)
                                        end
                                        if accept_token == 55 then
-                                               return new TObra.init_tk(location)
+                                               return new TCbra.init_tk(location)
                                        end
                                        if accept_token == 56 then
-                                               return new TCbra.init_tk(location)
+                                               return new TComma.init_tk(location)
                                        end
                                        if accept_token == 57 then
-                                               return new TComma.init_tk(location)
+                                               return new TColumn.init_tk(location)
                                        end
                                        if accept_token == 58 then
-                                               return new TColumn.init_tk(location)
+                                               return new TQuad.init_tk(location)
                                        end
                                        if accept_token == 59 then
-                                               return new TQuad.init_tk(location)
+                                               return new TAssign.init_tk(location)
                                        end
                                        if accept_token == 60 then
-                                               return new TAssign.init_tk(location)
+                                               return new TPluseq.init_tk(location)
                                        end
                                        if accept_token == 61 then
-                                               return new TPluseq.init_tk(location)
+                                               return new TMinuseq.init_tk(location)
                                        end
                                        if accept_token == 62 then
-                                               return new TMinuseq.init_tk(location)
+                                               return new TDotdotdot.init_tk(location)
                                        end
                                        if accept_token == 63 then
-                                               return new TDotdotdot.init_tk(location)
+                                               return new TDotdot.init_tk(location)
                                        end
                                        if accept_token == 64 then
-                                               return new TDotdot.init_tk(location)
+                                               return new TDot.init_tk(location)
                                        end
                                        if accept_token == 65 then
-                                               return new TDot.init_tk(location)
+                                               return new TPlus.init_tk(location)
                                        end
                                        if accept_token == 66 then
-                                               return new TPlus.init_tk(location)
+                                               return new TMinus.init_tk(location)
                                        end
                                        if accept_token == 67 then
-                                               return new TMinus.init_tk(location)
+                                               return new TStar.init_tk(location)
                                        end
                                        if accept_token == 68 then
-                                               return new TStar.init_tk(location)
+                                               return new TSlash.init_tk(location)
                                        end
                                        if accept_token == 69 then
-                                               return new TSlash.init_tk(location)
+                                               return new TPercent.init_tk(location)
                                        end
                                        if accept_token == 70 then
-                                               return new TPercent.init_tk(location)
+                                               return new TEq.init_tk(location)
                                        end
                                        if accept_token == 71 then
-                                               return new TEq.init_tk(location)
+                                               return new TNe.init_tk(location)
                                        end
                                        if accept_token == 72 then
-                                               return new TNe.init_tk(location)
+                                               return new TLt.init_tk(location)
                                        end
                                        if accept_token == 73 then
-                                               return new TLt.init_tk(location)
+                                               return new TLe.init_tk(location)
                                        end
                                        if accept_token == 74 then
-                                               return new TLe.init_tk(location)
+                                               return new TLl.init_tk(location)
                                        end
                                        if accept_token == 75 then
-                                               return new TLl.init_tk(location)
+                                               return new TGt.init_tk(location)
                                        end
                                        if accept_token == 76 then
-                                               return new TGt.init_tk(location)
+                                               return new TGe.init_tk(location)
                                        end
                                        if accept_token == 77 then
-                                               return new TGe.init_tk(location)
+                                               return new TGg.init_tk(location)
                                        end
                                        if accept_token == 78 then
-                                               return new TGg.init_tk(location)
+                                               return new TStarship.init_tk(location)
                                        end
                                        if accept_token == 79 then
-                                               return new TStarship.init_tk(location)
+                                               return new TBang.init_tk(location)
                                        end
                                        if accept_token == 80 then
-                                               return new TBang.init_tk(location)
+                                               return new TAt.init_tk(location)
                                        end
                                        if accept_token == 81 then
-                                               var token_text = string.substring(start_stream_pos, accept_length)
-                                               return new TClassid.init_tk(token_text, location)
+                                               return new TClassid.init_tk(location)
                                        end
                                        if accept_token == 82 then
-                                               var token_text = string.substring(start_stream_pos, accept_length)
-                                               return new TId.init_tk(token_text, location)
+                                               return new TId.init_tk(location)
                                        end
                                        if accept_token == 83 then
-                                               var token_text = string.substring(start_stream_pos, accept_length)
-                                               return new TAttrid.init_tk(token_text, location)
+                                               return new TAttrid.init_tk(location)
                                        end
                                        if accept_token == 84 then
-                                               var token_text = string.substring(start_stream_pos, accept_length)
-                                               return new TNumber.init_tk(token_text, location)
+                                               return new TNumber.init_tk(location)
                                        end
                                        if accept_token == 85 then
-                                               var token_text = string.substring(start_stream_pos, accept_length)
-                                               return new TFloat.init_tk(token_text, location)
+                                               return new TFloat.init_tk(location)
                                        end
                                        if accept_token == 86 then
-                                               var token_text = string.substring(start_stream_pos, accept_length)
-                                               return new TChar.init_tk(token_text, location)
+                                               return new TString.init_tk(location)
                                        end
                                        if accept_token == 87 then
-                                               var token_text = string.substring(start_stream_pos, accept_length)
-                                               return new TString.init_tk(token_text, location)
+                                               return new TStartString.init_tk(location)
                                        end
                                        if accept_token == 88 then
-                                               var token_text = string.substring(start_stream_pos, accept_length)
-                                               return new TStartString.init_tk(token_text, location)
+                                               return new TMidString.init_tk(location)
                                        end
                                        if accept_token == 89 then
-                                               var token_text = string.substring(start_stream_pos, accept_length)
-                                               return new TMidString.init_tk(token_text, location)
+                                               return new TEndString.init_tk(location)
                                        end
                                        if accept_token == 90 then
-                                               var token_text = string.substring(start_stream_pos, accept_length)
-                                               return new TEndString.init_tk(token_text, location)
+                                               return new TChar.init_tk(location)
                                        end
                                        if accept_token == 91 then
-                                               var token_text = string.substring(start_stream_pos, accept_length)
-                                               return new TBadChar.init_tk(token_text, location)
+                                               return new TBadString.init_tk(location)
                                        end
                                        if accept_token == 92 then
-                                               var token_text = string.substring(start_stream_pos, accept_length)
-                                               return new TBadString.init_tk(token_text, location)
+                                               return new TBadChar.init_tk(location)
+                                       end
+                                       if accept_token == 93 then
+                                               return new TExternCodeSegment.init_tk(location)
                                        end
                                else
                                        _stream_pos = sp
                                        var location = new Location(_file, start_line + 1, start_line + 1, start_pos + 1, start_pos + 1)
                                        if sp > start_stream_pos then
                                                var text = string.substring(start_stream_pos, sp-start_stream_pos)
-                                               var token = new AError.init_error("Syntax error: unknown token {text}.", location)
+                                               var token = new ALexerError.init_lexer_error("Syntax error: unknown token {text}.", location, text)
                                                return token
                                        else
-                                               var token = new EOF(location)
+                                               var token = new EOF.init_tk(location)
                                                return token
                                        end
                                end