tests: update sav/nitdoc
[nit.git] / src / parser / lexer.nit
index 3f2b3dc..3c975a6 100644 (file)
@@ -6,7 +6,16 @@ intrude import parser_nodes
 private import tables
 
 redef class Token
-    redef readable var _text: String
+    var _text: nullable String
+
+    redef fun text
+    do
+        var res = _text
+        if res != null then return res
+        res = location.text
+       _text = res
+       return res
+    end
 
     fun parser_index: Int is abstract
 end
@@ -17,9 +26,8 @@ redef class TEol
        return 0
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
                _location = loc
     end
 end
@@ -30,9 +38,8 @@ redef class TComment
        return 1
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
                _location = loc
     end
 end
@@ -43,9 +50,8 @@ redef class TKwmodule
        return 2
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
                _location = loc
     end
 end
@@ -58,7 +64,6 @@ redef class TKwimport
 
     init init_tk(loc: Location)
     do
-        _text = once "import"
                _location = loc
     end
 end
@@ -71,7 +76,6 @@ redef class TKwclass
 
     init init_tk(loc: Location)
     do
-        _text = once "class"
                _location = loc
     end
 end
@@ -84,7 +88,6 @@ redef class TKwabstract
 
     init init_tk(loc: Location)
     do
-        _text = once "abstract"
                _location = loc
     end
 end
@@ -97,7 +100,6 @@ redef class TKwinterface
 
     init init_tk(loc: Location)
     do
-        _text = once "interface"
                _location = loc
     end
 end
@@ -108,14 +110,13 @@ redef class TKwenum
        return 7
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
                _location = loc
     end
 end
 
-redef class TKwspecial
+redef class TKwend
     redef fun parser_index: Int
     do
        return 8
@@ -123,12 +124,11 @@ redef class TKwspecial
 
     init init_tk(loc: Location)
     do
-        _text = once "special"
                _location = loc
     end
 end
 
-redef class TKwend
+redef class TKwmeth
     redef fun parser_index: Int
     do
        return 9
@@ -136,12 +136,11 @@ redef class TKwend
 
     init init_tk(loc: Location)
     do
-        _text = once "end"
                _location = loc
     end
 end
 
-redef class TKwmeth
+redef class TKwtype
     redef fun parser_index: Int
     do
        return 10
@@ -149,12 +148,11 @@ redef class TKwmeth
 
     init init_tk(loc: Location)
     do
-        _text = once "fun"
                _location = loc
     end
 end
 
-redef class TKwtype
+redef class TKwinit
     redef fun parser_index: Int
     do
        return 11
@@ -162,12 +160,11 @@ redef class TKwtype
 
     init init_tk(loc: Location)
     do
-        _text = once "type"
                _location = loc
     end
 end
 
-redef class TKwinit
+redef class TKwredef
     redef fun parser_index: Int
     do
        return 12
@@ -175,12 +172,11 @@ redef class TKwinit
 
     init init_tk(loc: Location)
     do
-        _text = once "init"
                _location = loc
     end
 end
 
-redef class TKwredef
+redef class TKwis
     redef fun parser_index: Int
     do
        return 13
@@ -188,12 +184,11 @@ redef class TKwredef
 
     init init_tk(loc: Location)
     do
-        _text = once "redef"
                _location = loc
     end
 end
 
-redef class TKwis
+redef class TKwdo
     redef fun parser_index: Int
     do
        return 14
@@ -201,12 +196,11 @@ redef class TKwis
 
     init init_tk(loc: Location)
     do
-        _text = once "is"
                _location = loc
     end
 end
 
-redef class TKwdo
+redef class TKwreadable
     redef fun parser_index: Int
     do
        return 15
@@ -214,12 +208,11 @@ redef class TKwdo
 
     init init_tk(loc: Location)
     do
-        _text = once "do"
                _location = loc
     end
 end
 
-redef class TKwreadable
+redef class TKwwritable
     redef fun parser_index: Int
     do
        return 16
@@ -227,12 +220,11 @@ redef class TKwreadable
 
     init init_tk(loc: Location)
     do
-        _text = once "readable"
                _location = loc
     end
 end
 
-redef class TKwwritable
+redef class TKwvar
     redef fun parser_index: Int
     do
        return 17
@@ -240,12 +232,11 @@ redef class TKwwritable
 
     init init_tk(loc: Location)
     do
-        _text = once "writable"
                _location = loc
     end
 end
 
-redef class TKwvar
+redef class TKwintern
     redef fun parser_index: Int
     do
        return 18
@@ -253,12 +244,11 @@ redef class TKwvar
 
     init init_tk(loc: Location)
     do
-        _text = once "var"
                _location = loc
     end
 end
 
-redef class TKwintern
+redef class TKwextern
     redef fun parser_index: Int
     do
        return 19
@@ -266,12 +256,11 @@ redef class TKwintern
 
     init init_tk(loc: Location)
     do
-        _text = once "intern"
                _location = loc
     end
 end
 
-redef class TKwextern
+redef class TKwprotected
     redef fun parser_index: Int
     do
        return 20
@@ -279,12 +268,11 @@ redef class TKwextern
 
     init init_tk(loc: Location)
     do
-        _text = once "extern"
                _location = loc
     end
 end
 
-redef class TKwprotected
+redef class TKwprivate
     redef fun parser_index: Int
     do
        return 21
@@ -292,12 +280,11 @@ redef class TKwprotected
 
     init init_tk(loc: Location)
     do
-        _text = once "protected"
                _location = loc
     end
 end
 
-redef class TKwprivate
+redef class TKwintrude
     redef fun parser_index: Int
     do
        return 22
@@ -305,12 +292,11 @@ redef class TKwprivate
 
     init init_tk(loc: Location)
     do
-        _text = once "private"
                _location = loc
     end
 end
 
-redef class TKwintrude
+redef class TKwif
     redef fun parser_index: Int
     do
        return 23
@@ -318,12 +304,11 @@ redef class TKwintrude
 
     init init_tk(loc: Location)
     do
-        _text = once "intrude"
                _location = loc
     end
 end
 
-redef class TKwif
+redef class TKwthen
     redef fun parser_index: Int
     do
        return 24
@@ -331,12 +316,11 @@ redef class TKwif
 
     init init_tk(loc: Location)
     do
-        _text = once "if"
                _location = loc
     end
 end
 
-redef class TKwthen
+redef class TKwelse
     redef fun parser_index: Int
     do
        return 25
@@ -344,12 +328,11 @@ redef class TKwthen
 
     init init_tk(loc: Location)
     do
-        _text = once "then"
                _location = loc
     end
 end
 
-redef class TKwelse
+redef class TKwwhile
     redef fun parser_index: Int
     do
        return 26
@@ -357,12 +340,11 @@ redef class TKwelse
 
     init init_tk(loc: Location)
     do
-        _text = once "else"
                _location = loc
     end
 end
 
-redef class TKwwhile
+redef class TKwloop
     redef fun parser_index: Int
     do
        return 27
@@ -370,12 +352,11 @@ redef class TKwwhile
 
     init init_tk(loc: Location)
     do
-        _text = once "while"
                _location = loc
     end
 end
 
-redef class TKwloop
+redef class TKwfor
     redef fun parser_index: Int
     do
        return 28
@@ -383,12 +364,11 @@ redef class TKwloop
 
     init init_tk(loc: Location)
     do
-        _text = once "loop"
                _location = loc
     end
 end
 
-redef class TKwfor
+redef class TKwin
     redef fun parser_index: Int
     do
        return 29
@@ -396,12 +376,11 @@ redef class TKwfor
 
     init init_tk(loc: Location)
     do
-        _text = once "for"
                _location = loc
     end
 end
 
-redef class TKwin
+redef class TKwand
     redef fun parser_index: Int
     do
        return 30
@@ -409,12 +388,11 @@ redef class TKwin
 
     init init_tk(loc: Location)
     do
-        _text = once "in"
                _location = loc
     end
 end
 
-redef class TKwand
+redef class TKwor
     redef fun parser_index: Int
     do
        return 31
@@ -422,12 +400,11 @@ redef class TKwand
 
     init init_tk(loc: Location)
     do
-        _text = once "and"
                _location = loc
     end
 end
 
-redef class TKwor
+redef class TKwnot
     redef fun parser_index: Int
     do
        return 32
@@ -435,12 +412,11 @@ redef class TKwor
 
     init init_tk(loc: Location)
     do
-        _text = once "or"
                _location = loc
     end
 end
 
-redef class TKwnot
+redef class TKwreturn
     redef fun parser_index: Int
     do
        return 33
@@ -448,12 +424,11 @@ redef class TKwnot
 
     init init_tk(loc: Location)
     do
-        _text = once "not"
                _location = loc
     end
 end
 
-redef class TKwreturn
+redef class TKwcontinue
     redef fun parser_index: Int
     do
        return 34
@@ -461,12 +436,11 @@ redef class TKwreturn
 
     init init_tk(loc: Location)
     do
-        _text = once "return"
                _location = loc
     end
 end
 
-redef class TKwcontinue
+redef class TKwbreak
     redef fun parser_index: Int
     do
        return 35
@@ -474,12 +448,11 @@ redef class TKwcontinue
 
     init init_tk(loc: Location)
     do
-        _text = once "continue"
                _location = loc
     end
 end
 
-redef class TKwbreak
+redef class TKwabort
     redef fun parser_index: Int
     do
        return 36
@@ -487,12 +460,11 @@ redef class TKwbreak
 
     init init_tk(loc: Location)
     do
-        _text = once "break"
                _location = loc
     end
 end
 
-redef class TKwabort
+redef class TKwassert
     redef fun parser_index: Int
     do
        return 37
@@ -500,12 +472,11 @@ redef class TKwabort
 
     init init_tk(loc: Location)
     do
-        _text = once "abort"
                _location = loc
     end
 end
 
-redef class TKwassert
+redef class TKwnew
     redef fun parser_index: Int
     do
        return 38
@@ -513,12 +484,11 @@ redef class TKwassert
 
     init init_tk(loc: Location)
     do
-        _text = once "assert"
                _location = loc
     end
 end
 
-redef class TKwnew
+redef class TKwisa
     redef fun parser_index: Int
     do
        return 39
@@ -526,12 +496,11 @@ redef class TKwnew
 
     init init_tk(loc: Location)
     do
-        _text = once "new"
                _location = loc
     end
 end
 
-redef class TKwisa
+redef class TKwonce
     redef fun parser_index: Int
     do
        return 40
@@ -539,12 +508,11 @@ redef class TKwisa
 
     init init_tk(loc: Location)
     do
-        _text = once "isa"
                _location = loc
     end
 end
 
-redef class TKwonce
+redef class TKwsuper
     redef fun parser_index: Int
     do
        return 41
@@ -552,12 +520,11 @@ redef class TKwonce
 
     init init_tk(loc: Location)
     do
-        _text = once "once"
                _location = loc
     end
 end
 
-redef class TKwsuper
+redef class TKwself
     redef fun parser_index: Int
     do
        return 42
@@ -565,12 +532,11 @@ redef class TKwsuper
 
     init init_tk(loc: Location)
     do
-        _text = once "super"
                _location = loc
     end
 end
 
-redef class TKwself
+redef class TKwtrue
     redef fun parser_index: Int
     do
        return 43
@@ -578,12 +544,11 @@ redef class TKwself
 
     init init_tk(loc: Location)
     do
-        _text = once "self"
                _location = loc
     end
 end
 
-redef class TKwtrue
+redef class TKwfalse
     redef fun parser_index: Int
     do
        return 44
@@ -591,12 +556,11 @@ redef class TKwtrue
 
     init init_tk(loc: Location)
     do
-        _text = once "true"
                _location = loc
     end
 end
 
-redef class TKwfalse
+redef class TKwnull
     redef fun parser_index: Int
     do
        return 45
@@ -604,12 +568,11 @@ redef class TKwfalse
 
     init init_tk(loc: Location)
     do
-        _text = once "false"
                _location = loc
     end
 end
 
-redef class TKwnull
+redef class TKwas
     redef fun parser_index: Int
     do
        return 46
@@ -617,12 +580,11 @@ redef class TKwnull
 
     init init_tk(loc: Location)
     do
-        _text = once "null"
                _location = loc
     end
 end
 
-redef class TKwas
+redef class TKwnullable
     redef fun parser_index: Int
     do
        return 47
@@ -630,12 +592,11 @@ redef class TKwas
 
     init init_tk(loc: Location)
     do
-        _text = once "as"
                _location = loc
     end
 end
 
-redef class TKwnullable
+redef class TKwisset
     redef fun parser_index: Int
     do
        return 48
@@ -643,12 +604,11 @@ redef class TKwnullable
 
     init init_tk(loc: Location)
     do
-        _text = once "nullable"
                _location = loc
     end
 end
 
-redef class TKwisset
+redef class TKwlabel
     redef fun parser_index: Int
     do
        return 49
@@ -656,12 +616,11 @@ redef class TKwisset
 
     init init_tk(loc: Location)
     do
-        _text = once "isset"
                _location = loc
     end
 end
 
-redef class TKwlabel
+redef class TKwdebug
     redef fun parser_index: Int
     do
        return 50
@@ -669,7 +628,6 @@ redef class TKwlabel
 
     init init_tk(loc: Location)
     do
-        _text = once "label"
                _location = loc
     end
 end
@@ -682,7 +640,6 @@ redef class TOpar
 
     init init_tk(loc: Location)
     do
-        _text = once "("
                _location = loc
     end
 end
@@ -695,7 +652,6 @@ redef class TCpar
 
     init init_tk(loc: Location)
     do
-        _text = once ")"
                _location = loc
     end
 end
@@ -708,7 +664,6 @@ redef class TObra
 
     init init_tk(loc: Location)
     do
-        _text = once "["
                _location = loc
     end
 end
@@ -721,7 +676,6 @@ redef class TCbra
 
     init init_tk(loc: Location)
     do
-        _text = once "]"
                _location = loc
     end
 end
@@ -734,7 +688,6 @@ redef class TComma
 
     init init_tk(loc: Location)
     do
-        _text = once ","
                _location = loc
     end
 end
@@ -747,7 +700,6 @@ redef class TColumn
 
     init init_tk(loc: Location)
     do
-        _text = once ":"
                _location = loc
     end
 end
@@ -760,7 +712,6 @@ redef class TQuad
 
     init init_tk(loc: Location)
     do
-        _text = once "::"
                _location = loc
     end
 end
@@ -773,7 +724,6 @@ redef class TAssign
 
     init init_tk(loc: Location)
     do
-        _text = once "="
                _location = loc
     end
 end
@@ -786,7 +736,6 @@ redef class TPluseq
 
     init init_tk(loc: Location)
     do
-        _text = once "+="
                _location = loc
     end
 end
@@ -799,7 +748,6 @@ redef class TMinuseq
 
     init init_tk(loc: Location)
     do
-        _text = once "-="
                _location = loc
     end
 end
@@ -812,7 +760,6 @@ redef class TDotdotdot
 
     init init_tk(loc: Location)
     do
-        _text = once "..."
                _location = loc
     end
 end
@@ -825,7 +772,6 @@ redef class TDotdot
 
     init init_tk(loc: Location)
     do
-        _text = once ".."
                _location = loc
     end
 end
@@ -838,7 +784,6 @@ redef class TDot
 
     init init_tk(loc: Location)
     do
-        _text = once "."
                _location = loc
     end
 end
@@ -851,7 +796,6 @@ redef class TPlus
 
     init init_tk(loc: Location)
     do
-        _text = once "+"
                _location = loc
     end
 end
@@ -864,7 +808,6 @@ redef class TMinus
 
     init init_tk(loc: Location)
     do
-        _text = once "-"
                _location = loc
     end
 end
@@ -877,7 +820,6 @@ redef class TStar
 
     init init_tk(loc: Location)
     do
-        _text = once "*"
                _location = loc
     end
 end
@@ -890,7 +832,6 @@ redef class TSlash
 
     init init_tk(loc: Location)
     do
-        _text = once "/"
                _location = loc
     end
 end
@@ -903,7 +844,6 @@ redef class TPercent
 
     init init_tk(loc: Location)
     do
-        _text = once "%"
                _location = loc
     end
 end
@@ -916,7 +856,6 @@ redef class TEq
 
     init init_tk(loc: Location)
     do
-        _text = once "=="
                _location = loc
     end
 end
@@ -929,7 +868,6 @@ redef class TNe
 
     init init_tk(loc: Location)
     do
-        _text = once "!="
                _location = loc
     end
 end
@@ -942,7 +880,6 @@ redef class TLt
 
     init init_tk(loc: Location)
     do
-        _text = once "<"
                _location = loc
     end
 end
@@ -955,7 +892,6 @@ redef class TLe
 
     init init_tk(loc: Location)
     do
-        _text = once "<="
                _location = loc
     end
 end
@@ -968,7 +904,6 @@ redef class TLl
 
     init init_tk(loc: Location)
     do
-        _text = once "<<"
                _location = loc
     end
 end
@@ -981,7 +916,6 @@ redef class TGt
 
     init init_tk(loc: Location)
     do
-        _text = once ">"
                _location = loc
     end
 end
@@ -994,7 +928,6 @@ redef class TGe
 
     init init_tk(loc: Location)
     do
-        _text = once ">="
                _location = loc
     end
 end
@@ -1007,7 +940,6 @@ redef class TGg
 
     init init_tk(loc: Location)
     do
-        _text = once ">>"
                _location = loc
     end
 end
@@ -1020,7 +952,6 @@ redef class TStarship
 
     init init_tk(loc: Location)
     do
-        _text = once "<=>"
                _location = loc
     end
 end
@@ -1033,7 +964,6 @@ redef class TBang
 
     init init_tk(loc: Location)
     do
-        _text = once "!"
                _location = loc
     end
 end
@@ -1044,9 +974,8 @@ redef class TClassid
        return 79
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
                _location = loc
     end
 end
@@ -1057,9 +986,8 @@ redef class TId
        return 80
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
                _location = loc
     end
 end
@@ -1070,9 +998,8 @@ redef class TAttrid
        return 81
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
                _location = loc
     end
 end
@@ -1083,9 +1010,8 @@ redef class TNumber
        return 82
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
                _location = loc
     end
 end
@@ -1096,9 +1022,8 @@ redef class TFloat
        return 83
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
                _location = loc
     end
 end
@@ -1109,9 +1034,8 @@ redef class TChar
        return 84
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
                _location = loc
     end
 end
@@ -1122,9 +1046,8 @@ redef class TString
        return 85
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
                _location = loc
     end
 end
@@ -1135,9 +1058,8 @@ redef class TStartString
        return 86
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
                _location = loc
     end
 end
@@ -1148,9 +1070,8 @@ redef class TMidString
        return 87
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
                _location = loc
     end
 end
@@ -1161,9 +1082,44 @@ redef class TEndString
        return 88
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
+    do
+               _location = loc
+    end
+end
+
+redef class TBadChar
+    redef fun parser_index: Int
+    do
+       return 89
+    end
+
+    init init_tk(loc: Location)
+    do
+               _location = loc
+    end
+end
+
+redef class TBadString
+    redef fun parser_index: Int
+    do
+       return 90
+    end
+
+    init init_tk(loc: Location)
+    do
+               _location = loc
+    end
+end
+
+redef class TExternCodeSegment
+    redef fun parser_index: Int
+    do
+       return 91
+    end
+
+    init init_tk(loc: Location)
     do
-        _text = text
                _location = loc
     end
 end
@@ -1172,7 +1128,7 @@ end
 redef class EOF
     redef fun parser_index: Int
     do
-       return 89
+       return 92
     end
 
     init(loc: Location)
@@ -1347,16 +1303,13 @@ class Lexer
                                                return null
                                        end
                                        if accept_token == 1 then
-                                               var token_text = string.substring(start_stream_pos, accept_length)
-                                               return new TEol.init_tk(token_text, location)
+                                               return new TEol.init_tk(location)
                                        end
                                        if accept_token == 2 then
-                                               var token_text = string.substring(start_stream_pos, accept_length)
-                                               return new TComment.init_tk(token_text, location)
+                                               return new TComment.init_tk(location)
                                        end
                                        if accept_token == 3 then
-                                               var token_text = string.substring(start_stream_pos, accept_length)
-                                               return new TKwmodule.init_tk(token_text, location)
+                                               return new TKwmodule.init_tk(location)
                                        end
                                        if accept_token == 4 then
                                                return new TKwimport.init_tk(location)
@@ -1371,137 +1324,136 @@ class Lexer
                                                return new TKwinterface.init_tk(location)
                                        end
                                        if accept_token == 8 then
-                                               var token_text = string.substring(start_stream_pos, accept_length)
-                                               return new TKwenum.init_tk(token_text, location)
+                                               return new TKwenum.init_tk(location)
                                        end
                                        if accept_token == 9 then
-                                               return new TKwspecial.init_tk(location)
+                                               return new TKwend.init_tk(location)
                                        end
                                        if accept_token == 10 then
-                                               return new TKwend.init_tk(location)
+                                               return new TKwmeth.init_tk(location)
                                        end
                                        if accept_token == 11 then
-                                               return new TKwmeth.init_tk(location)
+                                               return new TKwtype.init_tk(location)
                                        end
                                        if accept_token == 12 then
-                                               return new TKwtype.init_tk(location)
+                                               return new TKwinit.init_tk(location)
                                        end
                                        if accept_token == 13 then
-                                               return new TKwinit.init_tk(location)
+                                               return new TKwredef.init_tk(location)
                                        end
                                        if accept_token == 14 then
-                                               return new TKwredef.init_tk(location)
+                                               return new TKwis.init_tk(location)
                                        end
                                        if accept_token == 15 then
-                                               return new TKwis.init_tk(location)
+                                               return new TKwdo.init_tk(location)
                                        end
                                        if accept_token == 16 then
-                                               return new TKwdo.init_tk(location)
+                                               return new TKwreadable.init_tk(location)
                                        end
                                        if accept_token == 17 then
-                                               return new TKwreadable.init_tk(location)
+                                               return new TKwwritable.init_tk(location)
                                        end
                                        if accept_token == 18 then
-                                               return new TKwwritable.init_tk(location)
+                                               return new TKwvar.init_tk(location)
                                        end
                                        if accept_token == 19 then
-                                               return new TKwvar.init_tk(location)
+                                               return new TKwintern.init_tk(location)
                                        end
                                        if accept_token == 20 then
-                                               return new TKwintern.init_tk(location)
+                                               return new TKwextern.init_tk(location)
                                        end
                                        if accept_token == 21 then
-                                               return new TKwextern.init_tk(location)
+                                               return new TKwprotected.init_tk(location)
                                        end
                                        if accept_token == 22 then
-                                               return new TKwprotected.init_tk(location)
+                                               return new TKwprivate.init_tk(location)
                                        end
                                        if accept_token == 23 then
-                                               return new TKwprivate.init_tk(location)
+                                               return new TKwintrude.init_tk(location)
                                        end
                                        if accept_token == 24 then
-                                               return new TKwintrude.init_tk(location)
+                                               return new TKwif.init_tk(location)
                                        end
                                        if accept_token == 25 then
-                                               return new TKwif.init_tk(location)
+                                               return new TKwthen.init_tk(location)
                                        end
                                        if accept_token == 26 then
-                                               return new TKwthen.init_tk(location)
+                                               return new TKwelse.init_tk(location)
                                        end
                                        if accept_token == 27 then
-                                               return new TKwelse.init_tk(location)
+                                               return new TKwwhile.init_tk(location)
                                        end
                                        if accept_token == 28 then
-                                               return new TKwwhile.init_tk(location)
+                                               return new TKwloop.init_tk(location)
                                        end
                                        if accept_token == 29 then
-                                               return new TKwloop.init_tk(location)
+                                               return new TKwfor.init_tk(location)
                                        end
                                        if accept_token == 30 then
-                                               return new TKwfor.init_tk(location)
+                                               return new TKwin.init_tk(location)
                                        end
                                        if accept_token == 31 then
-                                               return new TKwin.init_tk(location)
+                                               return new TKwand.init_tk(location)
                                        end
                                        if accept_token == 32 then
-                                               return new TKwand.init_tk(location)
+                                               return new TKwor.init_tk(location)
                                        end
                                        if accept_token == 33 then
-                                               return new TKwor.init_tk(location)
+                                               return new TKwnot.init_tk(location)
                                        end
                                        if accept_token == 34 then
-                                               return new TKwnot.init_tk(location)
+                                               return new TKwreturn.init_tk(location)
                                        end
                                        if accept_token == 35 then
-                                               return new TKwreturn.init_tk(location)
+                                               return new TKwcontinue.init_tk(location)
                                        end
                                        if accept_token == 36 then
-                                               return new TKwcontinue.init_tk(location)
+                                               return new TKwbreak.init_tk(location)
                                        end
                                        if accept_token == 37 then
-                                               return new TKwbreak.init_tk(location)
+                                               return new TKwabort.init_tk(location)
                                        end
                                        if accept_token == 38 then
-                                               return new TKwabort.init_tk(location)
+                                               return new TKwassert.init_tk(location)
                                        end
                                        if accept_token == 39 then
-                                               return new TKwassert.init_tk(location)
+                                               return new TKwnew.init_tk(location)
                                        end
                                        if accept_token == 40 then
-                                               return new TKwnew.init_tk(location)
+                                               return new TKwisa.init_tk(location)
                                        end
                                        if accept_token == 41 then
-                                               return new TKwisa.init_tk(location)
+                                               return new TKwonce.init_tk(location)
                                        end
                                        if accept_token == 42 then
-                                               return new TKwonce.init_tk(location)
+                                               return new TKwsuper.init_tk(location)
                                        end
                                        if accept_token == 43 then
-                                               return new TKwsuper.init_tk(location)
+                                               return new TKwself.init_tk(location)
                                        end
                                        if accept_token == 44 then
-                                               return new TKwself.init_tk(location)
+                                               return new TKwtrue.init_tk(location)
                                        end
                                        if accept_token == 45 then
-                                               return new TKwtrue.init_tk(location)
+                                               return new TKwfalse.init_tk(location)
                                        end
                                        if accept_token == 46 then
-                                               return new TKwfalse.init_tk(location)
+                                               return new TKwnull.init_tk(location)
                                        end
                                        if accept_token == 47 then
-                                               return new TKwnull.init_tk(location)
+                                               return new TKwas.init_tk(location)
                                        end
                                        if accept_token == 48 then
-                                               return new TKwas.init_tk(location)
+                                               return new TKwnullable.init_tk(location)
                                        end
                                        if accept_token == 49 then
-                                               return new TKwnullable.init_tk(location)
+                                               return new TKwisset.init_tk(location)
                                        end
                                        if accept_token == 50 then
-                                               return new TKwisset.init_tk(location)
+                                               return new TKwlabel.init_tk(location)
                                        end
                                        if accept_token == 51 then
-                                               return new TKwlabel.init_tk(location)
+                                               return new TKwdebug.init_tk(location)
                                        end
                                        if accept_token == 52 then
                                                return new TOpar.init_tk(location)
@@ -1588,44 +1540,43 @@ class Lexer
                                                return new TBang.init_tk(location)
                                        end
                                        if accept_token == 80 then
-                                               var token_text = string.substring(start_stream_pos, accept_length)
-                                               return new TClassid.init_tk(token_text, location)
+                                               return new TClassid.init_tk(location)
                                        end
                                        if accept_token == 81 then
-                                               var token_text = string.substring(start_stream_pos, accept_length)
-                                               return new TId.init_tk(token_text, location)
+                                               return new TId.init_tk(location)
                                        end
                                        if accept_token == 82 then
-                                               var token_text = string.substring(start_stream_pos, accept_length)
-                                               return new TAttrid.init_tk(token_text, location)
+                                               return new TAttrid.init_tk(location)
                                        end
                                        if accept_token == 83 then
-                                               var token_text = string.substring(start_stream_pos, accept_length)
-                                               return new TNumber.init_tk(token_text, location)
+                                               return new TNumber.init_tk(location)
                                        end
                                        if accept_token == 84 then
-                                               var token_text = string.substring(start_stream_pos, accept_length)
-                                               return new TFloat.init_tk(token_text, location)
+                                               return new TFloat.init_tk(location)
                                        end
                                        if accept_token == 85 then
-                                               var token_text = string.substring(start_stream_pos, accept_length)
-                                               return new TChar.init_tk(token_text, location)
+                                               return new TChar.init_tk(location)
                                        end
                                        if accept_token == 86 then
-                                               var token_text = string.substring(start_stream_pos, accept_length)
-                                               return new TString.init_tk(token_text, location)
+                                               return new TString.init_tk(location)
                                        end
                                        if accept_token == 87 then
-                                               var token_text = string.substring(start_stream_pos, accept_length)
-                                               return new TStartString.init_tk(token_text, location)
+                                               return new TStartString.init_tk(location)
                                        end
                                        if accept_token == 88 then
-                                               var token_text = string.substring(start_stream_pos, accept_length)
-                                               return new TMidString.init_tk(token_text, location)
+                                               return new TMidString.init_tk(location)
                                        end
                                        if accept_token == 89 then
-                                               var token_text = string.substring(start_stream_pos, accept_length)
-                                               return new TEndString.init_tk(token_text, location)
+                                               return new TEndString.init_tk(location)
+                                       end
+                                       if accept_token == 90 then
+                                               return new TBadChar.init_tk(location)
+                                       end
+                                       if accept_token == 91 then
+                                               return new TBadString.init_tk(location)
+                                       end
+                                       if accept_token == 92 then
+                                               return new TExternCodeSegment.init_tk(location)
                                        end
                                else
                                        _stream_pos = sp