X-Git-Url: http://nitlanguage.org

diff --git a/src/parser/lexer.nit b/src/parser/lexer.nit
index 452d452..7e07395 100644
--- a/src/parser/lexer.nit
+++ b/src/parser/lexer.nit
@@ -6,13 +6,23 @@ intrude import parser_nodes
 private import tables
 
 redef class Token
-    readable var _text: String
+    var _text: nullable String
 
-    fun parser_index: Int is abstract
+    redef fun text
+    do
+        var res = _text
+        if res != null then return res
+        res = location.text
+	_text = res
+	return res
+    end
 
-    redef fun to_s: String do
-        return "'{_text}'"
+    redef fun text=(text)
+    do
+        _text = text
     end
+
+    fun parser_index: Int is abstract
 end
 
 redef class TEol
@@ -21,9 +31,8 @@ redef class TEol
 	return 0
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
 		_location = loc
     end
 end
@@ -34,9 +43,8 @@ redef class TComment
 	return 1
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
 		_location = loc
     end
 end
@@ -47,9 +55,8 @@ redef class TKwmodule
 	return 2
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
 		_location = loc
     end
 end
@@ -62,7 +69,6 @@ redef class TKwimport
 
     init init_tk(loc: Location)
     do
-        _text = once "import"
 		_location = loc
     end
 end
@@ -75,7 +81,6 @@ redef class TKwclass
 
     init init_tk(loc: Location)
     do
-        _text = once "class"
 		_location = loc
     end
 end
@@ -88,7 +93,6 @@ redef class TKwabstract
 
     init init_tk(loc: Location)
     do
-        _text = once "abstract"
 		_location = loc
     end
 end
@@ -101,12 +105,11 @@ redef class TKwinterface
 
     init init_tk(loc: Location)
     do
-        _text = once "interface"
 		_location = loc
     end
 end
 
-redef class TKwuniversal
+redef class TKwenum
     redef fun parser_index: Int
     do
 	return 7
@@ -114,12 +117,11 @@ redef class TKwuniversal
 
     init init_tk(loc: Location)
     do
-        _text = once "universal"
 		_location = loc
     end
 end
 
-redef class TKwspecial
+redef class TKwend
     redef fun parser_index: Int
     do
 	return 8
@@ -127,12 +129,11 @@ redef class TKwspecial
 
     init init_tk(loc: Location)
     do
-        _text = once "special"
 		_location = loc
     end
 end
 
-redef class TKwend
+redef class TKwmeth
     redef fun parser_index: Int
     do
 	return 9
@@ -140,12 +141,11 @@ redef class TKwend
 
     init init_tk(loc: Location)
     do
-        _text = once "end"
 		_location = loc
     end
 end
 
-redef class TKwmeth
+redef class TKwtype
     redef fun parser_index: Int
     do
 	return 10
@@ -153,12 +153,11 @@ redef class TKwmeth
 
     init init_tk(loc: Location)
     do
-        _text = once "fun"
 		_location = loc
     end
 end
 
-redef class TKwtype
+redef class TKwinit
     redef fun parser_index: Int
     do
 	return 11
@@ -166,12 +165,11 @@ redef class TKwtype
 
     init init_tk(loc: Location)
     do
-        _text = once "type"
 		_location = loc
     end
 end
 
-redef class TKwinit
+redef class TKwredef
     redef fun parser_index: Int
     do
 	return 12
@@ -179,12 +177,11 @@ redef class TKwinit
 
     init init_tk(loc: Location)
     do
-        _text = once "init"
 		_location = loc
     end
 end
 
-redef class TKwredef
+redef class TKwis
     redef fun parser_index: Int
     do
 	return 13
@@ -192,12 +189,11 @@ redef class TKwredef
 
     init init_tk(loc: Location)
     do
-        _text = once "redef"
 		_location = loc
     end
 end
 
-redef class TKwis
+redef class TKwdo
     redef fun parser_index: Int
     do
 	return 14
@@ -205,12 +201,11 @@ redef class TKwis
 
     init init_tk(loc: Location)
     do
-        _text = once "is"
 		_location = loc
     end
 end
 
-redef class TKwdo
+redef class TKwreadable
     redef fun parser_index: Int
     do
 	return 15
@@ -218,12 +213,11 @@ redef class TKwdo
 
     init init_tk(loc: Location)
     do
-        _text = once "do"
 		_location = loc
     end
 end
 
-redef class TKwreadable
+redef class TKwwritable
     redef fun parser_index: Int
     do
 	return 16
@@ -231,12 +225,11 @@ redef class TKwreadable
 
     init init_tk(loc: Location)
     do
-        _text = once "readable"
 		_location = loc
     end
 end
 
-redef class TKwwritable
+redef class TKwvar
     redef fun parser_index: Int
     do
 	return 17
@@ -244,12 +237,11 @@ redef class TKwwritable
 
     init init_tk(loc: Location)
     do
-        _text = once "writable"
 		_location = loc
     end
 end
 
-redef class TKwvar
+redef class TKwintern
     redef fun parser_index: Int
     do
 	return 18
@@ -257,12 +249,11 @@ redef class TKwvar
 
     init init_tk(loc: Location)
     do
-        _text = once "var"
 		_location = loc
     end
 end
 
-redef class TKwintern
+redef class TKwextern
     redef fun parser_index: Int
     do
 	return 19
@@ -270,12 +261,11 @@ redef class TKwintern
 
     init init_tk(loc: Location)
     do
-        _text = once "intern"
 		_location = loc
     end
 end
 
-redef class TKwextern
+redef class TKwprotected
     redef fun parser_index: Int
     do
 	return 20
@@ -283,12 +273,11 @@ redef class TKwextern
 
     init init_tk(loc: Location)
     do
-        _text = once "extern"
 		_location = loc
     end
 end
 
-redef class TKwprotected
+redef class TKwprivate
     redef fun parser_index: Int
     do
 	return 21
@@ -296,12 +285,11 @@ redef class TKwprotected
 
     init init_tk(loc: Location)
     do
-        _text = once "protected"
 		_location = loc
     end
 end
 
-redef class TKwprivate
+redef class TKwintrude
     redef fun parser_index: Int
     do
 	return 22
@@ -309,12 +297,11 @@ redef class TKwprivate
 
     init init_tk(loc: Location)
     do
-        _text = once "private"
 		_location = loc
     end
 end
 
-redef class TKwintrude
+redef class TKwif
     redef fun parser_index: Int
     do
 	return 23
@@ -322,12 +309,11 @@ redef class TKwintrude
 
     init init_tk(loc: Location)
     do
-        _text = once "intrude"
 		_location = loc
     end
 end
 
-redef class TKwif
+redef class TKwthen
     redef fun parser_index: Int
     do
 	return 24
@@ -335,12 +321,11 @@ redef class TKwif
 
     init init_tk(loc: Location)
     do
-        _text = once "if"
 		_location = loc
     end
 end
 
-redef class TKwthen
+redef class TKwelse
     redef fun parser_index: Int
     do
 	return 25
@@ -348,12 +333,11 @@ redef class TKwthen
 
     init init_tk(loc: Location)
     do
-        _text = once "then"
 		_location = loc
     end
 end
 
-redef class TKwelse
+redef class TKwwhile
     redef fun parser_index: Int
     do
 	return 26
@@ -361,12 +345,11 @@ redef class TKwelse
 
     init init_tk(loc: Location)
     do
-        _text = once "else"
 		_location = loc
     end
 end
 
-redef class TKwwhile
+redef class TKwloop
     redef fun parser_index: Int
     do
 	return 27
@@ -374,12 +357,11 @@ redef class TKwwhile
 
     init init_tk(loc: Location)
     do
-        _text = once "while"
 		_location = loc
     end
 end
 
-redef class TKwloop
+redef class TKwfor
     redef fun parser_index: Int
     do
 	return 28
@@ -387,12 +369,11 @@ redef class TKwloop
 
     init init_tk(loc: Location)
     do
-        _text = once "loop"
 		_location = loc
     end
 end
 
-redef class TKwfor
+redef class TKwin
     redef fun parser_index: Int
     do
 	return 29
@@ -400,12 +381,11 @@ redef class TKwfor
 
     init init_tk(loc: Location)
     do
-        _text = once "for"
 		_location = loc
     end
 end
 
-redef class TKwin
+redef class TKwand
     redef fun parser_index: Int
     do
 	return 30
@@ -413,12 +393,11 @@ redef class TKwin
 
     init init_tk(loc: Location)
     do
-        _text = once "in"
 		_location = loc
     end
 end
 
-redef class TKwand
+redef class TKwor
     redef fun parser_index: Int
     do
 	return 31
@@ -426,12 +405,11 @@ redef class TKwand
 
     init init_tk(loc: Location)
     do
-        _text = once "and"
 		_location = loc
     end
 end
 
-redef class TKwor
+redef class TKwnot
     redef fun parser_index: Int
     do
 	return 32
@@ -439,12 +417,11 @@ redef class TKwor
 
     init init_tk(loc: Location)
     do
-        _text = once "or"
 		_location = loc
     end
 end
 
-redef class TKwnot
+redef class TKwreturn
     redef fun parser_index: Int
     do
 	return 33
@@ -452,12 +429,11 @@ redef class TKwnot
 
     init init_tk(loc: Location)
     do
-        _text = once "not"
 		_location = loc
     end
 end
 
-redef class TKwreturn
+redef class TKwcontinue
     redef fun parser_index: Int
     do
 	return 34
@@ -465,12 +441,11 @@ redef class TKwreturn
 
     init init_tk(loc: Location)
     do
-        _text = once "return"
 		_location = loc
     end
 end
 
-redef class TKwcontinue
+redef class TKwbreak
     redef fun parser_index: Int
     do
 	return 35
@@ -478,12 +453,11 @@ redef class TKwcontinue
 
     init init_tk(loc: Location)
     do
-        _text = once "continue"
 		_location = loc
     end
 end
 
-redef class TKwbreak
+redef class TKwabort
     redef fun parser_index: Int
     do
 	return 36
@@ -491,12 +465,11 @@ redef class TKwbreak
 
     init init_tk(loc: Location)
     do
-        _text = once "break"
 		_location = loc
     end
 end
 
-redef class TKwabort
+redef class TKwassert
     redef fun parser_index: Int
     do
 	return 37
@@ -504,12 +477,11 @@ redef class TKwabort
 
     init init_tk(loc: Location)
     do
-        _text = once "abort"
 		_location = loc
     end
 end
 
-redef class TKwassert
+redef class TKwnew
     redef fun parser_index: Int
     do
 	return 38
@@ -517,12 +489,11 @@ redef class TKwassert
 
     init init_tk(loc: Location)
     do
-        _text = once "assert"
 		_location = loc
     end
 end
 
-redef class TKwnew
+redef class TKwisa
     redef fun parser_index: Int
     do
 	return 39
@@ -530,12 +501,11 @@ redef class TKwnew
 
     init init_tk(loc: Location)
     do
-        _text = once "new"
 		_location = loc
     end
 end
 
-redef class TKwisa
+redef class TKwonce
     redef fun parser_index: Int
     do
 	return 40
@@ -543,12 +513,11 @@ redef class TKwisa
 
     init init_tk(loc: Location)
     do
-        _text = once "isa"
 		_location = loc
     end
 end
 
-redef class TKwonce
+redef class TKwsuper
     redef fun parser_index: Int
     do
 	return 41
@@ -556,12 +525,11 @@ redef class TKwonce
 
     init init_tk(loc: Location)
     do
-        _text = once "once"
 		_location = loc
     end
 end
 
-redef class TKwsuper
+redef class TKwself
     redef fun parser_index: Int
     do
 	return 42
@@ -569,12 +537,11 @@ redef class TKwsuper
 
     init init_tk(loc: Location)
     do
-        _text = once "super"
 		_location = loc
     end
 end
 
-redef class TKwself
+redef class TKwtrue
     redef fun parser_index: Int
     do
 	return 43
@@ -582,12 +549,11 @@ redef class TKwself
 
     init init_tk(loc: Location)
     do
-        _text = once "self"
 		_location = loc
     end
 end
 
-redef class TKwtrue
+redef class TKwfalse
     redef fun parser_index: Int
     do
 	return 44
@@ -595,12 +561,11 @@ redef class TKwtrue
 
     init init_tk(loc: Location)
     do
-        _text = once "true"
 		_location = loc
     end
 end
 
-redef class TKwfalse
+redef class TKwnull
     redef fun parser_index: Int
     do
 	return 45
@@ -608,12 +573,11 @@ redef class TKwfalse
 
     init init_tk(loc: Location)
     do
-        _text = once "false"
 		_location = loc
     end
 end
 
-redef class TKwnull
+redef class TKwas
     redef fun parser_index: Int
     do
 	return 46
@@ -621,12 +585,11 @@ redef class TKwnull
 
     init init_tk(loc: Location)
     do
-        _text = once "null"
 		_location = loc
     end
 end
 
-redef class TKwas
+redef class TKwnullable
     redef fun parser_index: Int
     do
 	return 47
@@ -634,12 +597,11 @@ redef class TKwas
 
     init init_tk(loc: Location)
     do
-        _text = once "as"
 		_location = loc
     end
 end
 
-redef class TKwnullable
+redef class TKwisset
     redef fun parser_index: Int
     do
 	return 48
@@ -647,12 +609,11 @@ redef class TKwnullable
 
     init init_tk(loc: Location)
     do
-        _text = once "nullable"
 		_location = loc
     end
 end
 
-redef class TKwisset
+redef class TKwlabel
     redef fun parser_index: Int
     do
 	return 49
@@ -660,12 +621,11 @@ redef class TKwisset
 
     init init_tk(loc: Location)
     do
-        _text = once "isset"
 		_location = loc
     end
 end
 
-redef class TKwlabel
+redef class TKwdebug
     redef fun parser_index: Int
     do
 	return 50
@@ -673,7 +633,6 @@ redef class TKwlabel
 
     init init_tk(loc: Location)
     do
-        _text = once "label"
 		_location = loc
     end
 end
@@ -686,7 +645,6 @@ redef class TOpar
 
     init init_tk(loc: Location)
     do
-        _text = once "("
 		_location = loc
     end
 end
@@ -699,7 +657,6 @@ redef class TCpar
 
     init init_tk(loc: Location)
     do
-        _text = once ")"
 		_location = loc
     end
 end
@@ -712,7 +669,6 @@ redef class TObra
 
     init init_tk(loc: Location)
     do
-        _text = once "["
 		_location = loc
     end
 end
@@ -725,7 +681,6 @@ redef class TCbra
 
     init init_tk(loc: Location)
     do
-        _text = once "]"
 		_location = loc
     end
 end
@@ -738,7 +693,6 @@ redef class TComma
 
     init init_tk(loc: Location)
     do
-        _text = once ","
 		_location = loc
     end
 end
@@ -751,7 +705,6 @@ redef class TColumn
 
     init init_tk(loc: Location)
     do
-        _text = once ":"
 		_location = loc
     end
 end
@@ -764,7 +717,6 @@ redef class TQuad
 
     init init_tk(loc: Location)
     do
-        _text = once "::"
 		_location = loc
     end
 end
@@ -777,7 +729,6 @@ redef class TAssign
 
     init init_tk(loc: Location)
     do
-        _text = once "="
 		_location = loc
     end
 end
@@ -790,7 +741,6 @@ redef class TPluseq
 
     init init_tk(loc: Location)
     do
-        _text = once "+="
 		_location = loc
     end
 end
@@ -803,7 +753,6 @@ redef class TMinuseq
 
     init init_tk(loc: Location)
     do
-        _text = once "-="
 		_location = loc
     end
 end
@@ -816,7 +765,6 @@ redef class TDotdotdot
 
     init init_tk(loc: Location)
     do
-        _text = once "..."
 		_location = loc
     end
 end
@@ -829,7 +777,6 @@ redef class TDotdot
 
     init init_tk(loc: Location)
     do
-        _text = once ".."
 		_location = loc
     end
 end
@@ -842,7 +789,6 @@ redef class TDot
 
     init init_tk(loc: Location)
     do
-        _text = once "."
 		_location = loc
     end
 end
@@ -855,7 +801,6 @@ redef class TPlus
 
     init init_tk(loc: Location)
     do
-        _text = once "+"
 		_location = loc
     end
 end
@@ -868,7 +813,6 @@ redef class TMinus
 
     init init_tk(loc: Location)
     do
-        _text = once "-"
 		_location = loc
     end
 end
@@ -881,7 +825,6 @@ redef class TStar
 
     init init_tk(loc: Location)
     do
-        _text = once "*"
 		_location = loc
     end
 end
@@ -894,7 +837,6 @@ redef class TSlash
 
     init init_tk(loc: Location)
     do
-        _text = once "/"
 		_location = loc
     end
 end
@@ -907,7 +849,6 @@ redef class TPercent
 
     init init_tk(loc: Location)
     do
-        _text = once "%"
 		_location = loc
     end
 end
@@ -920,7 +861,6 @@ redef class TEq
 
     init init_tk(loc: Location)
     do
-        _text = once "=="
 		_location = loc
     end
 end
@@ -933,7 +873,6 @@ redef class TNe
 
     init init_tk(loc: Location)
     do
-        _text = once "!="
 		_location = loc
     end
 end
@@ -946,7 +885,6 @@ redef class TLt
 
     init init_tk(loc: Location)
     do
-        _text = once "<"
 		_location = loc
     end
 end
@@ -959,7 +897,6 @@ redef class TLe
 
     init init_tk(loc: Location)
     do
-        _text = once "<="
 		_location = loc
     end
 end
@@ -972,7 +909,6 @@ redef class TLl
 
     init init_tk(loc: Location)
     do
-        _text = once "<<"
 		_location = loc
     end
 end
@@ -985,7 +921,6 @@ redef class TGt
 
     init init_tk(loc: Location)
     do
-        _text = once ">"
 		_location = loc
     end
 end
@@ -998,7 +933,6 @@ redef class TGe
 
     init init_tk(loc: Location)
     do
-        _text = once ">="
 		_location = loc
     end
 end
@@ -1011,7 +945,6 @@ redef class TGg
 
     init init_tk(loc: Location)
     do
-        _text = once ">>"
 		_location = loc
     end
 end
@@ -1024,7 +957,6 @@ redef class TStarship
 
     init init_tk(loc: Location)
     do
-        _text = once "<=>"
 		_location = loc
     end
 end
@@ -1037,85 +969,78 @@ redef class TBang
 
     init init_tk(loc: Location)
     do
-        _text = once "!"
 		_location = loc
     end
 end
 
-redef class TClassid
+redef class TAt
     redef fun parser_index: Int
     do
 	return 79
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
 		_location = loc
     end
 end
 
-redef class TId
+redef class TClassid
     redef fun parser_index: Int
     do
 	return 80
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
 		_location = loc
     end
 end
 
-redef class TAttrid
+redef class TId
     redef fun parser_index: Int
     do
 	return 81
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
 		_location = loc
     end
 end
 
-redef class TNumber
+redef class TAttrid
     redef fun parser_index: Int
     do
 	return 82
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
 		_location = loc
     end
 end
 
-redef class TFloat
+redef class TNumber
     redef fun parser_index: Int
     do
 	return 83
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
 		_location = loc
     end
 end
 
-redef class TChar
+redef class TFloat
     redef fun parser_index: Int
     do
 	return 84
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
 		_location = loc
     end
 end
@@ -1126,9 +1051,8 @@ redef class TString
 	return 85
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
 		_location = loc
     end
 end
@@ -1139,9 +1063,8 @@ redef class TStartString
 	return 86
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
 		_location = loc
     end
 end
@@ -1152,9 +1075,8 @@ redef class TMidString
 	return 87
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
     do
-        _text = text
 		_location = loc
     end
 end
@@ -1165,9 +1087,56 @@ redef class TEndString
 	return 88
     end
 
-    init init_tk(text: String, loc: Location)
+    init init_tk(loc: Location)
+    do
+		_location = loc
+    end
+end
+
+redef class TChar
+    redef fun parser_index: Int
+    do
+	return 89
+    end
+
+    init init_tk(loc: Location)
+    do
+		_location = loc
+    end
+end
+
+redef class TBadString
+    redef fun parser_index: Int
+    do
+	return 90
+    end
+
+    init init_tk(loc: Location)
+    do
+		_location = loc
+    end
+end
+
+redef class TBadChar
+    redef fun parser_index: Int
+    do
+	return 91
+    end
+
+    init init_tk(loc: Location)
+    do
+		_location = loc
+    end
+end
+
+redef class TExternCodeSegment
+    redef fun parser_index: Int
+    do
+	return 92
+    end
+
+    init init_tk(loc: Location)
     do
-        _text = text
 		_location = loc
     end
 end
@@ -1176,10 +1145,10 @@ end
 redef class EOF
     redef fun parser_index: Int
     do
-	return 89
+	return 93
     end
 
-    init(loc: Location)
+    init init_tk(loc: Location)
     do
         _text = ""
 		_location = loc
@@ -1191,33 +1160,46 @@ redef class AError
 
     init init_error(message: String, loc: Location)
     do
-		init(loc)
+		init_tk(loc)
 		_message = message
     end
 end
 
+redef class ALexerError
+    readable var _string: String
+
+    init init_lexer_error(message: String, loc: Location, string: String)
+    do
+		init_error(message, loc)
+		_string = string
+    end
+end
+
+redef class AParserError
+    readable var _token: Token
+
+    init init_parser_error(message: String, loc: Location, token: Token)
+    do
+		init_error(message, loc)
+		_token = token
+    end
+end
 
 # The lexer extract NIT tokens from an input stream.
 # It is better user with the Parser
 class Lexer
-special TablesCapable
+	super TablesCapable
 	# Last peeked token
 	var _token: nullable Token
 
 	# Lexer current state
 	var _state: Int = 0
 
-	# Name of the stream (as given to tokens)
-	readable var _filename: String
-
-	# Input stream where character are read
-	var _stream: IStream
+	# The source file
+	readable var _file: SourceFile
 
-	# Pushback buffer to store unread character
-	var _stream_buf: Buffer
-
-	# Number of character stored in the pushback buffer
-	var _stream_pos: Int
+	# Current character in the stream
+	var _stream_pos: Int = 0
 
 	# Current line number in the input stream
 	var _line: Int = 0
@@ -1228,23 +1210,13 @@ special TablesCapable
 	# Was the last character a cariage-return?
 	var _cr: Bool = false
 
-	# If the end of stream?
-	var _eof: Bool = false
-
-	# Current working text read from the input stream
-	var _text: Buffer
-
 	# Constante state values
 	private fun state_initial: Int do return 0 end
 
 	# Create a new lexer for a stream (and a name)
-	init(stream: IStream, fname: String)
+	init(file: SourceFile)
 	do
-		_filename = fname
-		_text = new Buffer
-		_stream = stream
-		_stream_pos = -1
-		_stream_buf = new Buffer
+		_file = file
 	end
 
 	# Give the next token (but do not consume it)
@@ -1264,7 +1236,7 @@ special TablesCapable
 			result = get_token
 		end
 		_token = null
-		return result.as(not null)
+		return result
 	end
 
 	# Get a token, or null if it is discarded
@@ -1272,8 +1244,12 @@ special TablesCapable
 	do
 		var dfa_state = 0
 
+		var sp = _stream_pos
+		var start_stream_pos = sp
 		var start_pos = _pos
 		var start_line = _line
+		var string = _file.string
+		var string_len = string.length
 
 		var accept_state = -1
 		var accept_token = -1
@@ -1281,34 +1257,35 @@ special TablesCapable
 		var accept_pos = -1
 		var accept_line = -1
 
-		var text = _text
-		text.clear
-
 		loop
-			var c = get_char
+			if sp >= string_len then
+				dfa_state = -1
+			else
+				var c = string[sp].ascii
+				sp += 1
 
-			if c != -1 then
 				var cr = _cr
 				var line = _line
 				var pos = _pos
 				if c == 10 then
 					if cr then
 						cr = false
+					        _file.line_starts[line] = sp
 					else
 						line = line + 1
 						pos = 0
+					        _file.line_starts[line] = sp
 					end
 				else if c == 13 then
 					line = line + 1
 					pos = 0
 					cr = true
+					_file.line_starts[line] = sp
 				else
 					pos = pos + 1
 					cr = false
 				end
 
-				text.add(c.ascii)
-
 				loop
 					var old_state = dfa_state
 					if dfa_state < -1 then
@@ -1341,8 +1318,6 @@ special TablesCapable
 				_cr = cr
 				_line = line
 				_pos = pos
-			else
-				dfa_state = -1
 			end
 
 			if dfa_state >= 0 then
@@ -1350,30 +1325,27 @@ special TablesCapable
 				if tok != -1 then
 					accept_state = dfa_state
 					accept_token = tok
-					accept_length = text.length
+					accept_length = sp - start_stream_pos
 					accept_pos = _pos
 					accept_line = _line
 				end
 			else
 				if accept_state != -1 then
-					var location = new Location(_filename, start_line + 1, accept_line + 1, start_pos + 1, accept_pos)
+					var location = new Location(_file, start_line + 1, accept_line + 1, start_pos + 1, accept_pos)
 					_pos = accept_pos
 					_line = accept_line
-					push_back(accept_length)
+					_stream_pos = start_stream_pos + accept_length
 					if accept_token == 0 then
 						return null
 					end
 					if accept_token == 1 then
-						var token_text = text.substring(0, accept_length)
-						return new TEol.init_tk(token_text, location)
+						return new TEol.init_tk(location)
 					end
 					if accept_token == 2 then
-						var token_text = text.substring(0, accept_length)
-						return new TComment.init_tk(token_text, location)
+						return new TComment.init_tk(location)
 					end
 					if accept_token == 3 then
-						var token_text = text.substring(0, accept_length)
-						return new TKwmodule.init_tk(token_text, location)
+						return new TKwmodule.init_tk(location)
 					end
 					if accept_token == 4 then
 						return new TKwimport.init_tk(location)
@@ -1388,136 +1360,136 @@ special TablesCapable
 						return new TKwinterface.init_tk(location)
 					end
 					if accept_token == 8 then
-						return new TKwuniversal.init_tk(location)
+						return new TKwenum.init_tk(location)
 					end
 					if accept_token == 9 then
-						return new TKwspecial.init_tk(location)
+						return new TKwend.init_tk(location)
 					end
 					if accept_token == 10 then
-						return new TKwend.init_tk(location)
+						return new TKwmeth.init_tk(location)
 					end
 					if accept_token == 11 then
-						return new TKwmeth.init_tk(location)
+						return new TKwtype.init_tk(location)
 					end
 					if accept_token == 12 then
-						return new TKwtype.init_tk(location)
+						return new TKwinit.init_tk(location)
 					end
 					if accept_token == 13 then
-						return new TKwinit.init_tk(location)
+						return new TKwredef.init_tk(location)
 					end
 					if accept_token == 14 then
-						return new TKwredef.init_tk(location)
+						return new TKwis.init_tk(location)
 					end
 					if accept_token == 15 then
-						return new TKwis.init_tk(location)
+						return new TKwdo.init_tk(location)
 					end
 					if accept_token == 16 then
-						return new TKwdo.init_tk(location)
+						return new TKwreadable.init_tk(location)
 					end
 					if accept_token == 17 then
-						return new TKwreadable.init_tk(location)
+						return new TKwwritable.init_tk(location)
 					end
 					if accept_token == 18 then
-						return new TKwwritable.init_tk(location)
+						return new TKwvar.init_tk(location)
 					end
 					if accept_token == 19 then
-						return new TKwvar.init_tk(location)
+						return new TKwintern.init_tk(location)
 					end
 					if accept_token == 20 then
-						return new TKwintern.init_tk(location)
+						return new TKwextern.init_tk(location)
 					end
 					if accept_token == 21 then
-						return new TKwextern.init_tk(location)
+						return new TKwprotected.init_tk(location)
 					end
 					if accept_token == 22 then
-						return new TKwprotected.init_tk(location)
+						return new TKwprivate.init_tk(location)
 					end
 					if accept_token == 23 then
-						return new TKwprivate.init_tk(location)
+						return new TKwintrude.init_tk(location)
 					end
 					if accept_token == 24 then
-						return new TKwintrude.init_tk(location)
+						return new TKwif.init_tk(location)
 					end
 					if accept_token == 25 then
-						return new TKwif.init_tk(location)
+						return new TKwthen.init_tk(location)
 					end
 					if accept_token == 26 then
-						return new TKwthen.init_tk(location)
+						return new TKwelse.init_tk(location)
 					end
 					if accept_token == 27 then
-						return new TKwelse.init_tk(location)
+						return new TKwwhile.init_tk(location)
 					end
 					if accept_token == 28 then
-						return new TKwwhile.init_tk(location)
+						return new TKwloop.init_tk(location)
 					end
 					if accept_token == 29 then
-						return new TKwloop.init_tk(location)
+						return new TKwfor.init_tk(location)
 					end
 					if accept_token == 30 then
-						return new TKwfor.init_tk(location)
+						return new TKwin.init_tk(location)
 					end
 					if accept_token == 31 then
-						return new TKwin.init_tk(location)
+						return new TKwand.init_tk(location)
 					end
 					if accept_token == 32 then
-						return new TKwand.init_tk(location)
+						return new TKwor.init_tk(location)
 					end
 					if accept_token == 33 then
-						return new TKwor.init_tk(location)
+						return new TKwnot.init_tk(location)
 					end
 					if accept_token == 34 then
-						return new TKwnot.init_tk(location)
+						return new TKwreturn.init_tk(location)
 					end
 					if accept_token == 35 then
-						return new TKwreturn.init_tk(location)
+						return new TKwcontinue.init_tk(location)
 					end
 					if accept_token == 36 then
-						return new TKwcontinue.init_tk(location)
+						return new TKwbreak.init_tk(location)
 					end
 					if accept_token == 37 then
-						return new TKwbreak.init_tk(location)
+						return new TKwabort.init_tk(location)
 					end
 					if accept_token == 38 then
-						return new TKwabort.init_tk(location)
+						return new TKwassert.init_tk(location)
 					end
 					if accept_token == 39 then
-						return new TKwassert.init_tk(location)
+						return new TKwnew.init_tk(location)
 					end
 					if accept_token == 40 then
-						return new TKwnew.init_tk(location)
+						return new TKwisa.init_tk(location)
 					end
 					if accept_token == 41 then
-						return new TKwisa.init_tk(location)
+						return new TKwonce.init_tk(location)
 					end
 					if accept_token == 42 then
-						return new TKwonce.init_tk(location)
+						return new TKwsuper.init_tk(location)
 					end
 					if accept_token == 43 then
-						return new TKwsuper.init_tk(location)
+						return new TKwself.init_tk(location)
 					end
 					if accept_token == 44 then
-						return new TKwself.init_tk(location)
+						return new TKwtrue.init_tk(location)
 					end
 					if accept_token == 45 then
-						return new TKwtrue.init_tk(location)
+						return new TKwfalse.init_tk(location)
 					end
 					if accept_token == 46 then
-						return new TKwfalse.init_tk(location)
+						return new TKwnull.init_tk(location)
 					end
 					if accept_token == 47 then
-						return new TKwnull.init_tk(location)
+						return new TKwas.init_tk(location)
 					end
 					if accept_token == 48 then
-						return new TKwas.init_tk(location)
+						return new TKwnullable.init_tk(location)
 					end
 					if accept_token == 49 then
-						return new TKwnullable.init_tk(location)
+						return new TKwisset.init_tk(location)
 					end
 					if accept_token == 50 then
-						return new TKwisset.init_tk(location)
+						return new TKwlabel.init_tk(location)
 					end
 					if accept_token == 51 then
-						return new TKwlabel.init_tk(location)
+						return new TKwdebug.init_tk(location)
 					end
 					if accept_token == 52 then
 						return new TOpar.init_tk(location)
@@ -1604,97 +1576,61 @@ special TablesCapable
 						return new TBang.init_tk(location)
 					end
 					if accept_token == 80 then
-						var token_text = text.substring(0, accept_length)
-						return new TClassid.init_tk(token_text, location)
+						return new TAt.init_tk(location)
 					end
 					if accept_token == 81 then
-						var token_text = text.substring(0, accept_length)
-						return new TId.init_tk(token_text, location)
+						return new TClassid.init_tk(location)
 					end
 					if accept_token == 82 then
-						var token_text = text.substring(0, accept_length)
-						return new TAttrid.init_tk(token_text, location)
+						return new TId.init_tk(location)
 					end
 					if accept_token == 83 then
-						var token_text = text.substring(0, accept_length)
-						return new TNumber.init_tk(token_text, location)
+						return new TAttrid.init_tk(location)
 					end
 					if accept_token == 84 then
-						var token_text = text.substring(0, accept_length)
-						return new TFloat.init_tk(token_text, location)
+						return new TNumber.init_tk(location)
 					end
 					if accept_token == 85 then
-						var token_text = text.substring(0, accept_length)
-						return new TChar.init_tk(token_text, location)
+						return new TFloat.init_tk(location)
 					end
 					if accept_token == 86 then
-						var token_text = text.substring(0, accept_length)
-						return new TString.init_tk(token_text, location)
+						return new TString.init_tk(location)
 					end
 					if accept_token == 87 then
-						var token_text = text.substring(0, accept_length)
-						return new TStartString.init_tk(token_text, location)
+						return new TStartString.init_tk(location)
 					end
 					if accept_token == 88 then
-						var token_text = text.substring(0, accept_length)
-						return new TMidString.init_tk(token_text, location)
+						return new TMidString.init_tk(location)
 					end
 					if accept_token == 89 then
-						var token_text = text.substring(0, accept_length)
-						return new TEndString.init_tk(token_text, location)
+						return new TEndString.init_tk(location)
+					end
+					if accept_token == 90 then
+						return new TChar.init_tk(location)
+					end
+					if accept_token == 91 then
+						return new TBadString.init_tk(location)
+					end
+					if accept_token == 92 then
+						return new TBadChar.init_tk(location)
+					end
+					if accept_token == 93 then
+						return new TExternCodeSegment.init_tk(location)
 					end
 				else
-					var location = new Location(_filename, start_line + 1, start_line + 1, start_pos + 1, start_pos + 1)
-					if text.length > 0 then
-						var token = new AError.init_error("Syntax error: unknown token {text}.", location)
+					_stream_pos = sp
+					var location = new Location(_file, start_line + 1, start_line + 1, start_pos + 1, start_pos + 1)
+					if sp > start_stream_pos then
+						var text = string.substring(start_stream_pos, sp-start_stream_pos)
+						var token = new ALexerError.init_lexer_error("Syntax error: unknown token {text}.", location, text)
 						return token
 					else
-						var token = new EOF(location)
+						var token = new EOF.init_tk(location)
 						return token
 					end
 				end
 			end
 		end
 	end
-
-	# Read the next character.
-	# The character is read from the stream of from the pushback buffer.
-	private fun get_char: Int
-	do
-		if _eof then
-			return -1
-		end
-
-		var result: Int
-
-		var sp = _stream_pos
-		if sp >= 0 then
-			var res = _stream_buf[_stream_pos]
-			_stream_pos = sp - 1
-			result = res.ascii
-		else
-			result = _stream.read_char
-		end
-
-		if result == -1 then
-			_eof = true
-		end
-
-		return result
-	end
-
-	# Unread some characters.
-	# Unread characters are stored in the pushback buffer.
-	private fun push_back(accept_length: Int)
-	do
-		var length = _text.length
-		var i = length - 1
-		while i >= accept_length do
-			_eof = false
-			_stream_pos = _stream_pos + 1
-			_stream_buf[_stream_pos] = _text[i]
-			i = i - 1
-		end
-	end
 end