- _token = null
- return result
- end
-
- # Get a token, or null if it is discarded
- private meth get_token: Token
- do
- var dfa_state = 0
-
- var start_pos = _pos
- var start_line = _line
-
- var accept_state = -1
- var accept_token = -1
- var accept_length = -1
- var accept_pos = -1
- var accept_line = -1
-
- var goto_table = _goto_table[_state]
- var accept = _accept_table[_state]
- _text.clear
-
- while true do
- var c = get_char
-
- if c != -1 then
- if c == 10 then
- if _cr then
- _cr = false
- else
- _line = _line + 1
- _pos = 0
- end
- else if c == 13 then
- _line = _line + 1
- _pos = 0
- _cr = true
- else
- _pos = _pos + 1
- _cr = false
- end
-
- _text.add(c.ascii)
-
- var first_loop = true # aka until
- while dfa_state < -1 or first_loop do
- var old_state = dfa_state
- if dfa_state < -1 then
- old_state = -2 - dfa_state
- end
-
- dfa_state = -1
-
- var tmp1 = goto_table[old_state]
- var low = 0
- var high = tmp1.length - 1
-
- while low <= high do
- var middle = (low + high) / 2
- var tmp2 = tmp1[middle]
-
- if c < tmp2[0] then
- high = middle - 1
- else if c > tmp2[1] then
- low = middle + 1
- else
- dfa_state = tmp2[2]
- low = high + 1 # aka break
- end
- end
- first_loop = false # aka until
- end
- else
- dfa_state = -1
- end
-
- if dfa_state >= 0 then
- if accept[dfa_state] != -1 then
- accept_state = dfa_state
- accept_token = accept[dfa_state]
- accept_length = _text.length
- accept_pos = _pos
- accept_line = _line
- end
- else
- if accept_state != -1 then
- if accept_token == 0 then
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return null
- end
- if accept_token == 1 then
- var token_text = _text.substring(0, accept_length)
- var token = new TEol.init_tk(token_text, _filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 2 then
- var token_text = _text.substring(0, accept_length)
- var token = new TComment.init_tk(token_text, _filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 3 then
- var token = new TKwpackage.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 4 then
- var token = new TKwimport.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 5 then
- var token = new TKwclass.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 6 then
- var token = new TKwabstract.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 7 then
- var token = new TKwinterface.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 8 then
- var token = new TKwuniversal.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 9 then
- var token = new TKwspecial.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 10 then
- var token = new TKwend.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 11 then
- var token_text = _text.substring(0, accept_length)
- var token = new TKwmeth.init_tk(token_text, _filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 12 then
- var token = new TKwtype.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 13 then
- var token = new TKwattr.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 14 then
- var token = new TKwinit.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 15 then
- var token = new TKwredef.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 16 then
- var token = new TKwis.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 17 then
- var token = new TKwdo.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 18 then
- var token = new TKwreadable.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 19 then
- var token = new TKwwritable.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 20 then
- var token = new TKwvar.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 21 then
- var token = new TKwintern.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 22 then
- var token = new TKwextern.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 23 then
- var token = new TKwprotected.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 24 then
- var token = new TKwprivate.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 25 then
- var token = new TKwintrude.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 26 then
- var token = new TKwif.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 27 then
- var token = new TKwthen.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 28 then
- var token = new TKwelse.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 29 then
- var token = new TKwwhile.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 30 then
- var token = new TKwfor.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 31 then
- var token = new TKwin.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 32 then
- var token = new TKwand.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 33 then
- var token = new TKwor.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 34 then
- var token = new TKwnot.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 35 then
- var token = new TKwreturn.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 36 then
- var token = new TKwcontinue.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 37 then
- var token = new TKwbreak.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 38 then
- var token = new TKwabort.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 39 then
- var token = new TKwassert.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 40 then
- var token = new TKwnew.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 41 then
- var token = new TKwisa.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 42 then
- var token = new TKwonce.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 43 then
- var token = new TKwsuper.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 44 then
- var token = new TKwself.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 45 then
- var token = new TKwtrue.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 46 then
- var token = new TKwfalse.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 47 then
- var token = new TKwnull.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 48 then
- var token = new TKwas.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 49 then
- var token = new TKwwith.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 50 then
- var token = new TKwnullable.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 51 then
- var token = new TKwisset.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 52 then
- var token = new TOpar.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 53 then
- var token = new TCpar.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 54 then
- var token = new TObra.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 55 then
- var token = new TCbra.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 56 then
- var token = new TComma.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 57 then
- var token = new TColumn.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 58 then
- var token = new TQuad.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 59 then
- var token = new TAssign.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 60 then
- var token = new TPluseq.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 61 then
- var token = new TMinuseq.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 62 then
- var token = new TDotdotdot.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 63 then
- var token = new TDotdot.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 64 then
- var token = new TDot.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 65 then
- var token = new TPlus.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 66 then
- var token = new TMinus.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 67 then
- var token = new TStar.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 68 then
- var token = new TSlash.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 69 then
- var token = new TPercent.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 70 then
- var token = new TEq.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 71 then
- var token = new TNe.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 72 then
- var token = new TLt.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 73 then
- var token = new TLe.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 74 then
- var token = new TGt.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 75 then
- var token = new TGe.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 76 then
- var token = new TStarship.init_tk(_filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 77 then
- var token_text = _text.substring(0, accept_length)
- var token = new TClassid.init_tk(token_text, _filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 78 then
- var token_text = _text.substring(0, accept_length)
- var token = new TId.init_tk(token_text, _filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 79 then
- var token_text = _text.substring(0, accept_length)
- var token = new TAttrid.init_tk(token_text, _filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 80 then
- var token_text = _text.substring(0, accept_length)
- var token = new TNumber.init_tk(token_text, _filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 81 then
- var token_text = _text.substring(0, accept_length)
- var token = new TFloat.init_tk(token_text, _filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 82 then
- var token_text = _text.substring(0, accept_length)
- var token = new TChar.init_tk(token_text, _filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 83 then
- var token_text = _text.substring(0, accept_length)
- var token = new TString.init_tk(token_text, _filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 84 then
- var token_text = _text.substring(0, accept_length)
- var token = new TStartString.init_tk(token_text, _filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 85 then
- var token_text = _text.substring(0, accept_length)
- var token = new TMidString.init_tk(token_text, _filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- if accept_token == 86 then
- var token_text = _text.substring(0, accept_length)
- var token = new TEndString.init_tk(token_text, _filename, start_line + 1, start_pos + 1)
- push_back(accept_length)
- _pos = accept_pos
- _line = accept_line
- return token
- end
- else
- if _text.length > 0 then
- var token = new PError.init_error(_filename, start_line + 1, start_pos + 1, "Unknown token: {_text}")
- return token
- else
- var token = new EOF(_filename, start_line + 1, start_pos + 1)
- return token
- end
- end
- end
- end
- return null
- end
-
- # Read the next character.
- # The character is read from the stream of from the pushback buffer.
- private meth get_char: Int
- do
- if _eof then
- return -1