src/parser/lexer.nit

   1 # Lexer and its tokens.
   2 # This file was generated by SableCC (http://www.sablecc.org/).
   3 package lexer
   4
   5 intrude import parser_nodes
   6 private import tables
   7
   8 redef class Token
   9     var _text: nullable String
  10
  11     redef fun text
  12     do
  13         var res = _text
  14         if res != null then return res
  15         res = location.text
  16         _text = res
  17         return res
  18     end
  19
  20     redef fun text=(text)
  21     do
  22         _text = text
  23     end
  24
  25     fun parser_index: Int is abstract
  26 end
  27
  28 redef class TEol
  29     redef fun parser_index: Int
  30     do
  31         return 0
  32     end
  33
  34     init init_tk(loc: Location)
  35     do
  36                 _location = loc
  37     end
  38 end
  39
  40 redef class TComment
  41     redef fun parser_index: Int
  42     do
  43         return 1
  44     end
  45
  46     init init_tk(loc: Location)
  47     do
  48                 _location = loc
  49     end
  50 end
  51
  52 redef class TKwmodule
  53     redef fun parser_index: Int
  54     do
  55         return 2
  56     end
  57
  58     init init_tk(loc: Location)
  59     do
  60                 _location = loc
  61     end
  62 end
  63
  64 redef class TKwimport
  65     redef fun parser_index: Int
  66     do
  67         return 3
  68     end
  69
  70     init init_tk(loc: Location)
  71     do
  72                 _location = loc
  73     end
  74 end
  75
  76 redef class TKwclass
  77     redef fun parser_index: Int
  78     do
  79         return 4
  80     end
  81
  82     init init_tk(loc: Location)
  83     do
  84                 _location = loc
  85     end
  86 end
  87
  88 redef class TKwabstract
  89     redef fun parser_index: Int
  90     do
  91         return 5
  92     end
  93
  94     init init_tk(loc: Location)
  95     do
  96                 _location = loc
  97     end
  98 end
  99
 100 redef class TKwinterface
 101     redef fun parser_index: Int
 102     do
 103         return 6
 104     end
 105
 106     init init_tk(loc: Location)
 107     do
 108                 _location = loc
 109     end
 110 end
 111
 112 redef class TKwenum
 113     redef fun parser_index: Int
 114     do
 115         return 7
 116     end
 117
 118     init init_tk(loc: Location)
 119     do
 120                 _location = loc
 121     end
 122 end
 123
 124 redef class TKwend
 125     redef fun parser_index: Int
 126     do
 127         return 8
 128     end
 129
 130     init init_tk(loc: Location)
 131     do
 132                 _location = loc
 133     end
 134 end
 135
 136 redef class TKwmeth
 137     redef fun parser_index: Int
 138     do
 139         return 9
 140     end
 141
 142     init init_tk(loc: Location)
 143     do
 144                 _location = loc
 145     end
 146 end
 147
 148 redef class TKwtype
 149     redef fun parser_index: Int
 150     do
 151         return 10
 152     end
 153
 154     init init_tk(loc: Location)
 155     do
 156                 _location = loc
 157     end
 158 end
 159
 160 redef class TKwinit
 161     redef fun parser_index: Int
 162     do
 163         return 11
 164     end
 165
 166     init init_tk(loc: Location)
 167     do
 168                 _location = loc
 169     end
 170 end
 171
 172 redef class TKwredef
 173     redef fun parser_index: Int
 174     do
 175         return 12
 176     end
 177
 178     init init_tk(loc: Location)
 179     do
 180                 _location = loc
 181     end
 182 end
 183
 184 redef class TKwis
 185     redef fun parser_index: Int
 186     do
 187         return 13
 188     end
 189
 190     init init_tk(loc: Location)
 191     do
 192                 _location = loc
 193     end
 194 end
 195
 196 redef class TKwdo
 197     redef fun parser_index: Int
 198     do
 199         return 14
 200     end
 201
 202     init init_tk(loc: Location)
 203     do
 204                 _location = loc
 205     end
 206 end
 207
 208 redef class TKwreadable
 209     redef fun parser_index: Int
 210     do
 211         return 15
 212     end
 213
 214     init init_tk(loc: Location)
 215     do
 216                 _location = loc
 217     end
 218 end
 219
 220 redef class TKwwritable
 221     redef fun parser_index: Int
 222     do
 223         return 16
 224     end
 225
 226     init init_tk(loc: Location)
 227     do
 228                 _location = loc
 229     end
 230 end
 231
 232 redef class TKwvar
 233     redef fun parser_index: Int
 234     do
 235         return 17
 236     end
 237
 238     init init_tk(loc: Location)
 239     do
 240                 _location = loc
 241     end
 242 end
 243
 244 redef class TKwintern
 245     redef fun parser_index: Int
 246     do
 247         return 18
 248     end
 249
 250     init init_tk(loc: Location)
 251     do
 252                 _location = loc
 253     end
 254 end
 255
 256 redef class TKwextern
 257     redef fun parser_index: Int
 258     do
 259         return 19
 260     end
 261
 262     init init_tk(loc: Location)
 263     do
 264                 _location = loc
 265     end
 266 end
 267
 268 redef class TKwprotected
 269     redef fun parser_index: Int
 270     do
 271         return 20
 272     end
 273
 274     init init_tk(loc: Location)
 275     do
 276                 _location = loc
 277     end
 278 end
 279
 280 redef class TKwprivate
 281     redef fun parser_index: Int
 282     do
 283         return 21
 284     end
 285
 286     init init_tk(loc: Location)
 287     do
 288                 _location = loc
 289     end
 290 end
 291
 292 redef class TKwintrude
 293     redef fun parser_index: Int
 294     do
 295         return 22
 296     end
 297
 298     init init_tk(loc: Location)
 299     do
 300                 _location = loc
 301     end
 302 end
 303
 304 redef class TKwif
 305     redef fun parser_index: Int
 306     do
 307         return 23
 308     end
 309
 310     init init_tk(loc: Location)
 311     do
 312                 _location = loc
 313     end
 314 end
 315
 316 redef class TKwthen
 317     redef fun parser_index: Int
 318     do
 319         return 24
 320     end
 321
 322     init init_tk(loc: Location)
 323     do
 324                 _location = loc
 325     end
 326 end
 327
 328 redef class TKwelse
 329     redef fun parser_index: Int
 330     do
 331         return 25
 332     end
 333
 334     init init_tk(loc: Location)
 335     do
 336                 _location = loc
 337     end
 338 end
 339
 340 redef class TKwwhile
 341     redef fun parser_index: Int
 342     do
 343         return 26
 344     end
 345
 346     init init_tk(loc: Location)
 347     do
 348                 _location = loc
 349     end
 350 end
 351
 352 redef class TKwloop
 353     redef fun parser_index: Int
 354     do
 355         return 27
 356     end
 357
 358     init init_tk(loc: Location)
 359     do
 360                 _location = loc
 361     end
 362 end
 363
 364 redef class TKwfor
 365     redef fun parser_index: Int
 366     do
 367         return 28
 368     end
 369
 370     init init_tk(loc: Location)
 371     do
 372                 _location = loc
 373     end
 374 end
 375
 376 redef class TKwin
 377     redef fun parser_index: Int
 378     do
 379         return 29
 380     end
 381
 382     init init_tk(loc: Location)
 383     do
 384                 _location = loc
 385     end
 386 end
 387
 388 redef class TKwand
 389     redef fun parser_index: Int
 390     do
 391         return 30
 392     end
 393
 394     init init_tk(loc: Location)
 395     do
 396                 _location = loc
 397     end
 398 end
 399
 400 redef class TKwor
 401     redef fun parser_index: Int
 402     do
 403         return 31
 404     end
 405
 406     init init_tk(loc: Location)
 407     do
 408                 _location = loc
 409     end
 410 end
 411
 412 redef class TKwnot
 413     redef fun parser_index: Int
 414     do
 415         return 32
 416     end
 417
 418     init init_tk(loc: Location)
 419     do
 420                 _location = loc
 421     end
 422 end
 423
 424 redef class TKwreturn
 425     redef fun parser_index: Int
 426     do
 427         return 33
 428     end
 429
 430     init init_tk(loc: Location)
 431     do
 432                 _location = loc
 433     end
 434 end
 435
 436 redef class TKwcontinue
 437     redef fun parser_index: Int
 438     do
 439         return 34
 440     end
 441
 442     init init_tk(loc: Location)
 443     do
 444                 _location = loc
 445     end
 446 end
 447
 448 redef class TKwbreak
 449     redef fun parser_index: Int
 450     do
 451         return 35
 452     end
 453
 454     init init_tk(loc: Location)
 455     do
 456                 _location = loc
 457     end
 458 end
 459
 460 redef class TKwabort
 461     redef fun parser_index: Int
 462     do
 463         return 36
 464     end
 465
 466     init init_tk(loc: Location)
 467     do
 468                 _location = loc
 469     end
 470 end
 471
 472 redef class TKwassert
 473     redef fun parser_index: Int
 474     do
 475         return 37
 476     end
 477
 478     init init_tk(loc: Location)
 479     do
 480                 _location = loc
 481     end
 482 end
 483
 484 redef class TKwnew
 485     redef fun parser_index: Int
 486     do
 487         return 38
 488     end
 489
 490     init init_tk(loc: Location)
 491     do
 492                 _location = loc
 493     end
 494 end
 495
 496 redef class TKwisa
 497     redef fun parser_index: Int
 498     do
 499         return 39
 500     end
 501
 502     init init_tk(loc: Location)
 503     do
 504                 _location = loc
 505     end
 506 end
 507
 508 redef class TKwonce
 509     redef fun parser_index: Int
 510     do
 511         return 40
 512     end
 513
 514     init init_tk(loc: Location)
 515     do
 516                 _location = loc
 517     end
 518 end
 519
 520 redef class TKwsuper
 521     redef fun parser_index: Int
 522     do
 523         return 41
 524     end
 525
 526     init init_tk(loc: Location)
 527     do
 528                 _location = loc
 529     end
 530 end
 531
 532 redef class TKwself
 533     redef fun parser_index: Int
 534     do
 535         return 42
 536     end
 537
 538     init init_tk(loc: Location)
 539     do
 540                 _location = loc
 541     end
 542 end
 543
 544 redef class TKwtrue
 545     redef fun parser_index: Int
 546     do
 547         return 43
 548     end
 549
 550     init init_tk(loc: Location)
 551     do
 552                 _location = loc
 553     end
 554 end
 555
 556 redef class TKwfalse
 557     redef fun parser_index: Int
 558     do
 559         return 44
 560     end
 561
 562     init init_tk(loc: Location)
 563     do
 564                 _location = loc
 565     end
 566 end
 567
 568 redef class TKwnull
 569     redef fun parser_index: Int
 570     do
 571         return 45
 572     end
 573
 574     init init_tk(loc: Location)
 575     do
 576                 _location = loc
 577     end
 578 end
 579
 580 redef class TKwas
 581     redef fun parser_index: Int
 582     do
 583         return 46
 584     end
 585
 586     init init_tk(loc: Location)
 587     do
 588                 _location = loc
 589     end
 590 end
 591
 592 redef class TKwnullable
 593     redef fun parser_index: Int
 594     do
 595         return 47
 596     end
 597
 598     init init_tk(loc: Location)
 599     do
 600                 _location = loc
 601     end
 602 end
 603
 604 redef class TKwisset
 605     redef fun parser_index: Int
 606     do
 607         return 48
 608     end
 609
 610     init init_tk(loc: Location)
 611     do
 612                 _location = loc
 613     end
 614 end
 615
 616 redef class TKwlabel
 617     redef fun parser_index: Int
 618     do
 619         return 49
 620     end
 621
 622     init init_tk(loc: Location)
 623     do
 624                 _location = loc
 625     end
 626 end
 627
 628 redef class TKwdebug
 629     redef fun parser_index: Int
 630     do
 631         return 50
 632     end
 633
 634     init init_tk(loc: Location)
 635     do
 636                 _location = loc
 637     end
 638 end
 639
 640 redef class TOpar
 641     redef fun parser_index: Int
 642     do
 643         return 51
 644     end
 645
 646     init init_tk(loc: Location)
 647     do
 648                 _location = loc
 649     end
 650 end
 651
 652 redef class TCpar
 653     redef fun parser_index: Int
 654     do
 655         return 52
 656     end
 657
 658     init init_tk(loc: Location)
 659     do
 660                 _location = loc
 661     end
 662 end
 663
 664 redef class TObra
 665     redef fun parser_index: Int
 666     do
 667         return 53
 668     end
 669
 670     init init_tk(loc: Location)
 671     do
 672                 _location = loc
 673     end
 674 end
 675
 676 redef class TCbra
 677     redef fun parser_index: Int
 678     do
 679         return 54
 680     end
 681
 682     init init_tk(loc: Location)
 683     do
 684                 _location = loc
 685     end
 686 end
 687
 688 redef class TComma
 689     redef fun parser_index: Int
 690     do
 691         return 55
 692     end
 693
 694     init init_tk(loc: Location)
 695     do
 696                 _location = loc
 697     end
 698 end
 699
 700 redef class TColumn
 701     redef fun parser_index: Int
 702     do
 703         return 56
 704     end
 705
 706     init init_tk(loc: Location)
 707     do
 708                 _location = loc
 709     end
 710 end
 711
 712 redef class TQuad
 713     redef fun parser_index: Int
 714     do
 715         return 57
 716     end
 717
 718     init init_tk(loc: Location)
 719     do
 720                 _location = loc
 721     end
 722 end
 723
 724 redef class TAssign
 725     redef fun parser_index: Int
 726     do
 727         return 58
 728     end
 729
 730     init init_tk(loc: Location)
 731     do
 732                 _location = loc
 733     end
 734 end
 735
 736 redef class TPluseq
 737     redef fun parser_index: Int
 738     do
 739         return 59
 740     end
 741
 742     init init_tk(loc: Location)
 743     do
 744                 _location = loc
 745     end
 746 end
 747
 748 redef class TMinuseq
 749     redef fun parser_index: Int
 750     do
 751         return 60
 752     end
 753
 754     init init_tk(loc: Location)
 755     do
 756                 _location = loc
 757     end
 758 end
 759
 760 redef class TDotdotdot
 761     redef fun parser_index: Int
 762     do
 763         return 61
 764     end
 765
 766     init init_tk(loc: Location)
 767     do
 768                 _location = loc
 769     end
 770 end
 771
 772 redef class TDotdot
 773     redef fun parser_index: Int
 774     do
 775         return 62
 776     end
 777
 778     init init_tk(loc: Location)
 779     do
 780                 _location = loc
 781     end
 782 end
 783
 784 redef class TDot
 785     redef fun parser_index: Int
 786     do
 787         return 63
 788     end
 789
 790     init init_tk(loc: Location)
 791     do
 792                 _location = loc
 793     end
 794 end
 795
 796 redef class TPlus
 797     redef fun parser_index: Int
 798     do
 799         return 64
 800     end
 801
 802     init init_tk(loc: Location)
 803     do
 804                 _location = loc
 805     end
 806 end
 807
 808 redef class TMinus
 809     redef fun parser_index: Int
 810     do
 811         return 65
 812     end
 813
 814     init init_tk(loc: Location)
 815     do
 816                 _location = loc
 817     end
 818 end
 819
 820 redef class TStar
 821     redef fun parser_index: Int
 822     do
 823         return 66
 824     end
 825
 826     init init_tk(loc: Location)
 827     do
 828                 _location = loc
 829     end
 830 end
 831
 832 redef class TSlash
 833     redef fun parser_index: Int
 834     do
 835         return 67
 836     end
 837
 838     init init_tk(loc: Location)
 839     do
 840                 _location = loc
 841     end
 842 end
 843
 844 redef class TPercent
 845     redef fun parser_index: Int
 846     do
 847         return 68
 848     end
 849
 850     init init_tk(loc: Location)
 851     do
 852                 _location = loc
 853     end
 854 end
 855
 856 redef class TEq
 857     redef fun parser_index: Int
 858     do
 859         return 69
 860     end
 861
 862     init init_tk(loc: Location)
 863     do
 864                 _location = loc
 865     end
 866 end
 867
 868 redef class TNe
 869     redef fun parser_index: Int
 870     do
 871         return 70
 872     end
 873
 874     init init_tk(loc: Location)
 875     do
 876                 _location = loc
 877     end
 878 end
 879
 880 redef class TLt
 881     redef fun parser_index: Int
 882     do
 883         return 71
 884     end
 885
 886     init init_tk(loc: Location)
 887     do
 888                 _location = loc
 889     end
 890 end
 891
 892 redef class TLe
 893     redef fun parser_index: Int
 894     do
 895         return 72
 896     end
 897
 898     init init_tk(loc: Location)
 899     do
 900                 _location = loc
 901     end
 902 end
 903
 904 redef class TLl
 905     redef fun parser_index: Int
 906     do
 907         return 73
 908     end
 909
 910     init init_tk(loc: Location)
 911     do
 912                 _location = loc
 913     end
 914 end
 915
 916 redef class TGt
 917     redef fun parser_index: Int
 918     do
 919         return 74
 920     end
 921
 922     init init_tk(loc: Location)
 923     do
 924                 _location = loc
 925     end
 926 end
 927
 928 redef class TGe
 929     redef fun parser_index: Int
 930     do
 931         return 75
 932     end
 933
 934     init init_tk(loc: Location)
 935     do
 936                 _location = loc
 937     end
 938 end
 939
 940 redef class TGg
 941     redef fun parser_index: Int
 942     do
 943         return 76
 944     end
 945
 946     init init_tk(loc: Location)
 947     do
 948                 _location = loc
 949     end
 950 end
 951
 952 redef class TStarship
 953     redef fun parser_index: Int
 954     do
 955         return 77
 956     end
 957
 958     init init_tk(loc: Location)
 959     do
 960                 _location = loc
 961     end
 962 end
 963
 964 redef class TBang
 965     redef fun parser_index: Int
 966     do
 967         return 78
 968     end
 969
 970     init init_tk(loc: Location)
 971     do
 972                 _location = loc
 973     end
 974 end
 975
 976 redef class TAt
 977     redef fun parser_index: Int
 978     do
 979         return 79
 980     end
 981
 982     init init_tk(loc: Location)
 983     do
 984                 _location = loc
 985     end
 986 end
 987
 988 redef class TClassid
 989     redef fun parser_index: Int
 990     do
 991         return 80
 992     end
 993
 994     init init_tk(loc: Location)
 995     do
 996                 _location = loc
 997     end
 998 end
 999
1000 redef class TId
1001     redef fun parser_index: Int
1002     do
1003         return 81
1004     end
1005
1006     init init_tk(loc: Location)
1007     do
1008                 _location = loc
1009     end
1010 end
1011
1012 redef class TAttrid
1013     redef fun parser_index: Int
1014     do
1015         return 82
1016     end
1017
1018     init init_tk(loc: Location)
1019     do
1020                 _location = loc
1021     end
1022 end
1023
1024 redef class TNumber
1025     redef fun parser_index: Int
1026     do
1027         return 83
1028     end
1029
1030     init init_tk(loc: Location)
1031     do
1032                 _location = loc
1033     end
1034 end
1035
1036 redef class TFloat
1037     redef fun parser_index: Int
1038     do
1039         return 84
1040     end
1041
1042     init init_tk(loc: Location)
1043     do
1044                 _location = loc
1045     end
1046 end
1047
1048 redef class TString
1049     redef fun parser_index: Int
1050     do
1051         return 85
1052     end
1053
1054     init init_tk(loc: Location)
1055     do
1056                 _location = loc
1057     end
1058 end
1059
1060 redef class TStartString
1061     redef fun parser_index: Int
1062     do
1063         return 86
1064     end
1065
1066     init init_tk(loc: Location)
1067     do
1068                 _location = loc
1069     end
1070 end
1071
1072 redef class TMidString
1073     redef fun parser_index: Int
1074     do
1075         return 87
1076     end
1077
1078     init init_tk(loc: Location)
1079     do
1080                 _location = loc
1081     end
1082 end
1083
1084 redef class TEndString
1085     redef fun parser_index: Int
1086     do
1087         return 88
1088     end
1089
1090     init init_tk(loc: Location)
1091     do
1092                 _location = loc
1093     end
1094 end
1095
1096 redef class TChar
1097     redef fun parser_index: Int
1098     do
1099         return 89
1100     end
1101
1102     init init_tk(loc: Location)
1103     do
1104                 _location = loc
1105     end
1106 end
1107
1108 redef class TBadString
1109     redef fun parser_index: Int
1110     do
1111         return 90
1112     end
1113
1114     init init_tk(loc: Location)
1115     do
1116                 _location = loc
1117     end
1118 end
1119
1120 redef class TBadChar
1121     redef fun parser_index: Int
1122     do
1123         return 91
1124     end
1125
1126     init init_tk(loc: Location)
1127     do
1128                 _location = loc
1129     end
1130 end
1131
1132 redef class TExternCodeSegment
1133     redef fun parser_index: Int
1134     do
1135         return 92
1136     end
1137
1138     init init_tk(loc: Location)
1139     do
1140                 _location = loc
1141     end
1142 end
1143
1144
1145 redef class EOF
1146     redef fun parser_index: Int
1147     do
1148         return 93
1149     end
1150
1151     init init_tk(loc: Location)
1152     do
1153         _text = ""
1154                 _location = loc
1155     end
1156 end
1157
1158 redef class AError
1159     readable var _message: String
1160
1161     init init_error(message: String, loc: Location)
1162     do
1163                 init_tk(loc)
1164                 _message = message
1165     end
1166 end
1167
1168 redef class ALexerError
1169     readable var _string: String
1170
1171     init init_lexer_error(message: String, loc: Location, string: String)
1172     do
1173                 init_error(message, loc)
1174                 _string = string
1175     end
1176 end
1177
1178 redef class AParserError
1179     readable var _token: Token
1180
1181     init init_parser_error(message: String, loc: Location, token: Token)
1182     do
1183                 init_error(message, loc)
1184                 _token = token
1185     end
1186 end
1187
1188 # The lexer extract NIT tokens from an input stream.
1189 # It is better user with the Parser
1190 class Lexer
1191         super TablesCapable
1192         # Last peeked token
1193         var _token: nullable Token
1194
1195         # Lexer current state
1196         var _state: Int = 0
1197
1198         # The source file
1199         readable var _file: SourceFile
1200
1201         # Current character in the stream
1202         var _stream_pos: Int = 0
1203
1204         # Current line number in the input stream
1205         var _line: Int = 0
1206
1207         # Current column in the input stream
1208         var _pos: Int = 0
1209
1210         # Was the last character a cariage-return?
1211         var _cr: Bool = false
1212
1213         # Constante state values
1214         private fun state_initial: Int do return 0 end
1215
1216         # Create a new lexer for a stream (and a name)
1217         init(file: SourceFile)
1218         do
1219                 _file = file
1220         end
1221
1222         # Give the next token (but do not consume it)
1223         fun peek: Token
1224         do
1225                 while _token == null do
1226                         _token = get_token
1227                 end
1228                 return _token.as(not null)
1229         end
1230
1231         # Give and consume the next token
1232         fun next: Token
1233         do
1234                 var result = _token
1235                 while result == null do
1236                         result = get_token
1237                 end
1238                 _token = null
1239                 return result
1240         end
1241
1242         # Primitive method to return a token, or return null if it is discarded
1243         # Is used to implement `peek` and `next`
1244         protected fun get_token: nullable Token
1245         do
1246                 var dfa_state = 0
1247
1248                 var sp = _stream_pos
1249                 var start_stream_pos = sp
1250                 var start_pos = _pos
1251                 var start_line = _line
1252                 var string = _file.string
1253                 var string_len = string.length
1254
1255                 var accept_state = -1
1256                 var accept_token = -1
1257                 var accept_length = -1
1258                 var accept_pos = -1
1259                 var accept_line = -1
1260
1261                 loop
1262                         if sp >= string_len then
1263                                 dfa_state = -1
1264                         else
1265                                 var c = string[sp].ascii
1266                                 sp += 1
1267
1268                                 var cr = _cr
1269                                 var line = _line
1270                                 var pos = _pos
1271                                 if c == 10 then
1272                                         if cr then
1273                                                 cr = false
1274                                                 _file.line_starts[line] = sp
1275                                         else
1276                                                 line = line + 1
1277                                                 pos = 0
1278                                                 _file.line_starts[line] = sp
1279                                         end
1280                                 else if c == 13 then
1281                                         line = line + 1
1282                                         pos = 0
1283                                         cr = true
1284                                         _file.line_starts[line] = sp
1285                                 else
1286                                         pos = pos + 1
1287                                         cr = false
1288                                 end
1289
1290                                 loop
1291                                         var old_state = dfa_state
1292                                         if dfa_state < -1 then
1293                                                 old_state = -2 - dfa_state
1294                                         end
1295
1296                                         dfa_state = -1
1297
1298                                         var low = 0
1299                                         var high = lexer_goto(old_state, 0) - 1
1300
1301                                         if high >= 0 then
1302                                                 while low <= high do
1303                                                         var middle = (low + high) / 2
1304                                                         var offset = middle * 3 + 1 # +1 because length is at 0
1305
1306                                                         if c < lexer_goto(old_state, offset) then
1307                                                                 high = middle - 1
1308                                                         else if c > lexer_goto(old_state, offset+1) then
1309                                                                 low = middle + 1
1310                                                         else
1311                                                                 dfa_state = lexer_goto(old_state, offset+2)
1312                                                                 break
1313                                                         end
1314                                                 end
1315                                         end
1316                                         if dfa_state > -2 then break
1317                                 end
1318
1319                                 _cr = cr
1320                                 _line = line
1321                                 _pos = pos
1322                         end
1323
1324                         if dfa_state >= 0 then
1325                                 var tok = lexer_accept(dfa_state)
1326                                 if tok != -1 then
1327                                         accept_state = dfa_state
1328                                         accept_token = tok
1329                                         accept_length = sp - start_stream_pos
1330                                         accept_pos = _pos
1331                                         accept_line = _line
1332                                 end
1333                         else
1334                                 if accept_state != -1 then
1335                                         var location = new Location(_file, start_line + 1, accept_line + 1, start_pos + 1, accept_pos)
1336                                         _pos = accept_pos
1337                                         _line = accept_line
1338                                         _stream_pos = start_stream_pos + accept_length
1339                                         if accept_token == 0 then
1340                                                 return null
1341                                         end
1342                                         if accept_token == 1 then
1343                                                 return new TEol.init_tk(location)
1344                                         end
1345                                         if accept_token == 2 then
1346                                                 return new TComment.init_tk(location)
1347                                         end
1348                                         if accept_token == 3 then
1349                                                 return new TKwmodule.init_tk(location)
1350                                         end
1351                                         if accept_token == 4 then
1352                                                 return new TKwimport.init_tk(location)
1353                                         end
1354                                         if accept_token == 5 then
1355                                                 return new TKwclass.init_tk(location)
1356                                         end
1357                                         if accept_token == 6 then
1358                                                 return new TKwabstract.init_tk(location)
1359                                         end
1360                                         if accept_token == 7 then
1361                                                 return new TKwinterface.init_tk(location)
1362                                         end
1363                                         if accept_token == 8 then
1364                                                 return new TKwenum.init_tk(location)
1365                                         end
1366                                         if accept_token == 9 then
1367                                                 return new TKwend.init_tk(location)
1368                                         end
1369                                         if accept_token == 10 then
1370                                                 return new TKwmeth.init_tk(location)
1371                                         end
1372                                         if accept_token == 11 then
1373                                                 return new TKwtype.init_tk(location)
1374                                         end
1375                                         if accept_token == 12 then
1376                                                 return new TKwinit.init_tk(location)
1377                                         end
1378                                         if accept_token == 13 then
1379                                                 return new TKwredef.init_tk(location)
1380                                         end
1381                                         if accept_token == 14 then
1382                                                 return new TKwis.init_tk(location)
1383                                         end
1384                                         if accept_token == 15 then
1385                                                 return new TKwdo.init_tk(location)
1386                                         end
1387                                         if accept_token == 16 then
1388                                                 return new TKwreadable.init_tk(location)
1389                                         end
1390                                         if accept_token == 17 then
1391                                                 return new TKwwritable.init_tk(location)
1392                                         end
1393                                         if accept_token == 18 then
1394                                                 return new TKwvar.init_tk(location)
1395                                         end
1396                                         if accept_token == 19 then
1397                                                 return new TKwintern.init_tk(location)
1398                                         end
1399                                         if accept_token == 20 then
1400                                                 return new TKwextern.init_tk(location)
1401                                         end
1402                                         if accept_token == 21 then
1403                                                 return new TKwprotected.init_tk(location)
1404                                         end
1405                                         if accept_token == 22 then
1406                                                 return new TKwprivate.init_tk(location)
1407                                         end
1408                                         if accept_token == 23 then
1409                                                 return new TKwintrude.init_tk(location)
1410                                         end
1411                                         if accept_token == 24 then
1412                                                 return new TKwif.init_tk(location)
1413                                         end
1414                                         if accept_token == 25 then
1415                                                 return new TKwthen.init_tk(location)
1416                                         end
1417                                         if accept_token == 26 then
1418                                                 return new TKwelse.init_tk(location)
1419                                         end
1420                                         if accept_token == 27 then
1421                                                 return new TKwwhile.init_tk(location)
1422                                         end
1423                                         if accept_token == 28 then
1424                                                 return new TKwloop.init_tk(location)
1425                                         end
1426                                         if accept_token == 29 then
1427                                                 return new TKwfor.init_tk(location)
1428                                         end
1429                                         if accept_token == 30 then
1430                                                 return new TKwin.init_tk(location)
1431                                         end
1432                                         if accept_token == 31 then
1433                                                 return new TKwand.init_tk(location)
1434                                         end
1435                                         if accept_token == 32 then
1436                                                 return new TKwor.init_tk(location)
1437                                         end
1438                                         if accept_token == 33 then
1439                                                 return new TKwnot.init_tk(location)
1440                                         end
1441                                         if accept_token == 34 then
1442                                                 return new TKwreturn.init_tk(location)
1443                                         end
1444                                         if accept_token == 35 then
1445                                                 return new TKwcontinue.init_tk(location)
1446                                         end
1447                                         if accept_token == 36 then
1448                                                 return new TKwbreak.init_tk(location)
1449                                         end
1450                                         if accept_token == 37 then
1451                                                 return new TKwabort.init_tk(location)
1452                                         end
1453                                         if accept_token == 38 then
1454                                                 return new TKwassert.init_tk(location)
1455                                         end
1456                                         if accept_token == 39 then
1457                                                 return new TKwnew.init_tk(location)
1458                                         end
1459                                         if accept_token == 40 then
1460                                                 return new TKwisa.init_tk(location)
1461                                         end
1462                                         if accept_token == 41 then
1463                                                 return new TKwonce.init_tk(location)
1464                                         end
1465                                         if accept_token == 42 then
1466                                                 return new TKwsuper.init_tk(location)
1467                                         end
1468                                         if accept_token == 43 then
1469                                                 return new TKwself.init_tk(location)
1470                                         end
1471                                         if accept_token == 44 then
1472                                                 return new TKwtrue.init_tk(location)
1473                                         end
1474                                         if accept_token == 45 then
1475                                                 return new TKwfalse.init_tk(location)
1476                                         end
1477                                         if accept_token == 46 then
1478                                                 return new TKwnull.init_tk(location)
1479                                         end
1480                                         if accept_token == 47 then
1481                                                 return new TKwas.init_tk(location)
1482                                         end
1483                                         if accept_token == 48 then
1484                                                 return new TKwnullable.init_tk(location)
1485                                         end
1486                                         if accept_token == 49 then
1487                                                 return new TKwisset.init_tk(location)
1488                                         end
1489                                         if accept_token == 50 then
1490                                                 return new TKwlabel.init_tk(location)
1491                                         end
1492                                         if accept_token == 51 then
1493                                                 return new TKwdebug.init_tk(location)
1494                                         end
1495                                         if accept_token == 52 then
1496                                                 return new TOpar.init_tk(location)
1497                                         end
1498                                         if accept_token == 53 then
1499                                                 return new TCpar.init_tk(location)
1500                                         end
1501                                         if accept_token == 54 then
1502                                                 return new TObra.init_tk(location)
1503                                         end
1504                                         if accept_token == 55 then
1505                                                 return new TCbra.init_tk(location)
1506                                         end
1507                                         if accept_token == 56 then
1508                                                 return new TComma.init_tk(location)
1509                                         end
1510                                         if accept_token == 57 then
1511                                                 return new TColumn.init_tk(location)
1512                                         end
1513                                         if accept_token == 58 then
1514                                                 return new TQuad.init_tk(location)
1515                                         end
1516                                         if accept_token == 59 then
1517                                                 return new TAssign.init_tk(location)
1518                                         end
1519                                         if accept_token == 60 then
1520                                                 return new TPluseq.init_tk(location)
1521                                         end
1522                                         if accept_token == 61 then
1523                                                 return new TMinuseq.init_tk(location)
1524                                         end
1525                                         if accept_token == 62 then
1526                                                 return new TDotdotdot.init_tk(location)
1527                                         end
1528                                         if accept_token == 63 then
1529                                                 return new TDotdot.init_tk(location)
1530                                         end
1531                                         if accept_token == 64 then
1532                                                 return new TDot.init_tk(location)
1533                                         end
1534                                         if accept_token == 65 then
1535                                                 return new TPlus.init_tk(location)
1536                                         end
1537                                         if accept_token == 66 then
1538                                                 return new TMinus.init_tk(location)
1539                                         end
1540                                         if accept_token == 67 then
1541                                                 return new TStar.init_tk(location)
1542                                         end
1543                                         if accept_token == 68 then
1544                                                 return new TSlash.init_tk(location)
1545                                         end
1546                                         if accept_token == 69 then
1547                                                 return new TPercent.init_tk(location)
1548                                         end
1549                                         if accept_token == 70 then
1550                                                 return new TEq.init_tk(location)
1551                                         end
1552                                         if accept_token == 71 then
1553                                                 return new TNe.init_tk(location)
1554                                         end
1555                                         if accept_token == 72 then
1556                                                 return new TLt.init_tk(location)
1557                                         end
1558                                         if accept_token == 73 then
1559                                                 return new TLe.init_tk(location)
1560                                         end
1561                                         if accept_token == 74 then
1562                                                 return new TLl.init_tk(location)
1563                                         end
1564                                         if accept_token == 75 then
1565                                                 return new TGt.init_tk(location)
1566                                         end
1567                                         if accept_token == 76 then
1568                                                 return new TGe.init_tk(location)
1569                                         end
1570                                         if accept_token == 77 then
1571                                                 return new TGg.init_tk(location)
1572                                         end
1573                                         if accept_token == 78 then
1574                                                 return new TStarship.init_tk(location)
1575                                         end
1576                                         if accept_token == 79 then
1577                                                 return new TBang.init_tk(location)
1578                                         end
1579                                         if accept_token == 80 then
1580                                                 return new TAt.init_tk(location)
1581                                         end
1582                                         if accept_token == 81 then
1583                                                 return new TClassid.init_tk(location)
1584                                         end
1585                                         if accept_token == 82 then
1586                                                 return new TId.init_tk(location)
1587                                         end
1588                                         if accept_token == 83 then
1589                                                 return new TAttrid.init_tk(location)
1590                                         end
1591                                         if accept_token == 84 then
1592                                                 return new TNumber.init_tk(location)
1593                                         end
1594                                         if accept_token == 85 then
1595                                                 return new TFloat.init_tk(location)
1596                                         end
1597                                         if accept_token == 86 then
1598                                                 return new TString.init_tk(location)
1599                                         end
1600                                         if accept_token == 87 then
1601                                                 return new TStartString.init_tk(location)
1602                                         end
1603                                         if accept_token == 88 then
1604                                                 return new TMidString.init_tk(location)
1605                                         end
1606                                         if accept_token == 89 then
1607                                                 return new TEndString.init_tk(location)
1608                                         end
1609                                         if accept_token == 90 then
1610                                                 return new TChar.init_tk(location)
1611                                         end
1612                                         if accept_token == 91 then
1613                                                 return new TBadString.init_tk(location)
1614                                         end
1615                                         if accept_token == 92 then
1616                                                 return new TBadChar.init_tk(location)
1617                                         end
1618                                         if accept_token == 93 then
1619                                                 return new TExternCodeSegment.init_tk(location)
1620                                         end
1621                                 else
1622                                         _stream_pos = sp
1623                                         var location = new Location(_file, start_line + 1, start_line + 1, start_pos + 1, start_pos + 1)
1624                                         if sp > start_stream_pos then
1625                                                 var text = string.substring(start_stream_pos, sp-start_stream_pos)
1626                                                 var token = new ALexerError.init_lexer_error("Syntax error: unknown token {text}.", location, text)
1627                                                 return token
1628                                         else
1629                                                 var token = new EOF.init_tk(location)
1630                                                 return token
1631                                         end
1632                                 end
1633                         end
1634                 end
1635         end
1636 end
1637