src/parser/lexer.nit

   1 # Lexer and its tokens.
   2 # This file was generated by SableCC (http://www.sablecc.org/).
   3 module lexer
   4
   5 intrude import parser_nodes
   6 private import tables
   7
   8 redef class Token
   9     var _text: nullable String
  10
  11     redef fun text
  12     do
  13         var res = _text
  14         if res != null then return res
  15         res = location.text
  16         _text = res
  17         return res
  18     end
  19
  20     redef fun text=(text)
  21     do
  22         _text = text
  23     end
  24
  25     fun parser_index: Int is abstract
  26 end
  27
  28 redef class TEol
  29     redef fun parser_index: Int
  30     do
  31         return 0
  32     end
  33
  34     init init_tk(loc: Location)
  35     do
  36                 _location = loc
  37     end
  38 end
  39
  40 redef class TComment
  41     redef fun parser_index: Int
  42     do
  43         return 1
  44     end
  45
  46     init init_tk(loc: Location)
  47     do
  48                 _location = loc
  49     end
  50 end
  51
  52 redef class TKwpackage
  53     redef fun parser_index: Int
  54     do
  55         return 2
  56     end
  57
  58     init init_tk(loc: Location)
  59     do
  60                 _location = loc
  61     end
  62 end
  63
  64 redef class TKwmodule
  65     redef fun parser_index: Int
  66     do
  67         return 3
  68     end
  69
  70     init init_tk(loc: Location)
  71     do
  72                 _location = loc
  73     end
  74 end
  75
  76 redef class TKwimport
  77     redef fun parser_index: Int
  78     do
  79         return 4
  80     end
  81
  82     init init_tk(loc: Location)
  83     do
  84                 _location = loc
  85     end
  86 end
  87
  88 redef class TKwclass
  89     redef fun parser_index: Int
  90     do
  91         return 5
  92     end
  93
  94     init init_tk(loc: Location)
  95     do
  96                 _location = loc
  97     end
  98 end
  99
 100 redef class TKwabstract
 101     redef fun parser_index: Int
 102     do
 103         return 6
 104     end
 105
 106     init init_tk(loc: Location)
 107     do
 108                 _location = loc
 109     end
 110 end
 111
 112 redef class TKwinterface
 113     redef fun parser_index: Int
 114     do
 115         return 7
 116     end
 117
 118     init init_tk(loc: Location)
 119     do
 120                 _location = loc
 121     end
 122 end
 123
 124 redef class TKwenum
 125     redef fun parser_index: Int
 126     do
 127         return 8
 128     end
 129
 130     init init_tk(loc: Location)
 131     do
 132                 _location = loc
 133     end
 134 end
 135
 136 redef class TKwend
 137     redef fun parser_index: Int
 138     do
 139         return 9
 140     end
 141
 142     init init_tk(loc: Location)
 143     do
 144                 _location = loc
 145     end
 146 end
 147
 148 redef class TKwmeth
 149     redef fun parser_index: Int
 150     do
 151         return 10
 152     end
 153
 154     init init_tk(loc: Location)
 155     do
 156                 _location = loc
 157     end
 158 end
 159
 160 redef class TKwtype
 161     redef fun parser_index: Int
 162     do
 163         return 11
 164     end
 165
 166     init init_tk(loc: Location)
 167     do
 168                 _location = loc
 169     end
 170 end
 171
 172 redef class TKwinit
 173     redef fun parser_index: Int
 174     do
 175         return 12
 176     end
 177
 178     init init_tk(loc: Location)
 179     do
 180                 _location = loc
 181     end
 182 end
 183
 184 redef class TKwredef
 185     redef fun parser_index: Int
 186     do
 187         return 13
 188     end
 189
 190     init init_tk(loc: Location)
 191     do
 192                 _location = loc
 193     end
 194 end
 195
 196 redef class TKwis
 197     redef fun parser_index: Int
 198     do
 199         return 14
 200     end
 201
 202     init init_tk(loc: Location)
 203     do
 204                 _location = loc
 205     end
 206 end
 207
 208 redef class TKwdo
 209     redef fun parser_index: Int
 210     do
 211         return 15
 212     end
 213
 214     init init_tk(loc: Location)
 215     do
 216                 _location = loc
 217     end
 218 end
 219
 220 redef class TKwreadable
 221     redef fun parser_index: Int
 222     do
 223         return 16
 224     end
 225
 226     init init_tk(loc: Location)
 227     do
 228                 _location = loc
 229     end
 230 end
 231
 232 redef class TKwwritable
 233     redef fun parser_index: Int
 234     do
 235         return 17
 236     end
 237
 238     init init_tk(loc: Location)
 239     do
 240                 _location = loc
 241     end
 242 end
 243
 244 redef class TKwvar
 245     redef fun parser_index: Int
 246     do
 247         return 18
 248     end
 249
 250     init init_tk(loc: Location)
 251     do
 252                 _location = loc
 253     end
 254 end
 255
 256 redef class TKwintern
 257     redef fun parser_index: Int
 258     do
 259         return 19
 260     end
 261
 262     init init_tk(loc: Location)
 263     do
 264                 _location = loc
 265     end
 266 end
 267
 268 redef class TKwextern
 269     redef fun parser_index: Int
 270     do
 271         return 20
 272     end
 273
 274     init init_tk(loc: Location)
 275     do
 276                 _location = loc
 277     end
 278 end
 279
 280 redef class TKwprotected
 281     redef fun parser_index: Int
 282     do
 283         return 21
 284     end
 285
 286     init init_tk(loc: Location)
 287     do
 288                 _location = loc
 289     end
 290 end
 291
 292 redef class TKwprivate
 293     redef fun parser_index: Int
 294     do
 295         return 22
 296     end
 297
 298     init init_tk(loc: Location)
 299     do
 300                 _location = loc
 301     end
 302 end
 303
 304 redef class TKwintrude
 305     redef fun parser_index: Int
 306     do
 307         return 23
 308     end
 309
 310     init init_tk(loc: Location)
 311     do
 312                 _location = loc
 313     end
 314 end
 315
 316 redef class TKwif
 317     redef fun parser_index: Int
 318     do
 319         return 24
 320     end
 321
 322     init init_tk(loc: Location)
 323     do
 324                 _location = loc
 325     end
 326 end
 327
 328 redef class TKwthen
 329     redef fun parser_index: Int
 330     do
 331         return 25
 332     end
 333
 334     init init_tk(loc: Location)
 335     do
 336                 _location = loc
 337     end
 338 end
 339
 340 redef class TKwelse
 341     redef fun parser_index: Int
 342     do
 343         return 26
 344     end
 345
 346     init init_tk(loc: Location)
 347     do
 348                 _location = loc
 349     end
 350 end
 351
 352 redef class TKwwhile
 353     redef fun parser_index: Int
 354     do
 355         return 27
 356     end
 357
 358     init init_tk(loc: Location)
 359     do
 360                 _location = loc
 361     end
 362 end
 363
 364 redef class TKwloop
 365     redef fun parser_index: Int
 366     do
 367         return 28
 368     end
 369
 370     init init_tk(loc: Location)
 371     do
 372                 _location = loc
 373     end
 374 end
 375
 376 redef class TKwfor
 377     redef fun parser_index: Int
 378     do
 379         return 29
 380     end
 381
 382     init init_tk(loc: Location)
 383     do
 384                 _location = loc
 385     end
 386 end
 387
 388 redef class TKwin
 389     redef fun parser_index: Int
 390     do
 391         return 30
 392     end
 393
 394     init init_tk(loc: Location)
 395     do
 396                 _location = loc
 397     end
 398 end
 399
 400 redef class TKwand
 401     redef fun parser_index: Int
 402     do
 403         return 31
 404     end
 405
 406     init init_tk(loc: Location)
 407     do
 408                 _location = loc
 409     end
 410 end
 411
 412 redef class TKwor
 413     redef fun parser_index: Int
 414     do
 415         return 32
 416     end
 417
 418     init init_tk(loc: Location)
 419     do
 420                 _location = loc
 421     end
 422 end
 423
 424 redef class TKwnot
 425     redef fun parser_index: Int
 426     do
 427         return 33
 428     end
 429
 430     init init_tk(loc: Location)
 431     do
 432                 _location = loc
 433     end
 434 end
 435
 436 redef class TKwimplies
 437     redef fun parser_index: Int
 438     do
 439         return 34
 440     end
 441
 442     init init_tk(loc: Location)
 443     do
 444                 _location = loc
 445     end
 446 end
 447
 448 redef class TKwreturn
 449     redef fun parser_index: Int
 450     do
 451         return 35
 452     end
 453
 454     init init_tk(loc: Location)
 455     do
 456                 _location = loc
 457     end
 458 end
 459
 460 redef class TKwcontinue
 461     redef fun parser_index: Int
 462     do
 463         return 36
 464     end
 465
 466     init init_tk(loc: Location)
 467     do
 468                 _location = loc
 469     end
 470 end
 471
 472 redef class TKwbreak
 473     redef fun parser_index: Int
 474     do
 475         return 37
 476     end
 477
 478     init init_tk(loc: Location)
 479     do
 480                 _location = loc
 481     end
 482 end
 483
 484 redef class TKwabort
 485     redef fun parser_index: Int
 486     do
 487         return 38
 488     end
 489
 490     init init_tk(loc: Location)
 491     do
 492                 _location = loc
 493     end
 494 end
 495
 496 redef class TKwassert
 497     redef fun parser_index: Int
 498     do
 499         return 39
 500     end
 501
 502     init init_tk(loc: Location)
 503     do
 504                 _location = loc
 505     end
 506 end
 507
 508 redef class TKwnew
 509     redef fun parser_index: Int
 510     do
 511         return 40
 512     end
 513
 514     init init_tk(loc: Location)
 515     do
 516                 _location = loc
 517     end
 518 end
 519
 520 redef class TKwisa
 521     redef fun parser_index: Int
 522     do
 523         return 41
 524     end
 525
 526     init init_tk(loc: Location)
 527     do
 528                 _location = loc
 529     end
 530 end
 531
 532 redef class TKwonce
 533     redef fun parser_index: Int
 534     do
 535         return 42
 536     end
 537
 538     init init_tk(loc: Location)
 539     do
 540                 _location = loc
 541     end
 542 end
 543
 544 redef class TKwsuper
 545     redef fun parser_index: Int
 546     do
 547         return 43
 548     end
 549
 550     init init_tk(loc: Location)
 551     do
 552                 _location = loc
 553     end
 554 end
 555
 556 redef class TKwself
 557     redef fun parser_index: Int
 558     do
 559         return 44
 560     end
 561
 562     init init_tk(loc: Location)
 563     do
 564                 _location = loc
 565     end
 566 end
 567
 568 redef class TKwtrue
 569     redef fun parser_index: Int
 570     do
 571         return 45
 572     end
 573
 574     init init_tk(loc: Location)
 575     do
 576                 _location = loc
 577     end
 578 end
 579
 580 redef class TKwfalse
 581     redef fun parser_index: Int
 582     do
 583         return 46
 584     end
 585
 586     init init_tk(loc: Location)
 587     do
 588                 _location = loc
 589     end
 590 end
 591
 592 redef class TKwnull
 593     redef fun parser_index: Int
 594     do
 595         return 47
 596     end
 597
 598     init init_tk(loc: Location)
 599     do
 600                 _location = loc
 601     end
 602 end
 603
 604 redef class TKwas
 605     redef fun parser_index: Int
 606     do
 607         return 48
 608     end
 609
 610     init init_tk(loc: Location)
 611     do
 612                 _location = loc
 613     end
 614 end
 615
 616 redef class TKwnullable
 617     redef fun parser_index: Int
 618     do
 619         return 49
 620     end
 621
 622     init init_tk(loc: Location)
 623     do
 624                 _location = loc
 625     end
 626 end
 627
 628 redef class TKwisset
 629     redef fun parser_index: Int
 630     do
 631         return 50
 632     end
 633
 634     init init_tk(loc: Location)
 635     do
 636                 _location = loc
 637     end
 638 end
 639
 640 redef class TKwlabel
 641     redef fun parser_index: Int
 642     do
 643         return 51
 644     end
 645
 646     init init_tk(loc: Location)
 647     do
 648                 _location = loc
 649     end
 650 end
 651
 652 redef class TKwdebug
 653     redef fun parser_index: Int
 654     do
 655         return 52
 656     end
 657
 658     init init_tk(loc: Location)
 659     do
 660                 _location = loc
 661     end
 662 end
 663
 664 redef class TOpar
 665     redef fun parser_index: Int
 666     do
 667         return 53
 668     end
 669
 670     init init_tk(loc: Location)
 671     do
 672                 _location = loc
 673     end
 674 end
 675
 676 redef class TCpar
 677     redef fun parser_index: Int
 678     do
 679         return 54
 680     end
 681
 682     init init_tk(loc: Location)
 683     do
 684                 _location = loc
 685     end
 686 end
 687
 688 redef class TObra
 689     redef fun parser_index: Int
 690     do
 691         return 55
 692     end
 693
 694     init init_tk(loc: Location)
 695     do
 696                 _location = loc
 697     end
 698 end
 699
 700 redef class TCbra
 701     redef fun parser_index: Int
 702     do
 703         return 56
 704     end
 705
 706     init init_tk(loc: Location)
 707     do
 708                 _location = loc
 709     end
 710 end
 711
 712 redef class TComma
 713     redef fun parser_index: Int
 714     do
 715         return 57
 716     end
 717
 718     init init_tk(loc: Location)
 719     do
 720                 _location = loc
 721     end
 722 end
 723
 724 redef class TColumn
 725     redef fun parser_index: Int
 726     do
 727         return 58
 728     end
 729
 730     init init_tk(loc: Location)
 731     do
 732                 _location = loc
 733     end
 734 end
 735
 736 redef class TQuad
 737     redef fun parser_index: Int
 738     do
 739         return 59
 740     end
 741
 742     init init_tk(loc: Location)
 743     do
 744                 _location = loc
 745     end
 746 end
 747
 748 redef class TAssign
 749     redef fun parser_index: Int
 750     do
 751         return 60
 752     end
 753
 754     init init_tk(loc: Location)
 755     do
 756                 _location = loc
 757     end
 758 end
 759
 760 redef class TPluseq
 761     redef fun parser_index: Int
 762     do
 763         return 61
 764     end
 765
 766     init init_tk(loc: Location)
 767     do
 768                 _location = loc
 769     end
 770 end
 771
 772 redef class TMinuseq
 773     redef fun parser_index: Int
 774     do
 775         return 62
 776     end
 777
 778     init init_tk(loc: Location)
 779     do
 780                 _location = loc
 781     end
 782 end
 783
 784 redef class TDotdotdot
 785     redef fun parser_index: Int
 786     do
 787         return 63
 788     end
 789
 790     init init_tk(loc: Location)
 791     do
 792                 _location = loc
 793     end
 794 end
 795
 796 redef class TDotdot
 797     redef fun parser_index: Int
 798     do
 799         return 64
 800     end
 801
 802     init init_tk(loc: Location)
 803     do
 804                 _location = loc
 805     end
 806 end
 807
 808 redef class TDot
 809     redef fun parser_index: Int
 810     do
 811         return 65
 812     end
 813
 814     init init_tk(loc: Location)
 815     do
 816                 _location = loc
 817     end
 818 end
 819
 820 redef class TPlus
 821     redef fun parser_index: Int
 822     do
 823         return 66
 824     end
 825
 826     init init_tk(loc: Location)
 827     do
 828                 _location = loc
 829     end
 830 end
 831
 832 redef class TMinus
 833     redef fun parser_index: Int
 834     do
 835         return 67
 836     end
 837
 838     init init_tk(loc: Location)
 839     do
 840                 _location = loc
 841     end
 842 end
 843
 844 redef class TStar
 845     redef fun parser_index: Int
 846     do
 847         return 68
 848     end
 849
 850     init init_tk(loc: Location)
 851     do
 852                 _location = loc
 853     end
 854 end
 855
 856 redef class TSlash
 857     redef fun parser_index: Int
 858     do
 859         return 69
 860     end
 861
 862     init init_tk(loc: Location)
 863     do
 864                 _location = loc
 865     end
 866 end
 867
 868 redef class TPercent
 869     redef fun parser_index: Int
 870     do
 871         return 70
 872     end
 873
 874     init init_tk(loc: Location)
 875     do
 876                 _location = loc
 877     end
 878 end
 879
 880 redef class TEq
 881     redef fun parser_index: Int
 882     do
 883         return 71
 884     end
 885
 886     init init_tk(loc: Location)
 887     do
 888                 _location = loc
 889     end
 890 end
 891
 892 redef class TNe
 893     redef fun parser_index: Int
 894     do
 895         return 72
 896     end
 897
 898     init init_tk(loc: Location)
 899     do
 900                 _location = loc
 901     end
 902 end
 903
 904 redef class TLt
 905     redef fun parser_index: Int
 906     do
 907         return 73
 908     end
 909
 910     init init_tk(loc: Location)
 911     do
 912                 _location = loc
 913     end
 914 end
 915
 916 redef class TLe
 917     redef fun parser_index: Int
 918     do
 919         return 74
 920     end
 921
 922     init init_tk(loc: Location)
 923     do
 924                 _location = loc
 925     end
 926 end
 927
 928 redef class TLl
 929     redef fun parser_index: Int
 930     do
 931         return 75
 932     end
 933
 934     init init_tk(loc: Location)
 935     do
 936                 _location = loc
 937     end
 938 end
 939
 940 redef class TGt
 941     redef fun parser_index: Int
 942     do
 943         return 76
 944     end
 945
 946     init init_tk(loc: Location)
 947     do
 948                 _location = loc
 949     end
 950 end
 951
 952 redef class TGe
 953     redef fun parser_index: Int
 954     do
 955         return 77
 956     end
 957
 958     init init_tk(loc: Location)
 959     do
 960                 _location = loc
 961     end
 962 end
 963
 964 redef class TGg
 965     redef fun parser_index: Int
 966     do
 967         return 78
 968     end
 969
 970     init init_tk(loc: Location)
 971     do
 972                 _location = loc
 973     end
 974 end
 975
 976 redef class TStarship
 977     redef fun parser_index: Int
 978     do
 979         return 79
 980     end
 981
 982     init init_tk(loc: Location)
 983     do
 984                 _location = loc
 985     end
 986 end
 987
 988 redef class TBang
 989     redef fun parser_index: Int
 990     do
 991         return 80
 992     end
 993
 994     init init_tk(loc: Location)
 995     do
 996                 _location = loc
 997     end
 998 end
 999
1000 redef class TAt
1001     redef fun parser_index: Int
1002     do
1003         return 81
1004     end
1005
1006     init init_tk(loc: Location)
1007     do
1008                 _location = loc
1009     end
1010 end
1011
1012 redef class TClassid
1013     redef fun parser_index: Int
1014     do
1015         return 82
1016     end
1017
1018     init init_tk(loc: Location)
1019     do
1020                 _location = loc
1021     end
1022 end
1023
1024 redef class TId
1025     redef fun parser_index: Int
1026     do
1027         return 83
1028     end
1029
1030     init init_tk(loc: Location)
1031     do
1032                 _location = loc
1033     end
1034 end
1035
1036 redef class TAttrid
1037     redef fun parser_index: Int
1038     do
1039         return 84
1040     end
1041
1042     init init_tk(loc: Location)
1043     do
1044                 _location = loc
1045     end
1046 end
1047
1048 redef class TNumber
1049     redef fun parser_index: Int
1050     do
1051         return 85
1052     end
1053
1054     init init_tk(loc: Location)
1055     do
1056                 _location = loc
1057     end
1058 end
1059
1060 redef class TFloat
1061     redef fun parser_index: Int
1062     do
1063         return 86
1064     end
1065
1066     init init_tk(loc: Location)
1067     do
1068                 _location = loc
1069     end
1070 end
1071
1072 redef class TString
1073     redef fun parser_index: Int
1074     do
1075         return 87
1076     end
1077
1078     init init_tk(loc: Location)
1079     do
1080                 _location = loc
1081     end
1082 end
1083
1084 redef class TStartString
1085     redef fun parser_index: Int
1086     do
1087         return 88
1088     end
1089
1090     init init_tk(loc: Location)
1091     do
1092                 _location = loc
1093     end
1094 end
1095
1096 redef class TMidString
1097     redef fun parser_index: Int
1098     do
1099         return 89
1100     end
1101
1102     init init_tk(loc: Location)
1103     do
1104                 _location = loc
1105     end
1106 end
1107
1108 redef class TEndString
1109     redef fun parser_index: Int
1110     do
1111         return 90
1112     end
1113
1114     init init_tk(loc: Location)
1115     do
1116                 _location = loc
1117     end
1118 end
1119
1120 redef class TChar
1121     redef fun parser_index: Int
1122     do
1123         return 91
1124     end
1125
1126     init init_tk(loc: Location)
1127     do
1128                 _location = loc
1129     end
1130 end
1131
1132 redef class TBadString
1133     redef fun parser_index: Int
1134     do
1135         return 92
1136     end
1137
1138     init init_tk(loc: Location)
1139     do
1140                 _location = loc
1141     end
1142 end
1143
1144 redef class TBadChar
1145     redef fun parser_index: Int
1146     do
1147         return 93
1148     end
1149
1150     init init_tk(loc: Location)
1151     do
1152                 _location = loc
1153     end
1154 end
1155
1156 redef class TExternCodeSegment
1157     redef fun parser_index: Int
1158     do
1159         return 94
1160     end
1161
1162     init init_tk(loc: Location)
1163     do
1164                 _location = loc
1165     end
1166 end
1167
1168
1169 redef class EOF
1170     redef fun parser_index: Int
1171     do
1172         return 95
1173     end
1174
1175     init init_tk(loc: Location)
1176     do
1177         _text = ""
1178                 _location = loc
1179     end
1180 end
1181
1182 redef class AError
1183     readable var _message: String
1184
1185     init init_error(message: String, loc: Location)
1186     do
1187                 init_tk(loc)
1188                 _message = message
1189     end
1190 end
1191
1192 redef class ALexerError
1193     readable var _string: String
1194
1195     init init_lexer_error(message: String, loc: Location, string: String)
1196     do
1197                 init_error(message, loc)
1198                 _string = string
1199     end
1200 end
1201
1202 redef class AParserError
1203     readable var _token: Token
1204
1205     init init_parser_error(message: String, loc: Location, token: Token)
1206     do
1207                 init_error(message, loc)
1208                 _token = token
1209     end
1210 end
1211
1212 # The lexer extract NIT tokens from an input stream.
1213 # It is better user with the Parser
1214 class Lexer
1215         super TablesCapable
1216         # Last peeked token
1217         var _token: nullable Token
1218
1219         # Lexer current state
1220         var _state: Int = 0
1221
1222         # The source file
1223         readable var _file: SourceFile
1224
1225         # Current character in the stream
1226         var _stream_pos: Int = 0
1227
1228         # Current line number in the input stream
1229         var _line: Int = 0
1230
1231         # Current column in the input stream
1232         var _pos: Int = 0
1233
1234         # Was the last character a cariage-return?
1235         var _cr: Bool = false
1236
1237         # Constante state values
1238         private fun state_initial: Int do return 0 end
1239
1240         # Create a new lexer for a stream (and a name)
1241         init(file: SourceFile)
1242         do
1243                 _file = file
1244         end
1245
1246         # The last peeked token to chain them
1247         private var last_token: nullable Token = null
1248
1249         # Give the next token (but do not consume it)
1250         fun peek: Token
1251         do
1252                 var t = _token
1253                 if t != null then return t
1254
1255                 t = get_token
1256                 while t == null do t = get_token
1257
1258                 var l = last_token
1259                 if l != null then
1260                         l.next_token = t
1261                         t.prev_token = l
1262                 end
1263
1264                 last_token = t
1265                 _token = t
1266                 return t
1267         end
1268
1269         # Give and consume the next token
1270         fun next: Token
1271         do
1272                 var result = peek
1273                 _token = null
1274                 return result
1275         end
1276
1277         # Primitive method to return a token, or return null if it is discarded
1278         # Is used to implement `peek` and `next`
1279         protected fun get_token: nullable Token
1280         do
1281                 var dfa_state = 0
1282
1283                 var sp = _stream_pos
1284                 var start_stream_pos = sp
1285                 var start_pos = _pos
1286                 var start_line = _line
1287                 var string = _file.string
1288                 var string_len = string.length
1289
1290                 var accept_state = -1
1291                 var accept_token = -1
1292                 var accept_length = -1
1293                 var accept_pos = -1
1294                 var accept_line = -1
1295
1296                 loop
1297                         if sp >= string_len then
1298                                 dfa_state = -1
1299                         else
1300                                 var c = string.chars[sp].ascii
1301                                 sp += 1
1302
1303                                 var cr = _cr
1304                                 var line = _line
1305                                 var pos = _pos
1306                                 if c == 10 then
1307                                         if cr then
1308                                                 cr = false
1309                                                 _file.line_starts[line] = sp
1310                                         else
1311                                                 line = line + 1
1312                                                 pos = 0
1313                                                 _file.line_starts[line] = sp
1314                                         end
1315                                 else if c == 13 then
1316                                         line = line + 1
1317                                         pos = 0
1318                                         cr = true
1319                                         _file.line_starts[line] = sp
1320                                 else
1321                                         pos = pos + 1
1322                                         cr = false
1323                                 end
1324
1325                                 loop
1326                                         var old_state = dfa_state
1327                                         if dfa_state < -1 then
1328                                                 old_state = -2 - dfa_state
1329                                         end
1330
1331                                         dfa_state = -1
1332
1333                                         var low = 0
1334                                         var high = lexer_goto(old_state, 0) - 1
1335
1336                                         if high >= 0 then
1337                                                 while low <= high do
1338                                                         var middle = (low + high) / 2
1339                                                         var offset = middle * 3 + 1 # +1 because length is at 0
1340
1341                                                         if c < lexer_goto(old_state, offset) then
1342                                                                 high = middle - 1
1343                                                         else if c > lexer_goto(old_state, offset+1) then
1344                                                                 low = middle + 1
1345                                                         else
1346                                                                 dfa_state = lexer_goto(old_state, offset+2)
1347                                                                 break
1348                                                         end
1349                                                 end
1350                                         end
1351                                         if dfa_state > -2 then break
1352                                 end
1353
1354                                 _cr = cr
1355                                 _line = line
1356                                 _pos = pos
1357                         end
1358
1359                         if dfa_state >= 0 then
1360                                 var tok = lexer_accept(dfa_state)
1361                                 if tok != -1 then
1362                                         accept_state = dfa_state
1363                                         accept_token = tok
1364                                         accept_length = sp - start_stream_pos
1365                                         accept_pos = _pos
1366                                         accept_line = _line
1367                                 end
1368                         else
1369                                 if accept_state != -1 then
1370                                         var location = new Location(_file, start_line + 1, accept_line + 1, start_pos + 1, accept_pos)
1371                                         _pos = accept_pos
1372                                         _line = accept_line
1373                                         _stream_pos = start_stream_pos + accept_length
1374                                         if accept_token == 0 then
1375                                                 return null
1376                                         end
1377                                         if accept_token == 1 then
1378                                                 return new TEol.init_tk(location)
1379                                         end
1380                                         if accept_token == 2 then
1381                                                 return new TComment.init_tk(location)
1382                                         end
1383                                         if accept_token == 3 then
1384                                                 return new TKwpackage.init_tk(location)
1385                                         end
1386                                         if accept_token == 4 then
1387                                                 return new TKwmodule.init_tk(location)
1388                                         end
1389                                         if accept_token == 5 then
1390                                                 return new TKwimport.init_tk(location)
1391                                         end
1392                                         if accept_token == 6 then
1393                                                 return new TKwclass.init_tk(location)
1394                                         end
1395                                         if accept_token == 7 then
1396                                                 return new TKwabstract.init_tk(location)
1397                                         end
1398                                         if accept_token == 8 then
1399                                                 return new TKwinterface.init_tk(location)
1400                                         end
1401                                         if accept_token == 9 then
1402                                                 return new TKwenum.init_tk(location)
1403                                         end
1404                                         if accept_token == 10 then
1405                                                 return new TKwend.init_tk(location)
1406                                         end
1407                                         if accept_token == 11 then
1408                                                 return new TKwmeth.init_tk(location)
1409                                         end
1410                                         if accept_token == 12 then
1411                                                 return new TKwtype.init_tk(location)
1412                                         end
1413                                         if accept_token == 13 then
1414                                                 return new TKwinit.init_tk(location)
1415                                         end
1416                                         if accept_token == 14 then
1417                                                 return new TKwredef.init_tk(location)
1418                                         end
1419                                         if accept_token == 15 then
1420                                                 return new TKwis.init_tk(location)
1421                                         end
1422                                         if accept_token == 16 then
1423                                                 return new TKwdo.init_tk(location)
1424                                         end
1425                                         if accept_token == 17 then
1426                                                 return new TKwreadable.init_tk(location)
1427                                         end
1428                                         if accept_token == 18 then
1429                                                 return new TKwwritable.init_tk(location)
1430                                         end
1431                                         if accept_token == 19 then
1432                                                 return new TKwvar.init_tk(location)
1433                                         end
1434                                         if accept_token == 20 then
1435                                                 return new TKwintern.init_tk(location)
1436                                         end
1437                                         if accept_token == 21 then
1438                                                 return new TKwextern.init_tk(location)
1439                                         end
1440                                         if accept_token == 22 then
1441                                                 return new TKwprotected.init_tk(location)
1442                                         end
1443                                         if accept_token == 23 then
1444                                                 return new TKwprivate.init_tk(location)
1445                                         end
1446                                         if accept_token == 24 then
1447                                                 return new TKwintrude.init_tk(location)
1448                                         end
1449                                         if accept_token == 25 then
1450                                                 return new TKwif.init_tk(location)
1451                                         end
1452                                         if accept_token == 26 then
1453                                                 return new TKwthen.init_tk(location)
1454                                         end
1455                                         if accept_token == 27 then
1456                                                 return new TKwelse.init_tk(location)
1457                                         end
1458                                         if accept_token == 28 then
1459                                                 return new TKwwhile.init_tk(location)
1460                                         end
1461                                         if accept_token == 29 then
1462                                                 return new TKwloop.init_tk(location)
1463                                         end
1464                                         if accept_token == 30 then
1465                                                 return new TKwfor.init_tk(location)
1466                                         end
1467                                         if accept_token == 31 then
1468                                                 return new TKwin.init_tk(location)
1469                                         end
1470                                         if accept_token == 32 then
1471                                                 return new TKwand.init_tk(location)
1472                                         end
1473                                         if accept_token == 33 then
1474                                                 return new TKwor.init_tk(location)
1475                                         end
1476                                         if accept_token == 34 then
1477                                                 return new TKwnot.init_tk(location)
1478                                         end
1479                                         if accept_token == 35 then
1480                                                 return new TKwimplies.init_tk(location)
1481                                         end
1482                                         if accept_token == 36 then
1483                                                 return new TKwreturn.init_tk(location)
1484                                         end
1485                                         if accept_token == 37 then
1486                                                 return new TKwcontinue.init_tk(location)
1487                                         end
1488                                         if accept_token == 38 then
1489                                                 return new TKwbreak.init_tk(location)
1490                                         end
1491                                         if accept_token == 39 then
1492                                                 return new TKwabort.init_tk(location)
1493                                         end
1494                                         if accept_token == 40 then
1495                                                 return new TKwassert.init_tk(location)
1496                                         end
1497                                         if accept_token == 41 then
1498                                                 return new TKwnew.init_tk(location)
1499                                         end
1500                                         if accept_token == 42 then
1501                                                 return new TKwisa.init_tk(location)
1502                                         end
1503                                         if accept_token == 43 then
1504                                                 return new TKwonce.init_tk(location)
1505                                         end
1506                                         if accept_token == 44 then
1507                                                 return new TKwsuper.init_tk(location)
1508                                         end
1509                                         if accept_token == 45 then
1510                                                 return new TKwself.init_tk(location)
1511                                         end
1512                                         if accept_token == 46 then
1513                                                 return new TKwtrue.init_tk(location)
1514                                         end
1515                                         if accept_token == 47 then
1516                                                 return new TKwfalse.init_tk(location)
1517                                         end
1518                                         if accept_token == 48 then
1519                                                 return new TKwnull.init_tk(location)
1520                                         end
1521                                         if accept_token == 49 then
1522                                                 return new TKwas.init_tk(location)
1523                                         end
1524                                         if accept_token == 50 then
1525                                                 return new TKwnullable.init_tk(location)
1526                                         end
1527                                         if accept_token == 51 then
1528                                                 return new TKwisset.init_tk(location)
1529                                         end
1530                                         if accept_token == 52 then
1531                                                 return new TKwlabel.init_tk(location)
1532                                         end
1533                                         if accept_token == 53 then
1534                                                 return new TKwdebug.init_tk(location)
1535                                         end
1536                                         if accept_token == 54 then
1537                                                 return new TOpar.init_tk(location)
1538                                         end
1539                                         if accept_token == 55 then
1540                                                 return new TCpar.init_tk(location)
1541                                         end
1542                                         if accept_token == 56 then
1543                                                 return new TObra.init_tk(location)
1544                                         end
1545                                         if accept_token == 57 then
1546                                                 return new TCbra.init_tk(location)
1547                                         end
1548                                         if accept_token == 58 then
1549                                                 return new TComma.init_tk(location)
1550                                         end
1551                                         if accept_token == 59 then
1552                                                 return new TColumn.init_tk(location)
1553                                         end
1554                                         if accept_token == 60 then
1555                                                 return new TQuad.init_tk(location)
1556                                         end
1557                                         if accept_token == 61 then
1558                                                 return new TAssign.init_tk(location)
1559                                         end
1560                                         if accept_token == 62 then
1561                                                 return new TPluseq.init_tk(location)
1562                                         end
1563                                         if accept_token == 63 then
1564                                                 return new TMinuseq.init_tk(location)
1565                                         end
1566                                         if accept_token == 64 then
1567                                                 return new TDotdotdot.init_tk(location)
1568                                         end
1569                                         if accept_token == 65 then
1570                                                 return new TDotdot.init_tk(location)
1571                                         end
1572                                         if accept_token == 66 then
1573                                                 return new TDot.init_tk(location)
1574                                         end
1575                                         if accept_token == 67 then
1576                                                 return new TPlus.init_tk(location)
1577                                         end
1578                                         if accept_token == 68 then
1579                                                 return new TMinus.init_tk(location)
1580                                         end
1581                                         if accept_token == 69 then
1582                                                 return new TStar.init_tk(location)
1583                                         end
1584                                         if accept_token == 70 then
1585                                                 return new TSlash.init_tk(location)
1586                                         end
1587                                         if accept_token == 71 then
1588                                                 return new TPercent.init_tk(location)
1589                                         end
1590                                         if accept_token == 72 then
1591                                                 return new TEq.init_tk(location)
1592                                         end
1593                                         if accept_token == 73 then
1594                                                 return new TNe.init_tk(location)
1595                                         end
1596                                         if accept_token == 74 then
1597                                                 return new TLt.init_tk(location)
1598                                         end
1599                                         if accept_token == 75 then
1600                                                 return new TLe.init_tk(location)
1601                                         end
1602                                         if accept_token == 76 then
1603                                                 return new TLl.init_tk(location)
1604                                         end
1605                                         if accept_token == 77 then
1606                                                 return new TGt.init_tk(location)
1607                                         end
1608                                         if accept_token == 78 then
1609                                                 return new TGe.init_tk(location)
1610                                         end
1611                                         if accept_token == 79 then
1612                                                 return new TGg.init_tk(location)
1613                                         end
1614                                         if accept_token == 80 then
1615                                                 return new TStarship.init_tk(location)
1616                                         end
1617                                         if accept_token == 81 then
1618                                                 return new TBang.init_tk(location)
1619                                         end
1620                                         if accept_token == 82 then
1621                                                 return new TAt.init_tk(location)
1622                                         end
1623                                         if accept_token == 83 then
1624                                                 return new TClassid.init_tk(location)
1625                                         end
1626                                         if accept_token == 84 then
1627                                                 return new TId.init_tk(location)
1628                                         end
1629                                         if accept_token == 85 then
1630                                                 return new TAttrid.init_tk(location)
1631                                         end
1632                                         if accept_token == 86 then
1633                                                 return new TNumber.init_tk(location)
1634                                         end
1635                                         if accept_token == 87 then
1636                                                 return new TFloat.init_tk(location)
1637                                         end
1638                                         if accept_token == 88 then
1639                                                 return new TString.init_tk(location)
1640                                         end
1641                                         if accept_token == 89 then
1642                                                 return new TStartString.init_tk(location)
1643                                         end
1644                                         if accept_token == 90 then
1645                                                 return new TMidString.init_tk(location)
1646                                         end
1647                                         if accept_token == 91 then
1648                                                 return new TEndString.init_tk(location)
1649                                         end
1650                                         if accept_token == 92 then
1651                                                 return new TChar.init_tk(location)
1652                                         end
1653                                         if accept_token == 93 then
1654                                                 return new TBadString.init_tk(location)
1655                                         end
1656                                         if accept_token == 94 then
1657                                                 return new TBadChar.init_tk(location)
1658                                         end
1659                                         if accept_token == 95 then
1660                                                 return new TExternCodeSegment.init_tk(location)
1661                                         end
1662                                 else
1663                                         _stream_pos = sp
1664                                         var location = new Location(_file, start_line + 1, start_line + 1, start_pos + 1, start_pos + 1)
1665                                         if sp > start_stream_pos then
1666                                                 var text = string.substring(start_stream_pos, sp-start_stream_pos)
1667                                                 var token = new ALexerError.init_lexer_error("Syntax error: unknown token {text}.", location, text)
1668                                                 return token
1669                                         else
1670                                                 var token = new EOF.init_tk(location)
1671                                                 return token
1672                                         end
1673                                 end
1674                         end
1675                 end
1676         end
1677 end
1678