src/parser/lexer.nit

   1 # Lexer and its tokens.
   2 # This file was generated by SableCC (http://www.sablecc.org/).
   3 package lexer
   4
   5 intrude import parser_nodes
   6 private import tables
   7
   8 redef class Token
   9     var _text: nullable String
  10
  11     redef fun text
  12     do
  13         var res = _text
  14         if res != null then return res
  15         res = location.text
  16         _text = res
  17         return res
  18     end
  19
  20     fun parser_index: Int is abstract
  21 end
  22
  23 redef class TEol
  24     redef fun parser_index: Int
  25     do
  26         return 0
  27     end
  28
  29     init init_tk(loc: Location)
  30     do
  31                 _location = loc
  32     end
  33 end
  34
  35 redef class TComment
  36     redef fun parser_index: Int
  37     do
  38         return 1
  39     end
  40
  41     init init_tk(loc: Location)
  42     do
  43                 _location = loc
  44     end
  45 end
  46
  47 redef class TKwmodule
  48     redef fun parser_index: Int
  49     do
  50         return 2
  51     end
  52
  53     init init_tk(loc: Location)
  54     do
  55                 _location = loc
  56     end
  57 end
  58
  59 redef class TKwimport
  60     redef fun parser_index: Int
  61     do
  62         return 3
  63     end
  64
  65     init init_tk(loc: Location)
  66     do
  67                 _location = loc
  68     end
  69 end
  70
  71 redef class TKwclass
  72     redef fun parser_index: Int
  73     do
  74         return 4
  75     end
  76
  77     init init_tk(loc: Location)
  78     do
  79                 _location = loc
  80     end
  81 end
  82
  83 redef class TKwabstract
  84     redef fun parser_index: Int
  85     do
  86         return 5
  87     end
  88
  89     init init_tk(loc: Location)
  90     do
  91                 _location = loc
  92     end
  93 end
  94
  95 redef class TKwinterface
  96     redef fun parser_index: Int
  97     do
  98         return 6
  99     end
 100
 101     init init_tk(loc: Location)
 102     do
 103                 _location = loc
 104     end
 105 end
 106
 107 redef class TKwenum
 108     redef fun parser_index: Int
 109     do
 110         return 7
 111     end
 112
 113     init init_tk(loc: Location)
 114     do
 115                 _location = loc
 116     end
 117 end
 118
 119 redef class TKwend
 120     redef fun parser_index: Int
 121     do
 122         return 8
 123     end
 124
 125     init init_tk(loc: Location)
 126     do
 127                 _location = loc
 128     end
 129 end
 130
 131 redef class TKwmeth
 132     redef fun parser_index: Int
 133     do
 134         return 9
 135     end
 136
 137     init init_tk(loc: Location)
 138     do
 139                 _location = loc
 140     end
 141 end
 142
 143 redef class TKwtype
 144     redef fun parser_index: Int
 145     do
 146         return 10
 147     end
 148
 149     init init_tk(loc: Location)
 150     do
 151                 _location = loc
 152     end
 153 end
 154
 155 redef class TKwinit
 156     redef fun parser_index: Int
 157     do
 158         return 11
 159     end
 160
 161     init init_tk(loc: Location)
 162     do
 163                 _location = loc
 164     end
 165 end
 166
 167 redef class TKwredef
 168     redef fun parser_index: Int
 169     do
 170         return 12
 171     end
 172
 173     init init_tk(loc: Location)
 174     do
 175                 _location = loc
 176     end
 177 end
 178
 179 redef class TKwis
 180     redef fun parser_index: Int
 181     do
 182         return 13
 183     end
 184
 185     init init_tk(loc: Location)
 186     do
 187                 _location = loc
 188     end
 189 end
 190
 191 redef class TKwdo
 192     redef fun parser_index: Int
 193     do
 194         return 14
 195     end
 196
 197     init init_tk(loc: Location)
 198     do
 199                 _location = loc
 200     end
 201 end
 202
 203 redef class TKwreadable
 204     redef fun parser_index: Int
 205     do
 206         return 15
 207     end
 208
 209     init init_tk(loc: Location)
 210     do
 211                 _location = loc
 212     end
 213 end
 214
 215 redef class TKwwritable
 216     redef fun parser_index: Int
 217     do
 218         return 16
 219     end
 220
 221     init init_tk(loc: Location)
 222     do
 223                 _location = loc
 224     end
 225 end
 226
 227 redef class TKwvar
 228     redef fun parser_index: Int
 229     do
 230         return 17
 231     end
 232
 233     init init_tk(loc: Location)
 234     do
 235                 _location = loc
 236     end
 237 end
 238
 239 redef class TKwintern
 240     redef fun parser_index: Int
 241     do
 242         return 18
 243     end
 244
 245     init init_tk(loc: Location)
 246     do
 247                 _location = loc
 248     end
 249 end
 250
 251 redef class TKwextern
 252     redef fun parser_index: Int
 253     do
 254         return 19
 255     end
 256
 257     init init_tk(loc: Location)
 258     do
 259                 _location = loc
 260     end
 261 end
 262
 263 redef class TKwprotected
 264     redef fun parser_index: Int
 265     do
 266         return 20
 267     end
 268
 269     init init_tk(loc: Location)
 270     do
 271                 _location = loc
 272     end
 273 end
 274
 275 redef class TKwprivate
 276     redef fun parser_index: Int
 277     do
 278         return 21
 279     end
 280
 281     init init_tk(loc: Location)
 282     do
 283                 _location = loc
 284     end
 285 end
 286
 287 redef class TKwintrude
 288     redef fun parser_index: Int
 289     do
 290         return 22
 291     end
 292
 293     init init_tk(loc: Location)
 294     do
 295                 _location = loc
 296     end
 297 end
 298
 299 redef class TKwif
 300     redef fun parser_index: Int
 301     do
 302         return 23
 303     end
 304
 305     init init_tk(loc: Location)
 306     do
 307                 _location = loc
 308     end
 309 end
 310
 311 redef class TKwthen
 312     redef fun parser_index: Int
 313     do
 314         return 24
 315     end
 316
 317     init init_tk(loc: Location)
 318     do
 319                 _location = loc
 320     end
 321 end
 322
 323 redef class TKwelse
 324     redef fun parser_index: Int
 325     do
 326         return 25
 327     end
 328
 329     init init_tk(loc: Location)
 330     do
 331                 _location = loc
 332     end
 333 end
 334
 335 redef class TKwwhile
 336     redef fun parser_index: Int
 337     do
 338         return 26
 339     end
 340
 341     init init_tk(loc: Location)
 342     do
 343                 _location = loc
 344     end
 345 end
 346
 347 redef class TKwloop
 348     redef fun parser_index: Int
 349     do
 350         return 27
 351     end
 352
 353     init init_tk(loc: Location)
 354     do
 355                 _location = loc
 356     end
 357 end
 358
 359 redef class TKwfor
 360     redef fun parser_index: Int
 361     do
 362         return 28
 363     end
 364
 365     init init_tk(loc: Location)
 366     do
 367                 _location = loc
 368     end
 369 end
 370
 371 redef class TKwin
 372     redef fun parser_index: Int
 373     do
 374         return 29
 375     end
 376
 377     init init_tk(loc: Location)
 378     do
 379                 _location = loc
 380     end
 381 end
 382
 383 redef class TKwand
 384     redef fun parser_index: Int
 385     do
 386         return 30
 387     end
 388
 389     init init_tk(loc: Location)
 390     do
 391                 _location = loc
 392     end
 393 end
 394
 395 redef class TKwor
 396     redef fun parser_index: Int
 397     do
 398         return 31
 399     end
 400
 401     init init_tk(loc: Location)
 402     do
 403                 _location = loc
 404     end
 405 end
 406
 407 redef class TKwnot
 408     redef fun parser_index: Int
 409     do
 410         return 32
 411     end
 412
 413     init init_tk(loc: Location)
 414     do
 415                 _location = loc
 416     end
 417 end
 418
 419 redef class TKwreturn
 420     redef fun parser_index: Int
 421     do
 422         return 33
 423     end
 424
 425     init init_tk(loc: Location)
 426     do
 427                 _location = loc
 428     end
 429 end
 430
 431 redef class TKwcontinue
 432     redef fun parser_index: Int
 433     do
 434         return 34
 435     end
 436
 437     init init_tk(loc: Location)
 438     do
 439                 _location = loc
 440     end
 441 end
 442
 443 redef class TKwbreak
 444     redef fun parser_index: Int
 445     do
 446         return 35
 447     end
 448
 449     init init_tk(loc: Location)
 450     do
 451                 _location = loc
 452     end
 453 end
 454
 455 redef class TKwabort
 456     redef fun parser_index: Int
 457     do
 458         return 36
 459     end
 460
 461     init init_tk(loc: Location)
 462     do
 463                 _location = loc
 464     end
 465 end
 466
 467 redef class TKwassert
 468     redef fun parser_index: Int
 469     do
 470         return 37
 471     end
 472
 473     init init_tk(loc: Location)
 474     do
 475                 _location = loc
 476     end
 477 end
 478
 479 redef class TKwnew
 480     redef fun parser_index: Int
 481     do
 482         return 38
 483     end
 484
 485     init init_tk(loc: Location)
 486     do
 487                 _location = loc
 488     end
 489 end
 490
 491 redef class TKwisa
 492     redef fun parser_index: Int
 493     do
 494         return 39
 495     end
 496
 497     init init_tk(loc: Location)
 498     do
 499                 _location = loc
 500     end
 501 end
 502
 503 redef class TKwonce
 504     redef fun parser_index: Int
 505     do
 506         return 40
 507     end
 508
 509     init init_tk(loc: Location)
 510     do
 511                 _location = loc
 512     end
 513 end
 514
 515 redef class TKwsuper
 516     redef fun parser_index: Int
 517     do
 518         return 41
 519     end
 520
 521     init init_tk(loc: Location)
 522     do
 523                 _location = loc
 524     end
 525 end
 526
 527 redef class TKwself
 528     redef fun parser_index: Int
 529     do
 530         return 42
 531     end
 532
 533     init init_tk(loc: Location)
 534     do
 535                 _location = loc
 536     end
 537 end
 538
 539 redef class TKwtrue
 540     redef fun parser_index: Int
 541     do
 542         return 43
 543     end
 544
 545     init init_tk(loc: Location)
 546     do
 547                 _location = loc
 548     end
 549 end
 550
 551 redef class TKwfalse
 552     redef fun parser_index: Int
 553     do
 554         return 44
 555     end
 556
 557     init init_tk(loc: Location)
 558     do
 559                 _location = loc
 560     end
 561 end
 562
 563 redef class TKwnull
 564     redef fun parser_index: Int
 565     do
 566         return 45
 567     end
 568
 569     init init_tk(loc: Location)
 570     do
 571                 _location = loc
 572     end
 573 end
 574
 575 redef class TKwas
 576     redef fun parser_index: Int
 577     do
 578         return 46
 579     end
 580
 581     init init_tk(loc: Location)
 582     do
 583                 _location = loc
 584     end
 585 end
 586
 587 redef class TKwnullable
 588     redef fun parser_index: Int
 589     do
 590         return 47
 591     end
 592
 593     init init_tk(loc: Location)
 594     do
 595                 _location = loc
 596     end
 597 end
 598
 599 redef class TKwisset
 600     redef fun parser_index: Int
 601     do
 602         return 48
 603     end
 604
 605     init init_tk(loc: Location)
 606     do
 607                 _location = loc
 608     end
 609 end
 610
 611 redef class TKwlabel
 612     redef fun parser_index: Int
 613     do
 614         return 49
 615     end
 616
 617     init init_tk(loc: Location)
 618     do
 619                 _location = loc
 620     end
 621 end
 622
 623 redef class TKwdebug
 624     redef fun parser_index: Int
 625     do
 626         return 50
 627     end
 628
 629     init init_tk(loc: Location)
 630     do
 631                 _location = loc
 632     end
 633 end
 634
 635 redef class TOpar
 636     redef fun parser_index: Int
 637     do
 638         return 51
 639     end
 640
 641     init init_tk(loc: Location)
 642     do
 643                 _location = loc
 644     end
 645 end
 646
 647 redef class TCpar
 648     redef fun parser_index: Int
 649     do
 650         return 52
 651     end
 652
 653     init init_tk(loc: Location)
 654     do
 655                 _location = loc
 656     end
 657 end
 658
 659 redef class TObra
 660     redef fun parser_index: Int
 661     do
 662         return 53
 663     end
 664
 665     init init_tk(loc: Location)
 666     do
 667                 _location = loc
 668     end
 669 end
 670
 671 redef class TCbra
 672     redef fun parser_index: Int
 673     do
 674         return 54
 675     end
 676
 677     init init_tk(loc: Location)
 678     do
 679                 _location = loc
 680     end
 681 end
 682
 683 redef class TComma
 684     redef fun parser_index: Int
 685     do
 686         return 55
 687     end
 688
 689     init init_tk(loc: Location)
 690     do
 691                 _location = loc
 692     end
 693 end
 694
 695 redef class TColumn
 696     redef fun parser_index: Int
 697     do
 698         return 56
 699     end
 700
 701     init init_tk(loc: Location)
 702     do
 703                 _location = loc
 704     end
 705 end
 706
 707 redef class TQuad
 708     redef fun parser_index: Int
 709     do
 710         return 57
 711     end
 712
 713     init init_tk(loc: Location)
 714     do
 715                 _location = loc
 716     end
 717 end
 718
 719 redef class TAssign
 720     redef fun parser_index: Int
 721     do
 722         return 58
 723     end
 724
 725     init init_tk(loc: Location)
 726     do
 727                 _location = loc
 728     end
 729 end
 730
 731 redef class TPluseq
 732     redef fun parser_index: Int
 733     do
 734         return 59
 735     end
 736
 737     init init_tk(loc: Location)
 738     do
 739                 _location = loc
 740     end
 741 end
 742
 743 redef class TMinuseq
 744     redef fun parser_index: Int
 745     do
 746         return 60
 747     end
 748
 749     init init_tk(loc: Location)
 750     do
 751                 _location = loc
 752     end
 753 end
 754
 755 redef class TDotdotdot
 756     redef fun parser_index: Int
 757     do
 758         return 61
 759     end
 760
 761     init init_tk(loc: Location)
 762     do
 763                 _location = loc
 764     end
 765 end
 766
 767 redef class TDotdot
 768     redef fun parser_index: Int
 769     do
 770         return 62
 771     end
 772
 773     init init_tk(loc: Location)
 774     do
 775                 _location = loc
 776     end
 777 end
 778
 779 redef class TDot
 780     redef fun parser_index: Int
 781     do
 782         return 63
 783     end
 784
 785     init init_tk(loc: Location)
 786     do
 787                 _location = loc
 788     end
 789 end
 790
 791 redef class TPlus
 792     redef fun parser_index: Int
 793     do
 794         return 64
 795     end
 796
 797     init init_tk(loc: Location)
 798     do
 799                 _location = loc
 800     end
 801 end
 802
 803 redef class TMinus
 804     redef fun parser_index: Int
 805     do
 806         return 65
 807     end
 808
 809     init init_tk(loc: Location)
 810     do
 811                 _location = loc
 812     end
 813 end
 814
 815 redef class TStar
 816     redef fun parser_index: Int
 817     do
 818         return 66
 819     end
 820
 821     init init_tk(loc: Location)
 822     do
 823                 _location = loc
 824     end
 825 end
 826
 827 redef class TSlash
 828     redef fun parser_index: Int
 829     do
 830         return 67
 831     end
 832
 833     init init_tk(loc: Location)
 834     do
 835                 _location = loc
 836     end
 837 end
 838
 839 redef class TPercent
 840     redef fun parser_index: Int
 841     do
 842         return 68
 843     end
 844
 845     init init_tk(loc: Location)
 846     do
 847                 _location = loc
 848     end
 849 end
 850
 851 redef class TEq
 852     redef fun parser_index: Int
 853     do
 854         return 69
 855     end
 856
 857     init init_tk(loc: Location)
 858     do
 859                 _location = loc
 860     end
 861 end
 862
 863 redef class TNe
 864     redef fun parser_index: Int
 865     do
 866         return 70
 867     end
 868
 869     init init_tk(loc: Location)
 870     do
 871                 _location = loc
 872     end
 873 end
 874
 875 redef class TLt
 876     redef fun parser_index: Int
 877     do
 878         return 71
 879     end
 880
 881     init init_tk(loc: Location)
 882     do
 883                 _location = loc
 884     end
 885 end
 886
 887 redef class TLe
 888     redef fun parser_index: Int
 889     do
 890         return 72
 891     end
 892
 893     init init_tk(loc: Location)
 894     do
 895                 _location = loc
 896     end
 897 end
 898
 899 redef class TLl
 900     redef fun parser_index: Int
 901     do
 902         return 73
 903     end
 904
 905     init init_tk(loc: Location)
 906     do
 907                 _location = loc
 908     end
 909 end
 910
 911 redef class TGt
 912     redef fun parser_index: Int
 913     do
 914         return 74
 915     end
 916
 917     init init_tk(loc: Location)
 918     do
 919                 _location = loc
 920     end
 921 end
 922
 923 redef class TGe
 924     redef fun parser_index: Int
 925     do
 926         return 75
 927     end
 928
 929     init init_tk(loc: Location)
 930     do
 931                 _location = loc
 932     end
 933 end
 934
 935 redef class TGg
 936     redef fun parser_index: Int
 937     do
 938         return 76
 939     end
 940
 941     init init_tk(loc: Location)
 942     do
 943                 _location = loc
 944     end
 945 end
 946
 947 redef class TStarship
 948     redef fun parser_index: Int
 949     do
 950         return 77
 951     end
 952
 953     init init_tk(loc: Location)
 954     do
 955                 _location = loc
 956     end
 957 end
 958
 959 redef class TBang
 960     redef fun parser_index: Int
 961     do
 962         return 78
 963     end
 964
 965     init init_tk(loc: Location)
 966     do
 967                 _location = loc
 968     end
 969 end
 970
 971 redef class TClassid
 972     redef fun parser_index: Int
 973     do
 974         return 79
 975     end
 976
 977     init init_tk(loc: Location)
 978     do
 979                 _location = loc
 980     end
 981 end
 982
 983 redef class TId
 984     redef fun parser_index: Int
 985     do
 986         return 80
 987     end
 988
 989     init init_tk(loc: Location)
 990     do
 991                 _location = loc
 992     end
 993 end
 994
 995 redef class TAttrid
 996     redef fun parser_index: Int
 997     do
 998         return 81
 999     end
1000
1001     init init_tk(loc: Location)
1002     do
1003                 _location = loc
1004     end
1005 end
1006
1007 redef class TNumber
1008     redef fun parser_index: Int
1009     do
1010         return 82
1011     end
1012
1013     init init_tk(loc: Location)
1014     do
1015                 _location = loc
1016     end
1017 end
1018
1019 redef class TFloat
1020     redef fun parser_index: Int
1021     do
1022         return 83
1023     end
1024
1025     init init_tk(loc: Location)
1026     do
1027                 _location = loc
1028     end
1029 end
1030
1031 redef class TChar
1032     redef fun parser_index: Int
1033     do
1034         return 84
1035     end
1036
1037     init init_tk(loc: Location)
1038     do
1039                 _location = loc
1040     end
1041 end
1042
1043 redef class TString
1044     redef fun parser_index: Int
1045     do
1046         return 85
1047     end
1048
1049     init init_tk(loc: Location)
1050     do
1051                 _location = loc
1052     end
1053 end
1054
1055 redef class TStartString
1056     redef fun parser_index: Int
1057     do
1058         return 86
1059     end
1060
1061     init init_tk(loc: Location)
1062     do
1063                 _location = loc
1064     end
1065 end
1066
1067 redef class TMidString
1068     redef fun parser_index: Int
1069     do
1070         return 87
1071     end
1072
1073     init init_tk(loc: Location)
1074     do
1075                 _location = loc
1076     end
1077 end
1078
1079 redef class TEndString
1080     redef fun parser_index: Int
1081     do
1082         return 88
1083     end
1084
1085     init init_tk(loc: Location)
1086     do
1087                 _location = loc
1088     end
1089 end
1090
1091 redef class TBadChar
1092     redef fun parser_index: Int
1093     do
1094         return 89
1095     end
1096
1097     init init_tk(loc: Location)
1098     do
1099                 _location = loc
1100     end
1101 end
1102
1103 redef class TBadString
1104     redef fun parser_index: Int
1105     do
1106         return 90
1107     end
1108
1109     init init_tk(loc: Location)
1110     do
1111                 _location = loc
1112     end
1113 end
1114
1115 redef class TExternCodeSegment
1116     redef fun parser_index: Int
1117     do
1118         return 91
1119     end
1120
1121     init init_tk(loc: Location)
1122     do
1123                 _location = loc
1124     end
1125 end
1126
1127
1128 redef class EOF
1129     redef fun parser_index: Int
1130     do
1131         return 92
1132     end
1133
1134     init init_tk(loc: Location)
1135     do
1136         _text = ""
1137                 _location = loc
1138     end
1139 end
1140
1141 redef class AError
1142     readable var _message: String
1143
1144     init init_error(message: String, loc: Location)
1145     do
1146                 init_tk(loc)
1147                 _message = message
1148     end
1149 end
1150
1151 redef class ALexerError
1152     readable var _string: String
1153
1154     init init_lexer_error(message: String, loc: Location, string: String)
1155     do
1156                 init_error(message, loc)
1157                 _string = string
1158     end
1159 end
1160
1161 redef class AParserError
1162     readable var _token: Token
1163
1164     init init_parser_error(message: String, loc: Location, token: Token)
1165     do
1166                 init_error(message, loc)
1167                 _token = token
1168     end
1169 end
1170
1171 # The lexer extract NIT tokens from an input stream.
1172 # It is better user with the Parser
1173 class Lexer
1174         super TablesCapable
1175         # Last peeked token
1176         var _token: nullable Token
1177
1178         # Lexer current state
1179         var _state: Int = 0
1180
1181         # The source file
1182         readable var _file: SourceFile
1183
1184         # Current character in the stream
1185         var _stream_pos: Int = 0
1186
1187         # Current line number in the input stream
1188         var _line: Int = 0
1189
1190         # Current column in the input stream
1191         var _pos: Int = 0
1192
1193         # Was the last character a cariage-return?
1194         var _cr: Bool = false
1195
1196         # Constante state values
1197         private fun state_initial: Int do return 0 end
1198
1199         # Create a new lexer for a stream (and a name)
1200         init(file: SourceFile)
1201         do
1202                 _file = file
1203         end
1204
1205         # Give the next token (but do not consume it)
1206         fun peek: Token
1207         do
1208                 while _token == null do
1209                         _token = get_token
1210                 end
1211                 return _token.as(not null)
1212         end
1213
1214         # Give and consume the next token
1215         fun next: Token
1216         do
1217                 var result = _token
1218                 while result == null do
1219                         result = get_token
1220                 end
1221                 _token = null
1222                 return result
1223         end
1224
1225         # Get a token, or null if it is discarded
1226         private fun get_token: nullable Token
1227         do
1228                 var dfa_state = 0
1229
1230                 var sp = _stream_pos
1231                 var start_stream_pos = sp
1232                 var start_pos = _pos
1233                 var start_line = _line
1234                 var string = _file.string
1235                 var string_len = string.length
1236
1237                 var accept_state = -1
1238                 var accept_token = -1
1239                 var accept_length = -1
1240                 var accept_pos = -1
1241                 var accept_line = -1
1242
1243                 loop
1244                         if sp >= string_len then
1245                                 dfa_state = -1
1246                         else
1247                                 var c = string[sp].ascii
1248                                 sp += 1
1249
1250                                 var cr = _cr
1251                                 var line = _line
1252                                 var pos = _pos
1253                                 if c == 10 then
1254                                         if cr then
1255                                                 cr = false
1256                                                 _file.line_starts[line] = sp
1257                                         else
1258                                                 line = line + 1
1259                                                 pos = 0
1260                                                 _file.line_starts[line] = sp
1261                                         end
1262                                 else if c == 13 then
1263                                         line = line + 1
1264                                         pos = 0
1265                                         cr = true
1266                                         _file.line_starts[line] = sp
1267                                 else
1268                                         pos = pos + 1
1269                                         cr = false
1270                                 end
1271
1272                                 loop
1273                                         var old_state = dfa_state
1274                                         if dfa_state < -1 then
1275                                                 old_state = -2 - dfa_state
1276                                         end
1277
1278                                         dfa_state = -1
1279
1280                                         var low = 0
1281                                         var high = lexer_goto(old_state, 0) - 1
1282
1283                                         if high >= 0 then
1284                                                 while low <= high do
1285                                                         var middle = (low + high) / 2
1286                                                         var offset = middle * 3 + 1 # +1 because length is at 0
1287
1288                                                         if c < lexer_goto(old_state, offset) then
1289                                                                 high = middle - 1
1290                                                         else if c > lexer_goto(old_state, offset+1) then
1291                                                                 low = middle + 1
1292                                                         else
1293                                                                 dfa_state = lexer_goto(old_state, offset+2)
1294                                                                 break
1295                                                         end
1296                                                 end
1297                                         end
1298                                         if dfa_state > -2 then break
1299                                 end
1300
1301                                 _cr = cr
1302                                 _line = line
1303                                 _pos = pos
1304                         end
1305
1306                         if dfa_state >= 0 then
1307                                 var tok = lexer_accept(dfa_state)
1308                                 if tok != -1 then
1309                                         accept_state = dfa_state
1310                                         accept_token = tok
1311                                         accept_length = sp - start_stream_pos
1312                                         accept_pos = _pos
1313                                         accept_line = _line
1314                                 end
1315                         else
1316                                 if accept_state != -1 then
1317                                         var location = new Location(_file, start_line + 1, accept_line + 1, start_pos + 1, accept_pos)
1318                                         _pos = accept_pos
1319                                         _line = accept_line
1320                                         _stream_pos = start_stream_pos + accept_length
1321                                         if accept_token == 0 then
1322                                                 return null
1323                                         end
1324                                         if accept_token == 1 then
1325                                                 return new TEol.init_tk(location)
1326                                         end
1327                                         if accept_token == 2 then
1328                                                 return new TComment.init_tk(location)
1329                                         end
1330                                         if accept_token == 3 then
1331                                                 return new TKwmodule.init_tk(location)
1332                                         end
1333                                         if accept_token == 4 then
1334                                                 return new TKwimport.init_tk(location)
1335                                         end
1336                                         if accept_token == 5 then
1337                                                 return new TKwclass.init_tk(location)
1338                                         end
1339                                         if accept_token == 6 then
1340                                                 return new TKwabstract.init_tk(location)
1341                                         end
1342                                         if accept_token == 7 then
1343                                                 return new TKwinterface.init_tk(location)
1344                                         end
1345                                         if accept_token == 8 then
1346                                                 return new TKwenum.init_tk(location)
1347                                         end
1348                                         if accept_token == 9 then
1349                                                 return new TKwend.init_tk(location)
1350                                         end
1351                                         if accept_token == 10 then
1352                                                 return new TKwmeth.init_tk(location)
1353                                         end
1354                                         if accept_token == 11 then
1355                                                 return new TKwtype.init_tk(location)
1356                                         end
1357                                         if accept_token == 12 then
1358                                                 return new TKwinit.init_tk(location)
1359                                         end
1360                                         if accept_token == 13 then
1361                                                 return new TKwredef.init_tk(location)
1362                                         end
1363                                         if accept_token == 14 then
1364                                                 return new TKwis.init_tk(location)
1365                                         end
1366                                         if accept_token == 15 then
1367                                                 return new TKwdo.init_tk(location)
1368                                         end
1369                                         if accept_token == 16 then
1370                                                 return new TKwreadable.init_tk(location)
1371                                         end
1372                                         if accept_token == 17 then
1373                                                 return new TKwwritable.init_tk(location)
1374                                         end
1375                                         if accept_token == 18 then
1376                                                 return new TKwvar.init_tk(location)
1377                                         end
1378                                         if accept_token == 19 then
1379                                                 return new TKwintern.init_tk(location)
1380                                         end
1381                                         if accept_token == 20 then
1382                                                 return new TKwextern.init_tk(location)
1383                                         end
1384                                         if accept_token == 21 then
1385                                                 return new TKwprotected.init_tk(location)
1386                                         end
1387                                         if accept_token == 22 then
1388                                                 return new TKwprivate.init_tk(location)
1389                                         end
1390                                         if accept_token == 23 then
1391                                                 return new TKwintrude.init_tk(location)
1392                                         end
1393                                         if accept_token == 24 then
1394                                                 return new TKwif.init_tk(location)
1395                                         end
1396                                         if accept_token == 25 then
1397                                                 return new TKwthen.init_tk(location)
1398                                         end
1399                                         if accept_token == 26 then
1400                                                 return new TKwelse.init_tk(location)
1401                                         end
1402                                         if accept_token == 27 then
1403                                                 return new TKwwhile.init_tk(location)
1404                                         end
1405                                         if accept_token == 28 then
1406                                                 return new TKwloop.init_tk(location)
1407                                         end
1408                                         if accept_token == 29 then
1409                                                 return new TKwfor.init_tk(location)
1410                                         end
1411                                         if accept_token == 30 then
1412                                                 return new TKwin.init_tk(location)
1413                                         end
1414                                         if accept_token == 31 then
1415                                                 return new TKwand.init_tk(location)
1416                                         end
1417                                         if accept_token == 32 then
1418                                                 return new TKwor.init_tk(location)
1419                                         end
1420                                         if accept_token == 33 then
1421                                                 return new TKwnot.init_tk(location)
1422                                         end
1423                                         if accept_token == 34 then
1424                                                 return new TKwreturn.init_tk(location)
1425                                         end
1426                                         if accept_token == 35 then
1427                                                 return new TKwcontinue.init_tk(location)
1428                                         end
1429                                         if accept_token == 36 then
1430                                                 return new TKwbreak.init_tk(location)
1431                                         end
1432                                         if accept_token == 37 then
1433                                                 return new TKwabort.init_tk(location)
1434                                         end
1435                                         if accept_token == 38 then
1436                                                 return new TKwassert.init_tk(location)
1437                                         end
1438                                         if accept_token == 39 then
1439                                                 return new TKwnew.init_tk(location)
1440                                         end
1441                                         if accept_token == 40 then
1442                                                 return new TKwisa.init_tk(location)
1443                                         end
1444                                         if accept_token == 41 then
1445                                                 return new TKwonce.init_tk(location)
1446                                         end
1447                                         if accept_token == 42 then
1448                                                 return new TKwsuper.init_tk(location)
1449                                         end
1450                                         if accept_token == 43 then
1451                                                 return new TKwself.init_tk(location)
1452                                         end
1453                                         if accept_token == 44 then
1454                                                 return new TKwtrue.init_tk(location)
1455                                         end
1456                                         if accept_token == 45 then
1457                                                 return new TKwfalse.init_tk(location)
1458                                         end
1459                                         if accept_token == 46 then
1460                                                 return new TKwnull.init_tk(location)
1461                                         end
1462                                         if accept_token == 47 then
1463                                                 return new TKwas.init_tk(location)
1464                                         end
1465                                         if accept_token == 48 then
1466                                                 return new TKwnullable.init_tk(location)
1467                                         end
1468                                         if accept_token == 49 then
1469                                                 return new TKwisset.init_tk(location)
1470                                         end
1471                                         if accept_token == 50 then
1472                                                 return new TKwlabel.init_tk(location)
1473                                         end
1474                                         if accept_token == 51 then
1475                                                 return new TKwdebug.init_tk(location)
1476                                         end
1477                                         if accept_token == 52 then
1478                                                 return new TOpar.init_tk(location)
1479                                         end
1480                                         if accept_token == 53 then
1481                                                 return new TCpar.init_tk(location)
1482                                         end
1483                                         if accept_token == 54 then
1484                                                 return new TObra.init_tk(location)
1485                                         end
1486                                         if accept_token == 55 then
1487                                                 return new TCbra.init_tk(location)
1488                                         end
1489                                         if accept_token == 56 then
1490                                                 return new TComma.init_tk(location)
1491                                         end
1492                                         if accept_token == 57 then
1493                                                 return new TColumn.init_tk(location)
1494                                         end
1495                                         if accept_token == 58 then
1496                                                 return new TQuad.init_tk(location)
1497                                         end
1498                                         if accept_token == 59 then
1499                                                 return new TAssign.init_tk(location)
1500                                         end
1501                                         if accept_token == 60 then
1502                                                 return new TPluseq.init_tk(location)
1503                                         end
1504                                         if accept_token == 61 then
1505                                                 return new TMinuseq.init_tk(location)
1506                                         end
1507                                         if accept_token == 62 then
1508                                                 return new TDotdotdot.init_tk(location)
1509                                         end
1510                                         if accept_token == 63 then
1511                                                 return new TDotdot.init_tk(location)
1512                                         end
1513                                         if accept_token == 64 then
1514                                                 return new TDot.init_tk(location)
1515                                         end
1516                                         if accept_token == 65 then
1517                                                 return new TPlus.init_tk(location)
1518                                         end
1519                                         if accept_token == 66 then
1520                                                 return new TMinus.init_tk(location)
1521                                         end
1522                                         if accept_token == 67 then
1523                                                 return new TStar.init_tk(location)
1524                                         end
1525                                         if accept_token == 68 then
1526                                                 return new TSlash.init_tk(location)
1527                                         end
1528                                         if accept_token == 69 then
1529                                                 return new TPercent.init_tk(location)
1530                                         end
1531                                         if accept_token == 70 then
1532                                                 return new TEq.init_tk(location)
1533                                         end
1534                                         if accept_token == 71 then
1535                                                 return new TNe.init_tk(location)
1536                                         end
1537                                         if accept_token == 72 then
1538                                                 return new TLt.init_tk(location)
1539                                         end
1540                                         if accept_token == 73 then
1541                                                 return new TLe.init_tk(location)
1542                                         end
1543                                         if accept_token == 74 then
1544                                                 return new TLl.init_tk(location)
1545                                         end
1546                                         if accept_token == 75 then
1547                                                 return new TGt.init_tk(location)
1548                                         end
1549                                         if accept_token == 76 then
1550                                                 return new TGe.init_tk(location)
1551                                         end
1552                                         if accept_token == 77 then
1553                                                 return new TGg.init_tk(location)
1554                                         end
1555                                         if accept_token == 78 then
1556                                                 return new TStarship.init_tk(location)
1557                                         end
1558                                         if accept_token == 79 then
1559                                                 return new TBang.init_tk(location)
1560                                         end
1561                                         if accept_token == 80 then
1562                                                 return new TClassid.init_tk(location)
1563                                         end
1564                                         if accept_token == 81 then
1565                                                 return new TId.init_tk(location)
1566                                         end
1567                                         if accept_token == 82 then
1568                                                 return new TAttrid.init_tk(location)
1569                                         end
1570                                         if accept_token == 83 then
1571                                                 return new TNumber.init_tk(location)
1572                                         end
1573                                         if accept_token == 84 then
1574                                                 return new TFloat.init_tk(location)
1575                                         end
1576                                         if accept_token == 85 then
1577                                                 return new TChar.init_tk(location)
1578                                         end
1579                                         if accept_token == 86 then
1580                                                 return new TString.init_tk(location)
1581                                         end
1582                                         if accept_token == 87 then
1583                                                 return new TStartString.init_tk(location)
1584                                         end
1585                                         if accept_token == 88 then
1586                                                 return new TMidString.init_tk(location)
1587                                         end
1588                                         if accept_token == 89 then
1589                                                 return new TEndString.init_tk(location)
1590                                         end
1591                                         if accept_token == 90 then
1592                                                 return new TBadChar.init_tk(location)
1593                                         end
1594                                         if accept_token == 91 then
1595                                                 return new TBadString.init_tk(location)
1596                                         end
1597                                         if accept_token == 92 then
1598                                                 return new TExternCodeSegment.init_tk(location)
1599                                         end
1600                                 else
1601                                         _stream_pos = sp
1602                                         var location = new Location(_file, start_line + 1, start_line + 1, start_pos + 1, start_pos + 1)
1603                                         if sp > start_stream_pos then
1604                                                 var text = string.substring(start_stream_pos, sp-start_stream_pos)
1605                                                 var token = new ALexerError.init_lexer_error("Syntax error: unknown token {text}.", location, text)
1606                                                 return token
1607                                         else
1608                                                 var token = new EOF.init_tk(location)
1609                                                 return token
1610                                         end
1611                                 end
1612                         end
1613                 end
1614         end
1615 end
1616