1 # Lexer and its tokens.
2 # This file was generated by SableCC (http://www.sablecc.org/).
5 intrude import parser_nodes
9 var _text
: nullable String
14 if res
!= null then return res
25 fun parser_index
: Int is abstract
29 redef fun parser_index
: Int
34 init init_tk
(loc
: Location)
41 redef fun parser_index
: Int
46 init init_tk
(loc
: Location)
52 redef class TKwpackage
53 redef fun parser_index
: Int
58 init init_tk
(loc
: Location)
65 redef fun parser_index
: Int
70 init init_tk
(loc
: Location)
77 redef fun parser_index
: Int
82 init init_tk
(loc
: Location)
89 redef fun parser_index
: Int
94 init init_tk
(loc
: Location)
100 redef class TKwabstract
101 redef fun parser_index
: Int
106 init init_tk
(loc
: Location)
112 redef class TKwinterface
113 redef fun parser_index
: Int
118 init init_tk
(loc
: Location)
125 redef fun parser_index
: Int
130 init init_tk
(loc
: Location)
137 redef fun parser_index
: Int
142 init init_tk
(loc
: Location)
149 redef fun parser_index
: Int
154 init init_tk
(loc
: Location)
161 redef fun parser_index
: Int
166 init init_tk
(loc
: Location)
173 redef fun parser_index
: Int
178 init init_tk
(loc
: Location)
185 redef fun parser_index
: Int
190 init init_tk
(loc
: Location)
197 redef fun parser_index
: Int
202 init init_tk
(loc
: Location)
209 redef fun parser_index
: Int
214 init init_tk
(loc
: Location)
220 redef class TKwreadable
221 redef fun parser_index
: Int
226 init init_tk
(loc
: Location)
232 redef class TKwwritable
233 redef fun parser_index
: Int
238 init init_tk
(loc
: Location)
245 redef fun parser_index
: Int
250 init init_tk
(loc
: Location)
256 redef class TKwintern
257 redef fun parser_index
: Int
262 init init_tk
(loc
: Location)
268 redef class TKwextern
269 redef fun parser_index
: Int
274 init init_tk
(loc
: Location)
280 redef class TKwpublic
281 redef fun parser_index
: Int
286 init init_tk
(loc
: Location)
292 redef class TKwprotected
293 redef fun parser_index
: Int
298 init init_tk
(loc
: Location)
304 redef class TKwprivate
305 redef fun parser_index
: Int
310 init init_tk
(loc
: Location)
316 redef class TKwintrude
317 redef fun parser_index
: Int
322 init init_tk
(loc
: Location)
329 redef fun parser_index
: Int
334 init init_tk
(loc
: Location)
341 redef fun parser_index
: Int
346 init init_tk
(loc
: Location)
353 redef fun parser_index
: Int
358 init init_tk
(loc
: Location)
365 redef fun parser_index
: Int
370 init init_tk
(loc
: Location)
377 redef fun parser_index
: Int
382 init init_tk
(loc
: Location)
389 redef fun parser_index
: Int
394 init init_tk
(loc
: Location)
401 redef fun parser_index
: Int
406 init init_tk
(loc
: Location)
413 redef fun parser_index
: Int
418 init init_tk
(loc
: Location)
425 redef fun parser_index
: Int
430 init init_tk
(loc
: Location)
437 redef fun parser_index
: Int
442 init init_tk
(loc
: Location)
448 redef class TKwimplies
449 redef fun parser_index
: Int
454 init init_tk
(loc
: Location)
460 redef class TKwreturn
461 redef fun parser_index
: Int
466 init init_tk
(loc
: Location)
472 redef class TKwcontinue
473 redef fun parser_index
: Int
478 init init_tk
(loc
: Location)
485 redef fun parser_index
: Int
490 init init_tk
(loc
: Location)
497 redef fun parser_index
: Int
502 init init_tk
(loc
: Location)
508 redef class TKwassert
509 redef fun parser_index
: Int
514 init init_tk
(loc
: Location)
521 redef fun parser_index
: Int
526 init init_tk
(loc
: Location)
533 redef fun parser_index
: Int
538 init init_tk
(loc
: Location)
545 redef fun parser_index
: Int
550 init init_tk
(loc
: Location)
557 redef fun parser_index
: Int
562 init init_tk
(loc
: Location)
569 redef fun parser_index
: Int
574 init init_tk
(loc
: Location)
581 redef fun parser_index
: Int
586 init init_tk
(loc
: Location)
593 redef fun parser_index
: Int
598 init init_tk
(loc
: Location)
605 redef fun parser_index
: Int
610 init init_tk
(loc
: Location)
617 redef fun parser_index
: Int
622 init init_tk
(loc
: Location)
628 redef class TKwnullable
629 redef fun parser_index
: Int
634 init init_tk
(loc
: Location)
641 redef fun parser_index
: Int
646 init init_tk
(loc
: Location)
653 redef fun parser_index
: Int
658 init init_tk
(loc
: Location)
665 redef fun parser_index
: Int
670 init init_tk
(loc
: Location)
677 redef fun parser_index
: Int
682 init init_tk
(loc
: Location)
689 redef fun parser_index
: Int
694 init init_tk
(loc
: Location)
701 redef fun parser_index
: Int
706 init init_tk
(loc
: Location)
713 redef fun parser_index
: Int
718 init init_tk
(loc
: Location)
725 redef fun parser_index
: Int
730 init init_tk
(loc
: Location)
737 redef fun parser_index
: Int
742 init init_tk
(loc
: Location)
749 redef fun parser_index
: Int
754 init init_tk
(loc
: Location)
761 redef fun parser_index
: Int
766 init init_tk
(loc
: Location)
773 redef fun parser_index
: Int
778 init init_tk
(loc
: Location)
785 redef fun parser_index
: Int
790 init init_tk
(loc
: Location)
796 redef class TDotdotdot
797 redef fun parser_index
: Int
802 init init_tk
(loc
: Location)
809 redef fun parser_index
: Int
814 init init_tk
(loc
: Location)
821 redef fun parser_index
: Int
826 init init_tk
(loc
: Location)
833 redef fun parser_index
: Int
838 init init_tk
(loc
: Location)
845 redef fun parser_index
: Int
850 init init_tk
(loc
: Location)
857 redef fun parser_index
: Int
862 init init_tk
(loc
: Location)
869 redef fun parser_index
: Int
874 init init_tk
(loc
: Location)
881 redef fun parser_index
: Int
886 init init_tk
(loc
: Location)
893 redef fun parser_index
: Int
898 init init_tk
(loc
: Location)
905 redef fun parser_index
: Int
910 init init_tk
(loc
: Location)
917 redef fun parser_index
: Int
922 init init_tk
(loc
: Location)
929 redef fun parser_index
: Int
934 init init_tk
(loc
: Location)
941 redef fun parser_index
: Int
946 init init_tk
(loc
: Location)
953 redef fun parser_index
: Int
958 init init_tk
(loc
: Location)
965 redef fun parser_index
: Int
970 init init_tk
(loc
: Location)
977 redef fun parser_index
: Int
982 init init_tk
(loc
: Location)
988 redef class TStarship
989 redef fun parser_index
: Int
994 init init_tk
(loc
: Location)
1001 redef fun parser_index
: Int
1006 init init_tk
(loc
: Location)
1013 redef fun parser_index
: Int
1018 init init_tk
(loc
: Location)
1024 redef class TClassid
1025 redef fun parser_index
: Int
1030 init init_tk
(loc
: Location)
1037 redef fun parser_index
: Int
1042 init init_tk
(loc
: Location)
1049 redef fun parser_index
: Int
1054 init init_tk
(loc
: Location)
1061 redef fun parser_index
: Int
1066 init init_tk
(loc
: Location)
1072 redef class THexNumber
1073 redef fun parser_index
: Int
1078 init init_tk
(loc
: Location)
1085 redef fun parser_index
: Int
1090 init init_tk
(loc
: Location)
1097 redef fun parser_index
: Int
1102 init init_tk
(loc
: Location)
1108 redef class TStartString
1109 redef fun parser_index
: Int
1114 init init_tk
(loc
: Location)
1120 redef class TMidString
1121 redef fun parser_index
: Int
1126 init init_tk
(loc
: Location)
1132 redef class TEndString
1133 redef fun parser_index
: Int
1138 init init_tk
(loc
: Location)
1145 redef fun parser_index
: Int
1150 init init_tk
(loc
: Location)
1156 redef class TBadString
1157 redef fun parser_index
: Int
1162 init init_tk
(loc
: Location)
1168 redef class TBadChar
1169 redef fun parser_index
: Int
1174 init init_tk
(loc
: Location)
1180 redef class TExternCodeSegment
1181 redef fun parser_index
: Int
1186 init init_tk
(loc
: Location)
1194 redef fun parser_index
: Int
1199 init init_tk
(loc
: Location)
1207 readable var _message
: String
1209 init init_error
(message
: String, loc
: Location)
1216 redef class ALexerError
1217 readable var _string
: String
1219 init init_lexer_error
(message
: String, loc
: Location, string
: String)
1221 init_error
(message
, loc
)
1226 redef class AParserError
1227 readable var _token
: Token
1229 init init_parser_error
(message
: String, loc
: Location, token
: Token)
1231 init_error
(message
, loc
)
1236 # The lexer extract NIT tokens from an input stream.
1237 # It is better user with the Parser
1241 var _token
: nullable Token
1243 # Lexer current state
1247 readable var _file
: SourceFile
1249 # Current character in the stream
1250 var _stream_pos
: Int = 0
1252 # Current line number in the input stream
1255 # Current column in the input stream
1258 # Was the last character a cariage-return?
1259 var _cr
: Bool = false
1261 # Constante state values
1262 private fun state_initial
: Int do return 0 end
1264 # Create a new lexer for a stream (and a name)
1265 init(file
: SourceFile)
1270 # The last peeked token to chain them
1271 private var last_token
: nullable Token = null
1273 # Give the next token (but do not consume it)
1277 if t
!= null then return t
1280 while t
== null do t
= get_token
1293 # Give and consume the next token
1301 # Primitive method to return a token, or return null if it is discarded
1302 # Is used to implement `peek` and `next`
1303 protected fun get_token
: nullable Token
1307 var sp
= _stream_pos
1308 var start_stream_pos
= sp
1309 var start_pos
= _pos
1310 var start_line
= _line
1311 var string
= _file
.string
1312 var string_len
= string
.length
1314 var accept_state
= -1
1315 var accept_token
= -1
1316 var accept_length
= -1
1318 var accept_line
= -1
1321 if sp
>= string_len
then
1324 var c
= string
.chars
[sp
].ascii
1333 _file
.line_starts
[line
] = sp
1337 _file
.line_starts
[line
] = sp
1339 else if c
== 13 then
1343 _file
.line_starts
[line
] = sp
1350 var old_state
= dfa_state
1351 if dfa_state
< -1 then
1352 old_state
= -2 - dfa_state
1358 var high
= lexer_goto
(old_state
, 0) - 1
1361 while low
<= high
do
1362 var middle
= (low
+ high
) / 2
1363 var offset
= middle
* 3 + 1 # +1 because length is at 0
1365 if c
< lexer_goto
(old_state
, offset
) then
1367 else if c
> lexer_goto
(old_state
, offset
+1) then
1370 dfa_state
= lexer_goto
(old_state
, offset
+2)
1375 if dfa_state
> -2 then break
1383 if dfa_state
>= 0 then
1384 var tok
= lexer_accept
(dfa_state
)
1386 accept_state
= dfa_state
1388 accept_length
= sp
- start_stream_pos
1393 if accept_state
!= -1 then
1394 var location
= new Location(_file
, start_line
+ 1, accept_line
+ 1, start_pos
+ 1, accept_pos
)
1397 _stream_pos
= start_stream_pos
+ accept_length
1398 if accept_token
== 0 then
1401 if accept_token
== 1 then
1402 return new TEol.init_tk
(location
)
1404 if accept_token
== 2 then
1405 return new TComment.init_tk
(location
)
1407 if accept_token
== 3 then
1408 return new TKwpackage.init_tk
(location
)
1410 if accept_token
== 4 then
1411 return new TKwmodule.init_tk
(location
)
1413 if accept_token
== 5 then
1414 return new TKwimport.init_tk
(location
)
1416 if accept_token
== 6 then
1417 return new TKwclass.init_tk
(location
)
1419 if accept_token
== 7 then
1420 return new TKwabstract.init_tk
(location
)
1422 if accept_token
== 8 then
1423 return new TKwinterface.init_tk
(location
)
1425 if accept_token
== 9 then
1426 return new TKwenum.init_tk
(location
)
1428 if accept_token
== 10 then
1429 return new TKwend.init_tk
(location
)
1431 if accept_token
== 11 then
1432 return new TKwmeth.init_tk
(location
)
1434 if accept_token
== 12 then
1435 return new TKwtype.init_tk
(location
)
1437 if accept_token
== 13 then
1438 return new TKwinit.init_tk
(location
)
1440 if accept_token
== 14 then
1441 return new TKwredef.init_tk
(location
)
1443 if accept_token
== 15 then
1444 return new TKwis.init_tk
(location
)
1446 if accept_token
== 16 then
1447 return new TKwdo.init_tk
(location
)
1449 if accept_token
== 17 then
1450 return new TKwreadable.init_tk
(location
)
1452 if accept_token
== 18 then
1453 return new TKwwritable.init_tk
(location
)
1455 if accept_token
== 19 then
1456 return new TKwvar.init_tk
(location
)
1458 if accept_token
== 20 then
1459 return new TKwintern.init_tk
(location
)
1461 if accept_token
== 21 then
1462 return new TKwextern.init_tk
(location
)
1464 if accept_token
== 22 then
1465 return new TKwpublic.init_tk
(location
)
1467 if accept_token
== 23 then
1468 return new TKwprotected.init_tk
(location
)
1470 if accept_token
== 24 then
1471 return new TKwprivate.init_tk
(location
)
1473 if accept_token
== 25 then
1474 return new TKwintrude.init_tk
(location
)
1476 if accept_token
== 26 then
1477 return new TKwif.init_tk
(location
)
1479 if accept_token
== 27 then
1480 return new TKwthen.init_tk
(location
)
1482 if accept_token
== 28 then
1483 return new TKwelse.init_tk
(location
)
1485 if accept_token
== 29 then
1486 return new TKwwhile.init_tk
(location
)
1488 if accept_token
== 30 then
1489 return new TKwloop.init_tk
(location
)
1491 if accept_token
== 31 then
1492 return new TKwfor.init_tk
(location
)
1494 if accept_token
== 32 then
1495 return new TKwin.init_tk
(location
)
1497 if accept_token
== 33 then
1498 return new TKwand.init_tk
(location
)
1500 if accept_token
== 34 then
1501 return new TKwor.init_tk
(location
)
1503 if accept_token
== 35 then
1504 return new TKwnot.init_tk
(location
)
1506 if accept_token
== 36 then
1507 return new TKwimplies.init_tk
(location
)
1509 if accept_token
== 37 then
1510 return new TKwreturn.init_tk
(location
)
1512 if accept_token
== 38 then
1513 return new TKwcontinue.init_tk
(location
)
1515 if accept_token
== 39 then
1516 return new TKwbreak.init_tk
(location
)
1518 if accept_token
== 40 then
1519 return new TKwabort.init_tk
(location
)
1521 if accept_token
== 41 then
1522 return new TKwassert.init_tk
(location
)
1524 if accept_token
== 42 then
1525 return new TKwnew.init_tk
(location
)
1527 if accept_token
== 43 then
1528 return new TKwisa.init_tk
(location
)
1530 if accept_token
== 44 then
1531 return new TKwonce.init_tk
(location
)
1533 if accept_token
== 45 then
1534 return new TKwsuper.init_tk
(location
)
1536 if accept_token
== 46 then
1537 return new TKwself.init_tk
(location
)
1539 if accept_token
== 47 then
1540 return new TKwtrue.init_tk
(location
)
1542 if accept_token
== 48 then
1543 return new TKwfalse.init_tk
(location
)
1545 if accept_token
== 49 then
1546 return new TKwnull.init_tk
(location
)
1548 if accept_token
== 50 then
1549 return new TKwas.init_tk
(location
)
1551 if accept_token
== 51 then
1552 return new TKwnullable.init_tk
(location
)
1554 if accept_token
== 52 then
1555 return new TKwisset.init_tk
(location
)
1557 if accept_token
== 53 then
1558 return new TKwlabel.init_tk
(location
)
1560 if accept_token
== 54 then
1561 return new TKwdebug.init_tk
(location
)
1563 if accept_token
== 55 then
1564 return new TOpar.init_tk
(location
)
1566 if accept_token
== 56 then
1567 return new TCpar.init_tk
(location
)
1569 if accept_token
== 57 then
1570 return new TObra.init_tk
(location
)
1572 if accept_token
== 58 then
1573 return new TCbra.init_tk
(location
)
1575 if accept_token
== 59 then
1576 return new TComma.init_tk
(location
)
1578 if accept_token
== 60 then
1579 return new TColumn.init_tk
(location
)
1581 if accept_token
== 61 then
1582 return new TQuad.init_tk
(location
)
1584 if accept_token
== 62 then
1585 return new TAssign.init_tk
(location
)
1587 if accept_token
== 63 then
1588 return new TPluseq.init_tk
(location
)
1590 if accept_token
== 64 then
1591 return new TMinuseq.init_tk
(location
)
1593 if accept_token
== 65 then
1594 return new TDotdotdot.init_tk
(location
)
1596 if accept_token
== 66 then
1597 return new TDotdot.init_tk
(location
)
1599 if accept_token
== 67 then
1600 return new TDot.init_tk
(location
)
1602 if accept_token
== 68 then
1603 return new TPlus.init_tk
(location
)
1605 if accept_token
== 69 then
1606 return new TMinus.init_tk
(location
)
1608 if accept_token
== 70 then
1609 return new TStar.init_tk
(location
)
1611 if accept_token
== 71 then
1612 return new TSlash.init_tk
(location
)
1614 if accept_token
== 72 then
1615 return new TPercent.init_tk
(location
)
1617 if accept_token
== 73 then
1618 return new TEq.init_tk
(location
)
1620 if accept_token
== 74 then
1621 return new TNe.init_tk
(location
)
1623 if accept_token
== 75 then
1624 return new TLt.init_tk
(location
)
1626 if accept_token
== 76 then
1627 return new TLe.init_tk
(location
)
1629 if accept_token
== 77 then
1630 return new TLl.init_tk
(location
)
1632 if accept_token
== 78 then
1633 return new TGt.init_tk
(location
)
1635 if accept_token
== 79 then
1636 return new TGe.init_tk
(location
)
1638 if accept_token
== 80 then
1639 return new TGg.init_tk
(location
)
1641 if accept_token
== 81 then
1642 return new TStarship.init_tk
(location
)
1644 if accept_token
== 82 then
1645 return new TBang.init_tk
(location
)
1647 if accept_token
== 83 then
1648 return new TAt.init_tk
(location
)
1650 if accept_token
== 84 then
1651 return new TClassid.init_tk
(location
)
1653 if accept_token
== 85 then
1654 return new TId.init_tk
(location
)
1656 if accept_token
== 86 then
1657 return new TAttrid.init_tk
(location
)
1659 if accept_token
== 87 then
1660 return new TNumber.init_tk
(location
)
1662 if accept_token
== 88 then
1663 return new THexNumber.init_tk
(location
)
1665 if accept_token
== 89 then
1666 return new TFloat.init_tk
(location
)
1668 if accept_token
== 90 then
1669 return new TString.init_tk
(location
)
1671 if accept_token
== 91 then
1672 return new TStartString.init_tk
(location
)
1674 if accept_token
== 92 then
1675 return new TMidString.init_tk
(location
)
1677 if accept_token
== 93 then
1678 return new TEndString.init_tk
(location
)
1680 if accept_token
== 94 then
1681 return new TChar.init_tk
(location
)
1683 if accept_token
== 95 then
1684 return new TBadString.init_tk
(location
)
1686 if accept_token
== 96 then
1687 return new TBadChar.init_tk
(location
)
1689 if accept_token
== 97 then
1690 return new TExternCodeSegment.init_tk
(location
)
1694 var location
= new Location(_file
, start_line
+ 1, start_line
+ 1, start_pos
+ 1, start_pos
+ 1)
1695 if sp
> start_stream_pos
then
1696 var text
= string
.substring
(start_stream_pos
, sp-start_stream_pos
)
1697 var token
= new ALexerError.init_lexer_error
("Syntax error: unknown token {text}.", location
, text
)
1700 var token
= new EOF.init_tk
(location
)