1 # Lexer and its tokens.
2 # This file was generated by SableCC (http://www.sablecc.org/).
5 intrude import parser_nodes
9 readable var _text
: String
11 fun parser_index
: Int is abstract
13 redef fun to_s
: String do
19 redef fun parser_index
: Int
24 init init_tk
(text
: String, loc
: Location)
32 redef fun parser_index
: Int
37 init init_tk
(text
: String, loc
: Location)
44 redef class TKwpackage
45 redef fun parser_index
: Int
50 init init_tk
(loc
: Location)
52 _text
= once
"package"
58 redef fun parser_index
: Int
63 init init_tk
(loc
: Location)
71 redef fun parser_index
: Int
76 init init_tk
(loc
: Location)
83 redef class TKwabstract
84 redef fun parser_index
: Int
89 init init_tk
(loc
: Location)
91 _text
= once
"abstract"
96 redef class TKwinterface
97 redef fun parser_index
: Int
102 init init_tk
(loc
: Location)
104 _text
= once
"interface"
109 redef class TKwuniversal
110 redef fun parser_index
: Int
115 init init_tk
(loc
: Location)
117 _text
= once
"universal"
122 redef class TKwspecial
123 redef fun parser_index
: Int
128 init init_tk
(loc
: Location)
130 _text
= once
"special"
136 redef fun parser_index
: Int
141 init init_tk
(loc
: Location)
149 redef fun parser_index
: Int
154 init init_tk
(loc
: Location)
162 redef fun parser_index
: Int
167 init init_tk
(loc
: Location)
175 redef fun parser_index
: Int
180 init init_tk
(loc
: Location)
188 redef fun parser_index
: Int
193 init init_tk
(loc
: Location)
201 redef fun parser_index
: Int
206 init init_tk
(loc
: Location)
214 redef fun parser_index
: Int
219 init init_tk
(loc
: Location)
226 redef class TKwreadable
227 redef fun parser_index
: Int
232 init init_tk
(loc
: Location)
234 _text
= once
"readable"
239 redef class TKwwritable
240 redef fun parser_index
: Int
245 init init_tk
(loc
: Location)
247 _text
= once
"writable"
253 redef fun parser_index
: Int
258 init init_tk
(loc
: Location)
265 redef class TKwintern
266 redef fun parser_index
: Int
271 init init_tk
(loc
: Location)
273 _text
= once
"intern"
278 redef class TKwextern
279 redef fun parser_index
: Int
284 init init_tk
(loc
: Location)
286 _text
= once
"extern"
291 redef class TKwprotected
292 redef fun parser_index
: Int
297 init init_tk
(loc
: Location)
299 _text
= once
"protected"
304 redef class TKwprivate
305 redef fun parser_index
: Int
310 init init_tk
(loc
: Location)
312 _text
= once
"private"
317 redef class TKwintrude
318 redef fun parser_index
: Int
323 init init_tk
(loc
: Location)
325 _text
= once
"intrude"
331 redef fun parser_index
: Int
336 init init_tk
(loc
: Location)
344 redef fun parser_index
: Int
349 init init_tk
(loc
: Location)
357 redef fun parser_index
: Int
362 init init_tk
(loc
: Location)
370 redef fun parser_index
: Int
375 init init_tk
(loc
: Location)
383 redef fun parser_index
: Int
388 init init_tk
(loc
: Location)
396 redef fun parser_index
: Int
401 init init_tk
(loc
: Location)
409 redef fun parser_index
: Int
414 init init_tk
(loc
: Location)
422 redef fun parser_index
: Int
427 init init_tk
(loc
: Location)
435 redef fun parser_index
: Int
440 init init_tk
(loc
: Location)
448 redef fun parser_index
: Int
453 init init_tk
(loc
: Location)
460 redef class TKwreturn
461 redef fun parser_index
: Int
466 init init_tk
(loc
: Location)
468 _text
= once
"return"
473 redef class TKwcontinue
474 redef fun parser_index
: Int
479 init init_tk
(loc
: Location)
481 _text
= once
"continue"
487 redef fun parser_index
: Int
492 init init_tk
(loc
: Location)
500 redef fun parser_index
: Int
505 init init_tk
(loc
: Location)
512 redef class TKwassert
513 redef fun parser_index
: Int
518 init init_tk
(loc
: Location)
520 _text
= once
"assert"
526 redef fun parser_index
: Int
531 init init_tk
(loc
: Location)
539 redef fun parser_index
: Int
544 init init_tk
(loc
: Location)
552 redef fun parser_index
: Int
557 init init_tk
(loc
: Location)
565 redef fun parser_index
: Int
570 init init_tk
(loc
: Location)
578 redef fun parser_index
: Int
583 init init_tk
(loc
: Location)
591 redef fun parser_index
: Int
596 init init_tk
(loc
: Location)
604 redef fun parser_index
: Int
609 init init_tk
(loc
: Location)
617 redef fun parser_index
: Int
622 init init_tk
(loc
: Location)
630 redef fun parser_index
: Int
635 init init_tk
(loc
: Location)
642 redef class TKwnullable
643 redef fun parser_index
: Int
648 init init_tk
(loc
: Location)
650 _text
= once
"nullable"
656 redef fun parser_index
: Int
661 init init_tk
(loc
: Location)
669 redef fun parser_index
: Int
674 init init_tk
(loc
: Location)
682 redef fun parser_index
: Int
687 init init_tk
(loc
: Location)
695 redef fun parser_index
: Int
700 init init_tk
(loc
: Location)
708 redef fun parser_index
: Int
713 init init_tk
(loc
: Location)
721 redef fun parser_index
: Int
726 init init_tk
(loc
: Location)
734 redef fun parser_index
: Int
739 init init_tk
(loc
: Location)
747 redef fun parser_index
: Int
752 init init_tk
(loc
: Location)
760 redef fun parser_index
: Int
765 init init_tk
(loc
: Location)
773 redef fun parser_index
: Int
778 init init_tk
(loc
: Location)
786 redef fun parser_index
: Int
791 init init_tk
(loc
: Location)
799 redef fun parser_index
: Int
804 init init_tk
(loc
: Location)
811 redef class TDotdotdot
812 redef fun parser_index
: Int
817 init init_tk
(loc
: Location)
825 redef fun parser_index
: Int
830 init init_tk
(loc
: Location)
838 redef fun parser_index
: Int
843 init init_tk
(loc
: Location)
851 redef fun parser_index
: Int
856 init init_tk
(loc
: Location)
864 redef fun parser_index
: Int
869 init init_tk
(loc
: Location)
877 redef fun parser_index
: Int
882 init init_tk
(loc
: Location)
890 redef fun parser_index
: Int
895 init init_tk
(loc
: Location)
903 redef fun parser_index
: Int
908 init init_tk
(loc
: Location)
916 redef fun parser_index
: Int
921 init init_tk
(loc
: Location)
929 redef fun parser_index
: Int
934 init init_tk
(loc
: Location)
942 redef fun parser_index
: Int
947 init init_tk
(loc
: Location)
955 redef fun parser_index
: Int
960 init init_tk
(loc
: Location)
968 redef fun parser_index
: Int
973 init init_tk
(loc
: Location)
981 redef fun parser_index
: Int
986 init init_tk
(loc
: Location)
994 redef fun parser_index
: Int
999 init init_tk
(loc
: Location)
1007 redef fun parser_index
: Int
1012 init init_tk
(loc
: Location)
1019 redef class TStarship
1020 redef fun parser_index
: Int
1025 init init_tk
(loc
: Location)
1033 redef fun parser_index
: Int
1038 init init_tk
(loc
: Location)
1045 redef class TClassid
1046 redef fun parser_index
: Int
1051 init init_tk
(text
: String, loc
: Location)
1059 redef fun parser_index
: Int
1064 init init_tk
(text
: String, loc
: Location)
1072 redef fun parser_index
: Int
1077 init init_tk
(text
: String, loc
: Location)
1085 redef fun parser_index
: Int
1090 init init_tk
(text
: String, loc
: Location)
1098 redef fun parser_index
: Int
1103 init init_tk
(text
: String, loc
: Location)
1111 redef fun parser_index
: Int
1116 init init_tk
(text
: String, loc
: Location)
1124 redef fun parser_index
: Int
1129 init init_tk
(text
: String, loc
: Location)
1136 redef class TStartString
1137 redef fun parser_index
: Int
1142 init init_tk
(text
: String, loc
: Location)
1149 redef class TMidString
1150 redef fun parser_index
: Int
1155 init init_tk
(text
: String, loc
: Location)
1162 redef class TEndString
1163 redef fun parser_index
: Int
1168 init init_tk
(text
: String, loc
: Location)
1177 redef fun parser_index
: Int
1190 readable var _message
: String
1192 init init_error
(message
: String, loc
: Location)
1200 # The lexer extract NIT tokens from an input stream.
1201 # It is better user with the Parser
1203 special TablesCapable
1205 var _token
: nullable Token
1207 # Lexer current state
1210 # Name of the stream (as given to tokens)
1211 readable var _filename
: String
1213 # Input stream where character are read
1214 var _stream
: IStream
1216 # Pushback buffer to store unread character
1217 var _stream_buf
: Buffer
1219 # Number of character stored in the pushback buffer
1220 var _stream_pos
: Int
1222 # Current line number in the input stream
1225 # Current column in the input stream
1228 # Was the last character a cariage-return?
1229 var _cr
: Bool = false
1231 # If the end of stream?
1232 var _eof
: Bool = false
1234 # Current working text read from the input stream
1237 # Constante state values
1238 private fun state_initial
: Int do return 0 end
1240 # Create a new lexer for a stream (and a name)
1241 init(stream
: IStream, fname
: String)
1247 _stream_buf
= new Buffer
1250 # Give the next token (but do not consume it)
1253 while _token
== null do
1256 return _token
.as(not null)
1259 # Give and consume the next token
1263 while result
== null do
1267 return result
.as(not null)
1270 # Get a token, or null if it is discarded
1271 private fun get_token
: nullable Token
1275 var start_pos
= _pos
1276 var start_line
= _line
1278 var accept_state
= -1
1279 var accept_token
= -1
1280 var accept_length
= -1
1282 var accept_line
= -1
1301 else if c
== 13 then
1313 var old_state
= dfa_state
1314 if dfa_state
< -1 then
1315 old_state
= -2 - dfa_state
1321 var high
= lexer_goto
(old_state
, 0) - 1
1324 while low
<= high
do
1325 var middle
= (low
+ high
) / 2
1326 var offset
= middle
* 3 + 1 # +1 because length is at 0
1328 if c
< lexer_goto
(old_state
, offset
) then
1330 else if c
> lexer_goto
(old_state
, offset
+1) then
1333 dfa_state
= lexer_goto
(old_state
, offset
+2)
1338 if dfa_state
> -2 then break
1348 if dfa_state
>= 0 then
1349 var tok
= lexer_accept
(dfa_state
)
1351 accept_state
= dfa_state
1353 accept_length
= text
.length
1358 if accept_state
!= -1 then
1359 var location
= new Location(_filename
, start_line
+ 1, accept_line
+ 1, start_pos
+ 1, accept_pos
)
1362 push_back
(accept_length
)
1363 if accept_token
== 0 then
1366 if accept_token
== 1 then
1367 var token_text
= text
.substring
(0, accept_length
)
1368 return new TEol.init_tk
(token_text
, location
)
1370 if accept_token
== 2 then
1371 var token_text
= text
.substring
(0, accept_length
)
1372 return new TComment.init_tk
(token_text
, location
)
1374 if accept_token
== 3 then
1375 return new TKwpackage.init_tk
(location
)
1377 if accept_token
== 4 then
1378 return new TKwimport.init_tk
(location
)
1380 if accept_token
== 5 then
1381 return new TKwclass.init_tk
(location
)
1383 if accept_token
== 6 then
1384 return new TKwabstract.init_tk
(location
)
1386 if accept_token
== 7 then
1387 return new TKwinterface.init_tk
(location
)
1389 if accept_token
== 8 then
1390 return new TKwuniversal.init_tk
(location
)
1392 if accept_token
== 9 then
1393 return new TKwspecial.init_tk
(location
)
1395 if accept_token
== 10 then
1396 return new TKwend.init_tk
(location
)
1398 if accept_token
== 11 then
1399 return new TKwmeth.init_tk
(location
)
1401 if accept_token
== 12 then
1402 return new TKwtype.init_tk
(location
)
1404 if accept_token
== 13 then
1405 return new TKwinit.init_tk
(location
)
1407 if accept_token
== 14 then
1408 return new TKwredef.init_tk
(location
)
1410 if accept_token
== 15 then
1411 return new TKwis.init_tk
(location
)
1413 if accept_token
== 16 then
1414 return new TKwdo.init_tk
(location
)
1416 if accept_token
== 17 then
1417 return new TKwreadable.init_tk
(location
)
1419 if accept_token
== 18 then
1420 return new TKwwritable.init_tk
(location
)
1422 if accept_token
== 19 then
1423 return new TKwvar.init_tk
(location
)
1425 if accept_token
== 20 then
1426 return new TKwintern.init_tk
(location
)
1428 if accept_token
== 21 then
1429 return new TKwextern.init_tk
(location
)
1431 if accept_token
== 22 then
1432 return new TKwprotected.init_tk
(location
)
1434 if accept_token
== 23 then
1435 return new TKwprivate.init_tk
(location
)
1437 if accept_token
== 24 then
1438 return new TKwintrude.init_tk
(location
)
1440 if accept_token
== 25 then
1441 return new TKwif.init_tk
(location
)
1443 if accept_token
== 26 then
1444 return new TKwthen.init_tk
(location
)
1446 if accept_token
== 27 then
1447 return new TKwelse.init_tk
(location
)
1449 if accept_token
== 28 then
1450 return new TKwwhile.init_tk
(location
)
1452 if accept_token
== 29 then
1453 return new TKwloop.init_tk
(location
)
1455 if accept_token
== 30 then
1456 return new TKwfor.init_tk
(location
)
1458 if accept_token
== 31 then
1459 return new TKwin.init_tk
(location
)
1461 if accept_token
== 32 then
1462 return new TKwand.init_tk
(location
)
1464 if accept_token
== 33 then
1465 return new TKwor.init_tk
(location
)
1467 if accept_token
== 34 then
1468 return new TKwnot.init_tk
(location
)
1470 if accept_token
== 35 then
1471 return new TKwreturn.init_tk
(location
)
1473 if accept_token
== 36 then
1474 return new TKwcontinue.init_tk
(location
)
1476 if accept_token
== 37 then
1477 return new TKwbreak.init_tk
(location
)
1479 if accept_token
== 38 then
1480 return new TKwabort.init_tk
(location
)
1482 if accept_token
== 39 then
1483 return new TKwassert.init_tk
(location
)
1485 if accept_token
== 40 then
1486 return new TKwnew.init_tk
(location
)
1488 if accept_token
== 41 then
1489 return new TKwisa.init_tk
(location
)
1491 if accept_token
== 42 then
1492 return new TKwonce.init_tk
(location
)
1494 if accept_token
== 43 then
1495 return new TKwsuper.init_tk
(location
)
1497 if accept_token
== 44 then
1498 return new TKwself.init_tk
(location
)
1500 if accept_token
== 45 then
1501 return new TKwtrue.init_tk
(location
)
1503 if accept_token
== 46 then
1504 return new TKwfalse.init_tk
(location
)
1506 if accept_token
== 47 then
1507 return new TKwnull.init_tk
(location
)
1509 if accept_token
== 48 then
1510 return new TKwas.init_tk
(location
)
1512 if accept_token
== 49 then
1513 return new TKwnullable.init_tk
(location
)
1515 if accept_token
== 50 then
1516 return new TKwisset.init_tk
(location
)
1518 if accept_token
== 51 then
1519 return new TKwlabel.init_tk
(location
)
1521 if accept_token
== 52 then
1522 return new TOpar.init_tk
(location
)
1524 if accept_token
== 53 then
1525 return new TCpar.init_tk
(location
)
1527 if accept_token
== 54 then
1528 return new TObra.init_tk
(location
)
1530 if accept_token
== 55 then
1531 return new TCbra.init_tk
(location
)
1533 if accept_token
== 56 then
1534 return new TComma.init_tk
(location
)
1536 if accept_token
== 57 then
1537 return new TColumn.init_tk
(location
)
1539 if accept_token
== 58 then
1540 return new TQuad.init_tk
(location
)
1542 if accept_token
== 59 then
1543 return new TAssign.init_tk
(location
)
1545 if accept_token
== 60 then
1546 return new TPluseq.init_tk
(location
)
1548 if accept_token
== 61 then
1549 return new TMinuseq.init_tk
(location
)
1551 if accept_token
== 62 then
1552 return new TDotdotdot.init_tk
(location
)
1554 if accept_token
== 63 then
1555 return new TDotdot.init_tk
(location
)
1557 if accept_token
== 64 then
1558 return new TDot.init_tk
(location
)
1560 if accept_token
== 65 then
1561 return new TPlus.init_tk
(location
)
1563 if accept_token
== 66 then
1564 return new TMinus.init_tk
(location
)
1566 if accept_token
== 67 then
1567 return new TStar.init_tk
(location
)
1569 if accept_token
== 68 then
1570 return new TSlash.init_tk
(location
)
1572 if accept_token
== 69 then
1573 return new TPercent.init_tk
(location
)
1575 if accept_token
== 70 then
1576 return new TEq.init_tk
(location
)
1578 if accept_token
== 71 then
1579 return new TNe.init_tk
(location
)
1581 if accept_token
== 72 then
1582 return new TLt.init_tk
(location
)
1584 if accept_token
== 73 then
1585 return new TLe.init_tk
(location
)
1587 if accept_token
== 74 then
1588 return new TLl.init_tk
(location
)
1590 if accept_token
== 75 then
1591 return new TGt.init_tk
(location
)
1593 if accept_token
== 76 then
1594 return new TGe.init_tk
(location
)
1596 if accept_token
== 77 then
1597 return new TGg.init_tk
(location
)
1599 if accept_token
== 78 then
1600 return new TStarship.init_tk
(location
)
1602 if accept_token
== 79 then
1603 return new TBang.init_tk
(location
)
1605 if accept_token
== 80 then
1606 var token_text
= text
.substring
(0, accept_length
)
1607 return new TClassid.init_tk
(token_text
, location
)
1609 if accept_token
== 81 then
1610 var token_text
= text
.substring
(0, accept_length
)
1611 return new TId.init_tk
(token_text
, location
)
1613 if accept_token
== 82 then
1614 var token_text
= text
.substring
(0, accept_length
)
1615 return new TAttrid.init_tk
(token_text
, location
)
1617 if accept_token
== 83 then
1618 var token_text
= text
.substring
(0, accept_length
)
1619 return new TNumber.init_tk
(token_text
, location
)
1621 if accept_token
== 84 then
1622 var token_text
= text
.substring
(0, accept_length
)
1623 return new TFloat.init_tk
(token_text
, location
)
1625 if accept_token
== 85 then
1626 var token_text
= text
.substring
(0, accept_length
)
1627 return new TChar.init_tk
(token_text
, location
)
1629 if accept_token
== 86 then
1630 var token_text
= text
.substring
(0, accept_length
)
1631 return new TString.init_tk
(token_text
, location
)
1633 if accept_token
== 87 then
1634 var token_text
= text
.substring
(0, accept_length
)
1635 return new TStartString.init_tk
(token_text
, location
)
1637 if accept_token
== 88 then
1638 var token_text
= text
.substring
(0, accept_length
)
1639 return new TMidString.init_tk
(token_text
, location
)
1641 if accept_token
== 89 then
1642 var token_text
= text
.substring
(0, accept_length
)
1643 return new TEndString.init_tk
(token_text
, location
)
1646 var location
= new Location(_filename
, start_line
+ 1, start_line
+ 1, start_pos
+ 1, start_pos
+ 1)
1647 if text
.length
> 0 then
1648 var token
= new AError.init_error
("Syntax error: unknown token {text}.", location
)
1651 var token
= new EOF(location
)
1656 if false then break # FIXME remove once unreach loop exits are in c_src
1658 return null # FIXME remove once unreach loop exits are in c_src
1661 # Read the next character.
1662 # The character is read from the stream of from the pushback buffer.
1663 private fun get_char
: Int
1671 var sp
= _stream_pos
1673 var res
= _stream_buf
[_stream_pos
]
1674 _stream_pos
= sp
- 1
1677 result
= _stream
.read_char
1680 if result
== -1 then
1687 # Unread some characters.
1688 # Unread characters are stored in the pushback buffer.
1689 private fun push_back
(accept_length
: Int)
1691 var length
= _text
.length
1693 while i
>= accept_length
do
1695 _stream_pos
= _stream_pos
+ 1
1696 _stream_buf
[_stream_pos
] = _text
[i
]