1 # Lexer and its tokens.
2 # This file was generated by SableCC (http://www.sablecc.org/).
3 module lexer
is generated
, no_warning
("missing-doc", "old-init")
5 intrude import parser_nodes
9 var text_cache
: nullable String
14 if res
!= null then return res
20 fun parser_index
: Int is abstract
24 redef fun parser_index
: Int
29 init init_tk
(loc
: Location)
36 redef fun parser_index
: Int
41 init init_tk
(loc
: Location)
48 redef fun parser_index
: Int
53 init init_tk
(loc
: Location)
60 redef fun parser_index
: Int
65 init init_tk
(loc
: Location)
72 redef fun parser_index
: Int
77 init init_tk
(loc
: Location)
84 redef fun parser_index
: Int
89 init init_tk
(loc
: Location)
96 redef fun parser_index
: Int
101 init init_tk
(loc
: Location)
108 redef fun parser_index
: Int
113 init init_tk
(loc
: Location)
120 redef fun parser_index
: Int
125 init init_tk
(loc
: Location)
132 redef fun parser_index
: Int
137 init init_tk
(loc
: Location)
144 redef fun parser_index
: Int
149 init init_tk
(loc
: Location)
156 redef fun parser_index
: Int
161 init init_tk
(loc
: Location)
168 redef fun parser_index
: Int
173 init init_tk
(loc
: Location)
179 redef class TTkAddrss
180 redef fun parser_index
: Int
185 init init_tk
(loc
: Location)
191 redef class TTkEquate
192 redef fun parser_index
: Int
197 init init_tk
(loc
: Location)
204 redef fun parser_index
: Int
209 init init_tk
(loc
: Location)
215 redef class TEndBlock
216 redef fun parser_index
: Int
221 init init_tk
(loc
: Location)
228 redef fun parser_index
: Int
233 init init_tk
(loc
: Location)
241 redef fun parser_index
: Int
256 init init_error
(message
: String, loc
: Location)
259 self.message
= message
264 # The lexer extract NIT tokens from an input stream.
265 # It is better user with the Parser
269 var token
: nullable Token
271 # Lexer current state
277 # Current character in the stream
278 var stream_pos
: Int = 0
280 # Current line number in the input stream
283 # Current column in the input stream
286 # Was the last character a cariage-return?
289 # Constante state values
290 private fun state_initial
: Int do return 0 end
292 # Create a new lexer for a stream (and a name)
293 init(file
: SourceFile)
298 # Give the next token (but do not consume it)
301 while _token
== null do
304 return _token
.as(not null)
307 # Give and consume the next token
311 while result
== null do
318 # Get a token, or null if it is discarded
319 private fun get_token
: nullable Token
324 var start_stream_pos
= sp
326 var start_line
= _line
327 var string
= _file
.string
328 var string_len
= string
.length
330 var accept_state
= -1
331 var accept_token
= -1
332 var accept_length
= -1
337 if sp
>= string_len
then
340 var c
= string
[sp
].code_point
341 if c
>= 255 then c
= 255
350 _file
.line_starts
[line
] = sp
354 _file
.line_starts
[line
] = sp
360 _file
.line_starts
[line
] = sp
367 var old_state
= dfa_state
368 if dfa_state
< -1 then
369 old_state
= -2 - dfa_state
375 var high
= lexer_goto
(old_state
, 0) - 1
379 var middle
= (low
+ high
) / 2
380 var offset
= middle
* 3 + 1 # +1 because length is at 0
382 if c
< lexer_goto
(old_state
, offset
) then
384 else if c
> lexer_goto
(old_state
, offset
+1) then
387 dfa_state
= lexer_goto
(old_state
, offset
+2)
392 if dfa_state
> -2 then break
400 if dfa_state
>= 0 then
401 var tok
= lexer_accept
(dfa_state
)
403 accept_state
= dfa_state
405 accept_length
= sp
- start_stream_pos
410 if accept_state
!= -1 then
411 var location
= new Location(_file
, start_line
+ 1, accept_line
+ 1, start_pos
+ 1, accept_pos
)
414 _stream_pos
= start_stream_pos
+ accept_length
415 if accept_token
== 0 then
418 if accept_token
== 1 then
419 return new TEol.init_tk
(location
)
421 if accept_token
== 2 then
422 return new TNumber.init_tk
(location
)
424 if accept_token
== 3 then
425 return new TFloat.init_tk
(location
)
427 if accept_token
== 4 then
428 return new TChar.init_tk
(location
)
430 if accept_token
== 5 then
431 return new TString.init_tk
(location
)
433 if accept_token
== 6 then
434 return new THex.init_tk
(location
)
436 if accept_token
== 7 then
437 return new TColon.init_tk
(location
)
439 if accept_token
== 8 then
440 return new TComma.init_tk
(location
)
442 if accept_token
== 9 then
443 return new TComment.init_tk
(location
)
445 if accept_token
== 10 then
446 return new TTkByte.init_tk
(location
)
448 if accept_token
== 11 then
449 return new TTkWord.init_tk
(location
)
451 if accept_token
== 12 then
452 return new TTkBlock.init_tk
(location
)
454 if accept_token
== 13 then
455 return new TTkAscii.init_tk
(location
)
457 if accept_token
== 14 then
458 return new TTkAddrss.init_tk
(location
)
460 if accept_token
== 15 then
461 return new TTkEquate.init_tk
(location
)
463 if accept_token
== 16 then
464 return new TTkBurn.init_tk
(location
)
466 if accept_token
== 17 then
467 return new TEndBlock.init_tk
(location
)
469 if accept_token
== 18 then
470 return new TId.init_tk
(location
)
474 var location
= new Location(_file
, start_line
+ 1, start_line
+ 1, start_pos
+ 1, start_pos
+ 1)
475 if sp
> start_stream_pos
then
476 var text
= string
.substring
(start_stream_pos
, sp-start_stream_pos
)
477 var token
= new AError.init_error
("Syntax error: unknown token {text}.", location
)
480 var token
= new EOF(location
)