1 # Lexer and its tokens.
2 # This file was generated by SableCC (http://www.sablecc.org/).
3 module lexer
is no_warning
("missing-doc", "old-init")
5 intrude import parser_nodes
9 var text_cache
: nullable String
14 if res
!= null then return res
20 fun parser_index
: Int is abstract
24 redef fun parser_index
: Int
29 init init_tk
(loc
: Location)
36 redef fun parser_index
: Int
41 init init_tk
(loc
: Location)
48 redef fun parser_index
: Int
53 init init_tk
(loc
: Location)
60 redef fun parser_index
: Int
65 init init_tk
(loc
: Location)
72 redef fun parser_index
: Int
77 init init_tk
(loc
: Location)
84 redef fun parser_index
: Int
89 init init_tk
(loc
: Location)
96 redef fun parser_index
: Int
101 init init_tk
(loc
: Location)
108 redef fun parser_index
: Int
113 init init_tk
(loc
: Location)
120 redef fun parser_index
: Int
125 init init_tk
(loc
: Location)
132 redef fun parser_index
: Int
137 init init_tk
(loc
: Location)
144 redef fun parser_index
: Int
149 init init_tk
(loc
: Location)
156 redef fun parser_index
: Int
161 init init_tk
(loc
: Location)
168 redef fun parser_index
: Int
173 init init_tk
(loc
: Location)
179 redef class TTkAddrss
180 redef fun parser_index
: Int
185 init init_tk
(loc
: Location)
191 redef class TTkEquate
192 redef fun parser_index
: Int
197 init init_tk
(loc
: Location)
204 redef fun parser_index
: Int
209 init init_tk
(loc
: Location)
215 redef class TEndBlock
216 redef fun parser_index
: Int
221 init init_tk
(loc
: Location)
228 redef fun parser_index
: Int
233 init init_tk
(loc
: Location)
241 redef fun parser_index
: Int
256 init init_error
(message
: String, loc
: Location)
259 self.message
= message
264 # The lexer extract NIT tokens from an input stream.
265 # It is better user with the Parser
269 var token
: nullable Token
271 # Lexer current state
277 # Current character in the stream
278 var stream_pos
: Int = 0
280 # Current line number in the input stream
283 # Current column in the input stream
286 # Was the last character a cariage-return?
289 # Constante state values
290 private fun state_initial
: Int do return 0 end
292 # Create a new lexer for a stream (and a name)
293 init(file
: SourceFile)
298 # Give the next token (but do not consume it)
301 while _token
== null do
304 return _token
.as(not null)
307 # Give and consume the next token
311 while result
== null do
318 # Get a token, or null if it is discarded
319 private fun get_token
: nullable Token
324 var start_stream_pos
= sp
326 var start_line
= _line
327 var string
= _file
.string
328 var string_len
= string
.length
330 var accept_state
= -1
331 var accept_token
= -1
332 var accept_length
= -1
337 if sp
>= string_len
then
340 var c
= string
[sp
].ascii
349 _file
.line_starts
[line
] = sp
353 _file
.line_starts
[line
] = sp
359 _file
.line_starts
[line
] = sp
366 var old_state
= dfa_state
367 if dfa_state
< -1 then
368 old_state
= -2 - dfa_state
374 var high
= lexer_goto
(old_state
, 0) - 1
378 var middle
= (low
+ high
) / 2
379 var offset
= middle
* 3 + 1 # +1 because length is at 0
381 if c
< lexer_goto
(old_state
, offset
) then
383 else if c
> lexer_goto
(old_state
, offset
+1) then
386 dfa_state
= lexer_goto
(old_state
, offset
+2)
391 if dfa_state
> -2 then break
399 if dfa_state
>= 0 then
400 var tok
= lexer_accept
(dfa_state
)
402 accept_state
= dfa_state
404 accept_length
= sp
- start_stream_pos
409 if accept_state
!= -1 then
410 var location
= new Location(_file
, start_line
+ 1, accept_line
+ 1, start_pos
+ 1, accept_pos
)
413 _stream_pos
= start_stream_pos
+ accept_length
414 if accept_token
== 0 then
417 if accept_token
== 1 then
418 return new TEol.init_tk
(location
)
420 if accept_token
== 2 then
421 return new TNumber.init_tk
(location
)
423 if accept_token
== 3 then
424 return new TFloat.init_tk
(location
)
426 if accept_token
== 4 then
427 return new TChar.init_tk
(location
)
429 if accept_token
== 5 then
430 return new TString.init_tk
(location
)
432 if accept_token
== 6 then
433 return new THex.init_tk
(location
)
435 if accept_token
== 7 then
436 return new TColon.init_tk
(location
)
438 if accept_token
== 8 then
439 return new TComma.init_tk
(location
)
441 if accept_token
== 9 then
442 return new TComment.init_tk
(location
)
444 if accept_token
== 10 then
445 return new TTkByte.init_tk
(location
)
447 if accept_token
== 11 then
448 return new TTkWord.init_tk
(location
)
450 if accept_token
== 12 then
451 return new TTkBlock.init_tk
(location
)
453 if accept_token
== 13 then
454 return new TTkAscii.init_tk
(location
)
456 if accept_token
== 14 then
457 return new TTkAddrss.init_tk
(location
)
459 if accept_token
== 15 then
460 return new TTkEquate.init_tk
(location
)
462 if accept_token
== 16 then
463 return new TTkBurn.init_tk
(location
)
465 if accept_token
== 17 then
466 return new TEndBlock.init_tk
(location
)
468 if accept_token
== 18 then
469 return new TId.init_tk
(location
)
473 var location
= new Location(_file
, start_line
+ 1, start_line
+ 1, start_pos
+ 1, start_pos
+ 1)
474 if sp
> start_stream_pos
then
475 var text
= string
.substring
(start_stream_pos
, sp-start_stream_pos
)
476 var token
= new AError.init_error
("Syntax error: unknown token {text}.", location
)
479 var token
= new EOF(location
)