X-Git-Url: http://nitlanguage.org diff --git a/src/parser/lexer_work.nit b/src/parser/lexer_work.nit index 1c626b8..0f06bce 100644 --- a/src/parser/lexer_work.nit +++ b/src/parser/lexer_work.nit @@ -19,34 +19,29 @@ intrude import parser_nodes private import tables redef class Token - var _text: nullable String + private var cached_text: nullable String redef fun text do - var res = _text + var res = _cached_text if res != null then return res res = location.text - _text = res + _cached_text = res return res end redef fun text=(text) do - _text = text + _cached_text = text end fun parser_index: Int is abstract end redef class EOF - redef fun parser_index: Int - do - return 97 - end - init init_tk(loc: Location) do - _text = "" + _cached_text = "" _location = loc end end @@ -85,36 +80,31 @@ end # It is better user with the Parser class Lexer super TablesCapable + # Last peeked token - var _token: nullable Token + var token: nullable Token = null # Lexer current state - var _state: Int = 0 + private var state: Int = 0 # The source file var file: SourceFile # Current character in the stream - var _stream_pos: Int = 0 + var stream_pos: Int = 0 # Current line number in the input stream - var _line: Int = 0 + var line: Int = 0 # Current column in the input stream - var _pos: Int = 0 + var pos: Int = 0 - # Was the last character a cariage-return? - var _cr: Bool = false + # Was the last character a carriage-return? + var cr: Bool = false # Constante state values private fun state_initial: Int do return 0 end - # Create a new lexer for a stream (and a name) - init(file: SourceFile) - do - self.file = file - end - # The last peeked token to chain them private var last_token: nullable Token = null @@ -127,7 +117,7 @@ class Lexer t = get_token while t == null do t = get_token - if t._location != null then + if isset t._location then var l = last_token if l != null then l.next_token = t @@ -244,20 +234,21 @@ class Lexer end else if accept_state != -1 then - var location = new Location(file, start_line + 1, accept_line + 1, start_pos + 1, accept_pos) _pos = accept_pos _line = accept_line _stream_pos = start_stream_pos + accept_length if accept_token == 0 then + # Ignored token (whitespaces) return null end + var location = new Location(file, start_line + 1, accept_line + 1, start_pos + 1, accept_pos) return make_token(accept_token, location) else _stream_pos = sp var location = new Location(file, start_line + 1, start_line + 1, start_pos + 1, start_pos + 1) if sp > start_stream_pos then var text = string.substring(start_stream_pos, sp-start_stream_pos) - var token = new ALexerError.init_lexer_error("Syntax error: unknown token {text}.", location, text) + var token = new ALexerError.init_lexer_error("Syntax Error: unknown token `{text}`.", location, text) file.last_token = token return token else