1 $ // This file is part of NIT ( http://www.nitlanguage.org ).
3 $ // Copyright 2008 Jean Privat <jean@pryen.org>
4 $ // Based on algorithms developped for ( http://www.sablecc.org/ ).
6 $ // Licensed under the Apache License, Version 2.0 (the "License");
7 $ // you may not use this file except in compliance with the License.
8 $ // You may obtain a copy of the License at
10 $ // http://www.apache.org/licenses/LICENSE-2.0
12 $ // Unless required by applicable law or agreed to in writing, software
13 $ // distributed under the License is distributed on an "AS IS" BASIS,
14 $ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 $ // See the License for the specific language governing permissions and
16 $ // limitations under the License.
18 $ template make_lexer()
20 # The lexer extract NIT tokens from an input stream.
21 # It is better user with the Parser
25 var _token: nullable Token
30 # Name of the stream (as given to tokens)
31 readable var _filename: String
33 # Input stream where character are read
36 # Pushback buffer to store unread character
37 var _stream_buf: Buffer
39 # Number of character stored in the pushback buffer
42 # Current line number in the input stream
45 # Current column in the input stream
48 # Was the last character a cariage-return?
51 # If the end of stream?
52 var _eof: Bool = false
54 # Current working text read from the input stream
57 $ foreach {lexer_data/state}
58 # Constante state values
59 private fun state_${translate(@name,"ABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwxyz")}: Int do return @id end
62 # Create a new lexer for a stream (and a name)
63 init(stream: IStream, fname: String)
69 _stream_buf = new Buffer
72 # Give the next token (but do not consume it)
75 while _token == null do
78 return _token.as(not null)
81 # Give and consume the next token
85 while result == null do
89 return result.as(not null)
92 # Get a token, or null if it is discarded
93 private fun get_token: nullable Token
98 var start_line = _line
100 var accept_state = -1
101 var accept_token = -1
102 var accept_length = -1
135 var old_state = dfa_state
136 if dfa_state < -1 then
137 old_state = -2 - dfa_state
143 var high = lexer_goto(old_state, 0) - 1
147 var middle = (low + high) / 2
148 var offset = middle * 3 + 1 # +1 because length is at 0
150 if c < lexer_goto(old_state, offset) then
152 else if c > lexer_goto(old_state, offset+1) then
155 dfa_state = lexer_goto(old_state, offset+2)
160 if dfa_state > -2 then break
170 if dfa_state >= 0 then
171 var tok = lexer_accept(dfa_state)
173 accept_state = dfa_state
175 accept_length = text.length
180 if accept_state != -1 then
181 var location = new Location(_filename, start_line + 1, accept_line + 1, start_pos + 1, accept_pos)
184 push_back(accept_length)
186 if accept_token == ${position()-1} then
187 $ if {count(transition[@from!=@to])!=0}
188 var state_id = _state
189 $ foreach transition in {transition[@from!=@to]}
190 if state_id == ${/parser/lexer_data/state[@name=$transition/@from]/@id} then
191 _state = state_${translate(@to,"ABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwxyz")}
197 var token_text = text.substring(0, accept_length)
198 return new @ename.init_tk(token_text, location)
200 return new @ename.init_tk(location)
208 var location = new Location(_filename, start_line + 1, start_line + 1, start_pos + 1, start_pos + 1)
209 if text.length > 0 then
210 var token = new PError.init_error("Syntax error: unknown token {text}.", location)
213 var token = new EOF(location)
218 if false then break # FIXME remove once unreach loop exits are in c_src
220 return null # FIXME remove once unreach loop exits are in c_src
223 # Read the next character.
224 # The character is read from the stream of from the pushback buffer.
225 private fun get_char: Int
235 var res = _stream_buf[_stream_pos]
239 result = _stream.read_char
249 # Unread some characters.
250 # Unread characters are stored in the pushback buffer.
251 private fun push_back(accept_length: Int)
253 var length = _text.length
255 while i >= accept_length do
257 _stream_pos = _stream_pos + 1
258 _stream_buf[_stream_pos] = _text[i]
268 $ template make_lexer_table()
269 $ foreach {lexer_data/goto_table/state}
271 $ if {count(goto)!=0}
272 static const int lexer_goto_row${position()}[] = {
275 @low, @high, @state[-sep ','-]
280 static const int lexer_goto_row_null[] = {0};
281 const int* const lexer_goto_table[] = {
283 $ if {count(goto)!=0}
284 lexer_goto_row${position()}[-sep ','-]
286 lexer_goto_row_null[-sep ','-]
292 $ foreach {lexer_data/accept_table/state}
293 const int lexer_accept_table[] = {
294 [-foreach {i}-]${.}[-sep ','-][-end foreach-]