1 /* This file is part of NIT ( http://www.nitlanguage.org ).
3 * Copyright 2008 Jean Privat <jean@pryen.org>
4 * Based on algorithms developped for ( http://www.sablecc.org/ ).
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
10 * http://www.apache.org/licenses/LICENSE-2.0
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
19 $ template make_lexer()
21 # The lexer extract NIT tokens from an input stream.
22 # It is better user with the Parser
30 # Name of the stream (as given to tokens)
31 readable attr _filename: String
33 # Input stream where character are read
36 # Pushback buffer to store unread character
37 attr _stream_buf: String
39 # Number of character stored in the pushback buffer
42 # Current line number in the input stream
45 # Current column in the input stream
48 # Was the last character a cariage-return?
51 # If the end of stream?
54 # Current working text read from the input stream
57 $ foreach {lexer_data/state}
58 # Constante state values
59 private meth state_${translate(@name,"ABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwxyz")}: Int do return @id end
62 # Create a new lexer for a stream (and a name)
63 init(stream: IStream, fname: String)
69 _stream_buf = new String
74 # Give the next token (but do not consume it)
77 while _token == null do
83 # Give and consume the next token
87 while result == null do
94 # Get a token, or null if it is discarded
95 private meth get_token: Token
100 var start_line = _line
102 var accept_state = -1
103 var accept_token = -1
104 var accept_length = -1
108 var goto_table = _goto_table[_state]
109 var accept = _accept_table[_state]
134 var first_loop = true # aka until
135 while dfa_state < -1 or first_loop do
136 var old_state = dfa_state
137 if dfa_state < -1 then
138 old_state = -2 - dfa_state
143 var tmp1 = goto_table[old_state]
145 var high = tmp1.length - 1
148 var middle = (low + high) / 2
149 var tmp2 = tmp1[middle]
153 else if c > tmp2[1] then
157 low = high + 1 # aka break
160 first_loop = false # aka until
166 if dfa_state >= 0 then
167 if accept[dfa_state] != -1 then
168 accept_state = dfa_state
169 accept_token = accept[dfa_state]
170 accept_length = _text.length
175 if accept_state != -1 then
177 if accept_token == ${position()-1} then
180 var token_text = _text.substring(0, accept_length)
181 var token = new @ename.init_tk(token_text, _filename, start_line + 1, start_pos + 1)
184 var token = new @ename.init_tk(_filename, start_line + 1, start_pos + 1)
186 push_back(accept_length)
189 $ if {count(transition[@from!=@to])!=0}
190 var state_id = _state
191 $ foreach transition in {transition[@from!=@to]}
192 if state_id == ${/parser/lexer_data/state[@name=$transition/@from]/@id} then
193 _state = state_${translate(@to,"ABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwxyz")}
205 if _text.length > 0 then
206 var token = new PError.init_error(_filename, start_line + 1, start_pos + 1, "Unknown token: {_text}")
209 var token = new EOF(_filename, start_line + 1, start_pos + 1)
218 # Read the next character.
219 # The character is read from the stream of from the pushback buffer.
220 private meth get_char: Int
230 var res = _stream_buf[_stream_pos]
234 result = _stream.read_char
244 # Unread some characters.
245 # Unread characters are stored in the pushback buffer.
246 private meth push_back(accept_length: Int)
248 var length = _text.length
250 while i >= accept_length do
252 _stream_pos = _stream_pos + 1
253 _stream_buf[_stream_pos] = _text[i]
258 attr _goto_table: Array[Array[Array[Array[Int]]]]
259 private meth build_goto_table
262 $ foreach {lexer_data/goto_table/state}
265 $ if {count(goto)!=0}
268 [@low, @high, @state] [-sep ','-]
272 nil_array [-sep ','-]
280 private meth nil_array: Array[Array[Int]]
282 return once new Array[Array[Int]]
285 attr _accept_table: Array[Array[Int]]
286 private meth build_accept_table do
287 _accept_table = once [
288 $ foreach {lexer_data/accept_table/state}
290 [-foreach {i}-]${.} [-sep ','-] [-end foreach-]