1 $ // This file is part of NIT ( http://www.nitlanguage.org ).
3 $ // Copyright 2008 Jean Privat <jean@pryen.org>
4 $ // Based on algorithms developped for ( http://www.sablecc.org/ ).
6 $ // Licensed under the Apache License, Version 2.0 (the "License");
7 $ // you may not use this file except in compliance with the License.
8 $ // You may obtain a copy of the License at
10 $ // http://www.apache.org/licenses/LICENSE-2.0
12 $ // Unless required by applicable law or agreed to in writing, software
13 $ // distributed under the License is distributed on an "AS IS" BASIS,
14 $ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 $ // See the License for the specific language governing permissions and
16 $ // limitations under the License.
18 $ template make_lexer()
20 # The lexer extract NIT tokens from an input stream.
21 # It is better user with the Parser
25 var _token: nullable Token
31 readable var _file: SourceFile
33 # Current character in the stream
34 var _stream_pos: Int = 0
36 # Current line number in the input stream
39 # Current column in the input stream
42 # Was the last character a cariage-return?
45 $ foreach {lexer_data/state}
46 # Constante state values
47 private fun state_${translate(@name,"ABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwxyz")}: Int do return @id end
50 # Create a new lexer for a stream (and a name)
51 init(file: SourceFile)
56 # The last peeked token to chain them
57 private var last_token: nullable Token = null
59 # Give the next token (but do not consume it)
63 if t != null then return t
66 while t == null do t = get_token
79 # Give and consume the next token
87 # Primitive method to return a token, or return null if it is discarded
88 # Is used to implement `peek` and `next`
89 protected fun get_token: nullable Token
94 var start_stream_pos = sp
96 var start_line = _line
97 var string = _file.string
98 var string_len = string.length
100 var accept_state = -1
101 var accept_token = -1
102 var accept_length = -1
107 if sp >= string_len then
110 var c = string[sp].ascii
119 _file.line_starts[line] = sp
123 _file.line_starts[line] = sp
129 _file.line_starts[line] = sp
136 var old_state = dfa_state
137 if dfa_state < -1 then
138 old_state = -2 - dfa_state
144 var high = lexer_goto(old_state, 0) - 1
148 var middle = (low + high) / 2
149 var offset = middle * 3 + 1 # +1 because length is at 0
151 if c < lexer_goto(old_state, offset) then
153 else if c > lexer_goto(old_state, offset+1) then
156 dfa_state = lexer_goto(old_state, offset+2)
161 if dfa_state > -2 then break
169 if dfa_state >= 0 then
170 var tok = lexer_accept(dfa_state)
172 accept_state = dfa_state
174 accept_length = sp - start_stream_pos
179 if accept_state != -1 then
180 var location = new Location(_file, start_line + 1, accept_line + 1, start_pos + 1, accept_pos)
183 _stream_pos = start_stream_pos + accept_length
185 if accept_token == ${position()-1} then
186 $ if {count(transition[@from!=@to])!=0}
187 var state_id = _state
188 $ foreach transition in {transition[@from!=@to]}
189 if state_id == ${/parser/lexer_data/state[@name=$transition/@from]/@id} then
190 _state = state_${translate(@to,"ABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwxyz")}
195 return new @ename.init_tk(location)
203 var location = new Location(_file, start_line + 1, start_line + 1, start_pos + 1, start_pos + 1)
204 if sp > start_stream_pos then
205 var text = string.substring(start_stream_pos, sp-start_stream_pos)
206 var token = new PLexerError.init_lexer_error("Syntax error: unknown token {text}.", location, text)
209 var token = new EOF.init_tk(location)
222 $ template make_lexer_table()
223 $ foreach {lexer_data/goto_table/state}
225 $ if {count(goto)!=0}
226 static const int lexer_goto_row${position()}[] = {
229 @low, @high, @state[-sep ','-]
234 static const int lexer_goto_row_null[] = {0};
235 const int* const lexer_goto_table[] = {
237 $ if {count(goto)!=0}
238 lexer_goto_row${position()}[-sep ','-]
240 lexer_goto_row_null[-sep ','-]
246 $ foreach {lexer_data/accept_table/state}
247 const int lexer_accept_table[] = {
248 [-foreach {i}-]${.}[-sep ','-][-end foreach-]