1 # This file is part of NIT ( http://www.nitlanguage.org ).
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 # Additional features on Nit AST
17 # Most of these feature require that the precomputation `parentize_tokens`
18 # is run on the root node of a AST.
20 # These aditionnal features are used either to have a better association between tokens and productions
21 # But also to query how productions and tokens are placed in the lines of code.
28 # Visit the AST and computes advanced AST attributes on Tokens and Prod
29 # This also force a parent on the detashed tokens
32 var v
= new PTokenVisitor
38 # The first token of the production in the AST
39 # Computed by `parentize_tokens`
40 var first_token
: nullable Token = null
42 # The last token of the production in the AST
43 # Computed by `parentize_tokens`
44 var last_token
: nullable Token = null
46 # Is the production contained in full block of line?
47 # Computed by `parentize_tokens`
50 if first_token
== null or last_token
== null then return false
51 #sys.stderr.write "{self}@{location}: start={first_token.is_starting_line} {first_token.inspect}@{first_token.location} ; end={last_token.is_ending_line} {last_token.inspect}@{last_token.location}\n"
52 return first_token
.is_starting_line
and last_token
.is_ending_line
55 # Is the production a part of a single line (without being a block)
56 # Computed by `parentize_tokens`
59 if first_token
== null or last_token
== null then return false
60 return not is_block
and location
.line_start
== location
.line_end
63 # A XML representation of the AST
64 # Productions and token become elements
66 # detached tokens and whitespaces are preserved in the XML
70 # var text = "y += foo"
71 # var ast = (new ToolContext).parse_something(text)
72 # assert ast isa AExpr
73 # ast.parentize_tokens
74 # assert ast.to_xml.write_to_string == """<ABlockExpr><ACallReassignExpr><TId>y</TId> <APlusAssignOp><TPluseq>+=</TPluseq></APlusAssignOp> <ACallExpr><TId>foo</TId></ACallExpr></ACallReassignExpr></ABlockExpr>"""
78 var res
= new HTMLTag("AST")
79 var stack
= new Array[HTMLTag]
82 if c
!= first_token
then res
.append
(c
.blank_before
)
83 var sp
= c
.starting_prods
84 if sp
!= null then for p
in sp
do
85 var tag
= new HTMLTag(p
.class_name
)
90 var tag
= new HTMLTag(c
.class_name
)
93 var ep
= c
.ending_prods
94 if ep
!= null then for p
in ep
do
98 if c
== last_token
then
104 assert stack
.is_empty
105 return res
.children
.first
110 # Is self the first AST token on its line in the source
111 # Computed by `parentize_tokens`
113 # Note, some tokens detached from the AST
114 # may precede `self` even if `is_starting_line` is true
115 # One can use `first_real_token_in_line` to get the real starting token
116 var is_starting_line
= false
118 # Is self the last AST token on its line in the source
119 # Computed by `parentize_tokens`
121 # Note, some tokens detached from the AST (like comments)
122 # may follow `self` even if `is_ending_line` is true.
123 # One can use `last_real_token_in_line` to get the real ending token
124 var is_ending_line
= false
126 # The first real token that starts the line of `self`
128 # This could return a token that is detached from the AST.
129 # See `first_token_in_line` if a AST token is required.
130 fun first_real_token_in_line
: Token
132 var line
= location
.line_start
136 if p
== null or p
.location
.line_start
!= line
then
143 # The first AST token that starts the line of `self`.
144 # May be null is the line contains only detached tokens (only comment)
146 # Computed by `parentize_tokens`
148 # ENSURE `result != null implies result.is_starting_line`
149 fun first_token_in_line
: nullable Token
151 return first_real_token_in_line
.first_ast_token
154 # The first AST token.
155 # This only work on the `first_real_token_in_line`
156 private var first_ast_token
: nullable Token
158 # The last read token that ends the line of `self`
160 # This usually return a detached token lake a TEol or a comment.
161 # See `last_token_in_line` if a AST token is required.
162 fun last_real_token_in_line
: Token
164 var line
= location
.line_start
168 if p
== null or p
.location
.line_start
!= line
then
175 # The last AST token that starts the line of `self`
176 # May be null is the line contains only detached tokens (only comment)
178 # Computed by `parentize_tokens`
180 # ENSURE `result.is_ending_line`
181 fun last_token_in_line
: nullable Token
183 return last_real_token_in_line
.last_ast_token
186 # The last AST token.
187 # This only work on the `last_real_token_in_line`
188 private var last_ast_token
: nullable Token
190 # The productions that starts with `self`, if any
191 # Productions goes from the most general to the most specific
193 # Computed by `parentize_tokens`
194 var starting_prods
: nullable Array[Prod]
196 # The productions that ends with `self`, if any
197 # Productions goes from the most specific to the most general
199 # Computed by `parentize_tokens`
200 var ending_prods
: nullable Array[Prod]
204 private class PTokenVisitor
207 var last_token
: nullable Token = null
209 # productions that need a fisrt token
210 var stack
= new Array[Prod]
215 # process remaining detashed tokens
218 c
.is_ending_line
= true
219 c
.last_real_token_in_line
.last_ast_token
= c
223 while c
!= null and c
.parent
== null do
231 if not n
isa Token then
235 if n
.first_token
== null then
236 # epsilon production, just discard
237 assert stack
.pop
== n
241 # last token ends the production
243 if t
.ending_prods
== null then t
.ending_prods
= new Array[Prod]
250 # We have a token, give it to prods that need one
251 if not stack
.is_empty
then
252 n
.starting_prods
= new Array[Prod]
255 n
.starting_prods
.add p
260 var last_token
= last_token
262 # n starts a new line
263 if last_token
== null or last_token
.location
.line_start
!= n
.location
.line_start
then
264 n
.is_starting_line
= true
265 n
.first_real_token_in_line
.first_ast_token
= n
268 # last_token ended a line
269 if last_token
!= null and last_token
.location
.line_start
!= n
.location
.line_start
then
270 last_token
.is_ending_line
= true
271 last_token
.last_real_token_in_line
.last_ast_token
= last_token
274 # Get the common parent
276 if last_token
== null then
279 p
= last_token
.common_parent
(n
)
282 # And apply it to detached tokens between `last_token` and `n`
284 while c
!= null and c
.parent
== null do