loader: can load a markdown file as a MDoc
[nit.git] / src / loader.nit
1 # This file is part of NIT ( http://www.nitlanguage.org ).
2 #
3 # Copyright 2012 Jean Privat <jean@pryen.org>
4 #
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
8 #
9 # http://www.apache.org/licenses/LICENSE-2.0
10 #
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
16
17 # Loading of Nit source files
18 module loader
19
20 import modelbuilder_base
21
22 redef class ToolContext
23 # Option --path
24 var opt_path = new OptionArray("Set include path for loaders (may be used more than once)", "-I", "--path")
25
26 # Option --only-metamodel
27 var opt_only_metamodel = new OptionBool("Stop after meta-model processing", "--only-metamodel")
28
29 # Option --only-parse
30 var opt_only_parse = new OptionBool("Only proceed to parse step of loaders", "--only-parse")
31
32 redef init
33 do
34 super
35 option_context.add_option(opt_path, opt_only_parse, opt_only_metamodel)
36 end
37 end
38
39 redef class ModelBuilder
40 redef init
41 do
42 super
43
44 # Setup the paths value
45 paths.append(toolcontext.opt_path.value)
46
47 var path_env = "NIT_PATH".environ
48 if not path_env.is_empty then
49 paths.append(path_env.split_with(':'))
50 end
51
52 var nit_dir = toolcontext.nit_dir
53 var libname = "{nit_dir}/lib"
54 if libname.file_exists then paths.add(libname)
55 end
56
57 # Load a bunch of modules.
58 # `modules` can contains filenames or module names.
59 # Imported modules are automatically loaded and modelized.
60 # The result is the corresponding model elements.
61 # Errors and warnings are printed with the toolcontext.
62 #
63 # Note: class and property model elements are not analysed.
64 fun parse(modules: Sequence[String]): Array[MModule]
65 do
66 var time0 = get_time
67 # Parse and recursively load
68 self.toolcontext.info("*** PARSE ***", 1)
69 var mmodules = new ArraySet[MModule]
70 for a in modules do
71 var nmodule = self.load_module(a)
72 if nmodule == null then continue # Skip error
73 # Load imported module
74 build_module_importation(nmodule)
75
76 mmodules.add(nmodule.mmodule.as(not null))
77 end
78 var time1 = get_time
79 self.toolcontext.info("*** END PARSE: {time1-time0} ***", 2)
80
81 self.toolcontext.check_errors
82
83 if toolcontext.opt_only_parse.value then
84 self.toolcontext.info("*** ONLY PARSE...", 1)
85 exit(0)
86 end
87
88 return mmodules.to_a
89 end
90
91 # Load recursively all modules of the group `mgroup`.
92 # See `parse` for details.
93 fun parse_group(mgroup: MGroup): Array[MModule]
94 do
95 var res = new Array[MModule]
96 visit_group(mgroup)
97 for mg in mgroup.in_nesting.smallers do
98 for mp in mg.module_paths do
99 var nmodule = self.load_module(mp.filepath)
100 if nmodule == null then continue # Skip error
101 # Load imported module
102 build_module_importation(nmodule)
103
104 res.add(nmodule.mmodule.as(not null))
105 end
106 end
107 return res
108 end
109
110 # Load a bunch of modules and groups.
111 # Each name can be a module or a group.
112 # If it is a group then recursively all its modules are parsed.
113 # See `parse` for details.
114 fun parse_full(names: Sequence[String]): Array[MModule]
115 do
116 var time0 = get_time
117 # Parse and recursively load
118 self.toolcontext.info("*** PARSE ***", 1)
119 var mmodules = new ArraySet[MModule]
120 for a in names do
121 var mgroup = self.get_mgroup(a)
122 if mgroup != null then
123 mmodules.add_all parse_group(mgroup)
124 continue
125 end
126 var nmodule = self.load_module(a)
127 if nmodule == null then continue # Skip error
128 # Load imported module
129 build_module_importation(nmodule)
130
131 mmodules.add(nmodule.mmodule.as(not null))
132 end
133 var time1 = get_time
134 self.toolcontext.info("*** END PARSE: {time1-time0} ***", 2)
135
136 self.toolcontext.check_errors
137
138 if toolcontext.opt_only_parse.value then
139 self.toolcontext.info("*** ONLY PARSE...", 1)
140 exit(0)
141 end
142
143 return mmodules.to_a
144 end
145
146 # The list of directories to search for top level modules
147 # The list is initially set with:
148 #
149 # * the toolcontext --path option
150 # * the NIT_PATH environment variable
151 # * `toolcontext.nit_dir`
152 # Path can be added (or removed) by the client
153 var paths = new Array[String]
154
155 # Like (and used by) `get_mmodule_by_name` but just return the ModulePath
156 fun search_mmodule_by_name(anode: nullable ANode, mgroup: nullable MGroup, name: String): nullable ModulePath
157 do
158 # First, look in groups
159 var c = mgroup
160 while c != null do
161 var dirname = c.filepath
162 if dirname == null then break # virtual group
163 if dirname.has_suffix(".nit") then break # singleton project
164
165 # Second, try the directory to find a file
166 var try_file = dirname + "/" + name + ".nit"
167 if try_file.file_exists then
168 var res = self.identify_file(try_file.simplify_path)
169 assert res != null
170 return res
171 end
172
173 # Third, try if the requested module is itself a group
174 try_file = dirname + "/" + name + "/" + name + ".nit"
175 if try_file.file_exists then
176 var res = self.identify_file(try_file.simplify_path)
177 assert res != null
178 return res
179 end
180
181 c = c.parent
182 end
183
184 # Look at some known directories
185 var lookpaths = self.paths
186
187 # Look in the directory of the group project also (even if not explicitly in the path)
188 if mgroup != null then
189 # path of the root group
190 var dirname = mgroup.mproject.root.filepath
191 if dirname != null then
192 dirname = dirname.join_path("..").simplify_path
193 if not lookpaths.has(dirname) and dirname.file_exists then
194 lookpaths = lookpaths.to_a
195 lookpaths.add(dirname)
196 end
197 end
198 end
199
200 var candidate = search_module_in_paths(anode.hot_location, name, lookpaths)
201
202 if candidate == null then
203 if mgroup != null then
204 error(anode, "Error: cannot find module {name} from {mgroup.name}. tried {lookpaths.join(", ")}")
205 else
206 error(anode, "Error: cannot find module {name}. tried {lookpaths.join(", ")}")
207 end
208 return null
209 end
210 return candidate
211 end
212
213 # Get a module by its short name; if required, the module is loaded, parsed and its hierarchies computed.
214 # If `mgroup` is set, then the module search starts from it up to the top level (see `paths`);
215 # if `mgroup` is null then the module is searched in the top level only.
216 # If no module exists or there is a name conflict, then an error on `anode` is displayed and null is returned.
217 fun get_mmodule_by_name(anode: nullable ANode, mgroup: nullable MGroup, name: String): nullable MModule
218 do
219 var path = search_mmodule_by_name(anode, mgroup, name)
220 if path == null then return null # Forward error
221 var res = self.load_module(path.filepath)
222 if res == null then return null # Forward error
223 # Load imported module
224 build_module_importation(res)
225 return res.mmodule.as(not null)
226 end
227
228 # Search a module `name` from path `lookpaths`.
229 # If found, the path of the file is returned
230 private fun search_module_in_paths(location: nullable Location, name: String, lookpaths: Collection[String]): nullable ModulePath
231 do
232 var candidate: nullable String = null
233 for dirname in lookpaths do
234 var try_file = (dirname + "/" + name + ".nit").simplify_path
235 if try_file.file_exists then
236 if candidate == null then
237 candidate = try_file
238 else if candidate != try_file then
239 # try to disambiguate conflicting modules
240 var abs_candidate = module_absolute_path(candidate)
241 var abs_try_file = module_absolute_path(try_file)
242 if abs_candidate != abs_try_file then
243 toolcontext.error(location, "Error: conflicting module file for {name}: {candidate} {try_file}")
244 end
245 end
246 end
247 try_file = (dirname + "/" + name + "/" + name + ".nit").simplify_path
248 if try_file.file_exists then
249 if candidate == null then
250 candidate = try_file
251 else if candidate != try_file then
252 # try to disambiguate conflicting modules
253 var abs_candidate = module_absolute_path(candidate)
254 var abs_try_file = module_absolute_path(try_file)
255 if abs_candidate != abs_try_file then
256 toolcontext.error(location, "Error: conflicting module file for {name}: {candidate} {try_file}")
257 end
258 end
259 end
260 end
261 if candidate == null then return null
262 return identify_file(candidate)
263 end
264
265 # Cache for `identify_file` by realpath
266 private var identified_files_by_path = new HashMap[String, nullable ModulePath]
267
268 # All the currently identified modules.
269 # See `identify_file`.
270 var identified_files = new Array[ModulePath]
271
272 # Identify a source file
273 # Load the associated project and groups if required
274 #
275 # Silently return `null` if `path` is not a valid module path.
276 fun identify_file(path: String): nullable ModulePath
277 do
278 # special case for not a nit file
279 if path.file_extension != "nit" then
280 # search dirless files in known -I paths
281 if path.dirname == "" then
282 var res = search_module_in_paths(null, path, self.paths)
283 if res != null then return res
284 end
285
286 # Found nothing? maybe it is a group...
287 var candidate = null
288 if path.file_exists then
289 var mgroup = get_mgroup(path)
290 if mgroup != null then
291 var owner_path = mgroup.filepath.join_path(mgroup.name + ".nit")
292 if owner_path.file_exists then candidate = owner_path
293 end
294 end
295
296 if candidate == null then
297 return null
298 end
299 path = candidate
300 end
301
302 # Fast track, the path is already known
303 var pn = path.basename(".nit")
304 var rp = module_absolute_path(path)
305 if identified_files_by_path.has_key(rp) then return identified_files_by_path[rp]
306
307 # Search for a group
308 var mgrouppath = path.join_path("..").simplify_path
309 var mgroup = get_mgroup(mgrouppath)
310
311 if mgroup == null then
312 # singleton project
313 var mproject = new MProject(pn, model)
314 mgroup = new MGroup(pn, mproject, null) # same name for the root group
315 mgroup.filepath = path
316 mproject.root = mgroup
317 toolcontext.info("found project `{pn}` at {path}", 2)
318 end
319
320 var res = new ModulePath(pn, path, mgroup)
321 mgroup.module_paths.add(res)
322
323 identified_files_by_path[rp] = res
324 identified_files.add(res)
325 return res
326 end
327
328 # Groups by path
329 private var mgroups = new HashMap[String, nullable MGroup]
330
331 # Return the mgroup associated to a directory path.
332 # If the directory is not a group null is returned.
333 #
334 # Note: `paths` is also used to look for mgroups
335 fun get_mgroup(dirpath: String): nullable MGroup
336 do
337 if not dirpath.file_exists then do
338 for p in paths do
339 var try = p / dirpath
340 if try.file_exists then
341 dirpath = try
342 break label
343 end
344 end
345 return null
346 end label
347
348 var rdp = module_absolute_path(dirpath)
349 if mgroups.has_key(rdp) then
350 return mgroups[rdp]
351 end
352
353 # Hack, a group is determined by:
354 # * the presence of a honomymous nit file
355 # * the fact that the directory is named `src`
356 var pn = rdp.basename(".nit")
357 var mp = dirpath.join_path(pn + ".nit").simplify_path
358
359 var dirpath2 = dirpath
360 if not mp.file_exists then
361 if pn == "src" then
362 # With a src directory, the group name is the name of the parent directory
363 dirpath2 = rdp.dirname
364 pn = dirpath2.basename("")
365 else
366 return null
367 end
368 end
369
370 # check parent directory
371 var parentpath = dirpath.join_path("..").simplify_path
372 var parent = get_mgroup(parentpath)
373
374 var mgroup
375 if parent == null then
376 # no parent, thus new project
377 var mproject = new MProject(pn, model)
378 mgroup = new MGroup(pn, mproject, null) # same name for the root group
379 mproject.root = mgroup
380 toolcontext.info("found project `{mproject}` at {dirpath}", 2)
381 else
382 mgroup = new MGroup(pn, parent.mproject, parent)
383 toolcontext.info("found sub group `{mgroup.full_name}` at {dirpath}", 2)
384 end
385 var readme = dirpath2.join_path("README.md")
386 if not readme.file_exists then readme = dirpath2.join_path("README")
387 if readme.file_exists then
388 var mdoc = load_markdown(readme)
389 mgroup.mdoc = mdoc
390 mdoc.original_mentity = mgroup
391 end
392 mgroup.filepath = dirpath
393 mgroups[rdp] = mgroup
394 return mgroup
395 end
396
397 # Load a markdown file as a documentation object
398 fun load_markdown(filepath: String): MDoc
399 do
400 var mdoc = new MDoc(new Location(new SourceFile.from_string(filepath, ""),0,0,0,0))
401 var s = new FileReader.open(filepath)
402 while not s.eof do
403 mdoc.content.add(s.read_line)
404 end
405 return mdoc
406 end
407
408 # Force the identification of all ModulePath of the group and sub-groups.
409 fun visit_group(mgroup: MGroup) do
410 var p = mgroup.filepath
411 for f in p.files do
412 var fp = p/f
413 var g = get_mgroup(fp)
414 if g != null then visit_group(g)
415 identify_file(fp)
416 end
417 end
418
419 # Transform relative paths (starting with '../') into absolute paths
420 private fun module_absolute_path(path: String): String do
421 return getcwd.join_path(path).simplify_path
422 end
423
424 # Try to load a module AST using a path.
425 # Display an error if there is a problem (IO / lexer / parser) and return null
426 fun load_module_ast(filename: String): nullable AModule
427 do
428 if filename.file_extension != "nit" then
429 self.toolcontext.error(null, "Error: file {filename} is not a valid nit module.")
430 return null
431 end
432 if not filename.file_exists then
433 self.toolcontext.error(null, "Error: file {filename} not found.")
434 return null
435 end
436
437 self.toolcontext.info("load module {filename}", 2)
438
439 # Load the file
440 var file = new FileReader.open(filename)
441 var lexer = new Lexer(new SourceFile(filename, file))
442 var parser = new Parser(lexer)
443 var tree = parser.parse
444 file.close
445
446 # Handle lexer and parser error
447 var nmodule = tree.n_base
448 if nmodule == null then
449 var neof = tree.n_eof
450 assert neof isa AError
451 error(neof, neof.message)
452 return null
453 end
454
455 return nmodule
456 end
457
458 # Try to load a module using a path.
459 # Display an error if there is a problem (IO / lexer / parser) and return null.
460 # Note: usually, you do not need this method, use `get_mmodule_by_name` instead.
461 #
462 # The MModule is created however, the importation is not performed,
463 # therefore you should call `build_module_importation`.
464 fun load_module(filename: String): nullable AModule
465 do
466 # Look for the module
467 var file = identify_file(filename)
468 if file == null then
469 toolcontext.error(null, "Error: cannot find module `{filename}`.")
470 return null
471 end
472
473 # Already known and loaded? then return it
474 var mmodule = file.mmodule
475 if mmodule != null then
476 return mmodule2nmodule[mmodule]
477 end
478
479 # Load it manually
480 var nmodule = load_module_ast(file.filepath)
481 if nmodule == null then return null # forward error
482
483 # build the mmodule and load imported modules
484 mmodule = build_a_mmodule(file.mgroup, file.name, nmodule)
485
486 if mmodule == null then return null # forward error
487
488 # Update the file information
489 file.mmodule = mmodule
490
491 return nmodule
492 end
493
494 # Injection of a new module without source.
495 # Used by the interpreter.
496 fun load_rt_module(parent: nullable MModule, nmodule: AModule, mod_name: String): nullable AModule
497 do
498 # Create the module
499
500 var mgroup = null
501 if parent != null then mgroup = parent.mgroup
502 var mmodule = new MModule(model, mgroup, mod_name, nmodule.location)
503 nmodule.mmodule = mmodule
504 nmodules.add(nmodule)
505 self.mmodule2nmodule[mmodule] = nmodule
506
507 if parent!= null then
508 var imported_modules = new Array[MModule]
509 imported_modules.add(parent)
510 mmodule.set_visibility_for(parent, intrude_visibility)
511 mmodule.set_imported_mmodules(imported_modules)
512 else
513 build_module_importation(nmodule)
514 end
515
516 return nmodule
517 end
518
519 # Visit the AST and create the `MModule` object
520 private fun build_a_mmodule(mgroup: nullable MGroup, mod_name: String, nmodule: AModule): nullable MModule
521 do
522 # Check the module name
523 var decl = nmodule.n_moduledecl
524 if decl != null then
525 var decl_name = decl.n_name.n_id.text
526 if decl_name != mod_name then
527 error(decl.n_name, "Error: module name missmatch; declared {decl_name} file named {mod_name}")
528 end
529 end
530
531 # Check for conflicting module names in the project
532 if mgroup != null then
533 var others = model.get_mmodules_by_name(mod_name)
534 if others != null then for other in others do
535 if other.mgroup!= null and other.mgroup.mproject == mgroup.mproject then
536 var node: ANode
537 if decl == null then node = nmodule else node = decl.n_name
538 error(node, "Error: A module named `{other.full_name}` already exists at {other.location}")
539 break
540 end
541 end
542 end
543
544 # Create the module
545 var mmodule = new MModule(model, mgroup, mod_name, nmodule.location)
546 nmodule.mmodule = mmodule
547 nmodules.add(nmodule)
548 self.mmodule2nmodule[mmodule] = nmodule
549
550 var source = nmodule.location.file
551 if source != null then
552 assert source.mmodule == null
553 source.mmodule = mmodule
554 end
555
556 if decl != null then
557 # Extract documentation
558 var ndoc = decl.n_doc
559 if ndoc != null then
560 var mdoc = ndoc.to_mdoc
561 mmodule.mdoc = mdoc
562 mdoc.original_mentity = mmodule
563 else
564 advice(decl, "missing-doc", "Documentation warning: Undocumented module `{mmodule}`")
565 end
566 # Is the module a test suite?
567 mmodule.is_test_suite = not decl.get_annotations("test_suite").is_empty
568 end
569
570 return mmodule
571 end
572
573 # Analyze the module importation and fill the module_importation_hierarchy
574 #
575 # Unless you used `load_module`, the importation is already done and this method does a no-op.
576 fun build_module_importation(nmodule: AModule)
577 do
578 if nmodule.is_importation_done then return
579 nmodule.is_importation_done = true
580 var mmodule = nmodule.mmodule.as(not null)
581 var stdimport = true
582 var imported_modules = new Array[MModule]
583 for aimport in nmodule.n_imports do
584 stdimport = false
585 if not aimport isa AStdImport then
586 continue
587 end
588 var mgroup = mmodule.mgroup
589 if aimport.n_name.n_quad != null then mgroup = null # Start from top level
590 for grp in aimport.n_name.n_path do
591 var path = search_mmodule_by_name(grp, mgroup, grp.text)
592 if path == null then return # Skip error
593 mgroup = path.mgroup
594 end
595 var mod_name = aimport.n_name.n_id.text
596 var sup = self.get_mmodule_by_name(aimport.n_name, mgroup, mod_name)
597 if sup == null then continue # Skip error
598 aimport.mmodule = sup
599 imported_modules.add(sup)
600 var mvisibility = aimport.n_visibility.mvisibility
601 if mvisibility == protected_visibility then
602 error(aimport.n_visibility, "Error: only properties can be protected.")
603 return
604 end
605 if sup == mmodule then
606 error(aimport.n_name, "Error: Dependency loop in module {mmodule}.")
607 end
608 if sup.in_importation < mmodule then
609 error(aimport.n_name, "Error: Dependency loop between modules {mmodule} and {sup}.")
610 return
611 end
612 mmodule.set_visibility_for(sup, mvisibility)
613 end
614 if stdimport then
615 var mod_name = "standard"
616 var sup = self.get_mmodule_by_name(nmodule, null, mod_name)
617 if sup != null then # Skip error
618 imported_modules.add(sup)
619 mmodule.set_visibility_for(sup, public_visibility)
620 end
621 end
622 self.toolcontext.info("{mmodule} imports {imported_modules.join(", ")}", 3)
623 mmodule.set_imported_mmodules(imported_modules)
624
625 # Force standard to be public if imported
626 for sup in mmodule.in_importation.greaters do
627 if sup.name == "standard" then
628 mmodule.set_visibility_for(sup, public_visibility)
629 end
630 end
631
632 # TODO: Correctly check for useless importation
633 # It is even doable?
634 var directs = mmodule.in_importation.direct_greaters
635 for nim in nmodule.n_imports do
636 if not nim isa AStdImport then continue
637 var im = nim.mmodule
638 if im == null then continue
639 if directs.has(im) then continue
640 # This generates so much noise that it is simpler to just comment it
641 #warning(nim, "Warning: possible useless importation of {im}")
642 end
643 end
644
645 # All the loaded modules
646 var nmodules = new Array[AModule]
647
648 # Register the nmodule associated to each mmodule
649 #
650 # Public clients need to use `mmodule2node` to access stuff.
651 private var mmodule2nmodule = new HashMap[MModule, AModule]
652
653 # Retrieve the associated AST node of a mmodule.
654 # This method is used to associate model entity with syntactic entities.
655 #
656 # If the module is not associated with a node, returns null.
657 fun mmodule2node(mmodule: MModule): nullable AModule
658 do
659 return mmodule2nmodule.get_or_null(mmodule)
660 end
661 end
662
663 # File-system location of a module (file) that is identified but not always loaded.
664 class ModulePath
665 # The name of the module
666 # (it's the basename of the filepath)
667 var name: String
668
669 # The human path of the module
670 var filepath: String
671
672 # The group (and the project) of the possible module
673 var mgroup: MGroup
674
675 # The loaded module (if any)
676 var mmodule: nullable MModule = null
677
678 redef fun to_s do return filepath
679 end
680
681 redef class MGroup
682 # Modules paths associated with the group
683 var module_paths = new Array[ModulePath]
684
685 # Is the group interesting for a final user?
686 #
687 # Groups are mandatory in the model but for simple projects they are not
688 # always interesting.
689 #
690 # A interesting group has, at least, one of the following true:
691 #
692 # * it has 2 modules or more
693 # * it has a subgroup
694 # * it has a documentation
695 fun is_interesting: Bool
696 do
697 return module_paths.length > 1 or mmodules.length > 1 or not in_nesting.direct_smallers.is_empty or mdoc != null
698 end
699
700 end
701
702 redef class SourceFile
703 # Associated mmodule, once created
704 var mmodule: nullable MModule = null
705 end
706
707 redef class AStdImport
708 # The imported module once determined
709 var mmodule: nullable MModule = null
710 end
711
712 redef class AModule
713 # The associated MModule once build by a `ModelBuilder`
714 var mmodule: nullable MModule
715 # Flag that indicate if the importation is already completed
716 var is_importation_done: Bool = false
717 end