X-Git-Url: http://nitlanguage.org diff --git a/src/loader.nit b/src/loader.nit index a1a5dc1..169d52e 100644 --- a/src/loader.nit +++ b/src/loader.nit @@ -15,20 +15,41 @@ # limitations under the License. # Loading of Nit source files +# +# The loader takes care of looking for module and projects in the file system, and the associated case of errors. +# The loading requires several steps: +# +# Identify: create an empty model entity associated to a name or a file path. +# Identification is used for instance when names are given in the command line. +# See `identify_module` and `identify_group`. +# +# Scan: visit directories and identify their contents. +# Scanning is done to enable the searching of modules in projects. +# See `scan_group` and `scan_full`. +# +# Parse: load the AST and associate it with the model entity. +# See `MModule::parse`. +# +# Import: means recursively load modules imported by a module. +# See `build_module_importation`. +# +# Load: means doing the full sequence: identify, parse and import. +# See `ModelBuilder::parse`, `ModelBuilder::parse_full`, `MModule::load` `ModelBuilder::load_module. module loader import modelbuilder_base import ini +import nitpm_shared redef class ToolContext # Option --path - var opt_path = new OptionArray("Set include path for loaders (may be used more than once)", "-I", "--path") + var opt_path = new OptionArray("Add an additional include path (may be used more than once)", "-I", "--path") # Option --only-metamodel var opt_only_metamodel = new OptionBool("Stop after meta-model processing", "--only-metamodel") # Option --only-parse - var opt_only_parse = new OptionBool("Only proceed to parse step of loaders", "--only-parse") + var opt_only_parse = new OptionBool("Only proceed to parse files", "--only-parse") redef init do @@ -45,16 +66,23 @@ redef class ModelBuilder # Setup the paths value paths.append(toolcontext.opt_path.value) + # Packages managed by nitpm, only use when not testing with tests.sh + if "NIT_TESTING_TESTS_SH".environ != "true" then + paths.add nitpm_lib_dir + end + var path_env = "NIT_PATH".environ if not path_env.is_empty then paths.append(path_env.split_with(':')) end var nit_dir = toolcontext.nit_dir - var libname = nit_dir/"lib" - if libname.file_exists then paths.add(libname) - libname = nit_dir/"contrib" - if libname.file_exists then paths.add(libname) + if nit_dir != null then + var libname = nit_dir/"lib" + if libname.file_exists then paths.add(libname) + libname = nit_dir/"contrib" + if libname.file_exists then paths.add(libname) + end end # Load a bunch of modules. @@ -84,7 +112,7 @@ redef class ModelBuilder if toolcontext.opt_only_parse.value then self.toolcontext.info("*** ONLY PARSE...", 1) - exit(0) + self.toolcontext.quit end return mmodules.to_a @@ -110,8 +138,10 @@ redef class ModelBuilder if stat != null and stat.is_dir then self.toolcontext.info("look in directory {a}", 2) var fs = a.files + alpha_comparator.sort(fs) # Try each entry as a group or a module for f in fs do + if f.first == '.' then continue var af = a/f mgroup = identify_group(af) if mgroup != null then @@ -131,6 +161,14 @@ redef class ModelBuilder var mmodule = identify_module(a) if mmodule == null then + var le = last_loader_error + if le != null then + toolcontext.error(null, le) + else if a.file_exists then + toolcontext.error(null, "Error: `{a}` is not a Nit source file.") + else + toolcontext.error(null, "Error: cannot find module `{a}`.") + end continue end @@ -173,7 +211,7 @@ redef class ModelBuilder if toolcontext.opt_only_parse.value then self.toolcontext.info("*** ONLY PARSE...", 1) - exit(0) + self.toolcontext.quit end return mmodules.to_a @@ -217,7 +255,14 @@ redef class ModelBuilder end end - var candidate = search_module_in_paths(anode.hot_location, name, lookpaths) + if mgroup != null then + var alias = mgroup.mpackage.import_alias(name) + if alias != null then name = alias + end + + var loc = null + if anode != null then loc = anode.hot_location + var candidate = search_module_in_paths(loc, name, lookpaths) if candidate == null then if mgroup != null then @@ -247,6 +292,11 @@ redef class ModelBuilder # If found, the module is returned. private fun search_module_in_paths(location: nullable Location, name: String, lookpaths: Collection[String]): nullable MModule do + var name_no_version + if name.has('=') then + name_no_version = name.split('=').first + else name_no_version = name + var res = new ArraySet[MModule] for dirname in lookpaths do # Try a single module file @@ -256,7 +306,7 @@ redef class ModelBuilder var g = identify_group((dirname/name).simplify_path) if g != null then scan_group(g) - res.add_all g.mmodules_by_name(name) + res.add_all g.mmodules_by_name(name_no_version) end end if res.is_empty then return null @@ -294,6 +344,14 @@ redef class ModelBuilder # A parsed module exists in the model but might be not yet analysed (no importation). var parsed_modules = new Array[MModule] + # Some `loader` services are silent and return `null` on error. + # + # Those services can set `last_loader_error` to precise an specific error message. + # if `last_loader_error == null` then a generic error message can be used. + # + # See `identified_modules` and `identify_group` for details. + var last_loader_error: nullable String = null + # Identify a source file and load the associated package and groups if required. # # This method does what the user expects when giving an argument to a Nit tool. @@ -306,13 +364,16 @@ redef class ModelBuilder # then the main module of the package `digraph` is searched in `paths` and returned. # # Silently return `null` if `path` does not exists or cannot be identified. + # If `null` is returned, `last_loader_error` can be set to a specific error message. # # On success, it returns a module that is possibly not yet parsed (no AST), or not yet analysed (no importation). # If the module was already identified, or loaded, it is returned. fun identify_module(path: String): nullable MModule do + last_loader_error = null + # special case for not a nit file - if not path.has_suffix(".nit") then + if not path.has_suffix(".nit") then do # search dirless files in known -I paths if not path.chars.has('/') then var res = search_module_in_paths(null, path, self.paths) @@ -320,19 +381,66 @@ redef class ModelBuilder end # Found nothing? maybe it is a group... - var candidate = null if path.file_exists then var mgroup = identify_group(path) if mgroup != null then var owner_path = mgroup.filepath.join_path(mgroup.name + ".nit") - if owner_path.file_exists then candidate = owner_path + if owner_path.file_exists then + path = owner_path + break + end end end - if candidate == null then - return null + # Found nothing? maybe it is a qualified name + if path.chars.has(':') then + var ids = path.split("::") + var g = identify_group(ids.first) + if g != null then + scan_group(g) + var ms = g.mmodules_by_name(ids.last) + + # Return exact match + for m in ms do + if m.full_name == path then + return m + end + end + + # Where there is only one or two names `foo::bar` + # then accept module that matches `foo::*::bar` + if ids.length <= 2 then + if ms.length == 1 then return ms.first + if ms.length > 1 then + var l = new Array[String] + for m in ms do + var fp = m.filepath + if fp != null then fp = " ({fp})" else fp = "" + l.add "`{m.full_name}`{fp}" + end + last_loader_error = "Error: conflicting module for `{path}`: {l.join(", ")} " + return null + end + end + + var bests = new BestDistance[String](path.length / 2) + # We found nothing. But propose something in the package? + for sg in g.mpackage.mgroups do + for m in sg.mmodules do + var d = path.levenshtein_distance(m.full_name) + bests.update(d, m.full_name) + end + end + var last_loader_error = "Error: cannot find module `{path}`." + if bests.best_items.not_empty then + last_loader_error += " Did you mean " + bests.best_items.join(", ", " or ") + "?" + end + self.last_loader_error = last_loader_error + return null + end end - path = candidate + + return null end # Does the file exists? @@ -351,26 +459,31 @@ redef class ModelBuilder var mgrouppath = path.join_path("..").simplify_path var mgroup = identify_group(mgrouppath) + if mgroup != null then + var mpackage = mgroup.mpackage + if not mpackage.accept(path) then + mgroup = null + toolcontext.info("module `{path}` excluded from package `{mpackage}`", 2) + end + end if mgroup == null then # singleton package - var mpackage = new MPackage(pn, model) - mgroup = new MGroup(pn, mpackage, null) # same name for the root group - mgroup.filepath = path + var loc = new Location.opaque_file(path) + var mpackage = new MPackage(pn, model, loc) + mgroup = new MGroup(pn, loc, mpackage, null) # same name for the root group mpackage.root = mgroup toolcontext.info("found singleton package `{pn}` at {path}", 2) # Attach homonymous `ini` file to the package var inipath = path.dirname / "{pn}.ini" if inipath.file_exists then - var ini = new ConfigTree(inipath) + var ini = new IniFile.from_file(inipath) mpackage.ini = ini end end - var src = new SourceFile.from_string(path, "") - var loc = new Location(src, 0, 0, 0, 0) + var loc = new Location.opaque_file(path) var res = new MModule(model, mgroup, pn, loc) - res.filepath = path identified_modules_by_path[rp] = res identified_modules_by_path[path] = res @@ -384,12 +497,19 @@ redef class ModelBuilder # Return the mgroup associated to a directory path. # If the directory is not a group null is returned. # + # Silently return `null` if `dirpath` does not exists, is not a directory, + # cannot be identified or cannot be attached to a mpackage. + # If `null` is returned, `last_loader_error` can be set to a specific error message. + # # Note: `paths` is also used to look for mgroups fun identify_group(dirpath: String): nullable MGroup do + # Reset error + last_loader_error = null + var stat = dirpath.file_stat - if stat == null then do + if stat == null or not stat.is_dir then do # search dirless directories in known -I paths if dirpath.chars.has('/') then return null for p in paths do @@ -405,6 +525,7 @@ redef class ModelBuilder # Filter out non-directories if not stat.is_dir then + last_loader_error = "Error: `{dirpath}` is not a directory." return null end @@ -422,7 +543,7 @@ redef class ModelBuilder var parent = null var inipath = dirpath / "package.ini" if inipath.file_exists then - ini = new ConfigTree(inipath) + ini = new IniFile.from_file(inipath) end if ini == null then @@ -431,6 +552,7 @@ redef class ModelBuilder # The root of the directory hierarchy in the file system. if rdp == "/" then mgroups[rdp] = null + last_loader_error = "Error: `{dirpath}` is not a Nit package." return null end @@ -438,6 +560,7 @@ redef class ModelBuilder if (dirpath/"packages.ini").file_exists then # dirpath cannot be a package since it is a package directory mgroups[rdp] = null + last_loader_error = "Error: `{dirpath}` is not a Nit package." return null end @@ -447,25 +570,34 @@ redef class ModelBuilder if not stopper.file_exists then # Recursively get the parent group parent = identify_group(parentpath) + if parent != null then do + var mpackage = parent.mpackage + if not mpackage.accept(dirpath) then + toolcontext.info("directory `{dirpath}` excluded from package `{mpackage}`", 2) + parent = null + end + end if parent == null then # Parent is not a group, thus we are not a group either mgroups[rdp] = null + last_loader_error = "Error: `{dirpath}` is not a Nit package." return null end end end + var loc = new Location.opaque_file(dirpath) var mgroup if parent == null then # no parent, thus new package if ini != null then pn = ini["package.name"] or else pn - var mpackage = new MPackage(pn, model) - mgroup = new MGroup(pn, mpackage, null) # same name for the root group + var mpackage = new MPackage(pn, model, loc) + mgroup = new MGroup(pn, loc, mpackage, null) # same name for the root group mpackage.root = mgroup toolcontext.info("found package `{mpackage}` at {dirpath}", 2) mpackage.ini = ini else - mgroup = new MGroup(pn, parent.mpackage, parent) + mgroup = new MGroup(pn, loc, parent.mpackage, parent) toolcontext.info("found sub group `{mgroup.full_name}` at {dirpath}", 2) end @@ -479,7 +611,6 @@ redef class ModelBuilder mdoc.original_mentity = mgroup end - mgroup.filepath = dirpath mgroups[rdp] = mgroup return mgroup end @@ -505,12 +636,12 @@ redef class ModelBuilder return mdoc end - # Force the identification of all ModulePath of the group and sub-groups in the file system. + # Force the identification of all MModule of the group and sub-groups in the file system. # # When a group is scanned, its sub-groups hierarchy is filled (see `MGroup::in_nesting`) - # and the potential modules (and nested modules) are identified (see `MGroup::module_paths`). + # and the potential modules (and nested modules) are identified (see `MGroup::modules`). # - # Basically, this recursively call `get_mgroup` and `identify_file` on each directory entry. + # Basically, this recursively call `identify_group` and `identify_module` on each directory entry. # # No-op if the group was already scanned (see `MGroup::scanned`). fun scan_group(mgroup: MGroup) do @@ -519,7 +650,10 @@ redef class ModelBuilder var p = mgroup.filepath # a virtual group has nothing to scan if p == null then return - for f in p.files do + var files = p.files + alpha_comparator.sort(files) + for f in files do + if f.first == '.' then continue var fp = p/f var g = identify_group(fp) # Recursively scan for groups of the same package @@ -538,6 +672,10 @@ redef class ModelBuilder # Try to load a module AST using a path. # Display an error if there is a problem (IO / lexer / parser) and return null + # + # The AST is loaded as is total independence of the model and its entities. + # + # AST are not cached or reused thus a new AST is returned on success. fun load_module_ast(filename: String): nullable AModule do if not filename.has_suffix(".nit") then @@ -578,6 +716,11 @@ redef class ModelBuilder var keep = new Array[String] var res = new Array[String] for a in args do + var stat = a.to_path.stat + if stat != null and stat.is_dir then + res.add a + continue + end var l = identify_module(a) if l == null then keep.add a @@ -600,7 +743,10 @@ redef class ModelBuilder # Look for the module var mmodule = identify_module(filename) if mmodule == null then - if filename.file_exists then + var le = last_loader_error + if le != null then + toolcontext.error(null, le) + else if filename.file_exists then toolcontext.error(null, "Error: `{filename}` is not a Nit source file.") else toolcontext.error(null, "Error: cannot find module `{filename}`.") @@ -614,7 +760,7 @@ redef class ModelBuilder # Injection of a new module without source. # Used by the interpreter. - fun load_rt_module(parent: nullable MModule, nmodule: AModule, mod_name: String): nullable AModule + fun load_rt_module(parent: nullable MModule, nmodule: AModule, mod_name: String): nullable MModule do # Create the module @@ -623,6 +769,7 @@ redef class ModelBuilder var mmodule = new MModule(model, mgroup, mod_name, nmodule.location) nmodule.mmodule = mmodule nmodules.add(nmodule) + parsed_modules.add mmodule self.mmodule2nmodule[mmodule] = nmodule if parent!= null then @@ -630,11 +777,10 @@ redef class ModelBuilder imported_modules.add(parent) mmodule.set_visibility_for(parent, intrude_visibility) mmodule.set_imported_mmodules(imported_modules) - else - build_module_importation(nmodule) end + build_module_importation(nmodule) - return nmodule + return mmodule end # Visit the AST and create the `MModule` object @@ -648,7 +794,7 @@ redef class ModelBuilder if decl != null then var decl_name = decl.n_name.n_id.text if decl_name != mmodule.name then - error(decl.n_name, "Error: module name mismatch; declared {decl_name} file named {mmodule.name}.") + warning(decl.n_name, "module-name-mismatch", "Error: module name mismatch; declared {decl_name} file named {mmodule.name}.") end end @@ -681,18 +827,16 @@ redef class ModelBuilder var mdoc = ndoc.to_mdoc mmodule.mdoc = mdoc mdoc.original_mentity = mmodule - else - advice(decl, "missing-doc", "Documentation warning: Undocumented module `{mmodule}`") end - # Is the module a test suite? - mmodule.is_test_suite = not decl.get_annotations("test_suite").is_empty + # Is the module generated? + mmodule.is_generated = not decl.get_annotations("generated").is_empty end end # Resolve the module identification for a given `AModuleName`. # # This method handles qualified names as used in `AModuleName`. - fun seach_module_by_amodule_name(n_name: AModuleName, mgroup: nullable MGroup): nullable MModule + fun search_module_by_amodule_name(n_name: AModuleName, mgroup: nullable MGroup): nullable MModule do var mod_name = n_name.n_id.text @@ -726,6 +870,13 @@ redef class ModelBuilder # If no module yet, then assume that the first element of the path # Is to be searched in the path. var root_name = n_name.n_path.first.text + + # Search for an alias in required external packages + if mgroup != null then + var alias = mgroup.mpackage.import_alias(root_name) + if alias != null then root_name = alias + end + var roots = search_group_in_paths(root_name, paths) if roots.is_empty then error(n_name, "Error: cannot find `{root_name}`. Tried: {paths.join(", ")}.") @@ -755,7 +906,7 @@ redef class ModelBuilder # Basically it check that `bar::foo` matches `bar/foo.nit` and `bar/baz/foo.nit` # but not `baz/foo.nit` nor `foo/bar.nit` # - # Is used by `seach_module_by_amodule_name` to validate qualified names. + # Is used by `search_module_by_amodule_name` to validate qualified names. private fun match_amodulename(n_name: AModuleName, m: MModule): Bool do var g: nullable MGroup = m.mgroup @@ -769,7 +920,10 @@ redef class ModelBuilder # Analyze the module importation and fill the module_importation_hierarchy # - # Unless you used `load_module`, the importation is already done and this method does a no-op. + # If the importation was already done (`nmodule.is_importation_done`), this method does a no-op. + # + # REQUIRE `nmodule.mmodule != null` + # ENSURE `nmodule.is_importation_done` fun build_module_importation(nmodule: AModule) do if nmodule.is_importation_done then return @@ -788,7 +942,7 @@ redef class ModelBuilder end # Load the imported module - var sup = seach_module_by_amodule_name(aimport.n_name, mmodule.mgroup) + var sup = search_module_by_amodule_name(aimport.n_name, mmodule.mgroup) if sup == null then mmodule.is_broken = true nmodule.mmodule = null # invalidate the module @@ -842,7 +996,7 @@ redef class ModelBuilder var atconditionals = aimport.get_annotations("conditional") if atconditionals.is_empty then continue - var suppath = seach_module_by_amodule_name(aimport.n_name, mmodule.mgroup) + var suppath = search_module_by_amodule_name(aimport.n_name, mmodule.mgroup) if suppath == null then continue # skip error for atconditional in atconditionals do @@ -853,7 +1007,7 @@ redef class ModelBuilder end # The rule - var rule = new Array[Object] + var rule = new Array[MModule] # First element is the goal, thus rule.add suppath @@ -911,14 +1065,11 @@ redef class ModelBuilder # It means that the first module is the module to automatically import. # The remaining modules are the conditions of the rule. # - # Each module is either represented by a MModule (if the module is already loaded) - # or by a ModulePath (if the module is not yet loaded). - # # Rules are declared by `build_module_importation` and are applied by `apply_conditional_importations` # (and `build_module_importation` that calls it). # # TODO (when the loader will be rewritten): use a better representation and move up rules in the model. - private var conditional_importations = new Array[SequenceRead[Object]] + var conditional_importations = new Array[SequenceRead[MModule]] # Extends the current importations according to imported rules about conditional importation fun apply_conditional_importations(mmodule: MModule) @@ -932,24 +1083,16 @@ redef class ModelBuilder for ci in conditional_importations do # Check conditions for i in [1..ci.length[ do - var rule_element = ci[i] - # An element of a rule is either a MModule or a ModulePath - # We need the mmodule to resonate on the importation - var m - if rule_element isa MModule then - m = rule_element - else - abort - end + var m = ci[i] # Is imported? - if not mmodule.in_importation.greaters.has(m) then continue label + if mmodule == m or not mmodule.in_importation.greaters.has(m) then continue label end # Still here? It means that all conditions modules are loaded and imported # Identify the module to automatically import - var suppath = ci.first.as(ModulePath) - var sup = load_module_path(suppath) - if sup == null then continue + var sup = ci.first + var ast = sup.load(self) + if ast == null then continue # Do nothing if already imported if mmodule.in_importation.greaters.has(sup) then continue label @@ -986,9 +1129,6 @@ redef class ModelBuilder end redef class MModule - # The path of the module source - var filepath: nullable String = null - # Force the parsing of the module using `modelbuilder`. # # If the module was already parsed, the existing ASI is returned. @@ -1012,6 +1152,7 @@ redef class MModule # build the mmodule nmodule.mmodule = self + self.location = nmodule.location modelbuilder.build_a_mmodule(mgroup, nmodule) modelbuilder.parsed_modules.add self @@ -1037,7 +1178,53 @@ redef class MPackage # The `ini` file is given as is and might contain invalid or missing information. # # Some packages, like stand-alone packages or virtual packages have no `ini` file associated. - var ini: nullable ConfigTree = null + var ini: nullable IniFile = null + + # Array of relative source paths excluded according to the `source.exclude` key of the `ini` + var excludes: nullable Array[String] is lazy do + var ini = self.ini + if ini == null then return null + var exclude = ini["source.exclude"] + if exclude == null then return null + var excludes = exclude.split(":") + return excludes + end + + # Does the source inclusion/inclusion rules of the package `ini` accept such path? + fun accept(filepath: String): Bool + do + var excludes = self.excludes + if excludes != null then + var relpath = root.filepath.relpath(filepath) + if excludes.has(relpath) then return false + end + return true + end + + # Get the name to search for, for a `root_name` declared as `import` in `ini` + fun import_alias(root_name: String): nullable String + do + var map = import_aliases_parsed + if map == null then return null + + var val = map.get_or_null(root_name) + if val == null then return null + + return val.dir_name + end + + private var import_aliases_parsed: nullable Map[String, ExternalPackage] is lazy do + var ini = ini + if ini == null then return null + + var import_line = ini["package.import"] + if import_line == null then return null + + var map = import_line.parse_import + if map.is_empty then return null + + return map + end end redef class MGroup