X-Git-Url: http://nitlanguage.org diff --git a/src/nitcatalog.nit b/src/nitcatalog.nit index e3e77f3..9dd3351 100644 --- a/src/nitcatalog.nit +++ b/src/nitcatalog.nit @@ -18,63 +18,13 @@ # # The tool scans packages and generates the HTML files of a catalog. # -# ## Features -# -# * [X] scan packages and their `.ini` -# * [X] generate lists of packages -# * [X] generate a page per package with the readme and most metadata -# * [ ] link/include/be included in the documentation -# * [ ] propose `related packages` -# * [X] show directory content (a la nitls) -# * [X] gather git information from the working directory -# * [ ] gather git information from the repository -# * [ ] gather package information from github -# * [ ] gather people information from github -# * [ ] reify people -# * [ ] separate information gathering from rendering -# * [ ] move up information gathering in (existing or new) service modules -# * [X] add command line options -# * [ ] harden HTML (escaping, path injection, etc) -# * [ ] nitcorn server with RESTful API -# -# ## Issues and limitations -# -# The tool works likee the other tools and expects to find valid Nit source code in the directories -# -# * cruft and temporary files will be collected -# * missing source file (e.g. not yet generated by nitcc) will make information -# incomplete (e.g. invalid module thus partial dependency and metrics) -# -# How to use the tool as the basis of a Nit code archive on the web usable with a package manager is not clear. +# See `catalog` for details module nitcatalog import loader # Scan&load packages, groups and modules -import doc::doc_down # Display mdoc -import md5 # To get gravatar images -import counter # For statistics -import modelize # To process and count classes and methods - -redef class MPackage - # Return the associated metadata from the `ini`, if any - fun metadata(key: String): nullable String - do - var ini = self.ini - if ini == null then return null - return ini[key] - end +import catalog - # The list of maintainers - var maintainers = new Array[String] - - # The list of contributors - var contributors = new Array[String] - - # The date of the most recent commit - var last_date: nullable String = null - - # The date of the oldest commit - var first_date: nullable String = null -end +import doc::templates::html_model # A HTML page in a catalog # @@ -83,6 +33,9 @@ end class CatalogPage super Template + # The associated catalog, used to groups options and other global data + var catalog: Catalog + # Placeholder to include additional things before the ``. var more_head = new Template @@ -131,6 +84,37 @@ class CatalogPage """ end + # Inject piwik HTML code if required + private fun add_piwik + do + var tracker_url = catalog.piwik_tracker + if tracker_url == null then return + + var site_id = catalog.piwik_site_id + + tracker_url = tracker_url.trim + if tracker_url.chars.last != '/' then tracker_url += "/" + add """ + + + + +""" + + end + redef fun rendering do add """ @@ -138,114 +122,86 @@ class CatalogPage +""" + add_piwik + add """ + """ end end -redef class Int - # Returns `log(self+1)`. Used to compute score of packages - fun score: Float do return (self+1).to_f.log -end - -# The main class of the calatog generator that has the knowledge -class Catalog - - # The modelbuilder - # used to access the files and count source lines of code - var modelbuilder: ModelBuilder - - # Packages by tag - var tag2proj = new MultiHashMap[String, MPackage] - - # Packages by category - var cat2proj = new MultiHashMap[String, MPackage] - - # Packages by maintainer - var maint2proj = new MultiHashMap[String, MPackage] - - # Packages by contributors - var contrib2proj = new MultiHashMap[String, MPackage] - - # Dependency between packages - var deps = new POSet[MPackage] - - # Number of modules by package - var mmodules = new Counter[MPackage] - - # Number of classes by package - var mclasses = new Counter[MPackage] - - # Number of methods by package - var mmethods = new Counter[MPackage] +redef class NitdocDecorator + redef fun add_image(v, link, name, comment) + do + # Keep absolute links as is + if link.has_prefix("http://") or link.has_prefix("https://") then + super + return + end - # Number of line of code by package - var loc = new Counter[MPackage] + do + # Get the directory of the doc object to deal with the relative link + var mdoc = current_mdoc + if mdoc == null then break + var source = mdoc.location.file + if source == null then break + var path = source.filename + var stat = path.file_stat + if stat == null then break + if not stat.is_dir then path = path.dirname + + # Get the full path to the local resource + var fulllink = path / link.to_s + stat = fulllink.file_stat + if stat == null then break + + # Get a collision-free catalog name for the resource + var hash = fulllink.md5 + var ext = fulllink.file_extension + if ext != null then hash = hash + "." + ext + + # Copy the local resource in the resource directory of the catalog + var res = catalog.outdir / "res" / hash + fulllink.file_copy_to(res) + + # Hijack the link in the html. + link = ".." / "res" / hash + super(v, link, name, comment) + return + end - # Number of commits by package - var commits = new Counter[MPackage] + # Something went bad + catalog.modelbuilder.toolcontext.error(current_mdoc.location, "Error: cannot find local image `{link}`") + super + end - # Score by package + # The registered catalog # - # The score is loosely computed using other metrics - var score = new Counter[MPackage] + # It is used to deal with relative links in images. + var catalog: Catalog is noautoinit +end - # Scan, register and add a contributor to a package - fun add_contrib(person: String, mpackage: MPackage, res: Template) +redef class Catalog + redef init do - var projs = contrib2proj[person] - if not projs.has(mpackage) then projs.add mpackage - var name = person - var email = null - var page = null - - # Regular expressions are broken, need to investigate. - # So split manually. - # - #var re = "([^<(]*?)(<([^>]*?)>)?(\\((.*)\\))?".to_re - #var m = (person+" ").search(re) - #print "{person}: `{m or else "?"}` `{m[1] or else "?"}` `{m[3] or else "?"}` `{m[5] or else "?"}`" - do - var sp1 = person.split_once_on("<") - if sp1.length < 2 then - break - end - var sp2 = sp1.last.split_once_on(">") - if sp2.length < 2 then - break - end - name = sp1.first.trim - email = sp2.first.trim - var sp3 = sp2.last.split_once_on("(") - if sp3.length < 2 then - break - end - var sp4 = sp3.last.split_once_on(")") - if sp4.length < 2 then - break - end - page = sp4.first.trim - end + # Register `self` to the global NitdocDecorator + # FIXME this is ugly. But no better idea at the moment. + modelbuilder.model.nitdoc_md_processor.decorator.as(NitdocDecorator).catalog = self + end - var e = name.html_escape - res.add "
  • " - if page != null then - res.add "" - end - if email != null then - # TODO get more things from github by using the email as a key - # "https://api.github.com/search/users?q={email}+in:email" - var md5 = email.md5.to_lower - res.add " " - end - res.add "{e}" - if page != null then res.add "" - res.add "
    • " + # The output directory where to generate pages + var outdir: String is noautoinit + + # Return a empty `CatalogPage`. + fun new_page(rootpath: String): CatalogPage + do + return new CatalogPage(self, rootpath) end # Recursively generate a level in the file tree of the *content* section - private fun gen_content_level(ot: OrderedTree[Object], os: Array[Object], res: Template) + private fun gen_content_level(ot: OrderedTree[MConcern], os: Array[Object], res: Template) do res.add "\n" end - # Compute information and generate a full HTML page for a package - fun package_page(mpackage: MPackage): Writable + # Generate a full HTML page for a package + fun generate_page(mpackage: MPackage): Writable do - var res = new CatalogPage("..") - var score = score[mpackage].to_f + var res = new_page("..") var name = mpackage.name.html_escape res.more_head.add """{{{name}}}""" - res.add """ -
    -

    {{{name}}}

    -""" + res.add """
    """ + var mdoc = mpackage.mdoc_or_fallback - if mdoc != null then - score += 100.0 + if mdoc == null then + res.add """

    {{{name}}}

    """ + else + res.add """ +
    +

    {{{name}}} - 

    +
    +""" res.add mdoc.html_documentation - score += mdoc.content.length.score end res.add "

    Content

    " - var ot = new OrderedTree[Object] + var ot = new OrderedTree[MConcern] for g in mpackage.mgroups do var pa = g.parent if g.is_interesting then ot.add(pa, g) pa = g end - for mp in g.module_paths do + for mp in g.mmodules do ot.add(pa, mp) end end @@ -313,46 +268,51 @@ class Catalog
      """ - var homepage = mpackage.metadata("upstream.homepage") + var tryit = mpackage.metadata.metadata("upstream.tryit") + if tryit != null then + var e = tryit.html_escape + res.add "
    • Trynit!
      • \n" + end + var apk = mpackage.metadata.metadata("upstream.apk") + if apk != null then + var e = apk.html_escape + res.add "
    • Android apk
      • \n" + end + + res.add """
    \n
      \n""" + + var homepage = mpackage.metadata.metadata("upstream.homepage") if homepage != null then - score += 5.0 var e = homepage.html_escape res.add "
    • {e}
      • \n" end - var maintainer = mpackage.metadata("package.maintainer") - if maintainer != null then - score += 5.0 - add_contrib(maintainer, mpackage, res) - mpackage.maintainers.add maintainer - var projs = maint2proj[maintainer] - if not projs.has(mpackage) then projs.add mpackage + for maintainer in mpackage.metadata.maintainers do + res.add "
    • {maintainer.to_html}
      • " end - var license = mpackage.metadata("package.license") + var license = mpackage.metadata.metadata("package.license") if license != null then - score += 5.0 var e = license.html_escape res.add "
    • {e} license
      • \n" end res.add "
    \n" res.add "

    Source Code

    \n
      \n" - var browse = mpackage.metadata("upstream.browse") + var browse = mpackage.metadata.metadata("upstream.browse") if browse != null then - score += 5.0 var e = browse.html_escape res.add "
    • {e}
      • \n" end - var git = mpackage.metadata("upstream.git") + var git = mpackage.metadata.metadata("upstream.git") if git != null then var e = git.html_escape res.add "
    • {e}
      • \n" end - var last_date = mpackage.last_date + var last_date = mpackage.metadata.last_date if last_date != null then var e = last_date.html_escape res.add "
    • most recent commit: {e}
      • \n" end - var first_date = mpackage.first_date + var first_date = mpackage.metadata.first_date if first_date != null then var e = first_date.html_escape res.add "
    • oldest commit: {e}
      • \n" @@ -363,30 +323,25 @@ class Catalog end res.add "
    \n" - res.add "

    Tags

    \n" - var tags = mpackage.metadata("package.tags") - var ts = new Array[String] - if tags != null then - for t in tags.split(",") do - t = t.trim - if t == "" then continue - ts.add t - end + res.add "

    Quality

    \n
      \n" + var errors = errors[mpackage] + if errors > 0 then + res.add "
    • {errors} errors
      • \n" end - if ts.is_empty then ts.add "none" + res.add "
    • {warnings[mpackage]} warnings ({warnings_per_kloc[mpackage]}/kloc)
      • \n" + res.add "
    • {documentation_score[mpackage]}% documented
      • \n" + res.add "
    \n" + + res.add "

    Tags

    \n" var ts2 = new Array[String] - for t in ts do - tag2proj[t].add mpackage + for t in mpackage.metadata.tags do t = t.html_escape ts2.add "{t}" end res.add_list(ts2, ", ", ", ") - var cat = ts.first - cat2proj[cat].add mpackage - score += ts.length.score - if deps.has(mpackage) then - var reqs = deps[mpackage].greaters.to_a + if deps.vertices.has(mpackage) then + var reqs = deps.get_all_successors(mpackage) reqs.remove(mpackage) alpha_comparator.sort(reqs) res.add "

    Requirements

    \n" @@ -395,7 +350,7 @@ class Catalog else var list = new Array[String] for r in reqs do - var direct = deps.has_direct_edge(mpackage, r) + var direct = deps.has_arc(mpackage, r) var s = "" if direct then s += "" s += r.to_s @@ -406,7 +361,7 @@ class Catalog res.add_list(list, ", ", " and ") end - reqs = deps[mpackage].smallers.to_a + reqs = deps.get_all_predecessors(mpackage) reqs.remove(mpackage) alpha_comparator.sort(reqs) res.add "

    Clients

    \n" @@ -415,7 +370,7 @@ class Catalog else var list = new Array[String] for r in reqs do - var direct = deps.has_direct_edge(r, mpackage) + var direct = deps.has_arc(r, mpackage) var s = "    " if direct then s += "" s += r.to_s @@ -425,71 +380,30 @@ class Catalog end res.add_list(list, ", ", " and ") end - - score += deps[mpackage].greaters.length.score - score += deps[mpackage].direct_greaters.length.score - score += deps[mpackage].smallers.length.score - score += deps[mpackage].direct_smallers.length.score end - var contributors = mpackage.contributors + var contributors = mpackage.metadata.contributors if not contributors.is_empty then res.add "

    Contributors

    \n
      " for c in contributors do - add_contrib(c, mpackage, res) + res.add "
    • {c.to_html}
      • " end res.add "
    " end - score += contributors.length.to_f - - var mmodules = 0 - var mclasses = 0 - var mmethods = 0 - var loc = 0 - for g in mpackage.mgroups do - mmodules += g.module_paths.length - for m in g.mmodules do - var am = modelbuilder.mmodule2node(m) - if am != null then - var file = am.location.file - if file != null then - loc += file.line_starts.length - 1 - end - end - for cd in m.mclassdefs do - mclasses += 1 - for pd in cd.mpropdefs do - if not pd isa MMethodDef then continue - mmethods += 1 - end - end - end - end - self.mmodules[mpackage] = mmodules - self.mclasses[mpackage] = mclasses - self.mmethods[mpackage] = mmethods - self.loc[mpackage] = loc - - #score += mmodules.score - score += mclasses.score - score += mmethods.score - score += loc.score res.add """

    Stats

      -
    • {{{mmodules}}} modules
      • -
    • {{{mclasses}}} classes
      • -
    • {{{mmethods}}} methods
      • -
    • {{{loc}}} lines of code
      • +
    • {{{mmodules[mpackage]}}} modules
      • +
    • {{{mclasses[mpackage]}}} classes
      • +
    • {{{mmethods[mpackage]}}} methods
      • +
    • {{{loc[mpackage]}}} lines of code
    """ res.add """
    """ - self.score[mpackage] = score.to_i - return res end @@ -513,18 +427,18 @@ class Catalog # # The list of keys is generated first to allow fast access to the correct `

    `. # `id_prefix` is used to give an id to the `

    ` element. - fun list_by(map: MultiHashMap[String, MPackage], id_prefix: String): Template + fun list_by(map: MultiHashMap[Object, MPackage], id_prefix: String): Template do var res = new Template var keys = map.keys.to_a alpha_comparator.sort(keys) - var list = [for x in keys do "{x.html_escape}"] + var list = [for x in keys do "{x.to_s.html_escape}"] res.add_list(list, ", ", " and ") for k in keys do var projs = map[k].to_a alpha_comparator.sort(projs) - var e = k.html_escape + var e = k.to_s.html_escape res.add "

    {e} ({projs.length})

    \n
      \n" for p in projs do res.add "
    • " @@ -554,43 +468,6 @@ class Catalog return res end - # Collect more information on a package using the `git` tool. - fun git_info(mpackage: MPackage) - do - var ini = mpackage.ini - if ini == null then return - - # TODO use real git info - #var repo = ini.get_or_null("upstream.git") - #var branch = ini.get_or_null("upstream.git.branch") - #var directory = ini.get_or_null("upstream.git.directory") - - var dirpath = mpackage.root.filepath - if dirpath == null then return - - # Collect commits info - var res = git_run("log", "--no-merges", "--follow", "--pretty=tformat:%ad;%aN <%aE>", "--", dirpath) - var contributors = new Counter[String] - var commits = res.split("\n") - if commits.not_empty and commits.last == "" then commits.pop - self.commits[mpackage] = commits.length - for l in commits do - var s = l.split_once_on(';') - if s.length != 2 or s.last == "" then continue - - # Collect date of last and first commit - if mpackage.last_date == null then mpackage.last_date = s.first - mpackage.first_date = s.first - - # Count contributors - contributors.inc(s.last) - end - for c in contributors.sort.reverse_iterator do - mpackage.contributors.add c - end - - end - # Produce a HTML table containig information on the packages # # `package_page` must have been called before so that information is computed. @@ -603,7 +480,7 @@ class Catalog res.add "name\n" res.add "maint\n" res.add "contrib\n" - if deps.not_empty then + if deps.vertices.not_empty then res.add "reqs\n" res.add "direct
      reqs\n" res.add "clients\n" @@ -614,41 +491,63 @@ class Catalog res.add "methods\n" res.add "lines\n" res.add "score\n" + res.add "errors\n" + res.add "warnings\n" + res.add "w/kloc\n" + res.add "doc\n" res.add "" for p in mpackages do res.add "" res.add "{p.name}" var maint = "?" - if p.maintainers.not_empty then maint = p.maintainers.first + if p.metadata.maintainers.not_empty then maint = p.metadata.maintainers.first.name.html_escape res.add "{maint}" - res.add "{p.contributors.length}" - if deps.not_empty then - res.add "{deps[p].greaters.length-1}" - res.add "{deps[p].direct_greaters.length}" - res.add "{deps[p].smallers.length-1}" - res.add "{deps[p].direct_smallers.length}" + res.add "{p.metadata.contributors.length}" + if deps.vertices.not_empty then + res.add "{deps.get_all_successors(p).length-1}" + res.add "{deps.successors(p).length}" + res.add "{deps.get_all_predecessors(p).length-1}" + res.add "{deps.predecessors(p).length}" end res.add "{mmodules[p]}" res.add "{mclasses[p]}" res.add "{mmethods[p]}" res.add "{loc[p]}" res.add "{score[p]}" + res.add "{errors[p]}" + res.add "{warnings[p]}" + res.add "{warnings_per_kloc[p]}" + res.add "{documentation_score[p]}" res.add "\n" end res.add "\n" return res end + + # Piwik tracker URL, if any + var piwik_tracker: nullable String = null + + # Piwik site ID + # Used when `piwik_tracker` is set + var piwik_site_id: Int = 1 end -# Execute a git command and return the result -fun git_run(command: String...): String -do - # print "git {command.join(" ")}" - var p = new ProcessReader("git", command...) - var res = p.read_all - p.close - p.wait - return res +redef class Person + redef fun to_html do + var res = "" + var e = name.html_escape + var page = self.page + if page != null then + res += "" + end + var gravatar = self.gravatar + if gravatar != null then + res += " " + end + res += e + if page != null then res += "" + return res + end end var model = new Model @@ -659,7 +558,13 @@ var opt_no_git = new OptionBool("Do not gather git information from the working var opt_no_parse = new OptionBool("Do not parse nit files (no importation information)", "--no-parse") var opt_no_model = new OptionBool("Do not analyse nit files (no class/method information)", "--no-model") -tc.option_context.add_option(opt_dir, opt_no_git, opt_no_parse, opt_no_model) +# Piwik tracker URL. +# If you want to monitor your visitors. +var opt_piwik_tracker = new OptionString("Piwik tracker URL (ex: `nitlanguage.org/piwik/`)", "--piwik-tracker") +# Piwik tracker site id. +var opt_piwik_site_id = new OptionString("Piwik site ID", "--piwik-site-id") + +tc.option_context.add_option(opt_dir, opt_no_git, opt_no_parse, opt_no_model, opt_piwik_tracker, opt_piwik_site_id) tc.process_options(sys.args) tc.keep_going = true @@ -667,33 +572,37 @@ tc.keep_going = true var modelbuilder = new ModelBuilder(model, tc) var catalog = new Catalog(modelbuilder) +catalog.piwik_tracker = opt_piwik_tracker.value +var piwik_site_id = opt_piwik_site_id.value +if piwik_site_id != null then + if catalog.piwik_tracker == null then + print_error "Warning: ignored `{opt_piwik_site_id}` because `{opt_piwik_tracker}` is not set." + else if piwik_site_id.is_int then + print_error "Warning: ignored `{opt_piwik_site_id}`, an integer is required." + else + catalog.piwik_site_id = piwik_site_id.to_i + end +end + + # Get files or groups -for a in tc.option_context.rest do - modelbuilder.get_mgroup(a) - modelbuilder.identify_file(a) +var args = tc.option_context.rest +var mmodules +if opt_no_parse.value then + mmodules = modelbuilder.scan_full(args) +else + mmodules = modelbuilder.parse_full(args) end +var mpackages = modelbuilder.model.mpackage_importation_graph.vertices # Scan packages and compute information -for p in model.mpackages do +for p in mpackages do var g = p.root assert g != null modelbuilder.scan_group(g) - - # Load the module to process importation information - if opt_no_parse.value then continue - modelbuilder.parse_group(g) - - catalog.deps.add_node(p) - for gg in p.mgroups do for m in gg.mmodules do - for im in m.in_importation.direct_greaters do - var ip = im.mpackage - if ip == null or ip == p then continue - catalog.deps.add_edge(p, ip) - end - end end -if not opt_no_git.value then for p in model.mpackages do +if not opt_no_git.value then for p in mpackages do catalog.git_info(p) end @@ -704,6 +613,9 @@ end var out = opt_dir.value or else "catalog.out" (out/"p").mkdir +(out/"res").mkdir + +catalog.outdir = out # Generate the css (hard coded) var css = """ @@ -807,15 +719,19 @@ css.write_to_file(out/"style.css") # PAGES -for p in model.mpackages do +for p in mpackages do # print p var f = "p/{p.name}.html" - catalog.package_page(p).write_to_file(out/f) + catalog.package_page(p) + catalog.generate_page(p).write_to_file(out/f) + # copy ini + var ini = p.ini + if ini != null then ini.write_to_file(out/"p/{p.name}.ini") end # INDEX -var index = new CatalogPage("") +var index = catalog.new_page("") index.more_head.add "Packages in Nit" index.add """ @@ -826,11 +742,11 @@ index.add """ index.add "

      Highlighted Packages

      \n" index.add catalog.list_best(catalog.score) -if catalog.deps.not_empty then +if catalog.deps.vertices.not_empty then index.add "

      Most Required

      \n" var reqs = new Counter[MPackage] - for p in model.mpackages do - reqs[p] = catalog.deps[p].smallers.length - 1 + for p in mpackages do + reqs[p] = catalog.deps.get_all_successors(p).length - 1 end index.add catalog.list_best(reqs) end @@ -846,7 +762,7 @@ index.add """

      Stats

        -
      • {{{model.mpackages.length}}} packages
        • +
      • {{{mpackages.length}}} packages
      • {{{catalog.maint2proj.length}}} maintainers
      • {{{catalog.contrib2proj.length}}} contributors
      • {{{catalog.tag2proj.length}}} tags
        • @@ -862,7 +778,7 @@ index.write_to_file(out/"index.html") # PEOPLE -var page = new CatalogPage("") +var page = catalog.new_page("") page.more_head.add "People of Nit" page.add """
        \n

        People of Nit

        \n""" page.add "

        By Maintainer

        \n" @@ -874,10 +790,10 @@ page.write_to_file(out/"people.html") # TABLE -page = new CatalogPage("") +page = catalog.new_page("") page.more_head.add "Projets of Nit" page.add """
        \n

        People of Nit

        \n""" page.add "

        Table of Projets

        \n" -page.add catalog.table_packages(model.mpackages) +page.add catalog.table_packages(mpackages.to_a) page.add "
        \n" page.write_to_file(out/"table.html")