#
# The tool scans packages and generates the HTML files of a catalog.
#
-# ## Features
-#
-# * [X] scan packages and their `.ini`
-# * [X] generate lists of packages
-# * [X] generate a page per package with the readme and most metadata
-# * [ ] link/include/be included in the documentation
-# * [ ] propose `related packages`
-# * [X] show directory content (a la nitls)
-# * [X] gather git information from the working directory
-# * [ ] gather git information from the repository
-# * [ ] gather package information from github
-# * [ ] gather people information from github
-# * [ ] reify people
-# * [ ] separate information gathering from rendering
-# * [ ] move up information gathering in (existing or new) service modules
-# * [X] add command line options
-# * [ ] harden HTML (escaping, path injection, etc)
-# * [ ] nitcorn server with RESTful API
-#
-# ## Issues and limitations
-#
-# The tool works likee the other tools and expects to find valid Nit source code in the directories
-#
-# * cruft and temporary files will be collected
-# * missing source file (e.g. not yet generated by nitcc) will make information
-# incomplete (e.g. invalid module thus partial dependency and metrics)
-#
-# How to use the tool as the basis of a Nit code archive on the web usable with a package manager is not clear.
+# See `catalog` for details
module nitcatalog
import loader # Scan&load packages, groups and modules
import doc::doc_down # Display mdoc
-import md5 # To get gravatar images
-import counter # For statistics
-import modelize # To process and count classes and methods
-
-redef class MPackage
- # Return the associated metadata from the `ini`, if any
- fun metadata(key: String): nullable String
- do
- var ini = self.ini
- if ini == null then return null
- return ini[key]
- end
-
- # The list of maintainers
- var maintainers = new Array[String]
-
- # The list of contributors
- var contributors = new Array[String]
-
- # The date of the most recent commit
- var last_date: nullable String = null
-
- # The date of the oldest commit
- var first_date: nullable String = null
-end
+import catalog
# A HTML page in a catalog
#
end
end
-redef class Int
- # Returns `log(self+1)`. Used to compute score of packages
- fun score: Float do return (self+1).to_f.log
-end
-
-# The main class of the calatog generator that has the knowledge
-class Catalog
-
- # The modelbuilder
- # used to access the files and count source lines of code
- var modelbuilder: ModelBuilder
-
- # Packages by tag
- var tag2proj = new MultiHashMap[String, MPackage]
-
- # Packages by category
- var cat2proj = new MultiHashMap[String, MPackage]
-
- # Packages by maintainer
- var maint2proj = new MultiHashMap[String, MPackage]
-
- # Packages by contributors
- var contrib2proj = new MultiHashMap[String, MPackage]
-
- # Dependency between packages
- var deps = new POSet[MPackage]
-
- # Number of modules by package
- var mmodules = new Counter[MPackage]
+redef class NitdocDecorator
+ redef fun add_image(v, link, name, comment)
+ do
+ # Keep absolute links as is
+ if link.has_prefix("http://") or link.has_prefix("https://") then
+ super
+ return
+ end
- # Number of classes by package
- var mclasses = new Counter[MPackage]
+ do
+ # Get the directory of the doc object to deal with the relative link
+ var mdoc = current_mdoc
+ if mdoc == null then break
+ var source = mdoc.location.file
+ if source == null then break
+ var path = source.filename
+ var stat = path.file_stat
+ if stat == null then break
+ if not stat.is_dir then path = path.dirname
+
+ # Get the full path to the local resource
+ var fulllink = path / link.to_s
+ stat = fulllink.file_stat
+ if stat == null then break
+
+ # Get a collision-free catalog name for the resource
+ var hash = fulllink.md5
+ var ext = fulllink.file_extension
+ if ext != null then hash = hash + "." + ext
+
+ # Copy the local resource in the resource directory of the catalog
+ var res = catalog.outdir / "res" / hash
+ fulllink.file_copy_to(res)
+
+ # Hijack the link in the html.
+ link = ".." / "res" / hash
+ super(v, link, name, comment)
+ return
+ end
- # Number of methods by package
- var mmethods = new Counter[MPackage]
+ # Something went bad
+ catalog.modelbuilder.toolcontext.error(current_mdoc.location, "Error: cannot find local image `{link}`")
+ super
+ end
- # Number of line of code by package
- var loc = new Counter[MPackage]
+ # The registered catalog
+ #
+ # It is used to deal with relative links in images.
+ var catalog: Catalog is noautoinit
+end
- # Number of commits by package
- var commits = new Counter[MPackage]
+redef class Catalog
+ redef init
+ do
+ # Register `self` to the global NitdocDecorator
+ # FIXME this is ugly. But no better idea at the moment.
+ modelbuilder.model.nitdoc_md_processor.emitter.decorator.as(NitdocDecorator).catalog = self
+ end
- # Score by package
- #
- # The score is loosely computed using other metrics
- var score = new Counter[MPackage]
+ # The output directory where to generate pages
+ var outdir: String is noautoinit
# Return a empty `CatalogPage`.
fun new_page(rootpath: String): CatalogPage
return new CatalogPage(self, rootpath)
end
- # Scan, register and add a contributor to a package
- fun add_contrib(person: String, mpackage: MPackage, res: Template)
- do
- var projs = contrib2proj[person]
- if not projs.has(mpackage) then projs.add mpackage
- var name = person
- var email = null
- var page = null
-
- # Regular expressions are broken, need to investigate.
- # So split manually.
- #
- #var re = "([^<(]*?)(<([^>]*?)>)?(\\((.*)\\))?".to_re
- #var m = (person+" ").search(re)
- #print "{person}: `{m or else "?"}` `{m[1] or else "?"}` `{m[3] or else "?"}` `{m[5] or else "?"}`"
- do
- var sp1 = person.split_once_on("<")
- if sp1.length < 2 then
- break
- end
- var sp2 = sp1.last.split_once_on(">")
- if sp2.length < 2 then
- break
- end
- name = sp1.first.trim
- email = sp2.first.trim
- var sp3 = sp2.last.split_once_on("(")
- if sp3.length < 2 then
- break
- end
- var sp4 = sp3.last.split_once_on(")")
- if sp4.length < 2 then
- break
- end
- page = sp4.first.trim
- end
-
- var e = name.html_escape
- res.add "<li>"
- if page != null then
- res.add "<a href=\"{page.html_escape}\">"
- end
- if email != null then
- # TODO get more things from github by using the email as a key
- # "https://api.github.com/search/users?q={email}+in:email"
- var md5 = email.md5.to_lower
- res.add "<img src=\"https://secure.gravatar.com/avatar/{md5}?size=20&default=retro\"> "
- end
- res.add "{e}"
- if page != null then res.add "</a>"
- res.add "</li>"
- end
-
# Recursively generate a level in the file tree of the *content* section
private fun gen_content_level(ot: OrderedTree[MConcern], os: Array[Object], res: Template)
do
res.add "</ul>\n"
end
- # Compute information and generate a full HTML page for a package
- fun package_page(mpackage: MPackage): Writable
+ # Generate a full HTML page for a package
+ fun generate_page(mpackage: MPackage): Writable
do
var res = new_page("..")
- var score = score[mpackage].to_f
var name = mpackage.name.html_escape
res.more_head.add """<title>{{{name}}}</title>"""
<h1 class="package-name">{{{name}}}</h1>
"""
var mdoc = mpackage.mdoc_or_fallback
- if mdoc != null then
- score += 100.0
- res.add mdoc.html_documentation
- score += mdoc.content.length.score
- end
+ if mdoc != null then res.add mdoc.html_documentation
res.add "<h2>Content</h2>"
var ot = new OrderedTree[MConcern]
"""
var tryit = mpackage.metadata("upstream.tryit")
if tryit != null then
- score += 1.0
var e = tryit.html_escape
res.add "<li><a href=\"{e}\">Try<span style=\"color:white\">n</span>it!</a></li>\n"
end
var apk = mpackage.metadata("upstream.apk")
if apk != null then
- score += 1.0
var e = apk.html_escape
res.add "<li><a href=\"{e}\">Android apk</a></li>\n"
end
var homepage = mpackage.metadata("upstream.homepage")
if homepage != null then
- score += 5.0
var e = homepage.html_escape
res.add "<li><a href=\"{e}\">{e}</a></li>\n"
end
- var maintainer = mpackage.metadata("package.maintainer")
- if maintainer != null then
- score += 5.0
- add_contrib(maintainer, mpackage, res)
- mpackage.maintainers.add maintainer
- var projs = maint2proj[maintainer]
- if not projs.has(mpackage) then projs.add mpackage
+ for maintainer in mpackage.maintainers do
+ res.add "<li>{maintainer.to_html}</li>"
end
var license = mpackage.metadata("package.license")
if license != null then
- score += 5.0
var e = license.html_escape
res.add "<li><a href=\"http://opensource.org/licenses/{e}\">{e}</a> license</li>\n"
end
res.add "<h3>Source Code</h3>\n<ul class=\"box\">\n"
var browse = mpackage.metadata("upstream.browse")
if browse != null then
- score += 5.0
var e = browse.html_escape
res.add "<li><a href=\"{e}\">{e}</a></li>\n"
end
end
res.add "</ul>\n"
- res.add "<h3>Tags</h3>\n"
- var tags = mpackage.metadata("package.tags")
- var ts = new Array[String]
- if tags != null then
- for t in tags.split(",") do
- t = t.trim
- if t == "" then continue
- ts.add t
- end
+ res.add "<h3>Quality</h3>\n<ul class=\"box\">\n"
+ var errors = errors[mpackage]
+ if errors > 0 then
+ res.add "<li>{errors} errors</li>\n"
end
- if ts.is_empty then ts.add "none"
- if tryit != null then ts.add "tryit"
- if apk != null then ts.add "apk"
+ res.add "<li>{warnings[mpackage]} warnings ({warnings_per_kloc[mpackage]}/kloc)</li>\n"
+ res.add "<li>{documentation_score[mpackage]}% documented</li>\n"
+ res.add "</ul>\n"
+
+ res.add "<h3>Tags</h3>\n"
var ts2 = new Array[String]
- for t in ts do
- tag2proj[t].add mpackage
+ for t in mpackage.tags do
t = t.html_escape
ts2.add "<a href=\"../index.html#tag_{t}\">{t}</a>"
end
res.add_list(ts2, ", ", ", ")
- var cat = ts.first
- cat2proj[cat].add mpackage
- score += ts.length.score
if deps.has(mpackage) then
var reqs = deps[mpackage].greaters.to_a
end
res.add_list(list, ", ", " and ")
end
-
- score += deps[mpackage].greaters.length.score
- score += deps[mpackage].direct_greaters.length.score
- score += deps[mpackage].smallers.length.score
- score += deps[mpackage].direct_smallers.length.score
end
var contributors = mpackage.contributors
- var more_contributors = mpackage.metadata("package.more_contributors")
- if more_contributors != null then
- for c in more_contributors.split(",") do
- contributors.add c.trim
- end
- end
if not contributors.is_empty then
res.add "<h3>Contributors</h3>\n<ul class=\"box\">"
for c in contributors do
- add_contrib(c, mpackage, res)
+ res.add "<li>{c.to_html}</li>"
end
res.add "</ul>"
end
- score += contributors.length.to_f
-
- var mmodules = 0
- var mclasses = 0
- var mmethods = 0
- var loc = 0
- for g in mpackage.mgroups do
- mmodules += g.mmodules.length
- for m in g.mmodules do
- var am = modelbuilder.mmodule2node(m)
- if am != null then
- var file = am.location.file
- if file != null then
- loc += file.line_starts.length - 1
- end
- end
- for cd in m.mclassdefs do
- mclasses += 1
- for pd in cd.mpropdefs do
- if not pd isa MMethodDef then continue
- mmethods += 1
- end
- end
- end
- end
- self.mmodules[mpackage] = mmodules
- self.mclasses[mpackage] = mclasses
- self.mmethods[mpackage] = mmethods
- self.loc[mpackage] = loc
-
- #score += mmodules.score
- score += mclasses.score
- score += mmethods.score
- score += loc.score
res.add """
<h3>Stats</h3>
<ul class="box">
-<li>{{{mmodules}}} modules</li>
-<li>{{{mclasses}}} classes</li>
-<li>{{{mmethods}}} methods</li>
-<li>{{{loc}}} lines of code</li>
+<li>{{{mmodules[mpackage]}}} modules</li>
+<li>{{{mclasses[mpackage]}}} classes</li>
+<li>{{{mmethods[mpackage]}}} methods</li>
+<li>{{{loc[mpackage]}}} lines of code</li>
</ul>
"""
res.add """
</div>
"""
- self.score[mpackage] = score.to_i
-
return res
end
#
# The list of keys is generated first to allow fast access to the correct `<h3>`.
# `id_prefix` is used to give an id to the `<h3>` element.
- fun list_by(map: MultiHashMap[String, MPackage], id_prefix: String): Template
+ fun list_by(map: MultiHashMap[Object, MPackage], id_prefix: String): Template
do
var res = new Template
var keys = map.keys.to_a
alpha_comparator.sort(keys)
- var list = [for x in keys do "<a href=\"#{id_prefix}{x.html_escape}\">{x.html_escape}</a>"]
+ var list = [for x in keys do "<a href=\"#{id_prefix}{x.to_s.html_escape}\">{x.to_s.html_escape}</a>"]
res.add_list(list, ", ", " and ")
for k in keys do
var projs = map[k].to_a
alpha_comparator.sort(projs)
- var e = k.html_escape
+ var e = k.to_s.html_escape
res.add "<h3 id=\"{id_prefix}{e}\">{e} ({projs.length})</h3>\n<ul>\n"
for p in projs do
res.add "<li>"
return res
end
- # Collect more information on a package using the `git` tool.
- fun git_info(mpackage: MPackage)
- do
- var ini = mpackage.ini
- if ini == null then return
-
- # TODO use real git info
- #var repo = ini.get_or_null("upstream.git")
- #var branch = ini.get_or_null("upstream.git.branch")
- #var directory = ini.get_or_null("upstream.git.directory")
-
- var dirpath = mpackage.root.filepath
- if dirpath == null then return
-
- # Collect commits info
- var res = git_run("log", "--no-merges", "--follow", "--pretty=tformat:%ad;%aN <%aE>", "--", dirpath)
- var contributors = new Counter[String]
- var commits = res.split("\n")
- if commits.not_empty and commits.last == "" then commits.pop
- self.commits[mpackage] = commits.length
- for l in commits do
- var s = l.split_once_on(';')
- if s.length != 2 or s.last == "" then continue
-
- # Collect date of last and first commit
- if mpackage.last_date == null then mpackage.last_date = s.first
- mpackage.first_date = s.first
-
- # Count contributors
- contributors.inc(s.last)
- end
- for c in contributors.sort.reverse_iterator do
- mpackage.contributors.add c
- end
-
- end
-
# Produce a HTML table containig information on the packages
#
# `package_page` must have been called before so that information is computed.
res.add "<th data-field=\"met\" data-sortable=\"true\">methods</th>\n"
res.add "<th data-field=\"loc\" data-sortable=\"true\">lines</th>\n"
res.add "<th data-field=\"score\" data-sortable=\"true\">score</th>\n"
+ res.add "<th data-field=\"errors\" data-sortable=\"true\">errors</th>\n"
+ res.add "<th data-field=\"warnings\" data-sortable=\"true\">warnings</th>\n"
+ res.add "<th data-field=\"warnings_per_kloc\" data-sortable=\"true\">w/kloc</th>\n"
+ res.add "<th data-field=\"doc\" data-sortable=\"true\">doc</th>\n"
res.add "</tr></thead>"
for p in mpackages do
res.add "<tr>"
res.add "<td><a href=\"p/{p.name}.html\">{p.name}</a></td>"
var maint = "?"
- if p.maintainers.not_empty then maint = p.maintainers.first
+ if p.maintainers.not_empty then maint = p.maintainers.first.name.html_escape
res.add "<td>{maint}</td>"
res.add "<td>{p.contributors.length}</td>"
if deps.not_empty then
res.add "<td>{mmethods[p]}</td>"
res.add "<td>{loc[p]}</td>"
res.add "<td>{score[p]}</td>"
+ res.add "<td>{errors[p]}</td>"
+ res.add "<td>{warnings[p]}</td>"
+ res.add "<td>{warnings_per_kloc[p]}</td>"
+ res.add "<td>{documentation_score[p]}</td>"
res.add "</tr>\n"
end
res.add "</table>\n"
var piwik_site_id: Int = 1
end
-# Execute a git command and return the result
-fun git_run(command: String...): String
-do
- # print "git {command.join(" ")}"
- var p = new ProcessReader("git", command...)
- var res = p.read_all
- p.close
- p.wait
- return res
-end
-
var model = new Model
var tc = new ToolContext
# Get files or groups
var args = tc.option_context.rest
+var mmodules
if opt_no_parse.value then
- modelbuilder.scan_full(args)
+ mmodules = modelbuilder.scan_full(args)
else
- modelbuilder.parse_full(args)
+ mmodules = modelbuilder.parse_full(args)
+end
+var mpackages = new Set[MPackage]
+for m in mmodules do
+ var p = m.mpackage
+ if p != null then mpackages.add p
end
# Scan packages and compute information
end
end
-if not opt_no_git.value then for p in model.mpackages do
+if not opt_no_git.value then for p in mpackages do
catalog.git_info(p)
end
var out = opt_dir.value or else "catalog.out"
(out/"p").mkdir
+(out/"res").mkdir
+
+catalog.outdir = out
# Generate the css (hard coded)
var css = """
# PAGES
-for p in model.mpackages do
+for p in mpackages do
# print p
var f = "p/{p.name}.html"
- catalog.package_page(p).write_to_file(out/f)
+ catalog.package_page(p)
+ catalog.generate_page(p).write_to_file(out/f)
end
# INDEX
if catalog.deps.not_empty then
index.add "<h2>Most Required</h2>\n"
var reqs = new Counter[MPackage]
- for p in model.mpackages do
+ for p in mpackages do
reqs[p] = catalog.deps[p].smallers.length - 1
end
index.add catalog.list_best(reqs)
<div class="sidebar">
<h3>Stats</h3>
<ul class="box">
-<li>{{{model.mpackages.length}}} packages</li>
+<li>{{{mpackages.length}}} packages</li>
<li>{{{catalog.maint2proj.length}}} maintainers</li>
<li>{{{catalog.contrib2proj.length}}} contributors</li>
<li>{{{catalog.tag2proj.length}}} tags</li>
page.more_head.add "<title>Projets of Nit</title>"
page.add """<div class="content">\n<h1>People of Nit</h1>\n"""
page.add "<h2>Table of Projets</h2>\n"
-page.add catalog.table_packages(model.mpackages)
+page.add catalog.table_packages(mpackages.to_a)
page.add "</div>\n"
page.write_to_file(out/"table.html")
nitlanguage / nit.git / blobdiff