# This file is part of NIT ( http://www.nitlanguage.org ).
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Basic catalog generator for Nit packages
#
# See:
#
# The tool scans packages and generates the HTML files of a catalog.
#
# ## Features
#
# * [X] scan packages and their `.ini`
# * [X] generate lists of packages
# * [X] generate a page per package with the readme and most metadata
# * [ ] link/include/be included in the documentation
# * [ ] propose `related packages`
# * [ ] show directory content (a la nitls)
# * [X] gather git information from the working directory
# * [ ] gather git information from the repository
# * [ ] gather package information from github
# * [ ] gather people information from github
# * [ ] reify people
# * [ ] separate information gathering from rendering
# * [ ] move up information gathering in (existing or new) service modules
# * [ ] add command line options
# * [ ] harden HTML (escaping, path injection, etc)
# * [ ] nitcorn server with RESTful API
#
# ## Issues and limitations
#
# The tool works likee the other tools and expects to find valid Nit source code in the directories
#
# * cruft and temporary files will be collected
# * missing source file (e.g. not yet generated by nitcc) will make information
# incomplete (e.g. invalid module thus partial dependency and metrics)
#
# How to use the tool as the basis of a Nit code archive on the web usable with a package manager is not clear.
module nitcatalog
import loader # Scan&load packages, groups and modules
import doc::doc_down # Display mdoc
import md5 # To get gravatar images
import counter # For statistics
import modelize # To process and count classes and methods
redef class MPackage
# Return the associated metadata from the `ini`, if any
fun metadata(key: String): nullable String
do
var ini = self.ini
if ini == null then return null
return ini[key]
end
# The list of maintainers
var maintainers = new Array[String]
# The list of contributors
var contributors = new Array[String]
# The date of the most recent commit
var last_date: nullable String = null
# The date of the oldest commit
var first_date: nullable String = null
end
# A HTML page in a catalog
#
# This is just a template with the header pre-filled and the footer injected at rendering.
# Therefore, once instantiated, the content can just be added to it.
class CatalogPage
super Template
# Placeholder to include additional things before the ``.
var more_head = new Template
redef init
do
add """
"""
add more_head
add """
"""
end
redef fun rendering
do
add """
"""
end
end
redef class Int
# Returns `log(self+1)`. Used to compute score of packages
fun score: Float do return (self+1).to_f.log
end
# The main class of the calatog generator that has the knowledge
class Catalog
# The modelbuilder
# used to access the files and count source lines of code
var modelbuilder: ModelBuilder
# Packages by tag
var tag2proj = new MultiHashMap[String, MPackage]
# Packages by category
var cat2proj = new MultiHashMap[String, MPackage]
# Packages by maintainer
var maint2proj = new MultiHashMap[String, MPackage]
# Packages by contributors
var contrib2proj = new MultiHashMap[String, MPackage]
# Dependency between packages
var deps = new POSet[MPackage]
# Number of modules by package
var mmodules = new Counter[MPackage]
# Number of classes by package
var mclasses = new Counter[MPackage]
# Number of methods by package
var mmethods = new Counter[MPackage]
# Number of line of code by package
var loc = new Counter[MPackage]
# Number of commits by package
var commits = new Counter[MPackage]
# Score by package
#
# The score is loosely computed using other metrics
var score = new Counter[MPackage]
# Scan, register and add a contributor to a package
fun add_contrib(person: String, mpackage: MPackage, res: Template)
do
var projs = contrib2proj[person]
if not projs.has(mpackage) then projs.add mpackage
var name = person
var email = null
var page = null
# Regular expressions are broken, need to investigate.
# So split manually.
#
#var re = "([^<(]*?)(<([^>]*?)>)?(\\((.*)\\))?".to_re
#var m = (person+" ").search(re)
#print "{person}: `{m or else "?"}` `{m[1] or else "?"}` `{m[3] or else "?"}` `{m[5] or else "?"}`"
do
var sp1 = person.split_once_on("<")
if sp1.length < 2 then
break
end
var sp2 = sp1.last.split_once_on(">")
if sp2.length < 2 then
break
end
name = sp1.first.trim
email = sp2.first.trim
var sp3 = sp2.last.split_once_on("(")
if sp3.length < 2 then
break
end
var sp4 = sp3.last.split_once_on(")")
if sp4.length < 2 then
break
end
page = sp4.first.trim
end
var e = name.html_escape
res.add "
"
end
# Compute information and generate a full HTML page for a package
fun package_page(mpackage: MPackage): Writable
do
var res = new CatalogPage
var score = score[mpackage].to_f
var name = mpackage.name.html_escape
res.more_head.add """{{{name}}}"""
res.add """
{{{name}}}
"""
var mdoc = mpackage.mdoc_or_fallback
if mdoc != null then
score += 100.0
res.add mdoc.html_documentation
score += mdoc.content.length.score
end
res.add """
"""
var homepage = mpackage.metadata("upstream.homepage")
if homepage != null then
score += 5.0
var e = homepage.html_escape
res.add "
\n"
end
var maintainer = mpackage.metadata("package.maintainer")
if maintainer != null then
score += 5.0
add_contrib(maintainer, mpackage, res)
mpackage.maintainers.add maintainer
var projs = maint2proj[maintainer]
if not projs.has(mpackage) then projs.add mpackage
end
var license = mpackage.metadata("package.license")
if license != null then
score += 5.0
var e = license.html_escape
res.add "
\n"
end
var git = mpackage.metadata("upstream.git")
if git != null then
var e = git.html_escape
res.add "
{e}
\n"
end
var last_date = mpackage.last_date
if last_date != null then
var e = last_date.html_escape
res.add "
most recent commit: {e}
\n"
end
var first_date = mpackage.first_date
if first_date != null then
var e = first_date.html_escape
res.add "
oldest commit: {e}
\n"
end
var commits = commits[mpackage]
if commits != 0 then
res.add "
{commits} commits
\n"
end
res.add "
\n"
res.add "
Tags
\n"
var tags = mpackage.metadata("package.tags")
var ts2 = new Array[String]
var cat = null
if tags != null then
var ts = tags.split(",")
for t in ts do
t = t.trim
if t == "" then continue
if cat == null then cat = t
tag2proj[t].add mpackage
t = t.html_escape
ts2.add "{t}"
end
res.add_list(ts2, ", ", ", ")
end
if ts2.is_empty then
var t = "none"
cat = t
tag2proj[t].add mpackage
res.add "{t}"
end
if cat != null then cat2proj[cat].add mpackage
score += ts2.length.score
var reqs = deps[mpackage].greaters.to_a
reqs.remove(mpackage)
alpha_comparator.sort(reqs)
res.add "
Requirements
\n"
if reqs.is_empty then
res.add "none"
else
var list = new Array[String]
for r in reqs do
var direct = deps.has_direct_edge(mpackage, r)
var s = ""
if direct then s += ""
s += r.to_s
if direct then s += ""
s += ""
list.add s
end
res.add_list(list, ", ", " and ")
end
reqs = deps[mpackage].smallers.to_a
reqs.remove(mpackage)
alpha_comparator.sort(reqs)
res.add "
Clients
\n"
if reqs.is_empty then
res.add "none"
else
var list = new Array[String]
for r in reqs do
var direct = deps.has_direct_edge(r, mpackage)
var s = ""
if direct then s += ""
s += r.to_s
if direct then s += ""
s += ""
list.add s
end
res.add_list(list, ", ", " and ")
end
score += deps[mpackage].greaters.length.score
score += deps[mpackage].direct_greaters.length.score
score += deps[mpackage].smallers.length.score
score += deps[mpackage].direct_smallers.length.score
var contributors = mpackage.contributors
if not contributors.is_empty then
res.add "
Contributors
\n
"
for c in contributors do
add_contrib(c, mpackage, res)
end
res.add "
"
end
score += contributors.length.to_f
var mmodules = 0
var mclasses = 0
var mmethods = 0
var loc = 0
for g in mpackage.mgroups do
mmodules += g.module_paths.length
for m in g.mmodules do
var am = modelbuilder.mmodule2node(m)
if am != null then
var file = am.location.file
if file != null then
loc += file.line_starts.length - 1
end
end
for cd in m.mclassdefs do
mclasses += 1
for pd in cd.mpropdefs do
if not pd isa MMethodDef then continue
mmethods += 1
end
end
end
end
self.mmodules[mpackage] = mmodules
self.mclasses[mpackage] = mclasses
self.mmethods[mpackage] = mmethods
self.loc[mpackage] = loc
#score += mmodules.score
score += mclasses.score
score += mmethods.score
score += loc.score
res.add """
Stats
{{{mmodules}}} modules
{{{mclasses}}} classes
{{{mmethods}}} methods
{{{loc}}} lines of code
"""
res.add """
"""
self.score[mpackage] = score.to_i
return res
end
# Return a short HTML sequence for a package
#
# Intended to use in lists.
fun li_package(p: MPackage): String
do
var res = ""
var f = "{p.name}.html"
res += "{p}"
var d = p.mdoc_or_fallback
if d != null then res += " - {d.html_synopsis.write_to_string}"
return res
end
# List packages by group.
#
# For each key of the `map` a `
` is generated.
# Each package is then listed.
#
# The list of keys is generated first to allow fast access to the correct `
`.
# `id_prefix` is used to give an id to the `
` element.
fun list_by(map: MultiHashMap[String, MPackage], id_prefix: String): Template
do
var res = new Template
var keys = map.keys.to_a
alpha_comparator.sort(keys)
var list = [for x in keys do "{x.html_escape}"]
res.add_list(list, ", ", " and ")
for k in keys do
var projs = map[k].to_a
alpha_comparator.sort(projs)
var e = k.html_escape
res.add "
{e} ({projs.length})
\n
\n"
for p in projs do
res.add "
"
res.add li_package(p)
res.add "
"
end
res.add "
"
end
return res
end
# List the 10 best packages from `cpt`
fun list_best(cpt: Counter[MPackage]): Template
do
var res = new Template
res.add "
"
var best = cpt.sort
for i in [1..10] do
if i > best.length then break
var p = best[best.length-i]
res.add "
"
return res
end
# Collect more information on a package using the `git` tool.
fun git_info(mpackage: MPackage)
do
var ini = mpackage.ini
if ini == null then return
# TODO use real git info
#var repo = ini.get_or_null("upstream.git")
#var branch = ini.get_or_null("upstream.git.branch")
#var directory = ini.get_or_null("upstream.git.directory")
var dirpath = mpackage.root.filepath
if dirpath == null then return
# Collect commits info
var res = git_run("log", "--no-merges", "--follow", "--pretty=tformat:%ad;%aN <%aE>", "--", dirpath)
var contributors = new Counter[String]
var commits = res.split("\n")
if commits.not_empty and commits.last == "" then commits.pop
self.commits[mpackage] = commits.length
for l in commits do
var s = l.split_once_on(';')
if s.length != 2 or s.last == "" then continue
# Collect date of last and first commit
if mpackage.last_date == null then mpackage.last_date = s.first
mpackage.first_date = s.first
# Count contributors
contributors.inc(s.last)
end
for c in contributors.sort.reverse_iterator do
mpackage.contributors.add c
end
end
# Produce a HTML table containig information on the packages
#
# `package_page` must have been called before so that information is computed.
fun table_packages(mpackages: Array[MPackage]): Template
do
alpha_comparator.sort(mpackages)
var res = new Template
res.add "
"
var maint = "?"
if p.maintainers.not_empty then maint = p.maintainers.first
res.add "
{maint}
"
res.add "
{p.contributors.length}
"
res.add "
{deps[p].greaters.length-1}
"
res.add "
{deps[p].direct_greaters.length}
"
res.add "
{deps[p].smallers.length-1}
"
res.add "
{deps[p].direct_smallers.length}
"
res.add "
{mmodules[p]}
"
res.add "
{mclasses[p]}
"
res.add "
{mmethods[p]}
"
res.add "
{loc[p]}
"
res.add "
{score[p]}
"
res.add "
\n"
end
res.add "
\n"
return res
end
end
# Execute a git command and return the result
fun git_run(command: String...): String
do
# print "git {command.join(" ")}"
var p = new ProcessReader("git", command...)
var res = p.read_all
p.close
p.wait
return res
end
var model = new Model
var tc = new ToolContext
tc.process_options(sys.args)
tc.keep_going = true
var modelbuilder = new ModelBuilder(model, tc)
var catalog = new Catalog(modelbuilder)
# Get files or groups
for a in tc.option_context.rest do
modelbuilder.get_mgroup(a)
modelbuilder.identify_file(a)
end
# Scan packages and compute information
for p in model.mpackages do
var g = p.root
assert g != null
modelbuilder.scan_group(g)
# Load the module to process importation information
modelbuilder.parse_group(g)
catalog.deps.add_node(p)
for gg in p.mgroups do for m in gg.mmodules do
for im in m.in_importation.direct_greaters do
var ip = im.mpackage
if ip == null or ip == p then continue
catalog.deps.add_edge(p, ip)
end
end
catalog.git_info(p)
end
# Run phases to modelize classes and properties (so we can count them)
#modelbuilder.run_phases
var out = "out"
out.mkdir
# Generate the css (hard coded)
var css = """
body {
margin-top: 15px;
background-color: #f8f8f8;
}
a {
color: #0D8921;
text-decoration: none;
}
a:hover {
color: #333;
text-decoration: none;
}
h1 {
font-weight: bold;
color: #0D8921;
font-size: 22px;
}
h2 {
color: #6C6C6C;
font-size: 18px;
border-bottom: solid 3px #CCC;
}
h3 {
color: #6C6C6C;
font-size: 15px;
border-bottom: solid 1px #CCC;
}
ul {
list-style-type: square;
}
dd {
color: #6C6C6C;
margin-top: 1em;
margin-bottom: 1em;
}
pre {
border: 1px solid #CCC;
font-family: Monospace;
color: #2d5003;
background-color: rgb(250, 250, 250);
}
code {
font-family: Monospace;
color: #2d5003;
}
footer {
margin-top: 20px;
}
.container {
margin: 0 auto;
padding: 0 20px;
}
.content {
float: left;
margin-top: 40px;
width: 65%;
}
.sidebar {
float: right;
margin-top: 40px;
width: 30%
}
.sidebar h3 {
color: #0D8921;
font-size: 18px;
border-bottom: 0px;
}
.box {
margin: 0;
padding: 0;
}
.box li {
line-height: 2.5;
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
padding-right: 10px;
border-bottom: 1px solid rgba(0,0,0,0.2);
}
"""
css.write_to_file(out/"style.css")
# PAGES
for p in model.mpackages do
# print p
var f = "{p.name}.html"
catalog.package_page(p).write_to_file(out/f)
end
# INDEX
var index = new CatalogPage
index.more_head.add "Packages in Nit"
index.add """
\n"
var reqs = new Counter[MPackage]
for p in model.mpackages do
reqs[p] = catalog.deps[p].smallers.length - 1
end
index.add catalog.list_best(reqs)
index.add "
![](\"https://secure.gravatar.com/avatar/{md5}?size=20&default=retro\")
"
end
res.add "{e}"
if page != null then res.add ""
res.add "