Merge: Added contributing guidelines and link from readme
[nit.git] / src / nitcatalog.nit
index ba0cb1c..42dc5dd 100644 (file)
 #
 # The tool scans packages and generates the HTML files of a catalog.
 #
-# ## Features
-#
-# * [X] scan packages and their `.ini`
-# * [X] generate lists of packages
-# * [X] generate a page per package with the readme and most metadata
-# * [ ] link/include/be included in the documentation
-# * [ ] propose `related packages`
-# * [X] show directory content (a la nitls)
-# * [X] gather git information from the working directory
-# * [ ] gather git information from the repository
-# * [ ] gather package information from github
-# * [ ] gather people information from github
-# * [ ] reify people
-# * [ ] separate information gathering from rendering
-# * [ ] move up information gathering in (existing or new) service modules
-# * [X] add command line options
-# * [ ] harden HTML (escaping, path injection, etc)
-# * [ ] nitcorn server with RESTful API
-#
-# ## Issues and limitations
-#
-# The tool works likee the other tools and expects to find valid Nit source code in the directories
-#
-# * cruft and temporary files will be collected
-# * missing source file (e.g. not yet generated by nitcc) will make information
-#   incomplete (e.g. invalid module thus partial dependency and metrics)
-#
-# How to use the tool as the basis of a Nit code archive on the web usable with a package manager is not clear.
+# See `catalog` for details
 module nitcatalog
 
 import loader # Scan&load packages, groups and modules
 import doc::doc_down # Display mdoc
-import md5 # To get gravatar images
-import counter # For statistics
-import modelize # To process and count classes and methods
-
-redef class MPackage
-       # Return the associated metadata from the `ini`, if any
-       fun metadata(key: String): nullable String
-       do
-               var ini = self.ini
-               if ini == null then return null
-               return ini[key]
-       end
-
-       # The list of maintainers
-       var maintainers = new Array[String]
-
-       # The list of contributors
-       var contributors = new Array[String]
-
-       # The date of the most recent commit
-       var last_date: nullable String = null
-
-       # The date of the oldest commit
-       var first_date: nullable String = null
-end
+import catalog
 
 # A HTML page in a catalog
 #
@@ -83,9 +32,18 @@ end
 class CatalogPage
        super Template
 
+       # The associated catalog, used to groups options and other global data
+       var catalog: Catalog
+
        # Placeholder to include additional things before the `</head>`.
        var more_head = new Template
 
+       # Relative path to the root directory (with the index file).
+       #
+       # Use "" for pages in the root directory
+       # Use ".." for pages in a subdirectory
+       var rootpath: String
+
        redef init
        do
                add """
@@ -94,7 +52,7 @@ class CatalogPage
 <head>
        <meta charset="utf-8">
        <link rel="stylesheet" media="all" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap.min.css">
-       <link rel="stylesheet" media="all" href="style.css">
+       <link rel="stylesheet" media="all" href="{{{rootpath / "style.css"}}}">
 """
                add more_head
 
@@ -116,7 +74,7 @@ class CatalogPage
     </div>
     <div class='collapse navbar-collapse' id='topmenu-collapse'>
      <ul class='nav navbar-nav'>
-      <li><a href="index.html">Catalog</a></li>
+      <li><a href="{{{rootpath / "index.html"}}}">Catalog</a></li>
      </ul>
     </div>
    </div>
@@ -125,6 +83,37 @@ class CatalogPage
 """
        end
 
+       # Inject piwik HTML code if required
+       private fun add_piwik
+       do
+               var tracker_url = catalog.piwik_tracker
+               if tracker_url == null then return
+
+               var site_id = catalog.piwik_site_id
+
+               tracker_url = tracker_url.trim
+               if tracker_url.chars.last != '/' then tracker_url += "/"
+               add """
+<!-- Piwik -->
+<script type="text/javascript">
+var _paq = _paq || [];
+_paq.push(['trackPageView']);
+_paq.push(['enableLinkTracking']);
+(function() {
+var u=(("https:" == document.location.protocol) ? "https" : "http") + "://{{{tracker_url.escape_to_c}}}";
+_paq.push(['setTrackerUrl', u+'piwik.php']);
+_paq.push(['setSiteId', {{{site_id}}}]);
+var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; g.type='text/javascript';
+g.defer=true; g.async=true; g.src=u+'piwik.js'; s.parentNode.insertBefore(g,s);
+})();
+
+</script>
+<noscript><p><img src="http://{{{tracker_url.html_escape}}}piwik.php?idsite={{{site_id}}}" style="border:0;" alt="" /></p></noscript>
+<!-- End Piwik Code -->
+"""
+
+       end
+
        redef fun rendering
        do
                add """
@@ -132,114 +121,86 @@ class CatalogPage
 <script src='https://code.jquery.com/jquery-latest.min.js'></script>
 <script src='https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/js/bootstrap.min.js'></script>
 <script src='https://cdnjs.cloudflare.com/ajax/libs/bootstrap-table/1.8.1/bootstrap-table-all.min.js'></script>
+"""
+               add_piwik
+               add """
+
 </body>
 </html>
 """
        end
 end
 
-redef class Int
-       # Returns `log(self+1)`. Used to compute score of packages
-       fun score: Float do return (self+1).to_f.log
-end
-
-# The main class of the calatog generator that has the knowledge
-class Catalog
-
-       # The modelbuilder
-       # used to access the files and count source lines of code
-       var modelbuilder: ModelBuilder
-
-       # Packages by tag
-       var tag2proj = new MultiHashMap[String, MPackage]
-
-       # Packages by category
-       var cat2proj = new MultiHashMap[String, MPackage]
-
-       # Packages by maintainer
-       var maint2proj = new MultiHashMap[String, MPackage]
-
-       # Packages by contributors
-       var contrib2proj = new MultiHashMap[String, MPackage]
-
-       # Dependency between packages
-       var deps = new POSet[MPackage]
-
-       # Number of modules by package
-       var mmodules = new Counter[MPackage]
-
-       # Number of classes by package
-       var mclasses = new Counter[MPackage]
-
-       # Number of methods by package
-       var mmethods = new Counter[MPackage]
+redef class NitdocDecorator
+       redef fun add_image(v, link, name, comment)
+       do
+               # Keep absolute links as is
+               if link.has_prefix("http://") or link.has_prefix("https://") then
+                       super
+                       return
+               end
 
-       # Number of line of code by package
-       var loc = new Counter[MPackage]
+               do
+                       # Get the directory of the doc object to deal with the relative link
+                       var mdoc = current_mdoc
+                       if mdoc == null then break
+                       var source = mdoc.location.file
+                       if source == null then break
+                       var path = source.filename
+                       var stat = path.file_stat
+                       if stat == null then break
+                       if not stat.is_dir then path = path.dirname
+
+                       # Get the full path to the local resource
+                       var fulllink = path / link.to_s
+                       stat = fulllink.file_stat
+                       if stat == null then break
+
+                       # Get a collision-free catalog name for the resource
+                       var hash = fulllink.md5
+                       var ext = fulllink.file_extension
+                       if ext != null then hash = hash + "." + ext
+
+                       # Copy the local resource in the resource directory of the catalog
+                       var res = catalog.outdir / "res" / hash
+                       fulllink.file_copy_to(res)
+
+                       # Hijack the link in the html.
+                       link = ".." / "res" / hash
+                       super(v, link, name, comment)
+                       return
+               end
 
-       # Number of commits by package
-       var commits = new Counter[MPackage]
+               # Something went bad
+               catalog.modelbuilder.toolcontext.error(current_mdoc.location, "Error: cannot find local image `{link}`")
+               super
+       end
 
-       # Score by package
+       # The registered catalog
        #
-       # The score is loosely computed using other metrics
-       var score = new Counter[MPackage]
+       # It is used to deal with relative links in images.
+       var catalog: Catalog is noautoinit
+end
 
-       # Scan, register and add a contributor to a package
-       fun add_contrib(person: String, mpackage: MPackage, res: Template)
+redef class Catalog
+       redef init
        do
-               var projs = contrib2proj[person]
-               if not projs.has(mpackage) then projs.add mpackage
-               var name = person
-               var email = null
-               var page = null
-
-               # Regular expressions are broken, need to investigate.
-               # So split manually.
-               #
-               #var re = "([^<(]*?)(<([^>]*?)>)?(\\((.*)\\))?".to_re
-               #var m = (person+" ").search(re)
-               #print "{person}: `{m or else "?"}` `{m[1] or else "?"}` `{m[3] or else "?"}` `{m[5] or else "?"}`"
-               do
-                       var sp1 = person.split_once_on("<")
-                       if sp1.length < 2 then
-                               break
-                       end
-                       var sp2 = sp1.last.split_once_on(">")
-                       if sp2.length < 2 then
-                               break
-                       end
-                       name = sp1.first.trim
-                       email = sp2.first.trim
-                       var sp3 = sp2.last.split_once_on("(")
-                       if sp3.length < 2 then
-                               break
-                       end
-                       var sp4 = sp3.last.split_once_on(")")
-                       if sp4.length < 2 then
-                               break
-                       end
-                       page = sp4.first.trim
-               end
+               # Register `self` to the global NitdocDecorator
+               # FIXME this is ugly. But no better idea at the moment.
+               modelbuilder.model.nitdoc_md_processor.emitter.decorator.as(NitdocDecorator).catalog = self
+       end
 
-               var e = name.html_escape
-               res.add "<li>"
-               if page != null then
-                       res.add "<a href=\"{page.html_escape}\">"
-               end
-               if email != null then
-                       # TODO get more things from github by using the email as a key
-                       # "https://api.github.com/search/users?q={email}+in:email"
-                       var md5 = email.md5.to_lower
-                       res.add "<img src=\"https://secure.gravatar.com/avatar/{md5}?size=20&amp;default=retro\">&nbsp;"
-               end
-               res.add "{e}"
-               if page != null then res.add "</a>"
-               res.add "</li>"
+       # The output directory where to generate pages
+       var outdir: String is noautoinit
+
+       # Return a empty `CatalogPage`.
+       fun new_page(rootpath: String): CatalogPage
+       do
+               return new CatalogPage(self, rootpath)
        end
 
        # Recursively generate a level in the file tree of the *content* section
-       private fun gen_content_level(ot: OrderedTree[Object], os: Array[Object], res: Template)
+       private fun gen_content_level(ot: OrderedTree[MConcern], os: Array[Object], res: Template)
        do
                res.add "<ul>\n"
                for o in os do
@@ -249,13 +210,10 @@ class Catalog
                                var mdoc = o.mdoc
                                if mdoc != null then d = ": {mdoc.html_synopsis.write_to_string}"
                                res.add "<strong>{o.name}</strong>{d} ({o.filepath.to_s})"
-                       else if o isa ModulePath then
+                       else if o isa MModule then
                                var d = ""
-                               var m = o.mmodule
-                               if m != null then
-                                       var mdoc = m.mdoc
-                                       if mdoc != null then d = ": {mdoc.html_synopsis.write_to_string}"
-                               end
+                               var mdoc = o.mdoc
+                               if mdoc != null then d = ": {mdoc.html_synopsis.write_to_string}"
                                res.add "<strong>{o.name}</strong>{d} ({o.filepath.to_s})"
                        else
                                abort
@@ -267,11 +225,10 @@ class Catalog
                res.add "</ul>\n"
        end
 
-       # Compute information and generate a full HTML page for a package
-       fun package_page(mpackage: MPackage): Writable
+       # Generate a full HTML page for a package
+       fun generate_page(mpackage: MPackage): Writable
        do
-               var res = new CatalogPage
-               var score = score[mpackage].to_f
+               var res = new_page("..")
                var name = mpackage.name.html_escape
                res.more_head.add """<title>{{{name}}}</title>"""
 
@@ -280,21 +237,17 @@ class Catalog
 <h1 class="package-name">{{{name}}}</h1>
 """
                var mdoc = mpackage.mdoc_or_fallback
-               if mdoc != null then
-                       score += 100.0
-                       res.add mdoc.html_documentation
-                       score += mdoc.content.length.score
-               end
+               if mdoc != null then res.add mdoc.html_documentation
 
                res.add "<h2>Content</h2>"
-               var ot = new OrderedTree[Object]
+               var ot = new OrderedTree[MConcern]
                for g in mpackage.mgroups do
                        var pa = g.parent
                        if g.is_interesting then
                                ot.add(pa, g)
                                pa = g
                        end
-                       for mp in g.module_paths do
+                       for mp in g.mmodules do
                                ot.add(pa, mp)
                        end
                end
@@ -307,23 +260,29 @@ class Catalog
 <div class="sidebar">
 <ul class="box">
 """
+               var tryit = mpackage.metadata("upstream.tryit")
+               if tryit != null then
+                       var e = tryit.html_escape
+                       res.add "<li><a href=\"{e}\">Try<span style=\"color:white\">n</span>it!</a></li>\n"
+               end
+               var apk = mpackage.metadata("upstream.apk")
+               if apk != null then
+                       var e = apk.html_escape
+                       res.add "<li><a href=\"{e}\">Android apk</a></li>\n"
+               end
+
+               res.add """</ul>\n<ul class="box">\n"""
+
                var homepage = mpackage.metadata("upstream.homepage")
                if homepage != null then
-                       score += 5.0
                        var e = homepage.html_escape
                        res.add "<li><a href=\"{e}\">{e}</a></li>\n"
                end
-               var maintainer = mpackage.metadata("package.maintainer")
-               if maintainer != null then
-                       score += 5.0
-                       add_contrib(maintainer, mpackage, res)
-                       mpackage.maintainers.add maintainer
-                       var projs = maint2proj[maintainer]
-                       if not projs.has(mpackage) then projs.add mpackage
+               for maintainer in mpackage.maintainers do
+                       res.add "<li>{maintainer.to_html}</li>"
                end
                var license = mpackage.metadata("package.license")
                if license != null then
-                       score += 5.0
                        var e = license.html_escape
                        res.add "<li><a href=\"http://opensource.org/licenses/{e}\">{e}</a> license</li>\n"
                end
@@ -332,7 +291,6 @@ class Catalog
                res.add "<h3>Source Code</h3>\n<ul class=\"box\">\n"
                var browse = mpackage.metadata("upstream.browse")
                if browse != null then
-                       score += 5.0
                        var e = browse.html_escape
                        res.add "<li><a href=\"{e}\">{e}</a></li>\n"
                end
@@ -357,30 +315,22 @@ class Catalog
                end
                res.add "</ul>\n"
 
+               res.add "<h3>Quality</h3>\n<ul class=\"box\">\n"
+               var errors = errors[mpackage]
+               if errors > 0 then
+                       res.add "<li>{errors} errors</li>\n"
+               end
+               res.add "<li>{warnings[mpackage]} warnings ({warnings_per_kloc[mpackage]}/kloc)</li>\n"
+               res.add "<li>{documentation_score[mpackage]}% documented</li>\n"
+               res.add "</ul>\n"
+
                res.add "<h3>Tags</h3>\n"
-               var tags = mpackage.metadata("package.tags")
                var ts2 = new Array[String]
-               var cat = null
-               if tags != null then
-                       var ts = tags.split(",")
-                       for t in ts do
-                               t = t.trim
-                               if t == "" then continue
-                               if cat == null then cat = t
-                               tag2proj[t].add mpackage
-                               t = t.html_escape
-                               ts2.add "<a href=\"index.html#tag_{t}\">{t}</a>"
-                       end
-                       res.add_list(ts2, ", ", ", ")
+               for t in mpackage.tags do
+                       t = t.html_escape
+                       ts2.add "<a href=\"../index.html#tag_{t}\">{t}</a>"
                end
-               if ts2.is_empty then
-                       var t = "none"
-                       cat = t
-                       tag2proj[t].add mpackage
-                       res.add "<a href=\"index.html#tag_{t}\">{t}</a>"
-               end
-               if cat != null then cat2proj[cat].add mpackage
-               score += ts2.length.score
+               res.add_list(ts2, ", ", ", ")
 
                if deps.has(mpackage) then
                        var reqs = deps[mpackage].greaters.to_a
@@ -422,71 +372,30 @@ class Catalog
                                end
                                res.add_list(list, ", ", " and ")
                        end
-
-                       score += deps[mpackage].greaters.length.score
-                       score += deps[mpackage].direct_greaters.length.score
-                       score += deps[mpackage].smallers.length.score
-                       score += deps[mpackage].direct_smallers.length.score
                end
 
                var contributors = mpackage.contributors
                if not contributors.is_empty then
                        res.add "<h3>Contributors</h3>\n<ul class=\"box\">"
                        for c in contributors do
-                               add_contrib(c, mpackage, res)
+                               res.add "<li>{c.to_html}</li>"
                        end
                        res.add "</ul>"
                end
-               score += contributors.length.to_f
-
-               var mmodules = 0
-               var mclasses = 0
-               var mmethods = 0
-               var loc = 0
-               for g in mpackage.mgroups do
-                       mmodules += g.module_paths.length
-                       for m in g.mmodules do
-                               var am = modelbuilder.mmodule2node(m)
-                               if am != null then
-                                       var file = am.location.file
-                                       if file != null then
-                                               loc += file.line_starts.length - 1
-                                       end
-                               end
-                               for cd in m.mclassdefs do
-                                       mclasses += 1
-                                       for pd in cd.mpropdefs do
-                                               if not pd isa MMethodDef then continue
-                                               mmethods += 1
-                                       end
-                               end
-                       end
-               end
-               self.mmodules[mpackage] = mmodules
-               self.mclasses[mpackage] = mclasses
-               self.mmethods[mpackage] = mmethods
-               self.loc[mpackage] = loc
-
-               #score += mmodules.score
-               score += mclasses.score
-               score += mmethods.score
-               score += loc.score
 
                res.add """
 <h3>Stats</h3>
 <ul class="box">
-<li>{{{mmodules}}} modules</li>
-<li>{{{mclasses}}} classes</li>
-<li>{{{mmethods}}} methods</li>
-<li>{{{loc}}} lines of code</li>
+<li>{{{mmodules[mpackage]}}} modules</li>
+<li>{{{mclasses[mpackage]}}} classes</li>
+<li>{{{mmethods[mpackage]}}} methods</li>
+<li>{{{loc[mpackage]}}} lines of code</li>
 </ul>
 """
 
                res.add """
 </div>
 """
-               self.score[mpackage] = score.to_i
-
                return res
        end
 
@@ -496,7 +405,7 @@ class Catalog
        fun li_package(p: MPackage): String
        do
                var res = ""
-               var f = "{p.name}.html"
+               var f = "p/{p.name}.html"
                res += "<a href=\"{f}\">{p}</a>"
                var d = p.mdoc_or_fallback
                if d != null then res += " - {d.html_synopsis.write_to_string}"
@@ -510,18 +419,18 @@ class Catalog
        #
        # The list of keys is generated first to allow fast access to the correct `<h3>`.
        # `id_prefix` is used to give an id to the `<h3>` element.
-       fun list_by(map: MultiHashMap[String, MPackage], id_prefix: String): Template
+       fun list_by(map: MultiHashMap[Object, MPackage], id_prefix: String): Template
        do
                var res = new Template
                var keys = map.keys.to_a
                alpha_comparator.sort(keys)
-               var list = [for x in keys do "<a href=\"#{id_prefix}{x.html_escape}\">{x.html_escape}</a>"]
+               var list = [for x in keys do "<a href=\"#{id_prefix}{x.to_s.html_escape}\">{x.to_s.html_escape}</a>"]
                res.add_list(list, ", ", " and ")
 
                for k in keys do
                        var projs = map[k].to_a
                        alpha_comparator.sort(projs)
-                       var e = k.html_escape
+                       var e = k.to_s.html_escape
                        res.add "<h3 id=\"{id_prefix}{e}\">{e} ({projs.length})</h3>\n<ul>\n"
                        for p in projs do
                                res.add "<li>"
@@ -551,43 +460,6 @@ class Catalog
                return res
        end
 
-       # Collect more information on a package using the `git` tool.
-       fun git_info(mpackage: MPackage)
-       do
-               var ini = mpackage.ini
-               if ini == null then return
-
-               # TODO use real git info
-               #var repo = ini.get_or_null("upstream.git")
-               #var branch = ini.get_or_null("upstream.git.branch")
-               #var directory = ini.get_or_null("upstream.git.directory")
-
-               var dirpath = mpackage.root.filepath
-               if dirpath == null then return
-
-               # Collect commits info
-               var res = git_run("log", "--no-merges", "--follow", "--pretty=tformat:%ad;%aN <%aE>", "--", dirpath)
-               var contributors = new Counter[String]
-               var commits = res.split("\n")
-               if commits.not_empty and commits.last == "" then commits.pop
-               self.commits[mpackage] = commits.length
-               for l in commits do
-                       var s = l.split_once_on(';')
-                       if s.length != 2 or s.last == "" then continue
-
-                       # Collect date of last and first commit
-                       if mpackage.last_date == null then mpackage.last_date = s.first
-                       mpackage.first_date = s.first
-
-                       # Count contributors
-                       contributors.inc(s.last)
-               end
-               for c in contributors.sort.reverse_iterator do
-                       mpackage.contributors.add c
-               end
-
-       end
-
        # Produce a HTML table containig information on the packages
        #
        # `package_page` must have been called before so that information is computed.
@@ -611,12 +483,16 @@ class Catalog
                res.add "<th data-field=\"met\" data-sortable=\"true\">methods</th>\n"
                res.add "<th data-field=\"loc\" data-sortable=\"true\">lines</th>\n"
                res.add "<th data-field=\"score\" data-sortable=\"true\">score</th>\n"
+               res.add "<th data-field=\"errors\" data-sortable=\"true\">errors</th>\n"
+               res.add "<th data-field=\"warnings\" data-sortable=\"true\">warnings</th>\n"
+               res.add "<th data-field=\"warnings_per_kloc\" data-sortable=\"true\">w/kloc</th>\n"
+               res.add "<th data-field=\"doc\" data-sortable=\"true\">doc</th>\n"
                res.add "</tr></thead>"
                for p in mpackages do
                        res.add "<tr>"
-                       res.add "<td><a href=\"{p.name}.html\">{p.name}</a></td>"
+                       res.add "<td><a href=\"p/{p.name}.html\">{p.name}</a></td>"
                        var maint = "?"
-                       if p.maintainers.not_empty then maint = p.maintainers.first
+                       if p.maintainers.not_empty then maint = p.maintainers.first.name.html_escape
                        res.add "<td>{maint}</td>"
                        res.add "<td>{p.contributors.length}</td>"
                        if deps.not_empty then
@@ -630,22 +506,22 @@ class Catalog
                        res.add "<td>{mmethods[p]}</td>"
                        res.add "<td>{loc[p]}</td>"
                        res.add "<td>{score[p]}</td>"
+                       res.add "<td>{errors[p]}</td>"
+                       res.add "<td>{warnings[p]}</td>"
+                       res.add "<td>{warnings_per_kloc[p]}</td>"
+                       res.add "<td>{documentation_score[p]}</td>"
                        res.add "</tr>\n"
                end
                res.add "</table>\n"
                return res
        end
-end
 
-# Execute a git command and return the result
-fun git_run(command: String...): String
-do
-       # print "git {command.join(" ")}"
-       var p = new ProcessReader("git", command...)
-       var res = p.read_all
-       p.close
-       p.wait
-       return res
+       # Piwik tracker URL, if any
+       var piwik_tracker: nullable String = null
+
+       # Piwik site ID
+       # Used when `piwik_tracker` is set
+       var piwik_site_id: Int = 1
 end
 
 var model = new Model
@@ -656,7 +532,13 @@ var opt_no_git = new OptionBool("Do not gather git information from the working
 var opt_no_parse = new OptionBool("Do not parse nit files (no importation information)", "--no-parse")
 var opt_no_model = new OptionBool("Do not analyse nit files (no class/method information)", "--no-model")
 
-tc.option_context.add_option(opt_dir, opt_no_git, opt_no_parse, opt_no_model)
+# Piwik tracker URL.
+# If you want to monitor your visitors.
+var opt_piwik_tracker = new OptionString("Piwik tracker URL (ex: `nitlanguage.org/piwik/`)", "--piwik-tracker")
+# Piwik tracker site id.
+var opt_piwik_site_id = new OptionString("Piwik site ID", "--piwik-site-id")
+
+tc.option_context.add_option(opt_dir, opt_no_git, opt_no_parse, opt_no_model, opt_piwik_tracker, opt_piwik_site_id)
 
 tc.process_options(sys.args)
 tc.keep_going = true
@@ -664,10 +546,31 @@ tc.keep_going = true
 var modelbuilder = new ModelBuilder(model, tc)
 var catalog = new Catalog(modelbuilder)
 
+catalog.piwik_tracker = opt_piwik_tracker.value
+var piwik_site_id = opt_piwik_site_id.value
+if piwik_site_id != null then
+       if catalog.piwik_tracker == null then
+               print_error "Warning: ignored `{opt_piwik_site_id}` because `{opt_piwik_tracker}` is not set."
+       else if piwik_site_id.is_int then
+               print_error "Warning: ignored `{opt_piwik_site_id}`, an integer is required."
+       else
+               catalog.piwik_site_id = piwik_site_id.to_i
+       end
+end
+
+
 # Get files or groups
-for a in tc.option_context.rest do
-       modelbuilder.get_mgroup(a)
-       modelbuilder.identify_file(a)
+var args = tc.option_context.rest
+var mmodules
+if opt_no_parse.value then
+       mmodules = modelbuilder.scan_full(args)
+else
+       mmodules = modelbuilder.parse_full(args)
+end
+var mpackages = new Set[MPackage]
+for m in mmodules do
+       var p = m.mpackage
+       if p != null then mpackages.add p
 end
 
 # Scan packages and compute information
@@ -678,7 +581,6 @@ for p in model.mpackages do
 
        # Load the module to process importation information
        if opt_no_parse.value then continue
-       modelbuilder.parse_group(g)
 
        catalog.deps.add_node(p)
        for gg in p.mgroups do for m in gg.mmodules do
@@ -690,7 +592,7 @@ for p in model.mpackages do
        end
 end
 
-if not opt_no_git.value then for p in model.mpackages do
+if not opt_no_git.value then for p in mpackages do
        catalog.git_info(p)
 end
 
@@ -700,7 +602,10 @@ if not opt_no_model.value then
 end
 
 var out = opt_dir.value or else "catalog.out"
-out.mkdir
+(out/"p").mkdir
+(out/"res").mkdir
+
+catalog.outdir = out
 
 # Generate the css (hard coded)
 var css = """
@@ -804,15 +709,16 @@ css.write_to_file(out/"style.css")
 
 # PAGES
 
-for p in model.mpackages do
+for p in mpackages do
        # print p
-       var f = "{p.name}.html"
-       catalog.package_page(p).write_to_file(out/f)
+       var f = "p/{p.name}.html"
+       catalog.package_page(p)
+       catalog.generate_page(p).write_to_file(out/f)
 end
 
 # INDEX
 
-var index = new CatalogPage
+var index = catalog.new_page("")
 index.more_head.add "<title>Packages in Nit</title>"
 
 index.add """
@@ -826,7 +732,7 @@ index.add catalog.list_best(catalog.score)
 if catalog.deps.not_empty then
        index.add "<h2>Most Required</h2>\n"
        var reqs = new Counter[MPackage]
-       for p in model.mpackages do
+       for p in mpackages do
                reqs[p] = catalog.deps[p].smallers.length - 1
        end
        index.add catalog.list_best(reqs)
@@ -843,7 +749,7 @@ index.add """
 <div class="sidebar">
 <h3>Stats</h3>
 <ul class="box">
-<li>{{{model.mpackages.length}}} packages</li>
+<li>{{{mpackages.length}}} packages</li>
 <li>{{{catalog.maint2proj.length}}} maintainers</li>
 <li>{{{catalog.contrib2proj.length}}} contributors</li>
 <li>{{{catalog.tag2proj.length}}} tags</li>
@@ -859,7 +765,7 @@ index.write_to_file(out/"index.html")
 
 # PEOPLE
 
-var page = new CatalogPage
+var page = catalog.new_page("")
 page.more_head.add "<title>People of Nit</title>"
 page.add """<div class="content">\n<h1>People of Nit</h1>\n"""
 page.add "<h2>By Maintainer</h2>\n"
@@ -871,10 +777,10 @@ page.write_to_file(out/"people.html")
 
 # TABLE
 
-page = new CatalogPage
+page = catalog.new_page("")
 page.more_head.add "<title>Projets of Nit</title>"
 page.add """<div class="content">\n<h1>People of Nit</h1>\n"""
 page.add "<h2>Table of Projets</h2>\n"
-page.add catalog.table_packages(model.mpackages)
+page.add catalog.table_packages(mpackages.to_a)
 page.add "</div>\n"
 page.write_to_file(out/"table.html")