From: Jean Privat Date: Sun, 17 Apr 2016 01:49:14 +0000 (-0400) Subject: nitc: split nitcatalog into a lib and a program X-Git-Url: http://nitlanguage.org nitc: split nitcatalog into a lib and a program Signed-off-by: Jean Privat --- diff --git a/src/catalog.nit b/src/catalog.nit new file mode 100644 index 0000000..1d6bb2a --- /dev/null +++ b/src/catalog.nit @@ -0,0 +1,300 @@ +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Basic catalog generator for Nit packages +# +# See: +# +# The tool scans packages and generates the HTML files of a catalog. +# +# ## Features +# +# * [X] scan packages and their `.ini` +# * [X] generate lists of packages +# * [X] generate a page per package with the readme and most metadata +# * [ ] link/include/be included in the documentation +# * [ ] propose `related packages` +# * [X] show directory content (a la nitls) +# * [X] gather git information from the working directory +# * [ ] gather git information from the repository +# * [ ] gather package information from github +# * [ ] gather people information from github +# * [ ] reify people +# * [X] separate information gathering from rendering +# * [ ] move up information gathering in (existing or new) service modules +# * [X] add command line options +# * [ ] harden HTML (escaping, path injection, etc) +# * [ ] nitcorn server with RESTful API +# +# ## Issues and limitations +# +# The tool works likee the other tools and expects to find valid Nit source code in the directories +# +# * cruft and temporary files will be collected +# * missing source file (e.g. not yet generated by nitcc) will make information +# incomplete (e.g. invalid module thus partial dependency and metrics) +# +# How to use the tool as the basis of a Nit code archive on the web usable with a package manager is not clear. +module catalog + +import md5 # To get gravatar images +import counter # For statistics +import modelize # To process and count classes and methods + +redef class MPackage + # Return the associated metadata from the `ini`, if any + fun metadata(key: String): nullable String + do + var ini = self.ini + if ini == null then return null + return ini[key] + end + + # The consolidated list of tags + var tags = new Array[String] + + # The list of maintainers + var maintainers = new Array[String] + + # The list of contributors + var contributors = new Array[String] + + # The date of the most recent commit + var last_date: nullable String = null + + # The date of the oldest commit + var first_date: nullable String = null +end + +redef class Int + # Returns `log(self+1)`. Used to compute score of packages + fun score: Float do return (self+1).to_f.log +end + +# The main class of the calatog generator that has the knowledge +class Catalog + + # The modelbuilder + # used to access the files and count source lines of code + var modelbuilder: ModelBuilder + + # Packages by tag + var tag2proj = new MultiHashMap[String, MPackage] + + # Packages by category + var cat2proj = new MultiHashMap[String, MPackage] + + # Packages by maintainer + var maint2proj = new MultiHashMap[String, MPackage] + + # Packages by contributors + var contrib2proj = new MultiHashMap[String, MPackage] + + # Dependency between packages + var deps = new POSet[MPackage] + + # Number of modules by package + var mmodules = new Counter[MPackage] + + # Number of classes by package + var mclasses = new Counter[MPackage] + + # Number of methods by package + var mmethods = new Counter[MPackage] + + # Number of line of code by package + var loc = new Counter[MPackage] + + # Number of commits by package + var commits = new Counter[MPackage] + + # Score by package + # + # The score is loosely computed using other metrics + var score = new Counter[MPackage] + + # Scan, register and add a contributor to a package + fun register_contrib(person: String, mpackage: MPackage) + do + var projs = contrib2proj[person] + if not projs.has(mpackage) then projs.add mpackage + end + + # Compute information for a package + fun package_page(mpackage: MPackage) + do + var score = score[mpackage].to_f + + var mdoc = mpackage.mdoc_or_fallback + if mdoc != null then + score += 100.0 + score += mdoc.content.length.score + end + + + var tryit = mpackage.metadata("upstream.tryit") + if tryit != null then + score += 1.0 + end + var apk = mpackage.metadata("upstream.apk") + if apk != null then + score += 1.0 + end + + var homepage = mpackage.metadata("upstream.homepage") + if homepage != null then + score += 5.0 + end + var maintainer = mpackage.metadata("package.maintainer") + if maintainer != null then + score += 5.0 + register_contrib(maintainer, mpackage) + mpackage.maintainers.add maintainer + var projs = maint2proj[maintainer] + if not projs.has(mpackage) then projs.add mpackage + end + var license = mpackage.metadata("package.license") + if license != null then + score += 5.0 + end + + var browse = mpackage.metadata("upstream.browse") + if browse != null then + score += 5.0 + end + + var tags = mpackage.metadata("package.tags") + var ts = mpackage.tags + if tags != null then + for t in tags.split(",") do + t = t.trim + if t == "" then continue + ts.add t + end + end + if ts.is_empty then ts.add "none" + if tryit != null then ts.add "tryit" + if apk != null then ts.add "apk" + for t in ts do + tag2proj[t].add mpackage + end + var cat = ts.first + cat2proj[cat].add mpackage + score += ts.length.score + + if deps.has(mpackage) then + score += deps[mpackage].greaters.length.score + score += deps[mpackage].direct_greaters.length.score + score += deps[mpackage].smallers.length.score + score += deps[mpackage].direct_smallers.length.score + end + + var contributors = mpackage.contributors + var more_contributors = mpackage.metadata("package.more_contributors") + if more_contributors != null then + for c in more_contributors.split(",") do + contributors.add c.trim + end + end + if not contributors.is_empty then + for c in contributors do + register_contrib(c, mpackage) + end + end + score += contributors.length.to_f + + var mmodules = 0 + var mclasses = 0 + var mmethods = 0 + var loc = 0 + for g in mpackage.mgroups do + mmodules += g.mmodules.length + for m in g.mmodules do + var am = modelbuilder.mmodule2node(m) + if am != null then + var file = am.location.file + if file != null then + loc += file.line_starts.length - 1 + end + end + for cd in m.mclassdefs do + mclasses += 1 + for pd in cd.mpropdefs do + if not pd isa MMethodDef then continue + mmethods += 1 + end + end + end + end + self.mmodules[mpackage] = mmodules + self.mclasses[mpackage] = mclasses + self.mmethods[mpackage] = mmethods + self.loc[mpackage] = loc + + #score += mmodules.score + score += mclasses.score + score += mmethods.score + score += loc.score + + self.score[mpackage] = score.to_i + end + + # Collect more information on a package using the `git` tool. + fun git_info(mpackage: MPackage) + do + var ini = mpackage.ini + if ini == null then return + + # TODO use real git info + #var repo = ini.get_or_null("upstream.git") + #var branch = ini.get_or_null("upstream.git.branch") + #var directory = ini.get_or_null("upstream.git.directory") + + var dirpath = mpackage.root.filepath + if dirpath == null then return + + # Collect commits info + var res = git_run("log", "--no-merges", "--follow", "--pretty=tformat:%ad;%aN <%aE>", "--", dirpath) + var contributors = new Counter[String] + var commits = res.split("\n") + if commits.not_empty and commits.last == "" then commits.pop + self.commits[mpackage] = commits.length + for l in commits do + var s = l.split_once_on(';') + if s.length != 2 or s.last == "" then continue + + # Collect date of last and first commit + if mpackage.last_date == null then mpackage.last_date = s.first + mpackage.first_date = s.first + + # Count contributors + contributors.inc(s.last) + end + for c in contributors.sort.reverse_iterator do + mpackage.contributors.add c + end + + end +end + +# Execute a git command and return the result +fun git_run(command: String...): String +do + # print "git {command.join(" ")}" + var p = new ProcessReader("git", command...) + var res = p.read_all + p.close + p.wait + return res +end diff --git a/src/nitcatalog.nit b/src/nitcatalog.nit index c604fa6..d2aff2d 100644 --- a/src/nitcatalog.nit +++ b/src/nitcatalog.nit @@ -18,63 +18,12 @@ # # The tool scans packages and generates the HTML files of a catalog. # -# ## Features -# -# * [X] scan packages and their `.ini` -# * [X] generate lists of packages -# * [X] generate a page per package with the readme and most metadata -# * [ ] link/include/be included in the documentation -# * [ ] propose `related packages` -# * [X] show directory content (a la nitls) -# * [X] gather git information from the working directory -# * [ ] gather git information from the repository -# * [ ] gather package information from github -# * [ ] gather people information from github -# * [ ] reify people -# * [ ] separate information gathering from rendering -# * [ ] move up information gathering in (existing or new) service modules -# * [X] add command line options -# * [ ] harden HTML (escaping, path injection, etc) -# * [ ] nitcorn server with RESTful API -# -# ## Issues and limitations -# -# The tool works likee the other tools and expects to find valid Nit source code in the directories -# -# * cruft and temporary files will be collected -# * missing source file (e.g. not yet generated by nitcc) will make information -# incomplete (e.g. invalid module thus partial dependency and metrics) -# -# How to use the tool as the basis of a Nit code archive on the web usable with a package manager is not clear. +# See `catalog` for details module nitcatalog import loader # Scan&load packages, groups and modules import doc::doc_down # Display mdoc -import md5 # To get gravatar images -import counter # For statistics -import modelize # To process and count classes and methods - -redef class MPackage - # Return the associated metadata from the `ini`, if any - fun metadata(key: String): nullable String - do - var ini = self.ini - if ini == null then return null - return ini[key] - end - - # The list of maintainers - var maintainers = new Array[String] - - # The list of contributors - var contributors = new Array[String] - - # The date of the most recent commit - var last_date: nullable String = null - - # The date of the oldest commit - var first_date: nullable String = null -end +import catalog # A HTML page in a catalog # @@ -182,64 +131,16 @@ g.defer=true; g.async=true; g.src=u+'piwik.js'; s.parentNode.insertBefore(g,s); end end -redef class Int - # Returns `log(self+1)`. Used to compute score of packages - fun score: Float do return (self+1).to_f.log -end - -# The main class of the calatog generator that has the knowledge -class Catalog - - # The modelbuilder - # used to access the files and count source lines of code - var modelbuilder: ModelBuilder - - # Packages by tag - var tag2proj = new MultiHashMap[String, MPackage] - - # Packages by category - var cat2proj = new MultiHashMap[String, MPackage] - - # Packages by maintainer - var maint2proj = new MultiHashMap[String, MPackage] - - # Packages by contributors - var contrib2proj = new MultiHashMap[String, MPackage] - - # Dependency between packages - var deps = new POSet[MPackage] - - # Number of modules by package - var mmodules = new Counter[MPackage] - - # Number of classes by package - var mclasses = new Counter[MPackage] - - # Number of methods by package - var mmethods = new Counter[MPackage] - - # Number of line of code by package - var loc = new Counter[MPackage] - - # Number of commits by package - var commits = new Counter[MPackage] - - # Score by package - # - # The score is loosely computed using other metrics - var score = new Counter[MPackage] - +redef class Catalog # Return a empty `CatalogPage`. fun new_page(rootpath: String): CatalogPage do return new CatalogPage(self, rootpath) end - # Scan, register and add a contributor to a package + # Add a contributor to a package fun add_contrib(person: String, mpackage: MPackage, res: Template) do - var projs = contrib2proj[person] - if not projs.has(mpackage) then projs.add mpackage var name = person var email = null var page = null @@ -314,11 +215,10 @@ class Catalog res.add "\n" end - # Compute information and generate a full HTML page for a package - fun package_page(mpackage: MPackage): Writable + # Generate a full HTML page for a package + fun generate_page(mpackage: MPackage): Writable do var res = new_page("..") - var score = score[mpackage].to_f var name = mpackage.name.html_escape res.more_head.add """{{{name}}}""" @@ -327,11 +227,7 @@ class Catalog

{{{name}}}

""" var mdoc = mpackage.mdoc_or_fallback - if mdoc != null then - score += 100.0 - res.add mdoc.html_documentation - score += mdoc.content.length.score - end + if mdoc != null then res.add mdoc.html_documentation res.add "

Content

" var ot = new OrderedTree[MConcern] @@ -356,13 +252,11 @@ class Catalog """ var tryit = mpackage.metadata("upstream.tryit") if tryit != null then - score += 1.0 var e = tryit.html_escape res.add "
  • Trynit!
    • \n" end var apk = mpackage.metadata("upstream.apk") if apk != null then - score += 1.0 var e = apk.html_escape res.add "
  • Android apk
    • \n" end @@ -371,21 +265,15 @@ class Catalog var homepage = mpackage.metadata("upstream.homepage") if homepage != null then - score += 5.0 var e = homepage.html_escape res.add "
  • {e}
    • \n" end var maintainer = mpackage.metadata("package.maintainer") if maintainer != null then - score += 5.0 add_contrib(maintainer, mpackage, res) - mpackage.maintainers.add maintainer - var projs = maint2proj[maintainer] - if not projs.has(mpackage) then projs.add mpackage end var license = mpackage.metadata("package.license") if license != null then - score += 5.0 var e = license.html_escape res.add "
  • {e} license
    • \n" end @@ -394,7 +282,6 @@ class Catalog res.add "

    Source Code

    \n
      \n" var browse = mpackage.metadata("upstream.browse") if browse != null then - score += 5.0 var e = browse.html_escape res.add "
    • {e}
      • \n" end @@ -420,28 +307,12 @@ class Catalog res.add "
    \n" res.add "

    Tags

    \n" - var tags = mpackage.metadata("package.tags") - var ts = new Array[String] - if tags != null then - for t in tags.split(",") do - t = t.trim - if t == "" then continue - ts.add t - end - end - if ts.is_empty then ts.add "none" - if tryit != null then ts.add "tryit" - if apk != null then ts.add "apk" var ts2 = new Array[String] - for t in ts do - tag2proj[t].add mpackage + for t in mpackage.tags do t = t.html_escape ts2.add "{t}" end res.add_list(ts2, ", ", ", ") - var cat = ts.first - cat2proj[cat].add mpackage - score += ts.length.score if deps.has(mpackage) then var reqs = deps[mpackage].greaters.to_a @@ -483,20 +354,9 @@ class Catalog end res.add_list(list, ", ", " and ") end - - score += deps[mpackage].greaters.length.score - score += deps[mpackage].direct_greaters.length.score - score += deps[mpackage].smallers.length.score - score += deps[mpackage].direct_smallers.length.score end var contributors = mpackage.contributors - var more_contributors = mpackage.metadata("package.more_contributors") - if more_contributors != null then - for c in more_contributors.split(",") do - contributors.add c.trim - end - end if not contributors.is_empty then res.add "

    Contributors

    \n
      " for c in contributors do @@ -504,56 +364,20 @@ class Catalog end res.add "
    " end - score += contributors.length.to_f - - var mmodules = 0 - var mclasses = 0 - var mmethods = 0 - var loc = 0 - for g in mpackage.mgroups do - mmodules += g.mmodules.length - for m in g.mmodules do - var am = modelbuilder.mmodule2node(m) - if am != null then - var file = am.location.file - if file != null then - loc += file.line_starts.length - 1 - end - end - for cd in m.mclassdefs do - mclasses += 1 - for pd in cd.mpropdefs do - if not pd isa MMethodDef then continue - mmethods += 1 - end - end - end - end - self.mmodules[mpackage] = mmodules - self.mclasses[mpackage] = mclasses - self.mmethods[mpackage] = mmethods - self.loc[mpackage] = loc - - #score += mmodules.score - score += mclasses.score - score += mmethods.score - score += loc.score res.add """

    Stats

      -
    • {{{mmodules}}} modules
      • -
    • {{{mclasses}}} classes
      • -
    • {{{mmethods}}} methods
      • -
    • {{{loc}}} lines of code
      • +
    • {{{mmodules[mpackage]}}} modules
      • +
    • {{{mclasses[mpackage]}}} classes
      • +
    • {{{mmethods[mpackage]}}} methods
      • +
    • {{{loc[mpackage]}}} lines of code
    """ res.add """ """ - self.score[mpackage] = score.to_i - return res end @@ -618,43 +442,6 @@ class Catalog return res end - # Collect more information on a package using the `git` tool. - fun git_info(mpackage: MPackage) - do - var ini = mpackage.ini - if ini == null then return - - # TODO use real git info - #var repo = ini.get_or_null("upstream.git") - #var branch = ini.get_or_null("upstream.git.branch") - #var directory = ini.get_or_null("upstream.git.directory") - - var dirpath = mpackage.root.filepath - if dirpath == null then return - - # Collect commits info - var res = git_run("log", "--no-merges", "--follow", "--pretty=tformat:%ad;%aN <%aE>", "--", dirpath) - var contributors = new Counter[String] - var commits = res.split("\n") - if commits.not_empty and commits.last == "" then commits.pop - self.commits[mpackage] = commits.length - for l in commits do - var s = l.split_once_on(';') - if s.length != 2 or s.last == "" then continue - - # Collect date of last and first commit - if mpackage.last_date == null then mpackage.last_date = s.first - mpackage.first_date = s.first - - # Count contributors - contributors.inc(s.last) - end - for c in contributors.sort.reverse_iterator do - mpackage.contributors.add c - end - - end - # Produce a HTML table containig information on the packages # # `package_page` must have been called before so that information is computed. @@ -711,17 +498,6 @@ class Catalog var piwik_site_id: Int = 1 end -# Execute a git command and return the result -fun git_run(command: String...): String -do - # print "git {command.join(" ")}" - var p = new ProcessReader("git", command...) - var res = p.read_all - p.close - p.wait - return res -end - var model = new Model var tc = new ToolContext @@ -901,7 +677,8 @@ css.write_to_file(out/"style.css") for p in model.mpackages do # print p var f = "p/{p.name}.html" - catalog.package_page(p).write_to_file(out/f) + catalog.package_page(p) + catalog.generate_page(p).write_to_file(out/f) end # INDEX