src/catalog.nit

   1 # This file is part of NIT ( http://www.nitlanguage.org ).
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 # Basic catalog generator for Nit packages
  16 #
  17 # See: <http://nitlanguage.org/catalog/>
  18 #
  19 # The tool scans packages and generates the HTML files of a catalog.
  20 #
  21 # ## Features
  22 #
  23 # * [X] scan packages and their `.ini`
  24 # * [X] generate lists of packages
  25 # * [X] generate a page per package with the readme and most metadata
  26 # * [ ] link/include/be included in the documentation
  27 # * [ ] propose `related packages`
  28 # * [X] show directory content (a la nitls)
  29 # * [X] gather git information from the working directory
  30 # * [ ] gather git information from the repository
  31 # * [ ] gather package information from github
  32 # * [ ] gather people information from github
  33 # * [ ] reify people
  34 # * [X] separate information gathering from rendering
  35 # * [ ] move up information gathering in (existing or new) service modules
  36 # * [X] add command line options
  37 # * [ ] harden HTML (escaping, path injection, etc)
  38 # * [ ] nitcorn server with RESTful API
  39 #
  40 # ## Issues and limitations
  41 #
  42 # The tool works likee the other tools and expects to find valid Nit source code in the directories
  43 #
  44 # * cruft and temporary files will be collected
  45 # * missing source file (e.g. not yet generated by nitcc) will make information
  46 #   incomplete (e.g. invalid module thus partial dependency and metrics)
  47 #
  48 # How to use the tool as the basis of a Nit code archive on the web usable with a package manager is not clear.
  49 module catalog
  50
  51 import md5 # To get gravatar images
  52 import counter # For statistics
  53 import modelize # To process and count classes and methods
  54
  55 redef class MPackage
  56         # Return the associated metadata from the `ini`, if any
  57         fun metadata(key: String): nullable String
  58         do
  59                 var ini = self.ini
  60                 if ini == null then return null
  61                 return ini[key]
  62         end
  63
  64         # The consolidated list of tags
  65         var tags = new Array[String]
  66
  67         # The list of maintainers
  68         var maintainers = new Array[String]
  69
  70         # The list of contributors
  71         var contributors = new Array[String]
  72
  73         # The date of the most recent commit
  74         var last_date: nullable String = null
  75
  76         # The date of the oldest commit
  77         var first_date: nullable String = null
  78 end
  79
  80 redef class Int
  81         # Returns `log(self+1)`. Used to compute score of packages
  82         fun score: Float do return (self+1).to_f.log
  83 end
  84
  85 # The main class of the calatog generator that has the knowledge
  86 class Catalog
  87
  88         # The modelbuilder
  89         # used to access the files and count source lines of code
  90         var modelbuilder: ModelBuilder
  91
  92         # Packages by tag
  93         var tag2proj = new MultiHashMap[String, MPackage]
  94
  95         # Packages by category
  96         var cat2proj = new MultiHashMap[String, MPackage]
  97
  98         # Packages by maintainer
  99         var maint2proj = new MultiHashMap[String, MPackage]
 100
 101         # Packages by contributors
 102         var contrib2proj = new MultiHashMap[String, MPackage]
 103
 104         # Dependency between packages
 105         var deps = new POSet[MPackage]
 106
 107         # Number of modules by package
 108         var mmodules = new Counter[MPackage]
 109
 110         # Number of classes by package
 111         var mclasses = new Counter[MPackage]
 112
 113         # Number of methods by package
 114         var mmethods = new Counter[MPackage]
 115
 116         # Number of line of code by package
 117         var loc = new Counter[MPackage]
 118
 119         # Number of commits by package
 120         var commits = new Counter[MPackage]
 121
 122         # Score by package
 123         #
 124         # The score is loosely computed using other metrics
 125         var score = new Counter[MPackage]
 126
 127         # Scan, register and add a contributor to a package
 128         fun register_contrib(person: String, mpackage: MPackage)
 129         do
 130                 var projs = contrib2proj[person]
 131                 if not projs.has(mpackage) then projs.add mpackage
 132         end
 133
 134         # Compute information for a package
 135         fun package_page(mpackage: MPackage)
 136         do
 137                 var score = score[mpackage].to_f
 138
 139                 var mdoc = mpackage.mdoc_or_fallback
 140                 if mdoc != null then
 141                         score += 100.0
 142                         score += mdoc.content.length.score
 143                 end
 144
 145
 146                 var tryit = mpackage.metadata("upstream.tryit")
 147                 if tryit != null then
 148                         score += 1.0
 149                 end
 150                 var apk = mpackage.metadata("upstream.apk")
 151                 if apk != null then
 152                         score += 1.0
 153                 end
 154
 155                 var homepage = mpackage.metadata("upstream.homepage")
 156                 if homepage != null then
 157                         score += 5.0
 158                 end
 159                 var maintainer = mpackage.metadata("package.maintainer")
 160                 if maintainer != null then
 161                         score += 5.0
 162                         register_contrib(maintainer, mpackage)
 163                         mpackage.maintainers.add maintainer
 164                         var projs = maint2proj[maintainer]
 165                         if not projs.has(mpackage) then projs.add mpackage
 166                 end
 167                 var license = mpackage.metadata("package.license")
 168                 if license != null then
 169                         score += 5.0
 170                 end
 171
 172                 var browse = mpackage.metadata("upstream.browse")
 173                 if browse != null then
 174                         score += 5.0
 175                 end
 176
 177                 var tags = mpackage.metadata("package.tags")
 178                 var ts = mpackage.tags
 179                 if tags != null then
 180                         for t in tags.split(",") do
 181                                 t = t.trim
 182                                 if t == "" then continue
 183                                 ts.add t
 184                         end
 185                 end
 186                 if ts.is_empty then ts.add "none"
 187                 if tryit != null then ts.add "tryit"
 188                 if apk != null then ts.add "apk"
 189                 for t in ts do
 190                         tag2proj[t].add mpackage
 191                 end
 192                 var cat = ts.first
 193                 cat2proj[cat].add mpackage
 194                 score += ts.length.score
 195
 196                 if deps.has(mpackage) then
 197                         score += deps[mpackage].greaters.length.score
 198                         score += deps[mpackage].direct_greaters.length.score
 199                         score += deps[mpackage].smallers.length.score
 200                         score += deps[mpackage].direct_smallers.length.score
 201                 end
 202
 203                 var contributors = mpackage.contributors
 204                 var more_contributors = mpackage.metadata("package.more_contributors")
 205                 if more_contributors != null then
 206                         for c in more_contributors.split(",") do
 207                                 contributors.add c.trim
 208                         end
 209                 end
 210                 if not contributors.is_empty then
 211                         for c in contributors do
 212                                 register_contrib(c, mpackage)
 213                         end
 214                 end
 215                 score += contributors.length.to_f
 216
 217                 var mmodules = 0
 218                 var mclasses = 0
 219                 var mmethods = 0
 220                 var loc = 0
 221                 for g in mpackage.mgroups do
 222                         mmodules += g.mmodules.length
 223                         for m in g.mmodules do
 224                                 var am = modelbuilder.mmodule2node(m)
 225                                 if am != null then
 226                                         var file = am.location.file
 227                                         if file != null then
 228                                                 loc += file.line_starts.length - 1
 229                                         end
 230                                 end
 231                                 for cd in m.mclassdefs do
 232                                         mclasses += 1
 233                                         for pd in cd.mpropdefs do
 234                                                 if not pd isa MMethodDef then continue
 235                                                 mmethods += 1
 236                                         end
 237                                 end
 238                         end
 239                 end
 240                 self.mmodules[mpackage] = mmodules
 241                 self.mclasses[mpackage] = mclasses
 242                 self.mmethods[mpackage] = mmethods
 243                 self.loc[mpackage] = loc
 244
 245                 #score += mmodules.score
 246                 score += mclasses.score
 247                 score += mmethods.score
 248                 score += loc.score
 249
 250                 self.score[mpackage] = score.to_i
 251         end
 252
 253         # Collect more information on a package using the `git` tool.
 254         fun git_info(mpackage: MPackage)
 255         do
 256                 var ini = mpackage.ini
 257                 if ini == null then return
 258
 259                 # TODO use real git info
 260                 #var repo = ini.get_or_null("upstream.git")
 261                 #var branch = ini.get_or_null("upstream.git.branch")
 262                 #var directory = ini.get_or_null("upstream.git.directory")
 263
 264                 var dirpath = mpackage.root.filepath
 265                 if dirpath == null then return
 266
 267                 # Collect commits info
 268                 var res = git_run("log", "--no-merges", "--follow", "--pretty=tformat:%ad;%aN <%aE>", "--", dirpath)
 269                 var contributors = new Counter[String]
 270                 var commits = res.split("\n")
 271                 if commits.not_empty and commits.last == "" then commits.pop
 272                 self.commits[mpackage] = commits.length
 273                 for l in commits do
 274                         var s = l.split_once_on(';')
 275                         if s.length != 2 or s.last == "" then continue
 276
 277                         # Collect date of last and first commit
 278                         if mpackage.last_date == null then mpackage.last_date = s.first
 279                         mpackage.first_date = s.first
 280
 281                         # Count contributors
 282                         contributors.inc(s.last)
 283                 end
 284                 for c in contributors.sort.reverse_iterator do
 285                         mpackage.contributors.add c
 286                 end
 287
 288         end
 289 end
 290
 291 # Execute a git command and return the result
 292 fun git_run(command: String...): String
 293 do
 294         # print "git {command.join(" ")}"
 295         var p = new ProcessReader("git", command...)
 296         var res = p.read_all
 297         p.close
 298         p.wait
 299         return res
 300 end