1 # This file is part of NIT ( http://www.nitlanguage.org ).
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 # Basic catalog generator for Nit packages
17 # See: <http://nitlanguage.org/catalog/>
19 # The tool scans packages and generates the HTML files of a catalog.
23 # * [X] scan packages and their `.ini`
24 # * [X] generate lists of packages
25 # * [X] generate a page per package with the readme and most metadata
26 # * [ ] link/include/be included in the documentation
27 # * [ ] propose `related packages`
28 # * [X] show directory content (a la nitls)
29 # * [X] gather git information from the working directory
30 # * [ ] gather git information from the repository
31 # * [ ] gather package information from github
32 # * [ ] gather people information from github
34 # * [X] separate information gathering from rendering
35 # * [ ] move up information gathering in (existing or new) service modules
36 # * [X] add command line options
37 # * [ ] harden HTML (escaping, path injection, etc)
38 # * [ ] nitcorn server with RESTful API
40 # ## Issues and limitations
42 # The tool works likee the other tools and expects to find valid Nit source code in the directories
44 # * cruft and temporary files will be collected
45 # * missing source file (e.g. not yet generated by nitcc) will make information
46 # incomplete (e.g. invalid module thus partial dependency and metrics)
48 # How to use the tool as the basis of a Nit code archive on the web usable with a package manager is not clear.
51 import md5
# To get gravatar images
52 import counter
# For statistics
53 import modelize
# To process and count classes and methods
56 # Return the associated metadata from the `ini`, if any
57 fun metadata
(key
: String): nullable String
60 if ini
== null then return null
64 # The consolidated list of tags
65 var tags
= new Array[String]
67 # The list of maintainers
68 var maintainers
= new Array[Person]
70 # The list of contributors
71 var contributors
= new Array[Person]
73 # The date of the most recent commit
74 var last_date
: nullable String = null
76 # The date of the oldest commit
77 var first_date
: nullable String = null
81 # Returns `log(self+1)`. Used to compute score of packages
82 fun score
: Float do return (self+1).to_f
.log
85 # A contributor/author/etc.
87 # It comes from git or the metadata
89 # TODO get more things from github by using the email as a key
90 # "https://api.github.com/search/users?q={email}+in:email"
92 # The name. Eg "John Doe"
93 var name
: String is writable
95 # The email, Eg "john.doe@example.com"
96 var email
: nullable String is writable
98 # Some homepage. Eg "http://example.com/~jdoe"
99 var page
: nullable String is writable
101 # Return a full-featured link to a person
105 var e
= name
.html_escape
108 res
+= "<a href=\"{page.html_escape}\
">"
110 var email
= self.email
111 if email
!= null then
112 var md5
= email
.md5
.to_lower
113 res
+= "<img src=\"https
://secure
.gravatar
.com
/avatar
/{md5}?size
=20&
;default
=retro\
"> "
116 if page
!= null then res
+= "</a>"
120 # The standard representation of a person.
123 # var jd = new Person("John Doe", "john.doe@example.com", "http://example.com/~jdoe")
124 # assert jd.to_s == "John Doe <john.doe@example.com> (http://example.com/~jdoe)"
127 # It can be used as the input of `parse`.
130 # var jd2 = new Person.parse(jd.to_s)
131 # assert jd2.to_s == jd.to_s
136 var email
= self.email
137 if email
!= null then res
+= " <{email}>"
139 if page
!= null then res
+= " ({page})"
143 # Crete a new person from its standard textual representation.
146 # var jd = new Person.parse("John Doe <john.doe@example.com> (http://example.com/~jdoe)")
147 # assert jd.name == "John Doe"
148 # assert jd.email == "john.doe@example.com"
149 # assert jd.page == "http://example.com/~jdoe"
152 # Emails and page are optional.
155 # var jd2 = new Person.parse("John Doe")
156 # assert jd2.name == "John Doe"
157 # assert jd2.email == null
158 # assert jd2.page == null
160 init parse
(person
: String)
165 # Regular expressions are broken, need to investigate.
168 #var re = "([^<(]*?)(<([^>]*?)>)?(\\((.*)\\))?".to_re
169 #var m = (person+" ").search(re)
170 #print "{person}: `{m or else "?"}` `{m[1] or else "?"}` `{m[3] or else "?"}` `{m[5] or else "?"}`"
172 var sp1
= person
.split_once_on
("<")
173 if sp1
.length
< 2 then
176 var sp2
= sp1
.last
.split_once_on
(">")
177 if sp2
.length
< 2 then
180 name
= sp1
.first
.trim
181 email
= sp2
.first
.trim
182 var sp3
= sp2
.last
.split_once_on
("(")
183 if sp3
.length
< 2 then
186 var sp4
= sp3
.last
.split_once_on
(")")
187 if sp4
.length
< 2 then
190 page
= sp4
.first
.trim
193 init(name
, email
, page
)
198 # The main class of the calatog generator that has the knowledge
202 # used to access the files and count source lines of code
203 var modelbuilder
: ModelBuilder
206 var tag2proj
= new MultiHashMap[String, MPackage]
208 # Packages by category
209 var cat2proj
= new MultiHashMap[String, MPackage]
211 # Packages by maintainer
212 var maint2proj
= new MultiHashMap[Person, MPackage]
214 # Packages by contributors
215 var contrib2proj
= new MultiHashMap[Person, MPackage]
217 # Dependency between packages
218 var deps
= new POSet[MPackage]
220 # Number of modules by package
221 var mmodules
= new Counter[MPackage]
223 # Number of classes by package
224 var mclasses
= new Counter[MPackage]
226 # Number of methods by package
227 var mmethods
= new Counter[MPackage]
229 # Number of line of code by package
230 var loc
= new Counter[MPackage]
233 var errors
= new Counter[MPackage]
235 # Number of warnings and advices
236 var warnings
= new Counter[MPackage]
238 # Number of warnings per 1000 lines of code (w/kloc)
239 var warnings_per_kloc
= new Counter[MPackage]
241 # Documentation score (between 0 and 100)
242 var documentation_score
= new Counter[MPackage]
244 # Number of commits by package
245 var commits
= new Counter[MPackage]
249 # The score is loosely computed using other metrics
250 var score
= new Counter[MPackage]
252 # List of known people
253 var persons
= new HashMap[String, Person]
255 # Scan, register and add a contributor to a package
256 fun register_contrib
(person
: String, mpackage
: MPackage): Person
258 var p
= persons
.get_or_null
(person
)
260 var new_p
= new Person.parse
(person
)
261 # Maybe, we already have this person in fact?
262 p
= persons
.get_or_null
(new_p
.to_s
)
268 var projs
= contrib2proj
[p
]
269 if not projs
.has
(mpackage
) then
271 mpackage
.contributors
.add p
276 # Compute information for a package
277 fun package_page
(mpackage
: MPackage)
279 var score
= score
[mpackage
].to_f
281 var mdoc
= mpackage
.mdoc_or_fallback
284 score
+= mdoc
.content
.length
.score
288 var tryit
= mpackage
.metadata
("upstream.tryit")
289 if tryit
!= null then
292 var apk
= mpackage
.metadata
("upstream.apk")
297 var homepage
= mpackage
.metadata
("upstream.homepage")
298 if homepage
!= null then
301 var maintainer
= mpackage
.metadata
("package.maintainer")
302 if maintainer
!= null then
304 var person
= register_contrib
(maintainer
, mpackage
)
305 mpackage
.maintainers
.add person
306 var projs
= maint2proj
[person
]
307 if not projs
.has
(mpackage
) then projs
.add mpackage
309 var license
= mpackage
.metadata
("package.license")
310 if license
!= null then
314 var browse
= mpackage
.metadata
("upstream.browse")
315 if browse
!= null then
319 var tags
= mpackage
.metadata
("package.tags")
320 var ts
= mpackage
.tags
322 for t
in tags
.split
(",") do
324 if t
== "" then continue
328 if ts
.is_empty
then ts
.add
"none"
329 if tryit
!= null then ts
.add
"tryit"
330 if apk
!= null then ts
.add
"apk"
332 tag2proj
[t
].add mpackage
335 cat2proj
[cat
].add mpackage
336 score
+= ts
.length
.score
338 if deps
.has
(mpackage
) then
339 score
+= deps
[mpackage
].greaters
.length
.score
340 score
+= deps
[mpackage
].direct_greaters
.length
.score
341 score
+= deps
[mpackage
].smallers
.length
.score
342 score
+= deps
[mpackage
].direct_smallers
.length
.score
345 var contributors
= mpackage
.contributors
346 var more_contributors
= mpackage
.metadata
("package.more_contributors")
347 if more_contributors
!= null then
348 for c
in more_contributors
.split
(",") do
349 register_contrib
(c
.trim
, mpackage
)
352 score
+= contributors
.length
.to_f
360 # The documentation value of each entity is ad hoc.
361 var entity_score
= 0.0
363 for g
in mpackage
.mgroups
do
364 mmodules
+= g
.mmodules
.length
367 if g
.mdoc
!= null then doc_score
+= gs
368 for m
in g
.mmodules
do
369 var source
= m
.location
.file
370 if source
!= null then
371 for msg
in source
.messages
do
372 if msg
.level
== 2 then
379 var am
= modelbuilder
.mmodule2node
(m
)
381 var file
= am
.location
.file
383 loc
+= file
.line_starts
.length
- 1
387 if m
.is_test_suite
then ms
/= 100.0
389 if m
.mdoc
!= null then doc_score
+= ms
else ms
/= 10.0
390 for cd
in m
.mclassdefs
do
392 if not cd
.is_intro
then cs
/= 100.0
393 if not cd
.mclass
.visibility
<= private_visibility
then cs
/= 100.0
395 if cd
.mdoc
!= null then doc_score
+= cs
397 for pd
in cd
.mpropdefs
do
399 if not pd
.is_intro
then ps
/= 100.0
400 if not pd
.mproperty
.visibility
<= private_visibility
then ps
/= 100.0
402 if pd
.mdoc
!= null then doc_score
+= ps
403 if not pd
isa MMethodDef then continue
409 self.mmodules
[mpackage
] = mmodules
410 self.mclasses
[mpackage
] = mclasses
411 self.mmethods
[mpackage
] = mmethods
412 self.loc
[mpackage
] = loc
413 self.errors
[mpackage
] = errors
414 self.warnings
[mpackage
] = warnings
416 self.warnings_per_kloc
[mpackage
] = warnings
* 1000 / loc
418 var documentation_score
= (100.0 * doc_score
/ entity_score
).to_i
419 self.documentation_score
[mpackage
] = documentation_score
421 #score += mmodules.score
422 score
+= mclasses
.score
423 score
+= mmethods
.score
425 score
+= documentation_score
.score
427 self.score
[mpackage
] = score
.to_i
430 # Collect more information on a package using the `git` tool.
431 fun git_info
(mpackage
: MPackage)
433 var ini
= mpackage
.ini
434 if ini
== null then return
436 # TODO use real git info
437 #var repo = ini.get_or_null("upstream.git")
438 #var branch = ini.get_or_null("upstream.git.branch")
439 #var directory = ini.get_or_null("upstream.git.directory")
441 var dirpath
= mpackage
.root
.filepath
442 if dirpath
== null then return
444 # Collect commits info
445 var res
= git_run
("log", "--no-merges", "--follow", "--pretty=tformat:%ad;%aN <%aE>", "--", dirpath
)
446 var contributors
= new Counter[String]
447 var commits
= res
.split
("\n")
448 if commits
.not_empty
and commits
.last
== "" then commits
.pop
449 self.commits
[mpackage
] = commits
.length
451 var s
= l
.split_once_on
(';')
452 if s
.length
!= 2 or s
.last
== "" then continue
454 # Collect date of last and first commit
455 if mpackage
.last_date
== null then mpackage
.last_date
= s
.first
456 mpackage
.first_date
= s
.first
459 contributors
.inc
(s
.last
)
461 for c
in contributors
.sort
.reverse_iterator
do
462 register_contrib
(c
, mpackage
)
468 # Execute a git command and return the result
469 fun git_run
(command
: String...): String
471 # print "git {command.join(" ")}"
472 var p
= new ProcessReader("git", command
...)