1 # This file is part of NIT ( http://www.nitlanguage.org ).
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 # Basic catalog generator for Nit packages
17 # See: <http://nitlanguage.org/catalog/>
19 # The tool scans packages and generates the HTML files of a catalog.
23 # * [X] scan packages and their `.ini`
24 # * [X] generate lists of packages
25 # * [X] generate a page per package with the readme and most metadata
26 # * [ ] link/include/be included in the documentation
27 # * [ ] propose `related packages`
28 # * [X] show directory content (a la nitls)
29 # * [X] gather git information from the working directory
30 # * [ ] gather git information from the repository
31 # * [ ] gather package information from github
32 # * [ ] gather people information from github
34 # * [X] separate information gathering from rendering
35 # * [ ] move up information gathering in (existing or new) service modules
36 # * [X] add command line options
37 # * [ ] harden HTML (escaping, path injection, etc)
38 # * [ ] nitcorn server with RESTful API
40 # ## Issues and limitations
42 # The tool works likee the other tools and expects to find valid Nit source code in the directories
44 # * cruft and temporary files will be collected
45 # * missing source file (e.g. not yet generated by nitcc) will make information
46 # incomplete (e.g. invalid module thus partial dependency and metrics)
48 # How to use the tool as the basis of a Nit code archive on the web usable with a package manager is not clear.
51 import md5
# To get gravatar images
52 import counter
# For statistics
53 import modelize
# To process and count classes and methods
57 # Metadata related to this package
58 var metadata
= new MPackageMetadata(self)
61 # The metadata extracted from a MPackage
62 class MPackageMetadata
64 # The mpacakge this metadata belongs to
65 var mpackage
: MPackage
67 # Return the associated metadata from the `ini`, if any
68 fun metadata
(key
: String): nullable String do
69 var ini
= mpackage
.ini
70 if ini
== null then return null
74 # The consolidated list of tags
75 var tags
: Array[String] is lazy
do
76 var tags
= new Array[String]
77 var string
= metadata
("package.tags")
78 if string
== null then return tags
79 for tag
in string
.split
(",") do
81 if tag
.is_empty
then continue
84 if tryit
!= null then tags
.add
"tryit"
85 if apk
!= null then tags
.add
"apk"
86 if tags
.is_empty
then tags
.add
"none"
90 # The list of all maintainers
91 var maintainers
= new Array[Person]
93 # The list of contributors
94 var contributors
= new Array[Person]
96 # The date of the most recent commit
97 var last_date
: nullable String = null
99 # The date of the oldest commit
100 var first_date
: nullable String = null
102 # Key: package.maintainer`
103 var maintainer
: nullable String is lazy
do return metadata
("package.maintainer")
105 # Key: `package.more_contributors`
106 var more_contributors
: Array[String] is lazy
do
107 var res
= new Array[String]
108 var string
= metadata
("package.more_contributors")
109 if string
== null then return res
110 for c
in string
.split
(",") do
112 if c
.is_empty
then continue
118 # Key: `package.license`
119 var license
: nullable String is lazy
do return metadata
("package.license")
121 # Key: `upstream.tryit`
122 var tryit
: nullable String is lazy
do return metadata
("upstream.tryit")
124 # Key: `upstream.apk`
125 var apk
: nullable String is lazy
do return metadata
("upstream.apk")
127 # Key: `upstream.homepage`
128 var homepage
: nullable String is lazy
do return metadata
("upstream.homepage")
130 # Key: `upstream.browse`
131 var browse
: nullable String is lazy
do return metadata
("upstream.browse")
133 # Package git clone address
134 var git
: nullable String is lazy
do return metadata
("upstream.git")
136 # Package issue tracker
137 var issues
: nullable String is lazy
do return metadata
("upstream.issues")
141 # Returns `log(self+1)`. Used to compute score of packages
142 fun score
: Float do return (self+1).to_f
.log
145 # A contributor/author/etc.
147 # It comes from git or the metadata
149 # TODO get more things from github by using the email as a key
150 # "https://api.github.com/search/users?q={email}+in:email"
152 # The name. Eg "John Doe"
153 var name
: String is writable
155 # The email, Eg "john.doe@example.com"
156 var email
: nullable String is writable
158 # Some homepage. Eg "http://example.com/~jdoe"
159 var page
: nullable String is writable
162 var gravatar
: nullable String is lazy
do
163 var email
= self.email
164 if email
== null then return null
165 return email
.md5
.to_lower
168 # The standard representation of a person.
171 # var jd = new Person("John Doe", "john.doe@example.com", "http://example.com/~jdoe")
172 # assert jd.to_s == "John Doe <john.doe@example.com> (http://example.com/~jdoe)"
175 # It can be used as the input of `parse`.
178 # var jd2 = new Person.parse(jd.to_s)
179 # assert jd2.to_s == jd.to_s
184 var email
= self.email
185 if email
!= null then res
+= " <{email}>"
187 if page
!= null then res
+= " ({page})"
191 # Crete a new person from its standard textual representation.
194 # var jd = new Person.parse("John Doe <john.doe@example.com> (http://example.com/~jdoe)")
195 # assert jd.name == "John Doe"
196 # assert jd.email == "john.doe@example.com"
197 # assert jd.page == "http://example.com/~jdoe"
200 # Emails and page are optional.
203 # var jd2 = new Person.parse("John Doe")
204 # assert jd2.name == "John Doe"
205 # assert jd2.email == null
206 # assert jd2.page == null
208 init parse
(person
: String)
213 # Regular expressions are broken, need to investigate.
216 #var re = "([^<(]*?)(<([^>]*?)>)?(\\((.*)\\))?".to_re
217 #var m = (person+" ").search(re)
218 #print "{person}: `{m or else "?"}` `{m[1] or else "?"}` `{m[3] or else "?"}` `{m[5] or else "?"}`"
220 var sp1
= person
.split_once_on
("<")
221 if sp1
.length
< 2 then
224 var sp2
= sp1
.last
.split_once_on
(">")
225 if sp2
.length
< 2 then
228 name
= sp1
.first
.trim
229 email
= sp2
.first
.trim
230 var sp3
= sp2
.last
.split_once_on
("(")
231 if sp3
.length
< 2 then
234 var sp4
= sp3
.last
.split_once_on
(")")
235 if sp4
.length
< 2 then
238 page
= sp4
.first
.trim
241 init(name
, email
, page
)
246 # The main class of the calatog generator that has the knowledge
250 # used to access the files and count source lines of code
251 var modelbuilder
: ModelBuilder
253 # List of all packages by their names
254 var mpackages
= new HashMap[String, MPackage]
257 var tag2proj
= new MultiHashMap[String, MPackage]
259 # Packages by category
260 var cat2proj
= new MultiHashMap[String, MPackage]
262 # Packages by maintainer
263 var maint2proj
= new MultiHashMap[Person, MPackage]
265 # Packages by contributors
266 var contrib2proj
= new MultiHashMap[Person, MPackage]
268 # Dependency between packages
269 fun deps
: HashDigraph[MPackage] do return modelbuilder
.model
.mpackage_importation_graph
271 # Number of modules by package
272 var mmodules
= new Counter[MPackage]
274 # Number of classes by package
275 var mclasses
= new Counter[MPackage]
277 # Number of methods by package
278 var mmethods
= new Counter[MPackage]
280 # Number of line of code by package
281 var loc
= new Counter[MPackage]
284 var errors
= new Counter[MPackage]
286 # Number of warnings and advices
287 var warnings
= new Counter[MPackage]
289 # Number of warnings per 1000 lines of code (w/kloc)
290 var warnings_per_kloc
= new Counter[MPackage]
292 # Documentation score (between 0 and 100)
293 var documentation_score
= new Counter[MPackage]
295 # Number of commits by package
296 var commits
= new Counter[MPackage]
300 # The score is loosely computed using other metrics
301 var score
= new Counter[MPackage]
303 # List of known people by their git string
304 var persons
= new HashMap[String, Person]
306 # Map person short names to person objects
307 var name2person
= new HashMap[String, Person]
309 # Package statistics cache
310 var mpackages_stats
= new HashMap[MPackage, MPackageStats]
312 # Scan, register and add a contributor to a package
313 fun register_contrib
(person
: String, mpackage
: MPackage): Person
315 var p
= persons
.get_or_null
(person
)
317 var new_p
= new Person.parse
(person
)
318 # Maybe, we already have this person in fact?
319 p
= persons
.get_or_null
(new_p
.to_s
)
325 var projs
= contrib2proj
[p
]
326 if not projs
.has
(mpackage
) then
328 mpackage
.metadata
.contributors
.add p
330 name2person
[p
.name
] = p
334 # Compute information for a package
335 fun package_page
(mpackage
: MPackage)
337 mpackages
[mpackage
.full_name
] = mpackage
339 var score
= score
[mpackage
].to_f
341 var mdoc
= mpackage
.mdoc_or_fallback
344 score
+= mdoc
.content
.length
.score
346 var metadata
= mpackage
.metadata
348 var tryit
= metadata
.tryit
349 if tryit
!= null then
352 var apk
= metadata
.apk
356 var homepage
= metadata
.homepage
357 if homepage
!= null then
360 var maintainer
= metadata
.maintainer
361 if maintainer
!= null then
363 var person
= register_contrib
(maintainer
, mpackage
)
364 mpackage
.metadata
.maintainers
.add person
365 var projs
= maint2proj
[person
]
366 if not projs
.has
(mpackage
) then projs
.add mpackage
368 var license
= metadata
.license
369 if license
!= null then
372 var browse
= metadata
.browse
373 if browse
!= null then
376 var tags
= metadata
.tags
378 tag2proj
[tag
].add mpackage
380 if tags
.not_empty
then
382 cat2proj
[cat
].add mpackage
383 score
+= tags
.length
.score
385 if deps
.has_vertex
(mpackage
) then
386 score
+= deps
.predecessors
(mpackage
).length
.score
387 score
+= deps
.get_all_predecessors
(mpackage
).length
.score
388 score
+= deps
.successors
(mpackage
).length
.score
389 score
+= deps
.get_all_successors
(mpackage
).length
.score
392 var contributors
= mpackage
.metadata
.contributors
393 var more_contributors
= metadata
.more_contributors
394 for c
in more_contributors
do
395 register_contrib
(c
, mpackage
)
397 score
+= contributors
.length
.to_f
404 # The documentation value of each entity is ad hoc.
405 var entity_score
= 0.0
407 for g
in mpackage
.mgroups
do
408 mmodules
+= g
.mmodules
.length
411 if g
.mdoc
!= null then doc_score
+= gs
412 for m
in g
.mmodules
do
413 var source
= m
.location
.file
414 if source
!= null then
415 for msg
in source
.messages
do
416 if msg
.level
== 2 then
423 var am
= modelbuilder
.mmodule2node
(m
)
425 var file
= am
.location
.file
427 loc
+= file
.line_starts
.length
- 1
431 if m
.is_test
then ms
/= 100.0
433 if m
.mdoc
!= null then doc_score
+= ms
else ms
/= 10.0
434 for cd
in m
.mclassdefs
do
436 if not cd
.is_intro
then cs
/= 100.0
437 if not cd
.mclass
.visibility
<= private_visibility
then cs
/= 100.0
439 if cd
.mdoc
!= null then doc_score
+= cs
441 for pd
in cd
.mpropdefs
do
443 if not pd
.is_intro
then ps
/= 100.0
444 if not pd
.mproperty
.visibility
<= private_visibility
then ps
/= 100.0
446 if pd
.mdoc
!= null then doc_score
+= ps
447 if not pd
isa MMethodDef then continue
453 self.mmodules
[mpackage
] = mmodules
454 self.mclasses
[mpackage
] = mclasses
455 self.mmethods
[mpackage
] = mmethods
456 self.loc
[mpackage
] = loc
457 self.errors
[mpackage
] = errors
458 self.warnings
[mpackage
] = warnings
460 self.warnings_per_kloc
[mpackage
] = warnings
* 1000 / loc
462 var documentation_score
= (100.0 * doc_score
/ entity_score
).to_i
463 self.documentation_score
[mpackage
] = documentation_score
464 #score += mmodules.score
465 score
+= mclasses
.score
466 score
+= mmethods
.score
468 score
+= documentation_score
.score
470 self.score
[mpackage
] = score
.to_i
473 # Collect more information on a package using the `git` tool.
474 fun git_info
(mpackage
: MPackage)
476 var ini
= mpackage
.ini
477 if ini
== null then return
479 var root
= mpackage
.root
480 if root
== null then return
482 # TODO use real git info
483 #var repo = ini.get_or_null("upstream.git")
484 #var branch = ini.get_or_null("upstream.git.branch")
485 #var directory = ini.get_or_null("upstream.git.directory")
487 var dirpath
= root
.filepath
488 if dirpath
== null then return
490 # Collect commits info
491 var res
= git_run
("log", "--no-merges", "--follow", "--pretty=tformat:%ad;%aN <%aE>", "--", dirpath
)
492 var contributors
= new Counter[String]
493 var commits
= res
.split
("\n")
494 if commits
.not_empty
and commits
.last
== "" then commits
.pop
495 self.commits
[mpackage
] = commits
.length
497 var s
= l
.split_once_on
(';')
498 if s
.length
!= 2 or s
.last
== "" then continue
500 # Collect date of last and first commit
501 if mpackage
.metadata
.last_date
== null then mpackage
.metadata
.last_date
= s
.first
502 mpackage
.metadata
.first_date
= s
.first
505 contributors
.inc
(s
.last
)
507 for c
in contributors
.sort
.reverse_iterator
do
508 register_contrib
(c
, mpackage
)
512 # Compose package stats
513 fun mpackage_stats
(mpackage
: MPackage): MPackageStats do
514 var stats
= new MPackageStats
515 stats
.mmodules
= mmodules
[mpackage
]
516 stats
.mclasses
= mclasses
[mpackage
]
517 stats
.mmethods
= mmethods
[mpackage
]
518 stats
.loc
= loc
[mpackage
]
519 stats
.errors
= errors
[mpackage
]
520 stats
.warnings
= warnings
[mpackage
]
521 stats
.warnings_per_kloc
= warnings_per_kloc
[mpackage
]
522 stats
.documentation_score
= documentation_score
[mpackage
]
523 stats
.commits
= commits
[mpackage
]
524 stats
.score
= score
[mpackage
]
526 mpackages_stats
[mpackage
] = stats
530 # Compose catalog stats
531 var catalog_stats
: CatalogStats is lazy
do
532 var stats
= new CatalogStats
533 stats
.packages
= mpackages
.length
534 stats
.maintainers
= maint2proj
.length
535 stats
.contributors
= contrib2proj
.length
536 stats
.tags
= tag2proj
.length
537 stats
.modules
= mmodules
.sum
538 stats
.classes
= mclasses
.sum
539 stats
.methods
= mmethods
.sum
551 # Number of maintainers
554 # Number of contributors
569 # Number of line of codes
572 # Return the stats as a Map associating each stat key to its value
573 fun to_map
: Map[String, Int] do
574 var map
= new HashMap[String, Int]
575 map
["packages"] = packages
576 map
["maintainers"] = maintainers
577 map
["contributors"] = contributors
579 map
["modules"] = modules
580 map
["classes"] = classes
581 map
["methods"] = methods
587 # MPackage statistics for the catalog
599 # Number of lines of code
605 # Number of warnings and advices
608 # Number of warnings per 1000 lines of code (w/kloc)
609 var warnings_per_kloc
= 0
611 # Documentation score (between 0 and 100)
612 var documentation_score
= 0
614 # Number of commits by package
619 # The score is loosely computed using other metrics
623 # Sort the mpackages by their score
624 class CatalogScoreSorter
627 # Catalog used to access scores
630 redef type COMPARED: MPackage
632 redef fun compare
(a
, b
) do
633 if not catalog
.mpackages_stats
.has_key
(a
) then return 1
634 if not catalog
.mpackages_stats
.has_key
(b
) then return -1
635 var astats
= catalog
.mpackages_stats
[a
]
636 var bstats
= catalog
.mpackages_stats
[b
]
637 return bstats
.score
<=> astats
.score
641 # Sort tabs alphabetically
642 class CatalogTagsSorter
645 redef type COMPARED: String
647 redef fun compare
(a
, b
) do return a
<=> b
650 # Execute a git command and return the result
651 fun git_run
(command
: String...): String
653 # print "git {command.join(" ")}"
654 var p
= new ProcessReader("git", command
...)