1 # This file is part of NIT ( http://www.nitlanguage.org ).
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 # Basic catalog generator for Nit projects
17 # See: <http://nitlanguage.org/catalog/>
19 # The tool scans projects and generates the HTML files of a catalog.
23 # * [X] scan projects and their `.ini`
24 # * [X] generate lists of projects
25 # * [X] generate a page per project with the readme and most metadata
26 # * [ ] link/include/be included in the documentation
27 # * [ ] propose `related projects`
28 # * [ ] show directory content (a la nitls)
29 # * [X] gather git information from the working directory
30 # * [ ] gather git information from the repository
31 # * [ ] gather project information from github
32 # * [ ] gather people information from github
34 # * [ ] separate information gathering from rendering
35 # * [ ] move up information gathering in (existing or new) service modules
36 # * [ ] add command line options
37 # * [ ] harden HTML (escaping, path injection, etc)
38 # * [ ] nitcorn server with RESTful API
40 # ## Issues and limitations
42 # The tool works likee the other tools and expects to find valid Nit source code in the directories
44 # * cruft and temporary files will be collected
45 # * missing source file (e.g. not yet generated by nitcc) will make information
46 # incomplete (e.g. invalid module thus partial dependency and metrics)
48 # How to use the tool as the basis of a Nit code archive on the web usable with a package manager is not clear.
51 import loader
# Scan&load projects, groups and modules
52 import doc
::doc_down
# Display mdoc
53 import md5
# To get gravatar images
54 import counter
# For statistics
55 import modelize
# To process and count classes and methods
58 # Return the associated metadata from the `ini`, if any
59 fun metadata
(key
: String): nullable String
62 if ini
== null then return null
66 # The list of maintainers
67 var maintainers
= new Array[String]
69 # The list of contributors
70 var contributors
= new Array[String]
72 # The date of the most recent commit
73 var last_date
: nullable String = null
75 # The date of the oldest commit
76 var first_date
: nullable String = null
79 # A HTML page in a catalog
81 # This is just a template with the header pre-filled and the footer injected at rendering.
82 # Therefore, once instantiated, the content can just be added to it.
86 # Placeholder to include additional things before the `</head>`.
87 var more_head
= new Template
95 <meta charset="utf-8">
96 <link rel="stylesheet" media="all" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap.min.css">
97 <link rel="stylesheet" media="all" href="style.css">
104 <div class='container-fluid'>
106 <nav id='topmenu' class='navbar navbar-default navbar-fixed-top' role='navigation'>
107 <div class='container-fluid'>
108 <div class='navbar-header'>
109 <button type='button' class='navbar-toggle' data-toggle='collapse' data-target='#topmenu-collapse'>
110 <span class='sr-only'>Toggle menu</span>
111 <span class='icon-bar'></span>
112 <span class='icon-bar'></span>
113 <span class='icon-bar'></span>
115 <span class='navbar-brand'><a href="http://nitlanguage.org/">Nitlanguage.org</a></span>
117 <div class='collapse navbar-collapse' id='topmenu-collapse'>
118 <ul class='nav navbar-nav'>
119 <li><a href="index.html">Catalog</a></li>
131 </div> <!-- container-fluid -->
132 <script src='https://code.jquery.com/jquery-latest.min.js'></script>
133 <script src='https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/js/bootstrap.min.js'></script>
134 <script src='https://cdnjs.cloudflare.com/ajax/libs/bootstrap-table/1.8.1/bootstrap-table-all.min.js'></script>
142 # Returns `log(self+1)`. Used to compute score of projects
143 fun score
: Float do return (self+1).to_f
.log
146 # The main class of the calatog generator that has the knowledge
150 # used to access the files and count source lines of code
151 var modelbuilder
: ModelBuilder
154 var tag2proj
= new MultiHashMap[String, MProject]
156 # Projects by category
157 var cat2proj
= new MultiHashMap[String, MProject]
159 # Projects by maintainer
160 var maint2proj
= new MultiHashMap[String, MProject]
162 # Projects by contributors
163 var contrib2proj
= new MultiHashMap[String, MProject]
165 # Dependency between projects
166 var deps
= new POSet[MProject]
168 # Number of modules by project
169 var mmodules
= new Counter[MProject]
171 # Number of classes by project
172 var mclasses
= new Counter[MProject]
174 # Number of methods by project
175 var mmethods
= new Counter[MProject]
177 # Number of line of code by project
178 var loc
= new Counter[MProject]
180 # Number of commits by project
181 var commits
= new Counter[MProject]
185 # The score is loosely computed using other metrics
186 var score
= new Counter[MProject]
188 # Scan, register and add a contributor to a project
189 fun add_contrib
(person
: String, mproject
: MProject, res
: Template)
191 var projs
= contrib2proj
[person
]
192 if not projs
.has
(mproject
) then projs
.add mproject
197 # Regular expressions are broken, need to investigate.
200 #var re = "([^<(]*?)(<([^>]*?)>)?(\\((.*)\\))?".to_re
201 #var m = (person+" ").search(re)
202 #print "{person}: `{m or else "?"}` `{m[1] or else "?"}` `{m[3] or else "?"}` `{m[5] or else "?"}`"
204 var sp1
= person
.split_once_on
("<")
205 if sp1
.length
< 2 then
208 var sp2
= sp1
.last
.split_once_on
(">")
209 if sp2
.length
< 2 then
212 name
= sp1
.first
.trim
213 email
= sp2
.first
.trim
214 var sp3
= sp2
.last
.split_once_on
("(")
215 if sp3
.length
< 2 then
218 var sp4
= sp3
.last
.split_once_on
(")")
219 if sp4
.length
< 2 then
222 page
= sp4
.first
.trim
225 var e
= name
.html_escape
228 res
.add
"<a href=\"{page.html_escape}\
">"
230 if email
!= null then
231 # TODO get more things from github by using the email as a key
232 # "https://api.github.com/search/users?q={email}+in:email"
233 var md5
= email
.md5
.to_lower
234 res
.add
"<img src=\"https
://secure
.gravatar
.com
/avatar
/{md5}?size
=20&
;default
=retro\
"> "
237 if page
!= null then res
.add
"</a>"
242 # Compute information and generate a full HTML page for a project
243 fun project_page
(mproject
: MProject): Writable
245 var res
= new CatalogPage
246 var score
= score
[mproject
].to_f
247 var name
= mproject
.name
.html_escape
248 res
.more_head
.add
"""<title>{{{name}}}</title>"""
251 <div class="content">
252 <h1 class="package-name">{{{name}}}</h1>
254 var mdoc
= mproject
.mdoc_or_fallback
257 res
.add mdoc
.html_documentation
258 score
+= mdoc
.content
.length
.score
262 <div class="sidebar">
265 var homepage
= mproject
.metadata
("upstream.homepage")
266 if homepage
!= null then
268 var e
= homepage
.html_escape
269 res
.add
"<li><a href=\"{e}\
">{e}</a></li>\n"
271 var maintainer
= mproject
.metadata
("project.maintainer")
272 if maintainer
!= null then
274 add_contrib
(maintainer
, mproject
, res
)
275 mproject
.maintainers
.add maintainer
276 var projs
= maint2proj
[maintainer
]
277 if not projs
.has
(mproject
) then projs
.add mproject
279 var license
= mproject
.metadata
("project.license")
280 if license
!= null then
282 var e
= license
.html_escape
283 res
.add
"<li><a href=\"http
://opensource
.org
/licenses
/{e}\
">{e}</a> license</li>\n"
287 res
.add
"<h3>Source Code</h3>\n<ul class=\"box\
">\n"
288 var browse
= mproject
.metadata
("upstream.browse")
289 if browse
!= null then
291 var e
= browse
.html_escape
292 res
.add
"<li><a href=\"{e}\
">{e}</a></li>\n"
294 var git
= mproject
.metadata
("upstream.git")
296 var e
= git
.html_escape
297 res
.add
"<li><tt>{e}</tt></li>\n"
299 var last_date
= mproject
.last_date
300 if last_date
!= null then
301 var e
= last_date
.html_escape
302 res
.add
"<li>most recent commit: {e}</li>\n"
304 var first_date
= mproject
.first_date
305 if first_date
!= null then
306 var e
= first_date
.html_escape
307 res
.add
"<li>oldest commit: {e}</li>\n"
309 var commits
= commits
[mproject
]
311 res
.add
"<li>{commits} commits</li>\n"
315 res
.add
"<h3>Tags</h3>\n"
316 var tags
= mproject
.metadata
("project.tags")
317 var ts2
= new Array[String]
320 var ts
= tags
.split
(",")
323 if t
== "" then continue
324 if cat
== null then cat
= t
325 tag2proj
[t
].add mproject
327 ts2
.add
"<a href=\"index
.html
#tag_{t}\">{t}</a>"
329 res
.add_list
(ts2
, ", ", ", ")
334 tag2proj
[t
].add mproject
335 res
.add
"<a href=\"index
.html
#tag_{t}\">{t}</a>"
337 if cat
!= null then cat2proj
[cat
].add mproject
338 score
+= ts2
.length
.score
340 var reqs
= deps
[mproject
].greaters
.to_a
341 reqs
.remove
(mproject
)
342 alpha_comparator
.sort
(reqs
)
343 res
.add
"<h3>Requirements</h3>\n"
344 if reqs
.is_empty
then
347 var list
= new Array[String]
349 var direct
= deps
.has_direct_edge
(mproject
, r
)
350 var s
= "<a href=\"{r}.html\
">"
351 if direct
then s
+= "<strong>"
353 if direct
then s
+= "</strong>"
357 res
.add_list
(list
, ", ", " and ")
360 reqs
= deps
[mproject
].smallers
.to_a
361 reqs
.remove
(mproject
)
362 alpha_comparator
.sort
(reqs
)
363 res
.add
"<h3>Clients</h3>\n"
364 if reqs
.is_empty
then
367 var list
= new Array[String]
369 var direct
= deps
.has_direct_edge
(r
, mproject
)
370 var s
= "<a href=\"{r}.html\
">"
371 if direct
then s
+= "<strong>"
373 if direct
then s
+= "</strong>"
377 res
.add_list
(list
, ", ", " and ")
380 score
+= deps
[mproject
].greaters
.length
.score
381 score
+= deps
[mproject
].direct_greaters
.length
.score
382 score
+= deps
[mproject
].smallers
.length
.score
383 score
+= deps
[mproject
].direct_smallers
.length
.score
385 var contributors
= mproject
.contributors
386 if not contributors
.is_empty
then
387 res
.add
"<h3>Contributors</h3>\n<ul class=\"box\
">"
388 for c
in contributors
do
389 add_contrib
(c
, mproject
, res
)
393 score
+= contributors
.length
.to_f
399 for g
in mproject
.mgroups
do
400 mmodules
+= g
.module_paths
.length
401 for m
in g
.mmodules
do
402 var am
= modelbuilder
.mmodule2node
(m
)
404 var file
= am
.location
.file
406 loc
+= file
.line_starts
.length
- 1
409 for cd
in m
.mclassdefs
do
411 for pd
in cd
.mpropdefs
do
412 if not pd
isa MMethodDef then continue
418 self.mmodules
[mproject
] = mmodules
419 self.mclasses
[mproject
] = mclasses
420 self.mmethods
[mproject
] = mmethods
421 self.loc
[mproject
] = loc
423 #score += mmodules.score
424 score
+= mclasses
.score
425 score
+= mmethods
.score
431 <li>{{{mmodules}}} modules</li>
432 <li>{{{mclasses}}} classes</li>
433 <li>{{{mmethods}}} methods</li>
434 <li>{{{loc}}} lines of code</li>
441 self.score
[mproject
] = score
.to_i
446 # Return a short HTML sequence for a project
448 # Intended to use in lists.
449 fun li_project
(p
: MProject): String
452 var f
= "{p.name}.html"
453 res
+= "<a href=\"{f}\
">{p}</a>"
454 var d
= p
.mdoc_or_fallback
455 if d
!= null then res
+= " - {d.html_synopsis.write_to_string}"
459 # List projects by group.
461 # For each key of the `map` a `<h3>` is generated.
462 # Each project is then listed.
464 # The list of keys is generated first to allow fast access to the correct `<h3>`.
465 # `id_prefix` is used to give an id to the `<h3>` element.
466 fun list_by
(map
: MultiHashMap[String, MProject], id_prefix
: String): Template
468 var res
= new Template
469 var keys
= map
.keys
.to_a
470 alpha_comparator
.sort
(keys
)
471 var list
= [for x
in keys
do "<a href=\"#{id_prefix}{x.html_escape}\">{x.html_escape}</a>"]
472 res
.add_list
(list
, ", ", " and ")
475 var projs
= map
[k
].to_a
476 alpha_comparator
.sort
(projs
)
477 var e
= k
.html_escape
478 res
.add
"<h3 id=\"{id_prefix}{e}\
">{e} ({projs.length})</h3>\n<ul>\n"
481 res
.add li_project
(p
)
489 # List the 10 best projects from `cpt`
490 fun list_best
(cpt
: Counter[MProject]): Template
492 var res
= new Template
496 if i
> best
.length
then break
497 var p
= best
[best
.length-i
]
499 res
.add li_project
(p
)
500 # res.add " ({cpt[p]})"
507 # Collect more information on a project using the `git` tool.
508 fun git_info
(mproject
: MProject)
510 var ini
= mproject
.ini
511 if ini
== null then return
513 # TODO use real git info
514 #var repo = ini.get_or_null("upstream.git")
515 #var branch = ini.get_or_null("upstream.git.branch")
516 #var directory = ini.get_or_null("upstream.git.directory")
518 var dirpath
= mproject
.root
.filepath
519 if dirpath
== null then return
521 # Collect commits info
522 var res
= git_run
("log", "--no-merges", "--follow", "--pretty=tformat:%ad;%aN <%aE>", "--", dirpath
)
523 var contributors
= new Counter[String]
524 var commits
= res
.split
("\n")
525 if commits
.not_empty
and commits
.last
== "" then commits
.pop
526 self.commits
[mproject
] = commits
.length
528 var s
= l
.split_once_on
(';')
529 if s
.length
!= 2 or s
.last
== "" then continue
531 # Collect date of last and first commit
532 if mproject
.last_date
== null then mproject
.last_date
= s
.first
533 mproject
.first_date
= s
.first
536 contributors
.inc
(s
.last
)
538 for c
in contributors
.sort
.reverse_iterator
do
539 mproject
.contributors
.add c
544 # Produce a HTML table containig information on the projects
546 # `project_page` must have been called before so that information is computed.
547 fun table_projects
(mprojects
: Array[MProject]): Template
549 alpha_comparator
.sort
(mprojects
)
550 var res
= new Template
551 res
.add
"<table data-toggle=\"table\
" data-sort-name=\"name\
" data-sort-order=\"desc\
" width=\"100%\
">\n"
552 res
.add
"<thead><tr>\n"
553 res
.add
"<th data-field=\"name\
" data-sortable=\"true\
">name</th>\n"
554 res
.add
"<th data-field=\"maint\
" data-sortable=\"true\
">maint</th>\n"
555 res
.add
"<th data-field=\"contrib\
" data-sortable=\"true\
">contrib</th>\n"
556 res
.add
"<th data-field=\"reqs\
" data-sortable=\"true\
">reqs</th>\n"
557 res
.add
"<th data-field=\"dreqs\
" data-sortable=\"true\
">direct<br>reqs</th>\n"
558 res
.add
"<th data-field=\"cli\
" data-sortable=\"true\
">clients</th>\n"
559 res
.add
"<th data-field=\"dcli\
" data-sortable=\"true\
">direct<br>clients</th>\n"
560 res
.add
"<th data-field=\"mod\
" data-sortable=\"true\
">modules</th>\n"
561 res
.add
"<th data-field=\"cla\
" data-sortable=\"true\
">classes</th>\n"
562 res
.add
"<th data-field=\"met\
" data-sortable=\"true\
">methods</th>\n"
563 res
.add
"<th data-field=\"loc\
" data-sortable=\"true\
">lines</th>\n"
564 res
.add
"<th data-field=\"score\
" data-sortable=\"true\
">score</th>\n"
565 res
.add
"</tr></thead>"
566 for p
in mprojects
do
568 res
.add
"<td><a href=\"{p.name}.html\
">{p.name}</a></td>"
570 if p
.maintainers
.not_empty
then maint
= p
.maintainers
.first
571 res
.add
"<td>{maint}</td>"
572 res
.add
"<td>{p.contributors.length}</td>"
573 res
.add
"<td>{deps[p].greaters.length-1}</td>"
574 res
.add
"<td>{deps[p].direct_greaters.length}</td>"
575 res
.add
"<td>{deps[p].smallers.length-1}</td>"
576 res
.add
"<td>{deps[p].direct_smallers.length}</td>"
577 res
.add
"<td>{mmodules[p]}</td>"
578 res
.add
"<td>{mclasses[p]}</td>"
579 res
.add
"<td>{mmethods[p]}</td>"
580 res
.add
"<td>{loc[p]}</td>"
581 res
.add
"<td>{score[p]}</td>"
589 # Execute a git command and return the result
590 fun git_run
(command
: String...): String
592 # print "git {command.join(" ")}"
593 var p
= new ProcessReader("git", command
...)
600 var model
= new Model
601 var tc
= new ToolContext
603 tc
.process_options
(sys
.args
)
606 var modelbuilder
= new ModelBuilder(model
, tc
)
607 var catalog
= new Catalog(modelbuilder
)
609 # Get files or groups
610 for a
in tc
.option_context
.rest
do
611 modelbuilder
.get_mgroup
(a
)
612 modelbuilder
.identify_file
(a
)
615 # Scan projects and compute information
616 for p
in model
.mprojects
do
619 modelbuilder
.scan_group
(g
)
621 # Load the module to process importation information
622 modelbuilder
.parse_group
(g
)
624 catalog
.deps
.add_node
(p
)
625 for gg
in p
.mgroups
do for m
in gg
.mmodules
do
626 for im
in m
.in_importation
.direct_greaters
do
628 if ip
== null or ip
== p
then continue
629 catalog
.deps
.add_edge
(p
, ip
)
636 # Run phases to modelize classes and properties (so we can count them)
637 #modelbuilder.run_phases
642 # Generate the css (hard coded)
646 background-color: #f8f8f8;
651 text-decoration: none;
656 text-decoration: none;
668 border-bottom: solid 3px #CCC;
674 border-bottom: solid 1px #CCC;
678 list-style-type: square;
688 border: 1px solid #CCC;
689 font-family: Monospace;
691 background-color: rgb(250, 250, 250);
695 font-family: Monospace;
735 text-overflow: ellipsis;
737 border-bottom: 1px solid rgba(0,0,0,0.2);
740 css
.write_to_file
(out
/"style.css")
744 for p
in model
.mprojects
do
746 var f
= "{p.name}.html"
747 catalog
.project_page
(p
).write_to_file
(out
/f
)
752 var index
= new CatalogPage
753 index
.more_head
.add
"<title>Projects in Nit</title>"
756 <div class="content">
757 <h1>Projects in Nit</h1>
760 index
.add
"<h2>Highlighted Projects</h2>\n"
761 index
.add catalog
.list_best
(catalog
.score
)
763 index
.add
"<h2>Most Required</h2>\n"
764 var reqs
= new Counter[MProject]
765 for p
in model
.mprojects
do
766 reqs
[p
] = catalog
.deps
[p
].smallers
.length
- 1
768 index
.add catalog
.list_best
(reqs
)
770 index
.add
"<h2>By First Tag</h2>\n"
771 index
.add catalog
.list_by
(catalog
.cat2proj
, "cat_")
773 index
.add
"<h2>By Any Tag</h2>\n"
774 index
.add catalog
.list_by
(catalog
.tag2proj
, "tag_")
778 <div class="sidebar">
781 <li>{{{model.mprojects.length}}} projects</li>
782 <li>{{{catalog.maint2proj.length}}} maintainers</li>
783 <li>{{{catalog.contrib2proj.length}}} contributors</li>
784 <li>{{{catalog.tag2proj.length}}} tags</li>
785 <li>{{{catalog.mmodules.sum}}} modules</li>
786 <li>{{{catalog.mclasses.sum}}} classes</li>
787 <li>{{{catalog.mmethods.sum}}} methods</li>
788 <li>{{{catalog.loc.sum}}} lines of code</li>
793 index
.write_to_file
(out
/"index.html")
797 var page
= new CatalogPage
798 page
.more_head
.add
"<title>People of Nit</title>"
799 page
.add
"""<div class="content">\n<h1>People of Nit</h1>\n"""
800 page
.add
"<h2>By Maintainer</h2>\n"
801 page
.add catalog
.list_by
(catalog
.maint2proj
, "maint_")
802 page
.add
"<h2>By Contributor</h2>\n"
803 page
.add catalog
.list_by
(catalog
.contrib2proj
, "contrib_")
805 page
.write_to_file
(out
/"people.html")
809 page
= new CatalogPage
810 page
.more_head
.add
"<title>Projets of Nit</title>"
811 page
.add
"""<div class="content">\n<h1>People of Nit</h1>\n"""
812 page
.add
"<h2>Table of Projets</h2>\n"
813 page
.add catalog
.table_projects
(model
.mprojects
)
815 page
.write_to_file
(out
/"table.html")