1 # This file is part of NIT ( http://www.nitlanguage.org ).
3 # Copyright 2016 Alexandre Terrasa <alexandre@moz-code.org>
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
22 import popcorn
::pop_repos
23 import popcorn
::pop_logging
25 # Loader configuration file
29 redef var default_config_file
= "loader.ini"
31 # Default database host string for MongoDb
32 var default_db_host
= "mongodb://localhost:27017/"
34 # Default database hostname
35 var default_db_name
= "github_loader"
38 var opt_db_host
= new OptionString("MongoDb host", "--db-host")
40 # MongoDb database name
41 var opt_db_name
= new OptionString("MongoDb database name", "--db-name")
44 var opt_verbose
= new OptionCount("Verbosity level", "-v", "--verbose")
47 var opt_no_colors
= new OptionBool("Do not use colors in output", "--no-colors")
50 var opt_tokens
= new OptionArray("Token list", "--tokens")
53 var opt_show_wallet
= new OptionBool("Show wallet status", "--show-wallet")
56 var opt_show_jobs
= new OptionBool("Show jobs status", "--show-jobs")
59 var opt_no_branches
= new OptionBool("Do not load branches", "--no-branches")
62 var opt_no_commits
= new OptionBool("Do not load commits from default branch", "--no-commits")
65 var opt_no_issues
= new OptionBool("Do not load issues", "--no-issues")
68 var opt_no_comments
= new OptionBool("Do not load issue comments", "--no-comments")
71 var opt_no_events
= new OptionBool("Do not load issues events", "--no-events")
74 var opt_start
= new OptionInt("Start loading issues from a number", 0, "--from")
77 var opt_clear
= new OptionBool("Clear job for given repo name", "--clear")
81 tool_description
= "Usage: loader <repo_name>"
82 add_option
(opt_db_host
, opt_db_name
)
83 add_option
(opt_tokens
, opt_show_wallet
)
84 add_option
(opt_verbose
, opt_no_colors
)
85 add_option
(opt_show_jobs
, opt_no_commits
, opt_no_issues
, opt_no_comments
, opt_no_events
)
86 add_option
(opt_start
, opt_clear
)
89 # MongoDB server used for data persistence
90 fun db_host
: String do
91 return opt_db_host
.value
or else ini
["db.host"] or else default_db_host
94 # MongoDB DB used for data persistence
95 fun db_name
: String do
96 return opt_db_name
.value
or else ini
["db.name"] or else default_db_name
100 var client
= new MongoClient(db_host
) is lazy
103 var db
: MongoDb = client
.database
(db_name
) is lazy
105 # Github tokens used to access data.
106 var tokens
: Array[String] is lazy
do
107 var arr
= opt_tokens
.value
109 var iarr
= ini
.at
("tokens")
110 if iarr
!= null then arr
= iarr
.values
.to_a
112 return arr
or else new Array[String]
115 # Github tokens wallet\13
116 var wallet
: GithubWallet is lazy
do
117 var wallet
= new GithubWallet.from_tokens
(tokens
)
118 wallet
.no_colors
= no_colors
122 # Use colors in console display
123 fun no_colors
: Bool do
124 if opt_no_colors
.value
then return true
125 return ini
["loader.no_colors"] == "true"
128 # Verbosity level (the higher the more verbose)
129 fun verbose_level
: Int do
130 var opt
= opt_start
.value
131 if opt
> 0 then return opt
132 var v
= ini
["loader.verbose"]
133 if v
!= null then return v
.to_i
137 # Logger used to print things
138 var logger
: ConsoleLog is lazy
do
139 var logger
= new ConsoleLog
140 logger
.level
= verbose_level
144 # Should we avoid loading branches?
145 fun no_branches
: Bool do
146 if opt_no_branches
.value
then return true
147 return ini
["loader.no_branches"] == "true"
150 # Should we avoid loading commits?
151 fun no_commits
: Bool do
152 if opt_no_commits
.value
then return true
153 return ini
["loader.no_commits"] == "true"
156 # Should we avoid loading issues?
157 fun no_issues
: Bool do
158 if opt_no_issues
.value
then return true
159 return ini
["loader.no_issues"] == "true"
162 # Should we avoid loading issue comments?
163 fun no_comments
: Bool do
164 if opt_no_comments
.value
then return true
165 return ini
["loader.no_comments"] == "true"
168 # Should we avoid loading events?
169 fun no_events
: Bool do
170 if opt_no_events
.value
then return true
171 return ini
["loader.no_events"] == "true"
174 # At which issue number should we start?
175 fun start_from_issue
: Int do
176 var opt
= opt_start
.value
177 if opt
> 0 then return opt
178 var v
= ini
["loader.start"]
179 if v
!= null then return v
.to_i
184 redef class GithubWallet
187 api
.enable_cache
= true
194 var config
= new LoaderConfig
197 var jobs
: LoaderJobRepo is lazy
do
198 return new LoaderJobRepo(config
.db
.collection
("loader_status"))
201 var repos
: RepoRepo is lazy
do
202 return new RepoRepo(config
.db
.collection
("repos"))
205 var branches
: BranchRepo is lazy
do
206 return new BranchRepo(config
.db
.collection
("branches"))
209 var commits
: CommitRepo is lazy
do
210 return new CommitRepo(config
.db
.collection
("commits"))
213 var issues
: IssueRepo is lazy
do
214 return new IssueRepo(config
.db
.collection
("issues"))
217 var pulls
: PullRequestRepo is lazy
do
218 return new PullRequestRepo(config
.db
.collection
("pull_requests"))
221 var issue_comments
: IssueCommentRepo is lazy
do
222 return new IssueCommentRepo(config
.db
.collection
("issue_comments"))
225 var issue_events
: IssueEventRepo is lazy
do
226 return new IssueEventRepo(config
.db
.collection
("issue_events"))
229 fun start
(repo_full_name
: String) do
230 var job
= jobs
.find_by_id
(repo_full_name
)
232 log
.info
"Creating new job for `{repo_full_name}`"
233 job
= add_job
(repo_full_name
)
235 log
.info
"Resuming pending job for `{repo_full_name}`"
237 print
"Load history for {job}..."
243 fun remove
(repo_full_name
: String) do
244 var job
= jobs
.find_by_id
(repo_full_name
)
246 log
.info
"No job found for `{repo_full_name}`"
248 jobs
.remove_by_id
(repo_full_name
)
249 log
.info
"Deleted job for `{repo_full_name}`"
254 fun show_wallet
do config
.wallet
.show_status
258 var jobs
= jobs
.find_all
259 print
"{jobs.length} jobs pending..."
263 print
"\nUse `loader <job> to start a new or resume a pending one"
267 fun add_job
(repo_full_name
: String): LoaderJob do
268 var repo
= config
.wallet
.api
.load_repo
(repo_full_name
)
269 assert repo
!= null else
270 error
"Repository `{repo_full_name}` not found"
273 var job
= new LoaderJob(repo
, config
.start_from_issue
)
279 fun finish_job
(job
: LoaderJob) do
280 print
"Finished job {job}"
281 jobs
.remove_by_id
(job
.id
)
284 fun load_branches
(job
: LoaderJob) do
285 if config
.no_branches
then return
287 var api
= config
.wallet
.api
289 for branch
in api
.load_repo_branches
(repo
) do
292 load_commits
(job
, branch
)
296 fun load_commits
(job
: LoaderJob, branch
: Branch) do
297 if config
.no_commits
then return
298 load_commit
(job
, branch
.commit
.sha
)
301 fun load_commit
(job
: LoaderJob, commit_sha
: String) do
302 if commits
.find_by_id
(commit_sha
) != null then return
303 var api
= config
.wallet
.api
304 var commit
= api
.load_commit
(job
.repo
, commit_sha
)
305 # print commit or else "NULL"
306 if commit
== null then return
307 var message
= commit
.message
or else "no message"
308 log
.info
"Load commit {commit_sha}: {message.split("\n").first}"
309 commit
.repo
= job
.repo
311 var parents
= commit
.parents
312 if parents
== null then return
313 for parent
in parents
do
314 load_commit
(job
, parent
.sha
)
318 # Load game for `repo_name`.
319 fun load_issues
(job
: LoaderJob) do
320 if config
.no_issues
then return
322 var i
= job
.last_issue
323 var last_issue
= load_last_issue
(job
)
324 if last_issue
!= null then
325 while i
<= last_issue
.number
do
334 # Load the `repo` last issue or abort.
335 private fun load_last_issue
(job
: LoaderJob): nullable Issue do
336 var api
= config
.wallet
.api
337 return api
.load_repo_last_issue
(job
.repo
)
340 # Load an issue or abort.
341 private fun load_issue
(job
: LoaderJob, issue_number
: Int) do
342 if issues
.find_by_id
("{job.repo.mongo_id}/{issue_number}") != null then return
344 var api
= config
.wallet
.api
345 var issue
= api
.load_issue
(job
.repo
, issue_number
)
346 assert issue
!= null else
347 check_error
(api
, "Issue #{issue_number} not found")
349 if issue
.is_pull_request
then
350 load_pull
(job
, issue
)
352 log
.info
"Load issue #{issue.number}: {issue.title.split("\n").first}"
353 issue
.repo
= job
.repo
355 load_issue_events
(job
, issue
)
357 load_issue_comments
(job
, issue
)
360 # Load issue comments.
361 private fun load_issue_comments
(job
: LoaderJob, issue
: Issue) do
362 if config
.no_comments
then return
363 var api
= config
.wallet
.api
364 for comment
in api
.load_issue_comments
(job
.repo
, issue
) do
365 comment
.repo
= job
.repo
366 issue_comments
.save comment
371 private fun load_issue_events
(job
: LoaderJob, issue
: Issue) do
372 if config
.no_events
then return
374 var api
= config
.wallet
.api
375 for event
in api
.load_issue_events
(job
.repo
, issue
) do
376 event
.repo
= job
.repo
377 issue_events
.save event
381 # Load a pull request or abort.
382 private fun load_pull
(job
: LoaderJob, issue
: Issue): PullRequest do
383 var api
= config
.wallet
.api
384 var pr
= api
.load_pull
(job
.repo
, issue
.number
)
385 assert pr
!= null else
386 check_error
(api
, "Pull request #{issue.number} not found")
388 log
.info
"Load pull request #{issue.number}: {pr.title.split("\n").first}"
391 load_pull_events
(job
, pr
)
396 private fun load_pull_events
(job
: LoaderJob, pull
: PullRequest) do
397 if config
.no_events
then return
399 var api
= config
.wallet
.api
400 for event
in api
.load_issue_events
(job
.repo
, pull
) do
401 event
.repo
= job
.repo
402 issue_events
.save event
406 # Check if the API is in error state then abort
407 fun check_error
(api
: GithubAPI, message
: nullable String) do
408 var err
= api
.last_error
410 error message
or else err
.message
415 fun log
: ConsoleLog do return config
.logger
417 # Display a error and exit
418 fun error
(msg
: String) do
419 log
.error
"Error: {msg}"
424 # Loader status by repo
429 # Repo this status is about
432 # Primary key: the repo id
433 redef var id
is lazy
, serialize_as
("_id") do return repo
.full_name
439 # Loader status repository
441 super MongoRepository[LoaderJob]
447 var repo
: nullable Repo = null is writable
453 var mongo_id
: String is lazy
, serialize_as
("_id") do return full_name
457 super MongoRepository[Repo]
464 var mongo_id
: String is lazy
, serialize_as
("_id") do
466 if repo
== null then return name
467 return "{repo.mongo_id}/{name}"
472 super MongoRepository[Branch]
474 fun find_by_repo
(repo
: Repo): Array[Branch] do
475 return find_all
((new MongoMatch).eq
("repo.full_name", repo
.full_name
))
483 var mongo_id
: String is lazy
, serialize_as
("_id") do return sha
487 super MongoRepository[Commit]
489 fun find_by_repo
(repo
: Repo): Array[Commit] do
490 return find_all
((new MongoMatch).eq
("repo.full_name", repo
.full_name
))
498 var mongo_id
: String is lazy
, serialize_as
("_id") do
500 if repo
== null then return number
.to_s
501 return "{repo.mongo_id}/{number}"
506 super MongoRepository[Issue]
508 fun find_by_repo
(repo
: Repo): Array[Issue] do
509 return find_all
((new MongoMatch).eq
("repo.full_name", repo
.full_name
))
513 class PullRequestRepo
514 super MongoRepository[PullRequest]
516 fun find_by_repo
(repo
: Repo): Array[Issue] do
517 return find_all
((new MongoMatch).eq
("repo.full_name", repo
.full_name
))
521 redef class IssueComment
525 var mongo_id
: String is lazy
, serialize_as
("_id") do return id
.to_s
528 class IssueCommentRepo
529 super MongoRepository[IssueComment]
531 fun find_by_repo
(repo
: Repo): Array[IssueComment] do
532 return find_all
((new MongoMatch).eq
("repo.full_name", repo
.full_name
))
536 redef class IssueEvent
540 var mongo_id
: String is lazy
, serialize_as
("_id") do return id
.to_s
544 super MongoRepository[IssueEvent]
546 fun find_by_repo
(repo
: Repo): Array[IssueEvent] do
547 return find_all
((new MongoMatch).eq
("repo.full_name", repo
.full_name
))
552 var loader
= new Loader
553 loader
.config
.parse_options
(args
)
558 loader
.branches
.clear
562 loader
.issue_comments
.clear
563 loader
.issue_events
.clear
565 if loader
.config
.help
then
570 if loader
.config
.opt_show_wallet
.value
then
574 var args
= loader
.config
.args
575 if loader
.config
.opt_show_jobs
.value
or args
.is_empty
then
579 if args
.is_empty
then return
581 if loader
.config
.opt_clear
.value
then
582 loader
.remove args
.first
584 loader
.start args
.first
586 var repo
= loader
.config
.wallet
.api
.load_repo
(args
.first
)
587 if repo
== null then return
589 print
"* {if loader.repos.find_by_id(args.first) != null then 1 else 0} repos"
590 print
"* {loader.branches.find_by_repo(repo).length} branches"
591 print
"* {loader.commits.find_by_repo(repo).length} commits"
592 print
"* {loader.issues.find_by_repo(repo).length} issues"
593 print
"* {loader.pulls.find_by_repo(repo).length} pulls"
594 print
"* {loader.issue_comments.find_by_repo(repo).length} comments"
595 print
"* {loader.issue_events.find_by_repo(repo).length} events"