1 # This file is part of NIT ( http://www.nitlanguage.org ).
3 # Copyright 2016 Alexandre Terrasa <alexandre@moz-code.org>
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
22 import popcorn
::pop_repos
23 import popcorn
::pop_logging
25 # Loader configuration file
29 redef var default_config_file
= "loader.ini"
31 # Default database host string for MongoDb
32 var default_db_host
= "mongodb://mongo:27017/"
34 # Default database hostname
35 var default_db_name
= "github_loader"
38 var opt_db_host
= new OptionString("MongoDb host", "--db-host")
40 # MongoDb database name
41 var opt_db_name
= new OptionString("MongoDb database name", "--db-name")
44 var opt_verbose
= new OptionCount("Verbosity level", "-v", "--verbose")
47 var opt_no_colors
= new OptionBool("Do not use colors in output", "--no-colors")
50 var opt_tokens
= new OptionArray("Token list", "--tokens")
53 var opt_show_wallet
= new OptionBool("Show wallet status", "--show-wallet")
56 var opt_show_jobs
= new OptionBool("Show jobs status", "--show-jobs")
59 var opt_no_branches
= new OptionBool("Do not load branches", "--no-branches")
62 var opt_no_commits
= new OptionBool("Do not load commits from default branch", "--no-commits")
65 var opt_no_issues
= new OptionBool("Do not load issues", "--no-issues")
68 var opt_no_comments
= new OptionBool("Do not load issue comments", "--no-comments")
71 var opt_no_events
= new OptionBool("Do not load issues events", "--no-events")
74 var opt_start
= new OptionInt("Start loading issues from a number", 0, "--from")
77 var opt_clear
= new OptionBool("Clear job for given repo name", "--clear")
81 tool_description
= "Usage: loader <repo_name>\nLoad a GitHub repo into a MongoDb."
82 add_option
(opt_db_host
, opt_db_name
)
83 add_option
(opt_tokens
, opt_show_wallet
)
84 add_option
(opt_verbose
, opt_no_colors
)
85 add_option
(opt_show_jobs
, opt_no_commits
, opt_no_issues
, opt_no_comments
, opt_no_events
)
86 add_option
(opt_start
, opt_clear
)
89 # MongoDB server used for data persistence
90 fun db_host
: String do
91 return opt_db_host
.value
or else ini
["db.host"] or else default_db_host
94 # MongoDB DB used for data persistence
95 fun db_name
: String do
96 return opt_db_name
.value
or else ini
["db.name"] or else default_db_name
100 var client
= new MongoClient(db_host
) is lazy
103 var db
: MongoDb = client
.database
(db_name
) is lazy
105 # Github tokens used to access data.
106 var tokens
: Array[String] is lazy
do
107 var opt_tokens
= self.opt_tokens
.value
108 if opt_tokens
.not_empty
then return opt_tokens
110 var res
= new Array[String]
111 var ini_tokens
= ini
.section
("tokens")
112 if ini_tokens
== null then return res
114 for token
in ini_tokens
.values
do
115 if token
== null then continue
121 # Github tokens wallet\13
122 var wallet
: GithubWallet is lazy
do
123 var wallet
= new GithubWallet.from_tokens
(tokens
)
124 wallet
.no_colors
= no_colors
128 # Use colors in console display
129 fun no_colors
: Bool do
130 if opt_no_colors
.value
then return true
131 return ini
["loader.no_colors"] == "true"
134 # Verbosity level (the higher the more verbose)
135 fun verbose_level
: Int do
136 var opt
= opt_start
.value
137 if opt
> 0 then return opt
138 var v
= ini
["loader.verbose"]
139 if v
!= null then return v
.to_i
143 # Logger used to print things
144 var logger
: ConsoleLog is lazy
do
145 var logger
= new ConsoleLog
146 logger
.level
= verbose_level
150 # Should we avoid loading branches?
151 fun no_branches
: Bool do
152 if opt_no_branches
.value
then return true
153 return ini
["loader.no_branches"] == "true"
156 # Should we avoid loading commits?
157 fun no_commits
: Bool do
158 if opt_no_commits
.value
then return true
159 return ini
["loader.no_commits"] == "true"
162 # Should we avoid loading issues?
163 fun no_issues
: Bool do
164 if opt_no_issues
.value
then return true
165 return ini
["loader.no_issues"] == "true"
168 # Should we avoid loading issue comments?
169 fun no_comments
: Bool do
170 if opt_no_comments
.value
then return true
171 return ini
["loader.no_comments"] == "true"
174 # Should we avoid loading events?
175 fun no_events
: Bool do
176 if opt_no_events
.value
then return true
177 return ini
["loader.no_events"] == "true"
180 # At which issue number should we start?
181 fun start_from_issue
: Int do
182 var opt
= opt_start
.value
183 if opt
> 0 then return opt
184 var v
= ini
["loader.start"]
185 if v
!= null then return v
.to_i
190 redef class GithubWallet
193 api
.enable_cache
= true
200 var config
= new LoaderConfig
203 var jobs
: LoaderJobRepo is lazy
do
204 return new LoaderJobRepo(config
.db
.collection
("loader_status"))
207 var repos
: RepoRepo is lazy
do
208 return new RepoRepo(config
.db
.collection
("repos"))
211 var branches
: BranchRepo is lazy
do
212 return new BranchRepo(config
.db
.collection
("branches"))
215 var commits
: CommitRepo is lazy
do
216 return new CommitRepo(config
.db
.collection
("commits"))
219 var issues
: IssueRepo is lazy
do
220 return new IssueRepo(config
.db
.collection
("issues"))
223 var pulls
: PullRequestRepo is lazy
do
224 return new PullRequestRepo(config
.db
.collection
("pull_requests"))
227 var issue_comments
: IssueCommentRepo is lazy
do
228 return new IssueCommentRepo(config
.db
.collection
("issue_comments"))
231 var issue_events
: IssueEventRepo is lazy
do
232 return new IssueEventRepo(config
.db
.collection
("issue_events"))
235 fun start
(repo_full_name
: String) do
236 var job
= jobs
.find_by_id
(repo_full_name
)
238 log
.info
"Creating new job for `{repo_full_name}`"
239 job
= add_job
(repo_full_name
)
241 log
.info
"Resuming pending job for `{repo_full_name}`"
243 print
"Load history for {job}..."
249 fun remove
(repo_full_name
: String) do
250 var job
= jobs
.find_by_id
(repo_full_name
)
252 log
.info
"No job found for `{repo_full_name}`"
254 jobs
.remove_by_id
(repo_full_name
)
255 log
.info
"Deleted job for `{repo_full_name}`"
260 fun show_wallet
do config
.wallet
.show_status
264 var jobs
= jobs
.find_all
265 print
"{jobs.length} jobs pending..."
269 print
"\nUse `loader <job> to start a new or resume a pending one"
273 fun add_job
(repo_full_name
: String): LoaderJob do
274 var repo
= config
.wallet
.api
.load_repo
(repo_full_name
)
275 assert repo
!= null else
276 error
"Repository `{repo_full_name}` not found"
279 var job
= new LoaderJob(repo
, config
.start_from_issue
)
285 fun finish_job
(job
: LoaderJob) do
286 print
"Finished job {job}"
287 jobs
.remove_by_id
(job
.id
)
290 fun load_branches
(job
: LoaderJob) do
291 if config
.no_branches
then return
293 var api
= config
.wallet
.api
295 for branch
in api
.load_repo_branches
(repo
) do
298 load_commits
(job
, branch
)
302 fun load_commits
(job
: LoaderJob, branch
: Branch) do
303 if config
.no_commits
then return
304 load_commit
(job
, branch
.commit
.sha
)
307 fun load_commit
(job
: LoaderJob, commit_sha
: String) do
308 if commits
.find_by_id
(commit_sha
) != null then return
309 var api
= config
.wallet
.api
310 var commit
= api
.load_commit
(job
.repo
, commit_sha
)
311 # print commit or else "NULL"
312 if commit
== null then return
313 var message
= commit
.message
or else "no message"
314 log
.info
"Load commit {commit_sha}: {message.split("\n").first}"
315 commit
.repo
= job
.repo
317 var parents
= commit
.parents
318 if parents
== null then return
319 for parent
in parents
do
320 load_commit
(job
, parent
.sha
)
324 # Load game for `repo_name`.
325 fun load_issues
(job
: LoaderJob) do
326 if config
.no_issues
then return
328 var i
= job
.last_issue
329 var last_issue
= load_last_issue
(job
)
330 if last_issue
!= null then
331 while i
<= last_issue
.number
do
340 # Load the `repo` last issue or abort.
341 private fun load_last_issue
(job
: LoaderJob): nullable Issue do
342 var api
= config
.wallet
.api
343 return api
.load_repo_last_issue
(job
.repo
)
346 # Load an issue or abort.
347 private fun load_issue
(job
: LoaderJob, issue_number
: Int) do
348 if issues
.find_by_id
("{job.repo.mongo_id}/{issue_number}") != null then return
350 var api
= config
.wallet
.api
351 var issue
= api
.load_issue
(job
.repo
, issue_number
)
352 assert issue
!= null else
353 check_error
(api
, "Issue #{issue_number} not found")
355 if issue
.is_pull_request
then
356 load_pull
(job
, issue
)
358 log
.info
"Load issue #{issue.number}: {issue.title.split("\n").first}"
359 issue
.repo
= job
.repo
361 load_issue_events
(job
, issue
)
363 load_issue_comments
(job
, issue
)
366 # Load issue comments.
367 private fun load_issue_comments
(job
: LoaderJob, issue
: Issue) do
368 if config
.no_comments
then return
369 var api
= config
.wallet
.api
370 for comment
in api
.load_issue_comments
(job
.repo
, issue
) do
371 comment
.repo
= job
.repo
372 issue_comments
.save comment
377 private fun load_issue_events
(job
: LoaderJob, issue
: Issue) do
378 if config
.no_events
then return
380 var api
= config
.wallet
.api
381 for event
in api
.load_issue_events
(job
.repo
, issue
) do
382 event
.repo
= job
.repo
383 issue_events
.save event
387 # Load a pull request or abort.
388 private fun load_pull
(job
: LoaderJob, issue
: Issue): PullRequest do
389 var api
= config
.wallet
.api
390 var pr
= api
.load_pull
(job
.repo
, issue
.number
)
391 assert pr
!= null else
392 check_error
(api
, "Pull request #{issue.number} not found")
394 log
.info
"Load pull request #{issue.number}: {pr.title.split("\n").first}"
397 load_pull_events
(job
, pr
)
402 private fun load_pull_events
(job
: LoaderJob, pull
: PullRequest) do
403 if config
.no_events
then return
405 var api
= config
.wallet
.api
406 for event
in api
.load_issue_events
(job
.repo
, pull
) do
407 event
.repo
= job
.repo
408 issue_events
.save event
412 # Check if the API is in error state then abort
413 fun check_error
(api
: GithubAPI, message
: nullable String) do
414 var err
= api
.last_error
416 error message
or else err
.message
421 fun log
: ConsoleLog do return config
.logger
423 # Display a error and exit
424 fun error
(msg
: String) do
425 log
.error
"Error: {msg}"
430 # Loader status by repo
435 # Repo this status is about
438 # Primary key: the repo id
439 redef var id
is lazy
, serialize_as
("_id") do return repo
.full_name
445 # Loader status repository
447 super MongoRepository[LoaderJob]
453 var repo
: nullable Repo = null is writable
459 var mongo_id
: String is lazy
, serialize_as
("_id") do return full_name
463 super MongoRepository[Repo]
470 var mongo_id
: String is lazy
, serialize_as
("_id") do
472 if repo
== null then return name
473 return "{repo.mongo_id}/{name}"
478 super MongoRepository[Branch]
480 fun find_by_repo
(repo
: Repo): Array[Branch] do
481 return find_all
((new MongoMatch).eq
("repo.full_name", repo
.full_name
))
489 var mongo_id
: String is lazy
, serialize_as
("_id") do return sha
493 super MongoRepository[Commit]
495 fun find_by_repo
(repo
: Repo): Array[Commit] do
496 return find_all
((new MongoMatch).eq
("repo.full_name", repo
.full_name
))
504 var mongo_id
: String is lazy
, serialize_as
("_id") do
506 if repo
== null then return number
.to_s
507 return "{repo.mongo_id}/{number}"
512 super MongoRepository[Issue]
514 fun find_by_repo
(repo
: Repo): Array[Issue] do
515 return find_all
((new MongoMatch).eq
("repo.full_name", repo
.full_name
))
519 class PullRequestRepo
520 super MongoRepository[PullRequest]
522 fun find_by_repo
(repo
: Repo): Array[Issue] do
523 return find_all
((new MongoMatch).eq
("repo.full_name", repo
.full_name
))
527 redef class IssueComment
531 var mongo_id
: String is lazy
, serialize_as
("_id") do return id
.to_s
534 class IssueCommentRepo
535 super MongoRepository[IssueComment]
537 fun find_by_repo
(repo
: Repo): Array[IssueComment] do
538 return find_all
((new MongoMatch).eq
("repo.full_name", repo
.full_name
))
542 redef class IssueEvent
546 var mongo_id
: String is lazy
, serialize_as
("_id") do return id
.to_s
550 super MongoRepository[IssueEvent]
552 fun find_by_repo
(repo
: Repo): Array[IssueEvent] do
553 return find_all
((new MongoMatch).eq
("repo.full_name", repo
.full_name
))
558 var loader
= new Loader
559 loader
.config
.parse_options
(args
)
564 loader
.branches
.clear
568 loader
.issue_comments
.clear
569 loader
.issue_events
.clear
571 if loader
.config
.help
then
576 if loader
.config
.opt_show_wallet
.value
then
580 var args
= loader
.config
.args
581 if loader
.config
.opt_show_jobs
.value
or args
.is_empty
then
585 if args
.is_empty
then return
587 if loader
.config
.opt_clear
.value
then
588 loader
.remove args
.first
590 loader
.start args
.first
592 var repo
= loader
.config
.wallet
.api
.load_repo
(args
.first
)
593 if repo
== null then return
595 print
"* {if loader.repos.find_by_id(args.first) != null then 1 else 0} repos"
596 print
"* {loader.branches.find_by_repo(repo).length} branches"
597 print
"* {loader.commits.find_by_repo(repo).length} commits"
598 print
"* {loader.issues.find_by_repo(repo).length} issues"
599 print
"* {loader.pulls.find_by_repo(repo).length} pulls"
600 print
"* {loader.issue_comments.find_by_repo(repo).length} comments"
601 print
"* {loader.issue_events.find_by_repo(repo).length} events"