1 # This file is part of NIT ( http://www.nitlanguage.org ).
3 # Copyright 2016 Alexandre Terrasa <alexandre@moz-code.org>
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
22 import popcorn
::pop_repos
23 import popcorn
::pop_logging
25 # Loader configuration file
29 redef var default_config_file
= "loader.ini"
31 # Default database host string for MongoDb
32 var default_db_host
= "mongodb://mongo:27017/"
34 # Default database hostname
35 var default_db_name
= "github_loader"
38 var opt_db_host
= new OptionString("MongoDb host", "--db-host")
40 # MongoDb database name
41 var opt_db_name
= new OptionString("MongoDb database name", "--db-name")
44 var opt_verbose
= new OptionCount("Verbosity level", "-v", "--verbose")
47 var opt_no_colors
= new OptionBool("Do not use colors in output", "--no-colors")
50 var opt_tokens
= new OptionArray("Token list", "--tokens")
53 var opt_show_wallet
= new OptionBool("Show wallet status", "--show-wallet")
56 var opt_show_jobs
= new OptionBool("Show jobs status", "--show-jobs")
59 var opt_no_branches
= new OptionBool("Do not load branches", "--no-branches")
62 var opt_no_commits
= new OptionBool("Do not load commits from default branch", "--no-commits")
65 var opt_no_issues
= new OptionBool("Do not load issues", "--no-issues")
68 var opt_no_comments
= new OptionBool("Do not load issue comments", "--no-comments")
71 var opt_no_events
= new OptionBool("Do not load issues events", "--no-events")
74 var opt_start
= new OptionInt("Start loading issues from a number", 0, "--from")
77 var opt_clear
= new OptionBool("Clear job for given repo name", "--clear")
81 tool_description
= "Usage: loader <repo_name>\nLoad a GitHub repo into a MongoDb."
82 add_option
(opt_db_host
, opt_db_name
)
83 add_option
(opt_tokens
, opt_show_wallet
)
84 add_option
(opt_verbose
, opt_no_colors
)
85 add_option
(opt_show_jobs
, opt_no_commits
, opt_no_issues
, opt_no_comments
, opt_no_events
)
86 add_option
(opt_start
, opt_clear
)
89 # MongoDB server used for data persistence
90 fun db_host
: String do
91 return opt_db_host
.value
or else ini
["db.host"] or else default_db_host
94 # MongoDB DB used for data persistence
95 fun db_name
: String do
96 return opt_db_name
.value
or else ini
["db.name"] or else default_db_name
100 var client
= new MongoClient(db_host
) is lazy
103 var db
: MongoDb = client
.database
(db_name
) is lazy
105 # Github tokens used to access data.
106 var tokens
: Array[String] is lazy
do
107 var opt_tokens
= self.opt_tokens
.value
108 if opt_tokens
.not_empty
then return opt_tokens
110 var res
= new Array[String]
111 var ini_tokens
= ini
.section
("tokens")
112 if ini_tokens
== null then return res
114 for token
in ini_tokens
.values
do
115 if token
== null then continue
121 # Github tokens wallet\13
122 var wallet
: GithubWallet is lazy
do
123 var wallet
= new GithubWallet.from_tokens
(tokens
)
124 wallet
.no_colors
= no_colors
128 # Use colors in console display
129 fun no_colors
: Bool do
130 if opt_no_colors
.value
then return true
131 return ini
["loader.no_colors"] == "true"
134 # Verbosity level (the higher the more verbose)
135 fun verbose_level
: Int do
136 var opt
= opt_start
.value
140 var v
= ini
["loader.verbose"]
141 if v
!= null and v
.to_i
> 0 then
147 # Logger used to print things
148 var logger
: PopLogger is lazy
do
149 var logger
= new PopLogger
150 logger
.level
= verbose_level
154 # Should we avoid loading branches?
155 fun no_branches
: Bool do
156 if opt_no_branches
.value
then return true
157 return ini
["loader.no_branches"] == "true"
160 # Should we avoid loading commits?
161 fun no_commits
: Bool do
162 if opt_no_commits
.value
then return true
163 return ini
["loader.no_commits"] == "true"
166 # Should we avoid loading issues?
167 fun no_issues
: Bool do
168 if opt_no_issues
.value
then return true
169 return ini
["loader.no_issues"] == "true"
172 # Should we avoid loading issue comments?
173 fun no_comments
: Bool do
174 if opt_no_comments
.value
then return true
175 return ini
["loader.no_comments"] == "true"
178 # Should we avoid loading events?
179 fun no_events
: Bool do
180 if opt_no_events
.value
then return true
181 return ini
["loader.no_events"] == "true"
184 # At which issue number should we start?
185 fun start_from_issue
: Int do
186 var opt
= opt_start
.value
187 if opt
> 0 then return opt
188 var v
= ini
["loader.start"]
189 if v
!= null then return v
.to_i
194 redef class GithubWallet
197 api
.enable_cache
= true
204 var config
= new LoaderConfig
207 var jobs
: LoaderJobRepo is lazy
do
208 return new LoaderJobRepo(config
.db
.collection
("loader_status"))
211 var repos
: RepoRepo is lazy
do
212 return new RepoRepo(config
.db
.collection
("repos"))
215 var branches
: BranchRepo is lazy
do
216 return new BranchRepo(config
.db
.collection
("branches"))
219 var commits
: CommitRepo is lazy
do
220 return new CommitRepo(config
.db
.collection
("commits"))
223 var issues
: IssueRepo is lazy
do
224 return new IssueRepo(config
.db
.collection
("issues"))
227 var pulls
: PullRequestRepo is lazy
do
228 return new PullRequestRepo(config
.db
.collection
("pull_requests"))
231 var issue_comments
: IssueCommentRepo is lazy
do
232 return new IssueCommentRepo(config
.db
.collection
("issue_comments"))
235 var issue_events
: IssueEventRepo is lazy
do
236 return new IssueEventRepo(config
.db
.collection
("issue_events"))
239 fun start
(repo_full_name
: String) do
240 var job
= jobs
.find_by_id
(repo_full_name
)
242 log
.info
"Creating new job for `{repo_full_name}`"
243 job
= add_job
(repo_full_name
)
245 log
.info
"Resuming pending job for `{repo_full_name}`"
247 print
"Load history for {job}..."
253 fun remove
(repo_full_name
: String) do
254 var job
= jobs
.find_by_id
(repo_full_name
)
256 log
.info
"No job found for `{repo_full_name}`"
258 jobs
.remove_by_id
(repo_full_name
)
259 log
.info
"Deleted job for `{repo_full_name}`"
264 fun show_wallet
do config
.wallet
.show_status
268 var jobs
= jobs
.find_all
269 print
"{jobs.length} jobs pending..."
273 print
"\nUse `loader <job> to start a new or resume a pending one"
277 fun add_job
(repo_full_name
: String): LoaderJob do
278 var repo
= config
.wallet
.api
.load_repo
(repo_full_name
)
279 assert repo
!= null else
280 error
"Repository `{repo_full_name}` not found"
283 var job
= new LoaderJob(repo
, config
.start_from_issue
)
289 fun finish_job
(job
: LoaderJob) do
290 print
"Finished job {job}"
291 jobs
.remove_by_id
(job
.id
)
294 fun load_branches
(job
: LoaderJob) do
295 if config
.no_branches
then return
297 var api
= config
.wallet
.api
299 for branch
in api
.load_repo_branches
(repo
) do
302 load_commits
(job
, branch
)
306 fun load_commits
(job
: LoaderJob, branch
: Branch) do
307 if config
.no_commits
then return
308 load_commit
(job
, branch
.commit
.sha
)
311 fun load_commit
(job
: LoaderJob, commit_sha
: String) do
312 if commits
.find_by_id
(commit_sha
) != null then return
313 var api
= config
.wallet
.api
314 var commit
= api
.load_commit
(job
.repo
, commit_sha
)
315 # print commit or else "NULL"
316 if commit
== null then return
317 var message
= commit
.message
or else "no message"
318 log
.info
"Load commit {commit_sha}: {message.split("\n").first}"
319 commit
.repo
= job
.repo
321 var parents
= commit
.parents
322 if parents
== null then return
323 for parent
in parents
do
324 load_commit
(job
, parent
.sha
)
328 # Load game for `repo_name`.
329 fun load_issues
(job
: LoaderJob) do
330 if config
.no_issues
then return
332 var i
= job
.last_issue
333 var last_issue
= load_last_issue
(job
)
334 if last_issue
!= null then
335 while i
<= last_issue
.number
do
344 # Load the `repo` last issue or abort.
345 private fun load_last_issue
(job
: LoaderJob): nullable Issue do
346 var api
= config
.wallet
.api
347 return api
.load_repo_last_issue
(job
.repo
)
350 # Load an issue or abort.
351 private fun load_issue
(job
: LoaderJob, issue_number
: Int) do
352 if issues
.find_by_id
("{job.repo.mongo_id}/{issue_number}") != null then return
354 var api
= config
.wallet
.api
355 var issue
= api
.load_issue
(job
.repo
, issue_number
)
356 assert issue
!= null else
357 check_error
(api
, "Issue #{issue_number} not found")
359 if issue
.is_pull_request
then
360 load_pull
(job
, issue
)
362 log
.info
"Load issue #{issue.number}: {issue.title.split("\n").first}"
363 issue
.repo
= job
.repo
365 load_issue_events
(job
, issue
)
367 load_issue_comments
(job
, issue
)
370 # Load issue comments.
371 private fun load_issue_comments
(job
: LoaderJob, issue
: Issue) do
372 if config
.no_comments
then return
373 var api
= config
.wallet
.api
374 for comment
in api
.load_issue_comments
(job
.repo
, issue
) do
375 comment
.repo
= job
.repo
376 issue_comments
.save comment
381 private fun load_issue_events
(job
: LoaderJob, issue
: Issue) do
382 if config
.no_events
then return
384 var api
= config
.wallet
.api
385 for event
in api
.load_issue_events
(job
.repo
, issue
) do
386 event
.repo
= job
.repo
387 issue_events
.save event
391 # Load a pull request or abort.
392 private fun load_pull
(job
: LoaderJob, issue
: Issue): PullRequest do
393 var api
= config
.wallet
.api
394 var pr
= api
.load_pull
(job
.repo
, issue
.number
)
395 assert pr
!= null else
396 check_error
(api
, "Pull request #{issue.number} not found")
398 log
.info
"Load pull request #{issue.number}: {pr.title.split("\n").first}"
401 load_pull_events
(job
, pr
)
406 private fun load_pull_events
(job
: LoaderJob, pull
: PullRequest) do
407 if config
.no_events
then return
409 var api
= config
.wallet
.api
410 for event
in api
.load_issue_events
(job
.repo
, pull
) do
411 event
.repo
= job
.repo
412 issue_events
.save event
416 # Check if the API is in error state then abort
417 fun check_error
(api
: GithubAPI, message
: nullable String) do
418 var err
= api
.last_error
420 error message
or else err
.message
425 fun log
: PopLogger do return config
.logger
427 # Display a error and exit
428 fun error
(msg
: String) do
429 log
.error
"Error: {msg}"
434 # Loader status by repo
439 # Repo this status is about
442 # Primary key: the repo id
443 redef var id
is lazy
, serialize_as
("_id") do return repo
.full_name
449 # Loader status repository
451 super MongoRepository[LoaderJob]
457 var repo
: nullable Repo = null is writable
463 var mongo_id
: String is lazy
, serialize_as
("_id") do return full_name
467 super MongoRepository[Repo]
474 var mongo_id
: String is lazy
, serialize_as
("_id") do
476 if repo
== null then return name
477 return "{repo.mongo_id}/{name}"
482 super MongoRepository[Branch]
484 fun find_by_repo
(repo
: Repo): Array[Branch] do
485 return find_all
((new MongoMatch).eq
("repo.full_name", repo
.full_name
))
493 var mongo_id
: String is lazy
, serialize_as
("_id") do return sha
497 super MongoRepository[Commit]
499 fun find_by_repo
(repo
: Repo): Array[Commit] do
500 return find_all
((new MongoMatch).eq
("repo.full_name", repo
.full_name
))
508 var mongo_id
: String is lazy
, serialize_as
("_id") do
510 if repo
== null then return number
.to_s
511 return "{repo.mongo_id}/{number}"
516 super MongoRepository[Issue]
518 fun find_by_repo
(repo
: Repo): Array[Issue] do
519 return find_all
((new MongoMatch).eq
("repo.full_name", repo
.full_name
))
523 class PullRequestRepo
524 super MongoRepository[PullRequest]
526 fun find_by_repo
(repo
: Repo): Array[Issue] do
527 return find_all
((new MongoMatch).eq
("repo.full_name", repo
.full_name
))
531 redef class IssueComment
535 var mongo_id
: String is lazy
, serialize_as
("_id") do return id
.to_s
538 class IssueCommentRepo
539 super MongoRepository[IssueComment]
541 fun find_by_repo
(repo
: Repo): Array[IssueComment] do
542 return find_all
((new MongoMatch).eq
("repo.full_name", repo
.full_name
))
546 redef class IssueEvent
550 var mongo_id
: String is lazy
, serialize_as
("_id") do return id
.to_s
554 super MongoRepository[IssueEvent]
556 fun find_by_repo
(repo
: Repo): Array[IssueEvent] do
557 return find_all
((new MongoMatch).eq
("repo.full_name", repo
.full_name
))
562 var loader
= new Loader
563 loader
.config
.parse_options
(args
)
568 loader
.branches
.clear
572 loader
.issue_comments
.clear
573 loader
.issue_events
.clear
575 if loader
.config
.help
then
580 if loader
.config
.opt_show_wallet
.value
then
584 var args
= loader
.config
.args
585 if loader
.config
.opt_show_jobs
.value
or args
.is_empty
then
589 if args
.is_empty
then return
591 if loader
.config
.opt_clear
.value
then
592 loader
.remove args
.first
594 loader
.start args
.first
596 var repo
= loader
.config
.wallet
.api
.load_repo
(args
.first
)
597 if repo
== null then return
599 print
"* {if loader.repos.find_by_id(args.first) != null then 1 else 0} repos"
600 print
"* {loader.branches.find_by_repo(repo).length} branches"
601 print
"* {loader.commits.find_by_repo(repo).length} commits"
602 print
"* {loader.issues.find_by_repo(repo).length} issues"
603 print
"* {loader.pulls.find_by_repo(repo).length} pulls"
604 print
"* {loader.issue_comments.find_by_repo(repo).length} comments"
605 print
"* {loader.issue_events.find_by_repo(repo).length} events"