Merge: fix ci nitunit some

[nit.git] / lib / github / loader.nit

# This file is part of NIT ( http://www.nitlanguage.org ).
#
# Copyright 2016 Alexandre Terrasa <alexandre@moz-code.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

module loader

import config
import github::wallet
import github::events
import popcorn::pop_repos
import popcorn::pop_logging

# Loader configuration file
class LoaderConfig
        super IniConfig

        redef var default_config_file = "loader.ini"

        # Default database host string for MongoDb
        var default_db_host = "mongodb://mongo:27017/"

        # Default database hostname
        var default_db_name = "github_loader"

        # MongoDb host name
        var opt_db_host = new OptionString("MongoDb host", "--db-host")

        # MongoDb database name
        var opt_db_name = new OptionString("MongoDb database name", "--db-name")

        # --verbose
        var opt_verbose = new OptionCount("Verbosity level", "-v", "--verbose")

        # --no-colors
        var opt_no_colors = new OptionBool("Do not use colors in output", "--no-colors")

        # --tokens
        var opt_tokens = new OptionArray("Token list", "--tokens")

        # --show-wallet
        var opt_show_wallet = new OptionBool("Show wallet status", "--show-wallet")

        # --show-jobs
        var opt_show_jobs = new OptionBool("Show jobs status", "--show-jobs")

        # --no-branches
        var opt_no_branches = new OptionBool("Do not load branches", "--no-branches")

        # --no-commits
        var opt_no_commits = new OptionBool("Do not load commits from default branch", "--no-commits")

        # --no-issues
        var opt_no_issues = new OptionBool("Do not load issues", "--no-issues")

        # --no-comments
        var opt_no_comments = new OptionBool("Do not load issue comments", "--no-comments")

        # --no-events
        var opt_no_events = new OptionBool("Do not load issues events", "--no-events")

        # --from
        var opt_start = new OptionInt("Start loading issues from a number", 0, "--from")

        # --clear
        var opt_clear = new OptionBool("Clear job for given repo name", "--clear")

        init do
                super
                tool_description = "Usage: loader <repo_name>\nLoad a GitHub repo into a MongoDb."
                add_option(opt_db_host, opt_db_name)
                add_option(opt_tokens, opt_show_wallet)
                add_option(opt_verbose, opt_no_colors)
                add_option(opt_show_jobs, opt_no_commits, opt_no_issues, opt_no_comments, opt_no_events)
                add_option(opt_start, opt_clear)
        end

        # MongoDB server used for data persistence
        fun db_host: String do
                return opt_db_host.value or else ini["db.host"] or else default_db_host
        end

        # MongoDB DB used for data persistence
        fun db_name: String do
                return opt_db_name.value or else ini["db.name"] or else default_db_name
        end

        # Mongo db client
        var client = new MongoClient(db_host) is lazy

        # Mongo db instance
        var db: MongoDb = client.database(db_name) is lazy

        # Github tokens used to access data.
        var tokens: Array[String] is lazy do
                var opt_tokens = self.opt_tokens.value
                if opt_tokens.not_empty then return opt_tokens

                var res = new Array[String]
                var ini_tokens = ini.section("tokens")
                if ini_tokens == null then return res

                for token in ini_tokens.values do
                        if token == null then continue
                        res.add token
                end
                return res
        end

        # Github tokens wallet\13
        var wallet: GithubWallet is lazy do
                return new GithubWallet(tokens)
        end

        # Use colors in console display
        fun no_colors: Bool do
                if opt_no_colors.value then return true
                return ini["loader.no_colors"] == "true"
        end

        # Verbosity level (the higher the more verbose)
        fun verbose_level: Int do
                var opt = opt_start.value
                if opt > 0 then
                        return info_level
                end
                var v = ini["loader.verbose"]
                if v != null and v.to_i > 0 then
                        return info_level
                end
                return warn_level
        end

        # Logger used to print things
        var logger: PopLogger is lazy do
                var logger = new PopLogger
                logger.level = verbose_level
                return logger
        end

        # Should we avoid loading branches?
        fun no_branches: Bool do
                if opt_no_branches.value then return true
                return ini["loader.no_branches"] == "true"
        end

        # Should we avoid loading commits?
        fun no_commits: Bool do
                if opt_no_commits.value then return true
                return ini["loader.no_commits"] == "true"
        end

        # Should we avoid loading issues?
        fun no_issues: Bool do
                if opt_no_issues.value then return true
                return ini["loader.no_issues"] == "true"
        end

        # Should we avoid loading issue comments?
        fun no_comments: Bool do
                if opt_no_comments.value then return true
                return ini["loader.no_comments"] == "true"
        end

        # Should we avoid loading events?
        fun no_events: Bool do
                if opt_no_events.value then return true
                return ini["loader.no_events"] == "true"
        end

        # At which issue number should we start?
        fun start_from_issue: Int do
                var opt = opt_start.value
                if opt > 0 then return opt
                var v = ini["loader.start"]
                if v != null then return v.to_i
                return 1
        end
end

redef class GithubWallet
        redef fun api do
                var api = super
                api.enable_cache = true
                return api
        end
end

class Loader

        var config = new LoaderConfig

        # Jobs repository
        var jobs: LoaderJobRepo is lazy do
                return new LoaderJobRepo(config.db.collection("loader_status"))
        end

        var repos: RepoRepo is lazy do
                return new RepoRepo(config.db.collection("repos"))
        end

        var branches: BranchRepo is lazy do
                return new BranchRepo(config.db.collection("branches"))
        end

        var commits: CommitRepo is lazy do
                return new CommitRepo(config.db.collection("commits"))
        end

        var issues: IssueRepo is lazy do
                return new IssueRepo(config.db.collection("issues"))
        end

        var pulls: PullRequestRepo is lazy do
                return new PullRequestRepo(config.db.collection("pull_requests"))
        end

        var issue_comments: IssueCommentRepo is lazy do
                return new IssueCommentRepo(config.db.collection("issue_comments"))
        end

        var issue_events: IssueEventRepo is lazy do
                return new IssueEventRepo(config.db.collection("issue_events"))
        end

        fun start(repo_slug: String) do
                var job = jobs.find_by_id(repo_slug)
                if job == null then
                        log.info "Creating new job for `{repo_slug}`"
                        job = add_job(repo_slug)
                else
                        log.info "Resuming pending job for `{repo_slug}`"
                end
                print "Load history for {job}..."
                get_branches(job)
                get_issues(job)
                finish_job(job)
        end

        fun remove(repo_slug: String) do
                var job = jobs.find_by_id(repo_slug)
                if job == null then
                        log.info "No job found for `{repo_slug}`"
                else
                        jobs.remove_by_id(repo_slug)
                        log.info "Deleted job for `{repo_slug}`"
                end
        end

        # Show wallet status
        fun show_wallet do config.wallet.show_status(config.no_colors)

        # Show jobs status
        fun show_jobs do
                var jobs = jobs.find_all
                print "{jobs.length} jobs pending..."
                for job in jobs do
                        print " * {job}"
                end
                print "\nUse `loader <job> to start a new or resume a pending one"
        end

        # Add a new job
        fun add_job(repo_slug: String): LoaderJob do
                var repo = config.wallet.api.get_repo(repo_slug)
                assert repo != null else
                        error "Repository `{repo_slug}` not found"
                end
                repos.save repo
                var job = new LoaderJob(repo, config.start_from_issue)
                jobs.save job
                return job
        end

        # Finish a job
        fun finish_job(job: LoaderJob) do
                print "Finished job {job}"
                jobs.remove_by_id(job.id)
        end

        fun get_branches(job: LoaderJob) do
                if config.no_branches then return

                var api = config.wallet.api
                var repo = job.repo
                for branch in api.get_repo_branches(repo.full_name) do
                        branch.repo = repo
                        branches.save branch
                        get_commits(job, branch)
                end
        end

        fun get_commits(job: LoaderJob, branch: Branch) do
                if config.no_commits then return
                get_commit(job, branch.commit.sha)
        end

        fun get_commit(job: LoaderJob, commit_sha: String) do
                if commits.find_by_id(commit_sha) != null then return
                var api = config.wallet.api
                var commit = api.get_commit(job.repo.full_name, commit_sha)
                # print commit or else "NULL"
                if commit == null then return
                var message = commit.message or else "no message"
                log.info "Load commit {commit_sha}: {message.split("\n").first}"
                commit.repo = job.repo
                commits.save commit
                var parents = commit.parents
                if parents == null then return
                for parent in parents do
                        get_commit(job, parent.sha)
                end
        end

        # Load game for `repo_name`.
        fun get_issues(job: LoaderJob) do
                if config.no_issues then return

                var api = config.wallet.api
                var page = 1
                var issues = api.get_repo_issues(job.repo.full_name, page, 100)
                while issues.not_empty do
                        for issue in issues do
                                get_issue(job, issue.number)
                                job.last_issue = issue.number
                                jobs.save job
                        end
                end
        end

        # Load an issue or abort.
        private fun get_issue(job: LoaderJob, issue_number: Int) do
                if issues.find_by_id("{job.repo.mongo_id}/{issue_number}") != null then return

                var api = config.wallet.api
                var issue = api.get_issue(job.repo.full_name, issue_number)
                assert issue != null else
                        check_error(api, "Issue #{issue_number} not found")
                end
                if issue.is_pull_request then
                        get_pull(job, issue)
                else
                        log.info "Load issue #{issue.number}: {issue.title.split("\n").first}"
                        issue.repo = job.repo
                        issues.save issue
                        get_issue_events(job, issue)
                end
                get_issue_comments(job, issue)
        end

        # Load issue comments.
        private fun get_issue_comments(job: LoaderJob, issue: Issue) do
                if config.no_comments then return
                var api = config.wallet.api
                for comment in api.get_issue_comments(job.repo.full_name, issue.number) do
                        comment.repo = job.repo
                        issue_comments.save comment
                end
        end

        # Load issue events.
        private fun get_issue_events(job: LoaderJob, issue: Issue) do
                if config.no_events then return

                var api = config.wallet.api
                for event in api.get_issue_events(job.repo.full_name, issue.number) do
                        event.repo = job.repo
                        issue_events.save event
                end
        end

        # Load a pull request or abort.
        private fun get_pull(job: LoaderJob, issue: Issue): PullRequest do
                var api = config.wallet.api
                var pr = api.get_pull(job.repo.full_name, issue.number)
                assert pr != null else
                        check_error(api, "Pull request #{issue.number} not found")
                end
                log.info "Load pull request #{issue.number}: {pr.title.split("\n").first}"
                pr.repo = job.repo
                pulls.save pr
                get_pull_events(job, pr)
                return pr
        end

        # Load pull events.
        private fun get_pull_events(job: LoaderJob, pull: PullRequest) do
                if config.no_events then return

                var api = config.wallet.api
                for event in api.get_issue_events(job.repo.full_name, pull.number) do
                        event.repo = job.repo
                        issue_events.save event
                end
        end

        # Check if the API is in error state then abort
        fun check_error(api: GithubAPI, message: nullable String) do
                var err = api.last_error
                if err != null then
                        error message or else err.message
                end
        end

        # Logger shortcut
        fun log: PopLogger do return config.logger

        # Display a error and exit
        fun error(msg: String) do
                log.error "Error: {msg}"
                exit 1
        end
end

# Loader status by repo
class LoaderJob
        super RepoObject
        serialize

        # Repo this status is about
        var repo: Repo

        # Primary key: the repo id
        redef var id is lazy, serialize_as("_id") do return repo.full_name

        # Last issue loaded
        var last_issue: Int
end

# Loader status repository
class LoaderJobRepo
        super MongoRepository[LoaderJob]
end

class RepoEntity
        serialize

        var repo: nullable Repo = null is writable
end

redef class Repo
        serialize

        var mongo_id: String is lazy, serialize_as("_id") do return full_name
end

class RepoRepo
        super MongoRepository[Repo]
end

redef class Branch
        super RepoEntity
        serialize

        var mongo_id: String is lazy, serialize_as("_id") do
                var repo = self.repo
                if repo == null then return name
                return "{repo.mongo_id}/{name}"
        end
end

class BranchRepo
        super MongoRepository[Branch]

        fun find_by_repo(repo: Repo): Array[Branch] do
                return find_all((new MongoMatch).eq("repo.full_name", repo.full_name))
        end
end

redef class Commit
        super RepoEntity
        serialize

        var mongo_id: String is lazy, serialize_as("_id") do return sha
end

class CommitRepo
        super MongoRepository[Commit]

        fun find_by_repo(repo: Repo): Array[Commit] do
                return find_all((new MongoMatch).eq("repo.full_name", repo.full_name))
        end
end

redef class Issue
        super RepoEntity
        serialize

        var mongo_id: String is lazy, serialize_as("_id") do
                var repo = self.repo
                if repo == null then return number.to_s
                return "{repo.mongo_id}/{number}"
        end
end

class IssueRepo
        super MongoRepository[Issue]

        fun find_by_repo(repo: Repo): Array[Issue] do
                return find_all((new MongoMatch).eq("repo.full_name", repo.full_name))
        end
end

class PullRequestRepo
        super MongoRepository[PullRequest]

        fun find_by_repo(repo: Repo): Array[Issue] do
                return find_all((new MongoMatch).eq("repo.full_name", repo.full_name))
        end
end

redef class IssueComment
        super RepoEntity
        serialize

        var mongo_id: String is lazy, serialize_as("_id") do return id.to_s
end

class IssueCommentRepo
        super MongoRepository[IssueComment]

        fun find_by_repo(repo: Repo): Array[IssueComment] do
                return find_all((new MongoMatch).eq("repo.full_name", repo.full_name))
        end
end

redef class IssueEvent
        super RepoEntity
        serialize

        var mongo_id: String is lazy, serialize_as("_id") do return id.to_s
end

class IssueEventRepo
        super MongoRepository[IssueEvent]

        fun find_by_repo(repo: Repo): Array[IssueEvent] do
                return find_all((new MongoMatch).eq("repo.full_name", repo.full_name))
        end
end

# Init options
var loader = new Loader
loader.config.parse_options(args)

# TODO TMP
loader.jobs.clear
loader.repos.clear
loader.branches.clear
loader.commits.clear
loader.issues.clear
loader.pulls.clear
loader.issue_comments.clear
loader.issue_events.clear

if loader.config.help then
        loader.config.usage
        exit 0
end

if loader.config.opt_show_wallet.value then
        loader.show_wallet
end

var args = loader.config.args
if loader.config.opt_show_jobs.value or args.is_empty then
        loader.show_jobs
end

if args.is_empty then return

if loader.config.opt_clear.value then
        loader.remove args.first
else
        loader.start args.first

        var repo = loader.config.wallet.api.get_repo(args.first)
        if repo == null then return
        print "Loaded"
        print "* {if loader.repos.find_by_id(args.first) != null then 1 else 0} repos"
        print "* {loader.branches.find_by_repo(repo).length} branches"
        print "* {loader.commits.find_by_repo(repo).length} commits"
        print "* {loader.issues.find_by_repo(repo).length} issues"
        print "* {loader.pulls.find_by_repo(repo).length} pulls"
        print "* {loader.issue_comments.find_by_repo(repo).length} comments"
        print "* {loader.issue_events.find_by_repo(repo).length} events"
end