Github OAuth tokens management

When using batch mode with the github API, we can rapidly reach the rate limit allowed by Github.

One solution consists in using a wallet of tokens so we can rely on more than one token and switch them when one become exhausted.

Using the Github wallet to check tokens

One functionality of the wallet is to check the validity of a token against the API. check_token will return false if a token is invalid or exhausted.

var wallet = new GithubWallet
assert not wallet.check_token("this is a bad token")

Storing tokens

The wallet can also be used to store tokens and check all of them.

wallet = new GithubWallet
wallet.add "some token"
wallet.add "some other token"

or

wallet = new GithubWallet(["token 1", "token 2"])

The show_status method can be used to display a summary of the validity of each token in the wallet.

wallet.show_status

Will display something like this:

Wallet (2 tokens):
 * [OK] token 1
 * [KO] token 2

Using the wallet to obtain a Github API client

Using the wallet you can cycle through tokens and obtain a new Github API client instance with a fresh rate limit.

wallet = new GithubWallet(["token 1", "token 2"])
var api = wallet.api

The wallet will automatically cycle through the registered tokens to find one that works.

If no valid token is found after all of them was tried, the wallet returns a client based on the last tried one.

Introduced classes

class GithubWallet

github :: GithubWallet

Github OAuth tokens wallet

All class definitions

class GithubWallet

github $ GithubWallet

Github OAuth tokens wallet
package_diagram github::wallet wallet github github github::wallet->github logger logger github::wallet->logger github->logger base64 base64 github->base64 curl curl github->curl json json github->json nitcorn nitcorn github->nitcorn popcorn popcorn github->popcorn ...base64 ... ...base64->base64 ...curl ... ...curl->curl ...json ... ...json->json ...nitcorn ... ...nitcorn->nitcorn ...popcorn ... ...popcorn->popcorn ...logger ... ...logger->logger github::loader loader github::loader->github::wallet a_star-m a_star-m a_star-m->github::loader a_star-m... ... a_star-m...->a_star-m

Ancestors

module abstract_collection

core :: abstract_collection

Abstract collection classes and services.
module abstract_text

core :: abstract_text

Abstract class for manipulation of sequences of characters
module api

github :: api

Nit object oriented interface to Github api.
module array

core :: array

This module introduces the standard array structure.
module base64

base64 :: base64

Offers the base 64 encoding and decoding algorithms
module bitset

core :: bitset

Services to handle BitSet
module bytes

core :: bytes

Services for byte streams and arrays
module cache

github :: cache

Enable caching on Github API accesses.
module caching

serialization :: caching

Services for caching serialization engines
module circular_array

core :: circular_array

Efficient data structure to access both end of the sequence.
module codec_base

core :: codec_base

Base for codecs to use with streams
module codecs

core :: codecs

Group module for all codec-related manipulations
module collection

core :: collection

This module define several collection classes.
module console

console :: console

Defines some ANSI Terminal Control Escape Sequences.
module core

core :: core

Standard classes and methods used by default by Nit programs and libraries.
module curl

curl :: curl

Data transfer powered by the native curl library
module engine_tools

serialization :: engine_tools

Advanced services for serialization engines
module environ

core :: environ

Access to the environment variables of the process
module error

json :: error

Intro JsonParseError which is exposed by all JSON reading APIs
module error

core :: error

Standard error-management infrastructure.
module exec

core :: exec

Invocation and management of operating system sub-processes.
module file

core :: file

File manipulations (create, read, write, etc.)
module fixed_ints

core :: fixed_ints

Basic integers of fixed-precision
module fixed_ints_text

core :: fixed_ints_text

Text services to complement fixed_ints
module flat

core :: flat

All the array-based text representations
module gc

core :: gc

Access to the Nit internal garbage collection mechanism
module hash_collection

core :: hash_collection

Introduce HashMap and HashSet.
module inspect

serialization :: inspect

Refine Serializable::inspect to show more useful information
module iso8859_1

core :: iso8859_1

Codec for ISO8859-1 I/O
module json

json :: json

Read and write JSON formatted text using the standard serialization services
module kernel

core :: kernel

Most basic classes and methods.
module list

core :: list

This module handle double linked lists
module math

core :: math

Mathematical operations
module meta

meta :: meta

Simple user-defined meta-level to manipulate types of instances as object.
module native

core :: native

Native structures for text and bytes
module native_curl

curl :: native_curl

Binding of C libCurl which allow us to interact with network.
module numeric

core :: numeric

Advanced services for Numeric types
module parser_base

parser_base :: parser_base

Simple base for hand-made parsers of all kinds
module poset

poset :: poset

Pre order sets and partial order set (ie hierarchies)
module protocol

core :: protocol

module queue

core :: queue

Queuing data structures and wrappers
module range

core :: range

Module for range of discrete objects.
module re

core :: re

Regular expression support for all services based on Pattern
module ropes

core :: ropes

Tree-based representation of a String.
module safe

serialization :: safe

Services for safer deserialization engines
module serialization

serialization :: serialization

General serialization services
module serialization_core

serialization :: serialization_core

Abstract services to serialize Nit objects to different formats
module serialization_read

json :: serialization_read

Services to read JSON: deserialize_json and JsonDeserializer
module serialization_write

json :: serialization_write

Services to write Nit objects to JSON strings: serialize_to_json and JsonSerializer
module sorter

core :: sorter

This module contains classes used to compare things and sorts arrays.
module static

json :: static

Static interface to read Nit objects from JSON strings
module store

json :: store

Store and load json data.
module stream

core :: stream

Input and output streams of characters
module text

core :: text

All the classes and methods related to the manipulation of text entities
module time

core :: time

Management of time and dates
module union_find

core :: union_find

union–find algorithm using an efficient disjoint-set data structure
module utf8

core :: utf8

Codec for UTF-8 I/O

Parents

module github

github :: github

Nit wrapper for Github API
module logger

logger :: logger

A simple logger for Nit

Children

module loader

github :: loader

Descendants

module a_star-m

a_star-m

# Github OAuth tokens management
#
# When using batch mode with the `github` API, we can rapidly reach the rate
# limit allowed by Github.
#
# One solution consists in using a wallet of tokens so we can rely on more than
# one token and switch them when one become exhausted.
#
# ## Using the Github wallet to check tokens
#
# One functionality of the wallet is to check the validity of a token against
# the API. `check_token` will return false if a token is invalid or exhausted.
#
# ~~~
# var wallet = new GithubWallet
# assert not wallet.check_token("this is a bad token")
# ~~~
#
# ## Storing tokens
#
# The wallet can also be used to store tokens and check all of them.
#
# ~~~
# wallet = new GithubWallet
# wallet.add "some token"
# wallet.add "some other token"
# ~~~
#
# or
#
# ~~~
# wallet = new GithubWallet(["token 1", "token 2"])
# ~~~
#
# The `show_status` method can be used to display a summary of the validity of
# each token in the wallet.
#
# ~~~
# wallet.show_status
# ~~~
#
# Will display something like this:
#
# ~~~raw
# Wallet (2 tokens):
#  * [OK] token 1
#  * [KO] token 2
# ~~~
#
# ## Using the wallet to obtain a Github API client
#
# Using the wallet you can cycle through tokens and obtain a new Github API client
# instance with a fresh rate limit.
#
# ~~~
# wallet = new GithubWallet(["token 1", "token 2"])
# var api = wallet.api
# ~~~
#
# The wallet will automatically cycle through the registered tokens to find one
# that works.
#
# If no valid token is found after all of them was tried, the wallet returns a
# client based on the last tried one.
module wallet

import github
import logger

# Github OAuth tokens wallet
class GithubWallet

	# Github API tokens
	var tokens = new Array[String] is optional

	# Logger used to display info about tokens state
	var logger = new Logger is optional, writable

	# Add a new token in the wallet
	fun add(token: String) do tokens.add token

	# Get an instance of GithubAPI based on the next available token.
	#
	# If no token is found, return an api based on the last exhausted token.
	fun api: GithubAPI do
		var token
		if tokens.is_empty then
			logger.warn "No tokens, using `get_github_oauth`"
			token = get_github_oauth
		else
			token = get_next_token
			var tried = 0
			while not check_token(token) do
				if tried >= tokens.length - 1 then
					logger.warn "Exhausted all tokens, using {token}"
					break
				end
				tried += 1
				token = get_next_token
			end
		end
		var api = new GithubAPI(token)
		api.enable_cache = true
		return api
	end

	# The current index in the `tokens` array
	private var current_index = 0

	# The current token in the `tokens` array based on `current_index`
	fun current_token: String do return tokens[current_index]

	# Get the next token in token `array` based on `current_token`.
	#
	# If the end of the list is reached, start again from the begining.
	fun get_next_token: String do
		if tokens.is_empty then
			return get_github_oauth
		end
		var token = current_token

		if current_index < tokens.length - 1 then
			current_index += 1
		else
			current_index = 0
		end
		return token
	end

	# Check if a token is valid
	fun check_token(token: String): Bool do
		logger.debug "Try token {token}"
		var api = new GithubAPI(token)
		api.get_auth_user
		return not api.was_error
	end

	# Show wallet status in console
	fun show_status(no_color: nullable Bool) do
		no_color = no_color or else false

		if tokens.is_empty then
			print "Wallet is empty"
			return
		end
		print "Wallet ({tokens.length} tokens):"
		for token in tokens do
			var status
			if check_token(token) then
				status = if no_color then "OK" else "OK".green
			else
				status = if no_color then "KO" else "KO".red
			end
			print " * [{status}] {token}"
		end
	end
end
lib/github/wallet.nit:17,1--173,3