From: Jean Privat Date: Thu, 21 Feb 2019 23:38:28 +0000 (-0500) Subject: Merge: Ci: move services to specific hostnames X-Git-Url: http://nitlanguage.org?hp=103325a7b6aad6f79b59bd22092590d5b482244f Merge: Ci: move services to specific hostnames Testing nit with various services is a PITA. The previous solution was to test them on localhost and requires that the services are available and configured on each test node (it is not always as easy as it seems). Another problem with `localhost` is that testing within docker is complex as running multiple services in a single container is discouraged. Here, we propose to simply move the service from localhost to specific hostnames. This is to be the current "good" practice and is supported out-of-the-box bu various CI infrastructure including gitlab-ci. Pull-Request: #2737 --- diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e792b05..4b0f2ed 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,13 @@ image: nitlang/nit-ci +services: + - mongo + - neo4j:2.3 + - postgres + +variables: + NEO4J_AUTH: none + cache: paths: - .ccache diff --git a/contrib/nitrpg/src/game.nit b/contrib/nitrpg/src/game.nit index b1d8916..5413f43 100644 --- a/contrib/nitrpg/src/game.nit +++ b/contrib/nitrpg/src/game.nit @@ -75,7 +75,7 @@ class Game redef var key = name is lazy # Mongo server url where this game data are stored. - var mongo_url = "mongodb://localhost:27017" is writable + var mongo_url = "mongodb://mongo:27017" is writable # Mongo db client. var client = new MongoClient(mongo_url) is lazy diff --git a/contrib/nitrpg/src/test_helper.nit b/contrib/nitrpg/src/test_helper.nit index 919ccfc..9d71ca3 100644 --- a/contrib/nitrpg/src/test_helper.nit +++ b/contrib/nitrpg/src/test_helper.nit @@ -31,7 +31,7 @@ abstract class NitrpgTestHelper end # Mongo API client - var mongo = new MongoClient("mongodb://localhost:27017/") + var mongo = new MongoClient("mongodb://mongo:27017/") # Load a new test database by with a name private fun load_db(name: String): MongoDb do return mongo.database(name) diff --git a/contrib/nitrpg/src/web.nit b/contrib/nitrpg/src/web.nit index c563af7..3d54672 100644 --- a/contrib/nitrpg/src/web.nit +++ b/contrib/nitrpg/src/web.nit @@ -67,7 +67,7 @@ class RpgAction fun load_games: Array[Game] do var res = new Array[Game] # TODO should be option - var mongo = new MongoClient("mongodb://localhost:27017") + var mongo = new MongoClient("mongodb://mongo:27017") var db = mongo.database("nitrpg") for obj in db.collection("games").find_all(new JsonObject) do var repo = api.load_repo(obj["name"].to_s) diff --git a/lib/github/loader.nit b/lib/github/loader.nit index b2ec70d..4ffa501 100644 --- a/lib/github/loader.nit +++ b/lib/github/loader.nit @@ -29,7 +29,7 @@ class LoaderConfig redef var default_config_file = "loader.ini" # Default database host string for MongoDb - var default_db_host = "mongodb://localhost:27017/" + var default_db_host = "mongodb://mongo:27017/" # Default database hostname var default_db_name = "github_loader" diff --git a/lib/mongodb/mongodb.nit b/lib/mongodb/mongodb.nit index 98b271d..020e0ce 100644 --- a/lib/mongodb/mongodb.nit +++ b/lib/mongodb/mongodb.nit @@ -22,7 +22,7 @@ # # ~~~ # # Opens the connexion with the Mongo server. -# var client = new MongoClient("mongodb://localhost:27017/") +# var client = new MongoClient("mongodb://mongo:27017/") # # # Select the database. # var db_suffix = "NIT_TESTING_ID".environ @@ -197,7 +197,7 @@ end # Usage: # # ~~~ -# var uri = "mongodb://localhost:27017/" +# var uri = "mongodb://mongo:27017/" # var client = new MongoClient(uri) # assert client.server_uri == uri # ~~~ @@ -216,7 +216,7 @@ class MongoClient # Returns `null` if an error occured. See `last_error`. # # ~~~ - # var client = new MongoClient("mongodb://localhost:27017/") + # var client = new MongoClient("mongodb://mongo:27017/") # assert client.server_status["process"] == "mongod" # ~~~ fun server_status: nullable JsonObject do @@ -230,7 +230,7 @@ class MongoClient # Lists available database names. # # ~~~ - # var client = new MongoClient("mongodb://localhost:27017/") + # var client = new MongoClient("mongodb://mongo:27017/") # var db_suffix = "NIT_TESTING_ID".environ # var db_name = "test_{db_suffix}" # var db = client.database(db_name) @@ -259,7 +259,7 @@ class MongoClient # There is no need to create a database manually. # # ~~~ - # var client = new MongoClient("mongodb://localhost:27017/") + # var client = new MongoClient("mongodb://mongo:27017/") # var db_suffix = "NIT_TESTING_ID".environ # var db_name = "test_{db_suffix}" # var db = client.database(db_name) @@ -321,7 +321,7 @@ class MongoDb # Returns `null` if an error occured. See `Sys::last_mongoc_error`. # # ~~~ - # var client = new MongoClient("mongodb://localhost:27017/") + # var client = new MongoClient("mongodb://mongo:27017/") # var db_suffix = "NIT_TESTING_ID".environ # var db_name = "test_{db_suffix}" # var db = client.database(db_name) @@ -346,7 +346,7 @@ class MongoDb # Loads or creates a collection by its `name`. # # ~~~ - # var client = new MongoClient("mongodb://localhost:27017/") + # var client = new MongoClient("mongodb://mongo:27017/") # var db_suffix = "NIT_TESTING_ID".environ # var db_name = "test_{db_suffix}" # var db = client.database(db_name) @@ -360,7 +360,7 @@ class MongoDb # Checks if a collection named `name` exists. # # ~~~ - # var client = new MongoClient("mongodb://localhost:27017/") + # var client = new MongoClient("mongodb://mongo:27017/") # var db_suffix = "NIT_TESTING_ID".environ # var db_name = "test_{db_suffix}" # var db = client.database(db_name) @@ -420,7 +420,7 @@ class MongoCollection # Returns `false` if an error occured. See `Sys::last_mongoc_error`. # # ~~~ - # var client = new MongoClient("mongodb://localhost:27017/") + # var client = new MongoClient("mongodb://mongo:27017/") # var db_suffix = "NIT_TESTING_ID".environ # var db_name = "test_{db_suffix}" # var db = client.database(db_name) @@ -455,7 +455,7 @@ class MongoCollection # Returns `false` if an error occured. See `Sys::last_mongoc_error`. # # ~~~ - # var client = new MongoClient("mongodb://localhost:27017/") + # var client = new MongoClient("mongodb://mongo:27017/") # var db_suffix = "NIT_TESTING_ID".environ # var db_name = "test_{db_suffix}" # var db = client.database(db_name) @@ -488,7 +488,7 @@ class MongoCollection # Returns `false` if an error occured. See `Sys::last_mongoc_error`. # # ~~~ - # var client = new MongoClient("mongodb://localhost:27017/") + # var client = new MongoClient("mongodb://mongo:27017/") # var db_suffix = "NIT_TESTING_ID".environ # var db_name = "test_{db_suffix}" # var db = client.database(db_name) @@ -513,7 +513,7 @@ class MongoCollection # No upsert is done, see `save` instead. # # ~~~ - # var client = new MongoClient("mongodb://localhost:27017/") + # var client = new MongoClient("mongodb://mongo:27017/") # var db_suffix = "NIT_TESTING_ID".environ # var db_name = "test_{db_suffix}" # var db = client.database(db_name) @@ -544,7 +544,7 @@ class MongoCollection # Returns `-1` if an error occured. See `Sys::last_mongoc_error`. # # ~~~ - # var client = new MongoClient("mongodb://localhost:27017/") + # var client = new MongoClient("mongodb://mongo:27017/") # var db_suffix = "NIT_TESTING_ID".environ # var db_name = "test_{db_suffix}" # var db = client.database(db_name) @@ -566,7 +566,7 @@ class MongoCollection # Returns `null` if an error occured. See `Sys::last_mongoc_error`. # # ~~~ - # var client = new MongoClient("mongodb://localhost:27017/") + # var client = new MongoClient("mongodb://mongo:27017/") # var db_suffix = "NIT_TESTING_ID".environ # var db_name = "test_{db_suffix}" # var db = client.database(db_name) @@ -599,7 +599,7 @@ class MongoCollection # * `limit` number of documents to return # # ~~~ - # var client = new MongoClient("mongodb://localhost:27017/") + # var client = new MongoClient("mongodb://mongo:27017/") # var db_suffix = "NIT_TESTING_ID".environ # var db_name = "test_{db_suffix}" # var db = client.database(db_name) @@ -625,7 +625,7 @@ class MongoCollection # Applies an aggregation `pipeline` over the collection. # # ~~~ - # var client = new MongoClient("mongodb://localhost:27017/") + # var client = new MongoClient("mongodb://mongo:27017/") # var db_suffix = "NIT_TESTING_ID".environ # var db_name = "test_{db_suffix}" # var db = client.database(db_name) @@ -665,7 +665,7 @@ class MongoCollection # Returns `null` if an error occured. See `Sys::last_mongoc_error`. # # ~~~ - # var client = new MongoClient("mongodb://localhost:27017/") + # var client = new MongoClient("mongodb://mongo:27017/") # var db_suffix = "NIT_TESTING_ID".environ # var db_name = "test_{db_suffix}" # var db = client.database(db_name) diff --git a/lib/neo4j/neo4j.nit b/lib/neo4j/neo4j.nit index b281c69..8c10f6e 100644 --- a/lib/neo4j/neo4j.nit +++ b/lib/neo4j/neo4j.nit @@ -14,16 +14,10 @@ # Neo4j connector through its JSON REST API using curl. # -# For ease of use and testing this module provide a wrapper to the `neo4j` command: -# -# # Start the Neo4j server -# var srv = new Neo4jServer -# assert srv.start_quiet -# # In order to connect to Neo4j you need a connector: # # # Create new Neo4j client -# var client = new Neo4jClient("http://localhost:7474") +# var client = new Neo4jClient("http://neo4j:7474") # assert client.is_ok # # The fundamental units that form a graph are nodes and relationships. @@ -66,39 +60,9 @@ module neo4j import curl_json import error -# Handles Neo4j server start and stop command -# -# `neo4j` binary must be in `PATH` in order to work -class Neo4jServer - - # Start the local Neo4j server instance - fun start: Bool do - sys.system("neo4j start console") - return true - end - - # Like `start` but redirect the console output to `/dev/null` - fun start_quiet: Bool do - sys.system("neo4j start console > /dev/null") - return true - end - - # Stop the local Neo4j server instance - fun stop: Bool do - sys.system("neo4j stop") - return true - end - - # Like `stop` but redirect the console output to `/dev/null` - fun stop_quiet: Bool do - sys.system("neo4j stop > /dev/null") - return true - end -end - # `Neo4jClient` is needed to communicate through the REST API # -# var client = new Neo4jClient("http://localhost:7474") +# var client = new Neo4jClient("http://neo4j:7474") # assert client.is_ok class Neo4jClient @@ -140,7 +104,7 @@ class Neo4jClient # Save the node in base # - # var client = new Neo4jClient("http://localhost:7474") + # var client = new Neo4jClient("http://neo4j:7474") # # # Create a node # var andres = new NeoNode @@ -196,7 +160,7 @@ class Neo4jClient # Save the edge in base # From and to nodes will be created. # - # var client = new Neo4jClient("http://localhost:7474") + # var client = new Neo4jClient("http://neo4j:7474") # # var andres = new NeoNode # var kate = new NeoNode @@ -245,7 +209,7 @@ class Neo4jClient # Retrieve all nodes with specified `lbl` # - # var client = new Neo4jClient("http://localhost:7474") + # var client = new Neo4jClient("http://neo4j:7474") # # var andres = new NeoNode # andres.labels.add_all(["Human", "Male"]) @@ -271,7 +235,7 @@ class Neo4jClient # Retrieve nodes belonging to all the specified `labels`. # - # var client = new Neo4jClient("http://localhost:7474") + # var client = new Neo4jClient("http://neo4j:7474") # # var andres = new NeoNode # andres.labels.add_all(["Human", "Male"]) @@ -387,7 +351,7 @@ end # # Example: # -# var client = new Neo4jClient("http://localhost:7474") +# var client = new Neo4jClient("http://neo4j:7474") # var query = new CypherQuery # query.nmatch("(n)-[r:LOVES]->(m)") # query.nwhere("n.name=\"Andres\"") @@ -506,7 +470,7 @@ end # Then we can link the entity to the base: # # # Init client -# var client = new Neo4jClient("http://localhost:7474") +# var client = new Neo4jClient("http://neo4j:7474") # client.save_node(andres) # # The node is now linked # assert andres.is_linked @@ -598,7 +562,7 @@ end # # Creating new nodes: # -# var client = new Neo4jClient("http://localhost:7474") +# var client = new Neo4jClient("http://neo4j:7474") # # var andres = new NeoNode # andres.labels.add "Person" @@ -711,7 +675,7 @@ end # # Create a relationship: # -# var client = new Neo4jClient("http://localhost:7474") +# var client = new Neo4jClient("http://neo4j:7474") # # Create nodes # var andres = new NeoNode # andres["name"] = "Andres" @@ -804,7 +768,7 @@ end # # Example: # -# var client = new Neo4jClient("http://localhost:7474") +# var client = new Neo4jClient("http://neo4j:7474") # # var node1 = new NeoNode # var node2 = new NeoNode @@ -1011,7 +975,7 @@ end # This is a representation of a neo job in JSON Format # # Each job description should contain a `to` attribute, with a value relative to the data API root -# (so http://localhost:7474/db/data/node becomes just /node), and a `method` attribute containing +# (so http://neo4j:7474/db/data/node becomes just /node), and a `method` attribute containing # HTTP verb to use. # # Optionally you may provide a `body` attribute, and an `id` attribute to help you keep track diff --git a/lib/popcorn/README.md b/lib/popcorn/README.md index bbc9c95..f7c6346 100644 --- a/lib/popcorn/README.md +++ b/lib/popcorn/README.md @@ -823,7 +823,7 @@ class UserForm end end -var mongo = new MongoClient("mongodb://localhost:27017/") +var mongo = new MongoClient("mongodb://mongo:27017/") var db = mongo.database("mongo_example") var app = new App diff --git a/lib/popcorn/examples/mongodb/example_mongodb.nit b/lib/popcorn/examples/mongodb/example_mongodb.nit index b7de4fe..00b1421 100644 --- a/lib/popcorn/examples/mongodb/example_mongodb.nit +++ b/lib/popcorn/examples/mongodb/example_mongodb.nit @@ -60,7 +60,7 @@ class UserList end end -var mongo = new MongoClient("mongodb://localhost:27017/") +var mongo = new MongoClient("mongodb://mongo:27017/") var db = mongo.database("mongo_example") var app = new App diff --git a/lib/popcorn/pop_repos.nit b/lib/popcorn/pop_repos.nit index 72003ec..c054d5a 100644 --- a/lib/popcorn/pop_repos.nit +++ b/lib/popcorn/pop_repos.nit @@ -106,7 +106,7 @@ # # Let's wrap it all together in a Popcorn app: # # # Init database -# var mongo = new MongoClient("mongodb://localhost:27017/") +# var mongo = new MongoClient("mongodb://mongo:27017/") # var db = mongo.database("tests_app_{100000.rand}") # var coll = db.collection("books") # @@ -126,7 +126,7 @@ import mongodb::queries redef class AppConfig # Default database host string for MongoDb - var default_db_host = "mongodb://localhost:27017/" + var default_db_host = "mongodb://mongo:27017/" # Default database hostname var default_db_name = "popcorn" @@ -290,7 +290,7 @@ end # # The repository can then be used with User instances: # # # Init database -# var mongo = new MongoClient("mongodb://localhost:27017/") +# var mongo = new MongoClient("mongodb://mongo:27017/") # var db = mongo.database("tests") # var coll = db.collection("test_pop_repo_{100000.rand}") # diff --git a/lib/postgresql/postgres.nit b/lib/postgresql/postgres.nit index d8a475e..6c209ef 100644 --- a/lib/postgresql/postgres.nit +++ b/lib/postgresql/postgres.nit @@ -37,7 +37,7 @@ # animals.add(turtle) # # var db_suffix = "NIT_TESTING_ID".environ -# var db = new Postgres.open("dbname=postgres") +# var db = new Postgres.open("host=postgres user=postgres dbname=postgres") # # assert db_is_open: not db.is_closed # assert create_table: db.create_table("IF NOT EXISTS animals_{db_suffix} (aname TEXT PRIMARY KEY, kind TEXT NOT NULL, age INT NOT NULL)") else print db.error diff --git a/misc/docker/ci-local/README.md b/misc/docker/ci-local/README.md new file mode 100644 index 0000000..4ac947a --- /dev/null +++ b/misc/docker/ci-local/README.md @@ -0,0 +1,34 @@ +# Local setup of services for tests + +Some tests and libs require specific external services like postgres. +To simplify the automation and the setup, we just assume these services are accessible at specific hostnames. + +When running and testing nit within a docker, it is easy to compose/link nit with other dockers/services. +See also .gitlab-ci.yml + +~~~ +$ docker run --link postgres [...] +~~~ + +When running and testing on a local host, one can setup those services locally then add aliases for localhost in `/etc/hosts`. + +~~~ +$ cat /etc/hosts +[...] +127.0.0.1 postgres +~~~ + +An other way is to run/test on a local host and have the services running in dockers. + +Because accessing a specific container by its name is cumbersome, a proposed way is to: + +* run these services with specific and fixed IPs +* add these IPs to `/etc/hosts` + +~~~ +$ cd misc/docker/ci-local +$ docker-compose up -d +$ cat /etc/hosts +[...] +172.16.238.4 postgres +~~~ diff --git a/misc/docker/ci-local/docker-compose.yml b/misc/docker/ci-local/docker-compose.yml new file mode 100644 index 0000000..aabc9f7 --- /dev/null +++ b/misc/docker/ci-local/docker-compose.yml @@ -0,0 +1,29 @@ +version: '2' +services: + mongo: + image: mongo + restart: always + networks: + nitci: + ipv4_address: 172.16.238.2 + neo4j: + image: neo4j:2.3 + restart: always + environment: + NEO4J_AUTH: none + networks: + nitci: + ipv4_address: 172.16.238.3 + postgres: + image: postgres + restart: always + networks: + nitci: + ipv4_address: 172.16.238.4 +networks: + nitci: + driver: bridge + ipam: + driver: default + config: + - subnet: 172.16.238.0/24 diff --git a/misc/docker/ci/Dockerfile b/misc/docker/ci/Dockerfile index 0b33333..000c0c1 100644 --- a/misc/docker/ci/Dockerfile +++ b/misc/docker/ci/Dockerfile @@ -47,7 +47,8 @@ RUN dpkg --add-architecture i386 \ libsqlite3-dev \ libx11-dev \ libxdg-basedir-dev \ - postgresql \ + netcat \ + psmisc \ # Packages needed for contrib, platforms and FFI ant \ clang \ diff --git a/tests/gitlab_ci.skip b/tests/gitlab_ci.skip index 8467aad..c75645c 100644 --- a/tests/gitlab_ci.skip +++ b/tests/gitlab_ci.skip @@ -1,10 +1,6 @@ emscripten java glsl -mongo mpi -neo objc -postgres -nitrpg wiringPi diff --git a/tests/test_neo.args b/tests/test_neo.args index a1ee088..46fe939 100644 --- a/tests/test_neo.args +++ b/tests/test_neo.args @@ -1 +1 @@ -localhost 7474 test_prog/test_prog.nit +neo4j 7474 test_prog/test_prog.nit diff --git a/tests/test_neo4j.nit b/tests/test_neo4j.nit index 974a80a..d240047 100644 --- a/tests/test_neo4j.nit +++ b/tests/test_neo4j.nit @@ -17,12 +17,9 @@ import neo4j # key used to loosely assume unicity and prevent conflicting db accesses var key = "NIT_TESTING_ID".environ.to_i -var srv = new Neo4jServer -srv.start_quiet - print "# Test local\n" -var client = new Neo4jClient("http://localhost:7474") +var client = new Neo4jClient("http://neo4j:7474") assert client.is_ok # Clear the previous objects, if any @@ -91,7 +88,7 @@ print "{kate["name"].to_s} IS LOVED BY {kate.in_nodes("LOVES").first["name"].to_ print "\n# Test lazy\n" -client = new Neo4jClient("http://localhost:7474/") +client = new Neo4jClient("http://neo4j:7474/") assert client.is_ok # Read Andres diff --git a/tests/test_neo4j_batch.nit b/tests/test_neo4j_batch.nit index 464eb2f..499f7ef 100644 --- a/tests/test_neo4j_batch.nit +++ b/tests/test_neo4j_batch.nit @@ -14,9 +14,6 @@ import neo4j -var srv = new Neo4jServer -srv.start_quiet - # key used to loosely assume unicity and prevent conflicting db accesses var key = "NIT_TESTING_ID".environ.to_i @@ -37,7 +34,7 @@ kate["status"] = false var loves = new NeoEdge(andres, "LOVES", kate) loves["since"] = 1999 -var client = new Neo4jClient("http://localhost:7474") +var client = new Neo4jClient("http://neo4j:7474") assert client.is_ok # Clear the previous objects, if any @@ -64,7 +61,7 @@ var andres_url = andres.url.to_s var kate_url = kate.url.to_s var loves_url = loves.url.to_s -client = new Neo4jClient("http://localhost:7474") +client = new Neo4jClient("http://neo4j:7474") assert client.is_ok # Read Andres diff --git a/tests/test_postgres_native.nit b/tests/test_postgres_native.nit index b7c5a49..d718469 100644 --- a/tests/test_postgres_native.nit +++ b/tests/test_postgres_native.nit @@ -19,7 +19,7 @@ module test_postgres_native import postgresql::native_postgres var db_suffix = "NIT_TESTING_ID".environ -var db = new NativePostgres.connectdb("dbname=postgres") +var db = new NativePostgres.connectdb("host=postgres user=postgres dbname=postgres") assert postgres_open: db.status.is_ok else print_error db.error var result = db.exec("CREATE TABLE IF NOT EXISTS animals_{db_suffix} (aname TEXT PRIMARY KEY, class TEXT NOT NULL, sex INTEGER)") diff --git a/tests/test_postgres_nity.nit b/tests/test_postgres_nity.nit index 4421a02..6714a04 100644 --- a/tests/test_postgres_nity.nit +++ b/tests/test_postgres_nity.nit @@ -19,7 +19,7 @@ module test_postgres_nity import postgresql::postgres var db_suffix = "NIT_TESTING_ID".environ -var db = new Postgres.open("dbname=postgres") +var db = new Postgres.open("host=postgres user=postgres dbname=postgres") assert open_db: not db.is_closed else print db.error assert create_table: db.create_table("IF NOT EXISTS users_{db_suffix} (uname TEXT PRIMARY KEY, pass TEXT NOT NULL, activated INTEGER, perc FLOAT)") else