From: Jean Privat Date: Thu, 21 Feb 2019 23:38:28 +0000 (-0500) Subject: Merge: Ci: move services to specific hostnames X-Git-Url: http://nitlanguage.org?hp=01a0edd36a994f0063960a1f39387a4a4397b2e4 Merge: Ci: move services to specific hostnames Testing nit with various services is a PITA. The previous solution was to test them on localhost and requires that the services are available and configured on each test node (it is not always as easy as it seems). Another problem with `localhost` is that testing within docker is complex as running multiple services in a single container is discouraged. Here, we propose to simply move the service from localhost to specific hostnames. This is to be the current "good" practice and is supported out-of-the-box bu various CI infrastructure including gitlab-ci. Pull-Request: #2737 --- diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0c4e0a7..4b0f2ed 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,6 +11,8 @@ variables: cache: paths: - .ccache + - .gradle/caches + - .gradle/wrapper key: "$CI_JOB_NAME" stages: @@ -22,16 +24,19 @@ stages: before_script: - date - export CCACHE_DIR=$PWD/.ccache + - export GRADLE_USER_HOME=$PWD/.gradle - export PATH=$PWD/bin:$PATH - pwd - ccache -s - ccache -M 500M + - du -sh .gradle || true - type -a nitc nitdoc || true # is there some nit tools? - "> status.txt" after_script: - export CCACHE_DIR=$PWD/.ccache - ccache -s + - du -sh .gradle || true - git status --ignored - date - tail status.txt @@ -89,7 +94,7 @@ nitunit_some: dependencies: - build_tools script: - - git diff --name-only origin/master..HEAD -- "*.nit" "*.res" "README.*" | grep -v "^tests/" > list0.txt + - git diff --name-only origin/master..HEAD -- "*.nit" "*.res" "README.*" | grep -v "^tests/" > list0.txt || true - xargs nitls -pP < list0.txt > list.txt - xargs nitunit < list.txt artifacts: @@ -106,9 +111,19 @@ nitpick_full: - build_tools script: - nitls lib src examples contrib - - nitls -Pp lib src examples | grep -v -f tests/gitlab_ci.skip > list.txt # filter what is skipped by tests.sh + - nitls -Pp lib src examples | grep -v -f tests/gitlab_ci.skip > list.txt || true # filter what is skipped by tests.sh - xargs nitpick < list.txt +basic_android: + stage: test + dependencies: + - build_tools + script: + - make -C contrib/asteronits android + artifacts: + paths: + - contrib/asteronits/bin/*.apk + # TEST FULL ######################################################### test_full_nitcs: @@ -208,7 +223,7 @@ nitunit_lib: dependencies: - build_tools script: - - nitls -Pp lib | grep -v -f tests/gitlab_ci.skip > list.txt # filter what is skipped by tests.sh + - nitls -Pp lib | grep -v -f tests/gitlab_ci.skip > list.txt || true # filter what is skipped by tests.sh - xargs nitunit -v < list.txt| tee log.txt - grep -e KO log.txt > status.txt || true - tail -3 log.txt >> status.txt @@ -225,7 +240,7 @@ nitunit_src: dependencies: - build_tools script: - - nitls -Pp src examples | grep -v -f tests/gitlab_ci.skip > list.txt # filter what is skipped by tests.sh + - nitls -Pp src examples | grep -v -f tests/gitlab_ci.skip > list.txt || true # filter what is skipped by tests.sh - xargs nitunit -v < list.txt| tee log.txt - grep -e KO log.txt > status.txt || true - tail -3 log.txt >> status.txt @@ -247,6 +262,21 @@ test_contribs: - test ! -s status.txt # no lines, no errors allow_failure: true +test_contribs_android: + stage: more_test + dependencies: + - build_tools + script: + - misc/jenkins/check_contrib.sh android + - grep 'error message' *.xml > status.txt || true + - mkdir -p apk + - find . -name '*.apk' -exec cp {} apk/ ";" + - test ! -s status.txt # no lines, no errors + artifacts: + paths: + - "apk" + when: always + build_oot: stage: more_test dependencies: diff --git a/lib/android/README.md b/lib/android/README.md index ffd961c..c948c48 100644 --- a/lib/android/README.md +++ b/lib/android/README.md @@ -23,33 +23,33 @@ it may be possible to support other platforms with some tweaks. You will probably need to tweak it to you system or update the download URL to the latest SDK tools from https://developer.android.com/studio/index.html#command-tools - ~~~ - # Fetch and extract SDK tools - mkdir -p ~/Android/Sdk - cd ~/Android/Sdk - wget https://dl.google.com/android/repository/sdk-tools-linux-3859397.zip - unzip sdk-tools-linux-3859397.zip + ~~~raw + # Fetch and extract SDK tools + mkdir -p ~/Android/Sdk + cd ~/Android/Sdk + wget https://dl.google.com/android/repository/sdk-tools-linux-3859397.zip + unzip sdk-tools-linux-3859397.zip - # Update tools - tools/bin/sdkmanager --update + # Update tools + tools/bin/sdkmanager --update - # Accept the licenses - tools/bin/sdkmanager --licenses + # Accept the licenses + tools/bin/sdkmanager --licenses - # Install the basic build tools - tools/bin/sdkmanager "build-tools;27.0.0" ndk-bundle - ~~~ + # Install the basic build tools + tools/bin/sdkmanager "build-tools;27.0.0" ndk-bundle + ~~~ 3. Set the environment variable ANDROID_HOME to the SDK installation directory, usually `~/Android/Sdk/`. Use the following command to setup the variable for bash. - ~~~ + ~~~raw echo "export ANDROID_HOME=~/Android/Sdk/" >> ~/.bashrc ~~~ 4. Install Java 8 JDK, on Debian/Ubuntu systems you can use the following command: - ~~~ + ~~~raw sudo apt install openjdk-8-jdk ~~~ @@ -71,7 +71,7 @@ and `android_manifest_activity`. Example usage to specify an extra permission: - ~~~ + ~~~raw android_manifest """""" ~~~ @@ -143,7 +143,7 @@ as they change between versions of the Java SDK. You should instead use a command similar to the following, replacing `KEYSTORE_PATH` and `KEY_ALIAS` with the desired values. - ~~~ + ~~~raw keytool -genkey -keystore KEYSTORE_PATH -alias KEY_ALIAS -sigalg MD5withRSA -keyalg RSA -keysize 1024 -validity 10000 ~~~ @@ -154,7 +154,7 @@ optionally `TSA_SERVER`. These settings can be set in a startup script such as You can use the following commands by replacing the right-hand values to your own configuration. - ~~~ + ~~~raw export KEYSTORE=keystore_path export KEY_ALIAS=key_alias export TSA_SERVER=timestamp_authority_server_url # Optional diff --git a/lib/curl/curl.nit b/lib/curl/curl.nit index 34b2576..18f05c8 100644 --- a/lib/curl/curl.nit +++ b/lib/curl/curl.nit @@ -16,27 +16,20 @@ # Data transfer powered by the native curl library # -# Download or upload over HTTP with `CurlHTTPRequest` and send emails with `CurlMail`. +# Download or upload data over HTTP with `CurlHTTPRequest` and send emails +# with `CurlMail`. Scripts can use the easier (but limited) services on `Text`, +# `http_get` and `http_download`, provided by `curl::extra`. module curl import native_curl -redef class Sys - # Shared Curl library handle - # - # Usually, you do not have to use this attribute, it instancied by `CurlHTTPRequest` and `CurlMail`. - # But in some cases you may want to finalize it to free some small resources. - # However, if Curl services are needed once again, this attribute must be manually set. - var curl: Curl = new Curl is lazy, writable -end - -# Curl library handle, it is initialized and released with this class -class Curl +# Curl library handle +private class Curl super FinalizableOnce - private var native = new NativeCurl.easy_init + var native = new NativeCurl.easy_init - # Check for correct initialization + # Is this instance correctly initialized? fun is_ok: Bool do return self.native.is_init redef fun finalize_once do if is_ok then native.easy_clean @@ -45,7 +38,7 @@ end # CURL Request class CurlRequest - private var curl: Curl = sys.curl + private var curl = new Curl # Shall this request be verbose? var verbose: Bool = false is writable @@ -71,6 +64,14 @@ class CurlRequest do return new CurlResponseFailed(error_code, error_msg) end + + # Close low-level resources associated to this request + # + # Once closed, this request can't be used again. + # + # If this service isn't called explicitly, low-level resources + # may be freed automatically by the GC. + fun close do curl.finalize end # HTTP request builder @@ -135,7 +136,6 @@ class CurlHTTPRequest do # Reset libcurl parameters as the lib is shared and options # might affect requests from one another. - self.curl.native = new NativeCurl.easy_init if not self.curl.is_ok then return answer_failure(0, "Curl instance is not correctly initialized") var success_response = new CurlResponseSuccess @@ -153,8 +153,6 @@ class CurlHTTPRequest var st_code = self.curl.native.easy_getinfo_long(new CURLInfoLong.response_code) if not st_code == null then success_response.status_code = st_code - self.curl.native.easy_clean - return success_response end @@ -279,6 +277,8 @@ class CurlHTTPRequest # Download to file given resource fun download_to_file(output_file_name: nullable String): CurlResponse do + if not self.curl.is_ok then return answer_failure(0, "Curl instance is not correctly initialized") + var success_response = new CurlFileResponseSuccess var callback_receiver: CurlCallbacks = success_response @@ -447,7 +447,6 @@ class CurlMail # Execute Mail request with settings configured through attribute fun execute: nullable CurlResponseFailed do - self.curl.native = new NativeCurl.easy_init if not self.curl.is_ok then return answer_failure(0, "Curl instance is not correctly initialized") var lines = new Array[String] @@ -521,8 +520,6 @@ class CurlMail var err_resp = perform if err_resp != null then return err_resp - self.curl.native.easy_clean - return null end end @@ -540,7 +537,10 @@ end class CurlResponseFailed super CurlResponse + # Curl error code var error_code: Int + + # Curl error message var error_msg: String redef fun to_s do return "{error_msg} ({error_code})" @@ -568,23 +568,27 @@ end class CurlResponseSuccess super CurlResponseSuccessIntern - var body_str = "" + # Server HTTP response code var status_code = 0 - # Receive body from request due to body callback registering - redef fun body_callback(line) do - self.body_str = "{self.body_str}{line}" - end + # Response body as a `String` + var body_str = "" + + # Accept part of the response body + redef fun body_callback(line) do self.body_str += line end # Success Response Class of a downloaded File class CurlFileResponseSuccess super CurlResponseSuccessIntern + # Server HTTP response code var status_code = 0 + var speed_download = 0.0 var size_download = 0.0 var total_time = 0.0 + private var file: nullable FileWriter = null # Receive bytes stream from request due to stream callback registering @@ -621,7 +625,7 @@ class HeaderMap # Get `self` as a single string for HTTP POST # # Require: `curl.is_ok` - fun to_url_encoded(curl: Curl): String + private fun to_url_encoded(curl: Curl): String do assert curl.is_ok diff --git a/lib/curl/extra.nit b/lib/curl/extra.nit new file mode 100644 index 0000000..93548d8 --- /dev/null +++ b/lib/curl/extra.nit @@ -0,0 +1,89 @@ +# This file is part of NIT ( http://www.nitlanguage.org ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Shortcut services for scripts: `http_get` and `http_download` +module extra + +import curl + +redef class Text + + # Execute a simple HTTP GET request to the URL `self` + # + # Set `accept_status_code` to the expected response HTTP code, defaults to 200. + # If a different status code is received, the return code is printed to stderr. + # + # Returns the response body on success and `null` on error. Prints the error + # message to stderr. + # + # For more control, set HTTP request headers, keep the response status code + # and much more, use `CurlHTTPRequest`. + # + # ~~~nitish + # assert "http://example.com/".http_get != null + # ~~~ + fun http_get(accept_status_code: nullable Int): nullable String + do + var req = new CurlHTTPRequest(self.to_s) + var resp = req.execute + req.close + + if resp isa CurlResponseSuccess then + if resp.status_code == (accept_status_code or else 200) then + return resp.body_str + else + print_error "HTTP request failed: server returned {resp.status_code}" + end + else if resp isa CurlResponseFailed then + print_error "HTTP request failed: {resp.error_msg}" + else abort + return null + end + + # Download the file at URL `self` to `output_path` with a simple HTTP request + # + # If not set, `output_path` defaults to `self.basename`. + # + # Set `accept_status_code` to the expected response HTTP code, defaults to 200. + # If a different status code is received, the return code is printed to stderr. + # + # Returns the path to the downloaded file on success and `null` on errors. + # Prints the error message to stderr. + # + # For more control, set HTTP request headers, keep the response status code + # and much more, use `CurlHTTPRequest`. + # + # ~~~nitish + # assert "http://example.com/".http_download("index.html") == "example.com" + # ~~~ + fun http_download(output_path: nullable Text, accept_status_code: nullable Int): nullable String + do + var path = (output_path or else self.basename).to_s + + var req = new CurlHTTPRequest(self.to_s) + var resp = req.download_to_file(path) + req.close + + if resp isa CurlFileResponseSuccess then + if resp.status_code == (accept_status_code or else 200) then + return path + else + print_error "HTTP request failed: server returned {resp.status_code}" + end + else if resp isa CurlResponseFailed then + print_error "HTTP request failed: {resp.error_msg}" + else abort + return null + end +end diff --git a/lib/neo4j/neo4j.nit b/lib/neo4j/neo4j.nit index a5f128a..8c10f6e 100644 --- a/lib/neo4j/neo4j.nit +++ b/lib/neo4j/neo4j.nit @@ -75,8 +75,6 @@ class Neo4jClient # REST service to send cypher requests private var cypher_url: String - private var curl = new Curl - init(base_url: String) do self.base_url = base_url var root = service_root diff --git a/lib/popcorn/pop_auth.nit b/lib/popcorn/pop_auth.nit index 2f84be7..3dd71fb 100644 --- a/lib/popcorn/pop_auth.nit +++ b/lib/popcorn/pop_auth.nit @@ -197,9 +197,6 @@ class GithubOAuthCallBack return end - # FIXME reinit curl before next request to avoid weird 404 - curl = new Curl - # Load github user var gh_api = new GithubAPI(access_token) var user = gh_api.load_auth_user diff --git a/misc/docker/ci/Dockerfile b/misc/docker/ci/Dockerfile index ca64f05..000c0c1 100644 --- a/misc/docker/ci/Dockerfile +++ b/misc/docker/ci/Dockerfile @@ -57,6 +57,7 @@ RUN dpkg --add-architecture i386 \ file \ gnupg \ gnuplot-nox \ + imagemagick \ inkscape \ libopenmpi-dev \ time \ @@ -72,11 +73,37 @@ RUN dpkg --add-architecture i386 \ && apt-get install -y nodejs \ && rm -rf /var/lib/apt/lists/* +# Install OpenGL validator +RUN git clone https://github.com/KhronosGroup/glslang.git --depth=1 \ + && mkdir -p glslang/build \ + && cd glslang/build \ + && cmake .. \ + && make \ + && make install + +# Install android sdk/ndk +RUN mkdir -p /opt \ + && cd /opt \ + # Android SDK + && curl https://dl.google.com/android/repository/sdk-tools-linux-3859397.zip -o android-sdk-linux.zip \ + && unzip -q android-sdk-linux.zip -d android-sdk-linux \ + # Download a specific ndk version because old versions are not available trough sdkmanager + && curl https://dl.google.com/android/repository/android-ndk-r17c-linux-x86_64.zip -o android-ndk-linux.zip \ + && unzip -q android-ndk-linux.zip \ + && mv android-ndk-r17c android-sdk-linux/ndk-bundle \ + && rm android-sdk-linux.zip android-ndk-linux.zip +RUN cd /opt \ + && yes | android-sdk-linux/tools/bin/sdkmanager "build-tools;27.0.0" "cmake;3.6.4111459" platform-tools tools --verbose\ + && yes | android-sdk-linux/tools/bin/sdkmanager --licenses --verbose +# TODO: predownload bwdgc and gradle? + # Setup environment variables +ENV ANDROID_HOME=/opt/android-sdk-linux/ ENV JAVA_HOME=/usr/lib/jvm/default-java/ ENV JNI_LIB_PATH=$JAVA_HOME/jre/lib/amd64/server/ ENV LD_LIBRARY_PATH=$JAVA_HOME/jre/lib/amd64/server/ +# Used by CI to render junit files to html RUN pip3 install junit2html # Prepare to install npm (npm is not packaged for debian:stretch) diff --git a/share/android-bdwgc/setup.sh b/share/android-bdwgc/setup.sh index 7ea0c27..334a54b 100755 --- a/share/android-bdwgc/setup.sh +++ b/share/android-bdwgc/setup.sh @@ -25,4 +25,4 @@ git clone --depth=1 -b android https://github.com/xymus/bdwgc.git || exit 1 # Setup libatomic_ops too cd bdwgc || exit 1 git submodule init || exit 1 -git submodule update --depth=1 || exit 1 +git submodule update || exit 1 diff --git a/tests/gitlab_ci.skip b/tests/gitlab_ci.skip index 5d4f23a..c75645c 100644 --- a/tests/gitlab_ci.skip +++ b/tests/gitlab_ci.skip @@ -1,9 +1,6 @@ -android emscripten java glsl mpi objc -action_nitro -asteronits wiringPi diff --git a/tests/test_curl.nit b/tests/test_curl.nit index 757c963..0d2b4d1 100644 --- a/tests/test_curl.nit +++ b/tests/test_curl.nit @@ -15,15 +15,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -import curl +intrude import curl class CallbackManager super CurlCallbacks - redef fun body_callback(line: String) do end + redef fun body_callback(line) do end end -fun error_manager(err: CURLCode) do if not err.is_ok then print err +private fun error_manager(err: CURLCode) do if not err.is_ok then print err var url = "http://example.org/" @@ -210,4 +210,4 @@ var hashMapRefined = new HeaderMap hashMapRefined["hello"] = "toto" hashMapRefined["hello"] = "tata" hashMapRefined["allo"] = "foo" -print hashMapRefined.to_url_encoded(sys.curl) +print hashMapRefined.to_url_encoded(new Curl)