Merge branch 'ci_tests_spawn' into 'develop'

Ci tests spawn

See merge request !46

.gitlab-ci.yaml

@@ -4,13 +4,17 @@ image: "python:3.7"
 stages:
 - build
 - test
+- data-supply
 - deploy
+- e2e-test
 
 variables:
   GIT_DEPTH: 1
   LC_ALL: "C"
   GIT_STRATEGY: clone
   GIT_SUBMODULE_STRATEGY: recursive
+  GIT_CLONE_PATH: $CI_BUILDS_DIR/$CI_CONCURRENT_ID/project-name
+
   HIVEMIND_SOURCE_HIVED_URL: $HIVEMIND_SOURCE_HIVED_URL
   HIVEMIND_DB_NAME: "hive_$CI_COMMIT_REF_SLUG"
   HIVEMIND_HTTP_PORT: $HIVEMIND_HTTP_PORT
@@ -21,20 +25,16 @@ variables:
   # official way to provide password to psql: http://www.postgresql.org/docs/9.3/static/libpq-envars.html
   PGPASSWORD: $HIVEMIND_POSTGRES_PASSWORD
 
-  PYTHONUSERBASE: ./local-site
-
 before_script:
   - echo "CI_NODE_TOTAL is $CI_NODE_TOTAL"
   - echo "CI_NODE_INDEX is $CI_NODE_INDEX"
   - echo "CI_CONCURRENT_ID is $CI_CONCURRENT_ID"
   - echo "CI_COMMIT_REF_SLUG is $CI_COMMIT_REF_SLUG"
-  - python3 --version
-  - python3 -m site --user-site
-  - pip3 install --user --upgrade pip setuptools
 
 hivemind_build:
   stage: build
   script:
+    - pip3 install --user --upgrade pip setuptools
     - git fetch --tags
     - git tag -f ci_implicit_tag
     - echo $PYTHONUSERBASE
@@ -48,34 +48,55 @@ hivemind_build:
      - hivemind
 
 hivemind_sync:
-  stage: deploy
-  environment: hive-4.pl.syncad.com
+  stage: data-supply
+  environment:
+      name: "hive sync built from branch $CI_COMMIT_REF_NAME targeting database $HIVEMIND_DB_NAME"
+
   needs:
     - job: hivemind_build
       artifacts: true
   variables:
     GIT_STRATEGY: none
+    PYTHONUSERBASE: ./local-site
 
   script:
+    - pip3 install --user --upgrade pip setuptools
     - scripts/ci_sync.sh "$HIVEMIND_DB_NAME" "$HIVEMIND_POSTGRESQL_CONNECTION_STRING" "$HIVEMIND_SOURCE_HIVED_URL" $HIVEMIND_MAX_BLOCK
-    - scripts/ci_start_server.sh "$HIVEMIND_DB_NAME" "$HIVEMIND_POSTGRESQL_CONNECTION_STRING" "$HIVEMIND_SOURCE_HIVED_URL" $HIVEMIND_HTTP_PORT
+
   artifacts:
     paths:
       - hivemind-sync.log
-      - hive_server.pid
-    expire_in: 1 week
 
+    expire_in: 1 week
+  when: manual
   tags:
      - hivemind
 
+hivemind_stop_server:
+  stage: deploy
+  environment:
+    name: "hive serve built from branch $CI_COMMIT_REF_NAME exposed on port $HIVEMIND_HTTP_PORT"
+    action: stop
+
+  variables:
+    GIT_STRATEGY: none
+  when: manual
+  script:
+    - scripts/ci_stop_server.sh hive_server.pid
+
 hivemind_start_server:
   stage: deploy
-  environment: hive-4.pl.syncad.com
+  environment:
+    name: "hive serve built from branch $CI_COMMIT_REF_NAME exposed on port $HIVEMIND_HTTP_PORT"
+    url: "http://hive-4.pl.syncad.com:$HIVEMIND_HTTP_PORT"
+    on_stop: hivemind_stop_server
+
   needs:
-    - job: hivemind_build
+    - job: hivemind_sync
       artifacts: true
   variables:
     GIT_STRATEGY: none
+    PYTHONUSERBASE: ./local-site
 
   script:
     - scripts/ci_start_server.sh "$HIVEMIND_DB_NAME" "$HIVEMIND_POSTGRESQL_CONNECTION_STRING" "$HIVEMIND_SOURCE_HIVED_URL" $HIVEMIND_HTTP_PORT
@@ -90,3 +111,24 @@ hivemind_start_server:
 
   tags:
      - hivemind
+
+hivemind_start_api_smoketest:
+  stage: e2e-test
+  environment: hive-4.pl.syncad.com
+  needs:
+    - job: hivemind_start_server
+      artifacts: true
+
+  variables:
+    GIT_STRATEGY: none
+
+  script:
+    - python3 -m site --user-site
+    - pip3 install --user --upgrade pip setuptools
+    - pip3 install --user pyresttest
+    - scripts/ci_start_api_smoketest.sh "http://localhost/$HIVEMIND_HTTP_PORT"
+
+  when: manual
+
+  tags:
+     - hivemind

scripts/ci_start_api_smoketest.sh

+#!/bin/bash 
+
+set -e
+
+cd tests/
+rm -rf ./build
+mkdir ./build
+cd build/
+cmake -DTEST_NODE="$1" ../
+ctest -R api/pyresttests/5000000 --output-on-failure

scripts/ci_start_server.sh

@@ -15,6 +15,12 @@ DB_NAME=${DB_NAME//]/_}
 
 DB_URL=$HIVEMIND_POSTGRESQL_CONNECTION_STRING/$DB_NAME
 
+ls -l dist/*
+rm -rf ./local-site
+mkdir -p `python3 -m site --user-site`
+python3 setup.py install --user --force
+./local-site/bin/hive -h 
+
 echo Attempting to start hive server listening on $HIVEMIND_HTTP_PORT port...
 if [ -f hive_server.pid ]; then kill -SIGINT `cat hive_server.pid`; fi;
 rm -f hive_server.pid

scripts/ci_stop_server.sh

+#!/bin/bash 
+
+set -e
+
+# Usage ci_stop_server.sh pid_file_name
+
+if [ -f $1 ]; then
+  PID=`cat $1`;
+  kill -SIGINT $PID;
+else
+  echo Specified pid file: $1 does not exists.;
+fi
+
+

scripts/ci_sync.sh

@@ -15,14 +15,23 @@ DB_NAME=${DB_NAME//]/_}
 DB_URL=$HIVEMIND_POSTGRESQL_CONNECTION_STRING/$DB_NAME
 echo Corrected db name $DB_NAME
 echo Corrected db url $DB_URL
+
+if [ -f hive_sync.pid ]; then
+  kill `cat hive_sync.pid`;
+  rm hive_sync.pid;
+fi
+
 ls -l dist/*
 rm -rf ./local-site
 mkdir -p `python3 -m site --user-site`
 python3 setup.py install --user --force
 ./local-site/bin/hive -h
+
 echo Attempting to recreate database $DB_NAME
 psql -U $POSTGRES_USER -h localhost -d postgres -c "DROP DATABASE IF EXISTS $DB_NAME;"
 psql -U $POSTGRES_USER -h localhost -d postgres -c "CREATE DATABASE $DB_NAME;"
+
 echo Attempting to starting hive sync using hived node: $HIVEMIND_SOURCE_HIVED_URL . Max sync block is: $HIVEMIND_MAX_BLOCK
 echo Attempting to access database $DB_URL
-./local-site/bin/hive sync --test-max-block=$HIVEMIND_MAX_BLOCK --exit-after-sync --test-profile=False --steemd-url "$HIVEMIND_SOURCE_HIVED_URL" --database-url $DB_URL 2>&1 | tee -i hivemind-sync.log
+./local-site/bin/hive sync --pid-file hive_sync.pid --test-max-block=$HIVEMIND_MAX_BLOCK --exit-after-sync --test-profile=False --steemd-url "$HIVEMIND_SOURCE_HIVED_URL" --database-url $DB_URL 2>&1 | tee -i hivemind-sync.log
+rm hive_sync.pid

tests/CMakeLists.txt

+cmake_minimum_required(VERSION 3.10)
+
 INCLUDE(tests_api/ApiTests.cmake)
 
 ENABLE_TESTING()