Skip to content
Snippets Groups Projects
Commit 7fe697f5 authored by Bartek Wrona's avatar Bartek Wrona
Browse files

Merge branch 'ci_tavern_prerequisites' into 'develop' 

Ci tavern prerequisites

See merge request !57
parents 4799bf54 1d4ab849
No related branches found
No related tags found
4 merge requests!456Release candidate v1 24,!230Setup monitoring with pghero,!135Enable postgres monitoring on CI server,!57Ci tavern prerequisites
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yaml
+ 102
− 25
View file @ 7fe697f5
...@@ -7,17 +7,18 @@ stages: ...@@ -7,17 +7,18 @@ stages:
- data-supply - data-supply
- deploy - deploy
- e2e-test - e2e-test
- post-deploy
variables: variables:
GIT_DEPTH: 1 GIT_DEPTH: 1
LC_ALL: "C" LC_ALL: "C"
GIT_STRATEGY: clone GIT_STRATEGY: clone
GIT_SUBMODULE_STRATEGY: recursive GIT_SUBMODULE_STRATEGY: recursive
GIT_CLONE_PATH: $CI_BUILDS_DIR/$CI_CONCURRENT_ID/project-name GIT_CLONE_PATH: $CI_BUILDS_DIR/$CI_COMMIT_REF_SLUG/$CI_CONCURRENT_ID/project-name
HIVEMIND_SOURCE_HIVED_URL: $HIVEMIND_SOURCE_HIVED_URL HIVEMIND_SOURCE_HIVED_URL: $HIVEMIND_SOURCE_HIVED_URL
HIVEMIND_DB_NAME: "hive_$CI_COMMIT_REF_SLUG" HIVEMIND_DB_NAME: "hive_$CI_COMMIT_REF_SLUG"
HIVEMIND_HTTP_PORT: $HIVEMIND_HTTP_PORT HIVEMIND_HTTP_PORT: $((HIVEMIND_HTTP_PORT + CI_CONCURRENT_ID))
# Configured at gitlab repository settings side # Configured at gitlab repository settings side
POSTGRES_USER: $HIVEMIND_POSTGRES_USER POSTGRES_USER: $HIVEMIND_POSTGRES_USER
POSTGRES_PASSWORD: $HIVEMIND_POSTGRES_PASSWORD POSTGRES_PASSWORD: $HIVEMIND_POSTGRES_PASSWORD
...@@ -26,6 +27,7 @@ variables: ...@@ -26,6 +27,7 @@ variables:
PGPASSWORD: $HIVEMIND_POSTGRES_PASSWORD PGPASSWORD: $HIVEMIND_POSTGRES_PASSWORD
before_script: before_script:
- pwd
- echo "CI_NODE_TOTAL is $CI_NODE_TOTAL" - echo "CI_NODE_TOTAL is $CI_NODE_TOTAL"
- echo "CI_NODE_INDEX is $CI_NODE_INDEX" - echo "CI_NODE_INDEX is $CI_NODE_INDEX"
- echo "CI_CONCURRENT_ID is $CI_CONCURRENT_ID" - echo "CI_CONCURRENT_ID is $CI_CONCURRENT_ID"
...@@ -44,11 +46,18 @@ hivemind_build: ...@@ -44,11 +46,18 @@ hivemind_build:
paths: paths:
- dist/ - dist/
expire_in: 1 week expire_in: 1 week
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: always
- when: always
tags: tags:
- hivemind - hivemind
hivemind_sync: hivemind_sync:
stage: data-supply stage: data-supply
environment: environment:
name: "hive sync built from branch $CI_COMMIT_REF_NAME targeting database $HIVEMIND_DB_NAME" name: "hive sync built from branch $CI_COMMIT_REF_NAME targeting database $HIVEMIND_DB_NAME"
...@@ -68,21 +77,13 @@ hivemind_sync: ...@@ -68,21 +77,13 @@ hivemind_sync:
- hivemind-sync.log - hivemind-sync.log
expire_in: 1 week expire_in: 1 week
when: manual
tags:
- hivemind
hivemind_stop_server: rules:
stage: deploy - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
environment: when: always
name: "hive serve built from branch $CI_COMMIT_REF_NAME exposed on port $HIVEMIND_HTTP_PORT" - if: '$CI_PIPELINE_SOURCE == "push"'
action: stop when: manual
- when: on_success
variables:
GIT_STRATEGY: none
when: manual
script:
- scripts/ci_stop_server.sh hive_server.pid
tags: tags:
- hivemind - hivemind
...@@ -95,8 +96,10 @@ hivemind_start_server: ...@@ -95,8 +96,10 @@ hivemind_start_server:
on_stop: hivemind_stop_server on_stop: hivemind_stop_server
needs: needs:
- job: hivemind_sync - job: hivemind_build
artifacts: true artifacts: true
# - job: hivemind_sync
# artifacts: true
variables: variables:
GIT_STRATEGY: none GIT_STRATEGY: none
PYTHONUSERBASE: ./local-site PYTHONUSERBASE: ./local-site
...@@ -110,12 +113,36 @@ hivemind_start_server: ...@@ -110,12 +113,36 @@ hivemind_start_server:
- hive_server.pid - hive_server.pid
expire_in: 1 week expire_in: 1 week
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: always
- if: '$CI_PIPELINE_SOURCE == "push"'
when: manual
- when: on_success
tags:
- hivemind
hivemind_stop_server:
stage: post-deploy
environment:
name: "hive serve built from branch $CI_COMMIT_REF_NAME exposed on port $HIVEMIND_HTTP_PORT"
action: stop
variables:
GIT_STRATEGY: none
when: manual when: manual
script:
- scripts/ci_stop_server.sh hive_server.pid
needs:
- job: hivemind_start_server
artifacts: true
tags: tags:
- hivemind - hivemind
hivemind_start_api_smoketest: .hivemind_start_api_smoketest: &common_api_smoketest_job
stage: e2e-test stage: e2e-test
environment: hive-4.pl.syncad.com environment: hive-4.pl.syncad.com
needs: needs:
...@@ -125,13 +152,63 @@ hivemind_start_api_smoketest: ...@@ -125,13 +152,63 @@ hivemind_start_api_smoketest:
variables: variables:
GIT_STRATEGY: none GIT_STRATEGY: none
script: rules:
- python3 -m site --user-site - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- pip3 install --user --upgrade pip setuptools when: always
- pip3 install --user pyresttest - if: '$CI_PIPELINE_SOURCE == "push"'
- scripts/ci_start_api_smoketest.sh "http://localhost:$HIVEMIND_HTTP_PORT" when: manual
- when: on_success
when: manual
tags: tags:
- hivemind - hivemind
bridge_api_smoketest:
<<: *common_api_smoketest_job
script:
- scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" test_bridge_api_patterns.tavern.yaml api_smoketest_bridge.xml
artifacts:
reports:
junit: api_smoketest_bridge.xml
condenser_api_smoketest:
<<: *common_api_smoketest_job
script:
- scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" test_condenser_api_patterns.tavern.yaml api_smoketest_condenser_api.xml
artifacts:
reports:
junit: api_smoketest_condenser_api.xml
database_api_smoketest:
<<: *common_api_smoketest_job
script:
- scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" test_database_api_patterns.tavern.yaml api_smoketest_database_api.xml
artifacts:
reports:
junit: api_smoketest_database_api.xml
follow_api_smoketest:
<<: *common_api_smoketest_job
script:
- scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" test_follow_api_patterns.tavern.yaml api_smoketest_follow_api.xml
artifacts:
reports:
junit: api_smoketest_follow_api.xml
tags_api_smoketest:
<<: *common_api_smoketest_job
script:
- scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" test_tags_api_patterns.tavern.yaml api_smoketest_tags_api.xml
artifacts:
reports:
junit: api_smoketest_tags_api.xml
scripts/ci_start_api_smoketest.sh
+ 16
− 6
View file @ 7fe697f5
...@@ -2,9 +2,19 @@ ...@@ -2,9 +2,19 @@
set -e set -e
cd tests/ cd tests/tests_api/hivemind/tavern
rm -rf ./build
mkdir ./build pip3 install --user jsondiff
cd build/ pip3 install --user tavern
cmake -DTEST_NODE="$1" ../ pip3 install --user pytest
ctest -R api/pyresttests/5000000 --output-on-failure
export HIVEMIND_ADDRESS=$1
export HIVEMIND_PORT=$2
echo Attempting to start tests on hivemind instance listeing on: $HIVEMIND_ADDRESS port: $HIVEMIND_PORT
echo "Selected test group (if empty all will be executed): $3"
python3 -m pytest -W ignore::pytest.PytestDeprecationWarning --junitxml=../../../../$4 $3
cd ../../../../
scripts/ci_start_server.sh
+ 28
− 6
View file @ 7fe697f5
#!/bin/bash #!/bin/bash
set -e set -xe
HIVEMIND_DB_NAME=$1 HIVEMIND_DB_NAME=$1
HIVEMIND_POSTGRESQL_CONNECTION_STRING=$2 HIVEMIND_POSTGRESQL_CONNECTION_STRING=$2
...@@ -15,16 +15,38 @@ DB_NAME=${DB_NAME//]/_} ...@@ -15,16 +15,38 @@ DB_NAME=${DB_NAME//]/_}
DB_URL=$HIVEMIND_POSTGRESQL_CONNECTION_STRING/$DB_NAME DB_URL=$HIVEMIND_POSTGRESQL_CONNECTION_STRING/$DB_NAME
# Reuse DB_NAME as name of symbolic link pointing local hive "binary".
HIVE_NAME=$DB_NAME
SAVED_PID=0
if [ -f hive_server.pid ]; then
SAVED_PID=`cat hive_server.pid`
kill -SIGINT $SAVED_PID || true;
sleep 5
kill -9 $SAVED_PID || true;
rm hive_server.pid;
fi
ls -l dist/* ls -l dist/*
rm -rf ./local-site rm -rf ./local-site
mkdir -p `python3 -m site --user-site` mkdir -p `python3 -m site --user-site`
python3 setup.py install --user --force python3 setup.py install --user --force
./local-site/bin/hive -h ln -sf ./local-site/bin/hive $HIVE_NAME
./$HIVE_NAME -h
rm -rf hive_server.log
echo Attempting to start hive server listening on $HIVEMIND_HTTP_PORT port... echo Attempting to start hive server listening on $HIVEMIND_HTTP_PORT port...
if [ -f hive_server.pid ]; then kill -SIGINT `cat hive_server.pid`; fi; screen -L -Logfile hive_server.log -dmS $HIVE_NAME ./$HIVE_NAME server --pid-file hive_server.pid --http-server-port $HIVEMIND_HTTP_PORT --steemd-url "$HIVEMIND_SOURCE_HIVED_URL" --database-url $DB_URL
rm -f hive_server.pid
screen -L -Logfile hive_server.log -dmS hive_server_$CI_JOB_ID ./local-site/bin/hive server --pid-file hive_server.pid --http-server-port $HIVEMIND_HTTP_PORT --steemd-url "$HIVEMIND_SOURCE_HIVED_URL" --database-url $DB_URL
for i in `seq 1 10`; do if [ -f hive_server.pid ]; then break; else sleep 1; fi; done for i in `seq 1 10`; do if [ -f hive_server.pid ]; then break; else sleep 1; fi; done
if [ -f hive_server.pid ]; then echo "Hive server started successfully..."; else "Hive server start failure..." ; fi;
SAVED_PID=`cat hive_server.pid`
LISTENING_PID=$(fuser $HIVEMIND_HTTP_PORT/tcp 2>/dev/null)
echo "Retrieved hive pid is: $SAVED_PID"
echo "Listening hive pid is: $LISTENING_PID"
cat hive_server.log
if [ "$SAVED_PID" != "$LISTENING_PID" ]; then echo "Saved pid: $SAVED_PID vs listening pid: $LISTENING_PID mismatch..."; fi
scripts/ci_sync.sh
+ 12
− 3
View file @ 7fe697f5
#!/bin/bash #!/bin/bash
set -e set -e
set -o pipefail
HIVEMIND_DB_NAME=$1 HIVEMIND_DB_NAME=$1
HIVEMIND_POSTGRESQL_CONNECTION_STRING=$2 HIVEMIND_POSTGRESQL_CONNECTION_STRING=$2
...@@ -16,16 +17,24 @@ DB_URL=$HIVEMIND_POSTGRESQL_CONNECTION_STRING/$DB_NAME ...@@ -16,16 +17,24 @@ DB_URL=$HIVEMIND_POSTGRESQL_CONNECTION_STRING/$DB_NAME
echo Corrected db name $DB_NAME echo Corrected db name $DB_NAME
echo Corrected db url $DB_URL echo Corrected db url $DB_URL
# Reuse DB_NAME as name of symbolic link pointing local hive "binary".
HIVE_NAME=$DB_NAME
if [ -f hive_sync.pid ]; then if [ -f hive_sync.pid ]; then
kill `cat hive_sync.pid`; kill -SIGINT `cat hive_sync.pid` || true;
rm hive_sync.pid; rm hive_sync.pid;
fi fi
kill -SIGINT `pgrep -f "$HIVE_NAME sync"` || true;
sleep 5
kill -9 `pgrep -f "$HIVE_NAME sync"` || true;
ls -l dist/* ls -l dist/*
rm -rf ./local-site rm -rf ./local-site
mkdir -p `python3 -m site --user-site` mkdir -p `python3 -m site --user-site`
python3 setup.py install --user --force python3 setup.py install --user --force
./local-site/bin/hive -h ln -sf ./local-site/bin/hive $HIVE_NAME
./$HIVE_NAME -h
echo Attempting to recreate database $DB_NAME echo Attempting to recreate database $DB_NAME
psql -U $POSTGRES_USER -h localhost -d postgres -c "DROP DATABASE IF EXISTS $DB_NAME;" psql -U $POSTGRES_USER -h localhost -d postgres -c "DROP DATABASE IF EXISTS $DB_NAME;"
...@@ -33,5 +42,5 @@ psql -U $POSTGRES_USER -h localhost -d postgres -c "CREATE DATABASE $DB_NAME;" ...@@ -33,5 +42,5 @@ psql -U $POSTGRES_USER -h localhost -d postgres -c "CREATE DATABASE $DB_NAME;"
echo Attempting to starting hive sync using hived node: $HIVEMIND_SOURCE_HIVED_URL . Max sync block is: $HIVEMIND_MAX_BLOCK echo Attempting to starting hive sync using hived node: $HIVEMIND_SOURCE_HIVED_URL . Max sync block is: $HIVEMIND_MAX_BLOCK
echo Attempting to access database $DB_URL echo Attempting to access database $DB_URL
./local-site/bin/hive sync --pid-file hive_sync.pid --test-max-block=$HIVEMIND_MAX_BLOCK --exit-after-sync --test-profile=False --steemd-url "$HIVEMIND_SOURCE_HIVED_URL" --database-url $DB_URL 2>&1 | tee -i hivemind-sync.log ./$HIVE_NAME sync --pid-file hive_sync.pid --test-max-block=$HIVEMIND_MAX_BLOCK --exit-after-sync --test-profile=False --steemd-url "$HIVEMIND_SOURCE_HIVED_URL" --database-url $DB_URL 2>&1 | tee -i hivemind-sync.log
rm hive_sync.pid rm hive_sync.pid
tests_api @ ef779872
Compare 7832473c...ef779872
Subproject commit 7832473c73dc0a8d60b780826196c51c07ee1346 Subproject commit ef779872ccc5f48fbc9dabe012644210626cada8
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment