diff --git a/.gitlab-ci.yaml b/.gitlab-ci.yaml index 84a907db34ecf1fe9a49b5904082b2457b70eaa8..63bdd35e35c875295515c9647a10ae69c9550426 100644 --- a/.gitlab-ci.yaml +++ b/.gitlab-ci.yaml @@ -7,17 +7,18 @@ stages: - data-supply - deploy - e2e-test +- post-deploy variables: GIT_DEPTH: 1 LC_ALL: "C" GIT_STRATEGY: clone GIT_SUBMODULE_STRATEGY: recursive - GIT_CLONE_PATH: $CI_BUILDS_DIR/$CI_CONCURRENT_ID/project-name + GIT_CLONE_PATH: $CI_BUILDS_DIR/$CI_COMMIT_REF_SLUG/$CI_CONCURRENT_ID/project-name HIVEMIND_SOURCE_HIVED_URL: $HIVEMIND_SOURCE_HIVED_URL HIVEMIND_DB_NAME: "hive_$CI_COMMIT_REF_SLUG" - HIVEMIND_HTTP_PORT: $HIVEMIND_HTTP_PORT + HIVEMIND_HTTP_PORT: $((HIVEMIND_HTTP_PORT + CI_CONCURRENT_ID)) # Configured at gitlab repository settings side POSTGRES_USER: $HIVEMIND_POSTGRES_USER POSTGRES_PASSWORD: $HIVEMIND_POSTGRES_PASSWORD @@ -26,6 +27,7 @@ variables: PGPASSWORD: $HIVEMIND_POSTGRES_PASSWORD before_script: + - pwd - echo "CI_NODE_TOTAL is $CI_NODE_TOTAL" - echo "CI_NODE_INDEX is $CI_NODE_INDEX" - echo "CI_CONCURRENT_ID is $CI_CONCURRENT_ID" @@ -44,11 +46,18 @@ hivemind_build: paths: - dist/ expire_in: 1 week + + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: always + - when: always + tags: - hivemind hivemind_sync: stage: data-supply + environment: name: "hive sync built from branch $CI_COMMIT_REF_NAME targeting database $HIVEMIND_DB_NAME" @@ -68,21 +77,13 @@ hivemind_sync: - hivemind-sync.log expire_in: 1 week - when: manual - tags: - - hivemind -hivemind_stop_server: - stage: deploy - environment: - name: "hive serve built from branch $CI_COMMIT_REF_NAME exposed on port $HIVEMIND_HTTP_PORT" - action: stop - - variables: - GIT_STRATEGY: none - when: manual - script: - - scripts/ci_stop_server.sh hive_server.pid + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: always + - if: '$CI_PIPELINE_SOURCE == "push"' + when: manual + - when: on_success tags: - hivemind @@ -95,8 +96,10 @@ hivemind_start_server: on_stop: hivemind_stop_server needs: - - job: hivemind_sync + - job: hivemind_build artifacts: true +# - job: hivemind_sync +# artifacts: true variables: GIT_STRATEGY: none PYTHONUSERBASE: ./local-site @@ -110,12 +113,36 @@ hivemind_start_server: - hive_server.pid expire_in: 1 week + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: always + - if: '$CI_PIPELINE_SOURCE == "push"' + when: manual + - when: on_success + + tags: + - hivemind + +hivemind_stop_server: + stage: post-deploy + environment: + name: "hive serve built from branch $CI_COMMIT_REF_NAME exposed on port $HIVEMIND_HTTP_PORT" + action: stop + + variables: + GIT_STRATEGY: none when: manual + script: + - scripts/ci_stop_server.sh hive_server.pid + + needs: + - job: hivemind_start_server + artifacts: true tags: - hivemind -hivemind_start_api_smoketest: +.hivemind_start_api_smoketest: &common_api_smoketest_job stage: e2e-test environment: hive-4.pl.syncad.com needs: @@ -125,13 +152,63 @@ hivemind_start_api_smoketest: variables: GIT_STRATEGY: none - script: - - python3 -m site --user-site - - pip3 install --user --upgrade pip setuptools - - pip3 install --user pyresttest - - scripts/ci_start_api_smoketest.sh "http://localhost:$HIVEMIND_HTTP_PORT" - - when: manual + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: always + - if: '$CI_PIPELINE_SOURCE == "push"' + when: manual + - when: on_success tags: - hivemind + +bridge_api_smoketest: + <<: *common_api_smoketest_job + + script: + - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" test_bridge_api_patterns.tavern.yaml api_smoketest_bridge.xml + + artifacts: + reports: + junit: api_smoketest_bridge.xml + +condenser_api_smoketest: + <<: *common_api_smoketest_job + + script: + - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" test_condenser_api_patterns.tavern.yaml api_smoketest_condenser_api.xml + + artifacts: + reports: + junit: api_smoketest_condenser_api.xml + +database_api_smoketest: + <<: *common_api_smoketest_job + + script: + - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" test_database_api_patterns.tavern.yaml api_smoketest_database_api.xml + + artifacts: + reports: + junit: api_smoketest_database_api.xml + +follow_api_smoketest: + <<: *common_api_smoketest_job + + script: + - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" test_follow_api_patterns.tavern.yaml api_smoketest_follow_api.xml + + artifacts: + reports: + junit: api_smoketest_follow_api.xml + +tags_api_smoketest: + <<: *common_api_smoketest_job + + script: + - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" test_tags_api_patterns.tavern.yaml api_smoketest_tags_api.xml + + artifacts: + reports: + junit: api_smoketest_tags_api.xml + diff --git a/scripts/ci_start_api_smoketest.sh b/scripts/ci_start_api_smoketest.sh index 1ea59d3900bd2d92a286c97bff5037045c44ffe9..def9b00273ca7f35964c8bd639e158690c86a406 100755 --- a/scripts/ci_start_api_smoketest.sh +++ b/scripts/ci_start_api_smoketest.sh @@ -2,9 +2,19 @@ set -e -cd tests/ -rm -rf ./build -mkdir ./build -cd build/ -cmake -DTEST_NODE="$1" ../ -ctest -R api/pyresttests/5000000 --output-on-failure +cd tests/tests_api/hivemind/tavern + +pip3 install --user jsondiff +pip3 install --user tavern +pip3 install --user pytest + +export HIVEMIND_ADDRESS=$1 +export HIVEMIND_PORT=$2 +echo Attempting to start tests on hivemind instance listeing on: $HIVEMIND_ADDRESS port: $HIVEMIND_PORT + +echo "Selected test group (if empty all will be executed): $3" + +python3 -m pytest -W ignore::pytest.PytestDeprecationWarning --junitxml=../../../../$4 $3 + +cd ../../../../ + diff --git a/scripts/ci_start_server.sh b/scripts/ci_start_server.sh index f7430b6bccb6aa0f036fc3a06ed8a5ee4dd4e933..fc7f60aa7b68a2a26fc34bfc2ada3a50a7d23912 100755 --- a/scripts/ci_start_server.sh +++ b/scripts/ci_start_server.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -e +set -xe HIVEMIND_DB_NAME=$1 HIVEMIND_POSTGRESQL_CONNECTION_STRING=$2 @@ -15,16 +15,38 @@ DB_NAME=${DB_NAME//]/_} DB_URL=$HIVEMIND_POSTGRESQL_CONNECTION_STRING/$DB_NAME +# Reuse DB_NAME as name of symbolic link pointing local hive "binary". +HIVE_NAME=$DB_NAME + +SAVED_PID=0 + +if [ -f hive_server.pid ]; then + SAVED_PID=`cat hive_server.pid` + kill -SIGINT $SAVED_PID || true; + sleep 5 + kill -9 $SAVED_PID || true; + + rm hive_server.pid; +fi + ls -l dist/* rm -rf ./local-site mkdir -p `python3 -m site --user-site` python3 setup.py install --user --force -./local-site/bin/hive -h +ln -sf ./local-site/bin/hive $HIVE_NAME +./$HIVE_NAME -h + +rm -rf hive_server.log echo Attempting to start hive server listening on $HIVEMIND_HTTP_PORT port... -if [ -f hive_server.pid ]; then kill -SIGINT `cat hive_server.pid`; fi; -rm -f hive_server.pid -screen -L -Logfile hive_server.log -dmS hive_server_$CI_JOB_ID ./local-site/bin/hive server --pid-file hive_server.pid --http-server-port $HIVEMIND_HTTP_PORT --steemd-url "$HIVEMIND_SOURCE_HIVED_URL" --database-url $DB_URL +screen -L -Logfile hive_server.log -dmS $HIVE_NAME ./$HIVE_NAME server --pid-file hive_server.pid --http-server-port $HIVEMIND_HTTP_PORT --steemd-url "$HIVEMIND_SOURCE_HIVED_URL" --database-url $DB_URL for i in `seq 1 10`; do if [ -f hive_server.pid ]; then break; else sleep 1; fi; done -if [ -f hive_server.pid ]; then echo "Hive server started successfully..."; else "Hive server start failure..." ; fi; + +SAVED_PID=`cat hive_server.pid` +LISTENING_PID=$(fuser $HIVEMIND_HTTP_PORT/tcp 2>/dev/null) +echo "Retrieved hive pid is: $SAVED_PID" +echo "Listening hive pid is: $LISTENING_PID" + +cat hive_server.log +if [ "$SAVED_PID" != "$LISTENING_PID" ]; then echo "Saved pid: $SAVED_PID vs listening pid: $LISTENING_PID mismatch..."; fi diff --git a/scripts/ci_sync.sh b/scripts/ci_sync.sh index c08fee4a9cc1b2bf7c92d2a02eaa58dc44bad988..b6be580d268a3e4db9a84fef6f012fc3144d2310 100755 --- a/scripts/ci_sync.sh +++ b/scripts/ci_sync.sh @@ -1,6 +1,7 @@ #!/bin/bash set -e +set -o pipefail HIVEMIND_DB_NAME=$1 HIVEMIND_POSTGRESQL_CONNECTION_STRING=$2 @@ -16,16 +17,24 @@ DB_URL=$HIVEMIND_POSTGRESQL_CONNECTION_STRING/$DB_NAME echo Corrected db name $DB_NAME echo Corrected db url $DB_URL +# Reuse DB_NAME as name of symbolic link pointing local hive "binary". +HIVE_NAME=$DB_NAME + if [ -f hive_sync.pid ]; then - kill `cat hive_sync.pid`; + kill -SIGINT `cat hive_sync.pid` || true; rm hive_sync.pid; fi +kill -SIGINT `pgrep -f "$HIVE_NAME sync"` || true; +sleep 5 +kill -9 `pgrep -f "$HIVE_NAME sync"` || true; + ls -l dist/* rm -rf ./local-site mkdir -p `python3 -m site --user-site` python3 setup.py install --user --force -./local-site/bin/hive -h +ln -sf ./local-site/bin/hive $HIVE_NAME +./$HIVE_NAME -h echo Attempting to recreate database $DB_NAME psql -U $POSTGRES_USER -h localhost -d postgres -c "DROP DATABASE IF EXISTS $DB_NAME;" @@ -33,5 +42,5 @@ psql -U $POSTGRES_USER -h localhost -d postgres -c "CREATE DATABASE $DB_NAME;" echo Attempting to starting hive sync using hived node: $HIVEMIND_SOURCE_HIVED_URL . Max sync block is: $HIVEMIND_MAX_BLOCK echo Attempting to access database $DB_URL -./local-site/bin/hive sync --pid-file hive_sync.pid --test-max-block=$HIVEMIND_MAX_BLOCK --exit-after-sync --test-profile=False --steemd-url "$HIVEMIND_SOURCE_HIVED_URL" --database-url $DB_URL 2>&1 | tee -i hivemind-sync.log +./$HIVE_NAME sync --pid-file hive_sync.pid --test-max-block=$HIVEMIND_MAX_BLOCK --exit-after-sync --test-profile=False --steemd-url "$HIVEMIND_SOURCE_HIVED_URL" --database-url $DB_URL 2>&1 | tee -i hivemind-sync.log rm hive_sync.pid diff --git a/tests/tests_api b/tests/tests_api index 7832473c73dc0a8d60b780826196c51c07ee1346..ef779872ccc5f48fbc9dabe012644210626cada8 160000 --- a/tests/tests_api +++ b/tests/tests_api @@ -1 +1 @@ -Subproject commit 7832473c73dc0a8d60b780826196c51c07ee1346 +Subproject commit ef779872ccc5f48fbc9dabe012644210626cada8