diff --git a/scripts/ci/hive-sync.sh b/scripts/ci/hive-sync.sh index 7e9bf57e06b6398e1067f13049feb18c2a82492d..5d9b9f08851752935385f753bc3d276cb9d47909 100755 --- a/scripts/ci/hive-sync.sh +++ b/scripts/ci/hive-sync.sh @@ -1,37 +1,37 @@ -#!/bin/bash - -set -euo pipefail - -# For debug only! -# RUNNER_HIVEMIND_SYNC_MAX_BLOCK=10000 -# RUNNER_HIVED_URL='{"default":"http://hived-node:8091"}' -# RUNNER_HIVED_URL='{"default":"http://172.17.0.1:8091"}' - -hive_sync() { - # Start hive sync process - - cat << EOF -Starting hive sync using hived url: ${RUNNER_HIVED_URL}. -Max sync block is: ${RUNNER_HIVEMIND_SYNC_MAX_BLOCK}. -EOF - - USER=${RUNNER_POSTGRES_APP_USER}:${RUNNER_POSTGRES_APP_USER_PASSWORD} - OPTIONS="host=${RUNNER_POSTGRES_HOST}&port=${RUNNER_POSTGRES_PORT}" - DATABASE_URL="postgresql://${USER}@/${HIVEMIND_DB_NAME}?${OPTIONS}" - - hive sync \ - --log-mask-sensitive-data \ - --pid-file hive_sync.pid \ - --test-max-block=${RUNNER_HIVEMIND_SYNC_MAX_BLOCK} \ - --exit-after-sync \ - --test-profile=False \ - --steemd-url "${RUNNER_HIVED_URL}" \ - --prometheus-port 11011 \ - --database-url "${DATABASE_URL}" \ - --mock-block-data-path mock_data/block_data/follow_op/mock_block_data_follow.json mock_data/block_data/community_op/mock_block_data_community.json mock_data/block_data/reblog_op/mock_block_data_reblog.json \ - --community-start-block 4999998 \ - 2>&1 | tee -i hivemind-sync.log - -} - -hive_sync +#!/bin/bash + +set -euo pipefail + +# For debug only! +# RUNNER_HIVEMIND_SYNC_MAX_BLOCK=10000 +# RUNNER_HIVED_URL='{"default":"http://hived-node:8091"}' +# RUNNER_HIVED_URL='{"default":"http://172.17.0.1:8091"}' + +hive_sync() { + # Start hive sync process + + cat << EOF +Starting hive sync using hived url: ${RUNNER_HIVED_URL}. +Max sync block is: ${RUNNER_HIVEMIND_SYNC_MAX_BLOCK}. +EOF + + USER=${RUNNER_POSTGRES_APP_USER}:${RUNNER_POSTGRES_APP_USER_PASSWORD} + OPTIONS="host=${RUNNER_POSTGRES_HOST}&port=${RUNNER_POSTGRES_PORT}" + DATABASE_URL="postgresql://${USER}@/${HIVEMIND_DB_NAME}?${OPTIONS}" + + hive sync \ + --log-mask-sensitive-data \ + --pid-file hive_sync.pid \ + --test-max-block=${RUNNER_HIVEMIND_SYNC_MAX_BLOCK} \ + --exit-after-sync \ + --test-profile=False \ + --steemd-url "${RUNNER_HIVED_URL}" \ + --prometheus-port 11011 \ + --database-url "${DATABASE_URL}" \ + --mock-block-data-path mock_data/block_data/follow_op/mock_block_data_follow.json mock_data/block_data/community_op/mock_block_data_community.json mock_data/block_data/reblog_op/mock_block_data_reblog.json \ + --community-start-block 4999998 \ + 2>&1 | tee -i hivemind-sync.log + +} + +hive_sync diff --git a/scripts/ci_sync.sh b/scripts/ci_sync.sh index fde79a3853094743436e3c3c13f33a564f4c21e7..c4aab0ab78131ef3ef59654f65b3f40f311a3cdd 100755 --- a/scripts/ci_sync.sh +++ b/scripts/ci_sync.sh @@ -1,62 +1,62 @@ -#!/bin/bash - -set -e -set -o pipefail - -HIVEMIND_DB_NAME=$1 -HIVEMIND_POSTGRESQL_CONNECTION_STRING=$2 -HIVEMIND_SOURCE_HIVED_URL=$3 -HIVEMIND_MAX_BLOCK=$4 -HIVEMIND_HTTP_PORT=$5 -HIVEMIND_ENABLE_DB_MONITORING=${6:-yes} - -PYTHONUSERBASE=./local-site - -DB_NAME=${HIVEMIND_DB_NAME//-/_} -DB_NAME=${DB_NAME//\[/_} -DB_NAME=${DB_NAME//]/_} -DB_URL=$HIVEMIND_POSTGRESQL_CONNECTION_STRING/$DB_NAME -echo Corrected db name $DB_NAME -echo Corrected db url $DB_URL - -# Reuse DB_NAME as name of symbolic link pointing local hive "binary". -HIVE_NAME=$DB_NAME - -if [ -f hive_sync.pid ]; then - kill -SIGINT `cat hive_sync.pid` || true; - rm hive_sync.pid; -fi - -kill -SIGINT `pgrep -f "$HIVE_NAME sync"` || true; -sleep 5 -kill -9 `pgrep -f "$HIVE_NAME sync"` || true; - -kill -SIGINT `pgrep -f "$HIVE_NAME server"` || true; -sleep 5 -kill -9 `pgrep -f "$HIVE_NAME server"` || true; - -fuser $HIVEMIND_HTTP_PORT/tcp -k -INT || true -sleep 5 - -fuser $HIVEMIND_HTTP_PORT/tcp -k -KILL || true -sleep 5 - -ls -l dist/* -rm -rf ./local-site -mkdir -p `python3 -m site --user-site` -python3 setup.py install --user --force -ln -sf ./local-site/bin/hive $HIVE_NAME -./$HIVE_NAME -h - -echo Attempting to recreate database $DB_NAME -psql -U $POSTGRES_USER -h localhost -d postgres -c "DROP DATABASE IF EXISTS $DB_NAME;" -if [ "$HIVEMIND_ENABLE_DB_MONITORING" = "yes" ]; then - psql -U $POSTGRES_USER -h localhost -d postgres -c "CREATE DATABASE $DB_NAME TEMPLATE template_monitoring;" -else - psql -U $POSTGRES_USER -h localhost -d postgres -c "CREATE DATABASE $DB_NAME" -fi - -echo Attempting to starting hive sync using hived node: $HIVEMIND_SOURCE_HIVED_URL . Max sync block is: $HIVEMIND_MAX_BLOCK -echo Attempting to access database $DB_URL -./$HIVE_NAME sync --pid-file hive_sync.pid --test-max-block=$HIVEMIND_MAX_BLOCK --exit-after-sync --test-profile=False --steemd-url "$HIVEMIND_SOURCE_HIVED_URL" --prometheus-port 11011 --database-url $DB_URL --mock-block-data-path mock_data/block_data/follow_op/mock_block_data_follow.json mock_data/block_data/community_op/mock_block_data_community.json mock_data/block_data/reblog_op/mock_block_data_reblog.json --community-start-block 4999998 2>&1 | tee -i hivemind-sync.log -rm hive_sync.pid +#!/bin/bash + +set -e +set -o pipefail + +HIVEMIND_DB_NAME=$1 +HIVEMIND_POSTGRESQL_CONNECTION_STRING=$2 +HIVEMIND_SOURCE_HIVED_URL=$3 +HIVEMIND_MAX_BLOCK=$4 +HIVEMIND_HTTP_PORT=$5 +HIVEMIND_ENABLE_DB_MONITORING=${6:-yes} + +PYTHONUSERBASE=./local-site + +DB_NAME=${HIVEMIND_DB_NAME//-/_} +DB_NAME=${DB_NAME//\[/_} +DB_NAME=${DB_NAME//]/_} +DB_URL=$HIVEMIND_POSTGRESQL_CONNECTION_STRING/$DB_NAME +echo Corrected db name $DB_NAME +echo Corrected db url $DB_URL + +# Reuse DB_NAME as name of symbolic link pointing local hive "binary". +HIVE_NAME=$DB_NAME + +if [ -f hive_sync.pid ]; then + kill -SIGINT `cat hive_sync.pid` || true; + rm hive_sync.pid; +fi + +kill -SIGINT `pgrep -f "$HIVE_NAME sync"` || true; +sleep 5 +kill -9 `pgrep -f "$HIVE_NAME sync"` || true; + +kill -SIGINT `pgrep -f "$HIVE_NAME server"` || true; +sleep 5 +kill -9 `pgrep -f "$HIVE_NAME server"` || true; + +fuser $HIVEMIND_HTTP_PORT/tcp -k -INT || true +sleep 5 + +fuser $HIVEMIND_HTTP_PORT/tcp -k -KILL || true +sleep 5 + +ls -l dist/* +rm -rf ./local-site +mkdir -p `python3 -m site --user-site` +python3 setup.py install --user --force +ln -sf ./local-site/bin/hive $HIVE_NAME +./$HIVE_NAME -h + +echo Attempting to recreate database $DB_NAME +psql -U $POSTGRES_USER -h localhost -d postgres -c "DROP DATABASE IF EXISTS $DB_NAME;" +if [ "$HIVEMIND_ENABLE_DB_MONITORING" = "yes" ]; then + psql -U $POSTGRES_USER -h localhost -d postgres -c "CREATE DATABASE $DB_NAME TEMPLATE template_monitoring;" +else + psql -U $POSTGRES_USER -h localhost -d postgres -c "CREATE DATABASE $DB_NAME" +fi + +echo Attempting to starting hive sync using hived node: $HIVEMIND_SOURCE_HIVED_URL . Max sync block is: $HIVEMIND_MAX_BLOCK +echo Attempting to access database $DB_URL +./$HIVE_NAME sync --pid-file hive_sync.pid --test-max-block=$HIVEMIND_MAX_BLOCK --exit-after-sync --test-profile=False --steemd-url "$HIVEMIND_SOURCE_HIVED_URL" --prometheus-port 11011 --database-url $DB_URL --mock-block-data-path mock_data/block_data/follow_op/mock_block_data_follow.json mock_data/block_data/community_op/mock_block_data_community.json mock_data/block_data/reblog_op/mock_block_data_reblog.json --community-start-block 4999998 2>&1 | tee -i hivemind-sync.log +rm hive_sync.pid