Commit fb9bd3dc authored by Bartek Wrona's avatar Bartek Wrona

Merge branch 'hivemind-ci-continue' into 'develop'

Hivemind CI implementation

See merge request !32
parents 31a4d740 ed236172
# https://hub.docker.com/r/library/python/tags/
image: "python:3.7"
stages:
- build
- test
......@@ -7,14 +10,75 @@ variables:
GIT_DEPTH: 1
LC_ALL: "C"
GIT_STRATEGY: clone
GIT_SUBMODULE_STRATEGY: recursiv
GIT_SUBMODULE_STRATEGY: recursive
HIVEMIND_SOURCE_HIVED_URL: $HIVEMIND_SOURCE_HIVED_URL
PYTHONUSERBASE: ./local-site
before_script:
- echo "CI_NODE_TOTAL is $CI_NODE_TOTAL"
- echo "CI_NODE_INDEX is $CI_NODE_INDEX"
- echo "CI_CONCURRENT_ID is $CI_CONCURRENT_ID"
- echo "CI_COMMIT_REF_SLUG is $CI_COMMIT_REF_SLUG"
- python3 --version
- python3 -m site --user-site
- pip3 install --user --upgrade pip setuptools
hivemind_build:
stage: build
script:
- git fetch --tags
- git tag -f ci_implicit_tag
- echo $PYTHONUSERBASE
- "python3 setup.py bdist_egg"
- ls -l dist/*
artifacts:
paths:
- dist/
expire_in: 1 week
tags:
- hivemind
hivemind_deploy:
stage: deploy
environment: hive-4.pl.syncad.com
needs:
- job: hivemind_build
artifacts: true
variables:
GIT_STRATEGY: none
HIVEMIND_DB_NAME: "hive_$CI_JOB_ID"
HIVEMIND_HTTP_PORT: $HIVEMIND_HTTP_PORT
# Configured at gitlab repository settings side
POSTGRES_USER: $HIVEMIND_POSTGRES_USER
POSTGRES_PASSWORD: $HIVEMIND_POSTGRES_PASSWORD
POSTGRES_HOST_AUTH_METHOD: trust
# official way to provide password to psql: http://www.postgresql.org/docs/9.3/static/libpq-envars.html
PGPASSWORD: $HIVEMIND_POSTGRES_PASSWORD
DB_URL: "$HIVEMIND_POSTGRESQL_CONNECTION_STRING/hive_$CI_JOB_ID"
PYTHONUSERBASE: ./local-site
script:
- python3 setup.py build
- python3 setup.py install --prefix "$CI_PROJECT_DIR/hivemind-install"
- ls -l dist/*
- rm -rf ./local-site
- mkdir -p `python3 -m site --user-site`
- python3 setup.py install --user --force
- ./local-site/bin/hive -h
- "echo Attempting to recreate database $HIVEMIND_DB_NAME"
- psql -U $POSTGRES_USER -h localhost -d postgres -c "DROP DATABASE IF EXISTS $HIVEMIND_DB_NAME;"
- psql -U $POSTGRES_USER -h localhost -d postgres -c "CREATE DATABASE $HIVEMIND_DB_NAME;"
- "echo Attempting to starting hive sync using hived node: $HIVEMIND_SOURCE_HIVED_URL . Max sync block is: $HIVEMIND_MAX_BLOCK"
- echo Attempting to access database $DB_URL
- ./local-site/bin/hive sync --test-max-block=$HIVEMIND_MAX_BLOCK --exit-after-sync --test-profile=False --steemd-url "$HIVEMIND_SOURCE_HIVED_URL" --database-url $DB_URL 2>&1 | tee -i hivemind-sync.log
- echo Attempting to start hive server listening on $HIVEMIND_HTTP_PORT port...
- screen -L -Logfile hive_server.log -dmS hive_server_$CI_JOB_ID ./local-site/bin/hive server --pid-file hive_server.pid --http-server-port $HIVEMIND_HTTP_PORT --steemd-url "$HIVEMIND_SOURCE_HIVED_URL" --database-url $DB_URL
- for i in `seq 1 10`; do if [ -f hive_server.pid ]; then break; else sleep 1; fi; done
- cat hive_server.pid
artifacts:
paths:
- hivemind-sync.log
- hive_server.pid
expire_in: 1 week
tags:
- hivemind
[submodule "tests/tests_api"]
path = tests/tests_api
url = git@gitlab.syncad.com:hive/tests_api.git
url = https://gitlab.syncad.com/hive/tests_api.git
......@@ -2,6 +2,7 @@
"""CLI service router"""
import os
import logging
from hive.conf import Conf
from hive.db.adapter import Db
......@@ -15,6 +16,17 @@ def run():
Db.set_shared_instance(conf.db())
mode = conf.mode()
pid_file_name = conf.pid_file()
if pid_file_name is not None:
fh = open(pid_file_name, 'w')
if fh is None:
print("Cannot write into specified pid_file: %s", pidpid_file_name)
else:
pid = os.getpid()
fh.write(str(pid))
fh.close()
if conf.get('test_profile'):
from hive.utils.profiler import Profiler
with Profiler():
......
......@@ -48,8 +48,11 @@ class Conf():
add('--log-level', env_var='LOG_LEVEL', default='INFO')
add('--test-disable-sync', type=strtobool, env_var='TEST_DISABLE_SYNC', help='(debug) skip sync and sweep; jump to block streaming', default=False)
add('--test-max-block', type=int, env_var='TEST_MAX_BLOCK', help='(debug) only sync to given block, for running sync test', default=None)
add('--exit-after-sync', help='exit when sync is completed', action='store_true')
add('--test-profile', type=strtobool, env_var='TEST_PROFILE', help='(debug) profile execution', default=False)
add('--pid-file', type=str, env_var='PID_FILE', help='Allows to dump current process pid into specified file', default=None)
# needed for e.g. tests - other args may be present
args = (parser.parse_args() if strict
else parser.parse_known_args()[0])
......@@ -118,3 +121,7 @@ class Conf():
def log_level(self):
"""Get `logger`s internal int level from config string."""
return int_log_level(self.get('log_level'))
def pid_file(self):
"""Get optional pid_file name to put current process pid in"""
return self._args.get("pid_file", None)
......@@ -189,6 +189,9 @@ class Sync:
def __init__(self, conf):
self._conf = conf
self._db = conf.db()
log.info("Using hived url: `%s'", self._conf.get('steemd_url'))
self._steem = conf.steem()
def run(self):
......@@ -217,6 +220,10 @@ class Sync:
return
DbState.finish_initial_sync()
if self._conf.get("exit_after_sync"):
log.info("Exiting after sync on user request...")
return
else:
# recover from fork
Blocks.verify_head(self._steem)
......
......@@ -59,7 +59,7 @@ def parse_amount(value, expected_unit=None):
raise Exception("invalid input amount %s" % repr(value))
if expected_unit:
assert unit == expected_unit
assert unit == expected_unit, "Unexpected unit: %s" % unit
return dec_amount
return (dec_amount, unit)
......
......@@ -20,11 +20,11 @@ tests_require = [
# yapf: disable
setup(
name='hivemind',
version='0.0.1',
version_format='0.0.1+{gitsha}',
description='Developer-friendly microservice powering social networks on the Steem blockchain.',
long_description=open('README.md').read(),
packages=find_packages(exclude=['scripts']),
setup_requires=['pytest-runner'],
setup_requires=['pytest-runner', 'setuptools-git-version'],
tests_require=tests_require,
install_requires=[
#'aiopg==0.16.0',
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment