Merge branch 'hivemind-ci-continue' into 'develop'

Hivemind CI implementation

See merge request !32

.gitlab-ci.yaml

+# https://hub.docker.com/r/library/python/tags/
+image: "python:3.7"
+
 stages:
 - build
 - test
@@ -7,14 +10,75 @@ variables:
   GIT_DEPTH: 1
   LC_ALL: "C"
   GIT_STRATEGY: clone
-  GIT_SUBMODULE_STRATEGY: recursiv
+  GIT_SUBMODULE_STRATEGY: recursive
+  HIVEMIND_SOURCE_HIVED_URL: $HIVEMIND_SOURCE_HIVED_URL
+  PYTHONUSERBASE: ./local-site
+
+before_script:
+  - echo "CI_NODE_TOTAL is $CI_NODE_TOTAL"
+  - echo "CI_NODE_INDEX is $CI_NODE_INDEX"
+  - echo "CI_CONCURRENT_ID is $CI_CONCURRENT_ID"
+  - echo "CI_COMMIT_REF_SLUG is $CI_COMMIT_REF_SLUG"
+  - python3 --version
+  - python3 -m site --user-site
+  - pip3 install --user --upgrade pip setuptools
 
 hivemind_build:
   stage: build
+  script:
+    - git fetch --tags
+    - git tag -f ci_implicit_tag
+    - echo $PYTHONUSERBASE
+    - "python3 setup.py bdist_egg"
+    - ls -l dist/*
+  artifacts:
+    paths:
+      - dist/
+    expire_in: 1 week
+  tags:
+     - hivemind
+
+hivemind_deploy:
+  stage: deploy
+  environment: hive-4.pl.syncad.com
+  needs:
+    - job: hivemind_build
+      artifacts: true
+  variables:
+    GIT_STRATEGY: none
+    HIVEMIND_DB_NAME: "hive_$CI_JOB_ID"
+    HIVEMIND_HTTP_PORT: $HIVEMIND_HTTP_PORT
+
+    # Configured at gitlab repository settings side
+    POSTGRES_USER: $HIVEMIND_POSTGRES_USER
+    POSTGRES_PASSWORD: $HIVEMIND_POSTGRES_PASSWORD
+    POSTGRES_HOST_AUTH_METHOD: trust
+    # official way to provide password to psql: http://www.postgresql.org/docs/9.3/static/libpq-envars.html
+    PGPASSWORD: $HIVEMIND_POSTGRES_PASSWORD
+    DB_URL: "$HIVEMIND_POSTGRESQL_CONNECTION_STRING/hive_$CI_JOB_ID"
+    PYTHONUSERBASE: ./local-site
 
   script:
-    - python3 setup.py build
-    - python3 setup.py install --prefix "$CI_PROJECT_DIR/hivemind-install"
+    - ls -l dist/*
+    - rm -rf ./local-site
+    - mkdir -p `python3 -m site --user-site`
+    - python3 setup.py install --user --force
+    - ./local-site/bin/hive -h
+    - "echo Attempting to recreate database $HIVEMIND_DB_NAME"
+    - psql -U $POSTGRES_USER -h localhost -d postgres -c "DROP DATABASE IF EXISTS $HIVEMIND_DB_NAME;"
+    - psql -U $POSTGRES_USER -h localhost -d postgres -c "CREATE DATABASE $HIVEMIND_DB_NAME;"
+    - "echo Attempting to starting hive sync using hived node: $HIVEMIND_SOURCE_HIVED_URL . Max sync block is: $HIVEMIND_MAX_BLOCK"
+    - echo Attempting to access database $DB_URL
+    - ./local-site/bin/hive sync --test-max-block=$HIVEMIND_MAX_BLOCK --exit-after-sync --test-profile=False --steemd-url "$HIVEMIND_SOURCE_HIVED_URL" --database-url $DB_URL 2>&1 | tee -i hivemind-sync.log
+    - echo Attempting to start hive server listening on $HIVEMIND_HTTP_PORT port...
+    - screen -L -Logfile hive_server.log -dmS hive_server_$CI_JOB_ID ./local-site/bin/hive server --pid-file hive_server.pid --http-server-port $HIVEMIND_HTTP_PORT --steemd-url "$HIVEMIND_SOURCE_HIVED_URL" --database-url $DB_URL
+    - for i in `seq 1 10`; do if [ -f hive_server.pid ]; then break; else sleep 1; fi;  done
+    - cat hive_server.pid
+  artifacts:
+    paths:
+      - hivemind-sync.log
+      - hive_server.pid
+    expire_in: 1 week
 
   tags:
      - hivemind

.gitmodules

 [submodule "tests/tests_api"]
 	path = tests/tests_api
-	url = git@gitlab.syncad.com:hive/tests_api.git
+	url = https://gitlab.syncad.com/hive/tests_api.git

hive/cli.py

@@ -2,6 +2,7 @@
 
 """CLI service router"""
 
+import os
 import logging
 from hive.conf import Conf
 from hive.db.adapter import Db
@@ -15,6 +16,17 @@ def run():
     Db.set_shared_instance(conf.db())
     mode = conf.mode()
 
+    pid_file_name = conf.pid_file()
+    if pid_file_name is not None:
+        fh = open(pid_file_name, 'w')
+        if fh is None:
+          print("Cannot write into specified pid_file: %s", pidpid_file_name)
+        else:
+          pid = os.getpid()
+          fh.write(str(pid))
+          fh.close()
+
+
     if conf.get('test_profile'):
         from hive.utils.profiler import Profiler
         with Profiler():

hive/conf.py

@@ -48,8 +48,11 @@ class Conf():
         add('--log-level', env_var='LOG_LEVEL', default='INFO')
         add('--test-disable-sync', type=strtobool, env_var='TEST_DISABLE_SYNC', help='(debug) skip sync and sweep; jump to block streaming', default=False)
         add('--test-max-block', type=int, env_var='TEST_MAX_BLOCK', help='(debug) only sync to given block, for running sync test', default=None)
+        add('--exit-after-sync', help='exit when sync is completed', action='store_true')
         add('--test-profile', type=strtobool, env_var='TEST_PROFILE', help='(debug) profile execution', default=False)
 
+        add('--pid-file', type=str, env_var='PID_FILE', help='Allows to dump current process pid into specified file', default=None)
+
         # needed for e.g. tests - other args may be present
         args = (parser.parse_args() if strict
                 else parser.parse_known_args()[0])
@@ -118,3 +121,7 @@ class Conf():
     def log_level(self):
         """Get `logger`s internal int level from config string."""
         return int_log_level(self.get('log_level'))
+
+    def pid_file(self):
+        """Get optional pid_file name to put current process pid in"""
+        return self._args.get("pid_file", None)

hive/indexer/sync.py

@@ -189,6 +189,9 @@ class Sync:
     def __init__(self, conf):
         self._conf = conf
         self._db = conf.db()
+
+        log.info("Using hived url: `%s'", self._conf.get('steemd_url'))
+
         self._steem = conf.steem()
 
     def run(self):
@@ -217,6 +220,10 @@ class Sync:
                 return
             DbState.finish_initial_sync()
 
+            if self._conf.get("exit_after_sync"):
+                log.info("Exiting after sync on user request...")
+                return
+
         else:
             # recover from fork
             Blocks.verify_head(self._steem)

hive/utils/normalize.py

@@ -59,7 +59,7 @@ def parse_amount(value, expected_unit=None):
         raise Exception("invalid input amount %s" % repr(value))
 
     if expected_unit:
-        assert unit == expected_unit
+        assert unit == expected_unit, "Unexpected unit: %s" % unit
         return dec_amount
 
     return (dec_amount, unit)

setup.py

@@ -20,11 +20,11 @@ tests_require = [
 # yapf: disable
 setup(
     name='hivemind',
-    version='0.0.1',
+    version_format='0.0.1+{gitsha}',
     description='Developer-friendly microservice powering social networks on the Steem blockchain.',
     long_description=open('README.md').read(),
     packages=find_packages(exclude=['scripts']),
-    setup_requires=['pytest-runner'],
+    setup_requires=['pytest-runner', 'setuptools-git-version'],
     tests_require=tests_require,
     install_requires=[
         #'aiopg==0.16.0',