Compare revisions

Bartek Wrona · Bartek Wrona · Bartek Wrona · Bartek Wrona · Bartek Wrona · Bartek Wrona
--- a/.gitlab-ci.yaml
+++ b/.gitlab-ci.yaml
@@ -12,6 +12,15 @@ variables:
  GIT_STRATEGY: clone
  GIT_SUBMODULE_STRATEGY: recursive
  HIVEMIND_SOURCE_HIVED_URL: $HIVEMIND_SOURCE_HIVED_URL
+  HIVEMIND_DB_NAME: "hive_$CI_COMMIT_REF_SLUG"
+  HIVEMIND_HTTP_PORT: $HIVEMIND_HTTP_PORT
+  # Configured at gitlab repository settings side
+  POSTGRES_USER: $HIVEMIND_POSTGRES_USER
+  POSTGRES_PASSWORD: $HIVEMIND_POSTGRES_PASSWORD
+  POSTGRES_HOST_AUTH_METHOD: trust
+  # official way to provide password to psql: http://www.postgresql.org/docs/9.3/static/libpq-envars.html
+  PGPASSWORD: $HIVEMIND_POSTGRES_PASSWORD
+
  PYTHONUSERBASE: ./local-site

 before_script:
@@ -38,7 +47,7 @@ hivemind_build:
  tags:
     - hivemind

-hivemind_deploy:
+hivemind_sync:
  stage: deploy
  environment: hive-4.pl.syncad.com
  needs:
@@ -46,34 +55,10 @@ hivemind_deploy:
      artifacts: true
  variables:
    GIT_STRATEGY: none
-    HIVEMIND_DB_NAME: "hive_$CI_JOB_ID"
-    HIVEMIND_HTTP_PORT: $HIVEMIND_HTTP_PORT
-
-    # Configured at gitlab repository settings side
-    POSTGRES_USER: $HIVEMIND_POSTGRES_USER
-    POSTGRES_PASSWORD: $HIVEMIND_POSTGRES_PASSWORD
-    POSTGRES_HOST_AUTH_METHOD: trust
-    # official way to provide password to psql: http://www.postgresql.org/docs/9.3/static/libpq-envars.html
-    PGPASSWORD: $HIVEMIND_POSTGRES_PASSWORD
-    DB_URL: "$HIVEMIND_POSTGRESQL_CONNECTION_STRING/hive_$CI_JOB_ID"
-    PYTHONUSERBASE: ./local-site

  script:
-    - ls -l dist/*
-    - rm -rf ./local-site
-    - mkdir -p `python3 -m site --user-site`
-    - python3 setup.py install --user --force
-    - ./local-site/bin/hive -h
-    - "echo Attempting to recreate database $HIVEMIND_DB_NAME"
-    - psql -U $POSTGRES_USER -h localhost -d postgres -c "DROP DATABASE IF EXISTS $HIVEMIND_DB_NAME;"
-    - psql -U $POSTGRES_USER -h localhost -d postgres -c "CREATE DATABASE $HIVEMIND_DB_NAME;"
-    - "echo Attempting to starting hive sync using hived node: $HIVEMIND_SOURCE_HIVED_URL . Max sync block is: $HIVEMIND_MAX_BLOCK"
-    - echo Attempting to access database $DB_URL
-    - ./local-site/bin/hive sync --test-max-block=$HIVEMIND_MAX_BLOCK --exit-after-sync --test-profile=False --steemd-url "$HIVEMIND_SOURCE_HIVED_URL" --database-url $DB_URL 2>&1 | tee -i hivemind-sync.log
-    - echo Attempting to start hive server listening on $HIVEMIND_HTTP_PORT port...
-    - screen -L -Logfile hive_server.log -dmS hive_server_$CI_JOB_ID ./local-site/bin/hive server --pid-file hive_server.pid --http-server-port $HIVEMIND_HTTP_PORT --steemd-url "$HIVEMIND_SOURCE_HIVED_URL" --database-url $DB_URL
-    - for i in `seq 1 10`; do if [ -f hive_server.pid ]; then break; else sleep 1; fi;  done
-    - cat hive_server.pid
+    - scripts/ci_sync.sh "$HIVEMIND_DB_NAME" "$HIVEMIND_POSTGRESQL_CONNECTION_STRING" "$HIVEMIND_SOURCE_HIVED_URL" $HIVEMIND_MAX_BLOCK
+    - scripts/ci_start_server.sh "$HIVEMIND_DB_NAME" "$HIVEMIND_POSTGRESQL_CONNECTION_STRING" "$HIVEMIND_SOURCE_HIVED_URL" $HIVEMIND_HTTP_PORT
  artifacts:
    paths:
      - hivemind-sync.log
@@ -82,3 +67,26 @@ hivemind_deploy:

  tags:
     - hivemind
+
+hivemind_start_server:
+  stage: deploy
+  environment: hive-4.pl.syncad.com
+  needs:
+    - job: hivemind_build
+      artifacts: true
+  variables:
+    GIT_STRATEGY: none
+
+  script:
+    - scripts/ci_start_server.sh "$HIVEMIND_DB_NAME" "$HIVEMIND_POSTGRESQL_CONNECTION_STRING" "$HIVEMIND_SOURCE_HIVED_URL" $HIVEMIND_HTTP_PORT
+
+  artifacts:
+    paths:
+      - hive_server.log
+      - hive_server.pid
+    expire_in: 1 week
+
+  when: manual
+
+  tags:
+     - hivemind
--- a/Pipfile
+++ b/Pipfile
@@ -11,7 +11,7 @@ toolz = "*"
 maya = "*"
 ujson = "*"
 prettytable = "*"
-jsonrpcserver = "*"
+jsonrpcserver = "4.1.3+8f3437a"
 aiohttp = "*"
 aiopg = "*"
 "psycopg2-binary" = "*"

--- a/hive/server/serve.py
+++ b/hive/server/serve.py
@@ -11,6 +11,8 @@ from aiohttp import web
 from jsonrpcserver.methods import Methods
 from jsonrpcserver import async_dispatch as dispatch

+import simplejson
+
 from hive.server.condenser_api import methods as condenser_api
 from hive.server.condenser_api.tags import get_trending_tags as condenser_api_get_trending_tags
 from hive.server.condenser_api.get_state import get_state as condenser_api_get_state
@@ -35,6 +37,12 @@ from hive.server.db import Db

 # pylint: disable=too-many-lines

+def decimal_serialize(obj):
+    return simplejson.dumps(obj=obj, use_decimal=True)
+
+def decimal_deserialize(s):
+    return simplejson.loads(s=s, use_decimal=True)
+
 async def db_head_state(context):
    """Status/health check."""
    db = context['db']
@@ -268,10 +276,10 @@ def run_server(conf):
        """Handles all hive jsonrpc API requests."""
        request = await request.text()
        # debug=True refs https://github.com/bcb/jsonrpcserver/issues/71
-        response = await dispatch(request, methods=methods, debug=True, context=app)
+        response = await dispatch(request, methods=methods, debug=True, context=app, serialize=decimal_serialize, deserialize=decimal_deserialize)
        if response.wanted:
            headers = {'Access-Control-Allow-Origin': '*'}
-            return web.json_response(response.deserialized(), status=200, headers=headers)
+            return web.json_response(response.deserialized(), status=200, headers=headers, dumps=decimal_serialize)
        return web.Response()

    if conf.get('sync_to_s3'):

--- a/requirements.txt
+++ b/requirements.txt
@@ -14,7 +14,8 @@ funcy==1.14
 -e git+git@gitlab.syncad.com:hive/hivemind.git@ba54fc07c08c2fd08b37475c9a15d61c64e380c9#egg=hivemind
 humanize==2.4.0
 idna==2.9
-jsonrpcserver==4.0.1
+-e git+https://github.com/bcb/jsonrpcserver.git@8f3437a19b6d1a8f600ee2c9b112116c85f17827#egg=jsonrpcserver-4.1.3+8f3437a
+simplejson
 jsonschema==2.6.0
 Mako==1.1.2
 Markdown==2.4.1

--- a/scripts/ci_start_server.sh
+++ b/scripts/ci_start_server.sh
+#!/bin/bash 
+
+set -e
+
+HIVEMIND_DB_NAME=$1
+HIVEMIND_POSTGRESQL_CONNECTION_STRING=$2
+HIVEMIND_SOURCE_HIVED_URL=$3
+HIVEMIND_HTTP_PORT=$4
+
+PYTHONUSERBASE=./local-site
+
+DB_NAME=${HIVEMIND_DB_NAME//-/_}
+DB_NAME=${DB_NAME//\[/_}
+DB_NAME=${DB_NAME//]/_}
+
+DB_URL=$HIVEMIND_POSTGRESQL_CONNECTION_STRING/$DB_NAME
+
+echo Attempting to start hive server listening on $HIVEMIND_HTTP_PORT port...
+if [ -f hive_server.pid ]; then kill -SIGINT `cat hive_server.pid`; fi;
+rm -f hive_server.pid
+screen -L -Logfile hive_server.log -dmS hive_server_$CI_JOB_ID ./local-site/bin/hive server --pid-file hive_server.pid --http-server-port $HIVEMIND_HTTP_PORT --steemd-url "$HIVEMIND_SOURCE_HIVED_URL" --database-url $DB_URL
+for i in `seq 1 10`; do if [ -f hive_server.pid ]; then break; else sleep 1; fi;  done
+if [ -f hive_server.pid ]; then echo "Hive server started successfully..."; else "Hive server start failure..." ; fi;
+
--- a/scripts/ci_sync.sh
+++ b/scripts/ci_sync.sh
+#!/bin/bash 
+
+set -e
+
+HIVEMIND_DB_NAME=$1
+HIVEMIND_POSTGRESQL_CONNECTION_STRING=$2
+HIVEMIND_SOURCE_HIVED_URL=$3
+HIVEMIND_MAX_BLOCK=$4
+
+PYTHONUSERBASE=./local-site
+
+DB_NAME=${HIVEMIND_DB_NAME//-/_}
+DB_NAME=${DB_NAME//\[/_}
+DB_NAME=${DB_NAME//]/_}
+DB_URL=$HIVEMIND_POSTGRESQL_CONNECTION_STRING/$DB_NAME
+echo Corrected db name $DB_NAME
+echo Corrected db url $DB_URL
+ls -l dist/*
+rm -rf ./local-site
+mkdir -p `python3 -m site --user-site`
+python3 setup.py install --user --force
+./local-site/bin/hive -h
+echo Attempting to recreate database $DB_NAME
+psql -U $POSTGRES_USER -h localhost -d postgres -c "DROP DATABASE IF EXISTS $DB_NAME;"
+psql -U $POSTGRES_USER -h localhost -d postgres -c "CREATE DATABASE $DB_NAME;"
+echo Attempting to starting hive sync using hived node: $HIVEMIND_SOURCE_HIVED_URL . Max sync block is: $HIVEMIND_MAX_BLOCK
+echo Attempting to access database $DB_URL
+./local-site/bin/hive sync --test-max-block=$HIVEMIND_MAX_BLOCK --exit-after-sync --test-profile=False --steemd-url "$HIVEMIND_SOURCE_HIVED_URL" --database-url $DB_URL 2>&1 | tee -i hivemind-sync.log
--- a/setup.py
+++ b/setup.py
@@ -26,10 +26,14 @@ setup(
    packages=find_packages(exclude=['scripts']),
    setup_requires=['pytest-runner', 'setuptools-git-version'],
    tests_require=tests_require,
+    dependency_links=[
+      'https://github.com/bcb/jsonrpcserver/tarball/8f3437a19b6d1a8f600ee2c9b112116c85f17827#egg=jsonrpcserver-4.1.3+8f3437a'
+    ],
    install_requires=[
        #'aiopg==0.16.0',
        'aiopg @ https://github.com/aio-libs/aiopg/tarball/862fff97e4ae465333451a4af2a838bfaa3dd0bc',
-        'jsonrpcserver==4.0.1',
+        'jsonrpcserver @ https://github.com/bcb/jsonrpcserver/tarball/8f3437a19b6d1a8f600ee2c9b112116c85f17827#egg=jsonrpcserver',
+        'simplejson',
        'aiohttp',
        'certifi',
        'sqlalchemy',

--- a/tests_api @ 6f9d9f14
+++ b/tests_api @ 6f9d9f14
-Subproject commit 4f27447a2c15a6e760a3f4c511652e3c89abb5c0
+Subproject commit 6f9d9f14f3188bb2b8119d97e3d31e778691b707
No results found