standardize vars, service scripts

Makefile

@@ -3,7 +3,7 @@ ROOT_DIR := $(shell pwd)
 
 PROJECT_NAME := hive
 PROJECT_DOCKER_TAG := steemit/$(PROJECT_NAME)
-PROJECT_DOCKER_RUN_ARGS := --link mysql:mysql
+PROJECT_DOCKER_RUN_ARGS := --link db:db
 
 default: build
 
@@ -18,9 +18,11 @@ run:
 compose:
 	docker-compose up -d
 
-mysql:
-	docker run -d --name steemit_mysql -p 3306:3306 -e MYSQL_ROOT_PASSWORD=root_password -e MYSQL_DATABASE=testdb mysql
+db:
+	docker run -d --name hive_db -p 5432:5432 -e POSTGRES_PASSWORD=root_password -e POSTGRES_DATABASE=hivepy postgres
 
+mysql:
+	docker run --env DATABASE_URL=mysql://root:root_password@mysql:3306/testdb -p 4000:8080 hive
 
 serve-local:
 	pipenv run python hive/server/serve.py --port 8080 --database_url='mysql://root:root_password@127.0.0.1:3306/testdb'

Pipfile

@@ -6,8 +6,6 @@ verify_ssl = true
 
 sqlalchemy = "*"
 mysqlclient = "*"
-click = "*"
-"click-spinner" = "*"
 funcy = "*"
 toolz = "*"
 maya = "*"
@@ -32,4 +30,4 @@ recommonmark = "*"
 "sphinxcontrib-restbuilder" = "*"
 yapf = "*"
 sphinx = "*"
-"autopep8" = "*"
+"autopep8" = "*"
\ No newline at end of file

hive/conf.py

@@ -11,8 +11,8 @@ class Conf():
         #pylint: disable=invalid-name,line-too-long
         p = configargparse.get_arg_parser(default_config_files=['./hive.conf'])
 
-        # runmodes: sync, status
-        p.add('--mode', default='sync')
+        # runmodes: sync, server, status
+        p.add('mode', nargs='*', default=['sync'])
 
         # common
         p.add('--database-url', env_var='DATABASE_URL', required=True, help='database connection url', default='postgresql://user:pass@localhost:5432/hive')
@@ -25,10 +25,14 @@ class Conf():
         p.add('--trail-blocks', type=int, env_var='TRAIL_BLOCKS', default=2)
 
         # specific to API server
-        p.add('--port', type=int, env_var='PORT', default=8080)
+        p.add('--http-server-port', type=int, env_var='HTTP_SERVER_PORT', default=8080)
 
         cls._args = p.parse_args()
 
+        if cls.get('log_level') == 'DEBUG':
+            print(cls._args)
+            print(p.format_values())
+
     @classmethod
     def args(cls):
         return cls._args

hive/indexer/core.py

@@ -11,7 +11,7 @@ from hive.indexer.cached_post import CachedPost
 log = logging.getLogger(__name__)
 
 
-def run():
+def run_sync():
     print("[HIVE] Welcome to hivemind")
 
     # make sure db schema is up to date, perform checks
@@ -46,7 +46,4 @@ def run():
 
 if __name__ == '__main__':
     Conf.init_argparse()
-    if Conf.get('mode') == 'status':
-        print(DbState.status())
-    else:
-        run()
+    run_sync()

hive/server/serve.py

@@ -14,127 +14,139 @@ from hive.conf import Conf
 from hive.server import condenser_api
 from hive.server import hive_api
 
-Conf.init_argparse()
-log_level = Conf.log_level()
-
-config.debug = (log_level == logging.DEBUG)
-logging.basicConfig(level=log_level)
-logger = logging.getLogger(__name__)
-logging.getLogger('jsonrpcserver.dispatcher.response').setLevel(log_level)
-
-
-hive_methods = (
-    hive_api.db_head_state,
-    hive_api.get_followers,
-    hive_api.get_following,
-    hive_api.get_follow_count,
-    hive_api.get_user_feed,
-    hive_api.get_blog_feed,
-    hive_api.get_discussions_by_sort_and_tag,
-    hive_api.get_related_posts,
-    hive_api.payouts_total,
-    hive_api.payouts_last_24h
-)
-
-condenser_methods = (
-    condenser_api.call,
-    condenser_api.get_followers,
-    condenser_api.get_following,
-    condenser_api.get_follow_count,
-    condenser_api.get_discussions_by_trending,
-    condenser_api.get_discussions_by_hot,
-    condenser_api.get_discussions_by_promoted,
-    condenser_api.get_discussions_by_created,
-    condenser_api.get_discussions_by_blog,
-    condenser_api.get_discussions_by_feed,
-    condenser_api.get_discussions_by_comments,
-    condenser_api.get_replies_by_last_update,
-    condenser_api.get_content,
-    condenser_api.get_content_replies,
-    condenser_api.get_state
-)
-
-# Register hive_api methods and (appbase) condenser_api methods
-methods = AsyncMethods()
-for m in hive_methods:
-    methods.add(m)
-    methods.add(m, 'hive_api.' + m.__name__)  # TODO: temp, for testing jussi-style path without jussi
-for m in condenser_methods:
-    # note: unclear if appbase expects condenser_api.call or call.condenser_api
-    methods.add(m, 'condenser_api.' + m.__name__)
-    methods.add(m, 'hive_api.condenser_api.' + m.__name__) # TODO: temp, for testing jussi-style path without jussi
-
-# Register non-appbase condenser_api endpoint (remove after appbase in prod)
-non_appbase_methods = AsyncMethods()
-non_appbase_methods.add(condenser_api.call, 'condenser_api.non_appb.call')
-non_appbase_methods.add(condenser_api.call, 'hive_api.condenser_api.non_appb.call') # TODO: temp, for testing jussi-style path without jussi
-for m in condenser_methods:
-    non_appbase_methods.add(m)
-
-app = web.Application()
-app['config'] = dict()
-app['config']['hive.MAX_DB_ROW_RESULTS'] = 100000
-app['config']['hive.DB_QUERY_LIMIT'] = app['config']['hive.MAX_DB_ROW_RESULTS'] + 1
-app['config']['hive.logger'] = logger
-
-async def init_db(app):
-    args = app['config']['args']
-    db = make_url(args.database_url)
-    engine = await create_engine(user=db.username,
-                                 database=db.database,
-                                 password=db.password,
-                                 host=db.host,
-                                 port=db.port,
-                                 **db.query)
-    app['db'] = engine
-
-async def close_db(app):
-    app['db'].close()
-    await app['db'].wait_closed()
-
-
-# Non JSON-RPC routes
-# -------------------
-async def health(request):
-    state = await hive_api.db_head_state()
-    max_head_age = (Conf.get('trail_blocks') + 1) * 3
-
-    if state['db_head_age'] > max_head_age:
-        status = 500
-        result = 'head block age (%s) > max (%s); head block num: %s' % (
-            state['db_head_age'], max_head_age, state['db_head_block'])
-    else:
-        status = 200
-        result = 'head block age is %d, head block num is %d' % (
-            state['db_head_age'], state['db_head_block'])
-
-    return web.json_response(status=status, data=dict(
-        state=state,
-        result=result,
-        status='OK' if status == 200 else 'WARN',
-        source_commit=os.environ.get('SOURCE_COMMIT'),
-        schema_hash=os.environ.get('SCHEMA_HASH'),
-        docker_tag=os.environ.get('DOCKER_TAG'),
-        timestamp=datetime.utcnow().isoformat()))
-
-async def jsonrpc_handler(request):
-    request = await request.text()
-    response = await methods.dispatch(request)
-    return web.json_response(response, status=200, headers={'Access-Control-Allow-Origin': '*'})
-
-async def non_appbase_handler(request):
-    request = await request.text()
-    response = await non_appbase_methods.dispatch(request)
-    return web.json_response(response, status=200, headers={'Access-Control-Allow-Origin': '*'})
-
-
-app.on_startup.append(init_db)
-app.on_cleanup.append(close_db)
-app.router.add_get('/health', health)
-app.router.add_post('/', jsonrpc_handler)
-app.router.add_post('/legacy', non_appbase_handler)
+
+def run_server():
+
+    log_level = Conf.log_level()
+
+    config.debug = (log_level == logging.DEBUG)
+    logging.basicConfig(level=log_level)
+    logger = logging.getLogger(__name__)
+    logging.getLogger('jsonrpcserver.dispatcher.response').setLevel(log_level)
+
+    hive_methods = (
+        hive_api.db_head_state,
+        hive_api.get_followers,
+        hive_api.get_following,
+        hive_api.get_follow_count,
+        hive_api.get_user_feed,
+        hive_api.get_blog_feed,
+        hive_api.get_discussions_by_sort_and_tag,
+        hive_api.get_related_posts,
+        hive_api.payouts_total,
+        hive_api.payouts_last_24h
+    )
+
+    condenser_methods = (
+        condenser_api.call,
+        condenser_api.get_followers,
+        condenser_api.get_following,
+        condenser_api.get_follow_count,
+        condenser_api.get_discussions_by_trending,
+        condenser_api.get_discussions_by_hot,
+        condenser_api.get_discussions_by_promoted,
+        condenser_api.get_discussions_by_created,
+        condenser_api.get_discussions_by_blog,
+        condenser_api.get_discussions_by_feed,
+        condenser_api.get_discussions_by_comments,
+        condenser_api.get_replies_by_last_update,
+        condenser_api.get_content,
+        condenser_api.get_content_replies,
+        condenser_api.get_state
+    )
+
+    # Register hive_api methods and (appbase) condenser_api methods
+    methods = AsyncMethods()
+    for method in hive_methods:
+        methods.add(method)
+
+        # TODO: temp, for testing jussi-style path without jussi
+        methods.add(method, 'hive_api.' + method.__name__)
+
+    for method in condenser_methods:
+        # note: unclear if appbase expects condenser_api.call or call.condenser_api
+        methods.add(method, 'condenser_api.' + method.__name__)
+
+        # TODO: temp, for testing jussi-style path without jussi
+        methods.add(method, 'hive_api.condenser_api.' + method.__name__)
+
+    # Register non-appbase condenser_api endpoint (remove after appbase in prod)
+    non_appbase_methods = AsyncMethods()
+    for method in condenser_methods:
+        non_appbase_methods.add(method)
+
+    # TODO: temp, for testing jussi-style path without jussi
+    non_appbase_methods.add(condenser_api.call, 'condenser_api.non_appb.call')
+    non_appbase_methods.add(condenser_api.call, 'hive_api.condenser_api.non_appb.call')
+    # -----
+
+    app = web.Application()
+    app['config'] = dict()
+    app['config']['args'] = Conf.args()
+    app['config']['hive.MAX_DB_ROW_RESULTS'] = 100000
+    app['config']['hive.DB_QUERY_LIMIT'] = app['config']['hive.MAX_DB_ROW_RESULTS'] + 1
+    app['config']['hive.logger'] = logger
+
+
+    async def init_db(app):
+        args = app['config']['args']
+        db = make_url(args.database_url)
+        engine = await create_engine(user=db.username,
+                                     database=db.database,
+                                     password=db.password,
+                                     host=db.host,
+                                     port=db.port,
+                                     **db.query)
+        app['db'] = engine
+
+    async def close_db(app):
+        app['db'].close()
+        await app['db'].wait_closed()
+
+    app.on_startup.append(init_db)
+    app.on_cleanup.append(close_db)
+
+
+    async def health(request):
+        #pylint: disable=unused-argument
+        state = await hive_api.db_head_state()
+        max_head_age = (Conf.get('trail_blocks') + 1) * 3
+
+        if state['db_head_age'] > max_head_age:
+            status = 500
+            result = 'head block age (%s) > max (%s); head block num: %s' % (
+                state['db_head_age'], max_head_age, state['db_head_block'])
+        else:
+            status = 200
+            result = 'head block age is %d, head block num is %d' % (
+                state['db_head_age'], state['db_head_block'])
+
+        return web.json_response(status=status, data=dict(
+            state=state,
+            result=result,
+            status='OK' if status == 200 else 'WARN',
+            source_commit=os.environ.get('SOURCE_COMMIT'),
+            schema_hash=os.environ.get('SCHEMA_HASH'),
+            docker_tag=os.environ.get('DOCKER_TAG'),
+            timestamp=datetime.utcnow().isoformat()))
+
+    async def jsonrpc_handler(request):
+        request = await request.text()
+        response = await methods.dispatch(request)
+        return web.json_response(response, status=200, headers={'Access-Control-Allow-Origin': '*'})
+
+    async def non_appbase_handler(request):
+        request = await request.text()
+        response = await non_appbase_methods.dispatch(request)
+        return web.json_response(response, status=200, headers={'Access-Control-Allow-Origin': '*'})
+
+    app.router.add_get('/health', health)
+    app.router.add_post('/', jsonrpc_handler)
+    app.router.add_post('/legacy', non_appbase_handler)
+
+    web.run_app(app, port=app['config']['args'].http_server_port)
 
 
 if __name__ == '__main__':
-    app['config']['args'] = Conf.args()
-    web.run_app(app, port=app['config']['args'].port)
+    Conf.init_argparse()
+    run_server()

run.py

+#!/usr/local/bin/python3
+
+from hive.conf import Conf
+from hive.db.db_state import DbState
+from hive.indexer.core import run_sync
+from hive.server.serve import run_server
+
+def run():
+    Conf.init_argparse()
+    mode = '/'.join(Conf.get('mode'))
+
+    if mode == 'server':
+        run_server()
+
+    elif mode == 'sync':
+        run_sync()
+
+    elif mode == 'status':
+        print(DbState.status())
+
+    else:
+        raise Exception("unknown run mode %s" % mode)
+
+if __name__ == '__main__':
+    run()

service/hive-indexer/run

@@ -2,4 +2,4 @@
 
 POPULATE_CMD="$(which hive)"
 
-exec "${POPULATE_CMD}" indexer run
+exec "${POPULATE_CMD}" sync

service/hive-web/run

@@ -2,4 +2,4 @@
 
 POPULATE_CMD="$(which hive)"
 
-python3 /app/hive/server/serve.py --database-url="${DATABASE_URL}" --port="${HTTP_SERVER_PORT}"
+python3 /app/hive/run.py server --database-url="${DATABASE_URL}" --http-server-port="${HTTP_SERVER_PORT}"

setup.py

@@ -30,19 +30,17 @@ setup(
         'aiohttp',
         'certifi',
         'sqlalchemy',
-        'click',
         'funcy',
         'toolz',
         'maya',
         'ujson',
         'urllib3',
-        'PrettyTable',
         'psycopg2',
         'aiocache',
         'configargparse',
     ],
     entry_points={
         'console_scripts': [
-            'hive=hive.cli:cli',
+            'hive=run:run',
         ]
     })