diff --git a/Makefile b/Makefile
index 6190379bdc3152bc521c2f20c4d9eb9e6d079c5b..4b4ec663db8e5e18cf06e0bf73647bcdba329e13 100644
--- a/Makefile
+++ b/Makefile
@@ -3,7 +3,7 @@ ROOT_DIR := $(shell pwd)
PROJECT_NAME := hive
PROJECT_DOCKER_TAG := steemit/$(PROJECT_NAME)
-PROJECT_DOCKER_RUN_ARGS := --link mysql:mysql
+PROJECT_DOCKER_RUN_ARGS := --link db:db
default: build
@@ -18,9 +18,11 @@ run:
compose:
docker-compose up -d
-mysql:
- docker run -d --name steemit_mysql -p 3306:3306 -e MYSQL_ROOT_PASSWORD=root_password -e MYSQL_DATABASE=testdb mysql
+db:
+ docker run -d --name hive_db -p 5432:5432 -e POSTGRES_PASSWORD=root_password -e POSTGRES_DATABASE=hivepy postgres
+mysql:
+ docker run --env DATABASE_URL=mysql://root:root_password@mysql:3306/testdb -p 4000:8080 hive
serve-local:
pipenv run python hive/server/serve.py --port 8080 --database_url='mysql://root:root_password@127.0.0.1:3306/testdb'
diff --git a/Pipfile b/Pipfile
index c224e9810d40db28740bab9bff1a4d742a2b6c6c..e4754682b599679542ec5b30d96c7f465ae87150 100644
--- a/Pipfile
+++ b/Pipfile
@@ -6,8 +6,6 @@ verify_ssl = true
sqlalchemy = "*"
mysqlclient = "*"
-click = "*"
-"click-spinner" = "*"
funcy = "*"
toolz = "*"
maya = "*"
@@ -32,4 +30,4 @@ recommonmark = "*"
"sphinxcontrib-restbuilder" = "*"
yapf = "*"
sphinx = "*"
-"autopep8" = "*"
\ No newline at end of file
+"autopep8" = "*"
diff --git a/hive/conf.py b/hive/conf.py
index c0aefc58fdf340d25cbded15170a313242de797d..81e1d3dbb843276e7913ede0593a1a0d7e936fbb 100644
--- a/hive/conf.py
+++ b/hive/conf.py
@@ -11,8 +11,8 @@ class Conf():
#pylint: disable=invalid-name,line-too-long
p = configargparse.get_arg_parser(default_config_files=['./hive.conf'])
- # runmodes: sync, status
- p.add('--mode', default='sync')
+ # runmodes: sync, server, status
+ p.add('mode', nargs='*', default=['sync'])
# common
p.add('--database-url', env_var='DATABASE_URL', required=True, help='database connection url', default='postgresql://user:pass@localhost:5432/hive')
@@ -25,10 +25,14 @@ class Conf():
p.add('--trail-blocks', type=int, env_var='TRAIL_BLOCKS', default=2)
# specific to API server
- p.add('--port', type=int, env_var='PORT', default=8080)
+ p.add('--http-server-port', type=int, env_var='HTTP_SERVER_PORT', default=8080)
cls._args = p.parse_args()
+ if cls.get('log_level') == 'DEBUG':
+ print(cls._args)
+ print(p.format_values())
+
@classmethod
def args(cls):
return cls._args
diff --git a/hive/indexer/core.py b/hive/indexer/core.py
index fd8df265f33531513bbbb7d045be62ccb243cf7e..aa1e411e21b920b25c6e19065c43b8147fc59256 100755
--- a/hive/indexer/core.py
+++ b/hive/indexer/core.py
@@ -11,7 +11,7 @@ from hive.indexer.cached_post import CachedPost
log = logging.getLogger(__name__)
-def run():
+def run_sync():
print("[HIVE] Welcome to hivemind")
# make sure db schema is up to date, perform checks
@@ -46,7 +46,4 @@ def run():
if __name__ == '__main__':
Conf.init_argparse()
- if Conf.get('mode') == 'status':
- print(DbState.status())
- else:
- run()
+ run_sync()
diff --git a/hive/server/serve.py b/hive/server/serve.py
index eff84f4ec91cbcf49c0151cc1cf88c725a61b9ae..a89ff74b75de70ec649acf105b2cb908a123ecc9 100644
--- a/hive/server/serve.py
+++ b/hive/server/serve.py
@@ -14,127 +14,139 @@ from hive.conf import Conf
from hive.server import condenser_api
from hive.server import hive_api
-Conf.init_argparse()
-log_level = Conf.log_level()
-
-config.debug = (log_level == logging.DEBUG)
-logging.basicConfig(level=log_level)
-logger = logging.getLogger(__name__)
-logging.getLogger('jsonrpcserver.dispatcher.response').setLevel(log_level)
-
-
-hive_methods = (
- hive_api.db_head_state,
- hive_api.get_followers,
- hive_api.get_following,
- hive_api.get_follow_count,
- hive_api.get_user_feed,
- hive_api.get_blog_feed,
- hive_api.get_discussions_by_sort_and_tag,
- hive_api.get_related_posts,
- hive_api.payouts_total,
- hive_api.payouts_last_24h
-)
-
-condenser_methods = (
- condenser_api.call,
- condenser_api.get_followers,
- condenser_api.get_following,
- condenser_api.get_follow_count,
- condenser_api.get_discussions_by_trending,
- condenser_api.get_discussions_by_hot,
- condenser_api.get_discussions_by_promoted,
- condenser_api.get_discussions_by_created,
- condenser_api.get_discussions_by_blog,
- condenser_api.get_discussions_by_feed,
- condenser_api.get_discussions_by_comments,
- condenser_api.get_replies_by_last_update,
- condenser_api.get_content,
- condenser_api.get_content_replies,
- condenser_api.get_state
-)
-
-# Register hive_api methods and (appbase) condenser_api methods
-methods = AsyncMethods()
-for m in hive_methods:
- methods.add(m)
- methods.add(m, 'hive_api.' + m.__name__) # TODO: temp, for testing jussi-style path without jussi
-for m in condenser_methods:
- # note: unclear if appbase expects condenser_api.call or call.condenser_api
- methods.add(m, 'condenser_api.' + m.__name__)
- methods.add(m, 'hive_api.condenser_api.' + m.__name__) # TODO: temp, for testing jussi-style path without jussi
-
-# Register non-appbase condenser_api endpoint (remove after appbase in prod)
-non_appbase_methods = AsyncMethods()
-non_appbase_methods.add(condenser_api.call, 'condenser_api.non_appb.call')
-non_appbase_methods.add(condenser_api.call, 'hive_api.condenser_api.non_appb.call') # TODO: temp, for testing jussi-style path without jussi
-for m in condenser_methods:
- non_appbase_methods.add(m)
-
-app = web.Application()
-app['config'] = dict()
-app['config']['hive.MAX_DB_ROW_RESULTS'] = 100000
-app['config']['hive.DB_QUERY_LIMIT'] = app['config']['hive.MAX_DB_ROW_RESULTS'] + 1
-app['config']['hive.logger'] = logger
-
-async def init_db(app):
- args = app['config']['args']
- db = make_url(args.database_url)
- engine = await create_engine(user=db.username,
- database=db.database,
- password=db.password,
- host=db.host,
- port=db.port,
- **db.query)
- app['db'] = engine
-
-async def close_db(app):
- app['db'].close()
- await app['db'].wait_closed()
-
-
-# Non JSON-RPC routes
-# -------------------
-async def health(request):
- state = await hive_api.db_head_state()
- max_head_age = (Conf.get('trail_blocks') + 1) * 3
-
- if state['db_head_age'] > max_head_age:
- status = 500
- result = 'head block age (%s) > max (%s); head block num: %s' % (
- state['db_head_age'], max_head_age, state['db_head_block'])
- else:
- status = 200
- result = 'head block age is %d, head block num is %d' % (
- state['db_head_age'], state['db_head_block'])
-
- return web.json_response(status=status, data=dict(
- state=state,
- result=result,
- status='OK' if status == 200 else 'WARN',
- source_commit=os.environ.get('SOURCE_COMMIT'),
- schema_hash=os.environ.get('SCHEMA_HASH'),
- docker_tag=os.environ.get('DOCKER_TAG'),
- timestamp=datetime.utcnow().isoformat()))
-
-async def jsonrpc_handler(request):
- request = await request.text()
- response = await methods.dispatch(request)
- return web.json_response(response, status=200, headers={'Access-Control-Allow-Origin': '*'})
-
-async def non_appbase_handler(request):
- request = await request.text()
- response = await non_appbase_methods.dispatch(request)
- return web.json_response(response, status=200, headers={'Access-Control-Allow-Origin': '*'})
-
-
-app.on_startup.append(init_db)
-app.on_cleanup.append(close_db)
-app.router.add_get('/health', health)
-app.router.add_post('/', jsonrpc_handler)
-app.router.add_post('/legacy', non_appbase_handler)
+
+def run_server():
+
+ log_level = Conf.log_level()
+
+ config.debug = (log_level == logging.DEBUG)
+ logging.basicConfig(level=log_level)
+ logger = logging.getLogger(__name__)
+ logging.getLogger('jsonrpcserver.dispatcher.response').setLevel(log_level)
+
+ hive_methods = (
+ hive_api.db_head_state,
+ hive_api.get_followers,
+ hive_api.get_following,
+ hive_api.get_follow_count,
+ hive_api.get_user_feed,
+ hive_api.get_blog_feed,
+ hive_api.get_discussions_by_sort_and_tag,
+ hive_api.get_related_posts,
+ hive_api.payouts_total,
+ hive_api.payouts_last_24h
+ )
+
+ condenser_methods = (
+ condenser_api.call,
+ condenser_api.get_followers,
+ condenser_api.get_following,
+ condenser_api.get_follow_count,
+ condenser_api.get_discussions_by_trending,
+ condenser_api.get_discussions_by_hot,
+ condenser_api.get_discussions_by_promoted,
+ condenser_api.get_discussions_by_created,
+ condenser_api.get_discussions_by_blog,
+ condenser_api.get_discussions_by_feed,
+ condenser_api.get_discussions_by_comments,
+ condenser_api.get_replies_by_last_update,
+ condenser_api.get_content,
+ condenser_api.get_content_replies,
+ condenser_api.get_state
+ )
+
+ # Register hive_api methods and (appbase) condenser_api methods
+ methods = AsyncMethods()
+ for method in hive_methods:
+ methods.add(method)
+
+ # TODO: temp, for testing jussi-style path without jussi
+ methods.add(method, 'hive_api.' + method.__name__)
+
+ for method in condenser_methods:
+ # note: unclear if appbase expects condenser_api.call or call.condenser_api
+ methods.add(method, 'condenser_api.' + method.__name__)
+
+ # TODO: temp, for testing jussi-style path without jussi
+ methods.add(method, 'hive_api.condenser_api.' + method.__name__)
+
+ # Register non-appbase condenser_api endpoint (remove after appbase in prod)
+ non_appbase_methods = AsyncMethods()
+ for method in condenser_methods:
+ non_appbase_methods.add(method)
+
+ # TODO: temp, for testing jussi-style path without jussi
+ non_appbase_methods.add(condenser_api.call, 'condenser_api.non_appb.call')
+ non_appbase_methods.add(condenser_api.call, 'hive_api.condenser_api.non_appb.call')
+ # -----
+
+ app = web.Application()
+ app['config'] = dict()
+ app['config']['args'] = Conf.args()
+ app['config']['hive.MAX_DB_ROW_RESULTS'] = 100000
+ app['config']['hive.DB_QUERY_LIMIT'] = app['config']['hive.MAX_DB_ROW_RESULTS'] + 1
+ app['config']['hive.logger'] = logger
+
+
+ async def init_db(app):
+ args = app['config']['args']
+ db = make_url(args.database_url)
+ engine = await create_engine(user=db.username,
+ database=db.database,
+ password=db.password,
+ host=db.host,
+ port=db.port,
+ **db.query)
+ app['db'] = engine
+
+ async def close_db(app):
+ app['db'].close()
+ await app['db'].wait_closed()
+
+ app.on_startup.append(init_db)
+ app.on_cleanup.append(close_db)
+
+
+ async def health(request):
+ #pylint: disable=unused-argument
+ state = await hive_api.db_head_state()
+ max_head_age = (Conf.get('trail_blocks') + 1) * 3
+
+ if state['db_head_age'] > max_head_age:
+ status = 500
+ result = 'head block age (%s) > max (%s); head block num: %s' % (
+ state['db_head_age'], max_head_age, state['db_head_block'])
+ else:
+ status = 200
+ result = 'head block age is %d, head block num is %d' % (
+ state['db_head_age'], state['db_head_block'])
+
+ return web.json_response(status=status, data=dict(
+ state=state,
+ result=result,
+ status='OK' if status == 200 else 'WARN',
+ source_commit=os.environ.get('SOURCE_COMMIT'),
+ schema_hash=os.environ.get('SCHEMA_HASH'),
+ docker_tag=os.environ.get('DOCKER_TAG'),
+ timestamp=datetime.utcnow().isoformat()))
+
+ async def jsonrpc_handler(request):
+ request = await request.text()
+ response = await methods.dispatch(request)
+ return web.json_response(response, status=200, headers={'Access-Control-Allow-Origin': '*'})
+
+ async def non_appbase_handler(request):
+ request = await request.text()
+ response = await non_appbase_methods.dispatch(request)
+ return web.json_response(response, status=200, headers={'Access-Control-Allow-Origin': '*'})
+
+ app.router.add_get('/health', health)
+ app.router.add_post('/', jsonrpc_handler)
+ app.router.add_post('/legacy', non_appbase_handler)
+
+ web.run_app(app, port=app['config']['args'].http_server_port)
if __name__ == '__main__':
- app['config']['args'] = Conf.args()
- web.run_app(app, port=app['config']['args'].port)
+ Conf.init_argparse()
+ run_server()
diff --git a/run.py b/run.py
new file mode 100755
index 0000000000000000000000000000000000000000..b52ac1554b379096e67f298162e224d69f4aa62d
--- /dev/null
+++ b/run.py
@@ -0,0 +1,25 @@
+#!/usr/local/bin/python3
+
+from hive.conf import Conf
+from hive.db.db_state import DbState
+from hive.indexer.core import run_sync
+from hive.server.serve import run_server
+
+def run():
+ Conf.init_argparse()
+ mode = '/'.join(Conf.get('mode'))
+
+ if mode == 'server':
+ run_server()
+
+ elif mode == 'sync':
+ run_sync()
+
+ elif mode == 'status':
+ print(DbState.status())
+
+ else:
+ raise Exception("unknown run mode %s" % mode)
+
+if __name__ == '__main__':
+ run()
diff --git a/service/hive-indexer/run b/service/hive-indexer/run
index c54c312c7f9acc179d341d5cc5525a2342a25ddd..748257ea4ee542e0e23781e97c00ddd547547885 100755
--- a/service/hive-indexer/run
+++ b/service/hive-indexer/run
@@ -2,4 +2,4 @@
POPULATE_CMD="$(which hive)"
-exec "${POPULATE_CMD}" indexer run
+exec "${POPULATE_CMD}" sync
diff --git a/service/hive-web/run b/service/hive-web/run
index 44891287b983a1b5c2ecd3aa7968e20c4594e1e5..ad077d1f5863bed219a96660155a484ea1c6b811 100755
--- a/service/hive-web/run
+++ b/service/hive-web/run
@@ -2,4 +2,4 @@
POPULATE_CMD="$(which hive)"
-python3 /app/hive/server/serve.py --database-url="${DATABASE_URL}" --port="${HTTP_SERVER_PORT}"
+python3 /app/hive/run.py server --database-url="${DATABASE_URL}" --http-server-port="${HTTP_SERVER_PORT}"
diff --git a/setup.py b/setup.py
index 37865809264026aff95a0e3b1c46a4945f3e5d59..82770c8bcf6665b57c0f13a75483a3ec9cc7f1aa 100644
--- a/setup.py
+++ b/setup.py
@@ -30,19 +30,17 @@ setup(
'aiohttp',
'certifi',
'sqlalchemy',
- 'click',
'funcy',
'toolz',
'maya',
'ujson',
'urllib3',
- 'PrettyTable',
'psycopg2',
'aiocache',
'configargparse',
],
entry_points={
'console_scripts': [
- 'hive=hive.cli:cli',
+ 'hive=run:run',
]
})