hivesync fixes, add sync-health check

hive/server/serve.py

@@ -101,6 +101,12 @@ def run_server():
     app.on_startup.append(init_db)
     app.on_cleanup.append(close_db)
 
+    async def head_age(request):
+        #pylint: disable=unused-argument
+        healthy_age = 3600 # one hour
+        curr_age = (await hive_api.db_head_state())['db_head_age']
+        status = 500 if curr_age > healthy_age else 200
+        return web.Response(status=status, text=str(curr_age))
 
     async def health(request):
         #pylint: disable=unused-argument
@@ -134,6 +140,7 @@ def run_server():
         return web.json_response(response, status=200, headers={'Access-Control-Allow-Origin': '*'})
 
     app.router.add_get('/.well-known/healthcheck.json', health)
+    app.router.add_get('/head_age', head_age)
     app.router.add_get('/health', health)
     app.router.add_post('/', jsonrpc_handler)
 

scripts/entrypoint.sh

@@ -42,6 +42,7 @@ else
     mkdir -p /etc/service/hivesync
     cp /usr/local/bin/hivesync.sh /etc/service/hivesync/run
     chmod +x /etc/service/hivesync/run
+    echo hivemind: starting hivesync service
     runsv /etc/service/hivesync
 fi
 

scripts/hivesync.sh

 #!/bin/bash
 
 # if the indexer dies by itself, kill runsv causing the container to exit
-HIVESYNC_PID=`pgrep -f sync`
+HIVESYNC_PID=`pgrep -f 'hive sync'`
 if [[ ! $? -eq 0 ]]; then
 	echo NOTIFYALERT! hivemindsync has quit unexpectedly, killing container and starting a new instance..
 	sleep 30
@@ -9,7 +9,8 @@ if [[ ! $? -eq 0 ]]; then
 	kill -9 $RUN_SV_PID
 fi
 
-HTTP_CODE=`curl -I -s -o /dev/null -w "%{http_code}" http://127.0.0.1/health/`
+# NOTE: this API endpoint returns head age in seconds, and a 200 code if it's less than 3600
+HTTP_CODE=`curl -I -s -o /dev/null -w "%{http_code}" http://127.0.0.1/head_age`
 
 # if we get a 200 then hive is synced, start syncing operation
 if [[ ${HTTP_CODE} -eq 200 ]]; then
@@ -44,4 +45,4 @@ if [[ ${HTTP_CODE} -eq 200 ]]; then
 fi
 
 # check every 60 seconds if synced
-sleep 60
+sleep 60
\ No newline at end of file