Allow the docker health check for the block processor to report health

as long as reputation tracker is actively processing blocks, even if
it's not yet synced

docker/scripts/block-processing-healthcheck.sh

@@ -8,6 +8,31 @@ trap 'trap - 2 15 && kill -- -$$' 2 15
 postgres_user=${POSTGRES_USER:-"haf_admin"}
 postgres_host=${POSTGRES_HOST:-"localhost"}
 postgres_port=${POSTGRES_PORT:-5432}
-POSTGRES_ACCESS=${POSTGRES_URL:-"postgresql://$postgres_user@$postgres_host:$postgres_port/haf_block_log"}
+POSTGRES_ACCESS=${POSTGRES_URL:-"postgresql://$postgres_user@$postgres_host:$postgres_port/haf_block_log?application_name=reptracker_health_check"}
 
-exec [ "$(psql "$POSTGRES_ACCESS" --quiet --no-align --tuples-only --command="SELECT hive.is_app_in_sync('reptracker_app');")" = t ]
+# this health check will return healthy if:
+# - reputation_tracker has processed a block in the last 60 seconds
+#   (as long as it was also after the container started, we don't want
+#    to report healthy immediately after a restart)
+# or
+# - reputation_tracker's head block has caught up to haf's irreversible block
+#   (so we don't mark reputation_tracker as unhealthy if HAF stops getting blocks)
+#
+# This check needs to know when the block processing started, so the docker entrypoint
+# must write this to a file like:
+#   date --utc --iso-8601=seconds > /tmp/block_processing_startup_time.txt
+if [ ! -f "/tmp/block_processing_startup_time.txt" ]; then
+  echo "file /tmp/block_processing_startup_time.txt does not exist, which means block"
+  echo "processing hasn't started yet"
+  exit 1
+fi
+STARTUP_TIME="$(cat /tmp/block_processing_startup_time.txt)"
+CHECK="SET TIME ZONE 'UTC'; \
+       SELECT ((now() - (SELECT last_active_at FROM hafd.contexts WHERE name = 'reptracker_app')) < interval '1 minute' \
+               AND (SELECT last_active_at FROM hafd.contexts WHERE name = 'reptracker_app') > '${STARTUP_TIME}'::timestamp) OR \
+              hive.is_app_in_sync('reptracker_app');"
+
+
+# the docker container probably won't have a locale set, do this to suppress the warning
+export LC_ALL=C
+exec [ "$(psql "$POSTGRES_ACCESS" --quiet --no-align --tuples-only --command="${CHECK}")" = t ]

scripts/process_blocks.sh

@@ -68,6 +68,8 @@ POSTGRES_ACCESS=${POSTGRES_URL:-"postgresql://$POSTGRES_USER@$POSTGRES_HOST:$POS
 process_blocks() {
     local n_blocks="${1:-null}"
     log_file="reptracker_sync.log"
+    # record the startup time for use in health checks
+    date -uIseconds > /tmp/block_processing_startup_time.txt
     psql "$POSTGRES_ACCESS" -v "ON_ERROR_STOP=on" -v REPTRACKER_SCHEMA="${REPTRACKER_SCHEMA}" -c "\timing" -c "SET SEARCH_PATH TO ${REPTRACKER_SCHEMA};" -c "CALL ${REPTRACKER_SCHEMA}.main('${REPTRACKER_SCHEMA}', $n_blocks);" 2>&1 | tee -i $log_file
 }