Do a vacuum full every time minutes during massive sync to keep size of...

Do a vacuum full every time minutes during massive sync to keep size of account_reputations table reasonable

db/database_schema.sql

@@ -46,6 +46,7 @@ CREATE TABLE IF NOT EXISTS account_reputations
     CONSTRAINT PK_account_reputations PRIMARY KEY (account_id)
 );
 
+
 PERFORM hive.app_register_table( __schema_name, 'account_reputations', __schema_name );
 
 DROP TYPE IF EXISTS AccountReputation CASCADE;
@@ -61,6 +62,10 @@ CREATE TYPE AccountReputation AS
 END
 $$;
 
+-- the current version of sqlfluff doesn't understand 'GRANT MAINTAIN'
+GRANT MAINTAIN ON ALL TABLES IN SCHEMA reptracker_app TO hived_group; -- noqa: PRS
+GRANT ALL ON SCHEMA reptracker_app TO hived_group;
+
 INSERT INTO reptracker_app_status
 (continue_processing, is_accounts_copied)
 VALUES

db/main_loop.sql

@@ -122,7 +122,7 @@ BEGIN
       CONTINUE;
     END IF;
 
-    PERFORM reptracker_process_blocks(_appContext, _blocks_range);
+    CALL reptracker_process_blocks(_appContext, _blocks_range);
   END LOOP;
 
   ASSERT FALSE, 'Cannot reach this point';

db/rep_helpers.sql

@@ -104,9 +104,8 @@ END
 $$;
 
 
-CREATE OR REPLACE FUNCTION reptracker_process_blocks(_context_name hive.context_name, _block_range hive.blocks_range, _logs BOOLEAN = true)
-RETURNS VOID
-LANGUAGE 'plpgsql' VOLATILE
+CREATE OR REPLACE PROCEDURE reptracker_process_blocks(_context_name hive.context_name, _block_range hive.blocks_range, _logs BOOLEAN = true)
+LANGUAGE 'plpgsql'
 AS
 $$
 BEGIN
@@ -117,6 +116,7 @@ BEGIN
     END IF;
 
     CALL reptracker_massive_processing(_block_range.first_block, _block_range.last_block, _logs);
+    PERFORM hive.app_request_table_vacuum('reptracker_app.account_reputations', interval '10 minutes'); --eventually fixup hard-coded schema name
     RETURN;
   END IF;
 

haf @ e7eeb458

-Subproject commit 5c8a432a43b799fbeeebd142b5f24cdea10f59ae
+Subproject commit e7eeb458e6f59a347b8bbcfa1f9f538b33261020

scripts/process_blocks.sh

@@ -70,8 +70,16 @@ process_blocks() {
     log_file="reptracker_sync.log"
     # record the startup time for use in health checks
     date -uIseconds > /tmp/block_processing_startup_time.txt
-    # wait untill reptracker indexes are created
-    psql "$POSTGRES_ACCESS" -v "ON_ERROR_STOP=on" -c "\timing" -c "SELECT hive.wait_till_registered_indexes_created('${REPTRACKER_SCHEMA}')"
+
+    # Loop until hive.check_if_registered_indexes_created returns true
+    while true; do
+        result=$(psql "$POSTGRES_ACCESS" -t -c "SELECT hive.check_if_registered_indexes_created('${REPTRACKER_SCHEMA}');" | tr -d '[:space:]')
+        if [ "$result" = "t" ]; then
+            break
+        else
+            sleep 10
+        fi
+    done
     psql "$POSTGRES_ACCESS" -v "ON_ERROR_STOP=on" -v REPTRACKER_SCHEMA="${REPTRACKER_SCHEMA}" -c "\timing" -c "SET SEARCH_PATH TO ${REPTRACKER_SCHEMA};" -c "CALL ${REPTRACKER_SCHEMA}.main('${REPTRACKER_SCHEMA}', $n_blocks);" 2>&1 | tee -i $log_file
 }