Proposal of the fix for performance issue

hive/indexer/blocks.py

@@ -91,6 +91,10 @@ class Blocks:
         cls._flush_blocks()
         Follow.flush(trx=False)
 
+        step_time = perf_counter()
+        Posts.flush()
+        log.info("[PROCESS MULTI] Comment payment flush in %fs", perf_counter() - step_time)
+
         DB.query("COMMIT")
         time_end = perf_counter()
         log.info("[PROCESS MULTI] %i blocks in %fs", len(blocks), time_end - time_start)

hive/indexer/posts.py

@@ -30,6 +30,8 @@ class Posts:
     _hits = 0
     _miss = 0
 
+    _comment_payout_ops = []
+
     @classmethod
     def last_id(cls):
         """Get the last indexed post id."""
@@ -146,8 +148,7 @@ class Posts:
             cls._insert_feed_cache(result, block_date)
 
     @classmethod
-    def comment_payout_op(cls, ops, date):
-        ops_stats = {}
+    def flush(cls):
         sql = """
               UPDATE hive_posts AS ihp SET
                   total_payout_value    = COALESCE( data_source.total_payout_value,                     ihp.total_payout_value ),
@@ -202,9 +203,23 @@ class Posts:
               INNER JOIN hive_permlink_data hpd_p ON hpd_p.permlink = t.permlink
               ) as data_source(author_id, permlink_id, total_payout_value)
               WHERE ihp.permlink_id = data_source.permlink_id and ihp.author_id = data_source.author_id
-              """
+        """
+
+        def chunks(lst, n):
+            """Yield successive n-sized chunks from lst."""
+            for i in range(0, len(lst), n):
+                yield lst[i:i + n]
+
+        for chunk in chunks(cls._comment_payout_ops, 1000):
+            values_str = ','.join(chunk)
+            actual_query = sql.format(values_str)
+
+            DB.query(actual_query)
 
-        values = []
+        cls._comment_payout_ops.clear()
+
+    @classmethod
+    def comment_payout_op(cls, ops, date):
         values_limit = 1000
 
         ops_stats = { 'author_reward_operation' : 0, 'comment_reward_operation' : 0, 'effective_comment_vote_operation' : 0, 'comment_payout_update_operation' : 0 }
@@ -289,7 +304,7 @@ class Posts:
                 payout_at = date  #Here should be `cashout_time`
                 last_payout = date
 
-            values.append("('{}', '{}', {}, {}, {}, {}, {}, {}, {}, {}, {}, '{}'::timestamp, {}, {}, {})".format(
+            cls._comment_payout_ops.append("('{}', '{}', {}, {}, {}, {}, {}, {}, {}, {}, {}, '{}'::timestamp, {}, {}, {})".format(
               author,
               permlink,
               "NULL" if ( total_payout_value is None ) else ( "'{}'".format( legacy_amount(total_payout_value) ) ),
@@ -308,19 +323,6 @@ class Posts:
 
               "NULL" if ( is_paidout is None ) else is_paidout ))
 
-            if len(values) >= values_limit:
-                values_str = ','.join(values)
-                actual_query = sql.format(values_str)
-
-                DB.query(actual_query)
-                values.clear()
-
-        if len(values) > 0:
-            values_str = ','.join(values)
-            actual_query = sql.format(values_str)
-
-            DB.query(actual_query)
-            values.clear()
         return ops_stats
 
     @classmethod