From 7810505bebdab1d0962cccc0550dd28183dae6bf Mon Sep 17 00:00:00 2001
From: Dariusz Kedzierski <dkedzierski@syncad.com>
Date: Thu, 30 Jul 2020 16:24:38 +0200
Subject: [PATCH] Proposal of the fix for performance issue
---
hive/indexer/blocks.py | 4 ++++
hive/indexer/posts.py | 38 ++++++++++++++++++++------------------
2 files changed, 24 insertions(+), 18 deletions(-)
diff --git a/hive/indexer/blocks.py b/hive/indexer/blocks.py
index a030a0feb..bc3bfc75e 100644
--- a/hive/indexer/blocks.py
+++ b/hive/indexer/blocks.py
@@ -91,6 +91,10 @@ class Blocks:
cls._flush_blocks()
Follow.flush(trx=False)
+ step_time = perf_counter()
+ Posts.flush()
+ log.info("[PROCESS MULTI] Comment payment flush in %fs", perf_counter() - step_time)
+
DB.query("COMMIT")
time_end = perf_counter()
log.info("[PROCESS MULTI] %i blocks in %fs", len(blocks), time_end - time_start)
diff --git a/hive/indexer/posts.py b/hive/indexer/posts.py
index 066ad72dd..b21596c9d 100644
--- a/hive/indexer/posts.py
+++ b/hive/indexer/posts.py
@@ -30,6 +30,8 @@ class Posts:
_hits = 0
_miss = 0
+ _comment_payout_ops = []
+
@classmethod
def last_id(cls):
"""Get the last indexed post id."""
@@ -146,8 +148,7 @@ class Posts:
cls._insert_feed_cache(result, block_date)
@classmethod
- def comment_payout_op(cls, ops, date):
- ops_stats = {}
+ def flush(cls):
sql = """
UPDATE hive_posts AS ihp SET
total_payout_value = COALESCE( data_source.total_payout_value, ihp.total_payout_value ),
@@ -202,9 +203,23 @@ class Posts:
INNER JOIN hive_permlink_data hpd_p ON hpd_p.permlink = t.permlink
) as data_source(author_id, permlink_id, total_payout_value)
WHERE ihp.permlink_id = data_source.permlink_id and ihp.author_id = data_source.author_id
- """
+ """
+
+ def chunks(lst, n):
+ """Yield successive n-sized chunks from lst."""
+ for i in range(0, len(lst), n):
+ yield lst[i:i + n]
+
+ for chunk in chunks(cls._comment_payout_ops, 1000):
+ values_str = ','.join(chunk)
+ actual_query = sql.format(values_str)
+
+ DB.query(actual_query)
- values = []
+ cls._comment_payout_ops.clear()
+
+ @classmethod
+ def comment_payout_op(cls, ops, date):
values_limit = 1000
ops_stats = { 'author_reward_operation' : 0, 'comment_reward_operation' : 0, 'effective_comment_vote_operation' : 0, 'comment_payout_update_operation' : 0 }
@@ -289,7 +304,7 @@ class Posts:
payout_at = date #Here should be `cashout_time`
last_payout = date
- values.append("('{}', '{}', {}, {}, {}, {}, {}, {}, {}, {}, {}, '{}'::timestamp, {}, {}, {})".format(
+ cls._comment_payout_ops.append("('{}', '{}', {}, {}, {}, {}, {}, {}, {}, {}, {}, '{}'::timestamp, {}, {}, {})".format(
author,
permlink,
"NULL" if ( total_payout_value is None ) else ( "'{}'".format( legacy_amount(total_payout_value) ) ),
@@ -308,19 +323,6 @@ class Posts:
"NULL" if ( is_paidout is None ) else is_paidout ))
- if len(values) >= values_limit:
- values_str = ','.join(values)
- actual_query = sql.format(values_str)
-
- DB.query(actual_query)
- values.clear()
-
- if len(values) > 0:
- values_str = ','.join(values)
- actual_query = sql.format(values_str)
-
- DB.query(actual_query)
- values.clear()
return ops_stats
@classmethod
--
GitLab