From bbce31d59da8ed25e167d164cb613fadef5ecbc7 Mon Sep 17 00:00:00 2001
From: Dariusz Kedzierski <dkedzierski@syncad.com>
Date: Mon, 22 Jun 2020 15:13:23 +0200
Subject: [PATCH] vops are now filtered, logging time for block processing
---
hive/indexer/blocks.py | 11 ++++++++---
hive/steem/client.py | 15 +++++++++++----
2 files changed, 19 insertions(+), 7 deletions(-)
diff --git a/hive/indexer/blocks.py b/hive/indexer/blocks.py
index bb6e0ec36..4f5f34b9b 100644
--- a/hive/indexer/blocks.py
+++ b/hive/indexer/blocks.py
@@ -11,6 +11,7 @@ from hive.indexer.payments import Payments
from hive.indexer.follow import Follow
from hive.indexer.votes import Votes
from hive.indexer.post_data_cache import PostDataCache
+from time import perf_counter
log = logging.getLogger(__name__)
@@ -35,15 +36,19 @@ class Blocks:
@classmethod
def process(cls, block, vops_in_block, hived):
"""Process a single block. Always wrap in a transaction!"""
+ time_start = perf_counter()
#assert is_trx_active(), "Block.process must be in a trx"
ret = cls._process(block, vops_in_block, hived, is_initial_sync=False)
PostDataCache.flush()
Votes.flush()
+ time_end = perf_counter()
+ log.info("[PROCESS BLOCK] %fs", time_end - time_start)
return ret
@classmethod
def process_multi(cls, blocks, vops, hived, is_initial_sync=False):
"""Batch-process blocks; wrapped in a transaction."""
+ time_start = perf_counter()
DB.query("START TRANSACTION")
last_num = 0
@@ -63,6 +68,8 @@ class Blocks:
Follow.flush(trx=False)
DB.query("COMMIT")
+ time_end = perf_counter()
+ log.info("[PROCESS MULTI] %i blocks in %fs", len(blocks), time_end - time_start)
@classmethod
def _process(cls, block, virtual_operations, hived, is_initial_sync=False):
@@ -137,9 +144,7 @@ class Blocks:
comment_payout_ops = {}
vops = []
if not is_initial_sync:
- ret = hived.get_virtual_operations(num)
- for vop in ret:
- vops.append(vop['op'])
+ vops = hived.get_virtual_operations(num)
else:
vops = virtual_operations[num]['ops'] if num in virtual_operations else []
for vop in vops:
diff --git a/hive/steem/client.py b/hive/steem/client.py
index 4baf983ee..af7a2e2ff 100644
--- a/hive/steem/client.py
+++ b/hive/steem/client.py
@@ -144,19 +144,26 @@ class SteemClient:
def get_virtual_operations(self, block):
""" Get virtual ops from block """
- ret = self.__exec('get_ops_in_block', {"block_num":block, "only_virtual":True})
- return ret['ops'] if 'ops' in ret else []
+ result = self.__exec('get_ops_in_block', {"block_num":block, "only_virtual":True})
+ tracked_ops = ['curation_reward_operation', 'author_reward_operation', 'comment_reward_operation', 'effective_comment_vote_operation']
+ ret = []
+ result = result['ops'] if 'ops' in result else []
+ for vop in result:
+ if vop['op']['type'] in tracked_ops:
+ ret.append(vop['op'])
+ return ret
def enum_virtual_ops(self, begin_block, end_block):
""" Get virtual ops for range of blocks """
result = self.__exec('enum_virtual_ops', {"block_range_begin":begin_block, "block_range_end":end_block})
ops = result['ops'] if 'ops' in result else []
+ tracked_ops = ['curation_reward_operation', 'author_reward_operation', 'comment_reward_operation', 'effective_comment_vote_operation']
ret = {}
for op in ops:
block = op['block']
- if block in ret:
+ if block in ret and op['op']['type'] in tracked_ops:
ret[block]['ops'].append(op['op'])
- else:
+ if block not in ret and op['op']['type'] in tracked_ops:
ret[block] = {'timestamp':op['timestamp'], 'ops':[op['op']]}
return ret
--
GitLab