vops are now filtered, logging time for block processing

hive/indexer/blocks.py

@@ -11,6 +11,7 @@ from hive.indexer.payments import Payments
 from hive.indexer.follow import Follow
 from hive.indexer.votes import Votes
 from hive.indexer.post_data_cache import PostDataCache
+from time import perf_counter
 
 log = logging.getLogger(__name__)
 
@@ -35,15 +36,19 @@ class Blocks:
     @classmethod
     def process(cls, block, vops_in_block, hived):
         """Process a single block. Always wrap in a transaction!"""
+        time_start = perf_counter()
         #assert is_trx_active(), "Block.process must be in a trx"
         ret = cls._process(block, vops_in_block, hived, is_initial_sync=False)
         PostDataCache.flush()
         Votes.flush()
+        time_end = perf_counter()
+        log.info("[PROCESS BLOCK] %fs", time_end - time_start)
         return ret
 
     @classmethod
     def process_multi(cls, blocks, vops, hived, is_initial_sync=False):
         """Batch-process blocks; wrapped in a transaction."""
+        time_start = perf_counter()
         DB.query("START TRANSACTION")
 
         last_num = 0
@@ -63,6 +68,8 @@ class Blocks:
         Follow.flush(trx=False)
 
         DB.query("COMMIT")
+        time_end = perf_counter()
+        log.info("[PROCESS MULTI] %i blocks in %fs", len(blocks), time_end - time_start)
 
     @classmethod
     def _process(cls, block, virtual_operations, hived, is_initial_sync=False):
@@ -137,9 +144,7 @@ class Blocks:
         comment_payout_ops = {}
         vops = []
         if not is_initial_sync:
-            ret = hived.get_virtual_operations(num)
-            for vop in ret:
-                vops.append(vop['op'])
+            vops = hived.get_virtual_operations(num)
         else:
             vops = virtual_operations[num]['ops'] if num in virtual_operations else []
         for vop in vops:

hive/steem/client.py

@@ -144,19 +144,26 @@ class SteemClient:
 
     def get_virtual_operations(self, block):
         """ Get virtual ops from block """
-        ret = self.__exec('get_ops_in_block', {"block_num":block, "only_virtual":True})
-        return ret['ops'] if 'ops' in ret else []
+        result = self.__exec('get_ops_in_block', {"block_num":block, "only_virtual":True})
+        tracked_ops = ['curation_reward_operation', 'author_reward_operation', 'comment_reward_operation', 'effective_comment_vote_operation']
+        ret = []
+        result = result['ops'] if 'ops' in result else []
+        for vop in result:
+            if vop['op']['type'] in tracked_ops:
+                ret.append(vop['op'])
+        return ret
 
     def enum_virtual_ops(self, begin_block, end_block):
         """ Get virtual ops for range of blocks """
         result = self.__exec('enum_virtual_ops', {"block_range_begin":begin_block, "block_range_end":end_block})
         ops = result['ops'] if 'ops' in result else []
+        tracked_ops = ['curation_reward_operation', 'author_reward_operation', 'comment_reward_operation', 'effective_comment_vote_operation']
         ret = {}
         for op in ops:
             block = op['block']
-            if block in ret:
+            if block in ret and op['op']['type'] in tracked_ops:
                 ret[block]['ops'].append(op['op'])
-            else:
+            if block not in ret and op['op']['type'] in tracked_ops:
                 ret[block] = {'timestamp':op['timestamp'], 'ops':[op['op']]}
         return ret