Improvements

- Removed double iterations trough transactions
- Account and community register methods changed to support single
  arguments instead lists
- Added missing block flush to process

hive/indexer/accounts.py

@@ -70,7 +70,7 @@ class Accounts:
         return False
 
     @classmethod
-    def register(cls, names, block_date):
+    def register(cls, name, block_date):
         """Block processing: register "candidate" names.
 
         There are four ops which can result in account creation:
@@ -79,24 +79,23 @@ class Accounts:
         the account they name does not already exist!
         """
 
+        if name is None:
+            return
+
         # filter out names which already registered
-        new_names = list(filter(lambda n: not cls.exists(n), set(names)))
-        if not new_names:
+        if cls.exists(name):
             return
 
-        for name in new_names:
-            DB.query("INSERT INTO hive_accounts (name, created_at) "
-                     "VALUES (:name, :date)", name=name, date=block_date)
+        DB.query("INSERT INTO hive_accounts (name, created_at) VALUES (:name, :date)", name=name, date=block_date)
 
         # pull newly-inserted ids and merge into our map
-        sql = "SELECT name, id FROM hive_accounts WHERE name IN :names"
-        for name, _id in DB.query_all(sql, names=tuple(new_names)):
-            cls._ids[name] = _id
+        sql = "SELECT id FROM hive_accounts WHERE name = :name"
+        cls._ids[name] = DB.query_one(sql, name=name)
 
         # post-insert: pass to communities to check for new registrations
         from hive.indexer.community import Community, START_DATE
         if block_date > START_DATE:
-            Community.register(new_names, block_date)
+            Community.register(name, block_date)
 
     # account cache methods
     # ---------------------

hive/indexer/blocks.py

@@ -64,6 +64,7 @@ class Blocks:
         PostDataCache.flush()
         Tags.flush()
         Votes.flush()
+        cls._flush_blocks()
         time_end = perf_counter()
         log.info("[PROCESS BLOCK] %fs", time_end - time_start)
         return ret
@@ -139,43 +140,30 @@ class Blocks:
 
         # head block date shall point to last imported block (not yet current one) to conform hived behavior.
         # that's why operations processed by node are included in the block being currently produced, so its processing time is equal to last produced block.
-        if(cls._head_block_date is None):
+        if cls._head_block_date is None:
             cls._head_block_date = block_date
 
-        # [DK] we will make two scans, first scan will register all accounts
-        account_names = set()
+        json_ops = []
+        update_comment_pending_payouts = []
         for tx_idx, tx in enumerate(block['transactions']):
             for operation in tx['operations']:
                 op_type = operation['type']
                 op = operation['value']
 
+                account_name = None
                 # account ops
                 if op_type == 'pow_operation':
-                    account_names.add(op['worker_account'])
+                    account_name = op['worker_account']
                 elif op_type == 'pow2_operation':
-                    account_names.add(op['work']['value']['input']['worker_account'])
+                    account_name = op['work']['value']['input']['worker_account']
                 elif op_type == 'account_create_operation':
-                    account_names.add(op['new_account_name'])
+                    account_name = op['new_account_name']
                 elif op_type == 'account_create_with_delegation_operation':
-                    account_names.add(op['new_account_name'])
+                    account_name = op['new_account_name']
                 elif op_type == 'create_claimed_account_operation':
-                    account_names.add(op['new_account_name'])
-
-        Accounts.register(account_names, cls._head_block_date)     # register any new names
-
-        # second scan will process all other ops
-        json_ops = []
-        update_comment_pending_payouts = []
-        for tx_idx, tx in enumerate(block['transactions']):
-            for operation in tx['operations']:
-                op_type = operation['type']
-                op = operation['value']
+                    account_name = op['new_account_name']
 
-                if(op_type != 'custom_json_operation'):
-                    if op_type in cls.ops_stats:
-                        cls.ops_stats[op_type] += 1
-                    else:
-                        cls.ops_stats[op_type] = 1
+                Accounts.register(account_name, cls._head_block_date)
 
                 # account metadata updates
                 if op_type == 'account_update_operation':
@@ -206,6 +194,12 @@ class Blocks:
                 elif op_type == 'custom_json_operation':
                     json_ops.append(op)
 
+                if op_type != 'custom_json_operation':
+                    if op_type in cls.ops_stats:
+                        cls.ops_stats[op_type] += 1
+                    else:
+                        cls.ops_stats[op_type] = 1
+
         # follow/reblog/community ops
         if json_ops:
             custom_ops_stats = CustomOp.process_ops(json_ops, num, cls._head_block_date)

hive/indexer/community.py

@@ -104,33 +104,32 @@ class Community:
     _names = {}
 
     @classmethod
-    def register(cls, names, block_date):
+    def register(cls, name, block_date):
         """Block processing: hooks into new account registration.
 
         `Accounts` calls this method with any newly registered names.
         This method checks for any valid community names and inserts them.
         """
 
-        for name in names:
-            #if not re.match(r'^hive-[123]\d{4,6}$', name):
-            if not re.match(r'^hive-[1]\d{4,6}$', name):
-                continue
-            type_id = int(name[5])
-            _id = Accounts.get_id(name)
-
-            # insert community
-            sql = """INSERT INTO hive_communities (id, name, type_id, created_at)
-                          VALUES (:id, :name, :type_id, :date)"""
-            DB.query(sql, id=_id, name=name, type_id=type_id, date=block_date)
-
-            # insert owner
-            sql = """INSERT INTO hive_roles (community_id, account_id, role_id, created_at)
-                         VALUES (:community_id, :account_id, :role_id, :date)"""
-            DB.query(sql, community_id=_id, account_id=_id,
-                     role_id=Role.owner.value, date=block_date)
-
-            Notify('new_community', src_id=None, dst_id=_id,
-                   when=block_date, community_id=_id).write()
+        #if not re.match(r'^hive-[123]\d{4,6}$', name):
+        if not re.match(r'^hive-[1]\d{4,6}$', name):
+            return
+        type_id = int(name[5])
+        _id = Accounts.get_id(name)
+
+        # insert community
+        sql = """INSERT INTO hive_communities (id, name, type_id, created_at)
+                        VALUES (:id, :name, :type_id, :date)"""
+        DB.query(sql, id=_id, name=name, type_id=type_id, date=block_date)
+
+        # insert owner
+        sql = """INSERT INTO hive_roles (community_id, account_id, role_id, created_at)
+                        VALUES (:community_id, :account_id, :role_id, :date)"""
+        DB.query(sql, community_id=_id, account_id=_id,
+                    role_id=Role.owner.value, date=block_date)
+
+        Notify('new_community', src_id=None, dst_id=_id,
+                when=block_date, community_id=_id).write()
 
     @classmethod
     def validated_id(cls, name):