From e6864f863a7598dea7587b9db013beb001b27ffb Mon Sep 17 00:00:00 2001
From: Dariusz Kedzierski <dkedzierski@syncad.com>
Date: Fri, 31 Jul 2020 15:26:53 +0200
Subject: [PATCH] Improvements
- Removed double iterations trough transactions
- Account and community register methods changed to support single
arguments instead lists
- Added missing block flush to process
---
hive/indexer/accounts.py | 19 +++++++++---------
hive/indexer/blocks.py | 40 ++++++++++++++++----------------------
hive/indexer/community.py | 41 +++++++++++++++++++--------------------
3 files changed, 46 insertions(+), 54 deletions(-)
diff --git a/hive/indexer/accounts.py b/hive/indexer/accounts.py
index 76fa2b1f1..5ae8bd2b7 100644
--- a/hive/indexer/accounts.py
+++ b/hive/indexer/accounts.py
@@ -70,7 +70,7 @@ class Accounts:
return False
@classmethod
- def register(cls, names, block_date):
+ def register(cls, name, block_date):
"""Block processing: register "candidate" names.
There are four ops which can result in account creation:
@@ -79,24 +79,23 @@ class Accounts:
the account they name does not already exist!
"""
+ if name is None:
+ return
+
# filter out names which already registered
- new_names = list(filter(lambda n: not cls.exists(n), set(names)))
- if not new_names:
+ if cls.exists(name):
return
- for name in new_names:
- DB.query("INSERT INTO hive_accounts (name, created_at) "
- "VALUES (:name, :date)", name=name, date=block_date)
+ DB.query("INSERT INTO hive_accounts (name, created_at) VALUES (:name, :date)", name=name, date=block_date)
# pull newly-inserted ids and merge into our map
- sql = "SELECT name, id FROM hive_accounts WHERE name IN :names"
- for name, _id in DB.query_all(sql, names=tuple(new_names)):
- cls._ids[name] = _id
+ sql = "SELECT id FROM hive_accounts WHERE name = :name"
+ cls._ids[name] = DB.query_one(sql, name=name)
# post-insert: pass to communities to check for new registrations
from hive.indexer.community import Community, START_DATE
if block_date > START_DATE:
- Community.register(new_names, block_date)
+ Community.register(name, block_date)
# account cache methods
# ---------------------
diff --git a/hive/indexer/blocks.py b/hive/indexer/blocks.py
index e1558f0fb..7629a089f 100644
--- a/hive/indexer/blocks.py
+++ b/hive/indexer/blocks.py
@@ -64,6 +64,7 @@ class Blocks:
PostDataCache.flush()
Tags.flush()
Votes.flush()
+ cls._flush_blocks()
time_end = perf_counter()
log.info("[PROCESS BLOCK] %fs", time_end - time_start)
return ret
@@ -139,43 +140,30 @@ class Blocks:
# head block date shall point to last imported block (not yet current one) to conform hived behavior.
# that's why operations processed by node are included in the block being currently produced, so its processing time is equal to last produced block.
- if(cls._head_block_date is None):
+ if cls._head_block_date is None:
cls._head_block_date = block_date
- # [DK] we will make two scans, first scan will register all accounts
- account_names = set()
+ json_ops = []
+ update_comment_pending_payouts = []
for tx_idx, tx in enumerate(block['transactions']):
for operation in tx['operations']:
op_type = operation['type']
op = operation['value']
+ account_name = None
# account ops
if op_type == 'pow_operation':
- account_names.add(op['worker_account'])
+ account_name = op['worker_account']
elif op_type == 'pow2_operation':
- account_names.add(op['work']['value']['input']['worker_account'])
+ account_name = op['work']['value']['input']['worker_account']
elif op_type == 'account_create_operation':
- account_names.add(op['new_account_name'])
+ account_name = op['new_account_name']
elif op_type == 'account_create_with_delegation_operation':
- account_names.add(op['new_account_name'])
+ account_name = op['new_account_name']
elif op_type == 'create_claimed_account_operation':
- account_names.add(op['new_account_name'])
-
- Accounts.register(account_names, cls._head_block_date) # register any new names
-
- # second scan will process all other ops
- json_ops = []
- update_comment_pending_payouts = []
- for tx_idx, tx in enumerate(block['transactions']):
- for operation in tx['operations']:
- op_type = operation['type']
- op = operation['value']
+ account_name = op['new_account_name']
- if(op_type != 'custom_json_operation'):
- if op_type in cls.ops_stats:
- cls.ops_stats[op_type] += 1
- else:
- cls.ops_stats[op_type] = 1
+ Accounts.register(account_name, cls._head_block_date)
# account metadata updates
if op_type == 'account_update_operation':
@@ -206,6 +194,12 @@ class Blocks:
elif op_type == 'custom_json_operation':
json_ops.append(op)
+ if op_type != 'custom_json_operation':
+ if op_type in cls.ops_stats:
+ cls.ops_stats[op_type] += 1
+ else:
+ cls.ops_stats[op_type] = 1
+
# follow/reblog/community ops
if json_ops:
custom_ops_stats = CustomOp.process_ops(json_ops, num, cls._head_block_date)
diff --git a/hive/indexer/community.py b/hive/indexer/community.py
index 91068bf5a..056f0e7e8 100644
--- a/hive/indexer/community.py
+++ b/hive/indexer/community.py
@@ -104,33 +104,32 @@ class Community:
_names = {}
@classmethod
- def register(cls, names, block_date):
+ def register(cls, name, block_date):
"""Block processing: hooks into new account registration.
`Accounts` calls this method with any newly registered names.
This method checks for any valid community names and inserts them.
"""
- for name in names:
- #if not re.match(r'^hive-[123]\d{4,6}$', name):
- if not re.match(r'^hive-[1]\d{4,6}$', name):
- continue
- type_id = int(name[5])
- _id = Accounts.get_id(name)
-
- # insert community
- sql = """INSERT INTO hive_communities (id, name, type_id, created_at)
- VALUES (:id, :name, :type_id, :date)"""
- DB.query(sql, id=_id, name=name, type_id=type_id, date=block_date)
-
- # insert owner
- sql = """INSERT INTO hive_roles (community_id, account_id, role_id, created_at)
- VALUES (:community_id, :account_id, :role_id, :date)"""
- DB.query(sql, community_id=_id, account_id=_id,
- role_id=Role.owner.value, date=block_date)
-
- Notify('new_community', src_id=None, dst_id=_id,
- when=block_date, community_id=_id).write()
+ #if not re.match(r'^hive-[123]\d{4,6}$', name):
+ if not re.match(r'^hive-[1]\d{4,6}$', name):
+ return
+ type_id = int(name[5])
+ _id = Accounts.get_id(name)
+
+ # insert community
+ sql = """INSERT INTO hive_communities (id, name, type_id, created_at)
+ VALUES (:id, :name, :type_id, :date)"""
+ DB.query(sql, id=_id, name=name, type_id=type_id, date=block_date)
+
+ # insert owner
+ sql = """INSERT INTO hive_roles (community_id, account_id, role_id, created_at)
+ VALUES (:community_id, :account_id, :role_id, :date)"""
+ DB.query(sql, community_id=_id, account_id=_id,
+ role_id=Role.owner.value, date=block_date)
+
+ Notify('new_community', src_id=None, dst_id=_id,
+ when=block_date, community_id=_id).write()
@classmethod
def validated_id(cls, name):
--
GitLab