diff --git a/hive/db/schema.py b/hive/db/schema.py index 6fe5e45ccde005aacbaf99b38ef77988a1a356e4..c27268e1c9a2a2b686ec223cea8ae547c191c749 100644 --- a/hive/db/schema.py +++ b/hive/db/schema.py @@ -64,11 +64,7 @@ def build_metadata(): sa.Column('rshares', sa.BigInteger, nullable=False), sa.Column('block_num', sa.Integer, nullable=False), -# sa.ForeignKeyConstraint(['voter_id'], ['hive_accounts.id']), -# sa.ForeignKeyConstraint(['author_id'], ['hive_accounts.id']), -# sa.ForeignKeyConstraint(['block_num'], ['hive_blocks.num']), - - sa.UniqueConstraint('author_id', 'permlink', 'voter_id', name='hive_reputation_data_uk') + sa.Index('hive_reputation_data_author_permlink_voter_idx', 'author_id', 'permlink', 'voter_id') ) sa.Table( @@ -1666,7 +1662,8 @@ def setup(db): $BODY$; """ - db.query_no_return(sql) +# db.query_no_return(sql) + sql = """ DROP FUNCTION IF EXISTS public.calculate_notify_vote_score(_payout hive_posts.payout%TYPE, _abs_rshares hive_posts_view.abs_rshares%TYPE, _rshares hive_votes.rshares%TYPE) CASCADE ; diff --git a/hive/indexer/reputations.py b/hive/indexer/reputations.py index e14a5484f90fead6fbd3d49c41a3722a469844b4..05924eb2eff56bfce7ce0879dbbd926db71c0169 100644 --- a/hive/indexer/reputations.py +++ b/hive/indexer/reputations.py @@ -8,41 +8,60 @@ log = logging.getLogger(__name__) CACHED_ITEMS_LIMIT = 200 class Reputations(DbAdapterHolder): - _queries = [] + _values = [] + _total_values = 0 @classmethod def process_vote(self, block_num, effective_vote_op): - return - self._queries.append("\nSELECT process_reputation_data({}, '{}', '{}', '{}', {});".format(block_num, effective_vote_op['author'], effective_vote_op['permlink'], - effective_vote_op['voter'], effective_vote_op['rshares'])) + tuple = "('{}', '{}', '{}', {}, {})".format(effective_vote_op['author'], effective_vote_op['voter'], + effective_vote_op['permlink'], effective_vote_op['rshares'], block_num) + self._values.append(tuple) @classmethod def flush(self): - if not self._queries: + if not self._values: + log.info("Written total reputation data records: {}".format(self._total_values)) return 0 + sql = """ + INSERT INTO hive_reputation_data + (voter_id, author_id, permlink, rshares, block_num) + + SELECT (SELECT ha_v.id FROM hive_accounts ha_v WHERE ha_v.name = t.voter) as voter_id, + (SELECT ha.id FROM hive_accounts ha WHERE ha.name = t.author) as author_id, + t.permlink as permlink, t.rshares, t.block_num + FROM + ( + VALUES + -- author, voter, permlink, rshares, block_num + {} + ) AS T(author, voter, permlink, rshares, block_num) + """ + self.beginTx() - query = "" - i = 0 - items = 0 - for s in self._queries: - query = query + str(self._queries[i]) + ";\n" - i = i + 1 - items = items + 1 - if items >= CACHED_ITEMS_LIMIT: - self.db.query_no_return(query) - query = "" - items = 0 - - if items >= CACHED_ITEMS_LIMIT: - self.db.query_no_return(query) - query = "" - items = 0 + begin = 0 + end = 0 + value_limit = 1000 + size = len(self._values) + while begin < size: + end = begin + value_limit + if end > size: + end = size - n = len(self._queries) - self._queries.clear() + param = ",".join(self._values[begin:end]) + query = sql.format(param) + self.db.query_no_return(query) + begin = end self.commitTx() + + n = len(self._values) + self._values.clear() + + self._total_values = self._total_values + n + + log.info("Written total reputation data records: {}".format(self._total_values)) + return n