Commit fc22287e authored by Bartek Wrona's avatar Bartek Wrona

Merge branch 'massive_reputation_data' into 'develop'

Massive import of reputation data

See merge request !157
parents 669ec675 33f94849
......@@ -64,11 +64,7 @@ def build_metadata():
sa.Column('rshares', sa.BigInteger, nullable=False),
sa.Column('block_num', sa.Integer, nullable=False),
# sa.ForeignKeyConstraint(['voter_id'], ['hive_accounts.id']),
# sa.ForeignKeyConstraint(['author_id'], ['hive_accounts.id']),
# sa.ForeignKeyConstraint(['block_num'], ['hive_blocks.num']),
sa.UniqueConstraint('author_id', 'permlink', 'voter_id', name='hive_reputation_data_uk')
sa.Index('hive_reputation_data_author_permlink_voter_idx', 'author_id', 'permlink', 'voter_id')
)
sa.Table(
......@@ -1666,7 +1662,8 @@ def setup(db):
$BODY$;
"""
db.query_no_return(sql)
# db.query_no_return(sql)
sql = """
DROP FUNCTION IF EXISTS public.calculate_notify_vote_score(_payout hive_posts.payout%TYPE, _abs_rshares hive_posts_view.abs_rshares%TYPE, _rshares hive_votes.rshares%TYPE) CASCADE
;
......
......@@ -8,41 +8,60 @@ log = logging.getLogger(__name__)
CACHED_ITEMS_LIMIT = 200
class Reputations(DbAdapterHolder):
_queries = []
_values = []
_total_values = 0
@classmethod
def process_vote(self, block_num, effective_vote_op):
return
self._queries.append("\nSELECT process_reputation_data({}, '{}', '{}', '{}', {});".format(block_num, effective_vote_op['author'], effective_vote_op['permlink'],
effective_vote_op['voter'], effective_vote_op['rshares']))
tuple = "('{}', '{}', '{}', {}, {})".format(effective_vote_op['author'], effective_vote_op['voter'],
effective_vote_op['permlink'], effective_vote_op['rshares'], block_num)
self._values.append(tuple)
@classmethod
def flush(self):
if not self._queries:
if not self._values:
log.info("Written total reputation data records: {}".format(self._total_values))
return 0
sql = """
INSERT INTO hive_reputation_data
(voter_id, author_id, permlink, rshares, block_num)
SELECT (SELECT ha_v.id FROM hive_accounts ha_v WHERE ha_v.name = t.voter) as voter_id,
(SELECT ha.id FROM hive_accounts ha WHERE ha.name = t.author) as author_id,
t.permlink as permlink, t.rshares, t.block_num
FROM
(
VALUES
-- author, voter, permlink, rshares, block_num
{}
) AS T(author, voter, permlink, rshares, block_num)
"""
self.beginTx()
query = ""
i = 0
items = 0
for s in self._queries:
query = query + str(self._queries[i]) + ";\n"
i = i + 1
items = items + 1
if items >= CACHED_ITEMS_LIMIT:
self.db.query_no_return(query)
query = ""
items = 0
if items >= CACHED_ITEMS_LIMIT:
self.db.query_no_return(query)
query = ""
items = 0
begin = 0
end = 0
value_limit = 1000
size = len(self._values)
while begin < size:
end = begin + value_limit
if end > size:
end = size
n = len(self._queries)
self._queries.clear()
param = ",".join(self._values[begin:end])
query = sql.format(param)
self.db.query_no_return(query)
begin = end
self.commitTx()
n = len(self._values)
self._values.clear()
self._total_values = self._total_values + n
log.info("Written total reputation data records: {}".format(self._total_values))
return n
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment