From ff333971ccba352df968f39e17bf617232a9bea8 Mon Sep 17 00:00:00 2001
From: Dariusz Kedzierski <dkedzierski@syncad.com>
Date: Mon, 22 Jun 2020 14:14:41 +0200
Subject: [PATCH] Cache for votes, unique constraint fix for feed_cache
---
hive/db/schema.py | 1 +
hive/indexer/blocks.py | 2 ++
hive/indexer/feed_cache.py | 2 +-
hive/indexer/votes.py | 68 ++++++++++++++++++++++++--------------
4 files changed, 48 insertions(+), 25 deletions(-)
diff --git a/hive/db/schema.py b/hive/db/schema.py
index 0548c6ea6..55b3bc09f 100644
--- a/hive/db/schema.py
+++ b/hive/db/schema.py
@@ -266,6 +266,7 @@ def build_metadata():
sa.Column('account_id', sa.Integer, nullable=False),
sa.Column('created_at', sa.DateTime, nullable=False),
sa.Index('hive_feed_cache_account_id', 'account_id'), # API (and rebuild?)
+ sa.UniqueConstraint('account_id', 'post_id', name='hive_feed_cache_ux1')
)
sa.Table(
diff --git a/hive/indexer/blocks.py b/hive/indexer/blocks.py
index 9ee315fe1..bb6e0ec36 100644
--- a/hive/indexer/blocks.py
+++ b/hive/indexer/blocks.py
@@ -38,6 +38,7 @@ class Blocks:
#assert is_trx_active(), "Block.process must be in a trx"
ret = cls._process(block, vops_in_block, hived, is_initial_sync=False)
PostDataCache.flush()
+ Votes.flush()
return ret
@classmethod
@@ -57,6 +58,7 @@ class Blocks:
# expensive. So is tracking follows at all; hence we track
# deltas in memory and update follow/er counts in bulk.
PostDataCache.flush()
+ Votes.flush()
cls._flush_blocks()
Follow.flush(trx=False)
diff --git a/hive/indexer/feed_cache.py b/hive/indexer/feed_cache.py
index d302a6020..f263b3833 100644
--- a/hive/indexer/feed_cache.py
+++ b/hive/indexer/feed_cache.py
@@ -22,7 +22,7 @@ class FeedCache:
assert not DbState.is_initial_sync(), 'writing to feed cache in sync'
sql = """INSERT INTO hive_feed_cache (account_id, post_id, created_at)
VALUES (:account_id, :id, :created_at)
- ON CONFLICT (account_id, post_id) DO NOTHING"""
+ ON CONFLICT ON CONSTRAINT hive_feed_cache_ux1 DO NOTHING"""
DB.query(sql, account_id=account_id, id=post_id, created_at=created_at)
@classmethod
diff --git a/hive/indexer/votes.py b/hive/indexer/votes.py
index 76f248951..9c55b7645 100644
--- a/hive/indexer/votes.py
+++ b/hive/indexer/votes.py
@@ -9,6 +9,7 @@ DB = Db.instance()
class Votes:
""" Class for managing posts votes """
+ _votes_data = {}
@classmethod
def get_vote_count(cls, author, permlink):
@@ -57,28 +58,47 @@ class Votes:
voter = vop['value']['voter']
author = vop['value']['author']
permlink = vop['value']['permlink']
- vote_percent = vop['value']['vote_percent']
- weight = vop['value']['weight']
- rshares = vop['value']['rshares']
- sql = """
- INSERT INTO hive_votes
- (post_id, voter_id, author_id, permlink_id, weight, rshares, vote_percent, last_update)
- SELECT hp.id, ha_v.id, ha_a.id, hpd_p.id, :weight, :rshares, :vote_percent, :last_update
- FROM hive_accounts ha_v,
- hive_posts hp
- INNER JOIN hive_accounts ha_a ON ha_a.id = hp.author_id
- INNER JOIN hive_permlink_data hpd_p ON hpd_p.id = hp.permlink_id
- WHERE ha_a.name = :author AND hpd_p.permlink = :permlink AND ha_v.name = :voter
- ON CONFLICT ON CONSTRAINT hive_votes_ux1 DO
- UPDATE
- SET
- weight = EXCLUDED.weight,
- rshares = EXCLUDED.rshares,
- vote_percent = EXCLUDED.vote_percent,
- last_update = EXCLUDED.last_update,
- num_changes = hive_votes.num_changes + 1
- WHERE hive_votes.id = EXCLUDED.id
- """
- DB.query(sql, voter=voter, author=author, permlink=permlink, weight=weight, rshares=rshares,
- vote_percent=vote_percent, last_update=date)
+ key = voter + "/" + author + "/" + permlink
+
+ cls._votes_data[key] = dict(voter=voter,
+ author=author,
+ permlink=permlink,
+ vote_percent=vop['value']['vote_percent'],
+ weight=vop['value']['weight'],
+ rshares=vop['value']['rshares'],
+ last_update=date)
+
+ @classmethod
+ def flush(cls):
+ """ Flush vote data from cache to database """
+ if cls._votes_data:
+ sql = """
+ INSERT INTO hive_votes
+ (post_id, voter_id, author_id, permlink_id, weight, rshares, vote_percent, last_update)
+ """
+ values = []
+ for _, vd in cls._votes_data.items():
+ values.append("""
+ SELECT hp.id, ha_v.id, ha_a.id, hpd_p.id, {}, {}, {}, '{}'::timestamp
+ FROM hive_accounts ha_v,
+ hive_posts hp
+ INNER JOIN hive_accounts ha_a ON ha_a.id = hp.author_id
+ INNER JOIN hive_permlink_data hpd_p ON hpd_p.id = hp.permlink_id
+ WHERE ha_a.name = '{}' AND hpd_p.permlink = '{}' AND ha_v.name = '{}'
+ """.format(vd['weight'], vd['rshares'], vd['vote_percent'], vd['last_update'], vd['author'], vd['permlink'], vd['voter']))
+ sql += ' UNION ALL '.join(values)
+
+ sql += """
+ ON CONFLICT ON CONSTRAINT hive_votes_ux1 DO
+ UPDATE
+ SET
+ weight = EXCLUDED.weight,
+ rshares = EXCLUDED.rshares,
+ vote_percent = EXCLUDED.vote_percent,
+ last_update = EXCLUDED.last_update,
+ num_changes = hive_votes.num_changes + 1
+ WHERE hive_votes.id = EXCLUDED.id
+ """
+ DB.query(sql)
+ cls._votes_data.clear()
--
GitLab