Cache for votes, unique constraint fix for feed_cache

hive/db/schema.py

@@ -266,6 +266,7 @@ def build_metadata():
         sa.Column('account_id', sa.Integer, nullable=False),
         sa.Column('created_at', sa.DateTime, nullable=False),
         sa.Index('hive_feed_cache_account_id', 'account_id'), # API (and rebuild?)
+        sa.UniqueConstraint('account_id', 'post_id', name='hive_feed_cache_ux1')
     )
 
     sa.Table(

hive/indexer/blocks.py

@@ -38,6 +38,7 @@ class Blocks:
         #assert is_trx_active(), "Block.process must be in a trx"
         ret = cls._process(block, vops_in_block, hived, is_initial_sync=False)
         PostDataCache.flush()
+        Votes.flush()
         return ret
 
     @classmethod
@@ -57,6 +58,7 @@ class Blocks:
         # expensive. So is tracking follows at all; hence we track
         # deltas in memory and update follow/er counts in bulk.
         PostDataCache.flush()
+        Votes.flush()
         cls._flush_blocks()
         Follow.flush(trx=False)
 

hive/indexer/feed_cache.py

@@ -22,7 +22,7 @@ class FeedCache:
         assert not DbState.is_initial_sync(), 'writing to feed cache in sync'
         sql = """INSERT INTO hive_feed_cache (account_id, post_id, created_at)
                       VALUES (:account_id, :id, :created_at)
-                 ON CONFLICT (account_id, post_id) DO NOTHING"""
+                 ON CONFLICT ON CONSTRAINT hive_feed_cache_ux1 DO NOTHING"""
         DB.query(sql, account_id=account_id, id=post_id, created_at=created_at)
 
     @classmethod

hive/indexer/votes.py

@@ -9,6 +9,7 @@ DB = Db.instance()
 
 class Votes:
     """ Class for managing posts votes """
+    _votes_data = {}
 
     @classmethod
     def get_vote_count(cls, author, permlink):
@@ -57,28 +58,47 @@ class Votes:
         voter = vop['value']['voter']
         author = vop['value']['author']
         permlink = vop['value']['permlink']
-        vote_percent = vop['value']['vote_percent']
-        weight = vop['value']['weight']
-        rshares = vop['value']['rshares']
 
-        sql = """
-            INSERT INTO hive_votes
-                  (post_id, voter_id, author_id, permlink_id, weight, rshares, vote_percent, last_update) 
-            SELECT hp.id, ha_v.id, ha_a.id, hpd_p.id, :weight, :rshares, :vote_percent, :last_update
-            FROM hive_accounts ha_v,
-                 hive_posts hp
-            INNER JOIN hive_accounts ha_a ON ha_a.id = hp.author_id
-            INNER JOIN hive_permlink_data hpd_p ON hpd_p.id = hp.permlink_id
-            WHERE ha_a.name = :author AND hpd_p.permlink = :permlink AND ha_v.name = :voter
-            ON CONFLICT ON CONSTRAINT hive_votes_ux1 DO
-                UPDATE
-                    SET
-                        weight = EXCLUDED.weight,
-                        rshares = EXCLUDED.rshares,
-                        vote_percent = EXCLUDED.vote_percent,
-                        last_update = EXCLUDED.last_update,
-                        num_changes = hive_votes.num_changes + 1
-                WHERE hive_votes.id = EXCLUDED.id
-        """
-        DB.query(sql, voter=voter, author=author, permlink=permlink, weight=weight, rshares=rshares,
-                 vote_percent=vote_percent, last_update=date)
+        key = voter + "/" + author + "/" + permlink
+
+        cls._votes_data[key] = dict(voter=voter,
+                                    author=author,
+                                    permlink=permlink,
+                                    vote_percent=vop['value']['vote_percent'],
+                                    weight=vop['value']['weight'],
+                                    rshares=vop['value']['rshares'],
+                                    last_update=date)
+
+    @classmethod
+    def flush(cls):
+        """ Flush vote data from cache to database """
+        if cls._votes_data:
+            sql = """
+                INSERT INTO hive_votes
+                    (post_id, voter_id, author_id, permlink_id, weight, rshares, vote_percent, last_update) 
+            """
+            values = []
+            for _, vd in cls._votes_data.items():
+                values.append("""
+                    SELECT hp.id, ha_v.id, ha_a.id, hpd_p.id, {}, {}, {}, '{}'::timestamp
+                    FROM hive_accounts ha_v,
+                        hive_posts hp
+                    INNER JOIN hive_accounts ha_a ON ha_a.id = hp.author_id
+                    INNER JOIN hive_permlink_data hpd_p ON hpd_p.id = hp.permlink_id
+                    WHERE ha_a.name = '{}' AND hpd_p.permlink = '{}' AND ha_v.name = '{}'
+                """.format(vd['weight'], vd['rshares'], vd['vote_percent'], vd['last_update'], vd['author'], vd['permlink'], vd['voter']))
+            sql += ' UNION ALL '.join(values)
+
+            sql += """
+                ON CONFLICT ON CONSTRAINT hive_votes_ux1 DO
+                    UPDATE
+                        SET
+                            weight = EXCLUDED.weight,
+                            rshares = EXCLUDED.rshares,
+                            vote_percent = EXCLUDED.vote_percent,
+                            last_update = EXCLUDED.last_update,
+                            num_changes = hive_votes.num_changes + 1
+                    WHERE hive_votes.id = EXCLUDED.id
+            """
+            DB.query(sql)
+            cls._votes_data.clear()