Merge with BW, merge fixes, simplification with post data cache

hive/db/schema.py

@@ -161,11 +161,11 @@ def build_metadata():
     sa.Table(
         'hive_post_data', metadata,
         sa.Column('id', sa.Integer, primary_key=True, autoincrement=False),
-        sa.Column('title', VARCHAR(255), server_default=''),
-        sa.Column('preview', VARCHAR(1024), server_default=''),
-        sa.Column('img_url', VARCHAR(1024), server_default=''),
-        sa.Column('body', TEXT, server_default=''),
-        sa.Column('json', TEXT, server_default='')
+        sa.Column('title', VARCHAR(255), nullable=False, server_default=''),
+        sa.Column('preview', VARCHAR(1024), nullable=False, server_default=''),
+        sa.Column('img_url', VARCHAR(1024), nullable=False, server_default=''),
+        sa.Column('body', TEXT, nullable=False, server_default=''),
+        sa.Column('json', TEXT, nullable=False, server_default='')
     )
 
     sa.Table(

hive/indexer/blocks.py

@@ -36,7 +36,9 @@ class Blocks:
     def process(cls, block, vops_in_block, hived):
         """Process a single block. Always wrap in a transaction!"""
         #assert is_trx_active(), "Block.process must be in a trx"
-        return cls._process(block, vops_in_block, hived, is_initial_sync=False)
+        ret = cls._process(block, vops_in_block, hived, is_initial_sync=False)
+        PostDataCache.flush()
+        return ret
 
     @classmethod
     def process_multi(cls, blocks, vops, hived, is_initial_sync=False):

hive/indexer/posts.py

@@ -107,34 +107,18 @@ class Posts:
 
         cls._set_id(op['author']+'/'+op['permlink'], result['id'])
 
-# add content data to hive_post_data
-        if DbState.is_initial_sync():
-            post_data = dict(title=op['title'], preview=op['preview'] if 'preview' in op else "",
-                             img_url=op['img_url'] if 'img_url' in op else "", body=op['body'],
-                             json=op['json_metadata'] if op['json_metadata'] else '{}')
-            PostDataCache.add_data(result['id'], post_data)
-        else:
-            sql = """
-                INSERT INTO hive_post_data (id, title, preview, img_url, body, json) 
-                VALUES (:id, :title, :preview, :img_url, :body, :json)
-                ON CONFLICT ON CONSTRAINT hive_post_data_pkey DO UPDATE SET
-                    title = :title,
-                    preview = :preview,
-                    img_url = :img_url,
-                    body = :body,
-                    json = :json
-                """
-            DB.query(sql, id=result['id'], title=op['title'],
-                     preview=op['preview'] if 'preview' in op else "",
-                     img_url=op['img_url'] if 'img_url' in op else "",
-                     body=op['body'], json=op['json_metadata'] if op['json_metadata'] else '{}')
+        # add content data to hive_post_data
+        post_data = dict(title=op['title'], preview=op['preview'] if 'preview' in op else "",
+                         img_url=op['img_url'] if 'img_url' in op else "", body=op['body'],
+                         json=op['json_metadata'] if op['json_metadata'] else '{}')
+        PostDataCache.add_data(result['id'], post_data)
 
         if not DbState.is_initial_sync():
             if error:
                 author_id = result['author_id']
-                Notify('error', dst_id=author_id, when=date,
+                Notify('error', dst_id=author_id, when=block_date,
                        post_id=result['id'], payload=error).write()
-            cls._insert_feed_cache(result, date)
+            cls._insert_feed_cache(result, block_date)
 
     @classmethod
     def comment_payout_op(cls, ops, date):

hive/indexer/votes.py

@@ -9,21 +9,6 @@ DB = Db.instance()
 
 class Votes:
     """ Class for managing posts votes """
-    @classmethod
-    def get_id(cls, voter, author, permlink):
-        """ Check if vote exists, if yes return its id, else return None """
-        sql = """
-            SELECT 
-                hv.id 
-            FROM 
-                hive_votes hv
-            INNER JOIN hive_accounts ha_v ON (ha_v.id = hv.voter_id)
-            INNER JOIN hive_accounts ha_a ON (ha_a.id = hv.author_id)
-            INNER JOIN hive_permlink_data hpd ON (hpd.id = hv.permlink_id)
-            WHERE ha_v.name = :voter AND ha_a.name = :author AND hpd.permlink = :permlink
-        """
-        ret = DB.query_row(sql, voter=voter, author=author, permlink=permlink)
-        return None if ret is None else int(ret.id)
 
     @classmethod
     def get_vote_count(cls, author, permlink):
@@ -72,51 +57,28 @@ class Votes:
         voter = vop['value']['voter']
         author = vop['value']['author']
         permlink = vop['value']['permlink']
-
-        vote_id = cls.get_id(voter, author, permlink)
-        # no vote so create new
-        if vote_id is None:
-            cls._insert(vop, date)
-        else:
-            cls._update(vote_id, vop, date)
-
-    @classmethod
-    def _insert(cls, vop, date):
-        """ Insert new vote """
-        voter = vop['value']['voter']
-        author = vop['value']['author']
-        permlink = vop['value']['permlink']
         vote_percent = vop['value']['vote_percent']
+        weight = vop['value']['weight']
+        rshares = vop['value']['rshares']
+
         sql = """
             INSERT INTO hive_votes
                   (post_id, voter_id, author_id, permlink_id, weight, rshares, vote_percent, last_update) 
             SELECT hp.id, ha_v.id, ha_a.id, hpd_p.id, :weight, :rshares, :vote_percent, :last_update
             FROM hive_accounts ha_v,
                  hive_posts hp
-            INNER JOIN hive_accounts ha_a ON ha_a.name = hp.author 
-            INNER JOIN hive_permlink_data hpd_p ON hpd_p.permlink = hp.permlink
+            INNER JOIN hive_accounts ha_a ON ha_a.id = hp.author_id
+            INNER JOIN hive_permlink_data hpd_p ON hpd_p.id = hp.permlink_id
             WHERE ha_a.name = :author AND hpd_p.permlink = :permlink AND ha_v.name = :voter
-            """
-        weight = vop['value']['weight']
-        rshares = vop['value']['rshares']
+            ON CONFLICT ON CONSTRAINT hive_votes_ux1 DO
+                UPDATE
+                    SET
+                        weight = EXCLUDED.weight,
+                        rshares = EXCLUDED.rshares,
+                        vote_percent = EXCLUDED.vote_percent,
+                        last_update = EXCLUDED.last_update,
+                        num_changes = hive_votes.num_changes + 1
+                WHERE hive_votes.id = EXCLUDED.id
+        """
         DB.query(sql, voter=voter, author=author, permlink=permlink, weight=weight, rshares=rshares,
                  vote_percent=vote_percent, last_update=date)
-
-    @classmethod
-    def _update(cls, vote_id, vop, date):
-        """ Update existing vote """
-        vote_percent = vop['value']['vote_percent']
-        sql = """
-            UPDATE hive_votes as hv
-            SET
-                weight = :weight,
-                rshares = :rshares,
-                vote_percent = :vote_percent,
-                last_update = :last_update,
-                num_changes = hv.num_changes + 1
-            WHERE hv.id = :id
-        """
-        weight = vop['value']['weight']
-        rshares = vop['value']['rshares']
-        DB.query(sql, weight=weight, rshares=rshares, vote_percent=vote_percent, last_update=date,
-                 id=vote_id)