hive/db/schema.py

@@ -602,7 +602,8 @@ def setup(db):
       "condenser_get_names_by_reblogged.sql",
       "condenser_get_discussions_by_comments.sql",
       "condenser_get_account_reputations.sql",
-      "update_follow_count.sql"
+      "update_follow_count.sql",
+      "delete_reblog_feed_cache.sql"
     ]
     from os.path import dirname, realpath
     dir_path = dirname(realpath(__file__))

hive/db/sql_scripts/bridge_get_by_feed_with_reblog.sql

@@ -72,10 +72,10 @@ BEGIN
       WHERE hfc.block_num > __cutoff AND hf.state = 1 AND hf.follower = __account_id
       GROUP BY hfc.post_id
       HAVING __post_id = 0 OR MIN(hfc.created_at) <= __min_date 
-      ORDER BY min_created DESC
+      ORDER BY min_created DESC, hfc.post_id DESC
       LIMIT _limit
     ) T ON hp.id =  T.post_id
-    ORDER BY T.min_created DESC;
+    ORDER BY T.min_created DESC, T.post_id DESC;
 END
 $BODY$
 ;

hive/db/sql_scripts/delete_reblog_feed_cache.sql

+
+DROP FUNCTION IF EXISTS delete_reblog_feed_cache(character varying,character varying,character varying)
+;
+
+CREATE OR REPLACE FUNCTION delete_reblog_feed_cache(
+  in _author hive_accounts.name%TYPE,
+  in _permlink hive_permlink_data.permlink%TYPE,
+  in _account hive_accounts.name%TYPE)
+RETURNS INTEGER
+LANGUAGE plpgsql
+AS
+$function$
+DECLARE
+  __account_id INT;
+  __post_id INT;
+BEGIN
+
+  __account_id = find_account_id( _account, False );
+  __post_id = find_comment_id( _author, _permlink, False );
+
+  IF __post_id = 0 THEN
+    RETURN 0;
+  END IF;
+
+  DELETE FROM hive_reblogs
+  WHERE blogger_id = __account_id AND post_id = __post_id;
+
+  DELETE FROM hive_feed_cache
+  WHERE account_id = __account_id AND post_id = __post_id;
+
+  RETURN 1;
+END
+$function$
+;

hive/db/sql_scripts/hive_post_operations.sql

@@ -150,26 +150,38 @@ CREATE OR REPLACE FUNCTION delete_hive_post(
   in _author hive_accounts.name%TYPE,
   in _permlink hive_permlink_data.permlink%TYPE,
   in _block_num hive_blocks.num%TYPE)
-RETURNS TABLE (id hive_posts.id%TYPE, depth hive_posts.depth%TYPE)
+RETURNS VOID
 LANGUAGE plpgsql
 AS
 $function$
+DECLARE
+  __post_id INT;
 BEGIN
-  RETURN QUERY UPDATE hive_posts AS hp
-    SET counter_deleted =
-    (
+
+  __post_id = find_comment_id( _author, _permlink, False );
+
+  IF __post_id = 0 THEN
+    RETURN;
+  END IF;
+
+  UPDATE hive_posts
+  SET counter_deleted =
+  (
       SELECT max( hps.counter_deleted ) + 1
       FROM hive_posts hps
       INNER JOIN hive_accounts ha ON hps.author_id = ha.id
       INNER JOIN hive_permlink_data hpd ON hps.permlink_id = hpd.id
       WHERE ha.name = _author AND hpd.permlink = _permlink
-    )
-    , block_num = _block_num
-  FROM hive_posts hp1
-  INNER JOIN hive_accounts ha ON hp1.author_id = ha.id
-  INNER JOIN hive_permlink_data hpd ON hp1.permlink_id = hpd.id
-  WHERE hp.id = hp1.id AND ha.name = _author AND hpd.permlink = _permlink AND hp1.counter_deleted = 0
-  RETURNING hp.id, hp.depth;
+  )
+  ,block_num = _block_num
+  WHERE id = __post_id;
+
+  DELETE FROM hive_reblogs
+  WHERE post_id = __post_id;
+
+  DELETE FROM hive_feed_cache
+  WHERE post_id = __post_id;
+
 END
 $function$
 ;

hive/indexer/feed_cache.py

-"""Maintains feed cache (blogs + reblogs)"""
-
-import logging
-from hive.db.adapter import Db
-from hive.indexer.db_adapter_holder import DbAdapterHolder
-
-log = logging.getLogger(__name__)
-
-DB = Db.instance()
-
-class FeedCache(DbAdapterHolder):
-    """Maintains `hive_feed_cache`, which merges posts and reports.
-
-    The feed cache allows for efficient querying of posts + reblogs,
-    savings us from expensive queries. Effectively a materialized view.
-    """
-    @classmethod
-    def delete(cls, post_id, account_id=None):
-        """Remove a post from feed cache.
-
-        If `account_id` is specified, we remove a single entry (e.g. a
-        singular un-reblog). Otherwise, we remove all instances of the
-        post (e.g. a post was deleted; its entry and all reblogs need
-        to be removed.
-        """
-        sql = "DELETE FROM hive_feed_cache WHERE post_id = :id"
-        if account_id:
-            sql = sql + " AND account_id = :account_id"
-        DB.query(sql, account_id=account_id, id=post_id)

hive/indexer/posts.py

@@ -10,7 +10,7 @@ from diff_match_patch import diff_match_patch
 from hive.db.adapter import Db
 from hive.db.db_state import DbState
 
-from hive.indexer.feed_cache import FeedCache
+from hive.indexer.reblog import Reblog
 from hive.indexer.community import Community
 from hive.indexer.notify import Notify
 from hive.indexer.post_data_cache import PostDataCache
@@ -390,22 +390,8 @@ class Posts(DbAdapterHolder):
     @classmethod
     def delete(cls, op):
         """Marks a post record as being deleted."""
-
-        sql = """
-              SELECT id, depth
-              FROM delete_hive_post((:author)::varchar, (:permlink)::varchar, (:block_num)::int);
-              """
-        row = DB.query_row(sql, author=op['author'], permlink = op['permlink'], block_num=op['block_num'])
-
-        result = dict(row)
-        pid = result['id']
-
-        depth = result['depth']
-
-        if depth == 0 and not DbState.is_initial_sync():
-            # TODO: delete from hive_reblogs -- otherwise feed cache gets
-            # populated with deleted posts somwrimas
-            FeedCache.delete(pid)
+        sql = "SELECT delete_hive_post((:author)::varchar, (:permlink)::varchar, (:block_num)::int );"
+        DB.query_no_return(sql, author=op['author'], permlink = op['permlink'], block_num=op['block_num'])
 
     @classmethod
     def _verify_post_against_community(cls, op, community_id, is_valid, is_muted):

hive/indexer/reblog.py

@@ -2,59 +2,69 @@
 
 import logging
 
+from hive.db.adapter import Db
 from hive.db.db_state import DbState
 
 from hive.indexer.accounts import Accounts
-from hive.indexer.feed_cache import FeedCache
 from hive.indexer.db_adapter_holder import DbAdapterHolder
 from hive.utils.normalize import escape_characters
 
 log = logging.getLogger(__name__)
-
-DELETE_SQL = """
-    WITH processing_set AS (
-        SELECT hp.id as post_id, ha.id as account_id
-        FROM hive_posts hp
-        INNER JOIN hive_accounts ha ON hp.author_id = ha.id
-        INNER JOIN hive_permlink_data hpd ON hp.permlink_id = hpd.id
-        WHERE ha.name = :a AND hpd.permlink = :permlink AND hp.depth = 0 AND hp.counter_deleted = 0
-    )
-    DELETE FROM hive_reblogs AS hr
-    WHERE hr.account = :a AND hr.post_id IN (SELECT ps.post_id FROM processing_set ps)
-    RETURNING hr.post_id, (SELECT ps.account_id FROM processing_set ps) AS account_id
-"""
+DB = Db.instance()
 
 class Reblog(DbAdapterHolder):
     """ Class for reblog operations """
-    reblog_items_to_flush = []
+    deleted_reblog_items = {}
+    reblog_items_to_flush = {}
 
     @classmethod
-    def reblog_op(cls, account, op_json, block_date, block_num):
-        """ Process reblog operation """
-        if 'account' not in op_json or \
-            'author' not in op_json or \
-            'permlink' not in op_json:
-            return
+    def _validated_op(cls, actor, op, block_date, block_num):
+        if 'account' not in op or \
+            'author' not in op or \
+            'permlink' not in op:
+            return None
+
+        if op['account'] != actor:
+            return None # impersonation
 
-        blogger = op_json['account']
-        author = op_json['author']
-        permlink = escape_characters(op_json['permlink'])
+        if not Accounts.exists(op['account']):
+            return None
+        if not Accounts.exists(op['author']):
+            return None
 
-        if blogger != account:
-            return  # impersonation
-        if not all(map(Accounts.exists, [author, blogger])):
+        _delete = True if ('delete' in op and op['delete'] == 'delete') else False
+
+        return dict(author = op['author'],
+                    permlink = op['permlink'],
+                    account = op['account'],
+                    block_date = block_date,
+                    block_num = block_num,
+                    delete = _delete )
+
+    @classmethod
+    def reblog_op(cls, actor, op, block_date, block_num):
+        """ Process reblog operation """
+        op = cls._validated_op(actor, op, block_date, block_num)
+        if not op:
             return
 
-        if 'delete' in op_json and op_json['delete'] == 'delete':
-            row = cls.db.query_row(DELETE_SQL, a=blogger, permlink=permlink)
-            if row is None:
-                log.debug("reblog: post not found: %s/%s", author, op_json['permlink'])
-                return
-            if not DbState.is_initial_sync():
-                result = dict(row)
-                FeedCache.delete(result['post_id'], result['account_id'])
+        key = "{}/{}/{}".format(op['author'], op['permlink'], op['account'])
+
+        if op['delete']:
+            cls.deleted_reblog_items[key] = {}
+            cls.delete( op['author'], op['permlink'], op['account'] )
         else:
-            cls.reblog_items_to_flush.append((blogger, author, permlink, block_date, block_num))
+            cls.reblog_items_to_flush[key] = { 'op': op }
+
+    @classmethod
+    def delete(cls, author, permlink, account ):
+        """Remove a reblog from hive_reblogs + feed from hive_feed_cache.
+        """
+        sql = "SELECT delete_reblog_feed_cache( (:author)::VARCHAR, (:permlink)::VARCHAR, (:account)::VARCHAR );"
+        status = DB.query_col(sql, author=author, permlink=permlink, account=account);
+        assert status is not None
+        if status == 0:
+          log.debug("reblog: post not found: %s/%s", author, permlink)
 
     @classmethod
     def flush(cls):
@@ -74,7 +84,7 @@ class Reblog(DbAdapterHolder):
                     INNER JOIN hive_accounts ha ON ha.name = t.author
                     INNER JOIN hive_accounts ha_b ON ha_b.name = t.blogger
                     INNER JOIN hive_permlink_data hpd ON hpd.permlink = t.permlink
-                    INNER JOIN hive_posts hp ON hp.author_id = ha.id AND hp.permlink_id = hpd.id
+                    INNER JOIN hive_posts hp ON hp.author_id = ha.id AND hp.permlink_id = hpd.id AND hp.counter_deleted = 0
             ) AS data_source (blogger_id, post_id, created_at, block_num)
             ON CONFLICT ON CONSTRAINT hive_reblogs_ux1 DO NOTHING
         """
@@ -85,24 +95,27 @@ class Reblog(DbAdapterHolder):
             limit = 1000
             count = 0
             cls.beginTx()
-            for reblog_item in cls.reblog_items_to_flush:
+            for k, v in cls.reblog_items_to_flush.items():
+                if k in cls.deleted_reblog_items:
+                  continue
+                reblog_item = v['op']
                 if count < limit:
-                    values.append("('{}', '{}', {}, '{}'::timestamp, {})".format(reblog_item[0],
-                                                                                   reblog_item[1],
-                                                                                   reblog_item[2],
-                                                                                   reblog_item[3],
-                                                                                   reblog_item[4]))
+                    values.append("({}, {}, {}, '{}'::timestamp, {})".format(escape_characters(reblog_item['account']),
+                                                                                escape_characters(reblog_item['author']),
+                                                                                escape_characters(reblog_item['permlink']),
+                                                                                reblog_item['block_date'],
+                                                                                reblog_item['block_num']))
                     count = count + 1
                 else:
                     values_str = ",".join(values)
                     query = sql_prefix.format(values_str, values_str)
                     cls.db.query(query)
                     values.clear()
-                    values.append("('{}', '{}', {}, '{}'::timestamp, {})".format(reblog_item[0],
-                                                                                   reblog_item[1],
-                                                                                   reblog_item[2],
-                                                                                   reblog_item[3],
-                                                                                   reblog_item[4]))
+                    values.append("({}, {}, {}, '{}'::timestamp, {})".format(escape_characters(reblog_item['account']),
+                                                                                escape_characters(reblog_item['author']),
+                                                                                escape_characters(reblog_item['permlink']),
+                                                                                reblog_item['block_date'],
+                                                                                reblog_item['block_num']))
                     count = 1
 
             if len(values) > 0:
@@ -112,4 +125,6 @@ class Reblog(DbAdapterHolder):
             cls.commitTx()
             cls.reblog_items_to_flush.clear()
 
+        cls.deleted_reblog_items.clear();
+
         return item_count

hive/server/condenser_api/objects.py

@@ -44,15 +44,15 @@ def _condenser_post_object(row, truncate_body=0, get_content_additions=False):
     """Given a hive_posts row, create a legacy-style post object."""
     paid = row['is_paidout']
 
-    # condenser#3424 mitigation
-    if not row['category']:
-        row['category'] = 'undefined'
-
     full_payout = row['pending_payout'] + row['payout'];
     post = {}
     post['author'] = row['author']
     post['permlink'] = row['permlink']
-    post['category'] = row['category']
+
+    if not row['category']:
+      post['category'] = 'undefined' # condenser#3424 mitigation
+    else:
+      post['category'] = row['category']
 
     post['title'] = row['title']
     post['body'] = row['body'][0:truncate_body] if truncate_body else row['body']

mock_data/block_data/reblog_op/flow.txt

 ***block 4999999***
 custom_json_operation("[\"reblog\",{\"account\":\"funny\",\"author\":\"steemit\",\"permlink\":\"firstpost\"}]") - very old post
 custom_json_operation("[\"reblog\",{\"account\":\"funny\",\"author\":\"steak\",\"permlink\":\"streak-test\"}]") - deleted post (should not be reblogged)
-custom_json_operation("[\"reblog\",{\"account\":\"funny\",\"author\":\"dollarvigilante\",\"permlink\":\"another-billionaire-warns-of-catastrophic-depths-not-seen-in-5-000-years-and-emphasizes-gold\"}]") - fresh post
+custom_json_operation("[\"reblog\",{\"account\":\"funny\",\"author\":\"dollarvigilante\",\"permlink\":\"another-billionaire-warns-of-catastrophic-depths-not-seen-in-5-000-years-and-emphasizes-gold\"}]") - fresh post
\ No newline at end of file
+account_create_operation( `test-reblog-01` )
+account_create_operation( `test-reblog-02` )
+account_create_operation( `test-reblog-03` )
+account_create_operation( `test-reblog-04` )
+account_create_operation( `test-reblog-05` )
+account_create_operation( `test-reblog-06` )
+comment_operation( `test-reblog-01`, `parrot-01`)
+comment_operation( `test-reblog-02`, `parrot-02`)
+comment_operation( `test-reblog-03`, `parrot-03`)
+comment_operation( `test-reblog-04`, `parrot-04`)
+comment_operation( `test-reblog-05`, `parrot-05`)
+comment_operation( `test-reblog-06`, `parrot-06`)
+***block 5000000***
+custom_json_operation("{\"account\":\"test-reblog-01\",\"author\":\"test-reblog-02\",\"permlink\":\"parrot-02\"}")
+custom_json_operation("{\"account\":\"test-reblog-01\",\"author\":\"test-reblog-03\",\"permlink\":\"parrot-03\"}")
+custom_json_operation("{\"account\":\"test-reblog-01\",\"author\":\"test-reblog-04\",\"permlink\":\"parrot-04\"}")
+custom_json_operation("{\"delete\":\"delete\",\"account\":\"test-reblog-01\",\"author\":\"test-reblog-04\",\"permlink\":\"parrot-04\"}")
+custom_json_operation("{\"delete\":\"delete\",\"account\":\"test-reblog-01\",\"author\":\"test-reblog-03\",\"permlink\":\"parrot-03\"}")
+custom_json_operation("{\"delete\":\"delete\",\"account\":\"test-reblog-01\",\"author\":\"test-reblog-02\",\"permlink\":\"parrot-02\"}")
+
+***block 5000001***
+custom_json_operation("{\"account\":\"test-reblog-02\",\"author\":\"test-reblog-01\",\"permlink\":\"parrot-01\"}")
+custom_json_operation("{\"account\":\"test-reblog-02\",\"author\":\"test-reblog-02\",\"permlink\":\"parrot-02\"}")
+custom_json_operation("{\"account\":\"test-reblog-02\",\"author\":\"test-reblog-03\",\"permlink\":\"parrot-03\"}")
+custom_json_operation("{\"account\":\"test-reblog-03\",\"author\":\"test-reblog-02\",\"permlink\":\"parrot-02\"}")
+custom_json_operation("{\"account\":\"test-reblog-03\",\"author\":\"test-reblog-03\",\"permlink\":\"parrot-03\"}")
+custom_json_operation("{\"account\":\"test-reblog-03\",\"author\":\"test-reblog-04\",\"permlink\":\"parrot-04\"}")
+***block 5000002***
+custom_json_operation("{\"account\":\"test-reblog-04\",\"author\":\"test-reblog-04\",\"permlink\":\"parrot-04\"}")
+custom_json_operation("{\"account\":\"test-reblog-05\",\"author\":\"test-reblog-05\",\"permlink\":\"parrot-05\"}")
+custom_json_operation("{\"account\":\"test-reblog-06\",\"author\":\"test-reblog-06\",\"permlink\":\"parrot-06\"}")
+custom_json_operation("{\"account\":\"test-reblog-06\",\"author\":\"test-reblog-05\",\"permlink\":\"parrot-05\"}")
+custom_json_operation("{\"delete\":\"delete\",\"account\":\"test-reblog-04\",\"author\":\"test-reblog-04\",\"permlink\":\"parrot-04\"}")
+custom_json_operation("{\"delete\":\"delete\",\"account\":\"test-reblog-06\",\"author\":\"test-reblog-06\",\"permlink\":\"parrot-06\"}")
+***block 5000003***
+custom_json_operation("{\"delete\":\"delete\",\"account\":\"test-reblog-05\",\"author\":\"test-reblog-05\",\"permlink\":\"parrot-05\"}")
+custom_json_operation("{\"delete\":\"delete\",\"account\":\"test-reblog-06\",\"author\":\"test-reblog-05\",\"permlink\":\"parrot-05\"}")
+***block 5000004***
+custom_json_operation("{\"account\":\"test-reblog-04\",\"author\":\"test-reblog-04\",\"permlink\":\"parrot-04\"}")
+custom_json_operation("{\"account\":\"test-reblog-05\",\"author\":\"test-reblog-05\",\"permlink\":\"parrot-05\"}")
+custom_json_operation("{\"account\":\"test-reblog-06\",\"author\":\"test-reblog-06\",\"permlink\":\"parrot-06\"}")
+custom_json_operation("{\"account\":\"test-reblog-06\",\"author\":\"test-reblog-05\",\"permlink\":\"parrot-05\"}")
+delete_comment_operation(`test-reblog-04`, `parrot-04`)
+delete_comment_operation(`test-reblog-05`, `parrot-05`)
+***block 5000005***
+delete_comment_operation(`test-reblog-06`, `parrot-06`)
+comment_operation( `test-reblog-01`, `monkey-01`)
+comment_operation( `test-reblog-01`, `monkey-02`)
+comment_operation( `test-reblog-01`, `monkey-03`)
+comment_operation( `test-reblog-01`, `monkey-04`)
+comment_operation( `test-reblog-01`, `monkey-05`)
+custom_json_operation("{\"account\":\"test-reblog-01\",\"author\":\"test-reblog-01\",\"permlink\":\"monkey-01\"}")
+delete_comment_operation(`test-reblog-01`, `monkey-01`)
+***block 5000006***
+custom_json_operation("{\"account\":\"test-reblog-01\",\"author\":\"test-reblog-01\",\"permlink\":\"monkey-02\"}")
+custom_json_operation("{\"account\":\"test-reblog-03\",\"author\":\"test-reblog-01\",\"permlink\":\"monkey-02\"}")
+custom_json_operation("{\"account\":\"test-reblog-04\",\"author\":\"test-reblog-01\",\"permlink\":\"monkey-02\"}")
+***block 5000007***
+delete_comment_operation(`test-reblog-01`, `monkey-02`)
+custom_json_operation("{\"delete\":\"delete\",\"account\":\"test-reblog-03\",\"author\":\"test-reblog-04\",\"permlink\":\"parrot-04\"}")
+custom_json_operation("{\"account\":\"test-reblog-01\",\"author\":\"test-reblog-01\",\"permlink\":\"monkey-03\"}")
+custom_json_operation("{\"account\":\"test-reblog-02\",\"author\":\"test-reblog-01\",\"permlink\":\"monkey-03\"}")
+custom_json_operation("{\"account\":\"test-reblog-03\",\"author\":\"test-reblog-01\",\"permlink\":\"monkey-03\"}")
+custom_json_operation("{\"account\":\"test-reblog-04\",\"author\":\"test-reblog-01\",\"permlink\":\"monkey-03\"}")
+custom_json_operation("{\"account\":\"test-reblog-05\",\"author\":\"test-reblog-01\",\"permlink\":\"monkey-03\"}")
+custom_json_operation("{\"account\":\"test-reblog-05\",\"author\":\"test-reblog-01\",\"permlink\":\"monkey-04\"}")
+custom_json_operation("{\"account\":\"test-reblog-05\",\"author\":\"test-reblog-01\",\"permlink\":\"monkey-05\"}")
+***block 5000008***
+custom_json_operation("{\"delete\":\"delete\",\"account\":\"test-reblog-05\",\"author\":\"test-reblog-01\",\"permlink\":\"monkey-04\"}")
+custom_json_operation("{\"account\":\"test-reblog-05\",\"author\":\"test-reblog-01\",\"permlink\":\"monkey-04\"}")
+custom_json_operation("{\"delete\":\"delete\",\"account\":\"test-reblog-05\",\"author\":\"test-reblog-01\",\"permlink\":\"monkey-04\"}")
+custom_json_operation("{\"delete\":\"delete\",\"account\":\"test-reblog-05\",\"author\":\"test-reblog-01\",\"permlink\":\"monkey-04\"}")
+***block 5000009***
+delete_comment_operation(`test-reblog-01`, `monkey-04`)
+***block 5000010***
+***block 5000011***

run_server_sync.sh

+#!/bin/bash
+
+# RUNNER_HIVEMIND_SYNC_MAX_BLOCK=5000017
+# RUNNER_HIVED_URL='{"default" : "192.168.6.136:8091"}''
+# RUNNER_POSTGRES_APP_USER=user
+# RUNNER_POSTGRES_APP_USER_PASSWORD=password
+# RUNNER_POSTGRES_HOST=localhost
+# RUNNER_POSTGRES_PORT=5432
+# HIVEMIND_DB_NAME=hivemind
+
+# ./run_server_sync.sh 5000017 '{"default" : "192.168.6.136:8091"}' mario_user mario_password localhost 5432 hivemind
+
+export RUNNER_HIVEMIND_SYNC_MAX_BLOCK=$1
+export RUNNER_HIVED_URL=$2
+export RUNNER_POSTGRES_APP_USER=$3
+export RUNNER_POSTGRES_APP_USER_PASSWORD=$4
+export RUNNER_POSTGRES_HOST=$5
+export RUNNER_POSTGRES_PORT=$6
+export HIVEMIND_DB_NAME=$7
+
+HIVE_SYNC_PATH="./scripts/ci/hive-sync.sh"
+
+"$HIVE_SYNC_PATH"

tests_api @ 395f7109

-Subproject commit 10288df4f5217ee9c38b5fb8e8c0b916500a7df9
+Subproject commit 395f7109b2544cd1ace33a6027a0078d65fb1722