hive/db/db_state.py

@@ -89,7 +89,7 @@ class DbState:
     def _disableable_indexes(cls):
         to_locate = [
             #'hive_posts_ix3', # (author, depth, id)
-            #'hive_posts_ix4', # (parent_id, id, is_deleted=0)
+            #'hive_posts_ix4', # (parent_id, id, counter_deleted=0)
             #'hive_posts_ix5', # (community_id>0, is_pinned=1)
             'hive_follows_ix5a', # (following, state, created_at, follower)
             'hive_follows_ix5b', # (follower, state, created_at, following)
@@ -304,8 +304,8 @@ class DbState:
         if cls._ver == 11:
             cls.db().query("DROP INDEX hive_posts_ix1")
             cls.db().query("DROP INDEX hive_posts_ix2")
-            cls.db().query("CREATE INDEX hive_posts_ix3 ON hive_posts (author, depth, id) WHERE is_deleted = '0'")
-            cls.db().query("CREATE INDEX hive_posts_ix4 ON hive_posts (parent_id, id) WHERE is_deleted = '0'")
+            cls.db().query("CREATE INDEX hive_posts_ix3 ON hive_posts (author, depth, id) WHERE counter_deleted = 0")
+            cls.db().query("CREATE INDEX hive_posts_ix4 ON hive_posts (parent_id, id) WHERE counter_deleted = 0")
             cls._set_ver(12)
 
         if cls._ver == 12: # community schema
@@ -324,8 +324,8 @@ class DbState:
             cls._set_ver(13)
 
         if cls._ver == 13:
-            sqls = ("CREATE INDEX hive_posts_ix5 ON hive_posts (id) WHERE is_pinned = '1' AND is_deleted = '0'",
-                    "CREATE INDEX hive_posts_ix6 ON hive_posts (community_id, id) WHERE community_id IS NOT NULL AND is_pinned = '1' AND is_deleted = '0'",)
+            sqls = ("CREATE INDEX hive_posts_ix5 ON hive_posts (id) WHERE is_pinned = '1' AND counter_deleted = 0",
+                    "CREATE INDEX hive_posts_ix6 ON hive_posts (community_id, id) WHERE community_id IS NOT NULL AND is_pinned = '1' AND counter_deleted = 0",)
                     #"CREATE INDEX hive_posts_cache_ix10 ON hive_posts_cache (post_id, payout) WHERE is_grayed = '1' AND payout > 0",
                     #"CREATE INDEX hive_posts_cache_ix30 ON hive_posts_cache (community_id, sc_trend,   post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND depth = 0",
                     #"CREATE INDEX hive_posts_cache_ix31 ON hive_posts_cache (community_id, sc_hot,     post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND depth = 0",

hive/db/schema.py

@@ -75,7 +75,7 @@ def build_metadata():
         sa.Column('community_id', sa.Integer, nullable=True),
         sa.Column('created_at', sa.DateTime, nullable=False),
         sa.Column('depth', SMALLINT, nullable=False),
-        sa.Column('is_deleted', BOOLEAN, nullable=False, server_default='0'),
+        sa.Column('counter_deleted', sa.Integer, nullable=False, server_default='0'),
         sa.Column('is_pinned', BOOLEAN, nullable=False, server_default='0'),
         sa.Column('is_muted', BOOLEAN, nullable=False, server_default='0'),
         sa.Column('is_valid', BOOLEAN, nullable=False, server_default='1'),
@@ -139,7 +139,7 @@ def build_metadata():
 
         sa.ForeignKeyConstraint(['author_id'], ['hive_accounts.id'], name='hive_posts_fk1'),
         sa.ForeignKeyConstraint(['parent_id'], ['hive_posts.id'], name='hive_posts_fk3'),
-        sa.UniqueConstraint('author_id', 'permlink_id', name='hive_posts_ux1'),
+        sa.UniqueConstraint('author_id', 'permlink_id', 'counter_deleted', name='hive_posts_ux1'),
         sa.Index('hive_posts_permlink_id', 'permlink_id'),
 
         sa.Index('hive_posts_depth_idx', 'depth'),
@@ -451,7 +451,7 @@ def setup(db):
              category_id,
              root_author_id, root_permlink_id,
              is_muted, is_valid,
-             author_id, permlink_id, created_at, updated_at, sc_hot, sc_trend,active, payout_at, cashout_time)
+             author_id, permlink_id, created_at, updated_at, sc_hot, sc_trend,active, payout_at, cashout_time, counter_deleted)
             SELECT php.id AS parent_id, php.author_id as parent_author_id,
                 php.permlink_id as parent_permlink_id, php.depth + 1 as depth,
                 (CASE
@@ -466,30 +466,21 @@ def setup(db):
                 ha.id as author_id, hpd.id as permlink_id, _date as created_at,
                 _date as updated_at, calculate_time_part_of_hot(_date) as sc_hot,
                  calculate_time_part_of_trending(_date) as sc_trend,
-                _date as active, (_date + INTERVAL '7 days') as payout_at, (_date + INTERVAL '7 days') as cashout_time
+                _date as active, (_date + INTERVAL '7 days') as payout_at, (_date + INTERVAL '7 days') as cashout_time, 0
             FROM hive_accounts ha,
                  hive_permlink_data hpd,
                  hive_posts php
             INNER JOIN hive_accounts pha ON pha.id = php.author_id
             INNER JOIN hive_permlink_data phpd ON phpd.id = php.permlink_id
             WHERE pha.name = _parent_author and phpd.permlink = _parent_permlink AND
-                   ha.name = _author and hpd.permlink = _permlink
+                   ha.name = _author and hpd.permlink = _permlink and php.counter_deleted = 0
 
             ON CONFLICT ON CONSTRAINT hive_posts_ux1 DO UPDATE SET
               --- During post update it is disallowed to change: parent-post, category, community-id
               --- then also depth, is_valid and is_muted is impossible to change
              --- post edit part
              updated_at = _date,
-             active = _date,
-
-              --- post undelete part (if was deleted)
-              is_deleted = false,
-              is_pinned = (CASE hp.is_deleted
-                              WHEN true THEN false
-                              ELSE hp.is_pinned --- no change
-                            END
-                           )
-
+             active = _date
             RETURNING (xmax = 0) as is_new_post, hp.id, hp.author_id, hp.permlink_id, (SELECT hcd.category FROM hive_category_data hcd WHERE hcd.id = hp.category_id) as post_category, hp.parent_id, hp.community_id, hp.is_valid, hp.is_muted, hp.depth, (hp.updated_at > hp.created_at) as is_edited
           ;
           ELSE
@@ -504,7 +495,7 @@ def setup(db):
              category_id,
              root_author_id, root_permlink_id,
              is_muted, is_valid,
-             author_id, permlink_id, created_at, updated_at, sc_hot, sc_trend, active, payout_at, cashout_time)
+             author_id, permlink_id, created_at, updated_at, sc_hot, sc_trend, active, payout_at, cashout_time, counter_deleted)
             SELECT 0 AS parent_id, 0 as parent_author_id, 0 as parent_permlink_id, 0 as depth,
                 (CASE
                   WHEN _date > _community_support_start_date THEN
@@ -518,7 +509,7 @@ def setup(db):
                 ha.id as author_id, hpd.id as permlink_id, _date as created_at,
                 _date as updated_at, calculate_time_part_of_hot(_date) as sc_hot,
                   calculate_time_part_of_trending(_date) as sc_trend,
-                _date as active, (_date + INTERVAL '7 days') as payout_at, (_date + INTERVAL '7 days') as cashout_time
+                _date as active, (_date + INTERVAL '7 days') as payout_at, (_date + INTERVAL '7 days') as cashout_time, 0
             FROM hive_accounts ha,
                  hive_permlink_data hpd
             WHERE ha.name = _author and hpd.permlink = _permlink
@@ -528,15 +519,7 @@ def setup(db):
               --- then also depth, is_valid and is_muted is impossible to change
               --- post edit part
               updated_at = _date,
-              active = _date,
-
-              --- post undelete part (if was deleted)
-              is_deleted = false,
-              is_pinned = (CASE hp.is_deleted
-                              WHEN true THEN false
-                              ELSE hp.is_pinned --- no change
-                            END
-                           )
+              active = _date
 
             RETURNING (xmax = 0) as is_new_post, hp.id, hp.author_id, hp.permlink_id, _parent_permlink as post_category, hp.parent_id, hp.community_id, hp.is_valid, hp.is_muted, hp.depth, (hp.updated_at > hp.created_at) as is_edited
             ;
@@ -558,11 +541,18 @@ def setup(db):
           $function$
           BEGIN
             RETURN QUERY UPDATE hive_posts AS hp
-              SET is_deleted = true
+              SET counter_deleted =
+              (
+                SELECT max( hps.counter_deleted ) + 1
+                FROM hive_posts hps
+                INNER JOIN hive_accounts ha ON hps.author_id = ha.id
+                INNER JOIN hive_permlink_data hpd ON hps.permlink_id = hpd.id
+                WHERE ha.name = _author AND hpd.permlink = _permlink
+              )
             FROM hive_posts hp1
             INNER JOIN hive_accounts ha ON hp1.author_id = ha.id
             INNER JOIN hive_permlink_data hpd ON hp1.permlink_id = hpd.id
-            WHERE hp.id = hp1.id AND ha.name = _author AND hpd.permlink = _permlink
+            WHERE hp.id = hp1.id AND ha.name = _author AND hpd.permlink = _permlink AND hp1.counter_deleted = 0
             RETURNING hp.id, hp.depth;
           END
           $function$
@@ -632,7 +622,7 @@ def setup(db):
             rpd.title AS root_title,
             hp.sc_trend,
             hp.sc_hot,
-            hp.is_deleted,
+            hp.counter_deleted,
             hp.is_pinned,
             hp.is_muted,
             hp.is_nsfw,
@@ -683,13 +673,13 @@ def setup(db):
               FROM
               (SELECT h1.Parent_Id AS queried_parent, h1.id
                FROM hive_posts h1
-               WHERE h1.depth > 0 AND NOT h1.is_deleted
+               WHERE h1.depth > 0 AND h1.counter_deleted = 0
                ORDER BY h1.depth DESC
               ) s
               UNION ALL
               SELECT tblChild.queried_parent, p.id FROM hive_posts p
               JOIN tblChild  ON p.Parent_Id = tblChild.Id
-              WHERE NOT p.is_deleted
+              WHERE p.counter_deleted = 0
             )
             SELECT queried_parent, cast(count(1) AS int) AS children_count
             FROM tblChild
@@ -800,7 +790,7 @@ def setup(db):
               hive_posts_view hp
           WHERE
               NOT hp.is_muted AND
-              NOT hp.is_deleted AND
+              hp.counter_deleted == 0 AND
               -- ABW: wrong! fat node required _author+_permlink to exist (when given) and sorted by ( cashout_time, comment_id )
               hp.cashout_time >= _cashout_time AND
               hp.id >= (SELECT id FROM hive_posts_view hp1 WHERE hp1.author >= _author AND hp1.permlink >= _permlink ORDER BY id LIMIT 1)
@@ -839,7 +829,7 @@ def setup(db):
               hive_posts_view hp
           WHERE
               NOT hp.is_muted AND
-              NOT hp.is_deleted AND
+              hp.counter_deleted = 0 AND
               hp.author > _author COLLATE "C" OR
               hp.author = _author AND hp.permlink >= _permlink COLLATE "C"
           ORDER BY
@@ -879,7 +869,7 @@ def setup(db):
               hive_posts_view hp
           WHERE
               NOT hp.is_muted AND
-              NOT hp.is_deleted AND
+              hp.counter_deleted = 0 AND
               -- ABW: wrong! fat node required both _root_author+_root_permlink and _start_post_author+start_post_permlink to exist (when given)
               -- and sorted by ( root_id, comment_id )
               root_author >= _root_author AND
@@ -923,7 +913,7 @@ def setup(db):
               hive_posts_view hp
           WHERE
               NOT hp.is_muted AND
-              NOT hp.is_deleted AND
+              hp.counter_deleted = 0 AND
               -- ABW: wrong! fat node required _start_post_author+_start_port_permlink to exist (when given) and sorted by ( parent_author, parent_permlink, comment_id )
               parent_author > _parent_author COLLATE "C" OR
               parent_author = _parent_author AND ( parent_permlink > _parent_permlink COLLATE "C" OR
@@ -967,7 +957,7 @@ def setup(db):
               hive_posts_view hp
           WHERE
               NOT hp.is_muted AND
-              NOT hp.is_deleted AND
+              hp.counter_deleted = 0 AND
               -- ABW: wrong! fat node required _start_post_author+_start_port_permlink to exist (when given) and sorted by ( _parent_author, updated_at, comment_id )
               hp.parent_author > _parent_author COLLATE "C" OR
               hp.parent_author = _parent_author AND hp.updated_at >= _updated_at AND 
@@ -1009,7 +999,7 @@ def setup(db):
               hive_posts_view hp
           WHERE
               NOT hp.is_muted AND
-              NOT hp.is_deleted AND
+              hp.counter_deleted = 0 AND
               -- ABW: wrong! fat node required _start_post_author+_start_post_permlink to exist (when given) and sorted just like
               -- in case of by_last_update (but in fat node) but should by ( _author, updated_at, comment_id )
               hp.author > _author COLLATE "C" OR
@@ -1032,7 +1022,7 @@ def setup(db):
     # hot and tranding functions
 
     sql = """
-       DROP FUNCTION IF EXISTS date_diff CASCADE
+       DROP FUNCTION IF EXISTS date_diff() CASCADE
        ;
        CREATE OR REPLACE FUNCTION date_diff (units VARCHAR(30), start_t TIMESTAMP, end_t TIMESTAMP)
          RETURNS INT AS $$

hive/indexer/blocks.py

@@ -106,7 +106,8 @@ class Blocks:
     @staticmethod
     def prepare_vops(comment_payout_ops, vopsList, date):
         vote_ops = {}
-        ops_stats = { 'author_reward_operation' : 0, 'comment_reward_operation' : 0, 'effective_comment_vote_operation' : 0, 'comment_payout_update_operation' : 0 }
+        ops_stats = { 'author_reward_operation' : 0, 'comment_reward_operation' : 0, 'effective_comment_vote_operation' : 0, 'comment_payout_update_operation' : 0, 'ineffective_delete_comment_operation' : 0 }
+        inefficient_deleted_ops = {}
 
         for vop in vopsList:
             key = None
@@ -151,8 +152,11 @@ class Blocks:
                   comment_payout_ops[key] = { 'author_reward_operation':None, 'comment_reward_operation':None, 'effective_comment_vote_operation':None, 'comment_payout_update_operation':None, 'date' : date }
 
                 comment_payout_ops[key][op_type] = op_value
+            elif op_type == 'ineffective_delete_comment_operation':
+              ops_stats[ 'ineffective_delete_comment_operation' ] += 1
+              inefficient_deleted_ops[key] = {}
 
-        return (vote_ops, ops_stats)
+        return (vote_ops, ops_stats, inefficient_deleted_ops)
 
 
     @classmethod
@@ -170,6 +174,17 @@ class Blocks:
         if cls._head_block_date is None:
             cls._head_block_date = cls._current_block_date
 
+        vote_ops                = None
+        comment_payout_stats    = None
+        inefficient_deleted_ops = None
+
+        if is_initial_sync:
+            if num in virtual_operations:
+              (vote_ops, comment_payout_stats, inefficient_deleted_ops ) = Blocks.prepare_vops(Posts.comment_payout_ops, virtual_operations[num], cls._current_block_date)
+        else:
+            vops = hived.get_virtual_operations(num)
+            (vote_ops, comment_payout_stats, inefficient_deleted_ops ) = Blocks.prepare_vops(Posts.comment_payout_ops, vops, cls._current_block_date)
+
         json_ops = []
         for tx_idx, tx in enumerate(block['transactions']):
             for operation in tx['operations']:
@@ -205,7 +220,9 @@ class Blocks:
                     if not is_initial_sync:
                         Accounts.dirty(op['author']) # lite - stats
                 elif op_type == 'delete_comment_operation':
-                    Posts.delete_op(op)
+                    key = "{}/{}".format(op['author'], op['permlink'])
+                    if ( inefficient_deleted_ops is None ) or ( key not in inefficient_deleted_ops ):
+                      Posts.delete_op(op)
                 elif op_type == 'comment_options_operation':
                     Posts.comment_options_op(op)
                 elif op_type == 'vote_operation':
@@ -231,16 +248,6 @@ class Blocks:
             custom_ops_stats = CustomOp.process_ops(json_ops, num, cls._head_block_date)
             cls.ops_stats = Blocks.merge_ops_stats(cls.ops_stats, custom_ops_stats)
 
-        vote_ops = None
-        comment_payout_stats = None
-
-        if is_initial_sync:
-            if num in virtual_operations:
-              (vote_ops, comment_payout_stats) = Blocks.prepare_vops(Posts.comment_payout_ops, virtual_operations[num], cls._current_block_date)
-        else:
-            vops = hived.get_virtual_operations(num)
-            (vote_ops, comment_payout_stats) = Blocks.prepare_vops(Posts.comment_payout_ops, vops, cls._current_block_date)
-
         if vote_ops is not None:
             for k, v in vote_ops.items():
                 Votes.effective_comment_vote_op(k, v)

hive/indexer/community.py

@@ -238,7 +238,7 @@ class Community:
                                FROM hive_posts
                               WHERE community_id IS NOT NULL
                                 AND is_paidout = '0'
-                                AND is_deleted = '0'
+                                AND counter_deleted = 0
                            GROUP BY community_id
                         ) p
                      ON community_id = id

hive/indexer/feed_cache.py

@@ -55,7 +55,7 @@ class FeedCache:
                  SELECT hive_accounts.id, hive_posts.id, hive_posts.created_at
                    FROM hive_posts
                    JOIN hive_accounts ON hive_posts.author_id = hive_accounts.id
-                  WHERE depth = 0 AND is_deleted = '0'
+                  WHERE depth = 0 AND counter_deleted = 0
             ON CONFLICT DO NOTHING
         """)
         lap_1 = time.perf_counter()

hive/indexer/posts.py

@@ -36,7 +36,7 @@ class Posts:
     @classmethod
     def last_id(cls):
         """Get the last indexed post id."""
-        sql = "SELECT MAX(id) FROM hive_posts WHERE is_deleted = '0'"
+        sql = "SELECT MAX(id) FROM hive_posts WHERE counter_deleted = 0"
         return DB.query_one(sql) or 0
 
     @classmethod

hive/indexer/votes.py

@@ -138,6 +138,7 @@ class Votes:
                 INNER JOIN hive_accounts ha_a ON ha_a.name = t.author
                 INNER JOIN hive_permlink_data hpd_p ON hpd_p.permlink = t.permlink
                 INNER JOIN hive_posts hp ON hp.author_id = ha_a.id AND hp.permlink_id = hpd_p.id
+                WHERE hp.counter_deleted = 0
                 ON CONFLICT ON CONSTRAINT hive_votes_ux1 DO
                 UPDATE
                   SET

hive/server/bridge_api/cursor.py

@@ -210,7 +210,7 @@ async def _pinned(db, community_id):
     """Get a list of pinned post `id`s in `community`."""
     sql = """SELECT id FROM hive_posts
               WHERE is_pinned = '1'
-                AND is_deleted = '0'
+                AND counter_deleted = 0
                 AND community_id = :community_id
             ORDER BY id DESC"""
     return await db.query_col(sql, community_id=community_id)
@@ -237,7 +237,7 @@ async def pids_by_blog(db, account: str, start_author: str = '',
     skip = """
         SELECT id FROM hive_posts
          WHERE author_id = (SELECT id FROM hive_accounts WHERE name = :account)
-           AND is_deleted = '0'
+           AND counter_deleted = 0
            AND depth = 0
            AND community_id IS NOT NULL
            AND id NOT IN (SELECT post_id FROM hive_reblogs
@@ -257,7 +257,7 @@ async def pids_by_blog(db, account: str, start_author: str = '',
     #    SELECT id
     #      FROM (
     #             SELECT id, author account, created_at FROM hive_posts
-    #              WHERE depth = 0 AND is_deleted = '0' AND community_id IS NULL
+    #              WHERE depth = 0 AND counter_deleted = 0 AND community_id IS NULL
     #              UNION ALL
     #             SELECT post_id id, account, created_at FROM hive_reblogs
     #           ) blog
@@ -319,7 +319,7 @@ async def pids_by_posts(db, account: str, start_permlink: str = '', limit: int =
     sql = """
         SELECT id FROM hive_posts
          WHERE author = (SELECT id FROM hive_accounts WHERE name = :account) %s
-           AND is_deleted = '0'
+           AND counter_deleted = 0
            AND depth = '0'
       ORDER BY id DESC
          LIMIT :limit
@@ -342,7 +342,7 @@ async def pids_by_comments(db, account: str, start_permlink: str = '', limit: in
     sql = """
         SELECT id FROM hive_posts
          WHERE author = (SELECT id FROM hive_accounts WHERE name = :account) %s
-           AND is_deleted = '0'
+           AND counter_deleted = 0
            AND depth > 0
       ORDER BY id DESC, depth
          LIMIT :limit
@@ -386,10 +386,10 @@ async def pids_by_replies(db, start_author: str, start_permlink: str = '',
        SELECT id FROM hive_posts
         WHERE parent_id IN (SELECT id FROM hive_posts
                              WHERE author_id = (SELECT id FROM hive_accounts WHERE name = :parent)
-                               AND is_deleted = '0'
+                               AND counter_deleted = 0
                           ORDER BY id DESC
                              LIMIT 10000) %s
-          AND is_deleted = '0'
+          AND counter_deleted = 0
      ORDER BY id DESC
         LIMIT :limit
     """ % seek

hive/server/bridge_api/methods.py

@@ -105,7 +105,7 @@ async def get_post(context, author, permlink, observer=None):
     if observer and context:
         blacklists_for_user = await Mutes.get_blacklists_for_observer(observer, context)
 
-    sql = "---bridge_api.get_post\n" + SQL_TEMPLATE + """ hp.author = :author AND hp.permlink = :permlink AND NOT hp.is_deleted """
+    sql = "---bridge_api.get_post\n" + SQL_TEMPLATE + """ hp.author = :author AND hp.permlink = :permlink AND hp.counter_deleted = 0 """
 
     result = await db.query_all(sql, author=author, permlink=permlink)
     assert len(result) == 1, 'invalid author/permlink or post not found in cache'
@@ -133,26 +133,26 @@ async def get_ranked_posts(context, sort, start_author='', start_permlink='',
     pinned_sql = ''
 
     if sort == 'trending':
-        sql = SQL_TEMPLATE + """ NOT hp.is_paidout AND hp.depth = 0 AND NOT hp.is_deleted
+        sql = SQL_TEMPLATE + """ NOT hp.is_paidout AND hp.depth = 0 AND hp.counter_deleted = 0
                                     %s ORDER BY hp.sc_trend DESC, hp.id LIMIT :limit """
     elif sort == 'hot':
-        sql = SQL_TEMPLATE + """ NOT hp.is_paidout AND hp.depth = 0 AND NOT hp.is_deleted
+        sql = SQL_TEMPLATE + """ NOT hp.is_paidout AND hp.depth = 0 AND hp.counter_deleted = 0
                                     %s ORDER BY hp.sc_hot DESC, hp.id LIMIT :limit """
     elif sort == 'created':
-        sql = SQL_TEMPLATE + """ hp.depth = 0 AND NOT hp.is_deleted AND NOT hp.is_grayed
+        sql = SQL_TEMPLATE + """ hp.depth = 0 AND hp.counter_deleted = 0 AND NOT hp.is_grayed
                                     %s ORDER BY hp.created_at DESC, hp.id LIMIT :limit """
     elif sort == 'promoted':
-        sql = SQL_TEMPLATE + """ hp.depth > 0 AND hp.promoted > 0 AND NOT hp.is_deleted
+        sql = SQL_TEMPLATE + """ hp.depth > 0 AND hp.promoted > 0 AND hp.counter_deleted = 0
                                     AND NOT hp.is_paidout %s ORDER BY hp.promoted DESC, hp.id LIMIT :limit """
     elif sort == 'payout':
-        sql = SQL_TEMPLATE + """ NOT hp.is_paidout AND NOT hp.is_deleted %s
+        sql = SQL_TEMPLATE + """ NOT hp.is_paidout AND hp.counter_deleted = 0 %s
                                     AND hp.payout_at BETWEEN now() + interval '12 hours' AND now() + interval '36 hours'
                                     ORDER BY hp.payout DESC, hp.id LIMIT :limit """
     elif sort == 'payout_comments':
-        sql = SQL_TEMPLATE + """ NOT hp.is_paidout AND NOT hp.is_deleted AND hp.depth > 0
+        sql = SQL_TEMPLATE + """ NOT hp.is_paidout AND hp.counter_deleted = 0 AND hp.depth > 0
                                     %s ORDER BY hp.payout DESC, hp.id LIMIT :limit """
     elif sort == 'muted':
-        sql = SQL_TEMPLATE + """ NOT hp.is_paidout AND NOT hp.is_deleted AND hp.is_grayed
+        sql = SQL_TEMPLATE + """ NOT hp.is_paidout AND hp.counter_deleted = 0 AND hp.is_grayed
                                     AND hp.payout > 0 %s ORDER BY hp.payout DESC, hp.id LIMIT :limit """
 
     sql = "---bridge_api.get_ranked_posts\n" + sql
@@ -290,11 +290,11 @@ async def get_account_posts(context, sort, account, start_author='', start_perml
                 post['reblogged_by'] = [account]
         return posts
     elif sort == 'posts':
-        sql = sql % """ hp.author = :account AND NOT hp.is_deleted AND hp.depth = 0 %s ORDER BY hp.id DESC LIMIT :limit"""
+        sql = sql % """ hp.author = :account AND hp.counter_deleted = 0 AND hp.depth = 0 %s ORDER BY hp.id DESC LIMIT :limit"""
     elif sort == 'comments':
-        sql = sql % """ hp.author = :account AND NOT hp.is_deleted AND hp.depth > 0 %s ORDER BY hp.id DESC, hp.depth LIMIT :limit"""
+        sql = sql % """ hp.author = :account AND hp.counter_deleted = 0 AND hp.depth > 0 %s ORDER BY hp.id DESC, hp.depth LIMIT :limit"""
     elif sort == 'payout':
-        sql = sql % """ hp.author = :account AND NOT hp.is_deleted AND NOT hp.is_paidout %s ORDER BY hp.payout DESC, hp.id LIMIT :limit"""
+        sql = sql % """ hp.author = :account AND hp.counter_deleted = 0 AND NOT hp.is_paidout %s ORDER BY hp.payout DESC, hp.id LIMIT :limit"""
     elif sort == 'feed':
         res = await cursor.pids_by_feed_with_reblog(db, account, *start, limit)
         return await load_posts_reblogs(context['db'], res)

hive/server/bridge_api/objects.py

@@ -147,7 +147,7 @@ async def load_posts_keyed(db, ids, truncate_body=0):
 
 
     sql = """SELECT id FROM hive_posts
-              WHERE id IN :ids AND is_pinned = '1' AND is_deleted = '0'"""
+              WHERE id IN :ids AND is_pinned = '1' AND counter_deleted = 0"""
     for pid in await db.query_col(sql, ids=tuple(ids)):
         if pid in posts_by_id:
             posts_by_id[pid]['stats']['is_pinned'] = True
@@ -170,16 +170,16 @@ async def load_posts(db, ids, truncate_body=0):
             ids.remove(_id)
             sql = """
                 SELECT
-                    hp.id, ha_a.name as author, hpd_p.permlink as permlink, depth, created_at, is_deleted
+                    hp.id, ha_a.name as author, hpd_p.permlink as permlink, depth, created_at, counter_deleted
                 FROM
                     hive_posts hp
                 INNER JOIN hive_accounts ha_a ON ha_a.id = hp.author_id
                 INNER JOIN hive_permlink_data hpd_p ON hpd_p.id = hp.permlink_id
-                WHERE id = :id"""
+                WHERE id = :id and counter_deleted = 0 """
             post = await db.query_row(sql, id=_id)
-            if not post['is_deleted']:
+            if post is None:
                 # TODO: This should never happen. See #173 for analysis
-                log.error("missing post: %s", dict(post))
+                log.error("missing post: id %i", _id)
             else:
                 log.info("requested deleted post: %s", dict(post))
 

hive/server/bridge_api/support.py

@@ -29,7 +29,7 @@ async def get_post_header(context, author, permlink):
         LEFT JOIN hive_category_data hcd ON hcd.id = hp.category_id
         WHERE ha_a.name = :author
             AND hpd_p.permlink = :permlink
-            AND is_deleted = '0'
+            AND counter_deleted = 0
     """
 
     row = await db.query_row(sql, author=author, permlink=permlink)

hive/server/bridge_api/thread.py

@@ -17,7 +17,7 @@ log = logging.getLogger(__name__)
 async def get_discussion(context, author, permlink, observer=None):
     """Modified `get_state` thread implementation."""
     # New index was created: hive_posts_parent_id_btree (CREATE INDEX "hive_posts_parent_id_btree" ON hive_posts btree(parent_id)
-    # We thougth this would be covered by "hive_posts_ix4" btree (parent_id, id) WHERE is_deleted = false but it was not
+    # We thougth this would be covered by "hive_posts_ix4" btree (parent_id, id) WHERE counter_deleted = 0 but it was not
     db = context['db']
 
     author = valid_account(author)
@@ -29,13 +29,13 @@ async def get_discussion(context, author, permlink, observer=None):
             id, parent_id
           FROM hive_posts_view hpv WHERE hpv.author = :author
             AND hpv.permlink = :permlink
-            AND NOT hpv.is_deleted AND NOT hpv.is_muted
+            AND hpv.counter_deleted = 0 AND NOT hpv.is_muted
           UNION ALL
           SELECT
             children.id, children.parent_id
           FROM hive_posts children
           INNER JOIN child_posts ON (children.parent_id = child_posts.id) 
-          WHERE NOT children.is_deleted AND NOT children.is_muted
+          WHERE children.counter_deleted = 0 AND NOT children.is_muted
         )
         SELECT
           cp.id,
@@ -80,7 +80,7 @@ async def get_discussion(context, author, permlink, observer=None):
           hpv.curator_payout_value
         FROM child_posts cp
         INNER JOIN hive_posts_view hpv ON (hpv.id = cp.id)
-        WHERE NOT hpv.is_deleted AND NOT hpv.is_muted
+        WHERE hpv.counter_deleted = 0 AND NOT hpv.is_muted
         ORDER BY cp.id
         LIMIT 2000
     """
@@ -135,7 +135,7 @@ async def _child_ids(db, parent_ids):
              SELECT parent_id, array_agg(id)
                FROM hive_posts
               WHERE parent_id IN :ids
-                AND is_deleted = '0'
+                AND counter_deleted = 0
            GROUP BY parent_id
     """
     rows = await db.query_all(sql, ids=tuple(parent_ids))

hive/server/common/payout_stats.py

@@ -44,7 +44,7 @@ class PayoutStats:
                    NULL authors
               FROM hive_posts
               INNER JOIN hive_accounts ha ON ha.id = hive_posts.author_id
-             WHERE is_paidout = '0' and is_deleted = false
+             WHERE is_paidout = '0' and counter_deleted = 0
           GROUP BY community_id, author
 
              UNION ALL
@@ -55,7 +55,7 @@ class PayoutStats:
                    COUNT(*) posts,
                    COUNT(DISTINCT(author_id)) authors
               FROM hive_posts
-             WHERE is_paidout = '0' and is_deleted = false
+             WHERE is_paidout = '0' and counter_deleted = 0
           GROUP BY community_id
         """
 

hive/server/condenser_api/cursor.py

@@ -21,12 +21,12 @@ async def get_post_id(db, author, permlink):
         INNER JOIN hive_accounts ha_a ON ha_a.id = hp.author_id
         INNER JOIN hive_permlink_data hpd_p ON hpd_p.id = hp.permlink_id
         WHERE ha_a.name = :author AND hpd_p.permlink = :permlink
-            AND is_deleted = '0' LIMIT 1"""
+            AND counter_deleted = 0 LIMIT 1"""
     return await db.query_one(sql, author=author, permlink=permlink)
 
 async def get_child_ids(db, post_id):
     """Given a parent post id, retrieve all child ids."""
-    sql = "SELECT id FROM hive_posts WHERE parent_id = :id AND is_deleted = '0'"
+    sql = "SELECT id FROM hive_posts WHERE parent_id = :id AND counter_deleted = 0"
     return await db.query_col(sql, id=post_id)
 
 async def _get_post_id(db, author, permlink):
@@ -317,7 +317,7 @@ async def pids_by_blog_without_reblog(db, account: str, start_permlink: str = ''
         SELECT id
           FROM hive_posts
          WHERE author_id = (SELECT id FROM hive_accounts WHERE name = :account) %s
-           AND is_deleted = '0'
+           AND counter_deleted = 0
            AND depth = 0
       ORDER BY id DESC
          LIMIT :limit
@@ -377,7 +377,7 @@ async def pids_by_account_comments(db, account: str, start_permlink: str = '', l
         SELECT id FROM hive_posts
          WHERE author_id = (SELECT id FROM hive_accounts WHERE name = :account) %s
            AND depth > 0
-           AND is_deleted = '0'
+           AND counter_deleted = 0
       ORDER BY id DESC, depth
          LIMIT :limit
     """ % seek
@@ -420,10 +420,10 @@ async def pids_by_replies_to_account(db, start_author: str, start_permlink: str
        SELECT id FROM hive_posts
         WHERE parent_id IN (SELECT id FROM hive_posts
                              WHERE author_id = (SELECT id FROM hive_accounts WHERE name = :parent)
-                               AND is_deleted = '0'
+                               AND counter_deleted = 0
                           ORDER BY id DESC
                              LIMIT 10000) %s
-          AND is_deleted = '0'
+          AND counter_deleted = 0
      ORDER BY id DESC
         LIMIT :limit
     """ % seek

hive/server/condenser_api/get_state.py

@@ -228,7 +228,7 @@ async def _child_ids(db, parent_ids):
              SELECT parent_id, array_agg(id)
                FROM hive_posts
               WHERE parent_id IN :ids
-                AND is_deleted = '0'
+                AND counter_deleted = 0
            GROUP BY parent_id
     """
     rows = await db.query_all(sql, ids=tuple(parent_ids))

hive/server/condenser_api/methods.py

@@ -148,7 +148,7 @@ async def get_content(context, author: str, permlink: str, observer=None):
     valid_permlink(permlink)
     #force copy
     sql = str(SQL_TEMPLATE)
-    sql += """ hp.author = :author AND hp.permlink = :permlink AND NOT hp.is_deleted """
+    sql += """ hp.author = :author AND hp.permlink = :permlink AND hp.counter_deleted = 0 """
 
     post = None
     result = await db.query_all(sql, author=author, permlink=permlink)
@@ -175,7 +175,7 @@ async def get_content_replies(context, author: str, permlink: str):
     #force copy
     sql = str(SQL_TEMPLATE)
     sql += """
-            hp.is_deleted = '0' AND hp.parent_author = :author AND hp.parent_permlink = :permlink
+            hp.counter_deleted = 0 AND hp.parent_author = :author AND hp.parent_permlink = :permlink
         ORDER BY hp.id LIMIT :limit
     """
 
@@ -224,7 +224,7 @@ async def get_discussions_by(discussion_type, context, start_author: str = '',
 
     sql = "---get_discussions_by_" + discussion_type + "\r\n" + str(SQL_TEMPLATE)
 
-    sql = sql + """ NOT hp.is_deleted """
+    sql = sql + """ hp.counter_deleted = 0 """
 
     if discussion_type == 'trending':
         sql = sql + """ AND NOT hp.is_paidout %s ORDER BY hp.sc_trend DESC LIMIT :limit """
@@ -365,7 +365,7 @@ async def get_discussions_by_blog(context, tag: str = None, start_author: str =
     #force copy
     sql = str(SQL_TEMPLATE)
     sql += """
-            NOT hp.is_deleted AND hp.id IN
+            hp.counter_deleted = 0 AND hp.id IN
                 (SELECT post_id FROM hive_feed_cache JOIN hive_accounts ON (hive_feed_cache.account_id = hive_accounts.id) WHERE hive_accounts.name = :author)
           """
     if start_author and start_permlink != '':
@@ -425,7 +425,7 @@ async def get_discussions_by_comments(context, start_author: str = None, start_p
     sql = str(SQL_TEMPLATE)
     sql += """
             hp.author = :start_author AND hp.depth > 0
-            AND NOT hp.is_deleted
+            AND hp.counter_deleted = 0
     """
 
     if start_permlink:

hive/server/condenser_api/objects.py

@@ -118,16 +118,16 @@ async def load_posts(db, ids, truncate_body=0):
             ids.remove(_id)
             sql = """
                 SELECT
-                    hp.id, ha_a.name as author, hpd_p.permlink as permlink, depth, created_at, is_deleted
+                    hp.id, ha_a.name as author, hpd_p.permlink as permlink, depth, created_at, counter_deleted
                 FROM
                     hive_posts hp
                 INNER JOIN hive_accounts ha_a ON ha_a.id = hp.author_id
                 INNER JOIN hive_permlink_data hpd_p ON hpd_p.id = hp.permlink_id
-                WHERE id = :id"""
+                WHERE id = :id and counter_deleted = 0 """
             post = await db.query_row(sql, id=_id)
-            if not post['is_deleted']:
+            if post is None:
                 # TODO: This should never happen. See #173 for analysis
-                log.error("missing post -- %s", dict(post))
+                log.error("missing post: id %i", _id)
             else:
                 log.info("requested deleted post: %s", dict(post))
 

hive/server/condenser_api/tags.py

@@ -32,7 +32,7 @@ async def get_trending_tags(context, start_tag: str = '', limit: int = 250):
           HAVING SUM(payout + pending_payout) <= (
             SELECT SUM(payout + pending_payout)
               FROM hive_posts
-             WHERE is_paidout = '0' and is_deleted = false
+             WHERE is_paidout = '0' and counter_deleted = 0
                AND category_id = (SELECT id FROM hive_category_data WHERE category = :start_tag))
         """
     else:
@@ -44,7 +44,7 @@ async def get_trending_tags(context, start_tag: str = '', limit: int = 250):
              SUM(CASE WHEN depth = 0 THEN 1 ELSE 0 END) AS top_posts,
              SUM(payout + pending_payout) AS total_payouts
         FROM hive_posts
-       WHERE is_paidout = '0' and is_deleted = false
+       WHERE is_paidout = '0' and counter_deleted = 0
     GROUP BY category %s
     ORDER BY SUM(payout + pending_payout) DESC
        LIMIT :limit

hive/server/database_api/methods.py

@@ -117,7 +117,7 @@ async def find_comments(context, comments: list):
             hive_posts_view hp
         JOIN (VALUES {}) AS t (author, permlink) ON hp.author = t.author AND hp.permlink = t.permlink
         WHERE
-            NOT hp.is_muted AND NOT hp.is_deleted
+            NOT hp.is_muted AND hp.counter_deleted = 0
     """
 
     idx = 0

hive/server/hive_api/notify.py

@@ -101,7 +101,7 @@ def _notifs_sql(where):
           LEFT JOIN hive_communities hc ON hn.community_id = hc.id
           WHERE %s
             AND score >= :min_score
-            AND COALESCE(hp.is_deleted, False) = False
+            AND COALESCE(hp.counter_deleted, 0) = 0
        ORDER BY hn.id DESC
           LIMIT :limit"""
     return sql % where