hive/db/schema.py

@@ -590,6 +590,7 @@ def setup(db):
             hp.depth,
             hp.promoted,
             hp.payout,
+            hp.pending_payout,
             hp.payout_at,
             hp.is_paidout,
             hp.children,
@@ -731,7 +732,7 @@ def setup(db):
     sql = """
       DROP TYPE IF EXISTS database_api_post CASCADE;
       CREATE TYPE database_api_post AS (
-        id INT, 
+        id INT,
         community_id INT,
         author VARCHAR(16),
         permlink VARCHAR(255),
@@ -743,7 +744,7 @@ def setup(db):
         payout DECIMAL(10,3),
         payout_at TIMESTAMP,
         is_paidout BOOLEAN,
-        children SMALLINT,
+        children INT,
         votes INT,
         created_at TIMESTAMP,
         updated_at TIMESTAMP,
@@ -785,7 +786,7 @@ def setup(db):
         AS
         $function$
         BEGIN
-          RETURN QUERY 
+          RETURN QUERY
           SELECT
               hp.id, hp.community_id, hp.author, hp.permlink, hp.title, hp.body,
               hp.category, hp.depth, hp.promoted, hp.payout, hp.payout_at, hp.is_paidout,
@@ -800,6 +801,7 @@ def setup(db):
           WHERE
               NOT hp.is_muted AND
               NOT hp.is_deleted AND
+              -- ABW: wrong! fat node required _author+_permlink to exist (when given) and sorted by ( cashout_time, comment_id )
               hp.cashout_time >= _cashout_time AND
               hp.id >= (SELECT id FROM hive_posts_view hp1 WHERE hp1.author >= _author AND hp1.permlink >= _permlink ORDER BY id LIMIT 1)
           ORDER BY
@@ -823,7 +825,7 @@ def setup(db):
         AS
         $function$
         BEGIN
-          RETURN QUERY 
+          RETURN QUERY
           SELECT
               hp.id, hp.community_id, hp.author, hp.permlink, hp.title, hp.body,
               hp.category, hp.depth, hp.promoted, hp.payout, hp.payout_at, hp.is_paidout,
@@ -838,8 +840,8 @@ def setup(db):
           WHERE
               NOT hp.is_muted AND
               NOT hp.is_deleted AND
-              hp.author >= _author COLLATE "C" AND
-              hp.permlink >= _permlink COLLATE "C"
+              hp.author > _author COLLATE "C" OR
+              hp.author = _author AND hp.permlink >= _permlink COLLATE "C"
           ORDER BY
               hp.author COLLATE "C" ASC,
               hp.permlink COLLATE "C" ASC
@@ -863,7 +865,7 @@ def setup(db):
         AS
         $function$
         BEGIN
-          RETURN QUERY 
+          RETURN QUERY
           SELECT
               hp.id, hp.community_id, hp.author, hp.permlink, hp.title, hp.body,
               hp.category, hp.depth, hp.promoted, hp.payout, hp.payout_at, hp.is_paidout,
@@ -878,14 +880,16 @@ def setup(db):
           WHERE
               NOT hp.is_muted AND
               NOT hp.is_deleted AND
+              -- ABW: wrong! fat node required both _root_author+_root_permlink and _start_post_author+start_post_permlink to exist (when given)
+              -- and sorted by ( root_id, comment_id )
               root_author >= _root_author AND
               root_permlink >= _root_permlink AND
               hp.id >= (SELECT id FROM hive_posts_view hp1 WHERE hp1.author >= _start_post_author AND hp1.permlink >= _start_post_permlink ORDER BY id LIMIT 1)
-          ORDER BY 
-              root_author ASC, 
+          ORDER BY
+              root_author ASC,
               root_permlink ASC,
               id ASC
-          LIMIT 
+          LIMIT
               _limit
           ;
         END
@@ -905,7 +909,7 @@ def setup(db):
         AS
         $function$
         BEGIN
-          RETURN QUERY 
+          RETURN QUERY
           SELECT
               hp.id, hp.community_id, hp.author, hp.permlink, hp.title, hp.body,
               hp.category, hp.depth, hp.promoted, hp.payout, hp.payout_at, hp.is_paidout,
@@ -920,9 +924,11 @@ def setup(db):
           WHERE
               NOT hp.is_muted AND
               NOT hp.is_deleted AND
-              parent_author >= _parent_author AND
-              parent_permlink >= _parent_permlink AND
-              hp.id >= (SELECT id FROM hive_posts_view hp1 WHERE hp1.author >= _start_post_author AND hp1.permlink >= _start_post_permlink ORDER BY id LIMIT 1)
+              -- ABW: wrong! fat node required _start_post_author+_start_port_permlink to exist (when given) and sorted by ( parent_author, parent_permlink, comment_id )
+              parent_author > _parent_author COLLATE "C" OR
+              parent_author = _parent_author AND ( parent_permlink > _parent_permlink COLLATE "C" OR
+              parent_permlink = _parent_permlink AND
+              hp.id >= (SELECT id FROM hive_posts_view hp1 WHERE hp1.author >= _start_post_author AND hp1.permlink >= _start_post_permlink ORDER BY id LIMIT 1) )
           ORDER BY
               parent_author ASC,
               parent_permlink ASC,
@@ -947,7 +953,7 @@ def setup(db):
         AS
         $function$
         BEGIN
-          RETURN QUERY 
+          RETURN QUERY
           SELECT
               hp.id, hp.community_id, hp.author, hp.permlink, hp.title, hp.body,
               hp.category, hp.depth, hp.promoted, hp.payout, hp.payout_at, hp.is_paidout,
@@ -962,14 +968,15 @@ def setup(db):
           WHERE
               NOT hp.is_muted AND
               NOT hp.is_deleted AND
-              hp.parent_author >= _parent_author AND 
-              hp.updated_at >= _updated_at AND 
+              -- ABW: wrong! fat node required _start_post_author+_start_port_permlink to exist (when given) and sorted by ( _parent_author, updated_at, comment_id )
+              hp.parent_author > _parent_author COLLATE "C" OR
+              hp.parent_author = _parent_author AND hp.updated_at >= _updated_at AND 
               hp.id >= (SELECT id FROM hive_posts_view hp1 WHERE hp1.author >= _start_post_author AND hp1.permlink >= _start_post_permlink ORDER BY id LIMIT 1)
-          ORDER BY 
-              hp.parent_author ASC, 
-              hp.updated_at ASC, 
-              hp.id ASC 
-          LIMIT 
+          ORDER BY
+              hp.parent_author ASC,
+              hp.updated_at ASC,
+              hp.id ASC
+          LIMIT
               _limit
           ;
         END
@@ -988,7 +995,7 @@ def setup(db):
         AS
         $function$
         BEGIN
-          RETURN QUERY 
+          RETURN QUERY
           SELECT
               hp.id, hp.community_id, hp.author, hp.permlink, hp.title, hp.body,
               hp.category, hp.depth, hp.promoted, hp.payout, hp.payout_at, hp.is_paidout,
@@ -1003,14 +1010,16 @@ def setup(db):
           WHERE
               NOT hp.is_muted AND
               NOT hp.is_deleted AND
-              hp.author >= _author AND 
-              hp.updated_at >= _updated_at AND 
-              hp.id >= (SELECT id FROM hive_posts_view hp1 WHERE hp1.author >= _start_post_author AND hp1.permlink >= _start_post_permlink ORDER BY id LIMIT 1)
-          ORDER BY 
-              hp.parent_author ASC, 
-              hp.updated_at ASC, 
-              hp.id ASC 
-          LIMIT 
+              -- ABW: wrong! fat node required _start_post_author+_start_post_permlink to exist (when given) and sorted just like
+              -- in case of by_last_update (but in fat node) but should by ( _author, updated_at, comment_id )
+              hp.author > _author COLLATE "C" OR
+              hp.author = _author AND hp.updated_at >= _updated_at AND 
+              hp.id >= (SELECT id FROM hive_posts_view hp1 WHERE hp1.author > _start_post_author COLLATE "C" OR hp1.author = _start_post_author AND hp1.permlink >= _start_post_permlink COLLATE "C" ORDER BY id LIMIT 1)
+          ORDER BY
+              hp.parent_author ASC,
+              hp.updated_at ASC,
+              hp.id ASC
+          LIMIT
               _limit
           ;
         END

hive/indexer/posts.py

@@ -50,10 +50,10 @@ class Posts:
         else:
             cls._miss += 1
             sql = """
-                SELECT hp.id 
-                FROM hive_posts hp 
-                INNER JOIN hive_accounts ha_a ON ha_a.id = hp.author_id 
-                INNER JOIN hive_permlink_data hpd_p ON hpd_p.id = hp.permlink_id 
+                SELECT hp.id
+                FROM hive_posts hp
+                INNER JOIN hive_accounts ha_a ON ha_a.id = hp.author_id
+                INNER JOIN hive_permlink_data hpd_p ON hpd_p.id = hp.permlink_id
                 WHERE ha_a.name = :a AND hpd_p.permlink = :p
             """
             _id = DB.query_one(sql, a=author, p=permlink)
@@ -130,16 +130,17 @@ class Posts:
         except Exception:
             pass
 
-        tags = [result['post_category']]
-        if md and 'tags' in md and isinstance(md['tags'], list):
-            tags = tags + md['tags']
-        tags = map(lambda tag: (str(tag) or '').strip('# ').lower()[:32], tags)
-        tags = filter(None, tags)
-        from funcy.seqs import distinct
-        tags = list(distinct(tags))[:5]
+        if not result['depth']:
+            tags = [result['post_category']]
+            if md and 'tags' in md and isinstance(md['tags'], list):
+                tags = tags + md['tags']
+            tags = map(lambda tag: (str(tag) or '').strip('# ').lower()[:32], tags)
+            tags = filter(None, tags)
+            from funcy.seqs import distinct
+            tags = list(distinct(tags))[:5]
 
-        for tag in tags:
-            Tags.add_tag(result['id'], tag)
+            for tag in tags:
+                Tags.add_tag(result['id'], tag)
 
         if not DbState.is_initial_sync():
             if error:
@@ -165,7 +166,7 @@ class Posts:
                   is_paidout            = COALESCE( CAST( data_source.is_paidout as BOOLEAN ),          ihp.is_paidout )
               FROM
               (
-              SELECT  ha_a.id as author_id, hpd_p.id as permlink_id, 
+              SELECT  ha_a.id as author_id, hpd_p.id as permlink_id,
                       t.total_payout_value,
                       t.curator_payout_value,
                       t.author_rewards,
@@ -320,17 +321,17 @@ class Posts:
     def update_child_count(cls, child_id, op='+'):
         """ Increase/decrease child count by 1 """
         sql = """
-            UPDATE 
-                hive_posts 
-            SET 
+            UPDATE
+                hive_posts
+            SET
                 children = GREATEST(0, (
-                    SELECT 
-                        CASE 
+                    SELECT
+                        CASE
                             WHEN children is NULL THEN 0
                             WHEN children=32762 THEN 0
                             ELSE children
                         END
-                    FROM 
+                    FROM
                         hive_posts
                     WHERE id = (SELECT parent_id FROM hive_posts WHERE id = :child_id)
                 )::int
@@ -365,7 +366,7 @@ class Posts:
                 allow_curation_rewards = :allow_curation_rewards,
                 beneficiaries = :beneficiaries
             WHERE
-            hp.author_id = (SELECT id FROM hive_accounts WHERE name = :author) AND 
+            hp.author_id = (SELECT id FROM hive_accounts WHERE name = :author) AND
             hp.permlink_id = (SELECT id FROM hive_permlink_data WHERE permlink = :permlink)
         """
         DB.query(sql, author=op['author'], permlink=op['permlink'], max_accepted_payout=max_accepted_payout,
@@ -389,7 +390,7 @@ class Posts:
             depth = result['depth']
 
             if depth == 0:
-                # TODO: delete from hive_reblogs -- otherwise feed cache gets 
+                # TODO: delete from hive_reblogs -- otherwise feed cache gets
                 # populated with deleted posts somwrimas
                 FeedCache.delete(pid)
 
@@ -442,7 +443,7 @@ class Posts:
             log.info("New body definition: {}".format(new_body_def))
             log.info("Old body definition: {}".format(old_body))
             new_body = new_body_def
-        
+
         return new_body
 
 

hive/server/bridge_api/methods.py

@@ -36,6 +36,7 @@ SQL_TEMPLATE = """
             hp.depth,
             hp.promoted, 
             hp.payout, 
+            hp.pending_payout,
             hp.payout_at, 
             hp.is_paidout, 
             hp.children, 

hive/server/bridge_api/objects.py

@@ -58,6 +58,7 @@ async def load_posts_keyed(db, ids, truncate_body=0):
             hp.depth,
             hp.promoted,
             hp.payout,
+            hp.pending_payout,
             hp.payout_at,
             hp.is_paidout,
             hp.children,
@@ -241,20 +242,19 @@ def _bridge_post_object(row, truncate_body=0):
 
     post['is_paidout'] = row['is_paidout']
     post['payout_at'] = json_date(row['payout_at'])
-    post['payout'] = float(row['payout'])
-    post['pending_payout_value'] = _amount(0 if paid else row['payout'])
-    post['author_payout_value'] = _amount(row['payout'] if paid else 0)
-    post['curator_payout_value'] = _amount(0)
+    post['payout'] = float(row['payout'] + row['pending_payout'])
+    post['pending_payout_value'] = _amount(0 if paid else post['payout'])
+    post['author_payout_value'] = _amount(0) # supplemented below
+    post['curator_payout_value'] = _amount(0) # supplemented below
     post['promoted'] = _amount(row['promoted'])
 
     post['replies'] = []
-# ABW: missing post['active_votes'] = _hydrate_active_votes(row['votes'])
     post['author_reputation'] = float(row['author_rep'])
 
     post['stats'] = {
         'hide': row['is_hidden'],
         'gray': row['is_grayed'],
-        'total_votes': Votes.get_vote_count(row['author'], row['permlink']), # ABW: incorrect calculation (possibly needs to exclude blacklisted votes)
+        'total_votes': Votes.get_vote_count(row['author'], row['permlink']),
         'flag_weight': float(row['flag_weight'])} # TODO: down_weight
 
 

hive/server/bridge_api/thread.py

@@ -38,8 +38,8 @@ async def get_discussion(context, author, permlink, observer=None):
           WHERE NOT children.is_deleted AND NOT children.is_muted
         )
         SELECT
-          child_posts.id,
-          child_posts.parent_id,
+          cp.id,
+          cp.parent_id,
           hpv.id as post_id,
           hpv.author,
           hpv.permlink,
@@ -49,6 +49,7 @@ async def get_discussion(context, author, permlink, observer=None):
           hpv.depth,
           hpv.promoted,
           hpv.payout,
+          hpv.pending_payout,
           hpv.payout_at,
           hpv.is_paidout,
           hpv.children,
@@ -77,9 +78,10 @@ async def get_discussion(context, author, permlink, observer=None):
           hpv.max_accepted_payout,
           hpv.percent_hbd,
           hpv.curator_payout_value
-        FROM child_posts 
-        INNER JOIN hive_posts_view hpv ON (hpv.id = child_posts.id)
+        FROM child_posts cp
+        INNER JOIN hive_posts_view hpv ON (hpv.id = cp.id)
         WHERE NOT hpv.is_deleted AND NOT hpv.is_muted
+        ORDER BY cp.id
         LIMIT 2000
     """
 
@@ -98,26 +100,24 @@ async def get_discussion(context, author, permlink, observer=None):
     root_post['replies'] = []
     all_posts[root_id] = root_post
 
-    id_to_parent_id_map = {}
-    id_to_parent_id_map[root_id] = None
+    parent_to_children_id_map = {}
 
     for index in range(1, len(rows)):
-        id_to_parent_id_map[rows[index]['id']] = rows[index]['parent_id']
+        parent_id = rows[index]['parent_id']
+        if parent_id not in parent_to_children_id_map:
+            parent_to_children_id_map[parent_id] = []
+        parent_to_children_id_map[parent_id].append(rows[index]['id'])
         post = _bridge_post_object(rows[index])
         post['active_votes'] = await find_votes({'db':db}, {'author':rows[index]['author'], 'permlink':rows[index]['permlink']}, VotesPresentation.BridgeApi)
         post = await append_statistics_to_post(post, rows[index], False, blacklists_for_user)
         post['replies'] = []
         all_posts[post['post_id']] = post
 
-    discussion_map = {}
-    build_discussion_map(root_id, id_to_parent_id_map, discussion_map)
-
-    for key in discussion_map:
-        children = discussion_map[key]
-        if children and len(children) > 0:
-            post = all_posts[key]
-            for child_id in children:
-                post['replies'].append(_ref(all_posts[child_id]))
+    for key in parent_to_children_id_map:
+        children = parent_to_children_id_map[key]
+        post = all_posts[key]
+        for child_id in children:
+            post['replies'].append(_ref(all_posts[child_id]))
 
     #result has to be in form of dictionary of dictionaries {post_ref: post}
     results = {}
@@ -126,21 +126,6 @@ async def get_discussion(context, author, permlink, observer=None):
         results[post_ref] = all_posts[key]
     return results
 
-def build_discussion_map(parent_id, posts, results):
-    results[parent_id] = get_children(parent_id, posts)
-    if results[parent_id] == []:
-        return
-
-    for post_id in results[parent_id]:
-        build_discussion_map(post_id, posts, results)
-
-def get_children(parent_id, posts):
-    results = []
-    for key in posts:
-        if posts[key] == parent_id:
-            results.append(key)
-    return results
-
 def _ref(post):
     return post['author'] + '/' + post['permlink']
 

hive/server/condenser_api/methods.py

@@ -34,6 +34,7 @@ SQL_TEMPLATE = """
         hp.promoted,
         hp.payout,
         hp.payout_at,
+        hp.pending_payout,
         hp.is_paidout,
         hp.children,
         hp.votes,

hive/server/condenser_api/objects.py

@@ -52,6 +52,7 @@ async def load_posts_keyed(db, ids, truncate_body=0):
         hp.depth,
         hp.promoted,
         hp.payout,
+        hp.pending_payout,
         hp.payout_at,
         hp.is_paidout,
         hp.children,
@@ -180,6 +181,7 @@ def _condenser_post_object(row, truncate_body=0):
     if not row['category']:
         row['category'] = 'undefined'
 
+    full_payout = row['pending_payout'] + row['payout'];
     post = {}
     post['post_id'] = row['id']
     post['author'] = row['author']
@@ -200,7 +202,7 @@ def _condenser_post_object(row, truncate_body=0):
     post['cashout_time'] = json_date(None if paid else row['payout_at'])
     post['total_payout_value'] = _amount(row['payout'] if paid else 0)
     post['curator_payout_value'] = _amount(0)
-    post['pending_payout_value'] = _amount(0 if paid else row['payout'])
+    post['pending_payout_value'] = _amount(0 if paid else full_payout)
     post['promoted'] = _amount(row['promoted'])
 
     post['replies'] = []

hive/server/database_api/methods.py

@@ -61,14 +61,14 @@ async def list_comments(context, start: list, limit: int, order: str):
         sql = "SELECT * FROM list_comments_by_author_last_update(:author, :updated_at, :start_post_author, :start_post_permlink, :limit)"
         result = await db.query_all(sql, author=author, updated_at=updated_at, start_post_author=start_post_author, start_post_permlink=start_post_permlink, limit=limit)
 
-    return [database_post_object(dict(row)) for row in result]
+    return { "comments": [database_post_object(dict(row)) for row in result] }
 
 @return_error_info
-async def find_comments(context, start: list, limit: int, order: str):
+async def find_comments(context, comments: list):
     """ Search for comments: limit and order is ignored in hive code """
-    comments = []
+    result = []
 
-    assert len(start) <= 1000, "Parameters count is greather than max allowed (1000)"
+    assert len(comments) <= 1000, "Parameters count is greather than max allowed (1000)"
     db = context['db']
 
     SQL_TEMPLATE = """
@@ -115,29 +115,26 @@ async def find_comments(context, start: list, limit: int, order: str):
             hp.reward_weight
         FROM
             hive_posts_view hp
+        JOIN (VALUES {}) AS t (author, permlink) ON hp.author = t.author AND hp.permlink = t.permlink
         WHERE
-            NOT hp.is_muted AND
-            NOT hp.is_deleted AND
+            NOT hp.is_muted AND NOT hp.is_deleted
     """
 
     idx = 0
-    sql = ""
-    for arg in start:
+    values = ""
+    for arg in comments:
         if idx > 0:
-            sql += " UNION ALL "
-        sql += str(SQL_TEMPLATE)
-        sql += """
-            hp.author = '{}' AND 
-            hp.permlink = '{}'
-        """.format(arg[0], arg[1])
+            values += ","
+        values += "('{}','{}')".format(arg[0], arg[1])
         idx += 1
+    sql = SQL_TEMPLATE.format(values)
 
-    result = await db.query_all(sql)
-    for row in result:
+    rows = await db.query_all(sql)
+    for row in rows:
         cpo = database_post_object(dict(row))
-        comments.append(cpo)
+        result.append(cpo)
 
-    return comments
+    return { "comments": result }
 
 class VotesPresentation(Enum):
     ActiveVotes = 1
@@ -214,7 +211,7 @@ async def list_votes(context, start: list, limit: int, order: str):
             hive_votes_accounts_permlinks_view
     """
 
-    if order == "by_comment_voter":
+    if order == "by_comment_voter": # ABW: wrong! fat node sorted by ( comment_id, voter_id )
         sql += """
             WHERE
                 author >= :author AND 
@@ -228,7 +225,7 @@ async def list_votes(context, start: list, limit: int, order: str):
                 :limit
         """
         return await db.query_all(sql, author=start[0], permlink=start[1], voter=start[2], limit=limit)
-    if order == "by_voter_comment":
+    if order == "by_voter_comment": # ABW: wrong! fat node sorted by ( voter_id, comment_id )
         sql += """
             WHERE
                 voter >= :voter AND 

hive/server/database_api/objects.py

@@ -9,12 +9,13 @@ def _amount(amount, asset='HBD'):
 
 def database_post_object(row, truncate_body=0):
     """Given a hive_posts row, create a legacy-style post object."""
+
     paid = row['is_paidout']
 
     post = {}
     post['active'] = json_date(row['active'])
     post['author_rewards'] = row['author_rewards']
-    post['post_id'] = row['id']
+    post['id'] = row['id']
     post['author'] = row['author']
     post['permlink'] = row['permlink']
     post['category'] = row['category'] if 'category' in row else 'undefined'

hive/version.py

 # generated by setup.py
 # contents will be overwritten
 VERSION = '0.0.1'
-GIT_REVISION = 'a1f4d0b'
+GIT_REVISION = '7a4e2b3'

tests_api @ bff5d817

-Subproject commit a88d42a79b9f006466d7a15650ccdc68f56004ea
+Subproject commit bff5d8175f96c1ba576c80e61fc43c9b6fa65ec6