Merge branch 'get_blog_fixes' into 'develop'

Reimplemented get_blog/get_blog_entries with fixes

See merge request !316

hive/db/schema.py

@@ -1600,7 +1600,9 @@ def setup(db):
       "bridge_get_account_posts_by_replies.sql",
       "bridge_get_relationship_between_accounts.sql",
       "bridge_get_post.sql",
+      "condenser_api_post_type.sql",
       "condenser_api_post_ex_type.sql",
+      "condenser_get_blog.sql",
       "condenser_get_content.sql",
       "condenser_get_discussions_by_created.sql",
       "condenser_get_discussions_by_blog.sql",

hive/db/sql_scripts/condenser_api_post_type.sql

+DROP TYPE IF EXISTS condenser_api_post CASCADE;
+-- type for regular condenser_api posts
+CREATE TYPE condenser_api_post AS (
+    id INT,
+    entry_id INT, -- used for paging with offset (otherwise can be any value)
+    author VARCHAR(16),
+    permlink VARCHAR(255),
+    author_rep BIGINT,
+    title VARCHAR(512),
+    body TEXT,
+    category VARCHAR(255),
+    depth SMALLINT,
+    promoted DECIMAL(10,3),
+    payout DECIMAL(10,3),
+    pending_payout DECIMAL(10,3),
+    payout_at TIMESTAMP,
+    is_paidout BOOLEAN,
+    children INT,
+    created_at TIMESTAMP,
+    updated_at TIMESTAMP,
+    reblogged_at TIMESTAMP, -- used when post data is combined with hive_feed_cache (otherwise can be date)
+    rshares NUMERIC,
+    json TEXT,
+    parent_author VARCHAR(16),
+    parent_permlink_or_category VARCHAR(255),
+    curator_payout_value VARCHAR(30),
+    max_accepted_payout VARCHAR(30),
+    percent_hbd INT,
+    beneficiaries JSON,
+    url TEXT,
+    root_title VARCHAR(512)
+);

hive/db/sql_scripts/condenser_get_blog.sql

+DROP FUNCTION IF EXISTS condenser_get_blog_helper CASCADE;
+CREATE FUNCTION condenser_get_blog_helper( in _blogger VARCHAR, in _last INT, in _limit INT,
+                                           out _account_id INT, out _offset INT, out _new_limit INT )
+AS
+$function$
+BEGIN
+  _account_id = find_account_id( _blogger, True );
+  IF _last < 0 THEN -- caller wants "most recent" page
+      SELECT INTO _last ( SELECT COUNT(1) - 1 FROM hive_feed_cache hfc WHERE hfc.account_id = _account_id );
+      _offset = _last - _limit + 1;
+      IF _offset < 0 THEN
+        _offset = 0;
+      END IF;
+      _new_limit = _limit;
+  ELSIF _last + 1 < _limit THEN -- bad call, but recoverable
+      _offset = 0;
+      _new_limit = _last + 1;
+  ELSE -- normal call
+      _offset = _last - _limit + 1;
+      _new_limit = _limit;
+  END IF;
+END
+$function$
+language plpgsql STABLE;
+
+DROP FUNCTION IF EXISTS condenser_get_blog;
+-- blog posts [ _last - _limit + 1, _last ] oldest first (reverted by caller)
+CREATE FUNCTION condenser_get_blog( in _blogger VARCHAR, in _last INT, in _limit INT )
+RETURNS SETOF condenser_api_post
+AS
+$function$
+DECLARE
+  __account_id INT;
+  __offset INT;
+BEGIN
+  SELECT h.* INTO __account_id, __offset, _limit FROM condenser_get_blog_helper( _blogger, _last, _limit ) h;
+  RETURN QUERY SELECT
+      hp.id,
+      blog.entry_id::INT,
+      hp.author,
+      hp.permlink,
+      hp.author_rep,
+      hp.title,
+      hp.body,
+      hp.category,
+      hp.depth,
+      hp.promoted,
+      hp.payout,
+      hp.pending_payout,
+      hp.payout_at,
+      hp.is_paidout,
+      hp.children,
+      hp.created_at,
+      hp.updated_at,
+      (
+        CASE hp.author_id = __account_id
+          WHEN True THEN '1970-01-01T00:00:00'::timestamp
+          ELSE blog.created_at
+        END
+      ) as reblogged_at,
+      hp.rshares,
+      hp.json,
+      hp.parent_author,
+      hp.parent_permlink_or_category,
+      hp.curator_payout_value,
+      hp.max_accepted_payout,
+      hp.percent_hbd,
+      hp.beneficiaries,
+      hp.url,
+      hp.root_title
+  FROM
+  (
+      SELECT
+          hfc.created_at, hfc.post_id, row_number() over (ORDER BY hfc.created_at ASC, hfc.post_id ASC) - 1 as entry_id
+      FROM
+          hive_feed_cache hfc
+      WHERE
+          hfc.account_id = __account_id
+      ORDER BY hfc.created_at ASC, hfc.post_id ASC
+      LIMIT _limit
+      OFFSET __offset
+  ) as blog
+  JOIN hive_posts_view hp ON hp.id = blog.post_id
+  ORDER BY blog.created_at ASC, blog.post_id ASC;
+END
+$function$
+language plpgsql STABLE;
+
+DROP FUNCTION IF EXISTS condenser_get_blog_entries;
+-- blog entries [ _last - _limit + 1, _last ] oldest first (reverted by caller)
+CREATE FUNCTION condenser_get_blog_entries( in _blogger VARCHAR, in _last INT, in _limit INT )
+RETURNS TABLE( entry_id INT, author hive_accounts.name%TYPE, permlink hive_permlink_data.permlink%TYPE, reblogged_at TIMESTAMP )
+AS
+$function$
+DECLARE
+  __account_id INT;
+  __offset INT;
+BEGIN
+  SELECT h.* INTO __account_id, __offset, _limit FROM condenser_get_blog_helper( _blogger, _last, _limit ) h;
+  RETURN QUERY SELECT
+      blog.entry_id::INT,
+      ha.name as author,
+      hpd.permlink,
+      (
+        CASE hp.author_id = __account_id
+          WHEN True THEN '1970-01-01T00:00:00'::timestamp
+          ELSE blog.created_at
+        END
+      ) as reblogged_at
+  FROM
+  (
+      SELECT
+          hfc.created_at, hfc.post_id, row_number() over (ORDER BY hfc.created_at ASC, hfc.post_id ASC) - 1 as entry_id
+      FROM
+          hive_feed_cache hfc
+      WHERE
+          hfc.account_id = __account_id
+      ORDER BY hfc.created_at ASC, hfc.post_id ASC
+      LIMIT _limit
+      OFFSET __offset
+  ) as blog
+  JOIN hive_posts hp ON hp.id = blog.post_id
+  JOIN hive_accounts ha ON ha.id = hp.author_id
+  JOIN hive_permlink_data hpd ON hpd.id = hp.permlink_id
+  ORDER BY blog.created_at ASC, blog.post_id ASC;
+END
+$function$
+language plpgsql STABLE;
+

hive/server/condenser_api/cursor.py

@@ -145,42 +145,6 @@ async def pids_by_blog(db, account: str, start_author: str = '',
     return await db.query_col(sql, account_id=account_id, start_id=start_id, limit=limit)
 
 
-async def pids_by_blog_by_index(db, account: str, start_index: int, limit: int = 20):
-    """Get post_ids for an author's blog (w/ reblogs), paged by index/limit.
-
-    Examples:
-    (acct, 2) = returns blog entries 0 up to 2 (3 oldest)
-    (acct, 0) = returns all blog entries (limit 0 means return all?)
-    (acct, 2, 1) = returns 1 post starting at idx 2
-    (acct, 2, 3) = returns 3 posts: idxs (2,1,0)
-    """
-
-    if start_index in (-1, 0):
-        sql = """SELECT COUNT(*) - 1 FROM hive_posts_view hp
-                  WHERE hp.author = :account"""
-        start_index = await db.query_one(sql, account=account)
-        if start_index < 0:
-            return (0, [])
-        if limit > start_index + 1:
-            limit = start_index + 1
-
-    offset = start_index - limit + 1
-    assert offset >= 0, ('start_index and limit combination is invalid (%d, %d)'
-                         % (start_index, limit))
-
-    sql = """
-        SELECT hp.id
-          FROM hive_posts_view hp
-         WHERE hp.author = :account
-      ORDER BY hp.created_at
-         LIMIT :limit
-        OFFSET :offset
-    """
-
-    ids = await db.query_col(sql, account=account, limit=limit, offset=offset)
-    return (start_index, list(reversed(ids)))
-
-
 async def pids_by_blog_without_reblog(db, account: str, start_permlink: str = '', limit: int = 20):
     """Get a list of post_ids for an author's blog without reblogs."""
 

hive/server/condenser_api/methods.py

@@ -8,6 +8,7 @@ from hive.server.condenser_api.objects import _mute_votes, _condenser_post_objec
 from hive.server.common.helpers import (
     ApiError,
     return_error_info,
+    json_date,
     valid_account,
     valid_permlink,
     valid_tag,
@@ -176,7 +177,7 @@ async def _get_content_impl(db, fat_node_style, author: str, permlink: str, obse
         if not observer:
             post['active_votes'] = _mute_votes(post['active_votes'], Mutes.all())
         else:
-            blacklists_for_user = await Mutes.get_blacklists_for_observer(observer, context)
+            blacklists_for_user = await Mutes.get_blacklists_for_observer(observer, {'db':db})
             post['active_votes'] = _mute_votes(post['active_votes'], blacklists_for_user.keys())
 
     return post
@@ -452,8 +453,43 @@ async def get_blog(context, account: str, start_entry_id: int = 0, limit: int =
     """Get posts for an author's blog (w/ reblogs), paged by index/limit.
 
     Equivalent to get_discussions_by_blog, but uses offset-based pagination.
+
+    Examples: (ABW: old description and examples were misleading as in many cases code worked differently, also now more cases actually work that gave error earlier)
+    (acct, -1, limit) for limit 1..500 - returns latest (no more than) limit posts
+    (acct, 0) - returns latest single post (ABW: this is a bug but I left it here because I'm afraid it was actively used - it should return oldest post)
+    (acct, 0, limit) for limit 1..500 - same as (acct, -1, limit) - see above
+    (acct, last_idx) for positive last_idx - returns last_idx oldest posts, or posts in range [last_idx..last_idx-500) when last_idx >= 500
+    (acct, last_idx, limit) for positive last_idx and limit 1..500 - returns posts in range [last_idx..last_idx-limit)
     """
-    return await _get_blog(context['db'], account, start_entry_id, limit)
+    db = context['db']
+
+    account = valid_account(account)
+    if not start_entry_id:
+        start_entry_id = -1
+    start_entry_id = valid_offset(start_entry_id)
+    if not limit:
+        limit = max(start_entry_id + 1, 1)
+        limit = min(limit, 500)
+    limit = valid_limit(limit, 500, None)
+
+    sql = "SELECT * FROM condenser_get_blog(:account, :last, :limit)"
+    result = await db.query_all(sql, account=account, last=start_entry_id, limit=limit)
+
+    muted_accounts = Mutes.all()
+    out = []
+    for row in result:
+        row = dict(row)
+        post = _condenser_post_object(row)
+
+        post['active_votes'] = await find_votes_impl(db, row['author'], row['permlink'], VotesPresentation.CondenserApi)
+        post['active_votes'] = _mute_votes(post['active_votes'], muted_accounts)
+
+        out.append({"blog": account,
+                    "entry_id": row['entry_id'],
+                    "comment": post,
+                    "reblogged_on": json_date(row['reblogged_at'])})
+
+    return list(reversed(out))
 
 @return_error_info
 @nested_query_compat
@@ -462,53 +498,30 @@ async def get_blog_entries(context, account: str, start_entry_id: int = 0, limit
 
     Interface identical to get_blog, but returns minimalistic post references.
     """
+    db = context['db']
 
-    entries = await _get_blog(context['db'], account, start_entry_id, limit)
-    for entry in entries:
-        # replace the comment body with just author/permlink
-        post = entry.pop('comment')
-        entry['author'] = post['author']
-        entry['permlink'] = post['permlink']
-
-    return entries
-
-async def _get_blog(db, account: str, start_index: int, limit: int = None):
-    """Get posts for an author's blog (w/ reblogs), paged by index/limit.
-
-    Examples:
-    (acct, 2) = returns blog entries 0 up to 2 (3 oldest)
-    (acct, 0) = returns all blog entries (limit 0 means return all?)
-    (acct, 2, 1) = returns 1 post starting at idx 2
-    (acct, 2, 3) = returns 3 posts: idxs (2,1,0)
-    (acct, -1, 10) = returns latest 10 posts
-    """
-
-    if start_index is None:
-        start_index = 0
-
+    account = valid_account(account)
+    if not start_entry_id:
+        start_entry_id = -1
+    start_entry_id = valid_offset(start_entry_id)
     if not limit:
-        limit = start_index + 1
+        limit = max(start_entry_id + 1, 1)
+        limit = min(limit, 500)
+    limit = valid_limit(limit, 500, None)
 
-    start_index, ids = await cursor.pids_by_blog_by_index(
-        db,
-        valid_account(account),
-        valid_offset(start_index),
-        valid_limit(limit, 500, None))
+    sql = "SELECT * FROM condenser_get_blog_entries(:account, :last, :limit)"
+    result = await db.query_all(sql, account=account, last=start_entry_id, limit=limit)
 
     out = []
-
-    idx = int(start_index)
-    for post in await load_posts(db, ids):
-        reblog = post['author'] != account
-        reblog_on = post['created'] if reblog else "1970-01-01T00:00:00"
-
+    for row in result:
+        row = dict(row)
         out.append({"blog": account,
-                    "entry_id": idx,
-                    "comment": post,
-                    "reblogged_on": reblog_on})
-        idx -= 1
+                    "entry_id": row['entry_id'],
+                    "author": row['author'],
+                    "permlink": row['permlink'],
+                    "reblogged_on": json_date(row['reblogged_at'])})
 
-    return out
+    return list(reversed(out))
 
 @return_error_info
 async def get_active_votes(context, author: str, permlink: str):

hive/server/condenser_api/objects.py

@@ -87,6 +87,7 @@ async def load_posts_keyed(db, ids, truncate_body=0):
         post = _condenser_post_object(row, truncate_body=truncate_body)
 
         post['active_votes'] = await find_votes_impl(db, row['author'], row['permlink'], VotesPresentation.CondenserApi)
+        post['active_votes'] = _mute_votes(post['active_votes'], muted_accounts)
         posts_by_id[row['id']] = post
 
     return posts_by_id

tests_api @ 3f308b5d

-Subproject commit 3d3daf0c67b9d429be51b2d66543a57c0f8fcf29
+Subproject commit 3f308b5d98b924cde5e33c65ae64ba96cb3c786d