Skip to content
Snippets Groups Projects
Commit 4624142e authored by Gandalf's avatar Gandalf
Browse files

[JES] Update sql queries for get_post and get_ranked_posts to be more efficient

parent dcea6ae7
No related branches found
No related tags found
5 merge requests!456Release candidate v1 24,!230Setup monitoring with pghero,!135Enable postgres monitoring on CI server,!16Dk issue 3 concurrent block query rebase,!15Dk issue 3 concurrent block query
Hide whitespace changes
Inline Side-by-side
hive/server/bridge_api/methods.py
+ 184
− 21
View file @ 4624142e
"""Bridge API public endpoints for posts""" """Bridge API public endpoints for posts"""
import hive.server.bridge_api.cursor as cursor import hive.server.bridge_api.cursor as cursor
from hive.server.bridge_api.objects import load_posts, load_posts_reblogs, load_profiles from hive.server.bridge_api.objects import load_posts, load_posts_reblogs, load_profiles, _condenser_post_object
from hive.server.common.helpers import ( from hive.server.common.helpers import (
return_error_info, return_error_info,
valid_account, valid_account,
...@@ -11,6 +11,10 @@ from hive.server.common.helpers import ( ...@@ -11,6 +11,10 @@ from hive.server.common.helpers import (
from hive.server.hive_api.common import get_account_id from hive.server.hive_api.common import get_account_id
from hive.server.hive_api.objects import _follow_contexts from hive.server.hive_api.objects import _follow_contexts
from hive.server.hive_api.community import list_top_communities from hive.server.hive_api.community import list_top_communities
from hive.server.common.mutes import Mutes
ROLES = {-2: 'muted', 0: 'guest', 2: 'member', 4: 'mod', 6: 'admin', 8: 'owner'}
#pylint: disable=too-many-arguments, no-else-return #pylint: disable=too-many-arguments, no-else-return
...@@ -61,35 +65,194 @@ async def get_post(context, author, permlink, observer=None): ...@@ -61,35 +65,194 @@ async def get_post(context, author, permlink, observer=None):
# pylint: disable=unused-variable # pylint: disable=unused-variable
#TODO: `observer` logic for user-post state #TODO: `observer` logic for user-post state
db = context['db'] db = context['db']
observer_id = await get_account_id(db, observer) if observer else None valid_account(author)
pid = await _get_post_id(db, valid_permlink(permlink)
valid_account(author), #observer_id = await get_account_id(db, observer) if observer else None
valid_permlink(permlink)) #pid = await _get_post_id(db,
posts = await load_posts(db, [pid]) # valid_account(author),
assert len(posts) == 1, 'cache post not found' # valid_permlink(permlink))
return posts[0] #posts = await load_posts(db, [pid])
#assert len(posts) == 1, 'cache post not found'
#return posts[0]
sql = """
SELECT hive_posts_cache.post_id, hive_posts_cache.community_id, hive_posts_cache.author, hive_posts_cache.permlink, hive_posts_cache.title, hive_posts_cache.body,
hive_posts_cache.category, hive_posts_cache.depth,
hive_posts_cache.promoted, hive_posts_cache.payout, hive_posts_cache.payout_at, hive_posts_cache.is_paidout, hive_posts_cache.children, hive_posts_cache.votes,
hive_posts_cache.created_at, hive_posts_cache.updated_at, hive_posts_cache.rshares, hive_posts_cache.raw_json, hive_posts_cache.json,
hive_posts_cache.is_hidden, hive_posts_cache.is_grayed, hive_posts_cache.total_votes, hive_posts_cache.flag_weight, hive_accounts.reputation AS author_rep
FROM hive_posts_cache JOIN hive_posts on (hive_posts_cache.post_id = hive_posts.id)
JOIN hive_accounts on (hive_posts_cache.author = hive_accounts.name)
WHERE hive_posts_cache.author = :author AND hive_posts_cache.permlink=:permlink AND NOT hive_posts.is_deleted;
"""
result = await db.query_all(sql, author=author, permlink=permlink)
assert len(result) == 1, 'invalid author/permlink or post not found in cache'
post = _condenser_post_object(result[0])
post['blacklists'] = Mutes.lists(post['author'], result[0]['author_rep'])
return post
@return_error_info @return_error_info
async def get_ranked_posts(context, sort, start_author='', start_permlink='', async def get_ranked_posts(context, sort, start_author='', start_permlink='',
limit=20, tag=None, observer=None): limit=20, tag=None, observer=None):
"""Query posts, sorted by given method.""" """Query posts, sorted by given method."""
db = context['db']
observer_id = await get_account_id(db, observer) if observer else None
assert sort in ['trending', 'hot', 'created', 'promoted', assert sort in ['trending', 'hot', 'created', 'promoted',
'payout', 'payout_comments', 'muted'], 'invalid sort' 'payout', 'payout_comments', 'muted'], 'invalid sort'
ids = await cursor.pids_by_ranked(
context['db'], valid_account(start_author, allow_empty=True)
sort, valid_permlink(start_permlink, allow_empty=True)
valid_account(start_author, allow_empty=True), valid_limit(limit, 100)
valid_permlink(start_permlink, allow_empty=True), valid_tag(tag, allow_empty=True)
valid_limit(limit, 100),
valid_tag(tag, allow_empty=True), db = context['db']
observer_id)
select_fragment = """
return await load_posts(context['db'], ids) SELECT hive_posts_cache.post_id, hive_posts_cache.author, hive_posts_cache.permlink,
hive_posts_cache.title, hive_posts_cache.body, hive_posts_cache.category, hive_posts_cache.depth,
hive_posts_cache.promoted, hive_posts_cache.payout, hive_posts_cache.payout_at,
hive_posts_cache.is_paidout, hive_posts_cache.children, hive_posts_cache.votes,
hive_posts_cache.created_at, hive_posts_cache.updated_at, hive_posts_cache.rshares,
hive_posts_cache.raw_json, hive_posts_cache.json, hive_accounts.reputation AS author_rep,
hive_posts_cache.is_hidden AS is_hidden, hive_posts_cache.is_grayed AS is_grayed,
hive_posts_cache.total_votes AS total_votes, hive_posts_cache.flag_weight AS flag_weight,
hive_posts_cache.sc_trend AS sc_trend, hive_accounts.id AS acct_author_id,
hive_roles.title as role_title, hive_communities.title AS community_title, hive_roles.role_id AS role_id,
hive_posts.is_pinned AS is_pinned,
FROM hive_posts_cache JOIN hive_posts ON (hive_posts_cache.post_id = hive_posts.id)
JOIN hive_accounts ON (hive_posts_cache.author = hive_accounts.name)
LEFT OUTER JOIN hive_communities ON (hive_posts_cache.community_id = hive_communities.id)
LEFT OUTER JOIN hive_roles ON (hive_accounts.id = hive_roles.account_id AND hive_posts_cache.community_id = hive_roles.community_id) """
ranked_by_trending_sql = select_fragment + """ WHERE NOT hive_posts_cache.is_paidout AND hive_posts_cache.depth = 0 AND NOT hive_posts.is_deleted
%s ORDER BY sc_trend desc, post_id LIMIT :limit """
ranked_by_hot_sql = select_fragment + """ WHERE NOT hive_posts_cache.is_paidout AND hive_posts_cache.depth = 0 AND NOT hive_posts.is_deleted
%s ORDER BY sc_hot desc, post_id LIMIT :limit """
ranked_by_created_sql = select_fragment + """ WHERE hive_posts_cache.depth = 0 AND NOT hive_posts.is_deleted
%s ORDER BY hive_posts_cache.created_at DESC, post_id LIMIT :limit """
ranked_by_promoted_sql = select_fragment + """ WHERE hive_posts_cache.depth > 0 AND hive_posts_cache.promoted > 0 AND NOT hive_posts.is_deleted
AND NOT hive_posts_cache.is_paidout %s ORDER BY hive_posts_cache.promoted DESC, post_id LIMIT :limit """
ranked_by_payout_sql = select_fragment + """ WHERE NOT hive_posts_cache.is_paidout AND NOT hive_posts.is_deleted %s
AND payout_at BETWEEN now() + interval '12 hours' AND now() + interval '36 hours'
ORDER BY hive_posts_cache.payout DESC, post_id LIMIT :limit """
ranked_by_payout_comments_sql = select_fragment + """ WHERE NOT hive_posts_cache.is_paidout AND NOT hive_posts.is_deleted AND hive_posts_cache.depth > 0
%s ORDER BY hive_posts_cache.payout DESC, post_id LIMIT :limit """
ranked_by_muted_sql = select_fragment + """ WHERE NOT hive_posts_cache.is_paidout AND NOT hive_posts.is_deleted AND hive_posts_cache.is_grayed
AND hive_posts_cache.payout > 0 %s ORDER BY hive_posts_cache.payout DESC, post_id LIMIT :limit """
sql = '';
if sort == 'trending':
sql = ranked_by_trending_sql
elif sort == 'hot':
sql = ranked_by_hot_sql
elif sort == 'created':
sql = ranked_by_created_sql
elif sort == 'promoted':
sql = ranked_by_promoted_sql
elif sort == 'payout':
sql = ranked_by_payout_sql
elif sort == 'payout_comments':
sql = ranked_by_payout_comments_sql
elif sort == 'muted':
sql = ranked_by_muted_sql
if not tag or tag == 'all':
if start_author and start_permlink:
if sort == 'trending':
sql = sql % """ AND hive_posts_cache.sc_trend <= (SELECT sc_trend FROM hive_posts_cache WHERE permlink = :permlink AND author = :author)
AND hive_posts_cache.post_id != (SELECT post_id FROM hive_posts_cache WHERE permlink = :permlink AND author=:author) """
elif sort == 'hot':
sql = sql % """ AND hive_posts_cache.sc_hot <= (SELECT sc_hot FROM hive_posts_cache WHERE permlink = :permlink AND author = :author)
AND hive_posts_cache.post_id != (SELECT post_id FROM hive_posts_cache WHERE permlink = :permlink AND author = :author) """
elif sort == 'created':
sql = sql % """ AND hive_posts_cache.post_id < (SELECT post_id FROM hive_posts_cache WHERE permlink = :permlink AND author = :author) """
elif sort == 'promoted':
sql = sql % """ AND hive_posts_cache.promoted <= (SELECT promoted FROM hive_posts_cache WHERE permlink = :permlink AND author = :author)
AND hive_posts_cache.post_id != (SELECT post_id FROM hive_posts_cache WHERE permlink = :permlink AND author = :author) """
else:
sql = sql % """ AND hive_posts_cache.payout <= (SELECT payout FROM hive_posts_cache WHERE permlink = :permlink AND author = :author)
AND hive_posts_cache.post_id != (SELECT post_id FROM hive_posts_cache WHERE permlink = :permlink AND author = :author) """
else:
sql = sql % """"""
elif tag == 'my':
if start_author and start_permlink:
if sort == 'trending':
sql = sql % """ AND hive_posts_cache.community_id IN (SELECT community_id FROM hive_roles WHERE account_id = hive_accounts.id )
AND hive_posts_cache.sc_trend <= (SELECT sc_trend FROM hive_posts_cache WHERE permlink = :permlink AND author = :author )
AND hive_posts_cache.post_id != (SELECT post_id FROM hive_posts_cache WHERE permlink = :permlink AND author = :author) """
elif sort == 'hot':
sql = sql % """ AND hive_posts_cache.community_id IN (SELECT community_id FROM hive_roles WHERE account_id = hive_accounts.id)
AND hive_posts_cache.sc_hot <= (SELECT sc_hot FROM hive_posts_cache WHERE permlink = :permlink AND author = :author)
AND hive_posts_cache.post_id != (SELECT post_id FROM hive_posts_cache WHERE permlink = :permlink AND author = :author) """
elif sort == 'created':
sql = sql % """ AND hive_posts_cache.community_id IN (SELECT community_id FROM hive_roles WHERE account_id = hive_accounts.id )
AND hive_posts_cache.post_id < (SELECT post_id FROM hive_posts_cache WHERE permlink = :permlink AND author = :author ) """
elif sort == 'promoted':
sql = sql % """ AND hive_posts_cache.community_id IN (SELECT community_id FROM hive_roles WHERE account_id = hive_accounts.id )
AND hive_posts_cache.promoted <= (SELECT promoted FROM hive_posts_cache WHERE permlink = :permlink AND author = :author )
AND hive_posts_cache.post_id != (SELECT post_id FROM hive_posts_cache WHERE permlink = :permlink AND author = :author) """
else:
sql = sql % """ AND hive_posts_cache.community_id IN (SELECT community_id FROM hive_roles WHERE account_id = hive_accounts.id )
AND hive_posts_cache.payout <= (SELECT payout FROM hive_posts_cache WHERE permlink = :permlink AND author = :author)
AND hive_posts_cache.post_id != (SELECT post_id FROM hive_posts_cache WHERE permlink = :permlink AND author = :author ) """
else:
sql = sql % """ AND hive_posts_cache.community_id IN (SELECT community_id FROM hive_roles WHERE account_id = hive_accounts.id ) """
elif tag[:5] == 'hive-':
if start_author and start_permlink:
if sort == 'trending':
sql = sql % """ AND hive_posts_cache.community_id = (SELECT community_id FROM communities WHERE name = :community_name )
AND hive_posts_cache.sc_trend <= (SELECT sc_trend FROM hive_posts_cache WHERE permlink = :permlink AND author = :author)
AND hive_posts_cache.post_id != (SELECT post_id FROM hive_posts_cache WHERE permlink = :permlink AND author = :author) """
elif sort == 'hot':
sql = sql % """ AND hive_posts_cache.community_id = (SELECT community_id FROM communities WHERE name = :community_name )
AND hive_posts_cache.sc_hot <= (SELECT sc_hot FROM hive_posts_cache WHERE permlink = :permlink AND author = :author)
AND hive_posts_cache.post_id != (SELECT post_id FROM hive_posts_cache WHERE permlink = :permlink AND author = :author) """
elif sort == 'created':
sql = sql % """ AND hive_posts_cache.community_id = (SELECT community_id FROM communities WHERE name = :community_name )
AND hive_posts_cache.post_id < (SELECT post_id FROM hive_posts_cache WHERE permlink = :permlink AND author = :author) """
elif sort == 'promoted':
sql = sql % """ AND hive_posts_cache.community_id = (SELECT community_id FROM communities WHERE name = :community_name )
AND hive_posts_cache.promoted <= (SELECT promoted FROM hive_posts_cache WHERE permlink = :permlink AND author = :author)
AND hive_posts_cache.post_id != (SELECT post_id FROM hive_posts_cache WHERE permlink = :permlink AND author = :author) """
else:
sql = sql % """ AND hive_posts_cache.community_id = (SELECT community_id FROM communities WHERE name = :community_name )
AND hive_posts_cache.payout <= (SELECT payout FROM hive_posts_cache WHERE permlink = :permlink AND author = :author)
AND hive_posts_cache.post_id != (SELECT post_id FROM hive_posts_cache WHERE permlink = :permlink AND author = :author) """
else:
sql = sql % """ AND hive_communities.name = :community_name """
else:
if sort in ['payout', 'payout_comments']:
sql = sql % """ AND hive_posts_cache.category = :tag"""
else:
sql = sql % """ AND hive_posts_cache.post_id IN (SELECT post_id FROM hive_post_tags WHERE tag = :tag)"""
sql_result = await db.query_all(sql, author=start_author, limit=limit, tag=tag, permlink=start_permlink, community_name=tag)
posts = []
for row in sql_result:
post = _condenser_post_object(row)
post['blacklists'] = Mutes.lists(row['author'], row['author_rep'])
if 'community_title' in row and row['community_title']:
post['community'] = row['category']
post['community_title'] = row['community_title']
if row['role_id']:
post['author_role'] = ROLES[row['role_id']]
post['author_title'] = row['role_title']
else:
post['author_role'] = 'guest'
post['author_title'] = ''
else:
post['stats']['gray'] = row['is_grayed']
post['stats']['hide'] = 'irredeemables' in post['blacklists']
if 'is_pinned' in row and row['is_pinned']:
post['stats']['is_pinned'] = True
posts.append(post)
return posts
@return_error_info @return_error_info
async def get_account_posts(context, sort, account, start_author='', start_permlink='', async def get_account_posts(context, sort, account, start_author='', start_permlink='',
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment