From dac76894b6e3472bf852377f631889505478d26f Mon Sep 17 00:00:00 2001
From: Dariusz Kedzierski <dkedzierski@syncad.com>
Date: Wed, 3 Jun 2020 18:19:00 +0200
Subject: [PATCH] [WIP] Works on server/sync
---
hive/db/schema.py | 173 ++++++++---------
hive/indexer/blocks.py | 2 +-
hive/indexer/cached_post.py | 1 -
hive/indexer/community.py | 5 +-
hive/indexer/posts.py | 18 +-
hive/server/bridge_api/cursor.py | 32 ++--
hive/server/bridge_api/objects.py | 61 ++++--
hive/server/common/objects.py | 6 +-
hive/server/condenser_api/cursor.py | 66 +++++--
hive/server/condenser_api/methods.py | 174 +++++++++++-------
hive/server/condenser_api/objects.py | 62 +++++--
hive/server/condenser_api/tags.py | 6 +-
hive/server/database_api/methods.py | 155 +++++++++++-----
hive/server/hive_api/community.py | 16 +-
hive/server/hive_api/objects.py | 43 ++++-
hive/server/hive_api/thread.py | 10 +-
hive/server/serve.py | 3 +-
hive/utils/post.py | 1 -
.../manual_tests/list_comments_by_permlink.py | 2 +-
tests/server/test_server_database_api.py | 8 +-
20 files changed, 527 insertions(+), 317 deletions(-)
diff --git a/hive/db/schema.py b/hive/db/schema.py
index 634339441..183bdbaff 100644
--- a/hive/db/schema.py
+++ b/hive/db/schema.py
@@ -10,7 +10,8 @@ from sqlalchemy.types import BOOLEAN
#pylint: disable=line-too-long, too-many-lines, bad-whitespace
-DB_VERSION = 17
+# [DK] we changed and removed some tables so i upgraded DB_VERSION to 18
+DB_VERSION = 18
def build_metadata():
"""Build schema def with SqlAlchemy"""
@@ -72,9 +73,9 @@ def build_metadata():
'hive_posts', metadata,
sa.Column('id', sa.Integer, primary_key=True),
sa.Column('parent_id', sa.Integer),
- sa.Column('author', VARCHAR(16), nullable=False),
- sa.Column('permlink', VARCHAR(255), nullable=False),
- sa.Column('category', VARCHAR(255), nullable=False, server_default=''),
+ sa.Column('author_id', sa.Integer, nullable=False),
+ sa.Column('permlink_id', sa.Integer, nullable=False),
+ sa.Column('category_id', sa.Integer, nullable=False),
sa.Column('community_id', sa.Integer, nullable=True),
sa.Column('created_at', sa.DateTime, nullable=False),
sa.Column('depth', SMALLINT, nullable=False),
@@ -84,13 +85,79 @@ def build_metadata():
sa.Column('is_valid', BOOLEAN, nullable=False, server_default='1'),
sa.Column('promoted', sa.types.DECIMAL(10, 3), nullable=False, server_default='0'),
- sa.ForeignKeyConstraint(['author'], ['hive_accounts.name'], name='hive_posts_fk1'),
+ sa.Column('children', SMALLINT, nullable=False, server_default='0'),
+
+ # basic/extended-stats
+ sa.Column('author_rep', sa.Float(precision=6), nullable=False, server_default='0'),
+ sa.Column('flag_weight', sa.Float(precision=6), nullable=False, server_default='0'),
+ sa.Column('total_votes', sa.Integer, nullable=False, server_default='0'),
+ sa.Column('up_votes', sa.Integer, nullable=False, server_default='0'),
+
+ # core stats/indexes
+ sa.Column('payout', sa.types.DECIMAL(10, 3), nullable=False, server_default='0'),
+ sa.Column('payout_at', sa.DateTime, nullable=False, server_default='1990-01-01'),
+ sa.Column('updated_at', sa.DateTime, nullable=False, server_default='1990-01-01'),
+ sa.Column('is_paidout', BOOLEAN, nullable=False, server_default='0'),
+
+ # ui flags/filters
+ sa.Column('is_nsfw', BOOLEAN, nullable=False, server_default='0'),
+ sa.Column('is_declined', BOOLEAN, nullable=False, server_default='0'),
+ sa.Column('is_full_power', BOOLEAN, nullable=False, server_default='0'),
+ sa.Column('is_hidden', BOOLEAN, nullable=False, server_default='0'),
+ sa.Column('is_grayed', BOOLEAN, nullable=False, server_default='0'),
+
+ # important indexes
+ sa.Column('rshares', sa.BigInteger, nullable=False, server_default='0'),
+ sa.Column('sc_trend', sa.Float(precision=6), nullable=False, server_default='0'),
+ sa.Column('sc_hot', sa.Float(precision=6), nullable=False, server_default='0'),
+
+ sa.Column('parent_author_id', sa.Integer, nullable=False),
+ sa.Column('parent_permlink_id', sa.Integer, nullable=False),
+ sa.Column('curator_payout_value', sa.String(16), nullable=False, server_default=''),
+ sa.Column('root_author_id', sa.Integer, nullable=False),
+ sa.Column('root_permlink_id', sa.Integer, nullable=False),
+ sa.Column('max_accepted_payout', sa.String(16), nullable=False, server_default=''),
+ sa.Column('percent_steem_dollars', sa.Integer, nullable=False, server_default='-1'),
+ sa.Column('allow_replies', BOOLEAN, nullable=False, server_default='1'),
+ sa.Column('allow_votes', BOOLEAN, nullable=False, server_default='1'),
+ sa.Column('allow_curation_rewards', BOOLEAN, nullable=False, server_default='1'),
+ sa.Column('beneficiaries', sa.JSON, nullable=False, server_default='[]'),
+ sa.Column('url', sa.Text, nullable=False, server_default=''),
+ sa.Column('root_title', sa.String(255), nullable=False, server_default=''),
+
+ sa.ForeignKeyConstraint(['author_id'], ['hive_accounts.id'], name='hive_posts_fk1'),
sa.ForeignKeyConstraint(['parent_id'], ['hive_posts.id'], name='hive_posts_fk3'),
sa.UniqueConstraint('author', 'permlink', name='hive_posts_ux1'),
sa.Index('hive_posts_ix3', 'author', 'depth', 'id', postgresql_where=sql_text("is_deleted = '0'")), # API: author blog/comments
sa.Index('hive_posts_ix4', 'parent_id DESC NULLS LAST', 'id'), #postgresql_where=sql_text("is_deleted = '0'")), # API: fetching children #[JES] We decided we want the full index since posts can be deleted/undeleted
sa.Index('hive_posts_ix5', 'id', postgresql_where=sql_text("is_pinned = '1' AND is_deleted = '0'")), # API: pinned post status
sa.Index('hive_posts_ix6', 'community_id', 'id', postgresql_where=sql_text("community_id IS NOT NULL AND is_pinned = '1' AND is_deleted = '0'")), # API: community pinned
+ sa.UniqueConstraint('author_id', 'permlink_id', name='hive_posts_ux1'),
+ )
+
+ sa.Table(
+ 'hive_post_data', metadata,
+ sa.column('id', sa.Integer, nullable=False),
+ sa.column('title', VARCHAR(255), nullable=False),
+ sa.column('preview', VARCHAR(1024), nullable=False),
+ sa.column('img_url', VARCHAR(1024), nullable=False),
+ sa.Column('body', TEXT),
+ sa.Column('votes', TEXT),
+ sa.Column('json', sa.JSON)
+ )
+
+ sa.Table(
+ 'hive_permlink_data', metadata,
+ sa.column('id', sa.Integer, primary_key=True),
+ sa.column('permlink', sa.String(255), nullable=False),
+ sa.UniqueConstraint('permlink', name='hive_permlink_data_permlink')
+ )
+
+ sa.Table(
+ 'hive_category_data', metadata,
+ sa.column('id', sa.Integer, primary_key=True),
+ sa.column('category', sa.String(255), nullable=False),
+ sa.UniqueConstraint('category', name='hive_category_data_category')
)
sa.Table(
@@ -150,101 +217,6 @@ def build_metadata():
sa.Index('hive_feed_cache_ix1', 'account_id', 'post_id', 'created_at'), # API (and rebuild?)
)
- sa.Table(
- 'hive_posts_cache', metadata,
- sa.Column('post_id', sa.Integer, primary_key=True, autoincrement=False),
- sa.Column('author', VARCHAR(16), nullable=False),
- sa.Column('permlink', VARCHAR(255), nullable=False),
- sa.Column('category', VARCHAR(255), nullable=False, server_default=''),
-
- # important/index
- sa.Column('community_id', sa.Integer, nullable=True),
- sa.Column('depth', SMALLINT, nullable=False, server_default='0'),
- sa.Column('children', SMALLINT, nullable=False, server_default='0'),
-
- # basic/extended-stats
- sa.Column('author_rep', sa.Float(precision=6), nullable=False, server_default='0'),
- sa.Column('flag_weight', sa.Float(precision=6), nullable=False, server_default='0'),
- sa.Column('total_votes', sa.Integer, nullable=False, server_default='0'),
- sa.Column('up_votes', sa.Integer, nullable=False, server_default='0'),
-
- # basic ui fields
- sa.Column('title', sa.String(255), nullable=False, server_default=''),
- sa.Column('preview', sa.String(1024), nullable=False, server_default=''),
- sa.Column('img_url', sa.String(1024), nullable=False, server_default=''),
-
- # core stats/indexes
- sa.Column('payout', sa.types.DECIMAL(10, 3), nullable=False, server_default='0'),
- sa.Column('promoted', sa.types.DECIMAL(10, 3), nullable=False, server_default='0'),
- sa.Column('created_at', sa.DateTime, nullable=False, server_default='1990-01-01'),
- sa.Column('payout_at', sa.DateTime, nullable=False, server_default='1990-01-01'),
- sa.Column('updated_at', sa.DateTime, nullable=False, server_default='1990-01-01'),
- sa.Column('is_paidout', BOOLEAN, nullable=False, server_default='0'),
-
- # ui flags/filters
- sa.Column('is_nsfw', BOOLEAN, nullable=False, server_default='0'),
- sa.Column('is_declined', BOOLEAN, nullable=False, server_default='0'),
- sa.Column('is_full_power', BOOLEAN, nullable=False, server_default='0'),
- sa.Column('is_hidden', BOOLEAN, nullable=False, server_default='0'),
- sa.Column('is_grayed', BOOLEAN, nullable=False, server_default='0'),
-
- # important indexes
- sa.Column('rshares', sa.BigInteger, nullable=False, server_default='0'),
- sa.Column('sc_trend', sa.Float(precision=6), nullable=False, server_default='0'),
- sa.Column('sc_hot', sa.Float(precision=6), nullable=False, server_default='0'),
-
- # bulk data
- sa.Column('body', TEXT),
- sa.Column('votes', TEXT),
- sa.Column('json', sa.Text),
- #sa.Column('raw_json', sa.Text),
-
- sa.Column('legacy_id', sa.Integer, nullable=False, server_default='-1'),
- sa.Column('parent_author', sa.String(16), nullable=False, server_default=''),
- sa.Column('parent_permlink', sa.String(255), nullable=False, server_default=''),
- sa.Column('curator_payout_value', sa.String(16), nullable=False, server_default=''),
- sa.Column('root_author', sa.String(16), nullable=False, server_default=''),
- sa.Column('root_permlink', sa.String(255), nullable=False, server_default=''),
- sa.Column('max_accepted_payout', sa.String(16), nullable=False, server_default=''),
- sa.Column('percent_steem_dollars', sa.Integer, nullable=False, server_default='-1'),
- sa.Column('allow_replies', BOOLEAN, nullable=False, server_default='1'),
- sa.Column('allow_votes', BOOLEAN, nullable=False, server_default='1'),
- sa.Column('allow_curation_rewards', BOOLEAN, nullable=False, server_default='1'),
- sa.Column('beneficiaries', sa.JSON, nullable=False, server_default=''),
- sa.Column('url', sa.Text, nullable=False, server_default=''),
- sa.Column('root_title', sa.String(255), nullable=False, server_default=''),
-
- sa.Column('author_permlink', sa.String(255 + 16, collation='C'), nullable=False, server_default=''),
-
- # index: misc
- sa.Index('hive_posts_cache_ix3', 'payout_at', 'post_id', postgresql_where=sql_text("is_paidout = '0'")), # core: payout sweep
- sa.Index('hive_posts_cache_ix8', 'category', 'payout', 'depth', postgresql_where=sql_text("is_paidout = '0'")), # API: tag stats
-
- # index: ranked posts
- sa.Index('hive_posts_cache_ix2', 'promoted', postgresql_where=sql_text("is_paidout = '0' AND promoted > 0")), # API: promoted
-
- sa.Index('hive_posts_cache_ix6a', 'sc_trend', 'post_id', postgresql_where=sql_text("is_paidout = '0'")), # API: trending todo: depth=0
- sa.Index('hive_posts_cache_ix7a', 'sc_hot', 'post_id', postgresql_where=sql_text("is_paidout = '0'")), # API: hot todo: depth=0
- sa.Index('hive_posts_cache_ix6b', 'post_id', 'sc_trend', postgresql_where=sql_text("is_paidout = '0'")), # API: trending, filtered todo: depth=0
- sa.Index('hive_posts_cache_ix7b', 'post_id', 'sc_hot', postgresql_where=sql_text("is_paidout = '0'")), # API: hot, filtered todo: depth=0
-
- sa.Index('hive_posts_cache_ix9a', 'depth', 'payout', 'post_id', postgresql_where=sql_text("is_paidout = '0'")), # API: payout todo: rem depth
- sa.Index('hive_posts_cache_ix9b', 'category', 'depth', 'payout', 'post_id', postgresql_where=sql_text("is_paidout = '0'")), # API: payout, filtered todo: rem depth
-
- sa.Index('hive_posts_cache_ix10', 'post_id', 'payout', postgresql_where=sql_text("is_grayed = '1' AND payout > 0")), # API: muted, by filter/date/payout
-
- # index: stats
- sa.Index('hive_posts_cache_ix20', 'community_id', 'author', 'payout', 'post_id', postgresql_where=sql_text("is_paidout = '0'")), # API: pending distribution; author payout
-
- # index: community ranked posts
- sa.Index('hive_posts_cache_ix30', 'community_id', 'sc_trend', 'post_id', postgresql_where=sql_text("community_id IS NOT NULL AND is_grayed = '0' AND depth = 0")), # API: community trend
- sa.Index('hive_posts_cache_ix31', 'community_id', 'sc_hot', 'post_id', postgresql_where=sql_text("community_id IS NOT NULL AND is_grayed = '0' AND depth = 0")), # API: community hot
- sa.Index('hive_posts_cache_ix32', 'community_id', 'created_at', 'post_id', postgresql_where=sql_text("community_id IS NOT NULL AND is_grayed = '0' AND depth = 0")), # API: community created
- sa.Index('hive_posts_cache_ix33', 'community_id', 'payout', 'post_id', postgresql_where=sql_text("community_id IS NOT NULL AND is_grayed = '0' AND is_paidout = '0'")), # API: community payout
- sa.Index('hive_posts_cache_ix34', 'community_id', 'payout', 'post_id', postgresql_where=sql_text("community_id IS NOT NULL AND is_grayed = '1' AND is_paidout = '0'")), # API: community muted
- sa.Index('hive_posts_cache_ix35', 'author', 'depth'),
- )
-
sa.Table(
'hive_state', metadata,
sa.Column('block_num', sa.Integer, primary_key=True, autoincrement=False),
@@ -370,7 +342,6 @@ def reset_autovac(db):
autovac_config = { # vacuum analyze
'hive_accounts': (50000, 100000),
- 'hive_posts_cache': (25000, 25000),
'hive_posts': (2500, 10000),
'hive_post_tags': (5000, 10000),
'hive_follows': (5000, 5000),
diff --git a/hive/indexer/blocks.py b/hive/indexer/blocks.py
index 58af5b48f..fa47f4af8 100644
--- a/hive/indexer/blocks.py
+++ b/hive/indexer/blocks.py
@@ -219,9 +219,9 @@ class Blocks:
# remove posts: core, tags, cache entries
if post_ids:
- DB.query("DELETE FROM hive_posts_cache WHERE post_id IN :ids", ids=post_ids)
DB.query("DELETE FROM hive_post_tags WHERE post_id IN :ids", ids=post_ids)
DB.query("DELETE FROM hive_posts WHERE id IN :ids", ids=post_ids)
+ DB.query("DELETE FROM hive_posts_data WHERE id IN :ids", ids=post_ids)
DB.query("DELETE FROM hive_payments WHERE block_num = :num", num=num)
DB.query("DELETE FROM hive_blocks WHERE num = :num", num=num)
diff --git a/hive/indexer/cached_post.py b/hive/indexer/cached_post.py
index 21ce636b3..e0db3053b 100644
--- a/hive/indexer/cached_post.py
+++ b/hive/indexer/cached_post.py
@@ -504,7 +504,6 @@ class CachedPost:
('json', json.dumps(basic['json_metadata'])),
#('raw_json', json.dumps(legacy_data)),
# TODO: check if writting fields below on every update is necessary!
- ('legacy_id', legacy_data['id']),
('parent_author', legacy_data['parent_author']),
('parent_permlink', legacy_data['parent_permlink']),
('curator_payout_value', legacy_data['curator_payout_value']),
diff --git a/hive/indexer/community.py b/hive/indexer/community.py
index d57c5acad..51df84475 100644
--- a/hive/indexer/community.py
+++ b/hive/indexer/community.py
@@ -235,10 +235,11 @@ class Community:
SELECT community_id,
COUNT(*) posts,
ROUND(SUM(payout)) payouts,
- COUNT(DISTINCT author) authors
- FROM hive_posts_cache
+ COUNT(DISTINCT author_id) authors
+ FROM hive_posts
WHERE community_id IS NOT NULL
AND is_paidout = '0'
+ AND is_deleted = '0'
GROUP BY community_id
) p
ON community_id = id
diff --git a/hive/indexer/posts.py b/hive/indexer/posts.py
index 9a72cd243..3917217a1 100644
--- a/hive/indexer/posts.py
+++ b/hive/indexer/posts.py
@@ -40,8 +40,13 @@ class Posts:
cls._ids[url] = _id
else:
cls._miss += 1
- sql = """SELECT id FROM hive_posts WHERE
- author = :a AND permlink = :p"""
+ sql = """
+ SELECT id
+ FROM hive_posts hp
+ LEFT JOIN hive_accounts ha_a ON ha_a.id = hp.author_id
+ LEFT JOIN hive_permlink_data hpd_p ON hpd_p.id = hp.permlink_id
+ WHERE ha_a.name = :a AND hpd_p.permlink = :p
+ """
_id = DB.query_one(sql, a=author, p=permlink)
if _id:
cls._set_id(url, _id)
@@ -112,10 +117,11 @@ class Posts:
@classmethod
def insert(cls, op, date):
"""Inserts new post records."""
- sql = """INSERT INTO hive_posts (is_valid, is_muted, parent_id, author,
- permlink, category, community_id, depth, created_at)
- VALUES (:is_valid, :is_muted, :parent_id, :author,
- :permlink, :category, :community_id, :depth, :date)"""
+ # TODO check if category and permlink exists
+ sql = """INSERT INTO hive_posts (is_valid, is_muted, parent_id, author_id,
+ permlink_id, category_id, community_id, depth, created_at)
+ VALUES (:is_valid, :is_muted, :parent_id, (SELECT id FROM hive_accounts WHERE name = :author),
+ (SELECT id FROM hive_permlink_data WHERE permlink = :permlink), (SELECT :category), :community_id, :depth, :date)"""
sql += ";SELECT currval(pg_get_serial_sequence('hive_posts','id'))"
post = cls._build_post(op, date)
result = DB.query(sql, **post)
diff --git a/hive/server/bridge_api/cursor.py b/hive/server/bridge_api/cursor.py
index 8d9477468..928bfbc89 100644
--- a/hive/server/bridge_api/cursor.py
+++ b/hive/server/bridge_api/cursor.py
@@ -104,7 +104,7 @@ async def pids_by_community(db, ids, sort, seek_id, limit):
# setup
field, pending, toponly, gray, promoted = definitions[sort]
- table = 'hive_posts_cache'
+ table = 'hive_posts'
where = ["community_id IN :ids"] if ids else ["community_id IS NOT NULL AND community_id != 1337319"]
# select
@@ -117,8 +117,8 @@ async def pids_by_community(db, ids, sort, seek_id, limit):
# seek
if seek_id:
- sval = "(SELECT %s FROM %s WHERE post_id = :seek_id)" % (field, table)
- sql = """((%s < %s) OR (%s = %s AND post_id > :seek_id))"""
+ sval = "(SELECT %s FROM %s WHERE id = :seek_id)" % (field, table)
+ sql = """((%s < %s) OR (%s = %s AND id > :seek_id))"""
where.append(sql % (field, sval, field, sval))
# simpler `%s <= %s` eval has edge case: many posts with payout 0
@@ -129,8 +129,8 @@ async def pids_by_community(db, ids, sort, seek_id, limit):
#where.append(sql % (field, sval, field, sval))
# build
- sql = ("""SELECT post_id FROM %s WHERE %s
- ORDER BY %s DESC, post_id LIMIT :limit
+ sql = ("""SELECT id FROM %s WHERE %s
+ ORDER BY %s DESC, id LIMIT :limit
""" % (table, ' AND '.join(where), field))
# execute
@@ -157,7 +157,7 @@ async def pids_by_category(db, tag, sort, last_id, limit):
'muted': ('payout', True, False, False, False),
}[sort]
- table = 'hive_posts_cache'
+ table = 'hive_post'
field = params[0]
where = []
@@ -172,17 +172,17 @@ async def pids_by_category(db, tag, sort, last_id, limit):
# filter by category or tag
if tag:
if sort in ['payout', 'payout_comments']:
- where.append('category = :tag')
+ where.append('category_id = (SELECT id FROM hive_category_data WHERE category = :tag)')
else:
sql = "SELECT post_id FROM hive_post_tags WHERE tag = :tag"
- where.append("post_id IN (%s)" % sql)
+ where.append("id IN (%s)" % sql)
if last_id:
- sval = "(SELECT %s FROM %s WHERE post_id = :last_id)" % (field, table)
- sql = """((%s < %s) OR (%s = %s AND post_id > :last_id))"""
+ sval = "(SELECT %s FROM %s WHERE id = :last_id)" % (field, table)
+ sql = """((%s < %s) OR (%s = %s AND id > :last_id))"""
where.append(sql % (field, sval, field, sval))
- sql = ("""SELECT post_id FROM %s WHERE %s
+ sql = ("""SELECT id FROM %s WHERE %s
ORDER BY %s DESC, post_id LIMIT :limit
""" % (table, ' AND '.join(where), field))
@@ -390,14 +390,14 @@ async def pids_by_payout(db, account: str, start_author: str = '',
start_id = None
if start_permlink:
start_id = await _get_post_id(db, start_author, start_permlink)
- last = "(SELECT payout FROM hive_posts_cache WHERE post_id = :start_id)"
- seek = ("""AND (payout < %s OR (payout = %s AND post_id > :start_id))"""
+ last = "(SELECT payout FROM hive_posts WHERE id = :start_id)"
+ seek = ("""AND (payout < %s OR (payout = %s AND id > :start_id))"""
% (last, last))
sql = """
- SELECT post_id
- FROM hive_posts_cache
- WHERE author = :account
+ SELECT id
+ FROM hive_posts
+ WHERE author_id = (SELECT id FROM hive_accounts WHERE name = :account)
AND is_paidout = '0' %s
ORDER BY payout DESC, post_id
LIMIT :limit
diff --git a/hive/server/bridge_api/objects.py b/hive/server/bridge_api/objects.py
index 64b845f93..956589340 100644
--- a/hive/server/bridge_api/objects.py
+++ b/hive/server/bridge_api/objects.py
@@ -43,14 +43,53 @@ async def load_posts_keyed(db, ids, truncate_body=0):
assert ids, 'no ids passed to load_posts_keyed'
# fetch posts and associated author reps
- sql = """SELECT post_id, community_id, author, permlink, title, body, category, depth,
- promoted, payout, payout_at, is_paidout, children, votes,
- created_at, updated_at, rshares, json,
- is_hidden, is_grayed, total_votes, flag_weight,
- legacy_id, parent_author, parent_permlink, curator_payout_value,
- root_author, root_permlink, max_accepted_payout, percent_steem_dollars,
- allow_replies, allow_votes, allow_curation_rewards, url, root_title
- FROM hive_posts_cache WHERE post_id IN :ids"""
+ sql = """
+ SELECT hp.id,
+ community_id,
+ ha_a.name as author,
+ hpd_p.permlink as permlink,
+ hpd.title as title,
+ hpd.body as body,
+ hcd.category as category,
+ depth,
+ promoted,
+ payout,
+ payout_at,
+ is_paidout,
+ children,
+ hpd.votes as votes,
+ hp.created_at,
+ updated_at,
+ rshares,
+ hpd.json as json,
+ is_hidden,
+ is_grayed,
+ total_votes,
+ flag_weight,
+ ha_pa.name as parent_author,
+ hpd_pp.permlink as parent_permlink,
+ curator_payout_value,
+ ha_ra.name as root_author,
+ hpd_rp.permlink as root_permlink,
+ max_accepted_payout,
+ percent_steem_dollars,
+ allow_replies,
+ allow_votes,
+ allow_curation_rewards,
+ beneficiaries,
+ url,
+ root_title
+ FROM hive_posts hp
+ LEFT JOIN hive_accounts ha_a ON ha_a.id = hp.author_id
+ LEFT JOIN hive_permlink_data hpd_p ON hpd_p.id = hp.permlink_id
+ LEFT JOIN hive_post_data hpd ON hpd.id = hp.id
+ LEFT JOIN hive_category_data hcd ON hcd.id = hp.category_id
+ LEFT JOIN hive_accounts ha_pa ON ha_pa.id = hp.parent_author_id
+ LEFT JOIN hive_permlink_data hpd_pp ON hpd_pp.id = hp.parent_permlink_id
+ LEFT JOIN hive_accounts ha_ra ON ha_ra.id = hp.root_author_id
+ LEFT JOIN hive_permlink_data hpd_rp ON hpd_rp.id = hp.root_permlink_id
+ WHERE id IN :ids
+ """
result = await db.query_all(sql, ids=tuple(ids))
author_map = await _query_author_map(db, result)
@@ -178,7 +217,7 @@ def _condenser_profile_object(row):
}}}
def _condenser_post_object(row, truncate_body=0):
- """Given a hive_posts_cache row, create a legacy-style post object."""
+ """Given a hive_posts row, create a legacy-style post object."""
paid = row['is_paidout']
# condenser#3424 mitigation
@@ -186,7 +225,7 @@ def _condenser_post_object(row, truncate_body=0):
row['category'] = 'undefined'
post = {}
- post['post_id'] = row['post_id']
+ post['post_id'] = row['id']
post['author'] = row['author']
post['permlink'] = row['permlink']
post['category'] = row['category']
@@ -222,8 +261,6 @@ def _condenser_post_object(row, truncate_body=0):
#post['author_reputation'] = rep_to_raw(row['author_rep'])
- post['legacy_id'] = row['legacy_id']
-
post['root_author'] = row['root_author']
post['root_permlink'] = row['root_permlink']
diff --git a/hive/server/common/objects.py b/hive/server/common/objects.py
index 1e731d960..f2497a6c3 100644
--- a/hive/server/common/objects.py
+++ b/hive/server/common/objects.py
@@ -28,7 +28,7 @@ async def query_author_map(db, posts):
return {r['name']: r for r in await db.query_all(sql, names=names)}
def condenser_post_object(row, truncate_body=0):
- """Given a hive_posts_cache row, create a legacy-style post object."""
+ """Given a hive_posts row, create a legacy-style post object."""
paid = row['is_paidout']
# condenser#3424 mitigation
@@ -36,7 +36,7 @@ def condenser_post_object(row, truncate_body=0):
row['category'] = 'undefined'
post = {}
- post['post_id'] = row['post_id']
+ post['post_id'] = row['id']
post['author'] = row['author']
post['permlink'] = row['permlink']
post['category'] = row['category']
@@ -63,8 +63,6 @@ def condenser_post_object(row, truncate_body=0):
post['active_votes'] = _hydrate_active_votes(row['votes'])
#post['author_reputation'] = rep_to_raw(row['author_rep'])
- post['legacy_id'] = row['legacy_id']
-
post['root_author'] = row['root_author']
post['root_permlink'] = row['root_permlink']
diff --git a/hive/server/condenser_api/cursor.py b/hive/server/condenser_api/cursor.py
index 04d65b452..633113383 100644
--- a/hive/server/condenser_api/cursor.py
+++ b/hive/server/condenser_api/cursor.py
@@ -141,7 +141,7 @@ async def pids_by_query(db, sort, start_author, start_permlink, limit, tag):
'payout_comments': ('payout', True, False, True, False),
}[sort]
- table = 'hive_posts_cache'
+ table = 'hive_posts'
field = params[0]
where = []
@@ -157,29 +157,67 @@ async def pids_by_query(db, sort, start_author, start_permlink, limit, tag):
# cid = get_community_id(tag)
# where.append('community_id = :cid')
if sort in ['payout', 'payout_comments']:
- where.append('category = :tag')
+ where.append('category_id = (SELECT id FROM hive_category_data WHERE category = :tag)')
else:
if tag[:5] == 'hive-':
- where.append('category = :tag')
+ where.append('category_id = (SELECT id FROM hive_category_data WHERE category = :tag)')
if sort in ('trending', 'hot'):
where.append('depth = 0')
sql = "SELECT post_id FROM hive_post_tags WHERE tag = :tag"
- where.append("post_id IN (%s)" % sql)
+ where.append("id IN (%s)" % sql)
start_id = None
if start_permlink and start_author:
- sql = "%s <= (SELECT %s FROM %s WHERE post_id = (SELECT post_id FROM hive_posts_cache WHERE author = :start_author AND permlink= :start_permlink))"
+ sql = "%s <= (SELECT %s FROM %s WHERE id = (SELECT id FROM hive_posts WHERE author_id = (SELECT id FROM hive_accounts WHERE name = :start_author) AND permlink_id = (SELECT id FROM hive_permlink_data WHERE permlink = :start_permlink)))"
where.append(sql % (field, field, table))
- columns = ['hive_posts_cache.post_id', 'hive_posts_cache.author', 'hive_posts_cache.permlink',
- 'hive_posts_cache.title', 'hive_posts_cache.body', 'hive_posts_cache.category',
- 'hive_posts_cache.depth', 'hive_posts_cache.promoted',
- 'hive_posts_cache.payout', 'hive_posts_cache.payout_at', 'hive_posts_cache.is_paidout',
- 'hive_posts_cache.children', 'hive_posts_cache.votes', 'hive_posts_cache.created_at',
- 'hive_posts_cache.updated_at', 'hive_posts_cache.rshares', 'hive_posts_cache.raw_json',
- 'hive_posts_cache.json']
- sql = ("SELECT %s FROM %s WHERE %s ORDER BY %s DESC LIMIT :limit"
- % (', '.join(columns), table, ' AND '.join(where), field))
+ sql = """
+ SELECT hp.id,
+ community_id,
+ ha_a.name as author,
+ hpd_p.permlink as permlink,
+ hpd.title as title,
+ hpd.body as body,
+ hcd.category as category,
+ depth,
+ promoted,
+ payout,
+ payout_at,
+ is_paidout,
+ children,
+ hpd.votes as votes,
+ hp.created_at,
+ updated_at,
+ rshares,
+ hpd.json as json,
+ is_hidden,
+ is_grayed,
+ total_votes,
+ flag_weight,
+ ha_pa.name as parent_author,
+ hpd_pp.permlink as parent_permlink,
+ curator_payout_value,
+ ha_ra.name as root_author,
+ hpd_rp.permlink as root_permlink,
+ max_accepted_payout,
+ percent_steem_dollars,
+ allow_replies,
+ allow_votes,
+ allow_curation_rewards,
+ beneficiaries,
+ url,
+ root_title
+ FROM hive_posts hp
+ LEFT JOIN hive_accounts ha_a ON ha_a.id = hp.author_id
+ LEFT JOIN hive_permlink_data hpd_p ON hpd_p.id = hp.permlink_id
+ LEFT JOIN hive_post_data hpd ON hpd.id = hp.id
+ LEFT JOIN hive_category_data hcd ON hcd.id = hp.category_id
+ LEFT JOIN hive_accounts ha_pa ON ha_pa.id = hp.parent_author_id
+ LEFT JOIN hive_permlink_data hpd_pp ON hpd_pp.id = hp.parent_permlink_id
+ LEFT JOIN hive_accounts ha_ra ON ha_ra.id = hp.root_author_id
+ LEFT JOIN hive_permlink_data hpd_rp ON hpd_rp.id = hp.root_permlink_id
+ WHERE %s ORDER BY %s DESC LIMIT :limit
+ """ % (' AND '.join(where), field)
#return await db.query_col(sql, tag=tag, start_id=start_id, limit=limit)
return [sql, tag, start_id, limit]
diff --git a/hive/server/condenser_api/methods.py b/hive/server/condenser_api/methods.py
index 3d50bde4f..56fa212a8 100644
--- a/hive/server/condenser_api/methods.py
+++ b/hive/server/condenser_api/methods.py
@@ -18,22 +18,53 @@ from hive.server.common.mutes import Mutes
# pylint: disable=too-many-arguments,line-too-long,too-many-lines
-SELECT_FRAGMENT = """
- SELECT hive_posts_cache.post_id, hive_posts_cache.author, hive_posts_cache.permlink,
- hive_posts_cache.title, hive_posts_cache.body, hive_posts_cache.category, hive_posts_cache.depth,
- hive_posts_cache.promoted, hive_posts_cache.payout, hive_posts_cache.payout_at,
- hive_posts_cache.is_paidout, hive_posts_cache.children, hive_posts_cache.votes,
- hive_posts_cache.created_at, hive_posts_cache.updated_at, hive_posts_cache.rshares,
- hive_posts_cache.raw_json, hive_posts_cache.json, hive_accounts.reputation AS author_rep,
- hive_posts_cache.is_hidden AS is_hidden, hive_posts_cache.is_grayed AS is_grayed,
- hive_posts_cache.total_votes AS total_votes, hive_posts_cache.flag_weight AS flag_weight,
- hive_posts_cache.sc_trend AS sc_trend, hive_accounts.id AS acct_author_id,
- hive_posts.is_pinned AS is_pinned
- FROM hive_posts_cache JOIN hive_posts ON (hive_posts_cache.post_id = hive_posts.id)
- JOIN hive_accounts ON (hive_posts_cache.author = hive_accounts.name)"""
-
-
-# Dummy
+SQL_TEMPLATE = """
+ SELECT hp.id,
+ community_id,
+ ha_a.name as author,
+ hpd_p.permlink as permlink,
+ hpd.title as title,
+ hpd.body as body,
+ hcd.category as category,
+ depth,
+ promoted,
+ payout,
+ payout_at,
+ is_paidout,
+ children,
+ hpd.votes as votes,
+ hp.created_at,
+ updated_at,
+ rshares,
+ hpd.json as json,
+ is_hidden,
+ is_grayed,
+ total_votes,
+ flag_weight,
+ ha_pa.name as parent_author,
+ hpd_pp.permlink as parent_permlink,
+ curator_payout_value,
+ ha_ra.name as root_author,
+ hpd_rp.permlink as root_permlink,
+ max_accepted_payout,
+ percent_steem_dollars,
+ allow_replies,
+ allow_votes,
+ allow_curation_rewards,
+ beneficiaries,
+ url,
+ root_title,
+ ha_a.reputation AS author_rep
+ FROM hive_posts hp
+ LEFT JOIN hive_accounts ha_a ON ha_a.id = hp.author_id
+ LEFT JOIN hive_permlink_data hpd_p ON hpd_p.id = hp.permlink_id
+ LEFT JOIN hive_post_data hpd ON hpd.id = hp.id
+ LEFT JOIN hive_category_data hcd ON hcd.id = hp.category_id
+ LEFT JOIN hive_accounts ha_pa ON ha_pa.id = hp.parent_author_id
+ LEFT JOIN hive_permlink_data hpd_pp ON hpd_pp.id = hp.parent_permlink_id
+ LEFT JOIN hive_accounts ha_ra ON ha_ra.id = hp.root_author_id
+ LEFT JOIN hive_permlink_data hpd_rp ON hpd_rp.id = hp.root_permlink_id
+"""
@return_error_info
async def get_account_votes(context, account):
@@ -118,9 +149,10 @@ async def get_content(context, author: str, permlink: str):
valid_account(author)
valid_permlink(permlink)
- sql = """ ---get_content\n""" + SELECT_FRAGMENT + """
- WHERE hive_posts_cache.author = :author AND hive_posts_cache.permlink = :permlink AND NOT hive_posts.is_deleted
- """
+ #force copy
+ sql = str(SQL_TEMPLATE)
+ sql += """ WHERE ha_a.name = :author AND hpd_p.permlink = :permlink AND NOT hp.is_deleted """
+
result = await db.query_all(sql, author=author, permlink=permlink)
result = dict(result[0])
post = _condenser_post_object(result, 0)
@@ -136,25 +168,24 @@ async def get_content_replies(context, author: str, permlink: str):
valid_account(author)
valid_permlink(permlink)
- sql = """
- --get_content_replies
- SELECT post_id, author, permlink, title, body, category, depth,
- promoted, payout, payout_at, is_paidout, children, votes,
- created_at, updated_at, rshares, json,
- legacy_id, parent_author, parent_permlink, curator_payout_value,
- root_author, root_permlink, max_accepted_payout, percent_steem_dollars,
- allow_replies, allow_votes, allow_curation_rewards, url, root_title
- FROM hive_posts_cache WHERE post_id IN (
- SELECT hp2.id FROM hive_posts hp2
- WHERE hp2.is_deleted = '0' AND
- hp2.parent_id = (SELECT id FROM hive_posts
- WHERE author = :author
- AND permlink = :permlink AND is_deleted = '0')
- LIMIT :limit
- )
- ORDER BY post_id"""
-
- result=await db.query_all(sql, author=author, permlink = permlink, limit=5000)
+ #force copy
+ sql = str(SQL_TEMPLATE)
+ sql += """
+ WHERE
+ hp.is_deleted = '0' AND
+ hp.parent_id = (
+ SELECT id
+ FROM hive_posts
+ WHERE
+ author_id = (SELECT id FROM hive_accounts WHERE name =:author)
+ AND permlink_id = (SELECT id FROM hive_permlink_data WHERE permlink = :permlink)
+ AND is_deleted = '0'
+ )
+ LIMIT :limit
+ ORDER BY id
+ """
+
+ result = await db.query_all(sql, author=author, permlink=permlink, limit=5000)
posts = await resultset_to_posts(db=db, resultset=result, truncate_body=0)
return posts
@@ -197,47 +228,47 @@ async def get_discussions_by(discussion_type, context, start_author: str = '',
valid_tag(tag, allow_empty=True)
db = context['db']
- sql = "---get_discussions_by_" + discussion_type + "\r\n" + SELECT_FRAGMENT
+ sql = "---get_discussions_by_" + discussion_type + "\r\n" + str(SQL_TEMPLATE)
- sql = sql + """ WHERE NOT hive_posts.is_deleted """
+ sql = sql + """ WHERE NOT hp.is_deleted """
if discussion_type == 'trending':
- sql = sql + """ AND NOT hive_posts_cache.is_paidout %s ORDER BY sc_trend DESC LIMIT :limit """
+ sql = sql + """ AND NOT hp.is_paidout %s ORDER BY sc_trend DESC LIMIT :limit """
elif discussion_type == 'hot':
- sql = sql + """ AND NOT hive_posts_cache.is_paidout %s ORDER BY sc_hot DESC LIMIT :limit """
+ sql = sql + """ AND NOT hp.is_paidout %s ORDER BY sc_hot DESC LIMIT :limit """
elif discussion_type == 'created':
- sql = sql + """ AND hive_posts.depth = 0 %s ORDER BY hive_posts_cache.created_at DESC LIMIT :limit """
+ sql = sql + """ AND hp.depth = 0 %s ORDER BY hp.created_at DESC LIMIT :limit """
elif discussion_type == 'promoted':
- sql = sql + """ AND NOT hive_posts_cache.is_paidout AND hive_posts.promoted > 0
- %s ORDER BY hive_posts_cache.promoted DESC LIMIT :limit """
+ sql = sql + """ AND NOT hp.is_paidout AND hp.promoted > 0
+ %s ORDER BY hp.promoted DESC LIMIT :limit """
elif discussion_type == 'payout':
- sql = sql + """ AND NOT hive_posts_cache.is_paidout AND hive_posts_cache.depth = 0
- %s ORDER BY hive_posts_cache.payout DESC LIMIT :limit """
+ sql = sql + """ AND NOT hp.is_paidout AND hp.depth = 0
+ %s ORDER BY hp.payout DESC LIMIT :limit """
elif discussion_type == 'payout_comments':
- sql = sql + """ AND NOT hive_posts_cache.is_paidout AND hive_posts_cache.depth > 0
- %s ORDER BY hive_posts_cache.payout DESC LIMIT :limit """
+ sql = sql + """ AND NOT hp.is_paidout AND hp.depth > 0
+ %s ORDER BY hp.payout DESC LIMIT :limit """
if tag and tag != 'all':
if tag[:5] == 'hive-':
- sql = sql % """ %s AND hive_posts_cache.category = :tag """
+ sql = sql % """ %s AND hp.category = :tag """
else:
- sql = sql % """ %s AND hive_posts_cache.post_id IN (SELECT post_id FROM hive_post_tags WHERE tag = :tag) """
+ sql = sql % """ %s AND hp.post_id IN (SELECT post_id FROM hive_post_tags WHERE tag = :tag) """
if start_author and start_permlink:
if discussion_type == 'trending':
- sql = sql % """ AND hive_posts_cache.sc_trend <= (SELECT sc_trend FROM hive_posts_cache WHERE permlink = :permlink AND author = :author)
- AND hive_posts_cache.post_id != (SELECT post_id FROM hive_posts_cache WHERE permlink = :permlink AND author = :author) """
+ sql = sql % """ AND hp.sc_trend <= (SELECT sc_trend FROM hp WHERE permlink_id = (SELECT id FROM hive_permlink_data WHERE permlink = :permlink) AND author_id = (SELECT id FROM hive_accounts WHERE name = :author))
+ AND hp.post_id != (SELECT post_id FROM hp WHERE permlink_id = (SELECT id FROM hive_permlink_data WHERE permlink = :permlink) AND author_id = (SELECT id FROM hive_accounts WHERE name = :author)) """
elif discussion_type == 'hot':
- sql = sql % """ AND hive_posts_cache.sc_hot <= (SELECT sc_hot FROM hive_posts_cache WHERE permlink = :permlink AND author = :author)
- AND hive_posts_cache.post_id != (SELECT post_id FROM hive_posts_cache WHERE permlink = :permlink AND author = :author) """
+ sql = sql % """ AND hp.sc_hot <= (SELECT sc_hot FROM hp WHERE permlink_id = (SELECT id FROM hive_permlink_data WHERE permlink = :permlink) AND author_id = (SELECT id FROM hive_accounts WHERE name = :author))
+ AND hp.post_id != (SELECT post_id FROM hp WHERE permlink_id = (SELECT id FROM hive_permlink_data WHERE permlink = :permlink) AND author_id = (SELECT id FROM hive_accounts WHERE name = :author)) """
elif discussion_type == 'created':
- sql = sql % """ AND hive_posts_cache.post_id < (SELECT post_id FROM hive_posts_cache WHERE permlink = :permlink AND author = :author) """
+ sql = sql % """ AND hp.post_id < (SELECT post_id FROM hp WHERE permlink_id = (SELECT id FROM hive_permlink_data WHERE permlink = :permlink) AND author_id = (SELECT id FROM hive_accounts WHERE name = :author)) """
elif discussion_type == 'promoted':
- sql = sql % """ AND hive_posts_cache.promoted <= (SELECT promoted FROM hive_posts_cache WHERE permlink = :permlink AND author = :author)
- AND hive_posts_cache.post_id != (SELECT post_id FROM hive_posts_cache WHERE permlink = :permlink AND author = :author) """
+ sql = sql % """ AND hp.promoted <= (SELECT promoted FROM hp WHERE permlink_id = (SELECT id FROM hive_permlink_data WHERE permlink = :permlink) AND author_id = (SELECT id FROM hive_accounts WHERE name = :author))
+ AND hp.post_id != (SELECT post_id FROM hp WHERE permlink_id = (SELECT id FROM hive_permlink_data WHERE permlink = :permlink) AND author_id = (SELECT id FROM hive_accounts WHERE name = :author)) """
else:
- sql = sql % """ AND hive_posts_cache.payout <= (SELECT payout FROM hive_posts_cache where permlink = :permlink AND author = :author)
- AND hive_posts_cache.post_id != (SELECT post_id FROM hive_posts_cache WHERE permlink = :permlink AND author = :author) """
+ sql = sql % """ AND hp.payout <= (SELECT payout FROM hp where permlink_id = (SELECT id FROM hive_permlink_data WHERE permlink = :permlink) AND author_id = (SELECT id FROM hive_accounts WHERE name = :author))
+ AND hp.post_id != (SELECT post_id FROM hp WHERE permlink_id = (SELECT id FROM hive_permlink_data WHERE permlink = :permlink) AND author_id = (SELECT id FROM hive_accounts WHERE name = :author)) """
else:
sql = sql % """ """
@@ -312,17 +343,19 @@ async def get_discussions_by_blog(context, tag: str = None, start_author: str =
valid_permlink(start_permlink, allow_empty=True)
valid_limit(limit, 100)
- sql = """ ---get_discussions_by_blog """ + SELECT_FRAGMENT + """
- WHERE NOT hive_posts.is_deleted AND hive_posts_cache.post_id IN
+ #force copy
+ sql = str(SQL_TEMPLATE)
+ sql += """
+ WHERE NOT hp.is_deleted AND hp.id IN
(SELECT post_id FROM hive_feed_cache JOIN hive_accounts ON (hive_feed_cache.account_id = hive_accounts.id) WHERE hive_accounts.name = :author)
"""
if start_author and start_permlink != '':
sql += """
- AND hive_posts_cache.created_at <= (SELECT created_at from hive_posts_cache where author = :start_author AND permlink = :start_permlink)
+ AND hp.created_at <= (SELECT created_at from hive_posts WHERE author_id = (SELECT id FROM hive_accounts WHERE name = :start_author) AND permlink_id = (SELECT id FROM hive_permlink_data WHERE permlink = :start_permlink))
"""
sql += """
- ORDER BY hive_posts_cache.created_at DESC
+ ORDER BY hp.created_at DESC
LIMIT :limit
"""
@@ -368,19 +401,20 @@ async def get_discussions_by_comments(context, start_author: str = None, start_p
valid_permlink(start_permlink, allow_empty=True)
valid_limit(limit, 100)
- sql = """ ---get_discussions_by_comments """ + SELECT_FRAGMENT + """
- WHERE hive_posts_cache.author = :start_author AND hive_posts_cache.depth > 0
- AND NOT hive_posts.is_deleted
+ #force copy
+ sql = str(SQL_TEMPLATE)
+ sql += """
+ WHERE ha_a.author = :start_author AND hp.depth > 0
+ AND NOT hp.is_deleted
"""
if start_permlink:
sql += """
- AND hive_posts_cache.post_id <= (SELECT hive_posts_cache.post_id FROM
- hive_posts_cache WHERE permlink = :start_permlink AND author=:start_author)
+ AND hp.id <= (SELECT hive_posts.id FROM hive_posts WHERE author_id = (SELECT id FROM hive_accounts WHERE name = :start_author) AND permlink_id = (SELECT id FROM hive_permlink_data WHERE permlink = :start_permlink))
"""
sql += """
- ORDER BY hive_posts_cache.post_id DESC, depth LIMIT :limit
+ ORDER BY hp.id DESC, depth LIMIT :limit
"""
posts = []
diff --git a/hive/server/condenser_api/objects.py b/hive/server/condenser_api/objects.py
index 71148d3c5..30c216ab9 100644
--- a/hive/server/condenser_api/objects.py
+++ b/hive/server/condenser_api/objects.py
@@ -40,13 +40,53 @@ async def load_posts_keyed(db, ids, truncate_body=0):
assert ids, 'no ids passed to load_posts_keyed'
# fetch posts and associated author reps
- sql = """SELECT post_id, author, permlink, title, body, category, depth,
- promoted, payout, payout_at, is_paidout, children, votes,
- created_at, updated_at, rshares, json,
- legacy_id, parent_author, parent_permlink, curator_payout_value,
- root_author, root_permlink, max_accepted_payout, percent_steem_dollars,
- allow_replies, allow_votes, allow_curation_rewards, url, root_title
- FROM hive_posts_cache WHERE post_id IN :ids"""
+ sql = """
+ SELECT hp.id,
+ community_id,
+ ha_a.name as author,
+ hpd_p.permlink as permlink,
+ hpd.title as title,
+ hpd.body as body,
+ hcd.category as category,
+ depth,
+ promoted,
+ payout,
+ payout_at,
+ is_paidout,
+ children,
+ hpd.votes as votes,
+ hp.created_at,
+ updated_at,
+ rshares,
+ hpd.json as json,
+ is_hidden,
+ is_grayed,
+ total_votes,
+ flag_weight,
+ ha_pa.name as parent_author,
+ hpd_pp.permlink as parent_permlink,
+ curator_payout_value,
+ ha_ra.name as root_author,
+ hpd_rp.permlink as root_permlink,
+ max_accepted_payout,
+ percent_steem_dollars,
+ allow_replies,
+ allow_votes,
+ allow_curation_rewards,
+ beneficiaries,
+ url,
+ root_title,
+ FROM hive_posts hp
+ LEFT JOIN hive_accounts ha_a ON ha_a.id = hp.author_id
+ LEFT JOIN hive_permlink_data hpd_p ON hpd_p.id = hp.permlink_id
+ LEFT JOIN hive_post_data hpd ON hpd.id = hp.id
+ LEFT JOIN hive_category_data hcd ON hcd.id = hp.category_id
+ LEFT JOIN hive_accounts ha_pa ON ha_pa.id = hp.parent_author_id
+ LEFT JOIN hive_permlink_data hpd_pp ON hpd_pp.id = hp.parent_permlink_id
+ LEFT JOIN hive_accounts ha_ra ON ha_ra.id = hp.root_author_id
+ LEFT JOIN hive_permlink_data hpd_rp ON hpd_rp.id = hp.root_permlink_id
+ WHERE post_id IN :ids"""
+
result = await db.query_all(sql, ids=tuple(ids))
author_reps = await _query_author_rep_map(db, result)
@@ -57,7 +97,7 @@ async def load_posts_keyed(db, ids, truncate_body=0):
row['author_rep'] = author_reps[row['author']]
post = _condenser_post_object(row, truncate_body=truncate_body)
post['active_votes'] = _mute_votes(post['active_votes'], muted_accounts)
- posts_by_id[row['post_id']] = post
+ posts_by_id[row['id']] = post
return posts_by_id
@@ -132,7 +172,7 @@ def _condenser_account_object(row):
}})}
def _condenser_post_object(row, truncate_body=0):
- """Given a hive_posts_cache row, create a legacy-style post object."""
+ """Given a hive_posts row, create a legacy-style post object."""
paid = row['is_paidout']
# condenser#3424 mitigation
@@ -140,7 +180,7 @@ def _condenser_post_object(row, truncate_body=0):
row['category'] = 'undefined'
post = {}
- post['post_id'] = row['post_id']
+ post['post_id'] = row['id']
post['author'] = row['author']
post['permlink'] = row['permlink']
post['category'] = row['category']
@@ -167,8 +207,6 @@ def _condenser_post_object(row, truncate_body=0):
post['active_votes'] = _hydrate_active_votes(row['votes'])
post['author_reputation'] = rep_to_raw(row['author_rep'])
- post['legacy_id'] = row['legacy_id']
-
post['root_author'] = row['root_author']
post['root_permlink'] = row['root_permlink']
diff --git a/hive/server/condenser_api/tags.py b/hive/server/condenser_api/tags.py
index 9a4f5b5b3..7cb83f44b 100644
--- a/hive/server/condenser_api/tags.py
+++ b/hive/server/condenser_api/tags.py
@@ -11,7 +11,7 @@ async def get_top_trending_tags_summary(context):
#return [tag['name'] for tag in await get_trending_tags('', 50)]
sql = """
SELECT category
- FROM hive_posts_cache
+ FROM hive_posts
WHERE is_paidout = '0'
GROUP BY category
ORDER BY SUM(payout) DESC
@@ -31,7 +31,7 @@ async def get_trending_tags(context, start_tag: str = '', limit: int = 250):
seek = """
HAVING SUM(payout) <= (
SELECT SUM(payout)
- FROM hive_posts_cache
+ FROM hive_posts
WHERE is_paidout = '0'
AND category = :start_tag)
"""
@@ -43,7 +43,7 @@ async def get_trending_tags(context, start_tag: str = '', limit: int = 250):
COUNT(*) AS total_posts,
SUM(CASE WHEN depth = 0 THEN 1 ELSE 0 END) AS top_posts,
SUM(payout) AS total_payouts
- FROM hive_posts_cache
+ FROM hive_posts
WHERE is_paidout = '0'
GROUP BY category %s
ORDER BY SUM(payout) DESC
diff --git a/hive/server/database_api/methods.py b/hive/server/database_api/methods.py
index 6bf3e7671..9368927ca 100644
--- a/hive/server/database_api/methods.py
+++ b/hive/server/database_api/methods.py
@@ -2,11 +2,69 @@
from hive.server.common.helpers import return_error_info, valid_limit
from hive.server.common.objects import condenser_post_object
+SQL_TEMPLATE = """
+ SELECT hp.id,
+ community_id,
+ ha_a.name as author,
+ hpd_p.permlink as permlink,
+ hpd.title as title,
+ hpd.body as body,
+ hcd.category as category,
+ depth,
+ promoted,
+ payout,
+ payout_at,
+ is_paidout,
+ children,
+ hpd.votes as votes,
+ hp.created_at,
+ updated_at,
+ rshares,
+ hpd.json as json,
+ is_hidden,
+ is_grayed,
+ total_votes,
+ flag_weight,
+ ha_pa.name as parent_author,
+ hpd_pp.permlink as parent_permlink,
+ curator_payout_value,
+ ha_ra.name as root_author,
+ hpd_rp.permlink as root_permlink,
+ max_accepted_payout,
+ percent_steem_dollars,
+ allow_replies,
+ allow_votes,
+ allow_curation_rewards,
+ beneficiaries,
+ url,
+ root_title
+ FROM hive_posts hp
+ LEFT JOIN hive_accounts ha_a ON ha_a.id = hp.author_id
+ LEFT JOIN hive_permlink_data hpd_p ON hpd_p.id = hp.permlink_id
+ LEFT JOIN hive_post_data hpd ON hpd.id = hp.id
+ LEFT JOIN hive_category_data hcd ON hcd.id = hp.category_id
+ LEFT JOIN hive_accounts ha_pa ON ha_pa.id = hp.parent_author_id
+ LEFT JOIN hive_permlink_data hpd_pp ON hpd_pp.id = hp.parent_permlink_id
+ LEFT JOIN hive_accounts ha_ra ON ha_ra.id = hp.root_author_id
+ LEFT JOIN hive_permlink_data hpd_rp ON hpd_rp.id = hp.root_permlink_id
+ WHERE
+"""
+
async def get_post_id_by_author_and_permlink(db, author: str, permlink: str, limit: int):
"""Return post ids for given author and permlink"""
- sql = """SELECT post_id FROM hive_posts_cache WHERE author = :author AND permlink = :permlink ORDER BY author ASC, permlink ASC, post_id ASC LIMIT :limit"""
+ sql = """
+ SELECT hp.id
+ FROM hive_posts hp
+ LEFT JOIN hive_accounts ha_a ON ha_a.id = hp.author_id
+ LEFT JOIN hive_permlink_data hpd_p ON hpd_p.id = hp.permlink_id
+ WHERE ha_a.name >= :author AND hpd_p.permlink >= :permlink
+ ORDER BY ha_a.name ASC
+ LIMIT :limit
+ """
result = await db.query_row(sql, author=author, permlink=permlink, limit=limit)
- return result.post_id
+ if result is not None:
+ return int(result.get('post_id', 0))
+ return 0
@return_error_info
async def list_comments(context, start: list, limit: int, order: str):
@@ -26,55 +84,38 @@ async def list_comments(context, start: list, limit: int, order: str):
if author or permlink:
post_id = await get_post_id_by_author_and_permlink(db, author, permlink, 1)
- sql = """SELECT post_id, community_id, author, permlink, title, body, category, depth,
- promoted, payout, payout_at, is_paidout, children, votes,
- created_at, updated_at, rshares, json,
- is_hidden, is_grayed, total_votes, flag_weight,
- legacy_id, parent_author, parent_permlink, curator_payout_value,
- root_author, root_permlink, max_accepted_payout, percent_steem_dollars,
- allow_replies, allow_votes, allow_curation_rewards, beneficiaries, url, root_title
- FROM hive_posts_cache WHERE payout_at >= :start AND post_id >= :post_id ORDER BY payout_at ASC, post_id ASC LIMIT :limit"""
+ sql = str(SQL_TEMPLATE)
+ sql += "hp.payout_at >= :start AND hp.id >= :post_id ORDER BY hp.payout_at ASC, hp.id ASC LIMIT :limit"
+
result = await db.query_all(sql, start=start[0], limit=limit, post_id=post_id)
for row in result:
comments.append(condenser_post_object(dict(row)))
elif order == 'by_permlink':
assert len(start) == 2, "Expecting two arguments"
- sql = """SELECT post_id, community_id, author, permlink, title, body, category, depth,
- promoted, payout, payout_at, is_paidout, children, votes,
- created_at, updated_at, rshares, json,
- is_hidden, is_grayed, total_votes, flag_weight,
- legacy_id, parent_author, parent_permlink, curator_payout_value,
- root_author, root_permlink, max_accepted_payout, percent_steem_dollars,
- allow_replies, allow_votes, allow_curation_rewards, beneficiaries, url, root_title
- FROM hive_posts_cache WHERE author >= :author AND permlink >= :permlink ORDER BY author ASC, permlink ASC LIMIT :limit"""
+
+ sql = str(SQL_TEMPLATE)
+ sql += 'ha_a.name >= :author COLLATE "C" AND hpd_p.permlink >= :permlink COLLATE "C" ORDER BY ha_a.name COLLATE "C" ASC LIMIT :limit'
+
result = await db.query_all(sql, author=start[0], permlink=start[1], limit=limit)
for row in result:
comments.append(condenser_post_object(dict(row)))
elif order == 'by_root':
assert len(start) == 4, "Expecting 4 arguments"
+ raise NotImplementedError('by_root')
+
+ sql = str(SQL_TEMPLATE)
+ sql += "get_rows_by_root(:root_author, :root_permlink, :child_author, :child_permlink) ORDER BY post_id ASC LIMIT :limit"
- sql = """SELECT post_id, community_id, author, permlink, title, body, category, depth,
- promoted, payout, payout_at, is_paidout, children, votes,
- created_at, updated_at, rshares, json,
- is_hidden, is_grayed, total_votes, flag_weight,
- legacy_id, parent_author, parent_permlink, curator_payout_value,
- root_author, root_permlink, max_accepted_payout, percent_steem_dollars,
- allow_replies, allow_votes, allow_curation_rewards, beneficiaries, url, root_title
- FROM get_rows_by_root(:root_author, :root_permlink, :child_author, :child_permlink) ORDER BY post_id ASC LIMIT :limit"""
result = await db.query_all(sql, root_author=start[0], root_permlink=start[1], child_author=start[2], child_permlink=start[3], limit=limit)
for row in result:
comments.append(condenser_post_object(dict(row)))
elif order == 'by_parent':
assert len(start) == 4, "Expecting 4 arguments"
+ raise NotImplementedError('by_parent')
+
+ sql = str(SQL_TEMPLATE)
+ sql += "get_rows_by_parent(:parent_author, :parent_permlink, :child_author, :child_permlink) LIMIT :limit"
- sql = """SELECT post_id, community_id, author, permlink, title, body, category, depth,
- promoted, payout, payout_at, is_paidout, children, votes,
- created_at, updated_at, rshares, json,
- is_hidden, is_grayed, total_votes, flag_weight,
- legacy_id, parent_author, parent_permlink, curator_payout_value,
- root_author, root_permlink, max_accepted_payout, percent_steem_dollars,
- allow_replies, allow_votes, allow_curation_rewards, beneficiaries, url, root_title
- FROM get_rows_by_parent(:parent_author, :parent_permlink, :child_author, :child_permlink) LIMIT :limit"""
result = await db.query_all(sql, parent_author=start[0], parent_permlink=start[1], child_author=start[2], child_permlink=start[3], limit=limit)
for row in result:
comments.append(condenser_post_object(dict(row)))
@@ -88,14 +129,9 @@ async def list_comments(context, start: list, limit: int, order: str):
if author or permlink:
post_id = await get_post_id_by_author_and_permlink(db, child_author, child_permlink, 1)
- sql = """SELECT post_id, community_id, author, permlink, title, body, category, depth,
- promoted, payout, payout_at, is_paidout, children, votes,
- created_at, updated_at, rshares, json,
- is_hidden, is_grayed, total_votes, flag_weight,
- legacy_id, parent_author, parent_permlink, curator_payout_value,
- root_author, root_permlink, max_accepted_payout, percent_steem_dollars,
- allow_replies, allow_votes, allow_curation_rewards, beneficiaries, url, root_title
- FROM hive_posts_cache WHERE parent_author >= :parent_author AND updated_at >= :updated_at AND post_id >= :post_id ORDER BY parent_author ASC, updated_at ASC, post_id ASC LIMIT :limit"""
+ sql = str(SQL_TEMPLATE)
+ sql += "ha_pa.name >= :parent_author AND hp.updated_at >= :updated_at AND hp.id >= :post_id ORDER BY ha_pa.name ASC, updated_at ASC, hp.id ASC LIMIT :limit"
+
result = await db.query_all(sql, parent_author=start[0], updated_at=start[1], post_id=post_id, limit=limit)
for row in result:
comments.append(condenser_post_object(dict(row)))
@@ -110,16 +146,35 @@ async def list_comments(context, start: list, limit: int, order: str):
if author or permlink:
post_id = await get_post_id_by_author_and_permlink(db, author, permlink, 1)
- sql = """SELECT post_id, community_id, author, permlink, title, body, category, depth,
- promoted, payout, payout_at, is_paidout, children, votes,
- created_at, updated_at, rshares, json,
- is_hidden, is_grayed, total_votes, flag_weight,
- legacy_id, parent_author, parent_permlink, curator_payout_value,
- root_author, root_permlink, max_accepted_payout, percent_steem_dollars,
- allow_replies, allow_votes, allow_curation_rewards, beneficiaries, url, root_title
- FROM hive_posts_cache WHERE author >= :author AND updated_at >= :updated_at AND post_id >= :post_id ORDER BY parent_author ASC, updated_at ASC, post_id ASC LIMIT :limit"""
+ sql = str(SQL_TEMPLATE)
+ sql += "ha_a.name >= :author AND hp.updated_at >= :updated_at AND hp.id >= :post_id ORDER BY ha_a.name ASC, hp.updated_at ASC, hp.id ASC LIMIT :limit"
+
result = await db.query_all(sql, author=start[0], updated_at=start[1], post_id=post_id, limit=limit)
for row in result:
comments.append(condenser_post_object(dict(row)))
return comments
+
+@return_error_info
+async def find_comments(context, start: list, limit: int, order: str):
+ """ Search for comments: limit and order is ignored in hive code """
+ comments = []
+
+ assert len(start) <= 1000, "Parameters count is greather than max allowed (1000)"
+ db = context['db']
+
+ # make a copy
+ sql = str(SQL_TEMPLATE)
+
+ idx = 0
+ for arg in start:
+ if idx > 0:
+ sql += " OR "
+ sql += "(ha_a.name = '{}' AND hpd_p.permlink = '{}')".format(arg[0], arg[1])
+ idx += 1
+
+ result = await db.query_all(sql)
+ for row in result:
+ comments.append(condenser_post_object(dict(row)))
+
+ return comments
diff --git a/hive/server/hive_api/community.py b/hive/server/hive_api/community.py
index 47963d1ce..fa572de00 100644
--- a/hive/server/hive_api/community.py
+++ b/hive/server/hive_api/community.py
@@ -353,8 +353,16 @@ async def top_community_muted(context, community):
async def _top_community_posts(db, community, limit=50):
# TODO: muted equivalent
- sql = """SELECT author, votes, payout FROM hive_posts_cache
- WHERE category = :community AND is_paidout = '0'
- AND post_id IN (SELECT id FROM hive_posts WHERE is_muted = '0')
- ORDER BY payout DESC LIMIT :limit"""
+ sql = """
+ SELECT ha_a.name as author,
+ hpd.votes as votes,
+ payout
+ FROM hive_posts hp
+ LEFT JOIN hive_accounts ha_a ON ha_a.id = hp.author_id
+ LEFT JOIN hive_post_data hpd ON hpd.id = hp.id
+ LEFT JOIN hive_category_data hcd ON hcd.id = hp.category_id
+ WHERE hcdcategory = :community AND is_paidout = '0'
+ AND post_id IN (SELECT id FROM hive_posts WHERE is_muted = '0')
+ ORDER BY payout DESC LIMIT :limit"""
+
return await db.query_all(sql, community=community, limit=limit)
diff --git a/hive/server/hive_api/objects.py b/hive/server/hive_api/objects.py
index c8c74a37e..05dfe19de 100644
--- a/hive/server/hive_api/objects.py
+++ b/hive/server/hive_api/objects.py
@@ -68,10 +68,22 @@ async def comments_by_id(db, ids, observer=None):
"""Given an array of post ids, returns comment objects keyed by id."""
assert ids, 'no ids passed to comments_by_id'
- sql = """SELECT post_id, author, permlink, body, depth,
- payout, payout_at, is_paidout, created_at, updated_at,
- rshares, is_hidden, is_grayed, votes
- FROM hive_posts_cache WHERE post_id IN :ids""" #votes
+ sql = """SELECT hp.id,
+ (SELECT name FROM hive_accounts ha WHERE ha.id = hp.author_id) as author,
+ (SELECT permlink FROM hive_permlink_data hpd WHERE hpd.id = hp.permlink_id) as permlink,
+ (SELECT body FROM hive_post_data hpa WHERE hpa.id = hp.id) as body,
+ hp.depth,
+ hp.payout,
+ hp.payout_at,
+ hp.is_paidout,
+ hp.created_at,
+ hp.updated_at,
+ hp.rshares,
+ hp.is_hidden,
+ hp.is_grayed,
+ hp.votes
+ FROM hive_posts hp
+ WHERE hp.id IN :ids""" #votes
result = await db.query_all(sql, ids=tuple(ids))
authors = set()
@@ -79,7 +91,7 @@ async def comments_by_id(db, ids, observer=None):
for row in result:
top_votes, observer_vote = _top_votes(row, 5, observer)
post = {
- 'id': row['post_id'],
+ 'id': row['id'],
'author': row['author'],
'url': row['author'] + '/' + row['permlink'],
'depth': row['depth'],
@@ -108,10 +120,23 @@ async def posts_by_id(db, ids, observer=None, lite=True):
"""Given a list of post ids, returns lite post objects in the same order."""
# pylint: disable=too-many-locals
- sql = """SELECT post_id, author, permlink, title, img_url, payout, promoted,
- created_at, payout_at, is_nsfw, rshares, votes,
- is_muted, is_invalid, %s
- FROM hive_posts_cache WHERE post_id IN :ids"""
+ sql = """SELECT hp.id,
+ author,
+ permlink,
+ title,
+ img_url,
+ hp.payout,
+ hp.promoted,
+ hp.created_at,
+ hp.payout_at,
+ hp.is_nsfw,
+ hp.rshares,
+ hp.votes,
+ hp.is_muted,
+ hp.is_invalid,
+ %s
+ FROM hive_posts hp
+ WHERE post_id IN :ids"""
fields = ['preview'] if lite else ['body', 'updated_at', 'json']
sql = sql % (', '.join(fields))
diff --git a/hive/server/hive_api/thread.py b/hive/server/hive_api/thread.py
index 27d984235..b3f03f7c9 100644
--- a/hive/server/hive_api/thread.py
+++ b/hive/server/hive_api/thread.py
@@ -29,7 +29,7 @@ async def fetch_more_children(context, root_id, last_sibling_id, sort='top',
valid_limit(limit, 50),
observer)
-_SORTS = dict(hot='sc_hot', top='payout', new='post_id')
+_SORTS = dict(hot='sc_hot', top='payout', new='id')
async def _fetch_children(db, root_id, start_id, sort, limit, observer=None):
"""Fetch truncated children from tree."""
mutes = set()
@@ -41,10 +41,10 @@ async def _fetch_children(db, root_id, start_id, sort, limit, observer=None):
# find most relevant ids in subset
seek = ''
if start_id:
- seek = """AND %s < (SELECT %s FROM hive_posts_cache
- WHERE post_id = :start_id)""" % (field, field)
- sql = """SELECT post_id FROM hive_posts_cache
- WHERE post_id IN :ids %s ORDER BY %s DESC
+ seek = """AND %s < (SELECT %s FROM hive_posts
+ WHERE id = :start_id)""" % (field, field)
+ sql = """SELECT id FROM hive_posts
+ WHERE id IN :ids %s ORDER BY %s DESC
LIMIT :limit""" % (seek, field)
relevant_ids = await db.query_col(sql, ids=tuple(parent.keys()),
start_id=start_id, limit=limit)
diff --git a/hive/server/serve.py b/hive/server/serve.py
index 873387dba..9593dce1c 100644
--- a/hive/server/serve.py
+++ b/hive/server/serve.py
@@ -140,7 +140,8 @@ def build_methods():
# database_api methods
methods.add(**{
- 'database_api.list_comments' : database_api.list_comments
+ 'database_api.list_comments' : database_api.list_comments,
+ 'database_api.find_comments' : database_api.find_comments
})
return methods
diff --git a/hive/utils/post.py b/hive/utils/post.py
index 9f299710f..8e4de28b7 100644
--- a/hive/utils/post.py
+++ b/hive/utils/post.py
@@ -60,7 +60,6 @@ def post_to_internal(post, post_id, level='insert', promoted=None):
('is_paidout', basic['is_paidout']),
('json', json.dumps(basic['json_metadata'])),
#('raw_json', json.dumps(legacy_data)),
- ('legacy_id', legacy_data['id']),
('parent_author', legacy_data['parent_author']),
('parent_permlink', legacy_data['parent_permlink']),
('curator_payout_value', legacy_data['curator_payout_value']),
diff --git a/tests/manual_tests/list_comments_by_permlink.py b/tests/manual_tests/list_comments_by_permlink.py
index a5ef3e333..8822f581b 100644
--- a/tests/manual_tests/list_comments_by_permlink.py
+++ b/tests/manual_tests/list_comments_by_permlink.py
@@ -1,5 +1,5 @@
#!/usr/bin/python3
-from .test_base import run_test
+from test_base import run_test
if __name__ == '__main__':
reference_hive_node_url = 'https://api.hive.blog'
diff --git a/tests/server/test_server_database_api.py b/tests/server/test_server_database_api.py
index 9e8c1691d..3bcb65bd2 100644
--- a/tests/server/test_server_database_api.py
+++ b/tests/server/test_server_database_api.py
@@ -12,7 +12,7 @@ def test_list_comments_by_cashout_time(client):
assert reference_data
assert test_data
assert len(reference_data) == len(test_data)
- to_compare = keys = ['author','permlink']
+ to_compare = ['author','permlink']
for idx in range(len(reference_data)):
for key in to_compare:
assert reference_data[idx][key] == test_data[idx][key]
@@ -24,7 +24,7 @@ def test_list_comments_by_permlink(client):
assert reference_data
assert test_data
assert len(reference_data) == len(test_data)
- to_compare = keys = ['author','permlink']
+ to_compare = ['author','permlink']
for idx in range(len(reference_data)):
for key in to_compare:
assert reference_data[idx][key] == test_data[idx][key]
@@ -35,7 +35,7 @@ def test_list_comments_by_root(client):
assert reference_data
assert test_data
assert len(reference_data) == len(test_data)
- to_compare = keys = ['author','permlink','root_author','root_permlink']
+ to_compare = ['author','permlink','root_author','root_permlink']
for idx in range(len(reference_data)):
for key in to_compare:
assert reference_data[idx][key] == test_data[idx][key]
@@ -46,7 +46,7 @@ def test_list_comments_by_parent(client):
assert reference_data
assert test_data
assert len(reference_data) == len(test_data)
- to_compare = keys = ['author','permlink','parent_author','parent_permlink']
+ to_compare = ['author','permlink','parent_author','parent_permlink']
for idx in range(len(reference_data)):
for key in to_compare:
assert reference_data[idx][key] == test_data[idx][key]
--
GitLab