From 538a27895e13ff485af3e0bb2c6b1a8a3cb635d0 Mon Sep 17 00:00:00 2001
From: Dariusz Kedzierski <dkedzierski@syncad.com>
Date: Mon, 8 Jun 2020 10:44:48 +0200
Subject: [PATCH] Initial sync fix, some minor fixes
---
hive/db/db_state.py | 64 ++++++++++++++---------------
hive/db/schema.py | 32 ++++++++++-----
hive/indexer/blocks.py | 3 +-
hive/indexer/posts.py | 6 +++
hive/server/database_api/methods.py | 2 +-
5 files changed, 61 insertions(+), 46 deletions(-)
diff --git a/hive/db/db_state.py b/hive/db/db_state.py
index bf57a8403..6d31b94ed 100644
--- a/hive/db/db_state.py
+++ b/hive/db/db_state.py
@@ -85,25 +85,25 @@ class DbState:
@classmethod
def _disableable_indexes(cls):
to_locate = [
- 'hive_posts_ix3', # (author, depth, id)
- 'hive_posts_ix4', # (parent_id, id, is_deleted=0)
- 'hive_posts_ix5', # (community_id>0, is_pinned=1)
+ #'hive_posts_ix3', # (author, depth, id)
+ #'hive_posts_ix4', # (parent_id, id, is_deleted=0)
+ #'hive_posts_ix5', # (community_id>0, is_pinned=1)
'hive_follows_ix5a', # (following, state, created_at, follower)
'hive_follows_ix5b', # (follower, state, created_at, following)
'hive_reblogs_ix1', # (post_id, account, created_at)
- 'hive_posts_cache_ix6a', # (sc_trend, post_id, paidout=0)
- 'hive_posts_cache_ix6b', # (post_id, sc_trend, paidout=0)
- 'hive_posts_cache_ix7a', # (sc_hot, post_id, paidout=0)
- 'hive_posts_cache_ix7b', # (post_id, sc_hot, paidout=0)
- 'hive_posts_cache_ix8', # (category, payout, depth, paidout=0)
- 'hive_posts_cache_ix9a', # (depth, payout, post_id, paidout=0)
- 'hive_posts_cache_ix9b', # (category, depth, payout, post_id, paidout=0)
- 'hive_posts_cache_ix10', # (post_id, payout, gray=1, payout>0)
- 'hive_posts_cache_ix30', # API: community trend
- 'hive_posts_cache_ix31', # API: community hot
- 'hive_posts_cache_ix32', # API: community created
- 'hive_posts_cache_ix33', # API: community payout
- 'hive_posts_cache_ix34', # API: community muted
+ #'hive_posts_cache_ix6a', # (sc_trend, post_id, paidout=0)
+ #'hive_posts_cache_ix6b', # (post_id, sc_trend, paidout=0)
+ #'hive_posts_cache_ix7a', # (sc_hot, post_id, paidout=0)
+ #'hive_posts_cache_ix7b', # (post_id, sc_hot, paidout=0)
+ #'hive_posts_cache_ix8', # (category, payout, depth, paidout=0)
+ #'hive_posts_cache_ix9a', # (depth, payout, post_id, paidout=0)
+ #'hive_posts_cache_ix9b', # (category, depth, payout, post_id, paidout=0)
+ #'hive_posts_cache_ix10', # (post_id, payout, gray=1, payout>0)
+ #'hive_posts_cache_ix30', # API: community trend
+ #'hive_posts_cache_ix31', # API: community hot
+ #'hive_posts_cache_ix32', # API: community created
+ #'hive_posts_cache_ix33', # API: community payout
+ #'hive_posts_cache_ix34', # API: community muted
'hive_accounts_ix3', # (vote_weight, name VPO)
'hive_accounts_ix4', # (id, name)
'hive_accounts_ix5', # (cached_at, name)
@@ -231,11 +231,11 @@ class DbState:
if cls._ver == 6:
cls.db().query("DROP INDEX hive_posts_cache_ix6")
- cls.db().query("CREATE INDEX hive_posts_cache_ix6a ON hive_posts_cache (sc_trend, post_id) WHERE is_paidout = '0'")
- cls.db().query("CREATE INDEX hive_posts_cache_ix6b ON hive_posts_cache (post_id, sc_trend) WHERE is_paidout = '0'")
- cls.db().query("DROP INDEX hive_posts_cache_ix7")
- cls.db().query("CREATE INDEX hive_posts_cache_ix7a ON hive_posts_cache (sc_hot, post_id) WHERE is_paidout = '0'")
- cls.db().query("CREATE INDEX hive_posts_cache_ix7b ON hive_posts_cache (post_id, sc_hot) WHERE is_paidout = '0'")
+ #cls.db().query("CREATE INDEX hive_posts_cache_ix6a ON hive_posts_cache (sc_trend, post_id) WHERE is_paidout = '0'")
+ #cls.db().query("CREATE INDEX hive_posts_cache_ix6b ON hive_posts_cache (post_id, sc_trend) WHERE is_paidout = '0'")
+ #cls.db().query("DROP INDEX hive_posts_cache_ix7")
+ #cls.db().query("CREATE INDEX hive_posts_cache_ix7a ON hive_posts_cache (sc_hot, post_id) WHERE is_paidout = '0'")
+ #cls.db().query("CREATE INDEX hive_posts_cache_ix7b ON hive_posts_cache (post_id, sc_hot) WHERE is_paidout = '0'")
cls._set_ver(7)
if cls._ver == 7:
@@ -257,9 +257,9 @@ class DbState:
cls._set_ver(10)
if cls._ver == 10:
- cls.db().query("CREATE INDEX hive_posts_cache_ix8 ON hive_posts_cache (category, payout, depth) WHERE is_paidout = '0'")
- cls.db().query("CREATE INDEX hive_posts_cache_ix9a ON hive_posts_cache (depth, payout, post_id) WHERE is_paidout = '0'")
- cls.db().query("CREATE INDEX hive_posts_cache_ix9b ON hive_posts_cache (category, depth, payout, post_id) WHERE is_paidout = '0'")
+ #cls.db().query("CREATE INDEX hive_posts_cache_ix8 ON hive_posts_cache (category, payout, depth) WHERE is_paidout = '0'")
+ #cls.db().query("CREATE INDEX hive_posts_cache_ix9a ON hive_posts_cache (depth, payout, post_id) WHERE is_paidout = '0'")
+ #cls.db().query("CREATE INDEX hive_posts_cache_ix9b ON hive_posts_cache (category, depth, payout, post_id) WHERE is_paidout = '0'")
cls._set_ver(11)
if cls._ver == 11:
@@ -286,13 +286,13 @@ class DbState:
if cls._ver == 13:
sqls = ("CREATE INDEX hive_posts_ix5 ON hive_posts (id) WHERE is_pinned = '1' AND is_deleted = '0'",
- "CREATE INDEX hive_posts_ix6 ON hive_posts (community_id, id) WHERE community_id IS NOT NULL AND is_pinned = '1' AND is_deleted = '0'",
- "CREATE INDEX hive_posts_cache_ix10 ON hive_posts_cache (post_id, payout) WHERE is_grayed = '1' AND payout > 0",
- "CREATE INDEX hive_posts_cache_ix30 ON hive_posts_cache (community_id, sc_trend, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND depth = 0",
- "CREATE INDEX hive_posts_cache_ix31 ON hive_posts_cache (community_id, sc_hot, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND depth = 0",
- "CREATE INDEX hive_posts_cache_ix32 ON hive_posts_cache (community_id, created_at, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND depth = 0",
- "CREATE INDEX hive_posts_cache_ix33 ON hive_posts_cache (community_id, payout, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND is_paidout = '0'",
- "CREATE INDEX hive_posts_cache_ix34 ON hive_posts_cache (community_id, payout, post_id) WHERE community_id IS NOT NULL AND is_grayed = '1' AND is_paidout = '0'")
+ "CREATE INDEX hive_posts_ix6 ON hive_posts (community_id, id) WHERE community_id IS NOT NULL AND is_pinned = '1' AND is_deleted = '0'",)
+ #"CREATE INDEX hive_posts_cache_ix10 ON hive_posts_cache (post_id, payout) WHERE is_grayed = '1' AND payout > 0",
+ #"CREATE INDEX hive_posts_cache_ix30 ON hive_posts_cache (community_id, sc_trend, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND depth = 0",
+ #"CREATE INDEX hive_posts_cache_ix31 ON hive_posts_cache (community_id, sc_hot, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND depth = 0",
+ #"CREATE INDEX hive_posts_cache_ix32 ON hive_posts_cache (community_id, created_at, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND depth = 0",
+ #"CREATE INDEX hive_posts_cache_ix33 ON hive_posts_cache (community_id, payout, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND is_paidout = '0'",
+ #"CREATE INDEX hive_posts_cache_ix34 ON hive_posts_cache (community_id, payout, post_id) WHERE community_id IS NOT NULL AND is_grayed = '1' AND is_paidout = '0'")
for sql in sqls:
cls.db().query(sql)
cls._set_ver(14)
@@ -302,7 +302,7 @@ class DbState:
cls.db().query("ALTER TABLE hive_communities ADD COLUMN category VARCHAR(32) NOT NULL DEFAULT ''")
cls.db().query("ALTER TABLE hive_communities ADD COLUMN avatar_url VARCHAR(1024) NOT NULL DEFAULT ''")
cls.db().query("ALTER TABLE hive_communities ADD COLUMN num_authors INTEGER NOT NULL DEFAULT 0")
- cls.db().query("CREATE INDEX hive_posts_cache_ix20 ON hive_posts_cache (community_id, author, payout, post_id) WHERE is_paidout = '0'")
+ #cls.db().query("CREATE INDEX hive_posts_cache_ix20 ON hive_posts_cache (community_id, author, payout, post_id) WHERE is_paidout = '0'")
cls._set_ver(15)
if cls._ver == 15:
diff --git a/hive/db/schema.py b/hive/db/schema.py
index 4d474b485..0d30b2e7c 100644
--- a/hive/db/schema.py
+++ b/hive/db/schema.py
@@ -129,18 +129,28 @@ def build_metadata():
sa.ForeignKeyConstraint(['parent_id'], ['hive_posts.id'], name='hive_posts_fk3'),
sa.UniqueConstraint('author_id', 'permlink_id', name='hive_posts_ux1'),
sa.Index('hive_posts_ix3', 'author_id', 'depth', 'id', postgresql_where=sql_text("is_deleted = '0'")), # API: author blog/comments
- sa.Index('hive_posts_ix4', 'parent_id DESC NULLS LAST', 'id'), #postgresql_where=sql_text("is_deleted = '0'")), # API: fetching children #[JES] We decided we want the full index since posts can be deleted/undeleted
sa.Index('hive_posts_ix5', 'id', postgresql_where=sql_text("is_pinned = '1' AND is_deleted = '0'")), # API: pinned post status
- sa.Index('hive_posts_ix6', 'community_id', 'id', postgresql_where=sql_text("community_id IS NOT NULL AND is_pinned = '1' AND is_deleted = '0'")), # API: community pinned
- sa.UniqueConstraint('author_id', 'permlink_id', name='hive_posts_ux1'),
+ sa.Index('hive_posts_depth_idx', 'depth'),
+ sa.Index('hive_posts_parent_id_idx', 'parent_id'),
+ sa.Index('hive_posts_community_id_idx', 'community_id'),
+
+ sa.Index('hive_posts_category_id_idx', 'category_id'),
+ sa.Index('hive_posts_payout_at_idx', 'payout_at'),
+ sa.Index('hive_posts_payout_at_idx2', 'payout_at', postgresql_where=sql_text("is_paidout = '0'")),
+ sa.Index('hive_posts_payout_idx', 'payout'),
+ sa.Index('hive_posts_promoted_idx', 'promoted'),
+ sa.Index('hive_posts_sc_trend_idx', 'sc_trend'),
+ sa.Index('hive_posts_sc_hot_idx', 'sc_hot'),
+ sa.Index('hive_posts_created_at_idx', 'created_at'),
+ sa.UniqueConstraint('author_id', 'permlink_id', name='hive_posts_ux1')
)
sa.Table(
'hive_post_data', metadata,
- sa.column('id', sa.Integer, nullable=False),
- sa.column('title', VARCHAR(255), nullable=False),
- sa.column('preview', VARCHAR(1024), nullable=False),
- sa.column('img_url', VARCHAR(1024), nullable=False),
+ sa.Column('id', sa.Integer, nullable=False),
+ sa.Column('title', VARCHAR(255), nullable=False),
+ sa.Column('preview', VARCHAR(1024), nullable=False),
+ sa.Column('img_url', VARCHAR(1024), nullable=False),
sa.Column('body', TEXT),
sa.Column('votes', TEXT),
sa.Column('json', sa.JSON)
@@ -148,15 +158,15 @@ def build_metadata():
sa.Table(
'hive_permlink_data', metadata,
- sa.column('id', sa.Integer, primary_key=True),
- sa.column('permlink', sa.String(255), nullable=False),
+ sa.Column('id', sa.Integer, primary_key=True),
+ sa.Column('permlink', sa.String(255), nullable=False),
sa.UniqueConstraint('permlink', name='hive_permlink_data_permlink')
)
sa.Table(
'hive_category_data', metadata,
- sa.column('id', sa.Integer, primary_key=True),
- sa.column('category', sa.String(255), nullable=False),
+ sa.Column('id', sa.Integer, primary_key=True),
+ sa.Column('category', sa.String(255), nullable=False),
sa.UniqueConstraint('category', name='hive_category_data_category')
)
diff --git a/hive/indexer/blocks.py b/hive/indexer/blocks.py
index bfee74c8d..74b9053f6 100644
--- a/hive/indexer/blocks.py
+++ b/hive/indexer/blocks.py
@@ -109,8 +109,7 @@ class Blocks:
if not is_initial_sync:
Accounts.dirty(op['author']) # lite - rep
Accounts.dirty(op['voter']) # lite - stats
- CachedPost.vote(op['author'], op['permlink'],
- None, op['voter'])
+ Posts.vote_op(op)
# misc ops
elif op_type == 'transfer_operation':
diff --git a/hive/indexer/posts.py b/hive/indexer/posts.py
index 2b8cb058b..68883b1ca 100644
--- a/hive/indexer/posts.py
+++ b/hive/indexer/posts.py
@@ -114,6 +114,12 @@ class Posts:
# post exists but was deleted. time to reinstate.
cls.undelete(op, block_date, pid)
+ @classmethod
+ def vote_op(cls, op):
+ """ Vote operation processing """
+ pid = cls.get_id(op['author'], op['permlink'])
+ assert pid, "Post does not exists in the database"
+
@classmethod
def insert(cls, op, date):
"""Inserts new post records."""
diff --git a/hive/server/database_api/methods.py b/hive/server/database_api/methods.py
index 9368927ca..096653b50 100644
--- a/hive/server/database_api/methods.py
+++ b/hive/server/database_api/methods.py
@@ -63,7 +63,7 @@ async def get_post_id_by_author_and_permlink(db, author: str, permlink: str, lim
"""
result = await db.query_row(sql, author=author, permlink=permlink, limit=limit)
if result is not None:
- return int(result.get('post_id', 0))
+ return int(result.get('id', 0))
return 0
@return_error_info
--
GitLab