Skip to content
Snippets Groups Projects
Commit 538a2789 authored by Dariusz Kędzierski's avatar Dariusz Kędzierski
Browse files

Initial sync fix, some minor fixes

parent e742b287
No related branches found
No related tags found
5 merge requests!456Release candidate v1 24,!230Setup monitoring with pghero,!135Enable postgres monitoring on CI server,!16Dk issue 3 concurrent block query rebase,!15Dk issue 3 concurrent block query
Hide whitespace changes
Inline Side-by-side
hive/db/db_state.py
+ 32
− 32
View file @ 538a2789
......@@ -85,25 +85,25 @@ class DbState:
@classmethod
def _disableable_indexes(cls):
to_locate = [
'hive_posts_ix3', # (author, depth, id)
'hive_posts_ix4', # (parent_id, id, is_deleted=0)
'hive_posts_ix5', # (community_id>0, is_pinned=1)
#'hive_posts_ix3', # (author, depth, id)
#'hive_posts_ix4', # (parent_id, id, is_deleted=0)
#'hive_posts_ix5', # (community_id>0, is_pinned=1)
'hive_follows_ix5a', # (following, state, created_at, follower)
'hive_follows_ix5b', # (follower, state, created_at, following)
'hive_reblogs_ix1', # (post_id, account, created_at)
'hive_posts_cache_ix6a', # (sc_trend, post_id, paidout=0)
'hive_posts_cache_ix6b', # (post_id, sc_trend, paidout=0)
'hive_posts_cache_ix7a', # (sc_hot, post_id, paidout=0)
'hive_posts_cache_ix7b', # (post_id, sc_hot, paidout=0)
'hive_posts_cache_ix8', # (category, payout, depth, paidout=0)
'hive_posts_cache_ix9a', # (depth, payout, post_id, paidout=0)
'hive_posts_cache_ix9b', # (category, depth, payout, post_id, paidout=0)
'hive_posts_cache_ix10', # (post_id, payout, gray=1, payout>0)
'hive_posts_cache_ix30', # API: community trend
'hive_posts_cache_ix31', # API: community hot
'hive_posts_cache_ix32', # API: community created
'hive_posts_cache_ix33', # API: community payout
'hive_posts_cache_ix34', # API: community muted
#'hive_posts_cache_ix6a', # (sc_trend, post_id, paidout=0)
#'hive_posts_cache_ix6b', # (post_id, sc_trend, paidout=0)
#'hive_posts_cache_ix7a', # (sc_hot, post_id, paidout=0)
#'hive_posts_cache_ix7b', # (post_id, sc_hot, paidout=0)
#'hive_posts_cache_ix8', # (category, payout, depth, paidout=0)
#'hive_posts_cache_ix9a', # (depth, payout, post_id, paidout=0)
#'hive_posts_cache_ix9b', # (category, depth, payout, post_id, paidout=0)
#'hive_posts_cache_ix10', # (post_id, payout, gray=1, payout>0)
#'hive_posts_cache_ix30', # API: community trend
#'hive_posts_cache_ix31', # API: community hot
#'hive_posts_cache_ix32', # API: community created
#'hive_posts_cache_ix33', # API: community payout
#'hive_posts_cache_ix34', # API: community muted
'hive_accounts_ix3', # (vote_weight, name VPO)
'hive_accounts_ix4', # (id, name)
'hive_accounts_ix5', # (cached_at, name)
......@@ -231,11 +231,11 @@ class DbState:
if cls._ver == 6:
cls.db().query("DROP INDEX hive_posts_cache_ix6")
cls.db().query("CREATE INDEX hive_posts_cache_ix6a ON hive_posts_cache (sc_trend, post_id) WHERE is_paidout = '0'")
cls.db().query("CREATE INDEX hive_posts_cache_ix6b ON hive_posts_cache (post_id, sc_trend) WHERE is_paidout = '0'")
cls.db().query("DROP INDEX hive_posts_cache_ix7")
cls.db().query("CREATE INDEX hive_posts_cache_ix7a ON hive_posts_cache (sc_hot, post_id) WHERE is_paidout = '0'")
cls.db().query("CREATE INDEX hive_posts_cache_ix7b ON hive_posts_cache (post_id, sc_hot) WHERE is_paidout = '0'")
#cls.db().query("CREATE INDEX hive_posts_cache_ix6a ON hive_posts_cache (sc_trend, post_id) WHERE is_paidout = '0'")
#cls.db().query("CREATE INDEX hive_posts_cache_ix6b ON hive_posts_cache (post_id, sc_trend) WHERE is_paidout = '0'")
#cls.db().query("DROP INDEX hive_posts_cache_ix7")
#cls.db().query("CREATE INDEX hive_posts_cache_ix7a ON hive_posts_cache (sc_hot, post_id) WHERE is_paidout = '0'")
#cls.db().query("CREATE INDEX hive_posts_cache_ix7b ON hive_posts_cache (post_id, sc_hot) WHERE is_paidout = '0'")
cls._set_ver(7)
if cls._ver == 7:
......@@ -257,9 +257,9 @@ class DbState:
cls._set_ver(10)
if cls._ver == 10:
cls.db().query("CREATE INDEX hive_posts_cache_ix8 ON hive_posts_cache (category, payout, depth) WHERE is_paidout = '0'")
cls.db().query("CREATE INDEX hive_posts_cache_ix9a ON hive_posts_cache (depth, payout, post_id) WHERE is_paidout = '0'")
cls.db().query("CREATE INDEX hive_posts_cache_ix9b ON hive_posts_cache (category, depth, payout, post_id) WHERE is_paidout = '0'")
#cls.db().query("CREATE INDEX hive_posts_cache_ix8 ON hive_posts_cache (category, payout, depth) WHERE is_paidout = '0'")
#cls.db().query("CREATE INDEX hive_posts_cache_ix9a ON hive_posts_cache (depth, payout, post_id) WHERE is_paidout = '0'")
#cls.db().query("CREATE INDEX hive_posts_cache_ix9b ON hive_posts_cache (category, depth, payout, post_id) WHERE is_paidout = '0'")
cls._set_ver(11)
if cls._ver == 11:
......@@ -286,13 +286,13 @@ class DbState:
if cls._ver == 13:
sqls = ("CREATE INDEX hive_posts_ix5 ON hive_posts (id) WHERE is_pinned = '1' AND is_deleted = '0'",
"CREATE INDEX hive_posts_ix6 ON hive_posts (community_id, id) WHERE community_id IS NOT NULL AND is_pinned = '1' AND is_deleted = '0'",
"CREATE INDEX hive_posts_cache_ix10 ON hive_posts_cache (post_id, payout) WHERE is_grayed = '1' AND payout > 0",
"CREATE INDEX hive_posts_cache_ix30 ON hive_posts_cache (community_id, sc_trend, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND depth = 0",
"CREATE INDEX hive_posts_cache_ix31 ON hive_posts_cache (community_id, sc_hot, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND depth = 0",
"CREATE INDEX hive_posts_cache_ix32 ON hive_posts_cache (community_id, created_at, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND depth = 0",
"CREATE INDEX hive_posts_cache_ix33 ON hive_posts_cache (community_id, payout, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND is_paidout = '0'",
"CREATE INDEX hive_posts_cache_ix34 ON hive_posts_cache (community_id, payout, post_id) WHERE community_id IS NOT NULL AND is_grayed = '1' AND is_paidout = '0'")
"CREATE INDEX hive_posts_ix6 ON hive_posts (community_id, id) WHERE community_id IS NOT NULL AND is_pinned = '1' AND is_deleted = '0'",)
#"CREATE INDEX hive_posts_cache_ix10 ON hive_posts_cache (post_id, payout) WHERE is_grayed = '1' AND payout > 0",
#"CREATE INDEX hive_posts_cache_ix30 ON hive_posts_cache (community_id, sc_trend, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND depth = 0",
#"CREATE INDEX hive_posts_cache_ix31 ON hive_posts_cache (community_id, sc_hot, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND depth = 0",
#"CREATE INDEX hive_posts_cache_ix32 ON hive_posts_cache (community_id, created_at, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND depth = 0",
#"CREATE INDEX hive_posts_cache_ix33 ON hive_posts_cache (community_id, payout, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND is_paidout = '0'",
#"CREATE INDEX hive_posts_cache_ix34 ON hive_posts_cache (community_id, payout, post_id) WHERE community_id IS NOT NULL AND is_grayed = '1' AND is_paidout = '0'")
for sql in sqls:
cls.db().query(sql)
cls._set_ver(14)
......@@ -302,7 +302,7 @@ class DbState:
cls.db().query("ALTER TABLE hive_communities ADD COLUMN category VARCHAR(32) NOT NULL DEFAULT ''")
cls.db().query("ALTER TABLE hive_communities ADD COLUMN avatar_url VARCHAR(1024) NOT NULL DEFAULT ''")
cls.db().query("ALTER TABLE hive_communities ADD COLUMN num_authors INTEGER NOT NULL DEFAULT 0")
cls.db().query("CREATE INDEX hive_posts_cache_ix20 ON hive_posts_cache (community_id, author, payout, post_id) WHERE is_paidout = '0'")
#cls.db().query("CREATE INDEX hive_posts_cache_ix20 ON hive_posts_cache (community_id, author, payout, post_id) WHERE is_paidout = '0'")
cls._set_ver(15)
if cls._ver == 15:
......
hive/db/schema.py
+ 21
− 11
View file @ 538a2789
......@@ -129,18 +129,28 @@ def build_metadata():
sa.ForeignKeyConstraint(['parent_id'], ['hive_posts.id'], name='hive_posts_fk3'),
sa.UniqueConstraint('author_id', 'permlink_id', name='hive_posts_ux1'),
sa.Index('hive_posts_ix3', 'author_id', 'depth', 'id', postgresql_where=sql_text("is_deleted = '0'")), # API: author blog/comments
sa.Index('hive_posts_ix4', 'parent_id DESC NULLS LAST', 'id'), #postgresql_where=sql_text("is_deleted = '0'")), # API: fetching children #[JES] We decided we want the full index since posts can be deleted/undeleted
sa.Index('hive_posts_ix5', 'id', postgresql_where=sql_text("is_pinned = '1' AND is_deleted = '0'")), # API: pinned post status
sa.Index('hive_posts_ix6', 'community_id', 'id', postgresql_where=sql_text("community_id IS NOT NULL AND is_pinned = '1' AND is_deleted = '0'")), # API: community pinned
sa.UniqueConstraint('author_id', 'permlink_id', name='hive_posts_ux1'),
sa.Index('hive_posts_depth_idx', 'depth'),
sa.Index('hive_posts_parent_id_idx', 'parent_id'),
sa.Index('hive_posts_community_id_idx', 'community_id'),
sa.Index('hive_posts_category_id_idx', 'category_id'),
sa.Index('hive_posts_payout_at_idx', 'payout_at'),
sa.Index('hive_posts_payout_at_idx2', 'payout_at', postgresql_where=sql_text("is_paidout = '0'")),
sa.Index('hive_posts_payout_idx', 'payout'),
sa.Index('hive_posts_promoted_idx', 'promoted'),
sa.Index('hive_posts_sc_trend_idx', 'sc_trend'),
sa.Index('hive_posts_sc_hot_idx', 'sc_hot'),
sa.Index('hive_posts_created_at_idx', 'created_at'),
sa.UniqueConstraint('author_id', 'permlink_id', name='hive_posts_ux1')
)
sa.Table(
'hive_post_data', metadata,
sa.column('id', sa.Integer, nullable=False),
sa.column('title', VARCHAR(255), nullable=False),
sa.column('preview', VARCHAR(1024), nullable=False),
sa.column('img_url', VARCHAR(1024), nullable=False),
sa.Column('id', sa.Integer, nullable=False),
sa.Column('title', VARCHAR(255), nullable=False),
sa.Column('preview', VARCHAR(1024), nullable=False),
sa.Column('img_url', VARCHAR(1024), nullable=False),
sa.Column('body', TEXT),
sa.Column('votes', TEXT),
sa.Column('json', sa.JSON)
......@@ -148,15 +158,15 @@ def build_metadata():
sa.Table(
'hive_permlink_data', metadata,
sa.column('id', sa.Integer, primary_key=True),
sa.column('permlink', sa.String(255), nullable=False),
sa.Column('id', sa.Integer, primary_key=True),
sa.Column('permlink', sa.String(255), nullable=False),
sa.UniqueConstraint('permlink', name='hive_permlink_data_permlink')
)
sa.Table(
'hive_category_data', metadata,
sa.column('id', sa.Integer, primary_key=True),
sa.column('category', sa.String(255), nullable=False),
sa.Column('id', sa.Integer, primary_key=True),
sa.Column('category', sa.String(255), nullable=False),
sa.UniqueConstraint('category', name='hive_category_data_category')
)
......
hive/indexer/blocks.py
+ 1
− 2
View file @ 538a2789
......@@ -109,8 +109,7 @@ class Blocks:
if not is_initial_sync:
Accounts.dirty(op['author']) # lite - rep
Accounts.dirty(op['voter']) # lite - stats
CachedPost.vote(op['author'], op['permlink'],
None, op['voter'])
Posts.vote_op(op)
# misc ops
elif op_type == 'transfer_operation':
......
hive/indexer/posts.py
+ 6
− 0
View file @ 538a2789
......@@ -114,6 +114,12 @@ class Posts:
# post exists but was deleted. time to reinstate.
cls.undelete(op, block_date, pid)
@classmethod
def vote_op(cls, op):
""" Vote operation processing """
pid = cls.get_id(op['author'], op['permlink'])
assert pid, "Post does not exists in the database"
@classmethod
def insert(cls, op, date):
"""Inserts new post records."""
......
hive/server/database_api/methods.py
+ 1
− 1
View file @ 538a2789
......@@ -63,7 +63,7 @@ async def get_post_id_by_author_and_permlink(db, author: str, permlink: str, lim
"""
result = await db.query_row(sql, author=author, permlink=permlink, limit=limit)
if result is not None:
return int(result.get('post_id', 0))
return int(result.get('id', 0))
return 0
@return_error_info
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment