Skip to content
Snippets Groups Projects
Commit 538a2789 authored by Dariusz Kędzierski's avatar Dariusz Kędzierski
Browse files

Initial sync fix, some minor fixes

parent e742b287
No related branches found
No related tags found
5 merge requests!456Release candidate v1 24,!230Setup monitoring with pghero,!135Enable postgres monitoring on CI server,!16Dk issue 3 concurrent block query rebase,!15Dk issue 3 concurrent block query
Hide whitespace changes
Inline Side-by-side
hive/db/db_state.py
+ 32
− 32
View file @ 538a2789
...@@ -85,25 +85,25 @@ class DbState: ...@@ -85,25 +85,25 @@ class DbState:
@classmethod @classmethod
def _disableable_indexes(cls): def _disableable_indexes(cls):
to_locate = [ to_locate = [
'hive_posts_ix3', # (author, depth, id) #'hive_posts_ix3', # (author, depth, id)
'hive_posts_ix4', # (parent_id, id, is_deleted=0) #'hive_posts_ix4', # (parent_id, id, is_deleted=0)
'hive_posts_ix5', # (community_id>0, is_pinned=1) #'hive_posts_ix5', # (community_id>0, is_pinned=1)
'hive_follows_ix5a', # (following, state, created_at, follower) 'hive_follows_ix5a', # (following, state, created_at, follower)
'hive_follows_ix5b', # (follower, state, created_at, following) 'hive_follows_ix5b', # (follower, state, created_at, following)
'hive_reblogs_ix1', # (post_id, account, created_at) 'hive_reblogs_ix1', # (post_id, account, created_at)
'hive_posts_cache_ix6a', # (sc_trend, post_id, paidout=0) #'hive_posts_cache_ix6a', # (sc_trend, post_id, paidout=0)
'hive_posts_cache_ix6b', # (post_id, sc_trend, paidout=0) #'hive_posts_cache_ix6b', # (post_id, sc_trend, paidout=0)
'hive_posts_cache_ix7a', # (sc_hot, post_id, paidout=0) #'hive_posts_cache_ix7a', # (sc_hot, post_id, paidout=0)
'hive_posts_cache_ix7b', # (post_id, sc_hot, paidout=0) #'hive_posts_cache_ix7b', # (post_id, sc_hot, paidout=0)
'hive_posts_cache_ix8', # (category, payout, depth, paidout=0) #'hive_posts_cache_ix8', # (category, payout, depth, paidout=0)
'hive_posts_cache_ix9a', # (depth, payout, post_id, paidout=0) #'hive_posts_cache_ix9a', # (depth, payout, post_id, paidout=0)
'hive_posts_cache_ix9b', # (category, depth, payout, post_id, paidout=0) #'hive_posts_cache_ix9b', # (category, depth, payout, post_id, paidout=0)
'hive_posts_cache_ix10', # (post_id, payout, gray=1, payout>0) #'hive_posts_cache_ix10', # (post_id, payout, gray=1, payout>0)
'hive_posts_cache_ix30', # API: community trend #'hive_posts_cache_ix30', # API: community trend
'hive_posts_cache_ix31', # API: community hot #'hive_posts_cache_ix31', # API: community hot
'hive_posts_cache_ix32', # API: community created #'hive_posts_cache_ix32', # API: community created
'hive_posts_cache_ix33', # API: community payout #'hive_posts_cache_ix33', # API: community payout
'hive_posts_cache_ix34', # API: community muted #'hive_posts_cache_ix34', # API: community muted
'hive_accounts_ix3', # (vote_weight, name VPO) 'hive_accounts_ix3', # (vote_weight, name VPO)
'hive_accounts_ix4', # (id, name) 'hive_accounts_ix4', # (id, name)
'hive_accounts_ix5', # (cached_at, name) 'hive_accounts_ix5', # (cached_at, name)
...@@ -231,11 +231,11 @@ class DbState: ...@@ -231,11 +231,11 @@ class DbState:
if cls._ver == 6: if cls._ver == 6:
cls.db().query("DROP INDEX hive_posts_cache_ix6") cls.db().query("DROP INDEX hive_posts_cache_ix6")
cls.db().query("CREATE INDEX hive_posts_cache_ix6a ON hive_posts_cache (sc_trend, post_id) WHERE is_paidout = '0'") #cls.db().query("CREATE INDEX hive_posts_cache_ix6a ON hive_posts_cache (sc_trend, post_id) WHERE is_paidout = '0'")
cls.db().query("CREATE INDEX hive_posts_cache_ix6b ON hive_posts_cache (post_id, sc_trend) WHERE is_paidout = '0'") #cls.db().query("CREATE INDEX hive_posts_cache_ix6b ON hive_posts_cache (post_id, sc_trend) WHERE is_paidout = '0'")
cls.db().query("DROP INDEX hive_posts_cache_ix7") #cls.db().query("DROP INDEX hive_posts_cache_ix7")
cls.db().query("CREATE INDEX hive_posts_cache_ix7a ON hive_posts_cache (sc_hot, post_id) WHERE is_paidout = '0'") #cls.db().query("CREATE INDEX hive_posts_cache_ix7a ON hive_posts_cache (sc_hot, post_id) WHERE is_paidout = '0'")
cls.db().query("CREATE INDEX hive_posts_cache_ix7b ON hive_posts_cache (post_id, sc_hot) WHERE is_paidout = '0'") #cls.db().query("CREATE INDEX hive_posts_cache_ix7b ON hive_posts_cache (post_id, sc_hot) WHERE is_paidout = '0'")
cls._set_ver(7) cls._set_ver(7)
if cls._ver == 7: if cls._ver == 7:
...@@ -257,9 +257,9 @@ class DbState: ...@@ -257,9 +257,9 @@ class DbState:
cls._set_ver(10) cls._set_ver(10)
if cls._ver == 10: if cls._ver == 10:
cls.db().query("CREATE INDEX hive_posts_cache_ix8 ON hive_posts_cache (category, payout, depth) WHERE is_paidout = '0'") #cls.db().query("CREATE INDEX hive_posts_cache_ix8 ON hive_posts_cache (category, payout, depth) WHERE is_paidout = '0'")
cls.db().query("CREATE INDEX hive_posts_cache_ix9a ON hive_posts_cache (depth, payout, post_id) WHERE is_paidout = '0'") #cls.db().query("CREATE INDEX hive_posts_cache_ix9a ON hive_posts_cache (depth, payout, post_id) WHERE is_paidout = '0'")
cls.db().query("CREATE INDEX hive_posts_cache_ix9b ON hive_posts_cache (category, depth, payout, post_id) WHERE is_paidout = '0'") #cls.db().query("CREATE INDEX hive_posts_cache_ix9b ON hive_posts_cache (category, depth, payout, post_id) WHERE is_paidout = '0'")
cls._set_ver(11) cls._set_ver(11)
if cls._ver == 11: if cls._ver == 11:
...@@ -286,13 +286,13 @@ class DbState: ...@@ -286,13 +286,13 @@ class DbState:
if cls._ver == 13: if cls._ver == 13:
sqls = ("CREATE INDEX hive_posts_ix5 ON hive_posts (id) WHERE is_pinned = '1' AND is_deleted = '0'", sqls = ("CREATE INDEX hive_posts_ix5 ON hive_posts (id) WHERE is_pinned = '1' AND is_deleted = '0'",
"CREATE INDEX hive_posts_ix6 ON hive_posts (community_id, id) WHERE community_id IS NOT NULL AND is_pinned = '1' AND is_deleted = '0'", "CREATE INDEX hive_posts_ix6 ON hive_posts (community_id, id) WHERE community_id IS NOT NULL AND is_pinned = '1' AND is_deleted = '0'",)
"CREATE INDEX hive_posts_cache_ix10 ON hive_posts_cache (post_id, payout) WHERE is_grayed = '1' AND payout > 0", #"CREATE INDEX hive_posts_cache_ix10 ON hive_posts_cache (post_id, payout) WHERE is_grayed = '1' AND payout > 0",
"CREATE INDEX hive_posts_cache_ix30 ON hive_posts_cache (community_id, sc_trend, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND depth = 0", #"CREATE INDEX hive_posts_cache_ix30 ON hive_posts_cache (community_id, sc_trend, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND depth = 0",
"CREATE INDEX hive_posts_cache_ix31 ON hive_posts_cache (community_id, sc_hot, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND depth = 0", #"CREATE INDEX hive_posts_cache_ix31 ON hive_posts_cache (community_id, sc_hot, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND depth = 0",
"CREATE INDEX hive_posts_cache_ix32 ON hive_posts_cache (community_id, created_at, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND depth = 0", #"CREATE INDEX hive_posts_cache_ix32 ON hive_posts_cache (community_id, created_at, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND depth = 0",
"CREATE INDEX hive_posts_cache_ix33 ON hive_posts_cache (community_id, payout, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND is_paidout = '0'", #"CREATE INDEX hive_posts_cache_ix33 ON hive_posts_cache (community_id, payout, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND is_paidout = '0'",
"CREATE INDEX hive_posts_cache_ix34 ON hive_posts_cache (community_id, payout, post_id) WHERE community_id IS NOT NULL AND is_grayed = '1' AND is_paidout = '0'") #"CREATE INDEX hive_posts_cache_ix34 ON hive_posts_cache (community_id, payout, post_id) WHERE community_id IS NOT NULL AND is_grayed = '1' AND is_paidout = '0'")
for sql in sqls: for sql in sqls:
cls.db().query(sql) cls.db().query(sql)
cls._set_ver(14) cls._set_ver(14)
...@@ -302,7 +302,7 @@ class DbState: ...@@ -302,7 +302,7 @@ class DbState:
cls.db().query("ALTER TABLE hive_communities ADD COLUMN category VARCHAR(32) NOT NULL DEFAULT ''") cls.db().query("ALTER TABLE hive_communities ADD COLUMN category VARCHAR(32) NOT NULL DEFAULT ''")
cls.db().query("ALTER TABLE hive_communities ADD COLUMN avatar_url VARCHAR(1024) NOT NULL DEFAULT ''") cls.db().query("ALTER TABLE hive_communities ADD COLUMN avatar_url VARCHAR(1024) NOT NULL DEFAULT ''")
cls.db().query("ALTER TABLE hive_communities ADD COLUMN num_authors INTEGER NOT NULL DEFAULT 0") cls.db().query("ALTER TABLE hive_communities ADD COLUMN num_authors INTEGER NOT NULL DEFAULT 0")
cls.db().query("CREATE INDEX hive_posts_cache_ix20 ON hive_posts_cache (community_id, author, payout, post_id) WHERE is_paidout = '0'") #cls.db().query("CREATE INDEX hive_posts_cache_ix20 ON hive_posts_cache (community_id, author, payout, post_id) WHERE is_paidout = '0'")
cls._set_ver(15) cls._set_ver(15)
if cls._ver == 15: if cls._ver == 15:
......
hive/db/schema.py
+ 21
− 11
View file @ 538a2789
...@@ -129,18 +129,28 @@ def build_metadata(): ...@@ -129,18 +129,28 @@ def build_metadata():
sa.ForeignKeyConstraint(['parent_id'], ['hive_posts.id'], name='hive_posts_fk3'), sa.ForeignKeyConstraint(['parent_id'], ['hive_posts.id'], name='hive_posts_fk3'),
sa.UniqueConstraint('author_id', 'permlink_id', name='hive_posts_ux1'), sa.UniqueConstraint('author_id', 'permlink_id', name='hive_posts_ux1'),
sa.Index('hive_posts_ix3', 'author_id', 'depth', 'id', postgresql_where=sql_text("is_deleted = '0'")), # API: author blog/comments sa.Index('hive_posts_ix3', 'author_id', 'depth', 'id', postgresql_where=sql_text("is_deleted = '0'")), # API: author blog/comments
sa.Index('hive_posts_ix4', 'parent_id DESC NULLS LAST', 'id'), #postgresql_where=sql_text("is_deleted = '0'")), # API: fetching children #[JES] We decided we want the full index since posts can be deleted/undeleted
sa.Index('hive_posts_ix5', 'id', postgresql_where=sql_text("is_pinned = '1' AND is_deleted = '0'")), # API: pinned post status sa.Index('hive_posts_ix5', 'id', postgresql_where=sql_text("is_pinned = '1' AND is_deleted = '0'")), # API: pinned post status
sa.Index('hive_posts_ix6', 'community_id', 'id', postgresql_where=sql_text("community_id IS NOT NULL AND is_pinned = '1' AND is_deleted = '0'")), # API: community pinned sa.Index('hive_posts_depth_idx', 'depth'),
sa.UniqueConstraint('author_id', 'permlink_id', name='hive_posts_ux1'), sa.Index('hive_posts_parent_id_idx', 'parent_id'),
sa.Index('hive_posts_community_id_idx', 'community_id'),
sa.Index('hive_posts_category_id_idx', 'category_id'),
sa.Index('hive_posts_payout_at_idx', 'payout_at'),
sa.Index('hive_posts_payout_at_idx2', 'payout_at', postgresql_where=sql_text("is_paidout = '0'")),
sa.Index('hive_posts_payout_idx', 'payout'),
sa.Index('hive_posts_promoted_idx', 'promoted'),
sa.Index('hive_posts_sc_trend_idx', 'sc_trend'),
sa.Index('hive_posts_sc_hot_idx', 'sc_hot'),
sa.Index('hive_posts_created_at_idx', 'created_at'),
sa.UniqueConstraint('author_id', 'permlink_id', name='hive_posts_ux1')
) )
sa.Table( sa.Table(
'hive_post_data', metadata, 'hive_post_data', metadata,
sa.column('id', sa.Integer, nullable=False), sa.Column('id', sa.Integer, nullable=False),
sa.column('title', VARCHAR(255), nullable=False), sa.Column('title', VARCHAR(255), nullable=False),
sa.column('preview', VARCHAR(1024), nullable=False), sa.Column('preview', VARCHAR(1024), nullable=False),
sa.column('img_url', VARCHAR(1024), nullable=False), sa.Column('img_url', VARCHAR(1024), nullable=False),
sa.Column('body', TEXT), sa.Column('body', TEXT),
sa.Column('votes', TEXT), sa.Column('votes', TEXT),
sa.Column('json', sa.JSON) sa.Column('json', sa.JSON)
...@@ -148,15 +158,15 @@ def build_metadata(): ...@@ -148,15 +158,15 @@ def build_metadata():
sa.Table( sa.Table(
'hive_permlink_data', metadata, 'hive_permlink_data', metadata,
sa.column('id', sa.Integer, primary_key=True), sa.Column('id', sa.Integer, primary_key=True),
sa.column('permlink', sa.String(255), nullable=False), sa.Column('permlink', sa.String(255), nullable=False),
sa.UniqueConstraint('permlink', name='hive_permlink_data_permlink') sa.UniqueConstraint('permlink', name='hive_permlink_data_permlink')
) )
sa.Table( sa.Table(
'hive_category_data', metadata, 'hive_category_data', metadata,
sa.column('id', sa.Integer, primary_key=True), sa.Column('id', sa.Integer, primary_key=True),
sa.column('category', sa.String(255), nullable=False), sa.Column('category', sa.String(255), nullable=False),
sa.UniqueConstraint('category', name='hive_category_data_category') sa.UniqueConstraint('category', name='hive_category_data_category')
) )
......
hive/indexer/blocks.py
+ 1
− 2
View file @ 538a2789
...@@ -109,8 +109,7 @@ class Blocks: ...@@ -109,8 +109,7 @@ class Blocks:
if not is_initial_sync: if not is_initial_sync:
Accounts.dirty(op['author']) # lite - rep Accounts.dirty(op['author']) # lite - rep
Accounts.dirty(op['voter']) # lite - stats Accounts.dirty(op['voter']) # lite - stats
CachedPost.vote(op['author'], op['permlink'], Posts.vote_op(op)
None, op['voter'])
# misc ops # misc ops
elif op_type == 'transfer_operation': elif op_type == 'transfer_operation':
......
hive/indexer/posts.py
+ 6
− 0
View file @ 538a2789
...@@ -114,6 +114,12 @@ class Posts: ...@@ -114,6 +114,12 @@ class Posts:
# post exists but was deleted. time to reinstate. # post exists but was deleted. time to reinstate.
cls.undelete(op, block_date, pid) cls.undelete(op, block_date, pid)
@classmethod
def vote_op(cls, op):
""" Vote operation processing """
pid = cls.get_id(op['author'], op['permlink'])
assert pid, "Post does not exists in the database"
@classmethod @classmethod
def insert(cls, op, date): def insert(cls, op, date):
"""Inserts new post records.""" """Inserts new post records."""
......
hive/server/database_api/methods.py
+ 1
− 1
View file @ 538a2789
...@@ -63,7 +63,7 @@ async def get_post_id_by_author_and_permlink(db, author: str, permlink: str, lim ...@@ -63,7 +63,7 @@ async def get_post_id_by_author_and_permlink(db, author: str, permlink: str, lim
""" """
result = await db.query_row(sql, author=author, permlink=permlink, limit=limit) result = await db.query_row(sql, author=author, permlink=permlink, limit=limit)
if result is not None: if result is not None:
return int(result.get('post_id', 0)) return int(result.get('id', 0))
return 0 return 0
@return_error_info @return_error_info
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment