Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • hive/hivemind
1 result
Show changes
Commits on Source (42)
Hide whitespace changes
Inline Side-by-side
Showing
with 262 additions and 185 deletions
.gitlab-ci.yaml
View file @ eec369bd
...@@ -50,6 +50,8 @@ hivemind_build: ...@@ -50,6 +50,8 @@ hivemind_build:
rules: rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: always when: always
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH == "develop"'
when: always
- when: always - when: always
tags: tags:
...@@ -70,7 +72,7 @@ hivemind_sync: ...@@ -70,7 +72,7 @@ hivemind_sync:
script: script:
- pip3 install --user --upgrade pip setuptools - pip3 install --user --upgrade pip setuptools
- scripts/ci_sync.sh "$HIVEMIND_DB_NAME" "$HIVEMIND_POSTGRESQL_CONNECTION_STRING" "$HIVEMIND_SOURCE_HIVED_URL" $HIVEMIND_MAX_BLOCK - scripts/ci_sync.sh "$HIVEMIND_DB_NAME" "$HIVEMIND_POSTGRESQL_CONNECTION_STRING" "$HIVEMIND_SOURCE_HIVED_URL" $HIVEMIND_MAX_BLOCK $HIVEMIND_HTTP_PORT
artifacts: artifacts:
paths: paths:
...@@ -81,6 +83,8 @@ hivemind_sync: ...@@ -81,6 +83,8 @@ hivemind_sync:
rules: rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: always when: always
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH == "develop"'
when: always
- if: '$CI_PIPELINE_SOURCE == "push"' - if: '$CI_PIPELINE_SOURCE == "push"'
when: manual when: manual
- when: on_success - when: on_success
...@@ -116,6 +120,8 @@ hivemind_start_server: ...@@ -116,6 +120,8 @@ hivemind_start_server:
rules: rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: always when: always
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH == "develop"'
when: always
- if: '$CI_PIPELINE_SOURCE == "push"' - if: '$CI_PIPELINE_SOURCE == "push"'
when: manual when: manual
- when: on_success - when: on_success
...@@ -131,7 +137,10 @@ hivemind_stop_server: ...@@ -131,7 +137,10 @@ hivemind_stop_server:
variables: variables:
GIT_STRATEGY: none GIT_STRATEGY: none
when: manual rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: always
- when: manual
script: script:
- scripts/ci_stop_server.sh hive_server.pid - scripts/ci_stop_server.sh hive_server.pid
......
hive/db/schema.py
View file @ eec369bd
...@@ -91,6 +91,7 @@ def build_metadata(): ...@@ -91,6 +91,7 @@ def build_metadata():
# core stats/indexes # core stats/indexes
sa.Column('payout', sa.types.DECIMAL(10, 3), nullable=False, server_default='0'), sa.Column('payout', sa.types.DECIMAL(10, 3), nullable=False, server_default='0'),
sa.Column('pending_payout', sa.types.DECIMAL(10, 3), nullable=False, server_default='0'),
sa.Column('payout_at', sa.DateTime, nullable=False, server_default='1990-01-01'), sa.Column('payout_at', sa.DateTime, nullable=False, server_default='1990-01-01'),
sa.Column('updated_at', sa.DateTime, nullable=False, server_default='1990-01-01'), sa.Column('updated_at', sa.DateTime, nullable=False, server_default='1990-01-01'),
sa.Column('is_paidout', BOOLEAN, nullable=False, server_default='0'), sa.Column('is_paidout', BOOLEAN, nullable=False, server_default='0'),
......
hive/indexer/blocks.py
View file @ eec369bd
...@@ -65,6 +65,7 @@ class Blocks: ...@@ -65,6 +65,7 @@ class Blocks:
Tags.flush() Tags.flush()
Votes.flush() Votes.flush()
cls._flush_blocks() cls._flush_blocks()
Posts.flush()
time_end = perf_counter() time_end = perf_counter()
log.info("[PROCESS BLOCK] %fs", time_end - time_start) log.info("[PROCESS BLOCK] %fs", time_end - time_start)
return ret return ret
...@@ -91,6 +92,7 @@ class Blocks: ...@@ -91,6 +92,7 @@ class Blocks:
Votes.flush() Votes.flush()
cls._flush_blocks() cls._flush_blocks()
Follow.flush(trx=False) Follow.flush(trx=False)
Posts.flush()
DB.query("COMMIT") DB.query("COMMIT")
time_end = perf_counter() time_end = perf_counter()
...@@ -99,37 +101,55 @@ class Blocks: ...@@ -99,37 +101,55 @@ class Blocks:
return cls.ops_stats return cls.ops_stats
@staticmethod @staticmethod
def prepare_vops(vopsList, date): def prepare_vops(comment_payout_ops, vopsList, date):
vote_ops = {} vote_ops = {}
comment_payout_ops = {} ops_stats = { 'author_reward_operation' : 0, 'comment_reward_operation' : 0, 'effective_comment_vote_operation' : 0, 'comment_payout_update_operation' : 0 }
for vop in vopsList: for vop in vopsList:
key = None key = None
val = None val = None
op_type = vop['type'] op_type = vop['type']
op_value = vop['value'] op_value = vop['value']
if op_type == 'curation_reward_operation': key = "{}/{}".format(op_value['author'], op_value['permlink'])
key = "{}/{}".format(op_value['comment_author'], op_value['comment_permlink'])
val = {'reward' : op_value['reward']} if op_type == 'author_reward_operation':
elif op_type == 'author_reward_operation': ops_stats[ 'author_reward_operation' ] += 1
key = "{}/{}".format(op_value['author'], op_value['permlink'])
val = {'hbd_payout':op_value['hbd_payout'], 'hive_payout':op_value['hive_payout'], 'vesting_payout':op_value['vesting_payout']} if key not in comment_payout_ops:
comment_payout_ops[key] = { 'author_reward_operation':None, 'comment_reward_operation':None, 'effective_comment_vote_operation':None, 'comment_payout_update_operation':None, 'date' : date }
comment_payout_ops[key][op_type] = op_value
elif op_type == 'comment_reward_operation': elif op_type == 'comment_reward_operation':
if('payout' not in op_value or op_value['payout'] is None): ops_stats[ 'comment_reward_operation' ] += 1
log.error("Broken op: `{}'".format(str(vop)))
key = "{}/{}".format(op_value['author'], op_value['permlink']) if key not in comment_payout_ops:
val = {'payout':op_value['payout'], 'author_rewards':op_value['author_rewards']} comment_payout_ops[key] = { 'author_reward_operation':None, 'comment_reward_operation':None, 'effective_comment_vote_operation':None, 'comment_payout_update_operation':None, 'date' : date }
comment_payout_ops[key]['effective_comment_vote_operation'] = None
comment_payout_ops[key][op_type] = op_value
elif op_type == 'effective_comment_vote_operation': elif op_type == 'effective_comment_vote_operation':
ops_stats[ 'effective_comment_vote_operation' ] += 1
key_vote = "{}/{}/{}".format(op_value['voter'], op_value['author'], op_value['permlink']) key_vote = "{}/{}/{}".format(op_value['voter'], op_value['author'], op_value['permlink'])
vote_ops[ key_vote ] = op_value vote_ops[ key_vote ] = op_value
if key is not None and val is not None: if key not in comment_payout_ops:
if key in comment_payout_ops: comment_payout_ops[key] = { 'author_reward_operation':None, 'comment_reward_operation':None, 'effective_comment_vote_operation':None, 'comment_payout_update_operation':None, 'date' : date }
comment_payout_ops[key].append({op_type:val})
else: comment_payout_ops[key][op_type] = op_value
comment_payout_ops[key] = [{op_type:val}]
elif op_type == 'comment_payout_update_operation':
ops_stats[ 'comment_payout_update_operation' ] += 1
if key not in comment_payout_ops:
comment_payout_ops[key] = { 'author_reward_operation':None, 'comment_reward_operation':None, 'effective_comment_vote_operation':None, 'comment_payout_update_operation':None, 'date' : date }
return (vote_ops, comment_payout_ops) comment_payout_ops[key][op_type] = op_value
return (vote_ops, ops_stats)
@classmethod @classmethod
...@@ -187,8 +207,7 @@ class Blocks: ...@@ -187,8 +207,7 @@ class Blocks:
if not is_initial_sync: if not is_initial_sync:
Accounts.dirty(op['author']) # lite - rep Accounts.dirty(op['author']) # lite - rep
Accounts.dirty(op['voter']) # lite - stats Accounts.dirty(op['voter']) # lite - stats
update_comment_pending_payouts.append([op['author'], op['permlink']]) Votes.vote_op(op)
Votes.vote_op(op)
# misc ops # misc ops
elif op_type == 'transfer_operation': elif op_type == 'transfer_operation':
...@@ -207,27 +226,21 @@ class Blocks: ...@@ -207,27 +226,21 @@ class Blocks:
custom_ops_stats = CustomOp.process_ops(json_ops, num, cls._head_block_date) custom_ops_stats = CustomOp.process_ops(json_ops, num, cls._head_block_date)
cls.ops_stats = Blocks.merge_ops_stats(cls.ops_stats, custom_ops_stats) cls.ops_stats = Blocks.merge_ops_stats(cls.ops_stats, custom_ops_stats)
if update_comment_pending_payouts: vote_ops = None
payout_ops_stat = Posts.update_comment_pending_payouts(hived, update_comment_pending_payouts) comment_payout_stats = None
cls.ops_stats = Blocks.merge_ops_stats(cls.ops_stats, payout_ops_stat)
# virtual ops
comment_payout_ops = {}
vote_ops = {}
empty_vops = (vote_ops, comment_payout_ops)
if is_initial_sync: if is_initial_sync:
(vote_ops, comment_payout_ops) = virtual_operations[num] if num in virtual_operations else empty_vops if num in virtual_operations:
(vote_ops, comment_payout_stats) = Blocks.prepare_vops(Posts.comment_payout_ops, virtual_operations[num], cls._head_block_date)
else: else:
vops = hived.get_virtual_operations(num) vops = hived.get_virtual_operations(num)
(vote_ops, comment_payout_ops) = Blocks.prepare_vops(vops, cls._head_block_date) (vote_ops, comment_payout_stats) = Blocks.prepare_vops(Posts.comment_payout_ops, vops, cls._head_block_date)
for k, v in vote_ops.items(): if vote_ops is not None:
Votes.effective_comment_vote_op(k, v, cls._head_block_date) for k, v in vote_ops.items():
Votes.effective_comment_vote_op(k, v, cls._head_block_date)
if comment_payout_ops: if Posts.comment_payout_ops:
comment_payout_stats = Posts.comment_payout_op(comment_payout_ops, cls._head_block_date)
cls.ops_stats = Blocks.merge_ops_stats(cls.ops_stats, comment_payout_stats) cls.ops_stats = Blocks.merge_ops_stats(cls.ops_stats, comment_payout_stats)
cls._head_block_date = block_date cls._head_block_date = block_date
......
hive/indexer/posts.py
View file @ eec369bd
...@@ -16,7 +16,7 @@ from hive.indexer.community import Community, START_DATE ...@@ -16,7 +16,7 @@ from hive.indexer.community import Community, START_DATE
from hive.indexer.notify import Notify from hive.indexer.notify import Notify
from hive.indexer.post_data_cache import PostDataCache from hive.indexer.post_data_cache import PostDataCache
from hive.indexer.tags import Tags from hive.indexer.tags import Tags
from hive.utils.normalize import legacy_amount, asset_to_hbd_hive from hive.utils.normalize import sbd_amount, legacy_amount, asset_to_hbd_hive
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
DB = Db.instance() DB = Db.instance()
...@@ -30,6 +30,9 @@ class Posts: ...@@ -30,6 +30,9 @@ class Posts:
_hits = 0 _hits = 0
_miss = 0 _miss = 0
comment_payout_ops = {}
_comment_payout_ops = []
@classmethod @classmethod
def last_id(cls): def last_id(cls):
"""Get the last indexed post id.""" """Get the last indexed post id."""
...@@ -146,21 +149,23 @@ class Posts: ...@@ -146,21 +149,23 @@ class Posts:
cls._insert_feed_cache(result, block_date) cls._insert_feed_cache(result, block_date)
@classmethod @classmethod
def comment_payout_op(cls, ops, date): def flush_into_db(cls):
ops_stats = {}
sql = """ sql = """
UPDATE hive_posts AS ihp SET UPDATE hive_posts AS ihp SET
total_payout_value = data_source.total_payout_value, total_payout_value = COALESCE( data_source.total_payout_value, ihp.total_payout_value ),
curator_payout_value = data_source.curator_payout_value, curator_payout_value = COALESCE( data_source.curator_payout_value, ihp.curator_payout_value ),
author_rewards = data_source.author_rewards, author_rewards = COALESCE( CAST( data_source.author_rewards as INT8 ), ihp.author_rewards ),
author_rewards_hive = data_source.author_rewards_hive, author_rewards_hive = COALESCE( CAST( data_source.author_rewards_hive as INT8 ), ihp.author_rewards_hive ),
author_rewards_hbd = data_source.author_rewards_hbd, author_rewards_hbd = COALESCE( CAST( data_source.author_rewards_hbd as INT8 ), ihp.author_rewards_hbd ),
author_rewards_vests = data_source.author_rewards_vests, author_rewards_vests = COALESCE( CAST( data_source.author_rewards_vests as INT8 ), ihp.author_rewards_vests ),
last_payout = data_source.last_payout, payout = COALESCE( CAST( data_source.payout as DECIMAL ), ihp.payout ),
cashout_time = data_source.cashout_time, pending_payout = COALESCE( CAST( data_source.pending_payout as DECIMAL ), ihp.pending_payout ),
is_paidout = true payout_at = COALESCE( CAST( data_source.payout_at as TIMESTAMP ), ihp.payout_at ),
updated_at = data_source.updated_at,
FROM last_payout = COALESCE( CAST( data_source.last_payout as TIMESTAMP ), ihp.last_payout ),
cashout_time = COALESCE( CAST( data_source.cashout_time as TIMESTAMP ), ihp.cashout_time ),
is_paidout = COALESCE( CAST( data_source.is_paidout as BOOLEAN ), ihp.is_paidout )
FROM
( (
SELECT ha_a.id as author_id, hpd_p.id as permlink_id, SELECT ha_a.id as author_id, hpd_p.id as permlink_id,
t.total_payout_value, t.total_payout_value,
...@@ -169,8 +174,13 @@ class Posts: ...@@ -169,8 +174,13 @@ class Posts:
t.author_rewards_hive, t.author_rewards_hive,
t.author_rewards_hbd, t.author_rewards_hbd,
t.author_rewards_vests, t.author_rewards_vests,
t.payout,
t.pending_payout,
t.payout_at,
t.updated_at,
t.last_payout, t.last_payout,
t.cashout_time t.cashout_time,
t.is_paidout
from from
( (
VALUES VALUES
...@@ -183,70 +193,142 @@ class Posts: ...@@ -183,70 +193,142 @@ class Posts:
author_rewards_hive, author_rewards_hive,
author_rewards_hbd, author_rewards_hbd,
author_rewards_vests, author_rewards_vests,
payout,
pending_payout,
payout_at,
updated_at,
last_payout, last_payout,
cashout_time) cashout_time,
is_paidout)
INNER JOIN hive_accounts ha_a ON ha_a.name = t.author INNER JOIN hive_accounts ha_a ON ha_a.name = t.author
INNER JOIN hive_permlink_data hpd_p ON hpd_p.permlink = t.permlink INNER JOIN hive_permlink_data hpd_p ON hpd_p.permlink = t.permlink
) as data_source(author_id, permlink_id, total_payout_value) ) as data_source(author_id, permlink_id, total_payout_value)
WHERE ihp.permlink_id = data_source.permlink_id and ihp.author_id = data_source.author_id WHERE ihp.permlink_id = data_source.permlink_id and ihp.author_id = data_source.author_id
""" """
def chunks(lst, n):
"""Yield successive n-sized chunks from lst."""
for i in range(0, len(lst), n):
yield lst[i:i + n]
for chunk in chunks(cls._comment_payout_ops, 1000):
values_str = ','.join(chunk)
actual_query = sql.format(values_str)
DB.query(actual_query)
cls._comment_payout_ops.clear()
values = [] @classmethod
def comment_payout_op(cls):
values_limit = 1000 values_limit = 1000
""" Process comment payment operations """ """ Process comment payment operations """
for k, v in ops.items(): for k, v in cls.comment_payout_ops.items():
author, permlink = k.split("/") author = None
# total payout to curators permlink = None
curator_rewards_sum = 0
# author payouts # author payouts
author_rewards = 0 author_rewards = None
author_rewards_hive = 0 author_rewards_hive = None
author_rewards_hbd = 0 author_rewards_hbd = None
author_rewards_vests = 0 author_rewards_vests = None
# total payout for comment # total payout for comment
comment_author_reward = None #comment_author_reward = None
for operation in v: curators_vesting_payout = None
for op, value in operation.items(): total_payout_value = None;
if op in ops_stats: curator_payout_value = None;
ops_stats[op] += 1 #beneficiary_payout_value = None;
else:
ops_stats[op] = 1 payout = None
pending_payout = None
if op == 'curation_reward_operation':
curator_rewards_sum = curator_rewards_sum + int(value['reward']['amount']) payout_at = None
elif op == 'author_reward_operation': last_payout = None
author_rewards_hive = value['hive_payout']['amount'] cashout_time = None
author_rewards_hbd = value['hbd_payout']['amount']
author_rewards_vests = value['vesting_payout']['amount'] is_paidout = None
elif op == 'comment_reward_operation':
comment_author_reward = value['payout'] date = v[ 'date' ]
author_rewards = value['author_rewards']
curator_rewards = {'amount' : str(curator_rewards_sum), 'precision': 6, 'nai': '@@000000037'} if v[ 'author_reward_operation' ] is not None:
value = v[ 'author_reward_operation' ]
values.append("('{}', '{}', '{}', '{}', {}, {}, {}, {}, '{}'::timestamp, '{}'::timestamp)".format(author, permlink, author_rewards_hive = value['hive_payout']['amount']
legacy_amount(comment_author_reward), # total_payout_value author_rewards_hbd = value['hbd_payout']['amount']
legacy_amount(curator_rewards), #curator_payout_value author_rewards_vests = value['vesting_payout']['amount']
author_rewards, curators_vesting_payout = value['curators_vesting_payout']['amount']
author_rewards_hive, if author is None:
author_rewards_hbd, author = value['author']
author_rewards_vests, permlink = value['permlink']
date, #last_payout
date #cashout_time if v[ 'comment_reward_operation' ] is not None:
)) value = v[ 'comment_reward_operation' ]
#comment_author_reward = value['payout']
if len(values) >= values_limit: author_rewards = value['author_rewards']
values_str = ','.join(values) total_payout_value = value['total_payout_value']
actual_query = sql.format(values_str) curator_payout_value = value['curator_payout_value']
DB.query(actual_query) #beneficiary_payout_value = value['beneficiary_payout_value']
values.clear()
payout = sum([ sbd_amount(total_payout_value), sbd_amount(curator_payout_value) ])
if len(values) > 0: pending_payout = 0
values_str = ','.join(values)
actual_query = sql.format(values_str) if author is None:
DB.query(actual_query) author = value['author']
values.clear() permlink = value['permlink']
return ops_stats
if v[ 'effective_comment_vote_operation' ] is not None:
value = v[ 'effective_comment_vote_operation' ]
pending_payout = sbd_amount( value['pending_payout'] )
if author is None:
author = value['author']
permlink = value['permlink']
if v[ 'comment_payout_update_operation' ] is not None:
value = v[ 'comment_payout_update_operation' ]
is_paidout = True
#Payout didn't generate any payments
if v[ 'comment_reward_operation' ] is None:
author_rewards = 0
total_payout_value = "0.000 HBD"
curator_payout_value = "0.000 HBD"
payout = 0
pending_payout = 0
if author is None:
author = value['author']
permlink = value['permlink']
#Calculations of all dates
if ( is_paidout is not None ):
payout_at = date
last_payout = date
cashout_time = "1969-12-31T23:59:59"
else:
if ( total_payout_value is not None ):
payout_at = date #Here should be `cashout_time`
last_payout = date
cls._comment_payout_ops.append("('{}', '{}', {}, {}, {}, {}, {}, {}, {}, {}, {}, '{}'::timestamp, {}, {}, {})".format(
author,
permlink,
"NULL" if ( total_payout_value is None ) else ( "'{}'".format( legacy_amount(total_payout_value) ) ),
"NULL" if ( curator_payout_value is None ) else ( "'{}'".format( legacy_amount(curator_payout_value) ) ),
"NULL" if ( author_rewards is None ) else author_rewards,
"NULL" if ( author_rewards_hive is None ) else author_rewards_hive,
"NULL" if ( author_rewards_hbd is None ) else author_rewards_hbd,
"NULL" if ( author_rewards_vests is None ) else author_rewards_vests,
"NULL" if ( payout is None ) else payout,
"NULL" if ( pending_payout is None ) else pending_payout,
"NULL" if ( payout_at is None ) else ( "'{}'::timestamp".format( payout_at ) ),
date,#updated_at
"NULL" if ( last_payout is None ) else ( "'{}'::timestamp".format( last_payout ) ),
"NULL" if ( cashout_time is None ) else ( "'{}'::timestamp".format( cashout_time ) ),
"NULL" if ( is_paidout is None ) else is_paidout ))
cls.comment_payout_ops.clear()
@classmethod @classmethod
def update_child_count(cls, child_id, op='+'): def update_child_count(cls, child_id, op='+'):
...@@ -328,60 +410,6 @@ class Posts: ...@@ -328,60 +410,6 @@ class Posts:
# force parent child recount when child is deleted # force parent child recount when child is deleted
cls.update_child_count(pid, '-') cls.update_child_count(pid, '-')
@classmethod
def update_comment_pending_payouts(cls, hived, posts):
comment_pending_payouts = hived.get_comment_pending_payouts(posts)
for comment_pending_payout in comment_pending_payouts:
if 'cashout_info' in comment_pending_payout:
cpp = comment_pending_payout['cashout_info']
sql = """UPDATE
hive_posts
SET
total_payout_value = :total_payout_value,
curator_payout_value = :curator_payout_value,
max_accepted_payout = :max_accepted_payout,
author_rewards = :author_rewards,
children_abs_rshares = :children_abs_rshares,
abs_rshares = :abs_rshares,
vote_rshares = :vote_rshares,
net_votes = :net_votes,
active = :active,
last_payout = :last_payout,
cashout_time = :cashout_time,
max_cashout_time = :max_cashout_time,
percent_hbd = :percent_hbd,
reward_weight = :reward_weight,
allow_replies = :allow_replies,
allow_votes = :allow_votes,
allow_curation_rewards = :allow_curation_rewards
WHERE id = (
SELECT hp.id
FROM hive_posts hp
INNER JOIN hive_accounts ha_a ON ha_a.id = hp.author_id
INNER JOIN hive_permlink_data hpd_p ON hpd_p.id = hp.permlink_id
WHERE ha_a.name = :author AND hpd_p.permlink = :permlink
)
"""
DB.query(sql, total_payout_value=legacy_amount(cpp['total_payout_value']),
curator_payout_value=legacy_amount(cpp['curator_payout_value']),
max_accepted_payout=legacy_amount(cpp['max_accepted_payout']),
author_rewards=cpp['author_rewards'],
children_abs_rshares=cpp['children_abs_rshares'],
abs_rshares=cpp['abs_rshares'],
vote_rshares=cpp['vote_rshares'],
net_votes=cpp['net_votes'],
active=cpp['active'],
last_payout=cpp['last_payout'],
cashout_time=cpp['cashout_time'],
max_cashout_time=cpp['max_cashout_time'],
percent_hbd=cpp['percent_hbd'],
reward_weight=cpp['reward_weight'],
allow_replies=cpp['allow_replies'],
allow_votes=cpp['allow_votes'],
allow_curation_rewards=cpp['allow_curation_rewards'],
author=comment_pending_payout['author'], permlink=comment_pending_payout['permlink'])
@classmethod @classmethod
def _insert_feed_cache(cls, result, date): def _insert_feed_cache(cls, result, date):
"""Insert the new post into feed cache if it's not a comment.""" """Insert the new post into feed cache if it's not a comment."""
...@@ -430,3 +458,9 @@ class Posts: ...@@ -430,3 +458,9 @@ class Posts:
new_body = new_body_def new_body = new_body_def
return new_body return new_body
@classmethod
def flush(cls):
cls.comment_payout_op()
cls.flush_into_db()
hive/indexer/sync.py
View file @ eec369bd
...@@ -45,7 +45,7 @@ def prepare_vops(vops_by_block): ...@@ -45,7 +45,7 @@ def prepare_vops(vops_by_block):
for blockNum, blockDict in vops_by_block.items(): for blockNum, blockDict in vops_by_block.items():
vopsList = blockDict['ops'] vopsList = blockDict['ops']
date = blockDict['timestamp'] date = blockDict['timestamp']
preparedVops[blockNum] = Blocks.prepare_vops(vopsList, date) preparedVops[blockNum] = vopsList
return preparedVops return preparedVops
......
hive/indexer/votes.py
View file @ eec369bd
...@@ -77,11 +77,12 @@ class Votes: ...@@ -77,11 +77,12 @@ class Votes:
log.info("Updating data in '_votes_data' using effective comment") log.info("Updating data in '_votes_data' using effective comment")
raise "Fatal error" raise "Fatal error"
if key in cls._votes_data: assert key in cls._votes_data
cls._votes_data[key]["vote_percent"] = vop["vote_percent"]
cls._votes_data[key]["weight"] = vop["weight"] cls._votes_data[key]["vote_percent"] = vop["vote_percent"]
cls._votes_data[key]["rshares"] = vop["rshares"] cls._votes_data[key]["weight"] = vop["weight"]
cls._votes_data[key]["last_update"] = date cls._votes_data[key]["rshares"] = vop["rshares"]
cls._votes_data[key]["last_update"] = date
@classmethod @classmethod
def flush(cls): def flush(cls):
......
hive/server/common/payout_stats.py
View file @ eec369bd
...@@ -39,23 +39,23 @@ class PayoutStats: ...@@ -39,23 +39,23 @@ class PayoutStats:
sql = """ sql = """
SELECT community_id, SELECT community_id,
ha.name as author, ha.name as author,
SUM(payout) payout, SUM( payout + pending_payout ) payout,
COUNT(*) posts, COUNT(*) posts,
NULL authors NULL authors
FROM hive_posts FROM hive_posts
INNER JOIN hive_accounts ha ON ha.id = hive_posts.author_id INNER JOIN hive_accounts ha ON ha.id = hive_posts.author_id
WHERE is_paidout = '0' WHERE is_paidout = '0' and is_deleted = false
GROUP BY community_id, author GROUP BY community_id, author
UNION ALL UNION ALL
SELECT community_id, SELECT community_id,
NULL author, NULL author,
SUM(payout) payout, SUM( payout + pending_payout ) payout,
COUNT(*) posts, COUNT(*) posts,
COUNT(DISTINCT(author_id)) authors COUNT(DISTINCT(author_id)) authors
FROM hive_posts FROM hive_posts
WHERE is_paidout = '0' WHERE is_paidout = '0' and is_deleted = false
GROUP BY community_id GROUP BY community_id
""" """
......
hive/server/condenser_api/tags.py
View file @ eec369bd
...@@ -29,10 +29,10 @@ async def get_trending_tags(context, start_tag: str = '', limit: int = 250): ...@@ -29,10 +29,10 @@ async def get_trending_tags(context, start_tag: str = '', limit: int = 250):
if start_tag: if start_tag:
seek = """ seek = """
HAVING SUM(payout) <= ( HAVING SUM(payout + pending_payout) <= (
SELECT SUM(payout) SELECT SUM(payout + pending_payout)
FROM hive_posts FROM hive_posts
WHERE is_paidout = '0' WHERE is_paidout = '0' and is_deleted = false
AND category_id = (SELECT id FROM hive_category_data WHERE category = :start_tag)) AND category_id = (SELECT id FROM hive_category_data WHERE category = :start_tag))
""" """
else: else:
...@@ -42,11 +42,11 @@ async def get_trending_tags(context, start_tag: str = '', limit: int = 250): ...@@ -42,11 +42,11 @@ async def get_trending_tags(context, start_tag: str = '', limit: int = 250):
SELECT (SELECT category FROM hive_category_data WHERE id = category_id) as category, SELECT (SELECT category FROM hive_category_data WHERE id = category_id) as category,
COUNT(*) AS total_posts, COUNT(*) AS total_posts,
SUM(CASE WHEN depth = 0 THEN 1 ELSE 0 END) AS top_posts, SUM(CASE WHEN depth = 0 THEN 1 ELSE 0 END) AS top_posts,
SUM(payout) AS total_payouts SUM(payout + pending_payout) AS total_payouts
FROM hive_posts FROM hive_posts
WHERE is_paidout = '0' WHERE is_paidout = '0' and is_deleted = false
GROUP BY category %s GROUP BY category %s
ORDER BY SUM(payout) DESC ORDER BY SUM(payout + pending_payout) DESC
LIMIT :limit LIMIT :limit
""" % seek """ % seek
......
hive/server/hive_api/stats.py
View file @ eec369bd
...@@ -46,4 +46,4 @@ async def get_payout_stats(context, limit=250): ...@@ -46,4 +46,4 @@ async def get_payout_stats(context, limit=250):
WHERE community_id IS NULL AND author IS NULL""" WHERE community_id IS NULL AND author IS NULL"""
blog_ttl = await db.query_one(sql) blog_ttl = await db.query_one(sql)
return dict(items=items, total=float(total), blogs=float(blog_ttl)) return dict(items=items, total=float(total if total is not None else 0.), blogs=float(blog_ttl if blog_ttl is not None else 0.))
hive/steem/client.py
View file @ eec369bd
...@@ -145,7 +145,7 @@ class SteemClient: ...@@ -145,7 +145,7 @@ class SteemClient:
def get_virtual_operations(self, block): def get_virtual_operations(self, block):
""" Get virtual ops from block """ """ Get virtual ops from block """
result = self.__exec('get_ops_in_block', {"block_num":block, "only_virtual":True}) result = self.__exec('get_ops_in_block', {"block_num":block, "only_virtual":True})
tracked_ops = ['curation_reward_operation', 'author_reward_operation', 'comment_reward_operation', 'effective_comment_vote_operation'] tracked_ops = ['author_reward_operation', 'comment_reward_operation', 'effective_comment_vote_operation', 'comment_payout_update_operation']
ret = [] ret = []
result = result['ops'] if 'ops' in result else [] result = result['ops'] if 'ops' in result else []
for vop in result: for vop in result:
...@@ -162,12 +162,12 @@ class SteemClient: ...@@ -162,12 +162,12 @@ class SteemClient:
#According to definition of hive::plugins::acount_history::enum_vops_filter: #According to definition of hive::plugins::acount_history::enum_vops_filter:
author_reward_operation = 0x000002 author_reward_operation = 0x000002
curation_reward_operation = 0x000004
comment_reward_operation = 0x000008 comment_reward_operation = 0x000008
effective_comment_vote_operation = 0x400000 effective_comment_vote_operation = 0x400000
comment_payout_update_operation = 0x000800
tracked_ops_filter = curation_reward_operation | author_reward_operation | comment_reward_operation | effective_comment_vote_operation tracked_ops_filter = author_reward_operation | comment_reward_operation | effective_comment_vote_operation | comment_payout_update_operation
tracked_ops = ['curation_reward_operation', 'author_reward_operation', 'comment_reward_operation', 'effective_comment_vote_operation'] tracked_ops = ['author_reward_operation', 'comment_reward_operation', 'effective_comment_vote_operation', 'comment_payout_update_operation']
resume_on_operation = 0 resume_on_operation = 0
......
scripts/ci_start_api_smoketest.sh
View file @ eec369bd
...@@ -4,9 +4,9 @@ set -e ...@@ -4,9 +4,9 @@ set -e
cd tests/tests_api/hivemind/tavern cd tests/tests_api/hivemind/tavern
pip3 install --user jsondiff pip3 install --user jsondiff==1.2.0
pip3 install --user tavern pip3 install --user tavern==1.2.2
pip3 install --user pytest pip3 install --user pytest==6.0.1
export HIVEMIND_ADDRESS=$1 export HIVEMIND_ADDRESS=$1
export HIVEMIND_PORT=$2 export HIVEMIND_PORT=$2
......
scripts/ci_start_server.sh
View file @ eec369bd
...@@ -29,6 +29,12 @@ if [ -f hive_server.pid ]; then ...@@ -29,6 +29,12 @@ if [ -f hive_server.pid ]; then
rm hive_server.pid; rm hive_server.pid;
fi fi
fuser $HIVEMIND_HTTP_PORT/tcp -k -INT || true
sleep 5
fuser $HIVEMIND_HTTP_PORT/tcp -k -KILL || true
sleep 5
ls -l dist/* ls -l dist/*
rm -rf ./local-site rm -rf ./local-site
mkdir -p `python3 -m site --user-site` mkdir -p `python3 -m site --user-site`
......
scripts/ci_stop_server.sh
View file @ eec369bd
...@@ -6,7 +6,9 @@ set -e ...@@ -6,7 +6,9 @@ set -e
if [ -f $1 ]; then if [ -f $1 ]; then
PID=`cat $1`; PID=`cat $1`;
kill -SIGINT $PID; kill -SIGINT $PID || true;
sleep 5
kill -9 $PID || true;
else else
echo Specified pid file: $1 does not exists.; echo Specified pid file: $1 does not exists.;
fi fi
......
scripts/ci_sync.sh
View file @ eec369bd
...@@ -7,6 +7,7 @@ HIVEMIND_DB_NAME=$1 ...@@ -7,6 +7,7 @@ HIVEMIND_DB_NAME=$1
HIVEMIND_POSTGRESQL_CONNECTION_STRING=$2 HIVEMIND_POSTGRESQL_CONNECTION_STRING=$2
HIVEMIND_SOURCE_HIVED_URL=$3 HIVEMIND_SOURCE_HIVED_URL=$3
HIVEMIND_MAX_BLOCK=$4 HIVEMIND_MAX_BLOCK=$4
HIVEMIND_HTTP_PORT=$5
PYTHONUSERBASE=./local-site PYTHONUSERBASE=./local-site
...@@ -29,6 +30,16 @@ kill -SIGINT `pgrep -f "$HIVE_NAME sync"` || true; ...@@ -29,6 +30,16 @@ kill -SIGINT `pgrep -f "$HIVE_NAME sync"` || true;
sleep 5 sleep 5
kill -9 `pgrep -f "$HIVE_NAME sync"` || true; kill -9 `pgrep -f "$HIVE_NAME sync"` || true;
kill -SIGINT `pgrep -f "$HIVE_NAME server"` || true;
sleep 5
kill -9 `pgrep -f "$HIVE_NAME server"` || true;
fuser $HIVEMIND_HTTP_PORT/tcp -k -INT || true
sleep 5
fuser $HIVEMIND_HTTP_PORT/tcp -k -KILL || true
sleep 5
ls -l dist/* ls -l dist/*
rm -rf ./local-site rm -rf ./local-site
mkdir -p `python3 -m site --user-site` mkdir -p `python3 -m site --user-site`
......
tests_api @ 113cb05c
Compare ef779872...113cb05c
Subproject commit ef779872ccc5f48fbc9dabe012644210626cada8 Subproject commit 113cb05cc2dc35fe6e6e149fc3ecb8ae286d4cc4