From df6fc91a6c75bb2fca1342dbd0ff5f415dd286e2 Mon Sep 17 00:00:00 2001
From: Dariusz Kedzierski <dkedzierski@syncad.com>
Date: Wed, 10 Jun 2020 01:24:55 +0200
Subject: [PATCH] [WIP] Support for comment payout virtual ops
---
hive/indexer/blocks.py | 25 +++++++++++++-
hive/indexer/posts.py | 51 ++++++++++++++++++++++++----
hive/indexer/sync.py | 4 ---
hive/server/bridge_api/methods.py | 1 -
hive/server/condenser_api/methods.py | 1 -
hive/server/database_api/methods.py | 2 +-
hive/steem/client.py | 18 ++++++----
hive/steem/http_client.py | 1 +
hive/utils/normalize.py | 9 +++++
hive/utils/stats.py | 3 +-
10 files changed, 94 insertions(+), 21 deletions(-)
diff --git a/hive/indexer/blocks.py b/hive/indexer/blocks.py
index 5da8f4f27..51c4a51ad 100644
--- a/hive/indexer/blocks.py
+++ b/hive/indexer/blocks.py
@@ -109,7 +109,7 @@ class Blocks:
if not is_initial_sync:
Accounts.dirty(op['author']) # lite - rep
Accounts.dirty(op['voter']) # lite - stats
- Posts.vote_op(op)
+ Posts.vote_op(hived, op)
# misc ops
elif op_type == 'transfer_operation':
@@ -117,6 +117,29 @@ class Blocks:
elif op_type == 'custom_json_operation':
json_ops.append(op)
+ comment_payout_ops = {}
+ for vop in hived.get_virtual_operations(num):
+ key = None
+ val = None
+ if vop['op']['type'] == 'curation_reward_operation':
+ key = "{}/{}".format(vop['op']['value']['comment_author'], vop['op']['value']['comment_permlink'])
+ val = {'reward' : vop['op']['value']['reward']}
+
+ if vop['op']['type'] == 'author_reward_operation':
+ key = "{}/{}".format(vop['op']['value']['author'], vop['op']['value']['permlink'])
+ val = {'hbd_payout':vop['op']['value']['hbd_payout'], 'hive_payout':vop['op']['value']['hive_payout'], 'vesting_payout':vop['op']['value']['vesting_payout']}
+
+ if vop['op']['type'] == 'comment_reward_operation':
+ key = "{}/{}".format(vop['op']['value']['author'], vop['op']['value']['permlink'])
+ val = {'payout':vop['op']['value']['payout']}
+
+ if key is not None and val is not None:
+ if key in comment_payout_ops:
+ comment_payout_ops[key].append({vop['op']['type']:val})
+ else:
+ comment_payout_ops[key] = [{vop['op']['type']:val}]
+ if comment_payout_ops:
+ Posts.comment_payout_op(comment_payout_ops, date)
CustomOp.process_ops(json_ops, num, date) # follow/reblog/community ops
return num
diff --git a/hive/indexer/posts.py b/hive/indexer/posts.py
index a52a08209..5cd140c3f 100644
--- a/hive/indexer/posts.py
+++ b/hive/indexer/posts.py
@@ -13,7 +13,7 @@ from hive.indexer.cached_post import CachedPost
from hive.indexer.feed_cache import FeedCache
from hive.indexer.community import Community, START_DATE
from hive.indexer.notify import Notify
-from hive.utils.normalize import legacy_amount
+from hive.utils.normalize import legacy_amount, parse_amount
log = logging.getLogger(__name__)
DB = Db.instance()
@@ -136,15 +136,57 @@ class Posts:
cls.undelete(op, block_date, pid)
@classmethod
- def vote_op(cls, op):
+ def vote_op(cls, hived, op):
""" Vote operation processing """
pid = cls.get_id(op['author'], op['permlink'])
assert pid, "Post does not exists in the database"
+ votes = hived.get_votes(op['author'], op['permlink'])
+ sql = """
+ UPDATE
+ hive_post_data
+ SET
+ votes = :votes
+ WHERE id = :id"""
+
+ DB.query(sql, id=pid, votes=dumps(votes))
+
+ @classmethod
+ def comment_payout_op(cls, ops, date):
+ """ Process comment payment operations """
+ for k, v in ops.items():
+ author, permlink = k.split("/")
+ pid = cls.get_id(author, permlink)
+ curator_rewards_sum = 0
+ author_rewards = ''
+ comment_author_reward = None
+ print(v)
+ for operation in v:
+ for op, value in operation.items():
+ if op == 'curation_reward_operation':
+ curator_rewards_sum = curator_rewards_sum + int(value['reward']['amount'])
+ if op == 'author_reward_operation':
+ author_rewards = "{}, {}, {}".format(legacy_amount(value['hbd_payout']), legacy_amount(value['hive_payout']), legacy_amount(value['vesting_payout']))
+ if op == 'comment_reward_operation':
+ comment_author_reward = value['payout']
+ curator_rewards = {'amount' : str(curator_rewards_sum), 'precision': 6, 'nai': '@@000000037'}
+ print("COMMENT OP REWARDS ==> {}/{} > AUTHOR > {} > CURATORS > {} > TOTAL > {}".format(author, permlink, author_rewards, legacy_amount(curator_rewards), legacy_amount(comment_author_reward)))
+ raise RuntimeError("Comment payout op")
+ sql = """UPDATE
+ hive_posts
+ SET
+ total_payout_value = :total_payout_value,
+ curator_payout_value = :curator_payout_value,
+ max_accepted_payout = :max_accepted_payout,
+ author_rewards = :author_rewards,
+ last_payout = :last_payout,
+ cashout_time = :cashout_time,
+ is_paidout = true
+ WHERE id = :id
+ """
@classmethod
def insert(cls, hived, op, date):
"""Inserts new post records."""
- print(op)
# inserting new post
# * Check for permlink, parent_permlink, root_permlink
@@ -153,7 +195,6 @@ class Posts:
# * insert post basic data
# * obtain id
# * insert post content data
- print(op)
# add permlinks to permlink table
for permlink in ['permlink', 'parent_permlink', 'root_permlink']:
@@ -190,8 +231,6 @@ class Posts:
)"""
sql += ";SELECT currval(pg_get_serial_sequence('hive_posts','id'))"
- print(post)
-
result = DB.query(sql, **post)
post['id'] = int(list(result)[0][0])
cls._set_id(op['author']+'/'+op['permlink'], post['id'])
diff --git a/hive/indexer/sync.py b/hive/indexer/sync.py
index e3734784e..142c32ec6 100644
--- a/hive/indexer/sync.py
+++ b/hive/indexer/sync.py
@@ -221,10 +221,6 @@ class Sync:
def _update_chain_state(self):
"""Update basic state props (head block, feed price) in db."""
state = self._steem.gdgp_extended()
- print("======================")
- print(state['steem_per_mvest'])
- print(state['usd_per_steem'])
- print(state['sbd_per_steem'])
self._db.query("""UPDATE hive_state SET block_num = :block_num,
steem_per_mvest = :spm, usd_per_steem = :ups,
sbd_per_steem = :sps, dgpo = :dgpo""",
diff --git a/hive/server/bridge_api/methods.py b/hive/server/bridge_api/methods.py
index a5489e367..9ec7a7205 100644
--- a/hive/server/bridge_api/methods.py
+++ b/hive/server/bridge_api/methods.py
@@ -285,7 +285,6 @@ async def get_account_posts(context, sort, account, start_author='', start_perml
sql = sql % """ """
posts = []
- print('sql is: ', sql)
sql_result = await db.query_all(sql, account=account, author=start_author, permlink=start_permlink, limit=limit)
for row in sql_result:
post = _condenser_post_object(row)
diff --git a/hive/server/condenser_api/methods.py b/hive/server/condenser_api/methods.py
index 56fa212a8..ea44ad5d6 100644
--- a/hive/server/condenser_api/methods.py
+++ b/hive/server/condenser_api/methods.py
@@ -554,7 +554,6 @@ async def _get_blog(db, account: str, start_index: int, limit: int = None):
@return_error_info
async def get_accounts(context, accounts: list):
"""Returns accounts data for accounts given in list"""
- print("Hivemind native get_accounts")
assert accounts, "Empty parameters are not supported"
assert len(accounts) < 1000, "Query exceeds limit"
diff --git a/hive/server/database_api/methods.py b/hive/server/database_api/methods.py
index 096653b50..afffc8c3f 100644
--- a/hive/server/database_api/methods.py
+++ b/hive/server/database_api/methods.py
@@ -69,7 +69,7 @@ async def get_post_id_by_author_and_permlink(db, author: str, permlink: str, lim
@return_error_info
async def list_comments(context, start: list, limit: int, order: str):
"""Returns all comments, starting with the specified options."""
- print("Hivemind native list_comments")
+
supported_order_list = ['by_cashout_time', 'by_permlink', 'by_root', 'by_parent', 'by_update', 'by_author_last_update']
assert order in supported_order_list, "Unsupported order, valid orders: {}".format(", ".join(supported_order_list))
limit = valid_limit(limit, 1000)
diff --git a/hive/steem/client.py b/hive/steem/client.py
index c5a281e7e..a86acc483 100644
--- a/hive/steem/client.py
+++ b/hive/steem/client.py
@@ -1,4 +1,5 @@
"""Tight and reliable steem API client for hive indexer."""
+import logging
from time import perf_counter as perf
from decimal import Decimal
@@ -8,6 +9,8 @@ from hive.utils.normalize import parse_amount, steem_amount, vests_amount
from hive.steem.http_client import HttpClient
from hive.steem.block.stream import BlockStream
+logger = logging.getLogger(__name__)
+
class SteemClient:
"""Handles upstream calls to jussi/steemd, with batching and retrying."""
# dangerous default value of url but it should be fine since we are not writting to it
@@ -21,10 +24,11 @@ class SteemClient:
self._max_workers = max_workers
self._client = dict()
for endpoint, endpoint_url in url.items():
- print("Endpoint {} will be routed to node {}".format(endpoint, endpoint_url))
+ logger.info("Endpoint %s will be routed to node %s" % (endpoint, endpoint_url))
self._client[endpoint] = HttpClient(nodes=[endpoint_url])
- def get_accounts(self, accounts):
+ def get_accounts(self, acc):
+ accounts = [v for v in acc if v != '']
"""Fetch multiple accounts by name."""
assert accounts, "no accounts passed to get_accounts"
assert len(accounts) <= 1000, "max 1000 accounts"
@@ -84,7 +88,6 @@ class SteemClient:
def gdgp_extended(self):
"""Get dynamic global props without the cruft plus useful bits."""
dgpo = self._gdgp()
- print(dgpo)
# remove unused/deprecated keys
unused = ['total_pow', 'num_pow_witnesses', 'confidential_supply',
@@ -102,7 +105,6 @@ class SteemClient:
@staticmethod
def _get_steem_per_mvest(dgpo):
- print("DGPO: ", dgpo)
steem = steem_amount(dgpo['total_vesting_fund_hive'])
mvests = vests_amount(dgpo['total_vesting_shares']) / Decimal(1e6)
return "%.6f" % (steem / mvests)
@@ -119,7 +121,7 @@ class SteemClient:
def _get_steem_price(self):
orders = self.__exec('get_order_book', [1])
- if orders['asks'] and orders[bids]:
+ if orders['asks'] and orders['bids']:
ask = Decimal(orders['asks'][0]['real_price'])
bid = Decimal(orders['bids'][0]['real_price'])
price = (ask + bid) / 2
@@ -140,10 +142,14 @@ class SteemClient:
return [blocks[x] for x in block_nums]
+ def get_virtual_operations(self, block):
+ """ Get virtual ops from block """
+ ret = self.__exec('get_ops_in_block', {"block_num":block, "only_virtual":True})
+ return ret['ops'] if 'ops' in ret else []
+
def get_comment_pending_payouts(self, comments):
""" Get comment pending payout data """
ret = self.__exec('get_comment_pending_payouts', {'comments':comments})
- print(ret)
return ret['cashout_infos']
def get_votes(self, author, permlink):
diff --git a/hive/steem/http_client.py b/hive/steem/http_client.py
index d1b37ff9e..eff65df2d 100644
--- a/hive/steem/http_client.py
+++ b/hive/steem/http_client.py
@@ -90,6 +90,7 @@ class HttpClient(object):
get_dynamic_global_properties='database_api',
list_votes='database_api',
get_comment_pending_payouts='database_api',
+ get_ops_in_block='account_history_api'
)
def __init__(self, nodes, **kwargs):
diff --git a/hive/utils/normalize.py b/hive/utils/normalize.py
index 4c8158535..10b3bcd51 100644
--- a/hive/utils/normalize.py
+++ b/hive/utils/normalize.py
@@ -180,3 +180,12 @@ def int_log_level(str_log_level):
if not isinstance(log_level, int):
raise ValueError('Invalid log level: %s' % str_log_level)
return log_level
+
+def asset_to_hbd_hive(price, asset):
+ """ Converts hive to hbd and hbd to hive based on price """
+ if asset['nai'] == price['base']['nai']:
+ result = int(asset['amount']) * int(price['quote']['amount']) / int(price['base']['amount'])
+ return {'amount' : result, 'nai' : price['quote']['nai'], 'precision' : price['quote']['precision']}
+ elif asset['nai'] == price['quote']['nai']:
+ result = int(asset['amount']) * int(price['base']['amount']) / int(price['quote']['amount'])
+ return {'amount' : result, 'nai' : price['base']['nai'], 'precision' : price['base']['precision']}
diff --git a/hive/utils/stats.py b/hive/utils/stats.py
index 5f6f8c7d4..5ae4f1f74 100644
--- a/hive/utils/stats.py
+++ b/hive/utils/stats.py
@@ -90,7 +90,8 @@ class SteemStats(StatsAbstract):
'get_feed_history': 20,
'lookup_accounts': 1000,
'list_votes':1000,
- 'get_comment_pending_payouts':1000
+ 'get_comment_pending_payouts':1000,
+ 'get_ops_in_block':500
}
def __init__(self):
--
GitLab