[WIP] Support for comment payout virtual ops

hive/indexer/blocks.py

@@ -109,7 +109,7 @@ class Blocks:
                     if not is_initial_sync:
                         Accounts.dirty(op['author']) # lite - rep
                         Accounts.dirty(op['voter']) # lite - stats
-                        Posts.vote_op(op)
+                        Posts.vote_op(hived, op)
 
                 # misc ops
                 elif op_type == 'transfer_operation':
@@ -117,6 +117,29 @@ class Blocks:
                 elif op_type == 'custom_json_operation':
                     json_ops.append(op)
 
+        comment_payout_ops = {}
+        for vop in hived.get_virtual_operations(num):
+            key = None
+            val = None
+            if vop['op']['type'] == 'curation_reward_operation':
+                key = "{}/{}".format(vop['op']['value']['comment_author'], vop['op']['value']['comment_permlink'])
+                val = {'reward' : vop['op']['value']['reward']}
+
+            if vop['op']['type'] == 'author_reward_operation':
+                key = "{}/{}".format(vop['op']['value']['author'], vop['op']['value']['permlink'])
+                val = {'hbd_payout':vop['op']['value']['hbd_payout'], 'hive_payout':vop['op']['value']['hive_payout'], 'vesting_payout':vop['op']['value']['vesting_payout']}
+
+            if vop['op']['type'] == 'comment_reward_operation':
+                key = "{}/{}".format(vop['op']['value']['author'], vop['op']['value']['permlink'])
+                val = {'payout':vop['op']['value']['payout']}
+
+            if key is not None and val is not None:
+                if key in comment_payout_ops:
+                    comment_payout_ops[key].append({vop['op']['type']:val})
+                else:
+                    comment_payout_ops[key] = [{vop['op']['type']:val}]
+        if comment_payout_ops:
+            Posts.comment_payout_op(comment_payout_ops, date)
         CustomOp.process_ops(json_ops, num, date)  # follow/reblog/community ops
 
         return num

hive/indexer/posts.py

@@ -13,7 +13,7 @@ from hive.indexer.cached_post import CachedPost
 from hive.indexer.feed_cache import FeedCache
 from hive.indexer.community import Community, START_DATE
 from hive.indexer.notify import Notify
-from hive.utils.normalize import legacy_amount
+from hive.utils.normalize import legacy_amount, parse_amount
 
 log = logging.getLogger(__name__)
 DB = Db.instance()
@@ -136,15 +136,57 @@ class Posts:
             cls.undelete(op, block_date, pid)
 
     @classmethod
-    def vote_op(cls, op):
+    def vote_op(cls, hived, op):
         """ Vote operation processing """
         pid = cls.get_id(op['author'], op['permlink'])
         assert pid, "Post does not exists in the database"
+        votes = hived.get_votes(op['author'], op['permlink'])
+        sql = """
+            UPDATE 
+                hive_post_data 
+            SET 
+                votes = :votes
+            WHERE id = :id"""
+
+        DB.query(sql, id=pid, votes=dumps(votes))
+
+    @classmethod
+    def comment_payout_op(cls, ops, date):
+        """ Process comment payment operations """
+        for k, v in ops.items():
+            author, permlink = k.split("/")
+            pid = cls.get_id(author, permlink)
+            curator_rewards_sum = 0
+            author_rewards = ''
+            comment_author_reward = None
+            print(v)
+            for operation in v:
+                for op, value in operation.items():
+                    if op == 'curation_reward_operation':
+                        curator_rewards_sum = curator_rewards_sum + int(value['reward']['amount'])
+                    if op == 'author_reward_operation':
+                        author_rewards = "{}, {}, {}".format(legacy_amount(value['hbd_payout']), legacy_amount(value['hive_payout']), legacy_amount(value['vesting_payout']))
+                    if op == 'comment_reward_operation':
+                        comment_author_reward = value['payout']
+                curator_rewards = {'amount' : str(curator_rewards_sum), 'precision': 6, 'nai': '@@000000037'}
+            print("COMMENT OP REWARDS ==> {}/{} > AUTHOR > {} > CURATORS > {} > TOTAL > {}".format(author, permlink, author_rewards, legacy_amount(curator_rewards), legacy_amount(comment_author_reward)))
+            raise RuntimeError("Comment payout op")
+            sql = """UPDATE
+                        hive_posts
+                    SET
+                        total_payout_value = :total_payout_value,
+                        curator_payout_value = :curator_payout_value,
+                        max_accepted_payout = :max_accepted_payout,
+                        author_rewards = :author_rewards,
+                        last_payout = :last_payout,
+                        cashout_time = :cashout_time,
+                        is_paidout = true
+                    WHERE id = :id
+            """
 
     @classmethod
     def insert(cls, hived, op, date):
         """Inserts new post records."""
-        print(op)
 
         # inserting new post
         # * Check for permlink, parent_permlink, root_permlink
@@ -153,7 +195,6 @@ class Posts:
         # * insert post basic data
         # * obtain id
         # * insert post content data
-        print(op)
 
         # add permlinks to permlink table
         for permlink in ['permlink', 'parent_permlink', 'root_permlink']:
@@ -190,8 +231,6 @@ class Posts:
             )"""
         sql += ";SELECT currval(pg_get_serial_sequence('hive_posts','id'))"
 
-        print(post)
-
         result = DB.query(sql, **post)
         post['id'] = int(list(result)[0][0])
         cls._set_id(op['author']+'/'+op['permlink'], post['id'])

hive/indexer/sync.py

@@ -221,10 +221,6 @@ class Sync:
     def _update_chain_state(self):
         """Update basic state props (head block, feed price) in db."""
         state = self._steem.gdgp_extended()
-        print("======================")
-        print(state['steem_per_mvest'])
-        print(state['usd_per_steem'])
-        print(state['sbd_per_steem'])
         self._db.query("""UPDATE hive_state SET block_num = :block_num,
                        steem_per_mvest = :spm, usd_per_steem = :ups,
                        sbd_per_steem = :sps, dgpo = :dgpo""",

hive/server/bridge_api/methods.py

@@ -285,7 +285,6 @@ async def get_account_posts(context, sort, account, start_author='', start_perml
         sql = sql % """ """
         
     posts = []
-    print('sql is: ', sql)
     sql_result = await db.query_all(sql, account=account, author=start_author, permlink=start_permlink, limit=limit)
     for row in sql_result:
         post = _condenser_post_object(row)

hive/server/condenser_api/methods.py

@@ -554,7 +554,6 @@ async def _get_blog(db, account: str, start_index: int, limit: int = None):
 @return_error_info
 async def get_accounts(context, accounts: list):
     """Returns accounts data for accounts given in list"""
-    print("Hivemind native get_accounts")
     assert accounts, "Empty parameters are not supported"
     assert len(accounts) < 1000, "Query exceeds limit"
 

hive/server/database_api/methods.py

@@ -69,7 +69,7 @@ async def get_post_id_by_author_and_permlink(db, author: str, permlink: str, lim
 @return_error_info
 async def list_comments(context, start: list, limit: int, order: str):
     """Returns all comments, starting with the specified options."""
-    print("Hivemind native list_comments")
+
     supported_order_list = ['by_cashout_time', 'by_permlink', 'by_root', 'by_parent', 'by_update', 'by_author_last_update']
     assert order in supported_order_list, "Unsupported order, valid orders: {}".format(", ".join(supported_order_list))
     limit = valid_limit(limit, 1000)

hive/steem/client.py

 """Tight and reliable steem API client for hive indexer."""
+import logging
 
 from time import perf_counter as perf
 from decimal import Decimal
@@ -8,6 +9,8 @@ from hive.utils.normalize import parse_amount, steem_amount, vests_amount
 from hive.steem.http_client import HttpClient
 from hive.steem.block.stream import BlockStream
 
+logger = logging.getLogger(__name__)
+
 class SteemClient:
     """Handles upstream calls to jussi/steemd, with batching and retrying."""
     # dangerous default value of url but it should be fine since we are not writting to it
@@ -21,10 +24,11 @@ class SteemClient:
         self._max_workers = max_workers
         self._client = dict()
         for endpoint, endpoint_url in url.items():
-            print("Endpoint {} will be routed to node {}".format(endpoint, endpoint_url))
+            logger.info("Endpoint %s will be routed to node %s" % (endpoint, endpoint_url))
             self._client[endpoint] = HttpClient(nodes=[endpoint_url])
 
-    def get_accounts(self, accounts):
+    def get_accounts(self, acc):
+        accounts = [v for v in acc if v != '']
         """Fetch multiple accounts by name."""
         assert accounts, "no accounts passed to get_accounts"
         assert len(accounts) <= 1000, "max 1000 accounts"
@@ -84,7 +88,6 @@ class SteemClient:
     def gdgp_extended(self):
         """Get dynamic global props without the cruft plus useful bits."""
         dgpo = self._gdgp()
-        print(dgpo)
 
         # remove unused/deprecated keys
         unused = ['total_pow', 'num_pow_witnesses', 'confidential_supply',
@@ -102,7 +105,6 @@ class SteemClient:
 
     @staticmethod
     def _get_steem_per_mvest(dgpo):
-        print("DGPO: ", dgpo)
         steem = steem_amount(dgpo['total_vesting_fund_hive'])
         mvests = vests_amount(dgpo['total_vesting_shares']) / Decimal(1e6)
         return "%.6f" % (steem / mvests)
@@ -119,7 +121,7 @@ class SteemClient:
 
     def _get_steem_price(self):
         orders = self.__exec('get_order_book', [1])
-        if orders['asks'] and orders[bids]:
+        if orders['asks'] and orders['bids']:
             ask = Decimal(orders['asks'][0]['real_price'])
             bid = Decimal(orders['bids'][0]['real_price'])
             price = (ask + bid) / 2
@@ -140,10 +142,14 @@ class SteemClient:
 
         return [blocks[x] for x in block_nums]
 
+    def get_virtual_operations(self, block):
+        """ Get virtual ops from block """
+        ret = self.__exec('get_ops_in_block', {"block_num":block, "only_virtual":True})
+        return ret['ops'] if 'ops' in ret else [] 
+
     def get_comment_pending_payouts(self, comments):
         """ Get comment pending payout data """
         ret = self.__exec('get_comment_pending_payouts', {'comments':comments})
-        print(ret)
         return ret['cashout_infos']
 
     def get_votes(self, author, permlink):

hive/steem/http_client.py

@@ -90,6 +90,7 @@ class HttpClient(object):
         get_dynamic_global_properties='database_api',
         list_votes='database_api',
         get_comment_pending_payouts='database_api',
+        get_ops_in_block='account_history_api'
     )
 
     def __init__(self, nodes, **kwargs):

hive/utils/normalize.py

@@ -180,3 +180,12 @@ def int_log_level(str_log_level):
     if not isinstance(log_level, int):
         raise ValueError('Invalid log level: %s' % str_log_level)
     return log_level
+
+def asset_to_hbd_hive(price, asset):
+    """ Converts hive to hbd and hbd to hive based on price """
+    if asset['nai'] == price['base']['nai']:
+        result = int(asset['amount']) * int(price['quote']['amount']) / int(price['base']['amount'])
+        return {'amount' : result, 'nai' : price['quote']['nai'], 'precision' : price['quote']['precision']}
+    elif asset['nai'] == price['quote']['nai']:
+        result = int(asset['amount']) * int(price['base']['amount']) / int(price['quote']['amount'])
+        return {'amount' : result, 'nai' : price['base']['nai'], 'precision' : price['base']['precision']}

hive/utils/stats.py

@@ -90,7 +90,8 @@ class SteemStats(StatsAbstract):
         'get_feed_history': 20,
         'lookup_accounts': 1000,
         'list_votes':1000,
-        'get_comment_pending_payouts':1000
+        'get_comment_pending_payouts':1000,
+        'get_ops_in_block':500
     }
 
     def __init__(self):