Issue #47 - still doesn't work but some fixes in virtual operations were applied

hive/db/schema.py

@@ -91,6 +91,7 @@ def build_metadata():
 
         # core stats/indexes
         sa.Column('payout', sa.types.DECIMAL(10, 3), nullable=False, server_default='0'),
+        sa.Column('pending_payout', sa.types.DECIMAL(10, 3), nullable=False, server_default='0'),
         sa.Column('payout_at', sa.DateTime, nullable=False, server_default='1990-01-01'),
         sa.Column('updated_at', sa.DateTime, nullable=False, server_default='1990-01-01'),
         sa.Column('is_paidout', BOOLEAN, nullable=False, server_default='0'),

hive/indexer/blocks.py

@@ -114,13 +114,18 @@ class Blocks:
                 key = "{}/{}".format(op_value['author'], op_value['permlink'])
                 val = {'hbd_payout':op_value['hbd_payout'], 'hive_payout':op_value['hive_payout'], 'vesting_payout':op_value['vesting_payout']}
             elif op_type == 'comment_reward_operation':
-                if('payout' not in op_value or op_value['payout'] is None):
-                    log.error("Broken op: `{}'".format(str(vop)))
                 key = "{}/{}".format(op_value['author'], op_value['permlink'])
-                val = {'payout':op_value['payout'], 'author_rewards':op_value['author_rewards']}
+                val = {'payout':op_value['payout'], 'author_rewards':op_value['author_rewards'], 'total_payout_value':op_value['total_payout_value'], 'curator_payout_value':op_value['curator_payout_value'], 'beneficiary_payout_value':op_value['beneficiary_payout_value'] }
+
             elif op_type == 'effective_comment_vote_operation':
-                key_vote = "{}/{}/{}".format(op_value['voter'], op_value['author'], op_value['permlink'])
-                vote_ops[ key_vote ] = op_value
+                key = "{}/{}".format(op_value['author'], op_value['permlink'])
+                val = {'pending_payout':op_value['pending_payout']}
+                vote_ops.append(vop)
+            elif op_type == 'comment_payout_update_operation':
+                key = "{}/{}".format(op_value['author'], op_value['permlink'])
+                #Later these values are not used
+                val = {'val':'key'}
+                vote_ops.append(vop)
 
             if key is not None and val is not None:
                 if key in comment_payout_ops:
@@ -166,7 +171,6 @@ class Blocks:
 
         # second scan will process all other ops
         json_ops = []
-        update_comment_pending_payouts = []
         for tx_idx, tx in enumerate(block['transactions']):
             for operation in tx['operations']:
                 op_type = operation['type']
@@ -199,8 +203,6 @@ class Blocks:
                     if not is_initial_sync:
                         Accounts.dirty(op['author']) # lite - rep
                         Accounts.dirty(op['voter']) # lite - stats
-                        update_comment_pending_payouts.append([op['author'], op['permlink']])
-                        Votes.vote_op(op)
 
                 # misc ops
                 elif op_type == 'transfer_operation':
@@ -213,10 +215,6 @@ class Blocks:
             custom_ops_stats = CustomOp.process_ops(json_ops, num, cls._head_block_date)
             cls.ops_stats = Blocks.merge_ops_stats(cls.ops_stats, custom_ops_stats)
 
-        if update_comment_pending_payouts:
-            payout_ops_stat = Posts.update_comment_pending_payouts(hived, update_comment_pending_payouts)
-            cls.ops_stats = Blocks.merge_ops_stats(cls.ops_stats, payout_ops_stat)
-
         # virtual ops
         comment_payout_ops = {}
         vote_ops = {}

hive/indexer/posts.py

@@ -150,16 +150,17 @@ class Posts:
         ops_stats = {}
         sql = """
               UPDATE hive_posts AS ihp SET
-                  total_payout_value = data_source.total_payout_value,
-                  curator_payout_value = data_source.curator_payout_value,
-                  author_rewards = data_source.author_rewards,
-                  author_rewards_hive = data_source.author_rewards_hive,
-                  author_rewards_hbd = data_source.author_rewards_hbd,
-                  author_rewards_vests = data_source.author_rewards_vests,
-                  last_payout = data_source.last_payout,
-                  cashout_time = data_source.cashout_time,
-                  is_paidout = true
-
+                  total_payout_value    = COALESCE( data_source.total_payout_value,                   ihp.total_payout_value ),
+                  curator_payout_value  = COALESCE( data_source.curator_payout_value,                 ihp.curator_payout_value ),
+                  author_rewards        = COALESCE( CAST( data_source.author_rewards as INT8 ),       ihp.author_rewards ),
+                  author_rewards_hive   = COALESCE( CAST( data_source.author_rewards_hive as INT8 ),  ihp.author_rewards_hive ),
+                  author_rewards_hbd    = COALESCE( CAST( data_source.author_rewards_hbd as INT8 ),   ihp.author_rewards_hbd ),
+                  author_rewards_vests  = COALESCE( CAST( data_source.author_rewards_vests as INT8 ), ihp.author_rewards_vests ),
+                  payout                = COALESCE( CAST( data_source.payout as DECIMAL ),            ihp.payout ),
+                  pending_payout        = COALESCE( CAST( data_source.pending_payout as DECIMAL ),    ihp.pending_payout ),
+                  last_payout           = data_source.last_payout,
+                  cashout_time          = data_source.cashout_time,
+                  is_paidout            = data_source.is_paidout
               FROM 
               (
               SELECT  ha_a.id as author_id, hpd_p.id as permlink_id, 
@@ -169,8 +170,11 @@ class Posts:
                       t.author_rewards_hive,
                       t.author_rewards_hbd,
                       t.author_rewards_vests,
+                      t.payout,
+                      t.pending_payout,
                       t.last_payout,
-                      t.cashout_time
+                      t.cashout_time,
+                      t.is_paidout
               from
               (
               VALUES
@@ -183,8 +187,11 @@ class Posts:
                       author_rewards_hive,
                       author_rewards_hbd,
                       author_rewards_vests,
+                      payout,
+                      pending_payout,
                       last_payout,
-                      cashout_time)
+                      cashout_time,
+                      is_paidout)
               INNER JOIN hive_accounts ha_a ON ha_a.name = t.author
               INNER JOIN hive_permlink_data hpd_p ON hpd_p.permlink = t.permlink
               ) as data_source(author_id, permlink_id, total_payout_value)
@@ -198,14 +205,26 @@ class Posts:
         for k, v in ops.items():
             author, permlink = k.split("/")
             # total payout to curators
-            curator_rewards_sum = 0
+            curator_rewards_sum = None
+
             # author payouts
-            author_rewards = 0
-            author_rewards_hive = 0
-            author_rewards_hbd = 0
-            author_rewards_vests = 0
+            author_rewards = None
+            author_rewards_hive = None
+            author_rewards_hbd = None
+            author_rewards_vests = None
+
             # total payout for comment
             comment_author_reward = None
+            curator_rewards = None
+            total_payout_value = None;
+            curator_payout_value = None;
+            beneficiary_payout_value = None;
+
+            payout = None
+            pending_payout = None
+
+            is_paidout = (date[0:4] == '1969')
+
             for operation in v:
                 for op, value in operation.items():
                     if op in ops_stats:
@@ -214,36 +233,55 @@ class Posts:
                         ops_stats[op] = 1
 
                     if op == 'curation_reward_operation':
+                        curator_rewards_sum = 0
                         curator_rewards_sum = curator_rewards_sum + int(value['reward']['amount'])
                     elif op == 'author_reward_operation':
                         author_rewards_hive = value['hive_payout']['amount']
                         author_rewards_hbd = value['hbd_payout']['amount']
                         author_rewards_vests = value['vesting_payout']['amount']
                     elif op == 'comment_reward_operation':
-                        comment_author_reward = value['payout']
-                        author_rewards = value['author_rewards']
-            curator_rewards = {'amount' : str(curator_rewards_sum), 'precision': 6, 'nai': '@@000000037'}
-
-            values.append("('{}', '{}', '{}', '{}', {}, {}, {}, {}, '{}'::timestamp, '{}'::timestamp)".format(author, permlink,
-               legacy_amount(comment_author_reward), # total_payout_value
-               legacy_amount(curator_rewards), #curator_payout_value
-               author_rewards,
-               author_rewards_hive,
-               author_rewards_hbd,
-               author_rewards_vests,
-               date, #last_payout
-               date #cashout_time
-               ))
+                        comment_author_reward     = value['payout']
+                        author_rewards            = value['author_rewards']
+                        total_payout_value        = value['total_payout_value']
+                        curator_payout_value      = value['curator_payout_value']
+                        beneficiary_payout_value  = value['beneficiary_payout_value']
+
+                    elif op == 'effective_comment_vote_operation':
+                        pending_payout = sbd_amount( value['pending_payout'] )
+
+            if curator_rewards_sum is not None:
+              curator_rewards = {'amount' : str(curator_rewards_sum), 'precision': 6, 'nai': '@@000000037'}
+
+            if ( total_payout_value is not None and curator_payout_value is not None ):
+              payout = sum([ sbd_amount(total_payout_value), sbd_amount(curator_payout_value) ])
+              pending_payout = 0
+
+            values.append("('{}', '{}', {}, {}, {}, {}, {}, {}, {}, {}, '{}'::timestamp, '{}'::timestamp, {})".format(
+              author,
+              permlink,
+              "NULL" if ( total_payout_value is None ) else ( "'{}'".format( legacy_amount(total_payout_value) ) ),
+              "NULL" if ( curator_rewards is None ) else ( "'{}'".format( legacy_amount(curator_rewards) ) ), #curator_payout_value
+              "NULL" if ( author_rewards is None ) else author_rewards,
+              "NULL" if ( author_rewards_hive is None ) else author_rewards_hive,
+              "NULL" if ( author_rewards_hbd is None ) else author_rewards_hbd,
+              "NULL" if ( author_rewards_vests is None ) else author_rewards_vests,
+              "NULL" if ( payout is None ) else payout,
+              "NULL" if ( pending_payout is None ) else pending_payout,
+              date, #last_payout
+              date, #cashout_time
+              is_paidout))
 
             if len(values) >= values_limit:
                 values_str = ','.join(values)
                 actual_query = sql.format(values_str)
+
                 DB.query(actual_query)
                 values.clear()
 
         if len(values) > 0:
             values_str = ','.join(values)
             actual_query = sql.format(values_str)
+
             DB.query(actual_query)
             values.clear()
         return ops_stats
@@ -328,60 +366,6 @@ class Posts:
         # force parent child recount when child is deleted
         cls.update_child_count(pid, '-')
 
-    @classmethod
-    def update_comment_pending_payouts(cls, hived, posts):
-        comment_pending_payouts = hived.get_comment_pending_payouts(posts)
-        for comment_pending_payout in comment_pending_payouts:
-            if 'cashout_info' in comment_pending_payout:
-                cpp = comment_pending_payout['cashout_info']
-                sql = """UPDATE
-                            hive_posts
-                        SET
-                            total_payout_value = :total_payout_value,
-                            curator_payout_value = :curator_payout_value,
-                            max_accepted_payout = :max_accepted_payout,
-                            author_rewards = :author_rewards,
-                            children_abs_rshares = :children_abs_rshares,
-                            abs_rshares = :abs_rshares,
-                            vote_rshares = :vote_rshares,
-                            net_votes = :net_votes,
-                            active = :active,
-                            last_payout = :last_payout,
-                            cashout_time = :cashout_time,
-                            max_cashout_time = :max_cashout_time,
-                            percent_hbd = :percent_hbd,
-                            reward_weight = :reward_weight,
-                            allow_replies = :allow_replies,
-                            allow_votes = :allow_votes,
-                            allow_curation_rewards = :allow_curation_rewards
-                        WHERE id = (
-                            SELECT hp.id 
-                            FROM hive_posts hp 
-                            INNER JOIN hive_accounts ha_a ON ha_a.id = hp.author_id 
-                            INNER JOIN hive_permlink_data hpd_p ON hpd_p.id = hp.permlink_id 
-                            WHERE ha_a.name = :author AND hpd_p.permlink = :permlink
-                        )
-                """
-
-                DB.query(sql, total_payout_value=legacy_amount(cpp['total_payout_value']),
-                         curator_payout_value=legacy_amount(cpp['curator_payout_value']),
-                         max_accepted_payout=legacy_amount(cpp['max_accepted_payout']),
-                         author_rewards=cpp['author_rewards'],
-                         children_abs_rshares=cpp['children_abs_rshares'],
-                         abs_rshares=cpp['abs_rshares'],
-                         vote_rshares=cpp['vote_rshares'],
-                         net_votes=cpp['net_votes'],
-                         active=cpp['active'],
-                         last_payout=cpp['last_payout'],
-                         cashout_time=cpp['cashout_time'],
-                         max_cashout_time=cpp['max_cashout_time'],
-                         percent_hbd=cpp['percent_hbd'],
-                         reward_weight=cpp['reward_weight'],
-                         allow_replies=cpp['allow_replies'],
-                         allow_votes=cpp['allow_votes'],
-                         allow_curation_rewards=cpp['allow_curation_rewards'],
-                         author=comment_pending_payout['author'], permlink=comment_pending_payout['permlink'])
-
     @classmethod
     def _insert_feed_cache(cls, result, date):
         """Insert the new post into feed cache if it's not a comment."""

hive/server/common/payout_stats.py

@@ -39,7 +39,7 @@ class PayoutStats:
         sql = """
             SELECT community_id,
                    ha.name as author,
-                   SUM(payout) payout,
+                   SUM( payout + pending_payout ) payout,
                    COUNT(*) posts,
                    NULL authors
               FROM hive_posts
@@ -51,7 +51,7 @@ class PayoutStats:
 
             SELECT community_id,
                    NULL author,
-                   SUM(payout) payout,
+                   SUM( payout + pending_payout ) payout,
                    COUNT(*) posts,
                    COUNT(DISTINCT(author_id)) authors
               FROM hive_posts

hive/server/hive_api/stats.py

@@ -46,4 +46,4 @@ async def get_payout_stats(context, limit=250):
               WHERE community_id IS NULL AND author IS NULL"""
     blog_ttl = await db.query_one(sql)
 
-    return dict(items=items, total=float(total), blogs=float(blog_ttl))
+    return dict(items=items, total=float(total if total is not None else 0.), blogs=float(blog_ttl if blog_ttl is not None else 0.))

hive/steem/client.py

@@ -145,7 +145,7 @@ class SteemClient:
     def get_virtual_operations(self, block):
         """ Get virtual ops from block """
         result = self.__exec('get_ops_in_block', {"block_num":block, "only_virtual":True})
-        tracked_ops = ['curation_reward_operation', 'author_reward_operation', 'comment_reward_operation', 'effective_comment_vote_operation']
+        tracked_ops = ['curation_reward_operation', 'author_reward_operation', 'comment_reward_operation', 'effective_comment_vote_operation', 'comment_payout_update_operation']
         ret = []
         result = result['ops'] if 'ops' in result else []
         for vop in result:
@@ -165,9 +165,10 @@ class SteemClient:
         curation_reward_operation               = 0x000004
         comment_reward_operation                = 0x000008
         effective_comment_vote_operation        = 0x400000
+        comment_payout_update_operation         = 0x000800
 
-        tracked_ops_filter = curation_reward_operation | author_reward_operation | comment_reward_operation | effective_comment_vote_operation
-        tracked_ops = ['curation_reward_operation', 'author_reward_operation', 'comment_reward_operation', 'effective_comment_vote_operation']
+        tracked_ops_filter = curation_reward_operation | author_reward_operation | comment_reward_operation | effective_comment_vote_operation | comment_payout_update_operation
+        tracked_ops = ['curation_reward_operation', 'author_reward_operation', 'comment_reward_operation', 'effective_comment_vote_operation', 'comment_payout_update_operation']
 
         resume_on_operation = 0
 

tests_api @ fda5911a

-Subproject commit 7832473c73dc0a8d60b780826196c51c07ee1346
+Subproject commit fda5911aa8900d1bed1958ab29132980dcc509c5