From 41ff225b86997fc6a007fc6f78391c3364ad4ad7 Mon Sep 17 00:00:00 2001
From: Dariusz Kedzierski <dkedzierski@syncad.com>
Date: Mon, 8 Jun 2020 23:09:08 +0200
Subject: [PATCH] [WIP] Testing in progress. Sync from 0 works. Small
rebranding.
---
hive/db/db_state.py | 3 +
hive/db/schema.py | 25 +++-
hive/indexer/blocks.py | 12 +-
hive/indexer/posts.py | 162 ++++++++++++++++++++++---
hive/indexer/sync.py | 8 +-
hive/server/bridge_api/objects.py | 2 +-
hive/server/common/objects.py | 2 +-
hive/server/condenser_api/get_state.py | 6 +-
hive/server/condenser_api/objects.py | 2 +-
hive/steem/client.py | 38 ++++--
hive/steem/http_client.py | 3 +-
hive/utils/stats.py | 2 +
scripts/update_hivemind_db.sql | 20 ++-
13 files changed, 239 insertions(+), 46 deletions(-)
diff --git a/hive/db/db_state.py b/hive/db/db_state.py
index 6d31b94ed..68f1ccc42 100644
--- a/hive/db/db_state.py
+++ b/hive/db/db_state.py
@@ -316,6 +316,9 @@ class DbState:
cls._set_ver(17)
if cls._ver == 17:
+ cls.db().query("INSERT INTO hive_accounts (name, created_at) VALUES ('', '1990-01-01T00:00:00') ON CONFLICT (name) DO NOTHING")
+ cls.db().query("INSERT INTO hive_permlink_data (permlink) VALUES ('') ON CONFLICT (permlink) DO NOTHING")
+ cls.db().query("INSERT INTO hive_category_data (category) VALUES ('') ON CONFLICT (category) DO NOTHING")
cls._set_ver(18)
reset_autovac(cls.db())
diff --git a/hive/db/schema.py b/hive/db/schema.py
index 0d30b2e7c..8320842dd 100644
--- a/hive/db/schema.py
+++ b/hive/db/schema.py
@@ -111,6 +111,20 @@ def build_metadata():
sa.Column('sc_trend', sa.Float(precision=6), nullable=False, server_default='0'),
sa.Column('sc_hot', sa.Float(precision=6), nullable=False, server_default='0'),
+ sa.Column('total_payout_value', sa.String(16), nullable=False, server_default=''),
+ sa.Column('author_rewards', sa.Integer, nullable=False, server_default='0'),
+ sa.Column('children_abs_rshares', sa.Integer, nullable=False, server_default='0'),
+ sa.Column('net_rshares', sa.Integer, nullable=False, server_default='0'),
+ sa.Column('abs_rshares', sa.Integer, nullable=False, server_default='0'),
+ sa.Column('vote_rshares', sa.Integer, nullable=False, server_default='0'),
+ sa.Column('net_votes', sa.Integer, nullable=False, server_default='0'),
+ sa.Column('active', sa.DateTime, nullable=False, server_default='1970-01-01 00:00:00'),
+ sa.Column('last_payout', sa.DateTime, nullable=False, server_default='1970-01-01 00:00:00'),
+ sa.Column('cashout_time', sa.DateTime, nullable=False, server_default='1970-01-01 00:00:00'),
+ sa.Column('max_cashout_time', sa.DateTime, nullable=False, server_default='1970-01-01 00:00:00'),
+ sa.Column('percent_hbd', sa.Integer, nullable=False, server_default='0'),
+ sa.Column('reward_weight', sa.Integer, nullable=False, server_default='0'),
+
sa.Column('parent_author_id', sa.Integer, nullable=False),
sa.Column('parent_permlink_id', sa.Integer, nullable=False),
sa.Column('curator_payout_value', sa.String(16), nullable=False, server_default=''),
@@ -231,9 +245,9 @@ def build_metadata():
'hive_state', metadata,
sa.Column('block_num', sa.Integer, primary_key=True, autoincrement=False),
sa.Column('db_version', sa.Integer, nullable=False),
- sa.Column('steem_per_mvest', sa.types.DECIMAL(8, 3), nullable=False),
- sa.Column('usd_per_steem', sa.types.DECIMAL(8, 3), nullable=False),
- sa.Column('sbd_per_steem', sa.types.DECIMAL(8, 3), nullable=False),
+ sa.Column('steem_per_mvest', sa.types.DECIMAL(14, 6), nullable=False),
+ sa.Column('usd_per_steem', sa.types.DECIMAL(14, 6), nullable=False),
+ sa.Column('sbd_per_steem', sa.types.DECIMAL(14, 6), nullable=False),
sa.Column('dgpo', sa.Text, nullable=False),
)
@@ -337,7 +351,10 @@ def setup(db):
"INSERT INTO hive_accounts (name, created_at) VALUES ('miners', '2016-03-24 16:05:00')",
"INSERT INTO hive_accounts (name, created_at) VALUES ('null', '2016-03-24 16:05:00')",
"INSERT INTO hive_accounts (name, created_at) VALUES ('temp', '2016-03-24 16:05:00')",
- "INSERT INTO hive_accounts (name, created_at) VALUES ('initminer', '2016-03-24 16:05:00')"]
+ "INSERT INTO hive_accounts (name, created_at) VALUES ('initminer', '2016-03-24 16:05:00')",
+ "INSERT INTO hive_accounts (name, created_at) VALUES ('', '1990-01-01T00:00:00') ON CONFLICT (name) DO NOTHING",
+ "INSERT INTO hive_permlink_data (permlink) VALUES ('') ON CONFLICT (permlink) DO NOTHING",
+ "INSERT INTO hive_category_data (category) VALUES ('') ON CONFLICT (category) DO NOTHING"]
for sql in sqls:
db.query(sql)
diff --git a/hive/indexer/blocks.py b/hive/indexer/blocks.py
index 74b9053f6..5da8f4f27 100644
--- a/hive/indexer/blocks.py
+++ b/hive/indexer/blocks.py
@@ -31,20 +31,20 @@ class Blocks:
return str(DB.query_one(sql) or '')
@classmethod
- def process(cls, block):
+ def process(cls, block, hived):
"""Process a single block. Always wrap in a transaction!"""
#assert is_trx_active(), "Block.process must be in a trx"
- return cls._process(block, is_initial_sync=False)
+ return cls._process(block, hived, is_initial_sync=False)
@classmethod
- def process_multi(cls, blocks, is_initial_sync=False):
+ def process_multi(cls, blocks, hived, is_initial_sync=False):
"""Batch-process blocks; wrapped in a transaction."""
DB.query("START TRANSACTION")
last_num = 0
try:
for block in blocks:
- last_num = cls._process(block, is_initial_sync)
+ last_num = cls._process(block, hived, is_initial_sync)
except Exception as e:
log.error("exception encountered block %d", last_num + 1)
raise e
@@ -57,7 +57,7 @@ class Blocks:
DB.query("COMMIT")
@classmethod
- def _process(cls, block, is_initial_sync=False):
+ def _process(cls, block, hived, is_initial_sync=False):
"""Process a single block. Assumes a trx is open."""
#pylint: disable=too-many-branches
num = cls._push(block)
@@ -100,7 +100,7 @@ class Blocks:
# post ops
elif op_type == 'comment_operation':
- Posts.comment_op(op, date)
+ Posts.comment_op(hived, op, date)
if not is_initial_sync:
Accounts.dirty(op['author']) # lite - stats
elif op_type == 'delete_comment_operation':
diff --git a/hive/indexer/posts.py b/hive/indexer/posts.py
index 68883b1ca..a52a08209 100644
--- a/hive/indexer/posts.py
+++ b/hive/indexer/posts.py
@@ -3,6 +3,8 @@
import logging
import collections
+from json import dumps
+
from hive.db.adapter import Db
from hive.db.db_state import DbState
@@ -11,6 +13,7 @@ from hive.indexer.cached_post import CachedPost
from hive.indexer.feed_cache import FeedCache
from hive.indexer.community import Community, START_DATE
from hive.indexer.notify import Notify
+from hive.utils.normalize import legacy_amount
log = logging.getLogger(__name__)
DB = Db.instance()
@@ -30,6 +33,24 @@ class Posts:
sql = "SELECT MAX(id) FROM hive_posts WHERE is_deleted = '0'"
return DB.query_one(sql) or 0
+ @classmethod
+ def find_root(cls, author, permlink):
+ """ Find root for post """
+ sql = """WITH parent AS
+ (
+ SELECT id, parent_id, 1 AS [level] from hive_posts WHERE id = (SELECT hp.id
+ FROM hive_posts hp
+ LEFT JOIN hive_accounts ha_a ON ha_a.id = hp.author_id
+ LEFT JOIN hive_permlink_data hpd_p ON hpd_p.id = hp.permlink_id
+ WHERE ha_a.name = :a AND hpd_p.permlink = :p)
+ UNION ALL
+ SELECT t.id, t.parent_id, [level] + 1 FROM parent
+ INNER JOIN hive_posts t ON t.id = parent.parent_id
+ )
+ SELECT TOP 1 id FROM parent ORDER BY [level] DESC"""
+ _id = DB.query_one(sql, a=author, p=permlink)
+ return _id
+
@classmethod
def get_id(cls, author, permlink):
"""Look up id by author/permlink, making use of LRU cache."""
@@ -101,15 +122,15 @@ class Posts:
cls.delete(op)
@classmethod
- def comment_op(cls, op, block_date):
+ def comment_op(cls, hived, op, block_date):
"""Register new/edited/undeleted posts; insert into feed cache."""
pid = cls.get_id(op['author'], op['permlink'])
if not pid:
# post does not exist, go ahead and process it.
- cls.insert(op, block_date)
+ cls.insert(hived, op, block_date)
elif not cls.is_pid_deleted(pid):
# post exists, not deleted, thus an edit. ignore.
- cls.update(op, block_date, pid)
+ cls.update(hived, op, block_date, pid)
else:
# post exists but was deleted. time to reinstate.
cls.undelete(op, block_date, pid)
@@ -121,8 +142,9 @@ class Posts:
assert pid, "Post does not exists in the database"
@classmethod
- def insert(cls, op, date):
+ def insert(cls, hived, op, date):
"""Inserts new post records."""
+ print(op)
# inserting new post
# * Check for permlink, parent_permlink, root_permlink
@@ -142,18 +164,19 @@ class Posts:
ON CONFLICT (permlink) DO NOTHING"""
DB.query(sql, permlink=op[permlink])
+ post = cls._build_post(op, date)
+
# add category to category table
- if 'category' in op:
- sql = """
- INSERT INTO hive_category_data (category)
- VALUES (:category)
- ON CONFLICT (category) DO NOTHING"""
- DB.query(sql, category=op['category'])
+ sql = """
+ INSERT INTO hive_category_data (category)
+ VALUES (:category)
+ ON CONFLICT (category) DO NOTHING"""
+ DB.query(sql, category=post['category'])
sql = """
INSERT INTO hive_posts (parent_id, author_id, permlink_id,
category_id, community_id, created_at, depth, is_muted,
- is_valid, parent_author_id, parent_permlink_id)
+ is_valid, parent_author_id, parent_permlink_id, root_author_id, root_permlink_id)
VALUES (:parent_id,
(SELECT id FROM hive_accounts WHERE name = :author),
(SELECT id FROM hive_permlink_data WHERE permlink = :permlink),
@@ -161,15 +184,66 @@ class Posts:
:community_id, :date, :depth,
:is_muted, :is_valid,
(SELECT id FROM hive_accounts WHERE name = :parent_author),
- (SELECT id FROM hive_permlink_data WHERE permlink = :parent_permlink)
+ (SELECT id FROM hive_permlink_data WHERE permlink = :parent_permlink),
+ (SELECT id FROM hive_accounts WHERE name = :root_author),
+ (SELECT id FROM hive_permlink_data WHERE permlink = :root_permlink)
)"""
sql += ";SELECT currval(pg_get_serial_sequence('hive_posts','id'))"
- post = cls._build_post(op, date)
+
+ print(post)
+
result = DB.query(sql, **post)
post['id'] = int(list(result)[0][0])
cls._set_id(op['author']+'/'+op['permlink'], post['id'])
+ comment_pending_payouts = hived.get_comment_pending_payouts([[op['author'], op['permlink']]])[0]
+ if 'cashout_info' in comment_pending_payouts:
+ sql = """UPDATE
+ hive_posts
+ SET
+ total_payout_value = :total_payout_value,
+ curator_payout_value = :curator_payout_value,
+ max_accepted_payout = :max_accepted_payout,
+ author_rewards = :author_rewards,
+ children_abs_rshares = :children_abs_rshares,
+ net_rshares = :net_rshares,
+ abs_rshares = :abs_rshares,
+ vote_rshares = :vote_rshares,
+ net_votes = :net_votes,
+ active = :active,
+ last_payout = :last_payout,
+ cashout_time = :cashout_time,
+ max_cashout_time = :max_cashout_time,
+ percent_hbd = :percent_hbd,
+ reward_weight = :reward_weight,
+ allow_replies = :allow_replies,
+ allow_votes = :allow_votes,
+ allow_curation_rewards = :allow_curation_rewards
+ WHERE id = :id
+ """
+ DB.query(sql, total_payout_value=legacy_amount(comment_pending_payouts['cashout_info']['total_payout_value']),
+ curator_payout_value=legacy_amount(comment_pending_payouts['cashout_info']['curator_payout_value']),
+ max_accepted_payout=legacy_amount(comment_pending_payouts['cashout_info']['max_accepted_payout']),
+ author_rewards=comment_pending_payouts['cashout_info']['author_rewards'],
+ children_abs_rshares=comment_pending_payouts['cashout_info']['children_abs_rshares'],
+ net_rshares=comment_pending_payouts['cashout_info']['net_rshares'],
+ abs_rshares=comment_pending_payouts['cashout_info']['abs_rshares'],
+ vote_rshares=comment_pending_payouts['cashout_info']['vote_rshares'],
+ net_votes=comment_pending_payouts['cashout_info']['net_votes'],
+ active=comment_pending_payouts['cashout_info']['active'],
+ last_payout=comment_pending_payouts['cashout_info']['last_payout'],
+ cashout_time=comment_pending_payouts['cashout_info']['cashout_time'],
+ max_cashout_time=comment_pending_payouts['cashout_info']['max_cashout_time'],
+ percent_hbd=comment_pending_payouts['cashout_info']['percent_hbd'],
+ reward_weight=comment_pending_payouts['cashout_info']['reward_weight'],
+ allow_replies=comment_pending_payouts['cashout_info']['allow_replies'],
+ allow_votes=comment_pending_payouts['cashout_info']['allow_votes'],
+ allow_curation_rewards=comment_pending_payouts['cashout_info']['allow_curation_rewards'],
+ id=post['id']
+ )
+
# add content data to hive_post_data
+ votes = hived.get_votes(op['author'], op['permlink'])
sql = """
INSERT INTO hive_post_data (id, title, preview, img_url, body,
votes, json)
@@ -177,7 +251,7 @@ class Posts:
DB.query(sql, id=post['id'], title=op['title'],
preview=op['preview'] if 'preview' in op else "",
img_url=op['img_url'] if 'img_url' in op else "",
- body=op['body'], votes=op['votes'] if 'votes' in op else "",
+ body=op['body'], votes=dumps(votes),
json=op['json_metadata'] if op['json_metadata'] else '{}')
if not DbState.is_initial_sync():
@@ -240,7 +314,7 @@ class Posts:
@classmethod
- def update(cls, op, date, pid):
+ def update(cls, hived, op, date, pid):
"""Handle post updates.
Here we could also build content diffs, but for now just used
@@ -273,8 +347,56 @@ class Posts:
parent_permlink_id = (SELECT id FROM hive_permlink_data WHERE permlink = :parent_permlink)
WHERE id = :id"""
post = cls._build_post(op, date)
+ post['id'] = pid
DB.query(sql, **post)
+ comment_pending_payouts = hived.get_comment_pending_payouts([[op['author'], op['permlink']]])[0]
+ if 'cashout_info' in comment_pending_payouts:
+ sql = """UPDATE
+ hive_posts
+ SET
+ total_payout_value = :total_payout_value,
+ curator_payout_value = :curator_payout_value,
+ max_accepted_payout = :max_accepted_payout,
+ author_rewards = :author_rewards,
+ children_abs_rshares = :children_abs_rshares,
+ net_rshares = :net_rshares,
+ abs_rshares = :abs_rshares,
+ vote_rshares = :vote_rshares,
+ net_votes = :net_votes,
+ active = :active,
+ last_payout = :last_payout,
+ cashout_time = :cashout_time,
+ max_cashout_time = :max_cashout_time,
+ percent_hbd = :percent_hbd,
+ reward_weight = :reward_weight,
+ allow_replies = :allow_replies,
+ allow_votes = :allow_votes,
+ allow_curation_rewards = :allow_curation_rewards
+ WHERE id = :id
+ """
+ DB.query(sql, total_payout_value=legacy_amount(comment_pending_payouts['cashout_info']['total_payout_value']),
+ curator_payout_value=legacy_amount(comment_pending_payouts['cashout_info']['curator_payout_value']),
+ max_accepted_payout=legacy_amount(comment_pending_payouts['cashout_info']['max_accepted_payout']),
+ author_rewards=comment_pending_payouts['cashout_info']['author_rewards'],
+ children_abs_rshares=comment_pending_payouts['cashout_info']['children_abs_rshares'],
+ net_rshares=comment_pending_payouts['cashout_info']['net_rshares'],
+ abs_rshares=comment_pending_payouts['cashout_info']['abs_rshares'],
+ vote_rshares=comment_pending_payouts['cashout_info']['vote_rshares'],
+ net_votes=comment_pending_payouts['cashout_info']['net_votes'],
+ active=comment_pending_payouts['cashout_info']['active'],
+ last_payout=comment_pending_payouts['cashout_info']['last_payout'],
+ cashout_time=comment_pending_payouts['cashout_info']['cashout_time'],
+ max_cashout_time=comment_pending_payouts['cashout_info']['max_cashout_time'],
+ percent_hbd=comment_pending_payouts['cashout_info']['percent_hbd'],
+ reward_weight=comment_pending_payouts['cashout_info']['reward_weight'],
+ allow_replies=comment_pending_payouts['cashout_info']['allow_replies'],
+ allow_votes=comment_pending_payouts['cashout_info']['allow_votes'],
+ allow_curation_rewards=comment_pending_payouts['cashout_info']['allow_curation_rewards'],
+ id=pid
+ )
+
+ votes = hived.get_votes(op['author'], op['permlink'])
sql = """
UPDATE
hive_post_data
@@ -290,7 +412,7 @@ class Posts:
DB.query(sql, id=pid, title=op['title'],
preview=op['preview'] if 'preview' in op else "",
img_url=op['img_url'] if 'img_url' in op else "",
- body=op['body'], votes=op['votes'] if 'votes' in op else "",
+ body=op['body'], votes=dumps(votes),
json=op['json_metadata'] if op['json_metadata'] else '{}')
@classmethod
@@ -340,6 +462,8 @@ class Posts:
community_id = Community.validated_id(category)
is_valid = True
is_muted = False
+ root_author = op['author']
+ root_permlink = op['permlink']
# this is a comment; inherit parent props.
else:
@@ -354,6 +478,8 @@ class Posts:
depth = parent_depth + 1
if not is_valid: error = 'replying to invalid post'
elif is_muted: error = 'replying to muted post'
+ #find root comment
+
# check post validity in specified context
error = None
@@ -367,6 +493,8 @@ class Posts:
depth=depth, date=date, error=error,
author=op['author'], permlink=op['permlink'],
parent_author=op['parent_author'],
- parent_permlink=op['parent_permlink'])
+ parent_permlink=op['parent_permlink'],
+ root_permlink=root_permlink,
+ root_author=root_author)
return ret
diff --git a/hive/indexer/sync.py b/hive/indexer/sync.py
index 0a0892efa..e3734784e 100644
--- a/hive/indexer/sync.py
+++ b/hive/indexer/sync.py
@@ -159,7 +159,7 @@ class Sync:
timer.batch_lap()
# process blocks
- Blocks.process_multi(blocks, is_initial_sync)
+ Blocks.process_multi(blocks, steemd, is_initial_sync)
timer.batch_finish(len(blocks))
_prefix = ("[SYNC] Got block %d @ %s" % (
@@ -192,7 +192,7 @@ class Sync:
start_time = perf()
self._db.query("START TRANSACTION")
- num = Blocks.process(block)
+ num = Blocks.process(block, steemd)
follows = Follow.flush(trx=False)
accts = Accounts.flush(steemd, trx=False, spread=8)
CachedPost.dirty_paidouts(block['timestamp'])
@@ -221,6 +221,10 @@ class Sync:
def _update_chain_state(self):
"""Update basic state props (head block, feed price) in db."""
state = self._steem.gdgp_extended()
+ print("======================")
+ print(state['steem_per_mvest'])
+ print(state['usd_per_steem'])
+ print(state['sbd_per_steem'])
self._db.query("""UPDATE hive_state SET block_num = :block_num,
steem_per_mvest = :spm, usd_per_steem = :ups,
sbd_per_steem = :sps, dgpo = :dgpo""",
diff --git a/hive/server/bridge_api/objects.py b/hive/server/bridge_api/objects.py
index e29ba5b3e..f4d7404f5 100644
--- a/hive/server/bridge_api/objects.py
+++ b/hive/server/bridge_api/objects.py
@@ -255,7 +255,7 @@ def _condenser_post_object(row, truncate_body=0):
post['promoted'] = _amount(row['promoted'])
post['replies'] = []
- post['active_votes'] = _hydrate_active_votes(row['votes'])
+ post['active_votes'] = json.loads(row['votes'])
post['author_reputation'] = row['author_rep']
post['stats'] = {
diff --git a/hive/server/common/objects.py b/hive/server/common/objects.py
index f2497a6c3..37f81b5e5 100644
--- a/hive/server/common/objects.py
+++ b/hive/server/common/objects.py
@@ -60,7 +60,7 @@ def condenser_post_object(row, truncate_body=0):
post['replies'] = []
post['body_length'] = len(row['body'])
- post['active_votes'] = _hydrate_active_votes(row['votes'])
+ post['active_votes'] = json.loads(row['votes'])
#post['author_reputation'] = rep_to_raw(row['author_rep'])
post['root_author'] = row['root_author']
diff --git a/hive/server/condenser_api/get_state.py b/hive/server/condenser_api/get_state.py
index a66502d54..61487be06 100644
--- a/hive/server/condenser_api/get_state.py
+++ b/hive/server/condenser_api/get_state.py
@@ -291,8 +291,8 @@ async def _get_props_lite(db):
# convert NAI amounts to legacy
nais = ['virtual_supply', 'current_supply', 'current_sbd_supply',
- 'pending_rewarded_vesting_steem', 'pending_rewarded_vesting_shares',
- 'total_vesting_fund_steem', 'total_vesting_shares']
+ 'pending_rewarded_vesting_hive', 'pending_rewarded_vesting_shares',
+ 'total_vesting_fund_hive', 'total_vesting_shares']
for k in nais:
if k in raw:
raw[k] = legacy_amount(raw[k])
@@ -303,6 +303,6 @@ async def _get_props_lite(db):
sbd_interest_rate=raw['sbd_interest_rate'],
head_block_number=raw['head_block_number'], #*
total_vesting_shares=raw['total_vesting_shares'],
- total_vesting_fund_steem=raw['total_vesting_fund_steem'],
+ total_vesting_fund_steem=raw['total_vesting_fund_hive'],
last_irreversible_block_num=raw['last_irreversible_block_num'], #*
)
diff --git a/hive/server/condenser_api/objects.py b/hive/server/condenser_api/objects.py
index 8c267f863..68ba8ceb7 100644
--- a/hive/server/condenser_api/objects.py
+++ b/hive/server/condenser_api/objects.py
@@ -210,7 +210,7 @@ def _condenser_post_object(row, truncate_body=0):
post['replies'] = []
post['body_length'] = len(row['body'])
- post['active_votes'] = _hydrate_active_votes(row['votes'])
+ post['active_votes'] = json.loads(row['votes'])
post['author_reputation'] = rep_to_raw(row['author_rep'])
post['root_author'] = row['root_author']
diff --git a/hive/steem/client.py b/hive/steem/client.py
index db2404945..c5a281e7e 100644
--- a/hive/steem/client.py
+++ b/hive/steem/client.py
@@ -84,13 +84,15 @@ class SteemClient:
def gdgp_extended(self):
"""Get dynamic global props without the cruft plus useful bits."""
dgpo = self._gdgp()
+ print(dgpo)
# remove unused/deprecated keys
unused = ['total_pow', 'num_pow_witnesses', 'confidential_supply',
'confidential_sbd_supply', 'total_reward_fund_steem',
'total_reward_shares2']
for key in unused:
- del dgpo[key]
+ if key in dgpo:
+ del dgpo[key]
return {
'dgpo': dgpo,
@@ -100,7 +102,8 @@ class SteemClient:
@staticmethod
def _get_steem_per_mvest(dgpo):
- steem = steem_amount(dgpo['total_vesting_fund_steem'])
+ print("DGPO: ", dgpo)
+ steem = steem_amount(dgpo['total_vesting_fund_hive'])
mvests = vests_amount(dgpo['total_vesting_shares']) / Decimal(1e6)
return "%.6f" % (steem / mvests)
@@ -108,15 +111,20 @@ class SteemClient:
# TODO: add latest feed price: get_feed_history.price_history[0]
feed = self.__exec('get_feed_history')['current_median_history']
units = dict([parse_amount(feed[k])[::-1] for k in ['base', 'quote']])
- price = units['HBD'] / units['HIVE']
+ if 'TBD' in units and 'TESTS' in units:
+ price = units['TBD'] / units['TESTS']
+ else:
+ price = units['HBD'] / units['HIVE']
return "%.6f" % price
def _get_steem_price(self):
orders = self.__exec('get_order_book', [1])
- ask = Decimal(orders['asks'][0]['real_price'])
- bid = Decimal(orders['bids'][0]['real_price'])
- price = (ask + bid) / 2
- return "%.6f" % price
+ if orders['asks'] and orders[bids]:
+ ask = Decimal(orders['asks'][0]['real_price'])
+ bid = Decimal(orders['bids'][0]['real_price'])
+ price = (ask + bid) / 2
+ return "%.6f" % price
+ return "0"
def get_blocks_range(self, lbound, ubound):
"""Retrieves blocks in the range of [lbound, ubound)."""
@@ -132,6 +140,22 @@ class SteemClient:
return [blocks[x] for x in block_nums]
+ def get_comment_pending_payouts(self, comments):
+ """ Get comment pending payout data """
+ ret = self.__exec('get_comment_pending_payouts', {'comments':comments})
+ print(ret)
+ return ret['cashout_infos']
+
+ def get_votes(self, author, permlink):
+ """ Get list of votes """
+ call = self.__exec("list_votes", {'start':[author, permlink, ""],
+ 'limit':1000, 'order':'by_comment_voter'})
+ ret = []
+ for vote in call['votes']:
+ if vote['author'] == author and vote['permlink'] == permlink:
+ ret.append(vote)
+ return ret
+
def __exec(self, method, params=None):
"""Perform a single steemd call."""
start = perf()
diff --git a/hive/steem/http_client.py b/hive/steem/http_client.py
index c0bb7ebf9..d1b37ff9e 100644
--- a/hive/steem/http_client.py
+++ b/hive/steem/http_client.py
@@ -84,11 +84,12 @@ class HttpClient(object):
METHOD_API = dict(
lookup_accounts='condenser_api',
get_block='block_api',
- get_content='condenser_api',
get_accounts='condenser_api',
get_order_book='condenser_api',
get_feed_history='condenser_api',
get_dynamic_global_properties='database_api',
+ list_votes='database_api',
+ get_comment_pending_payouts='database_api',
)
def __init__(self, nodes, **kwargs):
diff --git a/hive/utils/stats.py b/hive/utils/stats.py
index 0f3961ebb..5f6f8c7d4 100644
--- a/hive/utils/stats.py
+++ b/hive/utils/stats.py
@@ -89,6 +89,8 @@ class SteemStats(StatsAbstract):
'get_order_book': 20,
'get_feed_history': 20,
'lookup_accounts': 1000,
+ 'list_votes':1000,
+ 'get_comment_pending_payouts':1000
}
def __init__(self):
diff --git a/scripts/update_hivemind_db.sql b/scripts/update_hivemind_db.sql
index acdae4cde..42175c3db 100644
--- a/scripts/update_hivemind_db.sql
+++ b/scripts/update_hivemind_db.sql
@@ -71,7 +71,7 @@ CREATE TABLE IF NOT EXISTS hive_posts_new (
permlink_id INT NOT NULL,
category_id INT DEFAULT '1',
community_id INT,
- created_at DATE DEFAULT '1990-01-01T00:00:00',
+ created_at DATE DEFAULT '1970-01-01T00:00:00',
depth SMALLINT DEFAULT '-1',
is_deleted BOOLEAN DEFAULT '0',
is_pinned BOOLEAN DEFAULT '0',
@@ -90,8 +90,8 @@ CREATE TABLE IF NOT EXISTS hive_posts_new (
-- core stats/indexes
payout NUMERIC(10, 3) DEFAULT '0.0',
- payout_at DATE DEFAULT '1990-01-01T00:00:00',
- updated_at DATE DEFAULT '1990-01-01T00:00:00',
+ payout_at DATE DEFAULT '1970-01-01T00:00:00',
+ updated_at DATE DEFAULT '1970-01-01T00:00:00',
is_paidout BOOLEAN DEFAULT '0',
-- ui flags/filters
@@ -106,6 +106,20 @@ CREATE TABLE IF NOT EXISTS hive_posts_new (
sc_trend NUMERIC(6) DEFAULT '0.0',
sc_hot NUMERIC(6) DEFAULT '0.0',
+ total_payout_value VARCHAR(16) DEFAULT '',
+ author_rewards INT DEFAULT '0',
+ children_abs_rshares INT DEFAULT '0',
+ net_rshares INT DEFAULT '0',
+ abs_rshares INT DEFAULT '0',
+ vote_rshares INT DEFAULT '0',
+ net_votes INT DEFAULT '0',
+ active DATE DEFAULT '1970-01-01T00:00:00',
+ last_payout DATE DEFAULT '1970-01-01T00:00:00',
+ cashout_time DATE DEFAULT '1970-01-01T00:00:00',
+ max_cashout_time DATE DEFAULT '1970-01-01T00:00:00',
+ percent_hbd INT DEFAULT '0',
+ reward_weight INT DEFAULT '0',
+
-- columns from raw_json
parent_author_id INT DEFAULT '-1',
parent_permlink_id INT DEFAULT '-1',
--
GitLab