Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • hive/hivemind
1 result
Show changes
Commits on Source (27)
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 696 additions and 330 deletions
.gitignore
View file @ 9bf7b6ed
......@@ -130,3 +130,6 @@ tests/failed_blocks/
# pyrest tests
*.out.json
# version.py
hive/version.py
.gitlab-ci.yaml
View file @ 9bf7b6ed
......@@ -36,7 +36,7 @@ before_script:
hivemind_build:
stage: build
script:
- pip3 install --user --upgrade pip setuptools
- pip3 install --user --upgrade pip setuptools gitpython
- git fetch --tags
- git tag -f ci_implicit_tag
- echo $PYTHONUSERBASE
......@@ -71,7 +71,7 @@ hivemind_sync:
PYTHONUSERBASE: ./local-site
script:
- pip3 install --user --upgrade pip setuptools
- pip3 install --user --upgrade pip setuptools gitpython
- scripts/ci_sync.sh "$HIVEMIND_DB_NAME" "$HIVEMIND_POSTGRESQL_CONNECTION_STRING" "$HIVEMIND_SOURCE_HIVED_URL" $HIVEMIND_MAX_BLOCK $HIVEMIND_HTTP_PORT
artifacts:
......
Pipfile
View file @ 9bf7b6ed
......@@ -16,6 +16,7 @@ aiohttp = "*"
aiopg = "*"
"psycopg2-binary" = "*"
"diff-match-patch" = "*"
"gitpython" = "*"
[dev-packages]
......
hive/conf.py
View file @ 9bf7b6ed
......@@ -61,8 +61,13 @@ class Conf():
# configure logger and print config
root = logging.getLogger()
root.setLevel(conf.log_level())
root.info("loaded configuration:\n%s",
_sanitized_conf(parser))
from sys import argv
root.info("Used command line args: %s", " ".join(argv[1:]))
# uncomment for full list of program args
#args_list = ["--" + k + " " + str(v) for k,v in vars(args).items()]
#root.info("Full command line args: %s", " ".join(args_list))
if conf.mode() == 'server':
#DbStats.SLOW_QUERY_MS = 750
......
hive/db/db_state.py
View file @ 9bf7b6ed
......@@ -346,7 +346,7 @@ class DbState:
cls._set_ver(17)
if cls._ver == 17:
cls.db().query("INSERT INTO hive_accounts (name, created_at) VALUES ('', '1990-01-01T00:00:00') ON CONFLICT (name) DO NOTHING")
cls.db().query("INSERT INTO hive_accounts (name, created_at) VALUES ('', '1970-01-01T00:00:00') ON CONFLICT (name) DO NOTHING")
cls.db().query("INSERT INTO hive_permlink_data (permlink) VALUES ('') ON CONFLICT (permlink) DO NOTHING")
cls.db().query("INSERT INTO hive_category_data (category) VALUES ('') ON CONFLICT (category) DO NOTHING")
cls._set_ver(18)
......
hive/db/schema.py
View file @ 9bf7b6ed
This diff is collapsed.
hive/indexer/blocks.py
View file @ 9bf7b6ed
......@@ -23,14 +23,17 @@ class Blocks:
"""Processes blocks, dispatches work, manages `hive_blocks` table."""
blocks_to_flush = []
ops_stats = {}
_head_block_date = None
_head_block_date = None # timestamp of last fully processed block ("previous block")
_current_block_date = None # timestamp of block currently being processes ("current block")
def __init__(cls):
head_date = cls.head_date()
if(head_date == ''):
cls._head_block_date = None
cls._current_block_date = None
else:
cls._head_block_date = head_date
cls._current_block_date = head_date
@staticmethod
def merge_ops_stats(od1, od2):
......@@ -157,15 +160,17 @@ class Blocks:
"""Process a single block. Assumes a trx is open."""
#pylint: disable=too-many-branches
num = cls._push(block)
block_date = block['timestamp']
cls._current_block_date = block['timestamp']
# head block date shall point to last imported block (not yet current one) to conform hived behavior.
# that's why operations processed by node are included in the block being currently produced, so its processing time is equal to last produced block.
# unfortunately it is not true to all operations, most likely in case of dates that used to come from
# FatNode where it supplemented it with its-current head block, since it was already past block processing,
# it saw later block (equal to _current_block_date here)
if cls._head_block_date is None:
cls._head_block_date = block_date
cls._head_block_date = cls._current_block_date
json_ops = []
update_comment_pending_payouts = []
for tx_idx, tx in enumerate(block['transactions']):
for operation in tx['operations']:
op_type = operation['type']
......@@ -231,19 +236,19 @@ class Blocks:
if is_initial_sync:
if num in virtual_operations:
(vote_ops, comment_payout_stats) = Blocks.prepare_vops(Posts.comment_payout_ops, virtual_operations[num], cls._head_block_date)
(vote_ops, comment_payout_stats) = Blocks.prepare_vops(Posts.comment_payout_ops, virtual_operations[num], cls._current_block_date)
else:
vops = hived.get_virtual_operations(num)
(vote_ops, comment_payout_stats) = Blocks.prepare_vops(Posts.comment_payout_ops, vops, cls._head_block_date)
(vote_ops, comment_payout_stats) = Blocks.prepare_vops(Posts.comment_payout_ops, vops, cls._current_block_date)
if vote_ops is not None:
for k, v in vote_ops.items():
Votes.effective_comment_vote_op(k, v)
for k, v in vote_ops.items():
Votes.effective_comment_vote_op(k, v)
if Posts.comment_payout_ops:
cls.ops_stats = Blocks.merge_ops_stats(cls.ops_stats, comment_payout_stats)
cls._head_block_date = block_date
cls._head_block_date = cls._current_block_date
return num
......@@ -312,7 +317,9 @@ class Blocks:
"""
values = []
for block in cls.blocks_to_flush:
values.append("({}, '{}', '{}', {}, {}, '{}')".format(block['num'], block['hash'], block['prev'], block['txs'], block['ops'], block['date']))
values.append("({}, '{}', '{}', {}, {}, '{}')".format(block['num'], block['hash'],
block['prev'], block['txs'],
block['ops'], block['date']))
DB.query(query + ",".join(values))
cls.blocks_to_flush = []
......@@ -365,7 +372,7 @@ class Blocks:
if post_ids:
DB.query("DELETE FROM hive_post_tags WHERE post_id IN :ids", ids=post_ids)
DB.query("DELETE FROM hive_posts WHERE id IN :ids", ids=post_ids)
DB.query("DELETE FROM hive_posts_data WHERE id IN :ids", ids=post_ids)
DB.query("DELETE FROM hive_post_data WHERE id IN :ids", ids=post_ids)
DB.query("DELETE FROM hive_payments WHERE block_num = :num", num=num)
DB.query("DELETE FROM hive_blocks WHERE num = :num", num=num)
......
hive/indexer/posts.py
View file @ 9bf7b6ed
......@@ -161,7 +161,6 @@ class Posts:
payout = COALESCE( CAST( data_source.payout as DECIMAL ), ihp.payout ),
pending_payout = COALESCE( CAST( data_source.pending_payout as DECIMAL ), ihp.pending_payout ),
payout_at = COALESCE( CAST( data_source.payout_at as TIMESTAMP ), ihp.payout_at ),
last_payout = COALESCE( CAST( data_source.last_payout as TIMESTAMP ), ihp.last_payout ),
cashout_time = COALESCE( CAST( data_source.cashout_time as TIMESTAMP ), ihp.cashout_time ),
is_paidout = COALESCE( CAST( data_source.is_paidout as BOOLEAN ), ihp.is_paidout )
FROM
......@@ -176,7 +175,6 @@ class Posts:
t.payout,
t.pending_payout,
t.payout_at,
t.last_payout,
t.cashout_time,
t.is_paidout
from
......@@ -194,7 +192,6 @@ class Posts:
payout,
pending_payout,
payout_at,
last_payout,
cashout_time,
is_paidout)
INNER JOIN hive_accounts ha_a ON ha_a.name = t.author
......@@ -241,7 +238,6 @@ class Posts:
pending_payout = None
payout_at = None
last_payout = None
cashout_time = None
is_paidout = None
......@@ -300,14 +296,9 @@ class Posts:
#Calculations of all dates
if ( is_paidout is not None ):
payout_at = date
last_payout = date
cashout_time = "1969-12-31T23:59:59"
else:
if ( total_payout_value is not None ):
payout_at = date #Here should be `cashout_time`
last_payout = date
cls._comment_payout_ops.append("('{}', '{}', {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {})".format(
cls._comment_payout_ops.append("('{}', '{}', {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {})".format(
author,
permlink,
"NULL" if ( total_payout_value is None ) else ( "'{}'".format( legacy_amount(total_payout_value) ) ),
......@@ -320,7 +311,6 @@ class Posts:
"NULL" if ( pending_payout is None ) else pending_payout,
"NULL" if ( payout_at is None ) else ( "'{}'::timestamp".format( payout_at ) ),
"NULL" if ( last_payout is None ) else ( "'{}'::timestamp".format( last_payout ) ),
"NULL" if ( cashout_time is None ) else ( "'{}'::timestamp".format( cashout_time ) ),
"NULL" if ( is_paidout is None ) else is_paidout ))
......
hive/indexer/sync.py
View file @ 9bf7b6ed
......@@ -44,7 +44,6 @@ def prepare_vops(vops_by_block):
for blockNum, blockDict in vops_by_block.items():
vopsList = blockDict['ops']
date = blockDict['timestamp']
preparedVops[blockNum] = vopsList
return preparedVops
......
hive/indexer/votes.py
View file @ 9bf7b6ed
......@@ -46,6 +46,36 @@ class Votes:
ret = DB.query_row(sql, author=author, permlink=permlink)
return 0 if ret is None else int(ret.count)
@classmethod
def get_total_vote_weight(cls, author, permlink):
""" Get total vote weight for selected post """
sql = """
SELECT
sum(weight)
FROM
hive_votes_accounts_permlinks_view hv
WHERE
hv.author = :author AND
hv.permlink = :permlink
"""
ret = DB.query_row(sql, author=author, permlink=permlink)
return 0 if ret is None else int(0 if ret.sum is None else ret.sum)
@classmethod
def get_total_vote_rshares(cls, author, permlink):
""" Get total vote rshares for selected post """
sql = """
SELECT
sum(rshares)
FROM
hive_votes_accounts_permlinks_view hv
WHERE
hv.author = :author AND
hv.permlink = :permlink
"""
ret = DB.query_row(sql, author=author, permlink=permlink)
return 0 if ret is None else int(0 if ret.sum is None else ret.sum)
inside_flush = False
@classmethod
......@@ -56,15 +86,15 @@ class Votes:
permlink = vote_operation['permlink']
weight = vote_operation['weight']
if(cls.inside_flush):
log.info("Adding new vote-info into '_votes_data' dict")
raise "Fatal error"
if cls.inside_flush:
log.exception("Adding new vote-info into '_votes_data' dict")
raise RuntimeError("Fatal error")
key = voter + "/" + author + "/" + permlink
if key in cls._votes_data:
cls._votes_data[key]["vote_percent"]=weight
cls._votes_data[key]["last_update"]=date
cls._votes_data[key]["vote_percent"] = weight
cls._votes_data[key]["last_update"] = date
else:
cls._votes_data[key] = dict(voter=voter,
author=author,
......@@ -79,9 +109,9 @@ class Votes:
def effective_comment_vote_op(cls, key, vop):
""" Process effective_comment_vote_operation """
if(cls.inside_flush):
log.info("Updating data in '_votes_data' using effective comment")
raise "Fatal error"
if cls.inside_flush:
log.exception("Updating data in '_votes_data' using effective comment")
raise RuntimeError("Fatal error")
assert key in cls._votes_data
......@@ -140,18 +170,18 @@ class Votes:
if len(values) >= values_limit:
values_str = ','.join(values)
actual_query = sql.format(values_str,on_conflict_data_source,on_conflict_data_source)
actual_query = sql.format(values_str, on_conflict_data_source, on_conflict_data_source)
DB.query(actual_query)
values.clear()
if len(values_skip) > 0:
values_str = ','.join(values_skip)
actual_query = sql.format(values_str,'hive_votes','hive_votes')
actual_query = sql.format(values_str, 'hive_votes', 'hive_votes')
DB.query(actual_query)
values_skip.clear()
if len(values_override) > 0:
values_str = ','.join(values_override)
actual_query = sql.format(values_str,'EXCLUDED','EXCLUDED')
actual_query = sql.format(values_str, 'EXCLUDED', 'EXCLUDED')
DB.query(actual_query)
values_override.clear()
......
hive/server/bridge_api/methods.py
View file @ 9bf7b6ed
"""Bridge API public endpoints for posts"""
import hive.server.bridge_api.cursor as cursor
from hive.server.bridge_api.objects import load_posts, load_posts_reblogs, load_profiles, _condenser_post_object
from hive.server.bridge_api.objects import load_posts, load_posts_reblogs, load_profiles, _bridge_post_object
from hive.server.database_api.methods import find_votes, VotesPresentation
from hive.server.common.helpers import (
return_error_info,
valid_account,
......@@ -20,18 +21,15 @@ SQL_TEMPLATE = """
SELECT
hp.id,
hp.author,
hp.root_author,
hp.parent_author,
hp.author_rep,
hp.allow_replies,
hp.allow_votes,
hp.allow_curation_rewards,
hp.root_title,
hp.beneficiaries,
hp.max_accepted_payout,
hp.percent_hbd,
hp.url,
hp.permlink,
hp.root_permlink,
hp.parent_permlink,
hp.title,
hp.body,
hp.category,
......@@ -62,16 +60,6 @@ SQL_TEMPLATE = """
#pylint: disable=too-many-arguments, no-else-return
async def _get_post_id(db, author, permlink):
"""Get post_id from hive db."""
sql = """SELECT id FROM hive_posts
WHERE author_id = (SELECT id FROM hive_accounts WHERE name = :a)
AND permlink_id = (SELECT id FROM hive_permlink_data WHERE permlink = :p)
AND is_deleted = '0'"""
post_id = await db.query_one(sql, a=author, p=permlink)
assert post_id, 'invalid author/permlink'
return post_id
@return_error_info
async def get_profile(context, account, observer=None):
"""Load account/profile data."""
......@@ -120,7 +108,8 @@ async def get_post(context, author, permlink, observer=None):
result = await db.query_all(sql, author=author, permlink=permlink)
assert len(result) == 1, 'invalid author/permlink or post not found in cache'
post = _condenser_post_object(result[0])
post = _bridge_post_object(result[0])
post['active_votes'] = await find_votes({'db':db}, {'author':author, 'permlink':permlink}, VotesPresentation.BridgeApi)
post = await append_statistics_to_post(post, result[0], False, blacklists_for_user)
return post
......@@ -223,7 +212,8 @@ async def get_ranked_posts(context, sort, start_author='', start_permlink='',
if pinned_sql:
pinned_result = await db.query_all(pinned_sql, author=start_author, limit=limit, tag=tag, permlink=start_permlink, community_name=tag, observer=observer)
for row in pinned_result:
post = _condenser_post_object(row)
post = _bridge_post_object(row)
post['active_votes'] = await find_votes({'db':db}, {'author':row['author'], 'permlink':row['permlink']}, VotesPresentation.BridgeApi)
post = await append_statistics_to_post(post, row, True, blacklists_for_user)
limit = limit - 1
posts.append(post)
......@@ -231,7 +221,8 @@ async def get_ranked_posts(context, sort, start_author='', start_permlink='',
sql_result = await db.query_all(sql, author=start_author, limit=limit, tag=tag, permlink=start_permlink, community_name=tag, observer=observer)
for row in sql_result:
post = _condenser_post_object(row)
post = _bridge_post_object(row)
post['active_votes'] = await find_votes({'db':db}, {'author':row['author'], 'permlink':row['permlink']}, VotesPresentation.BridgeApi)
post = await append_statistics_to_post(post, row, False, blacklists_for_user)
if post['post_id'] in pinned_post_ids:
continue
......@@ -322,7 +313,8 @@ async def get_account_posts(context, sort, account, start_author='', start_perml
blacklists_for_user = await Mutes.get_blacklists_for_observer(observer, context)
sql_result = await db.query_all(sql, account=account, author=start_author, permlink=start_permlink, limit=limit)
for row in sql_result:
post = _condenser_post_object(row)
post = _bridge_post_object(row)
post['active_votes'] = await find_votes({'db':db}, {'author':row['author'], 'permlink':row['permlink']}, VotesPresentation.BridgeApi)
post = await append_statistics_to_post(post, row, False, blacklists_for_user)
posts.append(post)
return posts
......
hive/server/bridge_api/objects.py
View file @ 9bf7b6ed
......@@ -5,7 +5,7 @@ import ujson as json
from hive.server.common.mutes import Mutes
from hive.server.common.helpers import json_date
from hive.server.database_api.methods import find_votes
from hive.server.database_api.methods import find_votes, VotesPresentation
from hive.utils.normalize import sbd_amount
from hive.indexer.votes import Votes
......@@ -100,7 +100,7 @@ async def load_posts_keyed(db, ids, truncate_body=0):
author_ids[author['id']] = author['name']
row['author_rep'] = author['reputation']
post = _condenser_post_object(row, truncate_body=truncate_body)
post = _bridge_post_object(row, truncate_body=truncate_body)
post['active_votes'] = await find_votes({'db':db}, {'author':row['author'], 'permlink':row['permlink']})
post['blacklists'] = Mutes.lists(post['author'], author['reputation'])
......@@ -219,7 +219,7 @@ def _condenser_profile_object(row):
'profile_image': row['profile_image'],
}}}
def _condenser_post_object(row, truncate_body=0):
def _bridge_post_object(row, truncate_body=0):
"""Given a hive_posts row, create a legacy-style post object."""
paid = row['is_paidout']
......@@ -231,7 +231,7 @@ def _condenser_post_object(row, truncate_body=0):
post['title'] = row['title']
post['body'] = row['body'][0:truncate_body] if truncate_body else row['body']
post['json_metadata'] = row['json']
post['json_metadata'] = json.loads(row['json'])
post['created'] = json_date(row['created_at'])
post['updated'] = json_date(row['updated_at'])
......@@ -245,37 +245,30 @@ def _condenser_post_object(row, truncate_body=0):
post['pending_payout_value'] = _amount(0 if paid else row['payout'])
post['author_payout_value'] = _amount(row['payout'] if paid else 0)
post['curator_payout_value'] = _amount(0)
post['promoted'] = float(row['promoted'])
post['promoted'] = _amount(row['promoted'])
post['replies'] = []
# ABW: missing post['active_votes'] = _hydrate_active_votes(row['votes'])
post['author_reputation'] = float(row['author_rep'])
post['stats'] = {
'hide': row['is_hidden'],
'gray': row['is_grayed'],
'total_votes': Votes.get_vote_count(row['author'], row['permlink']),
'total_votes': Votes.get_vote_count(row['author'], row['permlink']), # ABW: incorrect calculation (possibly needs to exclude blacklisted votes)
'flag_weight': float(row['flag_weight'])} # TODO: down_weight
#post['author_reputation'] = rep_to_raw(row['author_rep'])
post['root_author'] = row['root_author']
post['root_permlink'] = row['root_permlink']
post['allow_replies'] = row['allow_replies']
post['allow_votes'] = row['allow_votes']
post['allow_curation_rewards'] = row['allow_curation_rewards']
post['url'] = row['url']
post['root_title'] = row['root_title']
post['beneficiaries'] = row['beneficiaries']
post['max_accepted_payout'] = row['max_accepted_payout']
post['percent_hbd'] = row['percent_hbd']
if paid:
curator_payout = sbd_amount(row['curator_payout_value'])
post['author_payout_value'] = _amount(row['payout'] - curator_payout)
post['curator_payout_value'] = _amount(curator_payout)
post['total_payout_value'] = _amount(row['payout'] - curator_payout)
# TODO: re-evaluate
if row['depth'] > 0:
......
hive/server/bridge_api/support.py
View file @ 9bf7b6ed
......@@ -4,7 +4,7 @@ import logging
#import ujson as json
import traceback
from hive.server.bridge_api.objects import _condenser_post_object
from hive.server.bridge_api.objects import _bridge_post_object
from hive.utils.post import post_to_internal
from hive.utils.normalize import sbd_amount
from hive.server.common.helpers import (
......@@ -96,7 +96,7 @@ async def normalize_post(context, post):
try:
if 'promoted' not in row: row['promoted'] = 0
row['author_rep'] = author['reputation']
ret = _condenser_post_object(row)
ret = _bridge_post_object(row)
except Exception as e:
log.error("post_to_internal: %s %s", repr(e), traceback.format_exc())
raise e
......
hive/server/bridge_api/thread.py
View file @ 9bf7b6ed
......@@ -2,8 +2,9 @@
import logging
from hive.server.bridge_api.objects import load_posts_keyed, _condenser_post_object
from hive.server.bridge_api.objects import load_posts_keyed, _bridge_post_object
from hive.server.bridge_api.methods import append_statistics_to_post
from hive.server.database_api.methods import find_votes, VotesPresentation
from hive.server.common.helpers import (
return_error_info,
valid_account,
......@@ -91,7 +92,8 @@ async def get_discussion(context, author, permlink, observer=None):
return {}
root_id = rows[0]['id']
all_posts = {}
root_post = _condenser_post_object(rows[0])
root_post = _bridge_post_object(rows[0])
root_post['active_votes'] = await find_votes({'db':db}, {'author':rows[0]['author'], 'permlink':rows[0]['permlink']}, VotesPresentation.BridgeApi)
root_post = await append_statistics_to_post(root_post, rows[0], False, blacklists_for_user)
root_post['replies'] = []
all_posts[root_id] = root_post
......@@ -101,7 +103,8 @@ async def get_discussion(context, author, permlink, observer=None):
for index in range(1, len(rows)):
id_to_parent_id_map[rows[index]['id']] = rows[index]['parent_id']
post = _condenser_post_object(rows[index])
post = _bridge_post_object(rows[index])
post['active_votes'] = await find_votes({'db':db}, {'author':rows[index]['author'], 'permlink':rows[index]['permlink']}, VotesPresentation.BridgeApi)
post = await append_statistics_to_post(post, rows[index], False, blacklists_for_user)
post['replies'] = []
all_posts[post['post_id']] = post
......@@ -138,20 +141,6 @@ def get_children(parent_id, posts):
results.append(key)
return results
async def _get_post_id(db, author, permlink):
"""Given an author/permlink, retrieve the id from db."""
sql = """
SELECT
id
FROM hive_posts hp
INNER JOIN hive_accounts ha_a ON ha_a.id = hp.author_id
INNER JOIN hive_permlink_data hpd_p ON hpd_p.id = hp.permlink_id
WHERE ha_a.name = :author
AND hpd_p.permlink = :permlink
AND is_deleted = '0'
LIMIT 1"""
return await db.query_one(sql, a=author, p=permlink)
def _ref(post):
return post['author'] + '/' + post['permlink']
......
hive/server/condenser_api/methods.py
View file @ 9bf7b6ed
......@@ -384,11 +384,13 @@ async def get_discussions_by_blog(context, tag: str = None, start_author: str =
for row in result:
row = dict(row)
post = _condenser_post_object(row, truncate_body=truncate_body)
post['active_votes'] = await find_votes(context, {'author':post['author'], 'permlink':post['permlink']})
post['active_votes'] = await find_votes(context, {'author':post['author'], 'permlink':post['permlink']}, votes_presentation = VotesPresentation.CondenserApi)
post['active_votes'] = _mute_votes(post['active_votes'], Mutes.all())
#posts_by_id[row['post_id']] = post
posts_by_id.append(post)
return posts_by_id
@return_error_info
@nested_query_compat
async def get_discussions_by_feed(context, tag: str = None, start_author: str = '',
......
hive/server/database_api/methods.py
View file @ 9bf7b6ed
......@@ -2,167 +2,66 @@
from enum import Enum
from hive.server.common.helpers import return_error_info, valid_limit, valid_account, valid_permlink
from hive.server.common.objects import condenser_post_object
from hive.server.database_api.objects import database_post_object
from hive.utils.normalize import rep_to_raw, number_to_json_value, time_string_with_t
SQL_TEMPLATE = """
SELECT
hp.id,
hp.community_id,
hp.author,
hp.permlink,
hp.title,
hp.body,
hp.category,
hp.depth,
hp.promoted,
hp.payout,
hp.payout_at,
hp.is_paidout,
hp.children,
hp.votes,
hp.created_at,
hp.updated_at,
hp.rshares,
hp.json,
hp.is_hidden,
hp.is_grayed,
hp.total_votes,
hp.flag_weight,
hp.parent_author,
hp.parent_permlink,
hp.curator_payout_value,
hp.root_author,
hp.root_permlink,
hp.max_accepted_payout,
hp.percent_hbd,
hp.allow_replies,
hp.allow_votes,
hp.allow_curation_rewards,
hp.beneficiaries,
hp.url,
hp.root_title
FROM hive_posts_view hp
WHERE
"""
async def get_post_id_by_author_and_permlink(db, author: str, permlink: str, limit: int):
"""Return post ids for given author and permlink"""
sql = """
SELECT hp.id
FROM hive_posts hp
INNER JOIN hive_accounts ha_a ON ha_a.id = hp.author_id
INNER JOIN hive_permlink_data hpd_p ON hpd_p.id = hp.permlink_id
WHERE ha_a.name >= :author AND hpd_p.permlink >= :permlink
ORDER BY ha_a.name ASC
LIMIT :limit
"""
result = await db.query_row(sql, author=author, permlink=permlink, limit=limit)
if result is not None:
return int(result.get('id', 0))
return 0
@return_error_info
async def list_comments(context, start: list, limit: int, order: str):
"""Returns all comments, starting with the specified options."""
supported_order_list = ['by_cashout_time', 'by_permlink', 'by_root', 'by_parent', 'by_update', 'by_author_last_update']
supported_order_list = ['by_cashout_time', 'by_permlink', 'by_root', 'by_parent', 'by_last_update', 'by_author_last_update']
assert order in supported_order_list, "Unsupported order, valid orders: {}".format(", ".join(supported_order_list))
limit = valid_limit(limit, 1000)
db = context['db']
comments = []
result = []
if order == 'by_cashout_time':
assert len(start) == 3, "Expecting three arguments"
cashout_time = start[0]
author = start[1]
permlink = start[2]
post_id = 0
if author or permlink:
post_id = await get_post_id_by_author_and_permlink(db, author, permlink, 1)
sql = str(SQL_TEMPLATE)
sql += "hp.payout_at >= :start AND hp.id >= :post_id ORDER BY hp.payout_at ASC, hp.id ASC LIMIT :limit"
result = await db.query_all(sql, start=start[0], limit=limit, post_id=post_id)
for row in result:
cpo = condenser_post_object(dict(row))
cpo['active_votes'] = await find_votes(context, {'author':cpo['author'], 'permlink':cpo['permlink']})
comments.append(cpo)
sql = "SELECT * FROM list_comments_by_cashout_time(:cashout_time, :author, :permlink, :limit)"
result = await db.query_all(sql, cashout_time=cashout_time, author=author, permlink=permlink, limit=limit)
elif order == 'by_permlink':
assert len(start) == 2, "Expecting two arguments"
sql = str(SQL_TEMPLATE)
sql += """ hp.id IN (SELECT hp1.id FROM hive_posts_a_p hp1 WHERE hp1.author >= :author COLLATE "C"
AND hp1.permlink >= :permlink COLLATE "C" ORDER BY hp1.author COLLATE "C" ASC LIMIT :limit)"""
result = await db.query_all(sql, author=start[0], permlink=start[1], limit=limit)
for row in result:
cpo = condenser_post_object(dict(row))
cpo['active_votes'] = await find_votes(context, {'author':cpo['author'], 'permlink':cpo['permlink']})
comments.append(cpo)
author = start[0]
permlink = start[1]
sql = "SELECT * FROM list_comments_by_permlink(:author, :permlink, :limit)"
result = await db.query_all(sql, author=author, permlink=permlink, limit=limit)
elif order == 'by_root':
assert len(start) == 4, "Expecting 4 arguments"
raise NotImplementedError('by_root')
sql = str(SQL_TEMPLATE)
sql += "get_rows_by_root(:root_author, :root_permlink, :child_author, :child_permlink) ORDER BY post_id ASC LIMIT :limit"
result = await db.query_all(sql, root_author=start[0], root_permlink=start[1], child_author=start[2], child_permlink=start[3], limit=limit)
for row in result:
cpo = condenser_post_object(dict(row))
cpo['active_votes'] = await find_votes(context, {'author':cpo['author'], 'permlink':cpo['permlink']})
comments.append(cpo)
root_author = start[0]
root_permlink = start[1]
start_post_author = start[2]
start_post_permlink = start[3]
sql = "SELECT * FROM list_comments_by_root(:root_author, :root_permlink, :start_post_author, :start_post_permlink, :limit)"
result = await db.query_all(sql, root_author=root_author, root_permlink=root_permlink, start_post_author=start_post_author, start_post_permlink=start_post_permlink, limit=limit)
elif order == 'by_parent':
assert len(start) == 4, "Expecting 4 arguments"
raise NotImplementedError('by_parent')
sql = str(SQL_TEMPLATE)
sql += "get_rows_by_parent(:parent_author, :parent_permlink, :child_author, :child_permlink) LIMIT :limit"
result = await db.query_all(sql, parent_author=start[0], parent_permlink=start[1], child_author=start[2], child_permlink=start[3], limit=limit)
for row in result:
cpo = condenser_post_object(dict(row))
cpo['active_votes'] = await find_votes(context, {'author':cpo['author'], 'permlink':cpo['permlink']})
comments.append(cpo)
elif order == 'by_update':
parent_author = start[0]
parent_permlink = start[1]
start_post_author = start[2]
start_post_permlink = start[3]
sql = "SELECT * FROM list_comments_by_parent(:parent_author, :parent_permlink, :start_post_author, :start_post_permlink, :limit)"
result = await db.query_all(sql, parent_author=parent_author, parent_permlink=parent_permlink, start_post_author=start_post_author, start_post_permlink=start_post_permlink, limit=limit)
elif order == 'by_last_update':
assert len(start) == 4, "Expecting 4 arguments"
child_author = start[2]
child_permlink = start[3]
post_id = 0
if author or permlink:
post_id = await get_post_id_by_author_and_permlink(db, child_author, child_permlink, 1)
sql = str(SQL_TEMPLATE)
sql += "hp.parent_author >= :parent_author AND hp.updated_at >= :updated_at AND hp.id >= :post_id ORDER BY hp.parent_author ASC, hp.updated_at ASC, hp.id ASC LIMIT :limit"
result = await db.query_all(sql, parent_author=start[0], updated_at=start[1], post_id=post_id, limit=limit)
for row in result:
cpo = condenser_post_object(dict(row))
cpo['active_votes'] = await find_votes(context, {'author':cpo['author'], 'permlink':cpo['permlink']})
comments.append(cpo)
parent_author = start[0]
updated_at = start[1]
start_post_author = start[2]
start_post_permlink = start[3]
sql = "SELECT * FROM list_comments_by_last_update(:parent_author, :updated_at, :start_post_author, :start_post_permlink, :limit)"
result = await db.query_all(sql, parent_author=parent_author, updated_at=updated_at, start_post_author=start_post_author, start_post_permlink=start_post_permlink, limit=limit)
elif order == 'by_author_last_update':
assert len(start) == 4, "Expecting 4 arguments"
author = start[0]
updated_at = start[1]
start_post_author = start[2]
start_post_permlink = start[3]
sql = "SELECT * FROM list_comments_by_author_last_update(:author, :updated_at, :start_post_author, :start_post_permlink, :limit)"
result = await db.query_all(sql, author=author, updated_at=updated_at, start_post_author=start_post_author, start_post_permlink=start_post_permlink, limit=limit)
author = start[2]
permlink = start[3]
post_id = 0
if author or permlink:
post_id = await get_post_id_by_author_and_permlink(db, author, permlink, 1)
sql = str(SQL_TEMPLATE)
sql += "hp.author >= :author AND hp.updated_at >= :updated_at AND hp.id >= :post_id ORDER BY hp.author ASC, hp.updated_at ASC, hp.id ASC LIMIT :limit"
result = await db.query_all(sql, author=start[0], updated_at=start[1], post_id=post_id, limit=limit)
for row in result:
cpo = condenser_post_object(dict(row))
cpo['active_votes'] = await find_votes(context, {'author':cpo['author'], 'permlink':cpo['permlink']})
comments.append(cpo)
return comments
return [database_post_object(dict(row)) for row in result]
@return_error_info
async def find_comments(context, start: list, limit: int, order: str):
......@@ -172,20 +71,70 @@ async def find_comments(context, start: list, limit: int, order: str):
assert len(start) <= 1000, "Parameters count is greather than max allowed (1000)"
db = context['db']
# make a copy
sql = str(SQL_TEMPLATE)
SQL_TEMPLATE = """
SELECT
hp.id,
hp.community_id,
hp.author,
hp.permlink,
hp.title,
hp.body,
hp.category,
hp.depth,
hp.promoted,
hp.payout,
hp.payout_at,
hp.is_paidout,
hp.children,
hp.votes,
hp.created_at,
hp.updated_at,
hp.rshares,
hp.json,
hp.is_hidden,
hp.is_grayed,
hp.total_votes,
hp.flag_weight,
hp.parent_author,
hp.parent_permlink,
hp.curator_payout_value,
hp.root_author,
hp.root_permlink,
hp.max_accepted_payout,
hp.percent_hbd,
hp.allow_replies,
hp.allow_votes,
hp.allow_curation_rewards,
hp.beneficiaries,
hp.url,
hp.root_title,
hp.abs_rshares,
hp.active,
hp.author_rewards,
hp.max_cashout_time,
hp.reward_weight
FROM
hive_posts_view hp
WHERE
NOT hp.is_muted AND
NOT hp.is_deleted AND
"""
idx = 0
sql = ""
for arg in start:
if idx > 0:
sql += " OR "
sql += "(hp.author = '{}' AND hp.permlink = '{}')".format(arg[0], arg[1])
sql += " UNION ALL "
sql += str(SQL_TEMPLATE)
sql += """
hp.author = '{}' AND
hp.permlink = '{}'
""".format(arg[0], arg[1])
idx += 1
result = await db.query_all(sql)
for row in result:
cpo = condenser_post_object(dict(row))
cpo['active_votes'] = await find_votes(context, {'author':cpo['author'], 'permlink':cpo['permlink']})
cpo = database_post_object(dict(row))
comments.append(cpo)
return comments
......@@ -194,6 +143,7 @@ class VotesPresentation(Enum):
ActiveVotes = 1
DatabaseApi = 2
CondenserApi = 3
BridgeApi = 4
@return_error_info
async def find_votes(context, params: dict, votes_presentation = VotesPresentation.DatabaseApi):
......@@ -216,30 +166,79 @@ async def find_votes(context, params: dict, votes_presentation = VotesPresentati
hive_votes_accounts_permlinks_view
WHERE
author = :author AND permlink = :permlink
ORDER BY voter_id
ORDER BY
voter_id
"""
ret = []
rows = await db.query_all(sql, author=params['author'], permlink=params['permlink'])
for row in rows:
if ( votes_presentation == VotesPresentation.DatabaseApi ):
if votes_presentation == VotesPresentation.DatabaseApi:
ret.append(dict(voter=row.voter, author=row.author, permlink=row.permlink,
weight=row.weight, rshares=row.rshares, vote_percent=row.percent,
last_update=str(row.time), num_changes=row.num_changes))
elif ( votes_presentation == VotesPresentation.CondenserApi ):
ret.append(dict(percent=str(row.percent), reputation=rep_to_raw(row.reputation), rshares=str(row.rshares), voter=row.voter))
weight=row.weight, rshares=row.rshares, vote_percent=row.percent,
last_update=str(row.time), num_changes=row.num_changes))
elif votes_presentation == VotesPresentation.CondenserApi:
ret.append(dict(percent=str(row.percent), reputation=rep_to_raw(row.reputation),
rshares=str(row.rshares), voter=row.voter))
elif votes_presentation == VotesPresentation.BridgeApi:
ret.append(dict(rshares=str(row.rshares), voter=row.voter))
else:
ret.append(dict(percent=row.percent, reputation=rep_to_raw(row.reputation),
rshares=number_to_json_value(row.rshares), time=time_string_with_t(row.time), voter=row.voter,
weight=number_to_json_value(row.weight),
rshares=number_to_json_value(row.rshares), time=time_string_with_t(row.time),
voter=row.voter, weight=number_to_json_value(row.weight)
))
return ret
@return_error_info
async def list_votes(context, start: list, limit: int, order: str):
""" Returns all votes, starting with the specified voter and/or author and permlink. """
supported_order_list = ["by_comment_voter", "by_voter_comment", "by_comment_voter", "by_voter_comment"]
supported_order_list = ["by_comment_voter", "by_voter_comment"]
assert order in supported_order_list, "Order {} is not supported".format(order)
limit = valid_limit(limit, 1000)
assert len(start) == 3, "Expecting 3 elements in start array"
db = context['db']
sql = """
SELECT
voter,
author,
permlink,
weight,
rshares,
percent,
time,
num_changes,
reputation
FROM
hive_votes_accounts_permlinks_view
"""
if order == "by_comment_voter":
sql += """
WHERE
author >= :author AND
permlink >= :permlink AND
voter >= :voter
ORDER BY
author ASC,
permlink ASC,
id ASC
LIMIT
:limit
"""
return await db.query_all(sql, author=start[0], permlink=start[1], voter=start[2], limit=limit)
if order == "by_voter_comment":
sql += """
WHERE
voter >= :voter AND
author >= :author AND
permlink >= :permlink
ORDER BY
voter ASC,
id ASC
LIMIT
:limit
"""
return await db.query_all(sql, author=start[1], permlink=start[2], voter=start[0], limit=limit)
return []
hive/server/common/objects.py → hive/server/database_api/objects.py
View file @ 9bf7b6ed
from hive.indexer.votes import Votes
from hive.server.common.helpers import json_date
from hive.utils.normalize import sbd_amount
from hive.utils.normalize import sbd_amount, to_nai
def _amount(amount, asset='HBD'):
"""Return a steem-style amount string given a (numeric, asset-str)."""
assert asset == 'HBD', 'unhandled asset %s' % asset
return "%.3f HBD" % amount
async def query_author_map(db, posts):
"""Given a list of posts, returns an author->reputation map."""
if not posts: return {}
names = tuple({post['author'] for post in posts})
sql = "SELECT id, name, reputation FROM hive_accounts WHERE name IN :names"
return {r['name']: r for r in await db.query_all(sql, names=names)}
def condenser_post_object(row, truncate_body=0):
def database_post_object(row, truncate_body=0):
"""Given a hive_posts row, create a legacy-style post object."""
paid = row['is_paidout']
# condenser#3424 mitigation
if not row['category']:
row['category'] = 'undefined'
post = {}
post['active'] = json_date(row['active'])
post['author_rewards'] = row['author_rewards']
post['post_id'] = row['id']
post['author'] = row['author']
post['permlink'] = row['permlink']
post['category'] = row['category']
post['category'] = row['category'] if 'category' in row else 'undefined'
post['title'] = row['title']
post['body'] = row['body'][0:truncate_body] if truncate_body else row['body']
......@@ -35,17 +27,16 @@ def condenser_post_object(row, truncate_body=0):
post['last_update'] = json_date(row['updated_at'])
post['depth'] = row['depth']
post['children'] = row['children']
post['children_abs_rshares'] = 0 # TODO
post['net_rshares'] = row['rshares']
post['last_payout'] = json_date(row['payout_at'] if paid else None)
post['cashout_time'] = json_date(None if paid else row['payout_at'])
post['total_payout_value'] = _amount(row['payout'] if paid else 0)
post['curator_payout_value'] = _amount(0)
post['pending_payout_value'] = _amount(0 if paid else row['payout'])
post['promoted'] = _amount(row['promoted'])
post['max_cashout_time'] = json_date(row['max_cashout_time'])
post['total_payout_value'] = to_nai(_amount(row['payout'] if paid else 0))
post['curator_payout_value'] = to_nai(_amount(0))
post['replies'] = []
post['body_length'] = len(row['body'])
post['reward_weight'] = row['reward_weight']
post['root_author'] = row['root_author']
post['root_permlink'] = row['root_permlink']
......@@ -61,15 +52,18 @@ def condenser_post_object(row, truncate_body=0):
post['parent_author'] = ''
post['parent_permlink'] = row['category']
post['url'] = row['url']
post['root_title'] = row['root_title']
post['beneficiaries'] = row['beneficiaries']
post['max_accepted_payout'] = row['max_accepted_payout']
post['max_accepted_payout'] = to_nai(row['max_accepted_payout'])
post['percent_hbd'] = row['percent_hbd']
post['abs_rshares'] = row['abs_rshares']
post['net_votes'] = Votes.get_vote_count(row['author'], row['permlink'])
if paid:
curator_payout = sbd_amount(row['curator_payout_value'])
post['curator_payout_value'] = _amount(curator_payout)
post['total_payout_value'] = _amount(row['payout'] - curator_payout)
post['curator_payout_value'] = to_nai(_amount(curator_payout))
post['total_payout_value'] = to_nai(_amount(row['payout'] - curator_payout))
post['total_vote_weight'] = Votes.get_total_vote_weight(row['author'], row['permlink'])
post['vote_rshares'] = Votes.get_total_vote_rshares(row['author'], row['permlink'])
return post
hive/server/hive_api/public.py
View file @ 9bf7b6ed
......@@ -10,6 +10,7 @@ from hive.server.condenser_api.cursor import get_followers, get_following
from hive.server.bridge_api.cursor import (
pids_by_blog, pids_by_comments, pids_by_feed_with_reblog)
from hive.db.schema import DB_VERSION as SCHEMA_DB_VERSION
log = logging.getLogger(__name__)
......@@ -108,3 +109,20 @@ async def list_account_feed(context, account, limit=10, observer=None, last_post
if rby: post['reblogged_by'] = list(rby)
return posts
async def get_info(context):
db = context['db']
sql = "SELECT num FROM hive_blocks ORDER BY num DESC LIMIT 1"
database_head_block = await db.query_one(sql)
from hive.version import VERSION, GIT_REVISION
ret = {
"hivemind_version" : VERSION,
"hivemind_git_rev" : GIT_REVISION,
"database_schema_version" : SCHEMA_DB_VERSION,
"database_head_block" : database_head_block
}
return ret
hive/server/serve.py
View file @ 9bf7b6ed
......@@ -27,6 +27,7 @@ from hive.server.bridge_api.support import get_post_header as bridge_api_get_pos
from hive.server.hive_api import community as hive_api_community
from hive.server.hive_api import notify as hive_api_notify
from hive.server.hive_api import stats as hive_api_stats
from hive.server.hive_api.public import get_info as hive_api_get_info
from hive.server.follow_api import methods as follow_api
from hive.server.tags_api import methods as tags_api
......@@ -62,6 +63,8 @@ def build_methods():
db_head_state,
)})
methods.add(**{'hive.get_info' : hive_api_get_info})
methods.add(**{'condenser_api.' + method.__name__: method for method in (
condenser_api.get_followers,
condenser_api.get_following,
......@@ -163,7 +166,7 @@ def build_methods():
# database_api methods
methods.add(**{
'database_api.list_comments' : database_api.list_comments,
'database_api.find_comments' : database_api.find_comments
'database_api.find_comments' : database_api.find_comments,
})
return methods
......@@ -220,7 +223,24 @@ def run_server(conf):
app['db'].close()
await app['db'].wait_closed()
async def show_info(app):
sql = "SELECT num FROM hive_blocks ORDER BY num DESC LIMIT 1"
database_head_block = await app['db'].query_one(sql)
import pkg_resources
hivemind_version, hivemind_git_rev = pkg_resources.get_distribution("hivemind").version.split("+")
from hive.version import VERSION, GIT_REVISION
log.info("hivemind_version : %s", VERSION)
log.info("hivemind_git_rev : %s", GIT_REVISION)
from hive.db.schema import DB_VERSION as SCHEMA_DB_VERSION
log.info("database_schema_version : %s", SCHEMA_DB_VERSION)
log.info("database_head_block : %s", database_head_block)
app.on_startup.append(init_db)
app.on_startup.append(show_info)
app.on_cleanup.append(close_db)
async def head_age(request):
......
hive/utils/normalize.py
View file @ 9bf7b6ed
......@@ -15,22 +15,57 @@ NAI_MAP = {
'@@000000037': 'VESTS',
}
dct={'0':'a','1':'b','2':'c','3':'d','4':'e',
'5':'f','6':'g','7':'h','8':'i','9':'j'}
NAI_PRECISION = {
'@@000000013': 3,
'@@000000021': 3,
'@@000000037': 6,
}
UNIT_NAI = {
'HBD' : '@@000000013',
'HIVE' : '@@000000021',
'VESTS' : '@@000000037'
}
# convert special chars into their octal formats recognized by sql
special_chars={
"\r":"\\015",
"\n":"\\012",
"\v":"\\013",
"\f": "\\014",
"\\":"\\134",
"'":"\\047",
"%":"\\045",
"_":"\\137",
":":"\\072"
SPECIAL_CHARS = {
"\r" : "\\015",
"\n" : "\\012",
"\v" : "\\013",
"\f" : "\\014",
"\\" : "\\134",
"'" : "\\047",
"%" : "\\045",
"_" : "\\137",
":" : "\\072"
}
def to_nai(value):
""" Convert various amount notation to nai notation """
ret = None
if isinstance(value, dict):
assert 'amount' in value, "amount not found in dict"
assert 'precision' in value, "precision not found in dict"
assert 'nai' in value, "nai not found in dict"
ret = value
elif isinstance(value, str):
raw_amount, unit = value.split(' ')
assert unit in UNIT_NAI, "Unknown unit {}".format(unit)
nai = UNIT_NAI[unit]
precision = NAI_PRECISION[nai]
satoshis = int(decimal.Decimal(raw_amount) * (10**precision))
ret = {'amount' : str(satoshis), 'nai' : nai, 'precision' : precision}
elif isinstance(value, list):
satoshis, precision, nai = value
assert nai in NAI_MAP, "Unknown NAI {}".format(nai)
else:
raise Exception("Invalid input amount %s" % repr(value))
return ret
def escape_characters(text):
""" Escape special charactes """
if len(text.strip()) == 0:
......@@ -39,12 +74,12 @@ def escape_characters(text):
ret = "E'"
for ch in text:
if ch.isprintable() or ch in special_chars:
if ch.isprintable() or ch in SPECIAL_CHARS:
try:
dw=special_chars[ch]
ret=ret+dw
except KeyError as k:
ret=ret+ch
dw = SPECIAL_CHARS[ch]
ret = ret + dw
except KeyError:
ret = ret + ch
else:
ordinal = ord(ch)
if ordinal == 0 or ordinal >= 0x80:
......