Skip to content
Snippets Groups Projects
Commit 23b3ea9c authored by Dariusz Kędzierski's avatar Dariusz Kędzierski Committed by Bartek Wrona
Browse files

Fixes in queries 

- fixed sql queries,
- removed matrialized view in sql query,
- fixed imports in manual tests
parent 8c232309
No related branches found
No related tags found
3 merge requests!456Release candidate v1 24,!230Setup monitoring with pghero,!135Enable postgres monitoring on CI server
Hide whitespace changes
Inline Side-by-side
Showing
with 598 additions and 252 deletions
.gitignore
+ 1
− 1
View file @ 23b3ea9c
......@@ -132,4 +132,4 @@ tests/failed_blocks/
*.out.json
# version.py
version.py
hive/version.py
hive/db/schema.py
+ 313
− 25
View file @ 23b3ea9c
......@@ -122,7 +122,7 @@ def build_metadata():
sa.Column('cashout_time', sa.DateTime, nullable=False, server_default='1970-01-01 00:00:00'),
sa.Column('max_cashout_time', sa.DateTime, nullable=False, server_default='1970-01-01 00:00:00'),
sa.Column('percent_hbd', sa.Integer, nullable=False, server_default='10000'),
sa.Column('reward_weight', sa.Integer, nullable=False, server_default='0'),
sa.Column('reward_weight', sa.Integer, nullable=False, server_default='10000'), # Seems to be always 10000
sa.Column('parent_author_id', sa.Integer, nullable=False),
sa.Column('parent_permlink_id', sa.BigInteger, nullable=False),
......@@ -154,6 +154,11 @@ def build_metadata():
sa.Index('hive_posts_sc_trend_idx', 'sc_trend'),
sa.Index('hive_posts_sc_hot_idx', 'sc_hot'),
sa.Index('hive_posts_created_at_idx', 'created_at'),
sa.Index('hive_posts_root_author_id', 'root_author_id'),
sa.Index('hive_posts_root_permlink_id', 'root_permlink_id'),
sa.Index('hive_posts_parent_author_id', 'parent_author_id'),
sa.Index('hive_posts_parent_permlink_id', 'parent_permlink_id')
)
sa.Table(
......@@ -446,7 +451,7 @@ def setup(db):
category_id,
root_author_id, root_permlink_id,
is_muted, is_valid,
author_id, permlink_id, created_at, updated_at)
author_id, permlink_id, created_at, updated_at, active)
SELECT php.id AS parent_id, php.author_id as parent_author_id,
php.permlink_id as parent_permlink_id, php.depth + 1 as depth,
(CASE
......@@ -459,7 +464,8 @@ def setup(db):
php.root_permlink_id as root_permlink_id,
php.is_muted as is_muted, php.is_valid as is_valid,
ha.id as author_id, hpd.id as permlink_id, _date as created_at,
_date as updated_at
_date as updated_at,
_date as active
FROM hive_accounts ha,
hive_permlink_data hpd,
hive_posts php
......@@ -473,6 +479,7 @@ def setup(db):
--- then also depth, is_valid and is_muted is impossible to change
--- post edit part
updated_at = _date,
active = _date,
--- post undelete part (if was deleted)
is_deleted = (CASE hp.is_deleted
......@@ -500,7 +507,7 @@ def setup(db):
category_id,
root_author_id, root_permlink_id,
is_muted, is_valid,
author_id, permlink_id, created_at, updated_at)
author_id, permlink_id, created_at, updated_at, active)
SELECT 0 AS parent_id, 0 as parent_author_id, 0 as parent_permlink_id, 0 as depth,
(CASE
WHEN _date > _community_support_start_date THEN
......@@ -512,7 +519,8 @@ def setup(db):
hpd.id as root_permlink_id, -- use perlink_id as root one if no parent
false as is_muted, true as is_valid,
ha.id as author_id, hpd.id as permlink_id, _date as created_at,
_date as updated_at
_date as updated_at,
_date as active
FROM hive_accounts ha,
hive_permlink_data hpd
WHERE ha.name = _author and hpd.permlink = _permlink
......@@ -522,6 +530,7 @@ def setup(db):
--- then also depth, is_valid and is_muted is impossible to change
--- post edit part
updated_at = _date,
active = _date,
--- post undelete part (if was deleted)
is_deleted = (CASE hp.is_deleted
......@@ -566,25 +575,6 @@ def setup(db):
"""
db.query_no_return(sql)
sql = """
DROP MATERIALIZED VIEW IF EXISTS hive_posts_a_p
;
CREATE MATERIALIZED VIEW hive_posts_a_p
AS
SELECT hp.id AS id,
ha_a.name AS author,
hpd_p.permlink AS permlink
FROM hive_posts hp
INNER JOIN hive_accounts ha_a ON ha_a.id = hp.author_id
INNER JOIN hive_permlink_data hpd_p ON hpd_p.id = hp.permlink_id
WITH DATA
;
DROP INDEX IF EXISTS hive_posts_a_p_idx
;
CREATE unique index hive_posts_a_p_idx ON hive_posts_a_p (author collate "C", permlink collate "C")
"""
db.query_no_return(sql)
sql = """
DROP VIEW IF EXISTS public.hive_posts_view;
......@@ -594,6 +584,8 @@ def setup(db):
hp.community_id,
hp.parent_id,
ha_a.name AS author,
hp.active,
hp.author_rewards,
hp.author_id,
hpd_p.permlink,
hpd.title,
......@@ -653,7 +645,11 @@ def setup(db):
hr.title AS role_title,
hr.role_id AS role_id,
hc.title AS community_title,
hc.name AS community_name
hc.name AS community_name,
hp.abs_rshares,
hp.cashout_time,
hp.max_cashout_time,
hp.reward_weight
FROM hive_posts hp
JOIN hive_posts rp ON rp.author_id = hp.root_author_id AND rp.permlink_id = hp.root_permlink_id
JOIN hive_post_data rpd ON rp.id = rpd.id
......@@ -736,6 +732,298 @@ def setup(db):
"""
db.query_no_return(sql)
sql = """
DROP TYPE IF EXISTS database_api_post CASCADE;
CREATE TYPE database_api_post AS (
id INT,
community_id INT,
author VARCHAR(16),
permlink VARCHAR(255),
title VARCHAR(512),
body TEXT,
category VARCHAR(255),
depth SMALLINT,
promoted DECIMAL(10,3),
payout DECIMAL(10,3),
payout_at TIMESTAMP,
is_paidout BOOLEAN,
children SMALLINT,
votes INT,
created_at TIMESTAMP,
updated_at TIMESTAMP,
rshares NUMERIC,
json TEXT,
is_hidden BOOLEAN,
is_grayed BOOLEAN,
total_votes INT,
flag_weight REAL,
parent_author VARCHAR(16),
parent_permlink VARCHAR(255),
curator_payout_value VARCHAR(30),
root_author VARCHAR(16),
root_permlink VARCHAR(255),
max_accepted_payout VARCHAR(30),
percent_hbd INT,
allow_replies BOOLEAN,
allow_votes BOOLEAN,
allow_curation_rewards BOOLEAN,
beneficiaries JSON,
url TEXT,
root_title VARCHAR(512),
abs_rshares BIGINT,
active TIMESTAMP,
author_rewards BIGINT,
max_cashout_time TIMESTAMP,
reward_weight INT
)
;
DROP FUNCTION IF EXISTS list_comments_by_cashout_time(timestamp, character varying, character varying, int)
;
CREATE OR REPLACE FUNCTION list_comments_by_cashout_time(
in _cashout_time timestamp,
in _author hive_accounts.name%TYPE,
in _permlink hive_permlink_data.permlink%TYPE,
in _limit INT)
RETURNS SETOF database_api_post
AS
$function$
BEGIN
RETURN QUERY
SELECT
hp.id, hp.community_id, hp.author, hp.permlink, hp.title, hp.body,
hp.category, hp.depth, hp.promoted, hp.payout, hp.payout_at, hp.is_paidout,
hp.children, hp.votes, hp.created_at, hp.updated_at, hp.rshares, hp.json,
hp.is_hidden, hp.is_grayed, hp.total_votes, hp.flag_weight, hp.parent_author,
hp.parent_permlink, hp.curator_payout_value, hp.root_author, hp.root_permlink,
hp.max_accepted_payout, hp.percent_hbd, hp.allow_replies, hp.allow_votes,
hp.allow_curation_rewards, hp.beneficiaries, hp.url, hp.root_title, hp.abs_rshares,
hp.active, hp.author_rewards, hp.max_cashout_time, hp.reward_weight
FROM
hive_posts_view hp
WHERE
NOT hp.is_muted AND
NOT hp.is_deleted AND
hp.cashout_time >= _cashout_time AND
hp.id >= (SELECT id FROM hive_posts_view hp1 WHERE hp1.author >= _author AND hp1.permlink >= _permlink ORDER BY id LIMIT 1)
ORDER BY
hp.cashout_time ASC,
hp.id ASC
LIMIT
_limit
;
END
$function$
LANGUAGE plpgsql
;
DROP FUNCTION IF EXISTS list_comments_by_permlink(character varying, character varying, int)
;
CREATE OR REPLACE FUNCTION list_comments_by_permlink(
in _author hive_accounts.name%TYPE,
in _permlink hive_permlink_data.permlink%TYPE,
in _limit INT)
RETURNS SETOF database_api_post
AS
$function$
BEGIN
RETURN QUERY
SELECT
hp.id, hp.community_id, hp.author, hp.permlink, hp.title, hp.body,
hp.category, hp.depth, hp.promoted, hp.payout, hp.payout_at, hp.is_paidout,
hp.children, hp.votes, hp.created_at, hp.updated_at, hp.rshares, hp.json,
hp.is_hidden, hp.is_grayed, hp.total_votes, hp.flag_weight, hp.parent_author,
hp.parent_permlink, hp.curator_payout_value, hp.root_author, hp.root_permlink,
hp.max_accepted_payout, hp.percent_hbd, hp.allow_replies, hp.allow_votes,
hp.allow_curation_rewards, hp.beneficiaries, hp.url, hp.root_title, hp.abs_rshares,
hp.active, hp.author_rewards, hp.max_cashout_time, hp.reward_weight
FROM
hive_posts_view hp
WHERE
NOT hp.is_muted AND
NOT hp.is_deleted AND
hp.author >= _author COLLATE "C" AND
hp.permlink >= _permlink COLLATE "C"
ORDER BY
hp.author COLLATE "C" ASC,
hp.permlink COLLATE "C" ASC
LIMIT
_limit
;
END
$function$
LANGUAGE plpgsql
;
DROP FUNCTION IF EXISTS list_comments_by_root(character varying, character varying, character varying, character varying, int)
;
CREATE OR REPLACE FUNCTION list_comments_by_root(
in _root_author hive_accounts.name%TYPE,
in _root_permlink hive_permlink_data.permlink%TYPE,
in _start_post_author hive_accounts.name%TYPE,
in _start_post_permlink hive_permlink_data.permlink%TYPE,
in _limit INT)
RETURNS SETOF database_api_post
AS
$function$
BEGIN
RETURN QUERY
SELECT
hp.id, hp.community_id, hp.author, hp.permlink, hp.title, hp.body,
hp.category, hp.depth, hp.promoted, hp.payout, hp.payout_at, hp.is_paidout,
hp.children, hp.votes, hp.created_at, hp.updated_at, hp.rshares, hp.json,
hp.is_hidden, hp.is_grayed, hp.total_votes, hp.flag_weight, hp.parent_author,
hp.parent_permlink, hp.curator_payout_value, hp.root_author, hp.root_permlink,
hp.max_accepted_payout, hp.percent_hbd, hp.allow_replies, hp.allow_votes,
hp.allow_curation_rewards, hp.beneficiaries, hp.url, hp.root_title, hp.abs_rshares,
hp.active, hp.author_rewards, hp.max_cashout_time, hp.reward_weight
FROM
hive_posts_view hp
WHERE
NOT hp.is_muted AND
NOT hp.is_deleted AND
root_author >= _root_author AND
root_permlink >= _root_permlink AND
hp.id >= (SELECT id FROM hive_posts_view hp1 WHERE hp1.author >= _start_post_author AND hp1.permlink >= _start_post_permlink ORDER BY id LIMIT 1)
ORDER BY
root_author ASC,
root_permlink ASC,
id ASC
LIMIT
_limit
;
END
$function$
LANGUAGE plpgsql
;
DROP FUNCTION IF EXISTS list_comments_by_parent(character varying, character varying, character varying, character varying, int)
;
CREATE OR REPLACE FUNCTION list_comments_by_parent(
in _parent_author hive_accounts.name%TYPE,
in _parent_permlink hive_permlink_data.permlink%TYPE,
in _start_post_author hive_accounts.name%TYPE,
in _start_post_permlink hive_permlink_data.permlink%TYPE,
in _limit INT)
RETURNS SETOF database_api_post
AS
$function$
BEGIN
RETURN QUERY
SELECT
hp.id, hp.community_id, hp.author, hp.permlink, hp.title, hp.body,
hp.category, hp.depth, hp.promoted, hp.payout, hp.payout_at, hp.is_paidout,
hp.children, hp.votes, hp.created_at, hp.updated_at, hp.rshares, hp.json,
hp.is_hidden, hp.is_grayed, hp.total_votes, hp.flag_weight, hp.parent_author,
hp.parent_permlink, hp.curator_payout_value, hp.root_author, hp.root_permlink,
hp.max_accepted_payout, hp.percent_hbd, hp.allow_replies, hp.allow_votes,
hp.allow_curation_rewards, hp.beneficiaries, hp.url, hp.root_title, hp.abs_rshares,
hp.active, hp.author_rewards, hp.max_cashout_time, hp.reward_weight
FROM
hive_posts_view hp
WHERE
NOT hp.is_muted AND
NOT hp.is_deleted AND
parent_author >= _parent_author AND
parent_permlink >= _parent_permlink AND
hp.id >= (SELECT id FROM hive_posts_view hp1 WHERE hp1.author >= _start_post_author AND hp1.permlink >= _start_post_permlink ORDER BY id LIMIT 1)
ORDER BY
parent_author ASC,
parent_permlink ASC,
id ASC
LIMIT
_limit
;
END
$function$
LANGUAGE plpgsql
;
DROP FUNCTION IF EXISTS list_comments_by_last_update(character varying, timestamp, character varying, character varying, int)
;
CREATE OR REPLACE FUNCTION list_comments_by_last_update(
in _parent_author hive_accounts.name%TYPE,
in _updated_at hive_posts.updated_at%TYPE,
in _start_post_author hive_accounts.name%TYPE,
in _start_post_permlink hive_permlink_data.permlink%TYPE,
in _limit INT)
RETURNS SETOF database_api_post
AS
$function$
BEGIN
RETURN QUERY
SELECT
hp.id, hp.community_id, hp.author, hp.permlink, hp.title, hp.body,
hp.category, hp.depth, hp.promoted, hp.payout, hp.payout_at, hp.is_paidout,
hp.children, hp.votes, hp.created_at, hp.updated_at, hp.rshares, hp.json,
hp.is_hidden, hp.is_grayed, hp.total_votes, hp.flag_weight, hp.parent_author,
hp.parent_permlink, hp.curator_payout_value, hp.root_author, hp.root_permlink,
hp.max_accepted_payout, hp.percent_hbd, hp.allow_replies, hp.allow_votes,
hp.allow_curation_rewards, hp.beneficiaries, hp.url, hp.root_title, hp.abs_rshares,
hp.active, hp.author_rewards, hp.max_cashout_time, hp.reward_weight
FROM
hive_posts_view hp
WHERE
NOT hp.is_muted AND
NOT hp.is_deleted AND
hp.parent_author >= _parent_author AND
hp.updated_at >= _updated_at AND
hp.id >= (SELECT id FROM hive_posts_view hp1 WHERE hp1.author >= _start_post_author AND hp1.permlink >= _start_post_permlink ORDER BY id LIMIT 1)
ORDER BY
hp.parent_author ASC,
hp.updated_at ASC,
hp.id ASC
LIMIT
_limit
;
END
$function$
LANGUAGE plpgsql
;
DROP FUNCTION IF EXISTS list_comments_by_author_last_update(character varying, timestamp, character varying, character varying, int)
;
CREATE OR REPLACE FUNCTION list_comments_by_author_last_update(
in _author hive_accounts.name%TYPE,
in _updated_at hive_posts.updated_at%TYPE,
in _start_post_author hive_accounts.name%TYPE,
in _start_post_permlink hive_permlink_data.permlink%TYPE,
in _limit INT)
RETURNS SETOF database_api_post
AS
$function$
BEGIN
RETURN QUERY
SELECT
hp.id, hp.community_id, hp.author, hp.permlink, hp.title, hp.body,
hp.category, hp.depth, hp.promoted, hp.payout, hp.payout_at, hp.is_paidout,
hp.children, hp.votes, hp.created_at, hp.updated_at, hp.rshares, hp.json,
hp.is_hidden, hp.is_grayed, hp.total_votes, hp.flag_weight, hp.parent_author,
hp.parent_permlink, hp.curator_payout_value, hp.root_author, hp.root_permlink,
hp.max_accepted_payout, hp.percent_hbd, hp.allow_replies, hp.allow_votes,
hp.allow_curation_rewards, hp.beneficiaries, hp.url, hp.root_title, hp.abs_rshares,
hp.active, hp.author_rewards, hp.max_cashout_time, hp.reward_weight
FROM
hive_posts_view hp
WHERE
NOT hp.is_muted AND
NOT hp.is_deleted AND
hp.author >= _author AND
hp.updated_at >= _updated_at AND
hp.id >= (SELECT id FROM hive_posts_view hp1 WHERE hp1.author >= _start_post_author AND hp1.permlink >= _start_post_permlink ORDER BY id LIMIT 1)
ORDER BY
hp.parent_author ASC,
hp.updated_at ASC,
hp.id ASC
LIMIT
_limit
;
END
$function$
LANGUAGE plpgsql
;
"""
db.query_no_return(sql)
def reset_autovac(db):
"""Initializes/resets per-table autovacuum/autoanalyze params.
......
hive/indexer/blocks.py
+ 5
− 4
View file @ 23b3ea9c
......@@ -171,7 +171,6 @@ class Blocks:
cls._head_block_date = cls._current_block_date
json_ops = []
update_comment_pending_payouts = []
for tx_idx, tx in enumerate(block['transactions']):
for operation in tx['operations']:
op_type = operation['type']
......@@ -243,8 +242,8 @@ class Blocks:
(vote_ops, comment_payout_stats) = Blocks.prepare_vops(Posts.comment_payout_ops, vops, cls._current_block_date)
if vote_ops is not None:
for k, v in vote_ops.items():
Votes.effective_comment_vote_op(k, v)
for k, v in vote_ops.items():
Votes.effective_comment_vote_op(k, v)
if Posts.comment_payout_ops:
cls.ops_stats = Blocks.merge_ops_stats(cls.ops_stats, comment_payout_stats)
......@@ -318,7 +317,9 @@ class Blocks:
"""
values = []
for block in cls.blocks_to_flush:
values.append("({}, '{}', '{}', {}, {}, '{}')".format(block['num'], block['hash'], block['prev'], block['txs'], block['ops'], block['date']))
values.append("({}, '{}', '{}', {}, {}, '{}')".format(block['num'], block['hash'],
block['prev'], block['txs'],
block['ops'], block['date']))
DB.query(query + ",".join(values))
cls.blocks_to_flush = []
......
hive/indexer/votes.py
+ 41
− 11
View file @ 23b3ea9c
......@@ -46,6 +46,36 @@ class Votes:
ret = DB.query_row(sql, author=author, permlink=permlink)
return 0 if ret is None else int(ret.count)
@classmethod
def get_total_vote_weight(cls, author, permlink):
""" Get total vote weight for selected post """
sql = """
SELECT
sum(weight)
FROM
hive_votes_accounts_permlinks_view hv
WHERE
hv.author = :author AND
hv.permlink = :permlink
"""
ret = DB.query_row(sql, author=author, permlink=permlink)
return 0 if ret is None else int(0 if ret.sum is None else ret.sum)
@classmethod
def get_total_vote_rshares(cls, author, permlink):
""" Get total vote rshares for selected post """
sql = """
SELECT
sum(rshares)
FROM
hive_votes_accounts_permlinks_view hv
WHERE
hv.author = :author AND
hv.permlink = :permlink
"""
ret = DB.query_row(sql, author=author, permlink=permlink)
return 0 if ret is None else int(0 if ret.sum is None else ret.sum)
inside_flush = False
@classmethod
......@@ -56,15 +86,15 @@ class Votes:
permlink = vote_operation['permlink']
weight = vote_operation['weight']
if(cls.inside_flush):
log.info("Adding new vote-info into '_votes_data' dict")
raise "Fatal error"
if cls.inside_flush:
log.exception("Adding new vote-info into '_votes_data' dict")
raise RuntimeError("Fatal error")
key = voter + "/" + author + "/" + permlink
if key in cls._votes_data:
cls._votes_data[key]["vote_percent"]=weight
cls._votes_data[key]["last_update"]=date
cls._votes_data[key]["vote_percent"] = weight
cls._votes_data[key]["last_update"] = date
else:
cls._votes_data[key] = dict(voter=voter,
author=author,
......@@ -79,9 +109,9 @@ class Votes:
def effective_comment_vote_op(cls, key, vop):
""" Process effective_comment_vote_operation """
if(cls.inside_flush):
log.info("Updating data in '_votes_data' using effective comment")
raise "Fatal error"
if cls.inside_flush:
log.exception("Updating data in '_votes_data' using effective comment")
raise RuntimeError("Fatal error")
assert key in cls._votes_data
......@@ -140,18 +170,18 @@ class Votes:
if len(values) >= values_limit:
values_str = ','.join(values)
actual_query = sql.format(values_str,on_conflict_data_source,on_conflict_data_source)
actual_query = sql.format(values_str, on_conflict_data_source, on_conflict_data_source)
DB.query(actual_query)
values.clear()
if len(values_skip) > 0:
values_str = ','.join(values_skip)
actual_query = sql.format(values_str,'hive_votes','hive_votes')
actual_query = sql.format(values_str, 'hive_votes', 'hive_votes')
DB.query(actual_query)
values_skip.clear()
if len(values_override) > 0:
values_str = ','.join(values_override)
actual_query = sql.format(values_str,'EXCLUDED','EXCLUDED')
actual_query = sql.format(values_str, 'EXCLUDED', 'EXCLUDED')
DB.query(actual_query)
values_override.clear()
......
hive/server/database_api/methods.py
+ 148
− 152
View file @ 23b3ea9c
......@@ -2,167 +2,66 @@
from enum import Enum
from hive.server.common.helpers import return_error_info, valid_limit, valid_account, valid_permlink
from hive.server.common.objects import condenser_post_object
from hive.server.database_api.objects import database_post_object
from hive.utils.normalize import rep_to_raw, number_to_json_value, time_string_with_t
SQL_TEMPLATE = """
SELECT
hp.id,
hp.community_id,
hp.author,
hp.permlink,
hp.title,
hp.body,
hp.category,
hp.depth,
hp.promoted,
hp.payout,
hp.payout_at,
hp.is_paidout,
hp.children,
hp.votes,
hp.created_at,
hp.updated_at,
hp.rshares,
hp.json,
hp.is_hidden,
hp.is_grayed,
hp.total_votes,
hp.flag_weight,
hp.parent_author,
hp.parent_permlink,
hp.curator_payout_value,
hp.root_author,
hp.root_permlink,
hp.max_accepted_payout,
hp.percent_hbd,
hp.allow_replies,
hp.allow_votes,
hp.allow_curation_rewards,
hp.beneficiaries,
hp.url,
hp.root_title
FROM hive_posts_view hp
WHERE
"""
async def get_post_id_by_author_and_permlink(db, author: str, permlink: str, limit: int):
"""Return post ids for given author and permlink"""
sql = """
SELECT hp.id
FROM hive_posts hp
INNER JOIN hive_accounts ha_a ON ha_a.id = hp.author_id
INNER JOIN hive_permlink_data hpd_p ON hpd_p.id = hp.permlink_id
WHERE ha_a.name >= :author AND hpd_p.permlink >= :permlink
ORDER BY ha_a.name ASC
LIMIT :limit
"""
result = await db.query_row(sql, author=author, permlink=permlink, limit=limit)
if result is not None:
return int(result.get('id', 0))
return 0
@return_error_info
async def list_comments(context, start: list, limit: int, order: str):
"""Returns all comments, starting with the specified options."""
supported_order_list = ['by_cashout_time', 'by_permlink', 'by_root', 'by_parent', 'by_update', 'by_author_last_update']
supported_order_list = ['by_cashout_time', 'by_permlink', 'by_root', 'by_parent', 'by_last_update', 'by_author_last_update']
assert order in supported_order_list, "Unsupported order, valid orders: {}".format(", ".join(supported_order_list))
limit = valid_limit(limit, 1000)
db = context['db']
comments = []
result = []
if order == 'by_cashout_time':
assert len(start) == 3, "Expecting three arguments"
cashout_time = start[0]
author = start[1]
permlink = start[2]
post_id = 0
if author or permlink:
post_id = await get_post_id_by_author_and_permlink(db, author, permlink, 1)
sql = str(SQL_TEMPLATE)
sql += "hp.payout_at >= :start AND hp.id >= :post_id ORDER BY hp.payout_at ASC, hp.id ASC LIMIT :limit"
result = await db.query_all(sql, start=start[0], limit=limit, post_id=post_id)
for row in result:
cpo = condenser_post_object(dict(row))
cpo['active_votes'] = await find_votes(context, {'author':cpo['author'], 'permlink':cpo['permlink']})
comments.append(cpo)
sql = "SELECT * FROM list_comments_by_cashout_time(:cashout_time, :author, :permlink, :limit)"
result = await db.query_all(sql, cashout_time=cashout_time, author=author, permlink=permlink, limit=limit)
elif order == 'by_permlink':
assert len(start) == 2, "Expecting two arguments"
sql = str(SQL_TEMPLATE)
sql += """ hp.id IN (SELECT hp1.id FROM hive_posts_a_p hp1 WHERE hp1.author >= :author COLLATE "C"
AND hp1.permlink >= :permlink COLLATE "C" ORDER BY hp1.author COLLATE "C" ASC LIMIT :limit)"""
result = await db.query_all(sql, author=start[0], permlink=start[1], limit=limit)
for row in result:
cpo = condenser_post_object(dict(row))
cpo['active_votes'] = await find_votes(context, {'author':cpo['author'], 'permlink':cpo['permlink']})
comments.append(cpo)
author = start[0]
permlink = start[1]
sql = "SELECT * FROM list_comments_by_permlink(:author, :permlink, :limit)"
result = await db.query_all(sql, author=author, permlink=permlink, limit=limit)
elif order == 'by_root':
assert len(start) == 4, "Expecting 4 arguments"
raise NotImplementedError('by_root')
sql = str(SQL_TEMPLATE)
sql += "get_rows_by_root(:root_author, :root_permlink, :child_author, :child_permlink) ORDER BY post_id ASC LIMIT :limit"
result = await db.query_all(sql, root_author=start[0], root_permlink=start[1], child_author=start[2], child_permlink=start[3], limit=limit)
for row in result:
cpo = condenser_post_object(dict(row))
cpo['active_votes'] = await find_votes(context, {'author':cpo['author'], 'permlink':cpo['permlink']})
comments.append(cpo)
root_author = start[0]
root_permlink = start[1]
start_post_author = start[2]
start_post_permlink = start[3]
sql = "SELECT * FROM list_comments_by_root(:root_author, :root_permlink, :start_post_author, :start_post_permlink, :limit)"
result = await db.query_all(sql, root_author=root_author, root_permlink=root_permlink, start_post_author=start_post_author, start_post_permlink=start_post_permlink, limit=limit)
elif order == 'by_parent':
assert len(start) == 4, "Expecting 4 arguments"
raise NotImplementedError('by_parent')
sql = str(SQL_TEMPLATE)
sql += "get_rows_by_parent(:parent_author, :parent_permlink, :child_author, :child_permlink) LIMIT :limit"
result = await db.query_all(sql, parent_author=start[0], parent_permlink=start[1], child_author=start[2], child_permlink=start[3], limit=limit)
for row in result:
cpo = condenser_post_object(dict(row))
cpo['active_votes'] = await find_votes(context, {'author':cpo['author'], 'permlink':cpo['permlink']})
comments.append(cpo)
elif order == 'by_update':
parent_author = start[0]
parent_permlink = start[1]
start_post_author = start[2]
start_post_permlink = start[3]
sql = "SELECT * FROM list_comments_by_parent(:parent_author, :parent_permlink, :start_post_author, :start_post_permlink, :limit)"
result = await db.query_all(sql, parent_author=parent_author, parent_permlink=parent_permlink, start_post_author=start_post_author, start_post_permlink=start_post_permlink, limit=limit)
elif order == 'by_last_update':
assert len(start) == 4, "Expecting 4 arguments"
child_author = start[2]
child_permlink = start[3]
post_id = 0
if author or permlink:
post_id = await get_post_id_by_author_and_permlink(db, child_author, child_permlink, 1)
sql = str(SQL_TEMPLATE)
sql += "hp.parent_author >= :parent_author AND hp.updated_at >= :updated_at AND hp.id >= :post_id ORDER BY hp.parent_author ASC, hp.updated_at ASC, hp.id ASC LIMIT :limit"
result = await db.query_all(sql, parent_author=start[0], updated_at=start[1], post_id=post_id, limit=limit)
for row in result:
cpo = condenser_post_object(dict(row))
cpo['active_votes'] = await find_votes(context, {'author':cpo['author'], 'permlink':cpo['permlink']})
comments.append(cpo)
parent_author = start[0]
updated_at = start[1]
start_post_author = start[2]
start_post_permlink = start[3]
sql = "SELECT * FROM list_comments_by_last_update(:parent_author, :updated_at, :start_post_author, :start_post_permlink, :limit)"
result = await db.query_all(sql, parent_author=parent_author, updated_at=updated_at, start_post_author=start_post_author, start_post_permlink=start_post_permlink, limit=limit)
elif order == 'by_author_last_update':
assert len(start) == 4, "Expecting 4 arguments"
author = start[0]
updated_at = start[1]
start_post_author = start[2]
start_post_permlink = start[3]
sql = "SELECT * FROM list_comments_by_author_last_update(:author, :updated_at, :start_post_author, :start_post_permlink, :limit)"
result = await db.query_all(sql, author=author, updated_at=updated_at, start_post_author=start_post_author, start_post_permlink=start_post_permlink, limit=limit)
author = start[2]
permlink = start[3]
post_id = 0
if author or permlink:
post_id = await get_post_id_by_author_and_permlink(db, author, permlink, 1)
sql = str(SQL_TEMPLATE)
sql += "hp.author >= :author AND hp.updated_at >= :updated_at AND hp.id >= :post_id ORDER BY hp.author ASC, hp.updated_at ASC, hp.id ASC LIMIT :limit"
result = await db.query_all(sql, author=start[0], updated_at=start[1], post_id=post_id, limit=limit)
for row in result:
cpo = condenser_post_object(dict(row))
cpo['active_votes'] = await find_votes(context, {'author':cpo['author'], 'permlink':cpo['permlink']})
comments.append(cpo)
return comments
return [database_post_object(dict(row)) for row in result]
@return_error_info
async def find_comments(context, start: list, limit: int, order: str):
......@@ -172,20 +71,70 @@ async def find_comments(context, start: list, limit: int, order: str):
assert len(start) <= 1000, "Parameters count is greather than max allowed (1000)"
db = context['db']
# make a copy
sql = str(SQL_TEMPLATE)
SQL_TEMPLATE = """
SELECT
hp.id,
hp.community_id,
hp.author,
hp.permlink,
hp.title,
hp.body,
hp.category,
hp.depth,
hp.promoted,
hp.payout,
hp.payout_at,
hp.is_paidout,
hp.children,
hp.votes,
hp.created_at,
hp.updated_at,
hp.rshares,
hp.json,
hp.is_hidden,
hp.is_grayed,
hp.total_votes,
hp.flag_weight,
hp.parent_author,
hp.parent_permlink,
hp.curator_payout_value,
hp.root_author,
hp.root_permlink,
hp.max_accepted_payout,
hp.percent_hbd,
hp.allow_replies,
hp.allow_votes,
hp.allow_curation_rewards,
hp.beneficiaries,
hp.url,
hp.root_title,
hp.abs_rshares,
hp.active,
hp.author_rewards,
hp.max_cashout_time,
hp.reward_weight
FROM
hive_posts_view hp
WHERE
NOT hp.is_muted AND
NOT hp.is_deleted AND
"""
idx = 0
sql = ""
for arg in start:
if idx > 0:
sql += " OR "
sql += "(hp.author = '{}' AND hp.permlink = '{}')".format(arg[0], arg[1])
sql += " UNION ALL "
sql += str(SQL_TEMPLATE)
sql += """
hp.author = '{}' AND
hp.permlink = '{}'
""".format(arg[0], arg[1])
idx += 1
result = await db.query_all(sql)
for row in result:
cpo = condenser_post_object(dict(row))
cpo['active_votes'] = await find_votes(context, {'author':cpo['author'], 'permlink':cpo['permlink']})
cpo = database_post_object(dict(row))
comments.append(cpo)
return comments
......@@ -216,30 +165,77 @@ async def find_votes(context, params: dict, votes_presentation = VotesPresentati
hive_votes_accounts_permlinks_view
WHERE
author = :author AND permlink = :permlink
ORDER BY voter_id
ORDER BY
voter_id
"""
ret = []
rows = await db.query_all(sql, author=params['author'], permlink=params['permlink'])
for row in rows:
if ( votes_presentation == VotesPresentation.DatabaseApi ):
if votes_presentation == VotesPresentation.DatabaseApi:
ret.append(dict(voter=row.voter, author=row.author, permlink=row.permlink,
weight=row.weight, rshares=row.rshares, vote_percent=row.percent,
last_update=str(row.time), num_changes=row.num_changes))
elif ( votes_presentation == VotesPresentation.CondenserApi ):
ret.append(dict(percent=str(row.percent), reputation=rep_to_raw(row.reputation), rshares=str(row.rshares), voter=row.voter))
weight=row.weight, rshares=row.rshares, vote_percent=row.percent,
last_update=str(row.time), num_changes=row.num_changes))
elif votes_presentation == VotesPresentation.CondenserApi:
ret.append(dict(percent=str(row.percent), reputation=rep_to_raw(row.reputation),
rshares=str(row.rshares), voter=row.voter))
else:
ret.append(dict(percent=row.percent, reputation=rep_to_raw(row.reputation),
rshares=number_to_json_value(row.rshares), time=time_string_with_t(row.time), voter=row.voter,
weight=number_to_json_value(row.weight),
rshares=number_to_json_value(row.rshares), time=time_string_with_t(row.time),
voter=row.voter, weight=number_to_json_value(row.weight)
))
return ret
@return_error_info
async def list_votes(context, start: list, limit: int, order: str):
""" Returns all votes, starting with the specified voter and/or author and permlink. """
supported_order_list = ["by_comment_voter", "by_voter_comment", "by_comment_voter", "by_voter_comment"]
supported_order_list = ["by_comment_voter", "by_voter_comment"]
assert order in supported_order_list, "Order {} is not supported".format(order)
limit = valid_limit(limit, 1000)
assert len(start) == 3, "Expecting 3 elements in start array"
db = context['db']
sql = """
SELECT
voter,
author,
permlink,
weight,
rshares,
percent,
time,
num_changes,
reputation
FROM
hive_votes_accounts_permlinks_view
"""
if order == "by_comment_voter":
sql += """
WHERE
author >= :author AND
permlink >= :permlink AND
voter >= :voter
ORDER BY
author ASC,
permlink ASC,
id ASC
LIMIT
:limit
"""
return await db.query_all(sql, author=start[0], permlink=start[1], voter=start[2], limit=limit)
if order == "by_voter_comment":
sql += """
WHERE
voter >= :voter AND
author >= :author AND
permlink >= :permlink
ORDER BY
voter ASC,
id ASC
LIMIT
:limit
"""
return await db.query_all(sql, author=start[1], permlink=start[2], voter=start[0], limit=limit)
return []
hive/server/common/objects.py → hive/server/database_api/objects.py
+ 19
− 25
View file @ 23b3ea9c
from hive.indexer.votes import Votes
from hive.server.common.helpers import json_date
from hive.utils.normalize import sbd_amount
from hive.utils.normalize import sbd_amount, to_nai
def _amount(amount, asset='HBD'):
"""Return a steem-style amount string given a (numeric, asset-str)."""
assert asset == 'HBD', 'unhandled asset %s' % asset
return "%.3f HBD" % amount
async def query_author_map(db, posts):
"""Given a list of posts, returns an author->reputation map."""
if not posts: return {}
names = tuple({post['author'] for post in posts})
sql = "SELECT id, name, reputation FROM hive_accounts WHERE name IN :names"
return {r['name']: r for r in await db.query_all(sql, names=names)}
def condenser_post_object(row, truncate_body=0):
def database_post_object(row, truncate_body=0):
"""Given a hive_posts row, create a legacy-style post object."""
paid = row['is_paidout']
# condenser#3424 mitigation
if not row['category']:
row['category'] = 'undefined'
post = {}
post['active'] = json_date(row['active'])
post['author_rewards'] = row['author_rewards']
post['post_id'] = row['id']
post['author'] = row['author']
post['permlink'] = row['permlink']
post['category'] = row['category']
post['category'] = row['category'] if 'category' in row else 'undefined'
post['title'] = row['title']
post['body'] = row['body'][0:truncate_body] if truncate_body else row['body']
......@@ -35,17 +27,16 @@ def condenser_post_object(row, truncate_body=0):
post['last_update'] = json_date(row['updated_at'])
post['depth'] = row['depth']
post['children'] = row['children']
post['children_abs_rshares'] = 0 # TODO
post['net_rshares'] = row['rshares']
post['last_payout'] = json_date(row['payout_at'] if paid else None)
post['cashout_time'] = json_date(None if paid else row['payout_at'])
post['total_payout_value'] = _amount(row['payout'] if paid else 0)
post['curator_payout_value'] = _amount(0)
post['pending_payout_value'] = _amount(0 if paid else row['payout'])
post['promoted'] = _amount(row['promoted'])
post['max_cashout_time'] = json_date(row['max_cashout_time'])
post['total_payout_value'] = to_nai(_amount(row['payout'] if paid else 0))
post['curator_payout_value'] = to_nai(_amount(0))
post['replies'] = []
post['body_length'] = len(row['body'])
post['reward_weight'] = row['reward_weight']
post['root_author'] = row['root_author']
post['root_permlink'] = row['root_permlink']
......@@ -61,15 +52,18 @@ def condenser_post_object(row, truncate_body=0):
post['parent_author'] = ''
post['parent_permlink'] = row['category']
post['url'] = row['url']
post['root_title'] = row['root_title']
post['beneficiaries'] = row['beneficiaries']
post['max_accepted_payout'] = row['max_accepted_payout']
post['max_accepted_payout'] = to_nai(row['max_accepted_payout'])
post['percent_hbd'] = row['percent_hbd']
post['abs_rshares'] = row['abs_rshares']
post['net_votes'] = Votes.get_vote_count(row['author'], row['permlink'])
if paid:
curator_payout = sbd_amount(row['curator_payout_value'])
post['curator_payout_value'] = _amount(curator_payout)
post['total_payout_value'] = _amount(row['payout'] - curator_payout)
post['curator_payout_value'] = to_nai(_amount(curator_payout))
post['total_payout_value'] = to_nai(_amount(row['payout'] - curator_payout))
post['total_vote_weight'] = Votes.get_total_vote_weight(row['author'], row['permlink'])
post['vote_rshares'] = Votes.get_total_vote_rshares(row['author'], row['permlink'])
return post
hive/server/serve.py
+ 1
− 1
View file @ 23b3ea9c
......@@ -166,7 +166,7 @@ def build_methods():
# database_api methods
methods.add(**{
'database_api.list_comments' : database_api.list_comments,
'database_api.find_comments' : database_api.find_comments
'database_api.find_comments' : database_api.find_comments,
})
return methods
......
hive/utils/normalize.py
+ 52
− 17
View file @ 23b3ea9c
......@@ -15,22 +15,57 @@ NAI_MAP = {
'@@000000037': 'VESTS',
}
dct={'0':'a','1':'b','2':'c','3':'d','4':'e',
'5':'f','6':'g','7':'h','8':'i','9':'j'}
NAI_PRECISION = {
'@@000000013': 3,
'@@000000021': 3,
'@@000000037': 6,
}
UNIT_NAI = {
'HBD' : '@@000000013',
'HIVE' : '@@000000021',
'VESTS' : '@@000000037'
}
# convert special chars into their octal formats recognized by sql
special_chars={
"\r":"\\015",
"\n":"\\012",
"\v":"\\013",
"\f": "\\014",
"\\":"\\134",
"'":"\\047",
"%":"\\045",
"_":"\\137",
":":"\\072"
SPECIAL_CHARS = {
"\r" : "\\015",
"\n" : "\\012",
"\v" : "\\013",
"\f" : "\\014",
"\\" : "\\134",
"'" : "\\047",
"%" : "\\045",
"_" : "\\137",
":" : "\\072"
}
def to_nai(value):
""" Convert various amount notation to nai notation """
ret = None
if isinstance(value, dict):
assert 'amount' in value, "amount not found in dict"
assert 'precision' in value, "precision not found in dict"
assert 'nai' in value, "nai not found in dict"
ret = value
elif isinstance(value, str):
raw_amount, unit = value.split(' ')
assert unit in UNIT_NAI, "Unknown unit {}".format(unit)
nai = UNIT_NAI[unit]
precision = NAI_PRECISION[nai]
satoshis = int(decimal.Decimal(raw_amount) * (10**precision))
ret = {'amount' : str(satoshis), 'nai' : nai, 'precision' : precision}
elif isinstance(value, list):
satoshis, precision, nai = value
assert nai in NAI_MAP, "Unknown NAI {}".format(nai)
else:
raise Exception("Invalid input amount %s" % repr(value))
return ret
def escape_characters(text):
""" Escape special charactes """
if len(text.strip()) == 0:
......@@ -39,12 +74,12 @@ def escape_characters(text):
ret = "E'"
for ch in text:
if ch.isprintable() or ch in special_chars:
if ch.isprintable() or ch in SPECIAL_CHARS:
try:
dw=special_chars[ch]
ret=ret+dw
except KeyError as k:
ret=ret+ch
dw = SPECIAL_CHARS[ch]
ret = ret + dw
except KeyError:
ret = ret + ch
else:
ordinal = ord(ch)
if ordinal == 0 or ordinal >= 0x80:
......
tests/manual_tests/list_comments_by_author_last_update_test.py
+ 3
− 3
View file @ 23b3ea9c
#!/usr/bin/python3
from .test_base import run_test
from test_base import run_test
if __name__ == '__main__':
reference_hive_node_url = 'https://api.hive.blog'
reference_hive_node_url = 'http://127.0.0.1:8090'
test_hive_node_url = 'http://127.0.0.1:8080'
payload = {
......@@ -17,4 +17,4 @@ if __name__ == '__main__':
"id":1
}
run_test(reference_hive_node_url, test_hive_node_url, payload, ['author', 'permlink', 'updated_at'])
run_test(reference_hive_node_url, test_hive_node_url, payload, ['author', 'permlink', 'last_update'])
tests/manual_tests/list_comments_by_cashout_test.py
+ 2
− 2
View file @ 23b3ea9c
#!/usr/bin/python3
from .test_base import run_test
from test_base import run_test
if __name__ == '__main__':
reference_hive_node_url = 'https://api.hive.blog'
reference_hive_node_url = 'http://127.0.0.1:8090'
test_hive_node_url = 'http://127.0.0.1:8080'
payload = {
......
tests/manual_tests/list_comments_by_parent_test.py
+ 2
− 2
View file @ 23b3ea9c
#!/usr/bin/python3
from .test_base import run_test
from test_base import run_test
if __name__ == '__main__':
reference_hive_node_url = 'https://api.hive.blog'
reference_hive_node_url = 'http://127.0.0.1:8090'
test_hive_node_url = 'http://127.0.0.1:8080'
payload = {
......
tests/manual_tests/list_comments_by_permlink.py
+ 2
− 2
View file @ 23b3ea9c
......@@ -2,14 +2,14 @@
from test_base import run_test
if __name__ == '__main__':
reference_hive_node_url = 'https://api.hive.blog'
reference_hive_node_url = 'http://127.0.0.1:8090'
test_hive_node_url = 'http://127.0.0.1:8080'
payload = {
"jsonrpc" : "2.0",
"method" : "database_api.list_comments",
"params" : {
"start" : ['', ''],
"start" : ['steemit', 'firstpost'],
"limit" : 10,
"order" : 'by_permlink'
},
......
tests/manual_tests/list_comments_by_root_test.py
+ 3
− 3
View file @ 23b3ea9c
#!/usr/bin/python3
from .test_base import run_test
from test_base import run_test
if __name__ == '__main__':
reference_hive_node_url = 'https://api.hive.blog'
reference_hive_node_url = 'http://127.0.0.1:8090'
test_hive_node_url = 'http://127.0.0.1:8080'
payload = {
......@@ -10,7 +10,7 @@ if __name__ == '__main__':
"method" : "database_api.list_comments",
"params" : {
"start" : ['steemit', 'firstpost', '', ''],
"limit" : 20,
"limit" : 10,
"order" : 'by_root'
},
"id":1
......
tests/manual_tests/list_comments_by_update_test.py
+ 4
− 4
View file @ 23b3ea9c
#!/usr/bin/python3
from .test_base import run_test
from test_base import run_test
if __name__ == '__main__':
reference_hive_node_url = 'https://api.hive.blog'
reference_hive_node_url = 'http://127.0.0.1:8090'
test_hive_node_url = 'http://127.0.0.1:8080'
payload = {
......@@ -11,9 +11,9 @@ if __name__ == '__main__':
"params" : {
"start" : ['steemit', '1970-01-01T00:00:00', '', ''],
"limit" : 10,
"order" : 'by_update'
"order" : 'by_last_update'
},
"id":1
}
run_test(reference_hive_node_url, test_hive_node_url, payload, ['author', 'permlink', 'parent_author', 'parent_permlink', 'updated_at'])
run_test(reference_hive_node_url, test_hive_node_url, payload, ['author', 'permlink', 'last_update'])
tests/manual_tests/test_base.py
+ 2
− 0
View file @ 23b3ea9c
......@@ -7,6 +7,7 @@ def run_test(reference_node_url, test_node_url, payload, table_keys):
resp = post(reference_node_url, dumps(payload))
json = resp.json()
#print(json)
table = prettytable.PrettyTable()
table.field_names = table_keys
for row in json['result']['comments']:
......@@ -17,6 +18,7 @@ def run_test(reference_node_url, test_node_url, payload, table_keys):
resp = post(test_node_url, dumps(payload))
json = resp.json()
#print(json)
table = prettytable.PrettyTable()
table.field_names = table_keys
for row in json['result']:
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment