Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • hive/hivemind
1 result
Show changes
Commits on Source (24)
Hide whitespace changes
Inline Side-by-side
Showing
with 236 additions and 258 deletions
.gitlab-ci.yaml
View file @ 06826c9d
......@@ -73,7 +73,8 @@ hivemind_sync:
script:
- pip3 install --user --upgrade pip setuptools
- scripts/ci_sync.sh "$HIVEMIND_DB_NAME" "$HIVEMIND_POSTGRESQL_CONNECTION_STRING" "$HIVEMIND_SOURCE_HIVED_URL" $HIVEMIND_MAX_BLOCK $HIVEMIND_HTTP_PORT
# WARNING: hardcoded 5000017 for max block
- scripts/ci_sync.sh "$HIVEMIND_DB_NAME" "$HIVEMIND_POSTGRESQL_CONNECTION_STRING" "$HIVEMIND_SOURCE_HIVED_URL" 5000017 $HIVEMIND_HTTP_PORT
artifacts:
paths:
......
.gitmodules
View file @ 06826c9d
[submodule "tests/tests_api"]
path = tests/tests_api
url = https://gitlab.syncad.com/hive/tests_api.git
url = ../../hive/tests_api.git
hive/db/db_state.py
View file @ 06826c9d
......@@ -358,6 +358,14 @@ class DbState:
time_end = perf_counter()
log.info("[INIT] update_notification_cache executed in %.4fs", time_end - time_start)
time_start = perf_counter()
sql = """
SELECT update_follow_count({}, {});
""".format(last_imported_block, current_imported_block)
DbState.db().query_no_return(sql)
time_end = perf_counter()
log.info("[INIT] update_follow_count executed in %.4fs", time_end - time_start)
# Update a block num immediately
DbState.db().query_no_return("UPDATE hive_state SET block_num = :block_num", block_num = current_imported_block)
......@@ -458,8 +466,8 @@ class DbState:
cls._set_ver(9)
if cls._ver == 9:
from hive.indexer.follow import Follow
Follow.force_recount()
#from hive.indexer.follow import Follow
#Follow.force_recount()
cls._set_ver(10)
if cls._ver == 10:
......
hive/db/schema.py
View file @ 06826c9d
......@@ -606,8 +606,8 @@ def setup(db):
"bridge_get_account_posts_by_blog.sql",
"condenser_get_names_by_reblogged.sql",
"condenser_get_discussions_by_comments.sql",
"condenser_get_account_reputations.sql"
"condenser_get_account_reputations.sql",
"update_follow_count.sql"
]
from os.path import dirname, realpath
dir_path = dirname(realpath(__file__))
......
hive/db/sql_scripts/bridge_get_account_posts_by_comments.sql
View file @ 06826c9d
......@@ -48,9 +48,14 @@ BEGIN
hp.is_pinned,
hp.curator_payout_value
FROM
hive_posts_view hp
WHERE
hp.author_id = __account_id AND hp.depth > 0 AND ( __post_id = 0 OR hp.id < __post_id )
(
SELECT hp1.id
FROM hive_posts hp1
WHERE hp1.author_id = __account_id AND hp1.counter_deleted = 0 AND hp1.depth > 0 AND ( __post_id = 0 OR hp1.id < __post_id )
ORDER BY hp1.id DESC
LIMIT _limit
) ds
JOIN hive_posts_view hp ON ds.id = hp.id
ORDER BY hp.id DESC
LIMIT _limit;
END
......
hive/db/sql_scripts/bridge_get_ranked_post_for_observer_communities.sql
View file @ 06826c9d
......@@ -327,12 +327,14 @@ AS
$function$
DECLARE
__post_id INT;
__account_id INT;
__trending_limit FLOAT := 0;
BEGIN
__post_id = find_comment_id( _author, _permlink, True );
IF __post_id <> 0 THEN
SELECT hp.sc_trend INTO __trending_limit FROM hive_posts hp WHERE hp.id = __post_id;
END IF;
__account_id = find_account_id( _observer, True );
RETURN QUERY SELECT
hp.id,
hp.author,
......@@ -378,9 +380,8 @@ BEGIN
FROM
hive_posts hp1
JOIN hive_subscriptions hs ON hp1.community_id = hs.community_id
JOIN hive_accounts ha ON ha.id = hs.account_id
WHERE
ha.name = _observer AND hp1.counter_deleted = 0 AND NOT hp1.is_paidout AND hp1.depth = 0
hs.account_id = __account_id AND hp1.counter_deleted = 0 AND NOT hp1.is_paidout AND hp1.depth = 0
AND ( __post_id = 0 OR hp1.sc_trend < __trending_limit OR ( hp1.sc_trend = __trending_limit AND hp1.id < __post_id ) )
ORDER BY hp1.sc_trend DESC, hp1.id DESC
LIMIT _limit
......
hive/db/sql_scripts/condenser_get_account_reputations.sql
View file @ 06826c9d
......@@ -2,7 +2,6 @@ DROP FUNCTION IF EXISTS condenser_get_account_reputations;
CREATE OR REPLACE FUNCTION condenser_get_account_reputations(
in _account_lower_bound VARCHAR,
in _without_lower_bound BOOLEAN,
in _limit INTEGER
)
RETURNS TABLE
......@@ -19,7 +18,7 @@ BEGIN
RETURN QUERY SELECT
ha.name, ha.reputation
FROM hive_accounts ha
WHERE _without_lower_bound OR ( ha.name >= _account_lower_bound )
WHERE ha.name >= _account_lower_bound AND ha.id != 0 -- don't include artificial empty account
ORDER BY name
LIMIT _limit;
......
hive/db/sql_scripts/db_upgrade.sh
View file @ 06826c9d
......@@ -59,7 +59,8 @@ for sql in postgres_handle_view_changes.sql \
bridge_get_account_posts_by_blog.sql \
condenser_get_names_by_reblogged.sql \
condenser_get_discussions_by_comments.sql \
condenser_get_account_reputations.sql
condenser_get_account_reputations.sql \
update_follow_count.sql
do
echo Executing psql -U $1 -d $2 -f $sql
......
hive/db/sql_scripts/update_follow_count.sql 0 → 100644
View file @ 06826c9d
DROP FUNCTION IF EXISTS update_follow_count(hive_blocks.num%TYPE, hive_blocks.num%TYPE);
CREATE OR REPLACE FUNCTION update_follow_count(
in _first_block hive_blocks.num%TYPE,
in _last_block hive_blocks.num%TYPE
)
RETURNS VOID
LANGUAGE 'plpgsql'
AS
$BODY$
BEGIN
UPDATE hive_accounts ha
SET
followers = data_set.followers_count,
following = data_set.following_count
FROM
(
WITH data_cfe(user_id) AS (
SELECT DISTINCT following FROM hive_follows WHERE block_num BETWEEN _first_block AND _last_block
UNION
SELECT DISTINCT follower FROM hive_follows WHERE block_num BETWEEN _first_block AND _last_block
)
SELECT
data_cfe.user_id AS user_id,
(SELECT COUNT(1) FROM hive_follows hf1 WHERE hf1.following = data_cfe.user_id AND hf1.state = 1) AS followers_count,
(SELECT COUNT(1) FROM hive_follows hf2 WHERE hf2.follower = data_cfe.user_id AND hf2.state = 1) AS following_count
FROM
data_cfe
) AS data_set(user_id, followers_count, following_count)
WHERE
ha.id = data_set.user_id;
END
$BODY$
;
\ No newline at end of file
hive/indexer/accounts.py
View file @ 06826c9d
......@@ -83,8 +83,6 @@ class Accounts(DbAdapterHolder):
"""Check if an account name exists."""
if isinstance(names, str):
return names in cls._ids
if isinstance(names, list):
return all(name in cls._ids for name in names)
return False
@classmethod
......
hive/indexer/blocks.py
View file @ 06826c9d
......@@ -441,7 +441,8 @@ class Blocks:
"SELECT update_hive_posts_api_helper({},{})".format(first_block, last_block),
"SELECT update_feed_cache({}, {})".format(first_block, last_block),
"SELECT update_hive_posts_mentions({}, {})".format(first_block, last_block),
"SELECT update_notification_cache({}, {}, {})".format(first_block, last_block, is_hour_action)
"SELECT update_notification_cache({}, {}, {})".format(first_block, last_block, is_hour_action),
"SELECT update_follow_count({}, {})".format(first_block, last_block),
#,"SELECT update_account_reputations({}, {})".format(first_block, last_block)
]
......
hive/indexer/follow.py
View file @ 06826c9d
"""Handles follow operations."""
import logging
from time import perf_counter as perf
from funcy.seqs import first
from hive.db.adapter import Db
from hive.db.db_state import DbState
from hive.utils.misc import chunks
from hive.indexer.accounts import Accounts
from hive.indexer.db_adapter_holder import DbAdapterHolder
from hive.utils.normalize import escape_characters
log = logging.getLogger(__name__)
FOLLOWERS = 'followers'
FOLLOWING = 'following'
DB = Db.instance()
def _flip_dict(dict_to_flip):
"""Swap keys/values. Returned dict values are array of keys."""
flipped = {}
for key, value in dict_to_flip.items():
if value in flipped:
flipped[value].append(key)
else:
flipped[value] = [key]
return flipped
class Follow(DbAdapterHolder):
"""Handles processing of incoming follow ups and flushing to db."""
......@@ -72,38 +59,40 @@ class Follow(DbAdapterHolder):
block_num=block_num)
cls.idx += 1
if not DbState.is_initial_sync():
for following in op['flg']:
new_state = state
old_state = cls._get_follow_db_state(op['flr'], following)
if new_state == 1:
Follow.follow(op['flr'], following)
if old_state == 1:
Follow.unfollow(op['flr'], following)
if state > 8:
# check if given state exists in dict
# if exists add follower to a list for a given state
# if not exists create list and set that list for given state
if state in cls.follow_update_items_to_flush:
cls.follow_update_items_to_flush[state].append(op['flr'])
cls.follow_update_items_to_flush[state].append((op['flr'], block_num))
else:
cls.follow_update_items_to_flush[state] = [op['flr']]
cls.follow_update_items_to_flush[state] = [(op['flr'], block_num)]
@classmethod
def _validated_op(cls, account, op, date):
"""Validate and normalize the operation."""
if ( not 'what' in op
if (not 'what' in op
or not isinstance(op['what'], list)
or not 'follower' in op
or not 'following' in op):
return None
# follower/following is empty
if not op['follower'] or not op['following']:
return None
op['following'] = op['following'] if isinstance(op['following'], list) else [op['following']]
# mimic original behaviour
# if following name does not exist do not process it: basically equal to drop op for single following entry
# follower/following is empty
if not op['follower'] or not op['following']:
op['following'] = [op for op in op['following'] if Accounts.exists(op)]
# if follower name does not exist drop op
if not Accounts.exists(op['follower']):
return None
if op['follower'] in op['following'] or op['follower'] != account:
return None
what = first(op['what']) or ''
......@@ -115,59 +104,13 @@ class Follow(DbAdapterHolder):
if what not in defs:
return None
all_accounts = list(op['following'])
all_accounts.append(op['follower'])
if (op['follower'] in op['following']
or op['follower'] != account):
return None
non_existent_names = Accounts.check_names(all_accounts)
if non_existent_names:
log.warning("Follow op validation, following names does not exists in database: {}".format(non_existent_names))
return dict(flr=escape_characters(op['follower']),
flg=[escape_characters(following) for following in op['following']],
state=defs[what],
at=date)
@classmethod
def _get_follow_db_state(cls, follower, following):
"""Retrieve current follow state of an account pair."""
sql = """
SELECT
state
FROM
hive_follows hf
INNER JOIN hive_accounts ha_flr ON ha_flr.id = hf.follower AND ha_flr.name = :follower
INNER JOIN hive_accounts ha_flg ON ha_flg.id = hf.following AND ha_flg.name = :following
"""
return cls.db.query_one(sql, follower=follower, following=following)
# -- stat tracking --
_delta = {FOLLOWERS: {}, FOLLOWING: {}}
@classmethod
def follow(cls, follower, following):
"""Applies follow count change the next flush."""
cls._apply_delta(follower, FOLLOWING, 1)
cls._apply_delta(following, FOLLOWERS, 1)
@classmethod
def unfollow(cls, follower, following):
"""Applies follow count change the next flush."""
cls._apply_delta(follower, FOLLOWING, -1)
cls._apply_delta(following, FOLLOWERS, -1)
@classmethod
def _apply_delta(cls, account, role, direction):
"""Modify an account's follow delta in specified direction."""
if not account in cls._delta[role]:
cls._delta[role][account] = 0
cls._delta[role][account] += direction
@classmethod
def _flush_follow_items(cls):
def flush(cls):
n = 0
if cls.follow_items_to_flush:
sql_prefix = """
......@@ -217,7 +160,8 @@ class Follow(DbAdapterHolder):
WHEN 7 THEN TRUE
WHEN 8 THEN FALSE
ELSE EXCLUDED.follow_muted
END)
END),
block_num = EXCLUDED.block_num
WHERE hf.following = EXCLUDED.following AND hf.follower = EXCLUDED.follower
"""
values = []
......@@ -270,7 +214,7 @@ class Follow(DbAdapterHolder):
for state, update_flush_items in cls.follow_update_items_to_flush.items():
for chunk in chunks(update_flush_items, 1000):
sql = None
query_values = ','.join(["({})".format(account) for account in chunk])
query_values = ','.join(["({}, {})".format(account[0], account[1]) for account in chunk])
# [DK] probaly not a bad idea to move that logic to SQL function
if state == 9:
#reset blacklists for follower
......@@ -278,18 +222,20 @@ class Follow(DbAdapterHolder):
UPDATE
hive_follows hf
SET
blacklisted = false
blacklisted = false,
block_num = ds.block_num
FROM
(
SELECT
ha.id as follower_id
ha.id as follower_id,
block_num
FROM
(
VALUES
{}
) AS T(name)
) AS T(name, block_num)
INNER JOIN hive_accounts ha ON ha.name = T.name
) AS ds (follower_id)
) AS ds (follower_id, block_num)
WHERE
hf.follower = ds.follower_id
""".format(query_values)
......@@ -299,18 +245,20 @@ class Follow(DbAdapterHolder):
UPDATE
hive_follows hf
SET
state = 0
state = 0,
block_num = ds.block_num
FROM
(
SELECT
ha.id as follower_id
ha.id as follower_id,
block_num
FROM
(
VALUES
{}
) AS T(name)
) AS T(name, block_num)
INNER JOIN hive_accounts ha ON ha.name = T.name
) AS ds (follower_id)
) AS ds (follower_id, block_num)
WHERE
hf.follower = ds.follower_id
AND hf.state = 1
......@@ -321,18 +269,20 @@ class Follow(DbAdapterHolder):
UPDATE
hive_follows hf
SET
state = 0
state = 0,
block_num = ds.block_num
FROM
(
SELECT
ha.id as follower_id
ha.id as follower_id,
block_num
FROM
(
VALUES
{}
) AS T(name)
) AS T(name, block_num)
INNER JOIN hive_accounts ha ON ha.name = T.name
) AS ds (follower_id)
) AS ds (follower_id, block_num)
WHERE
hf.follower = ds.follower_id
AND hf.state = 2
......@@ -343,36 +293,40 @@ class Follow(DbAdapterHolder):
UPDATE
hive_follows hf
SET
follow_blacklists = false
follow_blacklists = false,
block_num = ds.block_num
FROM
(
SELECT
ha.id as follower_id
ha.id as follower_id,
block_num
FROM
(
VALUES
{0}
) AS T(name)
) AS T(name, block_num)
INNER JOIN hive_accounts ha ON ha.name = T.name
) AS ds (follower_id)
) AS ds (follower_id, block_num)
WHERE
hf.follower = ds.follower_id;
UPDATE
hive_follows hf
SET
follow_blacklists = true
follow_blacklists = true,
block_num = ds.block_num
FROM
(
SELECT
ha.id as follower_id
ha.id as follower_id,
block_num
FROM
(
VALUES
{0}
) AS T(name)
) AS T(name, block_num)
INNER JOIN hive_accounts ha ON ha.name = T.name
) AS ds (follower_id)
) AS ds (follower_id, block_num)
WHERE
hf.follower = ds.follower_id
AND following = (SELECT id FROM hive_accounts WHERE name = 'null')
......@@ -384,36 +338,40 @@ class Follow(DbAdapterHolder):
UPDATE
hive_follows hf
SET
follow_muted = false
follow_muted = false,
block_num = ds.block_num
FROM
(
SELECT
ha.id as follower_id
ha.id as follower_id,
block_num
FROM
(
VALUES
{0}
) AS T(name)
) AS T(name, block_num)
INNER JOIN hive_accounts ha ON ha.name = T.name
) AS ds (follower_id)
) AS ds (follower_id, block_num)
WHERE
hf.follower = ds.follower_id;
UPDATE
hive_follows hf
SET
follow_muted = true
follow_muted = true,
block_num = ds.block_num
FROM
(
SELECT
ha.id as follower_id
ha.id as follower_id,
block_num
FROM
(
VALUES
{0}
) AS T(name)
) AS T(name, block_num)
INNER JOIN hive_accounts ha ON ha.name = T.name
) AS ds (follower_id)
) AS ds (follower_id, block_num)
WHERE
hf.follower = ds.follower_id
AND following = (SELECT id FROM hive_accounts WHERE name = 'null')
......@@ -427,18 +385,20 @@ class Follow(DbAdapterHolder):
blacklisted = false,
follow_blacklists = false,
follow_muted = false,
state = 0
state = 0,
block_num = ds.block_num
FROM
(
SELECT
ha.id as follower_id
ha.id as follower_id,
block_num
FROM
(
VALUES
{0}
) AS T(name)
) AS T(name, block_num)
INNER JOIN hive_accounts ha ON ha.name = T.name
) AS ds (follower_id)
) AS ds (follower_id, block_num)
WHERE
hf.follower = ds.follower_id;
......@@ -446,18 +406,20 @@ class Follow(DbAdapterHolder):
hive_follows hf
SET
follow_blacklists = true,
follow_muted = true
follow_muted = true,
block_num = ds.block_num
FROM
(
SELECT
ha.id as follower_id
ha.id as follower_id,
block_num
FROM
(
VALUES
{0}
) AS T(name)
) AS T(name, block_num)
INNER JOIN hive_accounts ha ON ha.name = T.name
) AS ds (follower_id)
) AS ds (follower_id, block_num)
WHERE
hf.follower = ds.follower_id
AND following = (SELECT id FROM hive_accounts WHERE name = 'null')
......@@ -470,95 +432,3 @@ class Follow(DbAdapterHolder):
cls.follow_update_items_to_flush.clear()
cls.idx = 0
return n
@classmethod
def flush(cls, trx=False):
"""Flushes pending follow count deltas."""
n = cls._flush_follow_items()
updated = 0
sqls = []
for col, deltas in cls._delta.items():
for delta, names in _flip_dict(deltas).items():
updated += len(names)
query_values = ','.join(["({})".format(account) for account in names])
sql = """
UPDATE
hive_accounts ha
SET
%s = %s + :mag
FROM
(
VALUES
%s
) AS T(name)
WHERE ha.name = T.name
"""
sqls.append((sql % (col, col, query_values), dict(mag=delta)))
if not updated:
return n
start = perf()
cls.db.batch_queries(sqls, trx=trx)
if trx:
log.info("[SYNC] flushed %d follow deltas in %ds",
updated, perf() - start)
cls._delta = {FOLLOWERS: {}, FOLLOWING: {}}
return updated + n
@classmethod
def flush_recount(cls):
"""Recounts follows/following counts for all queued accounts.
This is currently not used; this approach was shown to be too
expensive, but it's useful in case follow counts manage to get
out of sync.
"""
names = set([*cls._delta[FOLLOWERS].keys(),
*cls._delta[FOLLOWING].keys()])
query_values = ','.join(["({})".format(account) for account in names])
sql = """
UPDATE
hive_accounts ha
SET
followers = (SELECT COUNT(*) FROM hive_follows WHERE state = 1 AND following = ha.id),
following = (SELECT COUNT(*) FROM hive_follows WHERE state = 1 AND follower = ha.id)
FROM
(
VALUES
{}
) AS T(name)
WHERE ha.name = T.name
""".format(query_values)
cls.db.query(sql)
@classmethod
def force_recount(cls):
"""Recounts all follows after init sync."""
log.info("[SYNC] query follower counts")
sql = """
CREATE TEMPORARY TABLE following_counts AS (
SELECT ha.id account_id, COUNT(state) num
FROM hive_accounts ha
LEFT JOIN hive_follows hf ON ha.id = hf.follower AND state = 1
GROUP BY ha.id);
CREATE TEMPORARY TABLE follower_counts AS (
SELECT ha.id account_id, COUNT(state) num
FROM hive_accounts ha
LEFT JOIN hive_follows hf ON ha.id = hf.following AND state = 1
GROUP BY ha.id);
"""
cls.db.query(sql)
log.info("[SYNC] update follower counts")
sql = """
UPDATE hive_accounts SET followers = num FROM follower_counts
WHERE id = account_id AND followers != num;
UPDATE hive_accounts SET following = num FROM following_counts
WHERE id = account_id AND following != num;
"""
cls.db.query(sql)
hive/indexer/sync.py
View file @ 06826c9d
......@@ -297,9 +297,6 @@ class Sync:
if not CONTINUE_PROCESSING:
return
log.info("[INIT] *** Initial cache build ***")
Follow.force_recount()
def from_steemd(self, is_initial_sync=False, chunk_size=1000):
"""Fast sync strategy: read/process blocks in batches."""
steemd = self._steem
......
hive/server/bridge_api/methods.py
View file @ 06826c9d
......@@ -320,8 +320,6 @@ async def get_account_posts(context, sort:str, account:str, start_author:str='',
posts.append(post)
return posts
return await _get_posts(db, sort, account, start_author, start_permlink, limit, observer)
@return_error_info
async def get_relationship_between_accounts(context, account1, account2, observer=None):
......
hive/server/condenser_api/get_state.py
View file @ 06826c9d
......@@ -11,7 +11,6 @@ from hive.server.common.mutes import Mutes
from hive.server.condenser_api.objects import (
load_accounts,
_mute_votes,
_condenser_post_object)
from hive.server.common.helpers import (
ApiError,
......@@ -251,7 +250,6 @@ async def _load_discussion(db, author, permlink):
if post['author'] not in muted_accounts:
post['active_votes'] = await find_votes_impl(db, row['author'], row['permlink'], VotesPresentation.CondenserApi)
post['active_votes'] = _mute_votes(post['active_votes'], muted_accounts)
posts.append(post)
parent_key = _ref_parent(post)
......
hive/server/condenser_api/methods.py
View file @ 06826c9d
......@@ -2,7 +2,7 @@
from functools import wraps
import hive.server.condenser_api.cursor as cursor
from hive.server.condenser_api.objects import _mute_votes, _condenser_post_object
from hive.server.condenser_api.objects import _condenser_post_object
from hive.server.common.helpers import (
ApiError,
return_error_info,
......@@ -13,7 +13,6 @@ from hive.server.common.helpers import (
valid_offset,
valid_limit,
valid_follow_type)
from hive.server.common.mutes import Mutes
from hive.server.database_api.methods import find_votes_impl, VotesPresentation
# pylint: disable=too-many-arguments,line-too-long,too-many-lines
......@@ -84,16 +83,18 @@ async def get_reblogged_by(context, author: str, permlink: str):
valid_permlink(permlink))
@return_error_info
async def get_account_reputations(context, account_lower_bound: str = None, limit: int = None):
async def get_account_reputations(context, account_lower_bound: str = '', limit: int = 1000):
db = context['db']
return await _get_account_reputations_impl(db, True, account_lower_bound, limit)
async def _get_account_reputations_impl(db, fat_node_style, account_lower_bound, limit):
"""Enumerate account reputations."""
if not account_lower_bound:
account_lower_bound = ''
assert isinstance(account_lower_bound, str), "invalid account_lower_bound type"
limit = valid_limit(limit, 1000, 1000)
sql = "SELECT * FROM condenser_get_account_reputations( '{}', {}, {} )".format( account_lower_bound, account_lower_bound is None, limit )
sql = "SELECT * FROM condenser_get_account_reputations( (:start)::VARCHAR, :limit )"
rows = await db.query_all(sql, start=account_lower_bound, limit=limit)
if fat_node_style:
return [dict(account=r[0], reputation=r[1]) for r in rows]
......@@ -121,11 +122,6 @@ async def _get_content_impl(db, fat_node_style, author: str, permlink: str, obse
result = dict(result[0])
post = _condenser_post_object(result, 0, fat_node_style)
post['active_votes'] = await find_votes_impl(db, author, permlink, VotesPresentation.ActiveVotes if fat_node_style else VotesPresentation.CondenserApi)
if not observer:
post['active_votes'] = _mute_votes(post['active_votes'], Mutes.all())
else:
blacklists_for_user = await Mutes.get_blacklists_for_observer(observer, {'db':db})
post['active_votes'] = _mute_votes(post['active_votes'], blacklists_for_user.keys())
return post
......@@ -143,14 +139,11 @@ async def _get_content_replies_impl(db, fat_node_style, author: str, permlink: s
sql = "SELECT * FROM condenser_get_content_replies(:author, :permlink)"
result = await db.query_all(sql, author=author, permlink=permlink)
muted_accounts = Mutes.all()
posts = []
for row in result:
row = dict(row)
post = _condenser_post_object(row, get_content_additions=fat_node_style)
post['active_votes'] = await find_votes_impl(db, row['author'], row['permlink'], VotesPresentation.ActiveVotes if fat_node_style else VotesPresentation.CondenserApi)
post['active_votes'] = _mute_votes(post['active_votes'], muted_accounts)
posts.append(post)
return posts
......@@ -238,11 +231,9 @@ async def get_posts_by_given_sort(context, sort: str, start_author: str = '', st
sql_result = await db.query_all(sql, tag=tag, author=start_author, permlink=start_permlink, limit=limit )
muted_accounts = Mutes.all()
for row in sql_result:
post = _condenser_post_object(row, truncate_body)
post['active_votes'] = await find_votes_impl(db, row['author'], row['permlink'], VotesPresentation.CondenserApi)
post['active_votes'] = _mute_votes(post['active_votes'], muted_accounts)
posts.append(post)
return posts
......@@ -313,8 +304,6 @@ async def get_discussions_by_blog(context, tag: str = None, start_author: str =
row = dict(row)
post = _condenser_post_object(row, truncate_body=truncate_body)
post['active_votes'] = await find_votes_impl(db, post['author'], post['permlink'], VotesPresentation.CondenserApi)
post['active_votes'] = _mute_votes(post['active_votes'], Mutes.all())
#posts_by_id[row['post_id']] = post
posts_by_id.append(post)
return posts_by_id
......@@ -380,7 +369,6 @@ async def get_discussions_by_comments(context, start_author: str = None, start_p
row = dict(row)
post = _condenser_post_object(row, truncate_body=truncate_body)
post['active_votes'] = await find_votes_impl(db, post['author'], post['permlink'], VotesPresentation.CondenserApi)
post['active_votes'] = _mute_votes(post['active_votes'], Mutes.all())
posts.append(post)
return posts
......@@ -445,15 +433,12 @@ async def get_blog(context, account: str, start_entry_id: int = 0, limit: int =
sql = "SELECT * FROM condenser_get_blog(:account, :last, :limit)"
result = await db.query_all(sql, account=account, last=start_entry_id, limit=limit)
muted_accounts = Mutes.all()
out = []
for row in result:
row = dict(row)
post = _condenser_post_object(row)
post['active_votes'] = await find_votes_impl(db, row['author'], row['permlink'], VotesPresentation.CondenserApi)
post['active_votes'] = _mute_votes(post['active_votes'], muted_accounts)
out.append({"blog": account,
"entry_id": row['entry_id'],
"comment": post,
......
hive/server/condenser_api/objects.py
View file @ 06826c9d
......@@ -4,7 +4,6 @@ import logging
import ujson as json
from hive.utils.normalize import sbd_amount
from hive.server.common.mutes import Mutes
from hive.server.common.helpers import json_date, get_hive_accounts_info_view_query_string
from hive.server.database_api.methods import find_votes_impl, VotesPresentation
from hive.utils.account import safe_db_profile_metadata
......@@ -19,11 +18,6 @@ async def load_accounts(db, names, lite = False):
rows = await db.query_all(sql, names=tuple(names))
return [_condenser_account_object(row) for row in rows]
def _mute_votes(votes, muted_accounts):
if not muted_accounts:
return votes
return [v for v in votes if v['voter'] not in muted_accounts]
def _condenser_account_object(row):
"""Convert an internal account record into legacy-steemd style."""
#The member `vote_weight` from `hive_accounts` is removed, so currently the member `net_vesting_shares` is equals to zero.
......
hive/server/follow_api/methods.py
View file @ 06826c9d
......@@ -2,7 +2,7 @@ from hive.server.condenser_api.methods import _get_account_reputations_impl
from hive.server.common.helpers import return_error_info
@return_error_info
async def get_account_reputations(context, account_lower_bound: str = None, limit: int = None):
async def get_account_reputations(context, account_lower_bound: str = '', limit: int = 1000):
db = context['db']
return await _get_account_reputations_impl(db, False, account_lower_bound, limit)
......
hive/utils/normalize.py
View file @ 06826c9d
......@@ -68,6 +68,7 @@ def to_nai(value):
def escape_characters(text):
""" Escape special charactes """
assert isinstance(text, str), "Expected string got: {}".format(type(text))
if len(text.strip()) == 0:
return "'" + text + "'"
......
mock_data/block_data/follow_op/mock_block_data_follow.json
View file @ 06826c9d
......@@ -269,6 +269,17 @@
"json": "[\"follow\",{\"follower\":\"tester1\",\"following\":[\"t'es'ter3\", \"<html><body><p>PPPPP</p></body></html>\"],\"what\":[\"blog\"]}]"
}
},
{
"type": "custom_json_operation",
"value": {
"required_auths": [],
"required_posting_auths": [
"tester1"
],
"id": "follow",
"json": "[\"follow\",{\"follower\":\"tester1\",\"following\":[\"tester7\", \"<script>alert('hello world');</script>\"],\"what\":[\"blog\"]}]"
}
},
{
"type": "custom_json_operation",
"value": {
......@@ -352,7 +363,7 @@
"tester1"
],
"id": "follow",
"json": "[\"follow\",{\"follower\":\"tester1\",\"following\":[\"tester3\", \"tester4\"],\"what\":[\"blogo-doggo\"]}]"
"json": "[\"follow\",{\"follower\":\"tester1\",\"following\":[\"tester3\", \"gtg\"],\"what\":[\"blogo-doggo\"]}]"
}
},
{
......@@ -363,7 +374,84 @@
"te'%@ter1"
],
"id": "follow",
"json": "[\"follow\",{\"follower\":\"te'%@ter1\",\"following\":[\"tester3\", \"tester4\"],\"what\":[\"blog\"]}]"
"json": "[\"follow\",{\"follower\":\"te'%@ter1\",\"following\":[\"gtg\", \"tester4\"],\"what\":[\"blog\"]}]"
}
},
{
"type": "custom_json_operation",
"value": {
"required_auths": [],
"required_posting_auths": [
"{\"tester1\":\"tester1\"}"
],
"id": "follow",
"json": "[\"follow\",{\"follower\":{\"tester1\":\"tester1\"},\"following\":{\"gtg\":\"gtg\"},\"what\":[\"blog\"]}]"
}
},
{
"type": "custom_json_operation",
"value": {
"required_auths": [],
"required_posting_auths": [
"tester1"
],
"id": "follow",
"json": "[\"follow\",{\"follower\":\"tester1\",\"following\":{\"gtg\":\"gtg\"},\"what\":[\"blog\"]}]"
}
},
{
"type": "custom_json_operation",
"value": {
"required_auths": [],
"required_posting_auths": [
"tester1"
],
"id": "follow",
"json": "[\"follow\",{\"follower\":\"tester1\",\"following\":[\"tester3\", [\"gtg\"]],\"what\":[\"blog\"]}]"
}
},
{
"type": "custom_json_operation",
"value": {
"required_auths": [],
"required_posting_auths": [
"tester1"
],
"id": "follow",
"json": "[\"follow\",{\"follower\":[\"tester1\"],\"following\":[\"tester3\", [\"gtg\"]],\"what\":[\"blog\"]}]"
}
},
{
"type": "custom_json_operation",
"value": {
"required_auths": [],
"required_posting_auths": [
"[\"tester1\"]"
],
"id": "follow",
"json": "[\"follow\",{\"follower\":[\"tester1\"],\"following\":[\"tester3\", {\"gtg\":\"gtg\"}],\"what\":[\"blog\"]}]"
}
},
{
"type": "custom_json_operation",
"value": {
"required_auths": [],
"required_posting_auths": [
"tester1"
],
"id": "follow",
"json": "[\"follow\",{\"follower\":\"tester1\",\"following\":[\"tester3\", {\"gtg\":\"gtg\"}],\"what\":[\"blog\"]}]"
}
},
{
"type": "custom_json_operation",
"value": {
"required_auths": [],
"required_posting_auths": [
"tester1"
],
"id": "follow",
"json": "[\"follow\",{\"follower\":\"tester1\",\"following\":[\"tester7\", \"<script>alert('hello world');</script>\"],\"what\":[\"blog\"]}]"
}
}
]
......