.gitlab-ci.yaml

@@ -73,7 +73,8 @@ hivemind_sync:
 
   script:
     - pip3 install --user --upgrade pip setuptools
-    - scripts/ci_sync.sh "$HIVEMIND_DB_NAME" "$HIVEMIND_POSTGRESQL_CONNECTION_STRING" "$HIVEMIND_SOURCE_HIVED_URL" $HIVEMIND_MAX_BLOCK $HIVEMIND_HTTP_PORT
+    # WARNING: hardcoded 5000017 for max block
+    - scripts/ci_sync.sh "$HIVEMIND_DB_NAME" "$HIVEMIND_POSTGRESQL_CONNECTION_STRING" "$HIVEMIND_SOURCE_HIVED_URL" 5000017 $HIVEMIND_HTTP_PORT
 
   artifacts:
     paths:

.gitmodules

 [submodule "tests/tests_api"]
 	path = tests/tests_api
-	url = https://gitlab.syncad.com/hive/tests_api.git
+	url = ../../hive/tests_api.git

hive/db/db_state.py

@@ -358,6 +358,14 @@ class DbState:
         time_end = perf_counter()
         log.info("[INIT] update_notification_cache executed in %.4fs", time_end - time_start)
 
+        time_start = perf_counter()
+        sql = """
+            SELECT update_follow_count({}, {});
+        """.format(last_imported_block, current_imported_block)
+        DbState.db().query_no_return(sql)
+        time_end = perf_counter()
+        log.info("[INIT] update_follow_count executed in %.4fs", time_end - time_start)
+
         # Update a block num immediately
         DbState.db().query_no_return("UPDATE hive_state SET block_num = :block_num", block_num = current_imported_block)
 
@@ -458,8 +466,8 @@ class DbState:
             cls._set_ver(9)
 
         if cls._ver == 9:
-            from hive.indexer.follow import Follow
-            Follow.force_recount()
+            #from hive.indexer.follow import Follow
+            #Follow.force_recount()
             cls._set_ver(10)
 
         if cls._ver == 10:

hive/db/schema.py

@@ -606,8 +606,8 @@ def setup(db):
       "bridge_get_account_posts_by_blog.sql",
       "condenser_get_names_by_reblogged.sql",
       "condenser_get_discussions_by_comments.sql",
-      "condenser_get_account_reputations.sql"
-
+      "condenser_get_account_reputations.sql",
+      "update_follow_count.sql"
     ]
     from os.path import dirname, realpath
     dir_path = dirname(realpath(__file__))

hive/db/sql_scripts/bridge_get_account_posts_by_comments.sql

@@ -48,9 +48,14 @@ BEGIN
       hp.is_pinned,
       hp.curator_payout_value
   FROM
-      hive_posts_view hp
-  WHERE
-      hp.author_id = __account_id AND hp.depth > 0 AND ( __post_id = 0 OR hp.id < __post_id )
+  (
+    SELECT hp1.id
+    FROM hive_posts hp1 
+    WHERE hp1.author_id = __account_id AND hp1.counter_deleted = 0 AND hp1.depth > 0 AND ( __post_id = 0 OR hp1.id < __post_id )
+    ORDER BY hp1.id DESC
+    LIMIT _limit
+  ) ds
+  JOIN hive_posts_view hp ON ds.id = hp.id
   ORDER BY hp.id DESC
   LIMIT _limit;
 END

hive/db/sql_scripts/bridge_get_ranked_post_for_observer_communities.sql

@@ -327,12 +327,14 @@ AS
 $function$
 DECLARE
   __post_id INT;
+  __account_id INT;
   __trending_limit FLOAT := 0;
 BEGIN
   __post_id = find_comment_id( _author, _permlink, True );
   IF __post_id <> 0 THEN
       SELECT hp.sc_trend INTO __trending_limit FROM hive_posts hp WHERE hp.id = __post_id;
   END IF;
+  __account_id = find_account_id( _observer, True );
   RETURN QUERY SELECT
       hp.id,
       hp.author,
@@ -378,9 +380,8 @@ BEGIN
       FROM
           hive_posts hp1
           JOIN hive_subscriptions hs ON hp1.community_id = hs.community_id
-          JOIN hive_accounts ha ON ha.id = hs.account_id
       WHERE
-          ha.name = _observer AND hp1.counter_deleted = 0 AND NOT hp1.is_paidout AND hp1.depth = 0
+          hs.account_id = __account_id AND hp1.counter_deleted = 0 AND NOT hp1.is_paidout AND hp1.depth = 0
           AND ( __post_id = 0 OR hp1.sc_trend < __trending_limit OR ( hp1.sc_trend = __trending_limit AND hp1.id < __post_id ) )
       ORDER BY hp1.sc_trend DESC, hp1.id DESC
       LIMIT _limit

hive/db/sql_scripts/condenser_get_account_reputations.sql

@@ -2,7 +2,6 @@ DROP FUNCTION IF EXISTS condenser_get_account_reputations;
 
 CREATE OR REPLACE FUNCTION condenser_get_account_reputations(
   in _account_lower_bound VARCHAR,
-  in _without_lower_bound BOOLEAN,
   in _limit INTEGER
 )
 RETURNS TABLE
@@ -19,7 +18,7 @@ BEGIN
     RETURN QUERY SELECT
       ha.name, ha.reputation
     FROM hive_accounts ha
-    WHERE _without_lower_bound OR ( ha.name >= _account_lower_bound )
+    WHERE ha.name >= _account_lower_bound AND ha.id != 0 -- don't include artificial empty account
     ORDER BY name
     LIMIT _limit;
 

hive/db/sql_scripts/db_upgrade.sh

@@ -59,7 +59,8 @@ for sql in postgres_handle_view_changes.sql \
           bridge_get_account_posts_by_blog.sql \
           condenser_get_names_by_reblogged.sql \
           condenser_get_discussions_by_comments.sql \
-          condenser_get_account_reputations.sql
+          condenser_get_account_reputations.sql \
+          update_follow_count.sql
 
 do
     echo Executing psql -U $1 -d $2 -f $sql

hive/db/sql_scripts/update_follow_count.sql

+DROP FUNCTION IF EXISTS update_follow_count(hive_blocks.num%TYPE, hive_blocks.num%TYPE);
+CREATE OR REPLACE FUNCTION update_follow_count(
+  in _first_block hive_blocks.num%TYPE,
+  in _last_block hive_blocks.num%TYPE
+)
+RETURNS VOID
+LANGUAGE 'plpgsql'
+AS
+$BODY$
+BEGIN
+UPDATE hive_accounts ha
+SET
+  followers = data_set.followers_count,
+  following = data_set.following_count
+FROM
+  (
+    WITH data_cfe(user_id) AS (
+      SELECT DISTINCT following FROM hive_follows WHERE block_num BETWEEN _first_block AND _last_block
+      UNION 
+      SELECT DISTINCT follower FROM hive_follows WHERE block_num BETWEEN _first_block AND _last_block
+    )
+    SELECT
+        data_cfe.user_id AS user_id,
+        (SELECT COUNT(1) FROM hive_follows hf1 WHERE hf1.following = data_cfe.user_id AND hf1.state = 1) AS followers_count,
+        (SELECT COUNT(1) FROM hive_follows hf2 WHERE hf2.follower = data_cfe.user_id AND hf2.state = 1) AS following_count
+    FROM
+        data_cfe
+  ) AS data_set(user_id, followers_count, following_count)
+WHERE
+  ha.id = data_set.user_id;
+END
+$BODY$
+;
\ No newline at end of file

hive/indexer/accounts.py

@@ -83,8 +83,6 @@ class Accounts(DbAdapterHolder):
         """Check if an account name exists."""
         if isinstance(names, str):
             return names in cls._ids
-        if isinstance(names, list):
-            return all(name in cls._ids for name in names)
         return False
 
     @classmethod

hive/indexer/blocks.py

@@ -441,7 +441,8 @@ class Blocks:
             "SELECT update_hive_posts_api_helper({},{})".format(first_block, last_block),
             "SELECT update_feed_cache({}, {})".format(first_block, last_block),
             "SELECT update_hive_posts_mentions({}, {})".format(first_block, last_block),
-            "SELECT update_notification_cache({}, {}, {})".format(first_block, last_block, is_hour_action)
+            "SELECT update_notification_cache({}, {}, {})".format(first_block, last_block, is_hour_action),
+            "SELECT update_follow_count({}, {})".format(first_block, last_block),
             #,"SELECT update_account_reputations({}, {})".format(first_block, last_block)
         ]
 

hive/indexer/sync.py

@@ -297,9 +297,6 @@ class Sync:
         if not CONTINUE_PROCESSING:
             return
 
-        log.info("[INIT] *** Initial cache build ***")
-        Follow.force_recount()
-
     def from_steemd(self, is_initial_sync=False, chunk_size=1000):
         """Fast sync strategy: read/process blocks in batches."""
         steemd = self._steem

hive/server/bridge_api/methods.py

@@ -320,8 +320,6 @@ async def get_account_posts(context, sort:str, account:str, start_author:str='',
         posts.append(post)
     return posts
 
-    return await _get_posts(db, sort, account, start_author, start_permlink, limit, observer)
-
 
 @return_error_info
 async def get_relationship_between_accounts(context, account1, account2, observer=None):

hive/server/condenser_api/get_state.py

@@ -11,7 +11,6 @@ from hive.server.common.mutes import Mutes
 
 from hive.server.condenser_api.objects import (
     load_accounts,
-    _mute_votes,
     _condenser_post_object)
 from hive.server.common.helpers import (
     ApiError,
@@ -251,7 +250,6 @@ async def _load_discussion(db, author, permlink):
 
       if post['author'] not in muted_accounts:
         post['active_votes'] = await find_votes_impl(db, row['author'], row['permlink'], VotesPresentation.CondenserApi)
-        post['active_votes'] = _mute_votes(post['active_votes'], muted_accounts)
         posts.append(post)
 
         parent_key = _ref_parent(post)

hive/server/condenser_api/methods.py

@@ -2,7 +2,7 @@
 from functools import wraps
 
 import hive.server.condenser_api.cursor as cursor
-from hive.server.condenser_api.objects import _mute_votes, _condenser_post_object
+from hive.server.condenser_api.objects import _condenser_post_object
 from hive.server.common.helpers import (
     ApiError,
     return_error_info,
@@ -13,7 +13,6 @@ from hive.server.common.helpers import (
     valid_offset,
     valid_limit,
     valid_follow_type)
-from hive.server.common.mutes import Mutes
 from hive.server.database_api.methods import find_votes_impl, VotesPresentation
 
 # pylint: disable=too-many-arguments,line-too-long,too-many-lines
@@ -84,16 +83,18 @@ async def get_reblogged_by(context, author: str, permlink: str):
         valid_permlink(permlink))
 
 @return_error_info
-async def get_account_reputations(context, account_lower_bound: str = None, limit: int = None):
+async def get_account_reputations(context, account_lower_bound: str = '', limit: int = 1000):
     db = context['db']
     return await _get_account_reputations_impl(db, True, account_lower_bound, limit)
 
 async def _get_account_reputations_impl(db, fat_node_style, account_lower_bound, limit):
     """Enumerate account reputations."""
+    if not account_lower_bound:
+      account_lower_bound = ''
     assert isinstance(account_lower_bound, str), "invalid account_lower_bound type"
     limit = valid_limit(limit, 1000, 1000)
 
-    sql = "SELECT * FROM condenser_get_account_reputations( '{}', {}, {} )".format( account_lower_bound, account_lower_bound is None, limit )
+    sql = "SELECT * FROM condenser_get_account_reputations( (:start)::VARCHAR, :limit )"
     rows = await db.query_all(sql, start=account_lower_bound, limit=limit)
     if fat_node_style:
         return [dict(account=r[0], reputation=r[1]) for r in rows]
@@ -121,11 +122,6 @@ async def _get_content_impl(db, fat_node_style, author: str, permlink: str, obse
         result = dict(result[0])
         post = _condenser_post_object(result, 0, fat_node_style)
         post['active_votes'] = await find_votes_impl(db, author, permlink, VotesPresentation.ActiveVotes if fat_node_style else VotesPresentation.CondenserApi)
-        if not observer:
-            post['active_votes'] = _mute_votes(post['active_votes'], Mutes.all())
-        else:
-            blacklists_for_user = await Mutes.get_blacklists_for_observer(observer, {'db':db})
-            post['active_votes'] = _mute_votes(post['active_votes'], blacklists_for_user.keys())
 
     return post
 
@@ -143,14 +139,11 @@ async def _get_content_replies_impl(db, fat_node_style, author: str, permlink: s
     sql = "SELECT * FROM condenser_get_content_replies(:author, :permlink)"
     result = await db.query_all(sql, author=author, permlink=permlink)
 
-    muted_accounts = Mutes.all()
-
     posts = []
     for row in result:
         row = dict(row)
         post = _condenser_post_object(row, get_content_additions=fat_node_style)
         post['active_votes'] = await find_votes_impl(db, row['author'], row['permlink'], VotesPresentation.ActiveVotes if fat_node_style else VotesPresentation.CondenserApi)
-        post['active_votes'] = _mute_votes(post['active_votes'], muted_accounts)
         posts.append(post)
 
     return posts
@@ -238,11 +231,9 @@ async def get_posts_by_given_sort(context, sort: str, start_author: str = '', st
 
     sql_result = await db.query_all(sql, tag=tag, author=start_author, permlink=start_permlink, limit=limit )
 
-    muted_accounts = Mutes.all()
     for row in sql_result:
         post = _condenser_post_object(row, truncate_body)
         post['active_votes'] = await find_votes_impl(db, row['author'], row['permlink'], VotesPresentation.CondenserApi)
-        post['active_votes'] = _mute_votes(post['active_votes'], muted_accounts)
         posts.append(post)
     return posts
 
@@ -313,8 +304,6 @@ async def get_discussions_by_blog(context, tag: str = None, start_author: str =
         row = dict(row)
         post = _condenser_post_object(row, truncate_body=truncate_body)
         post['active_votes'] = await find_votes_impl(db, post['author'], post['permlink'], VotesPresentation.CondenserApi)
-        post['active_votes'] = _mute_votes(post['active_votes'], Mutes.all())
-        #posts_by_id[row['post_id']] = post
         posts_by_id.append(post)
 
     return posts_by_id
@@ -380,7 +369,6 @@ async def get_discussions_by_comments(context, start_author: str = None, start_p
         row = dict(row)
         post = _condenser_post_object(row, truncate_body=truncate_body)
         post['active_votes'] = await find_votes_impl(db, post['author'], post['permlink'], VotesPresentation.CondenserApi)
-        post['active_votes'] = _mute_votes(post['active_votes'], Mutes.all())
         posts.append(post)
 
     return posts
@@ -445,15 +433,12 @@ async def get_blog(context, account: str, start_entry_id: int = 0, limit: int =
     sql = "SELECT * FROM condenser_get_blog(:account, :last, :limit)"
     result = await db.query_all(sql, account=account, last=start_entry_id, limit=limit)
 
-    muted_accounts = Mutes.all()
     out = []
     for row in result:
         row = dict(row)
         post = _condenser_post_object(row)
 
         post['active_votes'] = await find_votes_impl(db, row['author'], row['permlink'], VotesPresentation.CondenserApi)
-        post['active_votes'] = _mute_votes(post['active_votes'], muted_accounts)
-
         out.append({"blog": account,
                     "entry_id": row['entry_id'],
                     "comment": post,

hive/server/condenser_api/objects.py

@@ -4,7 +4,6 @@ import logging
 import ujson as json
 
 from hive.utils.normalize import sbd_amount
-from hive.server.common.mutes import Mutes
 from hive.server.common.helpers import json_date, get_hive_accounts_info_view_query_string
 from hive.server.database_api.methods import find_votes_impl, VotesPresentation
 from hive.utils.account import safe_db_profile_metadata
@@ -19,11 +18,6 @@ async def load_accounts(db, names, lite = False):
     rows = await db.query_all(sql, names=tuple(names))
     return [_condenser_account_object(row) for row in rows]
 
-def _mute_votes(votes, muted_accounts):
-    if not muted_accounts:
-        return votes
-    return [v for v in votes if v['voter'] not in muted_accounts]
-
 def _condenser_account_object(row):
     """Convert an internal account record into legacy-steemd style."""
     #The member `vote_weight` from `hive_accounts` is removed, so currently the member `net_vesting_shares` is equals to zero.

hive/server/follow_api/methods.py

@@ -2,7 +2,7 @@ from hive.server.condenser_api.methods import _get_account_reputations_impl
 from hive.server.common.helpers import return_error_info
 
 @return_error_info
-async def get_account_reputations(context, account_lower_bound: str = None, limit: int = None):
+async def get_account_reputations(context, account_lower_bound: str = '', limit: int = 1000):
     db = context['db']
     return await _get_account_reputations_impl(db, False, account_lower_bound, limit)
 

hive/utils/normalize.py

@@ -68,6 +68,7 @@ def to_nai(value):
 
 def escape_characters(text):
     """ Escape special charactes """
+    assert isinstance(text, str), "Expected string got: {}".format(type(text))
     if len(text.strip()) == 0:
         return "'" + text + "'"
 

mock_data/block_data/follow_op/mock_block_data_follow.json

@@ -269,6 +269,17 @@
               "json": "[\"follow\",{\"follower\":\"tester1\",\"following\":[\"t'es'ter3\", \"<html><body><p>PPPPP</p></body></html>\"],\"what\":[\"blog\"]}]"
             }
           },
+          {
+            "type": "custom_json_operation",
+            "value": {
+              "required_auths": [],
+              "required_posting_auths": [
+                "tester1"
+              ],
+              "id": "follow",
+              "json": "[\"follow\",{\"follower\":\"tester1\",\"following\":[\"tester7\", \"<script>alert('hello world');</script>\"],\"what\":[\"blog\"]}]"
+            }
+          },
           {
             "type": "custom_json_operation",
             "value": {
@@ -352,7 +363,7 @@
                 "tester1"
               ],
               "id": "follow",
-              "json": "[\"follow\",{\"follower\":\"tester1\",\"following\":[\"tester3\", \"tester4\"],\"what\":[\"blogo-doggo\"]}]"
+              "json": "[\"follow\",{\"follower\":\"tester1\",\"following\":[\"tester3\", \"gtg\"],\"what\":[\"blogo-doggo\"]}]"
             }
           },
           {
@@ -363,7 +374,84 @@
                 "te'%@ter1"
               ],
               "id": "follow",
-              "json": "[\"follow\",{\"follower\":\"te'%@ter1\",\"following\":[\"tester3\", \"tester4\"],\"what\":[\"blog\"]}]"
+              "json": "[\"follow\",{\"follower\":\"te'%@ter1\",\"following\":[\"gtg\", \"tester4\"],\"what\":[\"blog\"]}]"
+            }
+          },
+          {
+            "type": "custom_json_operation",
+            "value": {
+              "required_auths": [],
+              "required_posting_auths": [
+                "{\"tester1\":\"tester1\"}"
+              ],
+              "id": "follow",
+              "json": "[\"follow\",{\"follower\":{\"tester1\":\"tester1\"},\"following\":{\"gtg\":\"gtg\"},\"what\":[\"blog\"]}]"
+            }
+          },
+          {
+            "type": "custom_json_operation",
+            "value": {
+              "required_auths": [],
+              "required_posting_auths": [
+                "tester1"
+              ],
+              "id": "follow",
+              "json": "[\"follow\",{\"follower\":\"tester1\",\"following\":{\"gtg\":\"gtg\"},\"what\":[\"blog\"]}]"
+            }
+          },
+          {
+            "type": "custom_json_operation",
+            "value": {
+              "required_auths": [],
+              "required_posting_auths": [
+                "tester1"
+              ],
+              "id": "follow",
+              "json": "[\"follow\",{\"follower\":\"tester1\",\"following\":[\"tester3\", [\"gtg\"]],\"what\":[\"blog\"]}]"
+            }
+          },
+          {
+            "type": "custom_json_operation",
+            "value": {
+              "required_auths": [],
+              "required_posting_auths": [
+                "tester1"
+              ],
+              "id": "follow",
+              "json": "[\"follow\",{\"follower\":[\"tester1\"],\"following\":[\"tester3\", [\"gtg\"]],\"what\":[\"blog\"]}]"
+            }
+          },
+          {
+            "type": "custom_json_operation",
+            "value": {
+              "required_auths": [],
+              "required_posting_auths": [
+                "[\"tester1\"]"
+              ],
+              "id": "follow",
+              "json": "[\"follow\",{\"follower\":[\"tester1\"],\"following\":[\"tester3\", {\"gtg\":\"gtg\"}],\"what\":[\"blog\"]}]"
+            }
+          },
+          {
+            "type": "custom_json_operation",
+            "value": {
+              "required_auths": [],
+              "required_posting_auths": [
+                "tester1"
+              ],
+              "id": "follow",
+              "json": "[\"follow\",{\"follower\":\"tester1\",\"following\":[\"tester3\", {\"gtg\":\"gtg\"}],\"what\":[\"blog\"]}]"
+            }
+          },
+          {
+            "type": "custom_json_operation",
+            "value": {
+              "required_auths": [],
+              "required_posting_auths": [
+                "tester1"
+              ],
+              "id": "follow",
+              "json": "[\"follow\",{\"follower\":\"tester1\",\"following\":[\"tester7\", \"<script>alert('hello world');</script>\"],\"what\":[\"blog\"]}]"
             }
           }
         ]