Fill mention notifications from indexer

hive/db/schema.py

@@ -452,6 +452,7 @@ def build_metadata_community(metadata=None):
         sa.Column('community', sa.String(16), nullable=True),
         sa.Column('payload', sa.String, nullable=True),
         sa.Index('hive_notification_cache_block_num_idx', 'block_num'),
+        sa.Index('hive_notification_cache_src_dst_post_id', 'src', 'dst', 'post_id', unique=True, postgresql_where=sql_text("type_id = 16")),  # mentions type
         sa.Index('hive_notification_cache_dst_score_idx', 'dst', 'score', postgresql_where=sql_text("dst IS NOT NULL")),
     )
 

hive/db/sql_scripts/hive_post_operations.sql

@@ -351,6 +351,71 @@ END
 $function$
 ;
 
+DROP FUNCTION IF EXISTS hivemind_app.process_hive_post_mentions;
+CREATE OR REPLACE FUNCTION hivemind_app.process_hive_post_mentions(_post_id hivemind_app.hive_posts.id%TYPE)
+RETURNS SETOF BIGINT
+LANGUAGE plpgsql
+AS
+$function$
+BEGIN
+  -- With clause is inlined, modified call to reptracker_endpoints.get_account_reputation.
+  -- Reputation is multiplied by 7.5 rather than 9 to bring the max value to 100 rather than 115.
+  -- In case of reputation being 0, the score is set to 25 rather than 0.
+  RETURN query
+    WITH log_account_rep AS
+    (
+        SELECT
+            account_id,
+            LOG(10, ABS(nullif(reputation, 0))) AS rep,
+            (CASE WHEN reputation < 0 THEN -1 ELSE 1 END) AS is_neg
+        FROM reptracker_app.account_reputations
+    ),
+    calculate_rep AS
+    (
+        SELECT
+            account_id,
+            GREATEST(lar.rep - 9, 0) * lar.is_neg AS rep
+        FROM log_account_rep lar
+    ),
+    final_rep AS
+    (
+        SELECT account_id, (cr.rep * 7.5 + 25)::INT AS rep FROM calculate_rep AS cr
+    ),
+    mentions AS
+    (
+        SELECT DISTINCT _post_id AS post_id, T.author_id, ha.id AS account_id, T.block_num
+        FROM
+          hivemind_app.hive_accounts ha
+        INNER JOIN
+        (
+          SELECT LOWER( ( SELECT trim( T.mention::text, '{""}') ) ) AS mention, T.author_id, T.block_num
+          FROM
+          (
+            SELECT
+              hp.id, REGEXP_MATCHES( hpd.body, '(?:^|[^a-zA-Z0-9_!#$%&*@\\/])(?:@)([a-zA-Z0-9\\.-]{1,16}[a-zA-Z0-9])(?![a-z])', 'g') AS mention, hp.author_id, hp.block_num
+            FROM hivemind_app.hive_posts hp
+            INNER JOIN hivemind_app.hive_post_data hpd ON hp.id = hpd.id
+            WHERE hp.id = _post_id
+          ) AS T
+        ) AS T ON ha.name = T.mention
+        WHERE ha.id != T.author_id
+        ORDER BY T.block_num, ha.id
+    )
+    INSERT INTO hivemind_app.hive_notification_cache
+    (block_num, type_id, created_at, src, dst, dst_post_id, post_id, score, payload, community, community_title)
+    SELECT hm.block_num, 16, (SELECT hb.created_at FROM hivemind_app.blocks_view hb WHERE hb.num = (hm.block_num - 1)) AS created_at, hm.author_id, hm.account_id, hm.post_id, hm.post_id, COALESCE(rep.rep, 25), '', '', ''
+    FROM mentions AS hm
+    JOIN hivemind_app.hive_accounts AS a ON hm.account_id = a.id
+    LEFT JOIN final_rep AS rep ON a.haf_id = rep.account_id
+    WHERE hm.block_num > hivemind_app.block_before_irreversible( '90 days' )
+        AND COALESCE(rep.rep, 25) > 0
+    ORDER BY hm.block_num, created_at, hm.author_id, hm.account_id
+    ON CONFLICT (src, dst, post_id) WHERE type_id=16 DO UPDATE
+    SET block_num=EXCLUDED.block_num, created_at=EXCLUDED.created_at
+    RETURNING id;
+END;
+$function$;
+
 DROP FUNCTION IF EXISTS hivemind_app.delete_hive_post(character varying,character varying,character varying, integer, timestamp)
 ;
 CREATE OR REPLACE FUNCTION hivemind_app.delete_hive_post(

hive/db/sql_scripts/notifications_view.sql

-DROP VIEW IF EXISTS hivemind_app.hive_accounts_rank_view CASCADE;
-
-CREATE OR REPLACE VIEW hivemind_app.hive_accounts_rank_view
- AS
-SELECT ha.id,
-	  case
-      WHEN ds.account_rank < 200 THEN 70
-      WHEN ds.account_rank < 1000 THEN 60
-      WHEN ds.account_rank < 6500 THEN 50
-      WHEN ds.account_rank < 25000 THEN 40
-      WHEN ds.account_rank < 100000 THEN 30
-      ELSE 20
-	  end AS score
-FROM hivemind_app.hive_accounts ha
-LEFT JOIN
-(
-  WITH base_rank_data AS
-  (
-  SELECT ha3.account_id, rank() OVER (ORDER BY ha3.reputation DESC) AS account_rank
-  FROM account_reputations ha3
-	ORDER BY ha3.reputation DESC
-	LIMIT 150000
-  )
-  SELECT ha.id,
-         brd.account_rank
-  FROM base_rank_data brd
-  JOIN hivemind_app.hive_accounts ha on ha.haf_id = brd.account_id
-
-  -- Conditions above (related to rank.position) eliminates all records having rank > 100k. So with inclding some
-  -- additional space for redundant accounts (having same reputation) lets assume we're limiting it to 150k
-  -- As another reason, it can be pointed that only 2% of account has the same reputations, it means only 2000
-  -- in 100000, but we get 150000 as 50% would repeat
-
-) ds on ds.id = ha.id
-; 
-
 DROP FUNCTION IF EXISTS hivemind_app.calculate_notify_vote_score(_payout hivemind_app.hive_posts.payout%TYPE, _abs_rshares hivemind_app.hive_posts.abs_rshares%TYPE, _rshares hivemind_app.hive_votes.rshares%TYPE) CASCADE
 ;
 CREATE OR REPLACE FUNCTION hivemind_app.calculate_notify_vote_score(_payout hivemind_app.hive_posts.payout%TYPE, _abs_rshares hivemind_app.hive_posts.abs_rshares%TYPE, _rshares hivemind_app.hive_votes.rshares%TYPE)
@@ -77,41 +41,6 @@ AS $BODY$
            END
 $BODY$;
 
-
--- View: hivemind_app.hive_raw_notifications_as_view
-
--- hive_posts, follows, hive_reblogs, hive_subscriptions, hive_mentions (these are scored by the src account's rank)
-DROP VIEW IF EXISTS hivemind_app.hive_raw_notifications_as_view CASCADE;
-CREATE OR REPLACE VIEW hivemind_app.hive_raw_notifications_as_view
- AS
- SELECT notifs.block_num,
-    notifs.post_id,
-    notifs.type_id,
-    notifs.created_at,
-    notifs.src,
-    notifs.dst,
-    notifs.dst_post_id,
-    notifs.community,
-    notifs.community_title,
-    notifs.payload,
-    harv.score
-   FROM (
- SELECT hm.block_num,
-    hm.post_id,
-    16 AS type_id,
-    (select hb.created_at from hivemind_app.blocks_view hb where hb.num = (hm.block_num - 1)) as created_at, -- use time of previous block to match head_block_time behavior at given block
-    hp.author_id AS src,
-    hm.account_id AS dst,
-    hm.post_id as dst_post_id,
-    ''::character varying(16) AS community,
-    ''::character varying AS community_title,
-    ''::character varying AS payload
-   FROM hivemind_app.hive_mentions hm  -- mentions
-   JOIN hivemind_app.hive_posts hp ON hm.post_id = hp.id
-) notifs
-JOIN hivemind_app.hive_accounts_rank_view harv ON harv.id = notifs.src
-;
-
 --vote has own score, new communities score as 35 (magic number), persistent notifications are already scored
 DROP VIEW IF EXISTS hivemind_app.hive_raw_notifications_view_no_account_score cascade;
 CREATE OR REPLACE VIEW hivemind_app.hive_raw_notifications_view_no_account_score
@@ -199,8 +128,6 @@ AS
 SELECT *
 FROM
   (
-  SELECT * FROM hivemind_app.hive_raw_notifications_as_view
-  UNION ALL
   SELECT * FROM hivemind_app.hive_raw_notifications_view_no_account_score
   ) as notifs
 WHERE notifs.score >= 0 AND notifs.src IS DISTINCT FROM notifs.dst;

hive/indexer/post_data_cache.py

@@ -8,7 +8,7 @@ log = logging.getLogger(__name__)
 
 
 class PostDataCache(DbAdapterHolder):
-    """Procides cache for DB operations on post data table in order to speed up massive sync"""
+    """Provides cache for DB operations on post data table in order to speed up massive sync"""
 
     _data = {}
 
@@ -61,11 +61,13 @@ class PostDataCache(DbAdapterHolder):
             cls.beginTx()
             if len(values_insert) > 0:
                 sql = f"""
-                    INSERT INTO
-                        {SCHEMA_NAME}.hive_post_data (id, title, body, json)
-                    VALUES
+                    WITH inserted AS (
+                        INSERT INTO {SCHEMA_NAME}.hive_post_data (id, title, body, json)
+                        VALUES {','.join(values_insert)}
+                        RETURNING id
+                    )
+                    SELECT {SCHEMA_NAME}.process_hive_post_mentions(id) FROM inserted
                 """
-                sql += ','.join(values_insert)
                 if print_query:
                     log.info(f"Executing query:\n{sql}")
                 cls.db.query_prepared(sql)
@@ -73,19 +75,20 @@ class PostDataCache(DbAdapterHolder):
 
             if len(values_update) > 0:
                 sql = f"""
-                    UPDATE {SCHEMA_NAME}.hive_post_data AS hpd SET
-                        title = COALESCE( data_source.title, hpd.title ),
-                        body = COALESCE( data_source.body, hpd.body ),
-                        json = COALESCE( data_source.json, hpd.json )
-                    FROM
-                    ( SELECT * FROM
-                    ( VALUES
-                """
-                sql += ','.join(values_update)
-                sql += """
-                    ) AS T(id, title, body, json)
-                    ) AS data_source
-                    WHERE hpd.id = data_source.id
+                    WITH updated AS (
+                        UPDATE {SCHEMA_NAME}.hive_post_data AS hpd SET
+                            title = COALESCE( data_source.title, hpd.title ),
+                            body = COALESCE( data_source.body, hpd.body ),
+                            json = COALESCE( data_source.json, hpd.json )
+                        FROM (
+                            SELECT *
+                            FROM (VALUES {','.join(values_update)})
+                            AS T(id, title, body, json)
+                        ) AS data_source
+                        WHERE hpd.id = data_source.id
+                        RETURNING hpd.id
+                    )
+                    SELECT {SCHEMA_NAME}.process_hive_post_mentions(id) FROM updated
                 """
                 if print_query:
                     log.info(f"Executing query:\n{sql}")

tests/api_tests/hivemind/tavern/bridge_api_patterns/unread_notifications/abit.pat.json

 {
   "lastread": "2016-04-29 04:11:57",
-  "unread": 1921
+  "unread": 1929
 }

tests/api_tests/hivemind/tavern/bridge_api_patterns/unread_notifications/alice.pat.json

 {
   "lastread": "2016-04-29 04:11:57",
-  "unread": 47
+  "unread": 48
 }

tests/api_tests/hivemind/tavern/bridge_api_patterns/unread_notifications/larrytom.pat.json

 {
   "lastread": "1970-01-01 00:00:00",
-  "unread": 26
+  "unread": 23
 }

tests/api_tests/hivemind/tavern/bridge_api_patterns/unread_notifications/min_score.pat.json

 {
   "lastread": "1970-01-01 00:00:00",
-  "unread": 2996
+  "unread": 3013
 }

tests/api_tests/hivemind/tavern/bridge_api_patterns/unread_notifications/steemit.pat.json

 {
   "lastread": "1970-01-01 00:00:00",
-  "unread": 673
+  "unread": 691
 }