Merge branch 'develop' of gitlab.syncad.com:hive/hivemind into jsalyers-muting-at-sql-level

hive/db/schema.py

@@ -115,6 +115,8 @@ def build_metadata():
         sa.Column('abs_rshares', sa.Numeric, nullable=False, server_default='0'),
         sa.Column('vote_rshares', sa.Numeric, nullable=False, server_default='0'),
         sa.Column('total_vote_weight', sa.Numeric, nullable=False, server_default='0'),
+        sa.Column('total_votes', sa.BigInteger, nullable=False, server_default='0'),
+        sa.Column('net_votes', sa.BigInteger, nullable=False, server_default='0'),
         sa.Column('active', sa.DateTime, nullable=False, server_default='1970-01-01 00:00:00'),
         sa.Column('cashout_time', sa.DateTime, nullable=False, server_default='1970-01-01 00:00:00'),
         sa.Column('percent_hbd', sa.Integer, nullable=False, server_default='10000'),
@@ -203,7 +205,7 @@ def build_metadata():
         sa.Index('hive_votes_voter_id_post_id_idx', 'voter_id', 'post_id'), # probably this index is redundant to hive_votes_voter_id_last_update_idx because of starting voter_id.
         sa.Index('hive_votes_voter_id_last_update_idx', 'voter_id', 'last_update'), # this index is critical for hive_accounts_info_view performance
         sa.Index('hive_votes_post_id_voter_id_idx', 'post_id', 'voter_id'),
-        sa.Index('hive_votes_block_num_idx', 'block_num') # this is also important for hive_accounts_info_view 
+        sa.Index('hive_votes_block_num_idx', 'block_num') # this is also important for hive_accounts_info_view
     )
 
     sa.Table(

hive/db/sql_scripts/hive_posts_view.sql

@@ -32,22 +32,8 @@ SELECT hp.id,
   hp.updated_at,
   hp.vote_rshares AS rshares,
   hp.abs_rshares AS abs_rshares,
-    COALESCE(
-      (
-        SELECT COUNT( 1 )
-        FROM hive_votes v
-        WHERE v.post_id = hp.id AND v.is_effective
-        GROUP BY v.post_id
-      ), 0
-    ) AS total_votes,
-    COALESCE(
-      (
-        SELECT SUM( CASE v.rshares > 0 WHEN True THEN 1 ELSE -1 END )
-        FROM hive_votes v
-        WHERE v.post_id = hp.id AND NOT v.rshares = 0
-        GROUP BY v.post_id
-      ), 0
-    ) AS net_votes,
+  hp.total_votes AS total_votes,
+  hp.net_votes as net_votes,
   hpd.json,
   ha_a.reputation AS author_rep,
   hp.is_hidden,

hive/db/sql_scripts/update_posts_rshares.sql

@@ -17,12 +17,20 @@ SET
   , vote_rshares = votes_rshares.rshares
   , sc_hot = CASE hp.is_paidout WHEN True Then 0 ELSE calculate_hot( votes_rshares.rshares, hp.created_at) END
   , sc_trend = CASE hp.is_paidout WHEN True Then 0 ELSE calculate_tranding( votes_rshares.rshares, hp.created_at) END
+  , total_votes = votes_rshares.total_votes
+  , net_votes = votes_rshares.net_votes
 FROM
   (
     SELECT
         hv.post_id
       , SUM( hv.rshares ) as rshares
       , SUM( ABS( hv.rshares ) ) as abs_rshares
+      , SUM( CASE hv.is_effective WHEN True THEN 1 ELSE 0 END ) as total_votes
+      , SUM( CASE
+              WHEN hv.rshares > 0 THEN 1
+              WHEN hv.rshares = 0 THEN 0
+              ELSE -1
+            END ) as net_votes
     FROM hive_votes hv
     WHERE EXISTS
       (
@@ -33,7 +41,12 @@ FROM
     GROUP BY hv.post_id
   ) as votes_rshares
 WHERE hp.id = votes_rshares.post_id
-AND (hp.abs_rshares != votes_rshares.abs_rshares OR hp.vote_rshares != votes_rshares.rshares);
+AND (
+  hp.abs_rshares != votes_rshares.abs_rshares
+  OR hp.vote_rshares != votes_rshares.rshares
+  OR hp.total_votes != votes_rshares.total_votes
+  OR hp.net_votes != votes_rshares.net_votes
+);
 RESET work_mem;
 RESET enable_seqscan;
 END;

hive/db/sql_scripts/upgrade/upgrade_runtime_migration.sql

@@ -132,7 +132,7 @@ TRUNCATE TABLE hive_db_data_migration;
 insert into hive_db_patch_level
 (patch_date, patched_to_revision)
 select ds.patch_date, ds.patch_revision
-from 
+from
 (
 values
 (now(), '7b8def051be224a5ebc360465f7a1522090c7125'),
@@ -146,7 +146,7 @@ values
 -- https://gitlab.syncad.com/hive/hivemind/-/merge_requests/257
 -- https://gitlab.syncad.com/hive/hivemind/-/merge_requests/251
 -- https://gitlab.syncad.com/hive/hivemind/-/merge_requests/265
--- 
+--
 (now(), '45c2883131472cc14a03fe4e355ba1435020d720'),
 (now(), '7cfc2b90a01b32688075b22a6ab173f210fc770f'), -- https://gitlab.syncad.com/hive/hivemind/-/merge_requests/286
 (now(), 'f2e5f656a421eb1dd71328a94a421934eda27a87')  -- https://gitlab.syncad.com/hive/hivemind/-/merge_requests/275
@@ -165,10 +165,10 @@ values
 ,(now(), '1847c75702384c7e34c624fc91f24d2ef20df91d') -- latest version of develop containing included changes.
 ,(now(), '1f23e1326f3010bc84353aba82d4aa7ff2f999e4') -- hive_posts_author_id_created_at_idx index def. to speedup hive_accounts_info_view.
 ,(now(), '2a274e586454968a4f298a855a7e60394ed90bde') -- get_number_of_unread_notifications speedup https://gitlab.syncad.com/hive/hivemind/-/merge_requests/348/diffs
+,(now(), '431fdaead7dcd69e4d2a45e7ce8a3186b8075515') -- https://gitlab.syncad.com/hive/hivemind/-/merge_requests/367
 ) ds (patch_date, patch_revision)
 where not exists (select null from hive_db_patch_level hpl where hpl.patched_to_revision = ds.patch_revision);
 
 COMMIT;
 
 ;
-

hive/db/sql_scripts/upgrade/upgrade_table_schema.sql

@@ -15,7 +15,7 @@ BEGIN
   EXECUTE 'ALTER DATABASE '||current_database()||' SET join_collapse_limit TO 16';
   EXECUTE 'ALTER DATABASE '||current_database()||' SET from_collapse_limit TO 16';
 END
-$$; 
+$$;
 
 SHOW join_collapse_limit;
 SHOW from_collapse_limit;
@@ -31,13 +31,13 @@ IF NOT EXISTS(SELECT data_type
     alter table ONlY hive_accounts
       add column is_implicit boolean,
       alter column is_implicit set default True;
-  
+
     --- reputations have to be recalculated from scratch.
     update hive_accounts set reputation = 0, is_implicit = True;
-  
+
     alter table ONlY hive_accounts
       alter column is_implicit set not null;
-  
+
     perform deps_restore_dependencies('public', 'hive_accounts');
 
     INSERT INTO hive_db_data_migration VALUES ('Reputation calculation');
@@ -187,7 +187,34 @@ IF EXISTS(SELECT data_type FROM information_schema.columns
 ELSE
   RAISE NOTICE 'SKIPPING hive_posts upgrade - adding a block_num_created column, type change for abs_rshares/vote_rshares columns';
 END IF;
+
+--- https://gitlab.syncad.com/hive/hivemind/-/merge_requests/367
+IF NOT EXISTS (SELECT data_type FROM information_schema.columns
+               WHERE table_name = 'hive_posts' AND column_name = 'total_votes')
+   AND NOT EXISTS (SELECT data_type FROM information_schema.columns
+                 WHERE table_name = 'hive_posts' AND column_name = 'net_votes') THEN
+  RAISE NOTICE 'Performing hive_posts upgrade - adding total_votes and net_votes columns';
+
+  PERFORM deps_save_and_drop_dependencies('public', 'hive_posts', true);
+
+  ALTER TABLE ONLY hive_posts
+    ADD COLUMN total_votes BIGINT,
+    ADD COLUMN net_votes BIGINT;
+
+  UPDATE hive_posts SET total_votes = 0, net_votes = 0; -- Artificial number, requires to start update_posts_rshares for all blocks
+
+  ALTER TABLE ONLY hive_posts
+    ALTER COLUMN total_votes SET NOT NULL,
+    ALTER COLUMN total_votes SET DEFAULT 0,
+    ALTER COLUMN net_votes SET NOT NULL,
+    ALTER COLUMN net_votes SET DEFAULT 0;
+
+  PERFORM deps_restore_dependencies('public', 'hive_posts');
+ELSE
+  RAISE NOTICE 'SKIPPING hive_posts upgrade - adding total_votes and net_votes columns';
+END IF;
 END
+
 $BODY$
 ;
 
@@ -200,9 +227,9 @@ DROP INDEX IF EXISTS hive_mentions_post_id_idx;
 
 -- updated up to 7b8def051be224a5ebc360465f7a1522090c7125
 -- updated up to 033619277eccea70118a5b8dc0c73b913da0025f
-INSERT INTO hive_db_data_migration 
+INSERT INTO hive_db_data_migration
 select 'update_posts_rshares( 0, head_block_number) execution'
-where not exists (select null from hive_db_patch_level where patched_to_revision = '033619277eccea70118a5b8dc0c73b913da0025f')
+where not exists (select null from hive_db_patch_level where patched_to_revision = '431fdaead7dcd69e4d2a45e7ce8a3186b8075515')
 ;
 
 -- updated to e8b65adf22654203f5a79937ff2a95c5c47e10c5 - See merge request hive/hivemind!251
@@ -228,13 +255,13 @@ IF NOT EXISTS(SELECT data_type
     alter table ONLY hive_follows
       add column follow_muted boolean,
       alter column follow_muted set default False;
-  
+
     --- Fill the default value for all existing records.
     update hive_follows set follow_muted = False;
-  
+
     alter table ONlY hive_follows
       alter column follow_muted set not null;
-  
+
     perform deps_restore_dependencies('public', 'hive_follows');
 ELSE
   RAISE NOTICE 'hive_follows::follow_muted migration skipped';
@@ -245,7 +272,7 @@ $BODY$;
 
 --- 4cdf5d19f6cfcb73d3fa504cac9467c4df31c02e - https://gitlab.syncad.com/hive/hivemind/-/merge_requests/295
 --- 9e126e9d762755f2b9a0fd68f076c9af6bb73b76 - https://gitlab.syncad.com/hive/hivemind/-/merge_requests/314 mentions fix
-INSERT INTO hive_db_data_migration 
+INSERT INTO hive_db_data_migration
 select 'update_hive_post_mentions refill execution'
 where not exists (select null from hive_db_patch_level where patched_to_revision = '9e126e9d762755f2b9a0fd68f076c9af6bb73b76' )
 ;
@@ -270,7 +297,7 @@ CREATE INDEX IF NOT EXISTS hive_votes_voter_id_last_update_idx ON hive_votes (vo
 
 --- https://gitlab.syncad.com/hive/hivemind/-/merge_requests/306 update posts children count fix
 --- 0e3c8700659d98b45f1f7146dc46a195f905fc2d
-INSERT INTO hive_db_data_migration 
+INSERT INTO hive_db_data_migration
 select 'update_hive_posts_children_count execution'
 where not exists (select null from hive_db_patch_level where patched_to_revision = '0e3c8700659d98b45f1f7146dc46a195f905fc2d' )
 ;