.gitlab-ci.yaml

@@ -31,18 +31,12 @@ variables:
   CI_DEBUG_SERVICES: "false" #All the service logs should be saved as artifacts, so it's fine to turn this off.
 
 include:
+  - template: Workflows/Branch-Pipelines.gitlab-ci.yml
   - project: hive/haf
-    ref: 4e9bfea03ef9511984f3ed54c6181d05b24cd960 # develop
+    ref: e988b4e0d221f4b8efcad2bc2e1b587e4c227669 # master
     file: /scripts/ci-helpers/prepare_data_image_job.yml # implicitly pulls base.gitlab-ci.yml from common-ci-configuration
 
 
-workflow:
-  rules:
-    - if: '$CI_MERGE_REQUEST_IID'
-    - if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS'
-      when: never 
-    - if: '$CI_COMMIT_BRANCH || $CI_COMMIT_TAG'
-
 #>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>| ANCHORS |>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
 
 .shared_tags:
@@ -545,8 +539,6 @@ Trigger benchmark-results-collector:
     - job: e2e_benchmark
       artifacts: true # Even though variables.env is not a regular artifact, but a dotenv report, this still needs to be set to true
   rules:
-    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
-      when: never
     - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
       when: on_success
     - if: '$CI_COMMIT_BRANCH == "develop"'

haf @ e988b4e0

-Subproject commit bf9782985235ec0a327ce202e806f1b58df4887f
+Subproject commit e988b4e0d221f4b8efcad2bc2e1b587e4c227669

hive/db/db_state.py

@@ -115,11 +115,14 @@ class DbState:
             'hive_feed_cache_created_at_idx',
             'hive_feed_cache_post_id_idx',
             'hive_feed_cache_account_id_created_at_post_id_idx',
-            'hive_follows_following_state_idx',  # (following, state, created_at, follower)
+            'hive_follows_following_state_id_idx',  # (following, state, id)
             'hive_follows_follower_state_idx',  # (follower, state, created_at, following)
             'hive_follows_follower_following_state_idx',
             'hive_follows_block_num_idx',
             'hive_follows_created_at_idx',
+            'hive_follows_follower_where_blacklisted_idx',
+            'hive_follows_follower_where_follow_muted_idx',
+            'hive_follows_follower_where_follow_blacklists_idx',
             'hive_posts_parent_id_id_idx',
             'hive_posts_depth_idx',
             'hive_posts_root_id_id_idx',

hive/db/schema.py

@@ -288,11 +288,14 @@ def build_metadata():
         sa.Column('follow_muted', BOOLEAN, nullable=False, server_default='0'),
         sa.Column('block_num', sa.Integer, nullable=False),
         sa.UniqueConstraint('following', 'follower', name='hive_follows_ux1'),  # core
-        sa.Index('hive_follows_following_state_idx', 'following', 'state'),
+        sa.Index('hive_follows_following_state_id_idx', 'following', 'state', 'id'), # index used by condenser_get_followers
         sa.Index('hive_follows_follower_state_idx', 'follower', 'state'),
         sa.Index('hive_follows_follower_following_state_idx', 'follower', 'following', 'state'),
         sa.Index('hive_follows_block_num_idx', 'block_num'),
         sa.Index('hive_follows_created_at_idx', 'created_at'),
+        sa.Index('hive_follows_follower_where_blacklisted_idx', 'follower', postgresql_where=sql_text('blacklisted')),
+        sa.Index('hive_follows_follower_where_follow_muted_idx', 'follower', postgresql_where=sql_text('follow_muted')),
+        sa.Index('hive_follows_follower_where_follow_blacklists_idx', 'follower', postgresql_where=sql_text('follow_blacklists')),
     )
 
     sa.Table(

hive/db/sql_scripts/hive_post_operations.sql

@@ -255,18 +255,24 @@ BEGIN
                             block_num = _block_num
                         RETURNING (xmax = 0) as is_new_post, hp.id, hp.author_id, hp.permlink_id, _parent_permlink as post_category, hp.parent_id, hp.community_id, hp.is_valid, hp.is_muted, hp.depth
                 ) -- WITH inserted_post
+            , tagsid_and_posts AS MATERIALIZED (
+                SELECT prepare_tags FROM hivemind_app.prepare_tags( ARRAY_APPEND(_metadata_tags, _parent_permlink ) )
+            ) -- WITH tagsid_and_posts
                , deleted_post_tags AS MATERIALIZED (
-                DELETE FROM hivemind_app.hive_post_tags hpt
-                    USING inserted_post as ip
-                    WHERE NOT ip.is_new_post AND hpt.post_id = ip.id
-                    RETURNING *
+                DELETE FROM hivemind_app.hive_post_tags hp
+                    USING hivemind_app.hive_post_tags as hpt
+                    JOIN inserted_post as ip ON hpt.post_id = ip.id AND NOT ip.is_new_post
+                    LEFT JOIN tagsid_and_posts as tap ON tap.prepare_tags = hpt.tag_id
+                    WHERE hpt.post_id = hp.post_id AND tap.prepare_tags IS NULL
+                    RETURNING hpt.post_id
             ) -- WITH deleted_post_tags
                , inserts_to_posts_and_tags AS MATERIALIZED (
                 INSERT INTO hivemind_app.hive_post_tags(post_id, tag_id)
                     SELECT ip.id, tags.prepare_tags
                     FROM inserted_post as ip
                     LEFT JOIN deleted_post_tags as dpt ON dpt.post_id = 0 -- there is no post 0, this is only to force execute the deleted_post_tags CTE
-                    JOIN ( SELECT prepare_tags FROM hivemind_app.prepare_tags( ARRAY_APPEND(_metadata_tags, _parent_permlink ) ) ) as tags ON TRUE
+                    JOIN tagsid_and_posts as tags ON TRUE
+                ON CONFLICT DO NOTHING
             )
             SELECT
                 ip.is_new_post,

hive/db/sql_scripts/upgrade/upgrade_runtime_migration.sql

@@ -2,6 +2,13 @@ SET ROLE hivemind;
 
 --- Put runtime data migration code here
 
+DO
+$$
+BEGIN
+
+IF EXISTS (SELECT column_name FROM information_schema.columns WHERE table_name='hivemind_app.hive_posts' and column_name='tags_ids') THEN
+  RAISE NOTICE 'Performing TAG-IDs migration';
+
 WITH data_source AS
 (
   SELECT p.id AS post_id, unnest(p.tags_ids) AS tag_id
@@ -24,6 +31,39 @@ TABLESPACE haf_tablespace
 ALTER TABLE hivemind_app.hive_post_tags INHERIT hive.hivemind_app;
 
 ANALYZE VERBOSE hivemind_app.hive_post_tags;
+ELSE
+  RAISE NOTICE 'TAG-IDs migration skipped';
+END IF;
+END
+$$
+;
+
+--- https://gitlab.syncad.com/hive/hivemind/-/merge_requests/686/
+
+CREATE INDEX IF NOT EXISTS hive_follows_follower_where_blacklisted_idx
+    ON hivemind_app.hive_follows USING btree
+    (follower ASC NULLS LAST)
+    TABLESPACE haf_tablespace
+    WHERE blacklisted;
+
+CREATE INDEX IF NOT EXISTS hive_follows_follower_where_follow_blacklists_idx
+    ON hivemind_app.hive_follows USING btree
+    (follower ASC NULLS LAST)
+    TABLESPACE haf_tablespace
+    WHERE follow_blacklists;
+
+CREATE INDEX IF NOT EXISTS hive_follows_follower_where_follow_muted_idx
+    ON hivemind_app.hive_follows USING btree
+    (follower ASC NULLS LAST)
+    TABLESPACE haf_tablespace
+    WHERE follow_muted;
+
+CREATE INDEX IF NOT EXISTS hive_follows_following_state_id_idx
+    ON hivemind_app.hive_follows USING btree
+    (following ASC NULLS LAST, state ASC NULLS LAST, id ASC NULLS LAST)
+    TABLESPACE haf_tablespace;
+
+DROP INDEX IF EXISTS hivemind_app.hive_follows_following_state_idx;
 
 --- Must be at the end
 TRUNCATE TABLE hivemind_app.hive_db_data_migration;

hive/server/condenser_api/methods.py

@@ -119,7 +119,6 @@ async def get_content(context, author: str, permlink: str, observer=None):
     return await _get_content_impl(db, True, author, permlink, observer)
 
 
-@return_error_info
 async def _get_content_impl(db, fat_node_style, author: str, permlink: str, observer=None):
     """Get a single post object."""
     valid_account(author)
@@ -145,7 +144,6 @@ async def get_content_replies(context, author: str, permlink: str):
     return await _get_content_replies_impl(db, True, author, permlink)
 
 
-@return_error_info
 async def _get_content_replies_impl(db, fat_node_style, author: str, permlink: str):
     """Get a list of post objects based on parent."""
     valid_account(author)

hive/server/database_api/methods.py

@@ -268,7 +268,6 @@ def api_vote_info(rows, votes_presentation):
     return ret
 
 
-@return_error_info
 async def find_votes_impl(db, author: str, permlink: str, votes_presentation, limit: int = 1000):
     sql = f"SELECT * FROM {SCHEMA_NAME}.find_votes(:author,:permlink,:limit)"
     rows = await db.query_all(sql, author=author, permlink=permlink, limit=limit)

mock_data/block_data/comment_op/mock_block_data_comments.json

@@ -55,7 +55,7 @@
 							"permlink": "test-beneficiaries",
 							"title": "Testing",
 							"body": "Much testing",
-							"json_metadata": "{}"
+							"json_metadata": "{\"tags\":[\"taverntest\",\"TavernTest\",\"# TavernTest\",\"taverntest\"]}"
 						}
 					},
 					{
@@ -106,6 +106,29 @@
 			}
 		]
 	},
+	"4984622": {
+			"transactions": [
+				{
+					"ref_block_num": 100000,
+					"ref_block_prefix": 1,
+					"expiration": "2020-03-23T12:17:03",
+					"operations": [
+						{
+							"type": "comment_operation",
+							"value": {
+								"parent_author": "",
+								"parent_permlink": "tag",
+								"author": "jouchelle",
+								"permlink": "test-beneficiaries",
+								"title": "Testing",
+								"body": "Much testing",
+								"json_metadata": "{\"tags\":[\"taverntest\",\"TavernTest\",\"# TavernTest\",\"taverntest\"]}"
+							}
+						}
+					]
+				}
+			]
+	},
 	"5000000": {
 		"transactions": []
 	}

tests/api_tests/hivemind/tavern/bridge_api_patterns/get_post/beneficiaries.pat.json

 {
-"promoted": "0.000 HBD",
-"replies": [],
-"reblogs": 0,
-"active_votes": [],
-"author": "jouchelle",
-"author_payout_value": "0.000 HBD",
-"author_reputation": 25,
-"beneficiaries": [
-{
-"account": "amering",
-"weight": 400
-},
-{
-"account": "batel1",
-"weight": 6800
-},
-{
-"account": "cloop3",
-"weight": 2000
-},
-{
-"account": "faddy3",
-"weight": 600
-},
-{
-"account": "fminerten5",
-"weight": 200
-}
-],
-"blacklists": [],
-"body": "Much testing",
-"category": "tag",
-"children": 0,
-"created": "2016-09-15T06:49:42",
-"curator_payout_value": "0.000 HBD",
-"depth": 0,
-"is_paidout": false,
-"json_metadata": {},
-"max_accepted_payout": "1000000.000 HBD",
-"net_rshares": 0,
-"payout": 29.016,
-"payout_at": "2016-09-22T06:49:42",
-"pending_payout_value": "29.016 HBD",
-"percent_hbd": 10000,
-"permlink": "test-beneficiaries",
-"post_id": 1251466,
-"stats": {
-"flag_weight": 0.0,
-"gray": false,
-"hide": false,
-"total_votes": 0
-},
-"title": "Testing",
-"updated": "2016-09-15T06:49:42",
-"url": "/tag/@jouchelle/test-beneficiaries"
+  "active_votes": [],
+  "author": "jouchelle",
+  "author_payout_value": "0.000 HBD",
+  "author_reputation": 25,
+  "beneficiaries": [
+    {
+      "account": "amering",
+      "weight": 400
+    },
+    {
+      "account": "batel1",
+      "weight": 6800
+    },
+    {
+      "account": "cloop3",
+      "weight": 2000
+    },
+    {
+      "account": "faddy3",
+      "weight": 600
+    },
+    {
+      "account": "fminerten5",
+      "weight": 200
+    }
+  ],
+  "blacklists": [],
+  "body": "Much testing",
+  "category": "tag",
+  "children": 0,
+  "created": "2016-09-15T06:49:42",
+  "curator_payout_value": "0.000 HBD",
+  "depth": 0,
+  "is_paidout": false,
+  "json_metadata": {
+    "tags": [
+      "taverntest",
+      "TavernTest",
+      "# TavernTest",
+      "taverntest"
+    ]
+  },
+  "max_accepted_payout": "1000000.000 HBD",
+  "net_rshares": 0,
+  "payout": 29.016,
+  "payout_at": "2016-09-22T06:49:42",
+  "pending_payout_value": "29.016 HBD",
+  "percent_hbd": 10000,
+  "permlink": "test-beneficiaries",
+  "post_id": 1251466,
+  "promoted": "0.000 HBD",
+  "reblogs": 0,
+  "replies": [],
+  "stats": {
+    "flag_weight": 0.0,
+    "gray": false,
+    "hide": false,
+    "total_votes": 0
+  },
+  "title": "Testing",
+  "updated": "2016-09-15T06:49:45",
+  "url": "/tag/@jouchelle/test-beneficiaries"
 }

tests/api_tests/hivemind/tavern/bridge_api_patterns/get_ranked_posts/created/tag_taverntest_duplicate.pat.json

+[
+  {
+    "active_votes": [],
+    "author": "jouchelle",
+    "author_payout_value": "0.000 HBD",
+    "author_reputation": 25,
+    "beneficiaries": [
+      {
+        "account": "amering",
+        "weight": 400
+      },
+      {
+        "account": "batel1",
+        "weight": 6800
+      },
+      {
+        "account": "cloop3",
+        "weight": 2000
+      },
+      {
+        "account": "faddy3",
+        "weight": 600
+      },
+      {
+        "account": "fminerten5",
+        "weight": 200
+      }
+    ],
+    "blacklists": [],
+    "body": "Much testing",
+    "category": "tag",
+    "children": 0,
+    "created": "2016-09-15T06:49:42",
+    "curator_payout_value": "0.000 HBD",
+    "depth": 0,
+    "is_paidout": false,
+    "json_metadata": {
+      "tags": [
+        "taverntest",
+        "TavernTest",
+        "# TavernTest",
+        "taverntest"
+      ]
+    },
+    "max_accepted_payout": "1000000.000 HBD",
+    "net_rshares": 0,
+    "payout": 29.016,
+    "payout_at": "2016-09-22T06:49:42",
+    "pending_payout_value": "29.016 HBD",
+    "percent_hbd": 10000,
+    "permlink": "test-beneficiaries",
+    "post_id": 1251464,
+    "promoted": "0.000 HBD",
+    "reblogs": 0,
+    "replies": [],
+    "stats": {
+      "flag_weight": 0.0,
+      "gray": false,
+      "hide": false,
+      "total_votes": 0
+    },
+    "title": "Testing",
+    "updated": "2016-09-15T06:49:45",
+    "url": "/tag/@jouchelle/test-beneficiaries"
+  }
+]

tests/api_tests/hivemind/tavern/bridge_api_patterns/get_ranked_posts/created/tag_taverntest_duplicate.tavern.yaml

+---
+  test_name: Hivemind
+
+  marks:
+    - patterntest
+
+  includes:
+    - !include ../../../common.yaml
+
+  stages:
+    - name: test
+      request:
+        url: "{service.proto:s}://{service.server:s}:{service.port}/"
+        method: POST
+        headers:
+          content-type: application/json
+        json:
+          jsonrpc: "2.0"
+          id: 1
+          method: "bridge.get_ranked_posts"
+          params: {"sort":"created","tag":"taverntest"}
+      response:
+        status_code: 200
+        verify_response_with:
+          function: validate_response:compare_response_with_pattern
+          extra_kwargs:
+            ignore_tags: "<bridge posts>"