From 2048080359ef78af326ccd87fb5d307f65bb7c91 Mon Sep 17 00:00:00 2001 From: Martin Lees <drov0@users.noreply.github.com> Date: Sat, 2 Apr 2022 16:16:30 -0400 Subject: [PATCH] Squashed commit of the following: commit f6824e05b47fb84d8099f6fa1265b3e730aefde3 Author: Martin Lees <drov0@users.noreply.github.com> Date: Sat Apr 2 13:53:06 2022 -0400 try other idea commit 587797618dca2da7ac82fbb3fbf0efde15594e37 Author: Martin Lees <drov0@users.noreply.github.com> Date: Sat Apr 2 13:51:27 2022 -0400 test wihout php.author_id commit 4a0c9dda3acdec835ee3d6ac56b80353e61a4ccd Author: Martin Lees <drov0@users.noreply.github.com> Date: Wed Mar 23 01:13:30 2022 -0400 test commit 1147b6c19d877d6b333f688b6d0b29ef9be8b7b4 Author: Martin Lees <drov0@users.noreply.github.com> Date: Wed Mar 23 01:04:26 2022 -0400 try to return a NULL::integer commit 5163b17932919f6a9e73b4a3d4694c2978f6e037 Author: Martin Lees <drov0@users.noreply.github.com> Date: Wed Mar 23 01:03:24 2022 -0400 test commit 8e737f13127db7d676891705fd48d03b7a9225c4 Author: Martin Lees <drov0@users.noreply.github.com> Date: Wed Mar 23 00:46:54 2022 -0400 fix query commit 1f3f40d5ae32d359c0ae80ba1b96b9ea4c3fbb1f Author: Martin Lees <drov0@users.noreply.github.com> Date: Wed Mar 23 00:41:36 2022 -0400 overhaul of the query commit a0c08b48e58c5851b980557c7d931452f6211ed7 Author: Martin Lees <drov0@users.noreply.github.com> Date: Tue Mar 22 22:23:32 2022 -0400 test commit 4128881322394879bc67715a93300f02961daf58 Author: Martin Lees <drov0@users.noreply.github.com> Date: Tue Mar 22 21:35:40 2022 -0400 typo commit 924dd89e6631736b8b9bb450860bc88a9026de64 Author: Martin Lees <drov0@users.noreply.github.com> Date: Tue Mar 22 21:31:19 2022 -0400 Decorated sql to fix conflict commit ebcda2ccfc94c64f5528e864038272ad81d670e0 Author: Martin Lees <drov0@users.noreply.github.com> Date: Tue Mar 22 21:16:39 2022 -0400 code style commit df102d8e151b1467df9053c803dc4c1f318e236b Author: Martin Lees <drov0@users.noreply.github.com> Date: Sun Mar 13 23:27:53 2022 -0400 fix query commit 77dc4fb29b3daf888821ddf2371562e46f3bacd9 Author: Martin Lees <drov0@users.noreply.github.com> Date: Sun Mar 13 17:10:09 2022 -0400 Fix format commit 46add4ed815fea64391ed8d58f7e7c67eeefe8bd Author: Martin Lees <drov0@users.noreply.github.com> Date: Sun Mar 13 17:04:05 2022 -0400 fix typo commit d1d08f626ced15c2db0fe0c5b87f8b8694b82c7d Author: Martin Lees <drov0@users.noreply.github.com> Date: Sun Mar 13 17:01:55 2022 -0400 added the flow + mocked ops commit 5a90840dd0ac4aca18915a1186bb58ee0d224243 Author: Martin Lees <drov0@users.noreply.github.com> Date: Tue Jan 25 23:56:05 2022 -0500 update community type feature --- hive/db/sql_scripts/hive_post_operations.sql | 66 +++- hive/indexer/community.py | 16 +- hive/indexer/posts.py | 8 +- mock_data/block_data/community_op/flow.txt | 16 + .../mock_block_data_community.json | 283 +++++++++++++++++- scripts/ci/start-api-benchmarks.sh | 2 +- 6 files changed, 365 insertions(+), 26 deletions(-) diff --git a/hive/db/sql_scripts/hive_post_operations.sql b/hive/db/sql_scripts/hive_post_operations.sql index 50645c1b7..de8b17757 100644 --- a/hive/db/sql_scripts/hive_post_operations.sql +++ b/hive/db/sql_scripts/hive_post_operations.sql @@ -25,6 +25,53 @@ BEGIN END $function$; +DROP FUNCTION IF EXISTS process_community_post; +CREATE OR REPLACE FUNCTION process_community_post(_block_num hive_posts.block_num%TYPE, _community_support_start_block hive_posts.block_num%TYPE, _community_id hive_posts.community_id%TYPE, _community_name hive_permlink_data.permlink%TYPE, _author_id hive_posts.author_id%TYPE, is_comment bool) +RETURNS TABLE(is_muted bool, community_id hive_posts.community_id%TYPE) +LANGUAGE plpgsql +as + $$ +declare + __community_type_id SMALLINT; + __role_id SMALLINT; + __member_role CONSTANT SMALLINT := 2; + __community_type_topic CONSTANT SMALLINT := 1; + __community_type_journal CONSTANT SMALLINT := 2; + __community_type_council CONSTANT SMALLINT := 3; +BEGIN + IF _block_num < _community_support_start_block THEN + RETURN QUERY ( SELECT FALSE, NULL::integer); + RETURN; -- extra return because a RETURN QUERY does not end the function + END IF; + + -- TODO: probably can be cleaned up with only one query instead of the IF + IF _community_id IS NOT NULL THEN + SELECT type_id INTO __community_type_id FROM hive_communities WHERE id = _community_id; + ELSE + SELECT type_id, id INTO __community_type_id, _community_id from hive_communities where name = _community_name; + END IF; + + IF __community_type_id = __community_type_topic THEN + RETURN QUERY ( SELECT TRUE, _community_id); -- Community type 1 allows everyone to post & comment + RETURN; -- extra return because a RETURN QUERY does not end the function + ELSE + IF __community_type_id = __community_type_journal AND is_comment = TRUE THEN + RETURN QUERY ( SELECT TRUE, _community_id); -- Community type journal allows everyone to comment + RETURN; -- extra return because a RETURN QUERY does not end the function + END IF; + select role_id into __role_id from hive_roles where hive_roles.community_id = _community_id AND account_id = _author_id; + IF __community_type_id = __community_type_journal AND is_comment = FALSE AND __role_id IS NOT NULL AND __role_id >= __member_role THEN + RETURN QUERY ( SELECT TRUE, _community_id); -- You have to be at least a member to post + RETURN; -- extra return because a RETURN QUERY does not end the function + ELSIF __community_type_id = __community_type_council AND __role_id IS NOT NULL AND __role_id >= __member_role THEN + RETURN QUERY ( SELECT TRUE, _community_id); -- You have to be at least a member to post or comment + RETURN; -- extra return because a RETURN QUERY does not end the function + END IF; + END IF; + RETURN QUERY ( SELECT FALSE, _community_id); + END; +$$; + DROP FUNCTION IF EXISTS process_hive_post_operation; ; CREATE OR REPLACE FUNCTION process_hive_post_operation( @@ -58,17 +105,14 @@ if _parent_author != '' THEN root_id, is_muted, is_valid, author_id, permlink_id, created_at, updated_at, sc_hot, sc_trend, active, payout_at, cashout_time, counter_deleted, block_num, block_num_created) SELECT php.id AS parent_id, php.depth + 1 AS depth, - (CASE - WHEN _block_num > _community_support_start_block THEN - COALESCE(php.community_id, (select hc.id from hive_communities hc where hc.name = _parent_permlink)) - ELSE NULL - END) AS community_id, + pcp.community_id AS community_id, COALESCE(php.category_id, (select hcg.id from hive_category_data hcg where hcg.category = _parent_permlink)) AS category_id, (CASE(php.root_id) WHEN 0 THEN php.id ELSE php.root_id END) AS root_id, - php.is_muted AS is_muted, php.is_valid AS is_valid, + pcp.is_muted AS is_muted, + php.is_valid AS is_valid, ha.id AS author_id, hpd.id AS permlink_id, _date AS created_at, _date AS updated_at, calculate_time_part_of_hot(_date) AS sc_hot, @@ -77,6 +121,7 @@ if _parent_author != '' THEN _block_num as block_num, _block_num as block_num_created FROM hive_accounts ha, hive_permlink_data hpd, + process_community_post(_block_num, _community_support_start_block, NULL, _parent_permlink, ha.id, false) pcp, hive_posts php INNER JOIN hive_accounts pha ON pha.id = php.author_id INNER JOIN hive_permlink_data phpd ON phpd.id = php.permlink_id @@ -106,14 +151,10 @@ ELSE active, payout_at, cashout_time, counter_deleted, block_num, block_num_created, tags_ids) SELECT 0 AS parent_id, 0 AS depth, - (CASE - WHEN _block_num > _community_support_start_block THEN - (select hc.id FROM hive_communities hc WHERE hc.name = _parent_permlink) - ELSE NULL - END) AS community_id, + pcp.community_id AS community_id, (SELECT hcg.id FROM hive_category_data hcg WHERE hcg.category = _parent_permlink) AS category_id, 0 as root_id, -- will use id as root one if no parent - false AS is_muted, true AS is_valid, + pcp.is_muted AS is_muted, true AS is_valid, ha.id AS author_id, hpd.id AS permlink_id, _date AS created_at, _date AS updated_at, calculate_time_part_of_hot(_date) AS sc_hot, @@ -125,6 +166,7 @@ ELSE FROM prepare_tags( ARRAY_APPEND(_metadata_tags, _parent_permlink ) ) ) as tags_ids FROM hive_accounts ha, + process_community_post(_block_num, _community_support_start_block, NULL, _parent_permlink, author_id, false) pcp, hive_permlink_data hpd WHERE ha.name = _author and hpd.permlink = _permlink diff --git a/hive/indexer/community.py b/hive/indexer/community.py index cd92f064d..93502ebe1 100644 --- a/hive/indexer/community.py +++ b/hive/indexer/community.py @@ -32,6 +32,7 @@ class Role(IntEnum): TYPE_TOPIC = 1 TYPE_JOURNAL = 2 TYPE_COUNCIL = 3 +valid_types = [TYPE_TOPIC, TYPE_JOURNAL, TYPE_COUNCIL] # https://en.wikipedia.org/wiki/ISO_639-1 LANGS = ( @@ -102,6 +103,12 @@ def read_key_dict(obj, key): assert isinstance(obj[key], dict), f'key `{key}` not a dict' return obj[key] +def read_key_integer(op, key): + """Reads a key from dict, ensuring valid bool if present.""" + if key in op: + assert isinstance(op[key], int), 'must be int: %s' % key + return op[key] + return None class Community: """Handles hive community registration and operations.""" @@ -122,8 +129,7 @@ class Community: This method checks for any valid community names and inserts them. """ - # if not re.match(r'^hive-[123]\d{4,6}$', name): - if not re.match(r'^hive-[1]\d{4,6}$', name): + if not re.match(r'^hive-[123]\d{4,6}$', name): return type_id = int(name[5]) _id = Accounts.get_id(name) @@ -532,7 +538,7 @@ class CommunityOp: def _read_props(self): # TODO: assert props changed? props = read_key_dict(self.op, 'props') - valid = ['title', 'about', 'lang', 'is_nsfw', 'description', 'flag_text', 'settings'] + valid = ['title', 'about', 'lang', 'is_nsfw', 'description', 'flag_text', 'settings', 'type_id'] assert_keys_match(props.keys(), valid, allow_missing=True) out = {} @@ -557,6 +563,10 @@ class CommunityOp: avatar_url = settings['avatar_url'] assert not avatar_url or _valid_url_proto(avatar_url) out['avatar_url'] = avatar_url + if 'type_id' in props: + community_type = read_key_integer(props, 'type_id') + assert community_type in valid_types, 'invalid community type' + out['type_id'] = community_type assert out, 'props were blank' self.props = out diff --git a/hive/indexer/posts.py b/hive/indexer/posts.py index b37a5c950..0a6c6a52e 100644 --- a/hive/indexer/posts.py +++ b/hive/indexer/posts.py @@ -88,8 +88,7 @@ class Posts(DbAdapterHolder): return result = dict(row) - # TODO we need to enhance checking related community post validation and honor is_muted. - error = cls._verify_post_against_community(op, result['community_id'], result['is_valid'], result['is_muted']) + error = cls._verify_post_against_community(op, result['community_id'], result['is_valid']) img_url = None if 'image' in md: @@ -393,12 +392,11 @@ class Posts(DbAdapterHolder): Votes.drop_votes_of_deleted_comment(op) @classmethod - def _verify_post_against_community(cls, op, community_id, is_valid, is_muted): + def _verify_post_against_community(cls, op, community_id, is_valid): error = None if community_id and is_valid and not Community.is_post_valid(community_id, op): error = 'not authorized' - # is_valid = False # TODO: reserved for future blacklist status? - is_muted = True + #is_valid = False # TODO: reserved for future blacklist status? return error @classmethod diff --git a/mock_data/block_data/community_op/flow.txt b/mock_data/block_data/community_op/flow.txt index 2e087cd58..bb9cd4b83 100644 --- a/mock_data/block_data/community_op/flow.txt +++ b/mock_data/block_data/community_op/flow.txt @@ -12,6 +12,9 @@ account_create_operation( `hive-103459` ) account_create_operation( `hive-188204` ) account_create_operation( `hive-149232` ) account_create_operation( `hive-104647` ) +account_create_operation( `hive-111111` ) +account_create_operation( `hive-222222` ) +account_create_operation( `hive-333333` ) comment_operation( `hive-135485`, `test-safari`, `secrets1`) comment_operation( `hive-135485`, `test-safari`, `secrets2`) transfer_opearation( `test-safari`, `null`, `0.010666 HBD`, `@test-safari/secrets2` ) - post promotion (with bad amount precision to see what happens - rounding occurs) @@ -22,6 +25,19 @@ comment_operation( `hive-117600`, `test-safari`, `secrets6`) custom_json_operation("[\"setRole\",{\"community\":\"hive-135485\",\"account\":\"test-safari\",\"role\":\"admin\"}]") comment_operation( `test-safari`, `muted-and-recreated`) comment_operation( `test-safari`, `muted-and-edited`) +comment_operation( `test-safari`, `displayed`) +comment_operation( `test-safari`, `hidden1`) +comment_operation( `test-safari`, `hidden2`) +custom_json_operation("[\"setRole\",{\"community\":\"hive-222222\",\"account\":\"test-safari\",\"role\":\"member\"}]") +custom_json_operation("[\"setRole\",{\"community\":\"hive-333333\",\"account\":\"test-safari\",\"role\":\"member\"}]") +comment_operation( `test-safari`, `display2`) +comment_operation( `blocktrades`, `display_comment`) +comment_operation( `test-safari`, `display3`) +comment_operation( `test-safari`, `display_comment1`) +comment_operation( `blocktrades`, `hidden_comment`) +comment_operation( `blocktrades`, `hidden`) +custom_json_operation("[\"updateProps\",{\"community\":\"hive-222222\",\"props\":{\"type_id\":1,\"title\":\"World News\",\"about\":\"A place for major news from around the world.\",\"is_nsfw\":true,\"description\":\"\",\"flag_text\":\"\"}}]") +comment_operation( `blocktrades`, `display4`) ***block 4998002*** custom_json_operation("[\"updateProps\",{\"community\":\"hive-135485\",\"props\":{\"title\":\"World News\",\"about\":\"A place for major news from around the world.\",\"is_nsfw\":true,\"description\":\"\",\"flag_text\":\"\"}}]") custom_json_operation("[\"setRole\",{\"community\":\"hive-135485\",\"account\":\"blocktrades\",\"role\":\"mod\"}]") diff --git a/mock_data/block_data/community_op/mock_block_data_community.json b/mock_data/block_data/community_op/mock_block_data_community.json index 2ce026a1d..18dcbb714 100644 --- a/mock_data/block_data/community_op/mock_block_data_community.json +++ b/mock_data/block_data/community_op/mock_block_data_community.json @@ -565,7 +565,127 @@ "json_metadata": "", "extensions": [] } - }, + }, + { + "type": "account_create_operation", + "value": { + "creator": "test-safari", + "new_account_name": "hive-111111", + "owner": { + "weight_threshold": 1, + "account_auths": [], + "key_auths": [ + [ + "", + 1 + ] + ] + }, + "active": { + "weight_threshold": 1, + "account_auths": [], + "key_auths": [ + [ + "", + 1 + ] + ] + }, + "posting": { + "weight_threshold": 1, + "account_auths": [], + "key_auths": [ + [ + "", + 1 + ] + ] + }, + "memo_key": "", + "json_metadata": "", + "extensions": [] + } + }, + { + "type": "account_create_operation", + "value": { + "creator": "test-safari", + "new_account_name": "hive-222222", + "owner": { + "weight_threshold": 1, + "account_auths": [], + "key_auths": [ + [ + "", + 1 + ] + ] + }, + "active": { + "weight_threshold": 1, + "account_auths": [], + "key_auths": [ + [ + "", + 1 + ] + ] + }, + "posting": { + "weight_threshold": 1, + "account_auths": [], + "key_auths": [ + [ + "", + 1 + ] + ] + }, + "memo_key": "", + "json_metadata": "", + "extensions": [] + } + }, + { + "type": "account_create_operation", + "value": { + "creator": "test-safari", + "new_account_name": "hive-333333", + "owner": { + "weight_threshold": 1, + "account_auths": [], + "key_auths": [ + [ + "", + 1 + ] + ] + }, + "active": { + "weight_threshold": 1, + "account_auths": [], + "key_auths": [ + [ + "", + 1 + ] + ] + }, + "posting": { + "weight_threshold": 1, + "account_auths": [], + "key_auths": [ + [ + "", + 1 + ] + ] + }, + "memo_key": "", + "json_metadata": "", + "extensions": [] + } + }, { "type": "comment_operation", "value": { @@ -681,6 +801,159 @@ "body": "This test is muted in community after creation, then edited - all in the same block for good measure. Test checks if muting survives edit (it should).", "json_metadata": "{}" } + }, + { + "type": "comment_operation", + "value": { + "parent_author": "", + "parent_permlink": "hive-111111", + "author": "test-safari", + "permlink": "displayed", + "title": "normal post", + "body": "This post is displayed", + "json_metadata": "{}" + } + }, + { + "type": "comment_operation", + "value": { + "parent_author": "", + "parent_permlink": "hive-222222", + "author": "test-safari", + "permlink": "hidden1", + "title": "This post is hidden", + "body": "This post is hidden", + "json_metadata": "{}" + } + }, + { + "type": "comment_operation", + "value": { + "parent_author": "", + "parent_permlink": "hive-333333", + "author": "test-safari", + "permlink": "hidden2", + "title": "This post is hidden", + "body": "This post is hidden", + "json_metadata": "{}" + } + }, + { + "type": "custom_json_operation", + "value": { + "required_auths": [], + "required_posting_auths": [ + "hive-222222" + ], + "id": "community", + "json": "[\"setRole\",{\"community\":\"hive-222222\",\"account\":\"test-safari\",\"role\":\"member\"}]" + } + }, + { + "type": "custom_json_operation", + "value": { + "required_auths": [], + "required_posting_auths": [ + "hive-333333" + ], + "id": "community", + "json": "[\"setRole\",{\"community\":\"hive-333333\",\"account\":\"test-safari\",\"role\":\"member\"}]" + } + }, + { + "type": "comment_operation", + "value": { + "parent_author": "", + "parent_permlink": "hive-222222", + "author": "test-safari", + "permlink": "display2", + "title": "This post is shown because I am a member", + "body": "This post is shown because I am a member", + "json_metadata": "{}" + } + }, + { + "type": "comment_operation", + "value": { + "parent_author": "test-safari", + "parent_permlink": "display2", + "author": "blocktrades", + "permlink": "display_comment", + "title": "This comment is shown even though I'm a guest because community type 2", + "body": "hello", + "json_metadata": "{}" + } + }, + { + "type": "comment_operation", + "value": { + "parent_author": "", + "parent_permlink": "hive-333333", + "author": "test-safari", + "permlink": "display3", + "title": "This post is shown because I am a member", + "body": "Hello", + "json_metadata": "{}" + } + }, + { + "type": "comment_operation", + "value": { + "parent_author": "test-safari", + "parent_permlink": "display3", + "author": "test-safari", + "permlink": "display_comment1", + "title": "This comment is shown because member and community type 3", + "body": "hello", + "json_metadata": "{}" + } + }, + { + "type": "comment_operation", + "value": { + "parent_author": "test-safari", + "parent_permlink": "display3", + "author": "blocktrades", + "permlink": "hidden_comment", + "title": "This comment is hidden because guest and community type 3", + "body": "hello", + "json_metadata": "{}" + } + }, + { + "type": "comment_operation", + "value": { + "parent_author": "", + "parent_permlink": "hive-222222", + "author": "blocktrades", + "permlink": "hidden", + "title": "This post is hidden because the community type is now 2", + "body": "Hello", + "json_metadata": "{}" + } + }, + { + "type": "custom_json_operation", + "value": { + "required_auths": [], + "required_posting_auths": [ + "hive-222222" + ], + "id": "community", + "json": "[\"updateProps\",{\"community\":\"hive-222222\",\"props\":{\"type_id\":1,\"title\":\"World News\",\"about\":\"A place for major news from around the world.\",\"is_nsfw\":true,\"description\":\"\",\"flag_text\":\"\"}}]" + } + }, + { + "type": "comment_operation", + "value": { + "parent_author": "", + "parent_permlink": "hive-222222", + "author": "blocktrades", + "permlink": "display4", + "title": "This post is shown because the community type is now 1", + "body": "Hello", + "json_metadata": "{}" + } } ], "extensions": [], @@ -871,7 +1144,7 @@ "extensions": [] } }, - { + { "type": "custom_json_operation", "value": { "required_auths": [], @@ -947,7 +1220,7 @@ "id": "community", "json": "[\"subscribe\",{\"community\":\"hive-157439\"}]" } - }, + }, { "type": "custom_json_operation", "value": { @@ -3952,7 +4225,7 @@ "4998013": { "transactions": [ { - "ref_block_num": 100001, + "ref_block_num": 100001, "ref_block_prefix": 1, "expiration": "2020-03-23T12:17:00", "operations": [ @@ -4267,4 +4540,4 @@ } ] } -} +} \ No newline at end of file diff --git a/scripts/ci/start-api-benchmarks.sh b/scripts/ci/start-api-benchmarks.sh index 6b3ffe23a..afe061210 100755 --- a/scripts/ci/start-api-benchmarks.sh +++ b/scripts/ci/start-api-benchmarks.sh @@ -10,7 +10,7 @@ ITERATIONS=${3:-5} JOBS=${4:-auto} export TAVERN_DIR="$(realpath $5)" -# since it working inside docker it shoud be fine to hardcode it to tmp +# since it working inside docker it should be fine to hardcode it to tmp export HIVEMIND_BENCHMARKS_IDS_FILE=/tmp/test_ids.csv export TAVERN_DISABLE_COMPARATOR=true -- GitLab