Skip to content
Snippets Groups Projects
Commit 7f9f79c3 authored by Marcin's avatar Marcin
Browse files

update state providers during updating haf 

If the definitions of start providers functions are the same as in the old hfm version, then
it is possible to make update of state providers runtime function. The updatable runtime code needs to
placed in state providers functions hive.runtimecode_provider_<provider_name>, all elements which are created there shall be placed in schema hive and will be removed during update procedure and recreated with their new versions
parent d3de8c81
No related branches found
No related tags found
6 merge requests!627merge in fix for get_current_block_age,!626Fix get_current_block_age function to avoid healthcheck fails,!622merge develop to master,!599merge ( with merge commit) develop to master,!597Merge develop to master for release,!577fixes for update procedure
Pipeline #111646 failed
Stage: build_and_test_phase_1
Stage: build_and_test_phase_2
Stage: cleanup
Hide whitespace changes
Inline Side-by-side
Showing
with 290 additions and 154 deletions
src/hive_fork_manager/CMakeLists.txt
+ 2
− 0
View file @ 7f9f79c3
......@@ -72,6 +72,8 @@ ADD_PSQL_EXTENSION(
application_loop/stages_functions.sql
application_loop/loop.sql
application_loop/contexts_log_api.sql
state_providers/update_providers.sql
)
ADD_SUBDIRECTORY( shared_lib )
......
src/hive_fork_manager/app_api.sql
+ 4
− 0
View file @ 7f9f79c3
......@@ -701,6 +701,10 @@ BEGIN
ON CONFLICT DO NOTHING', __context_id, _state_provider, _state_provider, _context
);
EXECUTE format(
'SELECT hive.runtimecode_provider_%s( %L )', _state_provider, _context
);
IF NOT hive.app_is_forking( _context ) THEN
RETURN;
END IF;
......
src/hive_fork_manager/application_loop/contexts_log_api.sql
+ 1
− 1
View file @ 7f9f79c3
......@@ -94,4 +94,4 @@ BEGIN
, __head_fork
;
END;
$$
\ No newline at end of file
$$;
src/hive_fork_manager/state_providers/accounts.sql
+ 11
− 0
View file @ 7f9f79c3
......@@ -33,6 +33,17 @@ END;
$BODY$
;
CREATE OR REPLACE FUNCTION hive.runtimecode_provider_accounts( _context hafd.context_name )
RETURNS VOID
LANGUAGE plpgsql
VOLATILE
AS
$BODY$
BEGIN
RETURN;
END;
$BODY$;
CREATE OR REPLACE FUNCTION hive.get_created_from_account_create_operations(IN _account_operation hafd.operation)
RETURNS TEXT
AS 'MODULE_PATHNAME', 'get_created_from_account_create_operations' LANGUAGE C;
......
src/hive_fork_manager/state_providers/keyauth.sql
+ 55
− 42
View file @ 7f9f79c3
......@@ -213,14 +213,31 @@ BEGIN
$$
, _context);
RETURN ARRAY[format('%1$s_keyauth_a', _context), format('%1$s_keyauth_k', _context), format('%1$s_accountauth_a', _context)];
END;
$BODY$
;
CREATE OR REPLACE FUNCTION hive.runtimecode_provider_keyauth( _context hafd.context_name )
RETURNS VOID
LANGUAGE plpgsql
VOLATILE
AS
$BODY$
DECLARE
__schema TEXT;
BEGIN
SELECT hc.schema INTo __schema
FROM hafd.contexts hc
WHERE hc.name = _context;
-- Persistent function definition for keyauth insertion
-- The 'hive.start_provider_keyauth_insert_into_keyauth_a' function is created here as a permanent
-- function rather than being dynamically generated during each call to 'hive.update_state_provider_keyauth'.
EXECUTE format(
$t$
$t$
CREATE OR REPLACE FUNCTION hafd.%1$s_insert_into_keyauth_a(
CREATE OR REPLACE FUNCTION hive.%1$s_insert_into_keyauth_a(
_first_block integer,
_last_block integer)
RETURNS void
......@@ -246,8 +263,8 @@ BEGIN
-- Consider relocating this logic from the current CTE to the actual 'start_provider_keyauth' execution for better efficiency.
WITH genesis_auth_records AS MATERIALIZED
(
SELECT
(SELECT a.id FROM %2$s.accounts_view a WHERE a.name = g.account_name) as account_id,
SELECT
(SELECT a.id FROM %2$s.accounts_view a WHERE a.name = g.account_name) as account_id,
g.account_name,
g.key_kind,
g.key_auth,
......@@ -259,14 +276,14 @@ BEGIN
(SELECT b.created_at FROM hafd.blocks b WHERE b.num = 1) as timestamp,
1
FROM hive.get_genesis_keyauths() as g
WHERE _first_block <= 1 AND 1 <= _last_block
WHERE _first_block <= 1 AND 1 <= _last_block
),
-- Hard fork 9 fixes some accounts that were compromised
HARDFORK_9_fixed_auth_records AS MATERIALIZED
(
SELECT
(SELECT a.id FROM %2$s.accounts_view a WHERE a.name = h.account_name) as account_id,
(SELECT a.id FROM %2$s.accounts_view a WHERE a.name = h.account_name) as account_id,
*,
__op_serial_id_dummy as op_serial_id,
__HARDFORK_9_block_num as block_num,
......@@ -279,7 +296,7 @@ BEGIN
HARDFORK_21_fixed_auth_records AS MATERIALIZED
(
SELECT
(SELECT a.id FROM %2$s.accounts_view a WHERE a.name = h.account_name) as account_id,
(SELECT a.id FROM %2$s.accounts_view a WHERE a.name = h.account_name) as account_id,
*,
__op_serial_id_dummy as op_serial_id,
__HARDFORK_21_block_num as block_num,
......@@ -292,7 +309,7 @@ BEGIN
HARDFORK_24_fixed_auth_records AS MATERIALIZED
(
SELECT
(SELECT a.id FROM %2$s.accounts_view a WHERE a.name = h.account_name) as account_id,
(SELECT a.id FROM %2$s.accounts_view a WHERE a.name = h.account_name) as account_id,
*,
__op_serial_id_dummy as op_serial_id,
__HARDFORK_24_block_num as block_num,
......@@ -354,7 +371,7 @@ BEGIN
),
-- Handle all other operations.
matching_op_types as materialized
matching_op_types as materialized
(
select ot.id from hafd.operation_types ot WHERE ot.name IN
(
......@@ -393,31 +410,31 @@ BEGIN
ov.id as op_stable_id
FROM matching_ops ov
),
min_block_per_pow_account AS
min_block_per_pow_account AS
(
SELECT
SELECT
r.account_name,
MIN(r.block_num) as min_block_num
FROM
FROM
raw_auth_records r
INNER JOIN
INNER JOIN
pow_extended_auth_records per ON r.account_name = per.account_name
GROUP BY
GROUP BY
r.account_name
),
min_block_num_from_stored_table_per_account_id AS
min_block_num_from_stored_table_per_account_id AS
(
SELECT
account_id,
SELECT
account_id,
MIN(block_num) AS min_block_num_from_stored_table
FROM
FROM
hafd.%1$s_keyauth_a
GROUP BY
GROUP BY
account_id
),
pow_extended_auth_records_with_min_block AS
(
SELECT
SELECT
pow.account_id,
pow.account_name,
pow.key_kind,
......@@ -432,17 +449,17 @@ BEGIN
mb.min_block_num,
pow_min_block_num,
mb_table.min_block_num_from_stored_table
FROM
FROM
pow_extended_auth_records pow
LEFT JOIN
LEFT JOIN
min_block_per_pow_account mb ON pow.account_name = mb.account_name
LEFT JOIN
LEFT JOIN
min_block_num_from_stored_table_per_account_id mb_table ON pow.account_id = mb_table.account_id
),
),
pow_extended_auth_records_filtered as materialized
(
SELECT
SELECT
account_id,
account_name,
key_kind,
......@@ -466,9 +483,9 @@ BEGIN
FROM raw_auth_records r
UNION ALL
SELECT
SELECT
*
FROM
FROM
pow_extended_auth_records_filtered
UNION ALL
......@@ -526,7 +543,7 @@ BEGIN
),
effective_key_or_account_auth_records as materialized
(
with effective_tuple_ids as materialized
with effective_tuple_ids as materialized
(
select s.account_id, s.key_kind, max(s.op_stable_id) as op_stable_id
from extended_auth_records s
......@@ -536,7 +553,7 @@ BEGIN
from extended_auth_records s1
join effective_tuple_ids e ON e.account_id = s1.account_id and e.key_kind = s1.key_kind and e.op_stable_id = s1.op_stable_id
),
--- PROCESSING OF KEY BASED AUTHORITIES ---
--- PROCESSING OF KEY BASED AUTHORITIES ---
supplement_key_dictionary as materialized
(
insert into hafd.%1$s_keyauth_k as dict (key)
......@@ -552,7 +569,7 @@ BEGIN
from effective_key_or_account_auth_records s
join supplement_key_dictionary kd on kd.key = s.key_auth
),
changed_key_authorities as materialized
changed_key_authorities as materialized
(
select distinct s.account_id as changed_account_id, s.key_kind as changed_key_kind
from effective_key_or_account_auth_records s
......@@ -597,12 +614,12 @@ BEGIN
) ds
WHERE ds.account_auth_id IS NOT NULL
),
changed_account_authorities as materialized
changed_account_authorities as materialized
(
select distinct s.account_id as changed_account_id, s.key_kind as changed_key_kind
from effective_key_or_account_auth_records s
),
delete_obsolete_account_auth_records as materialized
delete_obsolete_account_auth_records as materialized
(
DELETE FROM hafd.%1$s_accountauth_a as ae
using changed_account_authorities s
......@@ -628,9 +645,9 @@ BEGIN
SELECT
SELECT
(
select count(*) FROM
select count(*) FROM
store_account_auth_records
) as account_based_authority_entries,
(select count(*) FROM store_key_auth_records) AS key_based_authority_entries
......@@ -640,13 +657,9 @@ BEGIN
END;
$$;
$t$
, _context, __schema);
RETURN ARRAY[format('%1$s_keyauth_a', _context), format('%1$s_keyauth_k', _context), format('%1$s_accountauth_a', _context)];
, _context, __schema);
END;
$BODY$
;
$BODY$;
CREATE OR REPLACE FUNCTION hive.update_state_provider_keyauth(
_first_block hafd.blocks.num%TYPE,
......@@ -665,7 +678,7 @@ BEGIN
__context_id = hive.get_context_id( _context );
__template = $t$ SELECT hafd.%1$s_insert_into_keyauth_a(%L, %L) $t$;
__template = $t$ SELECT hive.%1$s_insert_into_keyauth_a(%L, %L) $t$;
EXECUTE format(__template, _context, _first_block, _last_block);
......@@ -693,7 +706,7 @@ BEGIN
EXECUTE format(
$$
DROP FUNCTION IF EXISTS hafd.%1$s_insert_into_keyauth_a
DROP FUNCTION IF EXISTS hive.%1$s_insert_into_keyauth_a
$$
, _context);
......
src/hive_fork_manager/state_providers/metadata.sql
+ 125
− 110
View file @ 7f9f79c3
......@@ -40,118 +40,134 @@ BEGIN
, PRIMARY KEY ( account_id )
)', __table_name);
EXECUTE format(
'CREATE OR REPLACE FUNCTION hafd.%I( _blockFrom INT, _blockTo INT )
RETURNS void
RETURN ARRAY[ __table_name ];
END;
$BODY$
;
CREATE OR REPLACE FUNCTION hive.runtimecode_provider_metadata( _context hafd.context_name )
RETURNS VOID
LANGUAGE plpgsql
VOLATILE
AS
$$
DECLARE
__state INT := 0;
BEGIN
IF COALESCE( ( SELECT _blockFrom > block_num FROM hafd.applied_hardforks WHERE hardfork_num = 21 ), FALSE ) THEN
__state := 1;
ELSIF COALESCE( ( SELECT _blockTo <= block_num FROM hafd.applied_hardforks WHERE hardfork_num = 21 ), FALSE ) THEN
__state := -1;
END IF;
WITH select_metadata AS MATERIALIZED (
SELECT
ov.body_binary,
ov.id,
ov.block_num
FROM
%s.operations_view ov
WHERE
ov.op_type_id in (
SELECT id FROM hafd.operation_types WHERE name IN
(''hive::protocol::account_create_operation'',
''hive::protocol::account_update_operation'',
''hive::protocol::create_claimed_account_operation'',
''hive::protocol::account_create_with_delegation_operation'',
''hive::protocol::account_update2_operation''))
AND ov.block_num BETWEEN _blockFrom AND _blockTo
), calculated_metadata AS MATERIALIZED
(
SELECT
(hive.get_metadata
AS
$BODY$
DECLARE
__schema TEXT;
BEGIN
SELECT hc.schema INTO __schema
FROM hafd.contexts hc
WHERE hc.name = _context;
EXECUTE format(
'CREATE OR REPLACE FUNCTION hive.%I( _blockFrom INT, _blockTo INT )
RETURNS void
LANGUAGE plpgsql
VOLATILE
AS
$$
DECLARE
__state INT := 0;
BEGIN
IF COALESCE( ( SELECT _blockFrom > block_num FROM hafd.applied_hardforks WHERE hardfork_num = 21 ), FALSE ) THEN
__state := 1;
ELSIF COALESCE( ( SELECT _blockTo <= block_num FROM hafd.applied_hardforks WHERE hardfork_num = 21 ), FALSE ) THEN
__state := -1;
END IF;
WITH select_metadata AS MATERIALIZED (
SELECT
ov.body_binary,
ov.id,
ov.block_num
FROM
%s.operations_view ov
WHERE
ov.op_type_id in (
SELECT id FROM hafd.operation_types WHERE name IN
(''hive::protocol::account_create_operation'',
''hive::protocol::account_update_operation'',
''hive::protocol::create_claimed_account_operation'',
''hive::protocol::account_create_with_delegation_operation'',
''hive::protocol::account_update2_operation''))
AND ov.block_num BETWEEN _blockFrom AND _blockTo
), calculated_metadata AS MATERIALIZED
(
SELECT
(hive.get_metadata
(
sm.body_binary,
CASE __state
WHEN 1 THEN TRUE
WHEN 0 THEN COALESCE( ( SELECT block_num < sm.block_num FROM hafd.applied_hardforks WHERE hardfork_num = 21 ), FALSE )
WHEN -1 THEN FALSE
END
)).*,
sm.id
FROM select_metadata sm
),
prepare_accounts AS MATERIALIZED
(
sm.body_binary,
CASE __state
WHEN 1 THEN TRUE
WHEN 0 THEN COALESCE( ( SELECT block_num < sm.block_num FROM hafd.applied_hardforks WHERE hardfork_num = 21 ), FALSE )
WHEN -1 THEN FALSE
END
)).*,
sm.id
FROM select_metadata sm
),
prepare_accounts AS MATERIALIZED
(
SELECT
metadata.account_name
FROM calculated_metadata as metadata
GROUP BY metadata.account_name
),
select_json_metadata AS MATERIALIZED
(
SELECT
DISTINCT ON (metadata.account_name) metadata.account_name,
metadata.json_metadata
FROM calculated_metadata as metadata
WHERE metadata.json_metadata != ''''
ORDER BY
metadata.account_name,
metadata.id DESC
),
select_posting_json_metadata AS MATERIALIZED
(
SELECT
DISTINCT ON (metadata.account_name) metadata.account_name,
metadata.posting_json_metadata
FROM calculated_metadata as metadata
WHERE metadata.posting_json_metadata != ''''
ORDER BY
metadata.account_name,
metadata.id DESC
)
INSERT INTO
hafd.%s_metadata(account_id, json_metadata, posting_json_metadata)
SELECT
av.id,
COALESCE(sjm.json_metadata, ''''),
COALESCE(pjm.posting_json_metadata, '''')
FROM prepare_accounts pa
LEFT JOIN select_json_metadata sjm ON sjm.account_name = pa.account_name
LEFT JOIN select_posting_json_metadata pjm ON pjm.account_name = pa.account_name
JOIN hive.accounts_view av ON av.name = pa.account_name
ON CONFLICT (account_id) DO UPDATE
SET
json_metadata =
(
CASE EXCLUDED.json_metadata
WHEN '''' THEN hafd.%s_metadata.json_metadata
ELSE EXCLUDED.json_metadata
END
),
posting_json_metadata =
(
CASE EXCLUDED.posting_json_metadata
WHEN '''' THEN hafd.%s_metadata.posting_json_metadata
ELSE EXCLUDED.posting_json_metadata
END
);
END
$$;'
, hive.get_metadata_update_function_name( _context ), __schema, _context, _context, _context
SELECT
metadata.account_name
FROM calculated_metadata as metadata
GROUP BY metadata.account_name
),
select_json_metadata AS MATERIALIZED
(
SELECT
DISTINCT ON (metadata.account_name) metadata.account_name,
metadata.json_metadata
FROM calculated_metadata as metadata
WHERE metadata.json_metadata != ''''
ORDER BY
metadata.account_name,
metadata.id DESC
),
select_posting_json_metadata AS MATERIALIZED
(
SELECT
DISTINCT ON (metadata.account_name) metadata.account_name,
metadata.posting_json_metadata
FROM calculated_metadata as metadata
WHERE metadata.posting_json_metadata != ''''
ORDER BY
metadata.account_name,
metadata.id DESC
)
INSERT INTO
hafd.%s_metadata(account_id, json_metadata, posting_json_metadata)
SELECT
av.id,
COALESCE(sjm.json_metadata, ''''),
COALESCE(pjm.posting_json_metadata, '''')
FROM prepare_accounts pa
LEFT JOIN select_json_metadata sjm ON sjm.account_name = pa.account_name
LEFT JOIN select_posting_json_metadata pjm ON pjm.account_name = pa.account_name
JOIN hive.accounts_view av ON av.name = pa.account_name
ON CONFLICT (account_id) DO UPDATE
SET
json_metadata =
(
CASE EXCLUDED.json_metadata
WHEN '''' THEN hafd.%s_metadata.json_metadata
ELSE EXCLUDED.json_metadata
END
),
posting_json_metadata =
(
CASE EXCLUDED.posting_json_metadata
WHEN '''' THEN hafd.%s_metadata.posting_json_metadata
ELSE EXCLUDED.posting_json_metadata
END
);
END
$$;'
, hive.get_metadata_update_function_name( _context ), __schema, _context, _context, _context
);
RETURN ARRAY[ __table_name ];
END;
$BODY$
;
$BODY$;
CREATE OR REPLACE FUNCTION hive.update_state_provider_metadata(
_first_block hafd.blocks.num%TYPE,
......@@ -165,7 +181,6 @@ AS
$BODY$
DECLARE
__context_id hafd.contexts.id%TYPE;
__table_name TEXT := _context || '_metadata';
BEGIN
__context_id = hive.get_context_id( _context );
......@@ -173,7 +188,7 @@ BEGIN
RAISE EXCEPTION 'No context with name %', _context;
END IF;
EXECUTE format(
'SELECT hafd.%I(%s, %s);'
'SELECT hive.%I(%s, %s);'
, hive.get_metadata_update_function_name( _context )
, _first_block
, _last_block
......@@ -200,7 +215,7 @@ BEGIN
EXECUTE format( 'DROP TABLE hafd.%I', __table_name );
EXECUTE format(
'DROP FUNCTION IF EXISTS hafd.%I'
'DROP FUNCTION IF EXISTS hive.%I'
, hive.get_metadata_update_function_name( _context )
);
END;
......
src/hive_fork_manager/state_providers/state_provider.template
+ 19
− 0
View file @ 7f9f79c3
......@@ -15,6 +15,25 @@ END;
$BODY$
;
CREATE OR REPLACE FUNCTION hive.runtimecode_provider_<provider_name>( _context hafd.context_name )
RETURNS TEXT[]
LANGUAGE plpgsql
VOLATILE
AS
$BODY$
DECLARE
BEGIN
-- HERE YOU NEED TO CREATE FUNCTIONS WHICH ARE USED BY PROVIDER
-- the function is executed at provider registration and then during each HAF update
-- the functions have to be create with CREATE OR REPLACE
-- it should be used only for function which are created only to fill state provider table
-- the functions names must conform pattern:
-- hive.<context_name>_<function_name>
-- it must be created in schema hive to be removed during hfm extension update
END;
$BODY$
;
CREATE OR REPLACE FUNCTION hive.update_state_provider_<provider_name>( _first_block hafd.blocks.num%TYPE, _last_block hafd.blocks.num%TYPE, _context hafd.context_name )
RETURNS void
LANGUAGE plpgsql
......
src/hive_fork_manager/state_providers/update_providers.sql 0 → 100644
+ 27
− 0
View file @ 7f9f79c3
CREATE OR REPLACE FUNCTION hive.state_provider_update_runtime( _provider hafd.state_providers, _context hafd.context_name)
RETURNS void
LANGUAGE plpgsql
AS
$BODY$
BEGIN
EXECUTE format('hive.runtimecode_provider_%s(%L)', _provider, _context );
END;
$BODY$
;
CREATE OR REPLACE FUNCTION hive.state_providers_update_runtime( )
RETURNS void
LANGUAGE plpgsql
AS
$BODY$
BEGIN
PERFORM
hive.state_provider_update_runtime(spr.state_provider, hc.name )
FROM hafd.state_providers_registered spr
JOIN hafd.contexts hc ON hc.id = spr.context_id;
END;
$BODY$
;
SELECT hive.state_providers_update_runtime();
src/hive_fork_manager/tools.sql
+ 1
− 0
View file @ 7f9f79c3
......@@ -19,3 +19,4 @@ BEGIN
END;
$BODY$
;
tests/integration/functional/hive_fork_manager/state_providers/import_state_provider.sql
+ 36
− 1
View file @ 7f9f79c3
CREATE OR REPLACE FUNCTION hafd.create_function_a()
RETURNS VOID
LANGUAGE plpgsql
AS
$BODY$
BEGIN
EXECUTE 'CREATE OR REPLACE FUNCTION hive.a()
RETURNS VOID
LANGUAGE plpgsql
AS
$BODY2$
BEGIN
END;
$BODY2$;
';
END;
$BODY$;
CREATE OR REPLACE PROCEDURE haf_admin_test_given()
LANGUAGE 'plpgsql'
AS
......@@ -51,8 +69,17 @@ BEGIN
$$
;';
---------------------------END OF TEST PROVIDER -------------------------------------------------------------------
EXECUTE 'CREATE OR REPLACE FUNCTION hive.runtimecode_provider_tests(_context hafd.context_name)
RETURNS VOID
LANGUAGE plpgsql
AS
$$
BEGIN
PERFORM hafd.create_function_a();
END;
$$;';
---------------------------END OF TEST PROVIDER -------------------------------------------------------------------
PERFORM hive.app_state_provider_import( 'ACCOUNTS', 'context' );
PERFORM hive.app_state_provider_import( 'TESTS', 'context' );
......@@ -72,6 +99,14 @@ BEGIN
ASSERT ( SELECT COUNT(*) FROM hafd.registered_tables WHERE origin_table_schema = 'hafd' AND origin_table_name = 'context_accounts' AND context_id = 1 ) = 1, 'State provider table is not registered';
ASSERT ( SELECT COUNT(*) FROM hafd.registered_tables WHERE origin_table_schema = 'hafd' AND origin_table_name = 'context_tests1' AND context_id = 1 ) = 1, 'State provider tests1 is not registered';
ASSERT ( SELECT COUNT(*) FROM hafd.registered_tables WHERE origin_table_schema = 'hafd' AND origin_table_name = 'context_tests2' AND context_id = 1 ) = 1, 'State provider tests2 is not registered';
ASSERT EXISTS (
SELECT 1
FROM pg_proc
JOIN pg_namespace ON pg_proc.pronamespace = pg_namespace.oid
WHERE pg_proc.proname = 'a' AND pg_namespace.nspname = 'hive'
), 'Function hive.a does not exists';
END;
$BODY$
;
......
tests/integration/functional/hive_fork_manager/state_providers/update_state_provider.sql
+ 9
− 0
View file @ 7f9f79c3
......@@ -29,6 +29,15 @@ BEGIN
END;
$$;';
EXECUTE 'CREATE OR REPLACE FUNCTION hive.runtimecode_provider_tests(_context hafd.context_name)
RETURNS VOID
LANGUAGE plpgsql
AS
$$
BEGIN
END;
$$;';
INSERT INTO hafd.operation_types
VALUES
( 1, 'hive::protocol::account_created_operation', TRUE )
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment