From 881ac31ec47bf76e354faee9c79ea51ea90542c7 Mon Sep 17 00:00:00 2001 From: Dan Notestein <dan@syncad.com> Date: Mon, 9 Dec 2024 18:52:56 -0500 Subject: [PATCH 01/33] Add functions to allow apps to request/wait for full vacuums --- src/hive_fork_manager/hived_api_impl.sql | 81 +++++++++++++++++++ .../hived_api_impl_indexes.sql | 9 +++ src/sql_serializer/indexes_controler.cpp | 26 ++++++ 3 files changed, 116 insertions(+) diff --git a/src/hive_fork_manager/hived_api_impl.sql b/src/hive_fork_manager/hived_api_impl.sql index a5c29e4e0..cf85c207a 100644 --- a/src/hive_fork_manager/hived_api_impl.sql +++ b/src/hive_fork_manager/hived_api_impl.sql @@ -785,3 +785,84 @@ BEGIN END; $BODY$ ; + +CREATE OR REPLACE FUNCTION hive.request_table_vacuum( + _table_name TEXT, + _min_interval INTERVAL DEFAULT NULL +) +RETURNS void +LANGUAGE plpgsql +AS +$BODY$ +BEGIN + IF _min_interval IS NOT NULL THEN + -- Check if the table has been vacuumed recently + IF EXISTS ( + SELECT 1 + FROM hafd.vacuum_requests + WHERE table_name = _table_name + AND last_vacuumed_time > NOW() - _min_interval + ) THEN + RAISE NOTICE 'Vacuum request for table % ignored due to recent vacuum.', _table_name; + RETURN; + END IF; + END IF; + + -- Insert or update the vacuum request + INSERT INTO hafd.vacuum_requests (table_name, status) + VALUES (_table_name, 'requested') + ON CONFLICT (table_name) DO UPDATE + SET status = 'requested'; + + RAISE NOTICE 'Vacuum request for table % submitted.', _table_name; +END; +$BODY$ +; + +CREATE OR REPLACE FUNCTION hive.wait_for_table_vacuum( + _table_name TEXT +) +RETURNS void +LANGUAGE plpgsql +AS +$BODY$ +DECLARE + __start_time TIMESTAMP; + __end_time TIMESTAMP; + __duration INTERVAL; +BEGIN + RAISE NOTICE 'Waiting for vacuum to complete for table %', _table_name; + __start_time := clock_timestamp(); + + LOOP + EXIT WHEN NOT EXISTS ( + SELECT 1 + FROM hafd.vacuum_requests + WHERE table_name = _table_name + AND status <> 'vacuumed' + ); + END LOOP; + + __end_time := clock_timestamp(); + __duration := __end_time - __start_time; + RAISE NOTICE 'Vacuum completed for table % in % seconds', _table_name, EXTRACT(EPOCH FROM __duration); +END; +$BODY$ +; + +CREATE OR REPLACE FUNCTION hive.test_vacuum_functions() +RETURNS void +LANGUAGE plpgsql +AS +$BODY$ +BEGIN + -- Request a vacuum for the table hafbe_app_keyauth_a + PERFORM hive.request_table_vacuum('hafbe_app_keyauth_a', '1 hour'::INTERVAL); + + -- Wait for the vacuum to complete + PERFORM hive.wait_for_table_vacuum('hafbe_app_keyauth_a'); + + RAISE NOTICE 'Test for vacuum functions on table hafbe_app_keyauth_a completed successfully.'; +END; +$BODY$ +; diff --git a/src/hive_fork_manager/hived_api_impl_indexes.sql b/src/hive_fork_manager/hived_api_impl_indexes.sql index bcafdc1d2..f35ec5ac4 100644 --- a/src/hive_fork_manager/hived_api_impl_indexes.sql +++ b/src/hive_fork_manager/hived_api_impl_indexes.sql @@ -12,3 +12,12 @@ CREATE TABLE IF NOT EXISTS hafd.indexes_constraints ( CONSTRAINT pk_hive_indexes_constraints UNIQUE( table_name, index_constraint_name ) ); SELECT pg_catalog.pg_extension_config_dump('hafd.indexes_constraints', ''); + +-- Only one vacuum request per table, and this is only for 'vacuum full analyze' requests +CREATE TYPE hafd.vacuum_status AS ENUM ('requested', 'vacuuming', 'vacuumed', 'failed'); +CREATE TABLE IF NOT EXISTS hafd.vacuum_requests ( + table_name text NOT NULL, + hafd.vacuum_status, + last_vacuumed_time timestamp, + CONSTRAINT pk_hive_vacuum_requests UNIQUE( table_name) +); diff --git a/src/sql_serializer/indexes_controler.cpp b/src/sql_serializer/indexes_controler.cpp index 76d765969..54bc9d1ae 100644 --- a/src/sql_serializer/indexes_controler.cpp +++ b/src/sql_serializer/indexes_controler.cpp @@ -178,6 +178,32 @@ void indexes_controler::poll_and_create_indexes() { while (!theApp.is_interrupt_request()) { dlog("Polling for tables with missing indexes..."); + + // Check for requested table vacuums + queries_commit_data_processor vacuum_requests_checker( + _db_url, + "Check for vacuum requests", + "index_ctrl", + [](const data_processor::data_chunk_ptr&, transaction_controllers::transaction& tx) -> data_processor::data_processing_status { + pqxx::result data = tx.exec( + "SELECT table_name FROM hafd.vacuum_requests WHERE status = 'requested';" + ); + for (const auto& record : data) { + std::string table_name = record["table_name"].as<std::string>(); + std::string vacuum_command = "VACUUM FULL ANALYZE " + table_name; + ilog("Performing vacuum: ${vacuum_command}", ("vacuum_command", vacuum_command)); + tx.exec(vacuum_command); + tx.exec("UPDATE hafd.vacuum_requests SET status = 'vacuumed', last_vacuumed_time = NOW() WHERE table_name = '" + table_name + "';"); + } + return data_processor::data_processing_status(); + }, + nullptr, + theApp + ); + + vacuum_requests_checker.trigger(data_processor::data_chunk_ptr(), 0); + vacuum_requests_checker.join(); + // Check for tables with missing indexes that are not currently being created queries_commit_data_processor missing_indexes_checker( _db_url, -- GitLab From 4c75411d5efe185de68e1efb21a306ff1e7e7dd9 Mon Sep 17 00:00:00 2001 From: Dan Notestein <dan@syncad.com> Date: Tue, 10 Dec 2024 01:42:13 +0000 Subject: [PATCH 02/33] add extension config dump call --- src/hive_fork_manager/hived_api_impl_indexes.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hive_fork_manager/hived_api_impl_indexes.sql b/src/hive_fork_manager/hived_api_impl_indexes.sql index f35ec5ac4..d48892cfb 100644 --- a/src/hive_fork_manager/hived_api_impl_indexes.sql +++ b/src/hive_fork_manager/hived_api_impl_indexes.sql @@ -21,3 +21,4 @@ CREATE TABLE IF NOT EXISTS hafd.vacuum_requests ( last_vacuumed_time timestamp, CONSTRAINT pk_hive_vacuum_requests UNIQUE( table_name) ); +SELECT pg_catalog.pg_extension_config_dump('hafd.vacuum_status', ''); -- GitLab From 3e3408675dd32d16c175a911f690c6de9cc75d1f Mon Sep 17 00:00:00 2001 From: Dan Notestein <dan@syncad.com> Date: Mon, 9 Dec 2024 23:05:15 -0500 Subject: [PATCH 03/33] fix typo --- src/hive_fork_manager/hived_api_impl_indexes.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hive_fork_manager/hived_api_impl_indexes.sql b/src/hive_fork_manager/hived_api_impl_indexes.sql index d48892cfb..cd63f7315 100644 --- a/src/hive_fork_manager/hived_api_impl_indexes.sql +++ b/src/hive_fork_manager/hived_api_impl_indexes.sql @@ -21,4 +21,4 @@ CREATE TABLE IF NOT EXISTS hafd.vacuum_requests ( last_vacuumed_time timestamp, CONSTRAINT pk_hive_vacuum_requests UNIQUE( table_name) ); -SELECT pg_catalog.pg_extension_config_dump('hafd.vacuum_status', ''); +SELECT pg_catalog.pg_extension_config_dump('hafd.vacuum_requests', ''); -- GitLab From cc024fd0500512ff37228ed9ce519bc90d2ba742 Mon Sep 17 00:00:00 2001 From: Dan Notestein <dan@syncad.com> Date: Tue, 10 Dec 2024 01:43:00 -0500 Subject: [PATCH 04/33] Grant permissions on vacuum_requests to hive_applications_group --- src/hive_fork_manager/authorization.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hive_fork_manager/authorization.sql b/src/hive_fork_manager/authorization.sql index d8b928816..3f9cc9805 100644 --- a/src/hive_fork_manager/authorization.sql +++ b/src/hive_fork_manager/authorization.sql @@ -68,6 +68,7 @@ GRANT ALL ON hafd.contexts_attachment TO hive_applications_group; GRANT ALL ON hafd.registered_tables TO hive_applications_group; GRANT ALL ON hafd.triggers TO hive_applications_group; GRANT ALL ON hafd.state_providers_registered TO hive_applications_group; +GRANT ALL ON hafd.vacuum_requests TO hive_applications_group; -- protect an application rows aginst other applications REVOKE UPDATE( is_forking, owner ) ON hafd.contexts FROM GROUP hive_applications_group; -- GitLab From 064f6d66daacbc3972af554fe6f6681b57412c57 Mon Sep 17 00:00:00 2001 From: Marcin Ickiewicz <mickiewicz@syncad.com> Date: Tue, 10 Dec 2024 14:01:35 +0100 Subject: [PATCH 05/33] add test for vacuum checker --- .../hive_fork_manager/CMakeLists.txt | 1 + .../app_api/vacuum_request.sql | 37 +++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 tests/integration/functional/hive_fork_manager/app_api/vacuum_request.sql diff --git a/tests/integration/functional/hive_fork_manager/CMakeLists.txt b/tests/integration/functional/hive_fork_manager/CMakeLists.txt index 83e8af5bc..28010b436 100644 --- a/tests/integration/functional/hive_fork_manager/CMakeLists.txt +++ b/tests/integration/functional/hive_fork_manager/CMakeLists.txt @@ -277,6 +277,7 @@ ADD_TEST( NAME test_update_script ADD_SQL_FUNCTIONAL_TEST( app_api/app_context_create_and_hole_in_events.sql ) ADD_SQL_FUNCTIONAL_TEST( app_api/operation_id.sql ) + ADD_SQL_FUNCTIONAL_TEST(app_api/vacuum_request.sql) ADD_SQL_FUNCTIONAL_TEST( application_loop/validate_stages.sql ) ADD_SQL_FUNCTIONAL_TEST( application_loop/current_stage.sql ) diff --git a/tests/integration/functional/hive_fork_manager/app_api/vacuum_request.sql b/tests/integration/functional/hive_fork_manager/app_api/vacuum_request.sql new file mode 100644 index 000000000..3a89888c4 --- /dev/null +++ b/tests/integration/functional/hive_fork_manager/app_api/vacuum_request.sql @@ -0,0 +1,37 @@ +CREATE OR REPLACE PROCEDURE alice_test_given() + LANGUAGE 'plpgsql' +AS +$BODY$ +BEGIN + CREATE SCHEMA A; + PERFORM hive.app_create_context( _name => 'context', _schema => 'a' ); + CREATE TABLE A.table1(id INTEGER ) INHERITS( a.context ); +END; +$BODY$ +; + +CREATE OR REPLACE PROCEDURE alice_impersonal_test_when() + LANGUAGE 'plpgsql' +AS +$BODY$ +BEGIN + PERFORM hive.request_table_vacuum('a.table1'); +END; +$BODY$ +; + +CREATE OR REPLACE PROCEDURE alice_test_then() + LANGUAGE 'plpgsql' +AS +$BODY$ +BEGIN + ASSERT + (SELECT COUNT(*) FROM hafd.vacuum_requests WHERE table_name = 'a.table1' ) = 1 + , 'a.table1 wrong number of requests'; + + ASSERT + (SELECT status FROM hafd.vacuum_requests WHERE table_name = 'a.table1' ) = 'requested' + , 'a.table1 status != requested'; +END; +$BODY$ +; -- GitLab From 91ae15f7f791251246c5cff5ac7d1944ece95664 Mon Sep 17 00:00:00 2001 From: Marcin Ickiewicz <mickiewicz@syncad.com> Date: Tue, 10 Dec 2024 16:29:07 +0100 Subject: [PATCH 06/33] vacuum executed in a separated connection --- hive | 2 +- src/sql_serializer/indexes_controler.cpp | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/hive b/hive index 2afff0e42..0037935d3 160000 --- a/hive +++ b/hive @@ -1 +1 @@ -Subproject commit 2afff0e42851ad0b58428abd690af9e648bf4745 +Subproject commit 0037935d34dfc75f7730fd77ddb17e7d81f0eb0a diff --git a/src/sql_serializer/indexes_controler.cpp b/src/sql_serializer/indexes_controler.cpp index 54bc9d1ae..25c93843b 100644 --- a/src/sql_serializer/indexes_controler.cpp +++ b/src/sql_serializer/indexes_controler.cpp @@ -184,15 +184,19 @@ void indexes_controler::poll_and_create_indexes() { _db_url, "Check for vacuum requests", "index_ctrl", - [](const data_processor::data_chunk_ptr&, transaction_controllers::transaction& tx) -> data_processor::data_processing_status { + [this](const data_processor::data_chunk_ptr&, transaction_controllers::transaction& tx) -> data_processor::data_processing_status { pqxx::result data = tx.exec( "SELECT table_name FROM hafd.vacuum_requests WHERE status = 'requested';" ); + // workaround, open separate connections for non-transactional start vacuum + auto vacuum_connection = std::make_unique<pqxx::connection>(_db_url); + pqxx::nontransaction vacuum_txn(*vacuum_connection); for (const auto& record : data) { std::string table_name = record["table_name"].as<std::string>(); std::string vacuum_command = "VACUUM FULL ANALYZE " + table_name; ilog("Performing vacuum: ${vacuum_command}", ("vacuum_command", vacuum_command)); - tx.exec(vacuum_command); + vacuum_txn.exec(vacuum_command); + tx.exec("UPDATE hafd.vacuum_requests SET status = 'vacuumed', last_vacuumed_time = NOW() WHERE table_name = '" + table_name + "';"); } return data_processor::data_processing_status(); -- GitLab From ec5b9a585f9f3533edf6626634a5c288e478bf2d Mon Sep 17 00:00:00 2001 From: Dan Notestein <dan@syncad.com> Date: Tue, 10 Dec 2024 17:10:59 -0500 Subject: [PATCH 07/33] udpate hive submodule --- hive | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive b/hive index 0037935d3..2afff0e42 160000 --- a/hive +++ b/hive @@ -1 +1 @@ -Subproject commit 0037935d34dfc75f7730fd77ddb17e7d81f0eb0a +Subproject commit 2afff0e42851ad0b58428abd690af9e648bf4745 -- GitLab From 3889cd2984b83e509b79ae1626ac58617d490636 Mon Sep 17 00:00:00 2001 From: Dan Notestein <dan@syncad.com> Date: Tue, 10 Dec 2024 17:12:47 -0500 Subject: [PATCH 08/33] temporary logging increase --- src/sql_serializer/indexes_controler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sql_serializer/indexes_controler.cpp b/src/sql_serializer/indexes_controler.cpp index 25c93843b..8981a9674 100644 --- a/src/sql_serializer/indexes_controler.cpp +++ b/src/sql_serializer/indexes_controler.cpp @@ -177,7 +177,7 @@ void indexes_controler::poll_and_create_indexes() { std::mutex mtx; // Protects threads_to_delete while (!theApp.is_interrupt_request()) { - dlog("Polling for tables with missing indexes..."); + ilog("Polling for tables with missing indexes..."); // Check for requested table vacuums queries_commit_data_processor vacuum_requests_checker( -- GitLab From 22f8360ea063acb85daa4cd7e070efcdb94283bf Mon Sep 17 00:00:00 2001 From: Dan Notestein <dan@syncad.com> Date: Mon, 9 Dec 2024 19:45:34 -0500 Subject: [PATCH 09/33] Fix SQL typo --- src/hive_fork_manager/hived_api_impl_indexes.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hive_fork_manager/hived_api_impl_indexes.sql b/src/hive_fork_manager/hived_api_impl_indexes.sql index cd63f7315..592fc8d34 100644 --- a/src/hive_fork_manager/hived_api_impl_indexes.sql +++ b/src/hive_fork_manager/hived_api_impl_indexes.sql @@ -14,10 +14,10 @@ CREATE TABLE IF NOT EXISTS hafd.indexes_constraints ( SELECT pg_catalog.pg_extension_config_dump('hafd.indexes_constraints', ''); -- Only one vacuum request per table, and this is only for 'vacuum full analyze' requests -CREATE TYPE hafd.vacuum_status AS ENUM ('requested', 'vacuuming', 'vacuumed', 'failed'); +CREATE TYPE hafd.vacuum_status AS ENUM ('requested', 'vacuumed'); CREATE TABLE IF NOT EXISTS hafd.vacuum_requests ( table_name text NOT NULL, - hafd.vacuum_status, + status hafd.vacuum_status NOT NULL DEFAULT 'requested', last_vacuumed_time timestamp, CONSTRAINT pk_hive_vacuum_requests UNIQUE( table_name) ); -- GitLab From 08a96de03b56e21733db447b95ac265d61c57130 Mon Sep 17 00:00:00 2001 From: Dan Notestein <dan@syncad.com> Date: Tue, 10 Dec 2024 19:39:43 -0500 Subject: [PATCH 10/33] add app_ in front of new functions, needs discussion --- src/hive_fork_manager/hived_api_impl.sql | 8 ++++---- .../hive_fork_manager/app_api/vacuum_request.sql | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/hive_fork_manager/hived_api_impl.sql b/src/hive_fork_manager/hived_api_impl.sql index cf85c207a..41433ecc7 100644 --- a/src/hive_fork_manager/hived_api_impl.sql +++ b/src/hive_fork_manager/hived_api_impl.sql @@ -786,7 +786,7 @@ END; $BODY$ ; -CREATE OR REPLACE FUNCTION hive.request_table_vacuum( +CREATE OR REPLACE FUNCTION hive.app_request_table_vacuum( _table_name TEXT, _min_interval INTERVAL DEFAULT NULL ) @@ -819,7 +819,7 @@ END; $BODY$ ; -CREATE OR REPLACE FUNCTION hive.wait_for_table_vacuum( +CREATE OR REPLACE FUNCTION hive.app_wait_for_table_vacuum( _table_name TEXT ) RETURNS void @@ -857,10 +857,10 @@ AS $BODY$ BEGIN -- Request a vacuum for the table hafbe_app_keyauth_a - PERFORM hive.request_table_vacuum('hafbe_app_keyauth_a', '1 hour'::INTERVAL); + PERFORM hive.app_request_table_vacuum('hafbe_app_keyauth_a', '1 hour'::INTERVAL); -- Wait for the vacuum to complete - PERFORM hive.wait_for_table_vacuum('hafbe_app_keyauth_a'); + PERFORM hive.app_wait_for_table_vacuum('hafbe_app_keyauth_a'); RAISE NOTICE 'Test for vacuum functions on table hafbe_app_keyauth_a completed successfully.'; END; diff --git a/tests/integration/functional/hive_fork_manager/app_api/vacuum_request.sql b/tests/integration/functional/hive_fork_manager/app_api/vacuum_request.sql index 3a89888c4..00200ab5f 100644 --- a/tests/integration/functional/hive_fork_manager/app_api/vacuum_request.sql +++ b/tests/integration/functional/hive_fork_manager/app_api/vacuum_request.sql @@ -15,7 +15,7 @@ CREATE OR REPLACE PROCEDURE alice_impersonal_test_when() AS $BODY$ BEGIN - PERFORM hive.request_table_vacuum('a.table1'); + PERFORM hive.app_request_table_vacuum('a.table1'); END; $BODY$ ; -- GitLab From fbeba66c2849de6160d41f3429b5035dbc70be65 Mon Sep 17 00:00:00 2001 From: Dan Notestein <dan@syncad.com> Date: Tue, 10 Dec 2024 21:57:43 -0500 Subject: [PATCH 11/33] add sleeps and log when we sleep --- src/hive_fork_manager/hived_api_impl.sql | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/hive_fork_manager/hived_api_impl.sql b/src/hive_fork_manager/hived_api_impl.sql index 41433ecc7..e392940a5 100644 --- a/src/hive_fork_manager/hived_api_impl.sql +++ b/src/hive_fork_manager/hived_api_impl.sql @@ -698,7 +698,8 @@ BEGIN SELECT 1 FROM hafd.indexes_constraints WHERE contexts @> ARRAY[(SELECT id FROM hafd.contexts WHERE name = _app_context)] AND status <> 'created' - ); + RAISE NOTICE 'Sleeping for 10 seconds waiting for indexes to be created'; + PERFORM pg_sleep(10); ); END LOOP; @@ -839,8 +840,10 @@ BEGIN SELECT 1 FROM hafd.vacuum_requests WHERE table_name = _table_name - AND status <> 'vacuumed' + AND status <> 'vacuumed' ); + RAISE NOTICE 'Sleeping for 10 seconds waiting for vacuum to be done'; + PERFORM pg_sleep(10); END LOOP; __end_time := clock_timestamp(); -- GitLab From 88e3923d3b97ff1d9c3bd4363b0d9e3b2557e342 Mon Sep 17 00:00:00 2001 From: Dan Notestein <dan@syncad.com> Date: Tue, 10 Dec 2024 22:01:06 -0500 Subject: [PATCH 12/33] Add logging for vacuum process in indexes_controler --- src/sql_serializer/indexes_controler.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sql_serializer/indexes_controler.cpp b/src/sql_serializer/indexes_controler.cpp index 8981a9674..4e7ed34b0 100644 --- a/src/sql_serializer/indexes_controler.cpp +++ b/src/sql_serializer/indexes_controler.cpp @@ -189,6 +189,7 @@ void indexes_controler::poll_and_create_indexes() { "SELECT table_name FROM hafd.vacuum_requests WHERE status = 'requested';" ); // workaround, open separate connections for non-transactional start vacuum + ilog("Checking for tables to vacuum..."); auto vacuum_connection = std::make_unique<pqxx::connection>(_db_url); pqxx::nontransaction vacuum_txn(*vacuum_connection); for (const auto& record : data) { @@ -196,7 +197,7 @@ void indexes_controler::poll_and_create_indexes() { std::string vacuum_command = "VACUUM FULL ANALYZE " + table_name; ilog("Performing vacuum: ${vacuum_command}", ("vacuum_command", vacuum_command)); vacuum_txn.exec(vacuum_command); - + ilog("Vacuumed table: ${table_name}", ("table_name", table_name)); tx.exec("UPDATE hafd.vacuum_requests SET status = 'vacuumed', last_vacuumed_time = NOW() WHERE table_name = '" + table_name + "';"); } return data_processor::data_processing_status(); -- GitLab From 5021f683d9d136dceea0b399b7fad7940b6903e1 Mon Sep 17 00:00:00 2001 From: Dan Notestein <dan@syncad.com> Date: Tue, 10 Dec 2024 22:17:22 -0500 Subject: [PATCH 13/33] fix sleep typo --- src/hive_fork_manager/hived_api_impl.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/hive_fork_manager/hived_api_impl.sql b/src/hive_fork_manager/hived_api_impl.sql index e392940a5..764b0eba4 100644 --- a/src/hive_fork_manager/hived_api_impl.sql +++ b/src/hive_fork_manager/hived_api_impl.sql @@ -699,7 +699,8 @@ BEGIN FROM hafd.indexes_constraints WHERE contexts @> ARRAY[(SELECT id FROM hafd.contexts WHERE name = _app_context)] AND status <> 'created' RAISE NOTICE 'Sleeping for 10 seconds waiting for indexes to be created'; - PERFORM pg_sleep(10); ); + ); + PERFORM pg_sleep(10); END LOOP; -- GitLab From b11c6ccb30691a3b7f307c05e708ea8108ebd190 Mon Sep 17 00:00:00 2001 From: Dan Notestein <dan@syncad.com> Date: Tue, 10 Dec 2024 22:47:43 -0500 Subject: [PATCH 14/33] forgot to commit this part --- src/hive_fork_manager/hived_api_impl.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hive_fork_manager/hived_api_impl.sql b/src/hive_fork_manager/hived_api_impl.sql index 764b0eba4..5c2e90b1d 100644 --- a/src/hive_fork_manager/hived_api_impl.sql +++ b/src/hive_fork_manager/hived_api_impl.sql @@ -698,7 +698,7 @@ BEGIN SELECT 1 FROM hafd.indexes_constraints WHERE contexts @> ARRAY[(SELECT id FROM hafd.contexts WHERE name = _app_context)] AND status <> 'created' - RAISE NOTICE 'Sleeping for 10 seconds waiting for indexes to be created'; + RAISE NOTICE 'Sleeping for 10 seconds waiting for indexes to be created' ); PERFORM pg_sleep(10); END LOOP; -- GitLab From 56de0453daba89f1c7746acaed27c31538672836 Mon Sep 17 00:00:00 2001 From: Dan Notestein <dan@syncad.com> Date: Tue, 10 Dec 2024 23:01:04 -0500 Subject: [PATCH 15/33] fix sleep typo --- src/hive_fork_manager/hived_api_impl.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hive_fork_manager/hived_api_impl.sql b/src/hive_fork_manager/hived_api_impl.sql index 5c2e90b1d..96c5acba0 100644 --- a/src/hive_fork_manager/hived_api_impl.sql +++ b/src/hive_fork_manager/hived_api_impl.sql @@ -698,8 +698,8 @@ BEGIN SELECT 1 FROM hafd.indexes_constraints WHERE contexts @> ARRAY[(SELECT id FROM hafd.contexts WHERE name = _app_context)] AND status <> 'created' - RAISE NOTICE 'Sleeping for 10 seconds waiting for indexes to be created' ); + RAISE NOTICE 'Sleeping for 10 seconds waiting for indexes to be created'; PERFORM pg_sleep(10); END LOOP; -- GitLab From 5b2a1a73530b1ef21e275a75f2101c64c0a7ebbb Mon Sep 17 00:00:00 2001 From: Dan Notestein <dan@syncad.com> Date: Wed, 11 Dec 2024 00:01:34 -0500 Subject: [PATCH 16/33] add logging for thread that runs vacuums --- src/sql_serializer/indexes_controler.cpp | 37 +++++++++++++++--------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/src/sql_serializer/indexes_controler.cpp b/src/sql_serializer/indexes_controler.cpp index 4e7ed34b0..370c2d685 100644 --- a/src/sql_serializer/indexes_controler.cpp +++ b/src/sql_serializer/indexes_controler.cpp @@ -185,21 +185,32 @@ void indexes_controler::poll_and_create_indexes() { "Check for vacuum requests", "index_ctrl", [this](const data_processor::data_chunk_ptr&, transaction_controllers::transaction& tx) -> data_processor::data_processing_status { - pqxx::result data = tx.exec( - "SELECT table_name FROM hafd.vacuum_requests WHERE status = 'requested';" - ); - // workaround, open separate connections for non-transactional start vacuum ilog("Checking for tables to vacuum..."); - auto vacuum_connection = std::make_unique<pqxx::connection>(_db_url); - pqxx::nontransaction vacuum_txn(*vacuum_connection); - for (const auto& record : data) { - std::string table_name = record["table_name"].as<std::string>(); - std::string vacuum_command = "VACUUM FULL ANALYZE " + table_name; - ilog("Performing vacuum: ${vacuum_command}", ("vacuum_command", vacuum_command)); - vacuum_txn.exec(vacuum_command); - ilog("Vacuumed table: ${table_name}", ("table_name", table_name)); - tx.exec("UPDATE hafd.vacuum_requests SET status = 'vacuumed', last_vacuumed_time = NOW() WHERE table_name = '" + table_name + "';"); + pqxx::result data; + try { data = tx.exec("SELECT table_name FROM hafd.vacuum_requests WHERE status = 'requested';"); } + catch (const pqxx::pqxx_exception& e) + { + elog("Error while checking for vacuum requests: ${e}", ("e", e.what())); + return data_processor::data_processing_status(); } + ilog("Found ${count} tables with vacuum requests.", ("count", data.size())); + + // workaround, open separate connections for non-transactional start vacuum + try + { + auto vacuum_connection = std::make_unique<pqxx::connection>(_db_url); + pqxx::nontransaction vacuum_txn(*vacuum_connection); + for (const auto& record : data) { + std::string table_name = record["table_name"].as<std::string>(); + std::string vacuum_command = "VACUUM FULL ANALYZE " + table_name; + ilog("Performing vacuum: ${vacuum_command}", ("vacuum_command", vacuum_command)); + vacuum_txn.exec(vacuum_command); + ilog("Vacuumed table: ${table_name}", ("table_name", table_name)); + tx.exec("UPDATE hafd.vacuum_requests SET status = 'vacuumed', last_vacuumed_time = NOW() WHERE table_name = '" + table_name + "';"); + ilog("Updated vacuum status for table: ${table_name}", ("table_name", table_name)); + } + } + catch (const pqxx::pqxx_exception& e) { elog("Error while vacuuming tables: ${e}", ("e", e.what())); } return data_processor::data_processing_status(); }, nullptr, -- GitLab From 42ced3518e4db72183394ae95753b5cb1dbfaf05 Mon Sep 17 00:00:00 2001 From: Dan Notestein <dan@syncad.com> Date: Wed, 11 Dec 2024 01:00:42 -0500 Subject: [PATCH 17/33] use proper exception class --- src/sql_serializer/indexes_controler.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sql_serializer/indexes_controler.cpp b/src/sql_serializer/indexes_controler.cpp index 370c2d685..5700f2a49 100644 --- a/src/sql_serializer/indexes_controler.cpp +++ b/src/sql_serializer/indexes_controler.cpp @@ -188,7 +188,7 @@ void indexes_controler::poll_and_create_indexes() { ilog("Checking for tables to vacuum..."); pqxx::result data; try { data = tx.exec("SELECT table_name FROM hafd.vacuum_requests WHERE status = 'requested';"); } - catch (const pqxx::pqxx_exception& e) + catch (const pqxx::sql_error& e) { elog("Error while checking for vacuum requests: ${e}", ("e", e.what())); return data_processor::data_processing_status(); @@ -210,7 +210,7 @@ void indexes_controler::poll_and_create_indexes() { ilog("Updated vacuum status for table: ${table_name}", ("table_name", table_name)); } } - catch (const pqxx::pqxx_exception& e) { elog("Error while vacuuming tables: ${e}", ("e", e.what())); } + catch (const pqxx::sql_error& e) { elog("Error while vacuuming tables: ${e}", ("e", e.what())); } return data_processor::data_processing_status(); }, nullptr, -- GitLab From 77503728ac08c1aef58896a3ea617f7abe6eef35 Mon Sep 17 00:00:00 2001 From: Dan Notestein <dan@syncad.com> Date: Wed, 11 Dec 2024 21:12:19 -0500 Subject: [PATCH 18/33] add timing and improved logging for index restoration and vacuum processes --- src/hive_fork_manager/hived_api_impl.sql | 20 +++++++++++++------- src/sql_serializer/indexes_controler.cpp | 20 +++++++++++++------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/hive_fork_manager/hived_api_impl.sql b/src/hive_fork_manager/hived_api_impl.sql index 96c5acba0..e561deca4 100644 --- a/src/hive_fork_manager/hived_api_impl.sql +++ b/src/hive_fork_manager/hived_api_impl.sql @@ -526,7 +526,9 @@ $function$ DECLARE __command TEXT; __original_command TEXT; - __cursor REFCURSOR; + __start_time TIMESTAMP; + __end_time TIMESTAMP; + __duration INTERVAL; BEGIN IF _table_name = 'hafd.account_operations' THEN @@ -534,10 +536,9 @@ BEGIN END IF; --restoring indexes, primary keys, unique constraints - OPEN __cursor FOR ( SELECT command FROM hafd.indexes_constraints WHERE table_name = _table_name AND is_foreign_key = FALSE AND status = 'missing' ); + FOR __original_command IN + SELECT command FROM hafd.indexes_constraints WHERE table_name = _table_name AND is_foreign_key = FALSE AND status = 'missing' LOOP - FETCH __cursor INTO __original_command; - EXIT WHEN NOT FOUND; IF concurrent THEN -- Modify the command to include CONCURRENTLY __command := regexp_replace(__original_command, 'CREATE INDEX', 'CREATE INDEX CONCURRENTLY', 'i'); @@ -546,10 +547,15 @@ BEGIN END IF; RAISE NOTICE 'Restoring index: %', __command; UPDATE hafd.indexes_constraints SET status = 'creating' WHERE command = __original_command; + COMMIT; + __start_time := clock_timestamp(); EXECUTE __command; + __end_time := clock_timestamp(); + __duration := __end_time - __start_time; + RAISE NOTICE 'Index % created in % seconds', __command, EXTRACT(EPOCH FROM __duration); UPDATE hafd.indexes_constraints SET status = 'created' WHERE command = __original_command; + COMMIT; END LOOP; - CLOSE __cursor; EXECUTE format( 'ANALYZE %s', _table_name ); @@ -843,8 +849,8 @@ BEGIN WHERE table_name = _table_name AND status <> 'vacuumed' ); - RAISE NOTICE 'Sleeping for 10 seconds waiting for vacuum to be done'; - PERFORM pg_sleep(10); + RAISE NOTICE 'Sleeping for 1 seconds waiting for vacuum to be done'; + PERFORM pg_sleep(1); END LOOP; __end_time := clock_timestamp(); diff --git a/src/sql_serializer/indexes_controler.cpp b/src/sql_serializer/indexes_controler.cpp index 5700f2a49..dba396808 100644 --- a/src/sql_serializer/indexes_controler.cpp +++ b/src/sql_serializer/indexes_controler.cpp @@ -177,7 +177,7 @@ void indexes_controler::poll_and_create_indexes() { std::mutex mtx; // Protects threads_to_delete while (!theApp.is_interrupt_request()) { - ilog("Polling for tables with missing indexes..."); + ilog("Polling for tables to vacuum or with missing indexes..."); // Check for requested table vacuums queries_commit_data_processor vacuum_requests_checker( @@ -185,7 +185,7 @@ void indexes_controler::poll_and_create_indexes() { "Check for vacuum requests", "index_ctrl", [this](const data_processor::data_chunk_ptr&, transaction_controllers::transaction& tx) -> data_processor::data_processing_status { - ilog("Checking for tables to vacuum..."); + dlog("Checking for tables to vacuum..."); pqxx::result data; try { data = tx.exec("SELECT table_name FROM hafd.vacuum_requests WHERE status = 'requested';"); } catch (const pqxx::sql_error& e) @@ -193,7 +193,7 @@ void indexes_controler::poll_and_create_indexes() { elog("Error while checking for vacuum requests: ${e}", ("e", e.what())); return data_processor::data_processing_status(); } - ilog("Found ${count} tables with vacuum requests.", ("count", data.size())); + dlog("Found ${count} tables with vacuum requests.", ("count", data.size())); // workaround, open separate connections for non-transactional start vacuum try @@ -204,10 +204,13 @@ void indexes_controler::poll_and_create_indexes() { std::string table_name = record["table_name"].as<std::string>(); std::string vacuum_command = "VACUUM FULL ANALYZE " + table_name; ilog("Performing vacuum: ${vacuum_command}", ("vacuum_command", vacuum_command)); + auto start_time = fc::time_point::now(); vacuum_txn.exec(vacuum_command); - ilog("Vacuumed table: ${table_name}", ("table_name", table_name)); + auto end_time = fc::time_point::now(); + fc::microseconds vacuum_duration = end_time - start_time; + ilog("Vacuumed table: ${table_name} in ${duration} seconds", ("table_name", table_name)("duration", vacuum_duration.to_seconds())); tx.exec("UPDATE hafd.vacuum_requests SET status = 'vacuumed', last_vacuumed_time = NOW() WHERE table_name = '" + table_name + "';"); - ilog("Updated vacuum status for table: ${table_name}", ("table_name", table_name)); + dlog("Updated vacuum status for table: ${table_name}", ("table_name", table_name)); } } catch (const pqxx::sql_error& e) { elog("Error while vacuuming tables: ${e}", ("e", e.what())); } @@ -250,12 +253,15 @@ void indexes_controler::poll_and_create_indexes() { ilog("NOTE: Starting a new thread to create indexes for table: ${table_name}", ("table_name", table_name)); active_threads[table_name] = std::thread([this, table_name, &threads_to_delete, &mtx]() { + auto start_time = fc::time_point::now(); auto processor = start_commit_sql(true, "hive.restore_indexes( '" + table_name + "', FALSE )", "restore indexes"); processor->join(); - ilog("Finished creating indexes for table: ${table_name}", ("table_name", table_name)); + auto end_time = fc::time_point::now(); + fc::microseconds index_creation_duration = end_time - start_time; + ilog("Finished creating indexes for table: ${table_name} in ${duration} seconds", ("table_name", table_name)("duration", index_creation_duration.to_seconds())); std::lock_guard g(mtx); threads_to_delete.insert(table_name); // Mark the thread for deletion - ilog("Thread for table: ${table_name} has been marked for deletion", ("table_name", table_name)); + dlog("Thread for table: ${table_name} has been marked for deletion", ("table_name", table_name)); }); } dlog("Finished processing tables with missing indexes."); -- GitLab From 9aba4ae803a4d6038002bff5c5814d1635de6183 Mon Sep 17 00:00:00 2001 From: Dan Notestein <dan@syncad.com> Date: Wed, 11 Dec 2024 22:16:34 -0500 Subject: [PATCH 19/33] can't commit in restore_indexes, consider making a procedure --- src/hive_fork_manager/hived_api_impl.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/hive_fork_manager/hived_api_impl.sql b/src/hive_fork_manager/hived_api_impl.sql index e561deca4..ca9aece5a 100644 --- a/src/hive_fork_manager/hived_api_impl.sql +++ b/src/hive_fork_manager/hived_api_impl.sql @@ -547,14 +547,12 @@ BEGIN END IF; RAISE NOTICE 'Restoring index: %', __command; UPDATE hafd.indexes_constraints SET status = 'creating' WHERE command = __original_command; - COMMIT; __start_time := clock_timestamp(); EXECUTE __command; __end_time := clock_timestamp(); __duration := __end_time - __start_time; RAISE NOTICE 'Index % created in % seconds', __command, EXTRACT(EPOCH FROM __duration); UPDATE hafd.indexes_constraints SET status = 'created' WHERE command = __original_command; - COMMIT; END LOOP; EXECUTE format( 'ANALYZE %s', _table_name ); -- GitLab From 0b65aac02291fbb302a589e56439831ebb360f43 Mon Sep 17 00:00:00 2001 From: Dan Notestein <dan@syncad.com> Date: Thu, 12 Dec 2024 18:03:05 -0500 Subject: [PATCH 20/33] create indexes concurrently in polling thread --- src/sql_serializer/indexes_controler.cpp | 198 ++++++++++++----------- 1 file changed, 104 insertions(+), 94 deletions(-) diff --git a/src/sql_serializer/indexes_controler.cpp b/src/sql_serializer/indexes_controler.cpp index dba396808..0d2e7f1b0 100644 --- a/src/sql_serializer/indexes_controler.cpp +++ b/src/sql_serializer/indexes_controler.cpp @@ -11,6 +11,7 @@ #include <mutex> #include <set> #include <map> +#include <regex> namespace hive { namespace plugins { namespace sql_serializer { @@ -171,117 +172,124 @@ indexes_controler::start_commit_sql( bool mode, const std::string& sql_function_ return processor; } -void indexes_controler::poll_and_create_indexes() { +void indexes_controler::poll_and_create_indexes() +{ std::map<std::string, std::thread> active_threads; // Doesn't need mutex, because it's modified by one thread at a time std::set<std::string> threads_to_delete; std::mutex mtx; // Protects threads_to_delete - while (!theApp.is_interrupt_request()) { - ilog("Polling for tables to vacuum or with missing indexes..."); - - // Check for requested table vacuums - queries_commit_data_processor vacuum_requests_checker( - _db_url, - "Check for vacuum requests", - "index_ctrl", - [this](const data_processor::data_chunk_ptr&, transaction_controllers::transaction& tx) -> data_processor::data_processing_status { - dlog("Checking for tables to vacuum..."); - pqxx::result data; - try { data = tx.exec("SELECT table_name FROM hafd.vacuum_requests WHERE status = 'requested';"); } - catch (const pqxx::sql_error& e) + const std::string thread_name = "haf_monitor"; + fc::set_thread_name(thread_name.c_str()); + fc::thread::current().set_name(thread_name); + + + while (!theApp.is_interrupt_request()) + { + ilog("Checking for table vacuum requests..."); + pqxx::connection conn(_db_url); + pqxx::nontransaction tx(conn); + try + { + pqxx::result data = tx.exec("SELECT table_name FROM hafd.vacuum_requests WHERE status = 'requested';"); + ilog("Found ${count} tables with vacuum requests.", ("count", data.size())); + + try + { + for (const auto& record : data) { - elog("Error while checking for vacuum requests: ${e}", ("e", e.what())); - return data_processor::data_processing_status(); + std::string table_name = record["table_name"].as<std::string>(); + std::string vacuum_command = "VACUUM FULL ANALYZE " + table_name; + ilog("Performing vacuum: ${vacuum_command}", (vacuum_command)); + auto start_time = fc::time_point::now(); + //vacuum_txn.exec(vacuum_command); + tx.exec(vacuum_command); + auto end_time = fc::time_point::now(); + fc::microseconds vacuum_duration = end_time - start_time; + ilog("Vacuumed table: ${table_name} in ${duration} seconds", (table_name)("duration", vacuum_duration.to_seconds())); + tx.exec("UPDATE hafd.vacuum_requests SET status = 'vacuumed', last_vacuumed_time = NOW() WHERE table_name = '" + table_name + "';"); + ilog("Updated vacuum status for table: ${table_name}", ("table_name", table_name)); } - dlog("Found ${count} tables with vacuum requests.", ("count", data.size())); - - // workaround, open separate connections for non-transactional start vacuum - try - { - auto vacuum_connection = std::make_unique<pqxx::connection>(_db_url); - pqxx::nontransaction vacuum_txn(*vacuum_connection); - for (const auto& record : data) { - std::string table_name = record["table_name"].as<std::string>(); - std::string vacuum_command = "VACUUM FULL ANALYZE " + table_name; - ilog("Performing vacuum: ${vacuum_command}", ("vacuum_command", vacuum_command)); - auto start_time = fc::time_point::now(); - vacuum_txn.exec(vacuum_command); - auto end_time = fc::time_point::now(); - fc::microseconds vacuum_duration = end_time - start_time; - ilog("Vacuumed table: ${table_name} in ${duration} seconds", ("table_name", table_name)("duration", vacuum_duration.to_seconds())); - tx.exec("UPDATE hafd.vacuum_requests SET status = 'vacuumed', last_vacuumed_time = NOW() WHERE table_name = '" + table_name + "';"); - dlog("Updated vacuum status for table: ${table_name}", ("table_name", table_name)); - } - } - catch (const pqxx::sql_error& e) { elog("Error while vacuuming tables: ${e}", ("e", e.what())); } - return data_processor::data_processing_status(); - }, - nullptr, - theApp - ); - - vacuum_requests_checker.trigger(data_processor::data_chunk_ptr(), 0); - vacuum_requests_checker.join(); + } + catch (const pqxx::sql_error& e) { elog("Error while vacuuming tables: ${e}", ("e", e.what())); } + } + catch (const pqxx::sql_error& e) + { + elog("Error while checking for vacuum requests: ${e}", ("e", e.what())); + } // Check for tables with missing indexes that are not currently being created - queries_commit_data_processor missing_indexes_checker( - _db_url, - "Check for missing indexes", - "index_ctrl", - [this, &active_threads, &threads_to_delete, &mtx](const data_processor::data_chunk_ptr&, transaction_controllers::transaction& tx) -> data_processor::data_processing_status { - dlog("Executing query to find tables with missing indexes..."); - pqxx::result data = tx.exec( - "SELECT DISTINCT table_name " - "FROM hafd.indexes_constraints " - "WHERE status = 'missing' " - "AND table_name NOT IN (" - " SELECT DISTINCT table_name " - " FROM hafd.indexes_constraints " - " WHERE status = 'creating'" - ");" + try + { + ilog("Executing query to find tables with missing indexes..."); + pqxx::result data = tx.exec( + "SELECT DISTINCT table_name " + "FROM hafd.indexes_constraints " + "WHERE status = 'missing' " + "AND table_name NOT IN (" + " SELECT DISTINCT table_name " + " FROM hafd.indexes_constraints " + " WHERE status = 'creating'" + ");" ); - dlog("Query executed. Found ${count} tables with missing indexes.", ("count", data.size())); - - for (const auto& record : data) { - std::string table_name = record["table_name"].as<std::string>(); - dlog("Processing table: ${table_name}", ("table_name", table_name)); - // Check if a thread is already running for this table - if (active_threads.find(table_name) != active_threads.end() && active_threads[table_name].joinable()) { - ilog("A thread is already running for table: ${table_name}", ("table_name", table_name)); - continue; - } + ilog("Query executed. Found ${count} tables with missing indexes.", ("count", data.size())); + for (const auto& record : data) //iterate over tables with missing indexes + { + std::string table_name = record["table_name"].as<std::string>(); + dlog("Processing table: ${table_name}", ("table_name", table_name)); + // Check if a thread is already running for this table + if (active_threads.find(table_name) != active_threads.end() && active_threads[table_name].joinable()) + { + ilog("A thread is already running for table: ${table_name}", ("table_name", table_name)); + continue; //check the next table + } - ilog("NOTE: Starting a new thread to create indexes for table: ${table_name}", ("table_name", table_name)); - active_threads[table_name] = std::thread([this, table_name, &threads_to_delete, &mtx]() { + ilog("NOTE: Starting a new thread to create indexes for table: ${table_name}", ("table_name", table_name)); + active_threads[table_name] = std::thread([this, table_name, &threads_to_delete, &mtx]() + { + const std::string thread_name = table_name; + if (thread_name.size() > 16) + thread_name.resize(16); + fc::set_thread_name(thread_name.c_str()); + fc::thread::current().set_name(thread_name); + + pqxx::connection conn(_db_url); + pqxx::nontransaction tx(conn); + pqxx::result data = tx.exec("SELECT command FROM hafd.indexes_constraints WHERE status = 'missing' AND table_name = '" + table_name + "';"); + for (const auto& index : data) //iterate over missing indexes and create them concurrently + { + std::string command = index["command"].as<std::string>(); + std::regex create_index_regex(R"((CREATE\s+UNIQUE\s+INDEX|CREATE\s+INDEX))", std::regex::icase); + command = std::regex_replace(command, create_index_regex, "$& CONCURRENTLY"); + ilog("Creating index: ${command}", (command)); auto start_time = fc::time_point::now(); - auto processor = start_commit_sql(true, "hive.restore_indexes( '" + table_name + "', FALSE )", "restore indexes"); - processor->join(); + tx.exec(command); auto end_time = fc::time_point::now(); fc::microseconds index_creation_duration = end_time - start_time; - ilog("Finished creating indexes for table: ${table_name} in ${duration} seconds", ("table_name", table_name)("duration", index_creation_duration.to_seconds())); - std::lock_guard g(mtx); - threads_to_delete.insert(table_name); // Mark the thread for deletion - dlog("Thread for table: ${table_name} has been marked for deletion", ("table_name", table_name)); - }); - } - dlog("Finished processing tables with missing indexes."); - return data_processor::data_processing_status(); - }, - nullptr, - theApp - ); + ilog("Finished creating index for table: ${table_name} in ${duration} seconds", (table_name)("duration", index_creation_duration.to_seconds())); + } + ilog("Finished creating all indexes for table: ${table_name}", (table_name)); + std::lock_guard g(mtx); + threads_to_delete.insert(table_name); // Mark the thread for deletion + ilog("Thread for table: ${table_name} has been marked for deletion", (table_name)); + }); + } //end for tables with missing indexes + dlog("Finished polling for tables with missing indexes, sleep for 10s."); + } + catch (const pqxx::sql_error& e) + { + elog("Error while checking for missing indexes: ${e}", ("e", e.what())); + } - missing_indexes_checker.trigger(data_processor::data_chunk_ptr(), 0); - missing_indexes_checker.join(); - dlog("Finished polling for tables with missing indexes, sleep for 10s."); // Sleep for 10 seconds before polling again fc::usleep(fc::seconds(10)); // Delete threads marked for deletion { std::lock_guard g(mtx); - for (const auto& table_name : threads_to_delete) { - if (active_threads[table_name].joinable()) { + for (const auto& table_name : threads_to_delete) + { + if (active_threads[table_name].joinable()) + { ilog("Joining thread for table: ${table_name}", ("table_name", table_name)); active_threads[table_name].join(); } @@ -292,12 +300,14 @@ void indexes_controler::poll_and_create_indexes() { } ilog("Interrupt request received, stopping polling for tables with missing indexes."); // Join all remaining threads before exiting - for (auto& [table_name, thread] : active_threads) { - if (thread.joinable()) { + for (auto& [table_name, thread] : active_threads) + { + if (thread.joinable()) + { ilog("Joining thread for table: ${table_name}", ("table_name", table_name)); thread.join(); } } -} +} //end poll_and_create_indexes }}} // namespace hive{ plugins { sql_serializer -- GitLab From c2be6df65c503a9995f1946a879740883e32bae9 Mon Sep 17 00:00:00 2001 From: Dan Notestein <dan@syncad.com> Date: Thu, 12 Dec 2024 18:22:00 -0500 Subject: [PATCH 21/33] add try catch around index creation step --- src/sql_serializer/indexes_controler.cpp | 25 ++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/sql_serializer/indexes_controler.cpp b/src/sql_serializer/indexes_controler.cpp index 0d2e7f1b0..24612e9d6 100644 --- a/src/sql_serializer/indexes_controler.cpp +++ b/src/sql_serializer/indexes_controler.cpp @@ -246,7 +246,7 @@ void indexes_controler::poll_and_create_indexes() ilog("NOTE: Starting a new thread to create indexes for table: ${table_name}", ("table_name", table_name)); active_threads[table_name] = std::thread([this, table_name, &threads_to_delete, &mtx]() { - const std::string thread_name = table_name; + std::string thread_name = table_name; if (thread_name.size() > 16) thread_name.resize(16); fc::set_thread_name(thread_name.c_str()); @@ -257,15 +257,20 @@ void indexes_controler::poll_and_create_indexes() pqxx::result data = tx.exec("SELECT command FROM hafd.indexes_constraints WHERE status = 'missing' AND table_name = '" + table_name + "';"); for (const auto& index : data) //iterate over missing indexes and create them concurrently { - std::string command = index["command"].as<std::string>(); - std::regex create_index_regex(R"((CREATE\s+UNIQUE\s+INDEX|CREATE\s+INDEX))", std::regex::icase); - command = std::regex_replace(command, create_index_regex, "$& CONCURRENTLY"); - ilog("Creating index: ${command}", (command)); - auto start_time = fc::time_point::now(); - tx.exec(command); - auto end_time = fc::time_point::now(); - fc::microseconds index_creation_duration = end_time - start_time; - ilog("Finished creating index for table: ${table_name} in ${duration} seconds", (table_name)("duration", index_creation_duration.to_seconds())); + try + { + std::string command = index["command"].as<std::string>(); + std::regex create_index_regex(R"((CREATE\s+UNIQUE\s+INDEX|CREATE\s+INDEX))", std::regex::icase); + command = std::regex_replace(command, create_index_regex, "$& CONCURRENTLY"); + ilog("Creating index: ${command}", (command)); + auto start_time = fc::time_point::now(); + tx.exec(command); + auto end_time = fc::time_point::now(); + fc::microseconds index_creation_duration = end_time - start_time; + ilog("Finished creating index for table: ${table_name} in ${duration} seconds", (table_name)("duration", index_creation_duration.to_seconds())); + } + catch (const pqxx::sql_error& e) { elog("Error while creating index: ${e}", ("e", e.what())); } + catch(std::exception& e ) { elog( e.what() ); } } ilog("Finished creating all indexes for table: ${table_name}", (table_name)); std::lock_guard g(mtx); -- GitLab From 3982ab062c5c6551c0c343a2e795e51a77a2ae92 Mon Sep 17 00:00:00 2001 From: Dan Notestein <dan@syncad.com> Date: Thu, 12 Dec 2024 21:39:40 -0500 Subject: [PATCH 22/33] update status when creating indexes --- src/sql_serializer/indexes_controler.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sql_serializer/indexes_controler.cpp b/src/sql_serializer/indexes_controler.cpp index 24612e9d6..95e6fcd66 100644 --- a/src/sql_serializer/indexes_controler.cpp +++ b/src/sql_serializer/indexes_controler.cpp @@ -263,11 +263,13 @@ void indexes_controler::poll_and_create_indexes() std::regex create_index_regex(R"((CREATE\s+UNIQUE\s+INDEX|CREATE\s+INDEX))", std::regex::icase); command = std::regex_replace(command, create_index_regex, "$& CONCURRENTLY"); ilog("Creating index: ${command}", (command)); + tx.exec("UPDATE hafd.indexes_constraints SET status = 'creating' WHERE command ='"+command+"';"); auto start_time = fc::time_point::now(); tx.exec(command); auto end_time = fc::time_point::now(); fc::microseconds index_creation_duration = end_time - start_time; ilog("Finished creating index for table: ${table_name} in ${duration} seconds", (table_name)("duration", index_creation_duration.to_seconds())); + tx.exec("UPDATE hafd.indexes_constraints SET status = 'created' WHERE command ='"+command+"';"); } catch (const pqxx::sql_error& e) { elog("Error while creating index: ${e}", ("e", e.what())); } catch(std::exception& e ) { elog( e.what() ); } -- GitLab From 6f3b5d7459c827057fe2bf851be9ff90533e5965 Mon Sep 17 00:00:00 2001 From: Dan Notestein <dan@syncad.com> Date: Thu, 12 Dec 2024 22:35:51 -0500 Subject: [PATCH 23/33] remove unneeded functions --- src/hive_fork_manager/hived_api_impl.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/hive_fork_manager/hived_api_impl.sql b/src/hive_fork_manager/hived_api_impl.sql index ca9aece5a..594c2e5ab 100644 --- a/src/hive_fork_manager/hived_api_impl.sql +++ b/src/hive_fork_manager/hived_api_impl.sql @@ -824,7 +824,7 @@ BEGIN END; $BODY$ ; - +/* CREATE OR REPLACE FUNCTION hive.app_wait_for_table_vacuum( _table_name TEXT ) @@ -874,3 +874,4 @@ BEGIN END; $BODY$ ; +*/ \ No newline at end of file -- GitLab From d4dfb2d739af6e33440881cfac7c5b42090257cd Mon Sep 17 00:00:00 2001 From: Dan Notestein <dan@syncad.com> Date: Thu, 12 Dec 2024 23:21:59 -0500 Subject: [PATCH 24/33] try to fix queries --- src/sql_serializer/indexes_controler.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/sql_serializer/indexes_controler.cpp b/src/sql_serializer/indexes_controler.cpp index 95e6fcd66..78cd6cd94 100644 --- a/src/sql_serializer/indexes_controler.cpp +++ b/src/sql_serializer/indexes_controler.cpp @@ -259,17 +259,20 @@ void indexes_controler::poll_and_create_indexes() { try { - std::string command = index["command"].as<std::string>(); + std::string index_constraint_name = index["index_constraint_name"].as<std::string>(); + std::string original_command = index["command"].as<std::string>(); std::regex create_index_regex(R"((CREATE\s+UNIQUE\s+INDEX|CREATE\s+INDEX))", std::regex::icase); - command = std::regex_replace(command, create_index_regex, "$& CONCURRENTLY"); + std::string command = std::regex_replace(original_command, create_index_regex, "$& CONCURRENTLY"); + std::string update_table = "UPDATE hafd.indexes_constraints SET status = 'creating' WHERE index_constraint_name ='"+index_constraint_name+"';" + idump(update_table); + tx.exec(update_table); ilog("Creating index: ${command}", (command)); - tx.exec("UPDATE hafd.indexes_constraints SET status = 'creating' WHERE command ='"+command+"';"); auto start_time = fc::time_point::now(); tx.exec(command); auto end_time = fc::time_point::now(); fc::microseconds index_creation_duration = end_time - start_time; ilog("Finished creating index for table: ${table_name} in ${duration} seconds", (table_name)("duration", index_creation_duration.to_seconds())); - tx.exec("UPDATE hafd.indexes_constraints SET status = 'created' WHERE command ='"+command+"';"); + tx.exec("UPDATE hafd.indexes_constraints SET status = 'created' WHERE index_constraint_name ='"+index_constraint_name+"';"); } catch (const pqxx::sql_error& e) { elog("Error while creating index: ${e}", ("e", e.what())); } catch(std::exception& e ) { elog( e.what() ); } -- GitLab From 3c3004be0c33e759c70bd9a17d7d3c0a36e8d8ee Mon Sep 17 00:00:00 2001 From: Dan Notestein <dan@syncad.com> Date: Thu, 12 Dec 2024 23:29:18 -0500 Subject: [PATCH 25/33] fix syntax error --- src/sql_serializer/indexes_controler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sql_serializer/indexes_controler.cpp b/src/sql_serializer/indexes_controler.cpp index 78cd6cd94..b965ed732 100644 --- a/src/sql_serializer/indexes_controler.cpp +++ b/src/sql_serializer/indexes_controler.cpp @@ -264,7 +264,7 @@ void indexes_controler::poll_and_create_indexes() std::regex create_index_regex(R"((CREATE\s+UNIQUE\s+INDEX|CREATE\s+INDEX))", std::regex::icase); std::string command = std::regex_replace(original_command, create_index_regex, "$& CONCURRENTLY"); std::string update_table = "UPDATE hafd.indexes_constraints SET status = 'creating' WHERE index_constraint_name ='"+index_constraint_name+"';" - idump(update_table); + idump((update_table)); tx.exec(update_table); ilog("Creating index: ${command}", (command)); auto start_time = fc::time_point::now(); -- GitLab From 02fb62d1bd4756ecc32f853230d366ccbbfdd730 Mon Sep 17 00:00:00 2001 From: Dan Notestein <dan@syncad.com> Date: Thu, 12 Dec 2024 23:41:13 -0500 Subject: [PATCH 26/33] fix typo --- src/sql_serializer/indexes_controler.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/sql_serializer/indexes_controler.cpp b/src/sql_serializer/indexes_controler.cpp index b965ed732..4c15b3b12 100644 --- a/src/sql_serializer/indexes_controler.cpp +++ b/src/sql_serializer/indexes_controler.cpp @@ -263,8 +263,9 @@ void indexes_controler::poll_and_create_indexes() std::string original_command = index["command"].as<std::string>(); std::regex create_index_regex(R"((CREATE\s+UNIQUE\s+INDEX|CREATE\s+INDEX))", std::regex::icase); std::string command = std::regex_replace(original_command, create_index_regex, "$& CONCURRENTLY"); - std::string update_table = "UPDATE hafd.indexes_constraints SET status = 'creating' WHERE index_constraint_name ='"+index_constraint_name+"';" - idump((update_table)); + std::string update_table = + "UPDATE hafd.indexes_constraints SET status = 'creating' WHERE index_constraint_name ='" + index_constraint_name + "';"; + ilog("SQL: ${update_table}",(update_table)); tx.exec(update_table); ilog("Creating index: ${command}", (command)); auto start_time = fc::time_point::now(); -- GitLab From 01f1352fe6784ade04833fdf566836773bfe48fe Mon Sep 17 00:00:00 2001 From: Dan Notestein <dan@syncad.com> Date: Fri, 13 Dec 2024 00:17:25 -0500 Subject: [PATCH 27/33] forgot to fetch column --- src/sql_serializer/indexes_controler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sql_serializer/indexes_controler.cpp b/src/sql_serializer/indexes_controler.cpp index 4c15b3b12..fa0c19be2 100644 --- a/src/sql_serializer/indexes_controler.cpp +++ b/src/sql_serializer/indexes_controler.cpp @@ -254,7 +254,7 @@ void indexes_controler::poll_and_create_indexes() pqxx::connection conn(_db_url); pqxx::nontransaction tx(conn); - pqxx::result data = tx.exec("SELECT command FROM hafd.indexes_constraints WHERE status = 'missing' AND table_name = '" + table_name + "';"); + pqxx::result data = tx.exec("SELECT index_constraint_name, command FROM hafd.indexes_constraints WHERE status = 'missing' AND table_name = '" + table_name + "';"); for (const auto& index : data) //iterate over missing indexes and create them concurrently { try -- GitLab From 43e74088ac325604f2630ab9ac4e02ea557b980a Mon Sep 17 00:00:00 2001 From: Dan Notestein <dan@syncad.com> Date: Fri, 13 Dec 2024 01:51:32 -0500 Subject: [PATCH 28/33] add hive.check_if_registered_indexes_created --- src/hive_fork_manager/hived_api_impl.sql | 31 ++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/hive_fork_manager/hived_api_impl.sql b/src/hive_fork_manager/hived_api_impl.sql index 594c2e5ab..2e16adfef 100644 --- a/src/hive_fork_manager/hived_api_impl.sql +++ b/src/hive_fork_manager/hived_api_impl.sql @@ -715,6 +715,37 @@ END; $BODY$ ; +CREATE OR REPLACE FUNCTION hive.check_if_registered_indexes_created( + _app_context TEXT +) +RETURNS BOOLEAN +LANGUAGE plpgsql +AS +$BODY$ +DECLARE + __context_id INT; +BEGIN + RAISE NOTICE 'Checking if registered indexes are created for context %', _app_context; + -- Lookup the context_id using context_name + SELECT id INTO __context_id + FROM hafd.contexts + WHERE name = _app_context; + + -- Abort with an error message if no context_id is found + IF __context_id IS NULL THEN + RAISE EXCEPTION 'Context % not found in hafd.contexts', _app_context; + END IF; + + -- Check if there are any indexes that are not created yet + RETURN NOT EXISTS ( + SELECT 1 + FROM hafd.indexes_constraints + WHERE contexts @> ARRAY[__context_id] AND status <> 'created' + ); +END; +$BODY$ +; + CREATE OR REPLACE FUNCTION hive.parse_create_index_command( create_index_command TEXT ) -- GitLab From 1f36c645069a458cc2e601f303645b9720104b2d Mon Sep 17 00:00:00 2001 From: Dan Notestein <dan@syncad.com> Date: Fri, 13 Dec 2024 03:26:26 -0500 Subject: [PATCH 29/33] replace wait_till with check_if index calls for now in tests --- src/hive_fork_manager/hived_api_impl.sql | 3 ++- .../haf_local_tools/system/haf/__init__.py | 7 ++++++- .../system/applications/test_application_index_many.py | 7 ++++++- .../system/applications/test_application_index_one.py | 9 +++++++-- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/hive_fork_manager/hived_api_impl.sql b/src/hive_fork_manager/hived_api_impl.sql index 2e16adfef..af3140dd7 100644 --- a/src/hive_fork_manager/hived_api_impl.sql +++ b/src/hive_fork_manager/hived_api_impl.sql @@ -680,7 +680,7 @@ BEGIN END; $BODY$ ; - +/* CREATE OR REPLACE FUNCTION hive.wait_till_registered_indexes_created( _app_context TEXT ) @@ -714,6 +714,7 @@ BEGIN END; $BODY$ ; +*/ CREATE OR REPLACE FUNCTION hive.check_if_registered_indexes_created( _app_context TEXT diff --git a/tests/integration/haf-local-tools/haf_local_tools/system/haf/__init__.py b/tests/integration/haf-local-tools/haf_local_tools/system/haf/__init__.py index fe0c2a519..b612de15e 100644 --- a/tests/integration/haf-local-tools/haf_local_tools/system/haf/__init__.py +++ b/tests/integration/haf-local-tools/haf_local_tools/system/haf/__init__.py @@ -57,7 +57,12 @@ def assert_index_does_not_exist(session, namespace, table, indexname): def wait_till_registered_indexes_created(haf_node, context): - haf_node.session.execute("select hive.wait_till_registered_indexes_created(:ctx)", {'ctx': context}) + while True: + result = haf_node.session.execute("SELECT hive.check_if_registered_indexes_created(:ctx)", {'ctx': context}).scalar() + if result: + break + tt.logger.info("Indexes not yet created. Sleeping for 10 seconds...") + tt.sleep(10) def register_index_dependency(haf_node, context, create_index_command): diff --git a/tests/integration/system/applications/test_application_index_many.py b/tests/integration/system/applications/test_application_index_many.py index 3711420b1..74f01c7f6 100644 --- a/tests/integration/system/applications/test_application_index_many.py +++ b/tests/integration/system/applications/test_application_index_many.py @@ -54,7 +54,12 @@ def test_application_index_many(haf_node): session.commit() # THEN - session.execute("select hive.wait_till_registered_indexes_created('application')") + while True: + result = session.execute("SELECT hive.check_if_registered_indexes_created('application')").scalar() + if result: + break + tt.logger.info("Indexes not yet created. Sleeping for 10 seconds...") + tt.sleep(10) assert_index_exists(session, 'hafd', 'operations', 'hive_operations_vote_author_permlink_1') assert_index_exists(session, 'hafd', 'operations', 'hive_operations_vote_author_permlink_2') diff --git a/tests/integration/system/applications/test_application_index_one.py b/tests/integration/system/applications/test_application_index_one.py index d8947183c..572aab2bc 100644 --- a/tests/integration/system/applications/test_application_index_one.py +++ b/tests/integration/system/applications/test_application_index_one.py @@ -2,7 +2,7 @@ import test_tools as tt from haf_local_tools import create_app from haf_local_tools.haf_node.monolithic_workaround import apply_block_log_type_to_monolithic_workaround -from haf_local_tools.system.haf import (connect_nodes, assert_index_exists, wait_till_registered_indexes_created, register_index_dependency) +from haf_local_tools.system.haf import (connect_nodes, assert_index_exists, register_index_dependency) def test_application_index_one(haf_node): @@ -33,6 +33,11 @@ def test_application_index_one(haf_node): session.commit() # THEN - wait_till_registered_indexes_created(haf_node, 'application') + while True: + result = session.execute("SELECT hive.check_if_registered_indexes_created('application')").scalar() + if result: + break + tt.logger.info("Indexes not yet created. Sleeping for 10 seconds...") + tt.sleep(10) assert_index_exists(session, 'hafd', 'operations', 'hive_operations_vote_author_permlink') -- GitLab From 4a05b0899ad43110cde03f1369670b3663ab9732 Mon Sep 17 00:00:00 2001 From: Dan Notestein <dan@syncad.com> Date: Fri, 13 Dec 2024 15:08:53 -0500 Subject: [PATCH 30/33] can't create an index concurrently from inside a function so removing option from hive.restore_indexes --- src/hive_fork_manager/authorization.sql | 2 +- src/hive_fork_manager/hived_api.sql | 4 ++-- src/hive_fork_manager/hived_api_impl.sql | 15 ++++----------- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/src/hive_fork_manager/authorization.sql b/src/hive_fork_manager/authorization.sql index 3f9cc9805..a09b86225 100644 --- a/src/hive_fork_manager/authorization.sql +++ b/src/hive_fork_manager/authorization.sql @@ -140,7 +140,7 @@ GRANT EXECUTE ON FUNCTION , hive.save_and_drop_indexes_constraints( in _schema TEXT, in _table TEXT ) , hive.save_and_drop_foreign_keys( in _table_schema TEXT, in _table_name TEXT ) , hive.recluster_account_operations_if_index_dropped() - , hive.restore_indexes( in _table_name TEXT, in concurrent BOOLEAN ) + , hive.restore_indexes( in _table_name TEXT ) , hive.restore_foreign_keys( in _table_name TEXT ) , hive.copy_blocks_to_irreversible( _head_block_of_irreversible_blocks INT, _new_irreversible_block INT ) , hive.copy_transactions_to_irreversible( _head_block_of_irreversible_blocks INT, _new_irreversible_block INT ) diff --git a/src/hive_fork_manager/hived_api.sql b/src/hive_fork_manager/hived_api.sql index e4fb56a44..2b557a53e 100644 --- a/src/hive_fork_manager/hived_api.sql +++ b/src/hive_fork_manager/hived_api.sql @@ -200,7 +200,7 @@ BEGIN PERFORM hive.save_and_drop_indexes_constraints( 'hafd', 'accounts' ); PERFORM hive.save_and_drop_indexes_constraints( 'hafd', 'account_operations' ); - PERFORM hive.reanalyze_indexes_with_expressions(); + PERFORM hive.reanalyze_indexes_with_expressions(); --I wonder if reanalyzing is really needed when indexes are dropped END; $BODY$ ; @@ -292,7 +292,7 @@ BEGIN PERFORM hive.save_and_drop_indexes_constraints( 'hafd', 'accounts_reversible' ); PERFORM hive.save_and_drop_indexes_constraints( 'hafd', 'account_operations_reversible' ); - PERFORM hive.reanalyze_indexes_with_expressions(); + PERFORM hive.reanalyze_indexes_with_expressions(); --I wonder if reanalyzing is really needed when indexes are dropped END; $BODY$ diff --git a/src/hive_fork_manager/hived_api_impl.sql b/src/hive_fork_manager/hived_api_impl.sql index af3140dd7..c03690a63 100644 --- a/src/hive_fork_manager/hived_api_impl.sql +++ b/src/hive_fork_manager/hived_api_impl.sql @@ -519,13 +519,12 @@ $function$ LANGUAGE plpgsql VOLATILE ; -CREATE OR REPLACE FUNCTION hive.restore_indexes( in _table_name TEXT, in concurrent BOOLEAN DEFAULT FALSE ) +CREATE OR REPLACE FUNCTION hive.restore_indexes( in _table_name TEXT) RETURNS VOID AS $function$ DECLARE __command TEXT; - __original_command TEXT; __start_time TIMESTAMP; __end_time TIMESTAMP; __duration INTERVAL; @@ -536,23 +535,17 @@ BEGIN END IF; --restoring indexes, primary keys, unique constraints - FOR __original_command IN + FOR __command IN SELECT command FROM hafd.indexes_constraints WHERE table_name = _table_name AND is_foreign_key = FALSE AND status = 'missing' LOOP - IF concurrent THEN - -- Modify the command to include CONCURRENTLY - __command := regexp_replace(__original_command, 'CREATE INDEX', 'CREATE INDEX CONCURRENTLY', 'i'); - ELSE - __command := __original_command; - END IF; RAISE NOTICE 'Restoring index: %', __command; - UPDATE hafd.indexes_constraints SET status = 'creating' WHERE command = __original_command; + UPDATE hafd.indexes_constraints SET status = 'creating' WHERE command = __command; __start_time := clock_timestamp(); EXECUTE __command; __end_time := clock_timestamp(); __duration := __end_time - __start_time; RAISE NOTICE 'Index % created in % seconds', __command, EXTRACT(EPOCH FROM __duration); - UPDATE hafd.indexes_constraints SET status = 'created' WHERE command = __original_command; + UPDATE hafd.indexes_constraints SET status = 'created' WHERE command = __command; END LOOP; EXECUTE format( 'ANALYZE %s', _table_name ); -- GitLab From c15bd2e29d49e966e2451bf70ad94e5ba8ce74db Mon Sep 17 00:00:00 2001 From: Dan Notestein <dan@syncad.com> Date: Fri, 13 Dec 2024 15:59:02 -0500 Subject: [PATCH 31/33] poll_and_create_indexes also analyzes a table after it creates indexes for it --- src/sql_serializer/indexes_controler.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/sql_serializer/indexes_controler.cpp b/src/sql_serializer/indexes_controler.cpp index fa0c19be2..e398f5e88 100644 --- a/src/sql_serializer/indexes_controler.cpp +++ b/src/sql_serializer/indexes_controler.cpp @@ -279,6 +279,15 @@ void indexes_controler::poll_and_create_indexes() catch(std::exception& e ) { elog( e.what() ); } } ilog("Finished creating all indexes for table: ${table_name}", (table_name)); + try + { + ilog("Analyzing table: ${table_name}", (table_name) ); + std::string analyze_table = "ANALYZE " + table_name + ";"; + tx.exec(analyze_table); + } + catch (const pqxx::sql_error& e) { elog("Error while analyzing table: ${e}", ("e", e.what())); } + catch(std::exception& e ) { elog( e.what() ); } + std::lock_guard g(mtx); threads_to_delete.insert(table_name); // Mark the thread for deletion ilog("Thread for table: ${table_name} has been marked for deletion", (table_name)); -- GitLab From c8fb708b4986f1a423d8a88de867bbe8a24c881e Mon Sep 17 00:00:00 2001 From: Dan Notestein <dan@syncad.com> Date: Fri, 13 Dec 2024 16:44:13 -0500 Subject: [PATCH 32/33] try to fix tests with correct sleep --- .../haf-local-tools/haf_local_tools/system/haf/__init__.py | 4 ++-- .../system/applications/test_application_index_one.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/haf-local-tools/haf_local_tools/system/haf/__init__.py b/tests/integration/haf-local-tools/haf_local_tools/system/haf/__init__.py index b612de15e..337b48156 100644 --- a/tests/integration/haf-local-tools/haf_local_tools/system/haf/__init__.py +++ b/tests/integration/haf-local-tools/haf_local_tools/system/haf/__init__.py @@ -5,7 +5,7 @@ import test_tools as tt from haf_local_tools.haf_node._haf_node import HafNode, Transaction, TransactionId from haf_local_tools.tables import BlocksView - +import time def connect_nodes(seed_node: tt.RawNode, peer_node: tt.RawNode) -> None: """ @@ -62,7 +62,7 @@ def wait_till_registered_indexes_created(haf_node, context): if result: break tt.logger.info("Indexes not yet created. Sleeping for 10 seconds...") - tt.sleep(10) + time.sleep(10) def register_index_dependency(haf_node, context, create_index_command): diff --git a/tests/integration/system/applications/test_application_index_one.py b/tests/integration/system/applications/test_application_index_one.py index 572aab2bc..623a754c5 100644 --- a/tests/integration/system/applications/test_application_index_one.py +++ b/tests/integration/system/applications/test_application_index_one.py @@ -3,7 +3,7 @@ import test_tools as tt from haf_local_tools import create_app from haf_local_tools.haf_node.monolithic_workaround import apply_block_log_type_to_monolithic_workaround from haf_local_tools.system.haf import (connect_nodes, assert_index_exists, register_index_dependency) - +import time def test_application_index_one(haf_node): tt.logger.info(f'Start test_application_index_one') @@ -38,6 +38,6 @@ def test_application_index_one(haf_node): if result: break tt.logger.info("Indexes not yet created. Sleeping for 10 seconds...") - tt.sleep(10) + time.sleep(10) assert_index_exists(session, 'hafd', 'operations', 'hive_operations_vote_author_permlink') -- GitLab From e7eeb458e6f59a347b8bbcfa1f9f538b33261020 Mon Sep 17 00:00:00 2001 From: Dan Notestein <dan@syncad.com> Date: Fri, 13 Dec 2024 17:17:03 -0500 Subject: [PATCH 33/33] missed one time call --- .../system/applications/test_application_index_many.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/system/applications/test_application_index_many.py b/tests/integration/system/applications/test_application_index_many.py index 74f01c7f6..f2c991f7d 100644 --- a/tests/integration/system/applications/test_application_index_many.py +++ b/tests/integration/system/applications/test_application_index_many.py @@ -3,7 +3,7 @@ import test_tools as tt from haf_local_tools import create_app from haf_local_tools.haf_node.monolithic_workaround import apply_block_log_type_to_monolithic_workaround from haf_local_tools.system.haf import (connect_nodes, assert_index_exists, register_index_dependency) - +import time def test_application_index_many(haf_node): tt.logger.info(f'Start test_application_index_many') @@ -59,7 +59,7 @@ def test_application_index_many(haf_node): if result: break tt.logger.info("Indexes not yet created. Sleeping for 10 seconds...") - tt.sleep(10) + time.sleep(10) assert_index_exists(session, 'hafd', 'operations', 'hive_operations_vote_author_permlink_1') assert_index_exists(session, 'hafd', 'operations', 'hive_operations_vote_author_permlink_2') -- GitLab