From 881ac31ec47bf76e354faee9c79ea51ea90542c7 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Mon, 9 Dec 2024 18:52:56 -0500
Subject: [PATCH 01/33] Add functions to allow apps to request/wait for full
 vacuums

---
 src/hive_fork_manager/hived_api_impl.sql      | 81 +++++++++++++++++++
 .../hived_api_impl_indexes.sql                |  9 +++
 src/sql_serializer/indexes_controler.cpp      | 26 ++++++
 3 files changed, 116 insertions(+)

diff --git a/src/hive_fork_manager/hived_api_impl.sql b/src/hive_fork_manager/hived_api_impl.sql
index a5c29e4e0..cf85c207a 100644
--- a/src/hive_fork_manager/hived_api_impl.sql
+++ b/src/hive_fork_manager/hived_api_impl.sql
@@ -785,3 +785,84 @@ BEGIN
 END;
 $BODY$
 ;
+
+CREATE OR REPLACE FUNCTION hive.request_table_vacuum(
+    _table_name TEXT,
+    _min_interval INTERVAL DEFAULT NULL
+)
+RETURNS void
+LANGUAGE plpgsql
+AS
+$BODY$
+BEGIN
+    IF _min_interval IS NOT NULL THEN
+        -- Check if the table has been vacuumed recently
+        IF EXISTS (
+            SELECT 1
+            FROM hafd.vacuum_requests
+            WHERE table_name = _table_name
+            AND last_vacuumed_time > NOW() - _min_interval
+        ) THEN
+            RAISE NOTICE 'Vacuum request for table % ignored due to recent vacuum.', _table_name;
+            RETURN;
+        END IF;
+    END IF;
+
+    -- Insert or update the vacuum request
+    INSERT INTO hafd.vacuum_requests (table_name, status)
+    VALUES (_table_name, 'requested')
+    ON CONFLICT (table_name) DO UPDATE
+    SET status = 'requested';
+
+    RAISE NOTICE 'Vacuum request for table % submitted.', _table_name;
+END;
+$BODY$
+;
+
+CREATE OR REPLACE FUNCTION hive.wait_for_table_vacuum(
+    _table_name TEXT
+)
+RETURNS void
+LANGUAGE plpgsql
+AS
+$BODY$
+DECLARE
+    __start_time TIMESTAMP;
+    __end_time TIMESTAMP;
+    __duration INTERVAL;
+BEGIN
+    RAISE NOTICE 'Waiting for vacuum to complete for table %', _table_name;
+    __start_time := clock_timestamp();
+
+    LOOP
+        EXIT WHEN NOT EXISTS (
+            SELECT 1
+            FROM hafd.vacuum_requests
+            WHERE table_name = _table_name
+            AND status <> 'vacuumed'
+        );
+    END LOOP;
+
+    __end_time := clock_timestamp();
+    __duration := __end_time - __start_time;
+    RAISE NOTICE 'Vacuum completed for table % in % seconds', _table_name, EXTRACT(EPOCH FROM __duration);
+END;
+$BODY$
+;
+
+CREATE OR REPLACE FUNCTION hive.test_vacuum_functions()
+RETURNS void
+LANGUAGE plpgsql
+AS
+$BODY$
+BEGIN
+    -- Request a vacuum for the table hafbe_app_keyauth_a
+    PERFORM hive.request_table_vacuum('hafbe_app_keyauth_a', '1 hour'::INTERVAL);
+
+    -- Wait for the vacuum to complete
+    PERFORM hive.wait_for_table_vacuum('hafbe_app_keyauth_a');
+
+    RAISE NOTICE 'Test for vacuum functions on table hafbe_app_keyauth_a completed successfully.';
+END;
+$BODY$
+;
diff --git a/src/hive_fork_manager/hived_api_impl_indexes.sql b/src/hive_fork_manager/hived_api_impl_indexes.sql
index bcafdc1d2..f35ec5ac4 100644
--- a/src/hive_fork_manager/hived_api_impl_indexes.sql
+++ b/src/hive_fork_manager/hived_api_impl_indexes.sql
@@ -12,3 +12,12 @@ CREATE TABLE IF NOT EXISTS hafd.indexes_constraints (
     CONSTRAINT pk_hive_indexes_constraints UNIQUE( table_name, index_constraint_name )
 );
 SELECT pg_catalog.pg_extension_config_dump('hafd.indexes_constraints', '');
+
+-- Only one vacuum request per table, and this is only for 'vacuum full analyze' requests
+CREATE TYPE hafd.vacuum_status AS ENUM ('requested', 'vacuuming', 'vacuumed', 'failed');
+CREATE TABLE IF NOT EXISTS hafd.vacuum_requests (
+    table_name text NOT NULL,
+    hafd.vacuum_status,
+    last_vacuumed_time timestamp,
+    CONSTRAINT pk_hive_vacuum_requests UNIQUE( table_name)
+);
diff --git a/src/sql_serializer/indexes_controler.cpp b/src/sql_serializer/indexes_controler.cpp
index 76d765969..54bc9d1ae 100644
--- a/src/sql_serializer/indexes_controler.cpp
+++ b/src/sql_serializer/indexes_controler.cpp
@@ -178,6 +178,32 @@ void indexes_controler::poll_and_create_indexes() {
 
   while (!theApp.is_interrupt_request()) {
     dlog("Polling for tables with missing indexes...");
+
+    // Check for requested table vacuums
+    queries_commit_data_processor vacuum_requests_checker(
+      _db_url,
+      "Check for vacuum requests",
+      "index_ctrl",
+      [](const data_processor::data_chunk_ptr&, transaction_controllers::transaction& tx) -> data_processor::data_processing_status {
+        pqxx::result data = tx.exec(
+          "SELECT table_name FROM hafd.vacuum_requests WHERE status = 'requested';"
+        );
+        for (const auto& record : data) {
+          std::string table_name = record["table_name"].as<std::string>();
+          std::string vacuum_command = "VACUUM FULL ANALYZE " + table_name;
+          ilog("Performing vacuum: ${vacuum_command}", ("vacuum_command", vacuum_command));
+          tx.exec(vacuum_command);
+          tx.exec("UPDATE hafd.vacuum_requests SET status = 'vacuumed', last_vacuumed_time = NOW() WHERE table_name = '" + table_name + "';");
+        }
+        return data_processor::data_processing_status();
+      },
+      nullptr,
+      theApp
+    );
+
+    vacuum_requests_checker.trigger(data_processor::data_chunk_ptr(), 0);
+    vacuum_requests_checker.join();
+
     // Check for tables with missing indexes that are not currently being created
     queries_commit_data_processor missing_indexes_checker(
       _db_url,
-- 
GitLab


From 4c75411d5efe185de68e1efb21a306ff1e7e7dd9 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Tue, 10 Dec 2024 01:42:13 +0000
Subject: [PATCH 02/33] add extension config dump call

---
 src/hive_fork_manager/hived_api_impl_indexes.sql | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/hive_fork_manager/hived_api_impl_indexes.sql b/src/hive_fork_manager/hived_api_impl_indexes.sql
index f35ec5ac4..d48892cfb 100644
--- a/src/hive_fork_manager/hived_api_impl_indexes.sql
+++ b/src/hive_fork_manager/hived_api_impl_indexes.sql
@@ -21,3 +21,4 @@ CREATE TABLE IF NOT EXISTS hafd.vacuum_requests (
     last_vacuumed_time timestamp,
     CONSTRAINT pk_hive_vacuum_requests UNIQUE( table_name)
 );
+SELECT pg_catalog.pg_extension_config_dump('hafd.vacuum_status', '');
-- 
GitLab


From 3e3408675dd32d16c175a911f690c6de9cc75d1f Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Mon, 9 Dec 2024 23:05:15 -0500
Subject: [PATCH 03/33] fix typo

---
 src/hive_fork_manager/hived_api_impl_indexes.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/hive_fork_manager/hived_api_impl_indexes.sql b/src/hive_fork_manager/hived_api_impl_indexes.sql
index d48892cfb..cd63f7315 100644
--- a/src/hive_fork_manager/hived_api_impl_indexes.sql
+++ b/src/hive_fork_manager/hived_api_impl_indexes.sql
@@ -21,4 +21,4 @@ CREATE TABLE IF NOT EXISTS hafd.vacuum_requests (
     last_vacuumed_time timestamp,
     CONSTRAINT pk_hive_vacuum_requests UNIQUE( table_name)
 );
-SELECT pg_catalog.pg_extension_config_dump('hafd.vacuum_status', '');
+SELECT pg_catalog.pg_extension_config_dump('hafd.vacuum_requests', '');
-- 
GitLab


From cc024fd0500512ff37228ed9ce519bc90d2ba742 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Tue, 10 Dec 2024 01:43:00 -0500
Subject: [PATCH 04/33] Grant permissions on vacuum_requests to
 hive_applications_group

---
 src/hive_fork_manager/authorization.sql | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/hive_fork_manager/authorization.sql b/src/hive_fork_manager/authorization.sql
index d8b928816..3f9cc9805 100644
--- a/src/hive_fork_manager/authorization.sql
+++ b/src/hive_fork_manager/authorization.sql
@@ -68,6 +68,7 @@ GRANT ALL ON hafd.contexts_attachment TO hive_applications_group;
 GRANT ALL ON hafd.registered_tables TO hive_applications_group;
 GRANT ALL ON hafd.triggers TO hive_applications_group;
 GRANT ALL ON hafd.state_providers_registered TO hive_applications_group;
+GRANT ALL ON hafd.vacuum_requests TO hive_applications_group;
 
 -- protect an application rows aginst other applications
 REVOKE UPDATE( is_forking, owner ) ON hafd.contexts FROM GROUP hive_applications_group;
-- 
GitLab


From 064f6d66daacbc3972af554fe6f6681b57412c57 Mon Sep 17 00:00:00 2001
From: Marcin Ickiewicz <mickiewicz@syncad.com>
Date: Tue, 10 Dec 2024 14:01:35 +0100
Subject: [PATCH 05/33] add test for vacuum checker

---
 .../hive_fork_manager/CMakeLists.txt          |  1 +
 .../app_api/vacuum_request.sql                | 37 +++++++++++++++++++
 2 files changed, 38 insertions(+)
 create mode 100644 tests/integration/functional/hive_fork_manager/app_api/vacuum_request.sql

diff --git a/tests/integration/functional/hive_fork_manager/CMakeLists.txt b/tests/integration/functional/hive_fork_manager/CMakeLists.txt
index 83e8af5bc..28010b436 100644
--- a/tests/integration/functional/hive_fork_manager/CMakeLists.txt
+++ b/tests/integration/functional/hive_fork_manager/CMakeLists.txt
@@ -277,6 +277,7 @@ ADD_TEST( NAME test_update_script
  ADD_SQL_FUNCTIONAL_TEST( app_api/app_context_create_and_hole_in_events.sql )
 
  ADD_SQL_FUNCTIONAL_TEST( app_api/operation_id.sql )
+ ADD_SQL_FUNCTIONAL_TEST(app_api/vacuum_request.sql)
 
  ADD_SQL_FUNCTIONAL_TEST( application_loop/validate_stages.sql )
  ADD_SQL_FUNCTIONAL_TEST( application_loop/current_stage.sql )
diff --git a/tests/integration/functional/hive_fork_manager/app_api/vacuum_request.sql b/tests/integration/functional/hive_fork_manager/app_api/vacuum_request.sql
new file mode 100644
index 000000000..3a89888c4
--- /dev/null
+++ b/tests/integration/functional/hive_fork_manager/app_api/vacuum_request.sql
@@ -0,0 +1,37 @@
+CREATE OR REPLACE PROCEDURE alice_test_given()
+    LANGUAGE 'plpgsql'
+AS
+$BODY$
+BEGIN
+    CREATE SCHEMA A;
+    PERFORM hive.app_create_context( _name =>  'context', _schema => 'a'  );
+    CREATE TABLE A.table1(id  INTEGER ) INHERITS( a.context );
+END;
+$BODY$
+;
+
+CREATE OR REPLACE PROCEDURE alice_impersonal_test_when()
+    LANGUAGE 'plpgsql'
+AS
+$BODY$
+BEGIN
+    PERFORM hive.request_table_vacuum('a.table1');
+END;
+$BODY$
+;
+
+CREATE OR REPLACE PROCEDURE alice_test_then()
+    LANGUAGE 'plpgsql'
+AS
+$BODY$
+BEGIN
+   ASSERT
+       (SELECT COUNT(*) FROM hafd.vacuum_requests WHERE table_name = 'a.table1' ) = 1
+        , 'a.table1 wrong number of requests';
+
+   ASSERT
+       (SELECT status FROM hafd.vacuum_requests WHERE table_name = 'a.table1' ) = 'requested'
+       , 'a.table1 status != requested';
+END;
+$BODY$
+;
-- 
GitLab


From 91ae15f7f791251246c5cff5ac7d1944ece95664 Mon Sep 17 00:00:00 2001
From: Marcin Ickiewicz <mickiewicz@syncad.com>
Date: Tue, 10 Dec 2024 16:29:07 +0100
Subject: [PATCH 06/33] vacuum executed in a separated connection

---
 hive                                     | 2 +-
 src/sql_serializer/indexes_controler.cpp | 8 ++++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/hive b/hive
index 2afff0e42..0037935d3 160000
--- a/hive
+++ b/hive
@@ -1 +1 @@
-Subproject commit 2afff0e42851ad0b58428abd690af9e648bf4745
+Subproject commit 0037935d34dfc75f7730fd77ddb17e7d81f0eb0a
diff --git a/src/sql_serializer/indexes_controler.cpp b/src/sql_serializer/indexes_controler.cpp
index 54bc9d1ae..25c93843b 100644
--- a/src/sql_serializer/indexes_controler.cpp
+++ b/src/sql_serializer/indexes_controler.cpp
@@ -184,15 +184,19 @@ void indexes_controler::poll_and_create_indexes() {
       _db_url,
       "Check for vacuum requests",
       "index_ctrl",
-      [](const data_processor::data_chunk_ptr&, transaction_controllers::transaction& tx) -> data_processor::data_processing_status {
+      [this](const data_processor::data_chunk_ptr&, transaction_controllers::transaction& tx) -> data_processor::data_processing_status {
         pqxx::result data = tx.exec(
           "SELECT table_name FROM hafd.vacuum_requests WHERE status = 'requested';"
         );
+        // workaround, open separate connections for non-transactional start vacuum
+        auto vacuum_connection = std::make_unique<pqxx::connection>(_db_url);
+        pqxx::nontransaction vacuum_txn(*vacuum_connection);
         for (const auto& record : data) {
           std::string table_name = record["table_name"].as<std::string>();
           std::string vacuum_command = "VACUUM FULL ANALYZE " + table_name;
           ilog("Performing vacuum: ${vacuum_command}", ("vacuum_command", vacuum_command));
-          tx.exec(vacuum_command);
+          vacuum_txn.exec(vacuum_command);
+
           tx.exec("UPDATE hafd.vacuum_requests SET status = 'vacuumed', last_vacuumed_time = NOW() WHERE table_name = '" + table_name + "';");
         }
         return data_processor::data_processing_status();
-- 
GitLab


From ec5b9a585f9f3533edf6626634a5c288e478bf2d Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Tue, 10 Dec 2024 17:10:59 -0500
Subject: [PATCH 07/33] udpate hive submodule

---
 hive | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hive b/hive
index 0037935d3..2afff0e42 160000
--- a/hive
+++ b/hive
@@ -1 +1 @@
-Subproject commit 0037935d34dfc75f7730fd77ddb17e7d81f0eb0a
+Subproject commit 2afff0e42851ad0b58428abd690af9e648bf4745
-- 
GitLab


From 3889cd2984b83e509b79ae1626ac58617d490636 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Tue, 10 Dec 2024 17:12:47 -0500
Subject: [PATCH 08/33] temporary logging increase

---
 src/sql_serializer/indexes_controler.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/sql_serializer/indexes_controler.cpp b/src/sql_serializer/indexes_controler.cpp
index 25c93843b..8981a9674 100644
--- a/src/sql_serializer/indexes_controler.cpp
+++ b/src/sql_serializer/indexes_controler.cpp
@@ -177,7 +177,7 @@ void indexes_controler::poll_and_create_indexes() {
   std::mutex mtx; // Protects threads_to_delete
 
   while (!theApp.is_interrupt_request()) {
-    dlog("Polling for tables with missing indexes...");
+    ilog("Polling for tables with missing indexes...");
 
     // Check for requested table vacuums
     queries_commit_data_processor vacuum_requests_checker(
-- 
GitLab


From 22f8360ea063acb85daa4cd7e070efcdb94283bf Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Mon, 9 Dec 2024 19:45:34 -0500
Subject: [PATCH 09/33] Fix SQL typo

---
 src/hive_fork_manager/hived_api_impl_indexes.sql | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/hive_fork_manager/hived_api_impl_indexes.sql b/src/hive_fork_manager/hived_api_impl_indexes.sql
index cd63f7315..592fc8d34 100644
--- a/src/hive_fork_manager/hived_api_impl_indexes.sql
+++ b/src/hive_fork_manager/hived_api_impl_indexes.sql
@@ -14,10 +14,10 @@ CREATE TABLE IF NOT EXISTS hafd.indexes_constraints (
 SELECT pg_catalog.pg_extension_config_dump('hafd.indexes_constraints', '');
 
 -- Only one vacuum request per table, and this is only for 'vacuum full analyze' requests
-CREATE TYPE hafd.vacuum_status AS ENUM ('requested', 'vacuuming', 'vacuumed', 'failed');
+CREATE TYPE hafd.vacuum_status AS ENUM ('requested', 'vacuumed');
 CREATE TABLE IF NOT EXISTS hafd.vacuum_requests (
     table_name text NOT NULL,
-    hafd.vacuum_status,
+    status hafd.vacuum_status NOT NULL DEFAULT 'requested',
     last_vacuumed_time timestamp,
     CONSTRAINT pk_hive_vacuum_requests UNIQUE( table_name)
 );
-- 
GitLab


From 08a96de03b56e21733db447b95ac265d61c57130 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Tue, 10 Dec 2024 19:39:43 -0500
Subject: [PATCH 10/33] add app_ in front of new functions, needs discussion

---
 src/hive_fork_manager/hived_api_impl.sql                  | 8 ++++----
 .../hive_fork_manager/app_api/vacuum_request.sql          | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/hive_fork_manager/hived_api_impl.sql b/src/hive_fork_manager/hived_api_impl.sql
index cf85c207a..41433ecc7 100644
--- a/src/hive_fork_manager/hived_api_impl.sql
+++ b/src/hive_fork_manager/hived_api_impl.sql
@@ -786,7 +786,7 @@ END;
 $BODY$
 ;
 
-CREATE OR REPLACE FUNCTION hive.request_table_vacuum(
+CREATE OR REPLACE FUNCTION hive.app_request_table_vacuum(
     _table_name TEXT,
     _min_interval INTERVAL DEFAULT NULL
 )
@@ -819,7 +819,7 @@ END;
 $BODY$
 ;
 
-CREATE OR REPLACE FUNCTION hive.wait_for_table_vacuum(
+CREATE OR REPLACE FUNCTION hive.app_wait_for_table_vacuum(
     _table_name TEXT
 )
 RETURNS void
@@ -857,10 +857,10 @@ AS
 $BODY$
 BEGIN
     -- Request a vacuum for the table hafbe_app_keyauth_a
-    PERFORM hive.request_table_vacuum('hafbe_app_keyauth_a', '1 hour'::INTERVAL);
+    PERFORM hive.app_request_table_vacuum('hafbe_app_keyauth_a', '1 hour'::INTERVAL);
 
     -- Wait for the vacuum to complete
-    PERFORM hive.wait_for_table_vacuum('hafbe_app_keyauth_a');
+    PERFORM hive.app_wait_for_table_vacuum('hafbe_app_keyauth_a');
 
     RAISE NOTICE 'Test for vacuum functions on table hafbe_app_keyauth_a completed successfully.';
 END;
diff --git a/tests/integration/functional/hive_fork_manager/app_api/vacuum_request.sql b/tests/integration/functional/hive_fork_manager/app_api/vacuum_request.sql
index 3a89888c4..00200ab5f 100644
--- a/tests/integration/functional/hive_fork_manager/app_api/vacuum_request.sql
+++ b/tests/integration/functional/hive_fork_manager/app_api/vacuum_request.sql
@@ -15,7 +15,7 @@ CREATE OR REPLACE PROCEDURE alice_impersonal_test_when()
 AS
 $BODY$
 BEGIN
-    PERFORM hive.request_table_vacuum('a.table1');
+    PERFORM hive.app_request_table_vacuum('a.table1');
 END;
 $BODY$
 ;
-- 
GitLab


From fbeba66c2849de6160d41f3429b5035dbc70be65 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Tue, 10 Dec 2024 21:57:43 -0500
Subject: [PATCH 11/33] add sleeps and log when we sleep

---
 src/hive_fork_manager/hived_api_impl.sql | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/hive_fork_manager/hived_api_impl.sql b/src/hive_fork_manager/hived_api_impl.sql
index 41433ecc7..e392940a5 100644
--- a/src/hive_fork_manager/hived_api_impl.sql
+++ b/src/hive_fork_manager/hived_api_impl.sql
@@ -698,7 +698,8 @@ BEGIN
         SELECT 1
         FROM hafd.indexes_constraints
         WHERE contexts @> ARRAY[(SELECT id FROM hafd.contexts WHERE name = _app_context)] AND status <> 'created'
-        );
+        RAISE NOTICE 'Sleeping for 10 seconds waiting for indexes to be created';
+        PERFORM pg_sleep(10);      );
     END LOOP;
 
 
@@ -839,8 +840,10 @@ BEGIN
             SELECT 1
             FROM hafd.vacuum_requests
             WHERE table_name = _table_name
-            AND status <> 'vacuumed'
+            AND status <> 'vacuumed'            
         );
+        RAISE NOTICE 'Sleeping for 10 seconds waiting for vacuum to be done';
+        PERFORM pg_sleep(10);
     END LOOP;
 
     __end_time := clock_timestamp();
-- 
GitLab


From 88e3923d3b97ff1d9c3bd4363b0d9e3b2557e342 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Tue, 10 Dec 2024 22:01:06 -0500
Subject: [PATCH 12/33] Add logging for vacuum process in indexes_controler

---
 src/sql_serializer/indexes_controler.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/sql_serializer/indexes_controler.cpp b/src/sql_serializer/indexes_controler.cpp
index 8981a9674..4e7ed34b0 100644
--- a/src/sql_serializer/indexes_controler.cpp
+++ b/src/sql_serializer/indexes_controler.cpp
@@ -189,6 +189,7 @@ void indexes_controler::poll_and_create_indexes() {
           "SELECT table_name FROM hafd.vacuum_requests WHERE status = 'requested';"
         );
         // workaround, open separate connections for non-transactional start vacuum
+        ilog("Checking for tables to vacuum...");
         auto vacuum_connection = std::make_unique<pqxx::connection>(_db_url);
         pqxx::nontransaction vacuum_txn(*vacuum_connection);
         for (const auto& record : data) {
@@ -196,7 +197,7 @@ void indexes_controler::poll_and_create_indexes() {
           std::string vacuum_command = "VACUUM FULL ANALYZE " + table_name;
           ilog("Performing vacuum: ${vacuum_command}", ("vacuum_command", vacuum_command));
           vacuum_txn.exec(vacuum_command);
-
+          ilog("Vacuumed table: ${table_name}", ("table_name", table_name));
           tx.exec("UPDATE hafd.vacuum_requests SET status = 'vacuumed', last_vacuumed_time = NOW() WHERE table_name = '" + table_name + "';");
         }
         return data_processor::data_processing_status();
-- 
GitLab


From 5021f683d9d136dceea0b399b7fad7940b6903e1 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Tue, 10 Dec 2024 22:17:22 -0500
Subject: [PATCH 13/33] fix sleep typo

---
 src/hive_fork_manager/hived_api_impl.sql | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/hive_fork_manager/hived_api_impl.sql b/src/hive_fork_manager/hived_api_impl.sql
index e392940a5..764b0eba4 100644
--- a/src/hive_fork_manager/hived_api_impl.sql
+++ b/src/hive_fork_manager/hived_api_impl.sql
@@ -699,7 +699,8 @@ BEGIN
         FROM hafd.indexes_constraints
         WHERE contexts @> ARRAY[(SELECT id FROM hafd.contexts WHERE name = _app_context)] AND status <> 'created'
         RAISE NOTICE 'Sleeping for 10 seconds waiting for indexes to be created';
-        PERFORM pg_sleep(10);      );
+        );
+        PERFORM pg_sleep(10);
     END LOOP;
 
 
-- 
GitLab


From b11c6ccb30691a3b7f307c05e708ea8108ebd190 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Tue, 10 Dec 2024 22:47:43 -0500
Subject: [PATCH 14/33] forgot to commit this part

---
 src/hive_fork_manager/hived_api_impl.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/hive_fork_manager/hived_api_impl.sql b/src/hive_fork_manager/hived_api_impl.sql
index 764b0eba4..5c2e90b1d 100644
--- a/src/hive_fork_manager/hived_api_impl.sql
+++ b/src/hive_fork_manager/hived_api_impl.sql
@@ -698,7 +698,7 @@ BEGIN
         SELECT 1
         FROM hafd.indexes_constraints
         WHERE contexts @> ARRAY[(SELECT id FROM hafd.contexts WHERE name = _app_context)] AND status <> 'created'
-        RAISE NOTICE 'Sleeping for 10 seconds waiting for indexes to be created';
+        RAISE NOTICE 'Sleeping for 10 seconds waiting for indexes to be created'
         );
         PERFORM pg_sleep(10);
     END LOOP;
-- 
GitLab


From 56de0453daba89f1c7746acaed27c31538672836 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Tue, 10 Dec 2024 23:01:04 -0500
Subject: [PATCH 15/33] fix sleep typo

---
 src/hive_fork_manager/hived_api_impl.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/hive_fork_manager/hived_api_impl.sql b/src/hive_fork_manager/hived_api_impl.sql
index 5c2e90b1d..96c5acba0 100644
--- a/src/hive_fork_manager/hived_api_impl.sql
+++ b/src/hive_fork_manager/hived_api_impl.sql
@@ -698,8 +698,8 @@ BEGIN
         SELECT 1
         FROM hafd.indexes_constraints
         WHERE contexts @> ARRAY[(SELECT id FROM hafd.contexts WHERE name = _app_context)] AND status <> 'created'
-        RAISE NOTICE 'Sleeping for 10 seconds waiting for indexes to be created'
         );
+        RAISE NOTICE 'Sleeping for 10 seconds waiting for indexes to be created';
         PERFORM pg_sleep(10);
     END LOOP;
 
-- 
GitLab


From 5b2a1a73530b1ef21e275a75f2101c64c0a7ebbb Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Wed, 11 Dec 2024 00:01:34 -0500
Subject: [PATCH 16/33] add logging for thread that runs vacuums

---
 src/sql_serializer/indexes_controler.cpp | 37 +++++++++++++++---------
 1 file changed, 24 insertions(+), 13 deletions(-)

diff --git a/src/sql_serializer/indexes_controler.cpp b/src/sql_serializer/indexes_controler.cpp
index 4e7ed34b0..370c2d685 100644
--- a/src/sql_serializer/indexes_controler.cpp
+++ b/src/sql_serializer/indexes_controler.cpp
@@ -185,21 +185,32 @@ void indexes_controler::poll_and_create_indexes() {
       "Check for vacuum requests",
       "index_ctrl",
       [this](const data_processor::data_chunk_ptr&, transaction_controllers::transaction& tx) -> data_processor::data_processing_status {
-        pqxx::result data = tx.exec(
-          "SELECT table_name FROM hafd.vacuum_requests WHERE status = 'requested';"
-        );
-        // workaround, open separate connections for non-transactional start vacuum
         ilog("Checking for tables to vacuum...");
-        auto vacuum_connection = std::make_unique<pqxx::connection>(_db_url);
-        pqxx::nontransaction vacuum_txn(*vacuum_connection);
-        for (const auto& record : data) {
-          std::string table_name = record["table_name"].as<std::string>();
-          std::string vacuum_command = "VACUUM FULL ANALYZE " + table_name;
-          ilog("Performing vacuum: ${vacuum_command}", ("vacuum_command", vacuum_command));
-          vacuum_txn.exec(vacuum_command);
-          ilog("Vacuumed table: ${table_name}", ("table_name", table_name));
-          tx.exec("UPDATE hafd.vacuum_requests SET status = 'vacuumed', last_vacuumed_time = NOW() WHERE table_name = '" + table_name + "';");
+        pqxx::result data;
+        try { data = tx.exec("SELECT table_name FROM hafd.vacuum_requests WHERE status = 'requested';"); } 
+        catch (const pqxx::pqxx_exception& e) 
+        {
+            elog("Error while checking for vacuum requests: ${e}", ("e", e.what()));
+            return data_processor::data_processing_status();
         }
+        ilog("Found ${count} tables with vacuum requests.", ("count", data.size()));
+
+        // workaround, open separate connections for non-transactional start vacuum
+        try 
+        {
+          auto vacuum_connection = std::make_unique<pqxx::connection>(_db_url);
+          pqxx::nontransaction vacuum_txn(*vacuum_connection);
+          for (const auto& record : data) {
+            std::string table_name = record["table_name"].as<std::string>();
+            std::string vacuum_command = "VACUUM FULL ANALYZE " + table_name;
+            ilog("Performing vacuum: ${vacuum_command}", ("vacuum_command", vacuum_command));
+            vacuum_txn.exec(vacuum_command);
+            ilog("Vacuumed table: ${table_name}", ("table_name", table_name));
+            tx.exec("UPDATE hafd.vacuum_requests SET status = 'vacuumed', last_vacuumed_time = NOW() WHERE table_name = '" + table_name + "';");
+            ilog("Updated vacuum status for table: ${table_name}", ("table_name", table_name));
+            }
+        } 
+        catch (const pqxx::pqxx_exception& e) { elog("Error while vacuuming tables: ${e}", ("e", e.what())); }
         return data_processor::data_processing_status();
       },
       nullptr,
-- 
GitLab


From 42ced3518e4db72183394ae95753b5cb1dbfaf05 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Wed, 11 Dec 2024 01:00:42 -0500
Subject: [PATCH 17/33] use proper exception class

---
 src/sql_serializer/indexes_controler.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/sql_serializer/indexes_controler.cpp b/src/sql_serializer/indexes_controler.cpp
index 370c2d685..5700f2a49 100644
--- a/src/sql_serializer/indexes_controler.cpp
+++ b/src/sql_serializer/indexes_controler.cpp
@@ -188,7 +188,7 @@ void indexes_controler::poll_and_create_indexes() {
         ilog("Checking for tables to vacuum...");
         pqxx::result data;
         try { data = tx.exec("SELECT table_name FROM hafd.vacuum_requests WHERE status = 'requested';"); } 
-        catch (const pqxx::pqxx_exception& e) 
+        catch (const pqxx::sql_error& e) 
         {
             elog("Error while checking for vacuum requests: ${e}", ("e", e.what()));
             return data_processor::data_processing_status();
@@ -210,7 +210,7 @@ void indexes_controler::poll_and_create_indexes() {
             ilog("Updated vacuum status for table: ${table_name}", ("table_name", table_name));
             }
         } 
-        catch (const pqxx::pqxx_exception& e) { elog("Error while vacuuming tables: ${e}", ("e", e.what())); }
+        catch (const pqxx::sql_error& e) { elog("Error while vacuuming tables: ${e}", ("e", e.what())); }
         return data_processor::data_processing_status();
       },
       nullptr,
-- 
GitLab


From 77503728ac08c1aef58896a3ea617f7abe6eef35 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Wed, 11 Dec 2024 21:12:19 -0500
Subject: [PATCH 18/33] add timing and improved logging for index restoration
 and vacuum processes

---
 src/hive_fork_manager/hived_api_impl.sql | 20 +++++++++++++-------
 src/sql_serializer/indexes_controler.cpp | 20 +++++++++++++-------
 2 files changed, 26 insertions(+), 14 deletions(-)

diff --git a/src/hive_fork_manager/hived_api_impl.sql b/src/hive_fork_manager/hived_api_impl.sql
index 96c5acba0..e561deca4 100644
--- a/src/hive_fork_manager/hived_api_impl.sql
+++ b/src/hive_fork_manager/hived_api_impl.sql
@@ -526,7 +526,9 @@ $function$
 DECLARE
   __command TEXT;
   __original_command TEXT;
-  __cursor REFCURSOR;
+  __start_time TIMESTAMP;
+  __end_time TIMESTAMP;
+  __duration INTERVAL;
 BEGIN
 
   IF _table_name = 'hafd.account_operations' THEN
@@ -534,10 +536,9 @@ BEGIN
   END IF;
 
   --restoring indexes, primary keys, unique constraints
-  OPEN __cursor FOR ( SELECT command FROM hafd.indexes_constraints WHERE table_name = _table_name AND is_foreign_key = FALSE AND status = 'missing' );
+  FOR __original_command IN
+    SELECT command FROM hafd.indexes_constraints WHERE table_name = _table_name AND is_foreign_key = FALSE AND status = 'missing'
   LOOP
-    FETCH __cursor INTO __original_command;
-    EXIT WHEN NOT FOUND;
     IF concurrent THEN
       -- Modify the command to include CONCURRENTLY
       __command := regexp_replace(__original_command, 'CREATE INDEX', 'CREATE INDEX CONCURRENTLY', 'i');
@@ -546,10 +547,15 @@ BEGIN
     END IF;
     RAISE NOTICE 'Restoring index: %', __command;
     UPDATE hafd.indexes_constraints SET status = 'creating' WHERE command = __original_command;
+    COMMIT;
+    __start_time := clock_timestamp();
     EXECUTE __command;
+    __end_time := clock_timestamp();
+    __duration := __end_time - __start_time;
+    RAISE NOTICE 'Index % created in % seconds', __command, EXTRACT(EPOCH FROM __duration);
     UPDATE hafd.indexes_constraints SET status = 'created' WHERE command = __original_command;
+    COMMIT;
   END LOOP;
-  CLOSE __cursor;
 
   EXECUTE format( 'ANALYZE %s',  _table_name );
 
@@ -843,8 +849,8 @@ BEGIN
             WHERE table_name = _table_name
             AND status <> 'vacuumed'            
         );
-        RAISE NOTICE 'Sleeping for 10 seconds waiting for vacuum to be done';
-        PERFORM pg_sleep(10);
+        RAISE NOTICE 'Sleeping for 1 seconds waiting for vacuum to be done';
+        PERFORM pg_sleep(1);
     END LOOP;
 
     __end_time := clock_timestamp();
diff --git a/src/sql_serializer/indexes_controler.cpp b/src/sql_serializer/indexes_controler.cpp
index 5700f2a49..dba396808 100644
--- a/src/sql_serializer/indexes_controler.cpp
+++ b/src/sql_serializer/indexes_controler.cpp
@@ -177,7 +177,7 @@ void indexes_controler::poll_and_create_indexes() {
   std::mutex mtx; // Protects threads_to_delete
 
   while (!theApp.is_interrupt_request()) {
-    ilog("Polling for tables with missing indexes...");
+    ilog("Polling for tables to vacuum or with missing indexes...");
 
     // Check for requested table vacuums
     queries_commit_data_processor vacuum_requests_checker(
@@ -185,7 +185,7 @@ void indexes_controler::poll_and_create_indexes() {
       "Check for vacuum requests",
       "index_ctrl",
       [this](const data_processor::data_chunk_ptr&, transaction_controllers::transaction& tx) -> data_processor::data_processing_status {
-        ilog("Checking for tables to vacuum...");
+        dlog("Checking for tables to vacuum...");
         pqxx::result data;
         try { data = tx.exec("SELECT table_name FROM hafd.vacuum_requests WHERE status = 'requested';"); } 
         catch (const pqxx::sql_error& e) 
@@ -193,7 +193,7 @@ void indexes_controler::poll_and_create_indexes() {
             elog("Error while checking for vacuum requests: ${e}", ("e", e.what()));
             return data_processor::data_processing_status();
         }
-        ilog("Found ${count} tables with vacuum requests.", ("count", data.size()));
+        dlog("Found ${count} tables with vacuum requests.", ("count", data.size()));
 
         // workaround, open separate connections for non-transactional start vacuum
         try 
@@ -204,10 +204,13 @@ void indexes_controler::poll_and_create_indexes() {
             std::string table_name = record["table_name"].as<std::string>();
             std::string vacuum_command = "VACUUM FULL ANALYZE " + table_name;
             ilog("Performing vacuum: ${vacuum_command}", ("vacuum_command", vacuum_command));
+            auto start_time = fc::time_point::now();
             vacuum_txn.exec(vacuum_command);
-            ilog("Vacuumed table: ${table_name}", ("table_name", table_name));
+            auto end_time = fc::time_point::now();
+            fc::microseconds vacuum_duration = end_time - start_time;
+            ilog("Vacuumed table: ${table_name} in ${duration} seconds", ("table_name", table_name)("duration", vacuum_duration.to_seconds()));
             tx.exec("UPDATE hafd.vacuum_requests SET status = 'vacuumed', last_vacuumed_time = NOW() WHERE table_name = '" + table_name + "';");
-            ilog("Updated vacuum status for table: ${table_name}", ("table_name", table_name));
+            dlog("Updated vacuum status for table: ${table_name}", ("table_name", table_name));
             }
         } 
         catch (const pqxx::sql_error& e) { elog("Error while vacuuming tables: ${e}", ("e", e.what())); }
@@ -250,12 +253,15 @@ void indexes_controler::poll_and_create_indexes() {
 
           ilog("NOTE: Starting a new thread to create indexes for table: ${table_name}", ("table_name", table_name));
           active_threads[table_name] = std::thread([this, table_name, &threads_to_delete, &mtx]() {
+            auto start_time = fc::time_point::now();
             auto processor = start_commit_sql(true, "hive.restore_indexes( '" + table_name + "', FALSE )", "restore indexes");
             processor->join();
-            ilog("Finished creating indexes for table: ${table_name}", ("table_name", table_name));
+            auto end_time = fc::time_point::now();
+            fc::microseconds index_creation_duration = end_time - start_time;
+            ilog("Finished creating indexes for table: ${table_name} in ${duration} seconds", ("table_name", table_name)("duration", index_creation_duration.to_seconds()));
             std::lock_guard g(mtx);
             threads_to_delete.insert(table_name); // Mark the thread for deletion
-            ilog("Thread for table: ${table_name} has been marked for deletion", ("table_name", table_name));
+            dlog("Thread for table: ${table_name} has been marked for deletion", ("table_name", table_name));
           });
         }
         dlog("Finished processing tables with missing indexes.");
-- 
GitLab


From 9aba4ae803a4d6038002bff5c5814d1635de6183 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Wed, 11 Dec 2024 22:16:34 -0500
Subject: [PATCH 19/33] can't commit in restore_indexes, consider making a
 procedure

---
 src/hive_fork_manager/hived_api_impl.sql | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/hive_fork_manager/hived_api_impl.sql b/src/hive_fork_manager/hived_api_impl.sql
index e561deca4..ca9aece5a 100644
--- a/src/hive_fork_manager/hived_api_impl.sql
+++ b/src/hive_fork_manager/hived_api_impl.sql
@@ -547,14 +547,12 @@ BEGIN
     END IF;
     RAISE NOTICE 'Restoring index: %', __command;
     UPDATE hafd.indexes_constraints SET status = 'creating' WHERE command = __original_command;
-    COMMIT;
     __start_time := clock_timestamp();
     EXECUTE __command;
     __end_time := clock_timestamp();
     __duration := __end_time - __start_time;
     RAISE NOTICE 'Index % created in % seconds', __command, EXTRACT(EPOCH FROM __duration);
     UPDATE hafd.indexes_constraints SET status = 'created' WHERE command = __original_command;
-    COMMIT;
   END LOOP;
 
   EXECUTE format( 'ANALYZE %s',  _table_name );
-- 
GitLab


From 0b65aac02291fbb302a589e56439831ebb360f43 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Thu, 12 Dec 2024 18:03:05 -0500
Subject: [PATCH 20/33] create indexes concurrently in polling thread

---
 src/sql_serializer/indexes_controler.cpp | 198 ++++++++++++-----------
 1 file changed, 104 insertions(+), 94 deletions(-)

diff --git a/src/sql_serializer/indexes_controler.cpp b/src/sql_serializer/indexes_controler.cpp
index dba396808..0d2e7f1b0 100644
--- a/src/sql_serializer/indexes_controler.cpp
+++ b/src/sql_serializer/indexes_controler.cpp
@@ -11,6 +11,7 @@
 #include <mutex>
 #include <set>
 #include <map>
+#include <regex>
 
 namespace hive { namespace plugins { namespace sql_serializer {
 
@@ -171,117 +172,124 @@ indexes_controler::start_commit_sql( bool mode, const std::string& sql_function_
   return processor;
 }
 
-void indexes_controler::poll_and_create_indexes() {
+void indexes_controler::poll_and_create_indexes() 
+{
   std::map<std::string, std::thread> active_threads; // Doesn't need mutex, because it's modified by one thread at a time
   std::set<std::string> threads_to_delete;
   std::mutex mtx; // Protects threads_to_delete
 
-  while (!theApp.is_interrupt_request()) {
-    ilog("Polling for tables to vacuum or with missing indexes...");
-
-    // Check for requested table vacuums
-    queries_commit_data_processor vacuum_requests_checker(
-      _db_url,
-      "Check for vacuum requests",
-      "index_ctrl",
-      [this](const data_processor::data_chunk_ptr&, transaction_controllers::transaction& tx) -> data_processor::data_processing_status {
-        dlog("Checking for tables to vacuum...");
-        pqxx::result data;
-        try { data = tx.exec("SELECT table_name FROM hafd.vacuum_requests WHERE status = 'requested';"); } 
-        catch (const pqxx::sql_error& e) 
+  const std::string thread_name = "haf_monitor";
+  fc::set_thread_name(thread_name.c_str());
+  fc::thread::current().set_name(thread_name);
+
+
+  while (!theApp.is_interrupt_request()) 
+  {    
+    ilog("Checking for table vacuum requests...");
+    pqxx::connection conn(_db_url);
+    pqxx::nontransaction tx(conn);
+    try 
+    { 
+      pqxx::result data = tx.exec("SELECT table_name FROM hafd.vacuum_requests WHERE status = 'requested';"); 
+      ilog("Found ${count} tables with vacuum requests.", ("count", data.size()));
+      
+      try 
+      {
+        for (const auto& record : data) 
         {
-            elog("Error while checking for vacuum requests: ${e}", ("e", e.what()));
-            return data_processor::data_processing_status();
+          std::string table_name = record["table_name"].as<std::string>();
+          std::string vacuum_command = "VACUUM FULL ANALYZE " + table_name;
+          ilog("Performing vacuum: ${vacuum_command}", (vacuum_command));
+          auto start_time = fc::time_point::now();
+          //vacuum_txn.exec(vacuum_command);
+          tx.exec(vacuum_command);
+          auto end_time = fc::time_point::now();
+          fc::microseconds vacuum_duration = end_time - start_time;
+          ilog("Vacuumed table: ${table_name} in ${duration} seconds", (table_name)("duration", vacuum_duration.to_seconds()));
+          tx.exec("UPDATE hafd.vacuum_requests SET status = 'vacuumed', last_vacuumed_time = NOW() WHERE table_name = '" + table_name + "';");
+          ilog("Updated vacuum status for table: ${table_name}", ("table_name", table_name));
         }
-        dlog("Found ${count} tables with vacuum requests.", ("count", data.size()));
-
-        // workaround, open separate connections for non-transactional start vacuum
-        try 
-        {
-          auto vacuum_connection = std::make_unique<pqxx::connection>(_db_url);
-          pqxx::nontransaction vacuum_txn(*vacuum_connection);
-          for (const auto& record : data) {
-            std::string table_name = record["table_name"].as<std::string>();
-            std::string vacuum_command = "VACUUM FULL ANALYZE " + table_name;
-            ilog("Performing vacuum: ${vacuum_command}", ("vacuum_command", vacuum_command));
-            auto start_time = fc::time_point::now();
-            vacuum_txn.exec(vacuum_command);
-            auto end_time = fc::time_point::now();
-            fc::microseconds vacuum_duration = end_time - start_time;
-            ilog("Vacuumed table: ${table_name} in ${duration} seconds", ("table_name", table_name)("duration", vacuum_duration.to_seconds()));
-            tx.exec("UPDATE hafd.vacuum_requests SET status = 'vacuumed', last_vacuumed_time = NOW() WHERE table_name = '" + table_name + "';");
-            dlog("Updated vacuum status for table: ${table_name}", ("table_name", table_name));
-            }
-        } 
-        catch (const pqxx::sql_error& e) { elog("Error while vacuuming tables: ${e}", ("e", e.what())); }
-        return data_processor::data_processing_status();
-      },
-      nullptr,
-      theApp
-    );
-
-    vacuum_requests_checker.trigger(data_processor::data_chunk_ptr(), 0);
-    vacuum_requests_checker.join();
+      } 
+      catch (const pqxx::sql_error& e) { elog("Error while vacuuming tables: ${e}", ("e", e.what())); }
+    } 
+    catch (const pqxx::sql_error& e) 
+    {
+      elog("Error while checking for vacuum requests: ${e}", ("e", e.what()));
+    }
 
     // Check for tables with missing indexes that are not currently being created
-    queries_commit_data_processor missing_indexes_checker(
-      _db_url,
-      "Check for missing indexes",
-      "index_ctrl",
-      [this, &active_threads, &threads_to_delete, &mtx](const data_processor::data_chunk_ptr&, transaction_controllers::transaction& tx) -> data_processor::data_processing_status {
-        dlog("Executing query to find tables with missing indexes...");
-        pqxx::result data = tx.exec(
-          "SELECT DISTINCT table_name "
-          "FROM hafd.indexes_constraints "
-          "WHERE status = 'missing' "
-          "AND table_name NOT IN ("
-          "  SELECT DISTINCT table_name "
-          "  FROM hafd.indexes_constraints "
-          "  WHERE status = 'creating'"
-          ");"
+    try 
+    { 
+      ilog("Executing query to find tables with missing indexes...");
+      pqxx::result data = tx.exec(
+            "SELECT DISTINCT table_name "
+            "FROM hafd.indexes_constraints "
+            "WHERE status = 'missing' "
+            "AND table_name NOT IN ("
+            "  SELECT DISTINCT table_name "
+            "  FROM hafd.indexes_constraints "
+            "  WHERE status = 'creating'"
+            ");"
         );
-        dlog("Query executed. Found ${count} tables with missing indexes.", ("count", data.size()));
-
-        for (const auto& record : data) {
-          std::string table_name = record["table_name"].as<std::string>();
-          dlog("Processing table: ${table_name}", ("table_name", table_name));
-          // Check if a thread is already running for this table
-          if (active_threads.find(table_name) != active_threads.end() && active_threads[table_name].joinable()) {
-            ilog("A thread is already running for table: ${table_name}", ("table_name", table_name));
-            continue;
-          }
+      ilog("Query executed. Found ${count} tables with missing indexes.", ("count", data.size()));
+      for (const auto& record : data) //iterate over tables with missing indexes
+      {
+        std::string table_name = record["table_name"].as<std::string>();
+        dlog("Processing table: ${table_name}", ("table_name", table_name));
+        // Check if a thread is already running for this table
+        if (active_threads.find(table_name) != active_threads.end() && active_threads[table_name].joinable()) 
+        {
+          ilog("A thread is already running for table: ${table_name}", ("table_name", table_name));
+          continue; //check the next table
+        }
 
-          ilog("NOTE: Starting a new thread to create indexes for table: ${table_name}", ("table_name", table_name));
-          active_threads[table_name] = std::thread([this, table_name, &threads_to_delete, &mtx]() {
+        ilog("NOTE: Starting a new thread to create indexes for table: ${table_name}", ("table_name", table_name));
+        active_threads[table_name] = std::thread([this, table_name, &threads_to_delete, &mtx]() 
+        {
+          const std::string thread_name = table_name;
+          if (thread_name.size() > 16)
+            thread_name.resize(16);
+          fc::set_thread_name(thread_name.c_str());
+          fc::thread::current().set_name(thread_name);
+
+          pqxx::connection conn(_db_url);
+          pqxx::nontransaction tx(conn);
+          pqxx::result data = tx.exec("SELECT command FROM hafd.indexes_constraints WHERE status = 'missing' AND table_name = '" + table_name + "';");
+          for (const auto& index : data) //iterate over missing indexes and create them concurrently
+          {
+            std::string command = index["command"].as<std::string>();
+            std::regex create_index_regex(R"((CREATE\s+UNIQUE\s+INDEX|CREATE\s+INDEX))", std::regex::icase);
+            command = std::regex_replace(command, create_index_regex, "$& CONCURRENTLY");
+            ilog("Creating index: ${command}", (command));
             auto start_time = fc::time_point::now();
-            auto processor = start_commit_sql(true, "hive.restore_indexes( '" + table_name + "', FALSE )", "restore indexes");
-            processor->join();
+            tx.exec(command);
             auto end_time = fc::time_point::now();
             fc::microseconds index_creation_duration = end_time - start_time;
-            ilog("Finished creating indexes for table: ${table_name} in ${duration} seconds", ("table_name", table_name)("duration", index_creation_duration.to_seconds()));
-            std::lock_guard g(mtx);
-            threads_to_delete.insert(table_name); // Mark the thread for deletion
-            dlog("Thread for table: ${table_name} has been marked for deletion", ("table_name", table_name));
-          });
-        }
-        dlog("Finished processing tables with missing indexes.");
-        return data_processor::data_processing_status();
-      },
-      nullptr,
-      theApp
-    );
+            ilog("Finished creating index for table: ${table_name} in ${duration} seconds", (table_name)("duration", index_creation_duration.to_seconds()));
+          }
+          ilog("Finished creating all indexes for table: ${table_name}", (table_name));
+          std::lock_guard g(mtx);
+          threads_to_delete.insert(table_name); // Mark the thread for deletion
+          ilog("Thread for table: ${table_name} has been marked for deletion", (table_name));
+        });
+      } //end for tables with missing indexes
+      dlog("Finished polling for tables with missing indexes, sleep for 10s.");
+    }
+    catch (const pqxx::sql_error& e) 
+    {
+      elog("Error while checking for missing indexes: ${e}", ("e", e.what()));
+    }
 
-    missing_indexes_checker.trigger(data_processor::data_chunk_ptr(), 0);
-    missing_indexes_checker.join();
-    dlog("Finished polling for tables with missing indexes, sleep for 10s.");
     // Sleep for 10 seconds before polling again
     fc::usleep(fc::seconds(10));
 
     // Delete threads marked for deletion
     {
       std::lock_guard g(mtx);
-      for (const auto& table_name : threads_to_delete) {
-        if (active_threads[table_name].joinable()) {
+      for (const auto& table_name : threads_to_delete) 
+      {
+        if (active_threads[table_name].joinable()) 
+        {
           ilog("Joining thread for table: ${table_name}", ("table_name", table_name));
           active_threads[table_name].join();
         }
@@ -292,12 +300,14 @@ void indexes_controler::poll_and_create_indexes() {
   }
   ilog("Interrupt request received, stopping polling for tables with missing indexes.");
   // Join all remaining threads before exiting
-  for (auto& [table_name, thread] : active_threads) {
-    if (thread.joinable()) {
+  for (auto& [table_name, thread] : active_threads) 
+  {
+    if (thread.joinable()) 
+    {
       ilog("Joining thread for table: ${table_name}", ("table_name", table_name));
       thread.join();
     }
   }
-}
+} //end poll_and_create_indexes
 
 }}} // namespace hive{ plugins { sql_serializer
-- 
GitLab


From c2be6df65c503a9995f1946a879740883e32bae9 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Thu, 12 Dec 2024 18:22:00 -0500
Subject: [PATCH 21/33] add try catch around index creation step

---
 src/sql_serializer/indexes_controler.cpp | 25 ++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/src/sql_serializer/indexes_controler.cpp b/src/sql_serializer/indexes_controler.cpp
index 0d2e7f1b0..24612e9d6 100644
--- a/src/sql_serializer/indexes_controler.cpp
+++ b/src/sql_serializer/indexes_controler.cpp
@@ -246,7 +246,7 @@ void indexes_controler::poll_and_create_indexes()
         ilog("NOTE: Starting a new thread to create indexes for table: ${table_name}", ("table_name", table_name));
         active_threads[table_name] = std::thread([this, table_name, &threads_to_delete, &mtx]() 
         {
-          const std::string thread_name = table_name;
+          std::string thread_name = table_name;
           if (thread_name.size() > 16)
             thread_name.resize(16);
           fc::set_thread_name(thread_name.c_str());
@@ -257,15 +257,20 @@ void indexes_controler::poll_and_create_indexes()
           pqxx::result data = tx.exec("SELECT command FROM hafd.indexes_constraints WHERE status = 'missing' AND table_name = '" + table_name + "';");
           for (const auto& index : data) //iterate over missing indexes and create them concurrently
           {
-            std::string command = index["command"].as<std::string>();
-            std::regex create_index_regex(R"((CREATE\s+UNIQUE\s+INDEX|CREATE\s+INDEX))", std::regex::icase);
-            command = std::regex_replace(command, create_index_regex, "$& CONCURRENTLY");
-            ilog("Creating index: ${command}", (command));
-            auto start_time = fc::time_point::now();
-            tx.exec(command);
-            auto end_time = fc::time_point::now();
-            fc::microseconds index_creation_duration = end_time - start_time;
-            ilog("Finished creating index for table: ${table_name} in ${duration} seconds", (table_name)("duration", index_creation_duration.to_seconds()));
+            try 
+            { 
+              std::string command = index["command"].as<std::string>();
+              std::regex create_index_regex(R"((CREATE\s+UNIQUE\s+INDEX|CREATE\s+INDEX))", std::regex::icase);
+              command = std::regex_replace(command, create_index_regex, "$& CONCURRENTLY");
+              ilog("Creating index: ${command}", (command));
+              auto start_time = fc::time_point::now();
+              tx.exec(command);
+              auto end_time = fc::time_point::now();
+              fc::microseconds index_creation_duration = end_time - start_time;
+              ilog("Finished creating index for table: ${table_name} in ${duration} seconds", (table_name)("duration", index_creation_duration.to_seconds()));
+            }
+            catch (const pqxx::sql_error& e) { elog("Error while creating index: ${e}", ("e", e.what())); }
+            catch(std::exception& e ) { elog( e.what() ); }
           }
           ilog("Finished creating all indexes for table: ${table_name}", (table_name));
           std::lock_guard g(mtx);
-- 
GitLab


From 3982ab062c5c6551c0c343a2e795e51a77a2ae92 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Thu, 12 Dec 2024 21:39:40 -0500
Subject: [PATCH 22/33] update status when creating indexes

---
 src/sql_serializer/indexes_controler.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/sql_serializer/indexes_controler.cpp b/src/sql_serializer/indexes_controler.cpp
index 24612e9d6..95e6fcd66 100644
--- a/src/sql_serializer/indexes_controler.cpp
+++ b/src/sql_serializer/indexes_controler.cpp
@@ -263,11 +263,13 @@ void indexes_controler::poll_and_create_indexes()
               std::regex create_index_regex(R"((CREATE\s+UNIQUE\s+INDEX|CREATE\s+INDEX))", std::regex::icase);
               command = std::regex_replace(command, create_index_regex, "$& CONCURRENTLY");
               ilog("Creating index: ${command}", (command));
+              tx.exec("UPDATE hafd.indexes_constraints SET status = 'creating' WHERE command ='"+command+"';");
               auto start_time = fc::time_point::now();
               tx.exec(command);
               auto end_time = fc::time_point::now();
               fc::microseconds index_creation_duration = end_time - start_time;
               ilog("Finished creating index for table: ${table_name} in ${duration} seconds", (table_name)("duration", index_creation_duration.to_seconds()));
+              tx.exec("UPDATE hafd.indexes_constraints SET status = 'created' WHERE command ='"+command+"';");
             }
             catch (const pqxx::sql_error& e) { elog("Error while creating index: ${e}", ("e", e.what())); }
             catch(std::exception& e ) { elog( e.what() ); }
-- 
GitLab


From 6f3b5d7459c827057fe2bf851be9ff90533e5965 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Thu, 12 Dec 2024 22:35:51 -0500
Subject: [PATCH 23/33] remove unneeded functions

---
 src/hive_fork_manager/hived_api_impl.sql | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/hive_fork_manager/hived_api_impl.sql b/src/hive_fork_manager/hived_api_impl.sql
index ca9aece5a..594c2e5ab 100644
--- a/src/hive_fork_manager/hived_api_impl.sql
+++ b/src/hive_fork_manager/hived_api_impl.sql
@@ -824,7 +824,7 @@ BEGIN
 END;
 $BODY$
 ;
-
+/*
 CREATE OR REPLACE FUNCTION hive.app_wait_for_table_vacuum(
     _table_name TEXT
 )
@@ -874,3 +874,4 @@ BEGIN
 END;
 $BODY$
 ;
+*/
\ No newline at end of file
-- 
GitLab


From d4dfb2d739af6e33440881cfac7c5b42090257cd Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Thu, 12 Dec 2024 23:21:59 -0500
Subject: [PATCH 24/33] try to fix queries

---
 src/sql_serializer/indexes_controler.cpp | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/sql_serializer/indexes_controler.cpp b/src/sql_serializer/indexes_controler.cpp
index 95e6fcd66..78cd6cd94 100644
--- a/src/sql_serializer/indexes_controler.cpp
+++ b/src/sql_serializer/indexes_controler.cpp
@@ -259,17 +259,20 @@ void indexes_controler::poll_and_create_indexes()
           {
             try 
             { 
-              std::string command = index["command"].as<std::string>();
+              std::string index_constraint_name = index["index_constraint_name"].as<std::string>();
+              std::string original_command = index["command"].as<std::string>();
               std::regex create_index_regex(R"((CREATE\s+UNIQUE\s+INDEX|CREATE\s+INDEX))", std::regex::icase);
-              command = std::regex_replace(command, create_index_regex, "$& CONCURRENTLY");
+              std::string command = std::regex_replace(original_command, create_index_regex, "$& CONCURRENTLY");
+              std::string update_table = "UPDATE hafd.indexes_constraints SET status = 'creating' WHERE index_constraint_name ='"+index_constraint_name+"';"
+              idump(update_table);
+              tx.exec(update_table);
               ilog("Creating index: ${command}", (command));
-              tx.exec("UPDATE hafd.indexes_constraints SET status = 'creating' WHERE command ='"+command+"';");
               auto start_time = fc::time_point::now();
               tx.exec(command);
               auto end_time = fc::time_point::now();
               fc::microseconds index_creation_duration = end_time - start_time;
               ilog("Finished creating index for table: ${table_name} in ${duration} seconds", (table_name)("duration", index_creation_duration.to_seconds()));
-              tx.exec("UPDATE hafd.indexes_constraints SET status = 'created' WHERE command ='"+command+"';");
+              tx.exec("UPDATE hafd.indexes_constraints SET status = 'created' WHERE index_constraint_name ='"+index_constraint_name+"';");
             }
             catch (const pqxx::sql_error& e) { elog("Error while creating index: ${e}", ("e", e.what())); }
             catch(std::exception& e ) { elog( e.what() ); }
-- 
GitLab


From 3c3004be0c33e759c70bd9a17d7d3c0a36e8d8ee Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Thu, 12 Dec 2024 23:29:18 -0500
Subject: [PATCH 25/33] fix syntax error

---
 src/sql_serializer/indexes_controler.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/sql_serializer/indexes_controler.cpp b/src/sql_serializer/indexes_controler.cpp
index 78cd6cd94..b965ed732 100644
--- a/src/sql_serializer/indexes_controler.cpp
+++ b/src/sql_serializer/indexes_controler.cpp
@@ -264,7 +264,7 @@ void indexes_controler::poll_and_create_indexes()
               std::regex create_index_regex(R"((CREATE\s+UNIQUE\s+INDEX|CREATE\s+INDEX))", std::regex::icase);
               std::string command = std::regex_replace(original_command, create_index_regex, "$& CONCURRENTLY");
               std::string update_table = "UPDATE hafd.indexes_constraints SET status = 'creating' WHERE index_constraint_name ='"+index_constraint_name+"';"
-              idump(update_table);
+              idump((update_table));
               tx.exec(update_table);
               ilog("Creating index: ${command}", (command));
               auto start_time = fc::time_point::now();
-- 
GitLab


From 02fb62d1bd4756ecc32f853230d366ccbbfdd730 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Thu, 12 Dec 2024 23:41:13 -0500
Subject: [PATCH 26/33] fix typo

---
 src/sql_serializer/indexes_controler.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/sql_serializer/indexes_controler.cpp b/src/sql_serializer/indexes_controler.cpp
index b965ed732..4c15b3b12 100644
--- a/src/sql_serializer/indexes_controler.cpp
+++ b/src/sql_serializer/indexes_controler.cpp
@@ -263,8 +263,9 @@ void indexes_controler::poll_and_create_indexes()
               std::string original_command = index["command"].as<std::string>();
               std::regex create_index_regex(R"((CREATE\s+UNIQUE\s+INDEX|CREATE\s+INDEX))", std::regex::icase);
               std::string command = std::regex_replace(original_command, create_index_regex, "$& CONCURRENTLY");
-              std::string update_table = "UPDATE hafd.indexes_constraints SET status = 'creating' WHERE index_constraint_name ='"+index_constraint_name+"';"
-              idump((update_table));
+              std::string update_table = 
+                "UPDATE hafd.indexes_constraints SET status = 'creating' WHERE index_constraint_name ='" + index_constraint_name + "';";
+              ilog("SQL: ${update_table}",(update_table));
               tx.exec(update_table);
               ilog("Creating index: ${command}", (command));
               auto start_time = fc::time_point::now();
-- 
GitLab


From 01f1352fe6784ade04833fdf566836773bfe48fe Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Fri, 13 Dec 2024 00:17:25 -0500
Subject: [PATCH 27/33] forgot to fetch column

---
 src/sql_serializer/indexes_controler.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/sql_serializer/indexes_controler.cpp b/src/sql_serializer/indexes_controler.cpp
index 4c15b3b12..fa0c19be2 100644
--- a/src/sql_serializer/indexes_controler.cpp
+++ b/src/sql_serializer/indexes_controler.cpp
@@ -254,7 +254,7 @@ void indexes_controler::poll_and_create_indexes()
 
           pqxx::connection conn(_db_url);
           pqxx::nontransaction tx(conn);
-          pqxx::result data = tx.exec("SELECT command FROM hafd.indexes_constraints WHERE status = 'missing' AND table_name = '" + table_name + "';");
+          pqxx::result data = tx.exec("SELECT index_constraint_name, command FROM hafd.indexes_constraints WHERE status = 'missing' AND table_name = '" + table_name + "';");
           for (const auto& index : data) //iterate over missing indexes and create them concurrently
           {
             try 
-- 
GitLab


From 43e74088ac325604f2630ab9ac4e02ea557b980a Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Fri, 13 Dec 2024 01:51:32 -0500
Subject: [PATCH 28/33] add hive.check_if_registered_indexes_created

---
 src/hive_fork_manager/hived_api_impl.sql | 31 ++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/src/hive_fork_manager/hived_api_impl.sql b/src/hive_fork_manager/hived_api_impl.sql
index 594c2e5ab..2e16adfef 100644
--- a/src/hive_fork_manager/hived_api_impl.sql
+++ b/src/hive_fork_manager/hived_api_impl.sql
@@ -715,6 +715,37 @@ END;
 $BODY$
 ;
 
+CREATE OR REPLACE FUNCTION hive.check_if_registered_indexes_created(
+    _app_context TEXT
+)
+RETURNS BOOLEAN
+LANGUAGE plpgsql
+AS
+$BODY$
+DECLARE
+    __context_id INT;
+BEGIN
+    RAISE NOTICE 'Checking if registered indexes are created for context %', _app_context;
+    -- Lookup the context_id using context_name
+    SELECT id INTO __context_id
+    FROM hafd.contexts
+    WHERE name = _app_context;
+
+    -- Abort with an error message if no context_id is found
+    IF __context_id IS NULL THEN
+        RAISE EXCEPTION 'Context % not found in hafd.contexts', _app_context;
+    END IF;
+
+    -- Check if there are any indexes that are not created yet
+    RETURN NOT EXISTS (
+        SELECT 1
+        FROM hafd.indexes_constraints
+        WHERE contexts @> ARRAY[__context_id] AND status <> 'created'
+    );
+END;
+$BODY$
+;
+
 CREATE OR REPLACE FUNCTION hive.parse_create_index_command(
     create_index_command TEXT
 )
-- 
GitLab


From 1f36c645069a458cc2e601f303645b9720104b2d Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Fri, 13 Dec 2024 03:26:26 -0500
Subject: [PATCH 29/33] replace wait_till with check_if index calls for now in
 tests

---
 src/hive_fork_manager/hived_api_impl.sql                 | 3 ++-
 .../haf_local_tools/system/haf/__init__.py               | 7 ++++++-
 .../system/applications/test_application_index_many.py   | 7 ++++++-
 .../system/applications/test_application_index_one.py    | 9 +++++++--
 4 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/src/hive_fork_manager/hived_api_impl.sql b/src/hive_fork_manager/hived_api_impl.sql
index 2e16adfef..af3140dd7 100644
--- a/src/hive_fork_manager/hived_api_impl.sql
+++ b/src/hive_fork_manager/hived_api_impl.sql
@@ -680,7 +680,7 @@ BEGIN
 END;
 $BODY$
 ;
-
+/*
 CREATE OR REPLACE FUNCTION hive.wait_till_registered_indexes_created(
     _app_context TEXT
 )
@@ -714,6 +714,7 @@ BEGIN
 END;
 $BODY$
 ;
+*/
 
 CREATE OR REPLACE FUNCTION hive.check_if_registered_indexes_created(
     _app_context TEXT
diff --git a/tests/integration/haf-local-tools/haf_local_tools/system/haf/__init__.py b/tests/integration/haf-local-tools/haf_local_tools/system/haf/__init__.py
index fe0c2a519..b612de15e 100644
--- a/tests/integration/haf-local-tools/haf_local_tools/system/haf/__init__.py
+++ b/tests/integration/haf-local-tools/haf_local_tools/system/haf/__init__.py
@@ -57,7 +57,12 @@ def assert_index_does_not_exist(session, namespace, table, indexname):
 
 
 def wait_till_registered_indexes_created(haf_node, context):
-    haf_node.session.execute("select hive.wait_till_registered_indexes_created(:ctx)", {'ctx': context})
+    while True:
+        result = haf_node.session.execute("SELECT hive.check_if_registered_indexes_created(:ctx)", {'ctx': context}).scalar()
+        if result:
+            break
+        tt.logger.info("Indexes not yet created. Sleeping for 10 seconds...")
+        tt.sleep(10)
 
 
 def register_index_dependency(haf_node, context, create_index_command):
diff --git a/tests/integration/system/applications/test_application_index_many.py b/tests/integration/system/applications/test_application_index_many.py
index 3711420b1..74f01c7f6 100644
--- a/tests/integration/system/applications/test_application_index_many.py
+++ b/tests/integration/system/applications/test_application_index_many.py
@@ -54,7 +54,12 @@ def test_application_index_many(haf_node):
     session.commit()
 
     # THEN
-    session.execute("select hive.wait_till_registered_indexes_created('application')")
+    while True:
+        result = session.execute("SELECT hive.check_if_registered_indexes_created('application')").scalar()
+        if result:
+            break
+        tt.logger.info("Indexes not yet created. Sleeping for 10 seconds...")
+        tt.sleep(10)
 
     assert_index_exists(session, 'hafd', 'operations', 'hive_operations_vote_author_permlink_1')
     assert_index_exists(session, 'hafd', 'operations', 'hive_operations_vote_author_permlink_2')
diff --git a/tests/integration/system/applications/test_application_index_one.py b/tests/integration/system/applications/test_application_index_one.py
index d8947183c..572aab2bc 100644
--- a/tests/integration/system/applications/test_application_index_one.py
+++ b/tests/integration/system/applications/test_application_index_one.py
@@ -2,7 +2,7 @@ import test_tools as tt
 
 from haf_local_tools import create_app
 from haf_local_tools.haf_node.monolithic_workaround import apply_block_log_type_to_monolithic_workaround
-from haf_local_tools.system.haf import (connect_nodes, assert_index_exists, wait_till_registered_indexes_created, register_index_dependency)
+from haf_local_tools.system.haf import (connect_nodes, assert_index_exists, register_index_dependency)
 
 
 def test_application_index_one(haf_node):
@@ -33,6 +33,11 @@ def test_application_index_one(haf_node):
     session.commit()
 
     # THEN
-    wait_till_registered_indexes_created(haf_node, 'application')
+    while True:
+        result = session.execute("SELECT hive.check_if_registered_indexes_created('application')").scalar()
+        if result:
+            break
+        tt.logger.info("Indexes not yet created. Sleeping for 10 seconds...")
+        tt.sleep(10)
 
     assert_index_exists(session, 'hafd', 'operations', 'hive_operations_vote_author_permlink')
-- 
GitLab


From 4a05b0899ad43110cde03f1369670b3663ab9732 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Fri, 13 Dec 2024 15:08:53 -0500
Subject: [PATCH 30/33] can't create an index concurrently from inside a
 function so removing option from hive.restore_indexes

---
 src/hive_fork_manager/authorization.sql  |  2 +-
 src/hive_fork_manager/hived_api.sql      |  4 ++--
 src/hive_fork_manager/hived_api_impl.sql | 15 ++++-----------
 3 files changed, 7 insertions(+), 14 deletions(-)

diff --git a/src/hive_fork_manager/authorization.sql b/src/hive_fork_manager/authorization.sql
index 3f9cc9805..a09b86225 100644
--- a/src/hive_fork_manager/authorization.sql
+++ b/src/hive_fork_manager/authorization.sql
@@ -140,7 +140,7 @@ GRANT EXECUTE ON FUNCTION
     , hive.save_and_drop_indexes_constraints( in _schema TEXT, in _table TEXT )
     , hive.save_and_drop_foreign_keys( in _table_schema TEXT, in _table_name TEXT )
     , hive.recluster_account_operations_if_index_dropped()
-    , hive.restore_indexes( in _table_name TEXT, in concurrent BOOLEAN )
+    , hive.restore_indexes( in _table_name TEXT )
     , hive.restore_foreign_keys( in _table_name TEXT )
     , hive.copy_blocks_to_irreversible( _head_block_of_irreversible_blocks INT, _new_irreversible_block INT )
     , hive.copy_transactions_to_irreversible( _head_block_of_irreversible_blocks INT, _new_irreversible_block INT )
diff --git a/src/hive_fork_manager/hived_api.sql b/src/hive_fork_manager/hived_api.sql
index e4fb56a44..2b557a53e 100644
--- a/src/hive_fork_manager/hived_api.sql
+++ b/src/hive_fork_manager/hived_api.sql
@@ -200,7 +200,7 @@ BEGIN
     PERFORM hive.save_and_drop_indexes_constraints( 'hafd', 'accounts' );
     PERFORM hive.save_and_drop_indexes_constraints( 'hafd', 'account_operations' );
 
-    PERFORM hive.reanalyze_indexes_with_expressions();
+    PERFORM hive.reanalyze_indexes_with_expressions(); --I wonder if reanalyzing is really needed when indexes are dropped
 END;
 $BODY$
 ;
@@ -292,7 +292,7 @@ BEGIN
     PERFORM hive.save_and_drop_indexes_constraints( 'hafd', 'accounts_reversible' );
     PERFORM hive.save_and_drop_indexes_constraints( 'hafd', 'account_operations_reversible' );
 
-    PERFORM hive.reanalyze_indexes_with_expressions();
+    PERFORM hive.reanalyze_indexes_with_expressions(); --I wonder if reanalyzing is really needed when indexes are dropped
 
 END;
 $BODY$
diff --git a/src/hive_fork_manager/hived_api_impl.sql b/src/hive_fork_manager/hived_api_impl.sql
index af3140dd7..c03690a63 100644
--- a/src/hive_fork_manager/hived_api_impl.sql
+++ b/src/hive_fork_manager/hived_api_impl.sql
@@ -519,13 +519,12 @@ $function$
 LANGUAGE plpgsql VOLATILE
 ;
 
-CREATE OR REPLACE FUNCTION hive.restore_indexes( in _table_name TEXT, in concurrent BOOLEAN DEFAULT FALSE )
+CREATE OR REPLACE FUNCTION hive.restore_indexes( in _table_name TEXT)
 RETURNS VOID
 AS
 $function$
 DECLARE
   __command TEXT;
-  __original_command TEXT;
   __start_time TIMESTAMP;
   __end_time TIMESTAMP;
   __duration INTERVAL;
@@ -536,23 +535,17 @@ BEGIN
   END IF;
 
   --restoring indexes, primary keys, unique constraints
-  FOR __original_command IN
+  FOR __command IN
     SELECT command FROM hafd.indexes_constraints WHERE table_name = _table_name AND is_foreign_key = FALSE AND status = 'missing'
   LOOP
-    IF concurrent THEN
-      -- Modify the command to include CONCURRENTLY
-      __command := regexp_replace(__original_command, 'CREATE INDEX', 'CREATE INDEX CONCURRENTLY', 'i');
-    ELSE
-      __command := __original_command;
-    END IF;
     RAISE NOTICE 'Restoring index: %', __command;
-    UPDATE hafd.indexes_constraints SET status = 'creating' WHERE command = __original_command;
+    UPDATE hafd.indexes_constraints SET status = 'creating' WHERE command = __command;
     __start_time := clock_timestamp();
     EXECUTE __command;
     __end_time := clock_timestamp();
     __duration := __end_time - __start_time;
     RAISE NOTICE 'Index % created in % seconds', __command, EXTRACT(EPOCH FROM __duration);
-    UPDATE hafd.indexes_constraints SET status = 'created' WHERE command = __original_command;
+    UPDATE hafd.indexes_constraints SET status = 'created' WHERE command = __command;
   END LOOP;
 
   EXECUTE format( 'ANALYZE %s',  _table_name );
-- 
GitLab


From c15bd2e29d49e966e2451bf70ad94e5ba8ce74db Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Fri, 13 Dec 2024 15:59:02 -0500
Subject: [PATCH 31/33] poll_and_create_indexes also analyzes a table after it
 creates indexes for it

---
 src/sql_serializer/indexes_controler.cpp | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/sql_serializer/indexes_controler.cpp b/src/sql_serializer/indexes_controler.cpp
index fa0c19be2..e398f5e88 100644
--- a/src/sql_serializer/indexes_controler.cpp
+++ b/src/sql_serializer/indexes_controler.cpp
@@ -279,6 +279,15 @@ void indexes_controler::poll_and_create_indexes()
             catch(std::exception& e ) { elog( e.what() ); }
           }
           ilog("Finished creating all indexes for table: ${table_name}", (table_name));
+          try
+          {
+            ilog("Analyzing table: ${table_name}", (table_name) );
+            std::string analyze_table = "ANALYZE " + table_name + ";";
+            tx.exec(analyze_table);
+          }
+          catch (const pqxx::sql_error& e) { elog("Error while analyzing table: ${e}", ("e", e.what())); }
+          catch(std::exception& e ) { elog( e.what() ); }
+          
           std::lock_guard g(mtx);
           threads_to_delete.insert(table_name); // Mark the thread for deletion
           ilog("Thread for table: ${table_name} has been marked for deletion", (table_name));
-- 
GitLab


From c8fb708b4986f1a423d8a88de867bbe8a24c881e Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Fri, 13 Dec 2024 16:44:13 -0500
Subject: [PATCH 32/33] try to fix tests with correct sleep

---
 .../haf-local-tools/haf_local_tools/system/haf/__init__.py    | 4 ++--
 .../system/applications/test_application_index_one.py         | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/integration/haf-local-tools/haf_local_tools/system/haf/__init__.py b/tests/integration/haf-local-tools/haf_local_tools/system/haf/__init__.py
index b612de15e..337b48156 100644
--- a/tests/integration/haf-local-tools/haf_local_tools/system/haf/__init__.py
+++ b/tests/integration/haf-local-tools/haf_local_tools/system/haf/__init__.py
@@ -5,7 +5,7 @@ import test_tools as tt
 
 from haf_local_tools.haf_node._haf_node import HafNode, Transaction, TransactionId
 from haf_local_tools.tables import BlocksView
-
+import time
 
 def connect_nodes(seed_node: tt.RawNode, peer_node: tt.RawNode) -> None:
     """
@@ -62,7 +62,7 @@ def wait_till_registered_indexes_created(haf_node, context):
         if result:
             break
         tt.logger.info("Indexes not yet created. Sleeping for 10 seconds...")
-        tt.sleep(10)
+        time.sleep(10)
 
 
 def register_index_dependency(haf_node, context, create_index_command):
diff --git a/tests/integration/system/applications/test_application_index_one.py b/tests/integration/system/applications/test_application_index_one.py
index 572aab2bc..623a754c5 100644
--- a/tests/integration/system/applications/test_application_index_one.py
+++ b/tests/integration/system/applications/test_application_index_one.py
@@ -3,7 +3,7 @@ import test_tools as tt
 from haf_local_tools import create_app
 from haf_local_tools.haf_node.monolithic_workaround import apply_block_log_type_to_monolithic_workaround
 from haf_local_tools.system.haf import (connect_nodes, assert_index_exists, register_index_dependency)
-
+import time
 
 def test_application_index_one(haf_node):
     tt.logger.info(f'Start test_application_index_one')
@@ -38,6 +38,6 @@ def test_application_index_one(haf_node):
         if result:
             break
         tt.logger.info("Indexes not yet created. Sleeping for 10 seconds...")
-        tt.sleep(10)
+        time.sleep(10)
 
     assert_index_exists(session, 'hafd', 'operations', 'hive_operations_vote_author_permlink')
-- 
GitLab


From e7eeb458e6f59a347b8bbcfa1f9f538b33261020 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Fri, 13 Dec 2024 17:17:03 -0500
Subject: [PATCH 33/33] missed one time call

---
 .../system/applications/test_application_index_many.py        | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/integration/system/applications/test_application_index_many.py b/tests/integration/system/applications/test_application_index_many.py
index 74f01c7f6..f2c991f7d 100644
--- a/tests/integration/system/applications/test_application_index_many.py
+++ b/tests/integration/system/applications/test_application_index_many.py
@@ -3,7 +3,7 @@ import test_tools as tt
 from haf_local_tools import create_app
 from haf_local_tools.haf_node.monolithic_workaround import apply_block_log_type_to_monolithic_workaround
 from haf_local_tools.system.haf import (connect_nodes, assert_index_exists, register_index_dependency)
-
+import time
 
 def test_application_index_many(haf_node):
     tt.logger.info(f'Start test_application_index_many')
@@ -59,7 +59,7 @@ def test_application_index_many(haf_node):
         if result:
             break
         tt.logger.info("Indexes not yet created. Sleeping for 10 seconds...")
-        tt.sleep(10)
+        time.sleep(10)
 
     assert_index_exists(session, 'hafd', 'operations', 'hive_operations_vote_author_permlink_1')
     assert_index_exists(session, 'hafd', 'operations', 'hive_operations_vote_author_permlink_2')
-- 
GitLab