Implement storing binary serialized operations in the HAF database

Related issue: #92 (closed)

Merge Request related to this one on the hive-side: hive!802 (merged)

Changes to the repo:

• Fixed hive_fork_manager being built always in the mainnet configuration on the CI
• Added new type: hive.operation
• Added operator class (using btree) for the hive.operation type
• Changed hive.operations.body column type from text to hive.operation
• Made CUSTOM_LOG variadic arguments optional
• Optimized escape_raw function
• Changed tests to work with the new type
• Made colect_data_and_fill_returned_recordset truly noexcept
• Created colect_operation_data_and_fill_returned_recordset function wrapping colect_data_and_fill_returned_recordset function specifically for the hive.operation type
• Changed sql_serializer to put binary data into the hive.operations table instead of the json representation
• Added HiveOperation type class for the sqlalchemy ORM

What should be added/changed/discussed before merging (not critical):

• Fix warnings generated by the sqlalchemy ORM related to the hive.operation type
• Make update_state_provider_keyauth function work with the binary data representation instead of text one
• Search for other possibilities to optimize the SQL code execution using binary data representation instead of the text one

@gandalf Please review and test full replay + live sync

src/hive_fork_manager/shared_lib/utilities.cpp

@@ -61,32 +61,18 @@ void extract_set_witness_properties_impl(extract_set_witness_properties_result_t
   helper.try_fill<fc::string>("url");
 }
 
-fc::string get_legacy_style_operation_impl( const char* raw_data, uint32_t data_length )
+fc::string get_legacy_style_operation_impl( const hive::protocol::operation& _op )
 {
-  if( !data_length )
-    return {};
-
-  using hive::protocol::operation;
-
-  operation op = fc::raw::unpack_from_char_array< operation >( raw_data, data_length );
-
   fc::variant v;
-  fc::to_variant( op, v );
+  fc::to_variant( _op, v );
 
   serialization_mode_controller::mode_guard guard( transaction_serialization_type::legacy );
 
-  return fc::json::to_string( op );
+  return fc::json::to_string( _op );
 }
 
-account_name_type get_account_from_accounts_operations_impl( const char* raw_data, uint32_t data_length )
+account_name_type get_account_from_accounts_operations_impl( const hive::protocol::operation& _op )
 {
-  if( !data_length )
-    return {};
-
-  using hive::protocol::operation;
-
-  operation _op = fc::raw::unpack_from_char_array< operation >( raw_data, data_length );
-
   return hive::app::get_account_from_accounts_operations( _op );
 }
 
@@ -102,13 +88,9 @@ flat_set<account_name_type> get_accounts( const char* raw_data, uint32_t data_le
   return _impacted;
 }
 
-impacted_balance_data collect_impacted_balances(const char* raw_data, uint32_t data_length, const bool is_hf01)
+impacted_balance_data collect_impacted_balances(const hive::protocol::operation& _op, const bool is_hf01)
 {
-  using hive::protocol::operation;
-
-  operation op = fc::raw::unpack_from_char_array< operation >( raw_data, data_length );
-
-  return hive::app::operation_get_impacted_balances(op, is_hf01);
+  return hive::app::operation_get_impacted_balances(_op, is_hf01);
 }
 
 extern "C" void issue_error(const char* msg);
@@ -118,10 +100,8 @@ void issue_error(const fc::string& msg)
   issue_error(msg.c_str());
 }
 
-collected_keyauth_collection_t collect_keyauths(const char *operation_body)
+collected_keyauth_collection_t collect_keyauths(const hive::protocol::operation& op)
 {
-    hive::protocol::operation op;
-    from_variant(fc::json::from_string(operation_body), op);
     return hive::app::operation_get_keyauths(op);
 }
 
@@ -219,8 +199,8 @@ TupleDesc build_tuple_descriptor(PG_FUNCTION_ARGS)
   return retvalDescription;
 }
 
-template<typename Collect, typename FillReturnTuple>
-Datum colect_data_and_fill_returned_recordset(Collect collect, FillReturnTuple fill_return_tuple,  const char* C_function_name, const char* arg1) noexcept 
+template<typename... Collect>
+Datum colect_data_and_fill_returned_recordset(const std::function<void()>& collect, const std::function<void()>& fill_return_tuple,  const char* C_function_name, const char* arg1) noexcept 
 {
   try 
   {
@@ -248,6 +228,21 @@ Datum colect_data_and_fill_returned_recordset(Collect collect, FillReturnTuple f
   return (Datum)0;
 }
 
+Datum colect_operation_data_and_fill_returned_recordset(const std::function<void(const operation&)>& collect,
+  const std::function<void()>& fill_return_tuple,  const char* C_function_name, const char* raw_data, uint32_t data_length) noexcept 
+{
+  using hive::protocol::operation;
+
+  operation op = fc::raw::unpack_from_char_array< operation >( raw_data, data_length );
+
+  fc::string op_name;
+  fc::get_legacy_static_variant_name{ op_name }( op );
+
+  return colect_data_and_fill_returned_recordset( [&]{
+    collect(op);
+  }, fill_return_tuple, C_function_name, op_name.c_str() );
+}
+
 template<typename ElemT, typename F>
 void fill_record_field(Datum* tuple_values, ElemT elem, const F& func, std::size_t counter)
 {
@@ -309,15 +304,15 @@ Datum get_account_from_accounts_operations(PG_FUNCTION_ARGS)
   _operation* op = PG_GETARG_HIVE_OPERATION_PP( 0 );
   auto _result = (Datum)0;
 
-  colect_data_and_fill_returned_recordset(
-    [=, &_result]()
+  colect_operation_data_and_fill_returned_recordset(
+    [=, &_result](const hive::protocol::operation& op)
     {
-      std::string account = get_account_from_accounts_operations_impl( VARDATA_ANY( op ), VARSIZE_ANY_EXHDR( op ) );
+      std::string account = get_account_from_accounts_operations_impl( op );
       _result = CStringGetTextDatum( account.c_str() );
     },
     [](){},
     __FUNCTION__,
-    ""); // XXX: Binary data is an argument
+    VARDATA_ANY( op ), VARSIZE_ANY_EXHDR( op ));
 
   return _result;
 }
@@ -358,15 +353,15 @@ Datum get_legacy_style_operation(PG_FUNCTION_ARGS)
   _operation* op = PG_GETARG_HIVE_OPERATION_PP( 0 );
   auto _result = (Datum)0;
 
-  colect_data_and_fill_returned_recordset(
-    [=, &_result]()
+  colect_operation_data_and_fill_returned_recordset(
+    [=, &_result](const hive::protocol::operation& op)
     {
-        fc::string _legacy_operation_body = get_legacy_style_operation_impl( VARDATA_ANY( op ), VARSIZE_ANY_EXHDR( op ) );
+        fc::string _legacy_operation_body = get_legacy_style_operation_impl( op );
         _result = CStringGetTextDatum( _legacy_operation_body.c_str() );
     },
     [](){},
     __FUNCTION__,
-    ""); // XXX: Binary data is an argument
+    VARDATA_ANY( op ), VARSIZE_ANY_EXHDR( op ));
 
   return _result;
 }
@@ -496,11 +491,11 @@ Datum get_impacted_balances(PG_FUNCTION_ARGS)
   _operation* operation_body = PG_GETARG_HIVE_OPERATION_PP( 0 );
   const bool is_hf01 = PG_GETARG_BOOL(1);
 
-  colect_data_and_fill_returned_recordset(
+  colect_operation_data_and_fill_returned_recordset(
 
-    [=, &collected_data]()
+    [=, &collected_data](const hive::protocol::operation& op)
     {
-        collected_data = collect_impacted_balances(VARDATA_ANY( operation_body ), VARSIZE_ANY_EXHDR( operation_body ), is_hf01);
+        collected_data = collect_impacted_balances(op, is_hf01);
     }, 
 
     [=, &collected_data]()
@@ -515,7 +510,7 @@ Datum get_impacted_balances(PG_FUNCTION_ARGS)
     
     __FUNCTION__,
 
-    ""); // XXX: Binary data is an argument
+    VARDATA_ANY( operation_body ), VARSIZE_ANY_EXHDR( operation_body ));