Commit 990e2c68 authored by Eric Frias's avatar Eric Frias Committed by Dan Notestein
Browse files

Enable block log compression by default.

Tweak compression format to only consume one bit for compression type.
Rename the option in the compress_block_log tool that decompresses the
block log to be more descriptive
parent d3454fe7
......@@ -319,7 +319,7 @@ namespace hive { namespace chain {
std::pair<uint64_t, block_log::block_attributes_t> split_block_start_pos_with_flags(uint64_t block_start_pos_with_flags)
{
block_log::block_attributes_t attributes;
attributes.flags = (block_log::block_flags)(block_start_pos_with_flags >> 62);
attributes.flags = (block_log::block_flags)(block_start_pos_with_flags >> 63);
if (block_start_pos_with_flags & 0x0100000000000000ull)
attributes.dictionary_number = (uint8_t)((block_start_pos_with_flags >> 48) & 0xff);
return std::make_pair(block_start_pos_with_flags & 0x0000ffffffffffffull, attributes);
......@@ -327,7 +327,7 @@ namespace hive { namespace chain {
uint64_t combine_block_start_pos_with_flags(uint64_t block_start_pos, block_log::block_attributes_t attributes)
{
return ((uint64_t)attributes.flags << 62) |
return ((uint64_t)attributes.flags << 63) |
(attributes.dictionary_number ? 0x0100000000000000ull : 0) |
((uint64_t)attributes.dictionary_number.value_or(0) << 48) |
block_start_pos;
......
......@@ -46,9 +46,10 @@ namespace hive { namespace chain {
// in the block log (and index), the positions are stored as 64-bit integers. We'll use the lower
// 48-bits as the actual position, and the upper 16 as flags that tell us how the block is stored
// hi lo|hi lo|hi | | | | | lo|
// cc.....d|<-dict->|<--------------------- position -------------------->|
// cc = block_flags, two bits specifying the compression method, or uncompressed
// d = one bit, if 1 the block uses a custom dictionary
// c......d|<-dict->|<--------------------- position -------------------->|
// c = block_flags, one bit specifying the compression method, or uncompressed
// (this was briefly two bits when we were testing other compression methods)
// d = one bit, if 1 the block uses a custom compression dictionary
// dict = the number specifying the dictionary used to compress the block, if d = 1, otherwise undefined
// . = unused
enum class block_flags {
......
......@@ -89,7 +89,7 @@ namespace chain {
fc::variant database_cfg;
bool replay_in_memory = false;
std::vector< std::string > replay_memory_indices{};
bool enable_block_log_compression = false;
bool enable_block_log_compression = true;
int block_log_compression_level = 15;
// The following fields are only used on reindexing
......
......@@ -126,7 +126,7 @@ class chain_plugin_impl
uint32_t flush_interval = 0;
bool replay_in_memory = false;
std::vector< std::string > replay_memory_indices{};
bool enable_block_log_compression = false;
bool enable_block_log_compression = true;
int block_log_compression_level = 15;
flat_map<uint32_t,block_id_type> loaded_checkpoints;
......@@ -755,7 +755,7 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip
("checkpoint,c", bpo::value<vector<string>>()->composing(), "Pairs of [BLOCK_NUM,BLOCK_ID] that should be enforced as checkpoints.")
("flush-state-interval", bpo::value<uint32_t>(),
"flush shared memory changes to disk every N blocks")
("enable-block-log-compression", boost::program_options::value<bool>()->default_value(false), "Compress blocks using zstd as they're added to the block log" )
("enable-block-log-compression", boost::program_options::value<bool>()->default_value(true), "Compress blocks using zstd as they're added to the block log" )
("block-log-compression-level", bpo::value<int>()->default_value(15), "Block log zstd compression level 0 (fast, low compression) - 22 (slow, high compression)" )
;
cli.add_options()
......
......@@ -347,7 +347,7 @@ int main(int argc, char** argv)
std::string zstd_levels_description = zstd_levels_description_stream.str();
boost::program_options::options_description options("Allowed options");
options.add_options()("enable-zstd", boost::program_options::value<std::string>()->default_value("yes"), "Whether to use zstd compression");
options.add_options()("decompress", boost::program_options::bool_switch()->default_value(false), "Instead of compressing the block log, decompress it");
options.add_options()("zstd-level", boost::program_options::value<int>()->default_value(15), zstd_levels_description.c_str());
options.add_options()("benchmark-decompression", "decompress each block and report the decompression times at the end");
options.add_options()("jobs,j", boost::program_options::value<int>()->default_value(1), "The number of threads to use for compression");
......@@ -365,7 +365,7 @@ int main(int argc, char** argv)
boost::program_options::variables_map options_map;
boost::program_options::store(boost::program_options::command_line_parser(argc, argv).options(options).positional(positional_options).run(), options_map);
enable_zstd = options_map["enable-zstd"].as<std::string>() == "yes";
enable_zstd = !options_map["decompress"].as<bool>();
zstd_level = options_map["zstd-level"].as<int>();
ilog("Compressing using zstd level ${zstd_level}", (zstd_level));
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment