diff --git a/.env.example b/.env.example index a418ad514367d3dafd24e4ec0e045420f9879baf..d137e5a49c6493d161a87032570e9b8fd045c715 100644 --- a/.env.example +++ b/.env.example @@ -180,6 +180,7 @@ ARGUMENTS="--replay-blockchain" # HIVESENSE_WORKERS=16 # HIVESENSE_EMBEDDING_BATCH_SIZE=100 # HIVESENSE_OLLAMA=http://hivesense-ollama:11434 +# HIVESENSE_MAINTENANCE_WORK_MEM=28 # HIVESENSE_STORE_HALFVEC_EMBEDDINGS=TRUE # HIVESENSE_USE_HALFVEC_INDEX=TRUE diff --git a/.gitignore b/.gitignore index 0eb16dd498efc9765df24458d410a65fcb1ee42e..49f06172f3b77af52f3c3c627805c151e942cdab 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ compose.override.yml docker-compose.override.yml compose.override.yaml docker-compose.override.yaml +.reduce_writebacks.original diff --git a/README.md b/README.md index 00355d9192b1b19b8b08397a563c39c3cc70a4f7..d311da6377d19ece1696b0396e83406a2385d32d 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,25 @@ is not compressed because hived directly manages compression of the block_log fi If you have a LOT of nvme storage (e.g. 6TB+), you can get better API performance at the cost of disk storage by disabling ZFS compression on the database dataset, but for most nodes this isn't recommended. -##### Speeding up the initial sync +##### Non-ZFS users + +If you're not using ZFS, you can run: +``` +sudo ./create_directories.sh +``` +to create the directory structure on a pre-existing filesystem. + +##### File Ownership + +The docker containers require different directories to have specific user/group ownership. The +create_zfs_datasets/create_directories scripts automatically set ownership correctly, and you may +never have a problem. But if you start getting permission errors, there's a +``` +sudo ./repair_permissions.sh +``` +script that will try to restore file ownership to what our stack expects. + +#### Speeding up the initial sync Following the instructions above will get you a working HAF node, but there are some things you can do to speed up the initial sync. @@ -136,14 +154,40 @@ ARGUMENTS="--replay-blockchain" Once the replay has finished, you can revert the `ARGUMENTS` line to the empty string +###### Reducing Writebacks + +You can tune your system to reduce how often the kernel flushes dirty pages to disk. This +will result in far fewer disk writes during sync. This reduces wear on your drives, and +significantly improves performance, giving you performance almost as good as putting hived's +shared memory file on a ramdisk, without the hassle of creating and destroying the ramdisk +and copying the file off after the sync. + +Note that unlike using a ramdisk, this changes global settings that will effect your whole +machine. Most of the benefit comes from preventing the OS from flushing hived's shared +memory file when it's being heavily written to during sync. After hived is in sync, you +can restore the original settings with little loss in performance while the rest of the apps +sync. + +There's a script `reduce_writebacks.sh` that will change your kernel parameters to the +suggested values. Run: + +``` +sudo ./reduce_writebacks.sh +``` + +After sync, you can restore the original values with: +``` +sudo ./reduce_writebacks.sh --restore +``` + ###### Shared Memory on Ramdisk If you have enough spare memory on your system, you can speed up the initial replay by placing the `shared_memory.bin` file on a ramdisk. -The current default shared memory filesize is 24G, so this will only work if you have 24G free -(that's in addition to the memory you expect to be used by hived and HAF's integrated PostgreSQL -instance). +The current default shared memory filesize is approximately 6G, so allowing for a little extra room, +this will only work if you have 8G free. (that's in addition to the memory you expect to be used by +hived and HAF's integrated PostgreSQL instance). If you have a 64GB system, ensure you have a big enough swapfile (32GB is recommended and 8GB is known to not be sufficient) to handle peak memory usage needs during the replay. @@ -364,4 +408,4 @@ Options: Example: ``` sudo ./rollback_zfs_datasets.sh --env-file=.env --zpool=haf-pool --top-level-dataset=haf-datadir snapshot_name -``` \ No newline at end of file +``` diff --git a/hivesense.yaml b/hivesense.yaml index 6d7de72cdfceab517776f6f4f36406edc6508b9b..b6e27705f4726b046a4a0ef496f16799e50d261b 100644 --- a/hivesense.yaml +++ b/hivesense.yaml @@ -87,6 +87,7 @@ services: - "--reduced-dim=${HIVESENSE_REDUCED_DIMENSION:-128}" - "--hnsw-m=${HIVESENSE_HNSW_M:-16}" - "--hnsw-ef-construction=${HIVESENSE_HNSW_EF_CONSTRUCTION:-200}" + - "--maintenance-work-mem=${HIVESENSE_MAINTENANCE_WORK_MEM:-28}" volumes: - type: bind source: ${HAF_DATA_DIRECTORY:-${TOP_LEVEL_DATASET_MOUNTPOINT}}/hivesense/config diff --git a/reduce_writebacks.sh b/reduce_writebacks.sh index 7bad99ef2221d94acdfa6f48aff0516861a8ec07..0df33d3132d26f90b0f81fb06262d62965cd96e6 100755 --- a/reduce_writebacks.sh +++ b/reduce_writebacks.sh @@ -1,5 +1,321 @@ -sudo sysctl -w vm.dirty_bytes=12000000000 #12GB -sudo sysctl -w vm.dirty_background_bytes=5000000000 #5GB -sudo sysctl -w vm.dirty_expire_centisecs=300000 #400000 #default 3000 #default causes too much writing -sudo sysctl -w vm.dirty_writeback_centisecs=50000 #500 #360000 #default 500 -sudo sysctl -w vm.swappiness=0 #a swappiness of zero is not sufficient to prevent writeback of dirty pages +#!/bin/bash + +# reduce_writebacks.sh - Optimize Linux VM parameters for initial HAF sync +# +# This script modifies kernel VM parameters to reduce write-back frequency, +# which significantly improves performance during initial blockchain sync. +# +# Usage: +# sudo ./reduce_writebacks.sh # Apply optimized settings +# sudo ./reduce_writebacks.sh --restore # Restore original settings +# ./reduce_writebacks.sh --help # Show this help + +set -euo pipefail + +# Configuration +SAVED_VALUES_FILE=".reduce_writebacks.original" + +# Color codes for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# VM parameters to configure +# Format: "parameter_name|new_value|description" +declare -a VM_PARAMS=( + "vm.dirty_bytes|12000000000|Maximum dirty memory before forced writeback (12GB)" + "vm.dirty_background_bytes|5000000000|Threshold for background writeback (5GB)" + "vm.dirty_expire_centisecs|300000|Age at which dirty data expires (5 min)" + "vm.dirty_writeback_centisecs|50000|Interval for writeback thread wakeup (50 sec)" + "vm.swappiness|0|Tendency to swap memory pages (0 = avoid swapping)" +) + +# Print functions +print_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +print_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +print_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +print_error() { + echo -e "${RED}[ERROR]${NC} $1" >&2 +} + +# Show help message +show_help() { + cat << EOF +Usage: sudo $0 [OPTIONS] + +Optimize Linux VM parameters for HAF initial sync performance. + +OPTIONS: + --restore Restore original VM parameter values + --dry-run Show what would be changed without applying + --help Show this help message + +VM PARAMETERS: +EOF + for param_info in "${VM_PARAMS[@]}"; do + IFS='|' read -r param value desc <<< "$param_info" + printf " %-30s %s\n" "$param" "$desc" + done + cat << EOF + +NOTES: + - Must be run as root + - Settings automatically reset on reboot + - Original values are saved to: $SAVED_VALUES_FILE + - Use --restore to manually revert changes + +EXAMPLES: + sudo $0 # Apply optimized settings + sudo $0 --restore # Restore original settings + sudo $0 --dry-run # Preview changes + +EOF +} + +# Check if running as root +check_root() { + if [[ $EUID -ne 0 ]]; then + print_error "This script must be run as root" + echo "Please run with: sudo $0 $*" + exit 1 + fi +} + +# Get current value of a sysctl parameter +get_sysctl_value() { + local param=$1 + sysctl -n "$param" 2>/dev/null || echo "unknown" +} + +# Save original values to file +# Parameters: +# $1 - skip_prompt (optional): if "true", don't prompt before overwriting +save_original_values() { + local skip_prompt=${1:-false} + local temp_file="${SAVED_VALUES_FILE}.tmp" + + if [[ -f "$SAVED_VALUES_FILE" ]] && [[ "$skip_prompt" == "false" ]]; then + print_warning "Original values file already exists: $SAVED_VALUES_FILE" + read -p "Overwrite? (y/N): " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + print_info "Keeping existing saved values file" + return 1 + fi + fi + + print_info "Saving original values to: $SAVED_VALUES_FILE" + + # Save with timestamp + echo "# Original sysctl values saved on $(date)" > "$temp_file" + + for param_info in "${VM_PARAMS[@]}"; do + IFS='|' read -r param _ _ <<< "$param_info" + local current_value + current_value=$(get_sysctl_value "$param") + echo "${param}=${current_value}" >> "$temp_file" + done + + mv "$temp_file" "$SAVED_VALUES_FILE" + print_success "Original values saved" + return 0 +} + +# Apply optimized settings +apply_settings() { + local dry_run=${1:-false} + local changes_needed=false + + print_info "Checking current VM parameter values..." + echo + + # First pass: check what needs to be changed + for param_info in "${VM_PARAMS[@]}"; do + IFS='|' read -r param new_value desc <<< "$param_info" + local current_value + current_value=$(get_sysctl_value "$param") + + if [[ "$current_value" == "$new_value" ]]; then + print_success "$param is already set to $new_value (no change needed)" + else + print_warning "$param: needs change from $current_value to $new_value" + changes_needed=true + fi + done + + echo + + if [[ "$changes_needed" == "false" ]]; then + print_success "All VM parameters are already optimized - no changes needed" + return 0 + fi + + if [[ "$dry_run" == "true" ]]; then + print_info "Dry run complete - no changes were applied" + return 0 + fi + + # Save original values BEFORE making any changes + if [[ -f "$SAVED_VALUES_FILE" ]]; then + print_warning "Original values file already exists: $SAVED_VALUES_FILE" + echo "We need to save the CURRENT values before changing them, but a backup file already exists." + read -p "Overwrite the existing backup with current values? (y/N): " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + save_original_values "true" || true # Skip prompt, we already asked + echo + else + print_warning "Keeping existing backup - you won't be able to restore to the current values!" + echo + fi + else + save_original_values "true" || true # No prompt needed, file doesn't exist + echo + fi + + # Second pass: apply the changes + print_info "Applying changes..." + echo + + for param_info in "${VM_PARAMS[@]}"; do + IFS='|' read -r param new_value desc <<< "$param_info" + local current_value + current_value=$(get_sysctl_value "$param") + + if [[ "$current_value" != "$new_value" ]]; then + if sysctl -w "${param}=${new_value}" > /dev/null 2>&1; then + print_success "$param: changed from $current_value to $new_value" + else + print_error "Failed to update $param" + fi + fi + done + + # Print reminder + echo + print_success "VM parameters have been optimized for HAF initial sync" + echo + print_warning "IMPORTANT NOTES:" + echo " • These settings will automatically reset on the next reboot" + echo " • To manually restore original values before rebooting, run:" + echo " sudo $0 --restore" + echo +} + +# Restore original settings +restore_settings() { + if [[ ! -f "$SAVED_VALUES_FILE" ]]; then + print_error "No saved values file found: $SAVED_VALUES_FILE" + echo "Run the script without --restore first to create a baseline" + exit 1 + fi + + print_info "Restoring original VM parameter values from: $SAVED_VALUES_FILE" + echo + + local changes_made=false + + while IFS='=' read -r param value; do + # Skip comments and empty lines + [[ "$param" =~ ^#.*$ ]] && continue + [[ -z "$param" ]] && continue + + local current_value + current_value=$(get_sysctl_value "$param") + + if [[ "$current_value" == "$value" ]]; then + print_success "$param is already set to $value (no change needed)" + else + print_warning "$param: restoring from $current_value to $value" + + # Special handling: can't set dirty_bytes/dirty_background_bytes to 0 directly + # Need to set ratio parameters instead to switch back to ratio mode + if [[ "$param" == "vm.dirty_bytes" ]] && [[ "$value" == "0" ]]; then + local default_ratio=20 + if sysctl -w "vm.dirty_ratio=${default_ratio}" > /dev/null 2>&1; then + print_success "$param restored (by setting vm.dirty_ratio=$default_ratio)" + changes_made=true + else + print_error "Failed to restore $param" + fi + elif [[ "$param" == "vm.dirty_background_bytes" ]] && [[ "$value" == "0" ]]; then + local default_ratio=10 + if sysctl -w "vm.dirty_background_ratio=${default_ratio}" > /dev/null 2>&1; then + print_success "$param restored (by setting vm.dirty_background_ratio=$default_ratio)" + changes_made=true + else + print_error "Failed to restore $param" + fi + else + # Normal restore + if sysctl -w "${param}=${value}" > /dev/null 2>&1; then + print_success "$param restored successfully" + changes_made=true + else + print_error "Failed to restore $param" + fi + fi + fi + done < "$SAVED_VALUES_FILE" + + echo + + if [[ "$changes_made" == "true" ]]; then + print_success "Original VM parameters have been restored" + else + print_success "All VM parameters were already at their original values" + fi +} + +# Main execution +main() { + local mode="apply" + local dry_run=false + + # Parse arguments + for arg in "$@"; do + case $arg in + --restore) + mode="restore" + ;; + --dry-run) + dry_run=true + ;; + --help|-h) + show_help + exit 0 + ;; + *) + print_error "Unknown option: $arg" + echo "Run '$0 --help' for usage information" + exit 1 + ;; + esac + done + + # Check root privileges + check_root "$@" + + # Execute requested mode + case $mode in + apply) + apply_settings "$dry_run" + ;; + restore) + restore_settings + ;; + esac +} + +main "$@"