From 952a7603a66746ff0741b0dbf288145cc851d0f3 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Sat, 3 Jan 2026 14:01:49 -0500
Subject: [PATCH 1/2] Add prepare_data_and_shm_dir.sh script

Move data preparation script from hive repo to common-ci-configuration
for use by multiple projects (hive, HAF, etc).

Includes fix for cp -u failing when source and target are the same file
(which happens when /cache/blockchain/block_log_5m is a symlink to
/blockchain/block_log_5m).
---
 scripts/prepare_data_and_shm_dir.sh | 104 ++++++++++++++++++++++++++++
 1 file changed, 104 insertions(+)
 create mode 100755 scripts/prepare_data_and_shm_dir.sh

diff --git a/scripts/prepare_data_and_shm_dir.sh b/scripts/prepare_data_and_shm_dir.sh
new file mode 100755
index 0000000..db8be1b
--- /dev/null
+++ b/scripts/prepare_data_and_shm_dir.sh
@@ -0,0 +1,104 @@
+#! /bin/bash
+set -xeuo pipefail
+shopt -s nullglob
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --data-base-dir=*)
+        DATA_BASE_DIR="${1#*=}"
+        echo "using DATA_BASE_DIR $DATA_BASE_DIR"
+        ;;
+    --block-log-source-dir=*)
+        BLOCK_LOG_SOURCE_DIR="${1#*=}"
+        echo "block-log-source-dir $BLOCK_LOG_SOURCE_DIR"
+        ;;
+    --config-ini-source=*)
+        CONFIG_INI_SOURCE="${1#*=}"
+        echo "config-ini $CONFIG_INI_SOURCE"
+        ;;
+    *)
+        echo "ERROR: '$1' is not a valid option/positional argument"
+        echo
+        exit 2
+    esac
+    shift
+done
+
+
+if [ -z $DATA_BASE_DIR ];
+then
+  echo "No DATA_BASE_DIR directory privided, skipping this script"
+  exit 1
+else
+  mkdir -p $DATA_BASE_DIR/datadir
+  mkdir -p $DATA_BASE_DIR/shm_dir
+fi
+
+function handle_single_file_of_block_log() {
+  local FILE_PATH=$1
+  local FILE_NAME=$(basename -- "$FILE_PATH")
+
+  mkdir -p $DATA_BASE_DIR/datadir/blockchain
+
+  if [ -n "${HIVE_NETWORK_TYPE+x}" ] && [ "$HIVE_NETWORK_TYPE" = mirrornet ];
+  then
+    echo "creating copy of block log file as mirrornet block log can't be shared between pipelines"
+    cp "$FILE_PATH" "$DATA_BASE_DIR/datadir/blockchain/"
+  else
+    local BLOCK_LOG_TARGET_DIR="$DATA_BASE_DIR/..$BLOCK_LOG_SOURCE_DIR"
+    echo "using $BLOCK_LOG_TARGET_DIR/$FILE_NAME as hardlink target"
+    mkdir -p "$BLOCK_LOG_TARGET_DIR"
+    # cp -u exits with error if source and target are the same file (e.g., when
+    # BLOCK_LOG_TARGET_DIR resolves to the same path via symlinks). This is fine -
+    # the file is already where we need it for hardlinking.
+    if ! cp -u "$FILE_PATH" "$BLOCK_LOG_TARGET_DIR/$FILE_NAME" 2>/dev/null; then
+      # Check if they're the same file (inode) - that's OK
+      if [ "$(stat -c %i "$FILE_PATH" 2>/dev/null)" = "$(stat -c %i "$BLOCK_LOG_TARGET_DIR/$FILE_NAME" 2>/dev/null)" ]; then
+        echo "Source and target are the same file (via symlink), skipping copy"
+      else
+        echo "ERROR: Failed to copy $FILE_PATH to $BLOCK_LOG_TARGET_DIR/$FILE_NAME"
+        exit 1
+      fi
+    fi
+    echo "creating hardlink of $BLOCK_LOG_TARGET_DIR/$FILE_NAME in $DATA_BASE_DIR/datadir/blockchain/"
+    ln "$BLOCK_LOG_TARGET_DIR/$FILE_NAME" "$DATA_BASE_DIR/datadir/blockchain/$FILE_NAME"
+  fi
+
+  if [ -e $FILE_PATH.artifacts ];
+  then
+    cp $FILE_PATH.artifacts $DATA_BASE_DIR/datadir/blockchain/$FILE_NAME.artifacts
+  fi
+}
+
+if [ -n "$BLOCK_LOG_SOURCE_DIR" ]; then
+  if [ -e $BLOCK_LOG_SOURCE_DIR/block_log ]; then
+    handle_single_file_of_block_log "$BLOCK_LOG_SOURCE_DIR/block_log"
+  fi
+  if ls $BLOCK_LOG_SOURCE_DIR/block_log_part.???? 1>/dev/null 2>&1; then
+    for TARGET_FILE in $BLOCK_LOG_SOURCE_DIR/block_log_part.????; do
+      handle_single_file_of_block_log "$TARGET_FILE"
+    done
+  fi
+fi
+
+
+# Copy config.ini if source is specified
+if [ -n "$CONFIG_INI_SOURCE" ]; then
+  echo "Copying config from: $CONFIG_INI_SOURCE"
+  if [ -f "$CONFIG_INI_SOURCE" ]; then
+    cp "$CONFIG_INI_SOURCE" "$DATA_BASE_DIR/datadir/config.ini"
+    if [ -f "$DATA_BASE_DIR/datadir/config.ini" ]; then
+      echo "Config copied successfully to $DATA_BASE_DIR/datadir/config.ini"
+      # Show key settings for verification
+      grep -E "shared-file-size|shared-file-full-threshold" "$DATA_BASE_DIR/datadir/config.ini" || true
+    else
+      echo "ERROR: Config copy failed - destination file not found"
+      exit 1
+    fi
+  else
+    echo "ERROR: Config source file not found: $CONFIG_INI_SOURCE"
+    exit 1
+  fi
+else
+  echo "WARNING: CONFIG_INI_SOURCE not specified - hived will use default config (larger shared memory)"
+fi
-- 
GitLab


From 8d2a4b727f9d147f1d60f6afc3629e0de7a08b91 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Sat, 3 Jan 2026 14:25:14 -0500
Subject: [PATCH 2/2] Fix cross-device hardlink failure: fall back to symlink

When source block_log is on a different filesystem (e.g., /blockchain/)
than the target cache dir (e.g., /cache/), hardlinks fail with
'Cross-device link' error. Fall back to symlink in this case.
---
 scripts/prepare_data_and_shm_dir.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scripts/prepare_data_and_shm_dir.sh b/scripts/prepare_data_and_shm_dir.sh
index db8be1b..fa1522e 100755
--- a/scripts/prepare_data_and_shm_dir.sh
+++ b/scripts/prepare_data_and_shm_dir.sh
@@ -61,7 +61,11 @@ function handle_single_file_of_block_log() {
       fi
     fi
     echo "creating hardlink of $BLOCK_LOG_TARGET_DIR/$FILE_NAME in $DATA_BASE_DIR/datadir/blockchain/"
-    ln "$BLOCK_LOG_TARGET_DIR/$FILE_NAME" "$DATA_BASE_DIR/datadir/blockchain/$FILE_NAME"
+    # Try hardlink first, fall back to symlink if cross-device (different filesystems)
+    if ! ln "$BLOCK_LOG_TARGET_DIR/$FILE_NAME" "$DATA_BASE_DIR/datadir/blockchain/$FILE_NAME" 2>/dev/null; then
+      echo "Hardlink failed (likely cross-device), creating symlink instead"
+      ln -s "$BLOCK_LOG_TARGET_DIR/$FILE_NAME" "$DATA_BASE_DIR/datadir/blockchain/$FILE_NAME"
+    fi
   fi
 
   if [ -e $FILE_PATH.artifacts ];
-- 
GitLab