Make zfs snapshot scripts more resilient

create_zfs_datasets.sh

@@ -45,9 +45,23 @@ while true; do
   esac
 done
 
-if [ ! -z "$ENV_FILE" ]; then
-  echo reading $ENV_FILE
-  . $ENV_FILE
+if [ -z "$ZPOOL" -o -z "$TOP_LEVEL_DATASET" ]; then
+  if [ ! -z "$ENV_FILE" ]; then
+    echo reading $ENV_FILE
+    . $ENV_FILE
+  elif [ -f .env ]; then
+    echo reading configuration from .env
+    . ./.env
+  else
+    echo "You must either provide an --env-file argument or both a --zpool and --top-level-dataset"
+    echo "argument to tell this script what to create."
+    exit 1
+  fi
+fi
+
+if [ -z "$ZPOOL" -o -z "$TOP_LEVEL_DATASET" ]; then
+  echo "Your environment file must define the ZPOOL and TOP_LEVEL_DATASET environment variables"
+  exit 1
 fi
 
 [ -z "$ZPOOL_MOUNT_POINT" ] && ZPOOL_MOUNT_POINT="/$ZPOOL"

snapshot_zfs_datasets.sh

@@ -45,17 +45,31 @@ while true; do
   esac
 done
 
+if [ -z "$ZPOOL" -o -z "$TOP_LEVEL_DATASET" ]; then
+  if [ ! -z "$ENV_FILE" ]; then
+    echo reading $ENV_FILE
+    . $ENV_FILE
+  elif [ -f .env ]; then
+    echo reading configuration from .env
+    . ./.env
+  else
+    echo "You must either provide an --env-file argument or both a --zpool and --top-level-dataset"
+    echo "argument to tell this script what to create."
+    exit 1
+  fi
+fi
+
+if [ -z "$ZPOOL" -o -z "$TOP_LEVEL_DATASET" ]; then
+  echo "Your environment file must define the ZPOOL and TOP_LEVEL_DATASET environment variables"
+  exit 1
+fi
+
 SNAPSHOT_NAME="$1"
 if [ -z "$SNAPSHOT_NAME" ]; then
   echo "No snapshot name provided"
   exit 1
 fi
 
-if [ ! -z "$ENV_FILE" ]; then
-  echo reading $ENV_FILE
-  . $ENV_FILE
-fi
-
 [ -z "$ZPOOL_MOUNT_POINT" ] && ZPOOL_MOUNT_POINT="/$ZPOOL"
 [ -z "$TOP_LEVEL_DATASET_MOUNTPOINT" ] && TOP_LEVEL_DATASET_MOUNTPOINT="${ZPOOL_MOUNT_POINT}/${TOP_LEVEL_DATASET}"
 
@@ -64,22 +78,64 @@ echo "zpool:             $ZPOOL"
 echo "  mounted on:      $ZPOOL_MOUNT_POINT"
 echo "top-level dataset: $TOP_LEVEL_DATASET"
 echo "  mounted on:      $TOP_LEVEL_DATASET_MOUNTPOINT"
+echo ""
+echo "This will unmount the HAF datasets, take a snapshot, then remount them."
+
+check_dataset_is_unmountable() {
+  echo -n "Checking $1..."
+  if lsof_result=$(lsof +f $1); then
+    echo " error, dataset in use"
+    echo "$lsof_result"
+    exit 1
+  fi
+  echo " ok"
+}
+
+echo "Verifying that all datasets are unmountable"
+check_dataset_is_unmountable "${ZPOOL}/${TOP_LEVEL_DATASET}/haf_db_store/pgdata/pg_wal"
+check_dataset_is_unmountable "${ZPOOL}/${TOP_LEVEL_DATASET}/haf_db_store/pgdata"
+check_dataset_is_unmountable "${ZPOOL}/${TOP_LEVEL_DATASET}/haf_db_store/tablespace"
+check_dataset_is_unmountable "${ZPOOL}/${TOP_LEVEL_DATASET}/haf_db_store/logs"
+check_dataset_is_unmountable "${ZPOOL}/${TOP_LEVEL_DATASET}/blockchain"
+check_dataset_is_unmountable "${ZPOOL}/${TOP_LEVEL_DATASET}"
+echo "All datasets appear unmountable"
 
 sync; sync; sync
 echo 3 > /proc/sys/vm/drop_caches
 
-zfs umount "${ZPOOL}/${TOP_LEVEL_DATASET}/haf_db_store/pgdata/pg_wal"
-zfs umount "${ZPOOL}/${TOP_LEVEL_DATASET}/haf_db_store/pgdata"
-zfs umount "${ZPOOL}/${TOP_LEVEL_DATASET}/haf_db_store/tablespace"
-zfs umount "${ZPOOL}/${TOP_LEVEL_DATASET}/haf_db_store/logs"
-zfs umount "${ZPOOL}/${TOP_LEVEL_DATASET}/blockchain"
-zfs umount "${ZPOOL}/${TOP_LEVEL_DATASET}"
+unmount() {
+  echo -n "Unmounting $1..."
+  zfs umount "$1"
+  echo " done"
+}
+
+unmount "${ZPOOL}/${TOP_LEVEL_DATASET}/haf_db_store/pgdata/pg_wal"
+unmount "${ZPOOL}/${TOP_LEVEL_DATASET}/haf_db_store/pgdata"
+unmount "${ZPOOL}/${TOP_LEVEL_DATASET}/haf_db_store/tablespace"
+unmount "${ZPOOL}/${TOP_LEVEL_DATASET}/haf_db_store/logs"
+unmount "${ZPOOL}/${TOP_LEVEL_DATASET}/blockchain"
+unmount "${ZPOOL}/${TOP_LEVEL_DATASET}"
 
+echo -n "Taking snapshot..."
 zfs snap -r "${ZPOOL}/${TOP_LEVEL_DATASET}@${SNAPSHOT_NAME}"
+echo " done"
+
+remount() {
+  echo -n "Re-mounting $1..."
+  zfs umount "$1"
+  echo " done"
+}
+
+remount "${ZPOOL}/${TOP_LEVEL_DATASET}"
+remount "${ZPOOL}/${TOP_LEVEL_DATASET}/blockchain"
+remount "${ZPOOL}/${TOP_LEVEL_DATASET}/haf_db_store/logs"
+remount "${ZPOOL}/${TOP_LEVEL_DATASET}/haf_db_store/tablespace"
+remount "${ZPOOL}/${TOP_LEVEL_DATASET}/haf_db_store/pgdata"
+remount "${ZPOOL}/${TOP_LEVEL_DATASET}/haf_db_store/pgdata/pg_wal"
 
-zfs mount "${ZPOOL}/${TOP_LEVEL_DATASET}"
-zfs mount "${ZPOOL}/${TOP_LEVEL_DATASET}/blockchain"
-zfs mount "${ZPOOL}/${TOP_LEVEL_DATASET}/haf_db_store/logs"
-zfs mount "${ZPOOL}/${TOP_LEVEL_DATASET}/haf_db_store/tablespace"
-zfs mount "${ZPOOL}/${TOP_LEVEL_DATASET}/haf_db_store/pgdata"
-zfs mount "${ZPOOL}/${TOP_LEVEL_DATASET}/haf_db_store/pgdata/pg_wal"
+zfs list "${ZPOOL}/${TOP_LEVEL_DATASET}@${SNAPSHOT_NAME}" \
+         "${ZPOOL}/${TOP_LEVEL_DATASET}/blockchain@${SNAPSHOT_NAME}" \
+         "${ZPOOL}/${TOP_LEVEL_DATASET}/haf_db_store/logs@${SNAPSHOT_NAME}" \
+         "${ZPOOL}/${TOP_LEVEL_DATASET}/haf_db_store/tablespace@${SNAPSHOT_NAME}" \
+         "${ZPOOL}/${TOP_LEVEL_DATASET}/haf_db_store/pgdata@${SNAPSHOT_NAME}" \
+         "${ZPOOL}/${TOP_LEVEL_DATASET}/haf_db_store/pgdata/pg_wal@${SNAPSHOT_NAME}"