retriable state loading

scripts/entrypoint.sh

@@ -21,8 +21,22 @@ if [[ "$RUN_IN_EB" ]]; then
     cd /var/lib/postgresql/9.5
 
     echo hivemind: attempting to pull in state file from s3://$S3_BUCKET/hivemind-$SCHEMA_HASH-latest.tar.lz4
-    s3cmd get s3://$S3_BUCKET/hivemind-$SCHEMA_HASH-latest.tar.lz4 - | lz4 -d | tar x
-    if [[ $? -ne 0 ]]; then
+
+    finished=0
+    count=1
+    while [[ $count -le 5 ]] && [[ $finished == 0 ]]
+    do
+      s3cmd get s3://$S3_BUCKET/hivemind-$SCHEMA_HASH-latest.tar.lz4 - | lz4 -d | tar x
+      if [[ $? -ne 0 ]]; then
+        sleep 1
+        echo notifyalert hivemind: unable to pull state from S3 - attempt $count
+        (( count++ ))
+      else
+        finished=1
+      fi
+    done
+
+    if [[ $finished == 0 ]]; then
       if [[ ! "$SYNC_TO_S3" ]]; then
         echo notifyalert hivemind: unable to pull state from S3 - exiting
         exit 1