From 52af37a5696db084c5fff998231087d1a55206ef Mon Sep 17 00:00:00 2001
From: Bartek Wrona <wrona@syncad.com>
Date: Tue, 4 Jul 2023 18:00:44 +0200
Subject: [PATCH 01/20] Base CI image switched to latest version existing in
 Hived repo: ubuntu22-04.5. Also included `prepare_data_image_job.yml` to use
 common predefined jobs for building Hived image.

---
 .gitlab-ci.yml | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e0c286b28f..2421e56047 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,5 +1,3 @@
-image: "registry.gitlab.syncad.com/hive/hive/ci-base-image:ubuntu22.04-3"
-
 stages:
   - static_code_analysis
   - build
@@ -22,6 +20,16 @@ variables:
   BEEKEEPER_BINARY: "$CI_PROJECT_DIR/beekeeper"
   HIVE_BUILD_ROOT_PATH: "$CI_PROJECT_DIR/hived"
 
+  # uses registry.gitlab.syncad.com/hive/hive/ci-base-image:ubuntu22.04-5
+  TEST_IMAGE_TAG: "@sha256:eaa6f32730f0f4c45a95a65b6d790bea2b71a59276eb28795c04be7703f003e6"
+
+include:
+  - project: 'hive/hive'
+    ref: fc899116bf691309cc618e51a9c2d8c6d5edb8cd #develop
+    file: '/scripts/ci-helpers/prepare_data_image_job.yml'
+
+image: "registry.gitlab.syncad.com/hive/hive/ci-base-image${TEST_IMAGE_TAG}"
+
 .shared_tags: &shared_tags
   - hived-for-tests
 
-- 
GitLab


From 2df48395fd660ffb61b0b5c7757fc189ad3c3a74 Mon Sep 17 00:00:00 2001
From: Bartek Wrona <wrona@syncad.com>
Date: Tue, 4 Jul 2023 18:05:44 +0200
Subject: [PATCH 02/20] Defined `common_prepare_hived_image` job and 2
 dependent: prepare_mainnet_hived_image and prepare_testnet_hived_image

---
 .gitlab-ci.yml | 27 +++++++++++++++++++++++++--
 1 file changed, 25 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2421e56047..bb0de0dea5 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -17,8 +17,9 @@ variables:
   # colors:
   TXT_BLUE: "\e[1;34m"
   TXT_CLEAR: "\e[0m"
-  BEEKEEPER_BINARY: "$CI_PROJECT_DIR/beekeeper"
-  HIVE_BUILD_ROOT_PATH: "$CI_PROJECT_DIR/hived"
+
+  HIVE_BUILD_ROOT_PATH: "hived-binaries"
+  BEEKEEPER_BINARY: "${HIVE_BUILD_ROOT_PATH}/mainnet/beekeeper"
 
   # uses registry.gitlab.syncad.com/hive/hive/ci-base-image:ubuntu22.04-5
   TEST_IMAGE_TAG: "@sha256:eaa6f32730f0f4c45a95a65b6d790bea2b71a59276eb28795c04be7703f003e6"
@@ -79,6 +80,28 @@ type_check_with_mypy:
 
 #>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>| BUILD |>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
 
+.common_prepare_hived_image:
+  extends: .prepare_hived_image
+  stage: build
+  variables:
+    REGISTRY_USER: "$HIVED_CI_IMGBUILDER_USER"
+    REGISTRY_PASS: $HIVED_CI_IMGBUILDER_PASSWORD
+    SUBMODULE_DIR: "${CI_PROJECT_DIR}/hive/"
+    BINARY_CACHE_PATH: "${HIVE_BUILD_ROOT_PATH}/${HIVE_NETWORK_TYPE}"
+  tags:
+    - public-runner-docker
+    - hived-for-tests
+
+prepare_mainnet_hived_image:
+  extends: .common_prepare_hived_image
+  variables:
+    HIVE_NETWORK_TYPE: mainnet
+
+prepare_testnet_hived_image:
+  extends: .common_prepare_hived_image
+  variables:
+    HIVE_NETWORK_TYPE: testnet
+
 .hive_based_build:
   stage: build
   needs: []
-- 
GitLab


From e5a9388552b350eb63fe70cad1cdb59eb1eeb862 Mon Sep 17 00:00:00 2001
From: Bartek Wrona <wrona@syncad.com>
Date: Tue, 4 Jul 2023 18:17:55 +0200
Subject: [PATCH 03/20] build_beekeeper job references replaced with
 prepare_mainnet_hived_image (beekeper tool can be always used in mainnet
 build)

---
 .gitlab-ci.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index bb0de0dea5..a62ef038f3 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -142,7 +142,7 @@ build_hived:
   stage: tests
   extends: .configuration
   needs:
-    - job: build_beekeeper
+    - job: prepare_mainnet_hived_image
       artifacts: true
     - job: build_hived
       artifacts: true
@@ -174,7 +174,7 @@ testing_clive:
   stage: build_docker
   needs:
     - job: pre_commit_checks
-    - job: build_beekeeper
+    - job: prepare_mainnet_hived_image
       artifacts: true
     - job: testing_clive
   variables:
@@ -207,7 +207,7 @@ build_and_push_docker_testnet_image:
   needs:
     - job: pre_commit_checks
     - job: testing_clive
-    - job: build_beekeeper
+    - job: prepare_mainnet_hived_image
       artifacts: true
     - job: build_hived
       artifacts: true
-- 
GitLab


From f91a8a30b90c14bb43bc9ef3868b715596fbe8f3 Mon Sep 17 00:00:00 2001
From: Bartek Wrona <wrona@syncad.com>
Date: Tue, 4 Jul 2023 18:26:29 +0200
Subject: [PATCH 04/20] build_hived job references replaced with
 prepare_testnet_hived_image

---
 .gitlab-ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a62ef038f3..3789d9821c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -144,7 +144,7 @@ build_hived:
   needs:
     - job: prepare_mainnet_hived_image
       artifacts: true
-    - job: build_hived
+    - job: prepare_testnet_hived_image
       artifacts: true
   variables:
     CLIVE_BEEKEEPER__PATH: $BEEKEEPER_BINARY
@@ -209,7 +209,7 @@ build_and_push_docker_testnet_image:
     - job: testing_clive
     - job: prepare_mainnet_hived_image
       artifacts: true
-    - job: build_hived
+    - job: prepare_testnet_hived_image
       artifacts: true
   rules:
     - if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_REF_NAME != "develop"'
-- 
GitLab


From d6ce072b5235c18563bf835c73584157a54a1b25 Mon Sep 17 00:00:00 2001
From: Bartek Wrona <wrona@syncad.com>
Date: Tue, 4 Jul 2023 18:27:45 +0200
Subject: [PATCH 05/20] Removed unused anymore .hive_based_build, build_hived
 and build_beekeeper job definitions.

---
 .gitlab-ci.yml | 32 --------------------------------
 1 file changed, 32 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3789d9821c..f76543593b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -102,38 +102,6 @@ prepare_testnet_hived_image:
   variables:
     HIVE_NETWORK_TYPE: testnet
 
-.hive_based_build:
-  stage: build
-  needs: []
-  variables:
-    TARGET: ""
-    SOURCE: ""
-    DESTINATION: ""
-  script:
-    - cd hive
-    - mkdir build && cd build
-    - cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_HIVE_TESTNET=ON -GNinja ..
-    - ninja $TARGET
-    - cp -r $SOURCE $DESTINATION
-  artifacts:
-    paths:
-      - $DESTINATION
-  tags: *shared_tags
-
-build_beekeeper:
-  extends: .hive_based_build
-  variables:
-    TARGET: "beekeeper"
-    SOURCE: "programs/beekeeper/beekeeper"
-    DESTINATION: "$BEEKEEPER_BINARY"
-
-build_hived:
-  extends: .hive_based_build
-  variables:
-    TARGET: "hived"
-    SOURCE: "."
-    DESTINATION: "$HIVE_BUILD_ROOT_PATH"
-
 #<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<| BUILD |<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
 
 #>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>| TESTS |>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
-- 
GitLab


From 88be03851ac06d4e44622dd97b96dbcaf8ffd440 Mon Sep 17 00:00:00 2001
From: Bartek Wrona <wrona@syncad.com>
Date: Tue, 4 Jul 2023 18:35:45 +0200
Subject: [PATCH 06/20] BEEKEEPER_LOCATION is explictly specified during
 building clive images.

---
 .gitlab-ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f76543593b..d06ffd1ebf 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -168,7 +168,7 @@ build_and_push_docker_image:
     - develop
     - master
   script:
-    - docker build -t $TAG_COMMIT -t $TAG_LATEST -f Dockerfile ..
+    - docker build --build-arg BEEKEEPER_LOCATION="${BEEKEEPER_BINARY}" -t $TAG_COMMIT -t $TAG_LATEST -f Dockerfile ..
 
 build_and_push_docker_testnet_image:
   extends: .build_and_push_base
@@ -187,6 +187,6 @@ build_and_push_docker_testnet_image:
   variables:
     BUILD_SUFFIX: "-testnet"
   script:
-    - docker build -t $TAG_COMMIT -t $TAG_LATEST -f Dockerfile.TESTNET ..
+    - docker build --build-arg BEEKEEPER_LOCATION="${BEEKEEPER_BINARY}" -t $TAG_COMMIT -t $TAG_LATEST -f Dockerfile.TESTNET ..
 
 #<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<| BUILD DOCKER |<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
-- 
GitLab


From 89dd71c44a8657a55b422345805889898bc017db Mon Sep 17 00:00:00 2001
From: Bartek Wrona <wrona@syncad.com>
Date: Tue, 4 Jul 2023 18:48:54 +0200
Subject: [PATCH 07/20] CI runner tags changed to `public-runner-docker` to use
 much more flexible runners

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d06ffd1ebf..d84b6af675 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -32,7 +32,7 @@ include:
 image: "registry.gitlab.syncad.com/hive/hive/ci-base-image${TEST_IMAGE_TAG}"
 
 .shared_tags: &shared_tags
-  - hived-for-tests
+  - public-runner-docker
 
 .configuration:
   before_script: &configuration_before_script # configure python
-- 
GitLab


From 4ab753a2e85e9299801e868bf95fbdf43edb01e4 Mon Sep 17 00:00:00 2001
From: Bartek Wrona <wrona@syncad.com>
Date: Tue, 4 Jul 2023 19:12:44 +0200
Subject: [PATCH 08/20] Hived path (testnet version binary) explicitly pointed
 to test-tools library using HIVED_PATH variable

---
 .gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d84b6af675..0f4143760f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -116,6 +116,7 @@ prepare_testnet_hived_image:
       artifacts: true
   variables:
     CLIVE_BEEKEEPER__PATH: $BEEKEEPER_BINARY
+    HIVED_PATH: "${CI_PROJECT_DIR}/${HIVE_BUILD_ROOT_PATH}/testnet/hived"
     PATH_TO_REPORT: "$CI_PROJECT_DIR/report.xml"
   artifacts:
     reports:
-- 
GitLab


From ea6a8d7f0634f444adb3a2e6821b8f8b0926c53d Mon Sep 17 00:00:00 2001
From: Bartek Wrona <wrona@syncad.com>
Date: Tue, 4 Jul 2023 19:14:30 +0200
Subject: [PATCH 09/20] Each job should be interruptible to cancel executing
 pipeline when new one is created

---
 .gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0f4143760f..57a66fa517 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -35,6 +35,7 @@ image: "registry.gitlab.syncad.com/hive/hive/ci-base-image${TEST_IMAGE_TAG}"
   - public-runner-docker
 
 .configuration:
+  interruptible: true
   before_script: &configuration_before_script # configure python
     - python3 -V
     - poetry self update
-- 
GitLab


From 09720187d926e2b91d23b3d8d8cf1c63ae700264 Mon Sep 17 00:00:00 2001
From: Bartek Wrona <wrona@syncad.com>
Date: Wed, 12 Jul 2023 23:05:30 +0200
Subject: [PATCH 10/20] Dockerfile definition adjusted to new image building
 scheme, where base image and beekeeper source image are specified as docker
 build args

---
 docker/Dockerfile | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index 8d1f22d665..e08980f3e9 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,15 +1,21 @@
-FROM registry.gitlab.syncad.com/hive/hive/ci-base-image:ubuntu22.04-3
+ARG BEEKEEPER_IMAGE
+ARG BASE_IMAGE
+
+ARG CI_REGISTRY_IMAGE=registry.gitlab.syncad.com/hive/clive/
+
+FROM $BEEKEEPER_IMAGE as beekeper_source
+
+FROM $BASE_IMAGE AS instance
 
 ADD --chown=hived_admin:users . /clive
+
 WORKDIR /clive
 
-ARG BEEKEEPER_LOCATION="./beekeeper"
 ENV BEEKEEPER_PATH="/clive/beekeeper"
-ADD --chown=hived_admin:users "${BEEKEEPER_LOCATION}" "${BEEKEEPER_PATH}"
-
-RUN poetry self update
+COPY --from=beekeper_source --chown=hived_admin:users "/home/hived/bin/beekeeper" "${BEEKEEPER_PATH}"
 
-RUN poetry install --only main
+RUN poetry self update && \
+    poetry install --only main
 
 # crucial for proper display
 ENV COLORTERM=truecolor
-- 
GitLab


From 4508db663af98ec612f7754396bc15f5e7bcbbff Mon Sep 17 00:00:00 2001
From: Bartek Wrona <wrona@syncad.com>
Date: Wed, 12 Jul 2023 23:06:42 +0200
Subject: [PATCH 11/20] Defined helper scripts: build_instance.sh,
 run_instance.sh to follow common Hive tools usage scheme

---
 scripts/ci-helpers/build_instance.sh | 92 ++++++++++++++++++++++++++++
 scripts/common.sh                    | 52 ++++++++++++++++
 scripts/run_instance.sh              | 90 +++++++++++++++++++++++++++
 3 files changed, 234 insertions(+)
 create mode 100755 scripts/ci-helpers/build_instance.sh
 create mode 100755 scripts/common.sh
 create mode 100755 scripts/run_instance.sh

diff --git a/scripts/ci-helpers/build_instance.sh b/scripts/ci-helpers/build_instance.sh
new file mode 100755
index 0000000000..ae40ac09e2
--- /dev/null
+++ b/scripts/ci-helpers/build_instance.sh
@@ -0,0 +1,92 @@
+#! /bin/bash
+set -euo pipefail
+
+SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+SCRIPTSDIR="$SCRIPTPATH/.."
+
+LOG_FILE=build_instance.log
+source "$SCRIPTSDIR/common.sh"
+
+BUILD_IMAGE_TAG=""
+REGISTRY=""
+SRCROOTDIR=""
+
+IMAGE_TAG_PREFIX=""
+
+BEEKEEPER_IMAGE=""
+BASE_IMAGE=""
+
+
+print_help () {
+    echo "Usage: $0 <image_tag> <src_dir> <registry_url> [OPTION[=VALUE]]..."
+    echo
+    echo "Allows to build docker image containing clive installation"
+    echo "OPTIONS:"
+    echo "  --beekeeper-source-image=image_name Allows to specify image name containing a prebuilt beekeper tool"
+    echo "  --base-image=image_name             Allows to specify an image name being use as a base of the one to be built"
+    echo "  --help                              Display this help screen and exit"
+    echo
+}
+
+EXPORT_PATH=""
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --beekeeper-source-image=*)
+      BEEKEEPER_IMAGE="${1#*=}"
+      ;;
+    --base-image=*)
+      BASE_IMAGE="${1#*=}"
+      ;;
+    --help)
+        print_help
+        exit 0
+        ;;
+    *)
+        if [ -z "$BUILD_IMAGE_TAG" ];
+        then
+          BUILD_IMAGE_TAG="${1}"
+        elif [ -z "$SRCROOTDIR" ];
+        then
+          SRCROOTDIR="${1}"
+        elif [ -z "$REGISTRY" ];
+        then
+          REGISTRY=${1}
+        else
+          echo "ERROR: '$1' is not a valid option/positional argument"
+          echo
+          print_help
+          exit 2
+        fi
+        ;;
+    esac
+    shift
+done
+
+TST_IMGTAG=${BUILD_IMAGE_TAG:?"Missing arg #1 to specify built image tag"}
+TST_SRCDIR=${SRCROOTDIR:?"Missing arg #2 to specify source directory"}
+TST_REGISTRY=${REGISTRY:?"Missing arg #3 to specify target container registry"}
+
+TST_BEEKEEPER_IMAGE=${BEEKEEPER_IMAGE:?"Missing --beekeeper-source-image to specify beekeeper binary source"}
+TST_BASE_IMAGE=${BASE_IMAGE:?"Missing --base-image option to specify base image"}
+
+# Supplement a registry path by trailing slash (if needed)
+[[ "${REGISTRY}" != */ ]] && REGISTRY="${REGISTRY}/"
+
+echo "Moving into source root directory: ${SRCROOTDIR}"
+
+pushd "$SRCROOTDIR"
+#pwd
+
+export DOCKER_BUILDKIT=1
+
+docker build --target=instance \
+  --build-arg CI_REGISTRY_IMAGE=$REGISTRY \
+  --build-arg BASE_IMAGE=${BASE_IMAGE} \
+  --build-arg BEEKEEPER_IMAGE=${BEEKEEPER_IMAGE} \
+  -t ${REGISTRY}${IMAGE_TAG_PREFIX}instance:${IMAGE_TAG_PREFIX}instance-${BUILD_IMAGE_TAG} \
+  -f docker/Dockerfile .
+
+popd
+
+echo "CLIVE_IMAGE_NAME=${REGISTRY}${IMAGE_TAG_PREFIX}instance:${IMAGE_TAG_PREFIX}instance-${BUILD_IMAGE_TAG}" > docker_image_name.env
diff --git a/scripts/common.sh b/scripts/common.sh
new file mode 100755
index 0000000000..7c32f8c7d4
--- /dev/null
+++ b/scripts/common.sh
@@ -0,0 +1,52 @@
+#! /bin/bash
+
+set -euo pipefail
+
+exec > >(tee -i "${LOG_FILE}") 2>&1
+
+log_exec_params() {
+  echo
+  echo -n "$0 parameters: "
+  for arg in "$@"; do echo -n "$arg "; done
+  echo
+}
+
+do_clone_commit() {
+  local commit="$1"
+  local src_dir=$2
+  local repo_url=$3
+
+  echo "Cloning commit: $commit from $repo_url into: $src_dir ..."
+  mkdir -p "$src_dir"
+  pushd "$src_dir"
+
+  git init
+  git remote add origin "$repo_url"
+  git fetch --depth 1 origin "$commit"
+  git checkout FETCH_HEAD
+  git submodule update --init --recursive
+
+  popd
+}
+
+do_clone_branch() {
+  local branch=$1
+  local src_dir="$2"
+  local repo_url="$3"
+  echo "Cloning branch: $branch from $repo_url ..."
+  git clone --recurse-submodules --shallow-submodules --single-branch --depth=1 --branch "$branch" -- "$repo_url" "$src_dir"
+}
+
+
+do_clone() {
+  local branch=$1
+  local src_dir="$2"
+  local repo_url="$3"
+  local commit="$4"
+
+  if [[ "$commit" != "" ]]; then
+    do_clone_commit $commit "$src_dir" $repo_url
+  else
+    do_clone_branch "$branch" "$src_dir" $repo_url
+  fi
+}
diff --git a/scripts/run_instance.sh b/scripts/run_instance.sh
new file mode 100755
index 0000000000..a32da2ebc3
--- /dev/null
+++ b/scripts/run_instance.sh
@@ -0,0 +1,90 @@
+#! /bin/bash
+
+SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+echo "$SCRIPTPATH"
+
+LOG_FILE=run_instance.log
+source "$SCRIPTPATH/common.sh"
+
+log_exec_params "$@"
+
+# Script reponsible for starting a docker container built for image specified at command line.
+
+print_help () {
+    echo "Usage: $0 <docker_img> [OPTION[=VALUE]]... [<hived_option>]..."
+    echo
+    echo "Allows to start docker container for a pointed clive docker image."
+    echo "OPTIONS:"
+    echo "  --name=CONTAINER_NAME                 Allows to specify a dedicated name to the spawned container instance"
+    echo "  --detach                              Allows to start container instance in detached mode. Otherwise, you can detach using Ctrl+p+q key binding"
+    echo "  --docker-option=OPTION                Allows to specify additional docker option, to be passed to underlying docker run spawn."
+    echo "  --help                                Display this help screen and exit"
+    echo
+}
+
+DOCKER_ARGS=()
+CLIVE_ARGS=()
+
+CONTAINER_NAME=clive-instance
+IMAGE_NAME=
+
+add_docker_arg() {
+  local arg="$1"
+#  echo "Processing docker argument: ${arg}"
+
+  DOCKER_ARGS+=("$arg")
+}
+
+add_clive_arg() {
+  local arg="$1"
+#  echo "Processing hived argument: ${arg}"
+
+  CLIVE_ARGS+=("$arg")
+}
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+     --name=*)
+        CONTAINER_NAME="${1#*=}"
+        echo "Container name is: $CONTAINER_NAME"
+        ;;
+    --detach)
+      add_docker_arg "--detach"
+      ;;
+
+    --docker-option=*)
+        option="${1#*=}"
+        add_docker_arg "$option"
+        ;;
+    --help)
+        print_help
+        exit 0
+        ;;
+     -*)
+        add_clive_arg "$1"
+        ;;
+     *)
+        IMAGE_NAME="${1}"
+        echo "Using image name: $IMAGE_NAME"
+        ;;
+    esac
+    shift
+done
+
+# Collect remaining command line args to pass to the container to run
+CMD_ARGS=("$@")
+
+if [ -z "$IMAGE_NAME" ]; then
+  echo "Error: Missing docker image name."
+  echo "Usage: $0 <docker_img> [OPTION[=VALUE]]... [<hived_option>]..."
+  echo
+  exit 1
+fi
+
+CMD_ARGS+=("${CLIVE_ARGS[@]}")
+
+#echo "Using docker image: $IMAGE_NAME"
+#echo "Additional hived args: ${CMD_ARGS[@]}"
+
+docker container rm -f -v "$CONTAINER_NAME" 2>/dev/null || true
+docker run --rm -it -e HIVED_UID=$(id -u) -e COLUMNS=$(tput cols) -e LINES=$(tput lines) --name "$CONTAINER_NAME" --stop-timeout=180 ${DOCKER_ARGS[@]} "${IMAGE_NAME}" "${CMD_ARGS[@]}"
-- 
GitLab


From 7ea9b0ff35e7c406409a161c29c787976e98847c Mon Sep 17 00:00:00 2001
From: Bartek Wrona <wrona@syncad.com>
Date: Wed, 12 Jul 2023 23:41:23 +0200
Subject: [PATCH 12/20] Defined docker image building job using
 build_instance.sh script and reusing common base specific to image building.

---
 .gitlab-ci.yml | 38 +++++++++++++++++++++++++++++++++++++-
 1 file changed, 37 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 57a66fa517..12242ee6aa 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -23,13 +23,14 @@ variables:
 
   # uses registry.gitlab.syncad.com/hive/hive/ci-base-image:ubuntu22.04-5
   TEST_IMAGE_TAG: "@sha256:eaa6f32730f0f4c45a95a65b6d790bea2b71a59276eb28795c04be7703f003e6"
+  CI_BASE_IMAGE: "registry.gitlab.syncad.com/hive/hive/ci-base-image${TEST_IMAGE_TAG}"
 
 include:
   - project: 'hive/hive'
     ref: fc899116bf691309cc618e51a9c2d8c6d5edb8cd #develop
     file: '/scripts/ci-helpers/prepare_data_image_job.yml'
 
-image: "registry.gitlab.syncad.com/hive/hive/ci-base-image${TEST_IMAGE_TAG}"
+image: ${CI_BASE_IMAGE}
 
 .shared_tags: &shared_tags
   - public-runner-docker
@@ -139,6 +140,41 @@ testing_clive:
 
 #>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>| BUILD DOCKER |>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
 
+.prepare_clive_image:
+  extends: .docker_image_builder_job
+  stage: build_docker
+  needs:
+    - job: pre_commit_checks
+    - job: prepare_mainnet_hived_image
+      artifacts: true
+
+  variables:
+    BASE_IMAGE: ""
+    BUILD_SUFFIX: ""
+
+    TAG_COMMIT: ${BUILD_SUFFIX}${CI_COMMIT_SHORT_SHA}
+    SCRIPTS_PATH: "$CI_PROJECT_DIR/scripts"
+
+  script:
+    - $SCRIPTS_PATH/ci-helpers/build_instance.sh "${TAG_COMMIT}" "${CI_PROJECT_DIR}" $CI_REGISTRY_IMAGE --beekeeper-source-image=${HIVED_IMAGE_NAME} --base-image=${BASE_IMAGE}
+    - echo $CI_BUILD_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY
+    - source docker_image_name.env
+    - docker push ${CLIVE_IMAGE_NAME}
+  artifacts:
+    reports:
+      dotenv: docker_image_name.env
+    paths:
+      - ./docker_image_name.env
+    expire_in: 6 hours
+
+  tags: *shared_tags
+
+build_and_push_clive_image:
+  extends: .prepare_clive_image
+  variables:
+    BASE_IMAGE: ${CI_BASE_IMAGE}
+
+
 .build_and_push_base:
   image: docker:20.10.13
   stage: build_docker
-- 
GitLab


From 107c7788c3ff69fd85e70c43ca45f9d3d0878171 Mon Sep 17 00:00:00 2001
From: Bartek Wrona <wrona@syncad.com>
Date: Thu, 13 Jul 2023 00:24:59 +0200
Subject: [PATCH 13/20] Explicitly defined GIT_STRATEGY=clone

---
 .gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 12242ee6aa..49a50f51cc 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -12,6 +12,7 @@ workflow:
     - if: $CI_COMMIT_BRANCH
 
 variables:
+  GIT_STRATEGY: clone
   GIT_SUBMODULE_STRATEGY: recursive
   PACKAGES_TO_CHECK: "clive/ tests/"
   # colors:
-- 
GitLab


From bc32aa23a46feb850705eab2ec0a59fb249ac709 Mon Sep 17 00:00:00 2001
From: Bartek Wrona <wrona@syncad.com>
Date: Thu, 13 Jul 2023 00:51:54 +0200
Subject: [PATCH 14/20] Defined job automatically additionally tagging built
 image using stable branch name (i.e. develop or master)

---
 .gitlab-ci.yml | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 49a50f51cc..2e81f1962e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -175,6 +175,23 @@ build_and_push_clive_image:
   variables:
     BASE_IMAGE: ${CI_BASE_IMAGE}
 
+build_and_push_clive_stable_image:
+  stage: build_docker
+  needs:
+    - job: build_and_push_clive_image
+      artifacts: true
+
+  only:
+    - develop
+    - master
+
+  variables:
+    BUILD_SUFFIX: ""
+    ADDITIONAL_IMAGE_NAME: ${CI_REGISTRY_IMAGE}instance:instance-${BUILD_SUFFIX}${CI_COMMIT_REF_NAME}
+  script:
+    - docker image tag ${CLIVE_IMAGE_NAME} ${ADDITIONAL_IMAGE_NAME}
+    - echo $CI_BUILD_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY
+    - docker push ${ADDITIONAL_IMAGE_NAME}
 
 .build_and_push_base:
   image: docker:20.10.13
-- 
GitLab


From ed53d9c93ba21b12b123815a23787084d74814b8 Mon Sep 17 00:00:00 2001
From: Bartek Wrona <wrona@syncad.com>
Date: Thu, 13 Jul 2023 18:08:34 +0200
Subject: [PATCH 15/20] Defined addtional Docker target to build a clive image
 having embedded Hive testnet installation.

---
 docker/Dockerfile | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index e08980f3e9..0d9a755b16 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -7,6 +7,8 @@ FROM $BEEKEEPER_IMAGE as beekeper_source
 
 FROM $BASE_IMAGE AS instance
 
+SHELL ["/bin/bash", "-c"]
+
 ADD --chown=hived_admin:users . /clive
 
 WORKDIR /clive
@@ -21,3 +23,21 @@ RUN poetry self update && \
 ENV COLORTERM=truecolor
 
 ENTRYPOINT ["poetry", "run", "clive"]
+
+# this target should be built using a testnet hived image as a base (to have embedded testnet)
+FROM instance as embedded_testnet_instance
+
+ARG CLIVE_SECRETS__DEFAULT_KEY="5KTNAYSHVzhnVPrwHpKhc5QqNQt6aW8JsrMT7T4hyrKydzYvYik"
+ENV CLIVE_SECRETS__DEFAULT_KEY=${CLIVE_SECRETS__DEFAULT_KEY}
+
+ENV HIVED_PATH="/home/hived/bin/hived"
+ENV CLI_WALLET_PATH="/home/hived/bin/cli_wallet"
+ENV GET_DEV_KEY_PATH="/home/hived/bin/get_dev_key"
+ENV COMPRESS_BLOCK_LOG_PATH="/home/hived/bin/compress_block_log"
+
+WORKDIR /clive
+
+# Run installation once again to supplement extras and development dependencies
+RUN poetry install
+
+ENTRYPOINT ["poetry", "run", "python", "testnet_node.py"]
-- 
GitLab


From 5dce17e162c653b2b92b6dacf92b72d56eb42f03 Mon Sep 17 00:00:00 2001
From: Bartek Wrona <wrona@syncad.com>
Date: Thu, 13 Jul 2023 18:09:53 +0200
Subject: [PATCH 16/20] build_instance.sh script supports embedded-testnet
 installation mode

---
 scripts/ci-helpers/build_instance.sh | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/scripts/ci-helpers/build_instance.sh b/scripts/ci-helpers/build_instance.sh
index ae40ac09e2..6013dec16d 100755
--- a/scripts/ci-helpers/build_instance.sh
+++ b/scripts/ci-helpers/build_instance.sh
@@ -16,6 +16,7 @@ IMAGE_TAG_PREFIX=""
 BEEKEEPER_IMAGE=""
 BASE_IMAGE=""
 
+DOCKER_TARGET="instance"
 
 print_help () {
     echo "Usage: $0 <image_tag> <src_dir> <registry_url> [OPTION[=VALUE]]..."
@@ -24,6 +25,7 @@ print_help () {
     echo "OPTIONS:"
     echo "  --beekeeper-source-image=image_name Allows to specify image name containing a prebuilt beekeper tool"
     echo "  --base-image=image_name             Allows to specify an image name being use as a base of the one to be built"
+    echo "  --embedded-testnet                  Allows to build a clive image having embedded a hived testnet inside (ready for immediate sanboxing run)"
     echo "  --help                              Display this help screen and exit"
     echo
 }
@@ -38,6 +40,10 @@ while [ $# -gt 0 ]; do
     --base-image=*)
       BASE_IMAGE="${1#*=}"
       ;;
+    --embedded-testnet)
+      DOCKER_TARGET="embedded_testnet_instance"
+      IMAGE_TAG_PREFIX="testnet-"
+      ;;
     --help)
         print_help
         exit 0
@@ -76,11 +82,10 @@ TST_BASE_IMAGE=${BASE_IMAGE:?"Missing --base-image option to specify base image"
 echo "Moving into source root directory: ${SRCROOTDIR}"
 
 pushd "$SRCROOTDIR"
-#pwd
 
 export DOCKER_BUILDKIT=1
 
-docker build --target=instance \
+docker build --target=${DOCKER_TARGET} \
   --build-arg CI_REGISTRY_IMAGE=$REGISTRY \
   --build-arg BASE_IMAGE=${BASE_IMAGE} \
   --build-arg BEEKEEPER_IMAGE=${BEEKEEPER_IMAGE} \
@@ -90,3 +95,5 @@ docker build --target=instance \
 popd
 
 echo "CLIVE_IMAGE_NAME=${REGISTRY}${IMAGE_TAG_PREFIX}instance:${IMAGE_TAG_PREFIX}instance-${BUILD_IMAGE_TAG}" > docker_image_name.env
+echo "CLIVE_IMAGE_PATH=${REGISTRY}${IMAGE_TAG_PREFIX}instance" >> docker_image_name.env
+echo "CLIVE_IMAGE_TAG_PREFIX=:${IMAGE_TAG_PREFIX}instance-" >>  docker_image_name.env
-- 
GitLab


From b6a428fb1e8a4293c20e4e2afaa7bb6ec53ca9b4 Mon Sep 17 00:00:00 2001
From: Bartek Wrona <wrona@syncad.com>
Date: Thu, 13 Jul 2023 18:11:49 +0200
Subject: [PATCH 17/20] testnet_node.py supports clive autolaunch

---
 testnet_node.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/testnet_node.py b/testnet_node.py
index ffa8e4b293..21861c898f 100644
--- a/testnet_node.py
+++ b/testnet_node.py
@@ -1,11 +1,16 @@
 from __future__ import annotations
 
+import sys
 import time
 from random import randint
 from typing import TYPE_CHECKING
 
 import test_tools as tt
 
+from clive.main import main as clive_main
+
+ARGS_COUNT = 2
+
 if TYPE_CHECKING:
     from typing import Any
 
@@ -53,7 +58,12 @@ tt.logger.info(f"{alice.name} public key: {alice.public_key}")
 tt.logger.info(f"{alice.name} private key: {alice.private_key}")
 tt.logger.info("done!")
 
-tt.logger.info("serving forever... press Ctrl+C to exit")
+# passing any argument to the script will skip clive autolaunch i.e. for debugging purposes.
+if len(sys.argv) < ARGS_COUNT:
+    tt.logger.info("Attempting to start a clive interactive mode - exit to finish")
+    clive_main()
+else:
+    tt.logger.info("serving forever... press Ctrl+C to exit")
 
-while True:
-    time.sleep(1)
+    while True:
+        time.sleep(1)
-- 
GitLab


From fe6875883de2f07c83c644c371880460640e9451 Mon Sep 17 00:00:00 2001
From: Bartek Wrona <wrona@syncad.com>
Date: Thu, 13 Jul 2023 18:12:39 +0200
Subject: [PATCH 18/20] Defined jobs to build a Clive image having embedded
 testnet installation

---
 .gitlab-ci.yml | 34 ++++++++++++++++++++++++++++++----
 1 file changed, 30 insertions(+), 4 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2e81f1962e..ee4894d754 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -151,13 +151,13 @@ testing_clive:
 
   variables:
     BASE_IMAGE: ""
-    BUILD_SUFFIX: ""
+    BUILD_INSTANCE_ARGS: ""
 
     TAG_COMMIT: ${BUILD_SUFFIX}${CI_COMMIT_SHORT_SHA}
     SCRIPTS_PATH: "$CI_PROJECT_DIR/scripts"
 
   script:
-    - $SCRIPTS_PATH/ci-helpers/build_instance.sh "${TAG_COMMIT}" "${CI_PROJECT_DIR}" $CI_REGISTRY_IMAGE --beekeeper-source-image=${HIVED_IMAGE_NAME} --base-image=${BASE_IMAGE}
+    - $SCRIPTS_PATH/ci-helpers/build_instance.sh "${TAG_COMMIT}" "${CI_PROJECT_DIR}" $CI_REGISTRY_IMAGE --beekeeper-source-image=${HIVED_IMAGE_NAME} --base-image=${BASE_IMAGE} ${BUILD_INSTANCE_ARGS}
     - echo $CI_BUILD_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY
     - source docker_image_name.env
     - docker push ${CLIVE_IMAGE_NAME}
@@ -175,6 +175,16 @@ build_and_push_clive_image:
   variables:
     BASE_IMAGE: ${CI_BASE_IMAGE}
 
+build_and_push_clive_testnet_image:
+  extends: .prepare_clive_image
+  needs:
+    - job: prepare_testnet_hived_image
+      artifacts: true
+
+  variables:
+    BASE_IMAGE: ${HIVED_IMAGE_NAME}
+    BUILD_INSTANCE_ARGS: "--embedded-testnet"
+
 build_and_push_clive_stable_image:
   stage: build_docker
   needs:
@@ -186,8 +196,24 @@ build_and_push_clive_stable_image:
     - master
 
   variables:
-    BUILD_SUFFIX: ""
-    ADDITIONAL_IMAGE_NAME: ${CI_REGISTRY_IMAGE}instance:instance-${BUILD_SUFFIX}${CI_COMMIT_REF_NAME}
+    ADDITIONAL_IMAGE_NAME: ${CLIVE_IMAGE_PATH}${CLIVE_IMAGE_TAG_PREFIX}${CI_COMMIT_REF_NAME}
+  script:
+    - docker image tag ${CLIVE_IMAGE_NAME} ${ADDITIONAL_IMAGE_NAME}
+    - echo $CI_BUILD_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY
+    - docker push ${ADDITIONAL_IMAGE_NAME}
+
+build_and_push_clive_stable_testnet_image:
+  stage: build_docker
+  needs:
+    - job: build_and_push_clive_testnet_image
+      artifacts: true
+
+  only:
+    - develop
+    - master
+
+  variables:
+    ADDITIONAL_IMAGE_NAME: ${CLIVE_IMAGE_PATH}${CLIVE_IMAGE_TAG_PREFIX}${CI_COMMIT_REF_NAME}
   script:
     - docker image tag ${CLIVE_IMAGE_NAME} ${ADDITIONAL_IMAGE_NAME}
     - echo $CI_BUILD_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY
-- 
GitLab


From 2d61534506c541e542a6b97ebc41cb62d51ec530 Mon Sep 17 00:00:00 2001
From: Bartek Wrona <wrona@syncad.com>
Date: Thu, 13 Jul 2023 19:28:37 +0200
Subject: [PATCH 19/20] Removed obsolete definitions of docker building jobs

---
 .gitlab-ci.yml | 52 --------------------------------------------------
 1 file changed, 52 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ee4894d754..3db9d46ed1 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -219,56 +219,4 @@ build_and_push_clive_stable_testnet_image:
     - echo $CI_BUILD_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY
     - docker push ${ADDITIONAL_IMAGE_NAME}
 
-.build_and_push_base:
-  image: docker:20.10.13
-  stage: build_docker
-  needs:
-    - job: pre_commit_checks
-    - job: prepare_mainnet_hived_image
-      artifacts: true
-    - job: testing_clive
-  variables:
-    DOCKER_DRIVER: overlay2
-    DOCKER_TLS_CERTDIR: ""
-    TAG_COMMIT: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_NAME$BUILD_SUFFIX:$CI_COMMIT_SHORT_SHA
-    TAG_LATEST: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_NAME$BUILD_SUFFIX:latest
-  services:
-    - docker:20.10.13-dind
-  before_script:
-    - ls -la
-    - cd docker/
-    - echo -e "${TXT_BLUE}Building a docker image started...${TXT_CLEAR}"
-    - echo $CI_BUILD_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY
-  after_script:
-    - docker push $TAG_COMMIT
-    - docker push $TAG_LATEST
-  tags: *shared_tags
-
-build_and_push_docker_image:
-  extends: .build_and_push_base
-  only:
-    - develop
-    - master
-  script:
-    - docker build --build-arg BEEKEEPER_LOCATION="${BEEKEEPER_BINARY}" -t $TAG_COMMIT -t $TAG_LATEST -f Dockerfile ..
-
-build_and_push_docker_testnet_image:
-  extends: .build_and_push_base
-  needs:
-    - job: pre_commit_checks
-    - job: testing_clive
-    - job: prepare_mainnet_hived_image
-      artifacts: true
-    - job: prepare_testnet_hived_image
-      artifacts: true
-  rules:
-    - if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_REF_NAME != "develop"'
-      when: manual
-    - when: always
-  allow_failure: true
-  variables:
-    BUILD_SUFFIX: "-testnet"
-  script:
-    - docker build --build-arg BEEKEEPER_LOCATION="${BEEKEEPER_BINARY}" -t $TAG_COMMIT -t $TAG_LATEST -f Dockerfile.TESTNET ..
-
 #<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<| BUILD DOCKER |<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
-- 
GitLab


From 418d78f9c17e946265bb06c9b1ec9a2332e41cf3 Mon Sep 17 00:00:00 2001
From: Bartek Wrona <wrona@syncad.com>
Date: Thu, 13 Jul 2023 22:35:20 +0200
Subject: [PATCH 20/20] Removed obsolete files

---
 docker/Dockerfile.TESTNET      | 24 ------------------------
 docker/entrypoint-testnet.bash |  7 -------
 2 files changed, 31 deletions(-)
 delete mode 100644 docker/Dockerfile.TESTNET
 delete mode 100644 docker/entrypoint-testnet.bash

diff --git a/docker/Dockerfile.TESTNET b/docker/Dockerfile.TESTNET
deleted file mode 100644
index 83cbfd2f9a..0000000000
--- a/docker/Dockerfile.TESTNET
+++ /dev/null
@@ -1,24 +0,0 @@
-FROM registry.gitlab.syncad.com/hive/hive/ci-base-image:ubuntu22.04-3
-
-ADD --chown=hived_admin:users . /clive
-WORKDIR /clive
-
-ARG HIVE_BUILD_ROOT_LOCATION="./hived/"
-ENV HIVE_BUILD_ROOT_PATH="/clive/hived/"
-ADD --chown=hived_admin:users "${HIVE_BUILD_ROOT_LOCATION}" "${HIVE_BUILD_ROOT_PATH}"
-
-ARG BEEKEEPER_LOCATION="./beekeeper"
-ENV BEEKEEPER_PATH="/clive/beekeeper"
-ADD --chown=hived_admin:users "${BEEKEEPER_LOCATION}" "${BEEKEEPER_PATH}"
-
-RUN poetry self update
-
-RUN poetry install
-
-# crucial for proper display
-ENV COLORTERM=truecolor
-
-ENV CLIVE_BEEKEEPER__PATH="${BEEKEEPER_PATH}"
-ENV CLIVE_SECRETS__DEFAULT_KEY="5KTNAYSHVzhnVPrwHpKhc5QqNQt6aW8JsrMT7T4hyrKydzYvYik"
-
-ENTRYPOINT bash docker/entrypoint-testnet.bash
diff --git a/docker/entrypoint-testnet.bash b/docker/entrypoint-testnet.bash
deleted file mode 100644
index 09c82f0eab..0000000000
--- a/docker/entrypoint-testnet.bash
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-poetry run python testnet_node.py &
-
-sleep 15
-
-poetry run clive
-- 
GitLab