From 52af37a5696db084c5fff998231087d1a55206ef Mon Sep 17 00:00:00 2001 From: Bartek Wrona <wrona@syncad.com> Date: Tue, 4 Jul 2023 18:00:44 +0200 Subject: [PATCH 01/20] Base CI image switched to latest version existing in Hived repo: ubuntu22-04.5. Also included `prepare_data_image_job.yml` to use common predefined jobs for building Hived image. --- .gitlab-ci.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e0c286b28f..2421e56047 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,3 @@ -image: "registry.gitlab.syncad.com/hive/hive/ci-base-image:ubuntu22.04-3" - stages: - static_code_analysis - build @@ -22,6 +20,16 @@ variables: BEEKEEPER_BINARY: "$CI_PROJECT_DIR/beekeeper" HIVE_BUILD_ROOT_PATH: "$CI_PROJECT_DIR/hived" + # uses registry.gitlab.syncad.com/hive/hive/ci-base-image:ubuntu22.04-5 + TEST_IMAGE_TAG: "@sha256:eaa6f32730f0f4c45a95a65b6d790bea2b71a59276eb28795c04be7703f003e6" + +include: + - project: 'hive/hive' + ref: fc899116bf691309cc618e51a9c2d8c6d5edb8cd #develop + file: '/scripts/ci-helpers/prepare_data_image_job.yml' + +image: "registry.gitlab.syncad.com/hive/hive/ci-base-image${TEST_IMAGE_TAG}" + .shared_tags: &shared_tags - hived-for-tests -- GitLab From 2df48395fd660ffb61b0b5c7757fc189ad3c3a74 Mon Sep 17 00:00:00 2001 From: Bartek Wrona <wrona@syncad.com> Date: Tue, 4 Jul 2023 18:05:44 +0200 Subject: [PATCH 02/20] Defined `common_prepare_hived_image` job and 2 dependent: prepare_mainnet_hived_image and prepare_testnet_hived_image --- .gitlab-ci.yml | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2421e56047..bb0de0dea5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,8 +17,9 @@ variables: # colors: TXT_BLUE: "\e[1;34m" TXT_CLEAR: "\e[0m" - BEEKEEPER_BINARY: "$CI_PROJECT_DIR/beekeeper" - HIVE_BUILD_ROOT_PATH: "$CI_PROJECT_DIR/hived" + + HIVE_BUILD_ROOT_PATH: "hived-binaries" + BEEKEEPER_BINARY: "${HIVE_BUILD_ROOT_PATH}/mainnet/beekeeper" # uses registry.gitlab.syncad.com/hive/hive/ci-base-image:ubuntu22.04-5 TEST_IMAGE_TAG: "@sha256:eaa6f32730f0f4c45a95a65b6d790bea2b71a59276eb28795c04be7703f003e6" @@ -79,6 +80,28 @@ type_check_with_mypy: #>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>| BUILD |>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +.common_prepare_hived_image: + extends: .prepare_hived_image + stage: build + variables: + REGISTRY_USER: "$HIVED_CI_IMGBUILDER_USER" + REGISTRY_PASS: $HIVED_CI_IMGBUILDER_PASSWORD + SUBMODULE_DIR: "${CI_PROJECT_DIR}/hive/" + BINARY_CACHE_PATH: "${HIVE_BUILD_ROOT_PATH}/${HIVE_NETWORK_TYPE}" + tags: + - public-runner-docker + - hived-for-tests + +prepare_mainnet_hived_image: + extends: .common_prepare_hived_image + variables: + HIVE_NETWORK_TYPE: mainnet + +prepare_testnet_hived_image: + extends: .common_prepare_hived_image + variables: + HIVE_NETWORK_TYPE: testnet + .hive_based_build: stage: build needs: [] -- GitLab From e5a9388552b350eb63fe70cad1cdb59eb1eeb862 Mon Sep 17 00:00:00 2001 From: Bartek Wrona <wrona@syncad.com> Date: Tue, 4 Jul 2023 18:17:55 +0200 Subject: [PATCH 03/20] build_beekeeper job references replaced with prepare_mainnet_hived_image (beekeper tool can be always used in mainnet build) --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bb0de0dea5..a62ef038f3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -142,7 +142,7 @@ build_hived: stage: tests extends: .configuration needs: - - job: build_beekeeper + - job: prepare_mainnet_hived_image artifacts: true - job: build_hived artifacts: true @@ -174,7 +174,7 @@ testing_clive: stage: build_docker needs: - job: pre_commit_checks - - job: build_beekeeper + - job: prepare_mainnet_hived_image artifacts: true - job: testing_clive variables: @@ -207,7 +207,7 @@ build_and_push_docker_testnet_image: needs: - job: pre_commit_checks - job: testing_clive - - job: build_beekeeper + - job: prepare_mainnet_hived_image artifacts: true - job: build_hived artifacts: true -- GitLab From f91a8a30b90c14bb43bc9ef3868b715596fbe8f3 Mon Sep 17 00:00:00 2001 From: Bartek Wrona <wrona@syncad.com> Date: Tue, 4 Jul 2023 18:26:29 +0200 Subject: [PATCH 04/20] build_hived job references replaced with prepare_testnet_hived_image --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a62ef038f3..3789d9821c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -144,7 +144,7 @@ build_hived: needs: - job: prepare_mainnet_hived_image artifacts: true - - job: build_hived + - job: prepare_testnet_hived_image artifacts: true variables: CLIVE_BEEKEEPER__PATH: $BEEKEEPER_BINARY @@ -209,7 +209,7 @@ build_and_push_docker_testnet_image: - job: testing_clive - job: prepare_mainnet_hived_image artifacts: true - - job: build_hived + - job: prepare_testnet_hived_image artifacts: true rules: - if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_REF_NAME != "develop"' -- GitLab From d6ce072b5235c18563bf835c73584157a54a1b25 Mon Sep 17 00:00:00 2001 From: Bartek Wrona <wrona@syncad.com> Date: Tue, 4 Jul 2023 18:27:45 +0200 Subject: [PATCH 05/20] Removed unused anymore .hive_based_build, build_hived and build_beekeeper job definitions. --- .gitlab-ci.yml | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3789d9821c..f76543593b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -102,38 +102,6 @@ prepare_testnet_hived_image: variables: HIVE_NETWORK_TYPE: testnet -.hive_based_build: - stage: build - needs: [] - variables: - TARGET: "" - SOURCE: "" - DESTINATION: "" - script: - - cd hive - - mkdir build && cd build - - cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_HIVE_TESTNET=ON -GNinja .. - - ninja $TARGET - - cp -r $SOURCE $DESTINATION - artifacts: - paths: - - $DESTINATION - tags: *shared_tags - -build_beekeeper: - extends: .hive_based_build - variables: - TARGET: "beekeeper" - SOURCE: "programs/beekeeper/beekeeper" - DESTINATION: "$BEEKEEPER_BINARY" - -build_hived: - extends: .hive_based_build - variables: - TARGET: "hived" - SOURCE: "." - DESTINATION: "$HIVE_BUILD_ROOT_PATH" - #<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<| BUILD |<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>| TESTS |>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> -- GitLab From 88be03851ac06d4e44622dd97b96dbcaf8ffd440 Mon Sep 17 00:00:00 2001 From: Bartek Wrona <wrona@syncad.com> Date: Tue, 4 Jul 2023 18:35:45 +0200 Subject: [PATCH 06/20] BEEKEEPER_LOCATION is explictly specified during building clive images. --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f76543593b..d06ffd1ebf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -168,7 +168,7 @@ build_and_push_docker_image: - develop - master script: - - docker build -t $TAG_COMMIT -t $TAG_LATEST -f Dockerfile .. + - docker build --build-arg BEEKEEPER_LOCATION="${BEEKEEPER_BINARY}" -t $TAG_COMMIT -t $TAG_LATEST -f Dockerfile .. build_and_push_docker_testnet_image: extends: .build_and_push_base @@ -187,6 +187,6 @@ build_and_push_docker_testnet_image: variables: BUILD_SUFFIX: "-testnet" script: - - docker build -t $TAG_COMMIT -t $TAG_LATEST -f Dockerfile.TESTNET .. + - docker build --build-arg BEEKEEPER_LOCATION="${BEEKEEPER_BINARY}" -t $TAG_COMMIT -t $TAG_LATEST -f Dockerfile.TESTNET .. #<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<| BUILD DOCKER |<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< -- GitLab From 89dd71c44a8657a55b422345805889898bc017db Mon Sep 17 00:00:00 2001 From: Bartek Wrona <wrona@syncad.com> Date: Tue, 4 Jul 2023 18:48:54 +0200 Subject: [PATCH 07/20] CI runner tags changed to `public-runner-docker` to use much more flexible runners --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d06ffd1ebf..d84b6af675 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -32,7 +32,7 @@ include: image: "registry.gitlab.syncad.com/hive/hive/ci-base-image${TEST_IMAGE_TAG}" .shared_tags: &shared_tags - - hived-for-tests + - public-runner-docker .configuration: before_script: &configuration_before_script # configure python -- GitLab From 4ab753a2e85e9299801e868bf95fbdf43edb01e4 Mon Sep 17 00:00:00 2001 From: Bartek Wrona <wrona@syncad.com> Date: Tue, 4 Jul 2023 19:12:44 +0200 Subject: [PATCH 08/20] Hived path (testnet version binary) explicitly pointed to test-tools library using HIVED_PATH variable --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d84b6af675..0f4143760f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -116,6 +116,7 @@ prepare_testnet_hived_image: artifacts: true variables: CLIVE_BEEKEEPER__PATH: $BEEKEEPER_BINARY + HIVED_PATH: "${CI_PROJECT_DIR}/${HIVE_BUILD_ROOT_PATH}/testnet/hived" PATH_TO_REPORT: "$CI_PROJECT_DIR/report.xml" artifacts: reports: -- GitLab From ea6a8d7f0634f444adb3a2e6821b8f8b0926c53d Mon Sep 17 00:00:00 2001 From: Bartek Wrona <wrona@syncad.com> Date: Tue, 4 Jul 2023 19:14:30 +0200 Subject: [PATCH 09/20] Each job should be interruptible to cancel executing pipeline when new one is created --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0f4143760f..57a66fa517 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,6 +35,7 @@ image: "registry.gitlab.syncad.com/hive/hive/ci-base-image${TEST_IMAGE_TAG}" - public-runner-docker .configuration: + interruptible: true before_script: &configuration_before_script # configure python - python3 -V - poetry self update -- GitLab From 09720187d926e2b91d23b3d8d8cf1c63ae700264 Mon Sep 17 00:00:00 2001 From: Bartek Wrona <wrona@syncad.com> Date: Wed, 12 Jul 2023 23:05:30 +0200 Subject: [PATCH 10/20] Dockerfile definition adjusted to new image building scheme, where base image and beekeeper source image are specified as docker build args --- docker/Dockerfile | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 8d1f22d665..e08980f3e9 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,15 +1,21 @@ -FROM registry.gitlab.syncad.com/hive/hive/ci-base-image:ubuntu22.04-3 +ARG BEEKEEPER_IMAGE +ARG BASE_IMAGE + +ARG CI_REGISTRY_IMAGE=registry.gitlab.syncad.com/hive/clive/ + +FROM $BEEKEEPER_IMAGE as beekeper_source + +FROM $BASE_IMAGE AS instance ADD --chown=hived_admin:users . /clive + WORKDIR /clive -ARG BEEKEEPER_LOCATION="./beekeeper" ENV BEEKEEPER_PATH="/clive/beekeeper" -ADD --chown=hived_admin:users "${BEEKEEPER_LOCATION}" "${BEEKEEPER_PATH}" - -RUN poetry self update +COPY --from=beekeper_source --chown=hived_admin:users "/home/hived/bin/beekeeper" "${BEEKEEPER_PATH}" -RUN poetry install --only main +RUN poetry self update && \ + poetry install --only main # crucial for proper display ENV COLORTERM=truecolor -- GitLab From 4508db663af98ec612f7754396bc15f5e7bcbbff Mon Sep 17 00:00:00 2001 From: Bartek Wrona <wrona@syncad.com> Date: Wed, 12 Jul 2023 23:06:42 +0200 Subject: [PATCH 11/20] Defined helper scripts: build_instance.sh, run_instance.sh to follow common Hive tools usage scheme --- scripts/ci-helpers/build_instance.sh | 92 ++++++++++++++++++++++++++++ scripts/common.sh | 52 ++++++++++++++++ scripts/run_instance.sh | 90 +++++++++++++++++++++++++++ 3 files changed, 234 insertions(+) create mode 100755 scripts/ci-helpers/build_instance.sh create mode 100755 scripts/common.sh create mode 100755 scripts/run_instance.sh diff --git a/scripts/ci-helpers/build_instance.sh b/scripts/ci-helpers/build_instance.sh new file mode 100755 index 0000000000..ae40ac09e2 --- /dev/null +++ b/scripts/ci-helpers/build_instance.sh @@ -0,0 +1,92 @@ +#! /bin/bash +set -euo pipefail + +SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +SCRIPTSDIR="$SCRIPTPATH/.." + +LOG_FILE=build_instance.log +source "$SCRIPTSDIR/common.sh" + +BUILD_IMAGE_TAG="" +REGISTRY="" +SRCROOTDIR="" + +IMAGE_TAG_PREFIX="" + +BEEKEEPER_IMAGE="" +BASE_IMAGE="" + + +print_help () { + echo "Usage: $0 <image_tag> <src_dir> <registry_url> [OPTION[=VALUE]]..." + echo + echo "Allows to build docker image containing clive installation" + echo "OPTIONS:" + echo " --beekeeper-source-image=image_name Allows to specify image name containing a prebuilt beekeper tool" + echo " --base-image=image_name Allows to specify an image name being use as a base of the one to be built" + echo " --help Display this help screen and exit" + echo +} + +EXPORT_PATH="" + +while [ $# -gt 0 ]; do + case "$1" in + --beekeeper-source-image=*) + BEEKEEPER_IMAGE="${1#*=}" + ;; + --base-image=*) + BASE_IMAGE="${1#*=}" + ;; + --help) + print_help + exit 0 + ;; + *) + if [ -z "$BUILD_IMAGE_TAG" ]; + then + BUILD_IMAGE_TAG="${1}" + elif [ -z "$SRCROOTDIR" ]; + then + SRCROOTDIR="${1}" + elif [ -z "$REGISTRY" ]; + then + REGISTRY=${1} + else + echo "ERROR: '$1' is not a valid option/positional argument" + echo + print_help + exit 2 + fi + ;; + esac + shift +done + +TST_IMGTAG=${BUILD_IMAGE_TAG:?"Missing arg #1 to specify built image tag"} +TST_SRCDIR=${SRCROOTDIR:?"Missing arg #2 to specify source directory"} +TST_REGISTRY=${REGISTRY:?"Missing arg #3 to specify target container registry"} + +TST_BEEKEEPER_IMAGE=${BEEKEEPER_IMAGE:?"Missing --beekeeper-source-image to specify beekeeper binary source"} +TST_BASE_IMAGE=${BASE_IMAGE:?"Missing --base-image option to specify base image"} + +# Supplement a registry path by trailing slash (if needed) +[[ "${REGISTRY}" != */ ]] && REGISTRY="${REGISTRY}/" + +echo "Moving into source root directory: ${SRCROOTDIR}" + +pushd "$SRCROOTDIR" +#pwd + +export DOCKER_BUILDKIT=1 + +docker build --target=instance \ + --build-arg CI_REGISTRY_IMAGE=$REGISTRY \ + --build-arg BASE_IMAGE=${BASE_IMAGE} \ + --build-arg BEEKEEPER_IMAGE=${BEEKEEPER_IMAGE} \ + -t ${REGISTRY}${IMAGE_TAG_PREFIX}instance:${IMAGE_TAG_PREFIX}instance-${BUILD_IMAGE_TAG} \ + -f docker/Dockerfile . + +popd + +echo "CLIVE_IMAGE_NAME=${REGISTRY}${IMAGE_TAG_PREFIX}instance:${IMAGE_TAG_PREFIX}instance-${BUILD_IMAGE_TAG}" > docker_image_name.env diff --git a/scripts/common.sh b/scripts/common.sh new file mode 100755 index 0000000000..7c32f8c7d4 --- /dev/null +++ b/scripts/common.sh @@ -0,0 +1,52 @@ +#! /bin/bash + +set -euo pipefail + +exec > >(tee -i "${LOG_FILE}") 2>&1 + +log_exec_params() { + echo + echo -n "$0 parameters: " + for arg in "$@"; do echo -n "$arg "; done + echo +} + +do_clone_commit() { + local commit="$1" + local src_dir=$2 + local repo_url=$3 + + echo "Cloning commit: $commit from $repo_url into: $src_dir ..." + mkdir -p "$src_dir" + pushd "$src_dir" + + git init + git remote add origin "$repo_url" + git fetch --depth 1 origin "$commit" + git checkout FETCH_HEAD + git submodule update --init --recursive + + popd +} + +do_clone_branch() { + local branch=$1 + local src_dir="$2" + local repo_url="$3" + echo "Cloning branch: $branch from $repo_url ..." + git clone --recurse-submodules --shallow-submodules --single-branch --depth=1 --branch "$branch" -- "$repo_url" "$src_dir" +} + + +do_clone() { + local branch=$1 + local src_dir="$2" + local repo_url="$3" + local commit="$4" + + if [[ "$commit" != "" ]]; then + do_clone_commit $commit "$src_dir" $repo_url + else + do_clone_branch "$branch" "$src_dir" $repo_url + fi +} diff --git a/scripts/run_instance.sh b/scripts/run_instance.sh new file mode 100755 index 0000000000..a32da2ebc3 --- /dev/null +++ b/scripts/run_instance.sh @@ -0,0 +1,90 @@ +#! /bin/bash + +SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +echo "$SCRIPTPATH" + +LOG_FILE=run_instance.log +source "$SCRIPTPATH/common.sh" + +log_exec_params "$@" + +# Script reponsible for starting a docker container built for image specified at command line. + +print_help () { + echo "Usage: $0 <docker_img> [OPTION[=VALUE]]... [<hived_option>]..." + echo + echo "Allows to start docker container for a pointed clive docker image." + echo "OPTIONS:" + echo " --name=CONTAINER_NAME Allows to specify a dedicated name to the spawned container instance" + echo " --detach Allows to start container instance in detached mode. Otherwise, you can detach using Ctrl+p+q key binding" + echo " --docker-option=OPTION Allows to specify additional docker option, to be passed to underlying docker run spawn." + echo " --help Display this help screen and exit" + echo +} + +DOCKER_ARGS=() +CLIVE_ARGS=() + +CONTAINER_NAME=clive-instance +IMAGE_NAME= + +add_docker_arg() { + local arg="$1" +# echo "Processing docker argument: ${arg}" + + DOCKER_ARGS+=("$arg") +} + +add_clive_arg() { + local arg="$1" +# echo "Processing hived argument: ${arg}" + + CLIVE_ARGS+=("$arg") +} + +while [ $# -gt 0 ]; do + case "$1" in + --name=*) + CONTAINER_NAME="${1#*=}" + echo "Container name is: $CONTAINER_NAME" + ;; + --detach) + add_docker_arg "--detach" + ;; + + --docker-option=*) + option="${1#*=}" + add_docker_arg "$option" + ;; + --help) + print_help + exit 0 + ;; + -*) + add_clive_arg "$1" + ;; + *) + IMAGE_NAME="${1}" + echo "Using image name: $IMAGE_NAME" + ;; + esac + shift +done + +# Collect remaining command line args to pass to the container to run +CMD_ARGS=("$@") + +if [ -z "$IMAGE_NAME" ]; then + echo "Error: Missing docker image name." + echo "Usage: $0 <docker_img> [OPTION[=VALUE]]... [<hived_option>]..." + echo + exit 1 +fi + +CMD_ARGS+=("${CLIVE_ARGS[@]}") + +#echo "Using docker image: $IMAGE_NAME" +#echo "Additional hived args: ${CMD_ARGS[@]}" + +docker container rm -f -v "$CONTAINER_NAME" 2>/dev/null || true +docker run --rm -it -e HIVED_UID=$(id -u) -e COLUMNS=$(tput cols) -e LINES=$(tput lines) --name "$CONTAINER_NAME" --stop-timeout=180 ${DOCKER_ARGS[@]} "${IMAGE_NAME}" "${CMD_ARGS[@]}" -- GitLab From 7ea9b0ff35e7c406409a161c29c787976e98847c Mon Sep 17 00:00:00 2001 From: Bartek Wrona <wrona@syncad.com> Date: Wed, 12 Jul 2023 23:41:23 +0200 Subject: [PATCH 12/20] Defined docker image building job using build_instance.sh script and reusing common base specific to image building. --- .gitlab-ci.yml | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 57a66fa517..12242ee6aa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,13 +23,14 @@ variables: # uses registry.gitlab.syncad.com/hive/hive/ci-base-image:ubuntu22.04-5 TEST_IMAGE_TAG: "@sha256:eaa6f32730f0f4c45a95a65b6d790bea2b71a59276eb28795c04be7703f003e6" + CI_BASE_IMAGE: "registry.gitlab.syncad.com/hive/hive/ci-base-image${TEST_IMAGE_TAG}" include: - project: 'hive/hive' ref: fc899116bf691309cc618e51a9c2d8c6d5edb8cd #develop file: '/scripts/ci-helpers/prepare_data_image_job.yml' -image: "registry.gitlab.syncad.com/hive/hive/ci-base-image${TEST_IMAGE_TAG}" +image: ${CI_BASE_IMAGE} .shared_tags: &shared_tags - public-runner-docker @@ -139,6 +140,41 @@ testing_clive: #>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>| BUILD DOCKER |>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +.prepare_clive_image: + extends: .docker_image_builder_job + stage: build_docker + needs: + - job: pre_commit_checks + - job: prepare_mainnet_hived_image + artifacts: true + + variables: + BASE_IMAGE: "" + BUILD_SUFFIX: "" + + TAG_COMMIT: ${BUILD_SUFFIX}${CI_COMMIT_SHORT_SHA} + SCRIPTS_PATH: "$CI_PROJECT_DIR/scripts" + + script: + - $SCRIPTS_PATH/ci-helpers/build_instance.sh "${TAG_COMMIT}" "${CI_PROJECT_DIR}" $CI_REGISTRY_IMAGE --beekeeper-source-image=${HIVED_IMAGE_NAME} --base-image=${BASE_IMAGE} + - echo $CI_BUILD_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY + - source docker_image_name.env + - docker push ${CLIVE_IMAGE_NAME} + artifacts: + reports: + dotenv: docker_image_name.env + paths: + - ./docker_image_name.env + expire_in: 6 hours + + tags: *shared_tags + +build_and_push_clive_image: + extends: .prepare_clive_image + variables: + BASE_IMAGE: ${CI_BASE_IMAGE} + + .build_and_push_base: image: docker:20.10.13 stage: build_docker -- GitLab From 107c7788c3ff69fd85e70c43ca45f9d3d0878171 Mon Sep 17 00:00:00 2001 From: Bartek Wrona <wrona@syncad.com> Date: Thu, 13 Jul 2023 00:24:59 +0200 Subject: [PATCH 13/20] Explicitly defined GIT_STRATEGY=clone --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 12242ee6aa..49a50f51cc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,6 +12,7 @@ workflow: - if: $CI_COMMIT_BRANCH variables: + GIT_STRATEGY: clone GIT_SUBMODULE_STRATEGY: recursive PACKAGES_TO_CHECK: "clive/ tests/" # colors: -- GitLab From bc32aa23a46feb850705eab2ec0a59fb249ac709 Mon Sep 17 00:00:00 2001 From: Bartek Wrona <wrona@syncad.com> Date: Thu, 13 Jul 2023 00:51:54 +0200 Subject: [PATCH 14/20] Defined job automatically additionally tagging built image using stable branch name (i.e. develop or master) --- .gitlab-ci.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 49a50f51cc..2e81f1962e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -175,6 +175,23 @@ build_and_push_clive_image: variables: BASE_IMAGE: ${CI_BASE_IMAGE} +build_and_push_clive_stable_image: + stage: build_docker + needs: + - job: build_and_push_clive_image + artifacts: true + + only: + - develop + - master + + variables: + BUILD_SUFFIX: "" + ADDITIONAL_IMAGE_NAME: ${CI_REGISTRY_IMAGE}instance:instance-${BUILD_SUFFIX}${CI_COMMIT_REF_NAME} + script: + - docker image tag ${CLIVE_IMAGE_NAME} ${ADDITIONAL_IMAGE_NAME} + - echo $CI_BUILD_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY + - docker push ${ADDITIONAL_IMAGE_NAME} .build_and_push_base: image: docker:20.10.13 -- GitLab From ed53d9c93ba21b12b123815a23787084d74814b8 Mon Sep 17 00:00:00 2001 From: Bartek Wrona <wrona@syncad.com> Date: Thu, 13 Jul 2023 18:08:34 +0200 Subject: [PATCH 15/20] Defined addtional Docker target to build a clive image having embedded Hive testnet installation. --- docker/Dockerfile | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index e08980f3e9..0d9a755b16 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -7,6 +7,8 @@ FROM $BEEKEEPER_IMAGE as beekeper_source FROM $BASE_IMAGE AS instance +SHELL ["/bin/bash", "-c"] + ADD --chown=hived_admin:users . /clive WORKDIR /clive @@ -21,3 +23,21 @@ RUN poetry self update && \ ENV COLORTERM=truecolor ENTRYPOINT ["poetry", "run", "clive"] + +# this target should be built using a testnet hived image as a base (to have embedded testnet) +FROM instance as embedded_testnet_instance + +ARG CLIVE_SECRETS__DEFAULT_KEY="5KTNAYSHVzhnVPrwHpKhc5QqNQt6aW8JsrMT7T4hyrKydzYvYik" +ENV CLIVE_SECRETS__DEFAULT_KEY=${CLIVE_SECRETS__DEFAULT_KEY} + +ENV HIVED_PATH="/home/hived/bin/hived" +ENV CLI_WALLET_PATH="/home/hived/bin/cli_wallet" +ENV GET_DEV_KEY_PATH="/home/hived/bin/get_dev_key" +ENV COMPRESS_BLOCK_LOG_PATH="/home/hived/bin/compress_block_log" + +WORKDIR /clive + +# Run installation once again to supplement extras and development dependencies +RUN poetry install + +ENTRYPOINT ["poetry", "run", "python", "testnet_node.py"] -- GitLab From 5dce17e162c653b2b92b6dacf92b72d56eb42f03 Mon Sep 17 00:00:00 2001 From: Bartek Wrona <wrona@syncad.com> Date: Thu, 13 Jul 2023 18:09:53 +0200 Subject: [PATCH 16/20] build_instance.sh script supports embedded-testnet installation mode --- scripts/ci-helpers/build_instance.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/scripts/ci-helpers/build_instance.sh b/scripts/ci-helpers/build_instance.sh index ae40ac09e2..6013dec16d 100755 --- a/scripts/ci-helpers/build_instance.sh +++ b/scripts/ci-helpers/build_instance.sh @@ -16,6 +16,7 @@ IMAGE_TAG_PREFIX="" BEEKEEPER_IMAGE="" BASE_IMAGE="" +DOCKER_TARGET="instance" print_help () { echo "Usage: $0 <image_tag> <src_dir> <registry_url> [OPTION[=VALUE]]..." @@ -24,6 +25,7 @@ print_help () { echo "OPTIONS:" echo " --beekeeper-source-image=image_name Allows to specify image name containing a prebuilt beekeper tool" echo " --base-image=image_name Allows to specify an image name being use as a base of the one to be built" + echo " --embedded-testnet Allows to build a clive image having embedded a hived testnet inside (ready for immediate sanboxing run)" echo " --help Display this help screen and exit" echo } @@ -38,6 +40,10 @@ while [ $# -gt 0 ]; do --base-image=*) BASE_IMAGE="${1#*=}" ;; + --embedded-testnet) + DOCKER_TARGET="embedded_testnet_instance" + IMAGE_TAG_PREFIX="testnet-" + ;; --help) print_help exit 0 @@ -76,11 +82,10 @@ TST_BASE_IMAGE=${BASE_IMAGE:?"Missing --base-image option to specify base image" echo "Moving into source root directory: ${SRCROOTDIR}" pushd "$SRCROOTDIR" -#pwd export DOCKER_BUILDKIT=1 -docker build --target=instance \ +docker build --target=${DOCKER_TARGET} \ --build-arg CI_REGISTRY_IMAGE=$REGISTRY \ --build-arg BASE_IMAGE=${BASE_IMAGE} \ --build-arg BEEKEEPER_IMAGE=${BEEKEEPER_IMAGE} \ @@ -90,3 +95,5 @@ docker build --target=instance \ popd echo "CLIVE_IMAGE_NAME=${REGISTRY}${IMAGE_TAG_PREFIX}instance:${IMAGE_TAG_PREFIX}instance-${BUILD_IMAGE_TAG}" > docker_image_name.env +echo "CLIVE_IMAGE_PATH=${REGISTRY}${IMAGE_TAG_PREFIX}instance" >> docker_image_name.env +echo "CLIVE_IMAGE_TAG_PREFIX=:${IMAGE_TAG_PREFIX}instance-" >> docker_image_name.env -- GitLab From b6a428fb1e8a4293c20e4e2afaa7bb6ec53ca9b4 Mon Sep 17 00:00:00 2001 From: Bartek Wrona <wrona@syncad.com> Date: Thu, 13 Jul 2023 18:11:49 +0200 Subject: [PATCH 17/20] testnet_node.py supports clive autolaunch --- testnet_node.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/testnet_node.py b/testnet_node.py index ffa8e4b293..21861c898f 100644 --- a/testnet_node.py +++ b/testnet_node.py @@ -1,11 +1,16 @@ from __future__ import annotations +import sys import time from random import randint from typing import TYPE_CHECKING import test_tools as tt +from clive.main import main as clive_main + +ARGS_COUNT = 2 + if TYPE_CHECKING: from typing import Any @@ -53,7 +58,12 @@ tt.logger.info(f"{alice.name} public key: {alice.public_key}") tt.logger.info(f"{alice.name} private key: {alice.private_key}") tt.logger.info("done!") -tt.logger.info("serving forever... press Ctrl+C to exit") +# passing any argument to the script will skip clive autolaunch i.e. for debugging purposes. +if len(sys.argv) < ARGS_COUNT: + tt.logger.info("Attempting to start a clive interactive mode - exit to finish") + clive_main() +else: + tt.logger.info("serving forever... press Ctrl+C to exit") -while True: - time.sleep(1) + while True: + time.sleep(1) -- GitLab From fe6875883de2f07c83c644c371880460640e9451 Mon Sep 17 00:00:00 2001 From: Bartek Wrona <wrona@syncad.com> Date: Thu, 13 Jul 2023 18:12:39 +0200 Subject: [PATCH 18/20] Defined jobs to build a Clive image having embedded testnet installation --- .gitlab-ci.yml | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2e81f1962e..ee4894d754 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -151,13 +151,13 @@ testing_clive: variables: BASE_IMAGE: "" - BUILD_SUFFIX: "" + BUILD_INSTANCE_ARGS: "" TAG_COMMIT: ${BUILD_SUFFIX}${CI_COMMIT_SHORT_SHA} SCRIPTS_PATH: "$CI_PROJECT_DIR/scripts" script: - - $SCRIPTS_PATH/ci-helpers/build_instance.sh "${TAG_COMMIT}" "${CI_PROJECT_DIR}" $CI_REGISTRY_IMAGE --beekeeper-source-image=${HIVED_IMAGE_NAME} --base-image=${BASE_IMAGE} + - $SCRIPTS_PATH/ci-helpers/build_instance.sh "${TAG_COMMIT}" "${CI_PROJECT_DIR}" $CI_REGISTRY_IMAGE --beekeeper-source-image=${HIVED_IMAGE_NAME} --base-image=${BASE_IMAGE} ${BUILD_INSTANCE_ARGS} - echo $CI_BUILD_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY - source docker_image_name.env - docker push ${CLIVE_IMAGE_NAME} @@ -175,6 +175,16 @@ build_and_push_clive_image: variables: BASE_IMAGE: ${CI_BASE_IMAGE} +build_and_push_clive_testnet_image: + extends: .prepare_clive_image + needs: + - job: prepare_testnet_hived_image + artifacts: true + + variables: + BASE_IMAGE: ${HIVED_IMAGE_NAME} + BUILD_INSTANCE_ARGS: "--embedded-testnet" + build_and_push_clive_stable_image: stage: build_docker needs: @@ -186,8 +196,24 @@ build_and_push_clive_stable_image: - master variables: - BUILD_SUFFIX: "" - ADDITIONAL_IMAGE_NAME: ${CI_REGISTRY_IMAGE}instance:instance-${BUILD_SUFFIX}${CI_COMMIT_REF_NAME} + ADDITIONAL_IMAGE_NAME: ${CLIVE_IMAGE_PATH}${CLIVE_IMAGE_TAG_PREFIX}${CI_COMMIT_REF_NAME} + script: + - docker image tag ${CLIVE_IMAGE_NAME} ${ADDITIONAL_IMAGE_NAME} + - echo $CI_BUILD_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY + - docker push ${ADDITIONAL_IMAGE_NAME} + +build_and_push_clive_stable_testnet_image: + stage: build_docker + needs: + - job: build_and_push_clive_testnet_image + artifacts: true + + only: + - develop + - master + + variables: + ADDITIONAL_IMAGE_NAME: ${CLIVE_IMAGE_PATH}${CLIVE_IMAGE_TAG_PREFIX}${CI_COMMIT_REF_NAME} script: - docker image tag ${CLIVE_IMAGE_NAME} ${ADDITIONAL_IMAGE_NAME} - echo $CI_BUILD_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY -- GitLab From 2d61534506c541e542a6b97ebc41cb62d51ec530 Mon Sep 17 00:00:00 2001 From: Bartek Wrona <wrona@syncad.com> Date: Thu, 13 Jul 2023 19:28:37 +0200 Subject: [PATCH 19/20] Removed obsolete definitions of docker building jobs --- .gitlab-ci.yml | 52 -------------------------------------------------- 1 file changed, 52 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ee4894d754..3db9d46ed1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -219,56 +219,4 @@ build_and_push_clive_stable_testnet_image: - echo $CI_BUILD_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY - docker push ${ADDITIONAL_IMAGE_NAME} -.build_and_push_base: - image: docker:20.10.13 - stage: build_docker - needs: - - job: pre_commit_checks - - job: prepare_mainnet_hived_image - artifacts: true - - job: testing_clive - variables: - DOCKER_DRIVER: overlay2 - DOCKER_TLS_CERTDIR: "" - TAG_COMMIT: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_NAME$BUILD_SUFFIX:$CI_COMMIT_SHORT_SHA - TAG_LATEST: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_NAME$BUILD_SUFFIX:latest - services: - - docker:20.10.13-dind - before_script: - - ls -la - - cd docker/ - - echo -e "${TXT_BLUE}Building a docker image started...${TXT_CLEAR}" - - echo $CI_BUILD_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY - after_script: - - docker push $TAG_COMMIT - - docker push $TAG_LATEST - tags: *shared_tags - -build_and_push_docker_image: - extends: .build_and_push_base - only: - - develop - - master - script: - - docker build --build-arg BEEKEEPER_LOCATION="${BEEKEEPER_BINARY}" -t $TAG_COMMIT -t $TAG_LATEST -f Dockerfile .. - -build_and_push_docker_testnet_image: - extends: .build_and_push_base - needs: - - job: pre_commit_checks - - job: testing_clive - - job: prepare_mainnet_hived_image - artifacts: true - - job: prepare_testnet_hived_image - artifacts: true - rules: - - if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_REF_NAME != "develop"' - when: manual - - when: always - allow_failure: true - variables: - BUILD_SUFFIX: "-testnet" - script: - - docker build --build-arg BEEKEEPER_LOCATION="${BEEKEEPER_BINARY}" -t $TAG_COMMIT -t $TAG_LATEST -f Dockerfile.TESTNET .. - #<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<| BUILD DOCKER |<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< -- GitLab From 418d78f9c17e946265bb06c9b1ec9a2332e41cf3 Mon Sep 17 00:00:00 2001 From: Bartek Wrona <wrona@syncad.com> Date: Thu, 13 Jul 2023 22:35:20 +0200 Subject: [PATCH 20/20] Removed obsolete files --- docker/Dockerfile.TESTNET | 24 ------------------------ docker/entrypoint-testnet.bash | 7 ------- 2 files changed, 31 deletions(-) delete mode 100644 docker/Dockerfile.TESTNET delete mode 100644 docker/entrypoint-testnet.bash diff --git a/docker/Dockerfile.TESTNET b/docker/Dockerfile.TESTNET deleted file mode 100644 index 83cbfd2f9a..0000000000 --- a/docker/Dockerfile.TESTNET +++ /dev/null @@ -1,24 +0,0 @@ -FROM registry.gitlab.syncad.com/hive/hive/ci-base-image:ubuntu22.04-3 - -ADD --chown=hived_admin:users . /clive -WORKDIR /clive - -ARG HIVE_BUILD_ROOT_LOCATION="./hived/" -ENV HIVE_BUILD_ROOT_PATH="/clive/hived/" -ADD --chown=hived_admin:users "${HIVE_BUILD_ROOT_LOCATION}" "${HIVE_BUILD_ROOT_PATH}" - -ARG BEEKEEPER_LOCATION="./beekeeper" -ENV BEEKEEPER_PATH="/clive/beekeeper" -ADD --chown=hived_admin:users "${BEEKEEPER_LOCATION}" "${BEEKEEPER_PATH}" - -RUN poetry self update - -RUN poetry install - -# crucial for proper display -ENV COLORTERM=truecolor - -ENV CLIVE_BEEKEEPER__PATH="${BEEKEEPER_PATH}" -ENV CLIVE_SECRETS__DEFAULT_KEY="5KTNAYSHVzhnVPrwHpKhc5QqNQt6aW8JsrMT7T4hyrKydzYvYik" - -ENTRYPOINT bash docker/entrypoint-testnet.bash diff --git a/docker/entrypoint-testnet.bash b/docker/entrypoint-testnet.bash deleted file mode 100644 index 09c82f0eab..0000000000 --- a/docker/entrypoint-testnet.bash +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -poetry run python testnet_node.py & - -sleep 15 - -poetry run clive -- GitLab