From 6e11e28cbfbf67ea55be34755f07fce7c2ed89dc Mon Sep 17 00:00:00 2001 From: Marcin Ickiewicz Date: Fri, 16 May 2025 10:08:27 +0200 Subject: [PATCH] Revert "use AI tooling delivered by the HAF image" This reverts commit 4f3c8919846d3a0bc75533cc3627312d6cabf443. It was merged by mistake https://gitlab.syncad.com/hive/hivesense/-/merge_requests/12 the HAF part has been not merged yet: https://gitlab.syncad.com/hive/haf/-/merge_requests/645 --- .gitlab-ci.yml | 18 +++- Dockerfile.haf_ai | 15 ++++ readme.md | 3 +- scripts/build_haf_ai_image.sh | 88 +++++++++++++++++++ scripts/ci-helpers/compose_variables.sh | 4 +- .../ci-helpers/start-ci-test-environment.sh | 2 +- submodules/haf | 2 +- 7 files changed, 126 insertions(+), 6 deletions(-) create mode 100644 Dockerfile.haf_ai create mode 100755 scripts/build_haf_ai_image.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1de1ff7..a71d163 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -110,13 +110,29 @@ build_images: tags: - public-runner-docker +build_haf_ai_images: + extends: .docker_image_builder_job_template + stage: build + before_script: + - !reference [.docker_image_builder_job_template, before_script] + - | + echo -e "\e[0Ksection_start:$(date +%s):login[collapsed=true]\r\e[0KLogging to Docker registry..." + docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + echo -e "\e[0Ksection_end:$(date +%s):login\r\e[0K" + script: + - $CI_PROJECT_DIR/scripts/build_haf_ai_image.sh --push + needs: + - job: prepare_haf_image + tags: + - public-runner-docker + sync: extends: .docker_image_builder_job_template stage: sync image: registry.gitlab.syncad.com/hive/haf_api_node/dind:1.27.11rc2 needs: - - prepare_haf_image - build_images + - build_haf_ai_images variables: DOCKER_COMPOSE_VERSION: 2.27.1 DATA_SOURCE: ${DATA_CACHE_HAF_PREFIX}_${HAF_COMMIT} diff --git a/Dockerfile.haf_ai b/Dockerfile.haf_ai new file mode 100644 index 0000000..94683bf --- /dev/null +++ b/Dockerfile.haf_ai @@ -0,0 +1,15 @@ +ARG HAF_TAG=70974c14 +ARG REGISTRY_IMAGE=registry.gitlab.syncad.com/hive/ +ARG NETWORK="haf" # mainnet + +FROM ${REGISTRY_IMAGE}${NETWORK}:${HAF_TAG} AS ai-instance + +SHELL ["/bin/bash", "-c"] + +USER root +WORKDIR /usr/local/src +COPY ./scripts/setup_ubuntu.sh /usr/local/src/scripts/ + +RUN bash -x ./scripts/setup_ubuntu.sh + +USER haf_admin \ No newline at end of file diff --git a/readme.md b/readme.md index 3208d7d..fe0d81e 100644 --- a/readme.md +++ b/readme.md @@ -222,7 +222,7 @@ The CI pipeline includes several key steps to ensure quality, build consistency, #### Docker Image Builds - Builds Docker images based on the current submodule versions for: - - `haf` + - `haf ai-instance` (with AI-related tools see [`/scripts/setup_ubuntu.sh`](./scripts/setup_ubuntu.sh) used by [`Dockerfile.haf_ai`](./Dockerfile.haf_ai) ) - `hivemind` - `haf_api_node` - `hivesense` @@ -244,6 +244,7 @@ To reproduce CI-related issues on your local machine, you can replicate the envi 1. **Build Docker Images** ```bash + ./scripts/build_haf_ai_image.sh # Builds the HAF Docker image with AI support. Uses the HAF submodule version as the base. ./scripts/build_images.sh # Builds Docker images for hivesense and the PostgREST rewriter. ``` diff --git a/scripts/build_haf_ai_image.sh b/scripts/build_haf_ai_image.sh new file mode 100755 index 0000000..6ed27f7 --- /dev/null +++ b/scripts/build_haf_ai_image.sh @@ -0,0 +1,88 @@ +#! /bin/bash -x + +set -euo pipefail + +SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +SCRIPTSDIR="$SCRIPTPATH/.." + +BUILD_IMAGE_TAG="" +NETWORK="" +SRCROOTDIR="${SCRIPTSDIR}" +REGISTRY="${CI_REGISTRY:-registry.gitlab.syncad.com}" +REGISTRY="${REGISTRY}/hive/" + +HAF_AI_INSTANCE_REGISTRY="registry.gitlab.syncad.com/hive/hivesense/haf/" + + +HAF_SUBMODULE_SHA=$(git -C submodules/haf describe --tags --exact-match HEAD 2>/dev/null || git -C submodules/haf rev-parse --short=8 HEAD) +BUILD_IMAGE_TAG=${HAF_SUBMODULE_SHA} + +print_help () { +cat <<-EOF + Usage: $0 [OPTION[=VALUE]]... + + Builds docker image containing Hived installation + OPTIONS: + --push Push built images to registry + --network-type=TYPE Allows to specify type of blockchain network supported by built hived. Allowed values: mainnet, testnet, mirrornet + --help|-h|-? Display this help screen and exit +EOF +} + +while [ $# -gt 0 ]; do + case "$1" in + --push) + PUSH=1 + ;; + --network-type=*) + type="${1#*=}" + + case $type in + "testnet"*) + NETWORK=testnet + ;; + "mirrornet"*) + NETWORK=mirrornet + ;; + "mainnet"*) + NETWORK="" + ;; + *) + echo "ERROR: '$type' is not a valid network type" + echo + exit 3 + esac + ;; + --help|-h|-?) + print_help + exit 0 + ;; + *) + if [ -z "$SRCROOTDIR" ]; + then + SRCROOTDIR="${1}" + else + echo "ERROR: '$1' is not a valid option/positional argument" + echo + print_help + exit 2 + fi + ;; + esac + shift +done + +AI_INSTANCE_IMAGE_PATH="${HAF_AI_INSTANCE_REGISTRY}${NETWORK}ai-instance:${BUILD_IMAGE_TAG}" + +docker build --progress=plain --target=ai-instance \ + --build-arg REGISTRY_IMAGE="${REGISTRY}" \ + --build-arg NETWORK="${NETWORK:-"haf"}" \ + --build-arg HAF_TAG="${BUILD_IMAGE_TAG}" \ + --tag "${AI_INSTANCE_IMAGE_PATH}" \ + --file Dockerfile.haf_ai "${SRCROOTDIR}" + +if [ -n "${PUSH:-}" ]; then + echo "Pushing image ${AI_INSTANCE_IMAGE_PATH}..." + docker push "${AI_INSTANCE_IMAGE_PATH}" + echo "Pushed image ${AI_INSTANCE_IMAGE_PATH}" +fi \ No newline at end of file diff --git a/scripts/ci-helpers/compose_variables.sh b/scripts/ci-helpers/compose_variables.sh index 59a595f..afe870f 100644 --- a/scripts/ci-helpers/compose_variables.sh +++ b/scripts/ci-helpers/compose_variables.sh @@ -26,7 +26,8 @@ HIVE_API_NODE_REGISTRY="registry.gitlab.syncad.com/hive" PUBLIC_HOSTNAME=${PUBLIC_HOSTNAME:-"localhost"} # HAF and Hivemind -HAF_VERSION="${HAF_SUBMODULE_SHA}" +HAF_IMAGE="registry.gitlab.syncad.com/hive/hivesense/haf/ai-instance" +HAF_VERSION="$HAF_SUBMODULE_SHA" ARGUMENTS="--replay-blockchain --block-stats-report-output=NOTIFY --block-stats-report-type=FULL --notifications-endpoint=hived-pme:9185 --stop-at-block=${NUMBER_OF_BLOCKS_TO_SYNC}" HIVEMIND_VERSION="$HIVEMIND_NODE_SUBMODULE_SHA" HIVEMIND_SYNC_ARGS="--test-max-block=${NUMBER_OF_BLOCKS_TO_SYNC}" @@ -39,7 +40,6 @@ HIVESENSE_REWRITER_IMAGE="registry.gitlab.syncad.com/hive/hivesense/rewiter" HIVESENSE_SYNC_ARGS="--stop-at-block=${NUMBER_OF_BLOCKS_TO_SYNC}" HIVESENSE_OLLAMA=http://hivesense-ollama:11434 -#HIVESENSE_OLLAMA=http://192.168.6.186:11434 HIVESENSE_MODEL=yxchia/multilingual-e5-base:F16 HIVESENSE_VECTOR_SIZE=768 HIVESENSE_START_BLOCK=1 diff --git a/scripts/ci-helpers/start-ci-test-environment.sh b/scripts/ci-helpers/start-ci-test-environment.sh index 2c24f6e..4ae20d4 100755 --- a/scripts/ci-helpers/start-ci-test-environment.sh +++ b/scripts/ci-helpers/start-ci-test-environment.sh @@ -1,4 +1,4 @@ -#!/bin/sh -x +#!/bin/sh # shellcheck disable=SC1091 diff --git a/submodules/haf b/submodules/haf index a95ca63..0e2bd7c 160000 --- a/submodules/haf +++ b/submodules/haf @@ -1 +1 @@ -Subproject commit a95ca631d74bdf9d871005ff092b85e95188c512 +Subproject commit 0e2bd7c69cc26928665281435165fd3288940db9 -- GitLab