Dockerized currently existing jobs - ref. hive#406

.gitattributes

+*.sh text eol=lf
\ No newline at end of file

.gitignore

+.vscode/
\ No newline at end of file

.gitlab-ci.yml

 stages:
   - validation
+  - build
 
 include:
-  template: Workflows/Branch-Pipelines.gitlab-ci.yml
+  - template: Workflows/Branch-Pipelines.gitlab-ci.yml
+  - local: templates/data_image_jobs.gitlab-ci.yml
 
 .validation_job:
   stage: validation
@@ -42,4 +44,55 @@ lint_ci_templates:
      - yamllint-parsable-result.txt
      - yamllint-junit-result.xml
     reports:
-      junit: yamllint-junit-result.xml
+      junit: yamllint-junit-result.xml
\ No newline at end of file
+
+lint_python_scripts:
+  extends: .validation_job
+  image: python:latest
+  before_script:
+    - pip install pylint_junit
+  script:
+    - pylint --output-format=text,pylint_junit.JUnitReporter:pylint-junit-result.xml scripts/python/*.py
+  artifacts:
+    paths: 
+     - pylint-junit-result.xml
+    reports:
+      junit: pylint-junit-result.xml
+
+.build_docker_image:
+  extends: .docker_image_builder_job
+  stage: build
+  variables:
+    BUILD_TARGET: ""
+  before_script:
+    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
+  script:
+    - |
+      if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then
+        echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'latest'"
+        docker build --target $BUILD_TARGET --pull -t "$CI_REGISTRY_IMAGE/$BUILD_TARGET:latest" .
+        docker push "$CI_REGISTRY_IMAGE/$BUILD_TARGET:latest"
+      fi
+      if [[ "$CI_COMMIT_BRANCH" == "main" ]]; then
+        echo "Running on main branch: tag = 'main-latest'"
+        docker build --target $BUILD_TARGET --pull -t "$CI_REGISTRY_IMAGE/$BUILD_TARGET:main-latest" .
+        docker push "$CI_REGISTRY_IMAGE/$BUILD_TARGET:main-latest"
+      fi
+      tag=":$CI_COMMIT_REF_SLUG"
+      echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag"
+      docker build --target $BUILD_TARGET --pull -t "$CI_REGISTRY_IMAGE/$BUILD_TARGET${tag}" .
+      docker push "$CI_REGISTRY_IMAGE/$BUILD_TARGET${tag}"
+  rules:
+    - if: $CI_COMMIT_BRANCH
+      exists:
+        - Dockerfile
+
+build_docker_builder_image:
+  extends: .build_docker_image
+  variables:
+    BUILD_TARGET: "docker-builder"
+
+build_image_remover_image:
+  extends: .build_docker_image
+  variables:
+    BUILD_TARGET: "image-remover"
\ No newline at end of file

.pylintrc

+[MASTER]
+load-plugins=pylint_junit
\ No newline at end of file

Dockerfile

+FROM docker:20.10.10 AS docker-builder
+
+COPY scripts/bash/*.sh /usr/local/bin
+RUN apk add --no-cache bash git ca-certificates curl
+
+FROM python:3.11.0 AS image-remover
+
+COPY scripts/python/delete-image.py /
+COPY scripts/python/requirements.txt /
+RUN pip install -r requirements.txt
+CMD [ "bash" ]
\ No newline at end of file

README.md

@@ -6,4 +6,10 @@ This project contains the common CI templates and scripts.
 
 - misc - miscellaneous files
 - scripts/bash - Bash scripts
+- scripts/python - Python scripts
 - templates - GitLab CI templates
+
+## Tmp
+
+docker build --target docker-builder --tag docker-builder .
+docker build --target image-remover --tag image-remover .

misc/docker-compose.dind.yml

+version: '3.9'
+services:
+  docker:
+    image: docker:20.10.10-dind
+    privileged: true
+    environment:
+      - DOCKER_TLS_CERTDIR=/certs
+    volumes:
+      - type: volume
+        source: docker-certs-ca
+        target: /certs/ca
+      - type: volume
+        source: docker-certs-client
+        target: /certs/client
+      - type: volume
+        source: docker-certs-server
+        target: /certs/server
+      - type: volume
+        source: docker-lib
+        target: /var/lib/docker
+    networks:
+      - docker
+volumes:
+  docker-certs-ca:
+  docker-certs-client:
+    name: docker-certs-client
+  docker-certs-server:
+  docker-lib:
+networks:
+  docker:    
+    name: docker
\ No newline at end of file

scripts/bash/build-data.sh

+#! /bin/bash
+# shellcheck source-path=SCRIPTDIR
+
+SCRIPTPATH=$(realpath "$0")
+SCRIPTDIR=$(dirname "$SCRIPTPATH")
+
+export LOG_FILE=build_data.log
+source "$SCRIPTDIR/common.sh"
+
+BUILD_IMAGE_TAG=${1:?"Missing argument #1: build image tag"}
+shift
+SRCROOTDIR=${1:?"Missing argument #2: source directory"}
+shift
+REGISTRY=${1:?"Missing argument #3: target image registry"}
+shift 
+
+# Supplement a registry path by trailing slash (if needed)
+[[ "${REGISTRY}" != */ ]] && REGISTRY="${REGISTRY}/"
+
+BLOCK_LOG_SUFFIX="-5m"
+
+"$SCRIPTDIR/build-instance.sh" "${BUILD_IMAGE_TAG}" "${SRCROOTDIR}" "${REGISTRY}" "${BLOCK_LOG_SUFFIX}" "$@"
+
+pushd "$SRCROOTDIR" ||exit 1 
+
+docker build --target=data \
+  --build-arg CI_REGISTRY_IMAGE="$REGISTRY" --build-arg BLOCK_LOG_SUFFIX="-5m" \
+  --build-arg BUILD_IMAGE_TAG="$BUILD_IMAGE_TAG" -t "${REGISTRY}data:data-${BUILD_IMAGE_TAG}" -f Dockerfile .
+
+popd || exit 1

scripts/bash/build-data4commit.sh

+#! /bin/bash
+# shellcheck source-path=SCRIPTDIR
+
+SCRIPTPATH=$(realpath "$0")
+SCRIPTDIR=$(dirname "$SCRIPTPATH")
+
+export LOG_FILE=build_data4commit.log
+source "$SCRIPTDIR/common.sh"
+
+COMMIT=${1:?"Missing argument #1: commit SHA"}
+shift
+REGISTRY=${1:?"Missing argument #2: target image registry"}
+shift
+REPOSITORY=${1:?"Missing argument #3: repository URL"}
+shift
+BRANCH="master"
+
+BUILD_IMAGE_TAG=$COMMIT
+
+readarray -d "/" -t REPOSITORY_URL_ARRAY <<< "$REPOSITORY"
+REPOSITORY_ARRAY_LENGTH=${#REPOSITORY_URL_ARRAY[*]}
+readarray -d "." -t SPLIT_REPOSITORY_NAME <<< "${REPOSITORY_URL_ARRAY[$REPOSITORY_ARRAY_LENGTH-1]}"
+PROJECT_NAME=${SPLIT_REPOSITORY_NAME[0]}
+
+do_clone "$BRANCH" "./${PROJECT_NAME}-${COMMIT}" "$REPOSITORY" "$COMMIT"
+
+"$SCRIPTDIR/build-data.sh" "$BUILD_IMAGE_TAG" "./${PROJECT_NAME}-${COMMIT}" "$REGISTRY" "$@"
+

scripts/bash/build-instance.sh

+#! /bin/bash
+# shellcheck source-path=SCRIPTDIR
+
+SCRIPTPATH=$(realpath "$0")
+SCRIPTDIR=$(dirname "$SCRIPTPATH")
+
+export LOG_FILE=build_instance.log
+source "$SCRIPTDIR/common.sh"
+
+BUILD_IMAGE_TAG=""
+REGISTRY=""
+SRCROOTDIR=""
+
+IMAGE_TAG_PREFIX=""
+
+BLOCK_LOG_SUFFIX=""
+
+BUILD_HIVE_TESTNET=OFF
+HIVE_CONVERTER_BUILD=OFF
+
+
+print_help () {
+    echo "Usage: $0 <image_tag> <src_dir> <registry_url> [<block_log_img_suffix>] [OPTION[=VALUE]]..."
+    echo
+    echo "Allows to build docker image containing Hived installation"
+    echo "OPTIONS:"
+    echo "  --network-type=TYPE       Allows to specify type of blockchain network supported by built hived. Allowed values: mainnet, testnet, mirrornet"
+    echo "  --export-binaries=PATH    Allows to specify a path where binaries shall be exported from built image."
+    echo "  --help                    Display this help screen and exit"
+    echo
+}
+
+EXPORT_PATH=""
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --network-type=*)
+        type="${1#*=}"
+
+        case $type in
+          "testnet"*)
+            BUILD_HIVE_TESTNET=ON
+            IMAGE_TAG_PREFIX=testnet-
+            ;;
+          "mirrornet"*)
+            BUILD_HIVE_TESTNET=OFF
+            HIVE_CONVERTER_BUILD=ON
+            IMAGE_TAG_PREFIX=mirror-
+            ;;
+          "mainnet"*)
+            BUILD_HIVE_TESTNET=OFF
+            HIVE_CONVERTER_BUILD=OFF
+            IMAGE_TAG_PREFIX=
+            ;;
+           *)
+            echo "ERROR: '$type' is not a valid network type"
+            echo
+            exit 3
+        esac
+        ;;
+    --export-binaries=*)
+        arg="${1#*=}"
+        EXPORT_PATH="$arg"
+        ;;
+    --help)
+        print_help
+        exit 0
+        ;;
+    *)
+        if [ -z "$BUILD_IMAGE_TAG" ];
+        then
+          BUILD_IMAGE_TAG="${1}"
+        elif [ -z "$SRCROOTDIR" ];
+        then
+          SRCROOTDIR="${1}"
+        elif [ -z "$REGISTRY" ];
+        then
+          REGISTRY=${1}
+        elif [ -z "$BLOCK_LOG_SUFFIX" ];
+        then
+          BLOCK_LOG_SUFFIX="${1}"
+        else
+          echo "ERROR: '$1' is not a valid option/positional argument"
+          echo
+          print_help
+          exit 2
+        fi
+        ;;
+    esac
+    shift
+done
+
+[[ -z "$BUILD_IMAGE_TAG" ]] && echo "Missing argument #1: build image tag." && exit 1
+[[ -z "$SRCROOTDIR" ]] && echo "Missing argument #2: source directory. Exiting." && exit 1
+[[ -z "$REGISTRY" ]] && echo "Missing argument #3: target image registry. Exiting." && exit 1
+
+# Supplement a registry path by trailing slash (if needed)
+[[ "${REGISTRY}" != */ ]] && REGISTRY="${REGISTRY}/"
+
+echo "Moving into source root directory: ${SRCROOTDIR}"
+pushd "$SRCROOTDIR" || exit 1
+pwd
+
+export DOCKER_BUILDKIT=1
+
+docker build --target=base_instance \
+  --build-arg BLOCK_LOG_SUFFIX="${BLOCK_LOG_SUFFIX}" \
+  --build-arg CI_REGISTRY_IMAGE="$REGISTRY" \
+  --build-arg BUILD_HIVE_TESTNET=$BUILD_HIVE_TESTNET \
+  --build-arg HIVE_CONVERTER_BUILD=$HIVE_CONVERTER_BUILD \
+  --build-arg BUILD_IMAGE_TAG="$BUILD_IMAGE_TAG" \
+  -t "${REGISTRY}base_instance${BLOCK_LOG_SUFFIX}:base_instance-${BUILD_IMAGE_TAG}" \
+  -f Dockerfile .
+
+# Build the image containing only binaries and be ready to start running hived instance, operating on mounted volumes pointing instance datadir and shm_dir
+docker build --target=instance \
+  --build-arg BLOCK_LOG_SUFFIX="${BLOCK_LOG_SUFFIX}" \
+  --build-arg CI_REGISTRY_IMAGE="$REGISTRY" \
+  --build-arg BUILD_HIVE_TESTNET=$BUILD_HIVE_TESTNET \
+  --build-arg HIVE_CONVERTER_BUILD=$HIVE_CONVERTER_BUILD \
+  --build-arg BUILD_IMAGE_TAG="$BUILD_IMAGE_TAG" \
+  -t "${REGISTRY}${IMAGE_TAG_PREFIX}instance${BLOCK_LOG_SUFFIX}:instance-${BUILD_IMAGE_TAG}" \
+  -f Dockerfile .
+
+
+popd || exit 1
+
+if [ -n "${EXPORT_PATH}" ] ;
+then
+  "$SCRIPTDIR/export-binaries.sh" "${REGISTRY}${IMAGE_TAG_PREFIX}instance${BLOCK_LOG_SUFFIX}:instance-${BUILD_IMAGE_TAG}" "${EXPORT_PATH}"
+fi
+

scripts/bash/common.sh

+#! /bin/bash
+
+set -euo pipefail 
+
+exec > >(tee "${LOG_FILE}") 2>&1
+
+log_exec_params() {
+  echo
+  echo -n "$0 parameters: "
+  for arg in "$@"; do echo -n "$arg "; done
+  echo
+}
+
+do_clone_commit() {
+  local commit="$1"
+  local src_dir=$2
+  local repo_url=$3
+  
+  echo "Cloning commit: $commit from $repo_url into: $src_dir ..."
+  mkdir -p "$src_dir"
+  pushd "$src_dir"
+  
+  git init
+  git remote add origin "$repo_url"
+  git fetch --depth 1 origin "$commit"
+  git checkout FETCH_HEAD
+  git submodule update --init --recursive
+  
+  popd
+}
+
+do_clone_branch() {
+  local branch=$1
+  local src_dir="$2"
+  local repo_url="$3"
+  echo "Cloning branch: $branch from $repo_url ..."
+  git clone --recurse-submodules --shallow-submodules --single-branch --depth=1 --branch "$branch" -- "$repo_url" "$src_dir"
+}
+
+
+do_clone() {
+  local branch=$1
+  local src_dir="$2"
+  local repo_url="$3"
+  local commit="$4"
+
+  if [[ "$commit" != "" ]]; then
+    do_clone_commit "$commit" "$src_dir" "$repo_url"
+  else
+    do_clone_branch "$branch" "$src_dir" "$repo_url"
+  fi
+}
+

scripts/bash/delete-image.sh

-#! /bin/bash
-
-set -ex
-
-REGISTRY_USER=${REGISTRY_USER:-gitlab-ci-token}
-REGISTRY_PASS=${REGISTRY_PASS:-${CI_JOB_TOKEN}}
-# DOCKER_IMAGE=${IMAGE_PATH:-${CI_REGISTRY}/${CI_PROJECT_PATH}}
-DOCKER_SERVER=${DOCKER_SERVER:-${CI_REGISTRY}}
-
-PROJECT_ID=${CI_PROJECT_ID}
-echo "Project ID: ${PROJECT_ID}"
-
-REGISTRY_ID=$( curl --request GET --url "https://gitlab.syncad.com/api/v4/projects/${PROJECT_ID}/registry/repositories" --header 'Accept: application/vnd.docker.distribution.manifest.v2+json' --header 'Authorization: Bearer' | jq -c ".[] | select (.location==\"${IMAGE_PATH}\") | .id" )
-echo "Repository ID: ${REGISTRY_ID}"
-
-# TST_REGISTRY_ID=${REGISTRY_ID:?"Missing a registry image matching to given path"}
-
-echo "Deleting '${IMAGE_PATH}:${IMAGE_TAG}' from registry..."
-
-curl --request DELETE --header "PRIVATE-TOKEN: ${REGISTRY_PASS}" \
-  "https://gitlab.syncad.com/api/v4/projects/${PROJECT_ID}/registry/repositories/${REGISTRY_ID}/tags/${IMAGE_TAG}"

scripts/bash/docker_image_utils.sh → scripts/bash/docker-image-utils.sh

@@ -85,5 +85,4 @@ extract_files_from_image() {
     COPY --from=${IMAGE_TAGGED_NAME} "${FILES[@]}" /
 EOF
 
-}
-
+}
\ No newline at end of file

scripts/bash/export-binaries.sh

+#! /bin/bash
+
+set -euo pipefail
+
+IMAGE_TAGGED_NAME=${1:-"Missing image name"}
+EXPORT_PATH=${2:-"Missing export target directory"}
+
+echo "Attempting to export built binaries from image: ${IMAGE_TAGGED_NAME} into directory: ${EXPORT_PATH}"
+
+export DOCKER_BUILDKIT=1
+
+docker build -o "${EXPORT_PATH}" - << EOF
+    FROM scratch
+    COPY --from=${IMAGE_TAGGED_NAME} /home/hived/bin/ /
+EOF

scripts/bash/get_image4submodule.sh → scripts/bash/get-image4submodule.sh

@@ -3,23 +3,26 @@
 
 set -euo pipefail
 
-SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+SCRIPTPATH=$(realpath "$0")
+SCRIPTDIR=$(dirname "$SCRIPTPATH")
 
 IMGNAME="data"
 
-source "$SCRIPTPATH/docker_image_utils.sh"
+source "$SCRIPTDIR/docker-image-utils.sh"
 
-submodule_path=${1:?"Missing arg 1 for submodule path variable"}
+SUBMODULE_PATH=${1:?"Missing argument #1: submodule path"}
 shift
-REGISTRY=${1:?"Missing arg 2 for REGISTRY variable"}
+REGISTRY=${1:?"Missing argument #2: image registry"}
 shift
-DOTENV_VAR_NAME=${1:?"Missing name of dot-env variable"}
+DOTENV_VAR_NAME=${1:?"Missing argument #3: dot-env name"}
 shift
-REGISTRY_USER=${1:?"Missing arg 4 for REGISTRY_USER variable"}
+REGISTRY_USER=${1:?"Missing argument #4: registry user"}
 shift
-REGISTRY_PASSWORD=${1:?"Missing arg 5 for REGISTRY_PASSWORD variable"}
+REGISTRY_PASSWORD=${1:?"Missing argument #5: registry password"}
 shift
-BINARY_CACHE_PATH=${1:?"Missing arg 6 specific to binary cache path"}
+BINARY_CACHE_PATH=${1:?"Missing argument #6: binary cache path"}
+shift
+REPOSITORY=${1:?"Missing argument #7: repository URL"}
 shift
 
 retrieve_submodule_commit () {
@@ -32,9 +35,9 @@ retrieve_submodule_commit () {
   echo "$commit"
 }
 
-echo "Attempting to get commit for: $submodule_path"
+echo "Attempting to get commit for: $SUBMODULE_PATH"
 
-commit=$( retrieve_submodule_commit "${submodule_path}" )
+commit=$( retrieve_submodule_commit "${SUBMODULE_PATH}" )
 
 img=$( build_image_name $IMGNAME "$commit" "$REGISTRY" )
 img_path=$( build_image_registry_path $IMGNAME "$commit" "$REGISTRY" )
@@ -49,11 +52,11 @@ docker_image_exists "$IMGNAME" "$commit" "$REGISTRY" image_exists
 if [ "$image_exists" -eq 1 ];
 then
   echo "Image already exists..."
-  "$SCRIPTPATH/export-binaries.sh" "${img}" "${BINARY_CACHE_PATH}"
+  "$SCRIPTDIR/export-binaries.sh" "${img}" "${BINARY_CACHE_PATH}"
 else
   # Here continue an image build.
   echo "${img} image is missing. Building it..."
-  "$SCRIPTPATH/build_data4commit.sh" "$commit" "$REGISTRY" --export-binaries="${BINARY_CACHE_PATH}"
+  "$SCRIPTDIR/build-data4commit.sh" "$commit" "$REGISTRY" "$REPOSITORY" --export-binaries="${BINARY_CACHE_PATH}"
   time docker push "$img"
 fi
 

scripts/python/delete-image.py

+# pylint: disable=C0103
+'''
+This script deletes a tag from GitLab's embedded docker repository.
+Run 'python delete-image.py -h' to see usage instructions.
+'''
+import argparse
+import gitlab
+
+parser = argparse.ArgumentParser(
+    description = 'Deletes a tag from GitLab\'s embedded Docker repository')
+parser.add_argument('--url',
+    default='https://gitlab.syncad.com',
+    help='GitLab instance URL (defaults to https://gitlab.syncad.com)')
+parser.add_argument('token',
+    help='Access token for authentication')
+parser.add_argument('project-id',
+    help='Project ID (eg. 121)')
+parser.add_argument('repository',
+    help='Repository location (eg. registry.gitlab.com/hive/haf/data)')
+parser.add_argument('tag',
+    help='Tag to be deleted (eg. latest)')
+
+args = parser.parse_args()
+
+server_url = args.url
+token = args.token
+project_id = getattr(args, 'project-id')
+repository_location = args.repository
+tag_name = args.tag
+
+gl = gitlab.Gitlab(server_url, private_token=token)
+
+def delete_tag(repository):
+    '''Deletes tag from repository'''
+    try:
+        tag = repository.tags.get(id=tag_name)
+        tag.delete()
+        print(f'Deleted tag \'{tag_name}\' from repository \'{repository_location}\'')
+    except gitlab.exceptions.GitlabGetError:
+        print(f'Tag \'{tag_name}\' does not exists in repository \'{repository_location}\'')
+
+def find_repository_and_delete_tag():
+    '''Looks up repository and invokes 'delete_tag' method on it'''
+    try:
+        repository_found = False
+        project = gl.projects.get(project_id)
+        repositories = project.repositories.list(all=True)
+
+        for repository in repositories:
+            if repository_location == repository.location:
+                repository_found = True
+                delete_tag(repository)
+
+        if not repository_found:
+            print(f'Registry repository \'{repository_location}\' ' +
+                'was not found in project \'{project.name}\'')
+    except gitlab.exceptions.GitlabGetError:
+        print(f'Project with ID \'{project_id}\' was not found in GitLab instance \'{server_url}\'')
+
+find_repository_and_delete_tag()

scripts/python/requirements.txt

+python-gitlab==3.11.0

templates/data_image_jobs.gitlab-ci.yml

@@ -7,36 +7,41 @@
   interruptible: true
   services:
     - docker:20.10.10-dind
-  before_script:
-    - apk update && apk add bash git ca-certificates curl
 
 .docker_image_cleanup_job:
-  image: alpine:3.16
+  image: registry.gitlab.syncad.com/hive/common-ci-configuration/image-remover:latest
   interruptible: true
-  before_script:
-    - apk update && apk add bash curl jq
   variables:
-    # Interface variables (to be overrided by derived jobs)
-    SUBMODULE_DIR: "$CI_PROJECT_DIR"
     REGISTRY: $CI_REGISTRY_IMAGE
-    REGISTRY_USER: ""
     REGISTRY_PASS: ""
     IMAGE_PATH: ""
     IMAGE_TAG: ""
-    SCRIPTS_PATH: "$SUBMODULE_DIR/scripts/bash"
   script:
     - echo "Attempting to cleanup an image $IMAGE_PATH using tag $IMAGE_TAG from $REGISTRY"
-    - "$SCRIPTS_PATH/delete-image.sh"
+    - python delete-image.py "$REGISTRY_PASS" "$CI_PROJECT_ID" "$IMAGE_PATH" "$IMAGE_TAG"
 
   when: always
 
 .prepare_data_5m_image:
   extends: .docker_image_builder_job
+  image: registry.gitlab.syncad.com/hive/common-ci-configuration/docker-builder:latest
   variables:
-    SUBMODULE_DIR: "$CI_PROJECT_DIR"
+    DOTENV_NAME: ""
     REGISTRY_USER: "$CI_IMG_BUILDER_USER"
     REGISTRY_PASS: "$CI_IMG_BUILDER_PASSWORD"
-    SCRIPTS_PATH: "$SUBMODULE_DIR/scripts/bash"
+    REGISTRY_URL: ""
+    REPOSITORY_URL: ""
+    SUBDIRECTORY: ""
+  script:
+    - |
+      get-image4submodule.sh "$SUBDIRECTORY" \
+        "$REGISTRY_URL" \
+        "$DOTENV_NAME" \
+        "$REGISTRY_USER" \
+        "$REGISTRY_PASS" \
+        "hived-mainnet-binaries" \
+        "$REPOSITORY_URL"
+    - ls -la ./hived-mainnet-binaries/*
   artifacts:
     reports:
       dotenv: docker_image_name.env
@@ -46,11 +51,17 @@
 
 .prepare_hived_data_5m_image:
   extends: .prepare_data_5m_image
-  script:
-    - $SCRIPTS_PATH/get_image4submodule.sh "$SUBMODULE_DIR" registry.gitlab.syncad.com/hive/hive/ "HIVED_IMAGE_NAME" "$REGISTRY_USER" "$REGISTRY_PASS" "hived-mainnet-binaries"
-    - ls -la ./hived-mainnet-binaries/*
+  variables:
+    DOTENV_NAME: "HIVED_IMAGE_NAME"
+    REGISTRY_URL: "registry.gitlab.syncad.com/hive/hive/"
+    REPOSITORY_URL: "https://gitlab.syncad.com/hive/hive.git"
+    SUBDIRECTORY: "hive"
+  
 
 .prepare_haf_data_5m_image:
   extends: .prepare_data_5m_image
-  script:
-    - $SCRIPTS_PATH/get_image4submodule.sh "$SUBMODULE_DIR" registry.gitlab.syncad.com/hive/haf/ "HAF_IMAGE_NAME" "$REGISTRY_USER" "$REGISTRY_PASS" "hived-mainnet-binaries"
+  variables:
+    DOTENV_NAME: "$HAF_IMAGE_NAME"
+    REGISTRY_URL: "registry.gitlab.syncad.com/hive/haf/"
+    REPOSITORY_URL: "https://gitlab.syncad.com/hive/haf.git"
+    SUBDIRECTORY: "haf"