Skip to content
Snippets Groups Projects
Verified Commit 332b451c authored by Konrad Botor's avatar Konrad Botor
Browse files

Updated base Docker images - ref. #205

parent 366f41d7
No related branches found
No related tags found
2 merge requests!827Merge develop changes to master,!605Resolve "Split sync_e2e_benchmark into two separate jobs"
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yaml
+ 60
− 13
View file @ 332b451c
......@@ -56,139 +56,186 @@ workflow:
#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>| BASH SCRIPTS |>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
.base_image_build_script: &base-image-build-script
- |
echo -e "\e[0Ksection_start:$(date +%s):login[collapsed=true]\r\e[0KLogging to Docker registry..."
docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
echo -e "\e[0Ksection_end:$(date +%s):login\r\e[0K"
echo -e "\e[0Ksection_start:$(date +%s):build[collapsed=true]\r\e[0KBuilding base Docker images..."
./scripts/ci/build_ci_base_image.sh
echo -e "\e[0Ksection_end:$(date +%s):build\r\e[0K"
.instance-build-script: &instance-build-script
- |
echo -e "\e[0Ksection_start:$(date +%s):login[collapsed=true]\r\e[0KLogging to Docker registry..."
docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
echo -e "\e[0Ksection_end:$(date +%s):login\r\e[0K"
echo -e "\e[0Ksection_start:$(date +%s):build[collapsed=true]\r\e[0KBuilding Hivemind Docker image..."
./scripts/ci-helpers/build_instance.sh \
"$CI_COMMIT_SHA" \
"$CI_PROJECT_DIR" \
"$CI_REGISTRY_IMAGE" \
--dot-env-filename=hivemind_image.env \
--dot-env-var-name=HIVEMIND_IMAGE
docker images
docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
source hivemind_image.env
docker push "$HIVEMIND_IMAGE"
cat hivemind_image.env
echo -e "\e[0Ksection_end:$(date +%s):build\r\e[0K"
.bridge_api_smoketest-script: &bridge_api_smoketest-script
- |
echo -e "\e[0Ksection_start:$(date +%s):bridge_api_smoketest[collapsed=true]\r\e[0KRunning bridge API smoketest..."
./scripts/ci/start-api-smoketest.sh \
$RUNNER_HIVEMIND_SMOKETEST_SERVER_HOSTNAME \
$RUNNER_HIVEMIND_SERVER_HTTP_PORT \
bridge_api_patterns/ \
api_smoketest_bridge.xml \
$RUNNER_PYTEST_WORKERS
echo -e "\e[0Ksection_end:$(date +%s):bridge_api_smoketest\r\e[0K"
.bridge_api_smoketest_negative-script: &bridge_api_smoketest_negative-script
- |
echo -e "\e[0Ksection_start:$(date +%s):bridge_api_smoketest_negative[collapsed=true]\r\e[0KRunning bridge API smoketest negative..."
./scripts/ci/start-api-smoketest.sh \
$RUNNER_HIVEMIND_SMOKETEST_SERVER_HOSTNAME \
$RUNNER_HIVEMIND_SERVER_HTTP_PORT \
bridge_api_negative/ \
api_smoketest_bridge_negative.xml \
$RUNNER_PYTEST_WORKERS
echo -e "\e[0Ksection_end:$(date +%s):bridge_api_smoketest_negative\r\e[0K"
.condenser_api_smoketest-script: &condenser_api_smoketest-script
- |
echo -e "\e[0Ksection_start:$(date +%s):condenser_api_smoketest[collapsed=true]\r\e[0KRunning condenser API smoketest..."
./scripts/ci/start-api-smoketest.sh \
$RUNNER_HIVEMIND_SMOKETEST_SERVER_HOSTNAME \
$RUNNER_HIVEMIND_SERVER_HTTP_PORT \
condenser_api_patterns/ \
api_smoketest_condenser_api.xml \
$RUNNER_PYTEST_WORKERS
echo -e "\e[0Ksection_end:$(date +%s):condenser_api_smoketest\r\e[0K"
.condenser_api_smoketest_negative-script: &condenser_api_smoketest_negative-script
- |
echo -e "\e[0Ksection_start:$(date +%s):condenser_api_smoketest_negative[collapsed=true]\r\e[0KRunning condenser API smoketest negative..."
./scripts/ci/start-api-smoketest.sh \
$RUNNER_HIVEMIND_SMOKETEST_SERVER_HOSTNAME \
$RUNNER_HIVEMIND_SERVER_HTTP_PORT \
condenser_api_negative/ \
api_smoketest_condenser_api_negative.xml \
$RUNNER_PYTEST_WORKERS
echo -e "\e[0Ksection_end:$(date +%s):condenser_api_smoketest_negative\r\e[0K"
.database_api_smoketest-script: &database_api_smoketest-script
- |
echo -e "\e[0Ksection_start:$(date +%s):database_api_smoketest[collapsed=true]\r\e[0KRunning database API smoketest..."
./scripts/ci/start-api-smoketest.sh \
$RUNNER_HIVEMIND_SMOKETEST_SERVER_HOSTNAME \
$RUNNER_HIVEMIND_SERVER_HTTP_PORT \
database_api_patterns/ \
api_smoketest_database_api.xml \
$RUNNER_PYTEST_WORKERS
echo -e "\e[0Ksection_end:$(date +%s):database_api_smoketest\r\e[0K"
.database_api_smoketest_negative-script: &database_api_smoketest_negative-script
- |
echo -e "\e[0Ksection_start:$(date +%s):database_api_smoketest_negative[collapsed=true]\r\e[0KRunning database API smoketest negative..."
./scripts/ci/start-api-smoketest.sh \
$RUNNER_HIVEMIND_SMOKETEST_SERVER_HOSTNAME \
$RUNNER_HIVEMIND_SERVER_HTTP_PORT \
database_api_negative/ \
api_smoketest_database_api_negative.xml \
$RUNNER_PYTEST_WORKERS
echo -e "\e[0Ksection_end:$(date +%s):database_api_smoketest_negative\r\e[0K"
.follow_api_smoketest-script: &follow_api_smoketest-script
- |
echo -e "\e[0Ksection_start:$(date +%s):follow_api_smoketest[collapsed=true]\r\e[0KRunning follow API smoketest..."
./scripts/ci/start-api-smoketest.sh \
$RUNNER_HIVEMIND_SMOKETEST_SERVER_HOSTNAME \
$RUNNER_HIVEMIND_SERVER_HTTP_PORT \
follow_api_patterns/ \
api_smoketest_follow_api.xml \
$RUNNER_PYTEST_WORKERS
echo -e "\e[0Ksection_end:$(date +%s):follow_api_smoketest\r\e[0K"
.follow_api_smoketest_negative-script: &follow_api_smoketest_negative-script
- |
echo -e "\e[0Ksection_start:$(date +%s):follow_api_smoketest_negative[collapsed=true]\r\e[0KRunning follow API smoketest negative..."
./scripts/ci/start-api-smoketest.sh \
$RUNNER_HIVEMIND_SMOKETEST_SERVER_HOSTNAME \
$RUNNER_HIVEMIND_SERVER_HTTP_PORT \
follow_api_negative/ \
api_smoketest_follow_api_negative.xml \
$RUNNER_PYTEST_WORKERS
echo -e "\e[0Ksection_end:$(date +%s):follow_api_smoketest_negative\r\e[0K"
.tags_api_smoketest-script: &tags_api_smoketest-script
- |
echo -e "\e[0Ksection_start:$(date +%s):tags_api_smoketest[collapsed=true]\r\e[0KRunning tags API smoketest..."
./scripts/ci/start-api-smoketest.sh \
$RUNNER_HIVEMIND_SMOKETEST_SERVER_HOSTNAME \
$RUNNER_HIVEMIND_SERVER_HTTP_PORT \
tags_api_negative/ \
api_smoketest_tags_api_negative.xml \
$RUNNER_PYTEST_WORKERS
echo -e "\e[0Ksection_end:$(date +%s):tags_api_smoketest\r\e[0K"
.tags_api_smoketest_negative-script: &tags_api_smoketest_negative-script
- |
echo -e "\e[0Ksection_start:$(date +%s):tags_api_smoketest_negative[collapsed=true]\r\e[0KRunning tags API smoketest negative..."
./scripts/ci/start-api-smoketest.sh \
$RUNNER_HIVEMIND_SMOKETEST_SERVER_HOSTNAME \
$RUNNER_HIVEMIND_SERVER_HTTP_PORT \
tags_api_patterns/ \
api_smoketest_tags_api.xml \
$RUNNER_PYTEST_WORKERS
echo -e "\e[0Ksection_end:$(date +%s):tags_api_smoketest_negative\r\e[0K"
.mock_tests-script: &mock_tests-script
- |
echo -e "\e[0Ksection_start:$(date +%s):mock_tests[collapsed=true]\r\e[0KRunning mock tests..."
./scripts/ci/start-api-smoketest.sh \
$RUNNER_HIVEMIND_SMOKETEST_SERVER_HOSTNAME \
$RUNNER_HIVEMIND_SERVER_HTTP_PORT \
mock_tests/ \
api_smoketest_mock_tests.xml \
$RUNNER_PYTEST_WORKERS
echo -e "\e[0Ksection_end:$(date +%s):mock_tests\r\e[0K"
.hive_api_smoketest-script: &hive_api_smoketest-script
- |
echo -e "\e[0Ksection_start:$(date +%s):hive_api_smoketest[collapsed=true]\r\e[0KRunning Hive API smoketests..."
./scripts/ci/start-api-smoketest.sh \
$RUNNER_HIVEMIND_SMOKETEST_SERVER_HOSTNAME \
$RUNNER_HIVEMIND_SERVER_HTTP_PORT \
hive_api_patterns/ \
api_smoketest_hive_api.xml \
$RUNNER_PYTEST_WORKERS
echo -e "\e[0Ksection_end:$(date +%s):hive_api_smoketest\r\e[0K"
.api-benchmark-script: &api-benchmark-script
- |
echo -e "\e[0Ksection_start:$(date +%s):api-benchmark[collapsed=true]\r\e[0KRunning API benchmark..."
./scripts/ci/start-api-benchmarks.sh \
$RUNNER_HIVEMIND_BENCHMARK_SERVER_HOSTNAME \
$RUNNER_HIVEMIND_SERVER_HTTP_PORT \
$RUNNER_BENCHMARK_ITERATIONS \
$RUNNER_PYTEST_WORKERS
$RUNNER_PYTEST_WORKERS
echo -e "\e[0Ksection_end:$(date +%s):api-benchmark\r\e[0K"
#<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<| BASH SCRIPTS |<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>| JOBS |>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
# Build base images if they're missing from registry
prepare_base_images:
stage: build
extends: .docker_image_builder_job_template
before_script:
- git config --global --add safe.directory $CI_PROJECT_DIR
script:
- *base-image-build-script
tags: *shared_tags
prepare_haf_image:
stage: build
extends: .prepare_haf_image
......@@ -216,6 +263,8 @@ prepare_haf_data:
prepare_hivemind_image:
stage: build
extends: .docker_image_builder_job_template
needs:
- prepare_base_images
before_script:
- git config --global --add safe.directory $CI_PROJECT_DIR
script:
......@@ -289,7 +338,7 @@ sync:
pushd +2
${WORKING_DIR}/app/ci/collect-db-stats.sh
after_script:
- cp "$DATA_CACHE_HAF/datadir/$CI_JOB_NAME.log" "haf-$CI_JOB_NAME.log" # in after_script, so it's done even if job fails
- cp "$DATA_CACHE_HAF/datadir/$CI_JOB_NAME.log" "haf-$CI_JOB_NAME.log" # in after_script, so it's done even if the job fails
artifacts:
when: always
expire_in: 7 days
......@@ -303,7 +352,7 @@ sync:
e2e_benchmark:
image: $CI_REGISTRY_IMAGE/ci_base_image:3.8
image: $CI_REGISTRY_IMAGE/ci-base-image:python-3.8-slim-2
stage: benchmark
interruptible: true
needs:
......@@ -375,10 +424,6 @@ e2e_benchmark:
AWAIT_SERVICE_ACCESS_TOKEN: $AWAIT_ACCESS_TOKEN
AWAIT_PACKAGE_URL: $CI_SERVER_URL/api/v4/projects/345/packages/generic/await/v1.3.2/await
LOG_SERVER_PORT: 8090
before_script:
- |
pip install --no-cache-dir --verbose --user tox==3.25.0
export PATH=/home/hivemind/.local/bin:$PATH
script:
- |
echo "HAF image name $HAF_IMAGE_NAME"
......@@ -407,20 +452,22 @@ e2e_benchmark:
- *api-benchmark-script
after_script:
- |
echo "Downloading logs..."
echo -e "\e[0Ksection_start:$(date +%s):logs[collapsed=true]\r\e[0KDownloading logs..."
cp "$DATA_CACHE_HAF/datadir/$CI_JOB_NAME.log" "haf-$CI_JOB_NAME.log"
wget $RUNNER_HIVEMIND_SMOKETEST_URL:$LOG_SERVER_PORT/request_process_times_smoketests.log || true
wget $RUNNER_HIVEMIND_SMOKETEST_URL:$LOG_SERVER_PORT/hivemind-server.log || true
wget $RUNNER_HIVEMIND_BENCHMARK_URL:$LOG_SERVER_PORT/request_process_times.log || true
wget $RUNNER_HIVEMIND_BENCHMARK_URL:$LOG_SERVER_PORT/hivemind-benchmark-server.log || true
echo -e "\e[0Ksection_end:$(date +%s):logs\r\e[0K"
- |
echo "Preparing dotenv file..."
echo -e "\e[0Ksection_start:$(date +%s):dotenv[collapsed=true]\r\e[0KPreparing dotenv file..."
{
echo "ARTIFACTS_JOB_ID=$CI_JOB_ID"
echo "APP_VERSION=$(python -c "with open('version.log') as f: exec(f.read()); print(__version__)")"
echo "SERVER_NAME=$CI_RUNNER_DESCRIPTION"
} > variables.env
cat variables.env
echo -e "\e[0Ksection_end:$(date +%s):dotenv\r\e[0K"
artifacts:
when: always
expire_in: 7 days
......
Dockerfile
+ 46
− 27
View file @ 332b451c
# Base docker file having defined environment for build and run of HAF instance.
# Use scripts/ci/build_ci_base_image.sh to rebuild new version of CI base image. It must be properly tagged and pushed to the container registry
ARG CI_REGISTRY_IMAGE=registry.gitlab.syncad.com/hive/hivemind/
ARG CI_IMAGE_TAG=:ubuntu20.04-1
# syntax=docker/dockerfile:1.5
# Base docker file having defined environment for build and run of a Hivemind instance.
# Use scripts/ci/build_ci_base_image.sh to build a new version of the CI base image. It must be properly tagged and pushed to the container registry.
FROM --platform=$BUILDPLATFORM python:3.8-slim as runtime
......@@ -13,36 +11,57 @@ ENV LANG=en_US.UTF-8
ENV TARGETPLATFORM=${TARGETPLATFORM}
ENV BUILDPLATFORM=${BUILDPLATFORM}
RUN apt update && DEBIAN_FRONTEND=noniteractive apt install -y \
bash \
joe \
sudo \
git \
ca-certificates \
postgresql-client \
wget \
procps \
xz-utils \
python3-cffi \
&& DEBIAN_FRONTEND=noniteractive apt-get clean && rm -rf /var/lib/apt/lists/* \
&& useradd -ms /bin/bash "haf_admin" && echo "haf_admin ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers \
&& useradd -ms /bin/bash "haf_app_admin" \
&& useradd -ms /bin/bash "hivemind"
COPY haf/scripts/setup_ubuntu.sh /root/setup_os.sh
RUN <<EOF
set -e
apt update && DEBIAN_FRONTEND=noniteractive apt install -y \
bash \
joe \
sudo \
git \
ca-certificates \
postgresql-client \
wget \
procps \
xz-utils \
python3-cffi \
DEBIAN_FRONTEND=noniteractive apt-get clean && rm -rf /var/lib/apt/lists/*
/root/setup_os.sh --haf-admin-account="haf_admin"
useradd -ms /bin/bash -c "HAF application admin account" -u 4001 -U "haf_app_admin" --groups users
# This user needs UID of 1000 to be able to save logs to cache when run in CI
useradd -ms /bin/bash -c "Hivemind service account" -u 1000 "hivemind" --groups users
EOF
SHELL ["/bin/bash", "-c"]
FROM ${CI_REGISTRY_IMAGE}runtime${CI_IMAGE_TAG} AS ci-base-image
FROM runtime AS ci-base-image
ENV LANG=en_US.UTF-8
SHELL ["/bin/bash", "-c"]
RUN apt update && DEBIAN_FRONTEND=noniteractive apt install -y gcc && \
git config --global --add safe.directory /home/hivemind/app
RUN <<EOF
set -e
FROM ${CI_REGISTRY_IMAGE}ci-base-image${CI_IMAGE_TAG} AS builder
apt update
DEBIAN_FRONTEND=noniteractive apt install -y gcc
git config --global --add safe.directory /home/hivemind/app
EOF
USER hivemind
ENV PATH=/home/hivemind/.local/bin:${PATH}
RUN <<EOF
pip install --no-cache-dir --verbose --user tox==3.25.0
EOF
FROM ci-base-image AS builder
WORKDIR /home/hivemind
SHELL ["/bin/bash", "-c"]
......@@ -51,7 +70,7 @@ COPY --chown=hivemind:hivemind . /home/hivemind/app
RUN ./app/scripts/ci/build.sh
FROM ${CI_REGISTRY_IMAGE}runtime${CI_IMAGE_TAG} AS instance
FROM runtime AS instance
ARG HTTP_PORT=8080
ENV HTTP_PORT=${HTTP_PORT}
......
scripts/ci-helpers/build_instance.sh
+ 10
− 2
View file @ 332b451c
......@@ -69,11 +69,19 @@ pwd
"$SRCROOTDIR/scripts/ci/fix_ci_tag.sh"
BUILD_OPTIONS=("--platform=amd64" "--target=instance" "--progress=plain")
CI_IMAGE_TAG=${CI_IMAGE_TAG:-"python-3.8-slim-1"} # see scripts/ci/build_ci_base_image.sh for the current tag
BUILD_OPTIONS=("--platform=linux/amd64" "--target=instance" "--progress=plain")
TAG="$REGISTRY/instance:instance-$BUILD_IMAGE_TAG"
# On CI push the images to the registry
if [[ -n "${CI:-}" ]]; then
BUILD_OPTIONS+=("--push")
else
BUILD_OPTIONS+=("--load")
fi
docker buildx build "${BUILD_OPTIONS[@]}" \
--build-arg CI_REGISTRY_IMAGE="$REGISTRY/" \
--build-context "runtime=docker-image://${REGISTRY}/runtime:${CI_IMAGE_TAG}" \
--tag "$TAG" \
--file Dockerfile .
......
scripts/ci/build_ci_base_image.sh
+ 94
− 9
View file @ 332b451c
#! /bin/bash
REGISTRY=${1:-registry.gitlab.syncad.com/hive/hivemind/}
CI_IMAGE_TAG=:ubuntu20.04-1
set -e
print_help () {
cat <<EOF
Usage: $0 [OPTION[=VALUE]]...
Script for building runtime and base Hivemind Docker images.
Options:
--registry=URL Registry to use as a part of image names (default: registry.gitlab.syncad.com/hive/hivemind)
--tag=TAG Image tag (default: python-3.8-slim-2)
EOF
}
function image-exists() {
local image=$1
docker manifest inspect "$image" > /dev/null
return $?
}
REGISTRY=${REGISTRY:-"registry.gitlab.syncad.com/hive/hivemind"}
CI_IMAGE_TAG=${CI_IMAGE_TAG:-"python-3.8-slim-2"}
while [ $# -gt 0 ]; do
case "$1" in
--help|-h|-?)
print_help
exit 0
;;
--registry=*)
REGISTRY="${1#*=}"
;;
--tag=*)
CI_IMAGE_TAG="${1#*=}"
;;
*)
echo -e " ERROR: '$1' is not a valid option/positional argument\n"
print_help
exit 1
;;
esac
shift
done
BUILD_OPTIONS=(
"--platform=linux/amd64"
"--progress=plain"
)
if [[ -n "${REGISTRY}" ]]; then
BUILD_OPTIONS+=("--build-arg")
BUILD_OPTIONS+=("CI_REGISTRY_IMAGE=$REGISTRY/")
fi
if [[ -n "${CI_IMAGE_TAG}" ]]; then
BUILD_OPTIONS+=("--build-arg")
BUILD_OPTIONS+=("CI_IMAGE_TAG=:$CI_IMAGE_TAG")
fi
# On CI push the images to the registry, outside of CI add
# them to `docker images`
if [[ -n "${CI:-}" ]]; then
BUILD_OPTIONS+=("--push")
else
BUILD_OPTIONS+=("--load")
fi
RUNTIME_TAG="${REGISTRY}/runtime:${CI_IMAGE_TAG}"
CI_BASE_IMAGE_TAG="${REGISTRY}/ci-base-image:${CI_IMAGE_TAG}"
# Skip building images on CI if they already exist in the repository
if [[ -n "${CI:-}" ]] && image-exists "$RUNTIME_TAG"; then
echo "Image $RUNTIME_TAG already exists. Skipping build..."
else
docker buildx build "${BUILD_OPTIONS[@]}" \
--target=runtime \
--tag "$RUNTIME_TAG" \
--file Dockerfile .
fi
if [[ -n "${CI:-}" ]] && image-exists "$CI_BASE_IMAGE_TAG"; then
echo "Image $CI_BASE_IMAGE_TAG already exists. Skipping build..."
else
# On CI explicitly use the runtiume image to build the ci-base image, locally just use
# the build cache to get the same result
if [[ -n "${CI:-}" ]]; then
BUILD_OPTIONS+=("--build-context")
BUILD_OPTIONS+=("runtime=docker-image://${RUNTIME_TAG}")
fi
docker buildx build "${BUILD_OPTIONS[@]}" \
--target=ci-base-image \
--tag "$CI_BASE_IMAGE_TAG" \
--file Dockerfile .
fi
export DOCKER_BUILDKIT=1
docker build --platform=amd64 --target=runtime \
--build-arg CI_REGISTRY_IMAGE=$REGISTRY --build-arg CI_IMAGE_TAG=$CI_IMAGE_TAG \
-t ${REGISTRY}runtime$CI_IMAGE_TAG -f Dockerfile .
docker build --platform=amd64 --target=ci-base-image \
--build-arg CI_REGISTRY_IMAGE=$REGISTRY --build-arg CI_IMAGE_TAG=$CI_IMAGE_TAG \
-t ${REGISTRY}ci-base-image$CI_IMAGE_TAG -f Dockerfile .
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment