diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6010ebb6bbad276f3ad1a4365fb51afd1bacb109..30cc8ac9e8d155f9c160994cbc828c33d52d0334 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,11 @@ +variables: + DOCKER_BUILDER_TAG: "$CI_COMMIT_SHA" + DOCKER_DIND_TAG: "$CI_COMMIT_SHA" + IMAGE_REMOVER_TAG: "$CI_COMMIT_SHA" + stages: - validation + - pre-build - build - example-build - example-cleanup @@ -64,14 +70,16 @@ lint_python_scripts: .build_docker_image: extends: .docker_image_builder_job - stage: build variables: BUILD_TARGET: "" before_script: - - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - script: - - tag="" - | + echo -e "\e[0Ksection_start:$(date +%s):login[collapsed=true]\r\e[0KLogging to Docker registry..." + docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + echo -e "\e[0Ksection_end:$(date +%s):login\r\e[0K" + script: + - | + tag="" echo -e "\e[0Ksection_start:$(date +%s):tag[collapsed=true]\r\e[0KDetermining tag for the new image..." if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'latest'" @@ -81,41 +89,54 @@ lint_python_scripts: tag=":$CI_COMMIT_REF_SLUG" fi echo -e "\e[0Ksection_end:$(date +%s):tag\r\e[0K" - - | echo -e "\e[0Ksection_start:$(date +%s):build[collapsed=true]\r\e[0KBuilding image "$CI_REGISTRY_IMAGE/$BUILD_TARGET${tag}"..." - docker build --target $BUILD_TARGET --pull \ + docker build --file "Dockerfile.$BUILD_TARGET" --pull \ -t "$CI_REGISTRY_IMAGE/$BUILD_TARGET${tag}" \ -t "$CI_REGISTRY_IMAGE/$BUILD_TARGET:$CI_COMMIT_SHA" . echo -e "\e[0Ksection_end:$(date +%s):build\r\e[0K" - - | echo -e "\e[0Ksection_start:$(date +%s):push1[collapsed=true]\r\e[0KPushing image "$CI_REGISTRY_IMAGE/$BUILD_TARGET${tag}"..." docker push "$CI_REGISTRY_IMAGE/$BUILD_TARGET${tag}" echo -e "\e[0Ksection_end:$(date +%s):push1\r\e[0K" - - | echo -e "\e[0Ksection_start:$(date +%s):push2[collapsed=true]\r\e[0KPushing image "$CI_REGISTRY_IMAGE/$BUILD_TARGET:$CI_COMMIT_SHA"..." docker push "$CI_REGISTRY_IMAGE/$BUILD_TARGET:$CI_COMMIT_SHA" echo -e "\e[0Ksection_end:$(date +%s):push2\r\e[0K" rules: - if: $CI_COMMIT_BRANCH exists: - - Dockerfile + - "Dockerfile.$BUILD_TARGET" tags: - public-runner-docker +build_docker_dind_image: + stage: pre-build + image: docker:20.10.10 + variables: + DOCKER_BUILDKIT: 1 + DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "/certs" + BUILD_TARGET: "docker-dind" + script: + - !reference [.build_docker_image, script] + rules: + - !reference [.build_docker_image, rules] + services: + - docker:20.10.10-dind + build_docker_builder_image: extends: .build_docker_image + stage: build variables: BUILD_TARGET: "docker-builder" build_image_remover_image: extends: .build_docker_image + stage: build variables: BUILD_TARGET: "image-remover" prepare_example_hived_data_5m_image: extends: .prepare_hived_data_5m_image stage: example-build - image: registry.gitlab.syncad.com/hive/common-ci-configuration/docker-builder:$CI_COMMIT_SHA variables: REGISTRY_USER: "$CI_REGISTRY_USER" REGISTRY_PASS: "$CI_REGISTRY_PASSWORD" @@ -131,7 +152,6 @@ prepare_example_hived_data_5m_image: prepare_example_haf_data_5m_image: extends: .prepare_haf_data_5m_image stage: example-build - image: registry.gitlab.syncad.com/hive/common-ci-configuration/docker-builder:$CI_COMMIT_SHA variables: REGISTRY_USER: "$CI_REGISTRY_USER" REGISTRY_PASS: "$CI_REGISTRY_PASSWORD" @@ -147,7 +167,6 @@ prepare_example_haf_data_5m_image: example_hived_data_image_cleanup: extends: .docker_image_cleanup_job stage: example-cleanup - image: registry.gitlab.syncad.com/hive/common-ci-configuration/image-remover:$CI_COMMIT_SHA variables: REGISTRY_PASS: "$REGISTRY_PASS" IMAGE_PATH: $HIVED_IMAGE_NAME_REGISTRY_PATH @@ -161,7 +180,6 @@ example_hived_data_image_cleanup: example_haf_data_image_cleanup: extends: .docker_image_cleanup_job stage: example-cleanup - image: registry.gitlab.syncad.com/hive/common-ci-configuration/image-remover:$CI_COMMIT_SHA variables: REGISTRY_PASS: "$REGISTRY_PASS" IMAGE_PATH: $HAF_IMAGE_NAME_REGISTRY_PATH diff --git a/Dockerfile.docker-builder b/Dockerfile.docker-builder new file mode 100644 index 0000000000000000000000000000000000000000..9d1681f0c695792316e27df35c71b665f9818103 --- /dev/null +++ b/Dockerfile.docker-builder @@ -0,0 +1,4 @@ +FROM docker:20.10.10 AS docker-builder + +COPY scripts/bash/*.sh /usr/local/bin +RUN apk add --no-cache bash git ca-certificates curl \ No newline at end of file diff --git a/Dockerfile.docker-dind b/Dockerfile.docker-dind new file mode 100644 index 0000000000000000000000000000000000000000..309b2d1d48a22d00e7e370b04d730e14ff9bd7c2 --- /dev/null +++ b/Dockerfile.docker-dind @@ -0,0 +1,14 @@ +# To workaround a gitlab healthcheck bug, expose just single port. +# See https://gitlab.com/gitlab-org/gitlab-runner/-/issues/29130#note_1028331564 and +# https://gitlab.com/search?search=Service+docker+dind+probably+didn%27t+start+properly&nav_source=navbar&project_id=250833&group_id=9970&scope=issues&sort=updated_desc +FROM docker:20.10.10-dind as upstream + +FROM scratch + +COPY --from=upstream / / + +VOLUME /var/lib/docker +EXPOSE 2376/tcp + +ENTRYPOINT ["dockerd-entrypoint.sh"] +CMD [] \ No newline at end of file diff --git a/Dockerfile b/Dockerfile.image-remover similarity index 55% rename from Dockerfile rename to Dockerfile.image-remover index 961b12ab3f0655775156a65f27ba63d743b54c5d..c59f4f71fd0ddea05b7206c91493908444c7577b 100644 --- a/Dockerfile +++ b/Dockerfile.image-remover @@ -1,8 +1,3 @@ -FROM docker:20.10.10 AS docker-builder - -COPY scripts/bash/*.sh /usr/local/bin -RUN apk add --no-cache bash git ca-certificates curl - FROM python:3.11.0 AS image-remover COPY scripts/python/delete-image.py / diff --git a/templates/data_image_jobs.gitlab-ci.yml b/templates/data_image_jobs.gitlab-ci.yml index c0ad65a0fd42b48af642d2744ded1c10589e1edb..8745b8cec274ac837000be8c101bef3231710199 100644 --- a/templates/data_image_jobs.gitlab-ci.yml +++ b/templates/data_image_jobs.gitlab-ci.yml @@ -1,15 +1,21 @@ +variables: + DOCKER_BUILDER_TAG: "latest" + DOCKER_DIND_TAG: "latest" + IMAGE_REMOVER_TAG: "latest" + .docker_image_builder_job: variables: DOCKER_BUILDKIT: 1 DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "/certs" - image: docker:20.10.10 + image: registry.gitlab.syncad.com/hive/common-ci-configuration/docker-builder:${DOCKER_BUILDER_TAG} interruptible: true services: - - docker:20.10.10-dind + - name: registry.gitlab.syncad.com/hive/common-ci-configuration/docker-dind:${DOCKER_DIND_TAG} + alias: docker .docker_image_cleanup_job: - image: registry.gitlab.syncad.com/hive/common-ci-configuration/image-remover:latest + image: registry.gitlab.syncad.com/hive/common-ci-configuration/image-remover:${IMAGE_REMOVER_TAG} interruptible: true variables: REGISTRY: $CI_REGISTRY_IMAGE @@ -24,7 +30,6 @@ .prepare_data_5m_image: extends: .docker_image_builder_job - image: registry.gitlab.syncad.com/hive/common-ci-configuration/docker-builder:latest variables: DOTENV_NAME: "" REGISTRY_USER: "$CI_IMG_BUILDER_USER"