From a84d60c83060557955ee8d78d2cfc317e056fac4 Mon Sep 17 00:00:00 2001
From: Konrad Botor <kbotor@syncad.com>
Date: Tue, 2 Apr 2024 10:32:01 +0200
Subject: [PATCH] Added Python script for deleting old BuildKit cache from
GitLab Docker registry
---
.gitlab-ci.yml | 4 +-
.pylintrc | 8 +-
...image-remover => Dockerfile.python-scripts | 4 +-
docker-bake.hcl | 10 +-
scripts/python/delete-image.py | 1 +
scripts/python/remove-buildkit-cache.py | 117 ++++++++++++++++++
scripts/python/requirements.txt | 1 +
templates/docker_image_jobs.gitlab-ci.yml | 18 ++-
8 files changed, 152 insertions(+), 11 deletions(-)
rename Dockerfile.image-remover => Dockerfile.python-scripts (54%)
create mode 100644 scripts/python/remove-buildkit-cache.py
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8247fc4..20d46c7 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -129,13 +129,13 @@ build_image_remover_image:
extends: .build_docker_image
stage: build
variables:
- BUILD_TARGET: "image-remover"
+ BUILD_TARGET: "python-scripts"
needs:
- build_docker_dind_image
rules:
- if: $CI_COMMIT_BRANCH
exists:
- - Dockerfile.image-remover
+ - Dockerfile.python-scripts
build_benchmark_test_runner_image:
extends: .build_docker_image
diff --git a/.pylintrc b/.pylintrc
index a1a3a2b..7500b1a 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -1,2 +1,8 @@
[MASTER]
-load-plugins=pylint_junit
\ No newline at end of file
+load-plugins=pylint_junit
+
+[BASIC]
+module-naming-style=any
+
+[FORMAT]
+max-line-length=150
\ No newline at end of file
diff --git a/Dockerfile.image-remover b/Dockerfile.python-scripts
similarity index 54%
rename from Dockerfile.image-remover
rename to Dockerfile.python-scripts
index c59f4f7..63f1d63 100644
--- a/Dockerfile.image-remover
+++ b/Dockerfile.python-scripts
@@ -1,6 +1,6 @@
-FROM python:3.11.0 AS image-remover
+FROM python:3.12.2 AS python-scripts
-COPY scripts/python/delete-image.py /
+COPY scripts/python/*.py /
COPY scripts/python/requirements.txt /
RUN pip install -r requirements.txt
CMD [ "bash" ]
\ No newline at end of file
diff --git a/docker-bake.hcl b/docker-bake.hcl
index f3095ee..e99b3e6 100644
--- a/docker-bake.hcl
+++ b/docker-bake.hcl
@@ -61,11 +61,11 @@ target "docker-dind" {
cache-to = generate-cache-to("docker-dind", "${tag}")
}
-target "image-remover" {
- dockerfile = "Dockerfile.image-remover"
- tags = generate-tags("image-remover", "${tag}")
- cache-from = generate-cache-from("image-remover", "${tag}")
- cache-to = generate-cache-to("image-remover", "${tag}")
+target "python-scripts" {
+ dockerfile = "Dockerfile.python-scripts"
+ tags = generate-tags("python-scripts", "${tag}")
+ cache-from = generate-cache-from("python-scripts", "${tag}")
+ cache-to = generate-cache-to("python-scripts", "${tag}")
}
target "tox-test-runner" {
diff --git a/scripts/python/delete-image.py b/scripts/python/delete-image.py
index 4023fd5..1f07dc5 100644
--- a/scripts/python/delete-image.py
+++ b/scripts/python/delete-image.py
@@ -1,3 +1,4 @@
+#!/usr/local/bin/python3
# pylint: disable=C0103
'''
This script deletes a tag from GitLab's embedded docker repository.
diff --git a/scripts/python/remove-buildkit-cache.py b/scripts/python/remove-buildkit-cache.py
new file mode 100644
index 0000000..0de5655
--- /dev/null
+++ b/scripts/python/remove-buildkit-cache.py
@@ -0,0 +1,117 @@
+#!/usr/local/bin/python3
+'''
+This script deletes old BuildKit cache from GitLab registry.
+It's meant to be run by CI as it uses the job token.
+
+Requires CI_API_V4_URL, CI_PROJECT_ID and CI_JOB_TOKEN to be set.
+Requires CACHE_REPOSITORIES to contains comma-separated list of cache repositories.
+'''
+import os
+import requests
+
+def append_names(names, data):
+ """Appends names from JSON to name list"""
+ for record in data:
+ names.append(record['name'])
+
+def can_tag_be_deleted(branches, branch_name):
+ """Checks if a tag should be deleted."""
+ for branch in branches:
+ if branch['name'] == branch_name and not branch['merged']:
+ return False
+ return True
+
+def process_cache_registry_repository(gitlab_api_url, project_id,
+ repository_id, job_token, branches):
+ """Processes cache registry repositories"""
+ repository_url = f"{gitlab_api_url}/projects/{project_id}/registry/repositories/{repository_id}"
+ tags_url = f"{repository_url}/tags"
+
+ page_num = 1
+ params = {
+ 'per_page': 100,
+ 'page': page_num
+ }
+
+ image_tags = []
+
+ print(f"Fetching first page of tag list for repository ID {repository_id}.")
+ response = requests.get(url=tags_url, params=params, timeout=60)
+ data = response.json()
+ headers = response.headers
+ total_pages = int(headers['x-total-pages'])
+ print(f"Total pages available: {total_pages}.")
+
+ append_names(image_tags, data)
+
+ while page_num < total_pages:
+ page_num += 1
+ params['page'] = page_num
+ print(f"Fetching page {page_num} of tag list.")
+ response = requests.get(url=tags_url, params=params, timeout=60)
+ data = response.json()
+ append_names(image_tags, data)
+
+ length=len(image_tags)
+ print(f"Found total {length} image tags.")
+
+ for image_tag in image_tags:
+ if image_tag == "latest":
+ continue
+ branch_name_1 = image_tag
+ branch_name_2 = image_tag.replace("-", "/", 1)
+ if can_tag_be_deleted(branches, branch_name_1) and can_tag_be_deleted(branches, branch_name_2):
+ print(f"Deleting cache tag {image_tag}")
+ response = requests.delete(f"{tags_url}/{image_tag}", headers={'JOB-TOKEN': job_token}, timeout=60)
+ rj=response.json()
+ print(f"Tag deletion request finished with code {response.status_code} and response {rj}")
+
+ print(f"Finished tag processing for repository ID {repository_id}.")
+
+def main():
+ """Main function"""
+ gitlab_api_url = os.environ['CI_API_V4_URL']
+ project_id = os.environ['CI_PROJECT_ID']
+ job_token = os.environ['CI_JOB_TOKEN']
+ cache_repositories = os.environ['CACHE_REPOSITORIES'].split(',')
+ cache_repository_ids = []
+
+ print("Fetching cache registry repositories list.")
+ repositories_url = f"{gitlab_api_url}/projects/{project_id}/registry/repositories"
+ response = requests.get(url=repositories_url, timeout=60)
+ data = response.json()
+ for repo in data:
+ if repo['name'] in cache_repositories:
+ cache_repository_ids.append(repo['id'])
+
+ print("Fetching first page of branch list.")
+ page_num = 1
+ params = {
+ 'per_page': 100,
+ 'page': page_num
+ }
+ branches_url = f"{gitlab_api_url}/projects/{project_id}/repository/branches"
+ response = requests.get(url=branches_url, params=params, timeout=60)
+ data = response.json()
+ headers = response.headers
+ total_pages = int(headers['x-total-pages'])
+ print(f"Total pages available: {total_pages}.")
+
+ branches = data.copy()
+
+ while page_num < total_pages:
+ page_num += 1
+ params['page'] = page_num
+ print(f"Fetching page {page_num} of branch list.")
+ response = requests.get(url=branches_url, params=params, timeout=60)
+ data = response.json()
+ branches.extend(data)
+
+ length=len(branches)
+ print(f"Found total {length} branches.")
+
+ for repository_id in cache_repository_ids:
+ process_cache_registry_repository(gitlab_api_url, project_id, repository_id, job_token, branches)
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/python/requirements.txt b/scripts/python/requirements.txt
index 30492a8..017884c 100644
--- a/scripts/python/requirements.txt
+++ b/scripts/python/requirements.txt
@@ -1 +1,2 @@
python-gitlab==3.11.0
+requests==2.31.0
\ No newline at end of file
diff --git a/templates/docker_image_jobs.gitlab-ci.yml b/templates/docker_image_jobs.gitlab-ci.yml
index 65e17d5..e818e62 100644
--- a/templates/docker_image_jobs.gitlab-ci.yml
+++ b/templates/docker_image_jobs.gitlab-ci.yml
@@ -24,7 +24,7 @@ include:
.docker_image_cleanup_job_template:
extends: .job-defaults
- image: registry.gitlab.syncad.com/hive/common-ci-configuration/image-remover:${IMAGE_REMOVER_TAG}
+ image: registry.gitlab.syncad.com/hive/common-ci-configuration/python-scripts:${IMAGE_REMOVER_TAG}
variables:
REGISTRY: $CI_REGISTRY_IMAGE
REGISTRY_PASS: ""
@@ -35,6 +35,22 @@ include:
- python /delete-image.py "$REGISTRY_PASS" "$CI_PROJECT_ID" "$IMAGE_PATH" "$IMAGE_TAG"
when: always
+.buildkit_cleanup_job_template:
+ extends: .job-defaults
+ image: registry.gitlab.syncad.com/hive/common-ci-configuration/python-scripts:${IMAGE_REMOVER_TAG}
+ variables:
+ CACHE_REPOSITORIES: ""
+ script:
+ - echo "Attempting to delete old BuildKit cache from the following repositories $CACHE_REPOSITORIES"
+ - python3 /remove-buildkit-cache.py
+ rules:
+ - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+ - if: $CI_COMMIT_BRANCH || $CI_COMMIT_TAG
+ when: manual
+ allow_failure: true
+ tags:
+ - public-runner-docker
+
.publish_docker_image_template:
extends: .docker_image_builder_job_template
needs: []
--
GitLab