Added Python script for deleting old BuildKit cache from GitLab Docker registry

.gitlab-ci.yml

@@ -129,13 +129,13 @@ build_image_remover_image:
   extends: .build_docker_image
   stage: build
   variables:
-    BUILD_TARGET: "image-remover"
+    BUILD_TARGET: "python-scripts"
   needs:
     - build_docker_dind_image
   rules:
     - if: $CI_COMMIT_BRANCH
       exists:
-        - Dockerfile.image-remover
+        - Dockerfile.python-scripts
 
 build_benchmark_test_runner_image:
   extends: .build_docker_image

.pylintrc

 [MASTER]
-load-plugins=pylint_junit
\ No newline at end of file
+load-plugins=pylint_junit
+
+[BASIC]
+module-naming-style=any
+
+[DESIGN]
+max-locals = 20
+
+[FORMAT]
+max-line-length=150
\ No newline at end of file

Dockerfile.image-remover → Dockerfile.python-scripts

-FROM python:3.11.0 AS image-remover
+FROM python:3.12.2 AS python-scripts
 
-COPY scripts/python/delete-image.py /
+COPY scripts/python/*.py /
 COPY scripts/python/requirements.txt /
 RUN pip install -r requirements.txt
 CMD [ "bash" ]
\ No newline at end of file

docker-bake.hcl

@@ -61,11 +61,11 @@ target "docker-dind" {
   cache-to = generate-cache-to("docker-dind", "${tag}")
 }
 
-target "image-remover" {
-  dockerfile = "Dockerfile.image-remover"
-  tags = generate-tags("image-remover", "${tag}")
-  cache-from = generate-cache-from("image-remover", "${tag}")
-  cache-to = generate-cache-to("image-remover", "${tag}")
+target "python-scripts" {
+  dockerfile = "Dockerfile.python-scripts"
+  tags = generate-tags("python-scripts", "${tag}")
+  cache-from = generate-cache-from("python-scripts", "${tag}")
+  cache-to = generate-cache-to("python-scripts", "${tag}")
 }
 
 target "tox-test-runner" {

scripts/python/delete-image.py

+#!/usr/local/bin/python3
 # pylint: disable=C0103
 '''
 This script deletes a tag from GitLab's embedded docker repository.

scripts/python/remove-buildkit-cache.py

+#!/usr/local/bin/python3
+'''
+This script deletes old BuildKit cache from GitLab registry.
+It's meant to be run by CI as it uses the job token.
+
+Requires CI_API_V4_URL, CI_PROJECT_ID and CI_JOB_TOKEN to be set.
+Requires CACHE_REPOSITORIES to contains comma-separated list of cache repositories.
+'''
+import os
+import requests
+
+def append_names(names, data):
+    """Appends names from JSON to name list"""
+    for record in data:
+        names.append(record['name'])
+
+def can_tag_be_deleted(branches, branch_name):
+    """Checks if a tag should be deleted."""
+    for branch in branches:
+        if branch['name'] == branch_name and not branch['merged']:
+            return False
+    return True
+
+def process_cache_registry_repository(gitlab_api_url, project_id,
+                                      repository_id, job_token, branches):
+    """Processes cache registry repositories"""
+    repository_url = f"{gitlab_api_url}/projects/{project_id}/registry/repositories/{repository_id}"
+    tags_url = f"{repository_url}/tags"
+
+    page_num = 1
+    params = {
+        'per_page': 100,
+        'page': page_num
+    }
+
+    image_tags = []
+
+    print(f"Fetching first page of tag list for repository ID {repository_id}.")
+    response = requests.get(url=tags_url, params=params, timeout=60)
+    data = response.json()
+    headers = response.headers
+    total_pages = int(headers['x-total-pages'])
+    print(f"Total pages available: {total_pages}.")
+
+    append_names(image_tags, data)
+
+    while page_num < total_pages:
+        page_num += 1
+        params['page'] = page_num
+        print(f"Fetching page {page_num} of tag list.")
+        response = requests.get(url=tags_url, params=params, timeout=60)
+        data = response.json()
+        append_names(image_tags, data)
+
+    length=len(image_tags)
+    print(f"Found total {length} image tags.")
+
+    for image_tag in image_tags:
+        if image_tag == "latest":
+            continue
+        branch_name_1 = image_tag
+        branch_name_2 = image_tag.replace("-", "/", 1)
+        if can_tag_be_deleted(branches, branch_name_1) and can_tag_be_deleted(branches, branch_name_2):
+            print(f"Deleting cache tag {image_tag}")
+            response = requests.delete(f"{tags_url}/{image_tag}", headers={'JOB-TOKEN': job_token}, timeout=60)
+            response_json=response.json()
+            print(f"Tag deletion request finished with code {response.status_code} and response {response_json}")
+
+    print(f"Finished tag processing for repository ID {repository_id}.")
+
+def main():
+    """Main function"""
+    gitlab_api_url = os.environ['CI_API_V4_URL']
+    project_id = os.environ['CI_PROJECT_ID']
+    job_token = os.environ['CI_JOB_TOKEN']
+    cache_repositories = os.environ['CACHE_REPOSITORIES'].split(',')
+    cache_repository_ids = []
+
+    print("Fetching cache registry repositories list.")
+    repositories_url = f"{gitlab_api_url}/projects/{project_id}/registry/repositories"
+    response = requests.get(url=repositories_url, timeout=60)
+    data = response.json()
+    for repo in data:
+        if repo['name'] in cache_repositories:
+            cache_repository_ids.append(repo['id'])
+
+    print("Fetching first page of branch list.")
+    page_num = 1
+    params = {
+        'per_page': 100,
+        'page': page_num
+    }
+    branches_url = f"{gitlab_api_url}/projects/{project_id}/repository/branches"
+    response = requests.get(url=branches_url, params=params, timeout=60)
+    data = response.json()
+    headers = response.headers
+    total_pages = int(headers['x-total-pages'])
+    print(f"Total pages available: {total_pages}.")
+
+    branches = data.copy()
+
+    while page_num < total_pages:
+        page_num += 1
+        params['page'] = page_num
+        print(f"Fetching page {page_num} of branch list.")
+        response = requests.get(url=branches_url, params=params, timeout=60)
+        data = response.json()
+        branches.extend(data)
+
+    length=len(branches)
+    print(f"Found total {length} branches.")
+
+    for repository_id in cache_repository_ids:
+        process_cache_registry_repository(gitlab_api_url, project_id, repository_id, job_token, branches)
+
+if __name__ == "__main__":
+    main()

scripts/python/requirements.txt

 python-gitlab==3.11.0
+requests==2.31.0
\ No newline at end of file

templates/docker_image_jobs.gitlab-ci.yml

@@ -24,7 +24,7 @@ include:
 
 .docker_image_cleanup_job_template:
   extends: .job-defaults
-  image: registry.gitlab.syncad.com/hive/common-ci-configuration/image-remover:${IMAGE_REMOVER_TAG}
+  image: registry.gitlab.syncad.com/hive/common-ci-configuration/python-scripts:${IMAGE_REMOVER_TAG}
   variables:
     REGISTRY: $CI_REGISTRY_IMAGE
     REGISTRY_PASS: ""
@@ -35,6 +35,22 @@ include:
     - python /delete-image.py "$REGISTRY_PASS" "$CI_PROJECT_ID" "$IMAGE_PATH" "$IMAGE_TAG"
   when: always
 
+.buildkit_cleanup_job_template:
+  extends: .job-defaults
+  image: registry.gitlab.syncad.com/hive/common-ci-configuration/python-scripts:${IMAGE_REMOVER_TAG}
+  variables:
+    CACHE_REPOSITORIES: ""
+  script:
+    - echo "Attempting to delete old BuildKit cache from the following repositories $CACHE_REPOSITORIES"
+    - python3 /remove-buildkit-cache.py
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: $CI_COMMIT_BRANCH || $CI_COMMIT_TAG
+      when: manual
+      allow_failure: true
+  tags:
+    - public-runner-docker
+
 .publish_docker_image_template:
   extends: .docker_image_builder_job_template
   needs: []