diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8247fc4bce0d90989974121ec85abdd67bf1cdb2..20d46c70fd1185365389895665f3f3a71f16eff8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -129,13 +129,13 @@ build_image_remover_image: extends: .build_docker_image stage: build variables: - BUILD_TARGET: "image-remover" + BUILD_TARGET: "python-scripts" needs: - build_docker_dind_image rules: - if: $CI_COMMIT_BRANCH exists: - - Dockerfile.image-remover + - Dockerfile.python-scripts build_benchmark_test_runner_image: extends: .build_docker_image diff --git a/.pylintrc b/.pylintrc index a1a3a2be940b15f11a2cf5a23863a9cfabd6218a..be3f3c1eb4c109924e2f684230bcb54fdfdd93ea 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,2 +1,11 @@ [MASTER] -load-plugins=pylint_junit \ No newline at end of file +load-plugins=pylint_junit + +[BASIC] +module-naming-style=any + +[DESIGN] +max-locals = 20 + +[FORMAT] +max-line-length=150 \ No newline at end of file diff --git a/Dockerfile.image-remover b/Dockerfile.python-scripts similarity index 54% rename from Dockerfile.image-remover rename to Dockerfile.python-scripts index c59f4f71fd0ddea05b7206c91493908444c7577b..63f1d63527ed05b84939662d5f40d796e983729b 100644 --- a/Dockerfile.image-remover +++ b/Dockerfile.python-scripts @@ -1,6 +1,6 @@ -FROM python:3.11.0 AS image-remover +FROM python:3.12.2 AS python-scripts -COPY scripts/python/delete-image.py / +COPY scripts/python/*.py / COPY scripts/python/requirements.txt / RUN pip install -r requirements.txt CMD [ "bash" ] \ No newline at end of file diff --git a/docker-bake.hcl b/docker-bake.hcl index f3095eec996eae3178885ad178b0dd67a589814c..e99b3e6c3cd23a417b1df9bed091419ee64ac223 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -61,11 +61,11 @@ target "docker-dind" { cache-to = generate-cache-to("docker-dind", "${tag}") } -target "image-remover" { - dockerfile = "Dockerfile.image-remover" - tags = generate-tags("image-remover", "${tag}") - cache-from = generate-cache-from("image-remover", "${tag}") - cache-to = generate-cache-to("image-remover", "${tag}") +target "python-scripts" { + dockerfile = "Dockerfile.python-scripts" + tags = generate-tags("python-scripts", "${tag}") + cache-from = generate-cache-from("python-scripts", "${tag}") + cache-to = generate-cache-to("python-scripts", "${tag}") } target "tox-test-runner" { diff --git a/scripts/python/delete-image.py b/scripts/python/delete-image.py index 4023fd57d2cbb422c8f55be996bd938d428790c9..1f07dc5af038f8b935f76db7d92bcb1792bbbbf1 100644 --- a/scripts/python/delete-image.py +++ b/scripts/python/delete-image.py @@ -1,3 +1,4 @@ +#!/usr/local/bin/python3 # pylint: disable=C0103 ''' This script deletes a tag from GitLab's embedded docker repository. diff --git a/scripts/python/remove-buildkit-cache.py b/scripts/python/remove-buildkit-cache.py new file mode 100644 index 0000000000000000000000000000000000000000..b855ce232aa10fbc528167ee2e45ddaa9c4ceaca --- /dev/null +++ b/scripts/python/remove-buildkit-cache.py @@ -0,0 +1,117 @@ +#!/usr/local/bin/python3 +''' +This script deletes old BuildKit cache from GitLab registry. +It's meant to be run by CI as it uses the job token. + +Requires CI_API_V4_URL, CI_PROJECT_ID and CI_JOB_TOKEN to be set. +Requires CACHE_REPOSITORIES to contains comma-separated list of cache repositories. +''' +import os +import requests + +def append_names(names, data): + """Appends names from JSON to name list""" + for record in data: + names.append(record['name']) + +def can_tag_be_deleted(branches, branch_name): + """Checks if a tag should be deleted.""" + for branch in branches: + if branch['name'] == branch_name and not branch['merged']: + return False + return True + +def process_cache_registry_repository(gitlab_api_url, project_id, + repository_id, job_token, branches): + """Processes cache registry repositories""" + repository_url = f"{gitlab_api_url}/projects/{project_id}/registry/repositories/{repository_id}" + tags_url = f"{repository_url}/tags" + + page_num = 1 + params = { + 'per_page': 100, + 'page': page_num + } + + image_tags = [] + + print(f"Fetching first page of tag list for repository ID {repository_id}.") + response = requests.get(url=tags_url, params=params, timeout=60) + data = response.json() + headers = response.headers + total_pages = int(headers['x-total-pages']) + print(f"Total pages available: {total_pages}.") + + append_names(image_tags, data) + + while page_num < total_pages: + page_num += 1 + params['page'] = page_num + print(f"Fetching page {page_num} of tag list.") + response = requests.get(url=tags_url, params=params, timeout=60) + data = response.json() + append_names(image_tags, data) + + length=len(image_tags) + print(f"Found total {length} image tags.") + + for image_tag in image_tags: + if image_tag == "latest": + continue + branch_name_1 = image_tag + branch_name_2 = image_tag.replace("-", "/", 1) + if can_tag_be_deleted(branches, branch_name_1) and can_tag_be_deleted(branches, branch_name_2): + print(f"Deleting cache tag {image_tag}") + response = requests.delete(f"{tags_url}/{image_tag}", headers={'JOB-TOKEN': job_token}, timeout=60) + response_json=response.json() + print(f"Tag deletion request finished with code {response.status_code} and response {response_json}") + + print(f"Finished tag processing for repository ID {repository_id}.") + +def main(): + """Main function""" + gitlab_api_url = os.environ['CI_API_V4_URL'] + project_id = os.environ['CI_PROJECT_ID'] + job_token = os.environ['CI_JOB_TOKEN'] + cache_repositories = os.environ['CACHE_REPOSITORIES'].split(',') + cache_repository_ids = [] + + print("Fetching cache registry repositories list.") + repositories_url = f"{gitlab_api_url}/projects/{project_id}/registry/repositories" + response = requests.get(url=repositories_url, timeout=60) + data = response.json() + for repo in data: + if repo['name'] in cache_repositories: + cache_repository_ids.append(repo['id']) + + print("Fetching first page of branch list.") + page_num = 1 + params = { + 'per_page': 100, + 'page': page_num + } + branches_url = f"{gitlab_api_url}/projects/{project_id}/repository/branches" + response = requests.get(url=branches_url, params=params, timeout=60) + data = response.json() + headers = response.headers + total_pages = int(headers['x-total-pages']) + print(f"Total pages available: {total_pages}.") + + branches = data.copy() + + while page_num < total_pages: + page_num += 1 + params['page'] = page_num + print(f"Fetching page {page_num} of branch list.") + response = requests.get(url=branches_url, params=params, timeout=60) + data = response.json() + branches.extend(data) + + length=len(branches) + print(f"Found total {length} branches.") + + for repository_id in cache_repository_ids: + process_cache_registry_repository(gitlab_api_url, project_id, repository_id, job_token, branches) + +if __name__ == "__main__": + main() diff --git a/scripts/python/requirements.txt b/scripts/python/requirements.txt index 30492a85eb3cc96ab989eba3fa2c5906362e9783..017884c3ba87118637679e1157a19d49ac24aa27 100644 --- a/scripts/python/requirements.txt +++ b/scripts/python/requirements.txt @@ -1 +1,2 @@ python-gitlab==3.11.0 +requests==2.31.0 \ No newline at end of file diff --git a/templates/docker_image_jobs.gitlab-ci.yml b/templates/docker_image_jobs.gitlab-ci.yml index 65e17d53feaaa0468a4b1672052628a0e763a166..e818e629da4c7591b46052fc4dbe9a8e9b233f83 100644 --- a/templates/docker_image_jobs.gitlab-ci.yml +++ b/templates/docker_image_jobs.gitlab-ci.yml @@ -24,7 +24,7 @@ include: .docker_image_cleanup_job_template: extends: .job-defaults - image: registry.gitlab.syncad.com/hive/common-ci-configuration/image-remover:${IMAGE_REMOVER_TAG} + image: registry.gitlab.syncad.com/hive/common-ci-configuration/python-scripts:${IMAGE_REMOVER_TAG} variables: REGISTRY: $CI_REGISTRY_IMAGE REGISTRY_PASS: "" @@ -35,6 +35,22 @@ include: - python /delete-image.py "$REGISTRY_PASS" "$CI_PROJECT_ID" "$IMAGE_PATH" "$IMAGE_TAG" when: always +.buildkit_cleanup_job_template: + extends: .job-defaults + image: registry.gitlab.syncad.com/hive/common-ci-configuration/python-scripts:${IMAGE_REMOVER_TAG} + variables: + CACHE_REPOSITORIES: "" + script: + - echo "Attempting to delete old BuildKit cache from the following repositories $CACHE_REPOSITORIES" + - python3 /remove-buildkit-cache.py + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_COMMIT_BRANCH || $CI_COMMIT_TAG + when: manual + allow_failure: true + tags: + - public-runner-docker + .publish_docker_image_template: extends: .docker_image_builder_job_template needs: []