From 4e066043d63192f044e2b7f08ff9a5edae434960 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Mon, 29 Dec 2025 22:59:53 -0500 Subject: [PATCH 1/7] Use common-ci-configuration HAF app testing templates - Add include for haf_app_testing.gitlab-ci.yml template - Replace validate_haf_commit with .haf_commit_validation template - Add CACHE_MANAGER variables for standardized cache-manager usage - Reference feature/haf-app-testing-templates branch for template testing --- .gitlab-ci.yml | 42 ++++++++++++------------------------------ 1 file changed, 12 insertions(+), 30 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eebd08e..517856d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -70,12 +70,19 @@ variables: # Auto-populated by detect_changes job: AUTO_SKIP_BUILD: "false" # Set to true when only tests/docs/SQL changed AUTO_CACHE_COMMIT: "" # Most recent cached HAF commit + # Cache manager configuration (from common-ci-configuration) + CACHE_MANAGER: "/tmp/cache-manager.sh" + CACHE_MANAGER_REF: "feature/haf-app-testing-templates" include: - template: Workflows/Branch-Pipelines.gitlab-ci.yml - project: hive/haf ref: 1ac5d12439b9cca33e6db383adc59967cae75fc4 # develop file: /scripts/ci-helpers/prepare_data_image_job.yml +# HAF app testing templates from common-ci-configuration +- project: 'hive/common-ci-configuration' + ref: feature/haf-app-testing-templates + file: '/templates/haf_app_testing.gitlab-ci.yml' default: hooks: @@ -256,37 +263,12 @@ lint_sql_scripts: - sql-lint.yaml validate_haf_commit: + extends: .haf_commit_validation stage: build - image: alpine:latest - script: - - | - # Validate that HAF_COMMIT variable matches both the submodule and include ref - # This prevents cache misses due to mismatched commits - SUBMODULE_COMMIT=$(cat .git/modules/haf/HEAD 2>/dev/null || git -C haf rev-parse HEAD) - # Use awk instead of head -1 to avoid SIGPIPE (exit 141) when grep writes more lines than needed - INCLUDE_REF=$(grep -A2 "project:.*hive/haf" .gitlab-ci.yml | grep "ref:" | awk 'NR==1' | sed 's/.*ref: *\([a-f0-9]*\).*/\1/') - - echo "HAF_COMMIT variable: $HAF_COMMIT" - echo "HAF submodule HEAD: $SUBMODULE_COMMIT" - echo "Include ref: $INCLUDE_REF" - - ERRORS=0 - if [ "$HAF_COMMIT" != "$SUBMODULE_COMMIT" ]; then - echo "ERROR: HAF_COMMIT variable does not match submodule commit!" - echo " Update HAF_COMMIT in .gitlab-ci.yml to: $SUBMODULE_COMMIT" - ERRORS=1 - fi - if [ "$HAF_COMMIT" != "$INCLUDE_REF" ]; then - echo "ERROR: HAF_COMMIT variable does not match include ref!" - echo " Both should be: $HAF_COMMIT" - ERRORS=1 - fi - if [ $ERRORS -eq 1 ]; then - echo "" - echo "To fix: ensure HAF_COMMIT, include ref, and submodule all use the same commit" - exit 1 - fi - echo "All HAF commit references are consistent" + variables: + HAF_COMMIT: "1ac5d12439b9cca33e6db383adc59967cae75fc4" + HAF_INCLUDE_REF: "1ac5d12439b9cca33e6db383adc59967cae75fc4" + HAF_SUBMODULE_PATH: "haf" tags: - public-runner-docker -- GitLab From 80bf9b3c044ff9a44875f730a9955b2ee39f25b0 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Mon, 29 Dec 2025 23:39:05 -0500 Subject: [PATCH 2/7] Migrate detect_changes to use common template Replace ~70 lines of inline script with template extension. Uses .haf_app_detect_changes from common-ci-configuration. --- .gitlab-ci.yml | 81 ++------------------------------------------------ 1 file changed, 3 insertions(+), 78 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 517856d..7d7ba16 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -141,87 +141,12 @@ default: # DETECT CHANGES - Determines if heavy jobs can be skipped # ============================================================================= detect_changes: - stage: detect - image: alpine:latest + extends: .haf_app_detect_changes variables: - GIT_SUBMODULE_STRATEGY: none + # Skip patterns: tests, docs, markdown files, CI config + HAF_APP_SKIP_PATTERNS: '^tests/|^docs/|\.md$|^README|^CHANGELOG|^LICENSE|^\.gitlab-ci\.yml$' tags: - public-runner-docker - rules: - # Skip for tags (always do full build for releases) - - if: $CI_COMMIT_TAG - when: never - - when: on_success - script: - - | - apk add --no-cache git - - # Initialize defaults - CAN_SKIP_BUILD=false - CACHE_COMMIT="" - - # Handle manual QUICK_TEST mode - if [ "$QUICK_TEST" = "true" ]; then - echo "=== Manual QUICK_TEST Mode ===" - CAN_SKIP_BUILD=true - CACHE_COMMIT="${QUICK_TEST_HAF_COMMIT}" - if [ -z "$CACHE_COMMIT" ]; then - echo "ERROR: QUICK_TEST_HAF_COMMIT must be set when QUICK_TEST=true" - echo "" - echo "Find available caches:" - echo " ssh hive-builder-10 'ls -lt /nfs/ci-cache/haf/*.tar | head -5'" - exit 1 - fi - echo "Using cached HAF data from: $CACHE_COMMIT" - else - # Automatic detection mode - echo "=== Detecting Changed Files ===" - - # Get changed files based on pipeline type - if [ -n "${CI_MERGE_REQUEST_DIFF_BASE_SHA:-}" ]; then - BASE_SHA="$CI_MERGE_REQUEST_DIFF_BASE_SHA" - echo "MR pipeline: comparing against target branch" - elif [ "${CI_PIPELINE_SOURCE:-}" = "push" ]; then - BASE_SHA="HEAD~1" - echo "Push pipeline: comparing against previous commit" - else - BASE_SHA=$(git merge-base HEAD origin/develop 2>/dev/null || echo "HEAD~1") - echo "Other pipeline: comparing against develop" - fi - - echo "Comparing $BASE_SHA to HEAD" - CHANGED_FILES=$(git diff --name-only "$BASE_SHA" HEAD 2>/dev/null || git diff --name-only HEAD~1 HEAD) - - echo "Changed files:" - echo "$CHANGED_FILES" | head -30 - - # Skip patterns: tests, docs, markdown files - SKIP_BUILD_PATTERNS="^tests/|^docs/|\.md$|^README|^CHANGELOG|^LICENSE|^\.gitlab-ci\.yml$" - - NEEDS_BUILD=$(echo "$CHANGED_FILES" | grep -vE "$SKIP_BUILD_PATTERNS" || true) - - if [ -z "$NEEDS_BUILD" ]; then - CAN_SKIP_BUILD=true - echo "" - echo "=== Can skip heavy jobs (only tests/docs/CI changed) ===" - else - echo "" - echo "=== Full build required ===" - echo "Files requiring build:" - echo "$NEEDS_BUILD" | head -10 - fi - fi - - # Export results - echo "AUTO_SKIP_BUILD=$CAN_SKIP_BUILD" >> detect_changes.env - echo "AUTO_CACHE_COMMIT=$CACHE_COMMIT" >> detect_changes.env - - echo "" - echo "=== Detection Results ===" - cat detect_changes.env - artifacts: - reports: - dotenv: detect_changes.env .lint_job: stage: lint -- GitLab From 459d53514f59b0221a6e1ef2f78c3002564c4952 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Mon, 29 Dec 2025 23:56:28 -0500 Subject: [PATCH 3/7] Use tavern test templates in pattern-test job - Use .tavern_test_variables for common variable defaults - Use .tavern_install_deps for pip install - Use .tavern_test_artifacts for artifacts config - Keep app-specific tests_api cloning/install --- .gitlab-ci.yml | 36 +++++++++--------------------------- 1 file changed, 9 insertions(+), 27 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7d7ba16..582bcd7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -748,7 +748,9 @@ performance-test: junit: tests/performance/junit-result.xml pattern-test: - extends: .test-with-docker-compose + extends: + - .test-with-docker-compose + - .tavern_test_variables stage: test needs: - job: sync @@ -760,50 +762,30 @@ pattern-test: - job: docker-ci-runner-build artifacts: true variables: - JUNIT_REPORT: $CI_PROJECT_DIR/tests/tavern/report.xml REPTRACKER_ADDRESS: docker REPTRACKER_PORT: 3000 - TAVERN_DIR: $CI_PROJECT_DIR/tests/tavern - PYTEST_NUMBER_OF_PROCESSES: 16 + PYTEST_WORKERS: "16" before_script: - !reference [.test-with-docker-compose, before_script] - | echo -e "\e[0Ksection_start:$(date +%s):tests_api[collapsed=true]\r\e[0KCloning tests_api repository..." cd "${CI_PROJECT_DIR}" - # Clone tests_api directly - submodule update doesn't work in DinD due to missing git remotes in extracted cache TESTS_API_DIR="${CI_PROJECT_DIR}/haf/hive/tests/python/hive-local-tools/tests_api" mkdir -p "$(dirname "$TESTS_API_DIR")" rm -rf "$TESTS_API_DIR" git clone --depth 1 https://gitlab.syncad.com/hive/tests_api.git "$TESTS_API_DIR" echo -e "\e[0Ksection_end:$(date +%s):tests_api\r\e[0K" + - !reference [.tavern_install_deps, script] - | - echo -e "\e[0Ksection_start:$(date +%s):venv[collapsed=true]\r\e[0KSetting up Python environment..." - - cd "${CI_PROJECT_DIR}" - # Install test dependencies directly - python3 -m venv venv/ - . venv/bin/activate - - echo "Installing test dependencies..." - pip install tavern==2.2.0 pytest-xdist==3.2.0 pyyaml requests deepdiff==6.3.0 prettytable==3.8.0 # Install tests_api for validate_response module (required by tavern tests) + . venv/bin/activate pip install -e "${CI_PROJECT_DIR}/haf/hive/tests/python/hive-local-tools/tests_api" - - echo -e "\e[0Ksection_end:$(date +%s):venv\r\e[0K" script: - | - cd "${CI_PROJECT_DIR}" . venv/bin/activate - cd tests/tavern - # In DinD, PostgREST is available at 'docker' host - REPTRACKER_ADDRESS=docker pytest -n $PYTEST_NUMBER_OF_PROCESSES --junitxml report.xml . - artifacts: - paths: - - "**/*.out.json" - - docker/container-logs.tar.gz - reports: - junit: tests/tavern/report.xml - when: always + cd "${TAVERN_DIR}" + REPTRACKER_ADDRESS=docker pytest -n ${PYTEST_WORKERS} --junitxml "${JUNIT_REPORT}" . + artifacts: !reference [.tavern_test_artifacts, artifacts] deploy_python_api_packages_to_gitlab: stage: publish -- GitLab From 56a4885b3cecc3141121ade0182cdfef5da39335 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Tue, 30 Dec 2025 00:26:31 -0500 Subject: [PATCH 4/7] Fix pattern-test paths after template migration - Use explicit cd to CI_PROJECT_DIR before activating venv - Use local report.xml path since we cd to TAVERN_DIR - Keep explicit artifacts config (template !reference had variable issues) --- .gitlab-ci.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 582bcd7..6f45178 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -782,10 +782,18 @@ pattern-test: pip install -e "${CI_PROJECT_DIR}/haf/hive/tests/python/hive-local-tools/tests_api" script: - | + cd "${CI_PROJECT_DIR}" . venv/bin/activate cd "${TAVERN_DIR}" - REPTRACKER_ADDRESS=docker pytest -n ${PYTEST_WORKERS} --junitxml "${JUNIT_REPORT}" . - artifacts: !reference [.tavern_test_artifacts, artifacts] + # In DinD, PostgREST is available at 'docker' host + REPTRACKER_ADDRESS=docker pytest -n ${PYTEST_WORKERS} --junitxml report.xml . + artifacts: + paths: + - "**/*.out.json" + - docker/container-logs.tar.gz + reports: + junit: tests/tavern/report.xml + when: always deploy_python_api_packages_to_gitlab: stage: publish -- GitLab From d2c98b3fc2d08fdcef0f400d7139f85f69bd54db Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Tue, 30 Dec 2025 00:44:11 -0500 Subject: [PATCH 5/7] Fix tavern test path issue: run pytest from project root validate_response module writes .out.json files to paths like tests/tavern/get_reputation/file.out.json. When pytest was run after cd'ing into tests/tavern, these paths couldn't be resolved correctly. Running pytest from project root with TAVERN_DIR as argument fixes this. --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6f45178..31c4917 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -784,9 +784,9 @@ pattern-test: - | cd "${CI_PROJECT_DIR}" . venv/bin/activate - cd "${TAVERN_DIR}" - # In DinD, PostgREST is available at 'docker' host - REPTRACKER_ADDRESS=docker pytest -n ${PYTEST_WORKERS} --junitxml report.xml . + # Run pytest from project root so validate_response can write to correct paths + # (validate_response writes to paths like tests/tavern/get_reputation/file.out.json) + REPTRACKER_ADDRESS=docker pytest -n ${PYTEST_WORKERS} --junitxml "${TAVERN_DIR}/report.xml" "${TAVERN_DIR}" artifacts: paths: - "**/*.out.json" -- GitLab From ecf9a46e83e8b9424a2605ac75ce2de3100dc9a4 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Tue, 30 Dec 2025 01:12:37 -0500 Subject: [PATCH 6/7] Refactor sync job to use common-ci-configuration templates Use composable templates for sync job: - .haf_app_sync_variables for common directory/compose variables - .haf_app_sync_setup for docker login and git safe.directory - .haf_app_fetch_haf_cache for HAF replay data from local/NFS - .haf_app_copy_blockchain for block_log copy - .haf_app_copy_datadir for datadir copy and ownership fix - .haf_app_sync_shutdown for checkpoint, logs, compose down - .haf_app_sync_save_cache for local and NFS cache push - .haf_app_sync_cleanup for after_script cleanup - .haf_app_sync_artifacts for artifacts configuration Reduces sync job from ~150 lines to ~50 lines while keeping app-specific startup scripts inline. --- .gitlab-ci.yml | 140 ++++++++----------------------------------------- 1 file changed, 21 insertions(+), 119 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 31c4917..12e5716 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -354,7 +354,9 @@ docker-setup-docker-image-build: - public-runner-docker sync: - extends: .docker_image_builder_job_template + extends: + - .docker_image_builder_job_template + - .haf_app_sync_variables stage: sync image: registry.gitlab.syncad.com/hive/reputation_tracker/ci-runner:docker-24.0.1-8 needs: @@ -372,135 +374,35 @@ sync: # Need recursive for haf/hive submodule (copy_datadir.sh symlink) GIT_SUBMODULE_STRATEGY: recursive DATA_SOURCE: ${DATA_CACHE_HAF_PREFIX}_${HAF_COMMIT} - DATADIR: ${CI_PROJECT_DIR}/${CI_JOB_ID}/datadir - SHM_DIR: ${CI_PROJECT_DIR}/${CI_JOB_ID}/shm_dir - HAF_DATA_DIRECTORY: ${DATADIR} - HAF_SHM_DIRECTORY: ${SHM_DIR} BACKEND_VERSION: "$CI_COMMIT_SHORT_SHA" POSTGRES_ACCESS: postgresql://haf_admin@docker:5432/haf_block_log COMPOSE_OPTIONS_STRING: --env-file ci.env --file docker-compose.yml --file overrides/dev.yml --ansi never + # Aliases for sync templates + APP_SYNC_CACHE_TYPE: "${REPTRACKER_SYNC_CACHE_TYPE}" + APP_CACHE_KEY: "${REPTRACKER_CACHE_KEY}" timeout: 1 hours before_script: - - | - echo -e "\e[0Ksection_start:$(date +%s):login[collapsed=true]\r\e[0KLogging to Docker registry..." - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - echo -e "\e[0Ksection_end:$(date +%s):login\r\e[0K" - echo -e "\e[0Ksection_start:$(date +%s):git[collapsed=true]\r\e[0KConfiguring Git..." - git config --global --add safe.directory "$CI_PROJECT_DIR" - git config --global --add safe.directory "$CI_PROJECT_DIR/haf" - echo -e "\e[0Ksection_end:$(date +%s):git\r\e[0K" - - | - # Ensure HAF replay data is available locally (fetch from NFS if needed) - LOCAL_HAF_CACHE="${DATA_CACHE_HAF_PREFIX}_${HAF_COMMIT}" - if [[ -d "${LOCAL_HAF_CACHE}/datadir" ]]; then - echo "Local HAF cache found at ${LOCAL_HAF_CACHE}" - else - echo "Local HAF cache not found, fetching from NFS..." - # Fetch cache-manager from common-ci-configuration - CACHE_MANAGER="/tmp/cache-manager.sh" - curl -fsSL "https://gitlab.syncad.com/hive/common-ci-configuration/-/raw/develop/scripts/cache-manager.sh" -o "$CACHE_MANAGER" - chmod +x "$CACHE_MANAGER" - if "$CACHE_MANAGER" get haf "${HAF_COMMIT}" "${LOCAL_HAF_CACHE}"; then - echo "Fetched HAF replay data from NFS cache" - else - echo "ERROR: Failed to fetch HAF replay data from NFS cache" - exit 1 - fi - fi + - !reference [.haf_app_sync_setup, script] + - !reference [.haf_app_fetch_haf_cache, script] script: + # Copy blockchain and datadir + - !reference [.haf_app_copy_blockchain, script] + - !reference [.haf_app_copy_datadir, script] + # Start app-specific environment - | - echo -e "\e[0Ksection_start:$(date +%s):compose[collapsed=true]\r\e[0KStarting the test environment..." - - cp "${BLOCK_LOG_SOURCE_DIR_5M}/block_log" "${CI_PROJECT_DIR}/docker/blockchain/block_log" - cp "${BLOCK_LOG_SOURCE_DIR_5M}/block_log.artifacts" "${CI_PROJECT_DIR}/docker/blockchain/block_log.artifacts" - chmod a+w docker/blockchain/block_log - - "${CI_PROJECT_DIR}/haf/scripts/copy_datadir.sh" - - # Fix pgdata ownership and permissions - use explicit UID 105 (postgres in HAF container) - # The copy_datadir.sh uses 'postgres:postgres' which may resolve to different UIDs on different systems - # PostgreSQL also requires pgdata to have mode 700 or 750 - if [[ -d "${DATADIR}/haf_db_store" ]]; then - echo "Fixing haf_db_store ownership to UID 105:109 (postgres in HAF container)" - sudo chown -R 105:109 "${DATADIR}/haf_db_store" - sudo chown -R 105:109 "${DATADIR}/haf_postgresql_conf.d" - # Fix pgdata permissions - PostgreSQL requires 700 or 750 - if [[ -d "${DATADIR}/haf_db_store/pgdata" ]]; then - echo "Fixing pgdata permissions to 700" - sudo chmod 700 "${DATADIR}/haf_db_store/pgdata" - fi - fi - + echo -e "\e[0Ksection_start:$(date +%s):start_env[collapsed=true]\r\e[0KStarting the test environment..." "${CI_PROJECT_DIR}/scripts/ci-helpers/start-ci-test-environment.sh" - - echo -e "\e[0Ksection_end:$(date +%s):compose\r\e[0K" - echo -e "\e[0Ksection_start:$(date +%s):wait[collapsed=true]\r\e[0K$MESSAGE" - + echo -e "\e[0Ksection_end:$(date +%s):start_env\r\e[0K" + - | + echo -e "\e[0Ksection_start:$(date +%s):wait[collapsed=true]\r\e[0KWaiting for reputation tracker..." "${CI_PROJECT_DIR}/scripts/ci-helpers/wait-for-rt-startup.sh" - echo -e "\e[0Ksection_end:$(date +%s):wait\r\e[0K" - - | - # Stop environment and save data to NFS (must be in script, not after_script, - # so downstream jobs can access the cache via 'needs: sync') - echo -e "\e[0Ksection_start:$(date +%s):save_cache[collapsed=true]\r\e[0KSaving sync data to NFS cache..." - - pushd docker - IFS=" " read -ra COMPOSE_OPTIONS <<< $COMPOSE_OPTIONS_STRING - - # Force PostgreSQL checkpoint before shutdown to ensure all reptracker data is written to disk - # Without this, data may be in WAL files that aren't included in the cache - echo "Forcing PostgreSQL checkpoint..." - docker compose "${COMPOSE_OPTIONS[@]}" exec -T haf psql -U haf_admin -d haf_block_log -c "CHECKPOINT;" || true - - docker compose "${COMPOSE_OPTIONS[@]}" logs haf > haf.log - docker compose "${COMPOSE_OPTIONS[@]}" logs backend-setup > backend-setup.log - docker compose "${COMPOSE_OPTIONS[@]}" logs backend-block-processing > backend-block-processing.log - docker compose "${COMPOSE_OPTIONS[@]}" logs backend-postgrest > backend-postgrest.log - - # Use longer timeout (60s) for graceful PostgreSQL shutdown - docker compose "${COMPOSE_OPTIONS[@]}" down --volumes --timeout 60 - popd - - tar -czvf docker/container-logs.tar.gz $(pwd)/docker/*.log - - sudo cp -a "${SHM_DIR}" "${DATADIR}/shm_dir" - mkdir -p "${DATADIR}/blockchain" - sudo cp -a "${CI_PROJECT_DIR}/docker/blockchain/block_log" "${DATADIR}/blockchain/block_log" - sudo cp -a "${CI_PROJECT_DIR}/docker/blockchain/block_log.artifacts" "${DATADIR}/blockchain/block_log.artifacts" - - # Save sync data to local cache with commit-based key (not pipeline ID) - LOCAL_REPTRACKER_CACHE="${DATA_CACHE_HAF_PREFIX}_${REPTRACKER_CACHE_KEY}" - mkdir -p "${LOCAL_REPTRACKER_CACHE}" - sudo cp -a "${DATADIR}" "${LOCAL_REPTRACKER_CACHE}" - - ls -lah "${DATADIR}" - ls -lah "${DATADIR}/blockchain" - ls -lah "${DATADIR}/shm_dir" - - ls -lah "${LOCAL_REPTRACKER_CACHE}" - ls -lah "${LOCAL_REPTRACKER_CACHE}/datadir/blockchain" || ls -lah "${LOCAL_REPTRACKER_CACHE}/blockchain" || true - ls -lah "${LOCAL_REPTRACKER_CACHE}/datadir/shm_dir" || ls -lah "${LOCAL_REPTRACKER_CACHE}/shm_dir" || true - - # Push sync data to NFS cache for sharing across builders - # Fetch cache-manager from common-ci-configuration - CACHE_MANAGER="/tmp/cache-manager.sh" - curl -fsSL "https://gitlab.syncad.com/hive/common-ci-configuration/-/raw/develop/scripts/cache-manager.sh" -o "$CACHE_MANAGER" - chmod +x "$CACHE_MANAGER" - - echo "Pushing sync data to NFS cache: ${REPTRACKER_SYNC_CACHE_TYPE}/${REPTRACKER_CACHE_KEY}" - "$CACHE_MANAGER" put "${REPTRACKER_SYNC_CACHE_TYPE}" "${REPTRACKER_CACHE_KEY}" "${LOCAL_REPTRACKER_CACHE}" || echo "Warning: Failed to push to NFS cache" - - echo -e "\e[0Ksection_end:$(date +%s):save_cache\r\e[0K" - after_script: - - | - # Cleanup only - downstream jobs already started, so this is just for disk space - sudo rm -rf ${CI_PROJECT_DIR}/${CI_JOB_ID} - artifacts: - paths: - - docker/container-logs.tar.gz - expire_in: 1 week - when: always + # Shutdown and save cache using templates + - !reference [.haf_app_sync_shutdown, script] + - !reference [.haf_app_sync_save_cache, script] + after_script: !reference [.haf_app_sync_cleanup, after_script] + artifacts: !reference [.haf_app_sync_artifacts, artifacts] tags: - data-cache-storage - fast -- GitLab From 89877da6958e7c73284ab6d023cb0602459712c4 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Tue, 30 Dec 2025 02:03:10 -0500 Subject: [PATCH 7/7] Update common-ci-configuration ref to develop Templates merged in common-ci-configuration!120 --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 12e5716..e313cb4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -72,7 +72,7 @@ variables: AUTO_CACHE_COMMIT: "" # Most recent cached HAF commit # Cache manager configuration (from common-ci-configuration) CACHE_MANAGER: "/tmp/cache-manager.sh" - CACHE_MANAGER_REF: "feature/haf-app-testing-templates" + CACHE_MANAGER_REF: "develop" include: - template: Workflows/Branch-Pipelines.gitlab-ci.yml @@ -81,7 +81,7 @@ include: file: /scripts/ci-helpers/prepare_data_image_job.yml # HAF app testing templates from common-ci-configuration - project: 'hive/common-ci-configuration' - ref: feature/haf-app-testing-templates + ref: develop file: '/templates/haf_app_testing.gitlab-ci.yml' default: -- GitLab