From 0386947a414d1b6acf09d44cd6d77fe2c3b6ab78 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Sat, 6 Dec 2025 22:11:28 -0500 Subject: [PATCH 01/50] Use NFS cache and remove builder affinity - Update HAF ref to include cache-manager.sh for NFS-based caching - Replace hive-builder-9 tags with 'fast' tag to allow jobs to run on any AMD 5950X builder (8, 9, 10) with NFS cache access --- .gitlab-ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 31b01a8..1e24aa0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -29,7 +29,7 @@ variables: include: - template: Workflows/Branch-Pipelines.gitlab-ci.yml - project: hive/haf - ref: bf820442979eff6c7cb7e387f26cd4ccf9345f3c # develop + ref: 68a06608fe308afc05c994b8e1c16564d34e8fac # feature/nfs-cache-manager (pending merge to develop) file: /scripts/ci-helpers/prepare_data_image_job.yml .lint_job: @@ -146,7 +146,7 @@ prepare_haf_data: CONFIG_INI_SOURCE: "$CI_PROJECT_DIR/haf/docker/config_5M.ini" tags: - data-cache-storage - - hive-builder-9 + - fast .docker-build-template: extends: .docker_image_builder_job_template @@ -297,7 +297,7 @@ sync: when: always tags: - data-cache-storage - - hive-builder-9 + - fast .hfm-only-service: &hfm-only-service name: $HAF_IMAGE_NAME @@ -368,7 +368,7 @@ regression-test: when: always tags: - data-cache-storage - - hive-builder-9 + - fast setup-scripts-test: image: registry.gitlab.syncad.com/hive/reputation_tracker/ci-runner:docker-24.0.1-3 @@ -394,7 +394,7 @@ setup-scripts-test: echo -e "\e[0Ksection_end:$(date +%s):tests\r\e[0K" tags: - data-cache-storage - - hive-builder-9 + - fast performance-test: image: registry.gitlab.syncad.com/hive/reputation_tracker/ci-runner:docker-24.0.1-3 @@ -431,7 +431,7 @@ performance-test: junit: tests/performance/junit-result.xml tags: - data-cache-storage - - hive-builder-9 + - fast pattern-test: extends: .pytest_based_template @@ -463,7 +463,7 @@ pattern-test: - "**/*.out.json" tags: - data-cache-storage - - hive-builder-9 + - fast deploy_python_api_packages_to_gitlab: stage: publish @@ -530,4 +530,4 @@ cleanup_haf_cache_manual: CLEANUP_PATH_PATTERN: "${DATA_CACHE_HAF_PREFIX}_*" tags: - data-cache-storage - - hive-builder-9 + - fast -- GitLab From 2233afd2b3f69f40075a3291321adf6b01f0769a Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Sat, 6 Dec 2025 22:38:51 -0500 Subject: [PATCH 02/50] Update HAF ref to merged develop commit --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1e24aa0..fd80bb8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -29,7 +29,7 @@ variables: include: - template: Workflows/Branch-Pipelines.gitlab-ci.yml - project: hive/haf - ref: 68a06608fe308afc05c994b8e1c16564d34e8fac # feature/nfs-cache-manager (pending merge to develop) + ref: 9384a5ef8ef95648cb01ca1a2d1d43f3e6267522 # develop (with NFS cache-manager) file: /scripts/ci-helpers/prepare_data_image_job.yml .lint_job: -- GitLab From e850c4c3ac8651e12ed9e3be660d27daf142593f Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Sat, 6 Dec 2025 22:40:38 -0500 Subject: [PATCH 03/50] Revert "Update HAF ref to merged develop commit" This reverts commit 2233afd2b3f69f40075a3291321adf6b01f0769a. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fd80bb8..1e24aa0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -29,7 +29,7 @@ variables: include: - template: Workflows/Branch-Pipelines.gitlab-ci.yml - project: hive/haf - ref: 9384a5ef8ef95648cb01ca1a2d1d43f3e6267522 # develop (with NFS cache-manager) + ref: 68a06608fe308afc05c994b8e1c16564d34e8fac # feature/nfs-cache-manager (pending merge to develop) file: /scripts/ci-helpers/prepare_data_image_job.yml .lint_job: -- GitLab From 5f135586718e280f819d1a1a65af6eca16defd65 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Sat, 6 Dec 2025 22:46:14 -0500 Subject: [PATCH 04/50] Update HAF ref to merged develop commit --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1e24aa0..5e3dd06 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -29,7 +29,7 @@ variables: include: - template: Workflows/Branch-Pipelines.gitlab-ci.yml - project: hive/haf - ref: 68a06608fe308afc05c994b8e1c16564d34e8fac # feature/nfs-cache-manager (pending merge to develop) + ref: e46ca9526605b452c24558a4180792634e52255d # develop (with NFS cache-manager) file: /scripts/ci-helpers/prepare_data_image_job.yml .lint_job: -- GitLab From bfc78903b096a30107f2f8a89883be162e7164de Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Sun, 7 Dec 2025 00:58:41 -0500 Subject: [PATCH 05/50] Add NFS sync cache for cross-builder test execution - Change sync cache key from CI_PIPELINE_ID to CI_COMMIT_SHORT_SHA - Push sync data to NFS via cache-manager after sync job completes - Add before_script to test jobs to fetch from NFS if local cache missing - Enables tests to run on any builder, not just the one that ran sync --- .gitlab-ci.yml | 69 ++++++++++++++++++++++++++++++++++++++++++-------- haf | 2 +- 2 files changed, 60 insertions(+), 11 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5e3dd06..0712eb8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,6 +20,10 @@ variables: GIT_SUBMODULE_UPDATE_FLAGS: --jobs 4 # HAF configuration DATA_CACHE_HAF_PREFIX: "/cache/replay_data_haf" + # NFS cache configuration for sync data sharing across builders + CACHE_NFS_PATH: "/nfs/ci-cache" + SYNC_CACHE_KEY: "${HAF_COMMIT}_${CI_COMMIT_SHORT_SHA}" + SYNC_CACHE_TYPE: "haf_sync" BLOCK_LOG_SOURCE_DIR_5M: /blockchain/block_log_5m FF_NETWORK_PER_BUILD: 1 # uses registry.gitlab.syncad.com/hive/haf/ci-base-image:ubuntu24.04-1 @@ -275,18 +279,29 @@ sync: cp -a "${CI_PROJECT_DIR}/docker/blockchain/block_log" "${DATADIR}/blockchain/block_log" cp -a "${CI_PROJECT_DIR}/docker/blockchain/block_log.artifacts" "${DATADIR}/blockchain/block_log.artifacts" - mkdir -p "${DATA_CACHE_HAF_PREFIX}_${HAF_COMMIT}_${CI_PIPELINE_ID}" - sudo cp -a "${DATADIR}" "${DATA_CACHE_HAF_PREFIX}_${HAF_COMMIT}_${CI_PIPELINE_ID}" + # Save sync data to local cache with commit-based key (not pipeline ID) + LOCAL_SYNC_CACHE="${DATA_CACHE_HAF_PREFIX}_${SYNC_CACHE_KEY}" + mkdir -p "${LOCAL_SYNC_CACHE}" + sudo cp -a "${DATADIR}" "${LOCAL_SYNC_CACHE}" ls -lah "${DATADIR}" ls -lah "${DATADIR}/blockchain" ls -lah "${DATADIR}/shm_dir" - ls -lah "${DATA_CACHE_HAF_PREFIX}_${HAF_COMMIT}_${CI_PIPELINE_ID}" - ls -lah "${DATA_CACHE_HAF_PREFIX}_${HAF_COMMIT}_${CI_PIPELINE_ID}/blockchain" - ls -lah "${DATA_CACHE_HAF_PREFIX}_${HAF_COMMIT}_${CI_PIPELINE_ID}/shm_dir" + ls -lah "${LOCAL_SYNC_CACHE}" + ls -lah "${LOCAL_SYNC_CACHE}/datadir/blockchain" || ls -lah "${LOCAL_SYNC_CACHE}/blockchain" || true + ls -lah "${LOCAL_SYNC_CACHE}/datadir/shm_dir" || ls -lah "${LOCAL_SYNC_CACHE}/shm_dir" || true - # Manually remove the copy of the repaly data to preserve disk space on the replay server + # Push sync data to NFS cache for sharing across builders + CACHE_MANAGER="${CI_PROJECT_DIR}/haf/scripts/ci-helpers/cache-manager.sh" + if [[ -x "$CACHE_MANAGER" ]]; then + echo "Pushing sync data to NFS cache: ${SYNC_CACHE_TYPE}/${SYNC_CACHE_KEY}" + "$CACHE_MANAGER" put "${SYNC_CACHE_TYPE}" "${SYNC_CACHE_KEY}" "${LOCAL_SYNC_CACHE}" || echo "Warning: Failed to push to NFS cache" + else + echo "Warning: cache-manager.sh not found, skipping NFS cache push" + fi + + # Manually remove the copy of the replay data to preserve disk space on the replay server sudo rm -rf ${CI_PROJECT_DIR}/${CI_JOB_ID} echo -e "\e[0Ksection_end:$(date +%s):compose2\r\e[0K" @@ -299,6 +314,36 @@ sync: - data-cache-storage - fast +# Ensure sync data is available locally (fetch from NFS if needed) +.ensure-sync-data: &ensure-sync-data +- | + LOCAL_SYNC_CACHE="${DATA_CACHE_HAF_PREFIX}_${SYNC_CACHE_KEY}" + if [[ -d "${LOCAL_SYNC_CACHE}/datadir" ]]; then + echo "Local sync cache found at ${LOCAL_SYNC_CACHE}" + else + echo "Local sync cache not found, checking NFS..." + CACHE_MANAGER="${CI_PROJECT_DIR}/haf/scripts/ci-helpers/cache-manager.sh" + NFS_CACHE_PATH="${CACHE_NFS_PATH}/${SYNC_CACHE_TYPE}/${SYNC_CACHE_KEY}" + if [[ -x "$CACHE_MANAGER" ]]; then + if "$CACHE_MANAGER" get "${SYNC_CACHE_TYPE}" "${SYNC_CACHE_KEY}" "${LOCAL_SYNC_CACHE}"; then + echo "Fetched sync data from NFS cache via cache-manager" + else + echo "ERROR: Failed to fetch sync data from NFS cache" + exit 1 + fi + elif [[ -d "$NFS_CACHE_PATH" ]]; then + echo "Using direct NFS copy (cache-manager not available)" + mkdir -p "${LOCAL_SYNC_CACHE}" + cp -a "${NFS_CACHE_PATH}"/* "${LOCAL_SYNC_CACHE}/" || rsync -a "${NFS_CACHE_PATH}/" "${LOCAL_SYNC_CACHE}/" + echo "Fetched sync data from NFS cache via direct copy" + else + echo "ERROR: No sync data available locally or on NFS" + echo " Local path checked: ${LOCAL_SYNC_CACHE}" + echo " NFS path checked: ${NFS_CACHE_PATH}" + exit 1 + fi + fi + .hfm-only-service: &hfm-only-service name: $HAF_IMAGE_NAME alias: hfm-only-instance @@ -353,7 +398,8 @@ regression-test: services: - *hfm-only-service variables: - DATA_SOURCE: ${DATA_CACHE_HAF_PREFIX}_${HAF_COMMIT}_${CI_PIPELINE_ID} + DATA_SOURCE: ${DATA_CACHE_HAF_PREFIX}_${SYNC_CACHE_KEY} + before_script: *ensure-sync-data script: - | echo -e "\e[0Ksection_start:$(date +%s):tests\r\e[0KRunning tests..." @@ -383,7 +429,8 @@ setup-scripts-test: services: - *hfm-only-service variables: - DATA_SOURCE: ${DATA_CACHE_HAF_PREFIX}_${HAF_COMMIT}_${CI_PIPELINE_ID} + DATA_SOURCE: ${DATA_CACHE_HAF_PREFIX}_${SYNC_CACHE_KEY} + before_script: *ensure-sync-data script: - | echo -e "\e[0Ksection_start:$(date +%s):tests\r\e[0KRunning tests..." @@ -410,7 +457,8 @@ performance-test: - *hfm-only-service - *postgrest-service variables: - DATA_SOURCE: ${DATA_CACHE_HAF_PREFIX}_${HAF_COMMIT}_${CI_PIPELINE_ID} + DATA_SOURCE: ${DATA_CACHE_HAF_PREFIX}_${SYNC_CACHE_KEY} + before_script: *ensure-sync-data script: - | echo -e "\e[0Ksection_start:$(date +%s):tests\r\e[0KRunning tests..." @@ -447,13 +495,14 @@ pattern-test: - *hfm-only-service - *postgrest-service variables: - DATA_SOURCE: ${DATA_CACHE_HAF_PREFIX}_${HAF_COMMIT}_${CI_PIPELINE_ID} + DATA_SOURCE: ${DATA_CACHE_HAF_PREFIX}_${SYNC_CACHE_KEY} JUNIT_REPORT: $CI_PROJECT_DIR/tests/tavern/report.xml PYTEST_BASED_IMAGE_NAME: $BUILDER_IMAGE_PATH POETRY_INSTALL_ROOT_DIR: $CI_PROJECT_DIR/haf/hive/tests/python/hive-local-tools REPTRACKER_ADDRESS: postgrest-server REPTRACKER_PORT: 3000 TAVERN_DIR: $CI_PROJECT_DIR/tests/tavern + before_script: *ensure-sync-data script: - | cd $CI_PROJECT_DIR/tests/tavern diff --git a/haf b/haf index bf82044..e46ca95 160000 --- a/haf +++ b/haf @@ -1 +1 @@ -Subproject commit bf820442979eff6c7cb7e387f26cd4ccf9345f3c +Subproject commit e46ca9526605b452c24558a4180792634e52255d -- GitLab From 91ed71dccd24eed679d5e76f32cff67c9d1450c0 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Sun, 7 Dec 2025 03:12:30 -0500 Subject: [PATCH 06/50] Fix pattern-test: don't override before_script The before_script: *ensure-sync-data was replacing the template's before_script that sets up venv and installs pytest via poetry. Move sync check to script section instead. --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0712eb8..657c87b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -502,8 +502,9 @@ pattern-test: REPTRACKER_ADDRESS: postgrest-server REPTRACKER_PORT: 3000 TAVERN_DIR: $CI_PROJECT_DIR/tests/tavern - before_script: *ensure-sync-data + # Note: Don't override before_script - template's poetry install runs pytest setup script: + - *ensure-sync-data - | cd $CI_PROJECT_DIR/tests/tavern pytest -n $PYTEST_NUMBER_OF_PROCESSES --junitxml report.xml . -- GitLab From 1376fb759f8ada072b22020b7dd74e5a3e491c41 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Sun, 7 Dec 2025 03:23:15 -0500 Subject: [PATCH 07/50] Update HAF submodule to include BusyBox flock fix --- haf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/haf b/haf index e46ca95..1e8fe7b 160000 --- a/haf +++ b/haf @@ -1 +1 @@ -Subproject commit e46ca9526605b452c24558a4180792634e52255d +Subproject commit 1e8fe7b1f587bac68f3184fc82afcefcf03834df -- GitLab From 62b4030466588d328a1ad28f5940ae5a7500845c Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Sun, 7 Dec 2025 03:56:46 -0500 Subject: [PATCH 08/50] Use NFS path directly for test jobs DATA_SOURCE Services start before scripts run, so ensure-sync-data can't fetch data in time. By pointing DATA_SOURCE to the NFS path directly, services can access the sync data that was pushed by the sync job. Changed jobs: - regression-test - setup-scripts-test - performance-test - pattern-test All now use: ${CACHE_NFS_PATH}/${SYNC_CACHE_TYPE}/${SYNC_CACHE_KEY} --- .gitlab-ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 657c87b..25d20c3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -398,8 +398,8 @@ regression-test: services: - *hfm-only-service variables: - DATA_SOURCE: ${DATA_CACHE_HAF_PREFIX}_${SYNC_CACHE_KEY} - before_script: *ensure-sync-data + # Use NFS path directly - services start before script, so data must already exist + DATA_SOURCE: ${CACHE_NFS_PATH}/${SYNC_CACHE_TYPE}/${SYNC_CACHE_KEY} script: - | echo -e "\e[0Ksection_start:$(date +%s):tests\r\e[0KRunning tests..." @@ -429,8 +429,8 @@ setup-scripts-test: services: - *hfm-only-service variables: - DATA_SOURCE: ${DATA_CACHE_HAF_PREFIX}_${SYNC_CACHE_KEY} - before_script: *ensure-sync-data + # Use NFS path directly - services start before script, so data must already exist + DATA_SOURCE: ${CACHE_NFS_PATH}/${SYNC_CACHE_TYPE}/${SYNC_CACHE_KEY} script: - | echo -e "\e[0Ksection_start:$(date +%s):tests\r\e[0KRunning tests..." @@ -457,8 +457,8 @@ performance-test: - *hfm-only-service - *postgrest-service variables: - DATA_SOURCE: ${DATA_CACHE_HAF_PREFIX}_${SYNC_CACHE_KEY} - before_script: *ensure-sync-data + # Use NFS path directly - services start before script, so data must already exist + DATA_SOURCE: ${CACHE_NFS_PATH}/${SYNC_CACHE_TYPE}/${SYNC_CACHE_KEY} script: - | echo -e "\e[0Ksection_start:$(date +%s):tests\r\e[0KRunning tests..." @@ -495,7 +495,8 @@ pattern-test: - *hfm-only-service - *postgrest-service variables: - DATA_SOURCE: ${DATA_CACHE_HAF_PREFIX}_${SYNC_CACHE_KEY} + # Use NFS path directly - services start before script, so data must already exist + DATA_SOURCE: ${CACHE_NFS_PATH}/${SYNC_CACHE_TYPE}/${SYNC_CACHE_KEY} JUNIT_REPORT: $CI_PROJECT_DIR/tests/tavern/report.xml PYTEST_BASED_IMAGE_NAME: $BUILDER_IMAGE_PATH POETRY_INSTALL_ROOT_DIR: $CI_PROJECT_DIR/haf/hive/tests/python/hive-local-tools @@ -504,7 +505,6 @@ pattern-test: TAVERN_DIR: $CI_PROJECT_DIR/tests/tavern # Note: Don't override before_script - template's poetry install runs pytest setup script: - - *ensure-sync-data - | cd $CI_PROJECT_DIR/tests/tavern pytest -n $PYTEST_NUMBER_OF_PROCESSES --junitxml report.xml . -- GitLab From 74b0aa5496c7429fda0a2040f508c3413b5d2dec Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Sun, 7 Dec 2025 11:04:03 -0500 Subject: [PATCH 09/50] Update HAF submodule to 8e6a5b98 with 1G shared-file-size Uses HAF commit with properly sized shared memory file (1G vs 8G) which matches the NFS cache. --- haf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/haf b/haf index 1e8fe7b..8e6a5b9 160000 --- a/haf +++ b/haf @@ -1 +1 @@ -Subproject commit 1e8fe7b1f587bac68f3184fc82afcefcf03834df +Subproject commit 8e6a5b9804f218159598a5684ba0d7350d291e1c -- GitLab From 190597a245b29bba98e8a0d63faf1297477c32cc Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Sun, 7 Dec 2025 11:22:34 -0500 Subject: [PATCH 10/50] Fix sync job to fetch HAF data from NFS when local cache missing Add before_script to sync job to fetch HAF replay data from NFS if local cache is not available (supports cross-builder execution) --- .gitlab-ci.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 25d20c3..0f2f016 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -242,6 +242,26 @@ sync: git config --global --add safe.directory "$CI_PROJECT_DIR" git config --global --add safe.directory "$CI_PROJECT_DIR/haf" echo -e "\e[0Ksection_end:$(date +%s):git\r\e[0K" + - | + # Ensure HAF replay data is available locally (fetch from NFS if needed) + LOCAL_HAF_CACHE="${DATA_CACHE_HAF_PREFIX}_${HAF_COMMIT}" + if [[ -d "${LOCAL_HAF_CACHE}/datadir" ]]; then + echo "Local HAF cache found at ${LOCAL_HAF_CACHE}" + else + echo "Local HAF cache not found, checking NFS..." + CACHE_MANAGER="${CI_PROJECT_DIR}/haf/scripts/ci-helpers/cache-manager.sh" + if [[ -x "$CACHE_MANAGER" ]]; then + if "$CACHE_MANAGER" get haf "${HAF_COMMIT}" "${LOCAL_HAF_CACHE}"; then + echo "Fetched HAF replay data from NFS cache" + else + echo "ERROR: Failed to fetch HAF replay data from NFS cache" + exit 1 + fi + else + echo "ERROR: cache-manager.sh not found and local cache missing" + exit 1 + fi + fi script: - | echo -e "\e[0Ksection_start:$(date +%s):compose[collapsed=true]\r\e[0KStarting the test environment..." -- GitLab From 1bfaddaae454d4fb624bf5cf17906d247faa53d8 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Sun, 7 Dec 2025 17:17:19 -0500 Subject: [PATCH 11/50] Add NFS fallback for test jobs, update HAF submodule - Add HAF_COMMIT global variable for service containers - Add validate_haf_commit job to check HAF_COMMIT matches submodule/include ref - Add haf-instance-with-nfs-fallback service with NFS fallback in entrypoint - Update test jobs (regression-test, setup-scripts-test, performance-test, pattern-test) to use NFS-fallback service - Update HAF submodule to e46ca95 (matching HAfAH) --- .gitlab-ci.yml | 142 ++++++++++++++++++++++++++++++------------------- haf | 2 +- 2 files changed, 88 insertions(+), 56 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0f2f016..8327b7d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,7 +21,7 @@ variables: # HAF configuration DATA_CACHE_HAF_PREFIX: "/cache/replay_data_haf" # NFS cache configuration for sync data sharing across builders - CACHE_NFS_PATH: "/nfs/ci-cache" + DATA_CACHE_NFS_PREFIX: "/nfs/ci-cache" SYNC_CACHE_KEY: "${HAF_COMMIT}_${CI_COMMIT_SHORT_SHA}" SYNC_CACHE_TYPE: "haf_sync" BLOCK_LOG_SOURCE_DIR_5M: /blockchain/block_log_5m @@ -29,6 +29,9 @@ variables: # uses registry.gitlab.syncad.com/hive/haf/ci-base-image:ubuntu24.04-1 BUILDER_IMAGE_TAG: "$TEST_HAF_IMAGE_TAG" BUILDER_IMAGE_PATH: "registry.gitlab.syncad.com/hive/haf/ci-base-image${BUILDER_IMAGE_TAG}" + # HAF submodule commit - must match the 'ref:' in the include section below + # This is needed for service containers which can't access dotenv artifacts + HAF_COMMIT: "e46ca9526605b452c24558a4180792634e52255d" include: - template: Workflows/Branch-Pipelines.gitlab-ci.yml @@ -74,6 +77,40 @@ lint_sql_scripts: paths: - sql-lint.yaml +validate_haf_commit: + stage: build + image: alpine:latest + script: + - | + # Validate that HAF_COMMIT variable matches both the submodule and include ref + # This prevents cache misses due to mismatched commits + SUBMODULE_COMMIT=$(cat .git/modules/haf/HEAD 2>/dev/null || git -C haf rev-parse HEAD) + INCLUDE_REF=$(grep -A2 "project:.*hive/haf" .gitlab-ci.yml | grep "ref:" | head -1 | sed 's/.*ref: *\([a-f0-9]*\).*/\1/') + + echo "HAF_COMMIT variable: $HAF_COMMIT" + echo "HAF submodule HEAD: $SUBMODULE_COMMIT" + echo "Include ref: $INCLUDE_REF" + + ERRORS=0 + if [ "$HAF_COMMIT" != "$SUBMODULE_COMMIT" ]; then + echo "ERROR: HAF_COMMIT variable does not match submodule commit!" + echo " Update HAF_COMMIT in .gitlab-ci.yml to: $SUBMODULE_COMMIT" + ERRORS=1 + fi + if [ "$HAF_COMMIT" != "$INCLUDE_REF" ]; then + echo "ERROR: HAF_COMMIT variable does not match include ref!" + echo " Both should be: $HAF_COMMIT" + ERRORS=1 + fi + if [ $ERRORS -eq 1 ]; then + echo "" + echo "To fix: ensure HAF_COMMIT, include ref, and submodule all use the same commit" + exit 1 + fi + echo "All HAF commit references are consistent" + tags: + - public-runner-docker + prepare_haf_image: stage: build extends: .prepare_haf_image @@ -334,47 +371,54 @@ sync: - data-cache-storage - fast -# Ensure sync data is available locally (fetch from NFS if needed) -.ensure-sync-data: &ensure-sync-data -- | - LOCAL_SYNC_CACHE="${DATA_CACHE_HAF_PREFIX}_${SYNC_CACHE_KEY}" - if [[ -d "${LOCAL_SYNC_CACHE}/datadir" ]]; then - echo "Local sync cache found at ${LOCAL_SYNC_CACHE}" - else - echo "Local sync cache not found, checking NFS..." - CACHE_MANAGER="${CI_PROJECT_DIR}/haf/scripts/ci-helpers/cache-manager.sh" - NFS_CACHE_PATH="${CACHE_NFS_PATH}/${SYNC_CACHE_TYPE}/${SYNC_CACHE_KEY}" - if [[ -x "$CACHE_MANAGER" ]]; then - if "$CACHE_MANAGER" get "${SYNC_CACHE_TYPE}" "${SYNC_CACHE_KEY}" "${LOCAL_SYNC_CACHE}"; then - echo "Fetched sync data from NFS cache via cache-manager" - else - echo "ERROR: Failed to fetch sync data from NFS cache" - exit 1 - fi - elif [[ -d "$NFS_CACHE_PATH" ]]; then - echo "Using direct NFS copy (cache-manager not available)" - mkdir -p "${LOCAL_SYNC_CACHE}" - cp -a "${NFS_CACHE_PATH}"/* "${LOCAL_SYNC_CACHE}/" || rsync -a "${NFS_CACHE_PATH}/" "${LOCAL_SYNC_CACHE}/" - echo "Fetched sync data from NFS cache via direct copy" - else - echo "ERROR: No sync data available locally or on NFS" - echo " Local path checked: ${LOCAL_SYNC_CACHE}" - echo " NFS path checked: ${NFS_CACHE_PATH}" - exit 1 - fi - fi - -.hfm-only-service: &hfm-only-service - name: $HAF_IMAGE_NAME - alias: hfm-only-instance +# HAF instance with NFS fallback for sync data +# Services start before before_script, so we need fallback logic in the entrypoint +.haf-instance-with-nfs-fallback: &haf-instance-with-nfs-fallback + name: ${HAF_IMAGE_NAME} + alias: haf-instance variables: - PGCTLTIMEOUT: 600 # give PostgreSQL more time to start if GitLab shut it down improperly after the sync job + PGCTLTIMEOUT: 600 PG_ACCESS: | "host all haf_admin 0.0.0.0/0 trust" "host all hived 0.0.0.0/0 trust" "host all reptracker_user 0.0.0.0/0 trust" "host all reptracker_owner 0.0.0.0/0 trust" "host all all 0.0.0.0/0 scram-sha-256" + DATA_SOURCE: "${DATA_CACHE_HAF_PREFIX}_${SYNC_CACHE_KEY}" + DATA_SOURCE_NFS_PREFIX: "${DATA_CACHE_NFS_PREFIX}" + DATA_SOURCE_NFS_TYPE: "${SYNC_CACHE_TYPE}" + DATA_SOURCE_NFS_KEY: "${SYNC_CACHE_KEY}" + entrypoint: + - '/bin/bash' + - '-c' + - | + set -xeuo pipefail + echo "Checking data source availability..." + ORIGINAL_SOURCE="${DATA_SOURCE}" + NFS_PREFIX="${DATA_SOURCE_NFS_PREFIX:-/nfs/ci-cache}" + NFS_TYPE="${DATA_SOURCE_NFS_TYPE:-haf_sync}" + NFS_KEY="${DATA_SOURCE_NFS_KEY}" + + # If original path doesn't exist, try NFS fallback + if [[ ! -d "${ORIGINAL_SOURCE}/datadir" ]]; then + echo "Local cache not found: ${ORIGINAL_SOURCE}/datadir" + NFS_PATH="${NFS_PREFIX}/${NFS_TYPE}/${NFS_KEY}" + + if [[ -d "${NFS_PATH}/datadir" ]]; then + echo "Found data on NFS: $NFS_PATH" + export DATA_SOURCE="$NFS_PATH" + else + echo "WARNING: Data not found in local or NFS cache" + echo "Checked: ${ORIGINAL_SOURCE}/datadir" + echo "Checked: ${NFS_PATH}/datadir" + fi + else + echo "Using local cache: ${ORIGINAL_SOURCE}" + fi + + # Run original entrypoint + exec /home/haf_admin/docker_entrypoint.sh "$@" + - '/bin/bash' command: ["--execute-maintenance-script=${HAF_SOURCE_DIR}/scripts/maintenance-scripts/sleep_infinity.sh"] .postgrest-service: &postgrest-service @@ -383,8 +427,8 @@ sync: variables: PGRST_ADMIN_SERVER_PORT: 3001 PGRST_SERVER_PORT: 3000 - # Pointing to the PostgreSQL service running in hfm-only-instance - PGRST_DB_URI: postgresql://haf_admin@hfm-only-instance:5432/haf_block_log + # Pointing to the PostgreSQL service running in haf-instance + PGRST_DB_URI: postgresql://haf_admin@haf-instance:5432/haf_block_log PGRST_DB_SCHEMA: reptracker_endpoints PGRST_DB_ANON_ROLE: reptracker_user PGRST_DB_POOL: 20 @@ -416,16 +460,13 @@ regression-test: - job: prepare_haf_image artifacts: true services: - - *hfm-only-service - variables: - # Use NFS path directly - services start before script, so data must already exist - DATA_SOURCE: ${CACHE_NFS_PATH}/${SYNC_CACHE_TYPE}/${SYNC_CACHE_KEY} + - *haf-instance-with-nfs-fallback script: - | echo -e "\e[0Ksection_start:$(date +%s):tests\r\e[0KRunning tests..." cd tests - ./account_dump_test.sh --host=hfm-only-instance + ./account_dump_test.sh --host=haf-instance echo -e "\e[0Ksection_end:$(date +%s):tests\r\e[0K" artifacts: @@ -447,16 +488,13 @@ setup-scripts-test: - job: prepare_haf_image artifacts: true services: - - *hfm-only-service - variables: - # Use NFS path directly - services start before script, so data must already exist - DATA_SOURCE: ${CACHE_NFS_PATH}/${SYNC_CACHE_TYPE}/${SYNC_CACHE_KEY} + - *haf-instance-with-nfs-fallback script: - | echo -e "\e[0Ksection_start:$(date +%s):tests\r\e[0KRunning tests..." cd tests/functional - ./test_scripts.sh --host=hfm-only-instance + ./test_scripts.sh --host=haf-instance echo -e "\e[0Ksection_end:$(date +%s):tests\r\e[0K" tags: @@ -474,11 +512,8 @@ performance-test: - job: prepare_haf_image artifacts: true services: - - *hfm-only-service + - *haf-instance-with-nfs-fallback - *postgrest-service - variables: - # Use NFS path directly - services start before script, so data must already exist - DATA_SOURCE: ${CACHE_NFS_PATH}/${SYNC_CACHE_TYPE}/${SYNC_CACHE_KEY} script: - | echo -e "\e[0Ksection_start:$(date +%s):tests\r\e[0KRunning tests..." @@ -512,18 +547,15 @@ pattern-test: - job: prepare_haf_image artifacts: true services: - - *hfm-only-service + - *haf-instance-with-nfs-fallback - *postgrest-service variables: - # Use NFS path directly - services start before script, so data must already exist - DATA_SOURCE: ${CACHE_NFS_PATH}/${SYNC_CACHE_TYPE}/${SYNC_CACHE_KEY} JUNIT_REPORT: $CI_PROJECT_DIR/tests/tavern/report.xml PYTEST_BASED_IMAGE_NAME: $BUILDER_IMAGE_PATH POETRY_INSTALL_ROOT_DIR: $CI_PROJECT_DIR/haf/hive/tests/python/hive-local-tools REPTRACKER_ADDRESS: postgrest-server REPTRACKER_PORT: 3000 TAVERN_DIR: $CI_PROJECT_DIR/tests/tavern - # Note: Don't override before_script - template's poetry install runs pytest setup script: - | cd $CI_PROJECT_DIR/tests/tavern diff --git a/haf b/haf index 8e6a5b9..e46ca95 160000 --- a/haf +++ b/haf @@ -1 +1 @@ -Subproject commit 8e6a5b9804f218159598a5684ba0d7350d291e1c +Subproject commit e46ca9526605b452c24558a4180792634e52255d -- GitLab From 636f4021485c1582cda350d28604d7c603d30925 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Sun, 7 Dec 2025 17:59:48 -0500 Subject: [PATCH 12/50] Pin sync and pattern-test to builder-9 for local cache access Pattern-test uses haf-instance service which copies data at startup. NFS copy exceeds 300s health check timeout, so both jobs must run on the same builder to use local cache. --- .gitlab-ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8327b7d..d6d21bb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -369,7 +369,8 @@ sync: when: always tags: - data-cache-storage - - fast + # Pin to builder-9 so pattern-test (also on builder-9) can use local cache + - hive-builder-9 # HAF instance with NFS fallback for sync data # Services start before before_script, so we need fallback logic in the entrypoint @@ -565,7 +566,8 @@ pattern-test: - "**/*.out.json" tags: - data-cache-storage - - fast + # Pin to same builder as sync to avoid NFS copy timeout during service startup + - hive-builder-9 deploy_python_api_packages_to_gitlab: stage: publish -- GitLab From e8201c0cf895a6358335002b5fbad02fd710105c Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Sun, 7 Dec 2025 18:33:28 -0500 Subject: [PATCH 13/50] Update HAF submodule with BusyBox flock fix --- .gitlab-ci.yml | 4 ++-- haf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d6d21bb..c140ee7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,12 +31,12 @@ variables: BUILDER_IMAGE_PATH: "registry.gitlab.syncad.com/hive/haf/ci-base-image${BUILDER_IMAGE_TAG}" # HAF submodule commit - must match the 'ref:' in the include section below # This is needed for service containers which can't access dotenv artifacts - HAF_COMMIT: "e46ca9526605b452c24558a4180792634e52255d" + HAF_COMMIT: "7a753d87a712ba333705b7732a76a151cd0d44ae" include: - template: Workflows/Branch-Pipelines.gitlab-ci.yml - project: hive/haf - ref: e46ca9526605b452c24558a4180792634e52255d # develop (with NFS cache-manager) + ref: 7a753d87a712ba333705b7732a76a151cd0d44ae # develop (with NFS cache-manager) file: /scripts/ci-helpers/prepare_data_image_job.yml .lint_job: diff --git a/haf b/haf index e46ca95..7a753d8 160000 --- a/haf +++ b/haf @@ -1 +1 @@ -Subproject commit e46ca9526605b452c24558a4180792634e52255d +Subproject commit 7a753d87a712ba333705b7732a76a151cd0d44ae -- GitLab From b94b21eb5c55dbfe50aec2c651802dad4a924470 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Sun, 7 Dec 2025 19:57:43 -0500 Subject: [PATCH 14/50] Update haf submodule: rename runner tags haf-build-* to build-* --- haf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/haf b/haf index 7a753d8..bfb9b34 160000 --- a/haf +++ b/haf @@ -1 +1 @@ -Subproject commit 7a753d87a712ba333705b7732a76a151cd0d44ae +Subproject commit bfb9b34590a9bf166d179c2ccaec5cbee81bc55e -- GitLab From ca531f04a5454b4a476369ec0a605deaf4482de6 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Mon, 8 Dec 2025 00:29:05 -0500 Subject: [PATCH 15/50] Update HAF submodule to 0457b5ff with chmod fix for NFS caching --- .gitlab-ci.yml | 4 ++-- haf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c140ee7..284ed91 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,12 +31,12 @@ variables: BUILDER_IMAGE_PATH: "registry.gitlab.syncad.com/hive/haf/ci-base-image${BUILDER_IMAGE_TAG}" # HAF submodule commit - must match the 'ref:' in the include section below # This is needed for service containers which can't access dotenv artifacts - HAF_COMMIT: "7a753d87a712ba333705b7732a76a151cd0d44ae" + HAF_COMMIT: "0457b5ffebdff0cf9067b8a770af9512c6a869c9" include: - template: Workflows/Branch-Pipelines.gitlab-ci.yml - project: hive/haf - ref: 7a753d87a712ba333705b7732a76a151cd0d44ae # develop (with NFS cache-manager) + ref: 0457b5ffebdff0cf9067b8a770af9512c6a869c9 # feature/nfs-cache-manager (with chmod fix) file: /scripts/ci-helpers/prepare_data_image_job.yml .lint_job: diff --git a/haf b/haf index bfb9b34..0457b5f 160000 --- a/haf +++ b/haf @@ -1 +1 @@ -Subproject commit bfb9b34590a9bf166d179c2ccaec5cbee81bc55e +Subproject commit 0457b5ffebdff0cf9067b8a770af9512c6a869c9 -- GitLab From febee2635b4fed31e7e297dc0b14e217e52e182b Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Mon, 8 Dec 2025 01:03:07 -0500 Subject: [PATCH 16/50] Update HAF submodule with subshell fix for build_data.sh exit --- .gitlab-ci.yml | 4 ++-- haf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 284ed91..70e41dd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,12 +31,12 @@ variables: BUILDER_IMAGE_PATH: "registry.gitlab.syncad.com/hive/haf/ci-base-image${BUILDER_IMAGE_TAG}" # HAF submodule commit - must match the 'ref:' in the include section below # This is needed for service containers which can't access dotenv artifacts - HAF_COMMIT: "0457b5ffebdff0cf9067b8a770af9512c6a869c9" + HAF_COMMIT: "5c8588d8b543cdbe22526aead91a60c3ecf46c4e" include: - template: Workflows/Branch-Pipelines.gitlab-ci.yml - project: hive/haf - ref: 0457b5ffebdff0cf9067b8a770af9512c6a869c9 # feature/nfs-cache-manager (with chmod fix) + ref: 5c8588d8b543cdbe22526aead91a60c3ecf46c4e # feature/nfs-cache-manager (with subshell fix) file: /scripts/ci-helpers/prepare_data_image_job.yml .lint_job: diff --git a/haf b/haf index 0457b5f..5c8588d 160000 --- a/haf +++ b/haf @@ -1 +1 @@ -Subproject commit 0457b5ffebdff0cf9067b8a770af9512c6a869c9 +Subproject commit 5c8588d8b543cdbe22526aead91a60c3ecf46c4e -- GitLab From 581a2fa5c2bb2891da0c45edf410d65eaa61335b Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Mon, 8 Dec 2025 01:10:13 -0500 Subject: [PATCH 17/50] Update HAF submodule with BusyBox flock fix --- .gitlab-ci.yml | 4 ++-- haf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 70e41dd..08ce87f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,12 +31,12 @@ variables: BUILDER_IMAGE_PATH: "registry.gitlab.syncad.com/hive/haf/ci-base-image${BUILDER_IMAGE_TAG}" # HAF submodule commit - must match the 'ref:' in the include section below # This is needed for service containers which can't access dotenv artifacts - HAF_COMMIT: "5c8588d8b543cdbe22526aead91a60c3ecf46c4e" + HAF_COMMIT: "d1fb5375e36ed6c64f11db94b094f86034752cbb" include: - template: Workflows/Branch-Pipelines.gitlab-ci.yml - project: hive/haf - ref: 5c8588d8b543cdbe22526aead91a60c3ecf46c4e # feature/nfs-cache-manager (with subshell fix) + ref: d1fb5375e36ed6c64f11db94b094f86034752cbb # feature/nfs-cache-manager (with subshell fix) file: /scripts/ci-helpers/prepare_data_image_job.yml .lint_job: diff --git a/haf b/haf index 5c8588d..d1fb537 160000 --- a/haf +++ b/haf @@ -1 +1 @@ -Subproject commit 5c8588d8b543cdbe22526aead91a60c3ecf46c4e +Subproject commit d1fb5375e36ed6c64f11db94b094f86034752cbb -- GitLab From 29b70df29806c0b7398342bf322ee72b1f175282 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Mon, 8 Dec 2025 01:53:50 -0500 Subject: [PATCH 18/50] Update HAF ref to a617806d with pgdata permission fix --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 08ce87f..8628bd7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,12 +31,12 @@ variables: BUILDER_IMAGE_PATH: "registry.gitlab.syncad.com/hive/haf/ci-base-image${BUILDER_IMAGE_TAG}" # HAF submodule commit - must match the 'ref:' in the include section below # This is needed for service containers which can't access dotenv artifacts - HAF_COMMIT: "d1fb5375e36ed6c64f11db94b094f86034752cbb" + HAF_COMMIT: "a617806da36c92cda2cc04f5c2cbba4780996220" include: - template: Workflows/Branch-Pipelines.gitlab-ci.yml - project: hive/haf - ref: d1fb5375e36ed6c64f11db94b094f86034752cbb # feature/nfs-cache-manager (with subshell fix) + ref: a617806da36c92cda2cc04f5c2cbba4780996220 # feature/nfs-cache-manager (with pgdata permission fix) file: /scripts/ci-helpers/prepare_data_image_job.yml .lint_job: -- GitLab From c1146279213cfe26c00fe7ac2bead0b13dfba2ab Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Mon, 8 Dec 2025 03:08:14 -0500 Subject: [PATCH 19/50] Update HAF submodule to 5808dd23 with replay_data_copy NFS fix --- .gitlab-ci.yml | 4 ++-- haf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8628bd7..bb4cab5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,12 +31,12 @@ variables: BUILDER_IMAGE_PATH: "registry.gitlab.syncad.com/hive/haf/ci-base-image${BUILDER_IMAGE_TAG}" # HAF submodule commit - must match the 'ref:' in the include section below # This is needed for service containers which can't access dotenv artifacts - HAF_COMMIT: "a617806da36c92cda2cc04f5c2cbba4780996220" + HAF_COMMIT: "5808dd2316a6055fb3c0ec85d5d250d5763efe15" include: - template: Workflows/Branch-Pipelines.gitlab-ci.yml - project: hive/haf - ref: a617806da36c92cda2cc04f5c2cbba4780996220 # feature/nfs-cache-manager (with pgdata permission fix) + ref: 5808dd2316a6055fb3c0ec85d5d250d5763efe15 # feature/nfs-cache-manager (with replay_data_copy NFS fix) file: /scripts/ci-helpers/prepare_data_image_job.yml .lint_job: diff --git a/haf b/haf index d1fb537..5808dd2 160000 --- a/haf +++ b/haf @@ -1 +1 @@ -Subproject commit d1fb5375e36ed6c64f11db94b094f86034752cbb +Subproject commit 5808dd2316a6055fb3c0ec85d5d250d5763efe15 -- GitLab From 45bbe249b68ded35465aecfd8f08df87c1530d82 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Mon, 8 Dec 2025 04:15:35 -0500 Subject: [PATCH 20/50] Update HAF submodule to e50db662 with block_log path fix --- .gitlab-ci.yml | 4 ++-- haf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bb4cab5..51f6aec 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,12 +31,12 @@ variables: BUILDER_IMAGE_PATH: "registry.gitlab.syncad.com/hive/haf/ci-base-image${BUILDER_IMAGE_TAG}" # HAF submodule commit - must match the 'ref:' in the include section below # This is needed for service containers which can't access dotenv artifacts - HAF_COMMIT: "5808dd2316a6055fb3c0ec85d5d250d5763efe15" + HAF_COMMIT: "e50db662040edc47eefdb103ca2f48414fe75046" include: - template: Workflows/Branch-Pipelines.gitlab-ci.yml - project: hive/haf - ref: 5808dd2316a6055fb3c0ec85d5d250d5763efe15 # feature/nfs-cache-manager (with replay_data_copy NFS fix) + ref: e50db662040edc47eefdb103ca2f48414fe75046 # feature/nfs-cache-manager (with block_log path fix) file: /scripts/ci-helpers/prepare_data_image_job.yml .lint_job: diff --git a/haf b/haf index 5808dd2..e50db66 160000 --- a/haf +++ b/haf @@ -1 +1 @@ -Subproject commit 5808dd2316a6055fb3c0ec85d5d250d5763efe15 +Subproject commit e50db662040edc47eefdb103ca2f48414fe75046 -- GitLab From b99bfbc7869778cd31afee5f1a3d8a44ec9ecad4 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Mon, 8 Dec 2025 09:48:12 -0500 Subject: [PATCH 21/50] Update HAF to 85fbba59 with pgdata ownership fix --- .gitlab-ci.yml | 4 ++-- haf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 51f6aec..0aa2882 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,12 +31,12 @@ variables: BUILDER_IMAGE_PATH: "registry.gitlab.syncad.com/hive/haf/ci-base-image${BUILDER_IMAGE_TAG}" # HAF submodule commit - must match the 'ref:' in the include section below # This is needed for service containers which can't access dotenv artifacts - HAF_COMMIT: "e50db662040edc47eefdb103ca2f48414fe75046" + HAF_COMMIT: "85fbba5957a9fcfd91a06165b1f4de11321bbbb5" include: - template: Workflows/Branch-Pipelines.gitlab-ci.yml - project: hive/haf - ref: e50db662040edc47eefdb103ca2f48414fe75046 # feature/nfs-cache-manager (with block_log path fix) + ref: 85fbba5957a9fcfd91a06165b1f4de11321bbbb5 # feature/nfs-cache-manager (with pgdata ownership fix) file: /scripts/ci-helpers/prepare_data_image_job.yml .lint_job: diff --git a/haf b/haf index e50db66..85fbba5 160000 --- a/haf +++ b/haf @@ -1 +1 @@ -Subproject commit e50db662040edc47eefdb103ca2f48414fe75046 +Subproject commit 85fbba5957a9fcfd91a06165b1f4de11321bbbb5 -- GitLab From 3cecac9d0f892ec5ecfa08b144238719d2974146 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Mon, 8 Dec 2025 17:29:37 -0500 Subject: [PATCH 22/50] Update HAF submodule and add 80m timeout for prepare_haf_data --- .gitlab-ci.yml | 5 +++-- haf | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0aa2882..03da69b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,12 +31,12 @@ variables: BUILDER_IMAGE_PATH: "registry.gitlab.syncad.com/hive/haf/ci-base-image${BUILDER_IMAGE_TAG}" # HAF submodule commit - must match the 'ref:' in the include section below # This is needed for service containers which can't access dotenv artifacts - HAF_COMMIT: "85fbba5957a9fcfd91a06165b1f4de11321bbbb5" + HAF_COMMIT: "97cc083e1aa96db494c81fc4e5e390702d5005e5" include: - template: Workflows/Branch-Pipelines.gitlab-ci.yml - project: hive/haf - ref: 85fbba5957a9fcfd91a06165b1f4de11321bbbb5 # feature/nfs-cache-manager (with pgdata ownership fix) + ref: 97cc083e1aa96db494c81fc4e5e390702d5005e5 # feature/nfs-cache-manager (80m timeout, spread-build-tags) file: /scripts/ci-helpers/prepare_data_image_job.yml .lint_job: @@ -181,6 +181,7 @@ prepare_haf_data: - job: prepare_haf_image artifacts: true stage: build + timeout: 80m variables: SUBMODULE_DIR: "$CI_PROJECT_DIR/haf" BLOCK_LOG_SOURCE_DIR: $BLOCK_LOG_SOURCE_DIR_5M diff --git a/haf b/haf index 85fbba5..97cc083 160000 --- a/haf +++ b/haf @@ -1 +1 @@ -Subproject commit 85fbba5957a9fcfd91a06165b1f4de11321bbbb5 +Subproject commit 97cc083e1e6823e08c221a2d1c9aaecccb094084 -- GitLab From 60ac07bde7b99792c7c4a13637f38b5399150dc9 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Mon, 8 Dec 2025 18:48:38 -0500 Subject: [PATCH 23/50] Update HAF to 85fbba59 with NFS cache fixes --- .gitlab-ci.yml | 4 ++-- haf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 03da69b..1d814dc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,12 +31,12 @@ variables: BUILDER_IMAGE_PATH: "registry.gitlab.syncad.com/hive/haf/ci-base-image${BUILDER_IMAGE_TAG}" # HAF submodule commit - must match the 'ref:' in the include section below # This is needed for service containers which can't access dotenv artifacts - HAF_COMMIT: "97cc083e1aa96db494c81fc4e5e390702d5005e5" + HAF_COMMIT: "85fbba5957a9fcfd91a06165b1f4de11321bbbb5" include: - template: Workflows/Branch-Pipelines.gitlab-ci.yml - project: hive/haf - ref: 97cc083e1aa96db494c81fc4e5e390702d5005e5 # feature/nfs-cache-manager (80m timeout, spread-build-tags) + ref: 85fbba5957a9fcfd91a06165b1f4de11321bbbb5 # feature/nfs-cache-manager file: /scripts/ci-helpers/prepare_data_image_job.yml .lint_job: diff --git a/haf b/haf index 97cc083..85fbba5 160000 --- a/haf +++ b/haf @@ -1 +1 @@ -Subproject commit 97cc083e1e6823e08c221a2d1c9aaecccb094084 +Subproject commit 85fbba5957a9fcfd91a06165b1f4de11321bbbb5 -- GitLab From da56bd050eb0a78086c34c460cc8a7e97817a194 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Mon, 8 Dec 2025 19:37:08 -0500 Subject: [PATCH 24/50] Update HAF submodule to 267c1abd with hive NFS fixes --- .gitlab-ci.yml | 4 ++-- haf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1d814dc..d95569e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,12 +31,12 @@ variables: BUILDER_IMAGE_PATH: "registry.gitlab.syncad.com/hive/haf/ci-base-image${BUILDER_IMAGE_TAG}" # HAF submodule commit - must match the 'ref:' in the include section below # This is needed for service containers which can't access dotenv artifacts - HAF_COMMIT: "85fbba5957a9fcfd91a06165b1f4de11321bbbb5" + HAF_COMMIT: "267c1abd75398fe8903f1e6eb3f5c2e596b9c28f" include: - template: Workflows/Branch-Pipelines.gitlab-ci.yml - project: hive/haf - ref: 85fbba5957a9fcfd91a06165b1f4de11321bbbb5 # feature/nfs-cache-manager + ref: 267c1abd75398fe8903f1e6eb3f5c2e596b9c28f # feature/nfs-cache-manager file: /scripts/ci-helpers/prepare_data_image_job.yml .lint_job: diff --git a/haf b/haf index 85fbba5..267c1ab 160000 --- a/haf +++ b/haf @@ -1 +1 @@ -Subproject commit 85fbba5957a9fcfd91a06165b1f4de11321bbbb5 +Subproject commit 267c1abd75398fe8903f1e6eb3f5c2e596b9c28f -- GitLab From d85a9b524f2a86eae096b294e15812c845f6f047 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Tue, 9 Dec 2025 01:02:39 -0500 Subject: [PATCH 25/50] Update HAF to df653a09 with pgdata permissions fix --- .gitlab-ci.yml | 12 +++++------- haf | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d95569e..d17ecd7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,12 +31,12 @@ variables: BUILDER_IMAGE_PATH: "registry.gitlab.syncad.com/hive/haf/ci-base-image${BUILDER_IMAGE_TAG}" # HAF submodule commit - must match the 'ref:' in the include section below # This is needed for service containers which can't access dotenv artifacts - HAF_COMMIT: "267c1abd75398fe8903f1e6eb3f5c2e596b9c28f" + HAF_COMMIT: "df653a09a8f16c8f96a852c40443635f4c93887d" include: - template: Workflows/Branch-Pipelines.gitlab-ci.yml - project: hive/haf - ref: 267c1abd75398fe8903f1e6eb3f5c2e596b9c28f # feature/nfs-cache-manager + ref: df653a09a8f16c8f96a852c40443635f4c93887d # feature/nfs-cache-manager file: /scripts/ci-helpers/prepare_data_image_job.yml .lint_job: @@ -122,8 +122,7 @@ prepare_haf_image: - git config --global --add safe.directory $CI_PROJECT_DIR/haf tags: - public-runner-docker - - hived-for-tests - - hived + - build-mainnet extract-swagger-json: extends: .filter_out_swagger_json @@ -230,8 +229,7 @@ prepare_haf_data: echo -e "\e[0Ksection_end:$(date +%s):build\r\e[0K" tags: - public-runner-docker - - hived-for-tests - - hived + - build-mainnet docker-ci-runner-build: extends: .docker-build-template @@ -599,7 +597,7 @@ build_and_publish_image: fi tags: - public-runner-docker - - hived-for-tests + - build-mainnet deploy-wax-spec-dev-package: extends: .npm_deploy_package_template diff --git a/haf b/haf index 267c1ab..df653a0 160000 --- a/haf +++ b/haf @@ -1 +1 @@ -Subproject commit 267c1abd75398fe8903f1e6eb3f5c2e596b9c28f +Subproject commit df653a09a8f16c8f96a852c40443635f4c93887d -- GitLab From 3625f6fe6cdd867cf94d00ae906d96408003d1d4 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Tue, 9 Dec 2025 01:34:49 -0500 Subject: [PATCH 26/50] Update HAF to e842bc87 (NFS-safe locking) --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d17ecd7..85fe9bf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,12 +31,12 @@ variables: BUILDER_IMAGE_PATH: "registry.gitlab.syncad.com/hive/haf/ci-base-image${BUILDER_IMAGE_TAG}" # HAF submodule commit - must match the 'ref:' in the include section below # This is needed for service containers which can't access dotenv artifacts - HAF_COMMIT: "df653a09a8f16c8f96a852c40443635f4c93887d" + HAF_COMMIT: "e842bc8773b4415077cb0fae028de9a03eb228ba" include: - template: Workflows/Branch-Pipelines.gitlab-ci.yml - project: hive/haf - ref: df653a09a8f16c8f96a852c40443635f4c93887d # feature/nfs-cache-manager + ref: e842bc8773b4415077cb0fae028de9a03eb228ba # feature/nfs-cache-manager file: /scripts/ci-helpers/prepare_data_image_job.yml .lint_job: -- GitLab From a4ea71453528a22cd4a14c2db07b1bed1ac70834 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Tue, 9 Dec 2025 01:35:42 -0500 Subject: [PATCH 27/50] Update HAF submodule to e842bc87 --- haf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/haf b/haf index df653a0..e842bc8 160000 --- a/haf +++ b/haf @@ -1 +1 @@ -Subproject commit df653a09a8f16c8f96a852c40443635f4c93887d +Subproject commit e842bc8773b4415077cb0fae028de9a03eb228ba -- GitLab From 815bc1bda34b53718eea916cd94e1b5f6f5d0aac Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Tue, 9 Dec 2025 02:17:00 -0500 Subject: [PATCH 28/50] Update HAF to 642330a5 (tablespace permissions fix) --- .gitlab-ci.yml | 4 ++-- haf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 85fe9bf..4154b43 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,12 +31,12 @@ variables: BUILDER_IMAGE_PATH: "registry.gitlab.syncad.com/hive/haf/ci-base-image${BUILDER_IMAGE_TAG}" # HAF submodule commit - must match the 'ref:' in the include section below # This is needed for service containers which can't access dotenv artifacts - HAF_COMMIT: "e842bc8773b4415077cb0fae028de9a03eb228ba" + HAF_COMMIT: "642330a54b44e7605423134e494eeaa40766399e" include: - template: Workflows/Branch-Pipelines.gitlab-ci.yml - project: hive/haf - ref: e842bc8773b4415077cb0fae028de9a03eb228ba # feature/nfs-cache-manager + ref: 642330a54b44e7605423134e494eeaa40766399e # feature/nfs-cache-manager file: /scripts/ci-helpers/prepare_data_image_job.yml .lint_job: diff --git a/haf b/haf index e842bc8..642330a 160000 --- a/haf +++ b/haf @@ -1 +1 @@ -Subproject commit e842bc8773b4415077cb0fae028de9a03eb228ba +Subproject commit 642330a54b44e7605423134e494eeaa40766399e -- GitLab From b56b681a4578d408e8e70c06b94ee24ceeedbd73 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Tue, 9 Dec 2025 04:00:27 -0500 Subject: [PATCH 29/50] Fix: Use explicit UID 105 for pgdata ownership in sync job The copy_datadir.sh script uses 'postgres:postgres' which resolves to the CI runner's postgres UID (70 on some systems). However, the HAF container expects postgres to be UID 105. This mismatch causes PostgreSQL to fail because it cannot read the pgdata directory. Fix by explicitly chowning haf_db_store to 105:109 after copy_datadir.sh runs, ensuring the HAF container's postgres user can access the data. --- .gitlab-ci.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4154b43..77b8548 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -308,6 +308,13 @@ sync: "${CI_PROJECT_DIR}/haf/scripts/copy_datadir.sh" + # Fix pgdata ownership - use explicit UID 105 (postgres in HAF container) + # The copy_datadir.sh uses 'postgres:postgres' which may resolve to different UIDs on different systems + if [[ -d "${DATADIR}/haf_db_store" ]]; then + echo "Fixing haf_db_store ownership to UID 105:109 (postgres in HAF container)" + sudo chown -R 105:109 "${DATADIR}/haf_db_store" + fi + "${CI_PROJECT_DIR}/scripts/ci-helpers/start-ci-test-environment.sh" echo -e "\e[0Ksection_end:$(date +%s):compose\r\e[0K" -- GitLab From 5bfdfc2a4c4ce7bcb55075bd100f389a62c36dfd Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Tue, 9 Dec 2025 04:24:03 -0500 Subject: [PATCH 30/50] Fix: Use sudo for copying files to datadir in sync after_script The blockchain directory permissions prevent the CI runner user from writing to it. Use sudo to copy the block_log and shm_dir files. --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 77b8548..870d8bb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -338,9 +338,9 @@ sync: tar -czvf docker/container-logs.tar.gz $(pwd)/docker/*.log - cp -a "${SHM_DIR}" "${DATADIR}/shm_dir" - cp -a "${CI_PROJECT_DIR}/docker/blockchain/block_log" "${DATADIR}/blockchain/block_log" - cp -a "${CI_PROJECT_DIR}/docker/blockchain/block_log.artifacts" "${DATADIR}/blockchain/block_log.artifacts" + sudo cp -a "${SHM_DIR}" "${DATADIR}/shm_dir" + sudo cp -a "${CI_PROJECT_DIR}/docker/blockchain/block_log" "${DATADIR}/blockchain/block_log" + sudo cp -a "${CI_PROJECT_DIR}/docker/blockchain/block_log.artifacts" "${DATADIR}/blockchain/block_log.artifacts" # Save sync data to local cache with commit-based key (not pipeline ID) LOCAL_SYNC_CACHE="${DATA_CACHE_HAF_PREFIX}_${SYNC_CACHE_KEY}" -- GitLab From 43a95bd47b23592204678caa027e4355aab13d42 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Tue, 9 Dec 2025 15:03:33 -0500 Subject: [PATCH 31/50] Update HAF to 465036e7 (fix pgdata ownership check) --- .gitlab-ci.yml | 4 ++-- haf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 870d8bb..ecfa5be 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,12 +31,12 @@ variables: BUILDER_IMAGE_PATH: "registry.gitlab.syncad.com/hive/haf/ci-base-image${BUILDER_IMAGE_TAG}" # HAF submodule commit - must match the 'ref:' in the include section below # This is needed for service containers which can't access dotenv artifacts - HAF_COMMIT: "642330a54b44e7605423134e494eeaa40766399e" + HAF_COMMIT: "465036e77ceb78cd71d1daeb377b6d0c0a21b857" include: - template: Workflows/Branch-Pipelines.gitlab-ci.yml - project: hive/haf - ref: 642330a54b44e7605423134e494eeaa40766399e # feature/nfs-cache-manager + ref: 465036e77ceb78cd71d1daeb377b6d0c0a21b857 # feature/nfs-cache-manager file: /scripts/ci-helpers/prepare_data_image_job.yml .lint_job: diff --git a/haf b/haf index 642330a..465036e 160000 --- a/haf +++ b/haf @@ -1 +1 @@ -Subproject commit 642330a54b44e7605423134e494eeaa40766399e +Subproject commit 465036e77ceb78cd71d1daeb377b6d0c0a21b857 -- GitLab From e3b5ed07898da078da3ea1af2dbfa664f152b8ca Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Tue, 9 Dec 2025 17:14:43 -0500 Subject: [PATCH 32/50] Replace hardcoded hive-builder-9 with flexible tags Use [data-cache-storage, fast] instead of pinning sync and pattern-test jobs to hive-builder-9. Jobs can now run on any builder with these tags and will use NFS fallback if local cache is not available. --- .gitlab-ci.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ecfa5be..f74da2a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -375,8 +375,7 @@ sync: when: always tags: - data-cache-storage - # Pin to builder-9 so pattern-test (also on builder-9) can use local cache - - hive-builder-9 + - fast # HAF instance with NFS fallback for sync data # Services start before before_script, so we need fallback logic in the entrypoint @@ -572,8 +571,7 @@ pattern-test: - "**/*.out.json" tags: - data-cache-storage - # Pin to same builder as sync to avoid NFS copy timeout during service startup - - hive-builder-9 + - fast deploy_python_api_packages_to_gitlab: stage: publish -- GitLab From 63690988d73d597c48986725cf176858cf0eb068 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Tue, 9 Dec 2025 20:07:57 -0500 Subject: [PATCH 33/50] Fix: Increase wait timeout from 10m to 30m for block processing --- scripts/ci-helpers/wait-for-rt-startup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci-helpers/wait-for-rt-startup.sh b/scripts/ci-helpers/wait-for-rt-startup.sh index 0ed84b7..f3a5cf3 100755 --- a/scripts/ci-helpers/wait-for-rt-startup.sh +++ b/scripts/ci-helpers/wait-for-rt-startup.sh @@ -60,6 +60,6 @@ export POSTGRES_ACCESS export COMMAND export MESSAGE -timeout -k 1m 10m bash -c wait-for-rt-startup +timeout -k 1m 30m bash -c wait-for-rt-startup echo "Block processing is finished." \ No newline at end of file -- GitLab From 69cd334fe1cc8ef23b90143ceb07e528c280c7ff Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Tue, 9 Dec 2025 20:47:05 -0500 Subject: [PATCH 34/50] Fix: Add pgdata permissions fix for PostgreSQL startup --- .gitlab-ci.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f74da2a..67b81e5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -308,11 +308,18 @@ sync: "${CI_PROJECT_DIR}/haf/scripts/copy_datadir.sh" - # Fix pgdata ownership - use explicit UID 105 (postgres in HAF container) + # Fix pgdata ownership and permissions - use explicit UID 105 (postgres in HAF container) # The copy_datadir.sh uses 'postgres:postgres' which may resolve to different UIDs on different systems + # PostgreSQL also requires pgdata to have mode 700 or 750 if [[ -d "${DATADIR}/haf_db_store" ]]; then echo "Fixing haf_db_store ownership to UID 105:109 (postgres in HAF container)" sudo chown -R 105:109 "${DATADIR}/haf_db_store" + sudo chown -R 105:109 "${DATADIR}/haf_postgresql_conf.d" + # Fix pgdata permissions - PostgreSQL requires 700 or 750 + if [[ -d "${DATADIR}/haf_db_store/pgdata" ]]; then + echo "Fixing pgdata permissions to 700" + sudo chmod 700 "${DATADIR}/haf_db_store/pgdata" + fi fi "${CI_PROJECT_DIR}/scripts/ci-helpers/start-ci-test-environment.sh" -- GitLab From 3f7984c1b25eb176065d98405aa17c1ec6bd43db Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Tue, 9 Dec 2025 21:55:41 -0500 Subject: [PATCH 35/50] Fix: Increase wait timeout from 30m to 45m for block processing --- scripts/ci-helpers/wait-for-rt-startup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci-helpers/wait-for-rt-startup.sh b/scripts/ci-helpers/wait-for-rt-startup.sh index f3a5cf3..75a1108 100755 --- a/scripts/ci-helpers/wait-for-rt-startup.sh +++ b/scripts/ci-helpers/wait-for-rt-startup.sh @@ -60,6 +60,6 @@ export POSTGRES_ACCESS export COMMAND export MESSAGE -timeout -k 1m 30m bash -c wait-for-rt-startup +timeout -k 1m 45m bash -c wait-for-rt-startup echo "Block processing is finished." \ No newline at end of file -- GitLab From 7701b05900e7f637942e76cd08e5e695451be7c8 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Tue, 9 Dec 2025 23:16:51 -0500 Subject: [PATCH 36/50] Fix: Increase wait timeout from 45m to 55m for block processing --- scripts/ci-helpers/wait-for-rt-startup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci-helpers/wait-for-rt-startup.sh b/scripts/ci-helpers/wait-for-rt-startup.sh index 75a1108..1e02769 100755 --- a/scripts/ci-helpers/wait-for-rt-startup.sh +++ b/scripts/ci-helpers/wait-for-rt-startup.sh @@ -60,6 +60,6 @@ export POSTGRES_ACCESS export COMMAND export MESSAGE -timeout -k 1m 45m bash -c wait-for-rt-startup +timeout -k 1m 55m bash -c wait-for-rt-startup echo "Block processing is finished." \ No newline at end of file -- GitLab From 17c5e5e2df5811e4694a15ed182fd945a03b03ed Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Wed, 10 Dec 2025 00:23:20 -0500 Subject: [PATCH 37/50] Fix: Add tar archive extraction support for NFS cache in test services --- .gitlab-ci.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 67b81e5..fbe1b09 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -416,14 +416,27 @@ sync: if [[ ! -d "${ORIGINAL_SOURCE}/datadir" ]]; then echo "Local cache not found: ${ORIGINAL_SOURCE}/datadir" NFS_PATH="${NFS_PREFIX}/${NFS_TYPE}/${NFS_KEY}" + NFS_TAR="${NFS_PATH}.tar" if [[ -d "${NFS_PATH}/datadir" ]]; then - echo "Found data on NFS: $NFS_PATH" + echo "Found data on NFS directory: $NFS_PATH" export DATA_SOURCE="$NFS_PATH" + elif [[ -f "${NFS_TAR}" ]]; then + echo "Found NFS tar archive: $NFS_TAR" + echo "Extracting to: ${ORIGINAL_SOURCE}" + mkdir -p "${ORIGINAL_SOURCE}" + tar xf "${NFS_TAR}" -C "${ORIGINAL_SOURCE}" + # Restore pgdata permissions for PostgreSQL + if [[ -d "${ORIGINAL_SOURCE}/datadir/haf_db_store/pgdata" ]]; then + chmod 700 "${ORIGINAL_SOURCE}/datadir/haf_db_store/pgdata" + fi + echo "Extracted NFS cache successfully" + export DATA_SOURCE="${ORIGINAL_SOURCE}" else echo "WARNING: Data not found in local or NFS cache" echo "Checked: ${ORIGINAL_SOURCE}/datadir" echo "Checked: ${NFS_PATH}/datadir" + echo "Checked: ${NFS_TAR}" fi else echo "Using local cache: ${ORIGINAL_SOURCE}" -- GitLab From e1d9ed3e89c4869539c5e5139674a150f6d0f275 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Wed, 10 Dec 2025 00:38:25 -0500 Subject: [PATCH 38/50] Fix: Use sudo to restore PostgreSQL permissions after NFS cache extraction --- .gitlab-ci.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fbe1b09..35ab4e6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -426,9 +426,13 @@ sync: echo "Extracting to: ${ORIGINAL_SOURCE}" mkdir -p "${ORIGINAL_SOURCE}" tar xf "${NFS_TAR}" -C "${ORIGINAL_SOURCE}" - # Restore pgdata permissions for PostgreSQL - if [[ -d "${ORIGINAL_SOURCE}/datadir/haf_db_store/pgdata" ]]; then - chmod 700 "${ORIGINAL_SOURCE}/datadir/haf_db_store/pgdata" + # Restore pgdata ownership and permissions for PostgreSQL + # PostgreSQL requires pgdata to be owned by postgres (UID 105) with mode 700 + if [[ -d "${ORIGINAL_SOURCE}/datadir/haf_db_store" ]]; then + echo "Fixing PostgreSQL data permissions..." + sudo chown -R 105:109 "${ORIGINAL_SOURCE}/datadir/haf_db_store" + sudo chown -R 105:109 "${ORIGINAL_SOURCE}/datadir/haf_postgresql_conf.d" 2>/dev/null || true + sudo chmod 700 "${ORIGINAL_SOURCE}/datadir/haf_db_store/pgdata" fi echo "Extracted NFS cache successfully" export DATA_SOURCE="${ORIGINAL_SOURCE}" -- GitLab From 90bf009841146d5371980612db4e72f38bb9d7de Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Wed, 10 Dec 2025 23:13:18 -0500 Subject: [PATCH 39/50] Remove custom NFS fallback logic, use copy_datadir.sh Removed 50+ lines of duplicate NFS tar extraction logic from service entrypoint. HAF's copy_datadir.sh now handles NFS fallback for all cache types via updated regex pattern. Updated HAF submodule to latest feature/nfs-cache-manager with generic cache-manager path support. --- .gitlab-ci.yml | 56 ++------------------------------------------------ haf | 2 +- 2 files changed, 3 insertions(+), 55 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 35ab4e6..3c33ccb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -384,8 +384,7 @@ sync: - data-cache-storage - fast -# HAF instance with NFS fallback for sync data -# Services start before before_script, so we need fallback logic in the entrypoint +# HAF instance with NFS fallback for sync data via copy_datadir.sh .haf-instance-with-nfs-fallback: &haf-instance-with-nfs-fallback name: ${HAF_IMAGE_NAME} alias: haf-instance @@ -397,58 +396,7 @@ sync: "host all reptracker_user 0.0.0.0/0 trust" "host all reptracker_owner 0.0.0.0/0 trust" "host all all 0.0.0.0/0 scram-sha-256" - DATA_SOURCE: "${DATA_CACHE_HAF_PREFIX}_${SYNC_CACHE_KEY}" - DATA_SOURCE_NFS_PREFIX: "${DATA_CACHE_NFS_PREFIX}" - DATA_SOURCE_NFS_TYPE: "${SYNC_CACHE_TYPE}" - DATA_SOURCE_NFS_KEY: "${SYNC_CACHE_KEY}" - entrypoint: - - '/bin/bash' - - '-c' - - | - set -xeuo pipefail - echo "Checking data source availability..." - ORIGINAL_SOURCE="${DATA_SOURCE}" - NFS_PREFIX="${DATA_SOURCE_NFS_PREFIX:-/nfs/ci-cache}" - NFS_TYPE="${DATA_SOURCE_NFS_TYPE:-haf_sync}" - NFS_KEY="${DATA_SOURCE_NFS_KEY}" - - # If original path doesn't exist, try NFS fallback - if [[ ! -d "${ORIGINAL_SOURCE}/datadir" ]]; then - echo "Local cache not found: ${ORIGINAL_SOURCE}/datadir" - NFS_PATH="${NFS_PREFIX}/${NFS_TYPE}/${NFS_KEY}" - NFS_TAR="${NFS_PATH}.tar" - - if [[ -d "${NFS_PATH}/datadir" ]]; then - echo "Found data on NFS directory: $NFS_PATH" - export DATA_SOURCE="$NFS_PATH" - elif [[ -f "${NFS_TAR}" ]]; then - echo "Found NFS tar archive: $NFS_TAR" - echo "Extracting to: ${ORIGINAL_SOURCE}" - mkdir -p "${ORIGINAL_SOURCE}" - tar xf "${NFS_TAR}" -C "${ORIGINAL_SOURCE}" - # Restore pgdata ownership and permissions for PostgreSQL - # PostgreSQL requires pgdata to be owned by postgres (UID 105) with mode 700 - if [[ -d "${ORIGINAL_SOURCE}/datadir/haf_db_store" ]]; then - echo "Fixing PostgreSQL data permissions..." - sudo chown -R 105:109 "${ORIGINAL_SOURCE}/datadir/haf_db_store" - sudo chown -R 105:109 "${ORIGINAL_SOURCE}/datadir/haf_postgresql_conf.d" 2>/dev/null || true - sudo chmod 700 "${ORIGINAL_SOURCE}/datadir/haf_db_store/pgdata" - fi - echo "Extracted NFS cache successfully" - export DATA_SOURCE="${ORIGINAL_SOURCE}" - else - echo "WARNING: Data not found in local or NFS cache" - echo "Checked: ${ORIGINAL_SOURCE}/datadir" - echo "Checked: ${NFS_PATH}/datadir" - echo "Checked: ${NFS_TAR}" - fi - else - echo "Using local cache: ${ORIGINAL_SOURCE}" - fi - - # Run original entrypoint - exec /home/haf_admin/docker_entrypoint.sh "$@" - - '/bin/bash' + DATA_SOURCE: "/cache/${SYNC_CACHE_TYPE}_${SYNC_CACHE_KEY}" command: ["--execute-maintenance-script=${HAF_SOURCE_DIR}/scripts/maintenance-scripts/sleep_infinity.sh"] .postgrest-service: &postgrest-service diff --git a/haf b/haf index 465036e..fd54402 160000 --- a/haf +++ b/haf @@ -1 +1 @@ -Subproject commit 465036e77ceb78cd71d1daeb377b6d0c0a21b857 +Subproject commit fd5440288396c42e02e3a995b50d2fff3a0f74c5 -- GitLab From b1b614c381f3cee10a90f9955f873f73b9403fce Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Wed, 10 Dec 2025 23:17:33 -0500 Subject: [PATCH 40/50] Update HAF ref to latest feature/nfs-cache-manager Updated both include ref and HAF_COMMIT variable to fd5440288 which includes all cache-manager refactoring. --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3c33ccb..fee10ee 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,12 +31,12 @@ variables: BUILDER_IMAGE_PATH: "registry.gitlab.syncad.com/hive/haf/ci-base-image${BUILDER_IMAGE_TAG}" # HAF submodule commit - must match the 'ref:' in the include section below # This is needed for service containers which can't access dotenv artifacts - HAF_COMMIT: "465036e77ceb78cd71d1daeb377b6d0c0a21b857" + HAF_COMMIT: "fd5440288396c42e02e3a995b50d2fff3a0f74c5" include: - template: Workflows/Branch-Pipelines.gitlab-ci.yml - project: hive/haf - ref: 465036e77ceb78cd71d1daeb377b6d0c0a21b857 # feature/nfs-cache-manager + ref: fd5440288396c42e02e3a995b50d2fff3a0f74c5 # feature/nfs-cache-manager file: /scripts/ci-helpers/prepare_data_image_job.yml .lint_job: -- GitLab From d036db46aad67b056fd597b67872bd5877ef6946 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Thu, 11 Dec 2025 01:41:43 -0500 Subject: [PATCH 41/50] Update HAF submodule with cache-manager permission fixes Updates HAF submodule from fd544028 to 24737036 to include critical cache-manager fixes: - Disabled tablespace symlink conversion (3e9b95c26) - Added haf_pipeline to permission/exclusion handlers (831f48a6a) - NFS cache directories chmod 777 (fc02aa903) - Local cache directories chmod 777 (247370363) These fixes prevent permission errors and lock timeouts when jobs run as different UIDs (syncad, root, gitlab-runner). --- haf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/haf b/haf index fd54402..2473703 160000 --- a/haf +++ b/haf @@ -1 +1 @@ -Subproject commit fd5440288396c42e02e3a995b50d2fff3a0f74c5 +Subproject commit 24737036315690b7f5d4093eeb71a4904b5e4a5b -- GitLab From df7f216b956de60541b8964d35337f5f9a142326 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Thu, 11 Dec 2025 02:08:13 -0500 Subject: [PATCH 42/50] Fix: update HAF_COMMIT and include ref to match submodule (24737036) --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fee10ee..bdd1d30 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,12 +31,12 @@ variables: BUILDER_IMAGE_PATH: "registry.gitlab.syncad.com/hive/haf/ci-base-image${BUILDER_IMAGE_TAG}" # HAF submodule commit - must match the 'ref:' in the include section below # This is needed for service containers which can't access dotenv artifacts - HAF_COMMIT: "fd5440288396c42e02e3a995b50d2fff3a0f74c5" + HAF_COMMIT: "24737036315690b7f5d4093eeb71a4904b5e4a5b" include: - template: Workflows/Branch-Pipelines.gitlab-ci.yml - project: hive/haf - ref: fd5440288396c42e02e3a995b50d2fff3a0f74c5 # feature/nfs-cache-manager + ref: 24737036315690b7f5d4093eeb71a4904b5e4a5b # feature/nfs-cache-manager file: /scripts/ci-helpers/prepare_data_image_job.yml .lint_job: -- GitLab From 21f995488c8a1e66cb96ac0f0c2158999e13946f Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Thu, 11 Dec 2025 02:28:43 -0500 Subject: [PATCH 43/50] Update HAF submodule to develop (61042b2aa) Switches from deleted feature/nfs-cache-manager branch to develop after MR !722 was squash-merged. The squash commit contains all cache-manager fixes including: - Disabled tablespace symlink conversion - Added haf_pipeline to permission/exclusion handlers - NFS and local cache directories chmod 777 - All NFS interactions via cache-manager Updates all 3 references: submodule, HAF_COMMIT, and include ref. --- .gitlab-ci.yml | 4 ++-- haf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bdd1d30..49948c4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,12 +31,12 @@ variables: BUILDER_IMAGE_PATH: "registry.gitlab.syncad.com/hive/haf/ci-base-image${BUILDER_IMAGE_TAG}" # HAF submodule commit - must match the 'ref:' in the include section below # This is needed for service containers which can't access dotenv artifacts - HAF_COMMIT: "24737036315690b7f5d4093eeb71a4904b5e4a5b" + HAF_COMMIT: "61042b2aab9f7f4967ff494c98108d55396c2756" include: - template: Workflows/Branch-Pipelines.gitlab-ci.yml - project: hive/haf - ref: 24737036315690b7f5d4093eeb71a4904b5e4a5b # feature/nfs-cache-manager + ref: 61042b2aab9f7f4967ff494c98108d55396c2756 # develop file: /scripts/ci-helpers/prepare_data_image_job.yml .lint_job: diff --git a/haf b/haf index 2473703..61042b2 160000 --- a/haf +++ b/haf @@ -1 +1 @@ -Subproject commit 24737036315690b7f5d4093eeb71a4904b5e4a5b +Subproject commit 61042b2aab9f7f4967ff494c98108d55396c2756 -- GitLab From 4be6bcfb380c97e4785aaca01da55917195dbdbc Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Thu, 11 Dec 2025 03:07:05 -0500 Subject: [PATCH 44/50] Fix: use nginx:latest base image The specific commit hash ecd325dd no longer exists in the registry. Switch to :latest tag like hafah and hivesense already do. --- Dockerfile.rewriter | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.rewriter b/Dockerfile.rewriter index 9d65a2c..58379da 100644 --- a/Dockerfile.rewriter +++ b/Dockerfile.rewriter @@ -1,4 +1,4 @@ -FROM registry.gitlab.syncad.com/hive/common-ci-configuration/nginx:ecd325dd43aee24562f59195ef51a20fa15514d4 AS without_tag +FROM registry.gitlab.syncad.com/hive/common-ci-configuration/nginx:latest AS without_tag COPY docker/reputation_tracker_nginx.conf.template /usr/local/openresty/nginx/conf/nginx.conf.template COPY rewrite_rules.conf /usr/local/openresty/nginx/conf/rewrite_rules.conf COPY docker/rewriter_entrypoint.sh /entrypoint.sh -- GitLab From 88663b2fb959ba16006f4e18a84351c65243af28 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Thu, 11 Dec 2025 04:16:20 -0500 Subject: [PATCH 45/50] Fix: service containers use NFS path for sync data Service containers were trying to use local /cache path which doesn't exist in their context. Changed DATA_SOURCE to use /nfs/ci-cache path so HAF setup script properly fetches from NFS cache. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 49948c4..c821c29 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -396,7 +396,7 @@ sync: "host all reptracker_user 0.0.0.0/0 trust" "host all reptracker_owner 0.0.0.0/0 trust" "host all all 0.0.0.0/0 scram-sha-256" - DATA_SOURCE: "/cache/${SYNC_CACHE_TYPE}_${SYNC_CACHE_KEY}" + DATA_SOURCE: "/nfs/ci-cache/${SYNC_CACHE_TYPE}/${SYNC_CACHE_KEY}" command: ["--execute-maintenance-script=${HAF_SOURCE_DIR}/scripts/maintenance-scripts/sleep_infinity.sh"] .postgrest-service: &postgrest-service -- GitLab From 1ebd7c34195f7ddb48a9215248db8c9b6921f9b6 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Thu, 11 Dec 2025 04:36:16 -0500 Subject: [PATCH 46/50] Fix: extract NFS tar to directory for service containers Service containers using copy_datadir.sh expect DATA_SOURCE/datadir to be a directory, but cache-manager stores data as tar files on NFS. After pushing to NFS cache, extract the tar to directory format so service containers can use it directly. --- .gitlab-ci.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c821c29..5e0d7b3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -367,6 +367,21 @@ sync: if [[ -x "$CACHE_MANAGER" ]]; then echo "Pushing sync data to NFS cache: ${SYNC_CACHE_TYPE}/${SYNC_CACHE_KEY}" "$CACHE_MANAGER" put "${SYNC_CACHE_TYPE}" "${SYNC_CACHE_KEY}" "${LOCAL_SYNC_CACHE}" || echo "Warning: Failed to push to NFS cache" + + # Also extract to directory format for service containers that can't use tar + # Service containers expect DATA_SOURCE/datadir to be a directory, not a tar file + NFS_TAR_FILE="/nfs/ci-cache/${SYNC_CACHE_TYPE}/${SYNC_CACHE_KEY}.tar" + NFS_CACHE_DIR="/nfs/ci-cache/${SYNC_CACHE_TYPE}/${SYNC_CACHE_KEY}" + if [[ -f "$NFS_TAR_FILE" && ! -d "${NFS_CACHE_DIR}/datadir" ]]; then + echo "Extracting tar to directory format for service containers: ${NFS_CACHE_DIR}" + mkdir -p "$NFS_CACHE_DIR" + chmod 777 "$NFS_CACHE_DIR" 2>/dev/null || true + tar xf "$NFS_TAR_FILE" -C "$NFS_CACHE_DIR" + chmod -R a+rX "$NFS_CACHE_DIR" 2>/dev/null || true + chmod 777 "${NFS_CACHE_DIR}/datadir" 2>/dev/null || true + echo "Extracted successfully" + ls -la "$NFS_CACHE_DIR" + fi else echo "Warning: cache-manager.sh not found, skipping NFS cache push" fi -- GitLab From c7068adce70c4317123bcc2ffc4c1abe25560d80 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Thu, 11 Dec 2025 04:39:00 -0500 Subject: [PATCH 47/50] Update HAF submodule with copy_datadir.sh tar fallback fix Service containers set DATA_SOURCE to NFS path but cache is stored as tar file. Updated copy_datadir.sh extracts from .tar when directory not found. --- haf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/haf b/haf index 61042b2..2473703 160000 --- a/haf +++ b/haf @@ -1 +1 @@ -Subproject commit 61042b2aab9f7f4967ff494c98108d55396c2756 +Subproject commit 24737036315690b7f5d4093eeb71a4904b5e4a5b -- GitLab From 4150ac21debd95fd73f55cd65f6593d1ab6cdeb5 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Thu, 11 Dec 2025 04:41:49 -0500 Subject: [PATCH 48/50] Update HAF submodule with hivemind_sync permission fix --- haf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/haf b/haf index 2473703..9611e89 160000 --- a/haf +++ b/haf @@ -1 +1 @@ -Subproject commit 24737036315690b7f5d4093eeb71a4904b5e4a5b +Subproject commit 9611e8909a601400522e6bdaadc4a04772b3be80 -- GitLab From 5b6afab3813b55cec51ad843a0f898b220940bc1 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Thu, 11 Dec 2025 04:41:52 -0500 Subject: [PATCH 49/50] Fix: sync HAF_COMMIT variable with submodule commit --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5e0d7b3..4889bd3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,12 +31,12 @@ variables: BUILDER_IMAGE_PATH: "registry.gitlab.syncad.com/hive/haf/ci-base-image${BUILDER_IMAGE_TAG}" # HAF submodule commit - must match the 'ref:' in the include section below # This is needed for service containers which can't access dotenv artifacts - HAF_COMMIT: "61042b2aab9f7f4967ff494c98108d55396c2756" + HAF_COMMIT: "24737036315690b7f5d4093eeb71a4904b5e4a5b" include: - template: Workflows/Branch-Pipelines.gitlab-ci.yml - project: hive/haf - ref: 61042b2aab9f7f4967ff494c98108d55396c2756 # develop + ref: 24737036315690b7f5d4093eeb71a4904b5e4a5b # develop file: /scripts/ci-helpers/prepare_data_image_job.yml .lint_job: -- GitLab From 8b81224d731e2c1824b723dc604d767e48c11c56 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Thu, 11 Dec 2025 04:44:27 -0500 Subject: [PATCH 50/50] Fix: sync HAF_COMMIT with updated submodule --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4889bd3..5660797 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,12 +31,12 @@ variables: BUILDER_IMAGE_PATH: "registry.gitlab.syncad.com/hive/haf/ci-base-image${BUILDER_IMAGE_TAG}" # HAF submodule commit - must match the 'ref:' in the include section below # This is needed for service containers which can't access dotenv artifacts - HAF_COMMIT: "24737036315690b7f5d4093eeb71a4904b5e4a5b" + HAF_COMMIT: "9611e8909a601400522e6bdaadc4a04772b3be80" include: - template: Workflows/Branch-Pipelines.gitlab-ci.yml - project: hive/haf - ref: 24737036315690b7f5d4093eeb71a4904b5e4a5b # develop + ref: 9611e8909a601400522e6bdaadc4a04772b3be80 # develop file: /scripts/ci-helpers/prepare_data_image_job.yml .lint_job: -- GitLab