From 4eef702bc7c64564bd6e5bdbd0ba1792342b1f1a Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Tue, 30 Dec 2025 05:07:25 -0500 Subject: [PATCH 1/5] Fix lint jobs: disable submodule fetching Lint jobs don't need submodules - they already exclude ./submodules from scanning. This prevents failures due to submodule fetch issues on public runners. --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 01a7cce..f68f45f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -104,6 +104,8 @@ default: .lint_job: extends: .job-defaults stage: lint + variables: + GIT_SUBMODULE_STRATEGY: none artifacts: name: lint-results when: always -- GitLab From d5cd9806ddb58077a400763773ff8da71b90b8b7 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Tue, 30 Dec 2025 05:54:29 -0500 Subject: [PATCH 2/5] Update HAF submodule to latest develop (b4225f9d2) Previous commit was force-pushed out of existence on HAF develop. --- .gitlab-ci.yml | 4 ++-- submodules/haf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f68f45f..6618725 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,7 +26,7 @@ variables: GIT_SUBMODULE_UPDATE_FLAGS: --jobs 4 # HAF submodule commit - must match the 'ref:' in the include section above # This is needed for service containers which can't access dotenv artifacts - HAF_COMMIT: "9add4c306ddc0067bf00857c2c70053f21525ae6" + HAF_COMMIT: "b4225f9d2591195b0e6aadf36bbef921d95f92b9" # HAF configuration DATA_CACHE_HAF_PREFIX: "/cache/replay_data_haf" # NFS cache configuration for sync data sharing across builders @@ -40,7 +40,7 @@ variables: include: - template: Workflows/Branch-Pipelines.gitlab-ci.yml - project: hive/haf - ref: 9add4c306ddc0067bf00857c2c70053f21525ae6 # develop + ref: b4225f9d2591195b0e6aadf36bbef921d95f92b9 # develop file: /scripts/ci-helpers/prepare_data_image_job.yml # implicitly pulls templates/base.gitlab-ci.yml from common-ci-configuration - project: 'hive/common-ci-configuration' ref: 6b9e9e75ec5263939450936a4f8348dfbca3666d diff --git a/submodules/haf b/submodules/haf index 9add4c3..b4225f9 160000 --- a/submodules/haf +++ b/submodules/haf @@ -1 +1 @@ -Subproject commit 9add4c306ddc0067bf00857c2c70053f21525ae6 +Subproject commit b4225f9d2591195b0e6aadf36bbef921d95f92b9 -- GitLab From f0dbf251c13b989ac09db203416c5738f22eb5f1 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Tue, 30 Dec 2025 06:24:45 -0500 Subject: [PATCH 3/5] Allow file:// protocol for nested submodules HAF's hive submodule uses relative paths (../hive.git) that resolve to file:// URLs during submodule operations. Git blocks these by default for security. Setting protocol.file.allow=always enables them for our CI. --- .gitlab-ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6618725..4b53cba 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,6 +24,11 @@ variables: # fetch workspaces during transition. Remove once all projects use fetch. GIT_CLONE_PATH: $CI_BUILDS_DIR/fetch/$CI_RUNNER_SHORT_TOKEN/$CI_CONCURRENT_ID/$CI_PROJECT_PATH GIT_SUBMODULE_UPDATE_FLAGS: --jobs 4 + # Allow file:// protocol for nested submodules with relative paths + # HAF's hive submodule uses relative paths that resolve to file:// URLs + GIT_CONFIG_COUNT: 1 + GIT_CONFIG_KEY_0: protocol.file.allow + GIT_CONFIG_VALUE_0: always # HAF submodule commit - must match the 'ref:' in the include section above # This is needed for service containers which can't access dotenv artifacts HAF_COMMIT: "b4225f9d2591195b0e6aadf36bbef921d95f92b9" -- GitLab From aba7db3c7d80ed4822cdc73e8913162705014cf8 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Tue, 30 Dec 2025 06:27:25 -0500 Subject: [PATCH 4/5] Clean stale HAF submodule in pre_get_sources When workspace has HAF submodule at different commit than needed, nested submodules can have broken remotes causing 'origin does not appear to be a git repository' errors. Clean HAF submodule when commit doesn't match to force fresh checkout. --- .gitlab-ci.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4b53cba..b3abdc5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -69,6 +69,17 @@ default: # Remove stale lock files that block git operations find .git -name "*.lock" -delete 2>/dev/null || true + # Clean stale HAF submodule if commit doesn't match expected + # This prevents "origin does not exist" errors in nested submodules + if [ -d "submodules/haf" ] && [ -n "${HAF_COMMIT:-}" ]; then + CURRENT_HAF=$(git ls-tree HEAD submodules/haf 2>/dev/null | awk '{print $3}') + if [ -n "$CURRENT_HAF" ] && [ "$CURRENT_HAF" != "$HAF_COMMIT" ]; then + echo "pre_get_sources: HAF submodule mismatch (have $CURRENT_HAF, need $HAF_COMMIT)" + echo "pre_get_sources: cleaning submodules/haf to force fresh checkout" + rm -rf submodules/haf .git/modules/submodules/haf + fi + fi + # Check if main repo is corrupt - if so, remove .git to force fresh clone if ! git rev-parse HEAD >/dev/null 2>&1; then echo "pre_get_sources: main repository corrupt, forcing fresh clone" @@ -85,6 +96,11 @@ default: if ! git -C "$submod" rev-parse HEAD >/dev/null 2>&1; then needs_clean=true fi + # Also check if origin remote is valid (broken remotes cause fetch failures) + if ! git -C "$submod" remote get-url origin >/dev/null 2>&1; then + echo "pre_get_sources: $submod has broken origin remote" + needs_clean=true + fi fi # Check if .git/modules exists but is corrupt (even if working dir is gone) if [ -d ".git/modules/$submod" ]; then -- GitLab From aeb01d9c5f15772df9261addd34ecd2244145cbe Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Tue, 30 Dec 2025 09:17:51 -0500 Subject: [PATCH 5/5] Fix validate_haf_commit SIGPIPE error (exit 141) Replace grep|head|sed pipeline with single awk command to avoid SIGPIPE when head closes the pipe before grep finishes writing. --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b3abdc5..37a6251 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -184,7 +184,8 @@ validate_haf_commit: script: - | SUBMODULE_COMMIT=$(git ls-tree HEAD submodules/haf | awk '{print $3}') - INCLUDE_REF=$(grep -A2 "project:.*hive/haf" .gitlab-ci.yml | grep "ref:" | head -1 | sed 's/.*ref: *\([a-f0-9]*\).*/\1/') + # Use awk instead of grep|head|sed to avoid SIGPIPE (exit code 141) + INCLUDE_REF=$(awk '/project:.*hive\/haf/{found=1} found && /ref:/{gsub(/.*ref: */, ""); gsub(/ .*/, ""); print; exit}' .gitlab-ci.yml) echo "HAF_COMMIT variable: $HAF_COMMIT" echo "HAF submodule: $SUBMODULE_COMMIT" echo "Include ref: $INCLUDE_REF" -- GitLab