From c53bae091a6280fb2fe49083c96e658e281066c8 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Fri, 9 Jan 2026 02:10:00 -0500 Subject: [PATCH] Add fallback logic to find-upstream-image for timing races When a squash merge lands on develop, the new commit's image may not exist yet (pipeline still building). Previously, downstream repos would fail immediately when the latest commit's image wasn't found. Now the script iterates through recent source commits (up to 10 by default) and uses the first one that has an available image. This prevents failures due to timing races between upstream and downstream pipelines. Changes: - find-upstream-image.sh: Check multiple commits, use first with image - Add --max-search option to control fallback depth (default: 10) - Add UPSTREAM_FALLBACK output variable to indicate when fallback used - Add UPSTREAM_OVERRIDE_TAG variable to skip lookup and use specific tag - Add UPSTREAM_MAX_SEARCH variable to template Fixes balance_tracker pipeline failures when HAF develop is updated but image isn't built yet. --- scripts/bash/find-upstream-image.sh | 168 +++++++++++++----- .../source_change_detection.gitlab-ci.yml | 57 +++++- 2 files changed, 182 insertions(+), 43 deletions(-) diff --git a/scripts/bash/find-upstream-image.sh b/scripts/bash/find-upstream-image.sh index 35e46bf..525cb5b 100755 --- a/scripts/bash/find-upstream-image.sh +++ b/scripts/bash/find-upstream-image.sh @@ -2,9 +2,10 @@ # # find-upstream-image.sh - Find the latest built image from an upstream repo # -# This script fetches an upstream repository, finds the last commit that changed -# source files, and checks if a Docker image exists for that commit. Used by -# downstream repos to find pre-built images from their dependencies. +# This script fetches an upstream repository, finds commits that changed source +# files, and checks if a Docker image exists for those commits. It automatically +# falls back to older commits if the latest doesn't have an image yet (e.g., when +# upstream pipeline is still building after a squash merge). # # Usage: # find-upstream-image.sh [OPTIONS] @@ -17,11 +18,12 @@ # Optional: # --branch=NAME Branch to check (default: develop) # --depth=N Git fetch depth (default: 100) +# --max-search=N Max commits to check for existing image (default: 10) # --image=NAME Image name within registry (default: none) # --output=FILE Output env file (default: upstream-image.env) # --work-dir=PATH Working directory for git clone (default: /tmp/upstream-repo-$$) # --keep-repo Don't delete cloned repo after completion -# --require-hit Exit with error if image not found +# --require-hit Exit with error if no image found after checking all commits # --quiet Suppress status messages # --help Show this help message # @@ -47,10 +49,11 @@ # UPSTREAM_IMAGE= Full image name with tag # UPSTREAM_REGISTRY= Registry path without tag # UPSTREAM_BRANCH= Branch that was checked +# UPSTREAM_FALLBACK=true|false Whether a fallback commit was used # # Exit Codes: -# 0 - Success -# 1 - Image not found (with --require-hit) or git/docker error +# 0 - Success (image found, possibly via fallback) +# 1 - No image found after checking all commits (with --require-hit) or git/docker error # 2 - Invalid arguments # @@ -64,6 +67,7 @@ REGISTRY="" PATTERNS="" BRANCH="develop" DEPTH=100 +MAX_SEARCH=10 IMAGE="" OUTPUT_FILE="upstream-image.env" WORK_DIR="" @@ -111,6 +115,9 @@ while [[ $# -gt 0 ]]; do --depth=*) DEPTH="${1#*=}" ;; + --max-search=*) + MAX_SEARCH="${1#*=}" + ;; --image=*) IMAGE="${1#*=}" ;; @@ -180,48 +187,80 @@ fi git clone --depth="$DEPTH" --branch="$BRANCH" --single-branch "$REPO_URL" "$WORK_DIR" 2>&1 | \ while IFS= read -r line; do log " $line"; done -# Find last source commit (full 40-char hash for cache keys; get-cached-image.sh will abbreviate for tags) -log "Finding last source commit for patterns: ${PATTERN_ARRAY[*]}" +# Find source commits (full 40-char hashes for cache keys) +# Get multiple commits so we can fall back if the latest doesn't have an image yet +log "Finding source commits for patterns: ${PATTERN_ARRAY[*]}" +log "Will check up to $MAX_SEARCH commits for existing images" + +cd "$WORK_DIR" -FIND_COMMIT_ARGS=(--dir="$WORK_DIR" --full --quiet) -COMMIT=$("$SCRIPT_DIR/find-last-source-commit.sh" "${FIND_COMMIT_ARGS[@]}" "${PATTERN_ARRAY[@]}") +# Get list of commits that changed source files (most recent first) +mapfile -t SOURCE_COMMITS < <(git log --pretty=format:"%H" -n "$MAX_SEARCH" -- "${PATTERN_ARRAY[@]}" 2>/dev/null || true) -if [[ -z "$COMMIT" ]]; then - error "Failed to find source commit" +if [[ ${#SOURCE_COMMITS[@]} -eq 0 ]]; then + error "Failed to find any source commits matching patterns" exit 1 fi -log "Found last source commit: $COMMIT" +log "Found ${#SOURCE_COMMITS[@]} source commit(s) to check" -# Check if image exists -log "Checking for image in registry: $REGISTRY" +# Iterate through commits, looking for one with an existing image +FOUND_IMAGE=false +USED_FALLBACK=false +CHECKED_COUNT=0 +FOUND_COMMIT="" -GET_IMAGE_ARGS=( - --commit="$COMMIT" - --registry="$REGISTRY" - --output="$OUTPUT_FILE.tmp" -) +for COMMIT in "${SOURCE_COMMITS[@]}"; do + CHECKED_COUNT=$((CHECKED_COUNT + 1)) -if [[ -n "$IMAGE" ]]; then - GET_IMAGE_ARGS+=(--image="$IMAGE") -fi + if [[ $CHECKED_COUNT -eq 1 ]]; then + log "Checking latest commit: $COMMIT" + else + log "Checking fallback commit $CHECKED_COUNT: $COMMIT" + fi -if [[ "$QUIET" == "true" ]]; then - GET_IMAGE_ARGS+=(--quiet) -fi + # Build args for get-cached-image.sh (without --require-hit, we handle that ourselves) + GET_IMAGE_ARGS=( + --commit="$COMMIT" + --registry="$REGISTRY" + --output="$OUTPUT_FILE.tmp" + ) -if [[ "$REQUIRE_HIT" == "true" ]]; then - GET_IMAGE_ARGS+=(--require-hit) -fi + if [[ -n "$IMAGE" ]]; then + GET_IMAGE_ARGS+=(--image="$IMAGE") + fi -"$SCRIPT_DIR/get-cached-image.sh" "${GET_IMAGE_ARGS[@]}" -GET_RESULT=$? + # Always use quiet mode for fallback checks to reduce noise + if [[ "$QUIET" == "true" ]] || [[ $CHECKED_COUNT -gt 1 ]]; then + GET_IMAGE_ARGS+=(--quiet) + fi + + # Check if image exists for this commit + "$SCRIPT_DIR/get-cached-image.sh" "${GET_IMAGE_ARGS[@]}" || true -# Read the temp output and rewrite with UPSTREAM_ prefix -if [[ -f "$OUTPUT_FILE.tmp" ]]; then + # Check the result + if [[ -f "$OUTPUT_FILE.tmp" ]] && grep -q "CACHE_HIT=true" "$OUTPUT_FILE.tmp"; then + FOUND_IMAGE=true + FOUND_COMMIT="$COMMIT" + if [[ $CHECKED_COUNT -gt 1 ]]; then + USED_FALLBACK=true + log "Found image at fallback commit $CHECKED_COUNT: $COMMIT" + else + log "Found image at latest commit: $COMMIT" + fi + break + else + log " No image found for $COMMIT" + rm -f "$OUTPUT_FILE.tmp" + fi +done + +# Handle the result +if [[ "$FOUND_IMAGE" == "true" ]]; then + # Read the temp output and rewrite with UPSTREAM_ prefix { - # Add branch info echo "UPSTREAM_BRANCH=$BRANCH" + echo "UPSTREAM_FALLBACK=$USED_FALLBACK" # Rename variables with UPSTREAM_ prefix sed 's/^CACHE_HIT=/UPSTREAM_CACHE_HIT=/; s/^IMAGE_COMMIT=/UPSTREAM_COMMIT=/; @@ -230,12 +269,59 @@ if [[ -f "$OUTPUT_FILE.tmp" ]]; then s/^IMAGE_REGISTRY=/UPSTREAM_REGISTRY=/' "$OUTPUT_FILE.tmp" } > "$OUTPUT_FILE" rm -f "$OUTPUT_FILE.tmp" -fi -log "" -log "Output written to: $OUTPUT_FILE" -if [[ "$QUIET" != "true" ]]; then - cat "$OUTPUT_FILE" >&2 -fi + if [[ "$USED_FALLBACK" == "true" ]]; then + log "" + log "NOTE: Using fallback image (latest commit's image not yet available)" + log " Latest source commit: ${SOURCE_COMMITS[0]}" + log " Using image from: $FOUND_COMMIT" + fi + + log "" + log "Output written to: $OUTPUT_FILE" + if [[ "$QUIET" != "true" ]]; then + cat "$OUTPUT_FILE" >&2 + fi + + exit 0 +else + # No image found for any commit + log "" + log "ERROR: No image found after checking $CHECKED_COUNT commit(s)" + log " Latest source commit: ${SOURCE_COMMITS[0]}" + log " This usually means the upstream pipeline hasn't finished building yet." + + # Write output with CACHE_HIT=false for the latest commit + LATEST_COMMIT="${SOURCE_COMMITS[0]}" + LATEST_TAG="${LATEST_COMMIT:0:8}" + if [[ -n "$IMAGE" ]]; then + FULL_IMAGE="${REGISTRY}/${IMAGE}:${LATEST_TAG}" + FULL_REGISTRY="${REGISTRY}/${IMAGE}" + else + FULL_IMAGE="${REGISTRY}:${LATEST_TAG}" + FULL_REGISTRY="${REGISTRY}" + fi -exit $GET_RESULT + cat > "$OUTPUT_FILE" << EOF +UPSTREAM_BRANCH=$BRANCH +UPSTREAM_FALLBACK=false +UPSTREAM_CACHE_HIT=false +UPSTREAM_COMMIT=$LATEST_COMMIT +UPSTREAM_TAG=$LATEST_TAG +UPSTREAM_IMAGE=$FULL_IMAGE +UPSTREAM_REGISTRY=$FULL_REGISTRY +EOF + + log "" + log "Output written to: $OUTPUT_FILE" + if [[ "$QUIET" != "true" ]]; then + cat "$OUTPUT_FILE" >&2 + fi + + if [[ "$REQUIRE_HIT" == "true" ]]; then + error "No upstream image available (--require-hit specified)" + exit 1 + fi + + exit 0 +fi diff --git a/templates/source_change_detection.gitlab-ci.yml b/templates/source_change_detection.gitlab-ci.yml index 4c119d2..cc63649 100644 --- a/templates/source_change_detection.gitlab-ci.yml +++ b/templates/source_change_detection.gitlab-ci.yml @@ -170,7 +170,16 @@ variables: # ============================================================================ # Template for finding pre-built images from upstream repositories -# Extend this and set the required variables +# Extend this and set the required variables. +# +# This template automatically falls back to older commits if the latest commit's +# image isn't available yet (e.g., after a squash merge when upstream pipeline +# is still building). This prevents failures due to timing races between repos. +# +# Override behavior: +# Set UPSTREAM_OVERRIDE_TAG to skip dynamic lookup and use a specific image tag. +# Example: UPSTREAM_OVERRIDE_TAG: "abc12345" to use registry/image:abc12345 +# This is useful for testing against a specific branch or pinning to a known good image. .find_upstream_image: stage: .pre image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/docker:24-cli @@ -183,12 +192,55 @@ variables: UPSTREAM_PATTERNS: "" UPSTREAM_IMAGE: "" UPSTREAM_DEPTH: "100" + # Max commits to check when looking for existing image (fallback depth) + UPSTREAM_MAX_SEARCH: "10" + # Set this to skip dynamic lookup and use a specific image tag + # Example: "abc12345" or "v1.27.0" - will use UPSTREAM_REGISTRY:UPSTREAM_OVERRIDE_TAG + UPSTREAM_OVERRIDE_TAG: "" # Output file name UPSTREAM_OUTPUT: "upstream-image.env" before_script: - apk add --no-cache git bash script: - | + # Check for override - skip dynamic lookup if specified + if [[ -n "$UPSTREAM_OVERRIDE_TAG" ]]; then + echo "Using override tag: $UPSTREAM_OVERRIDE_TAG (skipping dynamic lookup)" + + # Build the image name + if [[ -n "$UPSTREAM_IMAGE" ]]; then + FULL_IMAGE="${UPSTREAM_REGISTRY}/${UPSTREAM_IMAGE}:${UPSTREAM_OVERRIDE_TAG}" + FULL_REGISTRY="${UPSTREAM_REGISTRY}/${UPSTREAM_IMAGE}" + else + FULL_IMAGE="${UPSTREAM_REGISTRY}:${UPSTREAM_OVERRIDE_TAG}" + FULL_REGISTRY="${UPSTREAM_REGISTRY}" + fi + + # Check if the override image exists + if docker manifest inspect "$FULL_IMAGE" >/dev/null 2>&1; then + echo "Override image found: $FULL_IMAGE" + cat > "$UPSTREAM_OUTPUT" <