From 125cd1c9b6248c66e8b4682445a5f7d2e36f9c27 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Sat, 3 Jan 2026 12:36:09 -0500 Subject: [PATCH 1/3] Add lint, service config, and test helper templates for HAF apps New templates added to haf_app_testing.gitlab-ci.yml: Lint templates: - .haf_app_lint_bash - Bash script linting with shellcheck - .haf_app_lint_sql - SQL linting with sqlfluff (PostgreSQL) Service container configuration: - .haf_service_config - HAF instance configuration variables - .postgrest_service_config - PostgREST service configuration Test helpers (composable with !reference): - .haf_app_wait_for_postgres - Wait for PostgreSQL readiness - .haf_app_wait_for_postgrest - Wait for PostgREST API - .haf_app_wait_for_sync - Wait for HAF to reach target block - .haf_app_extract_test_cache - Extract cache for test jobs These templates reduce duplication across HAF apps (balance_tracker, reputation_tracker, haf_block_explorer) by providing reusable building blocks for common CI patterns. Also adds CI-IMPROVEMENT-PLAN.md documenting the analysis and future improvement roadmap for HAF app CI configurations. --- docs/CI-IMPROVEMENT-PLAN.md | 241 ++++++++++++++++++ docs/haf-app-testing.md | 136 ++++++++++ templates/haf_app_testing.gitlab-ci.yml | 315 ++++++++++++++++++++++++ 3 files changed, 692 insertions(+) create mode 100644 docs/CI-IMPROVEMENT-PLAN.md diff --git a/docs/CI-IMPROVEMENT-PLAN.md b/docs/CI-IMPROVEMENT-PLAN.md new file mode 100644 index 0000000..539c2a9 --- /dev/null +++ b/docs/CI-IMPROVEMENT-PLAN.md @@ -0,0 +1,241 @@ +# CI/CD Architecture Improvement Plan + +## Overview + +Analysis of CI configurations across hive, haf, hivemind, hafah, balance_tracker, reputation_tracker, and haf_block_explorer repositories. + +| Repository | CI Lines | Stages | Template Source | Caching Tiers | +|------------|----------|--------|-----------------|---------------| +| **hive** | 1,979 | 6 | common-ci-configuration + local | 3 (NFS, local, Docker) | +| **haf** | 1,397 | 7 | common-ci-configuration + hive | 2 (NFS, local) | +| **hivemind** | ~1,000 | 9 | common-ci-configuration + haf | 2 (NFS, local) | +| **hafah** | 872 | 5 | common-ci-configuration + haf | 2 (NFS, local) | +| **balance_tracker** | 1,620 | 7 | common-ci-configuration + haf | 2 (NFS, local) | +| **reputation_tracker** | ~1,200 | 7 | common-ci-configuration + haf | 2 (NFS, local) | +| **haf_block_explorer** | ~1,100 | 7 | common-ci-configuration + haf | 2 (NFS, local) | + +--- + +## Shared Patterns (Already Consistent) + +### 1. Template Inheritance Hierarchy +All repos use a consistent template chain: +``` +common-ci-configuration (base templates) + ↓ +hive (blockchain-specific templates) + ↓ +haf (HAF app templates: prepare_data_image_job.yml) + ↓ +Application repos (balance_tracker, hafah, etc.) +``` + +### 2. Two-Tier Caching Strategy +All repos implement: +- **NFS Cache** (`/nfs/ci-cache/...`) - Shared across builders +- **Local Cache** (`/cache/...`) - Per-builder extraction +- **cache-manager.sh** from common-ci-configuration for NFS operations +- Cache keys combining HAF commit + app commit: `${HAF_COMMIT}_${CI_COMMIT_SHORT_SHA}` + +### 3. Quick Test Mode +All HAF apps support: +```yaml +QUICK_TEST: "true" +QUICK_TEST_HAF_COMMIT: +``` +Skips heavy replay jobs, uses pre-cached data. + +### 4. Change Detection & Auto-Skip +All repos have `detect_changes` job that: +- Analyzes changed files +- Sets `AUTO_SKIP_BUILD` / `AUTO_SKIP_SYNC` flags +- Skips heavy jobs for docs/tests/SQL-only changes + +### 5. Git Strategy +Consistent across all repos: +```yaml +GIT_STRATEGY: fetch +GIT_DEPTH: 0 +GIT_SUBMODULE_STRATEGY: normal # Manual init for nested submodules +``` + +### 6. Git Corruption Recovery +All repos have `pre_get_sources` hooks that: +- Remove stale `.git` lock files +- Detect/clean corrupt submodules +- Handle directory-to-submodule transitions + +### 7. Docker-in-Docker Pattern +Test jobs use DinD with: +- `DOCKER_HOST: tcp://docker:2375` +- Cache extraction before service startup +- Marker files to prevent race conditions + +### 8. Service Container Architecture +All HAF apps use similar service structure: +- **haf-instance**: PostgreSQL + hived indexing +- **app-setup**: Schema installation +- **app/postgrest**: REST API gateway + +### 9. Runner Tag Strategy +Consistent tags: +- `public-runner-docker`: General jobs +- `data-cache-storage`: NFS cache access (builders 8-12) +- `fast`: High-performance builders + +### 10. HAF Commit Validation +All HAF apps have `validate_haf_commit` job ensuring consistency between: +- `.gitmodules` submodule ref +- `HAF_COMMIT` variable +- `include: ref:` statement + +--- + +## Key Differences + +### Pipeline Stage Organization + +| Repo | Unique Stages | +|------|---------------| +| **hive** | `static_code_analysis`, `deploy` (mirrornet trigger) | +| **haf** | `build_and_test_phase_1/2` (split build-test phases) | +| **hivemind** | `benchmark`, `collector` (performance tracking) | +| **hafah** | Standard 5-stage (simplest) | +| **balance_tracker** | `sync` separate from `test` | +| **reputation_tracker** | Similar to balance_tracker | +| **haf_block_explorer** | Most complex submodule dependencies (4 nested apps) | + +### Submodule Complexity + +| Repo | Submodule Depth | Notes | +|------|-----------------|-------| +| **hive** | 1 (test-tools, etc.) | Parent repo | +| **haf** | 2 (hive → test-tools) | Moderately complex | +| **hivemind** | 3 (hafah, reputation_tracker nested HAFs) | Very complex | +| **hafah** | 2 (haf → hive) | Standard HAF app | +| **balance_tracker** | 2 (haf → hive) | Standard HAF app | +| **reputation_tracker** | 2 (haf → hive) | Standard HAF app | +| **haf_block_explorer** | 4 (btracker, hafah, reptracker, each with HAF) | Most complex | + +### Cache Key Prefixes (Inconsistent!) + +| Repo | NFS Cache Prefix | +|------|------------------| +| **hive** | `/nfs/ci-cache/hive/replay_data_hive_*` | +| **haf** | `/nfs/ci-cache/haf/*` | +| **hivemind** | `/nfs/ci-cache/haf_hivemind_sync/*` | +| **hafah** | `/nfs/ci-cache/hif/*` (**inconsistent naming!**) | +| **balance_tracker** | `/nfs/ci-cache/haf_btracker_sync/*` | +| **reputation_tracker** | `/nfs/ci-cache/haf_reptracker_sync/*` | +| **haf_block_explorer** | `/nfs/ci-cache/haf_hafbe_sync/*` | + +--- + +## Improvement Actions + +### Action 1: Create Unified HAF App Template (HIGH PRIORITY) + +**Problem**: balance_tracker, reputation_tracker, hafah, and haf_block_explorer have ~70% identical CI code. + +**Solution**: Create `templates/haf_app_base.gitlab-ci.yml` with: +- Standard 7-stage pipeline structure +- Pre-configured sync job template +- Unified test job templates (regression, pattern, performance) +- WAX spec + Python API client generation + +**Expected reduction**: Each app's CI from ~1000+ lines to ~200-300 lines. + +**Implementation**: +1. Extract common patterns from existing apps +2. Create parameterized templates with variables for app-specific config +3. Migrate one app (balance_tracker) as pilot +4. Roll out to remaining apps + +### Action 2: Standardize Cache Naming (HIGH PRIORITY) + +**Problem**: HAfAH uses `/nfs/ci-cache/hif/` while others use `haf__sync` pattern. + +**Solution**: +- Rename hafah cache to `/nfs/ci-cache/haf_hafah_sync/` +- Document standard naming convention: `haf__sync` +- Update cache-manager prefixes + +**Implementation**: +1. Update hafah CI config +2. Clean up old cache directories +3. Document naming convention + +### Action 3: Implement Shared HAF Cache Warming (HIGH PRIORITY) + +**Problem**: Each app independently replays HAF 5M blocks, duplicating work. + +**Solution**: Create a central "HAF cache warmer" pipeline that: +- Runs nightly or on HAF develop changes +- Pre-warms NFS cache for all apps +- Apps only need to sync their application layer + +**Estimated savings**: 80 minutes × 5 apps = 400 minutes per HAF change. + +**Implementation**: +1. Create scheduled pipeline in HAF repo +2. Generate caches for mainnet, testnet, mirrornet +3. Update app pipelines to use pre-warmed caches +4. Add cache freshness validation + +--- + +## Additional Improvements (Medium Priority) + +### 4. Extract Common Submodule Initialization + +**Problem**: Each repo has slightly different `.init_submodules` logic with copy-paste drift. + +**Recommendation**: Create `scripts/init_haf_submodules.sh` that: +- Handles arbitrary nesting depth +- Properly resolves relative URLs +- Has consistent corruption detection/recovery + +### 5. Centralize Skip Rules + +**Problem**: `skip_rules.yml` exists in multiple repos with slight variations. + +**Recommendation**: Move to common-ci-configuration with parameterized patterns: +```yaml +include: + - project: hive/common-ci-configuration + file: templates/skip_rules.gitlab-ci.yml +variables: + HAF_APP_SKIP_PATTERNS: "tests/,docs/,*.md,CLAUDE.md" +``` + +### 6. Add Pipeline Status Dashboard + +**Problem**: No unified view of CI health across all repos. + +**Recommendation**: Extend existing Grafana dashboard to show: +- Pipeline success rates per repo +- Cache hit rates +- Average sync/test times +- Failed job patterns + +--- + +## Timeline + +| Phase | Action | Status | +|-------|--------|--------| +| 1 | Create haf_app_base.gitlab-ci.yml | Pending | +| 1 | Fix hafah cache naming | Pending | +| 2 | Implement HAF cache warmer | Pending | +| 3 | Extract common submodule init | Pending | +| 3 | Centralize skip rules | Pending | +| 4 | Add CI dashboard metrics | Pending | + +--- + +## Success Metrics + +- Reduce average HAF app CI config from ~1000 lines to ~300 lines +- Reduce duplicate HAF replay time by 80% via cache warming +- Achieve 100% cache naming consistency +- Zero copy-paste drift in submodule initialization diff --git a/docs/haf-app-testing.md b/docs/haf-app-testing.md index b915550..a826d67 100644 --- a/docs/haf-app-testing.md +++ b/docs/haf-app-testing.md @@ -177,6 +177,142 @@ cleanup_cache: HAF_APP_CACHE_TYPE: "haf_myapp_sync" ``` +## Lint Templates + +### .haf_app_lint_bash + +Bash script linting with shellcheck. Generates checkstyle XML reports for GitLab integration. + +```yaml +lint_bash: + extends: .haf_app_lint_bash + variables: + LINT_SCRIPTS_DIR: "scripts" # Optional, defaults to "scripts" +``` + +**Variables:** +| Variable | Default | Description | +|----------|---------|-------------| +| `LINT_SCRIPTS_DIR` | `scripts` | Directory to search for .sh files | + +**Artifacts:** +- `shellcheck.xml` - Checkstyle format report +- `shellcheck.html` - HTML formatted report + +### .haf_app_lint_sql + +SQL linting with sqlfluff for PostgreSQL. Outputs YAML format report. + +```yaml +lint_sql: + extends: .haf_app_lint_sql + variables: + SQLFLUFF_CONFIG: ".sqlfluff" # Optional + SQL_PATHS: "db/" # Optional, defaults to "." +``` + +**Variables:** +| Variable | Default | Description | +|----------|---------|-------------| +| `SQLFLUFF_DIALECT` | `postgres` | SQL dialect | +| `SQLFLUFF_CONFIG` | (empty) | Path to sqlfluff config file | +| `SQL_PATHS` | `.` | Paths to lint | + +**Artifacts:** +- `sqlfluff.yaml` - YAML format lint report + +## Test Helper Templates + +These templates provide composable scripts for test job setup. Use with `!reference` to build your before_script: + +```yaml +my_tests: + before_script: + - !reference [.haf_app_wait_for_postgres, script] + - !reference [.haf_app_wait_for_postgrest, script] + - # Your additional setup +``` + +### .haf_app_wait_for_postgres + +Waits for PostgreSQL to be ready. Works with both DinD (docker-compose) and service container patterns. + +**Variables:** +| Variable | Default | Description | +|----------|---------|-------------| +| `HAF_SERVICE_TIMEOUT` | `300` | Max wait time in seconds | +| `HAF_PG_HOST` | (auto-detect) | PostgreSQL host (for service containers) | +| `HAF_PG_PORT` | `5432` | PostgreSQL port | +| `HAF_PG_USER` | `haf_admin` | PostgreSQL user | + +### .haf_app_wait_for_postgrest + +Waits for PostgREST API to be ready. + +**Variables:** +| Variable | Default | Description | +|----------|---------|-------------| +| `PGRST_TIMEOUT` | `120` | Max wait time in seconds | +| `PGRST_HOST` | `http://postgrest:3000` | PostgREST URL | + +### .haf_app_wait_for_sync + +Waits for HAF to sync to a specific block number. Useful for tests that require data at a specific state. + +**Variables:** +| Variable | Default | Description | +|----------|---------|-------------| +| `HAF_TARGET_BLOCK` | `5000000` | Target block number | +| `HAF_SYNC_WAIT_TIMEOUT` | `1800` | Max wait time (30 min) | + +### .haf_app_extract_test_cache + +Extracts cached sync data for test jobs. Simplified version of cache extraction. + +**Required Variables:** +| Variable | Description | +|----------|-------------| +| `APP_SYNC_CACHE_TYPE` | Cache type (e.g., `haf_btracker_sync`) | +| `APP_CACHE_KEY` | Cache key | +| `HAF_DATA_DIRECTORY` | Target extraction directory | + +## Service Container Configuration + +Reference templates for GitLab CI service container configuration. + +**Note:** GitLab CI services cannot use `!reference` directly. Copy these patterns into your services section. + +### .haf_service_config + +HAF instance service configuration variables: + +```yaml +my_tests: + variables: + HAF_PG_ACCESS: "host all all 0.0.0.0/0 trust" + HAF_STOP_BLOCK: "5000024" + HAF_SERVICE_TIMEOUT: "300" + services: + - name: ${HAF_IMAGE_NAME} + alias: haf-instance + command: ["--stop-at-block=${HAF_STOP_BLOCK}", "--skip-hived"] +``` + +### .postgrest_service_config + +PostgREST service configuration variables: + +```yaml +my_tests: + variables: + PGRST_DB_URI: "postgres://haf_admin@haf-instance:5432/haf_block_log" + PGRST_DB_ANON_ROLE: "haf_app_user" + PGRST_DB_SCHEMA: "btracker_app" # Override per-app + services: + - name: postgrest/postgrest:v12.0.2 + alias: postgrest +``` + ## Rule Templates ### .skip_on_quick_test diff --git a/templates/haf_app_testing.gitlab-ci.yml b/templates/haf_app_testing.gitlab-ci.yml index dcb27b7..876bf08 100644 --- a/templates/haf_app_testing.gitlab-ci.yml +++ b/templates/haf_app_testing.gitlab-ci.yml @@ -983,3 +983,318 @@ include: allow_failure: true tags: - data-cache-storage + +# ============================================================================= +# LINT TEMPLATES +# ============================================================================= +# Reusable linting job templates for HAF applications. +# These templates provide consistent linting across all HAF apps. +# +# Usage: +# lint_bash: +# extends: .haf_app_lint_bash +# variables: +# LINT_SCRIPTS_DIR: "scripts" # Optional, defaults to "scripts" +# +# lint_sql: +# extends: .haf_app_lint_sql +# variables: +# SQLFLUFF_CONFIG: ".sqlfluff" # Optional + +# Bash script linting with shellcheck +# Generates checkstyle XML report for GitLab code quality integration +.haf_app_lint_bash: + extends: .job-defaults + stage: build + image: koalaman/shellcheck-alpine:stable + variables: + LINT_SCRIPTS_DIR: "scripts" + GIT_SUBMODULE_STRATEGY: none + before_script: + - apk add --no-cache findutils xmlstarlet + script: + - | + echo "=== Linting Bash Scripts in ${LINT_SCRIPTS_DIR}/ ===" + + # Find and lint all shell scripts + SCRIPTS=$(find "${LINT_SCRIPTS_DIR}/" -name "*.sh" -type f 2>/dev/null || true) + + if [[ -z "$SCRIPTS" ]]; then + echo "No .sh files found in ${LINT_SCRIPTS_DIR}/" + exit 0 + fi + + echo "Found $(echo "$SCRIPTS" | wc -l) script(s) to lint" + echo "$SCRIPTS" | head -20 + + # Run shellcheck with checkstyle output for CI integration + # Use xargs to handle file list properly + echo "$SCRIPTS" | xargs shellcheck -f checkstyle 2>&1 | tee shellcheck.xml || true + + # Convert to HTML report if xmlstarlet available + if command -v xmlstarlet >/dev/null 2>&1; then + xmlstarlet tr /usr/share/checkstyle/checkstyle-noframes.xsl shellcheck.xml > shellcheck.html 2>/dev/null || true + fi + + # Check for errors (not just style issues) + ERROR_COUNT=$(grep -c 'severity="error"' shellcheck.xml 2>/dev/null || echo "0") + if [[ "$ERROR_COUNT" -gt 0 ]]; then + echo "" + echo "Found $ERROR_COUNT error(s) in shell scripts" + exit 1 + fi + + echo "" + echo "Lint passed" + artifacts: + when: always + paths: + - shellcheck.xml + - shellcheck.html + expire_in: 1 week + allow_failure: true + tags: + - public-runner-docker + +# SQL linting with sqlfluff +# Supports PostgreSQL dialect, outputs YAML report +.haf_app_lint_sql: + extends: .job-defaults + stage: build + image: sqlfluff/sqlfluff:2.1.4 + variables: + SQLFLUFF_DIALECT: "postgres" + SQLFLUFF_CONFIG: "" + SQL_PATHS: "." + GIT_SUBMODULE_STRATEGY: none + script: + - | + echo "=== Linting SQL Files ===" + + # Build config option if provided + CONFIG_OPT="" + if [[ -n "${SQLFLUFF_CONFIG}" ]] && [[ -f "${SQLFLUFF_CONFIG}" ]]; then + CONFIG_OPT="--config ${SQLFLUFF_CONFIG}" + fi + + # Run sqlfluff lint + sqlfluff lint \ + --dialect "${SQLFLUFF_DIALECT}" \ + ${CONFIG_OPT} \ + --format yaml \ + ${SQL_PATHS} 2>&1 | tee sqlfluff.yaml || true + + # Check for violations + VIOLATION_COUNT=$(grep -c "^-" sqlfluff.yaml 2>/dev/null || echo "0") + echo "" + echo "Found $VIOLATION_COUNT violation(s)" + + # Fail on errors (L prefix = errors, not warnings) + if grep -qE "code: L0[0-9]{2}" sqlfluff.yaml 2>/dev/null; then + echo "SQL errors found - see sqlfluff.yaml for details" + exit 1 + fi + artifacts: + when: always + paths: + - sqlfluff.yaml + expire_in: 1 week + allow_failure: true + tags: + - public-runner-docker + +# ============================================================================= +# SERVICE CONTAINER TEMPLATES +# ============================================================================= +# Parameterized templates for GitLab CI services section. +# These provide standardized HAF and PostgREST service definitions. +# +# Note: GitLab CI services cannot use !reference directly. Instead, copy +# these definitions into your services section and customize the variables. +# +# Usage example: +# my_test_job: +# services: +# - name: ${HAF_IMAGE_NAME} +# alias: haf-instance +# command: ["--stop-at-block=5000024", "--skip-hived"] +# variables: +# PG_ACCESS: "host all all 0.0.0.0/0 trust" +# - name: postgrest/postgrest:v12.0.2 +# alias: postgrest +# variables: +# PGRST_DB_URI: "postgres://haf_admin@haf-instance:5432/haf_block_log" +# PGRST_DB_SCHEMA: "btracker_app" + +# HAF instance service configuration (for documentation/reference) +# Copy this pattern into your services section +.haf_service_config: + variables: + # PostgreSQL access for HAF container + HAF_PG_ACCESS: "host all all 0.0.0.0/0 trust" + # Default HAF block limit for tests + HAF_STOP_BLOCK: "5000024" + # Service wait timeout (seconds) + HAF_SERVICE_TIMEOUT: "300" + +# PostgREST service configuration (for documentation/reference) +.postgrest_service_config: + variables: + # PostgREST connection settings + PGRST_DB_URI: "postgres://haf_admin@haf-instance:5432/haf_block_log" + PGRST_DB_ANON_ROLE: "haf_app_user" + PGRST_DB_POOL: "10" + PGRST_SERVER_PORT: "3000" + # Override PGRST_DB_SCHEMA per-app + PGRST_DB_SCHEMA: "" + +# ============================================================================= +# TEST HELPER TEMPLATES +# ============================================================================= +# Utility scripts for waiting on services, health checks, etc. +# Use with !reference to compose test job before_script. +# +# Usage: +# my_test_job: +# before_script: +# - !reference [.haf_app_wait_for_postgres, script] +# - !reference [.haf_app_wait_for_postgrest, script] +# - # Your additional setup + +# Wait for PostgreSQL to be ready (for DinD with docker-compose) +# Requires: docker-compose with 'haf' service, or HAF_PG_HOST variable +.haf_app_wait_for_postgres: + script: + - | + echo -e "\e[0Ksection_start:$(date +%s):wait_pg[collapsed=true]\r\e[0KWaiting for PostgreSQL..." + + MAX_WAIT=${HAF_SERVICE_TIMEOUT:-300} + WAIT_INTERVAL=5 + ELAPSED=0 + + # Determine connection method + if [[ -n "${HAF_PG_HOST:-}" ]]; then + # Direct connection (service containers) + PG_CMD="pg_isready -h ${HAF_PG_HOST} -p ${HAF_PG_PORT:-5432} -U ${HAF_PG_USER:-haf_admin}" + elif command -v docker-compose >/dev/null 2>&1 || command -v docker >/dev/null 2>&1; then + # Docker compose (DinD) + PG_CMD="docker compose exec -T haf pg_isready -h localhost -U haf_admin" + else + echo "ERROR: No PostgreSQL connection method available" + exit 1 + fi + + while [[ $ELAPSED -lt $MAX_WAIT ]]; do + if $PG_CMD >/dev/null 2>&1; then + echo "PostgreSQL ready after ${ELAPSED}s" + break + fi + echo "Waiting for PostgreSQL... (${ELAPSED}s / ${MAX_WAIT}s)" + sleep $WAIT_INTERVAL + ELAPSED=$((ELAPSED + WAIT_INTERVAL)) + done + + if [[ $ELAPSED -ge $MAX_WAIT ]]; then + echo "ERROR: PostgreSQL not ready after ${MAX_WAIT}s" + exit 1 + fi + + echo -e "\e[0Ksection_end:$(date +%s):wait_pg\r\e[0K" + +# Wait for PostgREST to be ready +# Requires: PGRST_HOST or docker-compose with 'postgrest' service +.haf_app_wait_for_postgrest: + script: + - | + echo -e "\e[0Ksection_start:$(date +%s):wait_pgrst[collapsed=true]\r\e[0KWaiting for PostgREST..." + + MAX_WAIT=${PGRST_TIMEOUT:-120} + WAIT_INTERVAL=3 + ELAPSED=0 + + # Determine connection method + PGRST_URL="${PGRST_HOST:-http://postgrest:3000}" + + while [[ $ELAPSED -lt $MAX_WAIT ]]; do + if curl -sf "${PGRST_URL}/" >/dev/null 2>&1; then + echo "PostgREST ready after ${ELAPSED}s at ${PGRST_URL}" + break + fi + echo "Waiting for PostgREST... (${ELAPSED}s / ${MAX_WAIT}s)" + sleep $WAIT_INTERVAL + ELAPSED=$((ELAPSED + WAIT_INTERVAL)) + done + + if [[ $ELAPSED -ge $MAX_WAIT ]]; then + echo "ERROR: PostgREST not ready after ${MAX_WAIT}s" + exit 1 + fi + + echo -e "\e[0Ksection_end:$(date +%s):wait_pgrst\r\e[0K" + +# Wait for HAF sync to reach expected block +# Useful for test jobs that need HAF data to be at a specific state +.haf_app_wait_for_sync: + script: + - | + echo -e "\e[0Ksection_start:$(date +%s):wait_sync[collapsed=true]\r\e[0KWaiting for HAF sync..." + + TARGET_BLOCK=${HAF_TARGET_BLOCK:-5000000} + MAX_WAIT=${HAF_SYNC_WAIT_TIMEOUT:-1800} # 30 minutes default + WAIT_INTERVAL=10 + ELAPSED=0 + + # Build psql command + if [[ -n "${HAF_PG_HOST:-}" ]]; then + PSQL_CMD="psql -h ${HAF_PG_HOST} -p ${HAF_PG_PORT:-5432} -U ${HAF_PG_USER:-haf_admin} -d haf_block_log -tAc" + else + PSQL_CMD="docker compose exec -T haf psql -U haf_admin -d haf_block_log -tAc" + fi + + while [[ $ELAPSED -lt $MAX_WAIT ]]; do + CURRENT_BLOCK=$($PSQL_CMD "SELECT COALESCE(MAX(num), 0) FROM hive.blocks;" 2>/dev/null || echo "0") + + if [[ "$CURRENT_BLOCK" -ge "$TARGET_BLOCK" ]]; then + echo "HAF synced to block $CURRENT_BLOCK (target: $TARGET_BLOCK) after ${ELAPSED}s" + break + fi + + echo "HAF at block $CURRENT_BLOCK / $TARGET_BLOCK... (${ELAPSED}s / ${MAX_WAIT}s)" + sleep $WAIT_INTERVAL + ELAPSED=$((ELAPSED + WAIT_INTERVAL)) + done + + if [[ $ELAPSED -ge $MAX_WAIT ]]; then + echo "ERROR: HAF sync timeout - only at block $CURRENT_BLOCK (target: $TARGET_BLOCK)" + exit 1 + fi + + echo -e "\e[0Ksection_end:$(date +%s):wait_sync\r\e[0K" + +# Extract test cache for DinD jobs +# This is a simplified version of the cache extraction for test jobs +# Requires: APP_SYNC_CACHE_TYPE, APP_CACHE_KEY, HAF_DATA_DIRECTORY +.haf_app_extract_test_cache: + script: + - | + echo -e "\e[0Ksection_start:$(date +%s):extract_cache[collapsed=true]\r\e[0KExtracting test cache..." + + # Fetch cache-manager if needed + if [[ ! -x "$CACHE_MANAGER" ]]; then + mkdir -p "$(dirname "$CACHE_MANAGER")" + curl -fsSL "https://gitlab.syncad.com/hive/common-ci-configuration/-/raw/${CACHE_MANAGER_REF:-develop}/scripts/cache-manager.sh" -o "$CACHE_MANAGER" + chmod +x "$CACHE_MANAGER" + fi + + # Create target directory + mkdir -p "${HAF_DATA_DIRECTORY}" + + # Extract cache + echo "Extracting: ${APP_SYNC_CACHE_TYPE}/${APP_CACHE_KEY} -> ${HAF_DATA_DIRECTORY}" + if ! CACHE_HANDLING=haf "$CACHE_MANAGER" get "${APP_SYNC_CACHE_TYPE}" "${APP_CACHE_KEY}" "${HAF_DATA_DIRECTORY}"; then + echo "ERROR: Failed to extract cache" + exit 1 + fi + + ls -la "${HAF_DATA_DIRECTORY}/" + echo -e "\e[0Ksection_end:$(date +%s):extract_cache\r\e[0K" -- GitLab From 00002e3183e430bd3f279912335194d850a9e61f Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Sat, 3 Jan 2026 13:08:23 -0500 Subject: [PATCH 2/3] Add LINT_EXCLUDE_PATHS variable to bash lint template --- templates/haf_app_testing.gitlab-ci.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/templates/haf_app_testing.gitlab-ci.yml b/templates/haf_app_testing.gitlab-ci.yml index 876bf08..908b3c9 100644 --- a/templates/haf_app_testing.gitlab-ci.yml +++ b/templates/haf_app_testing.gitlab-ci.yml @@ -1009,6 +1009,8 @@ include: image: koalaman/shellcheck-alpine:stable variables: LINT_SCRIPTS_DIR: "scripts" + # Space-separated list of paths to exclude (e.g., "haf .git vendor") + LINT_EXCLUDE_PATHS: "" GIT_SUBMODULE_STRATEGY: none before_script: - apk add --no-cache findutils xmlstarlet @@ -1016,8 +1018,16 @@ include: - | echo "=== Linting Bash Scripts in ${LINT_SCRIPTS_DIR}/ ===" - # Find and lint all shell scripts - SCRIPTS=$(find "${LINT_SCRIPTS_DIR}/" -name "*.sh" -type f 2>/dev/null || true) + # Build find exclusion arguments + FIND_EXCLUDES="" + for EXCLUDE in ${LINT_EXCLUDE_PATHS}; do + FIND_EXCLUDES="${FIND_EXCLUDES} -path ./${EXCLUDE} -prune -o" + done + # Always exclude .git + FIND_EXCLUDES="-path ./.git -prune -o ${FIND_EXCLUDES}" + + # Find and lint all shell scripts (excluding specified paths) + SCRIPTS=$(eval "find ${LINT_SCRIPTS_DIR}/ ${FIND_EXCLUDES} -name '*.sh' -type f -print" 2>/dev/null || true) if [[ -z "$SCRIPTS" ]]; then echo "No .sh files found in ${LINT_SCRIPTS_DIR}/" -- GitLab From 214003c157350e5da1ee24347bccccbc2c7d6714 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Sat, 3 Jan 2026 13:11:29 -0500 Subject: [PATCH 3/3] Fix sqlfluff entrypoint in SQL lint template --- templates/haf_app_testing.gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/templates/haf_app_testing.gitlab-ci.yml b/templates/haf_app_testing.gitlab-ci.yml index 908b3c9..417d743 100644 --- a/templates/haf_app_testing.gitlab-ci.yml +++ b/templates/haf_app_testing.gitlab-ci.yml @@ -1071,7 +1071,9 @@ include: .haf_app_lint_sql: extends: .job-defaults stage: build - image: sqlfluff/sqlfluff:2.1.4 + image: + name: sqlfluff/sqlfluff:2.1.4 + entrypoint: [""] # Override default entrypoint to allow shell scripts variables: SQLFLUFF_DIALECT: "postgres" SQLFLUFF_CONFIG: "" -- GitLab