diff --git a/.gitlab-ci.yaml b/.gitlab-ci.yaml index 7fdbd2e6429f7627de50600b8dd72d96785350ea..c1b2795d0a1a0420b1702df8a527c40160a2cc22 100644 --- a/.gitlab-ci.yaml +++ b/.gitlab-ci.yaml @@ -17,6 +17,7 @@ include: variables: # HIVEMIND RUNNER_HIVEMIND_SERVER_HTTP_PORT: 8080 + RUNNER_PYTEST_WORKERS: 12 # pytest-xdist worker count - 12 is optimal for 32-core machine (leaves headroom for PostgreSQL and other processes) RUNNER_HIVEMIND_BENCHMARK_SERVER_HOSTNAME: hivemind-benchmark RUNNER_HIVEMIND_SMOKETEST_SERVER_HOSTNAME: hivemind-smoketest # HAF @@ -34,7 +35,7 @@ variables: GIT_STRATEGY: clone GIT_SUBMODULE_STRATEGY: recursive GIT_SUBMODULE_DEPTH: 1 - GIT_SUBMODULE_UPDATE_FLAGS: --jobs 4 + GIT_SUBMODULE_UPDATE_FLAGS: --jobs 8 IMAGE_REMOVER_TAG: "$CI_COMMON_JOB_VERSION" # Other @@ -91,8 +92,7 @@ variables: $RUNNER_HIVEMIND_SERVER_HTTP_PORT \ bridge_api_patterns/ \ api_smoketest_bridge.xml \ - $RUNNER_PYTEST_WORKERS \ - "auto" + $RUNNER_PYTEST_WORKERS echo -e "\e[0Ksection_end:$(date +%s):bridge_api_smoketest\r\e[0K" .bridge_api_smoketest_negative-script: &bridge_api_smoketest_negative-script @@ -103,8 +103,7 @@ variables: $RUNNER_HIVEMIND_SERVER_HTTP_PORT \ bridge_api_negative/ \ api_smoketest_bridge_negative.xml \ - $RUNNER_PYTEST_WORKERS \ - "auto" + $RUNNER_PYTEST_WORKERS echo -e "\e[0Ksection_end:$(date +%s):bridge_api_smoketest_negative\r\e[0K" .condenser_api_smoketest-script: &condenser_api_smoketest-script @@ -115,8 +114,7 @@ variables: $RUNNER_HIVEMIND_SERVER_HTTP_PORT \ condenser_api_patterns/ \ api_smoketest_condenser_api.xml \ - $RUNNER_PYTEST_WORKERS \ - "auto" + $RUNNER_PYTEST_WORKERS echo -e "\e[0Ksection_end:$(date +%s):condenser_api_smoketest\r\e[0K" .condenser_api_smoketest_negative-script: &condenser_api_smoketest_negative-script @@ -127,8 +125,7 @@ variables: $RUNNER_HIVEMIND_SERVER_HTTP_PORT \ condenser_api_negative/ \ api_smoketest_condenser_api_negative.xml \ - $RUNNER_PYTEST_WORKERS \ - "auto" + $RUNNER_PYTEST_WORKERS echo -e "\e[0Ksection_end:$(date +%s):condenser_api_smoketest_negative\r\e[0K" .database_api_smoketest-script: &database_api_smoketest-script @@ -139,8 +136,7 @@ variables: $RUNNER_HIVEMIND_SERVER_HTTP_PORT \ database_api_patterns/ \ api_smoketest_database_api.xml \ - $RUNNER_PYTEST_WORKERS \ - "auto" + $RUNNER_PYTEST_WORKERS echo -e "\e[0Ksection_end:$(date +%s):database_api_smoketest\r\e[0K" .database_api_smoketest_negative-script: &database_api_smoketest_negative-script @@ -151,8 +147,7 @@ variables: $RUNNER_HIVEMIND_SERVER_HTTP_PORT \ database_api_negative/ \ api_smoketest_database_api_negative.xml \ - $RUNNER_PYTEST_WORKERS \ - "auto" + $RUNNER_PYTEST_WORKERS echo -e "\e[0Ksection_end:$(date +%s):database_api_smoketest_negative\r\e[0K" .follow_api_smoketest-script: &follow_api_smoketest-script @@ -163,8 +158,7 @@ variables: $RUNNER_HIVEMIND_SERVER_HTTP_PORT \ follow_api_patterns/ \ api_smoketest_follow_api.xml \ - $RUNNER_PYTEST_WORKERS \ - "auto" + $RUNNER_PYTEST_WORKERS echo -e "\e[0Ksection_end:$(date +%s):follow_api_smoketest\r\e[0K" .follow_api_smoketest_negative-script: &follow_api_smoketest_negative-script @@ -175,8 +169,7 @@ variables: $RUNNER_HIVEMIND_SERVER_HTTP_PORT \ follow_api_negative/ \ api_smoketest_follow_api_negative.xml \ - $RUNNER_PYTEST_WORKERS \ - "auto" + $RUNNER_PYTEST_WORKERS echo -e "\e[0Ksection_end:$(date +%s):follow_api_smoketest_negative\r\e[0K" .tags_api_smoketest-script: &tags_api_smoketest-script @@ -187,8 +180,7 @@ variables: $RUNNER_HIVEMIND_SERVER_HTTP_PORT \ tags_api_patterns/ \ api_smoketest_tags_api.xml \ - $RUNNER_PYTEST_WORKERS \ - "auto" + $RUNNER_PYTEST_WORKERS echo -e "\e[0Ksection_end:$(date +%s):tags_api_smoketest\r\e[0K" .tags_api_smoketest_negative-script: &tags_api_smoketest_negative-script @@ -199,8 +191,7 @@ variables: $RUNNER_HIVEMIND_SERVER_HTTP_PORT \ tags_api_negative/ \ api_smoketest_tags_api_negative.xml \ - $RUNNER_PYTEST_WORKERS \ - "auto" + $RUNNER_PYTEST_WORKERS echo -e "\e[0Ksection_end:$(date +%s):tags_api_smoketest_negative\r\e[0K" .mock_tests-script: &mock_tests-script @@ -222,8 +213,7 @@ variables: $RUNNER_HIVEMIND_SERVER_HTTP_PORT \ hive_api_patterns/ \ api_smoketest_hive_api.xml \ - $RUNNER_PYTEST_WORKERS \ - "auto" + $RUNNER_PYTEST_WORKERS echo -e "\e[0Ksection_end:$(date +%s):hive_api_smoketest\r\e[0K" .search-api_smoketest-script: &search-api_smoketest-script @@ -234,8 +224,7 @@ variables: $RUNNER_HIVEMIND_SERVER_HTTP_PORT \ search-api_patterns/ \ api_smoketest_search-api.xml \ - $RUNNER_PYTEST_WORKERS \ - "auto" + $RUNNER_PYTEST_WORKERS echo -e "\e[0Ksection_end:$(date +%s):tags_api_smoketest_negative\r\e[0K" .api-benchmark-script: @@ -257,8 +246,7 @@ variables: $RUNNER_HIVEMIND_SERVER_HTTP_PORT \ postgrest_negative/ \ postgrest_negative.xml \ - $RUNNER_PYTEST_WORKERS \ - "auto" + $RUNNER_PYTEST_WORKERS echo -e "\e[0Ksection_end:$(date +%s):postgrest_negative\r\e[0K" .rest_api_smoketest-script: &rest_api_smoketest-script @@ -269,8 +257,7 @@ variables: $RUNNER_HIVEMIND_SERVER_HTTP_PORT \ rest_api_patterns/ \ api_smoketest_rest_api_patterns.xml \ - $RUNNER_PYTEST_WORKERS \ - "auto" + $RUNNER_PYTEST_WORKERS echo -e "\e[0Ksection_end:$(date +%s):rest_api_smoketest\r\e[0K" .rest_api_negative_smoketest-script: &rest_api_negative_smoketest-script @@ -281,8 +268,7 @@ variables: $RUNNER_HIVEMIND_SERVER_HTTP_PORT \ rest_api_negative/ \ api_smoketest_rest_api_negative.xml \ - $RUNNER_PYTEST_WORKERS \ - "auto" + $RUNNER_PYTEST_WORKERS echo -e "\e[0Ksection_end:$(date +%s):rest_api_negative_smoketest\r\e[0K" .hivemind-postgrest-serve-script: &hivemind-postgrest-serve-script | @@ -519,7 +505,7 @@ sync: USE_POSTGREST: 1 script: - | - sleep 20s + sleep 5s # Brief pause to ensure services are starting; await handles actual readiness cat ${WORKING_DIR}/.hivemind-venv/lib/python3.8/site-packages/hive/_version.py > version.log pushd ${WORKING_DIR}/app ${DATA_CACHE_HIVEMIND}/await -t 5m http://haf-instance:8091 -- echo "HAF ready" @@ -534,9 +520,16 @@ sync: -c "ALTER SYSTEM SET maintenance_work_mem = '8GB'" \ -c "ALTER SYSTEM SET fsync = 'off'" \ -c "ALTER SYSTEM SET full_page_writes = 'off'" \ + -c "ALTER SYSTEM SET synchronous_commit = 'off'" \ + -c "ALTER SYSTEM SET wal_buffers = '64MB'" \ + -c "ALTER SYSTEM SET checkpoint_completion_target = '0.9'" \ -c "ALTER SYSTEM SET max_parallel_workers_per_gather = 4" \ + -c "ALTER SYSTEM SET max_parallel_maintenance_workers = 4" \ + -c "ALTER SYSTEM SET max_parallel_workers = 8" \ + -c "ALTER SYSTEM SET random_page_cost = '1.1'" \ + -c "ALTER SYSTEM SET effective_io_concurrency = '200'" \ -c "SELECT pg_reload_conf()" - echo "PostgreSQL CI tuning applied (fsync=off, full_page_writes=off, work_mem=256MB)" + echo "PostgreSQL CI tuning applied (fsync=off, full_page_writes=off, synchronous_commit=off, SSD optimizations)" pushd ${WORKING_DIR} ${WORKING_DIR}/docker_entrypoint.sh setup \ --database-admin-url="${HAF_ADMIN_POSTGRES_URL}" \ @@ -578,6 +571,16 @@ e2e_benchmark_on_postgrest: image: $CI_REGISTRY_IMAGE/ci-base-image:python-3.12-slim-1 stage: benchmark interruptible: true + cache: + key: + files: + - tox.ini + - requirements.txt + prefix: pip-cache + paths: + - .tox/ + - .cache/pip/ + policy: pull-push needs: - job: replay_data_copy artifacts: true @@ -630,6 +633,7 @@ e2e_benchmark_on_postgrest: JOB_TOKEN: $CI_JOB_TOKEN RUNNER_HIVEMIND_BENCHMARK_URL: http://$RUNNER_HIVEMIND_BENCHMARK_SERVER_HOSTNAME RUNNER_HIVEMIND_SMOKETEST_URL: http://$RUNNER_HIVEMIND_SMOKETEST_SERVER_HOSTNAME + PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" # Enable pip caching for faster tox runs # TODO: As you complete the migration of methods to SQL, uncomment variables and add finished method names. # Missing variable enables the entire test group. RUN_TESTS_WITH_MARKER: "not postgrest_ignore"