From 59696070ae69be1722603be0803bf2f0e616aa26 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Fri, 28 Nov 2025 22:59:07 -0500 Subject: [PATCH 1/2] Switch data-cache-storage jobs to hive-builder-9 Builder-9 has better single-thread performance (32-core 5950X) compared to builder-5 (48-core 3960x), resulting in ~40% faster sync times. --- .gitlab-ci.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c43b9cd..e283191 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -145,6 +145,7 @@ prepare_haf_data: CONFIG_INI_SOURCE: "$CI_PROJECT_DIR/haf/docker/config_5M.ini" tags: - data-cache-storage + - hive-builder-9 .docker-build-template: extends: .docker_image_builder_job_template @@ -294,6 +295,7 @@ sync: when: always tags: - data-cache-storage + - hive-builder-9 .hfm-only-service: &hfm-only-service name: $HAF_IMAGE_NAME @@ -364,6 +366,7 @@ regression-test: when: always tags: - data-cache-storage + - hive-builder-9 setup-scripts-test: image: registry.gitlab.syncad.com/hive/reputation_tracker/ci-runner:docker-24.0.1-3 @@ -389,6 +392,7 @@ setup-scripts-test: echo -e "\e[0Ksection_end:$(date +%s):tests\r\e[0K" tags: - data-cache-storage + - hive-builder-9 performance-test: image: registry.gitlab.syncad.com/hive/reputation_tracker/ci-runner:docker-24.0.1-3 @@ -425,6 +429,7 @@ performance-test: junit: tests/performance/junit-result.xml tags: - data-cache-storage + - hive-builder-9 pattern-test: extends: .pytest_based_template @@ -456,6 +461,7 @@ pattern-test: - "**/*.out.json" tags: - data-cache-storage + - hive-builder-9 deploy_python_api_packages_to_gitlab: stage: publish @@ -522,3 +528,4 @@ cleanup_haf_cache_manual: CLEANUP_PATH_PATTERN: "${DATA_CACHE_HAF_PREFIX}_*" tags: - data-cache-storage + - hive-builder-9 -- GitLab From 040b53a75780cfa94a0e3d49c44977a92cffb159 Mon Sep 17 00:00:00 2001 From: Dan Notestein Date: Fri, 28 Nov 2025 23:13:48 -0500 Subject: [PATCH 2/2] Fix CTE scope bug: use temp table for cursor iteration CTEs are only accessible within their WITH statement, but the FOR loop was outside that scope. This caused 'relation votes_to_process does not exist'. Solution: Use a data-modifying CTE to INSERT votes into a pre-created temp table, then iterate over the temp table in the FOR loop. --- db/process_block_range.sql | 44 +++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/db/process_block_range.sql b/db/process_block_range.sql index da1ac67..a13ee85 100644 --- a/db/process_block_range.sql +++ b/db/process_block_range.sql @@ -18,6 +18,15 @@ DECLARE __upsert_votes INT; __vote_rec RECORD; BEGIN +-- Create temp table to materialize votes for sequential processing +CREATE TEMP TABLE IF NOT EXISTS _votes_to_process ( + author_id INT, + voter_id INT, + rshares BIGINT, + prev_rshares BIGINT, + source_op BIGINT +) ON COMMIT DROP; +TRUNCATE _votes_to_process; --------------------------------------------------------------------------------------- WITH vote_operations AS ( SELECT @@ -136,12 +145,6 @@ check_if_prev_balances_canceled AS ( FROM find_prev_votes_in_table ja ), --------------------------------------------------------------------------------------- --- Use cursor to guarantee sequential processing order for deterministic results -votes_to_process AS ( - SELECT author_id, voter_id, rshares, prev_rshares, source_op - FROM check_if_prev_balances_canceled - ORDER BY source_op -), delete_votes AS ( DELETE FROM active_votes av USING join_permlink_id_to_deletes dp @@ -152,42 +155,49 @@ delete_votes AS ( RETURNING av.author_id, av.permlink_serial_id ), upsert_votes AS ( - INSERT INTO active_votes AS av + INSERT INTO active_votes AS av (author_id, voter_id, permlink_serial_id, rshares) - SELECT + SELECT author_id, voter_id, permlink_id, rshares FROM ranked_data uv - WHERE - uv.row_num = 1 AND + WHERE + uv.row_num = 1 AND uv.op_type_id = 72 AND NOT EXISTS ( - SELECT NULL - FROM join_permlink_id_to_deletes dv + SELECT NULL + FROM join_permlink_id_to_deletes dv WHERE dv.author_id = uv.author_id AND dv.permlink_id = uv.permlink_id AND dv.source_op > uv.source_op LIMIT 1 ) ON CONFLICT ON CONSTRAINT pk_active_votes DO UPDATE SET rshares = EXCLUDED.rshares RETURNING av.author_id, av.voter_id, av.permlink_serial_id +), +-- Materialize votes data into temp table for sequential processing +materialize_votes AS ( + INSERT INTO _votes_to_process (author_id, voter_id, rshares, prev_rshares, source_op) + SELECT author_id, voter_id, rshares, prev_rshares, source_op + FROM check_if_prev_balances_canceled + RETURNING 1 ) SELECT - (SELECT count(*) FROM delete_votes) AS delete_votes, - (SELECT count(*) FROM upsert_votes) AS upsert_votes -INTO __delete_votes, __upsert_votes; + (SELECT count(*) FROM delete_votes), + (SELECT count(*) FROM upsert_votes), + (SELECT count(*) FROM materialize_votes) +INTO __delete_votes, __upsert_votes, __rep_change; -- Process reputation changes sequentially using cursor to guarantee order -FOR __vote_rec IN (SELECT * FROM votes_to_process) LOOP +FOR __vote_rec IN (SELECT * FROM _votes_to_process ORDER BY source_op) LOOP PERFORM reptracker_backend.calculate_account_reputations( __vote_rec.author_id, __vote_rec.voter_id, __vote_rec.rshares, __vote_rec.prev_rshares ); - __rep_change := __rep_change + 1; END LOOP; END -- GitLab