From 8964861df1d357e35dfadac7908d96faca44fe06 Mon Sep 17 00:00:00 2001
From: mtrela <mtrela@syncad.com>
Date: Thu, 30 Jul 2020 13:49:45 +0200
Subject: [PATCH] Votes are saved in more effective way
---
hive/indexer/votes.py | 105 ++++++++++++++----------------------------
1 file changed, 34 insertions(+), 71 deletions(-)
diff --git a/hive/indexer/votes.py b/hive/indexer/votes.py
index 2572c3810..274afad75 100644
--- a/hive/indexer/votes.py
+++ b/hive/indexer/votes.py
@@ -65,7 +65,10 @@ class Votes:
cls._votes_data[key] = dict(voter=voter,
author=author,
permlink=permlink,
- weight=vop['weight'])
+ vote_percent=0,
+ weight=0,
+ rshares=0,
+ last_update="1969-12-31T23:59:59")
@classmethod
def effective_comment_vote_op(cls, vop, date):
@@ -80,10 +83,7 @@ class Votes:
key = voter + "/" + author + "/" + permlink
- cls._effective_votes_data[key] = dict(voter=voter,
- author=author,
- permlink=permlink,
- vote_percent=vop['vote_percent'],
+ cls._effective_votes_data[key] = dict(vote_percent=vop['vote_percent'],
weight=vop['weight'],
rshares=vop['rshares'],
last_update=date)
@@ -91,25 +91,26 @@ class Votes:
@classmethod
def flush_votes(cls):
""" Flush vote data from cache to database """
+ cls.inside_flush = True
if cls._votes_data:
sql = """
INSERT INTO hive_votes
- (post_id, voter_id, author_id, permlink_id, weight)
- select data_source.post_id, data_source.voter_id, data_source.author_id, data_source.permlink_id, data_source.weight
+ (post_id, voter_id, author_id, permlink_id, weight, rshares, vote_percent, last_update)
+ select data_source.post_id, data_source.voter_id, data_source.author_id, data_source.permlink_id, data_source.weight, data_source.rshares, data_source.vote_percent, data_source.last_update
+ from
+ (
+ SELECT hp.id as post_id, ha_v.id as voter_id, ha_a.id as author_id, hpd_p.id as permlink_id, t.weight, t.rshares, t.vote_percent, t.last_update
from
(
- SELECT hp.id as post_id, ha_v.id as voter_id, ha_a.id as author_id, hpd_p.id as permlink_id, t.weight
- from
- (
- VALUES
- -- voter, author, permlink, weight
- {}
- ) AS T(voter, author, permlink, weight)
- INNER JOIN hive_accounts ha_v ON ha_v.name = T.voter
- INNER JOIN hive_accounts ha_a ON ha_a.name = T.author
- INNER JOIN hive_permlink_data hpd_p ON hpd_p.permlink = T.permlink
- INNER JOIN hive_posts hp ON hp.author_id = ha_a.id AND hp.permlink_id = hpd_p.id
- ) as data_source(post_id, voter_id, author_id, permlink_id, weight)
+ VALUES
+ -- voter, author, permlink, weight, rshares, vote_percent, last_update
+ {}
+ ) AS T(voter, author, permlink, weight, rshares, vote_percent, last_update)
+ INNER JOIN hive_accounts ha_v ON ha_v.name = t.voter
+ INNER JOIN hive_accounts ha_a ON ha_a.name = t.author
+ INNER JOIN hive_permlink_data hpd_p ON hpd_p.permlink = t.permlink
+ INNER JOIN hive_posts hp ON hp.author_id = ha_a.id AND hp.permlink_id = hpd_p.id
+ ) as data_source(post_id, voter_id, author_id, permlink_id, weight, rshares, vote_percent, last_update)
ON CONFLICT ON CONSTRAINT hive_votes_ux1 DO
UPDATE
SET
@@ -125,8 +126,8 @@ class Votes:
values_limit = 1000
for _, vd in cls._votes_data.items():
- values.append("('{}', '{}', '{}', {})".format(
- vd['voter'], vd['author'], vd['permlink'], vd['weight']))
+ values.append("('{}', '{}', '{}', {}, {}, {}, '{}'::timestamp)".format(
+ vd['voter'], vd['author'], vd['permlink'], vd['weight'], vd['rshares'], vd['vote_percent'], vd['last_update']))
if len(values) >= values_limit:
values_str = ','.join(values)
@@ -141,60 +142,22 @@ class Votes:
values.clear()
cls._votes_data.clear()
+ cls.inside_flush = False
@classmethod
- def flush_effective_votes(cls):
+ def process_effective_votes(cls):
""" Flush vote data from cache to database """
- if cls._effective_votes_data:
- sql = """
- UPDATE hive_votes AS hv SET
- weight = data_source.weight,
- rshares = data_source.rshares,
- vote_percent = data_source.vote_percent,
- last_update = data_source.last_update,
- num_changes = hv.num_changes + 1
- FROM
- (
- SELECT ha_v.id as voter_id, ha_a.id as author_id, hpd_p.id as permlink_id, t.weight, t.rshares, t.vote_percent, t.last_update
- FROM
- (
- VALUES
- -- voter, author, permlink, weight, rshares, vote_percent, last_update
- {}
- ) AS T(voter, author, permlink, weight, rshares, vote_percent, last_update)
- INNER JOIN hive_accounts ha_v ON ha_v.name = t.voter
- INNER JOIN hive_accounts ha_a ON ha_a.name = t.author
- INNER JOIN hive_permlink_data hpd_p ON hpd_p.permlink = t.permlink
- ) as data_source(voter_id, author_id, permlink_id, weight, rshares, vote_percent, last_update)
- WHERE hv.voter_id = data_source.voter_id and hv.author_id = data_source.author_id and hv.permlink_id = data_source.permlink_id;
- """
-
- values = []
- values_limit = 1000
-
- for _, vd in cls._effective_votes_data.items():
- values.append("('{}', '{}', '{}', {}, {}, {}, '{}'::timestamp)".format(
- vd['voter'], vd['author'], vd['permlink'], vd['weight'], vd['rshares'], vd['vote_percent'], vd['last_update']))
-
- if len(values) >= values_limit:
- values_str = ','.join(values)
- actual_query = sql.format(values_str)
- DB.query(actual_query)
- values.clear()
-
- if len(values) > 0:
- values_str = ','.join(values)
- actual_query = sql.format(values_str)
- DB.query(actual_query)
- values.clear()
-
- cls._effective_votes_data.clear()
+ for key, vd in cls._effective_votes_data.items():
+ assert key in cls._votes_data
+
+ cls._votes_data[key]["vote_percent"] = cls._effective_votes_data[key]["vote_percent"]
+ cls._votes_data[key]["weight"] = cls._effective_votes_data[key]["weight"]
+ cls._votes_data[key]["rshares"] = cls._effective_votes_data[key]["rshares"]
+ cls._votes_data[key]["last_update"] = cls._effective_votes_data[key]["last_update"]
+
+ cls._effective_votes_data.clear()
@classmethod
def flush(cls):
- cls.inside_flush = True
-
+ cls.process_effective_votes()
cls.flush_votes()
- cls.flush_effective_votes()
-
- cls.inside_flush = False
--
GitLab