Skip to content
Snippets Groups Projects
Commit 8964861d authored by Mariusz Trela's avatar Mariusz Trela
Browse files

Votes are saved in more effective way

parent d72db7b3
No related branches found
No related tags found
4 merge requests!456Release candidate v1 24,!230Setup monitoring with pghero,!135Enable postgres monitoring on CI server,!61Issue #47 - test database_api get_payout_stats fail
Hide whitespace changes
Inline Side-by-side
Showing
with 34 additions and 71 deletions
hive/indexer/votes.py
+ 34
− 71
View file @ 8964861d
...@@ -65,7 +65,10 @@ class Votes: ...@@ -65,7 +65,10 @@ class Votes:
cls._votes_data[key] = dict(voter=voter, cls._votes_data[key] = dict(voter=voter,
author=author, author=author,
permlink=permlink, permlink=permlink,
weight=vop['weight']) vote_percent=0,
weight=0,
rshares=0,
last_update="1969-12-31T23:59:59")
@classmethod @classmethod
def effective_comment_vote_op(cls, vop, date): def effective_comment_vote_op(cls, vop, date):
...@@ -80,10 +83,7 @@ class Votes: ...@@ -80,10 +83,7 @@ class Votes:
key = voter + "/" + author + "/" + permlink key = voter + "/" + author + "/" + permlink
cls._effective_votes_data[key] = dict(voter=voter, cls._effective_votes_data[key] = dict(vote_percent=vop['vote_percent'],
author=author,
permlink=permlink,
vote_percent=vop['vote_percent'],
weight=vop['weight'], weight=vop['weight'],
rshares=vop['rshares'], rshares=vop['rshares'],
last_update=date) last_update=date)
...@@ -91,25 +91,26 @@ class Votes: ...@@ -91,25 +91,26 @@ class Votes:
@classmethod @classmethod
def flush_votes(cls): def flush_votes(cls):
""" Flush vote data from cache to database """ """ Flush vote data from cache to database """
cls.inside_flush = True
if cls._votes_data: if cls._votes_data:
sql = """ sql = """
INSERT INTO hive_votes INSERT INTO hive_votes
(post_id, voter_id, author_id, permlink_id, weight) (post_id, voter_id, author_id, permlink_id, weight, rshares, vote_percent, last_update)
select data_source.post_id, data_source.voter_id, data_source.author_id, data_source.permlink_id, data_source.weight select data_source.post_id, data_source.voter_id, data_source.author_id, data_source.permlink_id, data_source.weight, data_source.rshares, data_source.vote_percent, data_source.last_update
from
(
SELECT hp.id as post_id, ha_v.id as voter_id, ha_a.id as author_id, hpd_p.id as permlink_id, t.weight, t.rshares, t.vote_percent, t.last_update
from from
( (
SELECT hp.id as post_id, ha_v.id as voter_id, ha_a.id as author_id, hpd_p.id as permlink_id, t.weight VALUES
from -- voter, author, permlink, weight, rshares, vote_percent, last_update
( {}
VALUES ) AS T(voter, author, permlink, weight, rshares, vote_percent, last_update)
-- voter, author, permlink, weight INNER JOIN hive_accounts ha_v ON ha_v.name = t.voter
{} INNER JOIN hive_accounts ha_a ON ha_a.name = t.author
) AS T(voter, author, permlink, weight) INNER JOIN hive_permlink_data hpd_p ON hpd_p.permlink = t.permlink
INNER JOIN hive_accounts ha_v ON ha_v.name = T.voter INNER JOIN hive_posts hp ON hp.author_id = ha_a.id AND hp.permlink_id = hpd_p.id
INNER JOIN hive_accounts ha_a ON ha_a.name = T.author ) as data_source(post_id, voter_id, author_id, permlink_id, weight, rshares, vote_percent, last_update)
INNER JOIN hive_permlink_data hpd_p ON hpd_p.permlink = T.permlink
INNER JOIN hive_posts hp ON hp.author_id = ha_a.id AND hp.permlink_id = hpd_p.id
) as data_source(post_id, voter_id, author_id, permlink_id, weight)
ON CONFLICT ON CONSTRAINT hive_votes_ux1 DO ON CONFLICT ON CONSTRAINT hive_votes_ux1 DO
UPDATE UPDATE
SET SET
...@@ -125,8 +126,8 @@ class Votes: ...@@ -125,8 +126,8 @@ class Votes:
values_limit = 1000 values_limit = 1000
for _, vd in cls._votes_data.items(): for _, vd in cls._votes_data.items():
values.append("('{}', '{}', '{}', {})".format( values.append("('{}', '{}', '{}', {}, {}, {}, '{}'::timestamp)".format(
vd['voter'], vd['author'], vd['permlink'], vd['weight'])) vd['voter'], vd['author'], vd['permlink'], vd['weight'], vd['rshares'], vd['vote_percent'], vd['last_update']))
if len(values) >= values_limit: if len(values) >= values_limit:
values_str = ','.join(values) values_str = ','.join(values)
...@@ -141,60 +142,22 @@ class Votes: ...@@ -141,60 +142,22 @@ class Votes:
values.clear() values.clear()
cls._votes_data.clear() cls._votes_data.clear()
cls.inside_flush = False
@classmethod @classmethod
def flush_effective_votes(cls): def process_effective_votes(cls):
""" Flush vote data from cache to database """ """ Flush vote data from cache to database """
if cls._effective_votes_data: for key, vd in cls._effective_votes_data.items():
sql = """ assert key in cls._votes_data
UPDATE hive_votes AS hv SET
weight = data_source.weight, cls._votes_data[key]["vote_percent"] = cls._effective_votes_data[key]["vote_percent"]
rshares = data_source.rshares, cls._votes_data[key]["weight"] = cls._effective_votes_data[key]["weight"]
vote_percent = data_source.vote_percent, cls._votes_data[key]["rshares"] = cls._effective_votes_data[key]["rshares"]
last_update = data_source.last_update, cls._votes_data[key]["last_update"] = cls._effective_votes_data[key]["last_update"]
num_changes = hv.num_changes + 1
FROM cls._effective_votes_data.clear()
(
SELECT ha_v.id as voter_id, ha_a.id as author_id, hpd_p.id as permlink_id, t.weight, t.rshares, t.vote_percent, t.last_update
FROM
(
VALUES
-- voter, author, permlink, weight, rshares, vote_percent, last_update
{}
) AS T(voter, author, permlink, weight, rshares, vote_percent, last_update)
INNER JOIN hive_accounts ha_v ON ha_v.name = t.voter
INNER JOIN hive_accounts ha_a ON ha_a.name = t.author
INNER JOIN hive_permlink_data hpd_p ON hpd_p.permlink = t.permlink
) as data_source(voter_id, author_id, permlink_id, weight, rshares, vote_percent, last_update)
WHERE hv.voter_id = data_source.voter_id and hv.author_id = data_source.author_id and hv.permlink_id = data_source.permlink_id;
"""
values = []
values_limit = 1000
for _, vd in cls._effective_votes_data.items():
values.append("('{}', '{}', '{}', {}, {}, {}, '{}'::timestamp)".format(
vd['voter'], vd['author'], vd['permlink'], vd['weight'], vd['rshares'], vd['vote_percent'], vd['last_update']))
if len(values) >= values_limit:
values_str = ','.join(values)
actual_query = sql.format(values_str)
DB.query(actual_query)
values.clear()
if len(values) > 0:
values_str = ','.join(values)
actual_query = sql.format(values_str)
DB.query(actual_query)
values.clear()
cls._effective_votes_data.clear()
@classmethod @classmethod
def flush(cls): def flush(cls):
cls.inside_flush = True cls.process_effective_votes()
cls.flush_votes() cls.flush_votes()
cls.flush_effective_votes()
cls.inside_flush = False
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment