sync to postgres branch

hive/db/methods.py

@@ -35,7 +35,7 @@ class QueryStats:
         for arr in sorted(cls.stats.items(), key=lambda x:-x[1][0])[0:40]:
             sql, vals = arr
             ms, calls = vals
-            print("% 5.1f%% % 10.2fms % 7.2favg % 7dx -- %s" % (100 * ms/ttl, ms, ms/calls, calls, sql[0:180]))
+            print("% 5.1f%% % 10.2fms % 7.2favg % 8dx -- %s" % (100 * ms/ttl, ms, ms/calls, calls, sql[0:180]))
         cls.stats = {}
         cls.ttltime = 0
 
@@ -58,6 +58,7 @@ def query(sql, **kwargs):
         logger.debug(res)
         return res
     except Exception as e:
+        print("[SQL] Error in query {} ({})".format(sql, kwargs))
         conn.close()
         logger.exception(e)
         raise e
@@ -158,7 +159,7 @@ async def payouts_total():
       WHERE is_paidout = 1 AND payout_at > '%s'
     """ % (precalc_date)
 
-    return precalc_sum + query_one(sql)
+    return float(precalc_sum + query_one(sql)) #TODO: decimal
 
 # sum of completed payouts last 24 hrs
 async def payouts_last_24h():
@@ -166,7 +167,7 @@ async def payouts_last_24h():
       SELECT SUM(payout) FROM hive_posts_cache
       WHERE is_paidout = 1 AND payout_at > DATE_SUB(NOW(), INTERVAL 24 HOUR)
     """
-    return query_one(sql)
+    return float(query_one(sql)) # TODO: decimal
 
 # unused
 def get_reblogs_since(account: str, since: str):
@@ -188,11 +189,11 @@ def get_posts(ids, context = None):
 
     reblogged_ids = []
     if context:
-        reblogged_ids = query_col("SELECT post_id FROM hive_reblogs WHERE account = :a AND post_id IN :ids", a=context, ids=ids)
+        reblogged_ids = query_col("SELECT post_id FROM hive_reblogs WHERE account = :a AND post_id IN :ids", a=context, ids=tuple(ids))
 
     # key by id so we can return sorted by input order
     posts_by_id = {}
-    for row in query(sql, ids=ids).fetchall():
+    for row in query(sql, ids=tuple(ids)).fetchall():
         obj = dict(row)
 
         if context:
@@ -218,7 +219,7 @@ def get_posts(ids, context = None):
 
 # builds SQL query to pull a list of posts for any sort order or tag
 # sort can be: trending hot new promoted
-def get_discussions_by_sort_and_tag(sort, tag, skip, limit, context = None):
+async def get_discussions_by_sort_and_tag(sort, tag, skip, limit, context = None):
     if skip > 5000:
         raise Exception("cannot skip {} results".format(skip))
     if limit > 100:

hive/indexer/accounts.py

@@ -40,7 +40,7 @@ class Accounts:
                     "VALUES (:name, :date)", name=name, date=block_date)
 
         sql = "SELECT name, id FROM hive_accounts WHERE name IN :names"
-        cls._ids = {**dict(query_all(sql, names=new_names)), **cls._ids}
+        cls._ids = {**dict(query_all(sql, names=tuple(new_names))), **cls._ids}
 
 
     # account cache methods
@@ -70,6 +70,8 @@ class Accounts:
 
     @classmethod
     def cache_dirty_follows(cls):
+        if not cls._dirty_follows:
+            return
         cls.update_follows(list(cls._dirty_follows))
         cls._dirty_follows = set()
 
@@ -126,40 +128,26 @@ class Accounts:
 
     @classmethod
     def update_follows(cls, accounts):
-        if not accounts:
-            return
-        from hive.indexer.cache import batch_queries
-        fstats = cls._get_accounts_follow_stats(accounts)
-        sqls = []
-        for name in accounts:
-            values = {
-                'name': name,
-                'followers': fstats['followers'][name],
-                'following': fstats['following'][name]
-            }
-
-            update = ', '.join([k+" = :"+k for k in values.keys()][1:])
-            sql = "UPDATE hive_accounts SET %s WHERE name = :name" % (update)
-            sqls.append([(sql, values)])
-        batch_queries(sqls)
+        sql = """
+            UPDATE hive_accounts
+               SET followers = (SELECT COUNT(*) FROM hive_follows WHERE state = 1 AND following = hive_accounts.name),
+                   following = (SELECT COUNT(*) FROM hive_follows WHERE state = 1 AND follower  = hive_accounts.name)
+             WHERE name IN :names
+        """
+        query(sql, names=tuple(accounts))
 
     @classmethod
     def _generate_cache_sqls(cls, accounts, block_date=None):
         if not block_date:
             block_date = get_adapter().head_time()
 
-        #fstats = cls._get_accounts_follow_stats(accounts)
         sqls = []
         for account in get_adapter().get_accounts(accounts):
-            name = account['name']
-
             values = {
-                'name': name,
+                'name': account['name'],
                 'proxy': account['proxy'],
                 'post_count': account['post_count'],
                 'reputation': rep_log10(account['reputation']),
-                #'followers': fstats['followers'][name],
-                #'following': fstats['following'][name],
                 'proxy_weight': amount(account['vesting_shares']),
                 'vote_weight': amount(account['vesting_shares']) + amount(account['received_vesting_shares']) - amount(account['delegated_vesting_shares']),
                 'kb_used': int(account['lifetime_bandwidth']) / 1e6 / 1024,
@@ -173,24 +161,6 @@ class Accounts:
             sqls.append([(sql, values)])
         return sqls
 
-    @classmethod
-    def _get_accounts_follow_stats(cls, accounts):
-        sql = """SELECT follower, COUNT(*) FROM hive_follows
-                WHERE follower IN :lst GROUP BY follower"""
-        following = dict(query(sql, lst=accounts).fetchall())
-        for name in accounts:
-            if name not in following:
-                following[name] = 0
-
-        sql = """SELECT following, COUNT(*) FROM hive_follows
-                WHERE following IN :lst GROUP BY following"""
-        followers = dict(query(sql, lst=accounts).fetchall())
-        for name in accounts:
-            if name not in followers:
-                followers[name] = 0
-
-        return {'followers': followers, 'following': following}
-
     @classmethod
     def _safe_account_metadata(cls, account):
         prof = {}

hive/server/serve.py

 # -*- coding: utf-8 -*-
+import os
 import logging
 from datetime import datetime
 

setup.py

@@ -38,7 +38,6 @@ setup(
         'ujson',
         'urllib3',
         'PrettyTable',
-        'progressbar2'
     ],
     entry_points={
         'console_scripts': [