accounts - remove bandwidth, fix bug #151 #152

hive/db/db_state.py

@@ -205,6 +205,16 @@ class DbState:
             cls.db().query(sql)
             cls._set_ver(5)
 
+        if cls._ver == 5:
+            # recover acct names lost to issue #151
+            from hive.steem.client import SteemClient
+            from hive.indexer.accounts import Accounts
+            names = SteemClient().get_all_account_names()
+            Accounts.load_ids()
+            Accounts.register(names, '1970-01-01T00:00:00')
+            Accounts.clear_ids()
+            cls._set_ver(6)
+
         assert cls._ver == DB_VERSION, "migration missing or invalid DB_VERSION"
         # Example migration:
         #if cls._ver == 1:

hive/db/schema.py

@@ -10,7 +10,7 @@ from sqlalchemy.types import BOOLEAN
 
 #pylint: disable=line-too-long, too-many-lines
 
-DB_VERSION = 5
+DB_VERSION = 6
 
 def build_metadata():
     """Build schema def with SqlAlchemy"""
@@ -53,7 +53,7 @@ def build_metadata():
         sa.Column('post_count', sa.Integer, nullable=False, server_default='0'),
         sa.Column('proxy_weight', sa.Float(precision=6), nullable=False, server_default='0'),
         sa.Column('vote_weight', sa.Float(precision=6), nullable=False, server_default='0'),
-        sa.Column('kb_used', sa.Integer, nullable=False, server_default='0'),
+        sa.Column('kb_used', sa.Integer, nullable=False, server_default='0'), # deprecated
         sa.Column('rank', sa.Integer, nullable=False, server_default='0'),
 
         sa.Column('active_at', sa.DateTime, nullable=False, server_default='1970-01-01 00:00:00'),

hive/indexer/accounts.py

@@ -35,6 +35,11 @@ class Accounts:
         assert not cls._ids, "id map already loaded"
         cls._ids = dict(DB.query_all("SELECT name, id FROM hive_accounts"))
 
+    @classmethod
+    def clear_ids(cls):
+        """Wipe id map. Only used for db migration #5."""
+        cls._ids = None
+
     @classmethod
     def get_id(cls, name):
         """Get account id by name. Throw if not found."""
@@ -157,15 +162,19 @@ class Accounts:
         profile = safe_profile_metadata(account)
         del account['json_metadata']
 
+        active_at = max(account['created'],
+                        account['last_post'],
+                        account['last_vote_time'])
+
         values = {
             'name':         account['name'],
+            'created_at':   account['created'],
             'proxy':        account['proxy'],
             'post_count':   account['post_count'],
             'reputation':   rep_log10(account['reputation']),
             'proxy_weight': vests_amount(account['vesting_shares']),
             'vote_weight':  vote_weight,
-            'kb_used':      int(account['lifetime_bandwidth']) / 1e6 / 1024,
-            'active_at':    account['last_bandwidth_update'],
+            'active_at':    active_at,
             'cached_at':    cached_at,
 
             'display_name':  profile['name'],

hive/indexer/blocks.py

@@ -80,13 +80,12 @@ class Blocks:
                     account_names.add(op['new_account_name'])
                 elif op_type == 'account_create_with_delegation_operation':
                     account_names.add(op['new_account_name'])
-                elif op_type == 'create_claimed_account':
+                elif op_type == 'create_claimed_account_operation':
                     account_names.add(op['new_account_name'])
 
                 # post ops
                 elif op_type == 'comment_operation':
                     comment_ops.append(op)
-                    account_names.add(op['author']) # temp: HF20 missing account bug
                 elif op_type == 'delete_comment_operation':
                     delete_ops.append(op)
                 elif op_type == 'vote_operation':

hive/steem/client.py

@@ -29,6 +29,15 @@ class SteemClient:
                                            % (len(accounts), len(ret)))
         return ret
 
+    def get_all_account_names(self):
+        """Fetch all account names."""
+        ret = []
+        names = self.__exec('lookup_accounts', ['', 1000])
+        while names:
+            ret.extend(names)
+            names = self.__exec('lookup_accounts', [names[-1], 1000])[1:]
+        return ret
+
     def get_content_batch(self, tuples):
         """Fetch multiple comment objects."""
         posts = self.__exec_batch('get_content', tuples)

hive/steem/http_client.py

@@ -82,6 +82,7 @@ class HttpClient(object):
     """Simple Steem JSON-HTTP-RPC API"""
 
     METHOD_API = dict(
+        lookup_accounts='condenser_api',
         get_block='block_api',
         get_content='condenser_api',
         get_accounts='condenser_api',

hive/utils/stats.py

@@ -89,6 +89,7 @@ class SteemStats(StatsAbstract):
         'get_content': 4,
         'get_order_book': 20,
         'get_feed_history': 20,
+        'lookup_accounts': 1000,
     }
 
     def __init__(self):