Merge pull request #224 from steemit/173-missing-cache-entries

retry insert when get_content fails

hive/db/adapter.py

@@ -177,8 +177,8 @@ class Db:
             Stats.log_db(sql, perf() - start)
             return result
         except Exception as e:
-            log.info("[SQL-ERR] %s in query %s (%s)",
-                     e.__class__.__name__, sql, kwargs)
+            log.warning("[SQL-ERR] %s in query %s (%s)",
+                        e.__class__.__name__, sql, kwargs)
             raise e
 
     @staticmethod

hive/indexer/accounts.py

@@ -95,7 +95,6 @@ class Accounts:
     @classmethod
     def dirty_oldest(cls, limit=50000):
         """Flag `limit` least-recently updated accounts for update."""
-        log.info("[HIVE] flagging %d oldest accounts for update", limit)
         sql = "SELECT name FROM hive_accounts ORDER BY cached_at LIMIT :limit"
         return cls.dirty(set(DB.query_col(sql, limit=limit)))
 

hive/indexer/cached_post.py

@@ -337,10 +337,17 @@ class CachedPost:
                     sql = """SELECT id, author, permlink, is_deleted
                                FROM hive_posts WHERE id = :id"""
                     row = DB.query_row(sql, id=pid)
-                    if level == 'insert' and not row['is_deleted']:
-                        log.warning("couldnt load post for %s: %s", level, row)
+                    if row['is_deleted']:
+                        log.info("found deleted post for %s: %s", level, row)
+                        if level == 'payout':
+                            log.warning("force delete %s", row)
+                            cls.delete(pid, row['author'], row['permlink'])
+                    elif level == 'insert':
+                        log.error("insert post not found -- DEFER %s", row)
+                        cls.insert(row['author'], row['permlink'], pid)
                     else:
-                        log.info("couldnt load post for %s: %s", level, row)
+                        log.warning("%s post not found -- DEFER %s", level, row)
+                        cls._dirty(level, row['author'], row['permlink'], pid)
 
                 cls._bump_last_id(pid)
 

hive/indexer/sync.py

@@ -60,7 +60,7 @@ class Sync:
         if self._conf.get('test_max_block'):
             # debug mode: partial sync
             return self.from_steemd()
-        elif self._conf.get('test_disable_sync'):
+        if self._conf.get('test_disable_sync'):
             # debug mode: no sync, just stream
             return self.listen()
 
@@ -193,8 +193,8 @@ class Sync:
             if num % 1200 == 0: #1hr
                 Accounts.fetch_ranks()
             if num % 100 == 0: #5min
+                log.info("[LIVE] flag 500 oldest accounts for update")
                 Accounts.dirty_oldest(500)
-                Accounts.flush(steemd, trx=True)
             if num % 20 == 0: #1min
                 self._update_chain_state()
 

hive/server/condenser_api/objects.py

@@ -73,7 +73,7 @@ async def load_posts(db, ids, truncate_body=0):
     # in rare cases of cache inconsistency, recover and warn
     missed = set(ids) - posts_by_id.keys()
     if missed:
-        log.warning("get_posts do not exist in cache: %s", repr(missed))
+        log.info("get_posts do not exist in cache: %s", repr(missed))
         for _id in missed:
             ids.remove(_id)
             sql = ("SELECT id, author, permlink, depth, created_at, is_deleted "
@@ -81,12 +81,9 @@ async def load_posts(db, ids, truncate_body=0):
             post = await db.query_row(sql, id=_id)
             if not post['is_deleted']:
                 # TODO: This should never happen. See #173 for analysis
-                log.error("missing post -- force insert %s", dict(post))
-                sql = """INSERT INTO hive_posts_cache (post_id, author, permlink)
-                              VALUES (:id, :author, :permlink)"""
-                await db.query(sql, **post)
+                log.error("missing post -- %s", dict(post))
             else:
-                log.warning("requested deleted post: %s", dict(post))
+                log.info("requested deleted post: %s", dict(post))
 
     return [posts_by_id[_id] for _id in ids]
 
@@ -94,7 +91,7 @@ async def _query_author_rep_map(db, posts):
     """Given a list of posts, returns an author->reputation map."""
     if not posts:
         return {}
-    names = tuple(set([post['author'] for post in posts]))
+    names = tuple({post['author'] for post in posts})
     sql = "SELECT name, reputation FROM hive_accounts WHERE name IN :names"
     return {r['name']: r['reputation'] for r in await db.query_all(sql, names=names)}
 

hive/server/db.py

@@ -97,8 +97,8 @@ class Db:
         try:
             return await conn.execute(self._sql_text(sql), **kwargs)
         except Exception as e:
-            log.info("[SQL-ERR] %s in query %s (%s)",
-                     e.__class__.__name__, sql, kwargs)
+            log.warning("[SQL-ERR] %s in query %s (%s)",
+                        e.__class__.__name__, sql, kwargs)
             raise e
 
     def _sql_text(self, sql):

hive/steem/http_client.py

@@ -164,7 +164,7 @@ class HttpClient(object):
 
                 if secs > 5:
                     log.warning('%s took %.1fs %s', what, secs, info)
-                if tries > 2:
+                if tries > 1:
                     log.warning('%s took %d tries %s', what, tries, info)
 
                 return result
@@ -176,8 +176,12 @@ class HttpClient(object):
                 if secs < 0: # request failed
                     secs = perf() - start
                     info = {'secs': round(secs, 3), 'try': tries}
-                log.error('%s failed in %.1fs. try %d. %s - %s',
-                          what, secs, tries, info, repr(e))
+                if tries > 1:
+                    log.warning('%s failed in %.1fs. try %d. %s - %s',
+                                what, secs, tries, info, repr(e))
+                else:
+                    log.info('%s failed in %.1fs. try %d. %s - %s',
+                             what, secs, tries, info, repr(e))
 
             if tries % 2 == 0:
                 self.next_node()