bump up LRU post map to 2M entries

README.md

@@ -68,7 +68,7 @@ Precedence: CLI over ENV over hive.conf. Check `hive --help` for details.
 ### Hardware
 
  - Focus on Postgres performance
- - 2GB of memory for hive itself (TODO: verify/limit max usage during initial sync)
+ - 2.5GB of memory for `hive sync` process
  - 200GB storage for database
 
 

hive/indexer/posts.py

@@ -18,6 +18,7 @@ class Posts:
     """Handles critical/core post ops and data."""
 
     # LRU cache for (author-permlink -> id) lookup
+    CACHE_SIZE = 2000000
     _ids = collections.OrderedDict()
     _hits = 0
     _miss = 0
@@ -56,7 +57,7 @@ class Posts:
     def _set_id(cls, url, pid):
         """Add an entry to the LRU, maintaining max size."""
         assert pid, "no pid provided for %s" % url
-        if len(cls._ids) > 1000000:
+        if len(cls._ids) > cls.CACHE_SIZE:
             cls._ids.popitem(last=False)
         cls._ids[url] = pid