Specialized endpoints support

README.md

@@ -63,7 +63,7 @@ Hivemind is deployed as a Docker container.
 Here is an example command that will initialize the DB schema and start the syncing process:
 
 ```
-docker run -d --name hivemind --env DATABASE_URL=postgresql://user:pass@hostname:5432/databasename --env STEEMD_URL=https://yoursteemnode --env SYNC_SERVICE=1 -p 8080:8080 steemit/hivemind:latest
+docker run -d --name hivemind --env DATABASE_URL=postgresql://user:pass@hostname:5432/databasename --env STEEMD_URL='{"default":"https://yoursteemnode"}' --env SYNC_SERVICE=1 -p 8080:8080 steemit/hivemind:latest
 ```
 
 Be sure to set `DATABASE_URL` to point to your postgres database and `STEEMD_URL` to point to your steemd node to sync from.
@@ -84,7 +84,7 @@ docker logs -f hivemind
 | `LOG_LEVEL`              | `--log-level`        | INFO    |
 | `HTTP_SERVER_PORT`       | `--http-server-port` | 8080    |
 | `DATABASE_URL`           | `--database-url`     | postgresql://user:pass@localhost:5432/hive |
-| `STEEMD_URL`             | `--steemd-url`       | https://api.steemit.com |
+| `STEEMD_URL`             | `--steemd-url`       | '{"default":"https://yoursteemnode"}' |
 | `MAX_BATCH`              | `--max-batch`        | 50      |
 | `MAX_WORKERS`            | `--max-workers`      | 4       |
 | `TRAIL_BLOCKS`           | `--trail-blocks`     | 2       |

docker-compose.yml

@@ -16,7 +16,7 @@ services:
     environment:
       DATABASE_URL: postgresql://testuser:testuserpass@db:5432/testdb
       LOG_LEVEL: INFO
-      STEEMD_URL: https://api.steemit.com
+      STEEMD_URL: '{"default" : "https://api.steemit.com"}'
     links:
       - db:db
     ports:

hive/conf.py

@@ -32,7 +32,7 @@ class Conf():
 
         # common
         add('--database-url', env_var='DATABASE_URL', required=False, help='database connection url', default='')
-        add('--steemd-url', env_var='STEEMD_URL', required=False, help='steemd/jussi endpoint', default='https://api.steemit.com')
+        add('--steemd-url', env_var='STEEMD_URL', required=False, help='steemd/jussi endpoint', default='{"default" : "https://api.steemit.com"}')
         add('--muted-accounts-url', env_var='MUTED_ACCOUNTS_URL', required=False, help='url to flat list of muted accounts', default='')
 
         # server
@@ -85,8 +85,9 @@ class Conf():
     def steem(self):
         """Get a SteemClient instance, lazily initialized"""
         if not self._steem:
+            from json import loads
             self._steem = SteemClient(
-                url=self.get('steemd_url'),
+                url=loads(self.get('steemd_url')),
                 max_batch=self.get('max_batch'),
                 max_workers=self.get('max_workers'))
         return self._steem

hive/steem/client.py

@@ -10,15 +10,19 @@ from hive.steem.block.stream import BlockStream
 
 class SteemClient:
     """Handles upstream calls to jussi/steemd, with batching and retrying."""
-
-    def __init__(self, url='https://api.steemit.com', max_batch=50, max_workers=1):
-        assert url, 'steem-API endpoint undefined'
+    # dangerous default value of url but it should be fine since we are not writting to it
+    def __init__(self, url={"default" : 'https://api.steemit.com'}, max_batch=50, max_workers=1):
+        assert url, 'steem-API endpoints undefined'
+        assert "default" in url, "Url should have default endpoint defined"
         assert max_batch > 0 and max_batch <= 5000
         assert max_workers > 0 and max_workers <= 64
 
         self._max_batch = max_batch
         self._max_workers = max_workers
-        self._client = HttpClient(nodes=[url])
+        self._client = dict()
+        for endpoint, endpoint_url in url.items():
+            print("Endpoint {} will be routed to node {}".format(endpoint, endpoint_url))
+            self._client[endpoint] = HttpClient(nodes=[endpoint_url])
 
     def get_accounts(self, accounts):
         """Fetch multiple accounts by name."""
@@ -135,7 +139,11 @@ class SteemClient:
     def __exec(self, method, params=None):
         """Perform a single steemd call."""
         start = perf()
-        result = self._client.exec(method, params)
+        result = None
+        if method in self._client:
+            result = self._client[method].exec(method, params)
+        else:
+            result = self._client["default"].exec(method, params)
         items = len(params[0]) if method == 'get_accounts' else 1
         Stats.log_steem(method, perf() - start, items)
         return result
@@ -145,12 +153,20 @@ class SteemClient:
         start = perf()
 
         result = []
-        for part in self._client.exec_multi(
-                method,
-                params,
-                max_workers=self._max_workers,
-                batch_size=self._max_batch):
-            result.extend(part)
+        if method in self._client:
+            for part in self._client[method].exec_multi(
+                    method,
+                    params,
+                    max_workers=self._max_workers,
+                    batch_size=self._max_batch):
+                result.extend(part)
+        else:
+            for part in self._client["default"].exec_multi(
+                    method,
+                    params,
+                    max_workers=self._max_workers,
+                    batch_size=self._max_batch):
+                result.extend(part)
 
         Stats.log_steem(method, perf() - start, len(params))
         return result