Add error handling and statds support

ae15ce81 · John Gerlock · bb576767 · ae15ce81 · bb576767 · ae15ce81
Commit ae15ce81 authored 7 years ago by John Gerlock
--- a/service/nginx/run
+++ b/service/nginx/run
@@ -2,4 +2,4 @@
 sv -w3 check jussi
 chown www-data /dev/stdout
 exec setuidgid www-data /usr/sbin/nginx -c /etc/service/nginx/nginx.conf
\ No newline at end of file
--- a/setup.cfg
+++ b/setup.cfg
-[aliases]
-test=pytest
-[build_sphinx]
-source-dir = docs/
-build-dir = docs/_build
-all_files = 1
-[pycodestyle]
-# formerly pep8
-ignore = E501
-[pep8]
-# backwards compat
-ignore = E501
-[style]
-# google yapf config
-[tool:pytest]
-norecursedirs=dist docs build .tox deploy
-addopts = --pylint
-[coverage:run]
-branch = True
-source = sbds
-[coverage:xml]
-output = coverage.xml
--- a/setup.py
+++ b/setup.py
 #!/usr/bin/env python
+# -*- coding: utf-8 -*-
 import os
 from setuptools import setup
 try:
    from pipenv.project import Project
    from pipenv.utils import convert_deps_to_pip
@@ -16,6 +19,7 @@ base_dir = os.path.dirname(__file__)
 about = {}
+# pylint: disable=exec-used
 with open(os.path.join(base_dir, "jussi", "__about__.py")) as f:
    exec(f.read(), about)

--- a/tests/100_batch_get_block.json
+++ b/tests/100_batch_get_block.json
 [{"id": 0, "jsonrpc": "2.0", "method": "get_block", "params": ["0"]}, {"id": 1, "jsonrpc": "2.0", "method": "get_block", "params": ["1"]}, {"id": 2, "jsonrpc": "2.0", "method": "get_block", "params": ["2"]}, {"id": 3, "jsonrpc": "2.0", "method": "get_block", "params": ["3"]}, {"id": 4, "jsonrpc": "2.0", "method": "get_block", "params": ["4"]}, {"id": 5, "jsonrpc": "2.0", "method": "get_block", "params": ["5"]}, {"id": 6, "jsonrpc": "2.0", "method": "get_block", "params": ["6"]}, {"id": 7, "jsonrpc": "2.0", "method": "get_block", "params": ["7"]}, {"id": 8, "jsonrpc": "2.0", "method": "get_block", "params": ["8"]}, {"id": 9, "jsonrpc": "2.0", "method": "get_block", "params": ["9"]}, {"id": 10, "jsonrpc": "2.0", "method": "get_block", "params": ["10"]}, {"id": 11, "jsonrpc": "2.0", "method": "get_block", "params": ["11"]}, {"id": 12, "jsonrpc": "2.0", "method": "get_block", "params": ["12"]}, {"id": 13, "jsonrpc": "2.0", "method": "get_block", "params": ["13"]}, {"id": 14, "jsonrpc": "2.0", "method": "get_block", "params": ["14"]}, {"id": 15, "jsonrpc": "2.0", "method": "get_block", "params": ["15"]}, {"id": 16, "jsonrpc": "2.0", "method": "get_block", "params": ["16"]}, {"id": 17, "jsonrpc": "2.0", "method": "get_block", "params": ["17"]}, {"id": 18, "jsonrpc": "2.0", "method": "get_block", "params": ["18"]}, {"id": 19, "jsonrpc": "2.0", "method": "get_block", "params": ["19"]}, {"id": 20, "jsonrpc": "2.0", "method": "get_block", "params": ["20"]}, {"id": 21, "jsonrpc": "2.0", "method": "get_block", "params": ["21"]}, {"id": 22, "jsonrpc": "2.0", "method": "get_block", "params": ["22"]}, {"id": 23, "jsonrpc": "2.0", "method": "get_block", "params": ["23"]}, {"id": 24, "jsonrpc": "2.0", "method": "get_block", "params": ["24"]}, {"id": 25, "jsonrpc": "2.0", "method": "get_block", "params": ["25"]}, {"id": 26, "jsonrpc": "2.0", "method": "get_block", "params": ["26"]}, {"id": 27, "jsonrpc": "2.0", "method": "get_block", "params": ["27"]}, {"id": 28, "jsonrpc": "2.0", "method": "get_block", "params": ["28"]}, {"id": 29, "jsonrpc": "2.0", "method": "get_block", "params": ["29"]}, {"id": 30, "jsonrpc": "2.0", "method": "get_block", "params": ["30"]}, {"id": 31, "jsonrpc": "2.0", "method": "get_block", "params": ["31"]}, {"id": 32, "jsonrpc": "2.0", "method": "get_block", "params": ["32"]}, {"id": 33, "jsonrpc": "2.0", "method": "get_block", "params": ["33"]}, {"id": 34, "jsonrpc": "2.0", "method": "get_block", "params": ["34"]}, {"id": 35, "jsonrpc": "2.0", "method": "get_block", "params": ["35"]}, {"id": 36, "jsonrpc": "2.0", "method": "get_block", "params": ["36"]}, {"id": 37, "jsonrpc": "2.0", "method": "get_block", "params": ["37"]}, {"id": 38, "jsonrpc": "2.0", "method": "get_block", "params": ["38"]}, {"id": 39, "jsonrpc": "2.0", "method": "get_block", "params": ["39"]}, {"id": 40, "jsonrpc": "2.0", "method": "get_block", "params": ["40"]}, {"id": 41, "jsonrpc": "2.0", "method": "get_block", "params": ["41"]}, {"id": 42, "jsonrpc": "2.0", "method": "get_block", "params": ["42"]}, {"id": 43, "jsonrpc": "2.0", "method": "get_block", "params": ["43"]}, {"id": 44, "jsonrpc": "2.0", "method": "get_block", "params": ["44"]}, {"id": 45, "jsonrpc": "2.0", "method": "get_block", "params": ["45"]}, {"id": 46, "jsonrpc": "2.0", "method": "get_block", "params": ["46"]}, {"id": 47, "jsonrpc": "2.0", "method": "get_block", "params": ["47"]}, {"id": 48, "jsonrpc": "2.0", "method": "get_block", "params": ["48"]}, {"id": 49, "jsonrpc": "2.0", "method": "get_block", "params": ["49"]}, {"id": 50, "jsonrpc": "2.0", "method": "get_block", "params": ["50"]}, {"id": 51, "jsonrpc": "2.0", "method": "get_block", "params": ["51"]}, {"id": 52, "jsonrpc": "2.0", "method": "get_block", "params": ["52"]}, {"id": 53, "jsonrpc": "2.0", "method": "get_block", "params": ["53"]}, {"id": 54, "jsonrpc": "2.0", "method": "get_block", "params": ["54"]}, {"id": 55, "jsonrpc": "2.0", "method": "get_block", "params": ["55"]}, {"id": 56, "jsonrpc": "2.0", "method": "get_block", "params": ["56"]}, {"id": 57, "jsonrpc": "2.0", "method": "get_block", "params": ["57"]}, {"id": 58, "jsonrpc": "2.0", "method": "get_block", "params": ["58"]}, {"id": 59, "jsonrpc": "2.0", "method": "get_block", "params": ["59"]}, {"id": 60, "jsonrpc": "2.0", "method": "get_block", "params": ["60"]}, {"id": 61, "jsonrpc": "2.0", "method": "get_block", "params": ["61"]}, {"id": 62, "jsonrpc": "2.0", "method": "get_block", "params": ["62"]}, {"id": 63, "jsonrpc": "2.0", "method": "get_block", "params": ["63"]}, {"id": 64, "jsonrpc": "2.0", "method": "get_block", "params": ["64"]}, {"id": 65, "jsonrpc": "2.0", "method": "get_block", "params": ["65"]}, {"id": 66, "jsonrpc": "2.0", "method": "get_block", "params": ["66"]}, {"id": 67, "jsonrpc": "2.0", "method": "get_block", "params": ["67"]}, {"id": 68, "jsonrpc": "2.0", "method": "get_block", "params": ["68"]}, {"id": 69, "jsonrpc": "2.0", "method": "get_block", "params": ["69"]}, {"id": 70, "jsonrpc": "2.0", "method": "get_block", "params": ["70"]}, {"id": 71, "jsonrpc": "2.0", "method": "get_block", "params": ["71"]}, {"id": 72, "jsonrpc": "2.0", "method": "get_block", "params": ["72"]}, {"id": 73, "jsonrpc": "2.0", "method": "get_block", "params": ["73"]}, {"id": 74, "jsonrpc": "2.0", "method": "get_block", "params": ["74"]}, {"id": 75, "jsonrpc": "2.0", "method": "get_block", "params": ["75"]}, {"id": 76, "jsonrpc": "2.0", "method": "get_block", "params": ["76"]}, {"id": 77, "jsonrpc": "2.0", "method": "get_block", "params": ["77"]}, {"id": 78, "jsonrpc": "2.0", "method": "get_block", "params": ["78"]}, {"id": 79, "jsonrpc": "2.0", "method": "get_block", "params": ["79"]}, {"id": 80, "jsonrpc": "2.0", "method": "get_block", "params": ["80"]}, {"id": 81, "jsonrpc": "2.0", "method": "get_block", "params": ["81"]}, {"id": 82, "jsonrpc": "2.0", "method": "get_block", "params": ["82"]}, {"id": 83, "jsonrpc": "2.0", "method": "get_block", "params": ["83"]}, {"id": 84, "jsonrpc": "2.0", "method": "get_block", "params": ["84"]}, {"id": 85, "jsonrpc": "2.0", "method": "get_block", "params": ["85"]}, {"id": 86, "jsonrpc": "2.0", "method": "get_block", "params": ["86"]}, {"id": 87, "jsonrpc": "2.0", "method": "get_block", "params": ["87"]}, {"id": 88, "jsonrpc": "2.0", "method": "get_block", "params": ["88"]}, {"id": 89, "jsonrpc": "2.0", "method": "get_block", "params": ["89"]}, {"id": 90, "jsonrpc": "2.0", "method": "get_block", "params": ["90"]}, {"id": 91, "jsonrpc": "2.0", "method": "get_block", "params": ["91"]}, {"id": 92, "jsonrpc": "2.0", "method": "get_block", "params": ["92"]}, {"id": 93, "jsonrpc": "2.0", "method": "get_block", "params": ["93"]}, {"id": 94, "jsonrpc": "2.0", "method": "get_block", "params": ["94"]}, {"id": 95, "jsonrpc": "2.0", "method": "get_block", "params": ["95"]}, {"id": 96, "jsonrpc": "2.0", "method": "get_block", "params": ["96"]}, {"id": 97, "jsonrpc": "2.0", "method": "get_block", "params": ["97"]}, {"id": 98, "jsonrpc": "2.0", "method": "get_block", "params": ["98"]}, {"id": 99, "jsonrpc": "2.0", "method": "get_block", "params": ["99"]}]
\ No newline at end of file
--- a/tests/10_batch_get_block.json
+++ b/tests/10_batch_get_block.json
 [{"id": 0, "jsonrpc": "2.0", "method": "get_block", "params": ["0"]}, {"id": 1, "jsonrpc": "2.0", "method": "get_block", "params": ["1"]}, {"id": 2, "jsonrpc": "2.0", "method": "get_block", "params": ["2"]}, {"id": 3, "jsonrpc": "2.0", "method": "get_block", "params": ["3"]}, {"id": 4, "jsonrpc": "2.0", "method": "get_block", "params": ["4"]}, {"id": 5, "jsonrpc": "2.0", "method": "get_block", "params": ["5"]}, {"id": 6, "jsonrpc": "2.0", "method": "get_block", "params": ["6"]}, {"id": 7, "jsonrpc": "2.0", "method": "get_block", "params": ["7"]}, {"id": 8, "jsonrpc": "2.0", "method": "get_block", "params": ["8"]}, {"id": 9, "jsonrpc": "2.0", "method": "get_block", "params": ["9"]}]
\ No newline at end of file
--- a/tests/10k_batch_get_block.json
+++ b/tests/10k_batch_get_block.json
--- a/tests/1k_batch_get_block.json
+++ b/tests/1k_batch_get_block.json
--- a/tests/conftests.py
+++ b/tests/conftests.py
-# coding=utf-8
--- a/tests/http_client.py
+++ b/tests/http_client.py
-# coding=utf-8
+# -*- coding: utf-8 -*-
+import concurrent.futures
 import json
-import os
 import logging
+import os
 import socket
-import concurrent.futures
 import time
 from functools import partial
 from functools import partialmethod
@@ -13,8 +13,6 @@ import certifi
 import urllib3
 from urllib3.connection import HTTPConnection
 logger = logging.getLogger(__name__)
@@ -25,6 +23,7 @@ class RPCError(Exception):
 class RPCConnectionError(Exception):
    pass
 def chunkify(iterable, chunksize=3000):
    i = 0
    chunk = []
@@ -35,9 +34,10 @@ def chunkify(iterable, chunksize=3000):
            yield chunk
            i = 0
            chunk = []
-    if len(chunk) > 0:
+    if chunk:
        yield chunk
 class SimpleSteemAPIClient(object):
    """Simple Steem JSON-HTTP-RPC API
@@ -64,8 +64,9 @@ class SimpleSteemAPIClient(object):
    """
-    def __init__(self, url=None, log_level=logging.INFO, **kwargs):
+    def __init__(self, url=None, **kwargs):
-        url = url or os.environ.get('STEEMD_HTTP_URL', 'https://steemd.steemitdev.com')
+        url = url or os.environ.get('STEEMD_HTTP_URL',
+                                    'https://steemd.steemitdev.com')
        self.url = url
        self.hostname = urlparse(url).hostname
        self.return_with_args = kwargs.get('return_with_args', False)
@@ -105,14 +106,12 @@ class SimpleSteemAPIClient(object):
        _logger = logging.getLogger('urllib3')
    @staticmethod
    def json_rpc_body(name, *args, as_json=True):
        body_dict = {"method": name, "params": args, "jsonrpc": "2.0", "id": 0}
        if as_json:
            return json.dumps(body_dict, ensure_ascii=False).encode('utf8')
-        else:
+        return body_dict
-            return body_dict
    def exec(self, name, *args, re_raise=None, return_with_args=None):
        body = SimpleSteemAPIClient.json_rpc_body(name, *args)
@@ -162,8 +161,7 @@ class SimpleSteemAPIClient(object):
                result = response_json.get('result', None)
        if return_with_args:
            return result, args
-        else:
+        return result
-            return result
    def exec_multi(self, name, params):
        body_gen = ({
@@ -173,29 +171,30 @@ class SimpleSteemAPIClient(object):
            "id": i
        } for i in params)
        for chunk in chunkify(body_gen):
-            batch_json_body = json.dumps(chunk, ensure_ascii=False).encode('utf8')
+            batch_json_body = json.dumps(
+                chunk, ensure_ascii=False).encode('utf8')
            r = self.request(body=batch_json_body).read()
            print(r)
-            batch_response = json.loads(self.request(body=batch_json_body).read())
+            batch_response = json.loads(
-            for i,resp in enumerate(batch_response):
+                self.request(body=batch_json_body).read())
+            for i, resp in enumerate(batch_response):
                yield self._return(
-                        response=resp,
+                    response=resp,
-                        args=batch_json_body[i]['params'],
+                    args=batch_json_body[i]['params'],
-                        return_with_args=True)
+                    return_with_args=True)
    def exec_batch(self, name, params):
        batch_requests = [{
-            "method":  name,
+            "method": name,
-            "params":   [str(i)],
+            "params": [str(i)],
            "jsonrpc": "2.0",
-            "id":      i
+            "id": i
        } for i in params]
        for chunk in chunkify(batch_requests):
            batch_json_body = json.dumps(chunk).encode()
            r = self.request(body=batch_json_body)
            batch_response = json.loads(r.data.decode())
-            for i,resp in enumerate(batch_response):
+            for resp in batch_response:
                yield json.dumps(resp)
    def exec_batch_with_futures(self, name, params, max_workers=None):
@@ -209,9 +208,6 @@ class SimpleSteemAPIClient(object):
                for item in future.result():
                    yield item
    get_dynamic_global_properties = partialmethod(
        exec, 'get_dynamic_global_properties')

--- a/tests/perf_test.py
+++ b/tests/perf_test.py
 # -*- coding: utf-8 -*-
 import argparse
+import logging
 import os
 import sys
-import logging
-sys.path.append(os.path.dirname(__file__))
-from multiprocessing import Pool
 from functools import partial
+from multiprocessing import Pool
 import http_client
-logger = logging.getLogger(__name__)
+sys.path.append(os.path.dirname(__file__))
+logger = logging.getLogger(__name__)
 # pylint: skip-file
 MAX_CHUNKSIZE = 1000000
@@ -33,11 +32,7 @@ def chunkify(iterable, chunksize=10000):
        yield chunk
-def fetch_blocks(block_nums,
+def fetch_blocks(block_nums, max_procs, max_threads, steemd_http_url):
-                            max_procs,
-                            max_threads,
-                            steemd_http_url):
    max_workers = max_procs or os.cpu_count() or 1
@@ -46,33 +41,27 @@ def fetch_blocks(block_nums,
        chunksize = 1
    map_func = partial(
-        block_adder_process_worker,
+        block_adder_process_worker, steemd_http_url, max_threads=max_threads)
-        steemd_http_url,
-        max_threads=max_threads)
    chunks = chunkify(block_nums, chunksize)
    with Pool(processes=max_workers) as pool:
        results = pool.map(map_func, chunks)
-    #print(results)
+    # print(results)
 def do_test(steemd_http_url, max_procs, max_threads, start=None, end=None):
    client = http_client.SimpleSteemAPIClient(url=steemd_http_url)
    try:
        start = start or 1
        end = end or client.block_height()
-        missing_block_nums = list(range(start,end))
+        missing_block_nums = list(range(start, end))
        # [2/2] adding missing blocks
-        fetch_blocks(
+        fetch_blocks(missing_block_nums, max_procs, max_threads,
-            missing_block_nums,
+                     steemd_http_url)
-            max_procs,
-            max_threads,
-            steemd_http_url)
    except KeyboardInterrupt:
        pass
@@ -87,16 +76,15 @@ def do_test(steemd_http_url, max_procs, max_threads, start=None, end=None):
 def block_fetcher_thread_worker(rpc_url, block_nums, max_threads=None):
    rpc = http_client.SimpleSteemAPIClient(rpc_url, return_with_args=False)
    # pylint: disable=unused-variable
-    for block in rpc.exec_batch('get_block', block_nums): #, max_workers=max_threads):
+    for block in rpc.exec_batch('get_block',
+                                block_nums):  # , max_workers=max_threads):
        yield block
-def block_adder_process_worker(
+def block_adder_process_worker(rpc_url, block_nums, max_threads=5):
-                               rpc_url,
-                               block_nums,
-                               max_threads=5):
    rpc = http_client.SimpleSteemAPIClient(rpc_url, return_with_args=False)
-    for block in rpc.exec_batch('get_block', block_nums): #, max_workers=max_threads):
+    for block in rpc.exec_batch('get_block',
+                                block_nums):  # , max_workers=max_threads):
        print(block)
@@ -104,10 +92,15 @@ def block_adder_process_worker(
 # using the click framework
 if __name__ == '__main__':
    parser = argparse.ArgumentParser('jussi perf test script')
-    parser.add_argument('url',type=str)
+    parser.add_argument('url', type=str)
-    parser.add_argument('--max_procs',type=int, default=os.cpu_count() -1)
+    parser.add_argument('--max_procs', type=int, default=os.cpu_count() - 1)
-    parser.add_argument('--max_threads',type=int, default=30) 
+    parser.add_argument('--max_threads', type=int, default=30)
-    parser.add_argument('--start',type=int, default=1) 
+    parser.add_argument('--start', type=int, default=1)
-    parser.add_argument('--end',type=int, default=None) 
+    parser.add_argument('--end', type=int, default=None)
    args = parser.parse_args()
-    do_test(args.url, max_procs=args.max_procs, max_threads=args.max_threads,start=args.start, end=args.end)
+    do_test(
+        args.url,
+        max_procs=args.max_procs,
+        max_threads=args.max_threads,
+        start=args.start,
+        end=args.end)