Skip to content
Snippets Groups Projects
Commit ae15ce81 authored by John Gerlock's avatar John Gerlock
Browse files

Add error handling and statds support

parent bb576767
No related branches found
No related tags found
No related merge requests found
......@@ -2,4 +2,4 @@
sv -w3 check jussi
chown www-data /dev/stdout
exec setuidgid www-data /usr/sbin/nginx -c /etc/service/nginx/nginx.conf
\ No newline at end of file
exec setuidgid www-data /usr/sbin/nginx -c /etc/service/nginx/nginx.conf
[aliases]
test=pytest
[build_sphinx]
source-dir = docs/
build-dir = docs/_build
all_files = 1
[pycodestyle]
# formerly pep8
ignore = E501
[pep8]
# backwards compat
ignore = E501
[style]
# google yapf config
[tool:pytest]
norecursedirs=dist docs build .tox deploy
addopts = --pylint
[coverage:run]
branch = True
source = sbds
[coverage:xml]
output = coverage.xml
setup.py 100644 → 100755
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
from setuptools import setup
try:
from pipenv.project import Project
from pipenv.utils import convert_deps_to_pip
......@@ -16,6 +19,7 @@ base_dir = os.path.dirname(__file__)
about = {}
# pylint: disable=exec-used
with open(os.path.join(base_dir, "jussi", "__about__.py")) as f:
exec(f.read(), about)
......
[{"id": 0, "jsonrpc": "2.0", "method": "get_block", "params": ["0"]}, {"id": 1, "jsonrpc": "2.0", "method": "get_block", "params": ["1"]}, {"id": 2, "jsonrpc": "2.0", "method": "get_block", "params": ["2"]}, {"id": 3, "jsonrpc": "2.0", "method": "get_block", "params": ["3"]}, {"id": 4, "jsonrpc": "2.0", "method": "get_block", "params": ["4"]}, {"id": 5, "jsonrpc": "2.0", "method": "get_block", "params": ["5"]}, {"id": 6, "jsonrpc": "2.0", "method": "get_block", "params": ["6"]}, {"id": 7, "jsonrpc": "2.0", "method": "get_block", "params": ["7"]}, {"id": 8, "jsonrpc": "2.0", "method": "get_block", "params": ["8"]}, {"id": 9, "jsonrpc": "2.0", "method": "get_block", "params": ["9"]}, {"id": 10, "jsonrpc": "2.0", "method": "get_block", "params": ["10"]}, {"id": 11, "jsonrpc": "2.0", "method": "get_block", "params": ["11"]}, {"id": 12, "jsonrpc": "2.0", "method": "get_block", "params": ["12"]}, {"id": 13, "jsonrpc": "2.0", "method": "get_block", "params": ["13"]}, {"id": 14, "jsonrpc": "2.0", "method": "get_block", "params": ["14"]}, {"id": 15, "jsonrpc": "2.0", "method": "get_block", "params": ["15"]}, {"id": 16, "jsonrpc": "2.0", "method": "get_block", "params": ["16"]}, {"id": 17, "jsonrpc": "2.0", "method": "get_block", "params": ["17"]}, {"id": 18, "jsonrpc": "2.0", "method": "get_block", "params": ["18"]}, {"id": 19, "jsonrpc": "2.0", "method": "get_block", "params": ["19"]}, {"id": 20, "jsonrpc": "2.0", "method": "get_block", "params": ["20"]}, {"id": 21, "jsonrpc": "2.0", "method": "get_block", "params": ["21"]}, {"id": 22, "jsonrpc": "2.0", "method": "get_block", "params": ["22"]}, {"id": 23, "jsonrpc": "2.0", "method": "get_block", "params": ["23"]}, {"id": 24, "jsonrpc": "2.0", "method": "get_block", "params": ["24"]}, {"id": 25, "jsonrpc": "2.0", "method": "get_block", "params": ["25"]}, {"id": 26, "jsonrpc": "2.0", "method": "get_block", "params": ["26"]}, {"id": 27, "jsonrpc": "2.0", "method": "get_block", "params": ["27"]}, {"id": 28, "jsonrpc": "2.0", "method": "get_block", "params": ["28"]}, {"id": 29, "jsonrpc": "2.0", "method": "get_block", "params": ["29"]}, {"id": 30, "jsonrpc": "2.0", "method": "get_block", "params": ["30"]}, {"id": 31, "jsonrpc": "2.0", "method": "get_block", "params": ["31"]}, {"id": 32, "jsonrpc": "2.0", "method": "get_block", "params": ["32"]}, {"id": 33, "jsonrpc": "2.0", "method": "get_block", "params": ["33"]}, {"id": 34, "jsonrpc": "2.0", "method": "get_block", "params": ["34"]}, {"id": 35, "jsonrpc": "2.0", "method": "get_block", "params": ["35"]}, {"id": 36, "jsonrpc": "2.0", "method": "get_block", "params": ["36"]}, {"id": 37, "jsonrpc": "2.0", "method": "get_block", "params": ["37"]}, {"id": 38, "jsonrpc": "2.0", "method": "get_block", "params": ["38"]}, {"id": 39, "jsonrpc": "2.0", "method": "get_block", "params": ["39"]}, {"id": 40, "jsonrpc": "2.0", "method": "get_block", "params": ["40"]}, {"id": 41, "jsonrpc": "2.0", "method": "get_block", "params": ["41"]}, {"id": 42, "jsonrpc": "2.0", "method": "get_block", "params": ["42"]}, {"id": 43, "jsonrpc": "2.0", "method": "get_block", "params": ["43"]}, {"id": 44, "jsonrpc": "2.0", "method": "get_block", "params": ["44"]}, {"id": 45, "jsonrpc": "2.0", "method": "get_block", "params": ["45"]}, {"id": 46, "jsonrpc": "2.0", "method": "get_block", "params": ["46"]}, {"id": 47, "jsonrpc": "2.0", "method": "get_block", "params": ["47"]}, {"id": 48, "jsonrpc": "2.0", "method": "get_block", "params": ["48"]}, {"id": 49, "jsonrpc": "2.0", "method": "get_block", "params": ["49"]}, {"id": 50, "jsonrpc": "2.0", "method": "get_block", "params": ["50"]}, {"id": 51, "jsonrpc": "2.0", "method": "get_block", "params": ["51"]}, {"id": 52, "jsonrpc": "2.0", "method": "get_block", "params": ["52"]}, {"id": 53, "jsonrpc": "2.0", "method": "get_block", "params": ["53"]}, {"id": 54, "jsonrpc": "2.0", "method": "get_block", "params": ["54"]}, {"id": 55, "jsonrpc": "2.0", "method": "get_block", "params": ["55"]}, {"id": 56, "jsonrpc": "2.0", "method": "get_block", "params": ["56"]}, {"id": 57, "jsonrpc": "2.0", "method": "get_block", "params": ["57"]}, {"id": 58, "jsonrpc": "2.0", "method": "get_block", "params": ["58"]}, {"id": 59, "jsonrpc": "2.0", "method": "get_block", "params": ["59"]}, {"id": 60, "jsonrpc": "2.0", "method": "get_block", "params": ["60"]}, {"id": 61, "jsonrpc": "2.0", "method": "get_block", "params": ["61"]}, {"id": 62, "jsonrpc": "2.0", "method": "get_block", "params": ["62"]}, {"id": 63, "jsonrpc": "2.0", "method": "get_block", "params": ["63"]}, {"id": 64, "jsonrpc": "2.0", "method": "get_block", "params": ["64"]}, {"id": 65, "jsonrpc": "2.0", "method": "get_block", "params": ["65"]}, {"id": 66, "jsonrpc": "2.0", "method": "get_block", "params": ["66"]}, {"id": 67, "jsonrpc": "2.0", "method": "get_block", "params": ["67"]}, {"id": 68, "jsonrpc": "2.0", "method": "get_block", "params": ["68"]}, {"id": 69, "jsonrpc": "2.0", "method": "get_block", "params": ["69"]}, {"id": 70, "jsonrpc": "2.0", "method": "get_block", "params": ["70"]}, {"id": 71, "jsonrpc": "2.0", "method": "get_block", "params": ["71"]}, {"id": 72, "jsonrpc": "2.0", "method": "get_block", "params": ["72"]}, {"id": 73, "jsonrpc": "2.0", "method": "get_block", "params": ["73"]}, {"id": 74, "jsonrpc": "2.0", "method": "get_block", "params": ["74"]}, {"id": 75, "jsonrpc": "2.0", "method": "get_block", "params": ["75"]}, {"id": 76, "jsonrpc": "2.0", "method": "get_block", "params": ["76"]}, {"id": 77, "jsonrpc": "2.0", "method": "get_block", "params": ["77"]}, {"id": 78, "jsonrpc": "2.0", "method": "get_block", "params": ["78"]}, {"id": 79, "jsonrpc": "2.0", "method": "get_block", "params": ["79"]}, {"id": 80, "jsonrpc": "2.0", "method": "get_block", "params": ["80"]}, {"id": 81, "jsonrpc": "2.0", "method": "get_block", "params": ["81"]}, {"id": 82, "jsonrpc": "2.0", "method": "get_block", "params": ["82"]}, {"id": 83, "jsonrpc": "2.0", "method": "get_block", "params": ["83"]}, {"id": 84, "jsonrpc": "2.0", "method": "get_block", "params": ["84"]}, {"id": 85, "jsonrpc": "2.0", "method": "get_block", "params": ["85"]}, {"id": 86, "jsonrpc": "2.0", "method": "get_block", "params": ["86"]}, {"id": 87, "jsonrpc": "2.0", "method": "get_block", "params": ["87"]}, {"id": 88, "jsonrpc": "2.0", "method": "get_block", "params": ["88"]}, {"id": 89, "jsonrpc": "2.0", "method": "get_block", "params": ["89"]}, {"id": 90, "jsonrpc": "2.0", "method": "get_block", "params": ["90"]}, {"id": 91, "jsonrpc": "2.0", "method": "get_block", "params": ["91"]}, {"id": 92, "jsonrpc": "2.0", "method": "get_block", "params": ["92"]}, {"id": 93, "jsonrpc": "2.0", "method": "get_block", "params": ["93"]}, {"id": 94, "jsonrpc": "2.0", "method": "get_block", "params": ["94"]}, {"id": 95, "jsonrpc": "2.0", "method": "get_block", "params": ["95"]}, {"id": 96, "jsonrpc": "2.0", "method": "get_block", "params": ["96"]}, {"id": 97, "jsonrpc": "2.0", "method": "get_block", "params": ["97"]}, {"id": 98, "jsonrpc": "2.0", "method": "get_block", "params": ["98"]}, {"id": 99, "jsonrpc": "2.0", "method": "get_block", "params": ["99"]}]
\ No newline at end of file
[{"id": 0, "jsonrpc": "2.0", "method": "get_block", "params": ["0"]}, {"id": 1, "jsonrpc": "2.0", "method": "get_block", "params": ["1"]}, {"id": 2, "jsonrpc": "2.0", "method": "get_block", "params": ["2"]}, {"id": 3, "jsonrpc": "2.0", "method": "get_block", "params": ["3"]}, {"id": 4, "jsonrpc": "2.0", "method": "get_block", "params": ["4"]}, {"id": 5, "jsonrpc": "2.0", "method": "get_block", "params": ["5"]}, {"id": 6, "jsonrpc": "2.0", "method": "get_block", "params": ["6"]}, {"id": 7, "jsonrpc": "2.0", "method": "get_block", "params": ["7"]}, {"id": 8, "jsonrpc": "2.0", "method": "get_block", "params": ["8"]}, {"id": 9, "jsonrpc": "2.0", "method": "get_block", "params": ["9"]}, {"id": 10, "jsonrpc": "2.0", "method": "get_block", "params": ["10"]}, {"id": 11, "jsonrpc": "2.0", "method": "get_block", "params": ["11"]}, {"id": 12, "jsonrpc": "2.0", "method": "get_block", "params": ["12"]}, {"id": 13, "jsonrpc": "2.0", "method": "get_block", "params": ["13"]}, {"id": 14, "jsonrpc": "2.0", "method": "get_block", "params": ["14"]}, {"id": 15, "jsonrpc": "2.0", "method": "get_block", "params": ["15"]}, {"id": 16, "jsonrpc": "2.0", "method": "get_block", "params": ["16"]}, {"id": 17, "jsonrpc": "2.0", "method": "get_block", "params": ["17"]}, {"id": 18, "jsonrpc": "2.0", "method": "get_block", "params": ["18"]}, {"id": 19, "jsonrpc": "2.0", "method": "get_block", "params": ["19"]}, {"id": 20, "jsonrpc": "2.0", "method": "get_block", "params": ["20"]}, {"id": 21, "jsonrpc": "2.0", "method": "get_block", "params": ["21"]}, {"id": 22, "jsonrpc": "2.0", "method": "get_block", "params": ["22"]}, {"id": 23, "jsonrpc": "2.0", "method": "get_block", "params": ["23"]}, {"id": 24, "jsonrpc": "2.0", "method": "get_block", "params": ["24"]}, {"id": 25, "jsonrpc": "2.0", "method": "get_block", "params": ["25"]}, {"id": 26, "jsonrpc": "2.0", "method": "get_block", "params": ["26"]}, {"id": 27, "jsonrpc": "2.0", "method": "get_block", "params": ["27"]}, {"id": 28, "jsonrpc": "2.0", "method": "get_block", "params": ["28"]}, {"id": 29, "jsonrpc": "2.0", "method": "get_block", "params": ["29"]}, {"id": 30, "jsonrpc": "2.0", "method": "get_block", "params": ["30"]}, {"id": 31, "jsonrpc": "2.0", "method": "get_block", "params": ["31"]}, {"id": 32, "jsonrpc": "2.0", "method": "get_block", "params": ["32"]}, {"id": 33, "jsonrpc": "2.0", "method": "get_block", "params": ["33"]}, {"id": 34, "jsonrpc": "2.0", "method": "get_block", "params": ["34"]}, {"id": 35, "jsonrpc": "2.0", "method": "get_block", "params": ["35"]}, {"id": 36, "jsonrpc": "2.0", "method": "get_block", "params": ["36"]}, {"id": 37, "jsonrpc": "2.0", "method": "get_block", "params": ["37"]}, {"id": 38, "jsonrpc": "2.0", "method": "get_block", "params": ["38"]}, {"id": 39, "jsonrpc": "2.0", "method": "get_block", "params": ["39"]}, {"id": 40, "jsonrpc": "2.0", "method": "get_block", "params": ["40"]}, {"id": 41, "jsonrpc": "2.0", "method": "get_block", "params": ["41"]}, {"id": 42, "jsonrpc": "2.0", "method": "get_block", "params": ["42"]}, {"id": 43, "jsonrpc": "2.0", "method": "get_block", "params": ["43"]}, {"id": 44, "jsonrpc": "2.0", "method": "get_block", "params": ["44"]}, {"id": 45, "jsonrpc": "2.0", "method": "get_block", "params": ["45"]}, {"id": 46, "jsonrpc": "2.0", "method": "get_block", "params": ["46"]}, {"id": 47, "jsonrpc": "2.0", "method": "get_block", "params": ["47"]}, {"id": 48, "jsonrpc": "2.0", "method": "get_block", "params": ["48"]}, {"id": 49, "jsonrpc": "2.0", "method": "get_block", "params": ["49"]}, {"id": 50, "jsonrpc": "2.0", "method": "get_block", "params": ["50"]}, {"id": 51, "jsonrpc": "2.0", "method": "get_block", "params": ["51"]}, {"id": 52, "jsonrpc": "2.0", "method": "get_block", "params": ["52"]}, {"id": 53, "jsonrpc": "2.0", "method": "get_block", "params": ["53"]}, {"id": 54, "jsonrpc": "2.0", "method": "get_block", "params": ["54"]}, {"id": 55, "jsonrpc": "2.0", "method": "get_block", "params": ["55"]}, {"id": 56, "jsonrpc": "2.0", "method": "get_block", "params": ["56"]}, {"id": 57, "jsonrpc": "2.0", "method": "get_block", "params": ["57"]}, {"id": 58, "jsonrpc": "2.0", "method": "get_block", "params": ["58"]}, {"id": 59, "jsonrpc": "2.0", "method": "get_block", "params": ["59"]}, {"id": 60, "jsonrpc": "2.0", "method": "get_block", "params": ["60"]}, {"id": 61, "jsonrpc": "2.0", "method": "get_block", "params": ["61"]}, {"id": 62, "jsonrpc": "2.0", "method": "get_block", "params": ["62"]}, {"id": 63, "jsonrpc": "2.0", "method": "get_block", "params": ["63"]}, {"id": 64, "jsonrpc": "2.0", "method": "get_block", "params": ["64"]}, {"id": 65, "jsonrpc": "2.0", "method": "get_block", "params": ["65"]}, {"id": 66, "jsonrpc": "2.0", "method": "get_block", "params": ["66"]}, {"id": 67, "jsonrpc": "2.0", "method": "get_block", "params": ["67"]}, {"id": 68, "jsonrpc": "2.0", "method": "get_block", "params": ["68"]}, {"id": 69, "jsonrpc": "2.0", "method": "get_block", "params": ["69"]}, {"id": 70, "jsonrpc": "2.0", "method": "get_block", "params": ["70"]}, {"id": 71, "jsonrpc": "2.0", "method": "get_block", "params": ["71"]}, {"id": 72, "jsonrpc": "2.0", "method": "get_block", "params": ["72"]}, {"id": 73, "jsonrpc": "2.0", "method": "get_block", "params": ["73"]}, {"id": 74, "jsonrpc": "2.0", "method": "get_block", "params": ["74"]}, {"id": 75, "jsonrpc": "2.0", "method": "get_block", "params": ["75"]}, {"id": 76, "jsonrpc": "2.0", "method": "get_block", "params": ["76"]}, {"id": 77, "jsonrpc": "2.0", "method": "get_block", "params": ["77"]}, {"id": 78, "jsonrpc": "2.0", "method": "get_block", "params": ["78"]}, {"id": 79, "jsonrpc": "2.0", "method": "get_block", "params": ["79"]}, {"id": 80, "jsonrpc": "2.0", "method": "get_block", "params": ["80"]}, {"id": 81, "jsonrpc": "2.0", "method": "get_block", "params": ["81"]}, {"id": 82, "jsonrpc": "2.0", "method": "get_block", "params": ["82"]}, {"id": 83, "jsonrpc": "2.0", "method": "get_block", "params": ["83"]}, {"id": 84, "jsonrpc": "2.0", "method": "get_block", "params": ["84"]}, {"id": 85, "jsonrpc": "2.0", "method": "get_block", "params": ["85"]}, {"id": 86, "jsonrpc": "2.0", "method": "get_block", "params": ["86"]}, {"id": 87, "jsonrpc": "2.0", "method": "get_block", "params": ["87"]}, {"id": 88, "jsonrpc": "2.0", "method": "get_block", "params": ["88"]}, {"id": 89, "jsonrpc": "2.0", "method": "get_block", "params": ["89"]}, {"id": 90, "jsonrpc": "2.0", "method": "get_block", "params": ["90"]}, {"id": 91, "jsonrpc": "2.0", "method": "get_block", "params": ["91"]}, {"id": 92, "jsonrpc": "2.0", "method": "get_block", "params": ["92"]}, {"id": 93, "jsonrpc": "2.0", "method": "get_block", "params": ["93"]}, {"id": 94, "jsonrpc": "2.0", "method": "get_block", "params": ["94"]}, {"id": 95, "jsonrpc": "2.0", "method": "get_block", "params": ["95"]}, {"id": 96, "jsonrpc": "2.0", "method": "get_block", "params": ["96"]}, {"id": 97, "jsonrpc": "2.0", "method": "get_block", "params": ["97"]}, {"id": 98, "jsonrpc": "2.0", "method": "get_block", "params": ["98"]}, {"id": 99, "jsonrpc": "2.0", "method": "get_block", "params": ["99"]}]
[{"id": 0, "jsonrpc": "2.0", "method": "get_block", "params": ["0"]}, {"id": 1, "jsonrpc": "2.0", "method": "get_block", "params": ["1"]}, {"id": 2, "jsonrpc": "2.0", "method": "get_block", "params": ["2"]}, {"id": 3, "jsonrpc": "2.0", "method": "get_block", "params": ["3"]}, {"id": 4, "jsonrpc": "2.0", "method": "get_block", "params": ["4"]}, {"id": 5, "jsonrpc": "2.0", "method": "get_block", "params": ["5"]}, {"id": 6, "jsonrpc": "2.0", "method": "get_block", "params": ["6"]}, {"id": 7, "jsonrpc": "2.0", "method": "get_block", "params": ["7"]}, {"id": 8, "jsonrpc": "2.0", "method": "get_block", "params": ["8"]}, {"id": 9, "jsonrpc": "2.0", "method": "get_block", "params": ["9"]}]
\ No newline at end of file
[{"id": 0, "jsonrpc": "2.0", "method": "get_block", "params": ["0"]}, {"id": 1, "jsonrpc": "2.0", "method": "get_block", "params": ["1"]}, {"id": 2, "jsonrpc": "2.0", "method": "get_block", "params": ["2"]}, {"id": 3, "jsonrpc": "2.0", "method": "get_block", "params": ["3"]}, {"id": 4, "jsonrpc": "2.0", "method": "get_block", "params": ["4"]}, {"id": 5, "jsonrpc": "2.0", "method": "get_block", "params": ["5"]}, {"id": 6, "jsonrpc": "2.0", "method": "get_block", "params": ["6"]}, {"id": 7, "jsonrpc": "2.0", "method": "get_block", "params": ["7"]}, {"id": 8, "jsonrpc": "2.0", "method": "get_block", "params": ["8"]}, {"id": 9, "jsonrpc": "2.0", "method": "get_block", "params": ["9"]}]
This diff is collapsed.
This diff is collapsed.
# coding=utf-8
# coding=utf-8
# -*- coding: utf-8 -*-
import concurrent.futures
import json
import os
import logging
import os
import socket
import concurrent.futures
import time
from functools import partial
from functools import partialmethod
......@@ -13,8 +13,6 @@ import certifi
import urllib3
from urllib3.connection import HTTPConnection
logger = logging.getLogger(__name__)
......@@ -25,6 +23,7 @@ class RPCError(Exception):
class RPCConnectionError(Exception):
pass
def chunkify(iterable, chunksize=3000):
i = 0
chunk = []
......@@ -35,9 +34,10 @@ def chunkify(iterable, chunksize=3000):
yield chunk
i = 0
chunk = []
if len(chunk) > 0:
if chunk:
yield chunk
class SimpleSteemAPIClient(object):
"""Simple Steem JSON-HTTP-RPC API
......@@ -64,8 +64,9 @@ class SimpleSteemAPIClient(object):
"""
def __init__(self, url=None, log_level=logging.INFO, **kwargs):
url = url or os.environ.get('STEEMD_HTTP_URL', 'https://steemd.steemitdev.com')
def __init__(self, url=None, **kwargs):
url = url or os.environ.get('STEEMD_HTTP_URL',
'https://steemd.steemitdev.com')
self.url = url
self.hostname = urlparse(url).hostname
self.return_with_args = kwargs.get('return_with_args', False)
......@@ -105,14 +106,12 @@ class SimpleSteemAPIClient(object):
_logger = logging.getLogger('urllib3')
@staticmethod
def json_rpc_body(name, *args, as_json=True):
body_dict = {"method": name, "params": args, "jsonrpc": "2.0", "id": 0}
if as_json:
return json.dumps(body_dict, ensure_ascii=False).encode('utf8')
else:
return body_dict
return body_dict
def exec(self, name, *args, re_raise=None, return_with_args=None):
body = SimpleSteemAPIClient.json_rpc_body(name, *args)
......@@ -162,8 +161,7 @@ class SimpleSteemAPIClient(object):
result = response_json.get('result', None)
if return_with_args:
return result, args
else:
return result
return result
def exec_multi(self, name, params):
body_gen = ({
......@@ -173,29 +171,30 @@ class SimpleSteemAPIClient(object):
"id": i
} for i in params)
for chunk in chunkify(body_gen):
batch_json_body = json.dumps(chunk, ensure_ascii=False).encode('utf8')
batch_json_body = json.dumps(
chunk, ensure_ascii=False).encode('utf8')
r = self.request(body=batch_json_body).read()
print(r)
batch_response = json.loads(self.request(body=batch_json_body).read())
for i,resp in enumerate(batch_response):
batch_response = json.loads(
self.request(body=batch_json_body).read())
for i, resp in enumerate(batch_response):
yield self._return(
response=resp,
args=batch_json_body[i]['params'],
return_with_args=True)
response=resp,
args=batch_json_body[i]['params'],
return_with_args=True)
def exec_batch(self, name, params):
batch_requests = [{
"method": name,
"params": [str(i)],
"method": name,
"params": [str(i)],
"jsonrpc": "2.0",
"id": i
"id": i
} for i in params]
for chunk in chunkify(batch_requests):
batch_json_body = json.dumps(chunk).encode()
r = self.request(body=batch_json_body)
batch_response = json.loads(r.data.decode())
for i,resp in enumerate(batch_response):
for resp in batch_response:
yield json.dumps(resp)
def exec_batch_with_futures(self, name, params, max_workers=None):
......@@ -209,9 +208,6 @@ class SimpleSteemAPIClient(object):
for item in future.result():
yield item
get_dynamic_global_properties = partialmethod(
exec, 'get_dynamic_global_properties')
......
# -*- coding: utf-8 -*-
import argparse
import logging
import os
import sys
import logging
sys.path.append(os.path.dirname(__file__))
from multiprocessing import Pool
from functools import partial
from multiprocessing import Pool
import http_client
logger = logging.getLogger(__name__)
sys.path.append(os.path.dirname(__file__))
logger = logging.getLogger(__name__)
# pylint: skip-file
MAX_CHUNKSIZE = 1000000
......@@ -33,11 +32,7 @@ def chunkify(iterable, chunksize=10000):
yield chunk
def fetch_blocks(block_nums,
max_procs,
max_threads,
steemd_http_url):
def fetch_blocks(block_nums, max_procs, max_threads, steemd_http_url):
max_workers = max_procs or os.cpu_count() or 1
......@@ -46,33 +41,27 @@ def fetch_blocks(block_nums,
chunksize = 1
map_func = partial(
block_adder_process_worker,
steemd_http_url,
max_threads=max_threads)
block_adder_process_worker, steemd_http_url, max_threads=max_threads)
chunks = chunkify(block_nums, chunksize)
with Pool(processes=max_workers) as pool:
results = pool.map(map_func, chunks)
#print(results)
# print(results)
def do_test(steemd_http_url, max_procs, max_threads, start=None, end=None):
client = http_client.SimpleSteemAPIClient(url=steemd_http_url)
try:
start = start or 1
end = end or client.block_height()
missing_block_nums = list(range(start,end))
missing_block_nums = list(range(start, end))
# [2/2] adding missing blocks
fetch_blocks(
missing_block_nums,
max_procs,
max_threads,
steemd_http_url)
fetch_blocks(missing_block_nums, max_procs, max_threads,
steemd_http_url)
except KeyboardInterrupt:
pass
......@@ -87,16 +76,15 @@ def do_test(steemd_http_url, max_procs, max_threads, start=None, end=None):
def block_fetcher_thread_worker(rpc_url, block_nums, max_threads=None):
rpc = http_client.SimpleSteemAPIClient(rpc_url, return_with_args=False)
# pylint: disable=unused-variable
for block in rpc.exec_batch('get_block', block_nums): #, max_workers=max_threads):
for block in rpc.exec_batch('get_block',
block_nums): # , max_workers=max_threads):
yield block
def block_adder_process_worker(
rpc_url,
block_nums,
max_threads=5):
def block_adder_process_worker(rpc_url, block_nums, max_threads=5):
rpc = http_client.SimpleSteemAPIClient(rpc_url, return_with_args=False)
for block in rpc.exec_batch('get_block', block_nums): #, max_workers=max_threads):
for block in rpc.exec_batch('get_block',
block_nums): # , max_workers=max_threads):
print(block)
......@@ -104,10 +92,15 @@ def block_adder_process_worker(
# using the click framework
if __name__ == '__main__':
parser = argparse.ArgumentParser('jussi perf test script')
parser.add_argument('url',type=str)
parser.add_argument('--max_procs',type=int, default=os.cpu_count() -1)
parser.add_argument('--max_threads',type=int, default=30)
parser.add_argument('--start',type=int, default=1)
parser.add_argument('--end',type=int, default=None)
parser.add_argument('url', type=str)
parser.add_argument('--max_procs', type=int, default=os.cpu_count() - 1)
parser.add_argument('--max_threads', type=int, default=30)
parser.add_argument('--start', type=int, default=1)
parser.add_argument('--end', type=int, default=None)
args = parser.parse_args()
do_test(args.url, max_procs=args.max_procs, max_threads=args.max_threads,start=args.start, end=args.end)
do_test(
args.url,
max_procs=args.max_procs,
max_threads=args.max_threads,
start=args.start,
end=args.end)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment