From 8cc6342d07e3b113929a3579bdd64de5666bfa5a Mon Sep 17 00:00:00 2001
From: Holger Nahrstaedt <holger@nahrstaedt.de>
Date: Fri, 27 Apr 2018 21:41:00 +0200
Subject: [PATCH] Memory consumption for graphenerpc reduced and other
improvements
Memo
* make prefix changeble
Transationbuilder
* sign() return the signed struct now
Graphenerpc
* Session are used for requests
* Singleton for websocket instance added
* Both measures reduce ram consumption when more than one Steem object is created.
Add missing scrypt package to dependency
---
beem/instance.py | 5 ++
beem/memo.py | 14 ++++--
beem/transactionbuilder.py | 1 +
beem/wallet.py | 7 +--
beemgrapheneapi/graphenerpc.py | 86 ++++++++++++++++++++++++++--------
examples/benchmark_beem.py | 2 +-
setup.py | 1 +
7 files changed, 86 insertions(+), 30 deletions(-)
diff --git a/beem/instance.py b/beem/instance.py
index 6cb1e139..2efe17fe 100644
--- a/beem/instance.py
+++ b/beem/instance.py
@@ -8,6 +8,7 @@ import beem as stm
class SharedInstance(object):
+ """Singelton for the Steem Instance"""
instance = None
config = {}
@@ -58,3 +59,7 @@ def set_shared_config(config):
if not isinstance(config, dict):
raise AssertionError()
SharedInstance.config = config
+ # if one is already set, delete
+ if SharedInstance.instance:
+ clear_cache()
+ SharedInstance.instance = None
diff --git a/beem/memo.py b/beem/memo.py
index 28a7ae89..df9342c5 100644
--- a/beem/memo.py
+++ b/beem/memo.py
@@ -177,12 +177,15 @@ class Memo(object):
if not memo_wif:
raise MissingKeyError("Memo key for %s missing!" % self.from_account["name"])
+ if not hasattr(self, 'chain_prefix'):
+ self.chain_prefix = self.steem.prefix
+
if bts_encrypt:
enc = BtsMemo.encode_memo_bts(
PrivateKey(memo_wif),
PublicKey(
self.to_account["memo_key"],
- prefix=self.steem.prefix
+ prefix=self.chain_prefix
),
nonce,
memo
@@ -199,11 +202,11 @@ class Memo(object):
PrivateKey(memo_wif),
PublicKey(
self.to_account["memo_key"],
- prefix=self.steem.prefix
+ prefix=self.chain_prefix
),
nonce,
memo,
- prefix=self.steem.prefix
+ prefix=self.chain_prefix
)
return {
@@ -255,6 +258,9 @@ class Memo(object):
"Need any of {}".format(
[memo_to["name"], memo_from["name"]]))
+ if not hasattr(self, 'chain_prefix'):
+ self.chain_prefix = self.steem.prefix
+
if message[0] == '#':
return BtsMemo.decode_memo(
PrivateKey(memo_wif),
@@ -263,7 +269,7 @@ class Memo(object):
else:
return BtsMemo.decode_memo_bts(
PrivateKey(memo_wif),
- PublicKey(pubkey, prefix=self.steem.prefix),
+ PublicKey(pubkey, prefix=self.chain_prefix),
nonce,
message
)
diff --git a/beem/transactionbuilder.py b/beem/transactionbuilder.py
index fc52398c..083bac5b 100644
--- a/beem/transactionbuilder.py
+++ b/beem/transactionbuilder.py
@@ -276,6 +276,7 @@ class TransactionBuilder(dict):
signedtx.sign(self.wifs, chain=self.steem.rpc.chain_params)
self["signatures"].extend(signedtx.json().get("signatures"))
+ return signedtx
def verify_authority(self):
""" Verify the authority of the signed transaction
diff --git a/beem/wallet.py b/beem/wallet.py
index f2899596..4889e2c7 100644
--- a/beem/wallet.py
+++ b/beem/wallet.py
@@ -180,12 +180,7 @@ class Wallet(object):
self.masterpassword = self.masterpwd.decrypted_master
def tryUnlockFromEnv(self):
- """Try to fetch the unlock password first from 'UNLOCK' environment variable.
- This is only done, when steem.config['password_storage'] == 'environment'.
- and then from the keyring module keyring.get_password('beem', 'wallet'),
- when steem.config['password_storage'] == 'keyring'
- In order to use this, you have to store the password in the 'UNLOCK' variable
- or in the keyring by python -m keyring set beem wallet
+ """ Try to fetch the unlock password from UNLOCK environment variable and keyring when no password is given.
"""
password_storage = self.steem.config["password_storage"]
if password_storage == "environment" and "UNLOCK" in os.environ:
diff --git a/beemgrapheneapi/graphenerpc.py b/beemgrapheneapi/graphenerpc.py
index 0df11163..3b9869f5 100644
--- a/beemgrapheneapi/graphenerpc.py
+++ b/beemgrapheneapi/graphenerpc.py
@@ -15,7 +15,6 @@ import threading
import re
import time
import warnings
-from requests.exceptions import ConnectionError
from .exceptions import (
UnauthorizedError, RPCConnection, RPCError, RPCErrorDoRetry, NumRetriesReached
)
@@ -38,6 +37,9 @@ REQUEST_MODULE = None
if not REQUEST_MODULE:
try:
import requests
+ from requests.adapters import HTTPAdapter
+ from requests.packages.urllib3.util.retry import Retry
+ from requests.exceptions import ConnectionError
REQUEST_MODULE = "requests"
except ImportError:
REQUEST_MODULE = None
@@ -46,6 +48,56 @@ if not REQUEST_MODULE:
log = logging.getLogger(__name__)
+def requests_retry_session(
+ retries=3,
+ backoff_factor=0.3,
+ status_forcelist=(500, 502, 504),
+ session=None,
+):
+ if REQUEST_MODULE is None:
+ raise Exception()
+ session = session or requests.Session()
+ retry = Retry(total=retries,
+ read=retries,
+ connect=retries,
+ backoff_factor=backoff_factor,
+ status_forcelist=status_forcelist)
+ adapter = HTTPAdapter(max_retries=retry)
+ session.mount('http://', adapter)
+ session.mount('https://', adapter)
+ return session
+
+
+class WebsocketInstance(object):
+ """Singelton for the Websocket Instance"""
+ instance_ws = None
+ instance_wss = None
+
+
+def shared_ws_instance(use_ssl=True):
+ """Get websocket instance"""
+ if WEBSOCKET_MODULE is None:
+ raise Exception()
+ if use_ssl and not WebsocketInstance.instance_wss:
+ ssl_defaults = ssl.get_default_verify_paths()
+ sslopt_ca_certs = {'ca_certs': ssl_defaults.cafile}
+ WebsocketInstance.instance_wss = websocket.WebSocket(sslopt=sslopt_ca_certs, enable_multithread=True)
+ elif not use_ssl and not WebsocketInstance.instance_ws:
+ WebsocketInstance.instance_ws = websocket.WebSocket(enable_multithread=True)
+ if use_ssl:
+ return WebsocketInstance.instance_wss
+ else:
+ return WebsocketInstance.instance_ws
+
+
+def set_ws_instance(ws_instance, use_ssl=True):
+ """Set websocket instance"""
+ if use_ssl:
+ WebsocketInstance.instance_wss = ws_instance
+ else:
+ WebsocketInstance.instance_ws = ws_instance
+
+
class GrapheneRPC(object):
"""
This class allows to call API methods synchronously, without callbacks.
@@ -105,6 +157,7 @@ class GrapheneRPC(object):
self.user = user
self.password = password
self.ws = None
+ self.session = None
self.rpc_queue = []
self.timeout = kwargs.get('timeout', 60)
self.num_retries = kwargs.get("num_retries", -1)
@@ -144,27 +197,23 @@ class GrapheneRPC(object):
self.error_cnt[self.url] = 0
log.debug("Trying to connect to node %s" % self.url)
if self.url[:3] == "wss":
- if WEBSOCKET_MODULE is None:
- raise Exception()
- ssl_defaults = ssl.get_default_verify_paths()
- sslopt_ca_certs = {'ca_certs': ssl_defaults.cafile}
- self.ws = websocket.WebSocket(sslopt=sslopt_ca_certs, enable_multithread=True)
+ self.ws = shared_ws_instance(use_ssl=True)
self.current_rpc = self.rpc_methods["ws"]
elif self.url[:2] == "ws":
- if WEBSOCKET_MODULE is None:
- raise Exception()
- self.ws = websocket.WebSocket(enable_multithread=True)
+ self.ws = shared_ws_instance(use_ssl=False)
self.current_rpc = self.rpc_methods["ws"]
else:
- if REQUEST_MODULE is None:
- raise Exception()
self.ws = None
+ self.session = requests_retry_session(retries=self.num_retries_call)
self.current_rpc = self.rpc_methods["jsonrpc"]
- self.headers = {'User-Agent': 'beem v%s' % (beem_version),
- 'content-type': 'application/json'}
+ self.session.auth = (self.user, self.password)
+ self.session.headers.update({'User-Agent': 'beem v%s' % (beem_version),
+ 'content-type': 'application/json'})
+
try:
if self.ws:
self.ws.connect(self.url)
+ self.rpclogin(self.user, self.password)
try:
props = None
props = self.get_config(api="database")
@@ -180,7 +229,6 @@ class GrapheneRPC(object):
else:
self.current_rpc = self.rpc_methods["appbase"]
self.chain_params = self.get_network(props)
- self.rpclogin(self.user, self.password)
break
except KeyboardInterrupt:
raise
@@ -199,13 +247,13 @@ class GrapheneRPC(object):
"""Close Websocket"""
if self.ws:
self.ws.close()
+ if self.session:
+ self.session.close()
def request_send(self, payload):
- response = requests.post(self.url,
- data=payload,
- headers=self.headers,
- timeout=self.timeout,
- auth=(self.user, self.password))
+ response = self.session.post(self.url,
+ data=payload,
+ timeout=self.timeout)
if response.status_code == 401:
raise UnauthorizedError
return response.text
diff --git a/examples/benchmark_beem.py b/examples/benchmark_beem.py
index a5cfad6c..eea39133 100644
--- a/examples/benchmark_beem.py
+++ b/examples/benchmark_beem.py
@@ -17,7 +17,7 @@ logging.basicConfig(level=logging.INFO)
if __name__ == "__main__":
- node_setup = 2
+ node_setup = 0
how_many_hours = 1
if node_setup == 0:
stm = Steem(node="https://api.steemit.com", num_retries=10)
diff --git a/setup.py b/setup.py
index ee8cb727..4218019c 100755
--- a/setup.py
+++ b/setup.py
@@ -27,6 +27,7 @@ requires = [
"websocket-client",
"appdirs",
"Events",
+ "scrypt",
"pylibscrypt",
"pycryptodomex",
"pytz",
--
GitLab