Memory consumption for graphenerpc reduced and other improvements

Memo
* make prefix changeble
Transationbuilder
* sign() return the signed struct now
Graphenerpc
* Session are used for requests
* Singleton for websocket instance added
* Both measures reduce ram consumption when more than one Steem object is created.
Add missing scrypt package to dependency

beem/instance.py

@@ -8,6 +8,7 @@ import beem as stm
 
 
 class SharedInstance(object):
+    """Singelton for the Steem Instance"""
     instance = None
     config = {}
 
@@ -58,3 +59,7 @@ def set_shared_config(config):
     if not isinstance(config, dict):
         raise AssertionError()
     SharedInstance.config = config
+    # if one is already set, delete
+    if SharedInstance.instance:
+        clear_cache()
+        SharedInstance.instance = None

beem/memo.py

@@ -177,12 +177,15 @@ class Memo(object):
         if not memo_wif:
             raise MissingKeyError("Memo key for %s missing!" % self.from_account["name"])
 
+        if not hasattr(self, 'chain_prefix'):
+            self.chain_prefix = self.steem.prefix
+
         if bts_encrypt:
             enc = BtsMemo.encode_memo_bts(
                 PrivateKey(memo_wif),
                 PublicKey(
                     self.to_account["memo_key"],
-                    prefix=self.steem.prefix
+                    prefix=self.chain_prefix
                 ),
                 nonce,
                 memo
@@ -199,11 +202,11 @@ class Memo(object):
                 PrivateKey(memo_wif),
                 PublicKey(
                     self.to_account["memo_key"],
-                    prefix=self.steem.prefix
+                    prefix=self.chain_prefix
                 ),
                 nonce,
                 memo,
-                prefix=self.steem.prefix
+                prefix=self.chain_prefix
             )
 
             return {
@@ -255,6 +258,9 @@ class Memo(object):
                     "Need any of {}".format(
                     [memo_to["name"], memo_from["name"]]))
 
+        if not hasattr(self, 'chain_prefix'):
+            self.chain_prefix = self.steem.prefix
+
         if message[0] == '#':
             return BtsMemo.decode_memo(
                 PrivateKey(memo_wif),
@@ -263,7 +269,7 @@ class Memo(object):
         else:
             return BtsMemo.decode_memo_bts(
                 PrivateKey(memo_wif),
-                PublicKey(pubkey, prefix=self.steem.prefix),
+                PublicKey(pubkey, prefix=self.chain_prefix),
                 nonce,
                 message
             )

beem/transactionbuilder.py

@@ -276,6 +276,7 @@ class TransactionBuilder(dict):
 
         signedtx.sign(self.wifs, chain=self.steem.rpc.chain_params)
         self["signatures"].extend(signedtx.json().get("signatures"))
+        return signedtx
 
     def verify_authority(self):
         """ Verify the authority of the signed transaction

beem/wallet.py

@@ -180,12 +180,7 @@ class Wallet(object):
                 self.masterpassword = self.masterpwd.decrypted_master
 
     def tryUnlockFromEnv(self):
-        """Try to fetch the unlock password first from 'UNLOCK' environment variable.
-            This is only done, when steem.config['password_storage'] == 'environment'.
-            and then from the keyring module keyring.get_password('beem', 'wallet'),
-            when steem.config['password_storage'] == 'keyring'
-            In order to use this, you have to store the password in the 'UNLOCK' variable
-            or in the keyring by python -m keyring set beem wallet
+        """ Try to fetch the unlock password from UNLOCK environment variable and keyring when no password is given.
         """
         password_storage = self.steem.config["password_storage"]
         if password_storage == "environment" and "UNLOCK" in os.environ:

beemgrapheneapi/graphenerpc.py

@@ -15,7 +15,6 @@ import threading
 import re
 import time
 import warnings
-from requests.exceptions import ConnectionError
 from .exceptions import (
     UnauthorizedError, RPCConnection, RPCError, RPCErrorDoRetry, NumRetriesReached
 )
@@ -38,6 +37,9 @@ REQUEST_MODULE = None
 if not REQUEST_MODULE:
     try:
         import requests
+        from requests.adapters import HTTPAdapter
+        from requests.packages.urllib3.util.retry import Retry
+        from requests.exceptions import ConnectionError
         REQUEST_MODULE = "requests"
     except ImportError:
         REQUEST_MODULE = None
@@ -46,6 +48,56 @@ if not REQUEST_MODULE:
 log = logging.getLogger(__name__)
 
 
+def requests_retry_session(
+    retries=3,
+    backoff_factor=0.3,
+    status_forcelist=(500, 502, 504),
+    session=None,
+):
+    if REQUEST_MODULE is None:
+        raise Exception()
+    session = session or requests.Session()
+    retry = Retry(total=retries,
+                  read=retries,
+                  connect=retries,
+                  backoff_factor=backoff_factor,
+                  status_forcelist=status_forcelist)
+    adapter = HTTPAdapter(max_retries=retry)
+    session.mount('http://', adapter)
+    session.mount('https://', adapter)
+    return session
+
+
+class WebsocketInstance(object):
+    """Singelton for the Websocket Instance"""
+    instance_ws = None
+    instance_wss = None
+
+
+def shared_ws_instance(use_ssl=True):
+    """Get websocket instance"""
+    if WEBSOCKET_MODULE is None:
+        raise Exception()
+    if use_ssl and not WebsocketInstance.instance_wss:
+        ssl_defaults = ssl.get_default_verify_paths()
+        sslopt_ca_certs = {'ca_certs': ssl_defaults.cafile}
+        WebsocketInstance.instance_wss = websocket.WebSocket(sslopt=sslopt_ca_certs, enable_multithread=True)
+    elif not use_ssl and not WebsocketInstance.instance_ws:
+        WebsocketInstance.instance_ws = websocket.WebSocket(enable_multithread=True)
+    if use_ssl:
+        return WebsocketInstance.instance_wss
+    else:
+        return WebsocketInstance.instance_ws
+
+
+def set_ws_instance(ws_instance, use_ssl=True):
+    """Set websocket instance"""
+    if use_ssl:
+        WebsocketInstance.instance_wss = ws_instance
+    else:
+        WebsocketInstance.instance_ws = ws_instance
+
+
 class GrapheneRPC(object):
     """
     This class allows to call API methods synchronously, without callbacks.
@@ -105,6 +157,7 @@ class GrapheneRPC(object):
         self.user = user
         self.password = password
         self.ws = None
+        self.session = None
         self.rpc_queue = []
         self.timeout = kwargs.get('timeout', 60)
         self.num_retries = kwargs.get("num_retries", -1)
@@ -144,27 +197,23 @@ class GrapheneRPC(object):
                     self.error_cnt[self.url] = 0
                 log.debug("Trying to connect to node %s" % self.url)
                 if self.url[:3] == "wss":
-                    if WEBSOCKET_MODULE is None:
-                        raise Exception()
-                    ssl_defaults = ssl.get_default_verify_paths()
-                    sslopt_ca_certs = {'ca_certs': ssl_defaults.cafile}
-                    self.ws = websocket.WebSocket(sslopt=sslopt_ca_certs, enable_multithread=True)
+                    self.ws = shared_ws_instance(use_ssl=True)
                     self.current_rpc = self.rpc_methods["ws"]
                 elif self.url[:2] == "ws":
-                    if WEBSOCKET_MODULE is None:
-                        raise Exception()
-                    self.ws = websocket.WebSocket(enable_multithread=True)
+                    self.ws = shared_ws_instance(use_ssl=False)
                     self.current_rpc = self.rpc_methods["ws"]
                 else:
-                    if REQUEST_MODULE is None:
-                        raise Exception()
                     self.ws = None
+                    self.session = requests_retry_session(retries=self.num_retries_call)
                     self.current_rpc = self.rpc_methods["jsonrpc"]
-                    self.headers = {'User-Agent': 'beem v%s' % (beem_version),
-                                    'content-type': 'application/json'}
+                    self.session.auth = (self.user, self.password)
+                    self.session.headers.update({'User-Agent': 'beem v%s' % (beem_version),
+                                                 'content-type': 'application/json'})
+
             try:
                 if self.ws:
                     self.ws.connect(self.url)
+                    self.rpclogin(self.user, self.password)
                 try:
                     props = None
                     props = self.get_config(api="database")
@@ -180,7 +229,6 @@ class GrapheneRPC(object):
                     else:
                         self.current_rpc = self.rpc_methods["appbase"]
                 self.chain_params = self.get_network(props)
-                self.rpclogin(self.user, self.password)
                 break
             except KeyboardInterrupt:
                 raise
@@ -199,13 +247,13 @@ class GrapheneRPC(object):
         """Close Websocket"""
         if self.ws:
             self.ws.close()
+        if self.session:
+            self.session.close()
 
     def request_send(self, payload):
-        response = requests.post(self.url,
-                                 data=payload,
-                                 headers=self.headers,
-                                 timeout=self.timeout,
-                                 auth=(self.user, self.password))
+        response = self.session.post(self.url,
+                                     data=payload,
+                                     timeout=self.timeout)
         if response.status_code == 401:
             raise UnauthorizedError
         return response.text

examples/benchmark_beem.py

@@ -17,7 +17,7 @@ logging.basicConfig(level=logging.INFO)
 
 
 if __name__ == "__main__":
-    node_setup = 2
+    node_setup = 0
     how_many_hours = 1
     if node_setup == 0:
         stm = Steem(node="https://api.steemit.com", num_retries=10)

setup.py

@@ -27,6 +27,7 @@ requires = [
     "websocket-client",
     "appdirs",
     "Events",
+    "scrypt",
     "pylibscrypt",
     "pycryptodomex",
     "pytz",