Initial commit

* introduced two new classes: MockBlockProvider and MockVopsProvider
* possibility to inject artificial data to block and vops using data
  from mock providers
* data for mock providers is stored in json files, examples provided
* added command line options to set paths for mock json files

hive/conf.py

@@ -53,6 +53,8 @@ class Conf():
         add('--exit-after-sync', help='exit when sync is completed', action='store_true')
         add('--test-profile', type=strtobool, env_var='TEST_PROFILE', help='(debug) profile execution', default=False)
         add('--log-virtual-op-calls', type=strtobool, env_var='LOG_VIRTUAL_OP_CALLS', help='(debug) log virtual op calls and responses', default=False)
+        add('--mock-block-data-path', type=str, env_var='MOCK_BLOCK_DATA_PATH', help='(debug/testing) load additional data from block data file')
+        add('--mock-vops-data-path', type=str, env_var='MOCK_VOPS_DATA_PATH', help='(debug/testing) load additional data from virtual operations data file')
 
         # logging
         add('--log-timestamp', help='Output timestamp in log', action='store_true')

hive/indexer/mock_block_provider.py

+""" Data provider for test operations """
+import logging
+from hive.indexer.mock_data_provider import MockDataProvider
+
+log = logging.getLogger(__name__)
+
+class MockBlockProvider(MockDataProvider):
+    """ Data provider for test ops """

hive/indexer/mock_data_provider.py

+""" Data provider for test operations """
+import logging
+log = logging.getLogger(__name__)
+
+from json import load
+
+class MockDataProvider():
+    """ Data provider for test operations """
+    block_data = {}
+
+    @classmethod
+    def get_max_block_number(cls):
+        return max([int(block) for block in cls.block_data.keys()])
+
+    @classmethod
+    def load_block_data(cls, data_path):
+        with open(data_path, "r") as data_file:
+            cls.block_data = load(data_file)
+
+    @classmethod
+    def add_block_data(cls, block_num, transactions):
+        if block_num in cls.block_data:
+            cls.block_data[block_num].extend(transactions)
+        else:
+            cls.block_data[block_num] = transactions
+
+    @classmethod
+    def get_block_data(cls, block_num, pop=False):
+        if pop:
+            return cls.block_data.pop(block_num, None)
+        return cls.block_data.get(block_num, None)

hive/indexer/mock_vops_provider.py

+""" Data provider for test vops """
+import logging
+from hive.indexer.mock_data_provider import MockDataProvider
+
+log = logging.getLogger(__name__)
+
+class MockVopsProvider(MockDataProvider):
+    """ Data provider for test vops """

hive/indexer/sync.py

@@ -28,6 +28,9 @@ from hive.utils.stats import PrometheusClient as PC
 from hive.utils.stats import BroadcastObject
 from hive.utils.communities_rank import update_communities_posts_and_rank
 
+from hive.indexer.mock_block_provider import MockBlockProvider
+from hive.indexer.mock_vops_provider import MockVopsProvider
+
 from datetime import datetime
 
 log = logging.getLogger(__name__)
@@ -231,6 +234,16 @@ class Sync:
             self._conf.get('blacklist_api_url'))
         Mutes.set_shared_instance(mutes)
 
+        mock_block_data_path = self._conf.get("mock_block_data_path")
+        if mock_block_data_path:
+            log.warning("Loading mock block data from: {}".format(mock_block_data_path))
+            MockBlockProvider.load_block_data(mock_block_data_path)
+
+        mock_vops_data_path = self._conf.get("mock_vops_data_path")
+        if mock_vops_data_path:
+            log.warning("Loading mock virtual ops data from: {}".format(mock_vops_data_path))
+            MockVopsProvider.load_block_data(mock_vops_data_path)
+
         # community stats
         update_communities_posts_and_rank()
 

hive/steem/block/stream.py

@@ -3,6 +3,7 @@
 import logging
 from time import sleep
 from hive.steem.block.schedule import BlockSchedule
+from hive.indexer.mock_block_provider import MockBlockProvider
 
 log = logging.getLogger(__name__)
 
@@ -91,6 +92,11 @@ class BlockStream:
                 sleep(0.5)
                 continue
 
+            data = MockBlockProvider.get_block_data(str(curr), True)
+            if data is not None:
+                block["transactions"].extend(data["transactions"])
+                block["transaction_ids"].extend(data["transaction_ids"])
+
             popped = queue.push(block)
             if popped:
                 yield popped

hive/steem/client.py

 """Tight and reliable steem API client for hive indexer."""
+
 import logging
 
 from time import perf_counter as perf
@@ -8,6 +9,8 @@ from hive.utils.stats import Stats
 from hive.utils.normalize import parse_amount, steem_amount, vests_amount
 from hive.steem.http_client import HttpClient
 from hive.steem.block.stream import BlockStream
+from hive.indexer.mock_block_provider import MockBlockProvider
+from hive.indexer.mock_vops_provider import MockVopsProvider
 
 logger = logging.getLogger(__name__)
 
@@ -140,6 +143,12 @@ class SteemClient:
             num = int(block['block_id'][:8], base=16)
             blocks[num] = block
 
+        for block_num in block_nums:
+            data = MockBlockProvider.get_block_data(str(block_num), True)
+            if data is not None:
+                blocks[block_num]["transactions"].extend(data["transactions"])
+                blocks[block_num]["transaction_ids"].extend(data["transaction_ids"])
+
         return [blocks[x] for x in block_nums]
 
     def get_virtual_operations(self, block):
@@ -176,6 +185,11 @@ class SteemClient:
                 , "group_by_block": True, "include_reversible": True, "operation_begin": resume_on_operation, "limit": 1000, "filter": tracked_ops_filter
             })
 
+            mock_vops = MockVopsProvider.get_block_data(str(from_block), True)
+            if mock_vops is not None:
+                call_result['ops_by_block'].extend(mock_vops['ops_by_block'])
+                call_result['ops'].extend(mock_vops['ops'])
+
             if conf.get('log_virtual_op_calls'):
                 call = """
                 Call enum_virtual_ops:

mock_block_data_example.json

+{
+  "200": {
+    "previous": "000000c78dd942bb3b207f3a25ca798330f47cb4",
+    "timestamp": "2016-03-24T16:15:30",
+    "witness": "initminer",
+    "transaction_merkle_root": "0000000000000000000000000000000000000000",
+    "extensions": [],
+    "witness_signature": "",
+    "transactions": [
+      {
+        "ref_block_num": 199,
+        "ref_block_prefix": 0,
+        "expiration": "2020-03-23T12:08:00",
+        "operations": [
+          {
+            "type": "create_claimed_account_operation",
+            "value": {
+              "creator": "esteemapp",
+              "new_account_name": "tester1",
+              "owner": {
+                "weight_threshold": 1,
+                "account_auths": [],
+                "key_auths": [
+                  [
+                    "",
+                    1
+                  ]
+                ]
+              },
+              "active": {
+                "weight_threshold": 1,
+                "account_auths": [],
+                "key_auths": [
+                  [
+                    "",
+                    1
+                  ]
+                ]
+              },
+              "posting": {
+                "weight_threshold": 1,
+                "account_auths": [],
+                "key_auths": [
+                  [
+                    "",
+                    1
+                  ]
+                ]
+              },
+              "memo_key": "",
+              "json_metadata": "",
+              "extensions": []
+            }
+          },
+          {
+            "type": "create_claimed_account_operation",
+            "value": {
+              "creator": "esteemapp",
+              "new_account_name": "tester2",
+              "owner": {
+                "weight_threshold": 1,
+                "account_auths": [],
+                "key_auths": [
+                  [
+                    "",
+                    1
+                  ]
+                ]
+              },
+              "active": {
+                "weight_threshold": 1,
+                "account_auths": [],
+                "key_auths": [
+                  [
+                    "",
+                    1
+                  ]
+                ]
+              },
+              "posting": {
+                "weight_threshold": 1,
+                "account_auths": [],
+                "key_auths": [
+                  [
+                    "",
+                    1
+                  ]
+                ]
+              },
+              "memo_key": "",
+              "json_metadata": "",
+              "extensions": []
+            }
+          }
+        ],
+        "extensions": [],
+        "signatures": [
+          ""
+        ]
+      }
+    ],
+    "block_id": "000000c811143b04d5a25a1dae5c1e6714745d9a",
+    "signing_key": "",
+    "transaction_ids": []
+  },
+  "300": {
+    "previous": "0000012b9edb72ce6ee6742f93a692d35edbdb46",
+    "timestamp": "2016-03-24T16:20:30",
+    "witness": "initminer",
+    "transaction_merkle_root": "0000000000000000000000000000000000000000",
+    "extensions": [],
+    "witness_signature": "",
+    "transactions": [
+      {
+        "ref_block_num": 299,
+        "ref_block_prefix": 1,
+        "expiration": "2020-03-23T12:17:00",
+        "operations": [
+          {
+            "type": "custom_json_operation",
+            "value": {
+              "required_auths": [],
+              "required_posting_auths": [
+                "tester1"
+              ],
+              "id": "follow",
+              "json": "[\"follow\",{\"follower\":\"tester1\",\"following\":\"tester2\",\"what\":[\"blog\"]}]"
+            }
+          }
+        ]
+      }
+    ],
+    "block_id": "0000012cd1d7cb9cfede098a07cf4981ec5506a0",
+    "signing_key": "",
+    "transaction_ids": []
+  }
+}
\ No newline at end of file

mock_vops_data_example.json

+{
+  "300": {
+    "ops": [
+      {
+        "trx_id": "0000000000000000000000000000000000000000",
+        "block": 300,
+        "trx_in_block": 4294967295,
+        "op_in_trx": 0,
+        "virtual_op": 1,
+        "timestamp": "2016-03-24T16:20:30",
+        "op": {
+          "type": "producer_reward_operation",
+          "value": {
+            "producer": "tester1",
+            "vesting_shares": {
+              "amount": "1000000",
+              "precision": 6,
+              "nai": "@@000000037"
+            }
+          }
+        },
+        "operation_id": "9223372039063639274"
+      }
+    ],
+    "ops_by_block": [],
+    "next_block_range_begin": 10977,
+    "next_operation_begin": 0
+  }
+}
\ No newline at end of file