Load from directory support. Bug fixes

hive/indexer/mock_block_provider.py

 """ Data provider for test operations """
 import logging
+import os
 from hive.indexer.mock_data_provider import MockDataProvider
 
 log = logging.getLogger(__name__)
 
 class MockBlockProvider(MockDataProvider):
     """ Data provider for test ops """
+    @classmethod
+    def load_block_data(cls, data_path):
+        if os.path.isdir(data_path):
+            log.warning("Loading mock block data from directory: {}".format(data_path))
+            cls.add_block_data_from_directory(data_path)
+        else:
+            log.warning("Loading mock block data from file: {}".format(data_path))
+            cls.add_block_data_from_file(data_path)
+
+    @classmethod
+    def add_block_data_from_directory(cls, dir_name):
+        for name in os.listdir(dir_name):
+            file_path = os.path.join(dir_name, name)
+            if os.path.isfile(file_path) and file_path.endswith(".json"):
+                cls.add_block_data_from_file(file_path)
+
+    @classmethod
+    def add_block_data_from_file(cls, file_name):
+        from json import load
+        data = {}
+        with open(file_name, "r") as src:
+            data = load(src)
+        for block_num, transactions in data.items():
+            cls.add_block_data(block_num, transactions)
+
     @classmethod
     def add_block_data(cls, block_num, transactions):
         if block_num in cls.block_data:

hive/indexer/mock_data_provider.py

 """ Data provider for test operations """
 import logging
+from json import load, dumps
 log = logging.getLogger(__name__)
 
-from json import load
-
 class MockDataProvider():
     """ Data provider for test operations """
     block_data = {}
 
     @classmethod
-    def load_block_data(cls, data_path):
-        with open(data_path, "r") as data_file:
-            cls.block_data = load(data_file)
+    def print_data(cls):
+        print(dumps(cls.block_data, indent=4, sort_keys=True))

hive/indexer/mock_vops_provider.py

 """ Data provider for test vops """
 import logging
+import os
 from hive.indexer.mock_data_provider import MockDataProvider
 
 log = logging.getLogger(__name__)
 
 class MockVopsProvider(MockDataProvider):
     """ Data provider for test vops """
+    @classmethod
+    def load_block_data(cls, data_path):
+        if os.path.isdir(data_path):
+            log.warning("Loading mock virtual ops data from directory: {}".format(data_path))
+            cls.add_block_data_from_directory(data_path)
+        else:
+            log.warning("Loading mock virtual ops data from file: {}".format(data_path))
+            cls.add_block_data_from_file(data_path)
+
+    @classmethod
+    def add_block_data_from_directory(cls, dir_name):
+        for name in os.listdir(dir_name):
+            file_path = os.path.join(dir_name, name)
+            if os.path.isfile(file_path) and file_path.endswith(".json"):
+                cls.add_block_data_from_file(file_path)
+
+    @classmethod
+    def add_block_data_from_file(cls, file_name):
+        from json import load
+        data = {}
+        with open(file_name, "r") as src:
+            data = load(src)
+        cls.add_block_data(data)
+
+    @classmethod
+    def add_block_data(cls, data):
+        if 'ops' in data:
+            if 'ops' in cls.block_data:
+                cls.block_data['ops'].extend(data['ops'])
+            else:
+                cls.block_data['ops'] = data['ops']
+
+        if 'ops_by_block' in data:
+            if 'ops_by_block' not in cls.block_data:
+                cls.block_data['ops_by_block'] = []
+
+        for ops in data['ops_by_block']:
+            for obb_ops in cls.block_data['ops_by_block']:
+                if ops['block'] == obb_ops['block']:
+                    obb_ops['ops'].extend(ops['ops'])
+
     @classmethod
     def get_block_data(cls, block_num):
-        ret = {'timestamp': "", 'ops' : [], 'ops_by_block' : []}
+        ret = {}
         if cls.block_data:
             for ops in cls.block_data['ops']:
                 if ops['block'] == block_num:
                     ret['timestamp'] = ops['timestamp']
-                    ret['ops'].append(ops)
+                    if 'ops' in ret:
+                        ret['ops'].append(ops)
+                    else:
+                        ret['ops'] = [ops]
             for ops in cls.block_data['ops_by_block']:
                 if ops['block'] == block_num:
                     ret['timestamp'] = ops['timestamp']
-                    ret['ops_by_block'].extend(ops['ops'])
+                    if 'ops_by_block' in ret:
+                        ret['ops_by_block'].extend(ops['ops'])
+                    else:
+                        ret['ops_by_block'] = ops['ops']
         return ret

hive/indexer/sync.py

@@ -223,13 +223,13 @@ class Sync:
 
         mock_block_data_path = self._conf.get("mock_block_data_path")
         if mock_block_data_path:
-            log.warning("Loading mock block data from: {}".format(mock_block_data_path))
             MockBlockProvider.load_block_data(mock_block_data_path)
+            MockBlockProvider.print_data()
 
         mock_vops_data_path = self._conf.get("mock_vops_data_path")
         if mock_vops_data_path:
-            log.warning("Loading mock virtual ops data from: {}".format(mock_vops_data_path))
             MockVopsProvider.load_block_data(mock_vops_data_path)
+            MockVopsProvider.print_data()
 
         # ensure db schema up to date, check app status
         DbState.initialize()

hive/steem/client.py

@@ -182,20 +182,17 @@ class SteemClient:
         def add_mock_vops(ret, from_block, end_block):
             for block_num in range(from_block, end_block):
                 mock_vops = MockVopsProvider.get_block_data(block_num)
-                if block_num in ret:
-                    if mock_vops['ops_by_block']:
-                        ret[block_num]['ops'].extend([op['op'] for op in mock_vops['ops_by_block']])
-                        print(ret)
-                    if mock_vops['ops']:
-                        ret[block_num]['ops'].extend([op['op'] for op in mock_vops['ops']])
-                        print(ret)
-                else:
-                    if mock_vops['ops_by_block']:
-                        ret[block_num] = {'timestamp':mock_vops['timestamp'], "ops" : [op['op'] for op in mock_vops['ops_by_block']]}
-                        print(ret)
-                    if mock_vops['ops']:
-                        ret[block_num] = {'timestamp':mock_vops['timestamp'], "ops" : [op['op'] for op in mock_vops['ops']]}
-                        print(ret)
+                if mock_vops:
+                    if block_num in ret:
+                        if 'ops_by_block' in mock_vops:
+                            ret[block_num]['ops'].extend([op['op'] for op in mock_vops['ops_by_block'] if op['block'] == block_num])
+                        if 'ops' in mock_vops:
+                            ret[block_num]['ops'].extend([op['op'] for op in mock_vops['ops'] if op['block'] == block_num])
+                    else:
+                        if 'ops_by_block' in mock_vops:
+                            ret[block_num] = {'timestamp':mock_vops['timestamp'], "ops" : [op['op'] for op in mock_vops['ops_by_block'] if op['block'] == block_num]}
+                        if 'ops' in mock_vops:
+                            ret[block_num] = {'timestamp':mock_vops['timestamp'], "ops" : [op['op'] for op in mock_vops['ops'] if op['block'] == block_num]}
 
         ret = {}
 

mock_data/examples/block_data/mock_block_data_example_002.json

+{
+  "5000007": {
+    "previous": "004c4b4600000000000000000000000000000000",
+    "timestamp": "2016-09-15T19:47:39",
+    "witness": "initminer",
+    "transaction_merkle_root": "0000000000000000000000000000000000000000",
+    "extensions": [],
+    "witness_signature": "",
+    "transactions": [],
+    "block_id": "004c4b4700000000000000000000000000000000",
+    "signing_key": "",
+    "transaction_ids": []
+  },
+  "5000008": {
+    "previous": "004c4b4700000000000000000000000000000000",
+    "timestamp": "2016-09-15T19:47:39",
+    "witness": "initminer",
+    "transaction_merkle_root": "0000000000000000000000000000000000000000",
+    "extensions": [],
+    "witness_signature": "",
+    "transactions": [],
+    "block_id": "004c4b4800000000000000000000000000000000",
+    "signing_key": "",
+    "transaction_ids": []
+  }
+}
\ No newline at end of file

mock_data/examples/vops_data/mock_vops_data_example_002.json

+{
+  "ops": [
+    {
+      "trx_id": "0000000000000000000000000000000000000000",
+      "block": 301,
+      "trx_in_block": 4294967295,
+      "op_in_trx": 0,
+      "virtual_op": 1,
+      "timestamp": "2016-03-24T16:20:33",
+      "op": {
+        "type": "producer_reward_operation",
+        "value": {
+          "producer": "tester1",
+          "vesting_shares": {
+            "amount": "1000000",
+            "precision": 6,
+            "nai": "@@000000037"
+          }
+        }
+      },
+      "operation_id": "9223372039063639274"
+    }
+  ],
+  "ops_by_block": [],
+  "next_block_range_begin": 10977,
+  "next_operation_begin": 0
+}
\ No newline at end of file