From 7d46be9137bcff5b7dc3fcab266ce6f37b3cf6af Mon Sep 17 00:00:00 2001
From: Dariusz Kedzierski <dkedzierski@syncad.com>
Date: Mon, 2 Nov 2020 10:44:41 +0100
Subject: [PATCH] Load from directory support. Bug fixes
---
hive/indexer/mock_block_provider.py | 26 +++++++++
hive/indexer/mock_data_provider.py | 8 ++-
hive/indexer/mock_vops_provider.py | 54 +++++++++++++++++--
hive/indexer/sync.py | 4 +-
hive/steem/client.py | 25 ++++-----
.../mock_block_data_example_001.json | 0
.../mock_block_data_example_002.json | 26 +++++++++
.../vops_data/mock_vops_data_example_001.json | 0
.../vops_data/mock_vops_data_example_002.json | 27 ++++++++++
9 files changed, 146 insertions(+), 24 deletions(-)
rename mock_block_data_example.json => mock_data/examples/block_data/mock_block_data_example_001.json (100%)
create mode 100644 mock_data/examples/block_data/mock_block_data_example_002.json
rename mock_vops_data_example.json => mock_data/examples/vops_data/mock_vops_data_example_001.json (100%)
create mode 100644 mock_data/examples/vops_data/mock_vops_data_example_002.json
diff --git a/hive/indexer/mock_block_provider.py b/hive/indexer/mock_block_provider.py
index f11aee6aa..b72fcd510 100644
--- a/hive/indexer/mock_block_provider.py
+++ b/hive/indexer/mock_block_provider.py
@@ -1,11 +1,37 @@
""" Data provider for test operations """
import logging
+import os
from hive.indexer.mock_data_provider import MockDataProvider
log = logging.getLogger(__name__)
class MockBlockProvider(MockDataProvider):
""" Data provider for test ops """
+ @classmethod
+ def load_block_data(cls, data_path):
+ if os.path.isdir(data_path):
+ log.warning("Loading mock block data from directory: {}".format(data_path))
+ cls.add_block_data_from_directory(data_path)
+ else:
+ log.warning("Loading mock block data from file: {}".format(data_path))
+ cls.add_block_data_from_file(data_path)
+
+ @classmethod
+ def add_block_data_from_directory(cls, dir_name):
+ for name in os.listdir(dir_name):
+ file_path = os.path.join(dir_name, name)
+ if os.path.isfile(file_path) and file_path.endswith(".json"):
+ cls.add_block_data_from_file(file_path)
+
+ @classmethod
+ def add_block_data_from_file(cls, file_name):
+ from json import load
+ data = {}
+ with open(file_name, "r") as src:
+ data = load(src)
+ for block_num, transactions in data.items():
+ cls.add_block_data(block_num, transactions)
+
@classmethod
def add_block_data(cls, block_num, transactions):
if block_num in cls.block_data:
diff --git a/hive/indexer/mock_data_provider.py b/hive/indexer/mock_data_provider.py
index db692f9b4..70f04d8ee 100644
--- a/hive/indexer/mock_data_provider.py
+++ b/hive/indexer/mock_data_provider.py
@@ -1,14 +1,12 @@
""" Data provider for test operations """
import logging
+from json import load, dumps
log = logging.getLogger(__name__)
-from json import load
-
class MockDataProvider():
""" Data provider for test operations """
block_data = {}
@classmethod
- def load_block_data(cls, data_path):
- with open(data_path, "r") as data_file:
- cls.block_data = load(data_file)
+ def print_data(cls):
+ print(dumps(cls.block_data, indent=4, sort_keys=True))
diff --git a/hive/indexer/mock_vops_provider.py b/hive/indexer/mock_vops_provider.py
index 2d5e3297e..d782a8387 100644
--- a/hive/indexer/mock_vops_provider.py
+++ b/hive/indexer/mock_vops_provider.py
@@ -1,21 +1,69 @@
""" Data provider for test vops """
import logging
+import os
from hive.indexer.mock_data_provider import MockDataProvider
log = logging.getLogger(__name__)
class MockVopsProvider(MockDataProvider):
""" Data provider for test vops """
+ @classmethod
+ def load_block_data(cls, data_path):
+ if os.path.isdir(data_path):
+ log.warning("Loading mock virtual ops data from directory: {}".format(data_path))
+ cls.add_block_data_from_directory(data_path)
+ else:
+ log.warning("Loading mock virtual ops data from file: {}".format(data_path))
+ cls.add_block_data_from_file(data_path)
+
+ @classmethod
+ def add_block_data_from_directory(cls, dir_name):
+ for name in os.listdir(dir_name):
+ file_path = os.path.join(dir_name, name)
+ if os.path.isfile(file_path) and file_path.endswith(".json"):
+ cls.add_block_data_from_file(file_path)
+
+ @classmethod
+ def add_block_data_from_file(cls, file_name):
+ from json import load
+ data = {}
+ with open(file_name, "r") as src:
+ data = load(src)
+ cls.add_block_data(data)
+
+ @classmethod
+ def add_block_data(cls, data):
+ if 'ops' in data:
+ if 'ops' in cls.block_data:
+ cls.block_data['ops'].extend(data['ops'])
+ else:
+ cls.block_data['ops'] = data['ops']
+
+ if 'ops_by_block' in data:
+ if 'ops_by_block' not in cls.block_data:
+ cls.block_data['ops_by_block'] = []
+
+ for ops in data['ops_by_block']:
+ for obb_ops in cls.block_data['ops_by_block']:
+ if ops['block'] == obb_ops['block']:
+ obb_ops['ops'].extend(ops['ops'])
+
@classmethod
def get_block_data(cls, block_num):
- ret = {'timestamp': "", 'ops' : [], 'ops_by_block' : []}
+ ret = {}
if cls.block_data:
for ops in cls.block_data['ops']:
if ops['block'] == block_num:
ret['timestamp'] = ops['timestamp']
- ret['ops'].append(ops)
+ if 'ops' in ret:
+ ret['ops'].append(ops)
+ else:
+ ret['ops'] = [ops]
for ops in cls.block_data['ops_by_block']:
if ops['block'] == block_num:
ret['timestamp'] = ops['timestamp']
- ret['ops_by_block'].extend(ops['ops'])
+ if 'ops_by_block' in ret:
+ ret['ops_by_block'].extend(ops['ops'])
+ else:
+ ret['ops_by_block'] = ops['ops']
return ret
diff --git a/hive/indexer/sync.py b/hive/indexer/sync.py
index 18f148423..f023e36cf 100644
--- a/hive/indexer/sync.py
+++ b/hive/indexer/sync.py
@@ -223,13 +223,13 @@ class Sync:
mock_block_data_path = self._conf.get("mock_block_data_path")
if mock_block_data_path:
- log.warning("Loading mock block data from: {}".format(mock_block_data_path))
MockBlockProvider.load_block_data(mock_block_data_path)
+ MockBlockProvider.print_data()
mock_vops_data_path = self._conf.get("mock_vops_data_path")
if mock_vops_data_path:
- log.warning("Loading mock virtual ops data from: {}".format(mock_vops_data_path))
MockVopsProvider.load_block_data(mock_vops_data_path)
+ MockVopsProvider.print_data()
# ensure db schema up to date, check app status
DbState.initialize()
diff --git a/hive/steem/client.py b/hive/steem/client.py
index 7efb7080f..9dcf494a3 100644
--- a/hive/steem/client.py
+++ b/hive/steem/client.py
@@ -182,20 +182,17 @@ class SteemClient:
def add_mock_vops(ret, from_block, end_block):
for block_num in range(from_block, end_block):
mock_vops = MockVopsProvider.get_block_data(block_num)
- if block_num in ret:
- if mock_vops['ops_by_block']:
- ret[block_num]['ops'].extend([op['op'] for op in mock_vops['ops_by_block']])
- print(ret)
- if mock_vops['ops']:
- ret[block_num]['ops'].extend([op['op'] for op in mock_vops['ops']])
- print(ret)
- else:
- if mock_vops['ops_by_block']:
- ret[block_num] = {'timestamp':mock_vops['timestamp'], "ops" : [op['op'] for op in mock_vops['ops_by_block']]}
- print(ret)
- if mock_vops['ops']:
- ret[block_num] = {'timestamp':mock_vops['timestamp'], "ops" : [op['op'] for op in mock_vops['ops']]}
- print(ret)
+ if mock_vops:
+ if block_num in ret:
+ if 'ops_by_block' in mock_vops:
+ ret[block_num]['ops'].extend([op['op'] for op in mock_vops['ops_by_block'] if op['block'] == block_num])
+ if 'ops' in mock_vops:
+ ret[block_num]['ops'].extend([op['op'] for op in mock_vops['ops'] if op['block'] == block_num])
+ else:
+ if 'ops_by_block' in mock_vops:
+ ret[block_num] = {'timestamp':mock_vops['timestamp'], "ops" : [op['op'] for op in mock_vops['ops_by_block'] if op['block'] == block_num]}
+ if 'ops' in mock_vops:
+ ret[block_num] = {'timestamp':mock_vops['timestamp'], "ops" : [op['op'] for op in mock_vops['ops'] if op['block'] == block_num]}
ret = {}
diff --git a/mock_block_data_example.json b/mock_data/examples/block_data/mock_block_data_example_001.json
similarity index 100%
rename from mock_block_data_example.json
rename to mock_data/examples/block_data/mock_block_data_example_001.json
diff --git a/mock_data/examples/block_data/mock_block_data_example_002.json b/mock_data/examples/block_data/mock_block_data_example_002.json
new file mode 100644
index 000000000..ce5d98e1d
--- /dev/null
+++ b/mock_data/examples/block_data/mock_block_data_example_002.json
@@ -0,0 +1,26 @@
+{
+ "5000007": {
+ "previous": "004c4b4600000000000000000000000000000000",
+ "timestamp": "2016-09-15T19:47:39",
+ "witness": "initminer",
+ "transaction_merkle_root": "0000000000000000000000000000000000000000",
+ "extensions": [],
+ "witness_signature": "",
+ "transactions": [],
+ "block_id": "004c4b4700000000000000000000000000000000",
+ "signing_key": "",
+ "transaction_ids": []
+ },
+ "5000008": {
+ "previous": "004c4b4700000000000000000000000000000000",
+ "timestamp": "2016-09-15T19:47:39",
+ "witness": "initminer",
+ "transaction_merkle_root": "0000000000000000000000000000000000000000",
+ "extensions": [],
+ "witness_signature": "",
+ "transactions": [],
+ "block_id": "004c4b4800000000000000000000000000000000",
+ "signing_key": "",
+ "transaction_ids": []
+ }
+}
\ No newline at end of file
diff --git a/mock_vops_data_example.json b/mock_data/examples/vops_data/mock_vops_data_example_001.json
similarity index 100%
rename from mock_vops_data_example.json
rename to mock_data/examples/vops_data/mock_vops_data_example_001.json
diff --git a/mock_data/examples/vops_data/mock_vops_data_example_002.json b/mock_data/examples/vops_data/mock_vops_data_example_002.json
new file mode 100644
index 000000000..040e8e044
--- /dev/null
+++ b/mock_data/examples/vops_data/mock_vops_data_example_002.json
@@ -0,0 +1,27 @@
+{
+ "ops": [
+ {
+ "trx_id": "0000000000000000000000000000000000000000",
+ "block": 301,
+ "trx_in_block": 4294967295,
+ "op_in_trx": 0,
+ "virtual_op": 1,
+ "timestamp": "2016-03-24T16:20:33",
+ "op": {
+ "type": "producer_reward_operation",
+ "value": {
+ "producer": "tester1",
+ "vesting_shares": {
+ "amount": "1000000",
+ "precision": 6,
+ "nai": "@@000000037"
+ }
+ }
+ },
+ "operation_id": "9223372039063639274"
+ }
+ ],
+ "ops_by_block": [],
+ "next_block_range_begin": 10977,
+ "next_operation_begin": 0
+}
\ No newline at end of file
--
GitLab