diff --git a/.gitlab-ci.yaml b/.gitlab-ci.yaml
index eecb7c851cc34c7745561275c5f700f2df35e0e3..ae286c45b5a7f99c8b5630c07a777a9b5c2bc9f6 100644
--- a/.gitlab-ci.yaml
+++ b/.gitlab-ci.yaml
@@ -73,7 +73,7 @@ hivemind_sync:
script:
- pip3 install --user --upgrade pip setuptools
- - scripts/ci_sync.sh "$HIVEMIND_DB_NAME" "$HIVEMIND_POSTGRESQL_CONNECTION_STRING" "$HIVEMIND_SOURCE_HIVED_URL" $HIVEMIND_MAX_BLOCK $HIVEMIND_HTTP_PORT
+ - scripts/ci_sync.sh "$HIVEMIND_DB_NAME" "$HIVEMIND_POSTGRESQL_CONNECTION_STRING" "$HIVEMIND_SOURCE_HIVED_URL" 5000017 $HIVEMIND_HTTP_PORT
artifacts:
paths:
diff --git a/hive/indexer/follow.py b/hive/indexer/follow.py
index 131f07dcd2a219a6635321f0f2a1fa68214d95c2..1dcc90cf6ab82e38770b39095d1eabb11866f096 100644
--- a/hive/indexer/follow.py
+++ b/hive/indexer/follow.py
@@ -10,6 +10,7 @@ from hive.utils.misc import chunks
from hive.indexer.accounts import Accounts
from hive.indexer.db_adapter_holder import DbAdapterHolder
+from hive.utils.normalize import escape_characters
log = logging.getLogger(__name__)
@@ -124,8 +125,8 @@ class Follow(DbAdapterHolder):
if non_existent_names:
log.warning("Follow op validation, following names does not exists in database: {}".format(non_existent_names))
- return dict(flr=op['follower'],
- flg=op['following'],
+ return dict(flr=escape_characters(op['follower']),
+ flg=[escape_characters(following) for following in op['following']],
state=defs[what],
at=date)
@@ -226,7 +227,7 @@ class Follow(DbAdapterHolder):
cls.beginTx()
for _, follow_item in cls.follow_items_to_flush.items():
if count < limit:
- values.append("({}, '{}', '{}', '{}'::timestamp, {}, {}, {}, {}, {})".format(follow_item['idx'],
+ values.append("({}, {}, {}, '{}'::timestamp, {}, {}, {}, {}, {})".format(follow_item['idx'],
follow_item['flr'],
follow_item['flg'],
follow_item['at'],
@@ -241,7 +242,7 @@ class Follow(DbAdapterHolder):
query += sql_postfix
cls.db.query(query)
values.clear()
- values.append("({}, '{}', '{}', '{}'::timestamp, {}, {}, {}, {}, {})".format(follow_item['idx'],
+ values.append("({}, {}, {}, '{}'::timestamp, {}, {}, {}, {}, {})".format(follow_item['idx'],
follow_item['flr'],
follow_item['flg'],
follow_item['at'],
@@ -269,7 +270,7 @@ class Follow(DbAdapterHolder):
for state, update_flush_items in cls.follow_update_items_to_flush.items():
for chunk in chunks(update_flush_items, 1000):
sql = None
- query_values = ','.join(["('{}')".format(account) for account in chunk])
+ query_values = ','.join(["({})".format(account) for account in chunk])
# [DK] probaly not a bad idea to move that logic to SQL function
if state == 9:
#reset blacklists for follower
@@ -360,7 +361,7 @@ class Follow(DbAdapterHolder):
UPDATE
hive_follows hf
SET
- hf.follow_blacklists = true
+ follow_blacklists = true
FROM
(
SELECT
@@ -481,7 +482,7 @@ class Follow(DbAdapterHolder):
for col, deltas in cls._delta.items():
for delta, names in _flip_dict(deltas).items():
updated += len(names)
- query_values = ','.join(["('{}')".format(account) for account in names])
+ query_values = ','.join(["({})".format(account) for account in names])
sql = """
UPDATE
hive_accounts ha
@@ -518,7 +519,7 @@ class Follow(DbAdapterHolder):
"""
names = set([*cls._delta[FOLLOWERS].keys(),
*cls._delta[FOLLOWING].keys()])
- query_values = ','.join(["('{}')".format(account) for account in names])
+ query_values = ','.join(["({})".format(account) for account in names])
sql = """
UPDATE
hive_accounts ha
diff --git a/hive/steem/client.py b/hive/steem/client.py
index 9dcf494a3946ab4d3735404e1afd028478e8bf49..cb48631cc0a9e973fcf63f52b88514ce3cc74c6c 100644
--- a/hive/steem/client.py
+++ b/hive/steem/client.py
@@ -153,16 +153,19 @@ class SteemClient:
batch_params = [{'block_num': i} for i in block_nums]
for result in self.__exec_batch('get_block', batch_params):
- assert 'block' in result, "result w/o block key: %s" % result
- block = result['block']
- num = int(block['block_id'][:8], base=16)
- blocks[num] = block
+ if 'block' in result:
+ block = result['block']
+ num = int(block['block_id'][:8], base=16)
+ blocks[num] = block
for block_num in block_nums:
data = MockBlockProvider.get_block_data(block_num, True)
if data is not None:
- blocks[block_num]["transactions"].extend(data["transactions"])
- blocks[block_num]["transaction_ids"].extend(data["transaction_ids"])
+ if block_num in blocks:
+ blocks[block_num]["transactions"].extend(data["transactions"])
+ blocks[block_num]["transaction_ids"].extend(data["transaction_ids"])
+ else:
+ blocks[block_num] = data
return [blocks[x] for x in block_nums]
diff --git a/mock_data/block_data/follow_op/mock_block_data_follow.json b/mock_data/block_data/follow_op/mock_block_data_follow.json
new file mode 100644
index 0000000000000000000000000000000000000000..1e5c5d3fdd8c2eaf6d52a5c61a5ca03715bd7a33
--- /dev/null
+++ b/mock_data/block_data/follow_op/mock_block_data_follow.json
@@ -0,0 +1,796 @@
+{
+ "100000": {
+ "previous": "",
+ "timestamp": "",
+ "witness": "",
+ "transaction_merkle_root": "",
+ "extensions": [],
+ "witness_signature": "",
+ "transactions": [
+ {
+ "ref_block_num": 100000,
+ "ref_block_prefix": 0,
+ "expiration": "2020-03-23T12:08:00",
+ "operations": [
+ {
+ "type": "create_claimed_account_operation",
+ "value": {
+ "creator": "esteemapp",
+ "new_account_name": "tester1",
+ "owner": {
+ "weight_threshold": 1,
+ "account_auths": [],
+ "key_auths": [
+ [
+ "",
+ 1
+ ]
+ ]
+ },
+ "active": {
+ "weight_threshold": 1,
+ "account_auths": [],
+ "key_auths": [
+ [
+ "",
+ 1
+ ]
+ ]
+ },
+ "posting": {
+ "weight_threshold": 1,
+ "account_auths": [],
+ "key_auths": [
+ [
+ "",
+ 1
+ ]
+ ]
+ },
+ "memo_key": "",
+ "json_metadata": "",
+ "extensions": []
+ }
+ },
+ {
+ "type": "create_claimed_account_operation",
+ "value": {
+ "creator": "esteemapp",
+ "new_account_name": "tester2",
+ "owner": {
+ "weight_threshold": 1,
+ "account_auths": [],
+ "key_auths": [
+ [
+ "",
+ 1
+ ]
+ ]
+ },
+ "active": {
+ "weight_threshold": 1,
+ "account_auths": [],
+ "key_auths": [
+ [
+ "",
+ 1
+ ]
+ ]
+ },
+ "posting": {
+ "weight_threshold": 1,
+ "account_auths": [],
+ "key_auths": [
+ [
+ "",
+ 1
+ ]
+ ]
+ },
+ "memo_key": "",
+ "json_metadata": "",
+ "extensions": []
+ }
+ },
+ {
+ "type": "create_claimed_account_operation",
+ "value": {
+ "creator": "esteemapp",
+ "new_account_name": "tester3",
+ "owner": {
+ "weight_threshold": 1,
+ "account_auths": [],
+ "key_auths": [
+ [
+ "",
+ 1
+ ]
+ ]
+ },
+ "active": {
+ "weight_threshold": 1,
+ "account_auths": [],
+ "key_auths": [
+ [
+ "",
+ 1
+ ]
+ ]
+ },
+ "posting": {
+ "weight_threshold": 1,
+ "account_auths": [],
+ "key_auths": [
+ [
+ "",
+ 1
+ ]
+ ]
+ },
+ "memo_key": "",
+ "json_metadata": "",
+ "extensions": []
+ }
+ },
+ {
+ "type": "create_claimed_account_operation",
+ "value": {
+ "creator": "esteemapp",
+ "new_account_name": "tester4",
+ "owner": {
+ "weight_threshold": 1,
+ "account_auths": [],
+ "key_auths": [
+ [
+ "",
+ 1
+ ]
+ ]
+ },
+ "active": {
+ "weight_threshold": 1,
+ "account_auths": [],
+ "key_auths": [
+ [
+ "",
+ 1
+ ]
+ ]
+ },
+ "posting": {
+ "weight_threshold": 1,
+ "account_auths": [],
+ "key_auths": [
+ [
+ "",
+ 1
+ ]
+ ]
+ },
+ "memo_key": "",
+ "json_metadata": "",
+ "extensions": []
+ }
+ },
+ {
+ "type": "create_claimed_account_operation",
+ "value": {
+ "creator": "esteemapp",
+ "new_account_name": "tester5",
+ "owner": {
+ "weight_threshold": 1,
+ "account_auths": [],
+ "key_auths": [
+ [
+ "",
+ 1
+ ]
+ ]
+ },
+ "active": {
+ "weight_threshold": 1,
+ "account_auths": [],
+ "key_auths": [
+ [
+ "",
+ 1
+ ]
+ ]
+ },
+ "posting": {
+ "weight_threshold": 1,
+ "account_auths": [],
+ "key_auths": [
+ [
+ "",
+ 1
+ ]
+ ]
+ },
+ "memo_key": "",
+ "json_metadata": "",
+ "extensions": []
+ }
+ }
+ ],
+ "extensions": [],
+ "signatures": [
+ ""
+ ]
+ }
+ ],
+ "block_id": "",
+ "signing_key": "",
+ "transaction_ids": []
+ },
+ "100001": {
+ "previous": "",
+ "timestamp": "",
+ "witness": "initminer",
+ "transaction_merkle_root": "",
+ "extensions": [],
+ "witness_signature": "",
+ "transactions": [
+ {
+ "ref_block_num": 100001,
+ "ref_block_prefix": 1,
+ "expiration": "2020-03-23T12:17:00",
+ "operations": [
+ {
+ "type": "custom_json_operation",
+ "value": {
+ "required_auths": [],
+ "required_posting_auths": [
+ "tester1"
+ ],
+ "id": "follow",
+ "json": "[\"follow\",{\"follower\":\"tester1\",\"following\":\"tester2\",\"what\":[\"blog\"]}]"
+ }
+ },
+ {
+ "type": "custom_json_operation",
+ "value": {
+ "required_auths": [],
+ "required_posting_auths": [
+ "tester1"
+ ],
+ "id": "follow",
+ "json": "[\"follow\",{\"follower\":\"tester1\",\"following\":[\"tester3\", \"tester4\"],\"what\":[\"blog\"]}]"
+ }
+ },
+ {
+ "type": "custom_json_operation",
+ "value": {
+ "required_auths": [],
+ "required_posting_auths": [
+ "tester1"
+ ],
+ "id": "follow",
+ "json": "[\"follow\",{\"follower\":\"tester1\",\"following\":[\"t'es'ter3\", \"<html><body><p>PPPPP</p></body></html>\"],\"what\":[\"blog\"]}]"
+ }
+ },
+ {
+ "type": "custom_json_operation",
+ "value": {
+ "required_auths": [],
+ "required_posting_auths": [
+ "tester1"
+ ],
+ "id": "follow",
+ "json": "[\"follow\",{\"follower\":\"tester1\",\"following\":[\"tester3\", \"tester4\"],\"what\":[\"blogo-doggo\"]}]"
+ }
+ },
+ {
+ "type": "custom_json_operation",
+ "value": {
+ "required_auths": [],
+ "required_posting_auths": [
+ "te'%@ter1"
+ ],
+ "id": "follow",
+ "json": "[\"follow\",{\"follower\":\"te'%@ter1\",\"following\":[\"tester3\", \"tester4\"],\"what\":[\"blog\"]}]"
+ }
+ }
+ ]
+ }
+ ],
+ "block_id": "",
+ "signing_key": "",
+ "transaction_ids": []
+ },
+ "5000001": {
+ "previous": "004c4b40245ffb07380a393fb2b3d841b76cdaec",
+ "timestamp": "2016-09-15T19:47:24",
+ "witness": "initminer",
+ "transaction_merkle_root": "",
+ "extensions": [],
+ "witness_signature": "",
+ "transactions": [
+ {
+ "ref_block_num": 100001,
+ "ref_block_prefix": 1,
+ "expiration": "2020-03-23T12:17:00",
+ "operations": [
+ {
+ "type": "custom_json_operation",
+ "value": {
+ "required_auths": [],
+ "required_posting_auths": [
+ "tester1"
+ ],
+ "id": "follow",
+ "json": "[\"follow\",{\"follower\":\"tester1\",\"following\":\"tester2\",\"what\":[\"blog\"]}]"
+ }
+ },
+ {
+ "type": "custom_json_operation",
+ "value": {
+ "required_auths": [],
+ "required_posting_auths": [
+ "tester1"
+ ],
+ "id": "follow",
+ "json": "[\"follow\",{\"follower\":\"tester1\",\"following\":[\"tester3\", \"tester4\"],\"what\":[\"blog\"]}]"
+ }
+ },
+ {
+ "type": "custom_json_operation",
+ "value": {
+ "required_auths": [],
+ "required_posting_auths": [
+ "tester1"
+ ],
+ "id": "follow",
+ "json": "[\"follow\",{\"follower\":\"tester1\",\"following\":[\"t'es'ter3\", \"<html><body><p>PPPPP</p></body></html>\"],\"what\":[\"blog\"]}]"
+ }
+ },
+ {
+ "type": "custom_json_operation",
+ "value": {
+ "required_auths": [],
+ "required_posting_auths": [
+ "tester1"
+ ],
+ "id": "follow",
+ "json": "[\"follow\",{\"follower\":\"tester1\",\"following\":[\"tester3\", \"tester4\"],\"what\":[\"blogo-doggo\"]}]"
+ }
+ },
+ {
+ "type": "custom_json_operation",
+ "value": {
+ "required_auths": [],
+ "required_posting_auths": [
+ "te'%@ter1"
+ ],
+ "id": "follow",
+ "json": "[\"follow\",{\"follower\":\"te'%@ter1\",\"following\":[\"tester3\", \"tester4\"],\"what\":[\"blog\"]}]"
+ }
+ }
+ ]
+ }
+ ],
+ "block_id": "004c4b4100000000000000000000000000000000",
+ "signing_key": "",
+ "transaction_ids": []
+ },
+ "5000002": {
+ "previous": "004c4b4100000000000000000000000000000000",
+ "timestamp": "2016-09-15T19:47:27",
+ "witness": "initminer",
+ "transaction_merkle_root": "0000000000000000000000000000000000000000",
+ "extensions": [],
+ "witness_signature": "",
+ "transactions": [
+ {
+ "ref_block_num": 100001,
+ "ref_block_prefix": 1,
+ "expiration": "2020-03-23T12:17:00",
+ "operations": [
+ {
+ "type": "custom_json_operation",
+ "value": {
+ "required_auths": [],
+ "required_posting_auths": [
+ "tester1"
+ ],
+ "id": "follow",
+ "json": "[\"follow\",{\"follower\":\"tester1\",\"following\":\"tester2\",\"what\":[\"ignore\"]}]"
+ }
+ }
+ ]
+ }
+ ],
+ "block_id": "004c4b4200000000000000000000000000000000",
+ "signing_key": "",
+ "transaction_ids": []
+ },
+ "5000003": {
+ "previous": "004c4b4200000000000000000000000000000000",
+ "timestamp": "2016-09-15T19:47:30",
+ "witness": "initminer",
+ "transaction_merkle_root": "0000000000000000000000000000000000000000",
+ "extensions": [],
+ "witness_signature": "",
+ "transactions": [
+ {
+ "ref_block_num": 100001,
+ "ref_block_prefix": 1,
+ "expiration": "2020-03-23T12:17:00",
+ "operations": [
+ {
+ "type": "custom_json_operation",
+ "value": {
+ "required_auths": [],
+ "required_posting_auths": [
+ "tester1"
+ ],
+ "id": "follow",
+ "json": "[\"follow\",{\"follower\":\"tester1\",\"following\":\"tester2\",\"what\":[\"blacklist\"]}]"
+ }
+ }
+ ]
+ }
+ ],
+ "block_id": "004c4b4300000000000000000000000000000000",
+ "signing_key": "",
+ "transaction_ids": []
+ },
+ "5000004": {
+ "previous": "004c4b4300000000000000000000000000000000",
+ "timestamp": "2016-09-15T19:47:33",
+ "witness": "initminer",
+ "transaction_merkle_root": "0000000000000000000000000000000000000000",
+ "extensions": [],
+ "witness_signature": "",
+ "transactions": [
+ {
+ "ref_block_num": 100001,
+ "ref_block_prefix": 1,
+ "expiration": "2020-03-23T12:17:00",
+ "operations": [
+ {
+ "type": "custom_json_operation",
+ "value": {
+ "required_auths": [],
+ "required_posting_auths": [
+ "tester1"
+ ],
+ "id": "follow",
+ "json": "[\"follow\",{\"follower\":\"tester1\",\"following\":\"tester2\",\"what\":[\"follow_blacklist\"]}]"
+ }
+ }
+ ]
+ }
+ ],
+ "block_id": "004c4b4400000000000000000000000000000000",
+ "signing_key": "",
+ "transaction_ids": []
+ },
+ "5000005": {
+ "previous": "004c4b4400000000000000000000000000000000",
+ "timestamp": "2016-09-15T19:47:36",
+ "witness": "initminer",
+ "transaction_merkle_root": "0000000000000000000000000000000000000000",
+ "extensions": [],
+ "witness_signature": "",
+ "transactions": [
+ {
+ "ref_block_num": 100001,
+ "ref_block_prefix": 1,
+ "expiration": "2020-03-23T12:17:00",
+ "operations": [
+ {
+ "type": "custom_json_operation",
+ "value": {
+ "required_auths": [],
+ "required_posting_auths": [
+ "tester1"
+ ],
+ "id": "follow",
+ "json": "[\"follow\",{\"follower\":\"tester1\",\"following\":\"tester2\",\"what\":[\"unblacklist\"]}]"
+ }
+ }
+ ]
+ }
+ ],
+ "block_id": "004c4b4500000000000000000000000000000000",
+ "signing_key": "",
+ "transaction_ids": []
+ },
+ "5000006": {
+ "previous": "004c4b4500000000000000000000000000000000",
+ "timestamp": "2016-09-15T19:47:39",
+ "witness": "initminer",
+ "transaction_merkle_root": "0000000000000000000000000000000000000000",
+ "extensions": [],
+ "witness_signature": "",
+ "transactions": [
+ {
+ "ref_block_num": 100001,
+ "ref_block_prefix": 1,
+ "expiration": "2020-03-23T12:17:00",
+ "operations": [
+ {
+ "type": "custom_json_operation",
+ "value": {
+ "required_auths": [],
+ "required_posting_auths": [
+ "tester1"
+ ],
+ "id": "follow",
+ "json": "[\"follow\",{\"follower\":\"tester1\",\"following\":\"tester2\",\"what\":[\"unfollow_blacklist\"]}]"
+ }
+ }
+ ]
+ }
+ ],
+ "block_id": "004c4b4600000000000000000000000000000000",
+ "signing_key": "",
+ "transaction_ids": []
+ },
+ "5000007": {
+ "previous": "004c4b4600000000000000000000000000000000",
+ "timestamp": "2016-09-15T19:47:42",
+ "witness": "initminer",
+ "transaction_merkle_root": "0000000000000000000000000000000000000000",
+ "extensions": [],
+ "witness_signature": "",
+ "transactions": [
+ {
+ "ref_block_num": 100001,
+ "ref_block_prefix": 1,
+ "expiration": "2020-03-23T12:17:00",
+ "operations": [
+ {
+ "type": "custom_json_operation",
+ "value": {
+ "required_auths": [],
+ "required_posting_auths": [
+ "tester1"
+ ],
+ "id": "follow",
+ "json": "[\"follow\",{\"follower\":\"tester1\",\"following\":\"tester2\",\"what\":[\"follow_muted\"]}]"
+ }
+ }
+ ]
+ }
+ ],
+ "block_id": "004c4b4700000000000000000000000000000000",
+ "signing_key": "",
+ "transaction_ids": []
+ },
+ "5000008": {
+ "previous": "004c4b4700000000000000000000000000000000",
+ "timestamp": "2016-09-15T19:47:45",
+ "witness": "initminer",
+ "transaction_merkle_root": "0000000000000000000000000000000000000000",
+ "extensions": [],
+ "witness_signature": "",
+ "transactions": [
+ {
+ "ref_block_num": 100001,
+ "ref_block_prefix": 1,
+ "expiration": "2020-03-23T12:17:00",
+ "operations": [
+ {
+ "type": "custom_json_operation",
+ "value": {
+ "required_auths": [],
+ "required_posting_auths": [
+ "tester1"
+ ],
+ "id": "follow",
+ "json": "[\"follow\",{\"follower\":\"tester1\",\"following\":\"tester2\",\"what\":[\"unfollow_muted\"]}]"
+ }
+ }
+ ]
+ }
+ ],
+ "block_id": "004c4b4800000000000000000000000000000000",
+ "signing_key": "",
+ "transaction_ids": []
+ },
+ "5000009": {
+ "previous": "004c4b4800000000000000000000000000000000",
+ "timestamp": "2016-09-15T19:47:48",
+ "witness": "initminer",
+ "transaction_merkle_root": "0000000000000000000000000000000000000000",
+ "extensions": [],
+ "witness_signature": "",
+ "transactions": [
+ {
+ "ref_block_num": 100001,
+ "ref_block_prefix": 1,
+ "expiration": "2020-03-23T12:17:00",
+ "operations": [
+ {
+ "type": "custom_json_operation",
+ "value": {
+ "required_auths": [],
+ "required_posting_auths": [
+ "tester1"
+ ],
+ "id": "follow",
+ "json": "[\"follow\",{\"follower\":\"tester1\",\"following\":\"tester2\",\"what\":[\"reset_blacklist\"]}]"
+ }
+ }
+ ]
+ }
+ ],
+ "block_id": "004c4b4900000000000000000000000000000000",
+ "signing_key": "",
+ "transaction_ids": []
+ },
+ "5000010": {
+ "previous": "004c4b4900000000000000000000000000000000",
+ "timestamp": "2016-09-15T19:47:51",
+ "witness": "initminer",
+ "transaction_merkle_root": "0000000000000000000000000000000000000000",
+ "extensions": [],
+ "witness_signature": "",
+ "transactions": [
+ {
+ "ref_block_num": 100001,
+ "ref_block_prefix": 1,
+ "expiration": "2020-03-23T12:17:00",
+ "operations": [
+ {
+ "type": "custom_json_operation",
+ "value": {
+ "required_auths": [],
+ "required_posting_auths": [
+ "tester1"
+ ],
+ "id": "follow",
+ "json": "[\"follow\",{\"follower\":\"tester1\",\"following\":\"tester2\",\"what\":[\"reset_following_list\"]}]"
+ }
+ }
+ ]
+ }
+ ],
+ "block_id": "004c4b4a00000000000000000000000000000000",
+ "signing_key": "",
+ "transaction_ids": []
+ },
+ "5000011": {
+ "previous": "004c4b4a00000000000000000000000000000000",
+ "timestamp": "2016-09-15T19:47:54",
+ "witness": "initminer",
+ "transaction_merkle_root": "0000000000000000000000000000000000000000",
+ "extensions": [],
+ "witness_signature": "",
+ "transactions": [
+ {
+ "ref_block_num": 100001,
+ "ref_block_prefix": 1,
+ "expiration": "2020-03-23T12:17:00",
+ "operations": [
+ {
+ "type": "custom_json_operation",
+ "value": {
+ "required_auths": [],
+ "required_posting_auths": [
+ "tester1"
+ ],
+ "id": "follow",
+ "json": "[\"follow\",{\"follower\":\"tester1\",\"following\":\"tester2\",\"what\":[\"reset_follow_blacklist\"]}]"
+ }
+ }
+ ]
+ }
+ ],
+ "block_id": "004c4b4b00000000000000000000000000000000",
+ "signing_key": "",
+ "transaction_ids": []
+ },
+ "5000012": {
+ "previous": "004c4b4b00000000000000000000000000000000",
+ "timestamp": "2016-09-15T19:47:57",
+ "witness": "initminer",
+ "transaction_merkle_root": "0000000000000000000000000000000000000000",
+ "extensions": [],
+ "witness_signature": "",
+ "transactions": [
+ {
+ "ref_block_num": 100001,
+ "ref_block_prefix": 1,
+ "expiration": "2020-03-23T12:17:00",
+ "operations": [
+ {
+ "type": "custom_json_operation",
+ "value": {
+ "required_auths": [],
+ "required_posting_auths": [
+ "tester1"
+ ],
+ "id": "follow",
+ "json": "[\"follow\",{\"follower\":\"tester1\",\"following\":\"tester2\",\"what\":[\"reset_follow_muted_list\"]}]"
+ }
+ }
+ ]
+ }
+ ],
+ "block_id": "004c4b4c00000000000000000000000000000000",
+ "signing_key": "",
+ "transaction_ids": []
+ },
+ "5000013": {
+ "previous": "004c4b4c00000000000000000000000000000000",
+ "timestamp": "2016-09-15T19:48:00",
+ "witness": "initminer",
+ "transaction_merkle_root": "0000000000000000000000000000000000000000",
+ "extensions": [],
+ "witness_signature": "",
+ "transactions": [
+ {
+ "ref_block_num": 100001,
+ "ref_block_prefix": 1,
+ "expiration": "2020-03-23T12:17:00",
+ "operations": [
+ {
+ "type": "custom_json_operation",
+ "value": {
+ "required_auths": [],
+ "required_posting_auths": [
+ "tester1"
+ ],
+ "id": "follow",
+ "json": "[\"follow\",{\"follower\":\"tester1\",\"following\":\"tester2\",\"what\":[\"reset_all_lists\"]}]"
+ }
+ }
+ ]
+ }
+ ],
+ "block_id": "004c4b4d00000000000000000000000000000000",
+ "signing_key": "",
+ "transaction_ids": []
+ },
+ "5000014": {
+ "previous": "004c4b4d00000000000000000000000000000000",
+ "timestamp": "2016-09-15T19:48:03",
+ "witness": "initminer",
+ "transaction_merkle_root": "0000000000000000000000000000000000000000",
+ "extensions": [],
+ "witness_signature": "",
+ "transactions": [],
+ "block_id": "004c4b4e00000000000000000000000000000000",
+ "signing_key": "",
+ "transaction_ids": []
+ },
+ "5000015": {
+ "previous": "004c4b4e00000000000000000000000000000000",
+ "timestamp": "2016-09-15T19:48:06",
+ "witness": "initminer",
+ "transaction_merkle_root": "0000000000000000000000000000000000000000",
+ "extensions": [],
+ "witness_signature": "",
+ "transactions": [],
+ "block_id": "004c4b4f00000000000000000000000000000000",
+ "signing_key": "",
+ "transaction_ids": []
+ },
+ "5000016": {
+ "previous": "004c4b4f00000000000000000000000000000000",
+ "timestamp": "2016-09-15T19:48:09",
+ "witness": "initminer",
+ "transaction_merkle_root": "0000000000000000000000000000000000000000",
+ "extensions": [],
+ "witness_signature": "",
+ "transactions": [],
+ "block_id": "004c4b5000000000000000000000000000000000",
+ "signing_key": "",
+ "transaction_ids": []
+ },
+ "5000017": {
+ "previous": "004c4b5000000000000000000000000000000000",
+ "timestamp": "2016-09-15T19:48:12",
+ "witness": "initminer",
+ "transaction_merkle_root": "0000000000000000000000000000000000000000",
+ "extensions": [],
+ "witness_signature": "",
+ "transactions": [],
+ "block_id": "004c4b5100000000000000000000000000000000",
+ "signing_key": "",
+ "transaction_ids": []
+ }
+}
\ No newline at end of file
diff --git a/scripts/ci_sync.sh b/scripts/ci_sync.sh
index e342c5e53dc442d225d0f4edf04514d59b7ec51a..9b71e585d5d06e11ed1b425e4aca58365f944408 100755
--- a/scripts/ci_sync.sh
+++ b/scripts/ci_sync.sh
@@ -58,5 +58,5 @@ fi
echo Attempting to starting hive sync using hived node: $HIVEMIND_SOURCE_HIVED_URL . Max sync block is: $HIVEMIND_MAX_BLOCK
echo Attempting to access database $DB_URL
-./$HIVE_NAME sync --pid-file hive_sync.pid --test-max-block=$HIVEMIND_MAX_BLOCK --exit-after-sync --test-profile=False --steemd-url "$HIVEMIND_SOURCE_HIVED_URL" --prometheus-port 11011 --database-url $DB_URL 2>&1 | tee -i hivemind-sync.log
+./$HIVE_NAME sync --pid-file hive_sync.pid --test-max-block=$HIVEMIND_MAX_BLOCK --exit-after-sync --test-profile=False --steemd-url "$HIVEMIND_SOURCE_HIVED_URL" --prometheus-port 11011 --database-url $DB_URL --mock-block-data-path mock_data/block_data/follow_op/mock_block_data_follow.json 2>&1 | tee -i hivemind-sync.log
rm hive_sync.pid
diff --git a/scripts/operation_extractor.py b/scripts/operation_extractor.py
new file mode 100755
index 0000000000000000000000000000000000000000..b26244331e3f9154030024b1a8fbc556cbaacf4b
--- /dev/null
+++ b/scripts/operation_extractor.py
@@ -0,0 +1,41 @@
+#!/usr/bin/python3
+
+from json import dumps
+from hive.steem.client import SteemClient
+
+if __name__ == "__main__":
+ import argparse
+ parser = argparse.ArgumentParser()
+
+ parser.add_argument("hivemind_address", type=str, help="Address of hivemind instance")
+ parser.add_argument("from_block", type=int, help="Scan from block")
+ parser.add_argument("to_block", type=int, help="Scan to block")
+ parser.add_argument("output_file", type=str, help="Prepared blocks will be saved in this file")
+ parser.add_argument("operations", type=str, nargs='+', help="Save selected operations")
+ parser.add_argument("--dump-ops-only", type=bool, default=False, help="Dump only selected ops without block data")
+
+ args = parser.parse_args()
+
+ client = SteemClient({"default":args.hivemind_address})
+ from_block = args.from_block
+ with open(args.output_file, "w") as output_file:
+ if not args.dump_ops_only:
+ output_file.write("{\n")
+ while from_block < args.to_block:
+ to_block = from_block + 1000
+ if to_block >= args.to_block:
+ to_block = args.to_block + 1
+ print("Processing range from: ", from_block, " to: ", to_block)
+ blocks = client.get_blocks_range(from_block, to_block)
+ for block in blocks:
+ block_num = int(block['block_id'][:8], base=16)
+ block_data = dict(block)
+ for idx in range(len(block_data['transactions'])):
+ block_data['transactions'][idx]['operations'] = [op for op in block_data['transactions'][idx]['operations'] if op['type'] in args.operations]
+ if args.dump_ops_only and block_data['transactions'][idx]['operations']:
+ output_file.write("{}\n".format(dumps(block_data['transactions'][idx]['operations'])))
+ if not args.dump_ops_only:
+ output_file.write('"{}":{},\n'.format(block_num, dumps(block_data)))
+ from_block = to_block
+ if not args.dump_ops_only:
+ output_file.write("}\n")
diff --git a/tests/tests_api b/tests/tests_api
index 819563bf5c43f0d7620b4be6e2a33df86dd168e4..ac95264adc33600c162ec0cd822a080694b22474 160000
--- a/tests/tests_api
+++ b/tests/tests_api
@@ -1 +1 @@
-Subproject commit 819563bf5c43f0d7620b4be6e2a33df86dd168e4
+Subproject commit ac95264adc33600c162ec0cd822a080694b22474