diff --git a/hive/db/schema.py b/hive/db/schema.py
index 7e350ef1a5e1d1724fcb273fec00d0604236795c..17cd773eaaadb24d74662c1fc6e30e3cc8c7ab16 100644
--- a/hive/db/schema.py
+++ b/hive/db/schema.py
@@ -584,6 +584,68 @@ def setup(db):
"""
db.query_no_return(sql)
+ sql = """
+ DROP VIEW IF EXISTS public.vw_hive_posts;
+
+ CREATE OR REPLACE VIEW public.vw_hive_posts
+ AS
+ SELECT hp.id,
+ hp.community_id,
+ ha_a.name AS author,
+ hpd_p.permlink,
+ hpd.title,
+ hpd.body,
+ hcd.category,
+ hp.depth,
+ hp.promoted,
+ hp.payout,
+ hp.payout_at,
+ hp.is_paidout,
+ hp.children,
+ 0 AS votes,
+ 0 AS active_votes,
+ hp.created_at,
+ hp.updated_at,
+ hp.rshares,
+ hpd.json,
+ hp.is_hidden,
+ hp.is_grayed,
+ hp.total_votes,
+ hp.flag_weight,
+ ha_pa.name AS parent_author,
+ hpd_pp.permlink AS parent_permlink,
+ hp.curator_payout_value,
+ ha_ra.name AS root_author,
+ hpd_rp.permlink AS root_permlink,
+ rcd.category as root_category,
+ hp.max_accepted_payout,
+ hp.percent_hbd,
+ hp.allow_replies,
+ hp.allow_votes,
+ hp.allow_curation_rewards,
+ hp.beneficiaries,
+ concat('/', rcd.category, '/@', ha_ra.name, '/', hpd_rp.permlink,
+ case (rp.id)
+ when hp.id then ''
+ else concat('#@', ha_a.name, '/', hpd_p.permlink)
+ end ) as url,
+ rpd.title AS root_title
+ FROM hive_posts hp
+ JOIN hive_posts rp ON rp.author_id = hp.root_author_id AND rp.permlink_id = hp.root_permlink_id
+ JOIN hive_post_data rpd ON rp.id = rpd.id
+ JOIN hive_accounts ha_a ON ha_a.id = hp.author_id
+ JOIN hive_permlink_data hpd_p ON hpd_p.id = hp.permlink_id
+ JOIN hive_post_data hpd ON hpd.id = hp.id
+ LEFT JOIN hive_category_data hcd ON hcd.id = hp.category_id
+ LEFT JOIN hive_category_data rcd ON rcd.id = rp.category_id
+ JOIN hive_accounts ha_pa ON ha_pa.id = hp.parent_author_id
+ JOIN hive_permlink_data hpd_pp ON hpd_pp.id = hp.parent_permlink_id
+ JOIN hive_accounts ha_ra ON ha_ra.id = hp.root_author_id
+ JOIN hive_permlink_data hpd_rp ON hpd_rp.id = hp.root_permlink_id;
+ ;
+ """
+ db.query_no_return(sql)
+
def reset_autovac(db):
"""Initializes/resets per-table autovacuum/autoanalyze params.
diff --git a/hive/indexer/post_data_cache.py b/hive/indexer/post_data_cache.py
index f6a4fab6d0995f792074145fbcde6debbb8c574f..1459f900dd6f63b8c4b8a29f28e6d8b7d5d1d7c2 100644
--- a/hive/indexer/post_data_cache.py
+++ b/hive/indexer/post_data_cache.py
@@ -30,11 +30,11 @@ class PostDataCache(object):
"""
values = []
for k, data in cls._data.items():
- title = "''" if not data['title'] else "'{}'".format(escape_characters(data['title']))
- preview = "''" if not data['preview'] else "'{}'".format(escape_characters(data['preview']))
- img_url = "''" if not data['img_url'] else "'{}'".format(escape_characters(data['img_url']))
- body = "''" if not data['body'] else "'{}'".format(escape_characters(data['body']))
- json = "'{}'" if not data['json'] else "'{}'".format(escape_characters(data['json']))
+ title = "''" if not data['title'] else "{}".format(escape_characters(data['title']))
+ preview = "''" if not data['preview'] else "{}".format(escape_characters(data['preview']))
+ img_url = "''" if not data['img_url'] else "{}".format(escape_characters(data['img_url']))
+ body = "''" if not data['body'] else "{}".format(escape_characters(data['body']))
+ json = "'{}'" if not data['json'] else "{}".format(escape_characters(data['json']))
values.append("({},{},{},{},{},{})".format(k, title, preview, img_url, body, json))
sql += ','.join(values)
sql += """
diff --git a/hive/indexer/tags.py b/hive/indexer/tags.py
index e63d198e0ac1b9b6088a3ef67cf955dd69d9a165..58809052633102265419bf3246cdc5cc6a458cc4 100644
--- a/hive/indexer/tags.py
+++ b/hive/indexer/tags.py
@@ -29,7 +29,7 @@ class Tags(object):
"""
values = []
for tag in cls._tags:
- values.append("('{}')".format(escape_characters(tag[1])))
+ values.append("({})".format(escape_characters(tag[1])))
if len(values) >= limit:
tag_query = str(sql)
DB.query(tag_query.format(','.join(values)))
@@ -59,7 +59,7 @@ class Tags(object):
"""
values = []
for tag in cls._tags:
- values.append("({}, '{}')".format(tag[0], escape_characters(tag[1])))
+ values.append("({}, {})".format(tag[0], escape_characters(tag[1])))
if len(values) >= limit:
tag_query = str(sql)
DB.query(tag_query.format(','.join(values)))
diff --git a/hive/server/condenser_api/objects.py b/hive/server/condenser_api/objects.py
index 0c52fba04cb168db623e9d3850b0df094458e298..830fe49ba777ddcceb1e896b99841faa8751b02f 100644
--- a/hive/server/condenser_api/objects.py
+++ b/hive/server/condenser_api/objects.py
@@ -44,49 +44,41 @@ async def load_posts_keyed(db, ids, truncate_body=0):
sql = """
SELECT hp.id,
hp.community_id,
- ha_a.name as author,
- hpd_p.permlink as permlink,
- hpd.title as title,
- hpd.body as body,
- hcd.category as category,
- depth,
- promoted,
- payout,
- payout_at,
- is_paidout,
- children,
- 0 as votes,
- 0 as active_votes,
+ hp.author,
+ hp.permlink,
+ hp.title,
+ hp.body,
+ hp.category,
+ hp.depth,
+ hp.promoted,
+ hp.payout,
+ hp.payout_at,
+ hp.is_paidout,
+ hp.children,
+ hp.votes,
+ hp.active_votes,
hp.created_at,
- updated_at,
- rshares,
- hpd.json as json,
- is_hidden,
- is_grayed,
- total_votes,
- flag_weight,
- ha_pa.name as parent_author,
- hpd_pp.permlink as parent_permlink,
- curator_payout_value,
- ha_ra.name as root_author,
- hpd_rp.permlink as root_permlink,
- max_accepted_payout,
- percent_hbd,
- allow_replies,
- allow_votes,
- allow_curation_rewards,
- beneficiaries,
- url,
- root_title
- FROM hive_posts hp
- INNER JOIN hive_accounts ha_a ON ha_a.id = hp.author_id
- INNER JOIN hive_permlink_data hpd_p ON hpd_p.id = hp.permlink_id
- LEFT JOIN hive_post_data hpd ON hpd.id = hp.id
- LEFT JOIN hive_category_data hcd ON hcd.id = hp.category_id
- INNER JOIN hive_accounts ha_pa ON ha_pa.id = hp.parent_author_id
- INNER JOIN hive_permlink_data hpd_pp ON hpd_pp.id = hp.parent_permlink_id
- INNER JOIN hive_accounts ha_ra ON ha_ra.id = hp.root_author_id
- INNER JOIN hive_permlink_data hpd_rp ON hpd_rp.id = hp.root_permlink_id
+ hp.updated_at,
+ hp.rshares,
+ hp.json as json,
+ hp.is_hidden,
+ hp.is_grayed,
+ hp.total_votes,
+ hp.flag_weight,
+ hp.parent_author,
+ hp.parent_permlink,
+ hp.curator_payout_value,
+ hp.root_author,
+ hp.root_permlink,
+ hp.max_accepted_payout,
+ hp.percent_hbd,
+ hp.allow_replies,
+ hp.allow_votes,
+ hp.allow_curation_rewards,
+ hp.beneficiaries,
+ hp.url,
+ hp.root_title
+ FROM vw_hive_posts hp
WHERE hp.id IN :ids"""
result = await db.query_all(sql, ids=tuple(ids))
@@ -214,13 +206,6 @@ def _condenser_post_object(row, truncate_body=0):
post['body_length'] = len(row['body'])
post['author_reputation'] = rep_to_raw(row['author_rep'])
- post['root_author'] = row['root_author']
- post['root_permlink'] = row['root_permlink']
-
- post['allow_replies'] = row['allow_replies']
- post['allow_votes'] = row['allow_votes']
- post['allow_curation_rewards'] = row['allow_curation_rewards']
-
if row['depth'] > 0:
post['parent_author'] = row['parent_author']
post['parent_permlink'] = row['parent_permlink']
diff --git a/hive/utils/normalize.py b/hive/utils/normalize.py
index 722fd8a38710d351e03d32b432534d65733736d5..64239d6ce39e1f348e4fd089e89392f7beb0e95e 100644
--- a/hive/utils/normalize.py
+++ b/hive/utils/normalize.py
@@ -3,6 +3,8 @@
import logging
import math
import decimal
+import time
+
from datetime import datetime
from pytz import utc
import ujson as json
@@ -13,14 +15,33 @@ NAI_MAP = {
'@@000000037': 'VESTS',
}
+dct={'0':'a','1':'b','2':'c','3':'d','4':'e',
+ '5':'f','6':'g','7':'h','8':'i','9':'j'}
+
+# convert special chars into their octal formats recognized by sql
+special_chars={
+ "\\":"\\134",
+ "'":"\\047",
+ "%":"\\045",
+ "_":"\\137",
+ ":":"\\072"
+}
+
def escape_characters(text):
""" Escape special charactes """
- ret = str(text)
- ret = ret.replace("\\", "\\\\")
- ret = ret.replace("'", "''")
- ret = ret.replace("%", '%%')
- ret = ret.replace("_", "\\_")
- ret = ret.replace(":", "\\:")
+ if len(text.strip()) == 0:
+ return "'" + text + "'"
+
+ ret = "E'"
+
+ for ch in text:
+ try:
+ dw=special_chars[ch]
+ ret=ret+dw
+ except KeyError as k:
+ ret=ret+ch
+
+ ret = ret + "'"
return ret
def vests_amount(value):
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 00af8dd4c1672bce08f66bd0294ce1a9da97e396..483ec9aecd1d218804320434f364753427ac47b3 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -76,7 +76,7 @@ ADD_API_PYREST_TEST(${CMAKE_BINARY_DIR}/tests/tests_api ${CMAKE_CURRENT_SOURCE_D
ADD_API_PYREST_TEST(${CMAKE_BINARY_DIR}/tests/tests_api ${CMAKE_CURRENT_SOURCE_DIR}/tests_api hivemind 5000000 database_api )
ADD_API_PYREST_TEST(${CMAKE_BINARY_DIR}/tests/tests_api ${CMAKE_CURRENT_SOURCE_DIR}/tests_api hivemind 5000000 follow_api )
ADD_API_PYREST_TEST(${CMAKE_BINARY_DIR}/tests/tests_api ${CMAKE_CURRENT_SOURCE_DIR}/tests_api hivemind 5000000 hive_api )
-
+ADD_API_PYREST_TEST(${CMAKE_BINARY_DIR}/tests/tests_api ${CMAKE_CURRENT_SOURCE_DIR}/tests_api hivemind 5000000 tags_api )