.gitlab-ci.yaml

@@ -174,51 +174,131 @@ hivemind_stop_server:
   tags:
      - hivemind
 
+bridge_api_smoketest_old:
+  <<: *common_api_smoketest_job
+
+  script:
+    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" test_bridge_api_patterns.tavern.yaml api_smoketest_bridge_old.xml
+
+  artifacts:
+    reports:
+      junit: api_smoketest_bridge_old.xml
+
 bridge_api_smoketest:
   <<: *common_api_smoketest_job
 
   script:
-    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" test_bridge_api_patterns.tavern.yaml api_smoketest_bridge.xml
+    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" bridge_api_patterns/ api_smoketest_bridge.xml
 
   artifacts:
     reports:
       junit: api_smoketest_bridge.xml
 
+bridge_api_smoketest_negative:
+  <<: *common_api_smoketest_job
+
+  script:
+    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" bridge_api_negative/ api_smoketest_bridge_negative.xml
+
+  artifacts:
+    reports:
+      junit: api_smoketest_bridge_negative.xml
+
+condenser_api_smoketest_old:
+  <<: *common_api_smoketest_job
+
+  script:
+    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" test_condenser_api_patterns.tavern.yaml api_smoketest_condenser_api_old.xml
+
+  artifacts:
+    reports:
+      junit: api_smoketest_condenser_api_old.xml
+
 condenser_api_smoketest:
   <<: *common_api_smoketest_job
 
   script:
-    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" test_condenser_api_patterns.tavern.yaml api_smoketest_condenser_api.xml
+    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" condenser_api_patterns/ api_smoketest_condenser_api.xml
 
   artifacts:
     reports:
       junit: api_smoketest_condenser_api.xml
 
+condenser_api_smoketest_negative:
+  <<: *common_api_smoketest_job
+
+  script:
+    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" condenser_api_negative/ api_smoketest_condenser_api_negative.xml
+
+  artifacts:
+    reports:
+      junit: api_smoketest_condenser_api_negative.xml
+
+database_api_smoketest_old:
+  <<: *common_api_smoketest_job
+
+  script:
+    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" test_database_api_patterns.tavern.yaml api_smoketest_database_api_old.xml
+
+  artifacts:
+    reports:
+      junit: api_smoketest_database_api_old.xml
+
 database_api_smoketest:
   <<: *common_api_smoketest_job
 
   script:
-    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" test_database_api_patterns.tavern.yaml api_smoketest_database_api.xml
+    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" database_api_patterns/ api_smoketest_database_api.xml
 
   artifacts:
     reports:
       junit: api_smoketest_database_api.xml
 
+database_api_smoketest_negative:
+  <<: *common_api_smoketest_job
+
+  script:
+    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" database_api_negative/ api_smoketest_database_api_negative.xml
+
+  artifacts:
+    reports:
+      junit: api_smoketest_database_api_negative.xml
+
+follow_api_smoketest_old:
+  <<: *common_api_smoketest_job
+
+  script:
+    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" test_follow_api_patterns.tavern.yaml api_smoketest_follow_api_old.xml
+
+  artifacts:
+    reports:
+      junit: api_smoketest_follow_api_old.xml
+
 follow_api_smoketest:
   <<: *common_api_smoketest_job
 
   script:
-    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" test_follow_api_patterns.tavern.yaml api_smoketest_follow_api.xml
+    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" follow_api_patterns/ api_smoketest_follow_api.xml
 
   artifacts:
     reports:
       junit: api_smoketest_follow_api.xml
 
+tags_api_smoketest_old:
+  <<: *common_api_smoketest_job
+
+  script:
+    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" test_tags_api_patterns.tavern.yaml api_smoketest_tags_api_old.xml
+
+  artifacts:
+    reports:
+      junit: api_smoketest_tags_api_old.xml
+
 tags_api_smoketest:
   <<: *common_api_smoketest_job
 
   script:
-    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" test_tags_api_patterns.tavern.yaml api_smoketest_tags_api.xml
+    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" tags_api_patterns/ api_smoketest_tags_api.xml
 
   artifacts:
     reports:

hive/db/schema.py

@@ -767,21 +767,31 @@ def setup(db):
     db.query_no_return(sql)
 
     sql = """
-      DROP FUNCTION IF EXISTS find_comment_id(character varying, character varying)
+      DROP FUNCTION IF EXISTS find_comment_id(character varying, character varying, boolean)
       ;
       CREATE OR REPLACE FUNCTION find_comment_id(
         in _author hive_accounts.name%TYPE,
-        in _permlink hive_permlink_data.permlink%TYPE)
-      RETURNS INT AS
+        in _permlink hive_permlink_data.permlink%TYPE,
+        in _check boolean)
+      RETURNS INT
+      LANGUAGE 'plpgsql'
+      AS
       $function$
-      SELECT COALESCE( (SELECT hp.id
-      FROM hive_posts hp
-      JOIN hive_accounts ha ON ha.id = hp.author_id
-      JOIN hive_permlink_data hpd ON hpd.id = hp.permlink_id
-      WHERE ha.name = _author AND hpd.permlink = _permlink AND hp.counter_deleted = 0
-      ), 0 );
+      DECLARE 
+        post_id INT;
+      BEGIN
+        SELECT INTO post_id COALESCE( (SELECT hp.id
+        FROM hive_posts hp
+        JOIN hive_accounts ha ON ha.id = hp.author_id
+        JOIN hive_permlink_data hpd ON hpd.id = hp.permlink_id
+        WHERE ha.name = _author AND hpd.permlink = _permlink AND hp.counter_deleted = 0
+        ), 0 );
+        IF _check AND (_author <> '' OR _permlink <> '') AND post_id = 0 THEN
+          RAISE EXCEPTION 'Post %/% does not exist', _author, _permlink;
+        END IF;
+        RETURN post_id;
+      END
       $function$
-      LANGUAGE sql
       ;
     """
     db.query_no_return(sql)
@@ -845,7 +855,7 @@ def setup(db):
         DECLARE
           __post_id INT;
         BEGIN
-          __post_id = find_comment_id(_author,_permlink);
+          __post_id = find_comment_id(_author,_permlink, False);
           RETURN QUERY
           SELECT
               hp.id, hp.community_id, hp.author, hp.permlink, hp.title, hp.body,
@@ -925,8 +935,8 @@ def setup(db):
           __root_id INT;
           __post_id INT;
         BEGIN
-          __root_id = find_comment_id(_root_author,_root_permlink);
-          __post_id = find_comment_id(_start_post_author,_start_post_permlink);
+          __root_id = find_comment_id(_root_author, _root_permlink, True);
+          __post_id = find_comment_id(_start_post_author, _start_post_permlink, True);
           RETURN QUERY
           SELECT
             hp.id, hp.community_id, hp.author, hp.permlink, hp.title, hp.body,
@@ -965,11 +975,10 @@ def setup(db):
         in _limit INT)
         RETURNS SETOF database_api_post 
         LANGUAGE sql
-
         COST 100
         STABLE
         ROWS 1000
-      AS $BODY$
+      AS $function$
         SELECT
           hp.id, hp.community_id, hp.author, hp.permlink, hp.title, hp.body,
           hp.category, hp.depth, hp.promoted, hp.payout, hp.last_payout_at, hp.cashout_time, hp.is_paidout,
@@ -988,7 +997,7 @@ def setup(db):
             WHERE
               h.parent_author > _parent_author OR
               h.parent_author = _parent_author AND ( h.parent_permlink_or_category > _parent_permlink OR
-              h.parent_permlink_or_category = _parent_permlink AND h.id >= find_comment_id(_start_post_author,_start_post_permlink) )
+              h.parent_permlink_or_category = _parent_permlink AND h.id >= find_comment_id(_start_post_author, _start_post_permlink, True) )
             ORDER BY
               h.parent_author ASC,
               h.parent_permlink_or_category ASC,
@@ -999,8 +1008,8 @@ def setup(db):
         WHERE
           NOT hp.is_muted
           ;
-      $BODY$;
-        ;
+      $function$
+      ;
 
       DROP FUNCTION IF EXISTS list_comments_by_last_update(character varying, timestamp, character varying, character varying, int)
       ;
@@ -1016,7 +1025,7 @@ def setup(db):
         DECLARE
           __post_id INT;
         BEGIN
-          __post_id = find_comment_id(_start_post_author,_start_post_permlink);
+          __post_id = find_comment_id(_start_post_author, _start_post_permlink, True);
           RETURN QUERY
           SELECT
               hp.id, hp.community_id, hp.author, hp.permlink, hp.title, hp.body,
@@ -1060,7 +1069,7 @@ def setup(db):
         DECLARE
           __post_id INT;
         BEGIN
-          __post_id = find_comment_id(_start_post_author,_start_post_permlink);
+          __post_id = find_comment_id(_start_post_author, _start_post_permlink, True);
           RETURN QUERY
           SELECT
               hp.id, hp.community_id, hp.author, hp.permlink, hp.title, hp.body,

hive/indexer/accounts.py

@@ -59,7 +59,7 @@ class Accounts:
     def get_id(cls, name):
         """Get account id by name. Throw if not found."""
         assert isinstance(name, str), "account name should be string"
-        assert name in cls._ids, "account does not exist or was not registered"
+        assert name in cls._ids, 'Account \'%s\' does not exist' % name
         return cls._ids[name]
 
     @classmethod

hive/indexer/community.py

@@ -461,7 +461,7 @@ class CommunityOp:
         _name = read_key_str(self.op, 'community', 16)
         assert _name, 'must name a community'
         _id = Community.validated_id(_name)
-        assert _id, 'community `%s` does not exist' % _name
+        assert _id, 'Community \'%s\' does not exist' % _name
 
         self.community = _name
         self.community_id = _id

hive/server/bridge_api/methods.py

@@ -66,7 +66,7 @@ async def get_profile(context, account, observer=None):
     """Load account/profile data."""
     db = context['db']
     ret = await load_profiles(db, [valid_account(account)])
-    assert ret, 'account \'{}\' does not exist'.format(account)
+    assert ret, 'Account \'{}\' does not exist'.format(account)
 
     observer_id = await get_account_id(db, observer) if observer else None
     if observer_id:

hive/server/bridge_api/support.py

@@ -34,7 +34,7 @@ async def get_post_header(context, author, permlink):
 
     row = await db.query_row(sql, author=author, permlink=permlink)
 
-    assert row, 'post \'@{}/{}\' does not exist'.format(author,permlink)
+    assert row, 'Post {}/{} does not exist'.format(author,permlink)
 
     return dict(
         author=row['author'],

hive/server/common/helpers.py

@@ -5,6 +5,7 @@ from functools import wraps
 import traceback
 import logging
 import datetime
+from psycopg2.errors import RaiseException
 
 log = logging.getLogger(__name__)
 
@@ -20,6 +21,9 @@ def return_error_info(function):
         """Catch ApiError and AssersionError (always due to user error)."""
         try:
             return await function(*args, **kwargs)
+        except (RaiseException) as e:
+            log.error("PGSQL: %s\n%s", repr(e), traceback.format_exc())
+            raise AssertionError(e.diag.message_primary)
         except (ApiError, AssertionError, TypeError, Exception) as e:
             if isinstance(e, KeyError):
                 #TODO: KeyError overloaded for method not found. Any KeyErrors
@@ -115,3 +119,23 @@ def valid_follow_type(follow_type: str):
     """Ensure follow type is valid steemd type."""
     assert follow_type in ['blog', 'ignore'], 'invalid follow_type `%s`' % follow_type
     return follow_type
+
+def valid_date(date, allow_empty=False):
+    """ Ensure that date is in correct format """
+    if not date:
+        assert allow_empty, 'Date is blank'
+    check_date = False
+    # check format "%Y-%m-%d %H:%M:%S"
+    try:
+        check_date = (date == datetime.datetime.strptime(date, "%Y-%m-%d %H:%M:%S").strftime('%Y-%m-%d %H:%M%S'))
+    except ValueError:
+        check_date = False
+    # if check failed for format above try another format
+    # check format "%Y-%m-%dT%H:%M:%S"
+    if not check_date:
+        try:
+            check_date = (date == datetime.datetime.strptime(date, "%Y-%m-%dT%H:%M:%S").strftime('%Y-%m-%dT%H:%M:%S'))
+        except ValueError:
+            pass
+
+    assert check_date, "Date should be in format Y-m-d H:M:S or Y-m-dTH:M:S"

hive/server/database_api/methods.py

 # pylint: disable=too-many-arguments,line-too-long,too-many-lines
 from enum import Enum
 
-from hive.server.common.helpers import return_error_info, valid_limit, valid_account, valid_permlink
+from hive.server.common.helpers import return_error_info, valid_limit, valid_account, valid_permlink, valid_date
 from hive.server.database_api.objects import database_post_object
 from hive.utils.normalize import rep_to_raw, number_to_json_value, time_string_with_t
 
@@ -20,48 +20,69 @@ async def list_comments(context, start: list, limit: int, order: str):
     if order == 'by_cashout_time':
         assert len(start) == 3, "Expecting three arguments"
         cashout_time = start[0]
+        valid_date(cashout_time)
         if cashout_time[0:4] == '1969':
             cashout_time = "infinity"
         author = start[1]
+        valid_account(author, allow_empty=True)
         permlink = start[2]
+        valid_permlink(permlink, allow_empty=True)
         sql = "SELECT * FROM list_comments_by_cashout_time(:cashout_time, :author, :permlink, :limit)"
         result = await db.query_all(sql, cashout_time=cashout_time, author=author, permlink=permlink, limit=limit)
     elif order == 'by_permlink':
         assert len(start) == 2, "Expecting two arguments"
         author = start[0]
+        valid_account(author, allow_empty=True)
         permlink = start[1]
+        valid_permlink(permlink, allow_empty=True)
         sql = "SELECT * FROM list_comments_by_permlink(:author, :permlink, :limit)"
         result = await db.query_all(sql, author=author, permlink=permlink, limit=limit)
     elif order == 'by_root':
         assert len(start) == 4, "Expecting 4 arguments"
         root_author = start[0]
+        valid_account(root_author, allow_empty=True)
         root_permlink = start[1]
+        valid_permlink(root_permlink, allow_empty=True)
         start_post_author = start[2]
+        valid_account(start_post_author, allow_empty=True)
         start_post_permlink = start[3]
+        valid_permlink(start_post_permlink, allow_empty=True)
         sql = "SELECT * FROM list_comments_by_root(:root_author, :root_permlink, :start_post_author, :start_post_permlink, :limit)"
         result = await db.query_all(sql, root_author=root_author, root_permlink=root_permlink, start_post_author=start_post_author, start_post_permlink=start_post_permlink, limit=limit)
     elif order == 'by_parent':
         assert len(start) == 4, "Expecting 4 arguments"
         parent_author = start[0]
+        valid_account(parent_author, allow_empty=True)
         parent_permlink = start[1]
+        valid_permlink(parent_permlink, allow_empty=True)
         start_post_author = start[2]
+        valid_account(start_post_author, allow_empty=True)
         start_post_permlink = start[3]
+        valid_permlink(start_post_permlink, allow_empty=True)
         sql = "SELECT * FROM list_comments_by_parent(:parent_author, :parent_permlink, :start_post_author, :start_post_permlink, :limit)"
         result = await db.query_all(sql, parent_author=parent_author, parent_permlink=parent_permlink, start_post_author=start_post_author, start_post_permlink=start_post_permlink, limit=limit)
     elif order == 'by_last_update':
         assert len(start) == 4, "Expecting 4 arguments"
         parent_author = start[0]
+        valid_account(parent_author, allow_empty=True)
         updated_at = start[1]
+        valid_date(updated_at)
         start_post_author = start[2]
+        valid_account(start_post_author, allow_empty=True)
         start_post_permlink = start[3]
+        valid_permlink(start_post_permlink, allow_empty=True)
         sql = "SELECT * FROM list_comments_by_last_update(:parent_author, :updated_at, :start_post_author, :start_post_permlink, :limit)"
         result = await db.query_all(sql, parent_author=parent_author, updated_at=updated_at, start_post_author=start_post_author, start_post_permlink=start_post_permlink, limit=limit)
     elif order == 'by_author_last_update':
         assert len(start) == 4, "Expecting 4 arguments"
         author = start[0]
+        valid_account(author, allow_empty=True)
         updated_at = start[1]
+        valid_date(updated_at)
         start_post_author = start[2]
+        valid_account(start_post_author, allow_empty=True)
         start_post_permlink = start[3]
+        valid_permlink(start_post_permlink, allow_empty=True)
         sql = "SELECT * FROM list_comments_by_author_last_update(:author, :updated_at, :start_post_author, :start_post_permlink, :limit)"
         result = await db.query_all(sql, author=author, updated_at=updated_at, start_post_author=start_post_author, start_post_permlink=start_post_permlink, limit=limit)
 

hive/utils/normalize.py

@@ -29,6 +29,7 @@ UNIT_NAI = {
 
 # convert special chars into their octal formats recognized by sql
 SPECIAL_CHARS = {
+    "\x00" : " ", # nul char cannot be stored in string column (ABW: if we ever find the need to store nul chars we'll need bytea, not text)
     "\r" : "\\015",
     "\n" : "\\012",
     "\v" : "\\013",
@@ -74,20 +75,21 @@ def escape_characters(text):
     ret = "E'"
 
     for ch in text:
-        if ch.isprintable() or ch in SPECIAL_CHARS:
-            try:
-                dw = SPECIAL_CHARS[ch]
-                ret = ret + dw
-            except KeyError:
-                ret = ret + ch
+        if ch in SPECIAL_CHARS:
+            dw = SPECIAL_CHARS[ch]
+            ret = ret + dw
+        elif ch.isprintable():
+            ret = ret + ch
         else:
+            # escaped_value = ch.encode('unicode-escape').decode('utf-8')
             ordinal = ord(ch)
-            if ordinal == 0 or ordinal >= 0x80:
-                escaped_value = 'u' + hex(ordinal)[2:]
-#                logging.info("Encoded unicode escape: {}".format(escaped_value))
-            else:
-                escaped_value = ch.encode('unicode-escape').decode('utf-8')
-
+            hexstr = hex(ordinal)[2:]
+            escaped_value = '\\u'
+            i = len(hexstr)
+            while i < 4:
+                escaped_value += '0'
+                i += 1
+            escaped_value += hexstr
             ret = ret + escaped_value
 
     ret = ret + "'"

tests_api @ 4ee51004

-Subproject commit c673b555aa055358e0f5a1e1401a4110f7f83ca3
+Subproject commit 4ee51004b4d83d2c12ca8f6e10faab762cc0262f