Merge branch 'escaping_fix' into 'develop'

fixed unicode escaping

See merge request !113

.gitlab-ci.yaml

@@ -194,11 +194,21 @@ condenser_api_smoketest:
     reports:
       junit: api_smoketest_condenser_api.xml
 
+database_api_smoketest_old:
+  <<: *common_api_smoketest_job
+
+  script:
+    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" test_database_api_patterns.tavern.yaml api_smoketest_database_api_old.xml
+
+  artifacts:
+    reports:
+      junit: api_smoketest_database_api_old.xml
+
 database_api_smoketest:
   <<: *common_api_smoketest_job
 
   script:
-    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" test_database_api_patterns.tavern.yaml api_smoketest_database_api.xml
+    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" database_api_patterns/ api_smoketest_database_api.xml
 
   artifacts:
     reports:
@@ -214,13 +224,24 @@ follow_api_smoketest:
     reports:
       junit: api_smoketest_follow_api.xml
 
+tags_api_smoketest_old:
+  <<: *common_api_smoketest_job
+
+  script:
+    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" test_tags_api_patterns.tavern.yaml api_smoketest_tags_api_old.xml
+
+  artifacts:
+    reports:
+      junit: api_smoketest_tags_api_old.xml
+
 tags_api_smoketest:
   <<: *common_api_smoketest_job
 
   script:
-    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" test_tags_api_patterns.tavern.yaml api_smoketest_tags_api.xml
+    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" tags_api_patterns/ api_smoketest_tags_api.xml
 
   artifacts:
     reports:
       junit: api_smoketest_tags_api.xml
 
+

hive/utils/normalize.py

@@ -29,6 +29,7 @@ UNIT_NAI = {
 
 # convert special chars into their octal formats recognized by sql
 SPECIAL_CHARS = {
+    "\x00" : " ", # nul char cannot be stored in string column (ABW: if we ever find the need to store nul chars we'll need bytea, not text)
     "\r" : "\\015",
     "\n" : "\\012",
     "\v" : "\\013",
@@ -74,20 +75,21 @@ def escape_characters(text):
     ret = "E'"
 
     for ch in text:
-        if ch.isprintable() or ch in SPECIAL_CHARS:
-            try:
-                dw = SPECIAL_CHARS[ch]
-                ret = ret + dw
-            except KeyError:
-                ret = ret + ch
+        if ch in SPECIAL_CHARS:
+            dw = SPECIAL_CHARS[ch]
+            ret = ret + dw
+        elif ch.isprintable():
+            ret = ret + ch
         else:
+            # escaped_value = ch.encode('unicode-escape').decode('utf-8')
             ordinal = ord(ch)
-            if ordinal == 0 or ordinal >= 0x80:
-                escaped_value = 'u' + hex(ordinal)[2:]
-#                logging.info("Encoded unicode escape: {}".format(escaped_value))
-            else:
-                escaped_value = ch.encode('unicode-escape').decode('utf-8')
-
+            hexstr = hex(ordinal)[2:]
+            escaped_value = '\\u'
+            i = len(hexstr)
+            while i < 4:
+                escaped_value += '0'
+                i += 1
+            escaped_value += hexstr
             ret = ret + escaped_value
 
     ret = ret + "'"

tests_api @ 7d925b4e

-Subproject commit c673b555aa055358e0f5a1e1401a4110f7f83ca3
+Subproject commit 7d925b4e88faafd6d0154725bfe26d7bdfaee23f