diff --git a/.gitlab-ci.yaml b/.gitlab-ci.yaml
index 3184bf82418b97b2a98082787df277f5f11648a8..6fdeae2ab2642d1513debf3dae0d7db91292798a 100644
--- a/.gitlab-ci.yaml
+++ b/.gitlab-ci.yaml
@@ -7,6 +7,7 @@ stages:
- data-supply
- deploy
- e2e-test
+- benchmark-tests
- post-deploy
variables:
@@ -274,3 +275,133 @@ tags_api_smoketest_negative:
reports:
junit: api_smoketest_tags_api_negative.xml
+.benchmark_tests: &common_api_benchmarks
+ stage: benchmark-tests
+ environment: hive-4.pl.syncad.com
+ needs:
+ - job: hivemind_start_server
+ artifacts: true
+
+ variables:
+ GIT_STRATEGY: none
+
+ rules:
+ - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+ when: always
+ - if: '$CI_PIPELINE_SOURCE == "push"'
+ when: manual
+ - when: on_success
+
+ tags:
+ - hivemind
+
+bridge_api_smoketest_benchmark:
+ <<: *common_api_benchmarks
+
+ script:
+ - scripts/ci_start_api_benchmark.sh localhost $HIVEMIND_HTTP_PORT tests/tests_api/hivemind/tavern/bridge_api_patterns/ api_benchmark_bridge
+
+ artifacts:
+ when: always
+ paths:
+ - tavern_benchmark_report_api_benchmark_bridge.html
+
+bridge_api_smoketest_negative_benchmark:
+ <<: *common_api_benchmarks
+
+ script:
+ - scripts/ci_start_api_benchmark.sh localhost $HIVEMIND_HTTP_PORT tests/tests_api/hivemind/tavern/bridge_api_negative/ api_benchmark_bridge_negative
+
+ artifacts:
+ when: always
+ paths:
+ - tavern_benchmark_report_api_benchmark_bridge_negative.html
+
+condenser_api_smoketest_benchmark:
+ <<: *common_api_benchmarks
+
+ script:
+ - scripts/ci_start_api_benchmark.sh localhost $HIVEMIND_HTTP_PORT tests/tests_api/hivemind/tavern/condenser_api_patterns/ api_benchmark_condenser
+
+ artifacts:
+ when: always
+ paths:
+ - tavern_benchmark_report_api_benchmark_condenser.html
+
+condenser_api_smoketest_negative_benchmark:
+ <<: *common_api_benchmarks
+
+ script:
+ - scripts/ci_start_api_benchmark.sh localhost $HIVEMIND_HTTP_PORT tests/tests_api/hivemind/tavern/condenser_api_negative/ api_benchmark_condenser_negative
+
+ artifacts:
+ when: always
+ paths:
+ - tavern_benchmark_report_api_benchmark_condenser_negative.html
+
+database_api_smoketest_benchmark:
+ <<: *common_api_benchmarks
+
+ script:
+ - scripts/ci_start_api_benchmark.sh localhost $HIVEMIND_HTTP_PORT tests/tests_api/hivemind/tavern/database_api_patterns/ api_benchmark_database
+
+ artifacts:
+ when: always
+ paths:
+ - tavern_benchmark_report_api_benchmark_database.html
+
+database_api_smoketest_negative_benchmark:
+ <<: *common_api_benchmarks
+
+ script:
+ - scripts/ci_start_api_benchmark.sh localhost $HIVEMIND_HTTP_PORT tests/tests_api/hivemind/tavern/database_api_negative/ api_benchmark_database_negative
+
+ artifacts:
+ when: always
+ paths:
+ - tavern_benchmark_report_api_benchmark_database_negative.html
+
+follow_api_smoketest_benchmark:
+ <<: *common_api_benchmarks
+
+ script:
+ - scripts/ci_start_api_benchmark.sh localhost $HIVEMIND_HTTP_PORT tests/tests_api/hivemind/tavern/follow_api_patterns/ api_benchmark_follow
+
+ artifacts:
+ when: always
+ paths:
+ - tavern_benchmark_report_api_benchmark_follow.html
+
+follow_api_smoketest_negative_benchmark:
+ <<: *common_api_benchmarks
+
+ script:
+ - scripts/ci_start_api_benchmark.sh localhost $HIVEMIND_HTTP_PORT tests/tests_api/hivemind/tavern/follow_api_negative/ api_benchmark_follow_negative
+
+ artifacts:
+ when: always
+ paths:
+ - tavern_benchmark_report_api_benchmark_follow_negative.html
+
+tags_api_smoketest_benchmark:
+ <<: *common_api_benchmarks
+
+ script:
+ - scripts/ci_start_api_benchmark.sh localhost $HIVEMIND_HTTP_PORT tests/tests_api/hivemind/tavern/tags_api_patterns/ api_benchmark_tags
+
+ artifacts:
+ when: always
+ paths:
+ - tavern_benchmark_report_api_benchmark_tags.html
+
+tags_api_smoketest_negative_benchmark:
+ <<: *common_api_benchmarks
+
+ script:
+ - scripts/ci_start_api_benchmark.sh localhost $HIVEMIND_HTTP_PORT tests/tests_api/hivemind/tavern/tags_api_negative/ api_benchmark_tags_negative
+
+ artifacts:
+ when: always
+ paths:
+ - tavern_benchmark_report_api_benchmark_tags_negative.html
+
diff --git a/scripts/ci_start_api_benchmark.sh b/scripts/ci_start_api_benchmark.sh
new file mode 100755
index 0000000000000000000000000000000000000000..4be4617875a17e1b545b7ba317b57923162fca17
--- /dev/null
+++ b/scripts/ci_start_api_benchmark.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# $1 - server address
+# $2 - server port
+# $3 - path to test directory
+# $4 - name of the benchmark script file
+
+set -e
+
+echo "========================= BENCHMARKS ================================="
+echo "Server address: $1"
+echo "Server port: $2"
+echo "Test directory to be processed: $3"
+echo "Benchmark test file name: $4.py"
+
+BASE_DIR=$(pwd)
+echo "Script base dir is: $BASE_DIR"
+
+pip3 install tox --user
+
+echo "Creating benchmark test file as: $4.py"
+$BASE_DIR/tests/tests_api/hivemind/benchmarks/benchmark_generator.py $3 "$4.py" "http://$1:$2"
+echo "Running benchmark tests on http://$1:$2"
+tox -e benchmark -- --benchmark-json="$4.json" "$4.py"
+echo "Creating html report from $4.json"
+$BASE_DIR/scripts/json_report_parser.py $3 "$4.json"
\ No newline at end of file
diff --git a/scripts/ci_start_api_smoketest.sh b/scripts/ci_start_api_smoketest.sh
index b85befd491532352b95bf4c7c45742bf804e2de7..17ff0f06f1d677b7b0cbf16cc58b97c9424d1d33 100755
--- a/scripts/ci_start_api_smoketest.sh
+++ b/scripts/ci_start_api_smoketest.sh
@@ -9,4 +9,4 @@ echo Attempting to start tests on hivemind instance listeing on: $HIVEMIND_ADDRE
echo "Selected test group (if empty all will be executed): $3"
-tox -- -W ignore::pytest.PytestDeprecationWarning -n auto --durations=0 --junitxml=../../../../$4 $3
+tox -e tavern -- -W ignore::pytest.PytestDeprecationWarning -n auto --durations=0 --junitxml=../../../../$4 $3
diff --git a/scripts/json_report_parser.py b/scripts/json_report_parser.py
new file mode 100755
index 0000000000000000000000000000000000000000..4a553dee6dfea5ccd0d2b0186b1adf6df733a077
--- /dev/null
+++ b/scripts/json_report_parser.py
@@ -0,0 +1,83 @@
+#!/usr/bin/python3
+
+import xml.dom.minidom
+import os
+from sys import exit
+from json import dumps, load
+
+def get_request_from_yaml(path_to_yaml):
+ import yaml
+ yaml_document = None
+ with open(path_to_yaml, "r") as yaml_file:
+ yaml_document = yaml.load(yaml_file, Loader=yaml.BaseLoader)
+ if "stages" in yaml_document:
+ if "request" in yaml_document["stages"][0]:
+ json_parameters = yaml_document["stages"][0]["request"].get("json", None)
+ assert json_parameters is not None, "Unable to find json parameters in request"
+ return dumps(json_parameters)
+ return ""
+
+def make_class_path_dict(root_dir):
+ import os
+ from fnmatch import fnmatch
+
+ pattern = "*.tavern.yaml"
+
+ ret = {}
+
+ for path, subdirs, files in os.walk(root_dir):
+ for name in files:
+ if fnmatch(name, pattern):
+ test_path = os.path.join(path, name)
+ ret[test_path.replace(".", "_").replace("-", "_").replace("/", "_")] = test_path
+ return ret
+
+def class_to_path(class_name, class_to_path_dic):
+ from fnmatch import fnmatch
+ for c, p in class_to_path_dic.items():
+ if fnmatch(c, "*" + class_name):
+ return p
+ return None
+
+if __name__ == '__main__':
+ above_treshold = False
+ import argparse
+ parser = argparse.ArgumentParser()
+ parser.add_argument("path_to_test_dir", type = str, help = "Path to test directory for given json benchmark file")
+ parser.add_argument("json_file", type = str, help = "Path to benchmark json file")
+ parser.add_argument("--time-threshold", dest="time_threshold", type=float, default=1.0, help="Time threshold for test execution time, tests with execution time greater than threshold will be marked on red.")
+ args = parser.parse_args()
+ html_file, _ = os.path.splitext(args.json_file)
+ html_file = "tavern_benchmark_report_" + html_file + ".html"
+ class_to_path_dic = make_class_path_dict(args.path_to_test_dir)
+ with open(html_file, "w") as ofile:
+ ofile.write("<html>\n")
+ ofile.write(" <head>\n")
+ ofile.write(" <style>\n")
+ ofile.write(" table, th, td {\n")
+ ofile.write(" border: 1px solid black;\n")
+ ofile.write(" border-collapse: collapse;\n")
+ ofile.write(" }\n")
+ ofile.write(" th, td {\n")
+ ofile.write(" padding: 15px;\n")
+ ofile.write(" }\n")
+ ofile.write(" </style>\n")
+ ofile.write(" </head>\n")
+ ofile.write(" <body>\n")
+ ofile.write(" <table>\n")
+ ofile.write(" <tr><th>Test name</th><th>Time [s]</th></tr>\n")
+ json_data = None
+ with open(args.json_file, "r") as json_file:
+ json_data = load(json_file)
+ for benchmark in json_data['benchmarks']:
+ if float(benchmark['stats']['mean']) > args.time_threshold:
+ ofile.write(" <tr><td>{}<br/>Parameters: {}</td><td bgcolor=\"red\">{:.4f}</td></tr>\n".format(benchmark['name'], get_request_from_yaml(class_to_path(benchmark['name'][5:], class_to_path_dic)), benchmark['stats']['mean']))
+ above_treshold = True
+ else:
+ ofile.write(" <tr><td>{}</td><td>{:.4f}</td></tr>\n".format(benchmark['name'], benchmark['stats']['mean']))
+ ofile.write(" </table>\n")
+ ofile.write(" </body>\n")
+ ofile.write("</html>\n")
+ if above_treshold:
+ exit(1)
+ exit(0)
diff --git a/tests/tests_api b/tests/tests_api
index 7bd064bf665df199064f318b41e7a2afa96e6c1f..380a6a4a2d478c055f25c0a5bf2e3c246c407916 160000
--- a/tests/tests_api
+++ b/tests/tests_api
@@ -1 +1 @@
-Subproject commit 7bd064bf665df199064f318b41e7a2afa96e6c1f
+Subproject commit 380a6a4a2d478c055f25c0a5bf2e3c246c407916
diff --git a/tox.ini b/tox.ini
index f475a0aa96a3b70954e319a7fe560131bebcf2eb..eb808212162539fbde0384cedea0860e2b8dbc40 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,7 +1,19 @@
[tox]
-envlist = py36
+envlist = py36, tavern, benchmark
[testenv]
+deps =
+ pytest
+
+[testenv:benchmark]
+deps =
+ {[testenv]deps}
+ pytest-benchmark
+ requests
+
+commands = pytest {posargs}
+
+[testenv:tavern]
setenv =
PYTHONPATH = {toxinidir}/tests/tests_api/hivemind/tavern:{env:PYTHONPATH:}
@@ -12,7 +24,7 @@ passenv =
changedir = tests/tests_api/hivemind/tavern
deps =
- pytest
+ {[testenv]deps}
pytest-cov
pytest-pylint
pytest-asyncio