Skip to content
Snippets Groups Projects
Commit 5ce6f29f authored by Dariusz Kędzierski's avatar Dariusz Kędzierski
Browse files

Add info about failing benchmarks to the console

parent b90ae0da
No related branches found
No related tags found
2 merge requests!456Release candidate v1 24,!246API tests execution time reports
Hide whitespace changes
Inline Side-by-side
scripts/ci_start_api_benchmark.sh
+ 1
− 0
View file @ 5ce6f29f
...@@ -17,6 +17,7 @@ BASE_DIR=$(pwd) ...@@ -17,6 +17,7 @@ BASE_DIR=$(pwd)
echo "Script base dir is: $BASE_DIR" echo "Script base dir is: $BASE_DIR"
pip3 install tox --user pip3 install tox --user
pip3 install prettytable --user
echo "Creating benchmark test file as: $4.py" echo "Creating benchmark test file as: $4.py"
$BASE_DIR/tests/tests_api/hivemind/benchmarks/benchmark_generator.py $3 "$4.py" "http://$1:$2" $BASE_DIR/tests/tests_api/hivemind/benchmarks/benchmark_generator.py $3 "$4.py" "http://$1:$2"
......
scripts/json_report_parser.py
+ 10
− 4
View file @ 5ce6f29f
#!/usr/bin/python3 #!/usr/bin/python3
import xml.dom.minidom
import os import os
from sys import exit from sys import exit
from json import dumps, load from json import dumps, load
...@@ -40,7 +39,7 @@ def class_to_path(class_name, class_to_path_dic): ...@@ -40,7 +39,7 @@ def class_to_path(class_name, class_to_path_dic):
return None return None
if __name__ == '__main__': if __name__ == '__main__':
above_treshold = False above_treshold = []
import argparse import argparse
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument("path_to_test_dir", type = str, help = "Path to test directory for given json benchmark file") parser.add_argument("path_to_test_dir", type = str, help = "Path to test directory for given json benchmark file")
...@@ -65,19 +64,26 @@ if __name__ == '__main__': ...@@ -65,19 +64,26 @@ if __name__ == '__main__':
ofile.write(" </head>\n") ofile.write(" </head>\n")
ofile.write(" <body>\n") ofile.write(" <body>\n")
ofile.write(" <table>\n") ofile.write(" <table>\n")
ofile.write(" <tr><th>Test name</th><th>Time [s]</th></tr>\n") ofile.write(" <tr><th>Test name</th><th>Mean time [s]</th></tr>\n")
json_data = None json_data = None
with open(args.json_file, "r") as json_file: with open(args.json_file, "r") as json_file:
json_data = load(json_file) json_data = load(json_file)
for benchmark in json_data['benchmarks']: for benchmark in json_data['benchmarks']:
if float(benchmark['stats']['mean']) > args.time_threshold: if float(benchmark['stats']['mean']) > args.time_threshold:
ofile.write(" <tr><td>{}<br/>Parameters: {}</td><td bgcolor=\"red\">{:.4f}</td></tr>\n".format(benchmark['name'], get_request_from_yaml(class_to_path(benchmark['name'][5:], class_to_path_dic)), benchmark['stats']['mean'])) ofile.write(" <tr><td>{}<br/>Parameters: {}</td><td bgcolor=\"red\">{:.4f}</td></tr>\n".format(benchmark['name'], get_request_from_yaml(class_to_path(benchmark['name'][5:], class_to_path_dic)), benchmark['stats']['mean']))
above_treshold = True above_treshold.append((benchmark['name'], benchmark['stats']['mean'], get_request_from_yaml(class_to_path(benchmark['name'][5:], class_to_path_dic))))
else: else:
ofile.write(" <tr><td>{}</td><td>{:.4f}</td></tr>\n".format(benchmark['name'], benchmark['stats']['mean'])) ofile.write(" <tr><td>{}</td><td>{:.4f}</td></tr>\n".format(benchmark['name'], benchmark['stats']['mean']))
ofile.write(" </table>\n") ofile.write(" </table>\n")
ofile.write(" </body>\n") ofile.write(" </body>\n")
ofile.write("</html>\n") ofile.write("</html>\n")
if above_treshold: if above_treshold:
from prettytable import PrettyTable
summary = PrettyTable()
print("########## Test failed with following tests above {}s threshold ##########".format(args.time_threshold))
summary.field_names = ['Test name', 'Mean time [s]', 'Call parameters']
for entry in above_treshold:
summary.add_row(entry)
print(summary)
exit(1) exit(1)
exit(0) exit(0)
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment