CI fix issue with gitlab timeout

Requires: common-ci-configuration!48 (merged)

gitlab job timeout causes job not to produce any artifacts. So when some test freezes/deadlock for some reason its unable to say which test failed because we have no info about it (artifacts or failing test name)

See: https://gitlab.syncad.com/hive/clive/-/jobs/1140805

Related:

• hive!1281 (merged)

.gitlab-ci.yml

@@ -18,6 +18,9 @@ variables:
   GIT_SUBMODULE_STRATEGY: recursive
   # static code analysis:
   PACKAGES_TO_CHECK: "clive/ tests/"
+  # tests:
+  PYTEST_NUMBER_OF_PROCESSES: 16
+  PYTEST_LOG_DURATIONS: 1
   # colors:
   TXT_GREEN: "\e[1;32m"
   TXT_BLUE: "\e[1;34m"
@@ -93,6 +96,58 @@ build_wheel:
 
 #>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>| TESTS |>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
 
+.run-pytest: &run-pytest
+  # Usage:
+  # 1. Add - *run-pytest to your script section.
+  # 2. Set the appropriate values for:
+  #    - PYTEST_TIMEOUT_MINUTES (required, should be set on the job level)
+  #    - PYTEST_NUMBER_OF_PROCESSES (default: defined via global CI variable, could be overridden on the job level)
+  #    - PYTEST_LOG_DURATIONS (default: defined via global CI variable, could be overridden on the job level)
+  #    - PYTEST_ARGS (default: "" - empty, should be set on the job level)
+  #    - PYTEST_JUNIT_REPORT (default: "report.xml")
+  #      Best to export PYTEST_ARGS as array (e.g. `export PYTEST_ARGS=(-m testnet)`). In some cases setting PYTEST_ARGS as string will fail.
+  - |
+    PYTEST_FULL_CMD=("pytest")
+
+    if [ -z "${PYTEST_TIMEOUT_MINUTES}" ]; then
+      echo "Required variable PYTEST_TIMEOUT_MINUTES was not set!"
+      exit 22
+    fi
+
+    # add pytest timeout to prevent gitlab CI killing job on timeout without any artifacts and no info about tests
+    PYTEST_FULL_CMD+=("--timeout=$((${PYTEST_TIMEOUT_MINUTES} * 60))")
+
+    if [[ ${PYTEST_NUMBER_OF_PROCESSES} == "auto" || ${PYTEST_NUMBER_OF_PROCESSES} -gt 1 ]]; then
+      PYTEST_FULL_CMD+=("-n=${PYTEST_NUMBER_OF_PROCESSES}")
+    fi
+
+    # durations=0 makes pytest print all tests durations
+    if [ ${PYTEST_LOG_DURATIONS:=0} -ge 1 ]; then
+      PYTEST_FULL_CMD+=("--durations=0")
+    fi
+
+    if [ -z "${PYTEST_JUNIT_REPORT}" ]; then
+      JUNIT_REPORT="report.xml"
+    else
+      JUNIT_REPORT="${PYTEST_JUNIT_REPORT}"
+    fi
+
+    PYTEST_FULL_CMD+=("--junitxml=${JUNIT_REPORT}")
+
+    PYTEST_FULL_CMD+=("${PYTEST_ARGS[@]}")
+
+    echo "Launching pytest with:
+         - timeout (minutes): ${PYTEST_TIMEOUT_MINUTES}
+         - processes: ${PYTEST_NUMBER_OF_PROCESSES}
+         - log durations: ${PYTEST_LOG_DURATIONS}
+         - additional arguments: ${PYTEST_ARGS[@]}"
+    echo
+    echo "Full pytest command:"
+    echo "${PYTEST_FULL_CMD[@]}"
+
+    # add extra bash timeout
+    timeout $(((${PYTEST_TIMEOUT_MINUTES} + 2) * 60)) "${PYTEST_FULL_CMD[@]}"
+
 .testing:
   stage: tests
   extends: .project_develop_configuration_template
@@ -122,36 +177,52 @@ build_wheel:
 
 testing_beekeeper:
   extends: .testing
+  variables:
+      PYTEST_TIMEOUT_MINUTES: 10
   script:
     - echo -e "${TXT_BLUE}Launching beekeeper tests...${TXT_CLEAR}"
-    - python3 -m pytest -n auto --durations 0 --junitxml=report.xml tests/functional/beekeeper
+    - export PYTEST_ARGS=(tests/functional/beekeeper)
+    - *run-pytest
 
 testing_clive:
   extends: .testing
+  variables:
+    PYTEST_TIMEOUT_MINUTES: 10
   script:
     - echo -e "${TXT_BLUE}Launching clive concurrent tests...${TXT_CLEAR}"
-    - python3 -m pytest
-      --ignore "tests/functional/beekeeper" --ignore "tests/tui" --ignore "tests/functional/cli"
+    - export PYTEST_ARGS=(
+      --ignore tests/functional/beekeeper --ignore tests/tui --ignore tests/functional/cli
       -k "not test_autocompletion_time"
-      -n auto --durations 0 --junitxml=report.xml tests/
+      )
+    - *run-pytest
 
 testing_clive_import_times_during_autocompletion:
   extends: .testing
+  variables:
+    PYTEST_NUMBER_OF_PROCESSES: 1
+    PYTEST_TIMEOUT_MINUTES: 3
   script:
     - echo -e "${TXT_BLUE}Launching clive test for autocompletion time...${TXT_CLEAR}"
-    - python3 -m pytest -k "test_autocompletion_time" --durations 0 --junitxml=report.xml tests/
+    - export PYTEST_ARGS=(-k test_autocompletion_time)
+    - *run-pytest
 
 testing_tui:
   extends: .testing
+  variables:
+    PYTEST_TIMEOUT_MINUTES: 10
   script:
     - echo -e "${TXT_BLUE}Launching tui tests...${TXT_CLEAR}"
-    - python3 -m pytest -n auto --durations 0 --junitxml=report.xml tests/tui
+    - export PYTEST_ARGS=(tests/tui)
+    - *run-pytest
 
 testing_cli:
   extends: .testing
+  variables:
+    PYTEST_TIMEOUT_MINUTES: 10
   script:
     - echo -e "${TXT_BLUE}Launching cli commands tests...${TXT_CLEAR}"
-    - python3 -m pytest -n auto --durations 0 --junitxml=report.xml tests/functional/cli
+    - export PYTEST_ARGS=(tests/functional/cli)
+    - *run-pytest
 
 testing_password_private_key_logging:
   stage: tests