New implementation of docker executors on CI server

scripts/ci/backup/.gitlab-ci-docker-backup.yaml

-stages:
-  - build
-  - data-supply
-  - e2e-test
-  - benchmark-tests
-
-variables:
-
-  GIT_DEPTH: 10
-  GIT_STRATEGY: fetch
-  GIT_SUBMODULE_STRATEGY: recursive
-
-  PIPENV_VENV_IN_PROJECT: 1
-  PIPENV_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pipenv"
-  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
-  POSTGRES_CLIENT_TOOLS_PATH: /usr/lib/postgresql
-  HIVEMIND_DB_NAME: "hive_${CI_COMMIT_REF_SLUG}_pipeline_id_${CI_PIPELINE_ID}"
-
-
-.setup-pip: &setup-pip
-  - python -m venv .venv
-  - source .venv/bin/activate
-  - time pip install --upgrade pip setuptools wheel
-  - pip --version
-  - easy_install --version
-  - wheel version
-  - pipenv --version
-  - poetry --version
-  - time pip install --editable .[dev]
-
-.setup-runner-env: &setup-runner-env
-  # Setup runner environment (to connect to correct postgres server, mainly).
-  - TMP_VAR=$(cat hive-sync-runner-id.txt 2>/dev/null || true); export HIVE_SYNC_RUNNER_ID=${TMP_VAR:-0}
-  - eval $(cat "$RUNNER_CONF" | ./scripts/ci/setup_env.py --current-runner-id=${CI_RUNNER_ID} --hive-sync-runner-id=${HIVE_SYNC_RUNNER_ID})
-
-.set-variables: &set-variables
-  # - export # List all variables and its values set by Gitlab CI.
-  - whoami
-  - echo "CI_RUNNER_ID is $CI_RUNNER_ID"
-  - echo "CI_PIPELINE_URL is $CI_PIPELINE_URL"
-  - echo "CI_PIPELINE_ID is $CI_PIPELINE_ID"
-  - echo "CI_COMMIT_SHORT_SHA is $CI_COMMIT_SHORT_SHA"
-  - echo "CI_COMMIT_REF_SLUG is $CI_COMMIT_REF_SLUG"
-  - export HIVEMIND_DB_NAME=${HIVEMIND_DB_NAME//[^a-zA-Z0-9_]/_}
-  - echo "HIVEMIND_DB_NAME is $HIVEMIND_DB_NAME"
-
-.fetch-git-tags: &fetch-git-tags
-  # - git fetch --tags # Looks to be unnecessary.
-  - git tag -f ci_implicit_tag # Needed to build python package
-
-.start_timer: &start-timer
-  - ./scripts/ci/timer.sh start
-
-.stop-timer: &stop-timer
-  - ./scripts/ci/timer.sh check
-
-.hive-sync-script-common: &hive-sync-script-common
-  - echo "${CI_RUNNER_ID}" > hive-sync-runner-id.txt
-  - ./scripts/ci/wait-for-postgres.sh "$RUNNER_POSTGRES_HOST" "$RUNNER_POSTGRES_PORT"
-  - export POSTGRES_MAJOR_VERSION=$(./scripts/ci/get-postgres-version.sh)
-  - ./scripts/ci/create-db.sh
-  - ./scripts/ci/hive-sync.sh
-  - ./scripts/ci/collect-db-stats.sh
-
-.rules-for-sync: &rules-for-sync
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
-      when: always
-    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH == "develop"'
-      when: always
-    - if: '$CI_PIPELINE_SOURCE == "push"'
-      when: manual
-    - when: on_success
-
-.rules-for-test: &rules-for-test
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
-      when: always
-    - if: '$CI_PIPELINE_SOURCE == "push"'
-      when: always
-    - when: on_success
-
-.default-docker:
-  image: hivemind/python:3.6
-  interruptible: true
-  cache: &global-cache
-    # Per-branch caching. CI_COMMIT_REF_SLUG is the same thing.
-    # key: "$CI_COMMIT_REF_NAME"
-    # Per project caching – use any key.
-    # Change this key, if you need to clear cache.
-    key: common-1
-    paths:
-      - .cache/
-      - .venv/
-      - .tox/
-  before_script:
-    - *start-timer
-    - *fetch-git-tags
-    - *set-variables
-    - *setup-pip
-    - *setup-runner-env
-  after_script:
-    - *stop-timer
-
-##### Jobs #####
-
-hivemind-sync:
-  # Postgres shared on host.
-  extends: .default-docker
-  <<: *rules-for-sync
-  stage: data-supply
-  needs: []
-  script:
-    - *hive-sync-script-common
-  artifacts:
-    paths:
-      - hivemind-sync.log
-      - pg-stats
-      - hive-sync-runner-id.txt
-    expire_in: 7 days
-  tags:
-    - hivemind-heavy-job
-
-.test-common:
-  extends: .default-docker
-  <<: *rules-for-test
-  needs:
-    - job: hivemind-sync
-      artifacts: true
-  allow_failure: true
-  before_script:
-    - *start-timer
-    - *fetch-git-tags
-    - *set-variables
-    - *setup-pip
-    - *setup-runner-env
-    - ./scripts/ci/wait-for-postgres.sh "$RUNNER_POSTGRES_HOST" "$RUNNER_POSTGRES_PORT"
-    - ./scripts/ci/hive-server.sh start
-  after_script:
-    - *stop-timer
-  tags:
-    - hivemind-light-job
-
-
-bridge_api_smoketest:
-  stage: e2e-test
-  extends: .test-common
-  script:
-    - |
-      ./scripts/ci/start-api-smoketest.sh \
-          localhost "$RUNNER_HIVEMIND_SERVER_HTTP_PORT" \
-          bridge_api_patterns/ api_smoketest_bridge.xml \
-          $RUNNER_TEST_JOBS
-  artifacts:
-    when: always
-    reports:
-      junit: api_smoketest_bridge.xml
-
-bridge_api_smoketest_negative:
-  stage: e2e-test
-  extends: .test-common
-  script:
-    - |
-      ./scripts/ci/start-api-smoketest.sh \
-          localhost "$RUNNER_HIVEMIND_SERVER_HTTP_PORT" \
-          bridge_api_negative/ api_smoketest_bridge_negative.xml \
-          $RUNNER_TEST_JOBS
-  artifacts:
-    when: always
-    reports:
-      junit: api_smoketest_bridge_negative.xml
-
-condenser_api_smoketest:
-  stage: e2e-test
-  extends: .test-common
-  script:
-    - |
-      ./scripts/ci/start-api-smoketest.sh \
-          localhost "$RUNNER_HIVEMIND_SERVER_HTTP_PORT" \
-          condenser_api_patterns/ api_smoketest_condenser_api.xml \
-          $RUNNER_TEST_JOBS
-  artifacts:
-    when: always
-    reports:
-      junit: api_smoketest_condenser_api.xml
-
-condenser_api_smoketest_negative:
-  stage: e2e-test
-  extends: .test-common
-  script:
-    - |
-      ./scripts/ci/start-api-smoketest.sh \
-          localhost "$RUNNER_HIVEMIND_SERVER_HTTP_PORT" \
-          condenser_api_negative/ api_smoketest_condenser_api_negative.xml \
-          $RUNNER_TEST_JOBS
-  artifacts:
-    when: always
-    reports:
-      junit: api_smoketest_condenser_api_negative.xml
-
-database_api_smoketest:
-  stage: e2e-test
-  extends: .test-common
-  script:
-    - |
-      ./scripts/ci/start-api-smoketest.sh \
-          localhost "$RUNNER_HIVEMIND_SERVER_HTTP_PORT" \
-          database_api_patterns/ api_smoketest_database_api.xml \
-          $RUNNER_TEST_JOBS
-  artifacts:
-    when: always
-    reports:
-      junit: api_smoketest_database_api.xml
-
-database_api_smoketest_negative:
-  stage: e2e-test
-  extends: .test-common
-  script:
-    - |
-      ./scripts/ci/start-api-smoketest.sh \
-          localhost "$RUNNER_HIVEMIND_SERVER_HTTP_PORT" \
-          database_api_negative/ api_smoketest_database_api_negative.xml \
-          $RUNNER_TEST_JOBS
-  artifacts:
-    when: always
-    reports:
-      junit: api_smoketest_database_api_negative.xml
-
-follow_api_smoketest:
-  stage: e2e-test
-  extends: .test-common
-  script:
-    - |
-      ./scripts/ci/start-api-smoketest.sh \
-          localhost "$RUNNER_HIVEMIND_SERVER_HTTP_PORT" \
-          follow_api_patterns/ api_smoketest_follow_api.xml \
-          $RUNNER_TEST_JOBS
-  artifacts:
-    when: always
-    reports:
-      junit: api_smoketest.xml
-
-follow_api_smoketest_negative:
-  stage: e2e-test
-  extends: .test-common
-  script:
-    - |
-      ./scripts/ci/start-api-smoketest.sh \
-          localhost "$RUNNER_HIVEMIND_SERVER_HTTP_PORT" \
-          follow_api_negative/ api_smoketest_follow_api_negative.xml \
-          $RUNNER_TEST_JOBS
-  artifacts:
-    when: always
-    reports:
-      junit: api_smoketest_follow_api_negative.xml
-
-tags_api_smoketest:
-  stage: e2e-test
-  extends: .test-common
-  script:
-    - |
-      ./scripts/ci/start-api-smoketest.sh \
-          localhost "$RUNNER_HIVEMIND_SERVER_HTTP_PORT" \
-          tags_api_negative/ api_smoketest_tags_api_negative.xml \
-          $RUNNER_TEST_JOBS
-  artifacts:
-    when: always
-    reports:
-      junit: api_smoketest_tags_api_negative.xml
-
-tags_api_smoketest_negative:
-  stage: e2e-test
-  extends: .test-common
-  script:
-    - |
-      ./scripts/ci/start-api-smoketest.sh \
-          localhost "$RUNNER_HIVEMIND_SERVER_HTTP_PORT" \
-          tags_api_patterns/ api_smoketest_tags_api.xml \
-          $RUNNER_TEST_JOBS
-  artifacts:
-    when: always
-    reports:
-      junit: api_smoketest_tags_api.xml
-
-
-
-api-smoketest-benchmark:
-  stage: benchmark-tests
-  extends: .test-common
-  script:
-    - |
-      ./scripts/ci/start-api-benchmarks.sh \
-          localhost $RUNNER_HIVEMIND_SERVER_HTTP_PORT 5 \
-          $RUNNER_TEST_JOBS
-    - ./scripts/xml_report_parser.py . ./tests/tests_api/hivemind/tavern
-  artifacts:
-    when: always
-    paths:
-      - tavern_benchmarks_report.html