stages:
  - build
  - data-supply
  - e2e-test


variables:

  PGPASSWORD: $HIVEMIND_POSTGRES_PASSWORD

  # GIT_DEPTH: 10
  GIT_DEPTH: 1

  # GIT_STRATEGY: fetch # Noticed errors with that.
  GIT_STRATEGY: clone
  # GIT_STRATEGY: none

  GIT_SUBMODULE_STRATEGY: recursive

  PIPENV_VENV_IN_PROJECT: 1
  PIPENV_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pipenv"
  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"

  POSTGRES_CLIENT_TOOLS_PATH: /usr/lib/postgresql

  # POSTGRES_HOST: 172.17.0.1 # Host
  # POSTGRES_HOST: postgres-10 # Docker service
  POSTGRES_PORT: 5432

  # Set on project level in Gitlab CI.
  # We need create role and create db privileges.
  # ADMIN_POSTGRES_USER: postgres
  # ADMIN_POSTGRES_USER_PASSWORD: postgres

  # Needed by old runner ssh-executor, probably.
  POSTGRES_USER: $HIVEMIND_POSTGRES_USER
  POSTGRES_PASSWORD: $HIVEMIND_POSTGRES_PASSWORD
  POSTGRES_HOST_AUTH_METHOD: trust

  HIVEMIND_DB_NAME: "hive_${CI_COMMIT_REF_SLUG}_pipeline_id_${CI_PIPELINE_ID}"
  HIVEMIND_EXEC_NAME: $DB_NAME

  # Set on project level in Gitlab CI.
  # HIVEMIND_POSTGRES_USER: hivemind_ci

  # Set on project level in Gitlab CI.
  HIVEMIND_POSTGRES_USER_PASSWORD: $HIVEMIND_POSTGRES_PASSWORD

  # Set on project level in Gitlab CI.
  # HIVEMIND_HTTP_PORT: 18080

  # Set on project level in Gitlab CI.
  # HIVEMIND_MAX_BLOCK: 10001
  # HIVEMIND_MAX_BLOCK: 5000001

  # Set on project level in Gitlab CI.
  # HIVEMIND_SOURCE_HIVED_URL: {"default":"http://hive-4.pl.syncad.com:8091"}
  # HIVEMIND_SOURCE_HIVED_URL: {"default":"192.168.6.136:8091"}
  # HIVEMIND_SOURCE_HIVED_URL: {"default":"http://172.17.0.1:8091"}


.postgres-10: &postgres-10
  name: hivemind/postgres:10
  alias: db
  command: [
      "postgres",
      "-c", "shared_preload_libraries=pg_stat_statements,pg_qualstats",
      "-c", "track_functions=pl",
      "-c", "track_io_timing=on",
      "-c", "track_activity_query_size=2048",
      "-c", "pg_stat_statements.max=10000",
      "-c", "pg_stat_statements.track=all",
      "-c", "max_connections=100",
      "-c", "shared_buffers=2GB",
      "-c", "effective_cache_size=6GB",
      "-c", "maintenance_work_mem=512MB",
      "-c", "checkpoint_completion_target=0.9",
      "-c", "wal_buffers=16MB",
      "-c", "default_statistics_target=100",
      "-c", "random_page_cost=1.1",
      "-c", "effective_io_concurrency=200",
      "-c", "work_mem=5242kB",
      "-c", "min_wal_size=2GB",
      "-c", "max_wal_size=8GB",
      "-c", "max_worker_processes=4",
      "-c", "max_parallel_workers_per_gather=2",
      "-c", "max_parallel_workers=4",
      ]

.postgres-12: &postgres-12
  name: hivemind/postgres:12
  alias: db
  command: [
      "postgres",
      "-c", "shared_preload_libraries=pg_stat_statements,pg_qualstats",
      "-c", "track_functions=pl",
      "-c", "track_io_timing=on",
      "-c", "track_activity_query_size=2048",
      "-c", "pg_stat_statements.max=10000",
      "-c", "pg_stat_statements.track=all",
      "-c", "max_connections=100",
      "-c", "shared_buffers=2GB",
      "-c", "effective_cache_size=6GB",
      "-c", "maintenance_work_mem=512MB",
      "-c", "checkpoint_completion_target=0.9",
      "-c", "wal_buffers=16MB",
      "-c", "default_statistics_target=100",
      "-c", "random_page_cost=1.1",
      "-c", "effective_io_concurrency=200",
      "-c", "work_mem=5242kB",
      "-c", "min_wal_size=2GB",
      "-c", "max_wal_size=8GB",
      "-c", "max_worker_processes=4",
      "-c", "max_parallel_workers_per_gather=2",
      "-c", "max_parallel_workers=4",
      ]

.setup-pip: &setup-pip
  - python -m venv .venv
  - source .venv/bin/activate
  - time pip install --upgrade pip setuptools wheel
  - pip --version
  - easy_install --version
  - wheel version
  - pipenv --version
  - poetry --version
  - time pip install --editable .

.setup-setuptools: &setup-setuptools
  - python -m venv .venv
  - source .venv/bin/activate
  - time pip install --upgrade pip setuptools wheel
  - pip --version
  - easy_install --version
  - wheel version
  - pipenv --version
  - poetry --version
  - time python setup.py develop

# no virtual environment
.setuptools: &setup-setuptools-no-venv
  # setuptools will install all dependencies to this directory.
  - export PYTHONUSERBASE=./local-site
  - time pip install --upgrade pip setuptools wheel
  - pip --version
  - easy_install --version
  - wheel version
  - pipenv --version
  - poetry --version
  - mkdir -p `python -m site --user-site`
  - python setup.py install --user --force
  # we can probably also run via: ./hive/cli.py
  - ln -sf ./local-site/bin/hive "$HIVEMIND_EXEC_NAME"

.setup-pipenv: &setup-pipenv
  ## Note, that Pipfile must exist.
  ## `--sequential` is slower, but doesn't emit messages about errors
  ## and need to repeat install.
  ## - pipenv sync --dev --bare --sequential
  ## It's faster than `--sequential`, but emits messages about errors
  ## and a need to repeat install, sometimes. However seems these
  ## errors are negligible.
  - time pipenv sync --dev --bare
  - source .venv/bin/activate
  - pip --version
  - easy_install --version
  - wheel version
  - pipenv --version
  - poetry --version

.set-variables: &set-variables
  - whoami
  # list all variables predefined by Gitlab CI
  # - export
  - echo "CI_PIPELINE_URL is $CI_PIPELINE_URL"
  - echo "CI_PIPELINE_ID is $CI_PIPELINE_ID"
  - echo "CI_COMMIT_SHORT_SHA is $CI_COMMIT_SHORT_SHA"
  - echo "CI_COMMIT_REF_SLUG is $CI_COMMIT_REF_SLUG"
  - export HIVEMIND_DB_NAME=${HIVEMIND_DB_NAME//[^a-zA-Z0-9_]/_}
  - echo "HIVEMIND_DB_NAME is $HIVEMIND_DB_NAME"
  - export HIVEMIND_POSTGRESQL_CONNECTION_STRING=postgresql://${HIVEMIND_POSTGRES_USER}:${HIVEMIND_POSTGRES_USER_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${HIVEMIND_DB_NAME}

.fetch-git-tags: &fetch-git-tags
  # - git fetch --tags
  - git tag -f ci_implicit_tag # Needed to build python package

.start_timer: &start-timer
  - ./scripts/ci/timer.sh start

.stop-timer: &stop-timer
  - ./scripts/ci/timer.sh check

.hive-sync-script-common: &hive-sync-script-common
  - ./scripts/ci/wait-for-postgres.sh ${POSTGRES_HOST} ${POSTGRES_PORT}
  - export POSTGRES_MAJOR_VERSION=$(./scripts/ci/get-postgres-version.sh)
  - ./scripts/ci/create-db.sh
  - ./scripts/ci/hive-sync.sh
  - ./scripts/ci/collect-db-stats.sh

.default-rules: &default-rules
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: always
    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH == "develop"'
      when: always
    - if: '$CI_PIPELINE_SOURCE == "push"'
      when: manual
    - when: on_success

default:
  image: hivemind/python:3.6
  # image: hivemind/python:3.8
  interruptible: false
  timeout: 2h
  cache: &global-cache
    # Per-branch caching. CI_COMMIT_REF_SLUG is the same thing.
    # key: "$CI_COMMIT_REF_NAME"
    # Per project caching – use any key. Change this key, if you need
    # to clear cache
    key: common-1
    paths:
      - .cache/
      - .venv/
      - .tox/
  before_script:
    - *start-timer
    - *fetch-git-tags
    - *set-variables
    - *setup-pip
  after_script:
    - *stop-timer

##### Jobs #####

.build-egg:
  stage: build
  needs: []
  script:
    - python setup.py bdist_egg
    - ls -l dist/*
  artifacts:
    paths:
      - dist/
    expire_in: 7 days
  tags:
    - hivemind-light-job

.build-wheel:
  stage: build
  needs: []
  script:
    - python setup.py bdist_wheel
    - ls -l dist/*
  artifacts:
    paths:
      - dist/
    expire_in: 7 days
  tags:
    - hivemind-light-job

# Postgres shared
hivemind-sync:
  <<: *default-rules
  stage: data-supply
  needs: []
  script:
    - *hive-sync-script-common
  artifacts:
    paths:
      - hivemind-sync.log
      - pg-stats
    expire_in: 7 days
  tags:
    - hivemind-heavy-job

# Postgres as service
.hivemind-sync:
  <<: *default-rules
  stage: data-supply
  services:
    - *postgres-10
    # - *postgres-12
  needs: []
  script:
    - *hive-sync-script-common
    # - ./scripts/ci/dump-db.sh
  artifacts:
    paths:
      - hivemind-sync.log
      - pg-stats
      - pg-dump-${HIVEMIND_DB_NAME}
    expire_in: 7 hours
  tags:
    - hivemind-heavy-job

.e2e-test-common:
  rules:
    - when: on_success
  needs:
    - job: hivemind-sync
      artifacts: false
  before_script:
    - *start-timer
    - *fetch-git-tags
    - *set-variables
    - *setup-pip
    - ./scripts/ci/wait-for-postgres.sh ${POSTGRES_HOST} ${POSTGRES_PORT}
    - ./scripts/ci/hive-server.sh start
  after_script:
    - ./scripts/ci/hive-server.sh stop
    - *stop-timer
  tags:
    - hivemind-light-job

bridge_api_smoketest:
  stage: e2e-test
  extends: .e2e-test-common
  script:
    - >
      scripts/ci/start-api-smoketest.sh localhost "$HIVEMIND_HTTP_PORT"
      bridge_api_patterns/ api_smoketest_bridge.xml
  artifacts:
    reports:
      junit: api_smoketest_bridge.xml

bridge_api_smoketest_negative:
  stage: e2e-test
  extends: .e2e-test-common
  script:
    - >
      scripts/ci/start-api-smoketest.sh localhost "$HIVEMIND_HTTP_PORT"
      bridge_api_negative/ api_smoketest_bridge_negative.xml
  artifacts:
    reports:
      junit: api_smoketest_bridge_negative.xml

condenser_api_smoketest:
  stage: e2e-test
  extends: .e2e-test-common
  script:
    - >
      scripts/ci/start-api-smoketest.sh localhost "$HIVEMIND_HTTP_PORT"
      condenser_api_patterns/ api_smoketest_condenser_api.xml
  artifacts:
    reports:
      junit: api_smoketest_condenser_api.xml

condenser_api_smoketest_negative:
  stage: e2e-test
  extends: .e2e-test-common
  script:
    - >
      scripts/ci/start-api-smoketest.sh localhost "$HIVEMIND_HTTP_PORT"
      condenser_api_negative/ api_smoketest_condenser_api_negative.xml
  artifacts:
    reports:
      junit: api_smoketest_condenser_api_negative.xml

database_api_smoketest:
  stage: e2e-test
  extends: .e2e-test-common
  script:
    - >
      scripts/ci/start-api-smoketest.sh localhost "$HIVEMIND_HTTP_PORT"
      database_api_patterns/ api_smoketest_database_api.xml
  artifacts:
    reports:
      junit: api_smoketest_database_api.xml

database_api_smoketest_negative:
  stage: e2e-test
  extends: .e2e-test-common
  script:
    - >
      scripts/ci/start-api-smoketest.sh localhost "$HIVEMIND_HTTP_PORT"
      database_api_negative/ api_smoketest_database_api_negative.xml
  artifacts:
    reports:
      junit: api_smoketest_database_api_negative.xml

follow_api_smoketest:
  stage: e2e-test
  extends: .e2e-test-common
  script:
    - >
      scripts/ci/start-api-smoketest.sh localhost "$HIVEMIND_HTTP_PORT"
      follow_api_patterns/ api_smoketest_follow_api.xml
  artifacts:
    reports:
      junit: api_smoketest_follow_api.xml

follow_api_smoketest_negative:
  stage: e2e-test
  extends: .e2e-test-common
  script:
    - >
      scripts/ci/start-api-smoketest.sh localhost "$HIVEMIND_HTTP_PORT"
      follow_api_negative/ api_smoketest_follow_api_negative.xml
  artifacts:
    reports:
      junit: api_smoketest_follow_api_negative.xml

tags_api_smoketest:
  stage: e2e-test
  extends: .e2e-test-common
  script:
    - >
      scripts/ci/start-api-smoketest.sh localhost "$HIVEMIND_HTTP_PORT"
      tags_api_patterns/ api_smoketest_tags_api.xml
  artifacts:
    reports:
      junit: api_smoketest_tags_api.xml

tags_api_smoketest_negative:
  stage: e2e-test
  extends: .e2e-test-common
  script:
    - >
      scripts/ci/start-api-smoketest.sh localhost "$HIVEMIND_HTTP_PORT"
      tags_api_negative/ api_smoketest_tags_api_negative.xml
  artifacts:
    reports:
      junit: api_smoketest_tags_api_negative.xml