From 18a2755936d639abb91f41be0b3324c40363ecad Mon Sep 17 00:00:00 2001
From: Wojciech Barcik <wbarcik@syncad.com>
Date: Fri, 30 Oct 2020 16:08:23 +0100
Subject: [PATCH] Restored old gitlab-ci.yaml
---
.gitlab-ci-docker.yaml | 423 +++++++++++++++++++++++++++++++++++
.gitlab-ci-old.yaml | 276 -----------------------
.gitlab-ci.yaml | 494 +++++++++++++++--------------------------
3 files changed, 596 insertions(+), 597 deletions(-)
create mode 100644 .gitlab-ci-docker.yaml
delete mode 100644 .gitlab-ci-old.yaml
diff --git a/.gitlab-ci-docker.yaml b/.gitlab-ci-docker.yaml
new file mode 100644
index 000000000..1a3b7f6ca
--- /dev/null
+++ b/.gitlab-ci-docker.yaml
@@ -0,0 +1,423 @@
+stages:
+ - build
+ - data-supply
+ - e2e-test
+
+
+variables:
+
+ PGPASSWORD: $HIVEMIND_POSTGRES_PASSWORD
+
+ # GIT_DEPTH: 10
+ GIT_DEPTH: 1
+
+ # GIT_STRATEGY: fetch # Noticed errors with that.
+ GIT_STRATEGY: clone
+ # GIT_STRATEGY: none
+
+ GIT_SUBMODULE_STRATEGY: recursive
+
+ PIPENV_VENV_IN_PROJECT: 1
+ PIPENV_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pipenv"
+ PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+
+ POSTGRES_CLIENT_TOOLS_PATH: /usr/lib/postgresql
+
+ # POSTGRES_HOST: 172.17.0.1 # Host
+ # POSTGRES_HOST: postgres-10 # Docker service
+ POSTGRES_PORT: 5432
+
+ # Set on project level in Gitlab CI.
+ # We need create role and create db privileges.
+ # ADMIN_POSTGRES_USER: postgres
+ # ADMIN_POSTGRES_USER_PASSWORD: postgres
+
+ # Needed by old runner ssh-executor, probably.
+ POSTGRES_USER: $HIVEMIND_POSTGRES_USER
+ POSTGRES_PASSWORD: $HIVEMIND_POSTGRES_PASSWORD
+ POSTGRES_HOST_AUTH_METHOD: trust
+
+ HIVEMIND_DB_NAME: "hive_${CI_COMMIT_REF_SLUG}_pipeline_id_${CI_PIPELINE_ID}"
+ HIVEMIND_EXEC_NAME: $DB_NAME
+
+ # Set on project level in Gitlab CI.
+ # HIVEMIND_POSTGRES_USER: hivemind_ci
+
+ # Set on project level in Gitlab CI.
+ HIVEMIND_POSTGRES_USER_PASSWORD: $HIVEMIND_POSTGRES_PASSWORD
+
+ # Set on project level in Gitlab CI.
+ # HIVEMIND_HTTP_PORT: 18080
+
+ # Set on project level in Gitlab CI.
+ # HIVEMIND_MAX_BLOCK: 10001
+ # HIVEMIND_MAX_BLOCK: 5000001
+
+ # Set on project level in Gitlab CI.
+ # HIVEMIND_SOURCE_HIVED_URL: {"default":"http://hive-4.pl.syncad.com:8091"}
+ # HIVEMIND_SOURCE_HIVED_URL: {"default":"192.168.6.136:8091"}
+ # HIVEMIND_SOURCE_HIVED_URL: {"default":"http://172.17.0.1:8091"}
+
+
+.postgres-10: &postgres-10
+ name: hivemind/postgres:10
+ alias: db
+ command: [
+ "postgres",
+ "-c", "shared_preload_libraries=pg_stat_statements,pg_qualstats",
+ "-c", "track_functions=pl",
+ "-c", "track_io_timing=on",
+ "-c", "track_activity_query_size=2048",
+ "-c", "pg_stat_statements.max=10000",
+ "-c", "pg_stat_statements.track=all",
+ "-c", "max_connections=100",
+ "-c", "shared_buffers=2GB",
+ "-c", "effective_cache_size=6GB",
+ "-c", "maintenance_work_mem=512MB",
+ "-c", "checkpoint_completion_target=0.9",
+ "-c", "wal_buffers=16MB",
+ "-c", "default_statistics_target=100",
+ "-c", "random_page_cost=1.1",
+ "-c", "effective_io_concurrency=200",
+ "-c", "work_mem=5242kB",
+ "-c", "min_wal_size=2GB",
+ "-c", "max_wal_size=8GB",
+ "-c", "max_worker_processes=4",
+ "-c", "max_parallel_workers_per_gather=2",
+ "-c", "max_parallel_workers=4",
+ ]
+
+.postgres-12: &postgres-12
+ name: hivemind/postgres:12
+ alias: db
+ command: [
+ "postgres",
+ "-c", "shared_preload_libraries=pg_stat_statements,pg_qualstats",
+ "-c", "track_functions=pl",
+ "-c", "track_io_timing=on",
+ "-c", "track_activity_query_size=2048",
+ "-c", "pg_stat_statements.max=10000",
+ "-c", "pg_stat_statements.track=all",
+ "-c", "max_connections=100",
+ "-c", "shared_buffers=2GB",
+ "-c", "effective_cache_size=6GB",
+ "-c", "maintenance_work_mem=512MB",
+ "-c", "checkpoint_completion_target=0.9",
+ "-c", "wal_buffers=16MB",
+ "-c", "default_statistics_target=100",
+ "-c", "random_page_cost=1.1",
+ "-c", "effective_io_concurrency=200",
+ "-c", "work_mem=5242kB",
+ "-c", "min_wal_size=2GB",
+ "-c", "max_wal_size=8GB",
+ "-c", "max_worker_processes=4",
+ "-c", "max_parallel_workers_per_gather=2",
+ "-c", "max_parallel_workers=4",
+ ]
+
+.setup-pip: &setup-pip
+ - python -m venv .venv
+ - source .venv/bin/activate
+ - time pip install --upgrade pip setuptools wheel
+ - pip --version
+ - easy_install --version
+ - wheel version
+ - pipenv --version
+ - poetry --version
+ - time pip install --editable .
+
+.setup-setuptools: &setup-setuptools
+ - python -m venv .venv
+ - source .venv/bin/activate
+ - time pip install --upgrade pip setuptools wheel
+ - pip --version
+ - easy_install --version
+ - wheel version
+ - pipenv --version
+ - poetry --version
+ - time python setup.py develop
+
+# no virtual environment
+.setuptools: &setup-setuptools-no-venv
+ # setuptools will install all dependencies to this directory.
+ - export PYTHONUSERBASE=./local-site
+ - time pip install --upgrade pip setuptools wheel
+ - pip --version
+ - easy_install --version
+ - wheel version
+ - pipenv --version
+ - poetry --version
+ - mkdir -p `python -m site --user-site`
+ - python setup.py install --user --force
+ # we can probably also run via: ./hive/cli.py
+ - ln -sf ./local-site/bin/hive "$HIVEMIND_EXEC_NAME"
+
+.setup-pipenv: &setup-pipenv
+ ## Note, that Pipfile must exist.
+ ## `--sequential` is slower, but doesn't emit messages about errors
+ ## and need to repeat install.
+ ## - pipenv sync --dev --bare --sequential
+ ## It's faster than `--sequential`, but emits messages about errors
+ ## and a need to repeat install, sometimes. However seems these
+ ## errors are negligible.
+ - time pipenv sync --dev --bare
+ - source .venv/bin/activate
+ - pip --version
+ - easy_install --version
+ - wheel version
+ - pipenv --version
+ - poetry --version
+
+.set-variables: &set-variables
+ - whoami
+ # list all variables predefined by Gitlab CI
+ # - export
+ - echo "CI_PIPELINE_URL is $CI_PIPELINE_URL"
+ - echo "CI_PIPELINE_ID is $CI_PIPELINE_ID"
+ - echo "CI_COMMIT_SHORT_SHA is $CI_COMMIT_SHORT_SHA"
+ - echo "CI_COMMIT_REF_SLUG is $CI_COMMIT_REF_SLUG"
+ - export HIVEMIND_DB_NAME=${HIVEMIND_DB_NAME//[^a-zA-Z0-9_]/_}
+ - echo "HIVEMIND_DB_NAME is $HIVEMIND_DB_NAME"
+ - export HIVEMIND_POSTGRESQL_CONNECTION_STRING=postgresql://${HIVEMIND_POSTGRES_USER}:${HIVEMIND_POSTGRES_USER_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${HIVEMIND_DB_NAME}
+
+.fetch-git-tags: &fetch-git-tags
+ # - git fetch --tags
+ - git tag -f ci_implicit_tag # Needed to build python package
+
+.start_timer: &start-timer
+ - ./scripts/ci/timer.sh start
+
+.stop-timer: &stop-timer
+ - ./scripts/ci/timer.sh check
+
+.hive-sync-script-common: &hive-sync-script-common
+ - ./scripts/ci/wait-for-postgres.sh ${POSTGRES_HOST} ${POSTGRES_PORT}
+ - export POSTGRES_MAJOR_VERSION=$(./scripts/ci/get-postgres-version.sh)
+ - ./scripts/ci/create-db.sh
+ - ./scripts/ci/hive-sync.sh
+ - ./scripts/ci/collect-db-stats.sh
+
+.default-rules: &default-rules
+ rules:
+ - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+ when: always
+ - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH == "develop"'
+ when: always
+ - if: '$CI_PIPELINE_SOURCE == "push"'
+ when: manual
+ - when: on_success
+
+default:
+ image: hivemind/python:3.6
+ # image: hivemind/python:3.8
+ interruptible: false
+ timeout: 2h
+ cache: &global-cache
+ # Per-branch caching. CI_COMMIT_REF_SLUG is the same thing.
+ # key: "$CI_COMMIT_REF_NAME"
+ # Per project caching – use any key. Change this key, if you need
+ # to clear cache
+ key: common-1
+ paths:
+ - .cache/
+ - .venv/
+ - .tox/
+ before_script:
+ - *start-timer
+ - *fetch-git-tags
+ - *set-variables
+ - *setup-pip
+ after_script:
+ - *stop-timer
+
+##### Jobs #####
+
+.build-egg:
+ stage: build
+ needs: []
+ script:
+ - python setup.py bdist_egg
+ - ls -l dist/*
+ artifacts:
+ paths:
+ - dist/
+ expire_in: 7 days
+ tags:
+ - hivemind-light-job
+
+.build-wheel:
+ stage: build
+ needs: []
+ script:
+ - python setup.py bdist_wheel
+ - ls -l dist/*
+ artifacts:
+ paths:
+ - dist/
+ expire_in: 7 days
+ tags:
+ - hivemind-light-job
+
+# Postgres shared
+hivemind-sync:
+ <<: *default-rules
+ stage: data-supply
+ needs: []
+ script:
+ - *hive-sync-script-common
+ artifacts:
+ paths:
+ - hivemind-sync.log
+ - pg-stats
+ expire_in: 7 days
+ tags:
+ - hivemind-heavy-job
+
+# Postgres as service
+.hivemind-sync:
+ <<: *default-rules
+ stage: data-supply
+ services:
+ - *postgres-10
+ # - *postgres-12
+ needs: []
+ script:
+ - *hive-sync-script-common
+ # - ./scripts/ci/dump-db.sh
+ artifacts:
+ paths:
+ - hivemind-sync.log
+ - pg-stats
+ - pg-dump-${HIVEMIND_DB_NAME}
+ expire_in: 7 hours
+ tags:
+ - hivemind-heavy-job
+
+.e2e-test-common:
+ rules:
+ - when: on_success
+ needs:
+ - job: hivemind-sync
+ artifacts: false
+ before_script:
+ - *start-timer
+ - *fetch-git-tags
+ - *set-variables
+ - *setup-pip
+ - ./scripts/ci/wait-for-postgres.sh ${POSTGRES_HOST} ${POSTGRES_PORT}
+ - ./scripts/ci/hive-server.sh start
+ after_script:
+ - ./scripts/ci/hive-server.sh stop
+ - *stop-timer
+ tags:
+ - hivemind-light-job
+
+bridge_api_smoketest:
+ stage: e2e-test
+ extends: .e2e-test-common
+ script:
+ - >
+ scripts/ci/start-api-smoketest.sh localhost "$HIVEMIND_HTTP_PORT"
+ bridge_api_patterns/ api_smoketest_bridge.xml
+ artifacts:
+ reports:
+ junit: api_smoketest_bridge.xml
+
+bridge_api_smoketest_negative:
+ stage: e2e-test
+ extends: .e2e-test-common
+ script:
+ - >
+ scripts/ci/start-api-smoketest.sh localhost "$HIVEMIND_HTTP_PORT"
+ bridge_api_negative/ api_smoketest_bridge_negative.xml
+ artifacts:
+ reports:
+ junit: api_smoketest_bridge_negative.xml
+
+condenser_api_smoketest:
+ stage: e2e-test
+ extends: .e2e-test-common
+ script:
+ - >
+ scripts/ci/start-api-smoketest.sh localhost "$HIVEMIND_HTTP_PORT"
+ condenser_api_patterns/ api_smoketest_condenser_api.xml
+ artifacts:
+ reports:
+ junit: api_smoketest_condenser_api.xml
+
+condenser_api_smoketest_negative:
+ stage: e2e-test
+ extends: .e2e-test-common
+ script:
+ - >
+ scripts/ci/start-api-smoketest.sh localhost "$HIVEMIND_HTTP_PORT"
+ condenser_api_negative/ api_smoketest_condenser_api_negative.xml
+ artifacts:
+ reports:
+ junit: api_smoketest_condenser_api_negative.xml
+
+database_api_smoketest:
+ stage: e2e-test
+ extends: .e2e-test-common
+ script:
+ - >
+ scripts/ci/start-api-smoketest.sh localhost "$HIVEMIND_HTTP_PORT"
+ database_api_patterns/ api_smoketest_database_api.xml
+ artifacts:
+ reports:
+ junit: api_smoketest_database_api.xml
+
+database_api_smoketest_negative:
+ stage: e2e-test
+ extends: .e2e-test-common
+ script:
+ - >
+ scripts/ci/start-api-smoketest.sh localhost "$HIVEMIND_HTTP_PORT"
+ database_api_negative/ api_smoketest_database_api_negative.xml
+ artifacts:
+ reports:
+ junit: api_smoketest_database_api_negative.xml
+
+follow_api_smoketest:
+ stage: e2e-test
+ extends: .e2e-test-common
+ script:
+ - >
+ scripts/ci/start-api-smoketest.sh localhost "$HIVEMIND_HTTP_PORT"
+ follow_api_patterns/ api_smoketest_follow_api.xml
+ artifacts:
+ reports:
+ junit: api_smoketest_follow_api.xml
+
+follow_api_smoketest_negative:
+ stage: e2e-test
+ extends: .e2e-test-common
+ script:
+ - >
+ scripts/ci/start-api-smoketest.sh localhost "$HIVEMIND_HTTP_PORT"
+ follow_api_negative/ api_smoketest_follow_api_negative.xml
+ artifacts:
+ reports:
+ junit: api_smoketest_follow_api_negative.xml
+
+tags_api_smoketest:
+ stage: e2e-test
+ extends: .e2e-test-common
+ script:
+ - >
+ scripts/ci/start-api-smoketest.sh localhost "$HIVEMIND_HTTP_PORT"
+ tags_api_patterns/ api_smoketest_tags_api.xml
+ artifacts:
+ reports:
+ junit: api_smoketest_tags_api.xml
+
+tags_api_smoketest_negative:
+ stage: e2e-test
+ extends: .e2e-test-common
+ script:
+ - >
+ scripts/ci/start-api-smoketest.sh localhost "$HIVEMIND_HTTP_PORT"
+ tags_api_negative/ api_smoketest_tags_api_negative.xml
+ artifacts:
+ reports:
+ junit: api_smoketest_tags_api_negative.xml
diff --git a/.gitlab-ci-old.yaml b/.gitlab-ci-old.yaml
deleted file mode 100644
index 3184bf824..000000000
--- a/.gitlab-ci-old.yaml
+++ /dev/null
@@ -1,276 +0,0 @@
-# https://hub.docker.com/r/library/python/tags/
-image: "python:3.7"
-
-stages:
-- build
-- test
-- data-supply
-- deploy
-- e2e-test
-- post-deploy
-
-variables:
- GIT_DEPTH: 1
- LC_ALL: "C"
- GIT_STRATEGY: clone
- GIT_SUBMODULE_STRATEGY: recursive
- GIT_CLONE_PATH: $CI_BUILDS_DIR/$CI_COMMIT_REF_SLUG/$CI_CONCURRENT_ID/project-name
-
- HIVEMIND_SOURCE_HIVED_URL: $HIVEMIND_SOURCE_HIVED_URL
- HIVEMIND_DB_NAME: "hive_$CI_COMMIT_REF_SLUG"
- HIVEMIND_HTTP_PORT: $((HIVEMIND_HTTP_PORT + CI_CONCURRENT_ID))
- # Configured at gitlab repository settings side
- POSTGRES_USER: $HIVEMIND_POSTGRES_USER
- POSTGRES_PASSWORD: $HIVEMIND_POSTGRES_PASSWORD
- POSTGRES_HOST_AUTH_METHOD: trust
- # official way to provide password to psql: http://www.postgresql.org/docs/9.3/static/libpq-envars.html
- PGPASSWORD: $HIVEMIND_POSTGRES_PASSWORD
-
-before_script:
- - pwd
- - echo "CI_NODE_TOTAL is $CI_NODE_TOTAL"
- - echo "CI_NODE_INDEX is $CI_NODE_INDEX"
- - echo "CI_CONCURRENT_ID is $CI_CONCURRENT_ID"
- - echo "CI_COMMIT_REF_SLUG is $CI_COMMIT_REF_SLUG"
-
-hivemind_build:
- stage: build
- script:
- - pip3 install --user --upgrade pip setuptools
- - git fetch --tags
- - git tag -f ci_implicit_tag
- - echo $PYTHONUSERBASE
- - "python3 setup.py bdist_egg"
- - ls -l dist/*
- artifacts:
- paths:
- - dist/
- expire_in: 1 week
-
- rules:
- - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- when: always
- - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH == "develop"'
- when: always
- - when: always
-
- tags:
- - hivemind
-
-hivemind_sync:
- stage: data-supply
-
- environment:
- name: "hive sync built from branch $CI_COMMIT_REF_NAME targeting database $HIVEMIND_DB_NAME"
-
- needs:
- - job: hivemind_build
- artifacts: true
- variables:
- GIT_STRATEGY: none
- PYTHONUSERBASE: ./local-site
-
- script:
- - pip3 install --user --upgrade pip setuptools
- - scripts/ci_sync.sh "$HIVEMIND_DB_NAME" "$HIVEMIND_POSTGRESQL_CONNECTION_STRING" "$HIVEMIND_SOURCE_HIVED_URL" $HIVEMIND_MAX_BLOCK $HIVEMIND_HTTP_PORT
-
- artifacts:
- paths:
- - hivemind-sync.log
-
- expire_in: 1 week
-
- rules:
- - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- when: always
- - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH == "develop"'
- when: always
- - if: '$CI_PIPELINE_SOURCE == "push"'
- when: manual
- - when: on_success
-
- tags:
- - hivemind
-
-hivemind_start_server:
- stage: deploy
- environment:
- name: "hive serve built from branch $CI_COMMIT_REF_NAME exposed on port $HIVEMIND_HTTP_PORT"
- url: "http://hive-4.pl.syncad.com:$HIVEMIND_HTTP_PORT"
- on_stop: hivemind_stop_server
-
- needs:
- - job: hivemind_build
- artifacts: true
-# - job: hivemind_sync
-# artifacts: true
- variables:
- GIT_STRATEGY: none
- PYTHONUSERBASE: ./local-site
-
- script:
- - scripts/ci_start_server.sh "$HIVEMIND_DB_NAME" "$HIVEMIND_POSTGRESQL_CONNECTION_STRING" "$HIVEMIND_SOURCE_HIVED_URL" $HIVEMIND_HTTP_PORT
-
- artifacts:
- paths:
- - hive_server.pid
- expire_in: 1 week
-
- rules:
- - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- when: always
- - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH == "develop"'
- when: always
- - if: '$CI_PIPELINE_SOURCE == "push"'
- when: manual
- - when: on_success
-
- tags:
- - hivemind
-
-hivemind_stop_server:
- stage: post-deploy
- environment:
- name: "hive serve built from branch $CI_COMMIT_REF_NAME exposed on port $HIVEMIND_HTTP_PORT"
- action: stop
-
- variables:
- GIT_STRATEGY: none
- rules:
- - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- when: always
- - when: manual
- script:
- - scripts/ci_stop_server.sh hive_server.pid
-
- needs:
- - job: hivemind_start_server
- artifacts: true
-
- tags:
- - hivemind
-
- artifacts:
- paths:
- - hive_server.log
-
-.hivemind_start_api_smoketest: &common_api_smoketest_job
- stage: e2e-test
- environment: hive-4.pl.syncad.com
- needs:
- - job: hivemind_start_server
- artifacts: true
-
- variables:
- GIT_STRATEGY: none
-
- rules:
- - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- when: always
- - if: '$CI_PIPELINE_SOURCE == "push"'
- when: manual
- - when: on_success
-
- tags:
- - hivemind
-
-bridge_api_smoketest:
- <<: *common_api_smoketest_job
-
- script:
- - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" bridge_api_patterns/ api_smoketest_bridge.xml
-
- artifacts:
- reports:
- junit: api_smoketest_bridge.xml
-
-bridge_api_smoketest_negative:
- <<: *common_api_smoketest_job
-
- script:
- - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" bridge_api_negative/ api_smoketest_bridge_negative.xml
-
- artifacts:
- reports:
- junit: api_smoketest_bridge_negative.xml
-
-condenser_api_smoketest:
- <<: *common_api_smoketest_job
-
- script:
- - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" condenser_api_patterns/ api_smoketest_condenser_api.xml
-
- artifacts:
- reports:
- junit: api_smoketest_condenser_api.xml
-
-condenser_api_smoketest_negative:
- <<: *common_api_smoketest_job
-
- script:
- - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" condenser_api_negative/ api_smoketest_condenser_api_negative.xml
-
- artifacts:
- reports:
- junit: api_smoketest_condenser_api_negative.xml
-
-database_api_smoketest:
- <<: *common_api_smoketest_job
-
- script:
- - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" database_api_patterns/ api_smoketest_database_api.xml
-
- artifacts:
- reports:
- junit: api_smoketest_database_api.xml
-
-database_api_smoketest_negative:
- <<: *common_api_smoketest_job
-
- script:
- - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" database_api_negative/ api_smoketest_database_api_negative.xml
-
- artifacts:
- reports:
- junit: api_smoketest_database_api_negative.xml
-
-follow_api_smoketest:
- <<: *common_api_smoketest_job
-
- script:
- - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" follow_api_patterns/ api_smoketest_follow_api.xml
-
- artifacts:
- reports:
- junit: api_smoketest_follow_api.xml
-
-follow_api_smoketest_negative:
- <<: *common_api_smoketest_job
-
- script:
- - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" follow_api_negative/ api_smoketest_follow_api_negative.xml
-
- artifacts:
- reports:
- junit: api_smoketest_follow_api_negative.xml
-
-tags_api_smoketest:
- <<: *common_api_smoketest_job
-
- script:
- - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" tags_api_patterns/ api_smoketest_tags_api.xml
-
- artifacts:
- reports:
- junit: api_smoketest_tags_api.xml
-
-tags_api_smoketest_negative:
- <<: *common_api_smoketest_job
-
- script:
- - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" tags_api_negative/ api_smoketest_tags_api_negative.xml
-
- artifacts:
- reports:
- junit: api_smoketest_tags_api_negative.xml
-
diff --git a/.gitlab-ci.yaml b/.gitlab-ci.yaml
index 1a3b7f6ca..e52857514 100644
--- a/.gitlab-ci.yaml
+++ b/.gitlab-ci.yaml
@@ -1,203 +1,85 @@
-stages:
- - build
- - data-supply
- - e2e-test
+# https://hub.docker.com/r/library/python/tags/
+image: "python:3.7"
+stages:
+- build
+- test
+- data-supply
+- deploy
+- e2e-test
+- post-deploy
variables:
-
- PGPASSWORD: $HIVEMIND_POSTGRES_PASSWORD
-
- # GIT_DEPTH: 10
GIT_DEPTH: 1
-
- # GIT_STRATEGY: fetch # Noticed errors with that.
+ LC_ALL: "C"
GIT_STRATEGY: clone
- # GIT_STRATEGY: none
-
GIT_SUBMODULE_STRATEGY: recursive
+ GIT_CLONE_PATH: $CI_BUILDS_DIR/$CI_COMMIT_REF_SLUG/$CI_CONCURRENT_ID/project-name
- PIPENV_VENV_IN_PROJECT: 1
- PIPENV_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pipenv"
- PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
-
- POSTGRES_CLIENT_TOOLS_PATH: /usr/lib/postgresql
-
- # POSTGRES_HOST: 172.17.0.1 # Host
- # POSTGRES_HOST: postgres-10 # Docker service
- POSTGRES_PORT: 5432
-
- # Set on project level in Gitlab CI.
- # We need create role and create db privileges.
- # ADMIN_POSTGRES_USER: postgres
- # ADMIN_POSTGRES_USER_PASSWORD: postgres
-
- # Needed by old runner ssh-executor, probably.
+ HIVEMIND_SOURCE_HIVED_URL: $HIVEMIND_SOURCE_HIVED_URL
+ HIVEMIND_DB_NAME: "hive_$CI_COMMIT_REF_SLUG"
+ HIVEMIND_HTTP_PORT: $((HIVEMIND_HTTP_PORT + CI_CONCURRENT_ID))
+ # Configured at gitlab repository settings side
POSTGRES_USER: $HIVEMIND_POSTGRES_USER
POSTGRES_PASSWORD: $HIVEMIND_POSTGRES_PASSWORD
POSTGRES_HOST_AUTH_METHOD: trust
+ # official way to provide password to psql: http://www.postgresql.org/docs/9.3/static/libpq-envars.html
+ PGPASSWORD: $HIVEMIND_POSTGRES_PASSWORD
- HIVEMIND_DB_NAME: "hive_${CI_COMMIT_REF_SLUG}_pipeline_id_${CI_PIPELINE_ID}"
- HIVEMIND_EXEC_NAME: $DB_NAME
-
- # Set on project level in Gitlab CI.
- # HIVEMIND_POSTGRES_USER: hivemind_ci
-
- # Set on project level in Gitlab CI.
- HIVEMIND_POSTGRES_USER_PASSWORD: $HIVEMIND_POSTGRES_PASSWORD
-
- # Set on project level in Gitlab CI.
- # HIVEMIND_HTTP_PORT: 18080
-
- # Set on project level in Gitlab CI.
- # HIVEMIND_MAX_BLOCK: 10001
- # HIVEMIND_MAX_BLOCK: 5000001
-
- # Set on project level in Gitlab CI.
- # HIVEMIND_SOURCE_HIVED_URL: {"default":"http://hive-4.pl.syncad.com:8091"}
- # HIVEMIND_SOURCE_HIVED_URL: {"default":"192.168.6.136:8091"}
- # HIVEMIND_SOURCE_HIVED_URL: {"default":"http://172.17.0.1:8091"}
-
-
-.postgres-10: &postgres-10
- name: hivemind/postgres:10
- alias: db
- command: [
- "postgres",
- "-c", "shared_preload_libraries=pg_stat_statements,pg_qualstats",
- "-c", "track_functions=pl",
- "-c", "track_io_timing=on",
- "-c", "track_activity_query_size=2048",
- "-c", "pg_stat_statements.max=10000",
- "-c", "pg_stat_statements.track=all",
- "-c", "max_connections=100",
- "-c", "shared_buffers=2GB",
- "-c", "effective_cache_size=6GB",
- "-c", "maintenance_work_mem=512MB",
- "-c", "checkpoint_completion_target=0.9",
- "-c", "wal_buffers=16MB",
- "-c", "default_statistics_target=100",
- "-c", "random_page_cost=1.1",
- "-c", "effective_io_concurrency=200",
- "-c", "work_mem=5242kB",
- "-c", "min_wal_size=2GB",
- "-c", "max_wal_size=8GB",
- "-c", "max_worker_processes=4",
- "-c", "max_parallel_workers_per_gather=2",
- "-c", "max_parallel_workers=4",
- ]
-
-.postgres-12: &postgres-12
- name: hivemind/postgres:12
- alias: db
- command: [
- "postgres",
- "-c", "shared_preload_libraries=pg_stat_statements,pg_qualstats",
- "-c", "track_functions=pl",
- "-c", "track_io_timing=on",
- "-c", "track_activity_query_size=2048",
- "-c", "pg_stat_statements.max=10000",
- "-c", "pg_stat_statements.track=all",
- "-c", "max_connections=100",
- "-c", "shared_buffers=2GB",
- "-c", "effective_cache_size=6GB",
- "-c", "maintenance_work_mem=512MB",
- "-c", "checkpoint_completion_target=0.9",
- "-c", "wal_buffers=16MB",
- "-c", "default_statistics_target=100",
- "-c", "random_page_cost=1.1",
- "-c", "effective_io_concurrency=200",
- "-c", "work_mem=5242kB",
- "-c", "min_wal_size=2GB",
- "-c", "max_wal_size=8GB",
- "-c", "max_worker_processes=4",
- "-c", "max_parallel_workers_per_gather=2",
- "-c", "max_parallel_workers=4",
- ]
-
-.setup-pip: &setup-pip
- - python -m venv .venv
- - source .venv/bin/activate
- - time pip install --upgrade pip setuptools wheel
- - pip --version
- - easy_install --version
- - wheel version
- - pipenv --version
- - poetry --version
- - time pip install --editable .
-
-.setup-setuptools: &setup-setuptools
- - python -m venv .venv
- - source .venv/bin/activate
- - time pip install --upgrade pip setuptools wheel
- - pip --version
- - easy_install --version
- - wheel version
- - pipenv --version
- - poetry --version
- - time python setup.py develop
-
-# no virtual environment
-.setuptools: &setup-setuptools-no-venv
- # setuptools will install all dependencies to this directory.
- - export PYTHONUSERBASE=./local-site
- - time pip install --upgrade pip setuptools wheel
- - pip --version
- - easy_install --version
- - wheel version
- - pipenv --version
- - poetry --version
- - mkdir -p `python -m site --user-site`
- - python setup.py install --user --force
- # we can probably also run via: ./hive/cli.py
- - ln -sf ./local-site/bin/hive "$HIVEMIND_EXEC_NAME"
-
-.setup-pipenv: &setup-pipenv
- ## Note, that Pipfile must exist.
- ## `--sequential` is slower, but doesn't emit messages about errors
- ## and need to repeat install.
- ## - pipenv sync --dev --bare --sequential
- ## It's faster than `--sequential`, but emits messages about errors
- ## and a need to repeat install, sometimes. However seems these
- ## errors are negligible.
- - time pipenv sync --dev --bare
- - source .venv/bin/activate
- - pip --version
- - easy_install --version
- - wheel version
- - pipenv --version
- - poetry --version
-
-.set-variables: &set-variables
- - whoami
- # list all variables predefined by Gitlab CI
- # - export
- - echo "CI_PIPELINE_URL is $CI_PIPELINE_URL"
- - echo "CI_PIPELINE_ID is $CI_PIPELINE_ID"
- - echo "CI_COMMIT_SHORT_SHA is $CI_COMMIT_SHORT_SHA"
+before_script:
+ - pwd
+ - echo "CI_NODE_TOTAL is $CI_NODE_TOTAL"
+ - echo "CI_NODE_INDEX is $CI_NODE_INDEX"
+ - echo "CI_CONCURRENT_ID is $CI_CONCURRENT_ID"
- echo "CI_COMMIT_REF_SLUG is $CI_COMMIT_REF_SLUG"
- - export HIVEMIND_DB_NAME=${HIVEMIND_DB_NAME//[^a-zA-Z0-9_]/_}
- - echo "HIVEMIND_DB_NAME is $HIVEMIND_DB_NAME"
- - export HIVEMIND_POSTGRESQL_CONNECTION_STRING=postgresql://${HIVEMIND_POSTGRES_USER}:${HIVEMIND_POSTGRES_USER_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${HIVEMIND_DB_NAME}
-.fetch-git-tags: &fetch-git-tags
- # - git fetch --tags
- - git tag -f ci_implicit_tag # Needed to build python package
+hivemind_build:
+ stage: build
+ script:
+ - pip3 install --user --upgrade pip setuptools
+ - git fetch --tags
+ - git tag -f ci_implicit_tag
+ - echo $PYTHONUSERBASE
+ - "python3 setup.py bdist_egg"
+ - ls -l dist/*
+ artifacts:
+ paths:
+ - dist/
+ expire_in: 1 week
+
+ rules:
+ - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+ when: always
+ - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH == "develop"'
+ when: always
+ - when: always
-.start_timer: &start-timer
- - ./scripts/ci/timer.sh start
+ tags:
+ - hivemind
-.stop-timer: &stop-timer
- - ./scripts/ci/timer.sh check
+hivemind_sync:
+ stage: data-supply
-.hive-sync-script-common: &hive-sync-script-common
- - ./scripts/ci/wait-for-postgres.sh ${POSTGRES_HOST} ${POSTGRES_PORT}
- - export POSTGRES_MAJOR_VERSION=$(./scripts/ci/get-postgres-version.sh)
- - ./scripts/ci/create-db.sh
- - ./scripts/ci/hive-sync.sh
- - ./scripts/ci/collect-db-stats.sh
+ environment:
+ name: "hive sync built from branch $CI_COMMIT_REF_NAME targeting database $HIVEMIND_DB_NAME"
+
+ needs:
+ - job: hivemind_build
+ artifacts: true
+ variables:
+ GIT_STRATEGY: none
+ PYTHONUSERBASE: ./local-site
+
+ script:
+ - pip3 install --user --upgrade pip setuptools
+ - scripts/ci_sync.sh "$HIVEMIND_DB_NAME" "$HIVEMIND_POSTGRESQL_CONNECTION_STRING" "$HIVEMIND_SOURCE_HIVED_URL" $HIVEMIND_MAX_BLOCK $HIVEMIND_HTTP_PORT
+
+ artifacts:
+ paths:
+ - hivemind-sync.log
+
+ expire_in: 1 week
-.default-rules: &default-rules
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: always
@@ -207,217 +89,187 @@ variables:
when: manual
- when: on_success
-default:
- image: hivemind/python:3.6
- # image: hivemind/python:3.8
- interruptible: false
- timeout: 2h
- cache: &global-cache
- # Per-branch caching. CI_COMMIT_REF_SLUG is the same thing.
- # key: "$CI_COMMIT_REF_NAME"
- # Per project caching – use any key. Change this key, if you need
- # to clear cache
- key: common-1
- paths:
- - .cache/
- - .venv/
- - .tox/
- before_script:
- - *start-timer
- - *fetch-git-tags
- - *set-variables
- - *setup-pip
- after_script:
- - *stop-timer
-
-##### Jobs #####
-
-.build-egg:
- stage: build
- needs: []
- script:
- - python setup.py bdist_egg
- - ls -l dist/*
- artifacts:
- paths:
- - dist/
- expire_in: 7 days
tags:
- - hivemind-light-job
+ - hivemind
+
+hivemind_start_server:
+ stage: deploy
+ environment:
+ name: "hive serve built from branch $CI_COMMIT_REF_NAME exposed on port $HIVEMIND_HTTP_PORT"
+ url: "http://hive-4.pl.syncad.com:$HIVEMIND_HTTP_PORT"
+ on_stop: hivemind_stop_server
+
+ needs:
+ - job: hivemind_build
+ artifacts: true
+# - job: hivemind_sync
+# artifacts: true
+ variables:
+ GIT_STRATEGY: none
+ PYTHONUSERBASE: ./local-site
-.build-wheel:
- stage: build
- needs: []
script:
- - python setup.py bdist_wheel
- - ls -l dist/*
+ - scripts/ci_start_server.sh "$HIVEMIND_DB_NAME" "$HIVEMIND_POSTGRESQL_CONNECTION_STRING" "$HIVEMIND_SOURCE_HIVED_URL" $HIVEMIND_HTTP_PORT
+
artifacts:
paths:
- - dist/
- expire_in: 7 days
+ - hive_server.pid
+ expire_in: 1 week
+
+ rules:
+ - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+ when: always
+ - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH == "develop"'
+ when: always
+ - if: '$CI_PIPELINE_SOURCE == "push"'
+ when: manual
+ - when: on_success
+
tags:
- - hivemind-light-job
+ - hivemind
-# Postgres shared
-hivemind-sync:
- <<: *default-rules
- stage: data-supply
- needs: []
+hivemind_stop_server:
+ stage: post-deploy
+ environment:
+ name: "hive serve built from branch $CI_COMMIT_REF_NAME exposed on port $HIVEMIND_HTTP_PORT"
+ action: stop
+
+ variables:
+ GIT_STRATEGY: none
+ rules:
+ - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+ when: always
+ - when: manual
script:
- - *hive-sync-script-common
- artifacts:
- paths:
- - hivemind-sync.log
- - pg-stats
- expire_in: 7 days
+ - scripts/ci_stop_server.sh hive_server.pid
+
+ needs:
+ - job: hivemind_start_server
+ artifacts: true
+
tags:
- - hivemind-heavy-job
+ - hivemind
-# Postgres as service
-.hivemind-sync:
- <<: *default-rules
- stage: data-supply
- services:
- - *postgres-10
- # - *postgres-12
- needs: []
- script:
- - *hive-sync-script-common
- # - ./scripts/ci/dump-db.sh
artifacts:
paths:
- - hivemind-sync.log
- - pg-stats
- - pg-dump-${HIVEMIND_DB_NAME}
- expire_in: 7 hours
- tags:
- - hivemind-heavy-job
+ - hive_server.log
+
+.hivemind_start_api_smoketest: &common_api_smoketest_job
+ stage: e2e-test
+ environment: hive-4.pl.syncad.com
+ needs:
+ - job: hivemind_start_server
+ artifacts: true
+
+ variables:
+ GIT_STRATEGY: none
-.e2e-test-common:
rules:
+ - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+ when: always
+ - if: '$CI_PIPELINE_SOURCE == "push"'
+ when: manual
- when: on_success
- needs:
- - job: hivemind-sync
- artifacts: false
- before_script:
- - *start-timer
- - *fetch-git-tags
- - *set-variables
- - *setup-pip
- - ./scripts/ci/wait-for-postgres.sh ${POSTGRES_HOST} ${POSTGRES_PORT}
- - ./scripts/ci/hive-server.sh start
- after_script:
- - ./scripts/ci/hive-server.sh stop
- - *stop-timer
+
tags:
- - hivemind-light-job
+ - hivemind
bridge_api_smoketest:
- stage: e2e-test
- extends: .e2e-test-common
+ <<: *common_api_smoketest_job
+
script:
- - >
- scripts/ci/start-api-smoketest.sh localhost "$HIVEMIND_HTTP_PORT"
- bridge_api_patterns/ api_smoketest_bridge.xml
+ - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" bridge_api_patterns/ api_smoketest_bridge.xml
+
artifacts:
reports:
junit: api_smoketest_bridge.xml
bridge_api_smoketest_negative:
- stage: e2e-test
- extends: .e2e-test-common
+ <<: *common_api_smoketest_job
+
script:
- - >
- scripts/ci/start-api-smoketest.sh localhost "$HIVEMIND_HTTP_PORT"
- bridge_api_negative/ api_smoketest_bridge_negative.xml
+ - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" bridge_api_negative/ api_smoketest_bridge_negative.xml
+
artifacts:
reports:
junit: api_smoketest_bridge_negative.xml
condenser_api_smoketest:
- stage: e2e-test
- extends: .e2e-test-common
+ <<: *common_api_smoketest_job
+
script:
- - >
- scripts/ci/start-api-smoketest.sh localhost "$HIVEMIND_HTTP_PORT"
- condenser_api_patterns/ api_smoketest_condenser_api.xml
+ - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" condenser_api_patterns/ api_smoketest_condenser_api.xml
+
artifacts:
reports:
junit: api_smoketest_condenser_api.xml
condenser_api_smoketest_negative:
- stage: e2e-test
- extends: .e2e-test-common
+ <<: *common_api_smoketest_job
+
script:
- - >
- scripts/ci/start-api-smoketest.sh localhost "$HIVEMIND_HTTP_PORT"
- condenser_api_negative/ api_smoketest_condenser_api_negative.xml
+ - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" condenser_api_negative/ api_smoketest_condenser_api_negative.xml
+
artifacts:
reports:
junit: api_smoketest_condenser_api_negative.xml
database_api_smoketest:
- stage: e2e-test
- extends: .e2e-test-common
+ <<: *common_api_smoketest_job
+
script:
- - >
- scripts/ci/start-api-smoketest.sh localhost "$HIVEMIND_HTTP_PORT"
- database_api_patterns/ api_smoketest_database_api.xml
+ - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" database_api_patterns/ api_smoketest_database_api.xml
+
artifacts:
reports:
junit: api_smoketest_database_api.xml
database_api_smoketest_negative:
- stage: e2e-test
- extends: .e2e-test-common
+ <<: *common_api_smoketest_job
+
script:
- - >
- scripts/ci/start-api-smoketest.sh localhost "$HIVEMIND_HTTP_PORT"
- database_api_negative/ api_smoketest_database_api_negative.xml
+ - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" database_api_negative/ api_smoketest_database_api_negative.xml
+
artifacts:
reports:
junit: api_smoketest_database_api_negative.xml
follow_api_smoketest:
- stage: e2e-test
- extends: .e2e-test-common
+ <<: *common_api_smoketest_job
+
script:
- - >
- scripts/ci/start-api-smoketest.sh localhost "$HIVEMIND_HTTP_PORT"
- follow_api_patterns/ api_smoketest_follow_api.xml
+ - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" follow_api_patterns/ api_smoketest_follow_api.xml
+
artifacts:
reports:
junit: api_smoketest_follow_api.xml
follow_api_smoketest_negative:
- stage: e2e-test
- extends: .e2e-test-common
+ <<: *common_api_smoketest_job
+
script:
- - >
- scripts/ci/start-api-smoketest.sh localhost "$HIVEMIND_HTTP_PORT"
- follow_api_negative/ api_smoketest_follow_api_negative.xml
+ - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" follow_api_negative/ api_smoketest_follow_api_negative.xml
+
artifacts:
reports:
junit: api_smoketest_follow_api_negative.xml
tags_api_smoketest:
- stage: e2e-test
- extends: .e2e-test-common
+ <<: *common_api_smoketest_job
+
script:
- - >
- scripts/ci/start-api-smoketest.sh localhost "$HIVEMIND_HTTP_PORT"
- tags_api_patterns/ api_smoketest_tags_api.xml
+ - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" tags_api_patterns/ api_smoketest_tags_api.xml
+
artifacts:
reports:
junit: api_smoketest_tags_api.xml
tags_api_smoketest_negative:
- stage: e2e-test
- extends: .e2e-test-common
+ <<: *common_api_smoketest_job
+
script:
- - >
- scripts/ci/start-api-smoketest.sh localhost "$HIVEMIND_HTTP_PORT"
- tags_api_negative/ api_smoketest_tags_api_negative.xml
+ - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" tags_api_negative/ api_smoketest_tags_api_negative.xml
+
artifacts:
reports:
junit: api_smoketest_tags_api_negative.xml
--
GitLab