Merge branch 'release_candidate_v1_24' into 'master'

Release candidate v1 24

Closes #103, #100, and #72

See merge request !456

.gitignore

@@ -37,6 +37,7 @@ var/
 *.egg-info/
 .installed.cfg
 *.egg
+pip-wheel-metadata
 
 # PyInstaller
 #  Usually these files are written by a python script from a template
@@ -124,3 +125,24 @@ tests/failed_blocks/
 /tests/envdir-to-envfile.sh
 /deploy/
 /scripts/hive.sqlite
+
+# vscode
+.vscode/*
+
+# pyrest tests
+*.out.json
+
+# version.py
+hive/version.py
+
+hivemind.port
+hive_server.pid
+hivemind-server.pid
+
+Pipfile.lock
+
+pghero.yml
+*~
+.tmp
+
+.private

.gitlab-ci-ssh.yaml

+stages:
+  - build
+  - test
+  - data-supply
+  - deploy
+  - e2e-test
+  - benchmark-tests
+  - post-deploy
+
+variables:
+  GIT_DEPTH: 1
+  LC_ALL: "C"
+  GIT_STRATEGY: clone
+  GIT_SUBMODULE_STRATEGY: recursive
+  GIT_CLONE_PATH: $CI_BUILDS_DIR/$CI_COMMIT_REF_SLUG/$CI_CONCURRENT_ID/project-name
+
+  HIVEMIND_SOURCE_HIVED_URL: $HIVEMIND_SOURCE_HIVED_URL
+  HIVEMIND_DB_NAME: "hive_${CI_COMMIT_REF_SLUG}"
+  HIVEMIND_HTTP_PORT: $((HIVEMIND_HTTP_PORT + CI_CONCURRENT_ID))
+  # Configured at gitlab repository settings side
+  POSTGRES_USER: $HIVEMIND_POSTGRES_USER
+  POSTGRES_PASSWORD: $HIVEMIND_POSTGRES_PASSWORD
+  POSTGRES_HOST_AUTH_METHOD: trust
+  # official way to provide password to psql: http://www.postgresql.org/docs/9.3/static/libpq-envars.html
+  PGPASSWORD: $HIVEMIND_POSTGRES_PASSWORD
+
+
+default:
+  before_script:
+    - pwd
+    - echo "CI_NODE_TOTAL is $CI_NODE_TOTAL"
+    - echo "CI_NODE_INDEX is $CI_NODE_INDEX"
+    - echo "CI_CONCURRENT_ID is $CI_CONCURRENT_ID"
+    - echo "CI_COMMIT_REF_SLUG is $CI_COMMIT_REF_SLUG"
+
+hivemind_build:
+  stage: build
+  script:
+    - pip3 install --user --upgrade pip setuptools
+    - git fetch --tags
+    - git tag -f ci_implicit_tag
+    - echo $PYTHONUSERBASE
+    - "python3 setup.py bdist_egg"
+    - ls -l dist/*
+  artifacts:
+    paths:
+      - dist/
+    expire_in: 1 week
+  tags:
+     - hivemind
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+      when: always
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+      when: always
+    - if: '$CI_COMMIT_BRANCH == "develop"'
+      when: always
+
+hivemind_sync:
+  stage: data-supply
+  environment:
+      name: "hive sync built from branch $CI_COMMIT_REF_NAME targeting database $HIVEMIND_DB_NAME"
+  needs:
+    - job: hivemind_build
+      artifacts: true
+  variables:
+    GIT_STRATEGY: none
+    PYTHONUSERBASE: ./local-site
+  script:
+    - pip3 install --user --upgrade pip setuptools
+    # WARNING!!! temporarily hardcoded 5000017 instead $HIVEMIND_MAX_BLOCK
+    # revert that change when $HIVEMIND_MAX_BLOCK will be set to 5000017
+    - scripts/ci_sync.sh "$HIVEMIND_DB_NAME" "$HIVEMIND_POSTGRESQL_CONNECTION_STRING" "$HIVEMIND_SOURCE_HIVED_URL" 5000017 $HIVEMIND_HTTP_PORT
+  artifacts:
+    paths:
+      - hivemind-sync.log
+    expire_in: 1 week
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+      when: always
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+      when: always
+    - if: '$CI_COMMIT_BRANCH == "develop"'
+      when: always
+    - if: '$CI_PIPELINE_SOURCE == "push"'
+      when: manual
+    - when: on_success
+  tags:
+     - hivemind
+
+hivemind_start_server:
+  stage: deploy
+  environment:
+    name: "hive serve built from branch $CI_COMMIT_REF_NAME exposed on port $HIVEMIND_HTTP_PORT"
+    url: "http://hive-4.pl.syncad.com:$HIVEMIND_HTTP_PORT"
+    on_stop: hivemind_stop_server
+  needs:
+    - job: hivemind_build
+      artifacts: true
+#    - job: hivemind_sync
+#      artifacts: true
+  variables:
+    GIT_STRATEGY: none
+    PYTHONUSERBASE: ./local-site
+  script:
+    - scripts/ci_start_server.sh "$HIVEMIND_DB_NAME" "$HIVEMIND_POSTGRESQL_CONNECTION_STRING" "$HIVEMIND_SOURCE_HIVED_URL" $HIVEMIND_HTTP_PORT
+  artifacts:
+    paths:
+      - hive_server.pid
+    expire_in: 1 week
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+      when: always
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+      when: always
+    - if: '$CI_COMMIT_BRANCH == "develop"'
+      when: always
+    - if: '$CI_PIPELINE_SOURCE == "push"'
+      when: manual
+    - when: on_success
+
+  tags:
+     - hivemind
+
+hivemind_stop_server:
+  stage: post-deploy
+  environment:
+    name: "hive serve built from branch $CI_COMMIT_REF_NAME exposed on port $HIVEMIND_HTTP_PORT"
+    action: stop
+  variables:
+    GIT_STRATEGY: none
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+      when: always
+  script:
+    - scripts/ci_stop_server.sh hive_server.pid
+  needs:
+    - job: hivemind_start_server
+      artifacts: true
+  tags:
+     - hivemind
+  artifacts:
+    paths:
+      - hive_server.log
+
+.hivemind_start_api_smoketest: &common_api_smoketest_job
+  stage: e2e-test
+  environment: hive-4.pl.syncad.com
+  needs:
+    - job: hivemind_start_server
+      artifacts: true
+  variables:
+    GIT_STRATEGY: none
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+      when: always
+    - if: '$CI_PIPELINE_SOURCE == "push"'
+      when: manual
+    - when: on_success
+  tags:
+     - hivemind
+
+bridge_api_smoketest:
+  <<: *common_api_smoketest_job
+  script:
+    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" bridge_api_patterns/ api_smoketest_bridge.xml
+  artifacts:
+    reports:
+      junit: api_smoketest_bridge.xml
+
+bridge_api_smoketest_negative:
+  <<: *common_api_smoketest_job
+  script:
+    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" bridge_api_negative/ api_smoketest_bridge_negative.xml
+  artifacts:
+    reports:
+      junit: api_smoketest_bridge_negative.xml
+
+condenser_api_smoketest:
+  <<: *common_api_smoketest_job
+  script:
+    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" condenser_api_patterns/ api_smoketest_condenser_api.xml
+  artifacts:
+    reports:
+      junit: api_smoketest_condenser_api.xml
+
+condenser_api_smoketest_negative:
+  <<: *common_api_smoketest_job
+  script:
+    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" condenser_api_negative/ api_smoketest_condenser_api_negative.xml
+  artifacts:
+    reports:
+      junit: api_smoketest_condenser_api_negative.xml
+
+database_api_smoketest:
+  <<: *common_api_smoketest_job
+  script:
+    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" database_api_patterns/ api_smoketest_database_api.xml
+  artifacts:
+    reports:
+      junit: api_smoketest_database_api.xml
+
+database_api_smoketest_negative:
+  <<: *common_api_smoketest_job
+  script:
+    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" database_api_negative/ api_smoketest_database_api_negative.xml
+  artifacts:
+    reports:
+      junit: api_smoketest_database_api_negative.xml
+
+follow_api_smoketest:
+  <<: *common_api_smoketest_job
+  script:
+    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" follow_api_patterns/ api_smoketest_follow_api.xml
+  artifacts:
+    reports:
+      junit: api_smoketest.xml
+
+follow_api_smoketest_negative:
+  <<: *common_api_smoketest_job
+  script:
+    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" follow_api_negative/ api_smoketest_follow_api_negative.xml
+  artifacts:
+    reports:
+      junit: api_smoketest_follow_api_negative.xml
+
+tags_api_smoketest:
+  <<: *common_api_smoketest_job
+  script:
+    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" tags_api_patterns/ api_smoketest_tags_api.xml
+  artifacts:
+    reports:
+      junit: api_smoketest_tags_api.xml
+
+tags_api_smoketest_negative:
+  <<: *common_api_smoketest_job
+  script:
+    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" tags_api_negative/ api_smoketest_tags_api_negative.xml
+
+mock_tests:
+  <<: *common_api_smoketest_job
+  script:
+    - scripts/ci_start_api_smoketest.sh localhost "$HIVEMIND_HTTP_PORT" mock_tests/ api_smoketest_mock_tests.xml
+
+api_smoketest_benchmark:
+  stage: benchmark-tests
+  environment: hive-4.pl.syncad.com
+  needs:
+    - job: hivemind_start_server
+      artifacts: true
+  allow_failure: true
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+      when: always
+    - if: '$CI_PIPELINE_SOURCE == "push"'
+      when: manual
+    - when: on_success
+  tags:
+     - hivemind
+  script:
+    - ./scripts/ci_start_api_benchmarks.sh localhost $HIVEMIND_HTTP_PORT 5
+  artifacts:
+    when: always
+    paths:
+      - tavern_benchmarks_report.html

.gitlab-ci.yaml

+stages:
+  - build
+  - test
+  - sync-e2e-benchmark
+  - data-supply
+  - deploy
+  - e2e-test
+  - benchmark-tests
+  - post-deploy
+
+.setup-pip: &setup-pip
+  - python -m venv .venv
+  - source .venv/bin/activate
+  - time pip install --upgrade pip setuptools wheel
+  - pip --version
+  - easy_install --version
+  - wheel version
+  - pipenv --version
+  - poetry --version
+  - time pip install --editable .[dev]
+
+.setup-runner-env: &setup-runner-env
+  # Setup runner environment (to connect to correct postgres server, mainly).
+  - TMP_VAR=$(cat hive-sync-runner-id.txt 2>/dev/null || true); export HIVE_SYNC_RUNNER_ID=${TMP_VAR:-0}
+  - eval $(cat "$RUNNER_CONF" | ./scripts/ci/setup_env.py --current-runner-id=${CI_RUNNER_ID} --hive-sync-runner-id=${HIVE_SYNC_RUNNER_ID})
+
+.set-variables: &set-variables
+  # - export # List all variables and its values set by Gitlab CI.
+  - whoami
+  - echo "CI_RUNNER_ID is $CI_RUNNER_ID"
+  - echo "CI_PIPELINE_URL is $CI_PIPELINE_URL"
+  - echo "CI_PIPELINE_ID is $CI_PIPELINE_ID"
+  - echo "CI_COMMIT_SHORT_SHA is $CI_COMMIT_SHORT_SHA"
+  - echo "CI_COMMIT_REF_SLUG is $CI_COMMIT_REF_SLUG"
+  - export HIVEMIND_DB_NAME=${HIVEMIND_DB_NAME//[^a-zA-Z0-9_]/_}
+  - echo "HIVEMIND_DB_NAME is $HIVEMIND_DB_NAME"
+
+.fetch-git-tags: &fetch-git-tags
+  # - git fetch --tags # Looks to be unnecessary.
+  - git tag -f ci_implicit_tag # Needed to build python package
+
+.start-timer: &start-timer
+  - ./scripts/ci/timer.sh start
+
+.stop-timer: &stop-timer
+  - ./scripts/ci/timer.sh check
+
+.hive-sync-script-common: &hive-sync-script-common
+  - echo "${CI_RUNNER_ID}" > hive-sync-runner-id.txt
+  - ./scripts/ci/wait-for-postgres.sh "$RUNNER_POSTGRES_HOST" "$RUNNER_POSTGRES_PORT"
+  - export POSTGRES_MAJOR_VERSION=$(./scripts/ci/get-postgres-version.sh)
+  - ./scripts/ci/drop-db.sh
+  - ./scripts/ci/create-db.sh
+  - ./scripts/ci/hive-sync.sh
+  - ./scripts/ci/collect-db-stats.sh
+
+.rules-for-sync: &rules-for-sync
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+      when: always
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+      when: always
+    - if: '$CI_COMMIT_BRANCH == "develop"'
+      when: always
+    - if: '$CI_PIPELINE_SOURCE == "push"'
+      when: manual
+    - when: manual
+
+.rules-for-test: &rules-for-test
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+      when: on_success
+    - if: '$CI_PIPELINE_SOURCE == "push"'
+      when: on_success
+    - when: on_success
+
+.default:
+  image: hivemind/python:3.6
+  interruptible: true
+  inherit:
+    default: false
+    variables: false
+  variables:
+    GIT_DEPTH: 10
+    GIT_STRATEGY: fetch
+    GIT_SUBMODULE_STRATEGY: recursive
+    PIPENV_VENV_IN_PROJECT: 1
+    PIPENV_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pipenv"
+    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+    POSTGRES_CLIENT_TOOLS_PATH: /usr/lib/postgresql
+    HIVEMIND_DB_NAME: "hive_${CI_COMMIT_REF_SLUG}"
+  cache: &global-cache
+    # Per-branch caching. CI_COMMIT_REF_SLUG is the same thing.
+    # key: "$CI_COMMIT_REF_NAME"
+    # Per project caching – use any key.
+    # Change this key, if you need to clear cache.
+    key: common-1
+    paths:
+      - .cache/
+      - .venv/
+      - .tox/
+  before_script:
+    - *start-timer
+    - *fetch-git-tags
+    - *set-variables
+    - *setup-pip
+    - *setup-runner-env
+  after_script:
+    - *stop-timer
+
+##### Jobs #####
+
+.hivemind-sync:
+  # Postgres shared on host.
+  extends: .default
+  <<: *rules-for-sync
+  stage: data-supply
+  needs: []
+  script:
+    - *hive-sync-script-common
+  artifacts:
+    paths:
+      - hivemind-sync.log
+      - pg-stats
+      - hive-sync-runner-id.txt
+    expire_in: 7 days
+  tags:
+    - hivemind-heavy-job
+
+.test-common:
+  extends: .default
+  <<: *rules-for-test
+  needs:
+    - job: hivemind-sync
+      artifacts: true
+  allow_failure: false
+  before_script:
+    - *start-timer
+    - *fetch-git-tags
+    - *set-variables
+    - *setup-pip
+    - *setup-runner-env
+    - ./scripts/ci/wait-for-postgres.sh "$RUNNER_POSTGRES_HOST" "$RUNNER_POSTGRES_PORT"
+    - ./scripts/ci/hive-server.sh start
+  after_script:
+    - *stop-timer
+  tags:
+    - hivemind-light-job
+
+
+.bridge_api_smoketest-script: &bridge_api_smoketest-script
+  - |
+    ./scripts/ci/start-api-smoketest.sh \
+        localhost "$RUNNER_HIVEMIND_SERVER_HTTP_PORT" \
+        bridge_api_patterns/ api_smoketest_bridge.xml \
+        $RUNNER_TEST_JOBS
+
+.bridge_api_smoketest:
+  stage: e2e-test
+  extends: .test-common
+  script:
+    - *bridge_api_smoketest-script
+  artifacts:
+    when: always
+    reports:
+      junit: api_smoketest_bridge.xml
+
+
+.bridge_api_smoketest_negative-script: &bridge_api_smoketest_negative-script
+  - |
+    ./scripts/ci/start-api-smoketest.sh \
+        localhost "$RUNNER_HIVEMIND_SERVER_HTTP_PORT" \
+        bridge_api_negative/ api_smoketest_bridge_negative.xml \
+        $RUNNER_TEST_JOBS
+
+.bridge_api_smoketest_negative:
+  stage: e2e-test
+  extends: .test-common
+  script:
+    - *bridge_api_smoketest_negative-script
+  artifacts:
+    when: always
+    reports:
+      junit: api_smoketest_bridge_negative.xml
+
+
+.condenser_api_smoketest-script: &condenser_api_smoketest-script
+  - |
+    ./scripts/ci/start-api-smoketest.sh \
+        localhost "$RUNNER_HIVEMIND_SERVER_HTTP_PORT" \
+        condenser_api_patterns/ api_smoketest_condenser_api.xml \
+        $RUNNER_TEST_JOBS
+
+.condenser_api_smoketest:
+  stage: e2e-test
+  extends: .test-common
+  script:
+    - *condenser_api_smoketest-script
+  artifacts:
+    when: always
+    reports:
+      junit: api_smoketest_condenser_api.xml
+
+
+.condenser_api_smoketest_negative-script: &condenser_api_smoketest_negative-script
+  - |
+    ./scripts/ci/start-api-smoketest.sh \
+        localhost "$RUNNER_HIVEMIND_SERVER_HTTP_PORT" \
+        condenser_api_negative/ api_smoketest_condenser_api_negative.xml \
+        $RUNNER_TEST_JOBS
+
+.condenser_api_smoketest_negative:
+  stage: e2e-test
+  extends: .test-common
+  script:
+    - *condenser_api_smoketest_negative-script
+  artifacts:
+    when: always
+    reports:
+      junit: api_smoketest_condenser_api_negative.xml
+
+
+.database_api_smoketest-script: &database_api_smoketest-script
+  - |
+    ./scripts/ci/start-api-smoketest.sh \
+        localhost "$RUNNER_HIVEMIND_SERVER_HTTP_PORT" \
+        database_api_patterns/ api_smoketest_database_api.xml \
+        $RUNNER_TEST_JOBS
+
+.database_api_smoketest:
+  stage: e2e-test
+  extends: .test-common
+  script:
+    - *database_api_smoketest-script
+  artifacts:
+    when: always
+    reports:
+      junit: api_smoketest_database_api.xml
+
+
+.database_api_smoketest_negative-script: &database_api_smoketest_negative-script
+  - |
+    ./scripts/ci/start-api-smoketest.sh \