scripts/ci/python/3.6/Dockerfile

@@ -39,12 +39,7 @@ ENV user ${user}
 ## Add user ##
 RUN groupadd --gid 1000 ${user} \
     && useradd --create-home --uid 1000 --gid ${user} ${user}
-
-# Gitlab CI accepts only root user, so we don't set user here.
-# You can (and should) run command in container as user `alice` this way:
-# docker-compose run --rm --user=alice --name=myrunner runner /bin/bash
-# USER ${user}
-
+USER ${user}
 WORKDIR /home/${user}
 RUN chown -R ${user}:${user} /home/${user}
 

scripts/ci/python/3.6/dev.dockerfile

+FROM python:3.6.12-buster
+
+# For running python as non-root user, e.g. on devel machine.
+
+# Setup python environment.
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONFAULTHANDLER 1
+
+# Install debian packages.
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        curl \
+        ca-certificates \
+        gnupg \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install debian pgdg repository.
+RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
+RUN echo "deb http://apt.postgresql.org/pub/repos/apt buster-pgdg main" \
+        > /etc/apt/sources.list.d/pgdg.list
+RUN apt-get update
+# Install postgresql client programs for various postgresl versions.
+RUN apt-get install -y --no-install-recommends \
+        postgresql-client-10 \
+        postgresql-client-11 \
+        postgresql-client-12 \
+        postgresql-client-13 \
+    && rm -rf /var/lib/apt/lists/*
+
+# Upgrade some crucial python packages.
+RUN pip install --upgrade pip setuptools wheel
+
+# Install python dependencies via pip.
+RUN pip install pipenv poetry
+
+ARG user
+ENV user ${user}
+
+## Add user ##
+RUN groupadd --gid 1000 ${user} \
+    && useradd --create-home --uid 1000 --gid ${user} ${user}
+
+RUN mkdir -p /home/${user}/src/hivemind
+RUN chown -R ${user}:${user} /home/${user}
+
+WORKDIR /home/${user}/src/hivemind
+USER ${user}
+
+CMD [ "python3" ]

scripts/ci/python/3.8/Dockerfile

@@ -39,12 +39,7 @@ ENV user ${user}
 ## Add user ##
 RUN groupadd --gid 1000 ${user} \
     && useradd --create-home --uid 1000 --gid ${user} ${user}
-
-# Gitlab CI accepts only root user, so we don't set user here.
-# You can (and should) run command in container as user `alice` this way:
-# docker-compose run --rm --user=alice --name=myrunner runner /bin/bash
-# USER ${user}
-
+USER ${user}
 WORKDIR /home/${user}
 RUN chown -R ${user}:${user} /home/${user}
 

scripts/ci/setup_env.py

+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+Tool for Gitlab runner to read environment from project variable
+and setup bash environment.
+When running on Gitlab CI you can  do this:
+```
+eval "$(cat $MY_ENV_VARIABLE | ./scripts/ci/setup_env.py)"
+echo "RUNNER_ID is $RUNNER_ID"
+```
+In bash you can do this:
+```
+eval "$(cat ./.tmp/env.yaml | ./scripts/ci/setup_env.py)"
+echo "RUNNER_ID is $RUNNER_ID"
+```
+"""
+
+import logging
+import sys
+import argparse
+import yaml
+
+FORMAT = '# %(asctime)s - %(name)s - %(levelname)s - %(message)s '
+logging.basicConfig(format=FORMAT)
+logger = logging.getLogger(__name__)
+
+
+def output(message, outfile, end='\n'):
+    """Print data to outfile"""
+    print(message, file=outfile, end=end)
+
+
+def read(infile):
+    """Read data from infile"""
+    if hasattr(infile, 'read'):
+        # data = json.loads(infile.read())
+        data = yaml.safe_load(infile.read())
+    else:
+        # data = json.loads(infile)
+        data = yaml.safe_load(infile)
+    return data
+
+
+def setup_env(current_runner_id, hive_sync_runner_id, infile, outfile, end, **kwargs):
+    """
+    Resolve and output environment for bash in pending CI job.
+    Assumption: all jobs in pipeline must use the same database.
+    We need to point current runner to the database used by runner,
+    that did hive sync (first stage in pipeline).
+    """
+
+    logger.debug('current_runner_id: %s', current_runner_id)
+    logger.debug('hive_sync_runner_id: %s', hive_sync_runner_id)
+
+    data = read(infile)
+    logger.debug('data: %s', data)
+
+    current_runner = data['runners'][str(current_runner_id)]
+    if hive_sync_runner_id == 0:
+        hive_sync_runner = current_runner
+    else:
+        hive_sync_runner = data['runners'][str(hive_sync_runner_id)]
+
+    if hive_sync_runner_id == 0:
+        # Do nothing, obviously. Current runner does hive sync itself.
+        logger.debug('case 1')
+        runner = current_runner
+    elif current_runner_id == hive_sync_runner_id:
+        # Do nothing, obviously. Current runner is the same, as runner
+        # that did hive sync.
+        logger.debug('case 2')
+        runner = current_runner
+    else:
+        if current_runner['host'] == hive_sync_runner['host']:
+            # We assume that all executors on the same machine
+            # use the same postgres server with the same credentials
+            # and unix socket connection configuration. So do nothing.
+            logger.debug('case 3')
+            runner = current_runner
+        else:
+            # Take postgres stuff from runner that did hive sync,
+            # but point current runner to postgres on the host of runner
+            # that did hive sync (exposed on network, we assume).
+            logger.debug('case 4')
+            runner = {}
+            for key, value in current_runner.items():
+                if key.startswith('postgres'):
+                    if key == 'postgres_host':
+                        runner[key] = hive_sync_runner['host']
+                    else:
+                        runner[key] = hive_sync_runner[key]
+                else:
+                    runner[key] = value
+
+    for key in runner:
+        output(
+            f'export RUNNER_{key.upper()}="{str(runner[key])}"',
+            outfile,
+            end,
+            )
+
+    for key in data['common']:
+        output(
+            f"export RUNNER_{key.upper()}=\"{str(data['common'][key])}\"",
+            outfile,
+            end,
+            )
+
+
+def parse_args():
+    """Parse command line arguments"""
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter
+        )
+    parser.add_argument(
+        'infile',
+        type=argparse.FileType('r'),
+        nargs='?',
+        default=sys.stdin,
+        help='Input file or pipe via STDIN'
+        )
+    parser.add_argument(
+        '-o', '--outfile',
+        type=argparse.FileType('w'),
+        default=sys.stdout,
+        help='Output file, STDOUT if not set'
+        )
+    parser.add_argument(
+        "-e", "--end",
+        dest='end',
+        default='\n',
+        help='String at the end of line in output'
+        )
+    parser.add_argument(
+        "-s", "--hive-sync-runner-id",
+        required=True,
+        type=int,
+        help='Id of runner which did hive sync, 0 when current runner does hive sync actually'
+        )
+    parser.add_argument(
+        "-c", "--current-runner-id",
+        required=True,
+        type=int,
+        help='Id of current runner'
+        )
+    parser.add_argument(
+        '--log-level',
+        default='INFO',
+        dest='log_level',
+        choices=['debug', 'info', 'warning', 'error'],
+        help='Log level (string)',
+        )
+
+    result = parser.parse_args()
+
+    # configure logger and print config
+    root = logging.getLogger()
+    root.setLevel(result.log_level.upper())
+
+    return result
+
+
+def main():
+    """Main dispatcher function"""
+    flags = parse_args()
+    setup_env(**vars(flags))
+
+
+if __name__ == '__main__':
+    main()

scripts/ci/start-api-benchmarks.sh

+#!/bin/bash
+
+set -e
+
+pip install tox
+
+export HIVEMIND_ADDRESS=$1
+export HIVEMIND_PORT=$2
+ITERATIONS=$3
+JOBS=${4:-auto}
+export TAVERN_DISABLE_COMPARATOR=true
+
+echo Attempting to start benchmarks on hivemind instance listening on: $HIVEMIND_ADDRESS port: $HIVEMIND_PORT
+
+for (( i=0; i<$ITERATIONS; i++ ))
+do
+  echo About to run iteration $i
+  tox -e tavern-benchmark -- \
+      -W ignore::pytest.PytestDeprecationWarning \
+      -n $JOBS \
+      --junitxml=../../../../benchmarks-$i.xml
+  echo Done!
+done

scripts/ci/start-api-smoketest.sh

 #!/bin/bash
 
 set -e
+
 pip install tox
 
 export HIVEMIND_ADDRESS=$1
 export HIVEMIND_PORT=$2
+TEST_GROUP=$3
+JUNITXML=$4
+JOBS=${5:-auto}
 export TAVERN_DIR="tests/tests_api/hivemind/tavern"
 
 echo "Starting tests on hivemind server running on ${HIVEMIND_ADDRESS}:${HIVEMIND_PORT}"
 
-echo "Selected test group (if empty all will be executed): $3"
+echo "Selected test group (if empty all will be executed): $TEST_GROUP"
 
-tox -e tavern -- -W ignore::pytest.PytestDeprecationWarning -n auto --junitxml=../../../../$4 $3
+tox -e tavern -- \
+    -W ignore::pytest.PytestDeprecationWarning \
+    -n $JOBS \
+    --junitxml=../../../../$JUNITXML $TEST_GROUP

scripts/ci/timer.sh

@@ -2,6 +2,8 @@
 
 set -euo pipefail
 
+JOB=$1
+
 start() {
   mkdir -p ".tmp"
   echo `date +%s` > ".tmp/timer-start"
@@ -18,4 +20,15 @@ check() {
     fi
 }
 
-"$1"
+main() {
+  if [ "$JOB" = "start" ]; then
+    start
+  elif [ "$JOB" = "check" ]; then
+    check
+  else
+    echo "Invalid argument"
+    exit 1
+  fi
+}
+
+main

scripts/ci/wait-for-postgres.sh

@@ -6,19 +6,26 @@
 
 set -e
 
-LIMIT=30 #seconds
-shift
-cmd="$@"
+LIMIT=10 #seconds
+
+HOST=$1
+PORT=$2
+
+if [ -z "$HOST" ]
+then
+    HOST="$RUNNER_POSTGRES_HOST"
+fi
+if [ -z "$PORT" ]
+then
+    PORT="$RUNNER_POSTGRES_PORT"
+fi
 
 wait_for_postgres() {
-    # wkedzierski@syncad.com work, but customized by wbarcik@syncad.com
     counter=0
-    echo "Waiting for postgres on ${POSTGRES_HOST}:${POSTGRES_PORT}. Timeout is ${LIMIT}s."
+    echo "Waiting for postgres on ${HOST}:${PORT}."
     while ! pg_isready \
-            --username $ADMIN_POSTGRES_USER \
-            --host $POSTGRES_HOST \
-            --port $POSTGRES_PORT \
-            --dbname postgres \
+            --host $HOST \
+            --port $PORT \
             --timeout=1 --quiet; do
         counter=$((counter+1))
         sleep 1
@@ -38,14 +45,15 @@ output_configuration() {
     echo "-------------------------------------------------"
     echo "Postgres version and configuration"
     echo "-------------------------------------------------"
-    PGPASSWORD=$ADMIN_POSTGRES_USER_PASSWORD psql \
-            --username "$ADMIN_POSTGRES_USER" \
-            --host "$POSTGRES_HOST" \
-            --port $POSTGRES_PORT \
+    PGPASSWORD=$RUNNER_POSTGRES_ADMIN_USER_PASSWORD psql \
+            --username "$RUNNER_POSTGRES_ADMIN_USER" \
+            --host "$HOST" \
+            --port $PORT \
             --dbname postgres <<EOF
 SELECT version();
-select name, setting, unit from pg_settings;
-\copy (select * from pg_settings) to '$DIR/pg_settings_on_start.csv' WITH CSV HEADER
+-- select name, setting, unit from pg_settings;
+-- show all;
+\copy (select name, setting, unit from pg_settings) to '$DIR/pg_settings_on_start.csv' WITH CSV HEADER
 \q
 EOF
     echo "-------------------------------------------------"

setup.py

@@ -91,9 +91,14 @@ if __name__ == "__main__":
             'pdoc',
             'diff-match-patch',
             'prometheus-client',
-            'psutil'
+            'psutil',
         ],
-
+        extras_require={
+            'dev': [
+                'pyYAML',
+                'prettytable',
+            ]
+        },
         entry_points={
             'console_scripts': [
                 'hive=hive.cli:run',

tests_api @ e8f2e181

-Subproject commit c9a3f6ecc01e3bed1098bf9698e95002ccdd47d6
+Subproject commit e8f2e1810071df5659edb45edf3d59af4b9be48e

tox.ini

@@ -3,7 +3,7 @@ envlist = py36, tavern, benchmark, tavern-benchmark
 skipsdist = true
 
 [testenv]
-deps = 
+deps =
   pytest
 
 [testenv:benchmark]
@@ -11,13 +11,13 @@ deps =
   {[testenv]deps}
   pytest-benchmark
   requests
-  pyyaml
+  pyYAML
   prettytable
 commands =
   python {toxinidir}/scripts/ci/start_api_benchmark.py {posargs}
 
 [testenv:tavern]
-setenv = 
+setenv =
   PYTHONPATH = {toxinidir}/tests/tests_api/hivemind/tavern:{env:PYTHONPATH:}
 
 passenv =
@@ -26,7 +26,7 @@ passenv =
 
 changedir = {env:TAVERN_DIR}
 
-deps = 
+deps =
   {[testenv]deps}
   pytest-xdist
   tavern
@@ -36,7 +36,7 @@ deps =
 commands = pytest {posargs}
 
 [testenv:tavern-benchmark]
-setenv = 
+setenv =
   {[testenv:tavern]setenv}
 
 passenv =
@@ -45,7 +45,7 @@ passenv =
 
 changedir = tests/tests_api/hivemind/tavern
 
-deps = 
+deps =
   {[testenv:tavern]deps}
 
 commands = pytest --durations=0 {posargs}