From 4c2baf611dad294d9a658c386459a1eab38308f5 Mon Sep 17 00:00:00 2001 From: Marcin Ickiewicz Date: Wed, 14 May 2025 09:56:17 +0200 Subject: [PATCH 1/6] docker images: add ai tools required by Hivesense --- Dockerfile | 13 +++++---- scripts/ci-helpers/build_ci_base_image.sh | 2 +- scripts/setup_ubuntu.sh | 33 ++++++++++++++++++++++- 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index f5669a480..18638442b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,12 +3,12 @@ # docker buildx build --progress=plain --target=ci-base-image --tag registry.gitlab.syncad.com/hive/haf/ci-base-image$CI_IMAGE_TAG --file Dockerfile . # To be started from cloned haf source directory. ARG CI_REGISTRY_IMAGE=registry.gitlab.syncad.com/hive/haf/ -ARG CI_IMAGE_TAG=ubuntu24.04-1 +ARG CI_IMAGE_TAG=ubuntu24.04-2 ARG BUILD_IMAGE_TAG ARG IMAGE_TAG_PREFIX -FROM registry.gitlab.syncad.com/hive/hive/minimal-runtime:ubuntu24.04-1 AS minimal-runtime +FROM registry.gitlab.syncad.com/hive/hive/minimal-runtime:ubuntu24.04-2 AS minimal-runtime ENV PATH="/home/haf_admin/.local/bin:$PATH" @@ -36,6 +36,9 @@ RUN apt-get update && \ apt-get remove -y gnupg && \ apt-get autoremove -y && \ busybox --install -s + +RUN bash -x ./scripts/setup_ubuntu.sh --ai --haf-admin-account="haf_admin" --hived-account="hived" && rm -rf /var/lib/apt/lists/* + # change the UID and GID to match the ones postgres is assigned in our non-minimal runtime RUN (chown -Rf --from=postgres 105 / || true) && (chown -Rf --from=:postgres :109 / || true) && usermod -u 105 postgres && groupmod -g 109 postgres RUN usermod -a -G users -c "PostgreSQL daemon account" postgres @@ -45,7 +48,7 @@ RUN useradd -r -s /usr/sbin/nologin -b /nonexistent -c "HAF maintenance service USER haf_admin WORKDIR /home/haf_admin -FROM registry.gitlab.syncad.com/hive/hive/ci-base-image:ubuntu24.04-1 AS ci-base-image +FROM registry.gitlab.syncad.com/hive/hive/ci-base-image:ubuntu24.04-2 AS ci-base-image ENV PATH="/home/haf_admin/.local/bin:$PATH" @@ -111,7 +114,7 @@ RUN <<-EOF sudo chown -R hived "${INSTALLATION_DIR}/"* EOF -FROM registry.gitlab.syncad.com/hive/haf/minimal-runtime:ubuntu24.04-1 AS instance +FROM registry.gitlab.syncad.com/hive/haf/minimal-runtime:ubuntu24.04-2 AS instance ARG BUILD_HIVE_TESTNET=OFF ENV BUILD_HIVE_TESTNET=${BUILD_HIVE_TESTNET} @@ -125,7 +128,7 @@ ENV HIVE_CONVERTER_BUILD=${HIVE_CONVERTER_BUILD} ARG HIVE_LINT=OFF ENV HIVE_LINT=${HIVE_LINT} -ENV BUILD_IMAGE_TAG=${BUILD_IMAGE_TAG:-:ubuntu24.04-1} +ENV BUILD_IMAGE_TAG=${BUILD_IMAGE_TAG:-:ubuntu24.04-2} ARG P2P_PORT=2001 ENV P2P_PORT=${P2P_PORT} diff --git a/scripts/ci-helpers/build_ci_base_image.sh b/scripts/ci-helpers/build_ci_base_image.sh index 59b3b3a37..1a08be868 100755 --- a/scripts/ci-helpers/build_ci_base_image.sh +++ b/scripts/ci-helpers/build_ci_base_image.sh @@ -1,7 +1,7 @@ #! /bin/bash REGISTRY=${1:-registry.gitlab.syncad.com/hive/haf} -CI_IMAGE_TAG=ubuntu24.04-1 +CI_IMAGE_TAG=ubuntu24.04-2 # exit when any command fails set -e diff --git a/scripts/setup_ubuntu.sh b/scripts/setup_ubuntu.sh index b925d1c3a..964c711de 100755 --- a/scripts/setup_ubuntu.sh +++ b/scripts/setup_ubuntu.sh @@ -15,6 +15,7 @@ print_help () { echo "Setup this machine for HAF installation." echo "OPTIONS:" echo " --dev Install packages required to build and run a HAF server." + echo " --ai Install pgai" echo " --user Install packages to a subdirectory of the user's home directory." echo " --haf-admin-account=NAME Specify the unix account name to be used for HAF administration (will be associated with the PostgreSQL role)." echo " --hived-account=NAME Specify the unix account name to be used for hived (will be associated with the PostgreSQL role)." @@ -32,6 +33,31 @@ assert_is_root() { fi } +install_ai_packages() { + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y \ + git \ + python3.12 python3.12-venv python3.12-dev python3-pip \ + postgresql-17-pgvector postgresql-plpython3-17 \ + curl + + # required by Hivesense as pgai + python3.12 -m pip install --break-system-packages langchain + + pushd /tmp + git clone https://github.com/timescale/pgai.git --branch extension-0.8.0 + pushd pgai + python3.12 -m venv venv/ + # shellcheck disable=SC1091 + . venv/bin/activate + python3.12 -m pip install --upgrade pip + projects/extension/build.py install + deactivate + popd + rm -r pgai + popd +} + install_all_dev_packages() { echo "Attempting to install all dev packages..." assert_is_root @@ -48,7 +74,8 @@ install_all_dev_packages() { /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y DEBIAN_FRONTEND=noninteractive apt-get install -y postgresql-17 postgresql-server-dev-17 postgresql-17-cron \ - netcat-openbsd # needed to correctly handle --skip-hived option + netcat-openbsd \ + git python3.12 python3.12-venv python3.12-dev python3-pip postgresql-17-pgvector postgresql-plpython3-17 curl # for hivesense apt-get clean rm -rf /var/lib/apt/lists/* @@ -103,8 +130,12 @@ while [ $# -gt 0 ]; do case "$1" in --dev) install_all_dev_packages + install_ai_packages create_maintenance_account ;; + --ai) + install_ai_packages + ;; --user) install_user_packages ;; -- GitLab From a0ea36b295cabd0f80bc0bdde0f5ef3014acb364 Mon Sep 17 00:00:00 2001 From: Marcin Ickiewicz Date: Wed, 14 May 2025 15:10:56 +0200 Subject: [PATCH 2/6] copy installed pg extensions from previos layer --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index 18638442b..382479dc9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -187,6 +187,10 @@ COPY --from=build --chown=hived_admin:users /home/hived_admin/hive_base_config/f /home/hived_admin/hive_base_config/faketime/src/ COPY --from=build --chown=root:root /usr/local/lib/faketime/* /usr/local/lib/faketime/ +COPY --from=build \ + /usr/share/postgresql/${POSTGRES_VERSION}/extension/* \ + /usr/share/postgresql/${POSTGRES_VERSION}/extension + COPY --from=build \ /home/haf_admin/build/extensions/hive_fork_manager/* \ /usr/share/postgresql/${POSTGRES_VERSION}/extension -- GitLab From c9753c0fc2c384e66aaf2871d9b495042abe2aad Mon Sep 17 00:00:00 2001 From: Marcin Ickiewicz Date: Tue, 20 May 2025 08:41:14 +0200 Subject: [PATCH 3/6] reduce image size --- Dockerfile | 6 +++--- scripts/ci-helpers/build_ci_base_image.sh | 6 +++--- scripts/setup_ubuntu.sh | 14 ++++++++++++++ 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 382479dc9..4e28c58e6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ # docker buildx build --progress=plain --target=ci-base-image --tag registry.gitlab.syncad.com/hive/haf/ci-base-image$CI_IMAGE_TAG --file Dockerfile . # To be started from cloned haf source directory. ARG CI_REGISTRY_IMAGE=registry.gitlab.syncad.com/hive/haf/ -ARG CI_IMAGE_TAG=ubuntu24.04-2 +ARG CI_IMAGE_TAG=ubuntu24.04-3 ARG BUILD_IMAGE_TAG ARG IMAGE_TAG_PREFIX @@ -114,7 +114,7 @@ RUN <<-EOF sudo chown -R hived "${INSTALLATION_DIR}/"* EOF -FROM registry.gitlab.syncad.com/hive/haf/minimal-runtime:ubuntu24.04-2 AS instance +FROM registry.gitlab.syncad.com/hive/haf/minimal-runtime:ubuntu24.04-3 AS instance ARG BUILD_HIVE_TESTNET=OFF ENV BUILD_HIVE_TESTNET=${BUILD_HIVE_TESTNET} @@ -128,7 +128,7 @@ ENV HIVE_CONVERTER_BUILD=${HIVE_CONVERTER_BUILD} ARG HIVE_LINT=OFF ENV HIVE_LINT=${HIVE_LINT} -ENV BUILD_IMAGE_TAG=${BUILD_IMAGE_TAG:-:ubuntu24.04-2} +ENV BUILD_IMAGE_TAG=${BUILD_IMAGE_TAG:-:ubuntu24.04-3} ARG P2P_PORT=2001 ENV P2P_PORT=${P2P_PORT} diff --git a/scripts/ci-helpers/build_ci_base_image.sh b/scripts/ci-helpers/build_ci_base_image.sh index 1a08be868..76e071cdd 100755 --- a/scripts/ci-helpers/build_ci_base_image.sh +++ b/scripts/ci-helpers/build_ci_base_image.sh @@ -1,7 +1,7 @@ #! /bin/bash REGISTRY=${1:-registry.gitlab.syncad.com/hive/haf} -CI_IMAGE_TAG=ubuntu24.04-2 +CI_IMAGE_TAG=ubuntu24.04-3 # exit when any command fails set -e @@ -9,9 +9,9 @@ set -e docker buildx build --progress=plain --target=minimal-runtime \ --build-arg CI_REGISTRY_IMAGE="$REGISTRY/" --build-arg CI_IMAGE_TAG=$CI_IMAGE_TAG \ --build-arg BUILD_IMAGE_TAG=$CI_IMAGE_TAG \ - --tag "${REGISTRY}minimal-runtime:$CI_IMAGE_TAG" --file Dockerfile . + --tag "${REGISTRY}/minimal-runtime:$CI_IMAGE_TAG" --file Dockerfile . docker buildx build --progress=plain --target=ci-base-image \ --build-arg CI_REGISTRY_IMAGE="$REGISTRY/" --build-arg CI_IMAGE_TAG=$CI_IMAGE_TAG \ --build-arg BUILD_IMAGE_TAG=$CI_IMAGE_TAG \ - -t "${REGISTRY}ci-base-image:$CI_IMAGE_TAG" -f Dockerfile . + -t "${REGISTRY}/ci-base-image:$CI_IMAGE_TAG" -f Dockerfile . diff --git a/scripts/setup_ubuntu.sh b/scripts/setup_ubuntu.sh index 964c711de..df347dfdb 100755 --- a/scripts/setup_ubuntu.sh +++ b/scripts/setup_ubuntu.sh @@ -56,6 +56,20 @@ install_ai_packages() { popd rm -r pgai popd + + apt-get clean + rm -rf /var/lib/apt/lists/* + rm -rf /root/.cache ~/.cache /tmp/* /var/tmp/* + find / -type d -name '__pycache__' -exec rm -rf {} + + rm -rf /usr/local/lib/pgai/0.8.0/google + rm -rf /usr/local/lib/pgai/0.8.0/litellm + + rm -rf /usr/local/lib/pgai/0.4.0 + rm -rf /usr/local/lib/pgai/0.4.1 + rm -rf /usr/local/lib/pgai/0.5.0 + rm -rf /usr/local/lib/pgai/0.6.0 + rm -rf /usr/local/lib/pgai/0.7.0 + } install_all_dev_packages() { -- GitLab From e4f14c64ab86e97896aff8bdefb3a835e0821e57 Mon Sep 17 00:00:00 2001 From: Marcin Ickiewicz Date: Tue, 20 May 2025 13:23:54 +0200 Subject: [PATCH 4/6] copy built pgai to minimal instance --- Dockerfile | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4e28c58e6..d71b20989 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,12 +32,11 @@ RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y postgresql-common gnupg && \ /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y && \ apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y curl postgresql-17 postgresql-17-cron libpq5 libboost-chrono1.83.0 libboost-context1.83.0 libboost-filesystem1.83.0 libboost-thread1.83.0 busybox netcat-openbsd && \ + DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y python3.12 python3-pip curl postgresql-17 postgresql-17-cron postgresql-17-pgvector postgresql-plpython3-17 libpq5 libboost-chrono1.83.0 libboost-context1.83.0 libboost-filesystem1.83.0 libboost-thread1.83.0 busybox netcat-openbsd && \ apt-get remove -y gnupg && \ apt-get autoremove -y && \ - busybox --install -s - -RUN bash -x ./scripts/setup_ubuntu.sh --ai --haf-admin-account="haf_admin" --hived-account="hived" && rm -rf /var/lib/apt/lists/* + busybox --install -s && \ + python3.12 -m pip install --break-system-packages langchain # change the UID and GID to match the ones postgres is assigned in our non-minimal runtime RUN (chown -Rf --from=postgres 105 / || true) && (chown -Rf --from=:postgres :109 / || true) && usermod -u 105 postgres && groupmod -g 109 postgres @@ -202,6 +201,10 @@ COPY --from=build \ /home/haf_admin/build/lib/libhfm-* \ /usr/lib/postgresql/${POSTGRES_VERSION}/lib +COPY --from=build \ + /usr/local/lib/pgai \ + /usr/local/lib/pgai + # set a variable telling the entrypoint not to try to install the extension from source, we just did it above ENV HAF_INSTALL_EXTENSION=no -- GitLab From 7ecae28f42fbdee63d98a9099c5232c3d4ef856d Mon Sep 17 00:00:00 2001 From: Marcin Ickiewicz Date: Tue, 20 May 2025 14:08:45 +0200 Subject: [PATCH 5/6] remove usused 50MB from a image layer --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d71b20989..5d72d498b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,7 +36,8 @@ RUN apt-get update && \ apt-get remove -y gnupg && \ apt-get autoremove -y && \ busybox --install -s && \ - python3.12 -m pip install --break-system-packages langchain + python3.12 -m pip install --break-system-packages langchain && \ + rm -rf /var/lib/apt/lists/* # change the UID and GID to match the ones postgres is assigned in our non-minimal runtime RUN (chown -Rf --from=postgres 105 / || true) && (chown -Rf --from=:postgres :109 / || true) && usermod -u 105 postgres && groupmod -g 109 postgres -- GitLab From a6d662af26b24d958d000fdadbcf9ab13da514c1 Mon Sep 17 00:00:00 2001 From: Marcin Ickiewicz Date: Tue, 20 May 2025 15:00:04 +0200 Subject: [PATCH 6/6] remove unused pyarrow (save 140MB) --- scripts/setup_ubuntu.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/setup_ubuntu.sh b/scripts/setup_ubuntu.sh index df347dfdb..3084a7883 100755 --- a/scripts/setup_ubuntu.sh +++ b/scripts/setup_ubuntu.sh @@ -69,6 +69,7 @@ install_ai_packages() { rm -rf /usr/local/lib/pgai/0.5.0 rm -rf /usr/local/lib/pgai/0.6.0 rm -rf /usr/local/lib/pgai/0.7.0 + rm -rf /usr/local/lib/pgai/0.8.0/pyarrow } -- GitLab