diff --git a/Dockerfile b/Dockerfile index f5669a48010a156157b0ec45e6ac263a9ea04a3b..5d72d498b1796957d700e2315eabdf481dd00f56 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,12 +3,12 @@ # docker buildx build --progress=plain --target=ci-base-image --tag registry.gitlab.syncad.com/hive/haf/ci-base-image$CI_IMAGE_TAG --file Dockerfile . # To be started from cloned haf source directory. ARG CI_REGISTRY_IMAGE=registry.gitlab.syncad.com/hive/haf/ -ARG CI_IMAGE_TAG=ubuntu24.04-1 +ARG CI_IMAGE_TAG=ubuntu24.04-3 ARG BUILD_IMAGE_TAG ARG IMAGE_TAG_PREFIX -FROM registry.gitlab.syncad.com/hive/hive/minimal-runtime:ubuntu24.04-1 AS minimal-runtime +FROM registry.gitlab.syncad.com/hive/hive/minimal-runtime:ubuntu24.04-2 AS minimal-runtime ENV PATH="/home/haf_admin/.local/bin:$PATH" @@ -32,10 +32,13 @@ RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y postgresql-common gnupg && \ /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y && \ apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y curl postgresql-17 postgresql-17-cron libpq5 libboost-chrono1.83.0 libboost-context1.83.0 libboost-filesystem1.83.0 libboost-thread1.83.0 busybox netcat-openbsd && \ + DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y python3.12 python3-pip curl postgresql-17 postgresql-17-cron postgresql-17-pgvector postgresql-plpython3-17 libpq5 libboost-chrono1.83.0 libboost-context1.83.0 libboost-filesystem1.83.0 libboost-thread1.83.0 busybox netcat-openbsd && \ apt-get remove -y gnupg && \ apt-get autoremove -y && \ - busybox --install -s + busybox --install -s && \ + python3.12 -m pip install --break-system-packages langchain && \ + rm -rf /var/lib/apt/lists/* + # change the UID and GID to match the ones postgres is assigned in our non-minimal runtime RUN (chown -Rf --from=postgres 105 / || true) && (chown -Rf --from=:postgres :109 / || true) && usermod -u 105 postgres && groupmod -g 109 postgres RUN usermod -a -G users -c "PostgreSQL daemon account" postgres @@ -45,7 +48,7 @@ RUN useradd -r -s /usr/sbin/nologin -b /nonexistent -c "HAF maintenance service USER haf_admin WORKDIR /home/haf_admin -FROM registry.gitlab.syncad.com/hive/hive/ci-base-image:ubuntu24.04-1 AS ci-base-image +FROM registry.gitlab.syncad.com/hive/hive/ci-base-image:ubuntu24.04-2 AS ci-base-image ENV PATH="/home/haf_admin/.local/bin:$PATH" @@ -111,7 +114,7 @@ RUN <<-EOF sudo chown -R hived "${INSTALLATION_DIR}/"* EOF -FROM registry.gitlab.syncad.com/hive/haf/minimal-runtime:ubuntu24.04-1 AS instance +FROM registry.gitlab.syncad.com/hive/haf/minimal-runtime:ubuntu24.04-3 AS instance ARG BUILD_HIVE_TESTNET=OFF ENV BUILD_HIVE_TESTNET=${BUILD_HIVE_TESTNET} @@ -125,7 +128,7 @@ ENV HIVE_CONVERTER_BUILD=${HIVE_CONVERTER_BUILD} ARG HIVE_LINT=OFF ENV HIVE_LINT=${HIVE_LINT} -ENV BUILD_IMAGE_TAG=${BUILD_IMAGE_TAG:-:ubuntu24.04-1} +ENV BUILD_IMAGE_TAG=${BUILD_IMAGE_TAG:-:ubuntu24.04-3} ARG P2P_PORT=2001 ENV P2P_PORT=${P2P_PORT} @@ -184,6 +187,10 @@ COPY --from=build --chown=hived_admin:users /home/hived_admin/hive_base_config/f /home/hived_admin/hive_base_config/faketime/src/ COPY --from=build --chown=root:root /usr/local/lib/faketime/* /usr/local/lib/faketime/ +COPY --from=build \ + /usr/share/postgresql/${POSTGRES_VERSION}/extension/* \ + /usr/share/postgresql/${POSTGRES_VERSION}/extension + COPY --from=build \ /home/haf_admin/build/extensions/hive_fork_manager/* \ /usr/share/postgresql/${POSTGRES_VERSION}/extension @@ -195,6 +202,10 @@ COPY --from=build \ /home/haf_admin/build/lib/libhfm-* \ /usr/lib/postgresql/${POSTGRES_VERSION}/lib +COPY --from=build \ + /usr/local/lib/pgai \ + /usr/local/lib/pgai + # set a variable telling the entrypoint not to try to install the extension from source, we just did it above ENV HAF_INSTALL_EXTENSION=no diff --git a/scripts/ci-helpers/build_ci_base_image.sh b/scripts/ci-helpers/build_ci_base_image.sh index 59b3b3a372b87d9396ce8b567c718f928ebf9ff9..76e071cddff2d0fe9a5cfa502947468e3d830150 100755 --- a/scripts/ci-helpers/build_ci_base_image.sh +++ b/scripts/ci-helpers/build_ci_base_image.sh @@ -1,7 +1,7 @@ #! /bin/bash REGISTRY=${1:-registry.gitlab.syncad.com/hive/haf} -CI_IMAGE_TAG=ubuntu24.04-1 +CI_IMAGE_TAG=ubuntu24.04-3 # exit when any command fails set -e @@ -9,9 +9,9 @@ set -e docker buildx build --progress=plain --target=minimal-runtime \ --build-arg CI_REGISTRY_IMAGE="$REGISTRY/" --build-arg CI_IMAGE_TAG=$CI_IMAGE_TAG \ --build-arg BUILD_IMAGE_TAG=$CI_IMAGE_TAG \ - --tag "${REGISTRY}minimal-runtime:$CI_IMAGE_TAG" --file Dockerfile . + --tag "${REGISTRY}/minimal-runtime:$CI_IMAGE_TAG" --file Dockerfile . docker buildx build --progress=plain --target=ci-base-image \ --build-arg CI_REGISTRY_IMAGE="$REGISTRY/" --build-arg CI_IMAGE_TAG=$CI_IMAGE_TAG \ --build-arg BUILD_IMAGE_TAG=$CI_IMAGE_TAG \ - -t "${REGISTRY}ci-base-image:$CI_IMAGE_TAG" -f Dockerfile . + -t "${REGISTRY}/ci-base-image:$CI_IMAGE_TAG" -f Dockerfile . diff --git a/scripts/setup_ubuntu.sh b/scripts/setup_ubuntu.sh index b925d1c3a242c82f758b88374d3ad172565dc2a3..3084a7883113d71d9a4b1b5218258129844ad3c0 100755 --- a/scripts/setup_ubuntu.sh +++ b/scripts/setup_ubuntu.sh @@ -15,6 +15,7 @@ print_help () { echo "Setup this machine for HAF installation." echo "OPTIONS:" echo " --dev Install packages required to build and run a HAF server." + echo " --ai Install pgai" echo " --user Install packages to a subdirectory of the user's home directory." echo " --haf-admin-account=NAME Specify the unix account name to be used for HAF administration (will be associated with the PostgreSQL role)." echo " --hived-account=NAME Specify the unix account name to be used for hived (will be associated with the PostgreSQL role)." @@ -32,6 +33,46 @@ assert_is_root() { fi } +install_ai_packages() { + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y \ + git \ + python3.12 python3.12-venv python3.12-dev python3-pip \ + postgresql-17-pgvector postgresql-plpython3-17 \ + curl + + # required by Hivesense as pgai + python3.12 -m pip install --break-system-packages langchain + + pushd /tmp + git clone https://github.com/timescale/pgai.git --branch extension-0.8.0 + pushd pgai + python3.12 -m venv venv/ + # shellcheck disable=SC1091 + . venv/bin/activate + python3.12 -m pip install --upgrade pip + projects/extension/build.py install + deactivate + popd + rm -r pgai + popd + + apt-get clean + rm -rf /var/lib/apt/lists/* + rm -rf /root/.cache ~/.cache /tmp/* /var/tmp/* + find / -type d -name '__pycache__' -exec rm -rf {} + + rm -rf /usr/local/lib/pgai/0.8.0/google + rm -rf /usr/local/lib/pgai/0.8.0/litellm + + rm -rf /usr/local/lib/pgai/0.4.0 + rm -rf /usr/local/lib/pgai/0.4.1 + rm -rf /usr/local/lib/pgai/0.5.0 + rm -rf /usr/local/lib/pgai/0.6.0 + rm -rf /usr/local/lib/pgai/0.7.0 + rm -rf /usr/local/lib/pgai/0.8.0/pyarrow + +} + install_all_dev_packages() { echo "Attempting to install all dev packages..." assert_is_root @@ -48,7 +89,8 @@ install_all_dev_packages() { /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y DEBIAN_FRONTEND=noninteractive apt-get install -y postgresql-17 postgresql-server-dev-17 postgresql-17-cron \ - netcat-openbsd # needed to correctly handle --skip-hived option + netcat-openbsd \ + git python3.12 python3.12-venv python3.12-dev python3-pip postgresql-17-pgvector postgresql-plpython3-17 curl # for hivesense apt-get clean rm -rf /var/lib/apt/lists/* @@ -103,8 +145,12 @@ while [ $# -gt 0 ]; do case "$1" in --dev) install_all_dev_packages + install_ai_packages create_maintenance_account ;; + --ai) + install_ai_packages + ;; --user) install_user_packages ;;