From 0b25a0ddd41218f81310f69feb2b887e8569429c Mon Sep 17 00:00:00 2001 From: Eric Frias Date: Wed, 20 Aug 2025 19:34:05 +0000 Subject: [PATCH 1/4] Revert "Checkout haf at 1.27.11 for haf_api_node" This reverts commit d30daba57c4f317188b59a9462e6dc8625c60bd6. --- db/schema.sql | 2 +- haf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/db/schema.sql b/db/schema.sql index 68d0cde..07c7b09 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -167,7 +167,7 @@ DECLARE synchronization_stages hive.application_stages; BEGIN IF NOT hive.app_context_exists('nfttracker_app') THEN - synchronization_stages := ARRAY[( 'MASSIVE_PROCESSING', 101, 10000 ), hive.live_stage()]::hive.application_stages; + synchronization_stages := ARRAY[( 'MASSIVE_PROCESSING', 101, 10000, '3 seconds' ), hive.live_stage()]::hive.application_stages; PERFORM hive.app_create_context( _name => 'nfttracker_app', diff --git a/haf b/haf index 0e2bd7c..86bd70a 160000 --- a/haf +++ b/haf @@ -1 +1 @@ -Subproject commit 0e2bd7c69cc26928665281435165fd3288940db9 +Subproject commit 86bd70a81252ce330489ed2a327e6f2f10964386 -- GitLab From 163a519e96ca1e09a4e73a17c68ea5fb26122b21 Mon Sep 17 00:00:00 2001 From: Eric Frias Date: Wed, 20 Aug 2025 16:09:28 -0400 Subject: [PATCH 2/4] Add CI image builds --- .gitlab-ci.yml | 113 +++++++++++++++++++++++++++++++++++++++++++++++++ .gitmodules | 2 +- 2 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..169dbbe --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,113 @@ +stages: +- build +- publish + +variables: + # Variables required by Common CI jobs + CI_COMMON_JOB_VERSION: "6b9e9e75ec5263939450936a4f8348dfbca3666d" + DOCKER_BUILDER_TAG: "$CI_COMMON_JOB_VERSION" + DOCKER_DIND_TAG: "$CI_COMMON_JOB_VERSION" + IMAGE_REMOVER_TAG: "$CI_COMMON_JOB_VERSION" + # Git configuration + GIT_STRATEGY: clone + GIT_SUBMODULE_STRATEGY: recursive + GIT_DEPTH: 1 + GIT_SUBMODULE_DEPTH: 1 + GIT_SUBMODULE_UPDATE_FLAGS: --jobs 4 + +include: +- template: Workflows/Branch-Pipelines.gitlab-ci.yml +- project: 'hive/common-ci-configuration' + ref: 6b9e9e75ec5263939450936a4f8348dfbca3666d + file: + - '/templates/base.gitlab-ci.yml' + - '/templates/docker_image_jobs.gitlab-ci.yml' + +build_images: + extends: .docker_image_builder_job_template + stage: build + before_script: + - !reference [.docker_image_builder_job_template, before_script] + - | + echo -e "\e[0Ksection_start:$(date +%s):login[collapsed=true]\r\e[0KLogging to Docker registry..." + docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + echo -e "\e[0Ksection_end:$(date +%s):login\r\e[0K" + script: + - | + # Build and push with commit SHA tag (this creates all three tags for GitLab registry) + $CI_PROJECT_DIR/scripts/ci-helpers/build_instance.sh \ + "$CI_COMMIT_SHORT_SHA" \ + "$CI_PROJECT_DIR" \ + "$CI_REGISTRY_IMAGE" + + # Push only the main image to registry.hive.blog (if credentials are available) + echo "Current branch: $CI_COMMIT_BRANCH" + echo "Is protected: $CI_COMMIT_REF_PROTECTED" + echo "Pipeline source: $CI_PIPELINE_SOURCE" + + if [[ -n "$BLOG_REGISTRY_USER" ]] && [[ -n "$BLOG_REGISTRY_PASSWORD" ]]; then + echo "Pushing to registry.hive.blog..." + docker tag "registry.gitlab.syncad.com/hive/nft_tracker:$CI_COMMIT_SHORT_SHA" "registry.hive.blog/nft_tracker:$CI_COMMIT_SHORT_SHA" + echo "Logging in to registry-upload.hive.blog as user: $BLOG_REGISTRY_USER" + echo "$BLOG_REGISTRY_PASSWORD" | docker login --username "$BLOG_REGISTRY_USER" --password-stdin registry-upload.hive.blog + docker push "registry.hive.blog/nft_tracker:$CI_COMMIT_SHORT_SHA" + else + echo "WARNING: BLOG_REGISTRY_USER or BLOG_REGISTRY_PASSWORD not set." + echo "Skipping push to registry.hive.blog (this is expected for non-protected branches)" + echo "Images are still available in GitLab registry at: registry.gitlab.syncad.com/hive/nft_tracker:$CI_COMMIT_SHORT_SHA" + fi + + # If on develop branch, also tag and push as 'develop' + if [[ "$CI_COMMIT_BRANCH" == "develop" ]]; then + echo "Tagging images with 'develop' tag..." + docker tag "registry.gitlab.syncad.com/hive/nft_tracker:$CI_COMMIT_SHORT_SHA" "registry.gitlab.syncad.com/hive/nft_tracker:develop" + docker push "registry.gitlab.syncad.com/hive/nft_tracker:develop" + + # Only push develop tag to registry.hive.blog if we have credentials + if [[ -n "$BLOG_REGISTRY_USER" ]] && [[ -n "$BLOG_REGISTRY_PASSWORD" ]]; then + docker tag "registry.gitlab.syncad.com/hive/nft_tracker:$CI_COMMIT_SHORT_SHA" "registry.hive.blog/nft_tracker:develop" + docker push "registry.hive.blog/nft_tracker:develop" + fi + fi + tags: + - public-runner-docker + +publish_release_images: + extends: .docker_image_builder_job_template + stage: publish + before_script: + - !reference [.docker_image_builder_job_template, before_script] + - | + echo -e "\e[0Ksection_start:$(date +%s):login[collapsed=true]\r\e[0KLogging to Docker registries..." + docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + echo "$BLOG_REGISTRY_PASSWORD" | docker login --username "$BLOG_REGISTRY_USER" --password-stdin registry-upload.hive.blog + echo -e "\e[0Ksection_end:$(date +%s):login\r\e[0K" + script: + - | + echo "Publishing release images for tag: $CI_COMMIT_TAG" + + # Build images with release tag for GitLab registry (creates all three tags) + $CI_PROJECT_DIR/scripts/ci-helpers/build_instance.sh \ + "$CI_COMMIT_TAG" \ + "$CI_PROJECT_DIR" \ + "$CI_REGISTRY_IMAGE" + + # Push only the main image to registry.hive.blog + docker tag "registry.gitlab.syncad.com/hive/nft_tracker:$CI_COMMIT_TAG" "registry.hive.blog/nft_tracker:$CI_COMMIT_TAG" + docker push "registry.hive.blog/nft_tracker:$CI_COMMIT_TAG" + + # Also tag with 'latest' if this is a release tag + docker tag "registry.gitlab.syncad.com/hive/nft_tracker:$CI_COMMIT_TAG" "registry.gitlab.syncad.com/hive/nft_tracker:latest" + docker tag "registry.gitlab.syncad.com/hive/nft_tracker:$CI_COMMIT_TAG" "registry.hive.blog/nft_tracker:latest" + docker push "registry.gitlab.syncad.com/hive/nft_tracker:latest" + docker push "registry.hive.blog/nft_tracker:latest" + + echo "Successfully published release images with tag: $CI_COMMIT_TAG" + rules: + - if: '$CI_COMMIT_TAG =~ /^v[0-9]+.*/ && $CI_COMMIT_REF_PROTECTED == "true"' + when: on_success + - if: '$CI_COMMIT_TAG =~ /^[0-9]+.*/ && $CI_COMMIT_REF_PROTECTED == "true"' + when: on_success + - when: never + tags: + - public-runner-docker \ No newline at end of file diff --git a/.gitmodules b/.gitmodules index d131a23..3f9daf5 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "haf"] path = haf - url = git@gitlab.syncad.com:hive/haf.git + url = ../haf.git -- GitLab From 7fdce2ecfd802dd2b90187bca156b7938912623f Mon Sep 17 00:00:00 2001 From: Eric Frias Date: Wed, 20 Aug 2025 16:54:11 -0400 Subject: [PATCH 3/4] Make installer sort of idempotent --- db/schema.sql | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/db/schema.sql b/db/schema.sql index 07c7b09..44518a7 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -2,16 +2,23 @@ SET ROLE nfttracker_owner; CREATE SCHEMA IF NOT EXISTS nfttracker_app AUTHORIZATION nfttracker_owner; +-- Drop and recreate domains to ensure idempotency +DROP DOMAIN IF EXISTS nfttracker_app.symbol_name CASCADE; CREATE DOMAIN nfttracker_app.symbol_name AS VARCHAR(10); +DROP DOMAIN IF EXISTS nfttracker_app.symbol_namespace CASCADE; CREATE DOMAIN nfttracker_app.symbol_namespace AS VARCHAR(16); +DROP DOMAIN IF EXISTS nfttracker_app.positive_integer CASCADE; CREATE DOMAIN nfttracker_app.positive_integer AS INTEGER CHECK (value > 0); +DROP DOMAIN IF EXISTS nfttracker_app.typename CASCADE; CREATE DOMAIN nfttracker_app.typename AS VARCHAR(255) CHECK (length(value) > 0); +-- Drop and recreate composite type +DROP TYPE IF EXISTS nfttracker_app.symbol CASCADE; CREATE TYPE nfttracker_app.symbol AS ( namespace nfttracker_app.symbol_namespace, name nfttracker_app.symbol_name @@ -48,12 +55,20 @@ BEGIN END; $$ LANGUAGE plpgsql IMMUTABLE STRICT; +-- Drop and recreate cast to ensure idempotency +DROP CAST IF EXISTS (TEXT AS nfttracker_app.symbol); CREATE CAST (TEXT AS nfttracker_app.symbol) WITH FUNCTION nfttracker_app.text_to_symbol(TEXT) AS implicit; +DROP DOMAIN IF EXISTS nfttracker_app.tags CASCADE; CREATE DOMAIN nfttracker_app.tags AS VARCHAR(8)[] CHECK (array_length(value, 1) <= 4); -CREATE TABLE IF NOT EXISTS nfttracker_app.types ( +-- Drop and recreate tables to ensure correct schema +DROP TABLE IF EXISTS nfttracker_app.instances CASCADE; +DROP TABLE IF EXISTS nfttracker_app.authorized_issuers CASCADE; +DROP TABLE IF EXISTS nfttracker_app.types CASCADE; + +CREATE TABLE nfttracker_app.types ( id BIGSERIAL PRIMARY KEY, creator INTEGER NOT NULL, owner INTEGER NOT NULL, @@ -120,13 +135,13 @@ FOR EACH ROW WHEN (NEW.max_count > OLD.max_count) EXECUTE FUNCTION nfttracker_app.prevent_increment_max_count(); -CREATE TABLE IF NOT EXISTS nfttracker_app.authorized_issuers ( +CREATE TABLE nfttracker_app.authorized_issuers ( type_id BIGINT NOT NULL REFERENCES nfttracker_app.types(id) ON DELETE CASCADE, account_id INTEGER NOT NULL, PRIMARY KEY (type_id, account_id) ); -CREATE TABLE IF NOT EXISTS nfttracker_app.instances ( +CREATE TABLE nfttracker_app.instances ( id BIGSERIAL PRIMARY KEY, type_id BIGINT NOT NULL REFERENCES nfttracker_app.types(id), holder INTEGER NOT NULL, @@ -136,7 +151,7 @@ CREATE TABLE IF NOT EXISTS nfttracker_app.instances ( created_at TIMESTAMP NOT NULL, updated_at TIMESTAMP NOT NULL ); -CREATE INDEX IF NOT EXISTS idx_nfts_instances_type_id ON nfttracker_app.instances(type_id); +CREATE INDEX idx_nfts_instances_type_id ON nfttracker_app.instances(type_id); CREATE OR REPLACE FUNCTION nfttracker_app.prevent_soulbound_unset() RETURNS TRIGGER AS $$ @@ -176,4 +191,4 @@ BEGIN _stages => synchronization_stages ); END IF; -END $$; +END $$; \ No newline at end of file -- GitLab From 905af9f5bf9ffd79733bc19fafcddf43a59e018e Mon Sep 17 00:00:00 2001 From: Eric Frias Date: Thu, 21 Aug 2025 10:08:28 -0400 Subject: [PATCH 4/4] Add initial postgrest API, mostly copied from hafbe --- .gitignore | 1 + .gitlab-ci.yml | 40 ++++++++- Dockerfile | 2 +- Dockerfile.rewriter | 33 +++++++ docker/nft_tracker_nginx.conf.template | 41 +++++++++ docker/rewriter_entrypoint.sh | 14 +++ endpoints/endpoint_schema.sql | 117 +++++++++++++++++++++++++ endpoints/get_version.sql | 64 ++++++++++++++ rewrite_rules.conf | 8 ++ scripts/install_app.sh | 9 ++ scripts/openapi_rewrite.sh | 100 +++++++++++++++++++++ 11 files changed, 427 insertions(+), 2 deletions(-) create mode 100644 Dockerfile.rewriter create mode 100644 docker/nft_tracker_nginx.conf.template create mode 100755 docker/rewriter_entrypoint.sh create mode 100644 endpoints/endpoint_schema.sql create mode 100644 endpoints/get_version.sql create mode 100644 rewrite_rules.conf create mode 100755 scripts/openapi_rewrite.sh diff --git a/.gitignore b/.gitignore index 748b066..d6a2381 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ tests/regression/results/*.out +endpoints_openapi diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 169dbbe..a4273fd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -40,6 +40,18 @@ build_images: "$CI_PROJECT_DIR" \ "$CI_REGISTRY_IMAGE" + # Build and push the rewriter image + docker build -t "registry.gitlab.syncad.com/hive/nft_tracker/postgrest-rewriter:$CI_COMMIT_SHORT_SHA" \ + -f Dockerfile.rewriter \ + --build-arg BUILD_TIME="$(date -uIseconds)" \ + --build-arg GIT_COMMIT_SHA="$CI_COMMIT_SHA" \ + --build-arg GIT_CURRENT_BRANCH="$CI_COMMIT_BRANCH" \ + --build-arg GIT_LAST_LOG_MESSAGE="$(git log -1 --pretty=%B)" \ + --build-arg GIT_LAST_COMMITTER="$(git log -1 --pretty='%an <%ae>')" \ + --build-arg GIT_LAST_COMMIT_DATE="$(git log -1 --pretty='%aI')" \ + . + docker push "registry.gitlab.syncad.com/hive/nft_tracker/postgrest-rewriter:$CI_COMMIT_SHORT_SHA" + # Push only the main image to registry.hive.blog (if credentials are available) echo "Current branch: $CI_COMMIT_BRANCH" echo "Is protected: $CI_COMMIT_REF_PROTECTED" @@ -48,9 +60,11 @@ build_images: if [[ -n "$BLOG_REGISTRY_USER" ]] && [[ -n "$BLOG_REGISTRY_PASSWORD" ]]; then echo "Pushing to registry.hive.blog..." docker tag "registry.gitlab.syncad.com/hive/nft_tracker:$CI_COMMIT_SHORT_SHA" "registry.hive.blog/nft_tracker:$CI_COMMIT_SHORT_SHA" + docker tag "registry.gitlab.syncad.com/hive/nft_tracker/postgrest-rewriter:$CI_COMMIT_SHORT_SHA" "registry.hive.blog/nft_tracker/postgrest-rewriter:$CI_COMMIT_SHORT_SHA" echo "Logging in to registry-upload.hive.blog as user: $BLOG_REGISTRY_USER" echo "$BLOG_REGISTRY_PASSWORD" | docker login --username "$BLOG_REGISTRY_USER" --password-stdin registry-upload.hive.blog docker push "registry.hive.blog/nft_tracker:$CI_COMMIT_SHORT_SHA" + docker push "registry.hive.blog/nft_tracker/postgrest-rewriter:$CI_COMMIT_SHORT_SHA" else echo "WARNING: BLOG_REGISTRY_USER or BLOG_REGISTRY_PASSWORD not set." echo "Skipping push to registry.hive.blog (this is expected for non-protected branches)" @@ -61,12 +75,16 @@ build_images: if [[ "$CI_COMMIT_BRANCH" == "develop" ]]; then echo "Tagging images with 'develop' tag..." docker tag "registry.gitlab.syncad.com/hive/nft_tracker:$CI_COMMIT_SHORT_SHA" "registry.gitlab.syncad.com/hive/nft_tracker:develop" + docker tag "registry.gitlab.syncad.com/hive/nft_tracker/postgrest-rewriter:$CI_COMMIT_SHORT_SHA" "registry.gitlab.syncad.com/hive/nft_tracker/postgrest-rewriter:develop" docker push "registry.gitlab.syncad.com/hive/nft_tracker:develop" + docker push "registry.gitlab.syncad.com/hive/nft_tracker/postgrest-rewriter:develop" # Only push develop tag to registry.hive.blog if we have credentials if [[ -n "$BLOG_REGISTRY_USER" ]] && [[ -n "$BLOG_REGISTRY_PASSWORD" ]]; then docker tag "registry.gitlab.syncad.com/hive/nft_tracker:$CI_COMMIT_SHORT_SHA" "registry.hive.blog/nft_tracker:develop" + docker tag "registry.gitlab.syncad.com/hive/nft_tracker/postgrest-rewriter:$CI_COMMIT_SHORT_SHA" "registry.hive.blog/nft_tracker/postgrest-rewriter:develop" docker push "registry.hive.blog/nft_tracker:develop" + docker push "registry.hive.blog/nft_tracker/postgrest-rewriter:develop" fi fi tags: @@ -92,15 +110,35 @@ publish_release_images: "$CI_PROJECT_DIR" \ "$CI_REGISTRY_IMAGE" - # Push only the main image to registry.hive.blog + # Build and push the rewriter image with release tag + docker build -t "registry.gitlab.syncad.com/hive/nft_tracker/postgrest-rewriter:$CI_COMMIT_TAG" \ + -f Dockerfile.rewriter \ + --target with_tag \ + --build-arg BUILD_TIME="$(date -uIseconds)" \ + --build-arg GIT_COMMIT_SHA="$CI_COMMIT_SHA" \ + --build-arg GIT_CURRENT_BRANCH="$CI_COMMIT_BRANCH" \ + --build-arg GIT_LAST_LOG_MESSAGE="$(git log -1 --pretty=%B)" \ + --build-arg GIT_LAST_COMMITTER="$(git log -1 --pretty='%an <%ae>')" \ + --build-arg GIT_LAST_COMMIT_DATE="$(git log -1 --pretty='%aI')" \ + --build-arg GIT_COMMIT_TAG="$CI_COMMIT_TAG" \ + . + docker push "registry.gitlab.syncad.com/hive/nft_tracker/postgrest-rewriter:$CI_COMMIT_TAG" + + # Push only the main images to registry.hive.blog docker tag "registry.gitlab.syncad.com/hive/nft_tracker:$CI_COMMIT_TAG" "registry.hive.blog/nft_tracker:$CI_COMMIT_TAG" + docker tag "registry.gitlab.syncad.com/hive/nft_tracker/postgrest-rewriter:$CI_COMMIT_TAG" "registry.hive.blog/nft_tracker/postgrest-rewriter:$CI_COMMIT_TAG" docker push "registry.hive.blog/nft_tracker:$CI_COMMIT_TAG" + docker push "registry.hive.blog/nft_tracker/postgrest-rewriter:$CI_COMMIT_TAG" # Also tag with 'latest' if this is a release tag docker tag "registry.gitlab.syncad.com/hive/nft_tracker:$CI_COMMIT_TAG" "registry.gitlab.syncad.com/hive/nft_tracker:latest" + docker tag "registry.gitlab.syncad.com/hive/nft_tracker/postgrest-rewriter:$CI_COMMIT_TAG" "registry.gitlab.syncad.com/hive/nft_tracker/postgrest-rewriter:latest" docker tag "registry.gitlab.syncad.com/hive/nft_tracker:$CI_COMMIT_TAG" "registry.hive.blog/nft_tracker:latest" + docker tag "registry.gitlab.syncad.com/hive/nft_tracker/postgrest-rewriter:$CI_COMMIT_TAG" "registry.hive.blog/nft_tracker/postgrest-rewriter:latest" docker push "registry.gitlab.syncad.com/hive/nft_tracker:latest" + docker push "registry.gitlab.syncad.com/hive/nft_tracker/postgrest-rewriter:latest" docker push "registry.hive.blog/nft_tracker:latest" + docker push "registry.hive.blog/nft_tracker/postgrest-rewriter:latest" echo "Successfully published release images with tag: $CI_COMMIT_TAG" rules: diff --git a/Dockerfile b/Dockerfile index 0d78e0f..b531e8f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,7 +39,7 @@ COPY scripts/install_app.sh /app/scripts/install_app.sh COPY scripts/uninstall_app.sh /app/scripts/uninstall_app.sh COPY scripts/process_blocks.sh /app/scripts/process_blocks.sh COPY db /app/db -# COPY endpoints /app/endpoints +COPY endpoints /app/endpoints COPY docker/scripts/block-processing-healthcheck.sh /app/block-processing-healthcheck.sh COPY docker/scripts/docker-entrypoint.sh /app/docker-entrypoint.sh diff --git a/Dockerfile.rewriter b/Dockerfile.rewriter new file mode 100644 index 0000000..1573fe9 --- /dev/null +++ b/Dockerfile.rewriter @@ -0,0 +1,33 @@ +FROM registry.gitlab.syncad.com/hive/common-ci-configuration/nginx:ecd325dd43aee24562f59195ef51a20fa15514d4 AS without_tag + +COPY docker/nft_tracker_nginx.conf.template /usr/local/openresty/nginx/conf/nginx.conf.template +COPY rewrite_rules.conf /usr/local/openresty/nginx/conf/rewrite_rules.conf +COPY docker/rewriter_entrypoint.sh /entrypoint.sh + +CMD ["/entrypoint.sh"] + +ARG BUILD_TIME +ARG GIT_COMMIT_SHA +ARG GIT_CURRENT_BRANCH +ARG GIT_LAST_LOG_MESSAGE +ARG GIT_LAST_COMMITTER +ARG GIT_LAST_COMMIT_DATE +LABEL org.opencontainers.image.created="$BUILD_TIME" +LABEL org.opencontainers.image.url="https://hive.io/" +LABEL org.opencontainers.image.documentation="https://gitlab.syncad.com/hive/nft_tracker" +LABEL org.opencontainers.image.source="https://gitlab.syncad.com/hive/nft_tracker" +#LABEL org.opencontainers.image.version="${VERSION}" +LABEL org.opencontainers.image.revision="$GIT_COMMIT_SHA" +LABEL org.opencontainers.image.licenses="MIT" +LABEL org.opencontainers.image.ref.name="NFT Tracker" +LABEL org.opencontainers.image.title="NFT Tracker PostgREST URL Rewriter Image" +LABEL org.opencontainers.image.description="Rewrites REST calls to provide more natural REST URLs than PostgREST alone allows" +LABEL io.hive.image.branch="$GIT_CURRENT_BRANCH" +LABEL io.hive.image.commit.log_message="$GIT_LAST_LOG_MESSAGE" +LABEL io.hive.image.commit.author="$GIT_LAST_COMMITTER" +LABEL io.hive.image.commit.date="$GIT_LAST_COMMIT_DATE" + +FROM without_tag AS with_tag + +ARG GIT_COMMIT_TAG +LABEL org.opencontainers.image.version="${GIT_COMMIT_TAG}" \ No newline at end of file diff --git a/docker/nft_tracker_nginx.conf.template b/docker/nft_tracker_nginx.conf.template new file mode 100644 index 0000000..55936fe --- /dev/null +++ b/docker/nft_tracker_nginx.conf.template @@ -0,0 +1,41 @@ +# +# Homepage and endpoints of the NFT Tracker API. +# +worker_processes auto; +error_log /dev/stdout info; +worker_rlimit_nofile 8192; + +events { + worker_connections 4096; +} +http { + access_log /dev/stdout; + server { + listen 0.0.0.0:80 default_server; + server_name _; + + location / { + include rewrite_rules.conf; + # ${REWRITE_LOG} will be replaced by the docker entrypoint script. + # Set REWRITE_LOG=on in the environment to enable rewrite logging, + # otherwise it will remain disabled + ${REWRITE_LOG} + + proxy_pass http://nft-tracker-postgrest:3000; # PostgREST endpoint + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_hide_header Content-Location; + proxy_set_header Connection ""; + proxy_http_version 1.1; + } + } + server { + listen 0.0.0.0:81; + + location /health { + return 204; + } + } +} \ No newline at end of file diff --git a/docker/rewriter_entrypoint.sh b/docker/rewriter_entrypoint.sh new file mode 100755 index 0000000..271dd78 --- /dev/null +++ b/docker/rewriter_entrypoint.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +# Default value for REWRITE_LOG is off, unless explicitly set to 'on' +if [ "$REWRITE_LOG" = "on" ]; then + REWRITE_LOG="rewrite_log on;" +else + REWRITE_LOG="# rewrite_log off;" +fi + +# Use sed to replace the placeholder in the nginx template file +sed "s|\${REWRITE_LOG}|$REWRITE_LOG|g" /usr/local/openresty/nginx/conf/nginx.conf.template > /usr/local/openresty/nginx/conf/nginx.conf + +# Start nginx +exec /usr/local/openresty/bin/openresty -g 'daemon off;' \ No newline at end of file diff --git a/endpoints/endpoint_schema.sql b/endpoints/endpoint_schema.sql new file mode 100644 index 0000000..38a8657 --- /dev/null +++ b/endpoints/endpoint_schema.sql @@ -0,0 +1,117 @@ +SET ROLE nfttracker_owner; + +/** openapi +openapi: 3.1.0 +info: + title: NFT Tracker + description: >- + NFT Tracker is an API for managing and tracking NFTs on the Hive blockchain + license: + name: MIT License + url: https://opensource.org/license/mit + version: 0.1.0 +externalDocs: + description: NFT Tracker gitlab repository + url: https://gitlab.syncad.com/hive/nft_tracker +tags: + - name: NFT + description: NFT management operations + - name: Other + description: General API information +servers: + - url: /nft-tracker-api + */ + +DO $__$ + DECLARE + __schema_name VARCHAR; + __swagger_url TEXT; + BEGIN + SHOW SEARCH_PATH INTO __schema_name; + __swagger_url := current_setting('custom.swagger_url', true)::TEXT; + IF __swagger_url IS NULL THEN + __swagger_url := 'localhost'; + END IF; + + CREATE SCHEMA IF NOT EXISTS nfttracker_endpoints AUTHORIZATION nfttracker_owner; + + EXECUTE FORMAT( + 'create or replace function nfttracker_endpoints.root() returns json as $_$ + declare + -- openapi-spec +-- openapi-generated-code-begin + openapi json = $$ +{ + "openapi": "3.1.0", + "info": { + "title": "NFT Tracker", + "description": "NFT Tracker is an API for managing and tracking NFTs on the Hive blockchain", + "license": { + "name": "MIT License", + "url": "https://opensource.org/license/mit" + }, + "version": "0.1.0" + }, + "externalDocs": { + "description": "NFT Tracker gitlab repository", + "url": "https://gitlab.syncad.com/hive/nft_tracker" + }, + "tags": [ + { + "name": "NFT", + "description": "NFT management operations" + }, + { + "name": "Other", + "description": "General API information" + } + ], + "servers": [ + { + "url": "/nft-tracker-api" + } + ], + "paths": { + "/version": { + "get": { + "tags": [ + "Other" + ], + "summary": "Get NFT Tracker''s version", + "description": "Get NFT Tracker''s last commit hash (versions set by hash value).\n\nSQL example\n* `SELECT * FROM nfttracker_endpoints.get_version();`\n\nREST call example\n* `GET ''https://%1$s/nft-tracker-api/version''`\n", + "operationId": "nfttracker_endpoints.get_version", + "responses": { + "200": { + "description": "NFT Tracker version\n\n* Returns `TEXT`\n", + "content": { + "application/json": { + "schema": { + "type": "string" + }, + "example": "c2fed8958584511ef1a66dab3dbac8c40f3518f0" + } + } + }, + "404": { + "description": "App not installed" + } + } + } + } + } +} +$$; +-- openapi-generated-code-end +begin + return openapi; +end +$_$ language plpgsql;' + , __swagger_url); + + -- Grant execute permission on the root function to nfttracker_user + GRANT EXECUTE ON FUNCTION nfttracker_endpoints.root() TO nfttracker_user; + + END +$__$; + +RESET ROLE; \ No newline at end of file diff --git a/endpoints/get_version.sql b/endpoints/get_version.sql new file mode 100644 index 0000000..6f4f28a --- /dev/null +++ b/endpoints/get_version.sql @@ -0,0 +1,64 @@ +SET ROLE nfttracker_owner; + +/** openapi:paths +/version: + get: + tags: + - Other + summary: Get NFT Tracker''s version + description: | + Get NFT Tracker''s last commit hash (versions set by hash value). + + SQL example + * `SELECT * FROM nfttracker_endpoints.get_version();` + + REST call example + * `GET ''https://%1$s/nft-tracker-api/version''` + operationId: nfttracker_endpoints.get_version + responses: + '200': + description: | + NFT Tracker version + + * Returns `TEXT` + content: + application/json: + schema: + type: string + example: c2fed8958584511ef1a66dab3dbac8c40f3518f0 + '404': + description: App not installed + */ +-- openapi-generated-code-begin +DROP FUNCTION IF EXISTS nfttracker_endpoints.get_version; +CREATE OR REPLACE FUNCTION nfttracker_endpoints.get_version() +RETURNS TEXT +-- openapi-generated-code-end +LANGUAGE 'plpgsql' STABLE +AS +$$ +DECLARE + _version TEXT; +BEGIN + -- Set cache headers for version endpoint (doesn't change often) + PERFORM set_config('response.headers', '[{"Cache-Control": "public, max-age=100000"}]', true); + + -- For now, return a placeholder version + -- TODO: This should be populated from a version table during deployment + _version := 'development'; + + -- Check if we have a version table (to be created later) + IF EXISTS (SELECT 1 FROM information_schema.tables + WHERE table_schema = 'nfttracker_app' + AND table_name = 'version') THEN + SELECT git_hash INTO _version FROM nfttracker_app.version LIMIT 1; + END IF; + + RETURN _version; +END +$$; + +-- Grant execute permission to nfttracker_user +GRANT EXECUTE ON FUNCTION nfttracker_endpoints.get_version() TO nfttracker_user; + +RESET ROLE; diff --git a/rewrite_rules.conf b/rewrite_rules.conf new file mode 100644 index 0000000..8d98c60 --- /dev/null +++ b/rewrite_rules.conf @@ -0,0 +1,8 @@ +rewrite ^/version /rpc/get_version break; +# endpoint for get /version + +rewrite ^/$ / break; +# endpoint for openapi spec itself + +rewrite ^/(.*)$ /rpc/$1 break; +# default endpoint for everything else diff --git a/scripts/install_app.sh b/scripts/install_app.sh index a359fb7..c99d0dc 100755 --- a/scripts/install_app.sh +++ b/scripts/install_app.sh @@ -64,5 +64,14 @@ psql "$POSTGRES_ACCESS" -v ON_ERROR_STOP=on -f "$SCRIPTPATH/../db/builtin_roles psql "$POSTGRES_ACCESS" -v ON_ERROR_STOP=on -f "$SCRIPTPATH/../db/schema.sql" psql "$POSTGRES_ACCESS" -v ON_ERROR_STOP=on -f "$SCRIPTPATH/../db/nft_actions.sql" psql "$POSTGRES_ACCESS" -v ON_ERROR_STOP=on -f "$SCRIPTPATH/../db/main_loop.sql" + +echo "Installing API endpoints..." +# Install endpoint schema and functions +psql "$POSTGRES_ACCESS" -v ON_ERROR_STOP=on -f "$SCRIPTPATH/../endpoints/endpoint_schema.sql" +psql "$POSTGRES_ACCESS" -v ON_ERROR_STOP=on -f "$SCRIPTPATH/../endpoints/get_version.sql" + +echo "Granting permissions..." psql "$POSTGRES_ACCESS" -v ON_ERROR_STOP=on -c "SET ROLE nfttracker_owner; GRANT USAGE ON SCHEMA ${NFTTRACKER_SCHEMA} to nfttracker_user;" psql "$POSTGRES_ACCESS" -v ON_ERROR_STOP=on -c "SET ROLE nfttracker_owner; GRANT SELECT ON ALL TABLES IN SCHEMA ${NFTTRACKER_SCHEMA} TO nfttracker_user;" +psql "$POSTGRES_ACCESS" -v ON_ERROR_STOP=on -c "SET ROLE nfttracker_owner; GRANT USAGE ON SCHEMA nfttracker_endpoints to nfttracker_user;" +psql "$POSTGRES_ACCESS" -v ON_ERROR_STOP=on -c "SET ROLE nfttracker_owner; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA nfttracker_endpoints TO nfttracker_user;" diff --git a/scripts/openapi_rewrite.sh b/scripts/openapi_rewrite.sh new file mode 100755 index 0000000..2da60f3 --- /dev/null +++ b/scripts/openapi_rewrite.sh @@ -0,0 +1,100 @@ +#!/bin/bash + +set -e +set -o pipefail + +SCRIPTDIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 || exit 1; pwd -P )" + +haf_dir="$SCRIPTDIR/../haf" +endpoints="endpoints" +rewrite_dir="${endpoints}_openapi" +input_file="rewrite_rules.conf" +temp_output_file=$(mktemp) + +# Default directories with fixed order if none provided +OUTPUT="$SCRIPTDIR/output" + +ENDPOINTS_IN_ORDER=" +../$endpoints/endpoint_schema.sql +../$endpoints/get_version.sql" + +# Function to reverse the lines for nginx rewrite rules +reverse_lines() { + awk ' + BEGIN { + RS = "" + FS = "\n" + } + { + for (i = 1; i <= NF; i++) { + if ($i ~ /^#/) { + comment = $i + } else if ($i ~ /^rewrite/) { + rewrite = $i + } + } + if (NR > 1) { + print "" + } + print comment + print rewrite + }' "$input_file" | tac +} + +# Function to install pip3 +install_pip() { + echo "pip3 is not installed. Installing now..." + # Ensure Python 3 is installed + if ! command -v python3 &> /dev/null; then + echo "Python 3 is not installed. Please install Python 3 first." + exit 1 + fi + # Try to install pip3 + sudo apt-get update + sudo apt-get install -y python3-pip + if ! command -v pip3 &> /dev/null; then + echo "pip3 installation failed. Please install pip3 manually." + exit 1 + fi +} + +# Check if pip3 is installed +if ! command -v pip3 &> /dev/null; then + install_pip +fi + +# Check if deepmerge is installed +if python3 -c "import deepmerge" &> /dev/null; then + echo "deepmerge is already installed." +else + echo "deepmerge is not installed. Installing now..." + pip3 install deepmerge + echo "deepmerge has been installed." +fi + +# Check if jsonpointer is installed +if python3 -c "import jsonpointer" &> /dev/null; then + echo "jsonpointer is already installed." +else + echo "jsonpointer is not installed. Installing now..." + pip3 install jsonpointer + echo "jsonpointer has been installed." +fi + +echo "Using endpoints directories" +echo "$ENDPOINTS_IN_ORDER" + +# run openapi rewrite script +# shellcheck disable=SC2086 +python3 $haf_dir/scripts/process_openapi.py $OUTPUT $ENDPOINTS_IN_ORDER + +# Create rewrite_rules.conf +reverse_lines > "$temp_output_file" +mv "$temp_output_file" "../$input_file" +rm "$input_file" + +# Move rewritten directory to root +rm -rf "$SCRIPTDIR/../$rewrite_dir" +mv "$OUTPUT/../$endpoints" "$SCRIPTDIR/../$rewrite_dir" +rm -rf "$SCRIPTDIR/output" +echo "Rewritten endpoint scripts saved in $rewrite_dir" \ No newline at end of file -- GitLab