From 2d66a22a89f721db54ec33cc776b15b6d8760d3c Mon Sep 17 00:00:00 2001
From: Konrad Botor <kbotor@syncad.com>
Date: Mon, 24 Jul 2023 13:40:54 +0200
Subject: [PATCH] Added Emscripten image build to CI - ref. !12

---
 .gitattributes                                |  3 +-
 .gitlab-ci.yml                                | 20 ++++--
 Dockerfile.emscripten                         | 37 +++++++----
 README.md                                     |  6 +-
 docker-bake.hcl                               | 63 ++++++++++++-------
 scripts/bash/build_emscripten_image.sh        |  4 +-
 scripts/bash/emscripten/prepare_boost.sh      |  2 +-
 scripts/bash/emscripten/prepare_openssl.sh    |  1 +
 .../bash/emscripten/prepare_secp256k1-zkp.sh  | 12 +++-
 9 files changed, 104 insertions(+), 44 deletions(-)

diff --git a/.gitattributes b/.gitattributes
index 857c41a..d6efeee 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,2 +1,3 @@
 *.sh text eol=lf
-Dockerfile.* text eol=lf
\ No newline at end of file
+Dockerfile.* text eol=lf
+*.patch text eol-lf
\ No newline at end of file
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c5ac946..3d386f6 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -33,7 +33,7 @@ lint_bash_scripts:
   before_script:
     - apk add xmlstarlet
   script:
-    - shellcheck -f checkstyle scripts/bash/*.sh > shellcheck-checkstyle-result.xml 
+    - shellcheck -f checkstyle scripts/**/*.sh > shellcheck-checkstyle-result.xml 
   after_script:
     - xmlstarlet tr misc/checkstyle2junit.xslt shellcheck-checkstyle-result.xml > shellcheck-junit-result.xml
   artifacts:
@@ -75,7 +75,7 @@ lint_python_scripts:
 
 .build_docker_image:
   extends: .docker_image_builder_job_template
-  image: docker:20.10.21
+  image: docker:24.0.1-cli
   variables:
     BUILD_TARGET: ""
   before_script:
@@ -96,7 +96,7 @@ lint_python_scripts:
       fi
       echo -e "\e[0Ksection_end:$(date +%s):tag\r\e[0K"
       echo -e "\e[0Ksection_start:$(date +%s):build[collapsed=true]\r\e[0KBaking image "$CI_REGISTRY_IMAGE/$BUILD_TARGET:${tag}"..."
-      docker buildx bake --progress=plain --push "$BUILD_TARGET"
+      docker buildx bake --progress=plain --provenance=false --push "$BUILD_TARGET"
       echo -e "\e[0Ksection_end:$(date +%s):build\r\e[0K"
   tags:
     - public-runner-docker
@@ -111,7 +111,7 @@ build_docker_dind_image:
       exists:
         - Dockerfile.docker-dind
   services:
-    - docker:20.10.21-dind
+    - docker:24.0.1-dind
 
 build_docker_builder_image:
   extends: .build_docker_image
@@ -161,6 +161,18 @@ build_tox_test_runner_image:
       exists:
         - Dockerfile.tox-test-runner
 
+build_emsdk_image:
+  extends: .build_docker_image
+  stage: build
+  variables:
+    BUILD_TARGET: "emsdk"
+  needs:
+    - build_docker_dind_image
+  rules:
+    - if: $CI_COMMIT_BRANCH
+      exists:
+        - Dockerfile.emscripten
+
 example_docker_image_builder_job:
   extends: .docker_image_builder_job_template
   stage: example-build
diff --git a/Dockerfile.emscripten b/Dockerfile.emscripten
index ffb7757..4997a9a 100644
--- a/Dockerfile.emscripten
+++ b/Dockerfile.emscripten
@@ -1,3 +1,4 @@
+# syntax=docker/dockerfile:1.5
 ARG EMSCRIPTEN_VERSION=3.1.43
 
 FROM emscripten/emsdk:${EMSCRIPTEN_VERSION} AS pure_emscripten_sdk
@@ -23,18 +24,30 @@ ENV OPENSSL_VERSION_TAG=${OPENSSL_VERSION_TAG}
 USER emscripten
 WORKDIR /home/emscripten
 
-RUN mkdir -vp tmp_src && cd tmp_src && \
-    git clone https://github.com/boostorg/boost.git && \
-    cd boost && \
-    git checkout tags/${BOOST_VERSION_TAG} && \
-    git submodule update --init --recursive && \
-    cd .. && \
-    git clone https://github.com/openssl/openssl.git && \
-    cd openssl && \
-    git checkout tags/${OPENSSL_VERSION_TAG} && \
-    git submodule update --init --recursive && \
-    cd .. && \
-    git clone https://github.com/ElementsProject/secp256k1-zkp.git
+RUN <<-EOF
+  set -e
+
+  git config --global advice.detachedHead false
+  
+  mkdir -vp tmp_src
+  cd tmp_src
+
+  git clone https://github.com/boostorg/boost.git
+
+  cd boost
+  git checkout tags/${BOOST_VERSION_TAG}
+  git submodule update --init --recursive
+
+  cd ..
+  git clone https://github.com/openssl/openssl.git
+
+  cd openssl
+  git checkout tags/${OPENSSL_VERSION_TAG}
+  git submodule update --init --recursive
+  
+  cd ..
+  git clone https://github.com/ElementsProject/secp256k1-zkp.git
+EOF
 
 FROM lib_source as emscripten_lib_builder
 
diff --git a/README.md b/README.md
index cc13324..84a492b 100644
--- a/README.md
+++ b/README.md
@@ -6,13 +6,15 @@ This project contains the common CI templates and scripts for Hive and Hive-rela
 
 - misc - miscellaneous files
 - scripts/bash - Bash scripts
+- scripts/emscripten - source for emsdk image
 - scripts/python - Python scripts
 - templates - GitLab CI templates
 
 ## Job templates
 
- - [docker_image_jobs.gitlab-ci.yml](templates/docker_image_jobs.gitlab-ci.yml) - templates for managing Docker images
- - [test_jobs.gitlab-ci.yml](templates/test_jobs.gitlab-ci.yml) - templates for running tests
+- [docker_image_jobs.gitlab-ci.yml](templates/docker_image_jobs.gitlab-ci.yml) - templates for managing Docker images
+- [test_jobs.gitlab-ci.yml](templates/test_jobs.gitlab-ci.yml) - templates for running tests
+
 ## Example jobs
 
 The GitLab CI configuration for this repository contains example jobs based on the templates defined in it. On top of that the Docker images are built by jobs also based on said templates.
diff --git a/docker-bake.hcl b/docker-bake.hcl
index 7ddf8e1..5ff8299 100644
--- a/docker-bake.hcl
+++ b/docker-bake.hcl
@@ -1,5 +1,14 @@
 variable "CI_REGISTRY_IMAGE" {}
 variable "CI_COMMIT_SHA" {}
+variable "EMSCRIPTEN_VERSION" {
+  default = "3.1.43"
+}
+variable "BOOST_VERSION_TAG" {
+  default = null
+}
+variable "OPENSSL_VERSION_TAG" {
+  default = null
+}
 variable "tag" {
   default = "latest"
 }
@@ -10,58 +19,70 @@ function "notempty" {
 }
 
 function "generate-tags" {
-  params = [target]
+  params = [target, local_tag]
   result = [
-    notempty(CI_REGISTRY_IMAGE) ? "${CI_REGISTRY_IMAGE}/${target}:${tag}" : "${target}:${tag}",
+    notempty(CI_REGISTRY_IMAGE) ? "${CI_REGISTRY_IMAGE}/${target}:${local_tag}" : "${target}:${local_tag}",
     notempty(CI_COMMIT_SHA) ? "${CI_REGISTRY_IMAGE}/${target}:${CI_COMMIT_SHA}": ""
   ]
 }
 
 function "generate-cache-from" {
-  params = [target]
+  params = [target, local_tag]
   result = [
-    notempty(CI_REGISTRY_IMAGE) ? "type=registry,ref=${CI_REGISTRY_IMAGE}/${target}:${tag}-cache" : "${target}:${tag}",
+    notempty(CI_REGISTRY_IMAGE) ? "type=registry,ref=${CI_REGISTRY_IMAGE}/${target}:${local_tag}-cache" : "${target}:${local_tag}",
   ]
 }
 
 function "generate-cache-to" {
-  params = [target]
+  params = [target, local_tag]
   result = [
-    notempty(CI_REGISTRY_IMAGE) ? "type=registry,mode=max,ref=${CI_REGISTRY_IMAGE}/${target}:${tag}-cache" : "type=inline",
+    notempty(CI_REGISTRY_IMAGE) ? "type=registry,mode=max,ref=${CI_REGISTRY_IMAGE}/${target}:${local_tag}-cache" : "type=inline",
   ]
 }
 
 target "benchmark-test-runner" {
   dockerfile = "Dockerfile.benchmark-test-runner"
-  tags = generate-tags("benchmark-test-runner")
-  cache-from = generate-cache-from("benchmark-test-runner")
-  cache-to = generate-cache-to("benchmark-test-runner")
+  tags = generate-tags("benchmark-test-runner", "${tag}")
+  cache-from = generate-cache-from("benchmark-test-runner", "${tag}")
+  cache-to = generate-cache-to("benchmark-test-runner", "${tag}")
 }
 
 target "docker-builder" {
   dockerfile = "Dockerfile.docker-builder"
-  tags = generate-tags("docker-builder")
-  cache-from = generate-cache-from("docker-builder")
-  cache-to = generate-cache-to("docker-builder")
+  tags = generate-tags("docker-builder", "${tag}")
+  cache-from = generate-cache-from("docker-builder", "${tag}")
+  cache-to = generate-cache-to("docker-builder", "${tag}")
 }
 
 target "docker-dind" {
   dockerfile = "Dockerfile.docker-dind"
-  tags = generate-tags("docker-dind")
-  cache-from = generate-cache-from("docker-dind")
-  cache-to = generate-cache-to("docker-dind")
+  tags = generate-tags("docker-dind", "${tag}")
+  cache-from = generate-cache-from("docker-dind", "${tag}")
+  cache-to = generate-cache-to("docker-dind", "${tag}")
 }
 
 target "image-remover" {
   dockerfile = "Dockerfile.image-remover"
-  tags = generate-tags("image-remover")
-  cache-from = generate-cache-from("image-remover")
-  cache-to = generate-cache-to("image-remover")
+  tags = generate-tags("image-remover", "${tag}")
+  cache-from = generate-cache-from("image-remover", "${tag}")
+  cache-to = generate-cache-to("image-remover", "${tag}")
 }
 
 target "tox-test-runner" {
   dockerfile = "Dockerfile.tox-test-runner"
-  tags = generate-tags("tox-test-runner")
-  cache-from = generate-cache-from("tox-test-runner")
-  cache-to = generate-cache-to("tox-test-runner")
+  tags = generate-tags("tox-test-runner", "${tag}")
+  cache-from = generate-cache-from("tox-test-runner", "${tag}")
+  cache-to = generate-cache-to("tox-test-runner", "${tag}")
+}
+
+target "emsdk" {
+  dockerfile = "Dockerfile.emscripten"
+  tags = [ notempty(CI_REGISTRY_IMAGE) ? "${CI_REGISTRY_IMAGE}/emsdk:${EMSCRIPTEN_VERSION}" : "emsdk:${EMSCRIPTEN_VERSION}" ]
+  cache-from = generate-cache-from("emsdk", "${EMSCRIPTEN_VERSION}")
+  cache-to = generate-cache-to("emsdk", "${EMSCRIPTEN_VERSION}")
+  args = {
+    EMSCRIPTEN_VERSION = "${EMSCRIPTEN_VERSION}",
+    BOOST_VERSION_TAG = "${BOOST_VERSION_TAG}",
+    OPENSSL_VERSION_TAG = "${OPENSSL_VERSION_TAG}"
+  }
 }
\ No newline at end of file
diff --git a/scripts/bash/build_emscripten_image.sh b/scripts/bash/build_emscripten_image.sh
index 803a09a..6487e33 100755
--- a/scripts/bash/build_emscripten_image.sh
+++ b/scripts/bash/build_emscripten_image.sh
@@ -10,6 +10,6 @@ EMSDK_VERSION=${1:-"3.1.43"}
 export DOCKER_BUILDKIT=1
 
 docker build --target=emscripten_builder \
-  --build-arg EMSCRIPTEN_VERSION=${EMSDK_VERSION} \
-  -t ${REGISTRY}emsdk:3.1.43 \
+  --build-arg "EMSCRIPTEN_VERSION=${EMSDK_VERSION}" \
+  -t "${REGISTRY}emsdk:3.1.43" \
   -f "${SRCDIR}/Dockerfile.emscripten" "${SRCDIR}"
diff --git a/scripts/bash/emscripten/prepare_boost.sh b/scripts/bash/emscripten/prepare_boost.sh
index 10b6e13..2642c18 100755
--- a/scripts/bash/emscripten/prepare_boost.sh
+++ b/scripts/bash/emscripten/prepare_boost.sh
@@ -29,7 +29,7 @@ printf "using clang : emscripten : emcc -s USE_ZLIB=1 -s USE_ICU=0 : <archiver>e
 ./b2 \
   --build-dir="${TMP_SRC}/boost_build/" \
   --prefix="${INSTALL_PREFIX}" \
-  -j $(nproc) \
+  -j "$(nproc)" \
   -q \
   runtime-link=static \
   link=static \
diff --git a/scripts/bash/emscripten/prepare_openssl.sh b/scripts/bash/emscripten/prepare_openssl.sh
index 8609dc8..499c710 100755
--- a/scripts/bash/emscripten/prepare_openssl.sh
+++ b/scripts/bash/emscripten/prepare_openssl.sh
@@ -23,6 +23,7 @@ emconfigure ./Configure \
   linux-x32 \
   -static
 
+# shellcheck disable=SC2016
 sed -i 's/$(CROSS_COMPILE)//' Makefile
 emmake make -j 8 
 emmake make install
diff --git a/scripts/bash/emscripten/prepare_secp256k1-zkp.sh b/scripts/bash/emscripten/prepare_secp256k1-zkp.sh
index 1391f1c..04c41e0 100755
--- a/scripts/bash/emscripten/prepare_secp256k1-zkp.sh
+++ b/scripts/bash/emscripten/prepare_secp256k1-zkp.sh
@@ -13,6 +13,16 @@ git checkout d22774e248c703a191049b78f8d04f37d6fcfa05
 
 export VERBOSE=1
 emconfigure ./autogen.sh
-emconfigure ./configure --prefix=${INSTALL_PREFIX} --with-asm=no --enable-shared=no --enable-tests=no --enable-benchmark=no --enable-exhaustive-tests=no --with-pic=no --with-valgrind=no --enable-module-recovery=yes --enable-module-rangeproof=yes --enable-experimental
+emconfigure ./configure --prefix="${INSTALL_PREFIX}" \
+  --with-asm=no \
+  --enable-shared=no \
+  --enable-tests=no \
+  --enable-benchmark=no \
+  --enable-exhaustive-tests=no \
+  --with-pic=no \
+  --with-valgrind=no \
+  --enable-module-recovery=yes \
+  --enable-module-rangeproof=yes \
+  --enable-experimental
 emmake make
 emmake make install
-- 
GitLab