Added Emscripten image build to CI - ref. !12

.gitattributes

 *.sh text eol=lf
-Dockerfile.* text eol=lf
\ No newline at end of file
+Dockerfile.* text eol=lf
+*.patch text eol-lf
\ No newline at end of file

.gitlab-ci.yml

@@ -33,7 +33,7 @@ lint_bash_scripts:
   before_script:
     - apk add xmlstarlet
   script:
-    - shellcheck -f checkstyle scripts/bash/*.sh > shellcheck-checkstyle-result.xml 
+    - shellcheck -f checkstyle scripts/**/*.sh > shellcheck-checkstyle-result.xml 
   after_script:
     - xmlstarlet tr misc/checkstyle2junit.xslt shellcheck-checkstyle-result.xml > shellcheck-junit-result.xml
   artifacts:
@@ -75,7 +75,7 @@ lint_python_scripts:
 
 .build_docker_image:
   extends: .docker_image_builder_job_template
-  image: docker:20.10.21
+  image: docker:24.0.1-cli
   variables:
     BUILD_TARGET: ""
   before_script:
@@ -96,7 +96,7 @@ lint_python_scripts:
       fi
       echo -e "\e[0Ksection_end:$(date +%s):tag\r\e[0K"
       echo -e "\e[0Ksection_start:$(date +%s):build[collapsed=true]\r\e[0KBaking image "$CI_REGISTRY_IMAGE/$BUILD_TARGET:${tag}"..."
-      docker buildx bake --progress=plain --push "$BUILD_TARGET"
+      docker buildx bake --progress=plain --provenance=false --push "$BUILD_TARGET"
       echo -e "\e[0Ksection_end:$(date +%s):build\r\e[0K"
   tags:
     - public-runner-docker
@@ -111,7 +111,7 @@ build_docker_dind_image:
       exists:
         - Dockerfile.docker-dind
   services:
-    - docker:20.10.21-dind
+    - docker:24.0.1-dind
 
 build_docker_builder_image:
   extends: .build_docker_image
@@ -161,6 +161,18 @@ build_tox_test_runner_image:
       exists:
         - Dockerfile.tox-test-runner
 
+build_emsdk_image:
+  extends: .build_docker_image
+  stage: build
+  variables:
+    BUILD_TARGET: "emsdk"
+  needs:
+    - build_docker_dind_image
+  rules:
+    - if: $CI_COMMIT_BRANCH
+      exists:
+        - Dockerfile.emscripten
+
 example_docker_image_builder_job:
   extends: .docker_image_builder_job_template
   stage: example-build

Dockerfile.emscripten

+# syntax=docker/dockerfile:1.5
 ARG EMSCRIPTEN_VERSION=3.1.43
 
 FROM emscripten/emsdk:${EMSCRIPTEN_VERSION} AS pure_emscripten_sdk
@@ -23,18 +24,30 @@ ENV OPENSSL_VERSION_TAG=${OPENSSL_VERSION_TAG}
 USER emscripten
 WORKDIR /home/emscripten
 
-RUN mkdir -vp tmp_src && cd tmp_src && \
-    git clone https://github.com/boostorg/boost.git && \
-    cd boost && \
-    git checkout tags/${BOOST_VERSION_TAG} && \
-    git submodule update --init --recursive && \
-    cd .. && \
-    git clone https://github.com/openssl/openssl.git && \
-    cd openssl && \
-    git checkout tags/${OPENSSL_VERSION_TAG} && \
-    git submodule update --init --recursive && \
-    cd .. && \
-    git clone https://github.com/ElementsProject/secp256k1-zkp.git
+RUN <<-EOF
+  set -e
+
+  git config --global advice.detachedHead false
+  
+  mkdir -vp tmp_src
+  cd tmp_src
+
+  git clone https://github.com/boostorg/boost.git
+
+  cd boost
+  git checkout tags/${BOOST_VERSION_TAG}
+  git submodule update --init --recursive
+
+  cd ..
+  git clone https://github.com/openssl/openssl.git
+
+  cd openssl
+  git checkout tags/${OPENSSL_VERSION_TAG}
+  git submodule update --init --recursive
+  
+  cd ..
+  git clone https://github.com/ElementsProject/secp256k1-zkp.git
+EOF
 
 FROM lib_source as emscripten_lib_builder
 

README.md

@@ -6,13 +6,15 @@ This project contains the common CI templates and scripts for Hive and Hive-rela
 
 - misc - miscellaneous files
 - scripts/bash - Bash scripts
+- scripts/emscripten - source for emsdk image
 - scripts/python - Python scripts
 - templates - GitLab CI templates
 
 ## Job templates
 
- - [docker_image_jobs.gitlab-ci.yml](templates/docker_image_jobs.gitlab-ci.yml) - templates for managing Docker images
- - [test_jobs.gitlab-ci.yml](templates/test_jobs.gitlab-ci.yml) - templates for running tests
+- [docker_image_jobs.gitlab-ci.yml](templates/docker_image_jobs.gitlab-ci.yml) - templates for managing Docker images
+- [test_jobs.gitlab-ci.yml](templates/test_jobs.gitlab-ci.yml) - templates for running tests
+
 ## Example jobs
 
 The GitLab CI configuration for this repository contains example jobs based on the templates defined in it. On top of that the Docker images are built by jobs also based on said templates.

docker-bake.hcl

 variable "CI_REGISTRY_IMAGE" {}
 variable "CI_COMMIT_SHA" {}
+variable "EMSCRIPTEN_VERSION" {
+  default = "3.1.43"
+}
+variable "BOOST_VERSION_TAG" {
+  default = null
+}
+variable "OPENSSL_VERSION_TAG" {
+  default = null
+}
 variable "tag" {
   default = "latest"
 }
@@ -10,58 +19,70 @@ function "notempty" {
 }
 
 function "generate-tags" {
-  params = [target]
+  params = [target, local_tag]
   result = [
-    notempty(CI_REGISTRY_IMAGE) ? "${CI_REGISTRY_IMAGE}/${target}:${tag}" : "${target}:${tag}",
+    notempty(CI_REGISTRY_IMAGE) ? "${CI_REGISTRY_IMAGE}/${target}:${local_tag}" : "${target}:${local_tag}",
     notempty(CI_COMMIT_SHA) ? "${CI_REGISTRY_IMAGE}/${target}:${CI_COMMIT_SHA}": ""
   ]
 }
 
 function "generate-cache-from" {
-  params = [target]
+  params = [target, local_tag]
   result = [
-    notempty(CI_REGISTRY_IMAGE) ? "type=registry,ref=${CI_REGISTRY_IMAGE}/${target}:${tag}-cache" : "${target}:${tag}",
+    notempty(CI_REGISTRY_IMAGE) ? "type=registry,ref=${CI_REGISTRY_IMAGE}/${target}:${local_tag}-cache" : "${target}:${local_tag}",
   ]
 }
 
 function "generate-cache-to" {
-  params = [target]
+  params = [target, local_tag]
   result = [
-    notempty(CI_REGISTRY_IMAGE) ? "type=registry,mode=max,ref=${CI_REGISTRY_IMAGE}/${target}:${tag}-cache" : "type=inline",
+    notempty(CI_REGISTRY_IMAGE) ? "type=registry,mode=max,ref=${CI_REGISTRY_IMAGE}/${target}:${local_tag}-cache" : "type=inline",
   ]
 }
 
 target "benchmark-test-runner" {
   dockerfile = "Dockerfile.benchmark-test-runner"
-  tags = generate-tags("benchmark-test-runner")
-  cache-from = generate-cache-from("benchmark-test-runner")
-  cache-to = generate-cache-to("benchmark-test-runner")
+  tags = generate-tags("benchmark-test-runner", "${tag}")
+  cache-from = generate-cache-from("benchmark-test-runner", "${tag}")
+  cache-to = generate-cache-to("benchmark-test-runner", "${tag}")
 }
 
 target "docker-builder" {
   dockerfile = "Dockerfile.docker-builder"
-  tags = generate-tags("docker-builder")
-  cache-from = generate-cache-from("docker-builder")
-  cache-to = generate-cache-to("docker-builder")
+  tags = generate-tags("docker-builder", "${tag}")
+  cache-from = generate-cache-from("docker-builder", "${tag}")
+  cache-to = generate-cache-to("docker-builder", "${tag}")
 }
 
 target "docker-dind" {
   dockerfile = "Dockerfile.docker-dind"
-  tags = generate-tags("docker-dind")
-  cache-from = generate-cache-from("docker-dind")
-  cache-to = generate-cache-to("docker-dind")
+  tags = generate-tags("docker-dind", "${tag}")
+  cache-from = generate-cache-from("docker-dind", "${tag}")
+  cache-to = generate-cache-to("docker-dind", "${tag}")
 }
 
 target "image-remover" {
   dockerfile = "Dockerfile.image-remover"
-  tags = generate-tags("image-remover")
-  cache-from = generate-cache-from("image-remover")
-  cache-to = generate-cache-to("image-remover")
+  tags = generate-tags("image-remover", "${tag}")
+  cache-from = generate-cache-from("image-remover", "${tag}")
+  cache-to = generate-cache-to("image-remover", "${tag}")
 }
 
 target "tox-test-runner" {
   dockerfile = "Dockerfile.tox-test-runner"
-  tags = generate-tags("tox-test-runner")
-  cache-from = generate-cache-from("tox-test-runner")
-  cache-to = generate-cache-to("tox-test-runner")
+  tags = generate-tags("tox-test-runner", "${tag}")
+  cache-from = generate-cache-from("tox-test-runner", "${tag}")
+  cache-to = generate-cache-to("tox-test-runner", "${tag}")
+}
+
+target "emsdk" {
+  dockerfile = "Dockerfile.emscripten"
+  tags = [ notempty(CI_REGISTRY_IMAGE) ? "${CI_REGISTRY_IMAGE}/emsdk:${EMSCRIPTEN_VERSION}" : "emsdk:${EMSCRIPTEN_VERSION}" ]
+  cache-from = generate-cache-from("emsdk", "${EMSCRIPTEN_VERSION}")
+  cache-to = generate-cache-to("emsdk", "${EMSCRIPTEN_VERSION}")
+  args = {
+    EMSCRIPTEN_VERSION = "${EMSCRIPTEN_VERSION}",
+    BOOST_VERSION_TAG = "${BOOST_VERSION_TAG}",
+    OPENSSL_VERSION_TAG = "${OPENSSL_VERSION_TAG}"
+  }
 }
\ No newline at end of file

scripts/bash/build_emscripten_image.sh

@@ -10,6 +10,6 @@ EMSDK_VERSION=${1:-"3.1.43"}
 export DOCKER_BUILDKIT=1
 
 docker build --target=emscripten_builder \
-  --build-arg EMSCRIPTEN_VERSION=${EMSDK_VERSION} \
-  -t ${REGISTRY}emsdk:3.1.43 \
+  --build-arg "EMSCRIPTEN_VERSION=${EMSDK_VERSION}" \
+  -t "${REGISTRY}emsdk:3.1.43" \
   -f "${SRCDIR}/Dockerfile.emscripten" "${SRCDIR}"

scripts/bash/emscripten/prepare_boost.sh

@@ -29,7 +29,7 @@ printf "using clang : emscripten : emcc -s USE_ZLIB=1 -s USE_ICU=0 : <archiver>e
 ./b2 \
   --build-dir="${TMP_SRC}/boost_build/" \
   --prefix="${INSTALL_PREFIX}" \
-  -j $(nproc) \
+  -j "$(nproc)" \
   -q \
   runtime-link=static \
   link=static \

scripts/bash/emscripten/prepare_openssl.sh

@@ -23,6 +23,7 @@ emconfigure ./Configure \
   linux-x32 \
   -static
 
+# shellcheck disable=SC2016
 sed -i 's/$(CROSS_COMPILE)//' Makefile
 emmake make -j 8 
 emmake make install

scripts/bash/emscripten/prepare_secp256k1-zkp.sh

@@ -13,6 +13,16 @@ git checkout d22774e248c703a191049b78f8d04f37d6fcfa05
 
 export VERBOSE=1
 emconfigure ./autogen.sh
-emconfigure ./configure --prefix=${INSTALL_PREFIX} --with-asm=no --enable-shared=no --enable-tests=no --enable-benchmark=no --enable-exhaustive-tests=no --with-pic=no --with-valgrind=no --enable-module-recovery=yes --enable-module-rangeproof=yes --enable-experimental
+emconfigure ./configure --prefix="${INSTALL_PREFIX}" \
+  --with-asm=no \
+  --enable-shared=no \
+  --enable-tests=no \
+  --enable-benchmark=no \
+  --enable-exhaustive-tests=no \
+  --with-pic=no \
+  --with-valgrind=no \
+  --enable-module-recovery=yes \
+  --enable-module-rangeproof=yes \
+  --enable-experimental
 emmake make
 emmake make install