From 219dfbabd58ff6008f8a85c291f8cf4f6da1318a Mon Sep 17 00:00:00 2001
From: Maxim De Clercq <maximdeclercq00@gmail.com>
Date: Sun, 23 Jan 2022 17:31:12 +0100
Subject: [PATCH 1/8] fix: pass RUSTC_WRAPPER to the cross container and
 enforce static builds

---
 .gitlab-ci.yml | 18 ++++++------------
 Cross.toml     |  3 ---
 cross/build.sh |  9 +++++----
 3 files changed, 11 insertions(+), 19 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b5a12f3d..fac678cf 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -35,8 +35,6 @@ variables:
   before_script:
     - 'echo "Building for target $TARGET"'
     - "rustup show && rustc --version && cargo --version" # Print version info for debugging
-    # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results:
-    - if [ -n "${SCCACHE_BIN_URL}" ]; then curl $SCCACHE_BIN_URL --output /sccache && chmod +x /sccache && export RUSTC_WRAPPER=/sccache; fi
     # install cross-compiling prerequisites
     - 'apt-get update && apt-get install -y docker.io && docker version' # install docker
     - 'cargo install cross && cross --version' # install cross
@@ -45,10 +43,14 @@ variables:
     - 'cp -r $CARGO_HOME/bin $SHARED_PATH/cargo'
     - 'cp -r $RUSTUP_HOME $SHARED_PATH'
     - 'export CARGO_HOME=$SHARED_PATH/cargo RUSTUP_HOME=$SHARED_PATH/rustup'
+    # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results:
+    - if [ -n "${SCCACHE_BIN_URL}" ]; then export RUSTC_WRAPPER=$SHARED_PATH/cargo/bin/sccache && curl $SCCACHE_BIN_URL --output $RUSTC_WRAPPER && chmod +x $RUSTC_WRAPPER; fi
   script:
     # cross-compile conduit for target
     - 'time ./cross/build.sh --locked --release'
     - 'mv "target/$TARGET/release/conduit" "conduit-$TARGET"'
+    # assert that the binary is statically linked
+    - 'file conduit-$TARGET | grep "static\(-pie\|ally\) linked"'
   cache:
     # https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci
     key: 'cargo-cache-$TARGET'
@@ -82,16 +84,6 @@ build:release:cargo:x86_64-unknown-linux-musl:
       - "conduit-x86_64-unknown-linux-musl"
     expose_as: "Conduit for x86_64-unknown-linux-musl"
 
-build:release:cargo:i686-unknown-linux-musl:
-  extends: .build-cargo-shared-settings
-  variables:
-    TARGET: "i686-unknown-linux-musl"
-  artifacts:
-    name: "conduit-i686-unknown-linux-musl"
-    paths:
-      - "conduit-i686-unknown-linux-musl"
-    expose_as: "Conduit for i686-unknown-linux-musl"
-
 build:release:cargo:arm-unknown-linux-musleabihf:
   extends: .build-cargo-shared-settings
   variables:
@@ -132,6 +124,8 @@ build:release:cargo:aarch64-unknown-linux-musl:
     # cross-compile conduit for target
     - 'time ./cross/build.sh --locked'
     - 'mv "target/$TARGET/debug/conduit" "conduit-debug-$TARGET"'
+    # assert that the binary is statically linked
+    - 'file conduit-debug-$TARGET | grep "static\(-pie\|ally\) linked"'
   artifacts:
     expire_in: 4 weeks
 
diff --git a/Cross.toml b/Cross.toml
index 22c84b97..491efcb7 100644
--- a/Cross.toml
+++ b/Cross.toml
@@ -7,8 +7,5 @@ image = "rust-cross:arm-unknown-linux-musleabihf"
 [target.armv7-unknown-linux-musleabihf]
 image = "rust-cross:armv7-unknown-linux-musleabihf"
 
-[target.i686-unknown-linux-musl]
-image = "rust-cross:i686-unknown-linux-musl"
-
 [target.x86_64-unknown-linux-musl]
 image = "rust-cross:x86_64-unknown-linux-musl"
diff --git a/cross/build.sh b/cross/build.sh
index 24a2224b..34082606 100755
--- a/cross/build.sh
+++ b/cross/build.sh
@@ -17,13 +17,14 @@ ENV TARGET_PREFIX="/usr/local/$(echo "${TARGET:?}" | sed -e 's/armv7/arm/' -e 's
 # Make sure that cc-rs links libc/libstdc++ statically when cross-compiling
 # See https://github.com/alexcrichton/cc-rs#external-configuration-via-environment-variables for more information
 ENV RUSTFLAGS="-L\$TARGET_PREFIX/lib" CXXSTDLIB="static=stdc++"
-# Forcefully linking against libatomic and libgcc is required for arm32, otherwise symbols are missing
-$([[ $TARGET =~ arm ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-lgcc -Clink-arg=-latomic"')
-# Forcefully linking against libc is required for 32-bit, otherwise symbols are missing
-$([[ $TARGET =~ arm|i686 ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -lstatic=c"')
+# Forcefully linking against libatomic, libc and libgcc is required for arm32, otherwise symbols are missing
+$([[ $TARGET =~ arm ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-lgcc -Clink-arg=-latomic -lstatic=c"')
 # Strip symbols while compiling in release mode
 $([[ $@ =~ -r ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-s"')
 
+# Support a rustc wrapper like sccache when cross-compiling
+ENV RUSTC_WRAPPER="$RUSTC_WRAPPER"
+
 # Make sure that rust-bindgen uses the correct include path when cross-compiling
 # See https://github.com/rust-lang/rust-bindgen#environment-variables for more information
 ENV BINDGEN_EXTRA_CLANG_ARGS="-I\$TARGET_PREFIX/include"

From c2ad2b3dd747e7a8baa5ff2f9ade8edb92204aa6 Mon Sep 17 00:00:00 2001
From: Maxim De Clercq <maximdeclercq00@gmail.com>
Date: Sun, 23 Jan 2022 17:38:13 +0100
Subject: [PATCH 2/8] fix: pass sccache variables to cross container with
 build.env.passthrough

---
 Cross.toml     | 12 ++++++++++++
 cross/build.sh |  3 ---
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/Cross.toml b/Cross.toml
index 491efcb7..a989a98f 100644
--- a/Cross.toml
+++ b/Cross.toml
@@ -1,3 +1,15 @@
+[build.env]
+# CI uses an S3 endpoint to store sccache artifacts, so their config needs to
+# be available in the cross container as well
+passthrough = [
+    "RUSTC_WRAPPER",
+    "AWS_ACCESS_KEY_ID",
+    "AWS_SECRET_ACCESS_KEY",
+    "SCCACHE_BUCKET",
+    "SCCACHE_ENDPOINT",
+    "SCCACHE_S3_USE_SSL",
+]
+
 [target.aarch64-unknown-linux-musl]
 image = "rust-cross:aarch64-unknown-linux-musl"
 
diff --git a/cross/build.sh b/cross/build.sh
index 34082606..4a6d4493 100755
--- a/cross/build.sh
+++ b/cross/build.sh
@@ -22,9 +22,6 @@ $([[ $TARGET =~ arm ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-lgcc -Clin
 # Strip symbols while compiling in release mode
 $([[ $@ =~ -r ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-s"')
 
-# Support a rustc wrapper like sccache when cross-compiling
-ENV RUSTC_WRAPPER="$RUSTC_WRAPPER"
-
 # Make sure that rust-bindgen uses the correct include path when cross-compiling
 # See https://github.com/rust-lang/rust-bindgen#environment-variables for more information
 ENV BINDGEN_EXTRA_CLANG_ARGS="-I\$TARGET_PREFIX/include"

From c7560b3502d27a49f935c347458df6421459c485 Mon Sep 17 00:00:00 2001
From: Maxim De Clercq <maximdeclercq00@gmail.com>
Date: Sun, 23 Jan 2022 18:09:14 +0100
Subject: [PATCH 3/8] fix: remove libgcc dependency in ci builds since the
 binary is ensured to be statically compiled

---
 docker/ci-binaries-packaging.Dockerfile | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/docker/ci-binaries-packaging.Dockerfile b/docker/ci-binaries-packaging.Dockerfile
index f4603105..a6339be3 100644
--- a/docker/ci-binaries-packaging.Dockerfile
+++ b/docker/ci-binaries-packaging.Dockerfile
@@ -9,6 +9,7 @@
 
 FROM docker.io/alpine:3.15.0 AS runner
 
+
 # Standard port on which Conduit launches.
 # You still need to map the port when using the docker command or docker-compose.
 EXPOSE 6167
@@ -18,10 +19,8 @@ ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml"
 
 # Conduit needs:
 #   ca-certificates: for https
-#   libgcc: Apparently this is needed, even if I (@jfowl) don't know exactly why. But whatever, it's not that big.
 RUN apk add --no-cache \
-    ca-certificates \
-    libgcc
+    ca-certificates
 
 
 ARG CREATED

From 64c25ea4a15739f75ebe2811e84dc00280ba5fb0 Mon Sep 17 00:00:00 2001
From: Maxim De Clercq <maximdeclercq00@gmail.com>
Date: Sun, 23 Jan 2022 18:31:40 +0100
Subject: [PATCH 4/8] fix: always print ELF information

---
 .gitlab-ci.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index fac678cf..defd66ee 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -50,7 +50,8 @@ variables:
     - 'time ./cross/build.sh --locked --release'
     - 'mv "target/$TARGET/release/conduit" "conduit-$TARGET"'
     # assert that the binary is statically linked
-    - 'file conduit-$TARGET | grep "static\(-pie\|ally\) linked"'
+    - 'ldd conduit-$TARGET' # print linking information
+    - 'file conduit-$TARGET | sed -e "/static\(-pie\|ally\) linked/!q1"' # print elf information
   cache:
     # https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci
     key: 'cargo-cache-$TARGET'
@@ -125,7 +126,8 @@ build:release:cargo:aarch64-unknown-linux-musl:
     - 'time ./cross/build.sh --locked'
     - 'mv "target/$TARGET/debug/conduit" "conduit-debug-$TARGET"'
     # assert that the binary is statically linked
-    - 'file conduit-debug-$TARGET | grep "static\(-pie\|ally\) linked"'
+    - 'ldd conduit-debug-$TARGET' # print linking information
+    - 'file conduit-debug-$TARGET | sed -e "/static\(-pie\|ally\) linked/!q1"' # print elf information
   artifacts:
     expire_in: 4 weeks
 

From 77ad4cb8f8c69b563c890494b5d203d96195253d Mon Sep 17 00:00:00 2001
From: Maxim De Clercq <maximdeclercq00@gmail.com>
Date: Sun, 23 Jan 2022 19:24:36 +0100
Subject: [PATCH 5/8] fix: use readelf for checking static compilation

---
 .gitlab-ci.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index defd66ee..9e584a2a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -49,9 +49,9 @@ variables:
     # cross-compile conduit for target
     - 'time ./cross/build.sh --locked --release'
     - 'mv "target/$TARGET/release/conduit" "conduit-$TARGET"'
-    # assert that the binary is statically linked
-    - 'ldd conduit-$TARGET' # print linking information
-    - 'file conduit-$TARGET | sed -e "/static\(-pie\|ally\) linked/!q1"' # print elf information
+    # print information about linking for debugging
+    - 'file conduit-$TARGET' # print file information
+    - 'readelf --dynamic conduit-$TARGET | sed -e "/NEEDED/q1"' # ensure statically linked
   cache:
     # https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci
     key: 'cargo-cache-$TARGET'
@@ -125,9 +125,9 @@ build:release:cargo:aarch64-unknown-linux-musl:
     # cross-compile conduit for target
     - 'time ./cross/build.sh --locked'
     - 'mv "target/$TARGET/debug/conduit" "conduit-debug-$TARGET"'
-    # assert that the binary is statically linked
-    - 'ldd conduit-debug-$TARGET' # print linking information
-    - 'file conduit-debug-$TARGET | sed -e "/static\(-pie\|ally\) linked/!q1"' # print elf information
+    # print information about linking for debugging
+    - 'file conduit-debug-$TARGET' # print file information
+    - 'readelf --dynamic conduit-debug-$TARGET | sed -e "/NEEDED/q1"' # ensure statically linked
   artifacts:
     expire_in: 4 weeks
 

From 067fcfc0e40ced1c2ed28f32b04940c6e74d6f5a Mon Sep 17 00:00:00 2001
From: Maxim De Clercq <maximdeclercq00@gmail.com>
Date: Sun, 23 Jan 2022 21:19:19 +0100
Subject: [PATCH 6/8] fix: remove trailing slash from shared path

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9e584a2a..aac773c2 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -28,7 +28,7 @@ variables:
     DOCKER_HOST: tcp://docker:2375/
     DOCKER_TLS_CERTDIR: ""
     DOCKER_DRIVER: overlay2
-    SHARED_PATH: $CI_PROJECT_DIR/shared/
+    SHARED_PATH: $CI_PROJECT_DIR/shared
     CARGO_PROFILE_RELEASE_LTO: "true"
     CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "1"
     CARGO_INCREMENTAL: "false" # https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow

From acf1585fc35e7851df9c30208f654d5c085267d6 Mon Sep 17 00:00:00 2001
From: Maxim De Clercq <maximdeclercq00@gmail.com>
Date: Mon, 24 Jan 2022 11:45:07 +0100
Subject: [PATCH 7/8] fix: make sure that libatomic is linked statically

---
 cross/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cross/build.sh b/cross/build.sh
index 4a6d4493..8f64ff87 100755
--- a/cross/build.sh
+++ b/cross/build.sh
@@ -18,7 +18,7 @@ ENV TARGET_PREFIX="/usr/local/$(echo "${TARGET:?}" | sed -e 's/armv7/arm/' -e 's
 # See https://github.com/alexcrichton/cc-rs#external-configuration-via-environment-variables for more information
 ENV RUSTFLAGS="-L\$TARGET_PREFIX/lib" CXXSTDLIB="static=stdc++"
 # Forcefully linking against libatomic, libc and libgcc is required for arm32, otherwise symbols are missing
-$([[ $TARGET =~ arm ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-lgcc -Clink-arg=-latomic -lstatic=c"')
+$([[ $TARGET =~ arm ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-static-libgcc -Clink-arg=-lgcc -lstatic=atomic -lstatic=c"')
 # Strip symbols while compiling in release mode
 $([[ $@ =~ -r ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-s"')
 

From ff167299766d41c361398243db6bf97e9d45fa65 Mon Sep 17 00:00:00 2001
From: Maxim De Clercq <maximdeclercq00@gmail.com>
Date: Tue, 25 Jan 2022 22:36:51 +0100
Subject: [PATCH 8/8] fix: correct RUSTC_WRAPPER path in cross container

---
 .gitlab-ci.yml | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index aac773c2..741b5327 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -8,6 +8,10 @@ variables:
   GIT_SUBMODULE_STRATEGY: recursive
   FF_USE_FASTZIP: 1
   CACHE_COMPRESSION_LEVEL: fastest
+  # Docker in Docker
+  DOCKER_HOST: tcp://docker:2375/
+  DOCKER_TLS_CERTDIR: ""
+  DOCKER_DRIVER: overlay2
 
 # --------------------------------------------------------------------- #
 #  Cargo: Compiling for different architectures                         #
@@ -25,9 +29,6 @@ variables:
   tags: ["docker"]
   services: ["docker:dind"]
   variables:
-    DOCKER_HOST: tcp://docker:2375/
-    DOCKER_TLS_CERTDIR: ""
-    DOCKER_DRIVER: overlay2
     SHARED_PATH: $CI_PROJECT_DIR/shared
     CARGO_PROFILE_RELEASE_LTO: "true"
     CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "1"
@@ -43,8 +44,9 @@ variables:
     - 'cp -r $CARGO_HOME/bin $SHARED_PATH/cargo'
     - 'cp -r $RUSTUP_HOME $SHARED_PATH'
     - 'export CARGO_HOME=$SHARED_PATH/cargo RUSTUP_HOME=$SHARED_PATH/rustup'
-    # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results:
-    - if [ -n "${SCCACHE_BIN_URL}" ]; then export RUSTC_WRAPPER=$SHARED_PATH/cargo/bin/sccache && curl $SCCACHE_BIN_URL --output $RUSTC_WRAPPER && chmod +x $RUSTC_WRAPPER; fi
+    # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results.
+    # The sccache binary is stored in the sysroot of the rustc installation since that directory is added to the path of the cross container.
+    - if [ -n "${SCCACHE_BIN_URL}" ]; then RUSTC_SYSROOT=$(rustc --print sysroot) && curl $SCCACHE_BIN_URL --output $RUSTC_SYSROOT/bin/sccache && chmod +x $RUSTC_SYSROOT/bin/sccache && export RUSTC_WRAPPER=sccache; fi
   script:
     # cross-compile conduit for target
     - 'time ./cross/build.sh --locked --release'
@@ -157,9 +159,6 @@ build:debug:cargo:x86_64-unknown-linux-musl:
     - "build:release:cargo:armv7-unknown-linux-musleabihf"
     - "build:release:cargo:aarch64-unknown-linux-musl"
   variables:
-    DOCKER_HOST: tcp://docker:2375/
-    DOCKER_TLS_CERTDIR: ""
-    DOCKER_DRIVER: overlay2
     PLATFORMS: "linux/arm/v6,linux/arm/v7,linux/arm64,linux/amd64"
     DOCKER_FILE: "docker/ci-binaries-packaging.Dockerfile"
   cache: