Merge branch 'ci-revamp-2022-10' into 'next'
Fix CI See merge request famedly/conduit!414
This commit is contained in:
commit
ccdaaceb33
5 changed files with 212 additions and 282 deletions
|
@ -25,4 +25,4 @@ docker-compose*
|
||||||
rustfmt.toml
|
rustfmt.toml
|
||||||
|
|
||||||
# Documentation
|
# Documentation
|
||||||
*.md
|
#*.md
|
||||||
|
|
359
.gitlab-ci.yml
359
.gitlab-ci.yml
|
@ -5,140 +5,10 @@ stages:
|
||||||
- upload artifacts
|
- upload artifacts
|
||||||
|
|
||||||
variables:
|
variables:
|
||||||
|
# Make GitLab CI go fast:
|
||||||
GIT_SUBMODULE_STRATEGY: recursive
|
GIT_SUBMODULE_STRATEGY: recursive
|
||||||
FF_USE_FASTZIP: 1
|
FF_USE_FASTZIP: 1
|
||||||
CACHE_COMPRESSION_LEVEL: fastest
|
CACHE_COMPRESSION_LEVEL: fastest
|
||||||
# Docker in Docker
|
|
||||||
DOCKER_HOST: tcp://docker:2375/
|
|
||||||
DOCKER_TLS_CERTDIR: ""
|
|
||||||
DOCKER_DRIVER: overlay2
|
|
||||||
|
|
||||||
# --------------------------------------------------------------------- #
|
|
||||||
# Cargo: Compiling for different architectures #
|
|
||||||
# --------------------------------------------------------------------- #
|
|
||||||
|
|
||||||
.build-cargo-shared-settings:
|
|
||||||
stage: "build"
|
|
||||||
needs: []
|
|
||||||
rules:
|
|
||||||
- if: '$CI_COMMIT_BRANCH == "master"'
|
|
||||||
- if: '$CI_COMMIT_BRANCH == "next"'
|
|
||||||
- if: "$CI_COMMIT_TAG"
|
|
||||||
- if: '($CI_MERGE_REQUEST_APPROVED == "true") || $BUILD_EVERYTHING' # Once MR is approved, test all builds. Or if BUILD_EVERYTHING is set.
|
|
||||||
interruptible: true
|
|
||||||
image: "registry.gitlab.com/jfowl/conduit-containers/rust-with-tools@sha256:69ab327974aef4cc0daf4273579253bf7ae5e379a6c52729b83137e4caa9d093"
|
|
||||||
tags: ["docker"]
|
|
||||||
services: ["docker:dind"]
|
|
||||||
variables:
|
|
||||||
SHARED_PATH: $CI_PROJECT_DIR/shared
|
|
||||||
CARGO_PROFILE_RELEASE_LTO: "true"
|
|
||||||
CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "1"
|
|
||||||
CARGO_INCREMENTAL: "false" # https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow
|
|
||||||
before_script:
|
|
||||||
- 'echo "Building for target $TARGET"'
|
|
||||||
- "rustup show && rustc --version && cargo --version" # Print version info for debugging
|
|
||||||
# fix cargo and rustup mounts from this container (https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41227)
|
|
||||||
- "mkdir -p $SHARED_PATH/cargo"
|
|
||||||
- "cp -r $CARGO_HOME/bin $SHARED_PATH/cargo"
|
|
||||||
- "cp -r $RUSTUP_HOME $SHARED_PATH"
|
|
||||||
- "export CARGO_HOME=$SHARED_PATH/cargo RUSTUP_HOME=$SHARED_PATH/rustup"
|
|
||||||
# If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results.
|
|
||||||
- if [ -n "${SCCACHE_ENDPOINT}" ]; then export RUSTC_WRAPPER=/sccache; fi
|
|
||||||
script:
|
|
||||||
# cross-compile conduit for target
|
|
||||||
- 'time cross build --target="$TARGET" --locked --release'
|
|
||||||
- 'mv "target/$TARGET/release/conduit" "conduit-$TARGET"'
|
|
||||||
# print information about linking for debugging
|
|
||||||
- "file conduit-$TARGET" # print file information
|
|
||||||
- 'readelf --dynamic conduit-$TARGET | sed -e "/NEEDED/q1"' # ensure statically linked
|
|
||||||
cache:
|
|
||||||
# https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci
|
|
||||||
key: "cargo-cache-$TARGET"
|
|
||||||
paths:
|
|
||||||
- $SHARED_PATH/cargo/registry/index
|
|
||||||
- $SHARED_PATH/cargo/registry/cache
|
|
||||||
- $SHARED_PATH/cargo/git/db
|
|
||||||
artifacts:
|
|
||||||
expire_in: never
|
|
||||||
|
|
||||||
build:release:cargo:x86_64-unknown-linux-musl-with-debug:
|
|
||||||
extends: .build-cargo-shared-settings
|
|
||||||
variables:
|
|
||||||
CARGO_PROFILE_RELEASE_DEBUG: 2 # Enable debug info for flamegraph profiling
|
|
||||||
TARGET: "x86_64-unknown-linux-musl"
|
|
||||||
after_script:
|
|
||||||
- "mv ./conduit-x86_64-unknown-linux-musl ./conduit-x86_64-unknown-linux-musl-with-debug"
|
|
||||||
artifacts:
|
|
||||||
name: "conduit-x86_64-unknown-linux-musl-with-debug"
|
|
||||||
paths:
|
|
||||||
- "conduit-x86_64-unknown-linux-musl-with-debug"
|
|
||||||
expose_as: "Conduit for x86_64-unknown-linux-musl-with-debug"
|
|
||||||
|
|
||||||
build:release:cargo:x86_64-unknown-linux-musl:
|
|
||||||
extends: .build-cargo-shared-settings
|
|
||||||
variables:
|
|
||||||
TARGET: "x86_64-unknown-linux-musl"
|
|
||||||
artifacts:
|
|
||||||
name: "conduit-x86_64-unknown-linux-musl"
|
|
||||||
paths:
|
|
||||||
- "conduit-x86_64-unknown-linux-musl"
|
|
||||||
expose_as: "Conduit for x86_64-unknown-linux-musl"
|
|
||||||
|
|
||||||
build:release:cargo:arm-unknown-linux-musleabihf:
|
|
||||||
extends: .build-cargo-shared-settings
|
|
||||||
variables:
|
|
||||||
TARGET: "arm-unknown-linux-musleabihf"
|
|
||||||
artifacts:
|
|
||||||
name: "conduit-arm-unknown-linux-musleabihf"
|
|
||||||
paths:
|
|
||||||
- "conduit-arm-unknown-linux-musleabihf"
|
|
||||||
expose_as: "Conduit for arm-unknown-linux-musleabihf"
|
|
||||||
|
|
||||||
build:release:cargo:armv7-unknown-linux-musleabihf:
|
|
||||||
extends: .build-cargo-shared-settings
|
|
||||||
variables:
|
|
||||||
TARGET: "armv7-unknown-linux-musleabihf"
|
|
||||||
artifacts:
|
|
||||||
name: "conduit-armv7-unknown-linux-musleabihf"
|
|
||||||
paths:
|
|
||||||
- "conduit-armv7-unknown-linux-musleabihf"
|
|
||||||
expose_as: "Conduit for armv7-unknown-linux-musleabihf"
|
|
||||||
|
|
||||||
build:release:cargo:aarch64-unknown-linux-musl:
|
|
||||||
extends: .build-cargo-shared-settings
|
|
||||||
variables:
|
|
||||||
TARGET: "aarch64-unknown-linux-musl"
|
|
||||||
artifacts:
|
|
||||||
name: "conduit-aarch64-unknown-linux-musl"
|
|
||||||
paths:
|
|
||||||
- "conduit-aarch64-unknown-linux-musl"
|
|
||||||
expose_as: "Conduit for aarch64-unknown-linux-musl"
|
|
||||||
|
|
||||||
.cargo-debug-shared-settings:
|
|
||||||
extends: ".build-cargo-shared-settings"
|
|
||||||
rules:
|
|
||||||
- when: "always"
|
|
||||||
cache:
|
|
||||||
key: "build_cache--$TARGET--$CI_COMMIT_BRANCH--debug"
|
|
||||||
script:
|
|
||||||
# cross-compile conduit for target
|
|
||||||
- 'time time cross build --target="$TARGET" --locked'
|
|
||||||
- 'mv "target/$TARGET/debug/conduit" "conduit-debug-$TARGET"'
|
|
||||||
# print information about linking for debugging
|
|
||||||
- "file conduit-debug-$TARGET" # print file information
|
|
||||||
- 'readelf --dynamic conduit-debug-$TARGET | sed -e "/NEEDED/q1"' # ensure statically linked
|
|
||||||
artifacts:
|
|
||||||
expire_in: 4 weeks
|
|
||||||
|
|
||||||
build:debug:cargo:x86_64-unknown-linux-musl:
|
|
||||||
extends: ".cargo-debug-shared-settings"
|
|
||||||
variables:
|
|
||||||
TARGET: "x86_64-unknown-linux-musl"
|
|
||||||
artifacts:
|
|
||||||
name: "conduit-debug-x86_64-unknown-linux-musl"
|
|
||||||
paths:
|
|
||||||
- "conduit-debug-x86_64-unknown-linux-musl"
|
|
||||||
expose_as: "Conduit DEBUG for x86_64-unknown-linux-musl"
|
|
||||||
|
|
||||||
# --------------------------------------------------------------------- #
|
# --------------------------------------------------------------------- #
|
||||||
# Create and publish docker image #
|
# Create and publish docker image #
|
||||||
|
@ -146,98 +16,106 @@ build:debug:cargo:x86_64-unknown-linux-musl:
|
||||||
|
|
||||||
.docker-shared-settings:
|
.docker-shared-settings:
|
||||||
stage: "build docker image"
|
stage: "build docker image"
|
||||||
image: jdrouet/docker-with-buildx:stable
|
image: jdrouet/docker-with-buildx:20.10.21-0.9.1
|
||||||
|
needs: []
|
||||||
tags: ["docker"]
|
tags: ["docker"]
|
||||||
|
variables:
|
||||||
|
# Docker in Docker:
|
||||||
|
DOCKER_HOST: tcp://docker:2375/
|
||||||
|
DOCKER_TLS_CERTDIR: ""
|
||||||
|
DOCKER_DRIVER: overlay2
|
||||||
services:
|
services:
|
||||||
- docker:dind
|
- docker:dind
|
||||||
needs:
|
|
||||||
- "build:release:cargo:x86_64-unknown-linux-musl"
|
|
||||||
- "build:release:cargo:arm-unknown-linux-musleabihf"
|
|
||||||
- "build:release:cargo:armv7-unknown-linux-musleabihf"
|
|
||||||
- "build:release:cargo:aarch64-unknown-linux-musl"
|
|
||||||
variables:
|
|
||||||
PLATFORMS: "linux/arm/v6,linux/arm/v7,linux/arm64,linux/amd64"
|
|
||||||
DOCKER_FILE: "docker/ci-binaries-packaging.Dockerfile"
|
|
||||||
cache:
|
|
||||||
paths:
|
|
||||||
- docker_cache
|
|
||||||
key: "$CI_JOB_NAME"
|
|
||||||
before_script:
|
|
||||||
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
|
|
||||||
# Only log in to Dockerhub if the credentials are given:
|
|
||||||
- if [ -n "${DOCKER_HUB}" ]; then docker login -u "$DOCKER_HUB_USER" -p "$DOCKER_HUB_PASSWORD" "$DOCKER_HUB"; fi
|
|
||||||
script:
|
script:
|
||||||
# Prepare buildx to build multiarch stuff:
|
- apk add openssh-client
|
||||||
- docker context create 'ci-context'
|
- eval $(ssh-agent -s)
|
||||||
- docker buildx create --name 'multiarch-builder' --use 'ci-context'
|
- mkdir -p ~/.ssh && chmod 700 ~/.ssh
|
||||||
# Copy binaries to their docker arch path
|
- printf "Host *\n\tStrictHostKeyChecking no\n\n" >> ~/.ssh/config
|
||||||
- mkdir -p linux/ && mv ./conduit-x86_64-unknown-linux-musl linux/amd64
|
- sh .gitlab/setup-buildx-remote-builders.sh
|
||||||
- mkdir -p linux/arm/ && mv ./conduit-arm-unknown-linux-musleabihf linux/arm/v6
|
# Authorize against this project's own image registry:
|
||||||
- mkdir -p linux/arm/ && mv ./conduit-armv7-unknown-linux-musleabihf linux/arm/v7
|
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
|
||||||
- mv ./conduit-aarch64-unknown-linux-musl linux/arm64
|
# Build multiplatform image and push to temporary tag:
|
||||||
- 'export CREATED=$(date -u +''%Y-%m-%dT%H:%M:%SZ'') && echo "Docker image creation date: $CREATED"'
|
|
||||||
# Build and push image:
|
|
||||||
- >
|
- >
|
||||||
docker buildx build
|
docker buildx build
|
||||||
|
--platform "linux/arm/v7,linux/arm64,linux/amd64"
|
||||||
--pull
|
--pull
|
||||||
|
--tag "$CI_REGISTRY_IMAGE/temporary-ci-images:$CI_JOB_ID"
|
||||||
--push
|
--push
|
||||||
--cache-from=type=local,src=$CI_PROJECT_DIR/docker_cache
|
--file "Dockerfile" .
|
||||||
--cache-to=type=local,dest=$CI_PROJECT_DIR/docker_cache
|
# Build multiplatform image to deb stage and extract their .deb files:
|
||||||
--build-arg CREATED=$CREATED
|
- >
|
||||||
--build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)
|
docker buildx build
|
||||||
--build-arg "GIT_REF=$CI_COMMIT_SHORT_SHA"
|
--platform "linux/arm/v7,linux/arm64,linux/amd64"
|
||||||
--platform "$PLATFORMS"
|
--target "packager-result"
|
||||||
--tag "$TAG"
|
--output="type=local,dest=/tmp/build-output"
|
||||||
--tag "$TAG-alpine"
|
--file "Dockerfile" .
|
||||||
--tag "$TAG-commit-$CI_COMMIT_SHORT_SHA"
|
# Build multiplatform image to binary stage and extract their binaries:
|
||||||
--file "$DOCKER_FILE" .
|
- >
|
||||||
|
docker buildx build
|
||||||
|
--platform "linux/arm/v7,linux/arm64,linux/amd64"
|
||||||
|
--target "builder-result"
|
||||||
|
--output="type=local,dest=/tmp/build-output"
|
||||||
|
--file "Dockerfile" .
|
||||||
|
# Copy to GitLab container registry:
|
||||||
|
- >
|
||||||
|
docker buildx imagetools create
|
||||||
|
--tag "$CI_REGISTRY_IMAGE/$TAG"
|
||||||
|
--tag "$CI_REGISTRY_IMAGE/$TAG-bullseye"
|
||||||
|
--tag "$CI_REGISTRY_IMAGE/$TAG-commit-$CI_COMMIT_SHORT_SHA"
|
||||||
|
"$CI_REGISTRY_IMAGE/temporary-ci-images:$CI_JOB_ID"
|
||||||
|
# if DockerHub credentials exist, also copy to dockerhub:
|
||||||
|
- if [ -n "${DOCKER_HUB}" ]; then docker login -u "$DOCKER_HUB_USER" -p "$DOCKER_HUB_PASSWORD" "$DOCKER_HUB"; fi
|
||||||
|
- >
|
||||||
|
if [ -n "${DOCKER_HUB}" ]; then
|
||||||
|
docker buildx imagetools create
|
||||||
|
--tag "$DOCKER_HUB_IMAGE/$TAG"
|
||||||
|
--tag "$DOCKER_HUB_IMAGE/$TAG-bullseye"
|
||||||
|
--tag "$DOCKER_HUB_IMAGE/$TAG-commit-$CI_COMMIT_SHORT_SHA"
|
||||||
|
"$CI_REGISTRY_IMAGE/temporary-ci-images:$CI_JOB_ID"
|
||||||
|
; fi
|
||||||
|
- mv /tmp/build-output ./
|
||||||
|
artifacts:
|
||||||
|
paths:
|
||||||
|
- "./build-output/"
|
||||||
|
|
||||||
docker:next:gitlab:
|
docker:next:
|
||||||
extends: .docker-shared-settings
|
extends: .docker-shared-settings
|
||||||
rules:
|
rules:
|
||||||
- if: '$CI_COMMIT_BRANCH == "next"'
|
- if: '$BUILD_SERVER_SSH_PRIVATE_KEY && $CI_COMMIT_BRANCH == "next"'
|
||||||
variables:
|
variables:
|
||||||
TAG: "$CI_REGISTRY_IMAGE/matrix-conduit:next"
|
TAG: "matrix-conduit:next"
|
||||||
|
|
||||||
docker:next:dockerhub:
|
docker:master:
|
||||||
extends: .docker-shared-settings
|
extends: .docker-shared-settings
|
||||||
rules:
|
rules:
|
||||||
- if: '$CI_COMMIT_BRANCH == "next" && $DOCKER_HUB'
|
- if: '$BUILD_SERVER_SSH_PRIVATE_KEY && $CI_COMMIT_BRANCH == "master"'
|
||||||
variables:
|
variables:
|
||||||
TAG: "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next"
|
TAG: "matrix-conduit:latest"
|
||||||
|
|
||||||
docker:master:gitlab:
|
docker:tags:
|
||||||
extends: .docker-shared-settings
|
extends: .docker-shared-settings
|
||||||
rules:
|
rules:
|
||||||
- if: '$CI_COMMIT_BRANCH == "master"'
|
- if: "$BUILD_SERVER_SSH_PRIVATE_KEY && $CI_COMMIT_TAG"
|
||||||
variables:
|
variables:
|
||||||
TAG: "$CI_REGISTRY_IMAGE/matrix-conduit:latest"
|
TAG: "matrix-conduit:$CI_COMMIT_TAG"
|
||||||
|
|
||||||
docker:master:dockerhub:
|
|
||||||
extends: .docker-shared-settings
|
|
||||||
rules:
|
|
||||||
- if: '$CI_COMMIT_BRANCH == "master" && $DOCKER_HUB'
|
|
||||||
variables:
|
|
||||||
TAG: "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:latest"
|
|
||||||
|
|
||||||
docker:tags:gitlab:
|
|
||||||
extends: .docker-shared-settings
|
|
||||||
rules:
|
|
||||||
- if: "$CI_COMMIT_TAG"
|
|
||||||
variables:
|
|
||||||
TAG: "$CI_REGISTRY_IMAGE/matrix-conduit:$CI_COMMIT_TAG"
|
|
||||||
|
|
||||||
docker:tags:dockerhub:
|
|
||||||
extends: .docker-shared-settings
|
|
||||||
rules:
|
|
||||||
- if: "$CI_COMMIT_TAG && $DOCKER_HUB"
|
|
||||||
variables:
|
|
||||||
TAG: "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:$CI_COMMIT_TAG"
|
|
||||||
|
|
||||||
# --------------------------------------------------------------------- #
|
# --------------------------------------------------------------------- #
|
||||||
# Run tests #
|
# Run tests #
|
||||||
# --------------------------------------------------------------------- #
|
# --------------------------------------------------------------------- #
|
||||||
|
|
||||||
|
cargo check:
|
||||||
|
stage: test
|
||||||
|
image: docker.io/rust:1.64.0-bullseye
|
||||||
|
needs: []
|
||||||
|
interruptible: true
|
||||||
|
before_script:
|
||||||
|
- "rustup show && rustc --version && cargo --version" # Print version info for debugging
|
||||||
|
- apt-get update && apt-get -y --no-install-recommends install libclang-dev # dependency for rocksdb
|
||||||
|
script:
|
||||||
|
- cargo check
|
||||||
|
|
||||||
|
|
||||||
.test-shared-settings:
|
.test-shared-settings:
|
||||||
stage: "test"
|
stage: "test"
|
||||||
needs: []
|
needs: []
|
||||||
|
@ -250,8 +128,7 @@ docker:tags:dockerhub:
|
||||||
test:cargo:
|
test:cargo:
|
||||||
extends: .test-shared-settings
|
extends: .test-shared-settings
|
||||||
before_script:
|
before_script:
|
||||||
# If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results:
|
- apt-get update && apt-get -y --no-install-recommends install libclang-dev # dependency for rocksdb
|
||||||
- if [ -n "${SCCACHE_ENDPOINT}" ]; then export RUSTC_WRAPPER=/usr/local/cargo/bin/sccache; fi
|
|
||||||
script:
|
script:
|
||||||
- rustc --version && cargo --version # Print version info for debugging
|
- rustc --version && cargo --version # Print version info for debugging
|
||||||
- "cargo test --color always --workspace --verbose --locked --no-fail-fast -- -Z unstable-options --format json | gitlab-report -p test > $CI_PROJECT_DIR/report.xml"
|
- "cargo test --color always --workspace --verbose --locked --no-fail-fast -- -Z unstable-options --format json | gitlab-report -p test > $CI_PROJECT_DIR/report.xml"
|
||||||
|
@ -260,14 +137,12 @@ test:cargo:
|
||||||
reports:
|
reports:
|
||||||
junit: report.xml
|
junit: report.xml
|
||||||
|
|
||||||
|
|
||||||
test:clippy:
|
test:clippy:
|
||||||
extends: .test-shared-settings
|
extends: .test-shared-settings
|
||||||
allow_failure: true
|
allow_failure: true
|
||||||
before_script:
|
before_script:
|
||||||
- rustup component add clippy
|
- rustup component add clippy
|
||||||
# If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results:
|
- apt-get update && apt-get -y --no-install-recommends install libclang-dev # dependency for rocksdb
|
||||||
- if [ -n "${SCCACHE_ENDPOINT}" ]; then export RUSTC_WRAPPER=/usr/local/cargo/bin/sccache; fi
|
|
||||||
script:
|
script:
|
||||||
- rustc --version && cargo --version # Print version info for debugging
|
- rustc --version && cargo --version # Print version info for debugging
|
||||||
- "cargo clippy --color always --verbose --message-format=json | gitlab-report -p clippy > $CI_PROJECT_DIR/gl-code-quality-report.json"
|
- "cargo clippy --color always --verbose --message-format=json | gitlab-report -p clippy > $CI_PROJECT_DIR/gl-code-quality-report.json"
|
||||||
|
@ -294,38 +169,6 @@ test:audit:
|
||||||
reports:
|
reports:
|
||||||
sast: gl-sast-report.json
|
sast: gl-sast-report.json
|
||||||
|
|
||||||
test:sytest:
|
|
||||||
stage: "test"
|
|
||||||
allow_failure: true
|
|
||||||
needs:
|
|
||||||
- "build:debug:cargo:x86_64-unknown-linux-musl"
|
|
||||||
image:
|
|
||||||
name: "valkum/sytest-conduit:latest"
|
|
||||||
entrypoint: [""]
|
|
||||||
tags: ["docker"]
|
|
||||||
variables:
|
|
||||||
PLUGINS: "https://github.com/valkum/sytest_conduit/archive/master.tar.gz"
|
|
||||||
interruptible: true
|
|
||||||
before_script:
|
|
||||||
- "mkdir -p /app"
|
|
||||||
- "cp ./conduit-debug-x86_64-unknown-linux-musl /app/conduit"
|
|
||||||
- "chmod +x /app/conduit"
|
|
||||||
- "rm -rf /src && ln -s $CI_PROJECT_DIR/ /src"
|
|
||||||
- "mkdir -p /work/server-0/database/ && mkdir -p /work/server-1/database/ && mkdir -p /work/server-2/database/"
|
|
||||||
- "cd /"
|
|
||||||
script:
|
|
||||||
- "SYTEST_EXIT_CODE=0"
|
|
||||||
- "/bootstrap.sh conduit || SYTEST_EXIT_CODE=1"
|
|
||||||
- 'perl /sytest/tap-to-junit-xml.pl --puretap --input /logs/results.tap --output $CI_PROJECT_DIR/sytest.xml "Sytest" && cp /logs/results.tap $CI_PROJECT_DIR/results.tap'
|
|
||||||
- "exit $SYTEST_EXIT_CODE"
|
|
||||||
artifacts:
|
|
||||||
when: always
|
|
||||||
paths:
|
|
||||||
- "$CI_PROJECT_DIR/sytest.xml"
|
|
||||||
- "$CI_PROJECT_DIR/results.tap"
|
|
||||||
reports:
|
|
||||||
junit: "$CI_PROJECT_DIR/sytest.xml"
|
|
||||||
|
|
||||||
test:dockerlint:
|
test:dockerlint:
|
||||||
stage: "test"
|
stage: "test"
|
||||||
needs: []
|
needs: []
|
||||||
|
@ -338,14 +181,12 @@ test:dockerlint:
|
||||||
hadolint
|
hadolint
|
||||||
--no-fail --verbose
|
--no-fail --verbose
|
||||||
./Dockerfile
|
./Dockerfile
|
||||||
./docker/ci-binaries-packaging.Dockerfile
|
|
||||||
# Then output the results into a json for GitLab to pretty-print this in the MR:
|
# Then output the results into a json for GitLab to pretty-print this in the MR:
|
||||||
- >
|
- >
|
||||||
hadolint
|
hadolint
|
||||||
--format gitlab_codeclimate
|
--format gitlab_codeclimate
|
||||||
--failure-threshold error
|
--failure-threshold error
|
||||||
./Dockerfile
|
./Dockerfile > dockerlint.json
|
||||||
./docker/ci-binaries-packaging.Dockerfile > dockerlint.json
|
|
||||||
artifacts:
|
artifacts:
|
||||||
when: always
|
when: always
|
||||||
reports:
|
reports:
|
||||||
|
@ -365,28 +206,26 @@ test:dockerlint:
|
||||||
# Store binaries as package so they have download urls #
|
# Store binaries as package so they have download urls #
|
||||||
# --------------------------------------------------------------------- #
|
# --------------------------------------------------------------------- #
|
||||||
|
|
||||||
publish:package:
|
# DISABLED FOR NOW, NEEDS TO BE FIXED AT A LATER TIME:
|
||||||
stage: "upload artifacts"
|
|
||||||
needs:
|
#publish:package:
|
||||||
- "build:release:cargo:x86_64-unknown-linux-musl"
|
# stage: "upload artifacts"
|
||||||
- "build:release:cargo:arm-unknown-linux-musleabihf"
|
# needs:
|
||||||
- "build:release:cargo:armv7-unknown-linux-musleabihf"
|
# - "docker:tags"
|
||||||
- "build:release:cargo:aarch64-unknown-linux-musl"
|
# rules:
|
||||||
# - "build:cargo-deb:x86_64-unknown-linux-gnu"
|
# - if: "$CI_COMMIT_TAG"
|
||||||
rules:
|
# image: curlimages/curl:latest
|
||||||
- if: '$CI_COMMIT_BRANCH == "master"'
|
# tags: ["docker"]
|
||||||
- if: '$CI_COMMIT_BRANCH == "next"'
|
# variables:
|
||||||
- if: "$CI_COMMIT_TAG"
|
# GIT_STRATEGY: "none" # Don't need a clean copy of the code, we just operate on artifacts
|
||||||
image: curlimages/curl:latest
|
# script:
|
||||||
tags: ["docker"]
|
# - 'BASE_URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/conduit-${CI_COMMIT_REF_SLUG}/build-${CI_PIPELINE_ID}"'
|
||||||
variables:
|
# - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file build-output/linux_amd64/conduit "${BASE_URL}/conduit-x86_64-unknown-linux-gnu"'
|
||||||
GIT_STRATEGY: "none" # Don't need a clean copy of the code, we just operate on artifacts
|
# - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file build-output/linux_arm_v7/conduit "${BASE_URL}/conduit-armv7-unknown-linux-gnu"'
|
||||||
script:
|
# - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file build-output/linux_arm64/conduit "${BASE_URL}/conduit-aarch64-unknown-linux-gnu"'
|
||||||
- 'BASE_URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/conduit-${CI_COMMIT_REF_SLUG}/build-${CI_PIPELINE_ID}"'
|
# - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file build-output/linux_amd64/conduit.deb "${BASE_URL}/conduit-x86_64-unknown-linux-gnu.deb"'
|
||||||
- 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-x86_64-unknown-linux-musl "${BASE_URL}/conduit-x86_64-unknown-linux-musl"'
|
# - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file build-output/linux_arm_v7/conduit.deb "${BASE_URL}/conduit-armv7-unknown-linux-gnu.deb"'
|
||||||
- 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-arm-unknown-linux-musleabihf "${BASE_URL}/conduit-arm-unknown-linux-musleabihf"'
|
# - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file build-output/linux_arm64/conduit.deb "${BASE_URL}/conduit-aarch64-unknown-linux-gnu.deb"'
|
||||||
- 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-armv7-unknown-linux-musleabihf "${BASE_URL}/conduit-armv7-unknown-linux-musleabihf"'
|
|
||||||
- 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-aarch64-unknown-linux-musl "${BASE_URL}/conduit-aarch64-unknown-linux-musl"'
|
|
||||||
|
|
||||||
# Avoid duplicate pipelines
|
# Avoid duplicate pipelines
|
||||||
# See: https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines
|
# See: https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines
|
||||||
|
|
37
.gitlab/setup-buildx-remote-builders.sh
Normal file
37
.gitlab/setup-buildx-remote-builders.sh
Normal file
|
@ -0,0 +1,37 @@
|
||||||
|
#!/bin/sh
|
||||||
|
set -eux
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------- #
|
||||||
|
# #
|
||||||
|
# Configures docker buildx to use a remote server for arm building. #
|
||||||
|
# Expects $SSH_PRIVATE_KEY to be a valid ssh ed25519 private key with #
|
||||||
|
# access to the server $ARM_SERVER_USER@$ARM_SERVER_IP #
|
||||||
|
# #
|
||||||
|
# This is expected to only be used in the official CI/CD pipeline! #
|
||||||
|
# #
|
||||||
|
# Requirements: openssh-client, docker buildx #
|
||||||
|
# Inspired by: https://depot.dev/blog/building-arm-containers #
|
||||||
|
# #
|
||||||
|
# --------------------------------------------------------------------- #
|
||||||
|
|
||||||
|
cat "$BUILD_SERVER_SSH_PRIVATE_KEY" | ssh-add -
|
||||||
|
|
||||||
|
# Test server connections:
|
||||||
|
ssh "$ARM_SERVER_USER@$ARM_SERVER_IP" "uname -a"
|
||||||
|
ssh "$AMD_SERVER_USER@$AMD_SERVER_IP" "uname -a"
|
||||||
|
|
||||||
|
# Connect remote arm64 server for all arm builds:
|
||||||
|
docker buildx create \
|
||||||
|
--name "multi" \
|
||||||
|
--driver "docker-container" \
|
||||||
|
--platform "linux/arm64,linux/arm/v7" \
|
||||||
|
"ssh://$ARM_SERVER_USER@$ARM_SERVER_IP"
|
||||||
|
|
||||||
|
# Connect remote amd64 server for adm64 builds:
|
||||||
|
docker buildx create --append \
|
||||||
|
--name "multi" \
|
||||||
|
--driver "docker-container" \
|
||||||
|
--platform "linux/amd64" \
|
||||||
|
"ssh://$AMD_SERVER_USER@$AMD_SERVER_IP"
|
||||||
|
|
||||||
|
docker buildx use multi
|
40
DEPLOY.md
40
DEPLOY.md
|
@ -12,21 +12,27 @@ only offer Linux binaries.
|
||||||
|
|
||||||
You may simply download the binary that fits your machine. Run `uname -m` to see what you need. Now copy the right url:
|
You may simply download the binary that fits your machine. Run `uname -m` to see what you need. Now copy the right url:
|
||||||
|
|
||||||
| CPU Architecture | Download stable version | Download development version |
|
| CPU Architecture | Download stable version | Download development version |
|
||||||
| ------------------------------------------- | ------------------------------ | ---------------------------- |
|
| ------------------------------------------- | --------------------------------------------------------------- | ----------------------------------------------------------- |
|
||||||
| x84_64 / amd64 (Most servers and computers) | [Download][x84_64-musl-master] | [Download][x84_64-musl-next] |
|
| x84_64 / amd64 (Most servers and computers) | [Binary][x84_64-glibc-master] / [.deb][x84_64-glibc-master-deb] | [Binary][x84_64-glibc-next] / [.deb][x84_64-glibc-next-deb] |
|
||||||
| armv6 | [Download][armv6-musl-master] | [Download][armv6-musl-next] |
|
| armv7 (e.g. Raspberry Pi by default) | [Binary][armv7-glibc-master] / [.deb][armv7-glibc-master-deb] | [Binary][armv7-glibc-next] / [.deb][armv7-glibc-next-deb] |
|
||||||
| armv7 (e.g. Raspberry Pi by default) | [Download][armv7-musl-master] | [Download][armv7-musl-next] |
|
| armv8 / aarch64 | [Binary][armv8-glibc-master] / [.deb][armv8-glibc-master-deb] | [Binary][armv8-glibc-next] / [.deb][armv8-glibc-next-deb] |
|
||||||
| armv8 / aarch64 | [Download][armv8-musl-master] | [Download][armv8-musl-next] |
|
|
||||||
|
|
||||||
[x84_64-musl-master]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-x86_64-unknown-linux-musl?job=build:release:cargo:x86_64-unknown-linux-musl
|
These builds were created on and linked against the glibc version shipped with Debian bullseye.
|
||||||
[armv6-musl-master]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-arm-unknown-linux-musleabihf?job=build:release:cargo:arm-unknown-linux-musleabihf
|
If you use a system with an older glibc version, you might need to compile Conduit yourself.
|
||||||
[armv7-musl-master]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-armv7-unknown-linux-musleabihf?job=build:release:cargo:armv7-unknown-linux-musleabihf
|
|
||||||
[armv8-musl-master]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-aarch64-unknown-linux-musl?job=build:release:cargo:aarch64-unknown-linux-musl
|
[x84_64-glibc-master]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/build-output/linux_amd64/conduit?job=docker:master
|
||||||
[x84_64-musl-next]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/next/raw/conduit-x86_64-unknown-linux-musl?job=build:release:cargo:x86_64-unknown-linux-musl
|
[armv7-glibc-master]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/build-output/linux_arm_v7/conduit?job=docker:master
|
||||||
[armv6-musl-next]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/next/raw/conduit-arm-unknown-linux-musleabihf?job=build:release:cargo:arm-unknown-linux-musleabihf
|
[armv8-glibc-master]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/build-output/linux_arm64/conduit?job=docker:master
|
||||||
[armv7-musl-next]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/next/raw/conduit-armv7-unknown-linux-musleabihf?job=build:release:cargo:armv7-unknown-linux-musleabihf
|
[x84_64-glibc-next]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/next/raw/build-output/linux_amd64/conduit?job=docker:next
|
||||||
[armv8-musl-next]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/next/raw/conduit-aarch64-unknown-linux-musl?job=build:release:cargo:aarch64-unknown-linux-musl
|
[armv7-glibc-next]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/next/raw/build-output/linux_arm_v7/conduit?job=docker:next
|
||||||
|
[armv8-glibc-next]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/next/raw/build-output/linux_arm64/conduit?job=docker:next
|
||||||
|
[x84_64-glibc-master-deb]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/build-output/linux_amd64/conduit.deb?job=docker:master
|
||||||
|
[armv7-glibc-master-deb]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/build-output/linux_arm_v7/conduit.deb?job=docker:master
|
||||||
|
[armv8-glibc-master-deb]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/build-output/linux_arm64/conduit.deb?job=docker:master
|
||||||
|
[x84_64-glibc-next-deb]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/next/raw/build-output/linux_amd64/conduit.deb?job=docker:next
|
||||||
|
[armv7-glibc-next-deb]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/next/raw/build-output/linux_arm_v7/conduit.deb?job=docker:next
|
||||||
|
[armv8-glibc-next-deb]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/next/raw/build-output/linux_arm64/conduit.deb?job=docker:next
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
$ sudo wget -O /usr/local/bin/matrix-conduit <url>
|
$ sudo wget -O /usr/local/bin/matrix-conduit <url>
|
||||||
|
@ -43,7 +49,6 @@ $ sudo apt install libclang-dev build-essential
|
||||||
$ cargo build --release
|
$ cargo build --release
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
||||||
If you want to cross compile Conduit to another architecture, read the [Cross-Compile Guide](cross/README.md).
|
If you want to cross compile Conduit to another architecture, read the [Cross-Compile Guide](cross/README.md).
|
||||||
|
|
||||||
## Adding a Conduit user
|
## Adding a Conduit user
|
||||||
|
@ -189,18 +194,21 @@ $ sudo systemctl reload apache2
|
||||||
```
|
```
|
||||||
|
|
||||||
### Caddy
|
### Caddy
|
||||||
|
|
||||||
Create `/etc/caddy/conf.d/conduit_caddyfile` and enter this (substitute for your server name).
|
Create `/etc/caddy/conf.d/conduit_caddyfile` and enter this (substitute for your server name).
|
||||||
|
|
||||||
```caddy
|
```caddy
|
||||||
your.server.name, your.server.name:8448 {
|
your.server.name, your.server.name:8448 {
|
||||||
reverse_proxy /_matrix/* 127.0.0.1:6167
|
reverse_proxy /_matrix/* 127.0.0.1:6167
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
That's it! Just start or enable the service and you're set.
|
That's it! Just start or enable the service and you're set.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
$ sudo systemctl enable caddy
|
$ sudo systemctl enable caddy
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
||||||
### Nginx
|
### Nginx
|
||||||
|
|
||||||
If you use Nginx and not Apache, add the following server section inside the http section of `/etc/nginx/nginx.conf`
|
If you use Nginx and not Apache, add the following server section inside the http section of `/etc/nginx/nginx.conf`
|
||||||
|
|
56
Dockerfile
56
Dockerfile
|
@ -1,5 +1,5 @@
|
||||||
# syntax=docker/dockerfile:1
|
# syntax=docker/dockerfile:1
|
||||||
FROM docker.io/rust:1.63-bullseye AS builder
|
FROM docker.io/rust:1.64-bullseye AS builder
|
||||||
WORKDIR /usr/src/conduit
|
WORKDIR /usr/src/conduit
|
||||||
|
|
||||||
# Install required packages to build Conduit and it's dependencies
|
# Install required packages to build Conduit and it's dependencies
|
||||||
|
@ -27,6 +27,49 @@ COPY src src
|
||||||
# Builds conduit and places the binary at /usr/src/conduit/target/release/conduit
|
# Builds conduit and places the binary at /usr/src/conduit/target/release/conduit
|
||||||
RUN touch src/main.rs && touch src/lib.rs && cargo build --release
|
RUN touch src/main.rs && touch src/lib.rs && cargo build --release
|
||||||
|
|
||||||
|
|
||||||
|
# ONLY USEFUL FOR CI: target stage to extract build artifacts
|
||||||
|
FROM scratch AS builder-result
|
||||||
|
COPY --from=builder /usr/src/conduit/target/release/conduit /conduit
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------------------------------------------
|
||||||
|
# Build cargo-deb, a tool to package up rust binaries into .deb packages for Debian/Ubuntu based systems:
|
||||||
|
# ---------------------------------------------------------------------------------------------------------------
|
||||||
|
FROM docker.io/rust:1.64-bullseye AS build-cargo-deb
|
||||||
|
|
||||||
|
RUN apt-get update && \
|
||||||
|
apt-get install -y --no-install-recommends \
|
||||||
|
dpkg \
|
||||||
|
dpkg-dev \
|
||||||
|
liblzma-dev
|
||||||
|
|
||||||
|
RUN cargo install cargo-deb
|
||||||
|
# => binary is in /usr/local/cargo/bin/cargo-deb
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------------------------------------------
|
||||||
|
# Package conduit build-result into a .deb package:
|
||||||
|
# ---------------------------------------------------------------------------------------------------------------
|
||||||
|
FROM builder AS packager
|
||||||
|
WORKDIR /usr/src/conduit
|
||||||
|
|
||||||
|
COPY ./LICENSE ./LICENSE
|
||||||
|
COPY ./README.md ./README.md
|
||||||
|
COPY debian/README.Debian ./debian/
|
||||||
|
COPY --from=build-cargo-deb /usr/local/cargo/bin/cargo-deb /usr/local/cargo/bin/cargo-deb
|
||||||
|
|
||||||
|
# --no-build makes cargo-deb reuse already compiled project
|
||||||
|
RUN cargo deb --no-build
|
||||||
|
# => Package is in /usr/src/conduit/target/debian/<project_name>_<version>_<arch>.deb
|
||||||
|
|
||||||
|
|
||||||
|
# ONLY USEFUL FOR CI: target stage to extract build artifacts
|
||||||
|
FROM scratch AS packager-result
|
||||||
|
COPY --from=packager /usr/src/conduit/target/debian/*.deb /conduit.deb
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------------------------------------------
|
||||||
# Stuff below this line actually ends up in the resulting docker image
|
# Stuff below this line actually ends up in the resulting docker image
|
||||||
# ---------------------------------------------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------------------------------------------
|
||||||
|
@ -45,9 +88,11 @@ ENV CONDUIT_PORT=6167 \
|
||||||
# └─> Set no config file to do all configuration with env vars
|
# └─> Set no config file to do all configuration with env vars
|
||||||
|
|
||||||
# Conduit needs:
|
# Conduit needs:
|
||||||
|
# dpkg: to install conduit.deb
|
||||||
# ca-certificates: for https
|
# ca-certificates: for https
|
||||||
# iproute2 & wget: for the healthcheck script
|
# iproute2 & wget: for the healthcheck script
|
||||||
RUN apt-get update && apt-get -y --no-install-recommends install \
|
RUN apt-get update && apt-get -y --no-install-recommends install \
|
||||||
|
dpkg \
|
||||||
ca-certificates \
|
ca-certificates \
|
||||||
iproute2 \
|
iproute2 \
|
||||||
wget \
|
wget \
|
||||||
|
@ -57,8 +102,9 @@ RUN apt-get update && apt-get -y --no-install-recommends install \
|
||||||
COPY ./docker/healthcheck.sh /srv/conduit/healthcheck.sh
|
COPY ./docker/healthcheck.sh /srv/conduit/healthcheck.sh
|
||||||
HEALTHCHECK --start-period=5s --interval=5s CMD ./healthcheck.sh
|
HEALTHCHECK --start-period=5s --interval=5s CMD ./healthcheck.sh
|
||||||
|
|
||||||
# Copy over the actual Conduit binary from the builder stage
|
# Install conduit.deb:
|
||||||
COPY --from=builder /usr/src/conduit/target/release/conduit /srv/conduit/conduit
|
COPY --from=packager /usr/src/conduit/target/debian/*.deb /srv/conduit/
|
||||||
|
RUN dpkg -i /srv/conduit/*.deb
|
||||||
|
|
||||||
# Improve security: Don't run stuff as root, that does not need to run as root
|
# Improve security: Don't run stuff as root, that does not need to run as root
|
||||||
# Most distros also use 1000:1000 for the first real user, so this should resolve volume mounting problems.
|
# Most distros also use 1000:1000 for the first real user, so this should resolve volume mounting problems.
|
||||||
|
@ -73,7 +119,7 @@ RUN chown -cR conduit:conduit /srv/conduit && \
|
||||||
chmod +x /srv/conduit/healthcheck.sh && \
|
chmod +x /srv/conduit/healthcheck.sh && \
|
||||||
mkdir -p ${DEFAULT_DB_PATH} && \
|
mkdir -p ${DEFAULT_DB_PATH} && \
|
||||||
chown -cR conduit:conduit ${DEFAULT_DB_PATH}
|
chown -cR conduit:conduit ${DEFAULT_DB_PATH}
|
||||||
|
|
||||||
# Change user to conduit, no root permissions afterwards:
|
# Change user to conduit, no root permissions afterwards:
|
||||||
USER conduit
|
USER conduit
|
||||||
# Set container home directory
|
# Set container home directory
|
||||||
|
@ -81,4 +127,4 @@ WORKDIR /srv/conduit
|
||||||
|
|
||||||
# Run Conduit and print backtraces on panics
|
# Run Conduit and print backtraces on panics
|
||||||
ENV RUST_BACKTRACE=1
|
ENV RUST_BACKTRACE=1
|
||||||
ENTRYPOINT [ "/srv/conduit/conduit" ]
|
ENTRYPOINT [ "/usr/sbin/matrix-conduit" ]
|
||||||
|
|
Loading…
Reference in a new issue