Matthias/conduit
1
0
Fork 0
Code Issues 179 Pull requests 44 Projects Releases 5 Packages Wiki Activity

Merge branch 'cross' into 'next'

Browse source 
cross compile static binaries for x86_64 and aarch64

See merge request famedly/conduit!569
This commit is contained in:
Charles Hall 2024-01-26 04:12:15 +00:00
commit bf48c10d28
7 changed files with 405 additions and 88 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.envrc
View file

@ -1,3 +1,5 @@
#!/usr/bin/env bash
use flake
PATH_add bin

72
.gitlab-ci.yml
View file

@ -10,6 +10,14 @@ before_script:
# Enable nix-command and flakes
- if command -v nix > /dev/null; then echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf; fi
# Add our own binary cache
- if command -v nix > /dev/null; then echo "extra-substituters = https://nix.computer.surgery/conduit" >> /etc/nix/nix.conf; fi
- if command -v nix > /dev/null; then echo "extra-trusted-public-keys = conduit:ZGAf6P6LhNvnoJJ3Me3PRg7tlLSrPxcQ2RiE5LIppjo=" >> /etc/nix/nix.conf; fi
# Add crane binary cache
- if command -v nix > /dev/null; then echo "extra-substituters = https://crane.cachix.org" >> /etc/nix/nix.conf; fi
- if command -v nix > /dev/null; then echo "extra-trusted-public-keys = crane.cachix.org-1:8Scfpmn9w+hGdXH/Q9tTLiYAE/2dnJYRJP7kl80GuRk=" >> /etc/nix/nix.conf; fi
# Add nix-community binary cache
- if command -v nix > /dev/null; then echo "extra-substituters = https://nix-community.cachix.org" >> /etc/nix/nix.conf; fi
- if command -v nix > /dev/null; then echo "extra-trusted-public-keys = nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" >> /etc/nix/nix.conf; fi
@ -34,20 +42,70 @@ ci:
- target
- .gitlab-ci.d
docker:
static:x86_64-unknown-linux-musl:
stage: artifacts
image: nixos/nix:2.19.2
script:
- nix build .#oci-image
# Push artifacts and build requirements to binary cache
- ./bin/nix-build-and-cache .#static-x86_64-unknown-linux-musl
# Make the output less difficult to find
- cp result docker-image.tar.gz
- cp result/bin/conduit conduit
artifacts:
paths:
- docker-image.tar.gz
- conduit
debian:
static:aarch64-unknown-linux-musl:
stage: artifacts
image: nixos/nix:2.19.2
script:
# Push artifacts and build requirements to binary cache
- ./bin/nix-build-and-cache .#static-aarch64-unknown-linux-musl
# Make the output less difficult to find
- cp result/bin/conduit conduit
artifacts:
paths:
- conduit
oci-image:x86_64-unknown-linux-gnu:
stage: artifacts
image: nixos/nix:2.19.2
script:
# Push artifacts and build requirements to binary cache
#
# Since the OCI image package is based on the binary package, this has the
# fun side effect of uploading the normal binary too. Conduit users who are
# deploying with Nix can leverage this fact by adding our binary cache to
# their systems.
- ./bin/nix-build-and-cache .#oci-image
# Make the output less difficult to find
- cp result oci-image.tar.gz
artifacts:
paths:
- oci-image.tar.gz
oci-image:aarch64-unknown-linux-musl:
stage: artifacts
needs:
# Wait for the static binary job to finish before starting so we don't have
# to build that twice for no reason
- static:aarch64-unknown-linux-musl
image: nixos/nix:2.19.2
script:
# Push artifacts and build requirements to binary cache
- ./bin/nix-build-and-cache .#oci-image-aarch64-unknown-linux-musl
# Make the output less difficult to find
- cp result oci-image.tar.gz
artifacts:
paths:
- oci-image.tar.gz
debian:x86_64-unknown-linux-gnu:
stage: artifacts
# See also `rust-toolchain.toml`
image: rust:1.70.0
script:
- apt-get update && apt-get install -y --no-install-recommends libclang-dev
@ -55,10 +113,10 @@ debian:
- cargo deb
# Make the output less difficult to find
- mv target/debian/*.deb .
- mv target/debian/*.deb conduit.deb
artifacts:
paths:
- "*.deb"
- conduit.deb
cache:
key: debian
paths:

7
Cargo.toml
View file

@ -9,12 +9,7 @@ readme = "README.md"
version = "0.7.0-alpha"
edition = "2021"
# When changing this, make sure to update the hash near the text "THE
# rust-version HASH" in `flake.nix`. If you don't have Nix installed or
# otherwise don't know how to do this, ping `@charles:computer.surgery` or
# `@dusk:gaze.systems` in the matrix room.
#
# Also make sure to update the docker image tags in `.gitlab-ci.yml`.
# See also `rust-toolchain.toml`
rust-version = "1.70.0"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

31
bin/nix-build-and-cache Executable file
View file

@ -0,0 +1,31 @@
#!/usr/bin/env bash
set -euo pipefail
# The first argument must be the desired installable
INSTALLABLE="$1"
# Build the installable and forward any other arguments too
nix build "$@"
if [ ! -z ${ATTIC_TOKEN+x} ]; then
nix run --inputs-from . attic -- login \
conduit \
https://nix.computer.surgery/conduit \
"$ATTIC_TOKEN"
push_args=(
# Attic and its build dependencies
"$(nix path-info --inputs-from . attic)"
"$(nix path-info --inputs-from . attic --derivation)"
# The target installable and its build dependencies
"$(nix path-info "$INSTALLABLE" --derivation)"
"$(nix path-info "$INSTALLABLE")"
)
nix run --inputs-from . attic -- push conduit "${push_args[@]}"
else
echo "\$ATTIC_TOKEN is unset, skipping uploading to the binary cache"
fi

125
flake.lock
View file

@ -1,21 +1,66 @@
{
"nodes": {
"attic": {
"inputs": {
"crane": "crane",
"flake-compat": "flake-compat",
"flake-utils": "flake-utils",
"nixpkgs": "nixpkgs",
"nixpkgs-stable": "nixpkgs-stable"
},
"locked": {
"lastModified": 1705617092,
"narHash": "sha256-n9PK4O4X4S1JkwpkMuYm1wHZYJzRqif8g3RuVIPD+rY=",
"owner": "zhaofengli",
"repo": "attic",
"rev": "fbe252a5c21febbe920c025560cbd63b20e24f3b",
"type": "github"
},
"original": {
"owner": "zhaofengli",
"ref": "main",
"repo": "attic",
"type": "github"
}
},
"crane": {
"inputs": {
"nixpkgs": [
"attic",
"nixpkgs"
]
},
"locked": {
"lastModified": 1702918879,
"narHash": "sha256-tWJqzajIvYcaRWxn+cLUB9L9Pv4dQ3Bfit/YjU5ze3g=",
"owner": "ipetkov",
"repo": "crane",
"rev": "7195c00c272fdd92fc74e7d5a0a2844b9fadb2fb",
"type": "github"
},
"original": {
"owner": "ipetkov",
"repo": "crane",
"type": "github"
}
},
"crane_2": {
"inputs": {
"nixpkgs": [
"nixpkgs"
]
},
"locked": {
"lastModified": 1705597458,
"narHash": "sha256-vJ8Ib9ruxbaBxGEcA0d7dHqxpc6Z+SGR2XIxVeSMuLM=",
"owner": "ipetkov",
"lastModified": 1706070683,
"narHash": "sha256-iQoXIFviH5QbSKDkWyFJKKUtTcSiCvl1tW18ax+pKfI=",
"owner": "CobaltCause",
"repo": "crane",
"rev": "742170d82cd65c925dcddc5c3d6185699fbbad08",
"rev": "8060686afabc5b042e8982200cb2cc3fd22cf416",
"type": "github"
},
"original": {
"owner": "ipetkov",
"owner": "CobaltCause",
"ref": "crimes-for-cross",
"repo": "crane",
"type": "github"
}
@ -41,7 +86,38 @@
"type": "github"
}
},
"flake-compat": {
"flake": false,
"locked": {
"lastModified": 1673956053,
"narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
"owner": "edolstra",
"repo": "flake-compat",
"rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
"type": "github"
},
"original": {
"owner": "edolstra",
"repo": "flake-compat",
"type": "github"
}
},
"flake-utils": {
"locked": {
"lastModified": 1667395993,
"narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"flake-utils_2": {
"inputs": {
"systems": "systems"
},
@ -75,6 +151,38 @@
}
},
"nixpkgs": {
"locked": {
"lastModified": 1702539185,
"narHash": "sha256-KnIRG5NMdLIpEkZTnN5zovNYc0hhXjAgv6pfd5Z4c7U=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "aa9d4729cbc99dabacb50e3994dcefb3ea0f7447",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixpkgs-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs-stable": {
"locked": {
"lastModified": 1702780907,
"narHash": "sha256-blbrBBXjjZt6OKTcYX1jpe9SRof2P9ZYWPzq22tzXAA=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "1e2e384c5b7c50dbf8e9c441a9e58d85f408b01f",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-23.11",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs_2": {
"locked": {
"lastModified": 1705496572,
"narHash": "sha256-rPIe9G5EBLXdBdn9ilGc0nq082lzQd0xGGe092R/5QE=",
@ -92,11 +200,12 @@
},
"root": {
"inputs": {
"crane": "crane",
"attic": "attic",
"crane": "crane_2",
"fenix": "fenix",
"flake-utils": "flake-utils",
"flake-utils": "flake-utils_2",
"nix-filter": "nix-filter",
"nixpkgs": "nixpkgs"
"nixpkgs": "nixpkgs_2"
}
},
"rust-analyzer-src": {

234
flake.nix
View file

@ -9,9 +9,13 @@
inputs.nixpkgs.follows = "nixpkgs";
};
crane = {
url = "github:ipetkov/crane";
# TODO: Switch back to upstream after [this issue][0] is fixed
#
# [0]: https://github.com/ipetkov/crane/issues/497
url = "github:CobaltCause/crane?ref=crimes-for-cross";
inputs.nixpkgs.follows = "nixpkgs";
};
attic.url = "github:zhaofengli/attic?ref=main";
};
outputs =
@ -22,59 +26,122 @@
, fenix
, crane
, ...
}: flake-utils.lib.eachDefaultSystem (system:
let
pkgs = nixpkgs.legacyPackages.${system};
# Use mold on Linux
stdenv = if pkgs.stdenv.isLinux then
pkgs.stdenvAdapters.useMoldLinker pkgs.stdenv
else
pkgs.stdenv;
pkgsHost = nixpkgs.legacyPackages.${system};
# Nix-accessible `Cargo.toml`
cargoToml = builtins.fromTOML (builtins.readFile ./Cargo.toml);
# The Rust toolchain to use
toolchain = fenix.packages.${system}.toolchainOf {
# Use the Rust version defined in `Cargo.toml`
channel = cargoToml.package.rust-version;
toolchain = fenix.packages.${system}.fromToolchainFile {
file = ./rust-toolchain.toml;
# THE rust-version HASH
# See also `rust-toolchain.toml`
sha256 = "sha256-gdYqng0y9iHYzYPAdkC/ka3DRny3La/S5G8ASj0Ayyc=";
};
mkToolchain = fenix.packages.${system}.combine;
builder = pkgs:
((crane.mkLib pkgs).overrideToolchain toolchain).buildPackage;
buildToolchain = mkToolchain (with toolchain; [
cargo
rustc
]);
nativeBuildInputs = pkgs: [
# bindgen needs the build platform's libclang. Apparently due to
# "splicing weirdness", pkgs.rustPlatform.bindgenHook on its own doesn't
# quite do the right thing here.
pkgs.buildPackages.rustPlatform.bindgenHook
];
devToolchain = mkToolchain (with toolchain; [
cargo
clippy
rust-src
rustc
# Always use nightly rustfmt because most of its options are unstable
fenix.packages.${system}.latest.rustfmt
]);
builder =
((crane.mkLib pkgs).overrideToolchain buildToolchain).buildPackage;
nativeBuildInputs = (with pkgs.rustPlatform; [
bindgenHook
]);
env = {
env = pkgs: {
ROCKSDB_INCLUDE_DIR = "${pkgs.rocksdb}/include";
ROCKSDB_LIB_DIR = "${pkgs.rocksdb}/lib";
};
in
{
packages.default = builder {
}
// pkgs.lib.optionalAttrs pkgs.stdenv.hostPlatform.isStatic {
ROCKSDB_STATIC = "";
}
// {
CARGO_BUILD_RUSTFLAGS = let inherit (pkgs) lib stdenv; in
lib.concatStringsSep " " ([]
++ lib.optionals
# This disables PIE for static builds, which isn't great in terms
# of security. Unfortunately, my hand is forced because nixpkgs'
# `libstdc++.a` is built without `-fPIE`, which precludes us from
# leaving PIE enabled.
stdenv.hostPlatform.isStatic
["-C" "relocation-model=static"]
++ lib.optionals
(stdenv.buildPlatform.config != pkgs.stdenv.hostPlatform.config)
["-l" "c"]
++ lib.optionals
# This check has to match the one [here][0]. We only need to set
# these flags when using a different linker. Don't ask me why,
# though, because I don't know. All I know is it breaks otherwise.
#
# [0]: https://github.com/NixOS/nixpkgs/blob/612f97239e2cc474c13c9dafa0df378058c5ad8d/pkgs/build-support/rust/lib/default.nix#L36-L39
(
pkgs.stdenv.hostPlatform.isAarch64
&& pkgs.stdenv.hostPlatform.isStatic
&& !pkgs.stdenv.isDarwin
&& !pkgs.stdenv.cc.bintools.isLLVM
)
[
"-l"
"stdc++"
"-L"
"${stdenv.cc.cc.lib}/${stdenv.hostPlatform.config}/lib"
]
);
}
# What follows is stolen from [here][0]. Its purpose is to properly
# configure compilers and linkers for various stages of the build, and
# even covers the case of build scripts that need native code compiled and
# run on the build platform (I think).
#
# [0]: https://github.com/NixOS/nixpkgs/blob/612f97239e2cc474c13c9dafa0df378058c5ad8d/pkgs/build-support/rust/lib/default.nix#L64-L78
// (
let
inherit (pkgs.rust.lib) envVars;
in
pkgs.lib.optionalAttrs
(pkgs.stdenv.targetPlatform.rust.rustcTarget
!= pkgs.stdenv.hostPlatform.rust.rustcTarget)
(
let
inherit (pkgs.stdenv.targetPlatform.rust) cargoEnvVarTarget;
in
{
"CC_${cargoEnvVarTarget}" = envVars.ccForTarget;
"CXX_${cargoEnvVarTarget}" = envVars.cxxForTarget;
"CARGO_TARGET_${cargoEnvVarTarget}_LINKER" =
envVars.linkerForTarget;
}
)
// (
let
inherit (pkgs.stdenv.hostPlatform.rust) cargoEnvVarTarget rustcTarget;
in
{
"CC_${cargoEnvVarTarget}" = envVars.ccForHost;
"CXX_${cargoEnvVarTarget}" = envVars.cxxForHost;
"CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.linkerForHost;
CARGO_BUILD_TARGET = rustcTarget;
}
)
// (
let
inherit (pkgs.stdenv.buildPlatform.rust) cargoEnvVarTarget;
in
{
"CC_${cargoEnvVarTarget}" = envVars.ccForBuild;
"CXX_${cargoEnvVarTarget}" = envVars.cxxForBuild;
"CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.linkerForBuild;
HOST_CC = "${pkgs.buildPackages.stdenv.cc}/bin/cc";
HOST_CXX = "${pkgs.buildPackages.stdenv.cc}/bin/c++";
}
));
package = pkgs: builder pkgs {
src = nix-filter {
root = ./.;
include = [
@ -87,46 +154,79 @@
# This is redundant with CI
doCheck = false;
inherit
env
nativeBuildInputs
stdenv;
env = env pkgs;
nativeBuildInputs = nativeBuildInputs pkgs;
meta.mainProgram = cargoToml.package.name;
};
packages.oci-image =
let
package = self.packages.${system}.default;
in
pkgs.dockerTools.buildImage {
name = package.pname;
tag = "latest";
config = {
# Use the `tini` init system so that signals (e.g. ctrl+c/SIGINT) are
# handled as expected
Entrypoint = [
"${pkgs.lib.getExe' pkgs.tini "tini"}"
"--"
];
Cmd = [
"${pkgs.lib.getExe package}"
];
mkOciImage = pkgs: package:
pkgs.dockerTools.buildImage {
name = package.pname;
tag = "latest";
config = {
# Use the `tini` init system so that signals (e.g. ctrl+c/SIGINT)
# are handled as expected
Entrypoint = [
"${pkgs.lib.getExe' pkgs.tini "tini"}"
"--"
];
Cmd = [
"${pkgs.lib.getExe package}"
];
};
};
};
in
{
packages = {
default = package pkgsHost;
devShells.default = (pkgs.mkShell.override { inherit stdenv; }) {
env = env // {
oci-image = mkOciImage pkgsHost self.packages.${system}.default;
# Build an OCI image from the musl aarch64 build so we don't have to
# build for aarch64 twice (to make a gnu version specifically for the
# OCI image)
oci-image-aarch64-unknown-linux-musl = mkOciImage
pkgsHost
self.packages.${system}.static-aarch64-unknown-linux-musl;
# Don't build a musl x86_64 OCI image because that would be pointless.
# Just use the gnu one (i.e. `self.packages."x86_64-linux".oci-image`).
} // builtins.listToAttrs (
builtins.map
(crossSystem: {
name = "static-${crossSystem}";
value = package (import nixpkgs {
inherit system;
crossSystem = {
config = crossSystem;
};
}).pkgsStatic;
})
[
"x86_64-unknown-linux-musl"
"aarch64-unknown-linux-musl"
]
);
devShells.default = pkgsHost.mkShell {
env = env pkgsHost // {
# Rust Analyzer needs to be able to find the path to default crate
# sources, and it can read this environment variable to do so. The
# `rust-src` component is required in order for this to work.
RUST_SRC_PATH = "${devToolchain}/lib/rustlib/src/rust/library";
RUST_SRC_PATH = "${toolchain}/lib/rustlib/src/rust/library";
};
# Development tools
nativeBuildInputs = nativeBuildInputs ++ [
devToolchain
] ++ (with pkgs; [
nativeBuildInputs = nativeBuildInputs pkgsHost ++ [
# Always use nightly rustfmt because most of its options are unstable
#
# This needs to come before `toolchain` in this list, otherwise
# `$PATH` will have stable rustfmt instead.
fenix.packages.${system}.latest.rustfmt
toolchain
] ++ (with pkgsHost; [
engage
]);
};

22
rust-toolchain.toml Normal file
View file

@ -0,0 +1,22 @@
# This is the authoritiative configuration of this project's Rust toolchain.
#
# Other files that need upkeep when this changes:
#
# * `.gitlab-ci.yml`
# * `Cargo.toml`
# * `flake.nix`
#
# Search in those files for `rust-toolchain.toml` to find the relevant places.
# If you're having trouble making the relevant changes, bug a maintainer.
[toolchain]
channel = "1.70.0"
components = [
# For rust-analyzer
"rust-src",
]
targets = [
"x86_64-unknown-linux-gnu",
"x86_64-unknown-linux-musl",
"aarch64-unknown-linux-musl",
]