Merge branch 'release/samedec-0.4.1'

Author: cbs228

.github/container/README.md

@@ -0,0 +1,49 @@
+# Rust Cross-Compiling Containers
+
+[cross-rs](https://github.com/cross-rs/cross) splits the build environment into:
+
+1. An "outer" build environment, which contains `cargo` and a toolchain.
+
+2. An "inner" build environment, which includes a gcc cross-compiler, foreign-architecture runtime libraries, and qemu.
+
+If the outer build environment is containerized, this introduces problems. The inner build environment (2) needs access to the host's Docker or podman socket. This can be done, but it does not mesh well with Github Actions' `jobs:*:container` option.
+
+Further, with containers, we want all parts of the build environment:
+
+* Included and ready-to-use
+* Fully-defined by the `Containerfile` and the image's `sha256:` hash
+
+Depending on an external container at runtime defeats some of these design goals.
+
+Instead of using cross-rs, we create our own environment that includes all of these tools. See
+
+```
+./build.sh
+```
+
+to build it. The build instructions are heavily influenced by the cross-rs project but do not depend on it.
+
+You can use any of these containers offline to build samedec. Define the following volumes:
+
+* `/src`: the repository root of a Rust project
+* `/install`: destination for binaries installed with `cargo install`
+* `/src/target` (**optional**): a build directory.
+* `/cargohome` (**optional**): a persistent `$CARGO_HOME` directory
+
+Example:
+
+```bash
+mkdir -p out/aarch64-unknown-linux-gnu
+
+podman run \
+  --security-opt label=disable \
+  --userns=keep-id:uid=1001,gid=1001 \
+  --rm -it \
+  --volume .:/src:ro \
+  --volume ./target:/src/target:rw \
+  --volume ./out/aarch64-unknown-linux-gnu:/install \
+  ghcr.io/cbs228/sameold/builder/aarch64-unknown-linux-gnu \
+  cargo install --path crates/samedec
+```
+
+You should run your build as UID 1001 within the container. The above `--userns` mapping will accomplish this.

.github/container/aarch64-unknown-linux-gnu/Dockerfile

@@ -0,0 +1,35 @@
+#
+# Debian base image, with arm64 toolchains
+#
+
+FROM ghcr.io/cbs228/sameold/builder/rust:latest
+
+ARG SOURCE_DATE_EPOCH
+
+RUN --mount=target=/var/lib/apt,type=cache,sharing=locked \
+    --mount=target=/var/cache/apt,type=cache,sharing=locked \
+    --mount=destination=/var/log,type=tmpfs \
+    [ "$(dpkg --print-architecture)" != arm64 ] || exit 0; \
+    set -eux; \
+    apt-get update; \
+    apt-get install -y \
+        binutils-aarch64-linux-gnu \
+        gcc-aarch64-linux-gnu \
+        libc6-dev-arm64-cross \
+        libc6:arm64 \
+    ; \
+    rm -f -- /etc/machine-id /var/cache/ldconfig/aux-cache
+
+ARG RUST_TARGET="aarch64-unknown-linux-gnu"
+
+RUN set -eux; \
+    export CARGO_HOME=/usr/local/cargo; \
+    rustup target add "$RUST_TARGET";
+
+ENV CARGO_BUILD_TARGET="$RUST_TARGET" \
+    CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc \
+    CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUNNER=qemu-run-maybe
+
+USER builder
+
+LABEL org.opencontainers.image.description="A Debian-based Rust cross-compiling environment."

.github/container/armv7-unknown-linux-gnueabihf/Dockerfile

@@ -0,0 +1,35 @@
+#
+# Debian base image, with armhf toolchains
+#
+
+FROM ghcr.io/cbs228/sameold/builder/rust:latest
+
+ARG SOURCE_DATE_EPOCH
+
+RUN --mount=target=/var/lib/apt,type=cache,sharing=locked \
+    --mount=target=/var/cache/apt,type=cache,sharing=locked \
+    --mount=destination=/var/log,type=tmpfs \
+    [ "$(dpkg --print-architecture)" != armhf ] || exit 0; \
+    set -eux; \
+    apt-get update; \
+    apt-get install -y \
+        binutils-arm-linux-gnueabi \
+        gcc-arm-linux-gnueabihf \
+        libc6-dev-armhf-cross \
+        libc6:armhf \
+    ; \
+    rm -f -- /etc/machine-id /var/cache/ldconfig/aux-cache
+
+ARG RUST_TARGET="armv7-unknown-linux-gnueabihf"
+
+RUN set -eux; \
+    export CARGO_HOME=/usr/local/cargo; \
+    rustup target add "$RUST_TARGET";
+
+ENV CARGO_BUILD_TARGET="$RUST_TARGET" \
+    CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER=arm-linux-gnueabihf-gcc \
+    CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_RUNNER=qemu-run-maybe
+
+USER builder
+
+LABEL org.opencontainers.image.description="A Debian-based Rust cross-compiling environment."

.github/container/base/Dockerfile

@@ -0,0 +1,67 @@
+#
+# Debian base image
+#
+
+# or buster-20240612-slim for a snapshot-capable, reproducible image
+ARG DEBIAN_TAG=buster-slim
+
+FROM docker.io/library/debian:${DEBIAN_TAG}
+
+ARG DEBIAN_TAG
+ARG SOURCE_DATE_EPOCH
+
+# Create a non-root user for running builds
+# GHA wants uid 1001
+ARG UID=1001
+
+RUN --mount=target=/var/lib/apt,type=cache,sharing=locked \
+    --mount=target=/var/cache/apt,type=cache,sharing=locked \
+    --mount=destination=/var/log,type=tmpfs \
+    set -eux; \
+    # create an unprivileged user for builds
+    groupadd builder -g "$UID"; \
+    useradd builder -u "$UID" -g "$UID" --create-home; \
+    # build directories for any user
+    mkdir -m 1777 /src /install /cargo; \
+    # re-enable apt caching (we have a cache mount)
+    rm -f /etc/apt/apt.conf.d/docker-clean; \
+    # if tag contains numerics, like buster-20240612-slim, use the
+    # snapshot URL that's baked into the image
+    if echo "${DEBIAN_TAG}" | grep -q "[0-9]"; then \
+        sed -i -r \
+                -e 's/^deb/# deb/' \
+                -e 's|^#\s*(.*http://snapshot\.)|\1|' \
+            /etc/apt/sources.list; \
+        cat >&2 /etc/apt/sources.list; \
+        echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/use-snapshot.conf; \
+        echo 'Acquire::Retries "10";' >> /etc/apt/apt.conf.d/use-snapshot.conf; \
+        echo 'Acquire::Retries::Delay::Maximum "600";' >> /etc/apt/apt.conf.d/use-snapshot.conf; \
+    fi; \
+    # enable packages from multiple architectures
+    dpkg --add-architecture amd64; \
+    dpkg --add-architecture armhf; \
+    dpkg --add-architecture arm64; \
+    dpkg --add-architecture i386; \
+    # install native C compilers, CI utilities, and qemu
+    apt-get update; \
+    apt-get install -y --no-install-recommends \
+        build-essential \
+        ca-certificates \
+        curl \
+        gcc \
+        git \
+        libc6-dev \
+        qemu-user \
+        qemu-user-binfmt \
+        tar \
+        zstd; \
+    git config --system --add safe.directory '*'; \
+    rm -f -- /etc/machine-id /var/cache/ldconfig/aux-cache
+
+# Directories for sources and installed binaries
+VOLUME ["/src", "/install"]
+
+LABEL org.opencontainers.image.source="https://github.com/cbs228/sameold"
+LABEL org.opencontainers.image.description="A minimal debian cross-compiling environment with good glibc compatibility."
+
+WORKDIR "/src"

.github/container/build.sh

@@ -0,0 +1,196 @@
+#!/usr/bin/env bash
+#
+# (re)build the CI container image
+#
+# run with --push to push the images.
+
+# set to disable cache
+NO_CACHE="${NO_CACHE:-}"
+
+DEBIAN_TAG="buster-20240612-slim"
+CONTAINER_PREFIX="ghcr.io/cbs228"
+CONTAINER_FQNAME="${CONTAINER_PREFIX}/sameold/builder/%s"
+
+RUST_VERSIONS=("1.84.0")
+
+usage() {
+  cat <<EOF
+Usage: $0 [--push]
+
+Build container images for the CI environment. To select
+a particular container platform tool like podman, set
+
+    BUILDER=podman
+
+Prior to pushing, make sure to
+
+    podman login "$CONTAINER_PREFIX"
+
+or equivalent.
+EOF
+}
+
+run() {
+  # Echo and run
+  echo >&2 "$@"
+  "$@"
+}
+
+container_builder() {
+  # Usage: automatically detect container platform command
+
+  if [ -x "${BUILDER:-}" ]; then
+    echo "${BUILDER}"
+  elif [ -n "${BUILDER:-}" ]; then
+    command -v "${BUILDER}"
+  else
+    {
+      command -v podman || \
+      command -v docker
+    } 2>/dev/null || {
+      echo >&2 "FATAL: container platform tools not found"
+      return 1;
+    }
+  fi
+}
+
+container_name() {
+  # Usage: container_name SUFFIX
+
+  #shellcheck disable=SC2059
+  printf "$CONTAINER_FQNAME" "$1"
+}
+
+buildcontainer() {
+  # Usage: buildcontainer ARGS
+  #
+  # Run the $BUILDER to build a container image. Some standardized
+  # arguments are passed to every build.
+
+  if [[ $BUILDER =~ podman$ ]]; then
+    run "$BUILDER" build \
+      --build-arg SOURCE_DATE_EPOCH="$SOURCE_DATE_EPOCH" \
+      --timestamp "$SOURCE_DATE_EPOCH" \
+      ${NO_CACHE:+--no-cache} \
+      "$@"
+  else
+    # docker mode; 100% untested
+    run "$BUILDER" buildx build \
+      --build-arg SOURCE_DATE_EPOCH="$SOURCE_DATE_EPOCH" \
+			--output type=docker,rewrite-timestamp=true \
+      ${NO_CACHE:+--no-cache} \
+      "$@"
+  fi
+}
+
+tagall() {
+  # Usage: tagall SHORTNAME TAG0 TAG1 ...
+  #
+  # Apply all tags to the given image, which must already be tagged as
+  # "current/container/prefix/$SHORTNAME:$TAG0"
+
+  local prefix
+  prefix="$(container_name "${1?}")"
+  shift
+
+  run "$BUILDER" tag "${@/#/"$prefix:"}"
+}
+
+pushall() {
+  # Usage: pushall SHORTNAME TAG0 TAG1 ...
+  #
+  # Push the given image "current/container/prefix/$SHORTNAME" and
+  # all the tags specified on the command line. The pushes are not
+  # atomic and will happen sequentially.
+
+  local prefix
+  prefix="$(container_name "${1?}")"
+  shift
+
+  local tag
+  for tag in "$@"; do
+    run "$BUILDER" push "${prefix}:${tag}"
+  done
+}
+
+# return if sourced
+(return 0 2>/dev/null) && return 0
+
+set -euo pipefail
+
+if ! options="$(getopt -o 'hp' --long help,push -- "$@")"; then
+  usage >&2
+  exit 1
+fi
+
+eval set -- "${options:-}"
+push_images=''
+while true; do
+  case "${1:-}" in
+    (-h | --help)
+      usage
+      exit 0 ;;
+    (-p | --push)
+      push_images=y ;;
+    ('') ;;
+  esac
+  shift || break
+done
+
+BUILDER="$(container_builder)"
+selfdir="$(dirname "$(realpath -e "${0?}")")"
+
+# set SOURCE_DATE_EPOCH if possible
+SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git log -1 --pretty=%ct -- "$selfdir" || date +%s)}"
+export SOURCE_DATE_EPOCH
+
+# tag with "latest" and SOURCE_DATE_EPOCH
+CONTAINER_TAGS=("latest" "$(date --date '@'"$SOURCE_DATE_EPOCH" +'%Y-%m-%d')")
+
+# build the base image
+base_tag="$(container_name base):${CONTAINER_TAGS[0]}"
+
+buildcontainer \
+  --build-arg DEBIAN_TAG="$DEBIAN_TAG" \
+  --tag "$base_tag" \
+  "${selfdir?}/base"
+
+# add rust to the base image
+rust_tag="$(container_name rust):${CONTAINER_TAGS[0]}"
+
+buildcontainer \
+  --from "$base_tag" \
+  --build-arg RUST_VERSIONS="${RUST_VERSIONS[*]}" \
+  --tag "$rust_tag" \
+  "${selfdir?}/rust"
+
+# build architecture-specific images
+for containerdir in "${selfdir?}/"*-*-*; do
+  [ -d "$containerdir" ] || continue
+
+  platform_triple="$(basename ${containerdir})"
+  cur_tag="$(container_name "$platform_triple"):${CONTAINER_TAGS[0]}"
+
+  buildcontainer \
+    --from "$rust_tag" \
+    --tag "${cur_tag}" \
+    "${containerdir}"
+done
+
+# if all builds succeed, apply remaining tags...
+tagall base "${CONTAINER_TAGS[@]}"
+tagall rust "${CONTAINER_TAGS[@]}"
+for containerdir in "${selfdir?}/"*-*-*; do
+  [ -d "$containerdir" ] || continue
+
+  tagall "$(basename "$containerdir")" "${CONTAINER_TAGS[@]}"
+done
+
+[ -n "${push_images:-}" ] || exit 0
+
+# ... and push
+for containerdir in "${selfdir?}/"*; do
+  [ -d "$containerdir" ] || continue
+
+  pushall "$(basename "$containerdir")" "${CONTAINER_TAGS[@]}"
+done