Merge branch 'radicle-dev:master' into master

.gitignore

@@ -3,6 +3,7 @@ node_modules/
 NOTES
 config/local*
 http-server/target
+http-server/build/artifacts
 
 # KaTeX files
 *.min.css

http-server/build/Dockerfile

@@ -0,0 +1,90 @@
+# Builds release binaries for Radicle.
+FROM rust:1.77.2-alpine3.19 as builder
+LABEL maintainer="Radicle Team <team@radicle.xyz>"
+WORKDIR /src
+COPY . .
+
+# Copy cargo configuration we're going to use to specify compiler options.
+RUN mkdir -p .cargo && cp build/config.toml .cargo/config.toml
+# Install dependencies.
+RUN apk update && apk add --no-cache git musl-dev minisign curl xz asciidoctor
+# Build man pages and strip metadata. Removes all comments, since they include
+# non-reproducible information, such as version numbers.
+RUN asciidoctor --doctype manpage --backend manpage --destination-dir . *.1.adoc && \
+    find . -maxdepth 1 -type f -name '*.1' -exec sed -i '/^.\\\"/d' '{}' \;
+# Add cargo targets.
+RUN rustup target add \
+    x86_64-unknown-linux-musl \
+    aarch64-unknown-linux-musl \
+    x86_64-apple-darwin \
+    aarch64-apple-darwin
+
+# Install dependencies for cross-compiling to macOS.
+# We use Zig as the linker to perform the compilation from a Linux host.
+# Zig is not yet available on Debian, so we download the official binary.
+# Compilation is done via `cargo-zigbuild` which is a wrapper around `zig`.
+RUN curl -sSf -o zig.tar.xz         https://ziglang.org/download/0.12.0/zig-linux-x86_64-0.12.0.tar.xz && \
+    curl -sSf -o zig.tar.xz.minisig https://ziglang.org/download/0.12.0/zig-linux-x86_64-0.12.0.tar.xz.minisig && \
+    minisign -Vm zig.tar.xz -P RWSGOq2NVecA2UPNdBUZykf1CCb147pkmdtYxgb3Ti+JO/wCYvhbAb/U && \
+    xz -d -c zig.tar.xz | tar -x && \
+    mv zig-linux-x86_64-0.12.0/zig /usr/bin/zig && \
+    mv zig-linux-x86_64-0.12.0/lib /usr/lib/zig && \
+    cargo install cargo-zigbuild@0.18.3
+
+
+# Parts of the macOS SDK are required to build Radicle, we make these available
+# here. So far only `CoreFoundation` and `Security` frameworks are needed.
+RUN xz -d -c build/macos-sdk-11.3.tar.xz | tar -x
+# This env var is used by `cargo-zigbuild` to find the SDK.
+ENV SDKROOT /src/macos-sdk-11.3
+
+# Build binaries.
+RUN cargo zigbuild --locked --release \
+    --target=x86_64-apple-darwin \
+    --target=aarch64-apple-darwin \
+    --target=aarch64-unknown-linux-musl \
+    --target=x86_64-unknown-linux-musl \
+    -p radicle-httpd 
+
+# Now copy the files to a new image without all the intermediary artifacts to
+# save some space.
+FROM alpine:3.19 as packager
+COPY --from=builder \
+    /src/target/x86_64-unknown-linux-musl/release/rad-web \
+    /src/target/x86_64-unknown-linux-musl/release/radicle-httpd \
+    /builds/x86_64-unknown-linux-musl/bin/
+COPY --from=builder \
+    /src/target/aarch64-unknown-linux-musl/release/rad-web \
+    /src/target/aarch64-unknown-linux-musl/release/radicle-httpd \
+    /builds/aarch64-unknown-linux-musl/bin/
+COPY --from=builder \
+    /src/target/aarch64-apple-darwin/release/rad-web \
+    /src/target/aarch64-apple-darwin/release/radicle-httpd \
+    /builds/aarch64-apple-darwin/bin/
+COPY --from=builder \
+    /src/target/x86_64-apple-darwin/release/rad-web \
+    /src/target/x86_64-apple-darwin/release/radicle-httpd \
+    /builds/x86_64-apple-darwin/bin/
+COPY --from=builder /src/*.1 /builds/x86_64-unknown-linux-musl/man/man1/
+COPY --from=builder /src/*.1 /builds/aarch64-unknown-linux-musl/man/man1/
+COPY --from=builder /src/*.1 /builds/aarch64-apple-darwin/man/man1/
+COPY --from=builder /src/*.1 /builds/x86_64-apple-darwin/man/man1/
+
+# Create and compress reproducible archive.
+WORKDIR /builds
+RUN apk update && apk add --no-cache tar xz
+RUN find * -maxdepth 0 -type d -exec mv '{}' "radicle-$RADICLE_VERSION-{}" \; && \
+    find * -maxdepth 0 -type d -exec tar \
+    --sort=name \
+    --verbose \
+    --mtime="@$GIT_COMMIT_TIME" \
+    --owner=0 \
+    --group=0 \
+    --numeric-owner \
+    --format=posix \
+    --pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime \
+    --mode='go+u,go-w' \
+    --remove-files \
+    --create --xz \
+    --file="{}.tar.xz" \
+    '{}' \;

http-server/build/README.md

@@ -0,0 +1,70 @@
+# Builds
+
+Radicle uses a [reproducible build][rb] pipeline to make binary verification
+easier and more secure.
+
+[rb]: https://reproducible-builds.org/
+
+This build pipeline is designed to be run on an x86_64 machine running Linux.
+The output is a set of `.tar.xz` archives containing binaries for the supported
+platforms and signed by the user's Radicle key.
+
+These binaries are statically linked to be maximally portable, and designed to
+be reproducible, byte for byte.
+
+To run the build, simply enter the following command from the repository root:
+
+    build/build
+
+This will build all targets and place the output in `build/artifacts` with
+one sub-directory per build target.
+
+Note that it will use `git describe` to get a version number for the build.
+You *must* have a commit tagged with a version in your history or the build
+will fail, eg. `v1.0.0`.
+
+When the build completes, the SHA-256 checksums of the artifacts are output.
+For a given Radicle version and source tree, the same set of checksums should
+always be output, no matter where or when the build is run. If they do not
+match, either the build pipeline has a bug, making it non-reproducible, or one
+of the machines is compromised.
+
+Here's an example output for a development version of Radicle:
+
+    b9aa75bba175e18e05df4f6b39ec097414bbf56ccdeb4a2229b557f8b8e05404  radicle-1.0.0-rc.4-3-gb299f3b5-aarch64-apple-darwin.tar.xz
+    c7070806bf2d17a8a0d3b329e4d57b1e544b7b82cb58e2863074d96348a2ab0d  radicle-1.0.0-rc.4-3-gb299f3b5-aarch64-unknown-linux-musl.tar.xz
+    1a8327854f16ea90491fb90e0c3291a63c4b2ab01742c8435faec7d370cacb79  radicle-1.0.0-rc.4-3-gb299f3b5-x86_64-apple-darwin.tar.xz
+    709ac67541ff0c0c570ac22ab2de9f98320e0cc2cc9b67f1909c014a2bb5bd49  radicle-1.0.0-rc.4-3-gb299f3b5-x86_64-unknown-linux-musl.tar.xz
+
+A script is included in `build/checksums` to output these checksums after
+the artifacts are built.
+
+## Requirements
+
+The following software is required for the build:
+
+  * `podman`
+  * `rad` (The Radicle CLI)
+  * `sha256sum`
+
+## macOS
+
+macOS binaries are not signed or notarized, so they have to be downloaded via
+the CLI to avoid issues. A copy of a small subset of the Apple SDK is included
+here to be able to cross-compile.
+
+## Podman
+
+We use `podman` to make the build reproducible on any machine by controlling
+the build environment. We prefer `podman` to `docker` because it doesn't
+require a background process to run and can be run without root access out of
+the box.
+
+The first time you run `podman`, you may have to give yourself some extra UIDs
+for `podman` to use, with:
+
+    sudo usermod --add-subuids 100000-165535 --add-subgids 100000-165535 $USER
+
+Then update `podman` with:
+
+    podman system migrate

http-server/build/TARGETS

@@ -0,0 +1,4 @@
+x86_64-unknown-linux-musl
+aarch64-unknown-linux-musl
+x86_64-apple-darwin
+aarch64-apple-darwin

http-server/build/build

@@ -0,0 +1,115 @@
+#!/bin/sh
+set -e
+
+main() {
+  # Use UTC time for everything.
+  export TZ=UTC0
+  # Set minimal locale.
+  export LC_ALL=C
+  # Set source date. This is honored by `asciidoctor` and other tools.
+  export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
+  # Define user OS for podman
+  export OS=$(uname)
+
+  if ! command -v rad >/dev/null; then
+    echo "fatal: rad is not installed" >&2
+    exit 1
+  fi
+
+  if ! command -v podman >/dev/null; then
+    echo "fatal: podman is not installed" >&2
+    exit 1
+  fi
+
+  if ! command -v sha256sum >/dev/null; then
+    echo "fatal: sha256sum is not installed" >&2
+    exit 1
+  fi
+
+  rev="$(git rev-parse --short HEAD)"
+  tempdir="$(mktemp -d)"
+  gitarchive="$tempdir/heartwood-$rev.tar.gz"
+  keypath="$(rad path)/keys/radicle.pub"
+  version="$(build/version)"
+  image=radicle-build-$version
+
+  if [ ! -f "$keypath" ]; then
+    echo "fatal: no key found at $keypath" >&2
+    exit 1
+  fi
+  # Authenticate user for signing
+  rad auth
+
+  echo "Building Radicle $version.."
+  echo "Creating archive of repository at $rev in $gitarchive.."
+  git archive --format tar.gz -o "$gitarchive" HEAD
+
+  echo "Building image ($image).."
+
+  case "$OS" in
+  Darwin)
+    podman build \
+      --env SOURCE_DATE_EPOCH \
+      --env TZ \
+      --env LC_ALL \
+      --env GIT_COMMIT_TIME=$SOURCE_DATE_EPOCH \
+      --env GIT_HEAD=$rev \
+      --env RADICLE_VERSION=$version \
+      --arch aarch64 --tag $image -f ./build/Dockerfile - <$gitarchive
+
+    echo "Creating container (radicle-build-container).."
+    podman create --ulimit=host --replace --name radicle-build-container $image
+    ;;
+  *)
+    podman --cgroup-manager=cgroupfs build \
+      --env SOURCE_DATE_EPOCH \
+      --env TZ \
+      --env LC_ALL \
+      --env GIT_COMMIT_TIME=$SOURCE_DATE_EPOCH \
+      --env GIT_HEAD=$rev \
+      --env RADICLE_VERSION=$version \
+      --arch amd64 --tag $image -f ./build/Dockerfile - <$gitarchive
+
+    echo "Creating container (radicle-build-container).."
+    podman --cgroup-manager=cgroupfs create --ulimit=host --replace --name radicle-build-container $image
+    ;;
+  esac
+
+  # Copy build artifacts to output folder.
+  outdir=build/artifacts
+  mkdir -p $outdir
+  podman cp --overwrite radicle-build-container:/builds/. $outdir/
+
+  for target in $(cat build/TARGETS); do
+    echo "Signing artifacts for $target.."
+
+    filename="radicle-$version-$target.tar.xz"
+    filepath="$outdir/$filename"
+
+    # Output SHA256 digest of archive.
+    checksum="$(cd $outdir && sha256sum $filename)"
+    echo "Checksum of $filepath is $(echo "$checksum" | cut -d' ' -f1)"
+    echo "$checksum" >$filepath.sha256
+
+    # Sign archive and verify archive.
+    rm -f $filepath.sig # Delete existing signature
+    ssh-keygen -Y sign -n file -f $keypath $filepath
+    ssh-keygen -Y check-novalidate -n file -s $filepath.sig <$filepath
+  done
+
+  # Remove build artifacts that aren't needed anymore.
+  rm -f $gitarchive
+  podman rm radicle-build-container >/dev/null
+  podman rmi --ignore localhost/$image
+}
+
+# Run build.
+echo "Running build.."
+main "$@"
+
+# Show artifact checksums.
+echo
+build/checksums
+echo
+
+echo "Build successful."

http-server/build/checksums

@@ -0,0 +1,2 @@
+#!/bin/sh
+find build/artifacts -type f -name '*.sha256' -exec cat {} +

http-server/build/config.toml

@@ -0,0 +1,13 @@
+[target.x86_64-unknown-linux-musl]
+rustflags = [
+    "-C", "codegen-units=1",
+    "-C", "incremental=false",
+    "-C", "opt-level=3",
+]
+
+[target.aarch64-unknown-linux-musl]
+rustflags = [
+    "-C", "codegen-units=1",
+    "-C", "incremental=false",
+    "-C", "opt-level=3",
+]

http-server/build/tag

@@ -0,0 +1,38 @@
+#!/bin/sh
+set -e
+
+if [ $# -ne 1 ]; then
+  echo "Usage: $0 <version-number>"
+  exit 1
+fi
+
+version="$1"
+tag="v$version"
+commit="$(git rev-parse HEAD)"
+signing_key=$(git config user.signingKey)
+
+git show "$commit"
+
+if [ "$signing_key" != "$(rad self --ssh-key)" ]; then
+  echo "The Git signing key does not match the output of 'rad self --ssh-key'."
+  exit 1
+fi
+
+printf "\n"
+printf "Tag the above commit with \033[35m$tag\033[0m, using \033[35m$(rad self --did)\033[0m? [y/N] "
+read confirmation
+rad auth
+
+case "$confirmation" in
+  [Yy]*)
+    if git tag --annotate --sign "$tag" -m "Release $version" "$commit"; then
+      echo "Tag $tag created and signed over $commit."
+    else
+      echo "Failed to create tag."
+      exit 1
+    fi ;;
+  *)
+    echo "Operation aborted."
+    exit 1 ;;
+esac
+

http-server/build/upload

@@ -0,0 +1,47 @@
+#!/bin/sh
+set -e
+
+SSH_LOGIN=${SSH_LOGIN:-release}
+SSH_ADDRESS=${SSH_ADDRESS:-$SSH_LOGIN@files.radicle.xyz}
+SSH_KEY="$(rad path)/keys/radicle"
+
+main() {
+  version="$(build/version)"
+
+  echo "Uploading Radicle $version..."
+
+  if [ -z "$version" ]; then
+    echo "fatal: empty version number" >&2
+    exit 1
+  fi
+
+  # Create remote folder.
+  ssh -i $SSH_KEY $SSH_ADDRESS mkdir -p /mnt/radicle/files/releases/radicle-httpd/$version
+  # Copy files over.
+  scp -i $SSH_KEY build/artifacts/radicle-$version* $SSH_ADDRESS:/mnt/radicle/files/releases/radicle-httpd/$version
+
+  for target in $(cat build/TARGETS); do
+    archive=/mnt/radicle/files/releases/radicle-httpd/$version/radicle-$version-$target.tar.xz
+    symlink=/mnt/radicle/files/releases/radicle-httpd/$version/radicle-$target.tar.xz
+
+    echo "Creating symlinks for $target.."
+
+    ssh -i $SSH_KEY $SSH_ADDRESS ln -snf $archive $symlink
+    ssh -i $SSH_KEY $SSH_ADDRESS ln -snf $archive.sig $symlink.sig
+    ssh -i $SSH_KEY $SSH_ADDRESS ln -snf $archive.sha256 $symlink.sha256
+  done
+
+  if git describe --exact-match --match='v*' 2>/dev/null; then
+    echo "Creating 'latest' symlink.."
+    ssh -i $SSH_KEY $SSH_ADDRESS ln -snf /mnt/radicle/files/releases/radicle-httpd/$version /mnt/radicle/files/releases/radicle-httpd/latest
+  else
+    echo "Skipping 'latest' symlink creation for development build."
+  fi
+
+  echo "Pushing tags.."
+  git push rad --tags
+
+  echo "Done."
+}
+
+main "$@"