Merge bitcoin/bitcoin#21462: guix: Add guix-{attest,verify} scripts

d420e5c1c0 guix-attest: Avoid incomplete sigdirs with ERR traps (Carl Dong)
feda2c8e31 guix: Skip attesting to dist-archive (Carl Dong)
d522d8006b guix: Attest to inputs in inputs.SHA256SUMS (Carl Dong)
f9e2960c01 guix: Construct $OUTDIR in ${DISTSRC}/output (Carl Dong)
022abc85fc guix: Minor quoting fix in libexec/build.sh (Carl Dong)
c83c4fa5b7 guix-attest: Allow skipping GPG signing with NO_SIGN (Carl Dong)
0e1c2e448c guix-attest: Use ascii-armor signatures (Carl Dong)
b5fd89c4c8 guix-attest: Only use cross-platform flags for find+xargs (Carl Dong)
5926432ba6 guix: Add guix-verify script (Carl Dong)
30daf76a97 guix: Add guix-attest script (Carl Dong)

Pull request description:

  Adds replacements for `gsign` and `gverify`.

  Personally I'm not a big fan of using the word "sign" as it's been used to refer to both codesigning and GPG signing.

ACKs for top commit:
  laanwj:
    Code review and tested ACK d420e5c1c0

Tree-SHA512: 93d82d201f4596eaea0e3825aa55b013dfb91790e6ccee79893833d37921513d7b4e735f0641103e1e2ea8308abe4cb6218b73160924708802f2e0e3f7f6caf1

contrib/guix/guix-attest

@@ -0,0 +1,207 @@
+#!/usr/bin/env bash
+export LC_ALL=C
+set -e -o pipefail
+
+# Source the common prelude, which:
+#   1. Checks if we're at the top directory of the Bitcoin Core repository
+#   2. Defines a few common functions and variables
+#
+# shellcheck source=libexec/prelude.bash
+source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash"
+
+
+###################
+## Sanity Checks ##
+###################
+
+################
+# Required non-builtin commands should be invokable
+################
+
+check_tools cat env basename mkdir xargs find
+if [ -z "$NO_SIGN" ]; then
+    check_tools gpg
+fi
+
+################
+# Required env vars should be non-empty
+################
+
+cmd_usage() {
+cat <<EOF
+Synopsis:
+
+    env GUIX_SIGS_REPO=<path/to/guix.sigs> \\
+        SIGNER=GPG_KEY_NAME[=SIGNER_NAME] \\
+        [ NO_SIGN=1 ]
+      ./contrib/guix/guix-attest
+
+Example w/o overriding signing name:
+
+    env GUIX_SIGS_REPO=/home/achow101/guix.sigs \\
+        SIGNER=achow101 \\
+      ./contrib/guix/guix-attest
+
+Example overriding signing name:
+
+    env GUIX_SIGS_REPO=/home/dongcarl/guix.sigs \\
+        SIGNER=0x96AB007F1A7ED999=dongcarl \\
+      ./contrib/guix/guix-attest
+
+Example w/o signing, just creating SHA256SUMS:
+
+    env GUIX_SIGS_REPO=/home/achow101/guix.sigs \\
+        SIGNER=achow101 \\
+        NO_SIGN=1 \\
+      ./contrib/guix/guix-attest
+
+EOF
+}
+
+if [ -z "$GUIX_SIGS_REPO" ] || [ -z "$SIGNER" ]; then
+    cmd_usage
+    exit 1
+fi
+
+################
+# GUIX_SIGS_REPO should exist as a directory
+################
+
+if [ ! -d "$GUIX_SIGS_REPO" ]; then
+cat << EOF
+ERR: The specified GUIX_SIGS_REPO is not an existent directory:
+
+    '$GUIX_SIGS_REPO'
+
+Hint: Please clone the guix.sigs repository and point to it with the
+      GUIX_SIGS_REPO environment variable.
+
+EOF
+cmd_usage
+exit 1
+fi
+
+################
+# The key specified in SIGNER should be usable
+################
+
+IFS='=' read -r gpg_key_name signer_name <<< "$SIGNER"
+if [ -z "${signer_name}" ]; then
+    signer_name="$gpg_key_name"
+fi
+
+if [ -z "$NO_SIGN" ] && ! gpg --dry-run --list-secret-keys "${gpg_key_name}" >/dev/null 2>&1; then
+    echo "ERR: GPG can't seem to find any key named '${gpg_key_name}'"
+    exit 1
+fi
+
+################
+# We should be able to find at least one output
+################
+
+echo "Looking for build output directories in ${OUTDIR_BASE}"
+
+shopt -s nullglob
+OUTDIRS=( "${OUTDIR_BASE}"/* ) # This expands to an array of directories...
+shopt -u nullglob
+
+if (( ${#OUTDIRS[@]} )); then
+    echo "Found build output directories:"
+    for outdir in "${OUTDIRS[@]}"; do
+        echo "    '$outdir'"
+    done
+    echo
+else
+    echo "ERR: Could not find any build output directories in ${OUTDIR_BASE}"
+    exit 1
+fi
+
+
+##############
+##  Attest  ##
+##############
+
+# Usage: out_name $outdir
+#
+#   HOST: The output directory being attested
+#
+out_name() {
+    basename "$1"
+}
+
+# Usage: out_sig_dir $outdir
+#
+#   outdir: The output directory being attested
+#
+out_sig_dir() {
+    echo "$GUIX_SIGS_REPO/$VERSION/$(out_name "$1")/$signer_name"
+}
+
+# Accumulate a list of signature directories that already exist...
+outdirs_already_attested_to=()
+
+echo "Attesting to build outputs for version: '${VERSION}'"
+echo ""
+
+# MAIN LOGIC: Loop through each output for VERSION and attest to output in
+#             GUIX_SIGS_REPO as SIGNER, if attestation does not exist
+for outdir in "${OUTDIRS[@]}"; do
+    if [ -e "${outdir}/SKIPATTEST.TAG" ]; then
+        echo "${outname}: SKIPPING: Output directory marked with SKIPATTEST.TAG file"
+        continue
+    fi
+    outname="$(out_name "$outdir")"
+    outsigdir="$(out_sig_dir "$outdir")"
+    if [ -e "$outsigdir" ]; then
+        echo "${outname}: SKIPPING: Signature directory already exists in the specified guix.sigs repository"
+        outdirs_already_attested_to+=("$outdir")
+    else
+        # Clean up incomplete sigdir if something fails (likely gpg)
+        trap 'rm -rf "$outsigdir"' ERR
+
+        mkdir -p "$outsigdir"
+
+        (
+            cd "$outdir"
+
+            if [ -e inputs.SHA256SUMS ]; then
+                echo "${outname}: Including existent input SHA256SUMS"
+                cat inputs.SHA256SUMS >> "$outsigdir"/SHA256SUMS
+            fi
+
+            echo "${outname}: Hashing build outputs to produce SHA256SUMS"
+            files="$(find -L . -type f ! -iname '*.SHA256SUMS')"
+            if [ -n "$files" ]; then
+                cut -c3- <<< "$files" | env LC_ALL=C sort | xargs sha256sum >> "$outsigdir"/SHA256SUMS
+            else
+                echo "ERR: ${outname}: No outputs found in '${outdir}'"
+                exit 1
+            fi
+        )
+        if [ -z "$NO_SIGN" ]; then
+            echo "${outname}: Signing SHA256SUMS to produce SHA256SUMS.asc"
+            gpg --detach-sign --local-user "$gpg_key_name" --armor --output "$outsigdir"/SHA256SUMS.asc "$outsigdir"/SHA256SUMS
+        else
+            echo "${outname}: Not signing SHA256SUMS as \$NO_SIGN is not empty"
+        fi
+        echo ""
+
+        trap - ERR  # Reset ERR trap
+    fi
+done
+
+if (( ${#outdirs_already_attested_to[@]} )); then
+# ...so that we can print them out nicely in a warning message
+cat << EOF
+
+WARN: Signature directories from '$signer_name' already exist in the specified
+      guix.sigs repository for the following output directories and were
+      skipped:
+
+EOF
+for outdir in "${outdirs_already_attested_to[@]}"; do
+    echo "    '${outdir}'"
+    echo "    Corresponds to: '$(out_sig_dir "$outdir")'"
+    echo ""
+done
+fi

contrib/guix/guix-verify

@@ -0,0 +1,113 @@
+#!/usr/bin/env bash
+export LC_ALL=C
+set -e -o pipefail
+
+# Source the common prelude, which:
+#   1. Checks if we're at the top directory of the Bitcoin Core repository
+#   2. Defines a few common functions and variables
+#
+# shellcheck source=libexec/prelude.bash
+source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash"
+
+
+###################
+## Sanity Checks ##
+###################
+
+################
+# Required non-builtin commands should be invokable
+################
+
+check_tools cat diff gpg
+
+################
+# Required env vars should be non-empty
+################
+
+cmd_usage() {
+cat <<EOF
+Synopsis:
+
+    env GUIX_SIGS_REPO=<path/to/guix.sigs> ./contrib/guix/guix-verify
+
+EOF
+}
+
+if [ -z "$GUIX_SIGS_REPO" ]; then
+    cmd_usage
+    exit 1
+fi
+
+################
+# GUIX_SIGS_REPO should exist as a directory
+################
+
+if [ ! -d "$GUIX_SIGS_REPO" ]; then
+cat << EOF
+ERR: The specified GUIX_SIGS_REPO is not an existent directory:
+
+    '$GUIX_SIGS_REPO'
+
+Hint: Please clone the guix.sigs repository and point to it with the
+      GUIX_SIGS_REPO environment variable.
+
+EOF
+cmd_usage
+exit 1
+fi
+
+################
+# We should be able to find at least one output
+################
+
+OUTSIGDIR_BASE="${GUIX_SIGS_REPO}/${VERSION}"
+echo "Looking for output signature directories in '${OUTSIGDIR_BASE}'"
+
+shopt -s nullglob
+OUTSIGDIRS=( "$OUTSIGDIR_BASE"/* ) # This expands to an array of directories...
+shopt -u nullglob
+
+if (( ${#OUTSIGDIRS[@]} )); then
+    echo "Found output signature directories:"
+    for outsigdir in "${OUTSIGDIRS[@]}"; do
+        echo "    '$outsigdir'"
+    done
+    echo
+else
+    echo "ERR: Could not find any output signature directories in ${OUTSIGDIR_BASE}"
+    exit 1
+fi
+
+
+##############
+##  Verify  ##
+##############
+
+# MAIN LOGIC: Loop through each output for VERSION and check that the SHA256SUMS
+#             and SHA256SUMS.asc file match between signers, using the first
+#             available signer as the arbitrary comparison base.
+for outsigdir in "${OUTSIGDIRS[@]}"; do
+    echo "BEGIN: Checking output signatures for $(basename "$outsigdir")"
+    echo ""
+    signer_dirs=( "$outsigdir"/* ) # This expands to an array of directories...
+    compare_signer_dir="${signer_dirs[0]}" # ...we just want the first one
+    for current_signer_dir in "${signer_dirs[@]}"; do
+        if ! gpg --quiet --batch --verify "$current_signer_dir"/SHA256SUMS.asc "$current_signer_dir"/SHA256SUMS; then
+            echo "ERR: Failed to verify GPG signature in '${current_signer_dir}/SHA256SUMS.asc'"
+            echo ""
+            echo "Hint: Either the signature is invalid or the public key is missing"
+            echo ""
+        elif ! diff --report-identical "$compare_signer_dir"/SHA256SUMS "$current_signer_dir"/SHA256SUMS; then
+            echo "ERR: The SHA256SUMS attestation in these two directories differ:"
+            echo "    '${compare_signer_dir}'"
+            echo "    '${current_signer_dir}'"
+            echo ""
+        else
+            echo "Verified: '${current_signer_dir}'"
+            echo ""
+        fi
+    done
+    echo "DONE: Checking output signatures for $(basename "$outsigdir")"
+    echo ""
+    echo ""
+done

contrib/guix/libexec/build.sh

@@ -33,6 +33,9 @@ Required environment variables as seen inside the container:
     OUTDIR: ${OUTDIR:?not set}
 EOF
 
+ACTUAL_OUTDIR="${OUTDIR}"
+OUTDIR="${DISTSRC}/output"
+
 #####################
 # Environment Setup #
 #####################
@@ -224,9 +227,25 @@ GIT_ARCHIVE="${DIST_ARCHIVE_BASE}/${DISTNAME}.tar.gz"
 # Create the source tarball if not already there
 if [ ! -e "$GIT_ARCHIVE" ]; then
     mkdir -p "$(dirname "$GIT_ARCHIVE")"
+    touch "${DIST_ARCHIVE_BASE}"/SKIPATTEST.TAG
     git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD
 fi
 
+# tmpdir="$(mktemp -d)"
+# (
+#     cd "$tmpdir"
+#     mkdir -p inputs
+#     ln -sf --target-directory=inputs "$GIT_ARCHIVE"
+
+#     mkdir -p "$OUTDIR"
+#     find -L inputs -type f -print0 | xargs -0 sha256sum > "${OUTDIR}/inputs.SHA256SUMS"
+# )
+
+mkdir -p "$OUTDIR"
+cat << EOF > "$OUTDIR"/inputs.SHA256SUMS
+$(sha256sum "$GIT_ARCHIVE" | cut -d' ' -f1)  inputs/$(basename "$GIT_ARCHIVE")
+EOF
+
 ###########################
 # Binary Tarball Building #
 ###########################
@@ -292,7 +311,8 @@ mkdir -p "$DISTSRC"
     # version symbols for Linux distro back-compatibility.
     make -C src --jobs=1 check-symbols  ${V:+V=1}
 
-    mkdir -p ${OUTDIR}
+    mkdir -p "$OUTDIR"
+
     # Make the os-specific installers
     case "$HOST" in
         *mingw*)
@@ -427,3 +447,5 @@ mkdir -p "$DISTSRC"
             ;;
     esac
 )  # $DISTSRC
+
+mv --no-target-directory "$OUTDIR" "$ACTUAL_OUTDIR"