From 22437fc72e78ba3845a3953853d40093de32c395 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Fri, 25 Sep 2020 21:18:01 -0400 Subject: [PATCH 1/3] build: Run libdmg-hfsplus's DMG tool in make deploy Previously, the compression of the .iso file to a .dmg file was done outside of `make deploy' in order to use the faketime-wrapped version of libdmg-hfsplus's DMG tool. Specifying the faketime-wrapped version of the DMG tool to ./configure fixes this and simplifies build scripts. --- .gitignore | 1 + Makefile.am | 7 ++++++- configure.ac | 1 + contrib/gitian-descriptors/gitian-osx.yml | 6 ++---- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 5726b18928..810ef5db6b 100644 --- a/.gitignore +++ b/.gitignore @@ -76,6 +76,7 @@ src/qt/bitcoin-qt.includes *.log *.trs *.dmg +*.iso *.json.h *.raw.h diff --git a/Makefile.am b/Makefile.am index 958d68061f..07c56a46ae 100644 --- a/Makefile.am +++ b/Makefile.am @@ -35,6 +35,7 @@ space := $(empty) $(empty) OSX_APP=Bitcoin-Qt.app OSX_VOLNAME = $(subst $(space),-,$(PACKAGE_NAME)) OSX_DMG = $(OSX_VOLNAME).dmg +OSX_TEMP_ISO = $(OSX_DMG:.dmg=).temp.iso OSX_BACKGROUND_SVG=background.svg OSX_BACKGROUND_IMAGE=background.tiff OSX_BACKGROUND_IMAGE_DPIS=36 72 @@ -135,9 +136,13 @@ $(APP_DIST_DIR)/Applications: $(APP_DIST_EXTRAS): $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Bitcoin-Qt -$(OSX_DMG): $(APP_DIST_EXTRAS) +.INTERMEDIATE: $(OSX_TEMP_ISO) +$(OSX_TEMP_ISO): $(APP_DIST_EXTRAS) $(GENISOIMAGE) -no-cache-inodes -D -l -probe -V "$(OSX_VOLNAME)" -no-pad -r -dir-mode 0755 -apple -o $@ dist +$(OSX_DMG): $(OSX_TEMP_ISO) + $(DMG) dmg "$<" "$@" + dpi%.$(OSX_BACKGROUND_IMAGE): contrib/macdeploy/$(OSX_BACKGROUND_SVG) sed 's/PACKAGE_NAME/$(PACKAGE_NAME)/' < "$<" | $(RSVG_CONVERT) -f png -d $* -p $* | $(IMAGEMAGICK_CONVERT) - $@ OSX_BACKGROUND_IMAGE_DPIFILES := $(foreach dpi,$(OSX_BACKGROUND_IMAGE_DPIS),dpi$(dpi).$(OSX_BACKGROUND_IMAGE)) diff --git a/configure.ac b/configure.ac index 9d951afba1..bdfbd91445 100644 --- a/configure.ac +++ b/configure.ac @@ -670,6 +670,7 @@ case $host in AC_PATH_TOOL([INSTALLNAMETOOL], [install_name_tool], install_name_tool) AC_PATH_TOOL([OTOOL], [otool], otool) AC_PATH_PROGS([GENISOIMAGE], [genisoimage mkisofs],genisoimage) + AC_PATH_PROGS([DMG], [dmg], dmg) AC_PATH_PROGS([RSVG_CONVERT], [rsvg-convert rsvg],rsvg-convert) AC_PATH_PROGS([IMAGEMAGICK_CONVERT], [convert],convert) AC_PATH_PROGS([TIFFCP], [tiffcp],tiffcp) diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index 4119e88003..55b1dc2661 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -38,7 +38,7 @@ script: | WRAP_DIR=$HOME/wrapped HOSTS="x86_64-apple-darwin18" - CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests GENISOIMAGE=$WRAP_DIR/genisoimage" + CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests GENISOIMAGE=$WRAP_DIR/genisoimage DMG=$WRAP_DIR/dmg" FAKETIME_HOST_PROGS="" FAKETIME_PROGS="ar ranlib date dmg genisoimage" @@ -132,7 +132,6 @@ script: | make osx_volname make deploydir - OSX_VOLNAME="$(cat osx_volname)" mkdir -p unsigned-app-${i} cp osx_volname unsigned-app-${i}/ cp contrib/macdeploy/detached-sig-apply.sh unsigned-app-${i} @@ -145,8 +144,7 @@ script: | find . | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz popd - make deploy - ${WRAP_DIR}/dmg dmg "${OSX_VOLNAME}.dmg" ${OUTDIR}/${DISTNAME}-osx-unsigned.dmg + make deploy OSX_DMG="${OUTDIR}/${DISTNAME}-osx-unsigned.dmg" cd installed find . -name "lib*.la" -delete From 0df98191268fe641dd5d12e3a9f517c0c5cfacdd Mon Sep 17 00:00:00 2001 From: fanquake Date: Sat, 15 Feb 2020 20:38:18 +0800 Subject: [PATCH 2/3] build: Replace genisoimage with xorriso xorriso and its mkisofs/genisoimage emulation alter-ego xorrisofs are more maintained, and has the right toggles for us to achieve output determinism without using blunt tools like faketime. In this commit, we use xorrisofs from the build environment rather than building it ourselves using depends. This is not necessary and can be changed in the future. From https://wiki.debian.org/genisoimage?action=recall&rev=11 : > The classical command line interface for production of ISO 9660 > filesystem images is the option set established by program mkisofs. > For reasons of licensing and other problems with its author, Debian > ships a fork of mkisofs, called genisoimage, which was split off in > 2006 and then developed independently. > > Meanwhile, genisoimage gets no new features and not even bug fixes. It > is first choice only if its options -udf or -hfs are needed. > > Replacement in most uses cases, especially for bootable ISO 9660 > filesystems, archiving, and backup, is xorrisofs which starts the -as > mkisofs emulation mode of program xorriso. --- Makefile.am | 2 +- ci/test/00_setup_env_mac.sh | 2 +- configure.ac | 2 +- contrib/gitian-descriptors/gitian-osx-signer.yml | 5 +++-- contrib/gitian-descriptors/gitian-osx.yml | 7 ++++--- contrib/macdeploy/README.md | 14 +++++--------- 6 files changed, 15 insertions(+), 17 deletions(-) diff --git a/Makefile.am b/Makefile.am index 07c56a46ae..6573f91d97 100644 --- a/Makefile.am +++ b/Makefile.am @@ -138,7 +138,7 @@ $(APP_DIST_EXTRAS): $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Bitcoin-Qt .INTERMEDIATE: $(OSX_TEMP_ISO) $(OSX_TEMP_ISO): $(APP_DIST_EXTRAS) - $(GENISOIMAGE) -no-cache-inodes -D -l -probe -V "$(OSX_VOLNAME)" -no-pad -r -dir-mode 0755 -apple -o $@ dist + $(XORRISOFS) -D -l -V "$(OSX_VOLNAME)" -no-pad -r -dir-mode 0755 -o $@ dist $(OSX_DMG): $(OSX_TEMP_ISO) $(DMG) dmg "$<" "$@" diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh index 5bda37d46d..b0de2ec0bb 100644 --- a/ci/test/00_setup_env_mac.sh +++ b/ci/test/00_setup_env_mac.sh @@ -9,7 +9,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_macos_cross export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic can cross-compile to macos (bionic is used in the gitian build as well) export HOST=x86_64-apple-darwin18 -export PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools" +export PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools xorriso" export XCODE_VERSION=11.3.1 export XCODE_BUILD_ID=11C505 export RUN_UNIT_TESTS=false diff --git a/configure.ac b/configure.ac index bdfbd91445..4d176bdd3b 100644 --- a/configure.ac +++ b/configure.ac @@ -669,7 +669,7 @@ case $host in AC_PATH_TOOL([DSYMUTIL], [dsymutil], dsymutil) AC_PATH_TOOL([INSTALLNAMETOOL], [install_name_tool], install_name_tool) AC_PATH_TOOL([OTOOL], [otool], otool) - AC_PATH_PROGS([GENISOIMAGE], [genisoimage mkisofs],genisoimage) + AC_PATH_PROGS([XORRISOFS], [xorrisofs], xorrisofs) AC_PATH_PROGS([DMG], [dmg], dmg) AC_PATH_PROGS([RSVG_CONVERT], [rsvg-convert rsvg],rsvg-convert) AC_PATH_PROGS([IMAGEMAGICK_CONVERT], [convert],convert) diff --git a/contrib/gitian-descriptors/gitian-osx-signer.yml b/contrib/gitian-descriptors/gitian-osx-signer.yml index a4f3219c22..2330ff7736 100644 --- a/contrib/gitian-descriptors/gitian-osx-signer.yml +++ b/contrib/gitian-descriptors/gitian-osx-signer.yml @@ -7,6 +7,7 @@ architectures: - "amd64" packages: - "faketime" +- "xorriso" remotes: - "url": "https://github.com/bitcoin-core/bitcoin-detached-sigs.git" "dir": "signature" @@ -18,7 +19,7 @@ script: | WRAP_DIR=$HOME/wrapped mkdir -p ${WRAP_DIR} export PATH="$PWD":$PATH - FAKETIME_PROGS="dmg genisoimage" + FAKETIME_PROGS="dmg xorrisofs" # Create global faketime wrappers for prog in ${FAKETIME_PROGS}; do @@ -36,5 +37,5 @@ script: | tar -xf ${UNSIGNED} OSX_VOLNAME="$(cat osx_volname)" ./detached-sig-apply.sh ${UNSIGNED} signature/osx - ${WRAP_DIR}/genisoimage -no-cache-inodes -D -l -probe -V "${OSX_VOLNAME}" -no-pad -r -dir-mode 0755 -apple -o uncompressed.dmg signed-app + ${WRAP_DIR}/xorrisofs -D -l -V "${OSX_VOLNAME}" -no-pad -r -dir-mode 0755 -o uncompressed.dmg signed-app ${WRAP_DIR}/dmg dmg uncompressed.dmg ${OUTDIR}/${SIGNED} diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index 55b1dc2661..9a7dd13c9c 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -28,6 +28,7 @@ packages: - "python3-dev" - "python3-setuptools" - "fonts-tuffy" +- "xorriso" remotes: - "url": "https://github.com/bitcoin/bitcoin.git" "dir": "bitcoin" @@ -38,9 +39,9 @@ script: | WRAP_DIR=$HOME/wrapped HOSTS="x86_64-apple-darwin18" - CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests GENISOIMAGE=$WRAP_DIR/genisoimage DMG=$WRAP_DIR/dmg" + CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests XORRISOFS=${WRAP_DIR}/xorrisofs DMG=${WRAP_DIR}/dmg" FAKETIME_HOST_PROGS="" - FAKETIME_PROGS="ar ranlib date dmg genisoimage" + FAKETIME_PROGS="ar ranlib date dmg xorrisofs" export QT_RCC_TEST=1 export QT_RCC_SOURCE_DATE_OVERRIDE=1 @@ -136,7 +137,7 @@ script: | cp osx_volname unsigned-app-${i}/ cp contrib/macdeploy/detached-sig-apply.sh unsigned-app-${i} cp contrib/macdeploy/detached-sig-create.sh unsigned-app-${i} - cp ${BASEPREFIX}/${i}/native/bin/dmg ${BASEPREFIX}/${i}/native/bin/genisoimage unsigned-app-${i} + cp ${BASEPREFIX}/${i}/native/bin/dmg unsigned-app-${i} cp ${BASEPREFIX}/${i}/native/bin/${i}-codesign_allocate unsigned-app-${i}/codesign_allocate cp ${BASEPREFIX}/${i}/native/bin/${i}-pagestuff unsigned-app-${i}/pagestuff mv dist unsigned-app-${i} diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index 6c3db2620b..2d9a4a2153 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -92,19 +92,15 @@ created using these tools. The build process has been designed to avoid includin SDK's files in Gitian's outputs. All interim tarballs are fully deterministic and may be freely redistributed. -`genisoimage` is used to create the initial DMG. It is not deterministic as-is, so it has been -patched. A system `genisoimage` will work fine, but it will not be deterministic because -the file-order will change between invocations. The patch can be seen here: [cdrkit-deterministic.patch](https://github.com/bitcoin/bitcoin/blob/master/depends/patches/native_cdrkit/cdrkit-deterministic.patch). -No effort was made to fix this cleanly, so it likely leaks memory badly, however it's only used for -a single invocation, so that's no real concern. +[`xorrisofs`](https://www.gnu.org/software/xorriso/) is used to create the DMG. -`genisoimage` cannot compress DMGs, so afterwards, the DMG tool from the -`libdmg-hfsplus` project is used to compress it. There are several bugs in this tool and its -maintainer has seemingly abandoned the project. +`xorrisofs` cannot compress DMGs, so afterwards, the DMG tool from the +`libdmg-hfsplus` project is used to compress it. There are several bugs in this +tool and its maintainer has seemingly abandoned the project. The DMG tool has the ability to create DMGs from scratch as well, but this functionality is broken. Only the compression feature is currently used. Ideally, the creation could be fixed -and `genisoimage` would no longer be necessary. +and `xorrisofs` would no longer be necessary. Background images and other features can be added to DMG files by inserting a `.DS_Store` during creation. From 7587d11ec959f15f469bd396d4ad2697729b4ccd Mon Sep 17 00:00:00 2001 From: fanquake Date: Sat, 15 Feb 2020 20:41:14 +0800 Subject: [PATCH 3/3] build: remove cdrkit package from depends --- depends/packages/native_cdrkit.mk | 28 ------ depends/packages/packages.mk | 2 +- .../native_cdrkit/cdrkit-deterministic.patch | 86 ------------------- 3 files changed, 1 insertion(+), 115 deletions(-) delete mode 100644 depends/packages/native_cdrkit.mk delete mode 100644 depends/patches/native_cdrkit/cdrkit-deterministic.patch diff --git a/depends/packages/native_cdrkit.mk b/depends/packages/native_cdrkit.mk deleted file mode 100644 index 7bdf2d7dfd..0000000000 --- a/depends/packages/native_cdrkit.mk +++ /dev/null @@ -1,28 +0,0 @@ -package=native_cdrkit -$(package)_version=1.1.11 -$(package)_download_path=https://distro.ibiblio.org/fatdog/source/600/c -$(package)_file_name=cdrkit-$($(package)_version).tar.bz2 -$(package)_sha256_hash=b50d64c214a65b1a79afe3a964c691931a4233e2ba605d793eb85d0ac3652564 -$(package)_patches=cdrkit-deterministic.patch - -define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/cdrkit-deterministic.patch -endef - -# Starting with 10.1, GCC defaults to -fno-common, resulting in linking errors. -# Pass -fcommon to retain the legacy behaviour. -define $(package)_config_cmds - $($(package)_cmake) -DCMAKE_C_FLAGS="$$($(1)_cflags) -fcommon" -endef - -define $(package)_build_cmds - $(MAKE) genisoimage -endef - -define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) -C genisoimage install -endef - -define $(package)_postprocess_cmds - rm bin/isovfy bin/isoinfo bin/isodump bin/isodebug bin/devdump -endef diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 4b4cc7d9ff..d4fd23a47b 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -25,5 +25,5 @@ darwin_native_packages = native_ds_store native_mac_alias $(host_arch)_$(host_os)_native_packages += native_b2 ifneq ($(build_os),darwin) -darwin_native_packages += native_cctools native_cdrkit native_libdmg-hfsplus +darwin_native_packages += native_cctools native_libdmg-hfsplus endif diff --git a/depends/patches/native_cdrkit/cdrkit-deterministic.patch b/depends/patches/native_cdrkit/cdrkit-deterministic.patch deleted file mode 100644 index 8ab0993dc4..0000000000 --- a/depends/patches/native_cdrkit/cdrkit-deterministic.patch +++ /dev/null @@ -1,86 +0,0 @@ ---- cdrkit-1.1.11.old/genisoimage/tree.c 2008-10-21 19:57:47.000000000 -0400 -+++ cdrkit-1.1.11/genisoimage/tree.c 2013-12-06 00:23:18.489622668 -0500 -@@ -1139,8 +1139,9 @@ - scan_directory_tree(struct directory *this_dir, char *path, - struct directory_entry *de) - { -- DIR *current_dir; -+ int current_file; - char whole_path[PATH_MAX]; -+ struct dirent **d_list; - struct dirent *d_entry; - struct directory *parent; - int dflag; -@@ -1164,7 +1165,8 @@ - this_dir->dir_flags |= DIR_WAS_SCANNED; - - errno = 0; /* Paranoia */ -- current_dir = opendir(path); -+ //current_dir = opendir(path); -+ current_file = scandir(path, &d_list, NULL, alphasort); - d_entry = NULL; - - /* -@@ -1173,12 +1175,12 @@ - */ - old_path = path; - -- if (current_dir) { -+ if (current_file >= 0) { - errno = 0; -- d_entry = readdir(current_dir); -+ d_entry = d_list[0]; - } - -- if (!current_dir || !d_entry) { -+ if (current_file < 0 || !d_entry) { - int ret = 1; - - #ifdef USE_LIBSCHILY -@@ -1191,8 +1193,8 @@ - de->isorec.flags[0] &= ~ISO_DIRECTORY; - ret = 0; - } -- if (current_dir) -- closedir(current_dir); -+ if(d_list) -+ free(d_list); - return (ret); - } - #ifdef ABORT_DEEP_ISO_ONLY -@@ -1208,7 +1210,7 @@ - errmsgno(EX_BAD, "use Rock Ridge extensions via -R or -r,\n"); - errmsgno(EX_BAD, "or allow deep ISO9660 directory nesting via -D.\n"); - } -- closedir(current_dir); -+ free(d_list); - return (1); - } - #endif -@@ -1250,13 +1252,13 @@ - * The first time through, skip this, since we already asked - * for the first entry when we opened the directory. - */ -- if (dflag) -- d_entry = readdir(current_dir); -+ if (dflag && current_file >= 0) -+ d_entry = d_list[current_file]; - dflag++; - -- if (!d_entry) -+ if (current_file < 0) - break; -- -+ current_file--; - /* OK, got a valid entry */ - - /* If we do not want all files, then pitch the backups. */ -@@ -1348,7 +1350,7 @@ - insert_file_entry(this_dir, whole_path, d_entry->d_name); - #endif /* APPLE_HYB */ - } -- closedir(current_dir); -+ free(d_list); - - #ifdef APPLE_HYB - /*