foster/bitcoin-bitcoin-core
0
0
Fork 0
mirror of https://github.com/bitcoin/bitcoin.git synced 2025-02-02 09:46:52 -05:00
Code Issues Activity

Merge #20470: build: Replace genisoimage with xorriso

Browse source 
7587d11ec9 build: remove cdrkit package from depends (fanquake)
0df9819126 build: Replace genisoimage with xorriso (fanquake)
22437fc72e build: Run libdmg-hfsplus's DMG tool in make deploy (Carl Dong)

Pull request description:

  This is a redo of fanquake's https://github.com/bitcoin/bitcoin/pull/18151, which, aside from switching us from the deprecated `genisoimage` to the maintained `xorriso`, is also necessary for Guix to achieve determinism without using faketime.

  > xorriso and its mkisofs/genisoimage emulation alter-ego xorrisofs are
  > more maintained, and has the right toggles for us to achieve output
  > determinism without using blunt tools like faketime.
  >
  > In this commit, we use xorrisofs from the build environment rather than
  > building it ourselves using depends. This is not necessary and can be
  > changed in the future.
  >
  > From wiki.debian.org/genisoimage?action=recall&rev=11 :
  >
  > > The classical command line interface for production of ISO 9660
  > > filesystem images is the option set established by program mkisofs.
  > > For reasons of licensing and other problems with its author, Debian
  > > ships a fork of mkisofs, called genisoimage, which was split off in
  > > 2006 and then developed independently.
  > >
  > > Meanwhile, genisoimage gets no new features and not even bug fixes. It
  > > is first choice only if its options -udf or -hfs are needed.
  > >
  > > Replacement in most uses cases, especially for bootable ISO 9660
  > > filesystems, archiving, and backup, is xorrisofs which starts the -as
  > > mkisofs emulation mode of program xorriso.

ACKs for top commit:
  laanwj:
    ACK 7587d11ec9

Tree-SHA512: 62f3aad08fa8bf21192e951d7dd33b24975586d76834cfa3498f4b8cdb586cefec8cab2c073d1951a0884b5e182fd71ef2cf3accad98f84455016776ad3c5422
This commit is contained in:
Wladimir J. van der Laan 2020-12-16 21:03:31 +01:00
commit 4acbcfa97d
No known key found for this signature in database
GPG key ID: 1E4AED62986CD25D
10 changed files with 25 additions and 136 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -76,6 +76,7 @@ src/qt/bitcoin-qt.includes
*.log
*.trs
*.dmg
*.iso
*.json.h
*.raw.h

9
Makefile.am
View file

@ -35,6 +35,7 @@ space := $(empty) $(empty)
OSX_APP=Bitcoin-Qt.app
OSX_VOLNAME = $(subst $(space),-,$(PACKAGE_NAME))
OSX_DMG = $(OSX_VOLNAME).dmg
OSX_TEMP_ISO = $(OSX_DMG:.dmg=).temp.iso
OSX_BACKGROUND_SVG=background.svg
OSX_BACKGROUND_IMAGE=background.tiff
OSX_BACKGROUND_IMAGE_DPIS=36 72
@ -135,8 +136,12 @@ $(APP_DIST_DIR)/Applications:
$(APP_DIST_EXTRAS): $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Bitcoin-Qt
$(OSX_DMG): $(APP_DIST_EXTRAS)
$(GENISOIMAGE) -no-cache-inodes -D -l -probe -V "$(OSX_VOLNAME)" -no-pad -r -dir-mode 0755 -apple -o $@ dist
.INTERMEDIATE: $(OSX_TEMP_ISO)
$(OSX_TEMP_ISO): $(APP_DIST_EXTRAS)
$(XORRISOFS) -D -l -V "$(OSX_VOLNAME)" -no-pad -r -dir-mode 0755 -o $@ dist
$(OSX_DMG): $(OSX_TEMP_ISO)
$(DMG) dmg "$<" "$@"
dpi%.$(OSX_BACKGROUND_IMAGE): contrib/macdeploy/$(OSX_BACKGROUND_SVG)
sed 's/PACKAGE_NAME/$(PACKAGE_NAME)/' < "$<" | $(RSVG_CONVERT) -f png -d $* -p $* | $(IMAGEMAGICK_CONVERT) - $@

2
ci/test/00_setup_env_mac.sh
View file

@ -9,7 +9,7 @@ export LC_ALL=C.UTF-8
export CONTAINER_NAME=ci_macos_cross
export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic can cross-compile to macos (bionic is used in the gitian build as well)
export HOST=x86_64-apple-darwin18
export PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools"
export PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools xorriso"
export XCODE_VERSION=11.3.1
export XCODE_BUILD_ID=11C505
export RUN_UNIT_TESTS=false

3
configure.ac
View file

@ -675,7 +675,8 @@ case $host in
AC_PATH_TOOL([DSYMUTIL], [dsymutil], dsymutil)
AC_PATH_TOOL([INSTALLNAMETOOL], [install_name_tool], install_name_tool)
AC_PATH_TOOL([OTOOL], [otool], otool)
AC_PATH_PROGS([GENISOIMAGE], [genisoimage mkisofs],genisoimage)
AC_PATH_PROGS([XORRISOFS], [xorrisofs], xorrisofs)
AC_PATH_PROGS([DMG], [dmg], dmg)
AC_PATH_PROGS([RSVG_CONVERT], [rsvg-convert rsvg],rsvg-convert)
AC_PATH_PROGS([IMAGEMAGICK_CONVERT], [convert],convert)
AC_PATH_PROGS([TIFFCP], [tiffcp],tiffcp)

5
contrib/gitian-descriptors/gitian-osx-signer.yml
View file

@ -7,6 +7,7 @@ architectures:
- "amd64"
packages:
- "faketime"
- "xorriso"
remotes:
- "url": "https://github.com/bitcoin-core/bitcoin-detached-sigs.git"
"dir": "signature"
@ -18,7 +19,7 @@ script: |
WRAP_DIR=$HOME/wrapped
mkdir -p ${WRAP_DIR}
export PATH="$PWD":$PATH
FAKETIME_PROGS="dmg genisoimage"
FAKETIME_PROGS="dmg xorrisofs"
# Create global faketime wrappers
for prog in ${FAKETIME_PROGS}; do
@ -36,5 +37,5 @@ script: |
tar -xf ${UNSIGNED}
OSX_VOLNAME="$(cat osx_volname)"
./detached-sig-apply.sh ${UNSIGNED} signature/osx
${WRAP_DIR}/genisoimage -no-cache-inodes -D -l -probe -V "${OSX_VOLNAME}" -no-pad -r -dir-mode 0755 -apple -o uncompressed.dmg signed-app
${WRAP_DIR}/xorrisofs -D -l -V "${OSX_VOLNAME}" -no-pad -r -dir-mode 0755 -o uncompressed.dmg signed-app
${WRAP_DIR}/dmg dmg uncompressed.dmg ${OUTDIR}/${SIGNED}

11
contrib/gitian-descriptors/gitian-osx.yml
View file

@ -28,6 +28,7 @@ packages:
- "python3-dev"
- "python3-setuptools"
- "fonts-tuffy"
- "xorriso"
remotes:
- "url": "https://github.com/bitcoin/bitcoin.git"
"dir": "bitcoin"
@ -38,9 +39,9 @@ script: |
WRAP_DIR=$HOME/wrapped
HOSTS="x86_64-apple-darwin18"
CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests GENISOIMAGE=$WRAP_DIR/genisoimage"
CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests XORRISOFS=${WRAP_DIR}/xorrisofs DMG=${WRAP_DIR}/dmg"
FAKETIME_HOST_PROGS=""
FAKETIME_PROGS="ar ranlib date dmg genisoimage"
FAKETIME_PROGS="ar ranlib date dmg xorrisofs"
export QT_RCC_TEST=1
export QT_RCC_SOURCE_DATE_OVERRIDE=1
@ -132,12 +133,11 @@ script: |
make osx_volname
make deploydir
OSX_VOLNAME="$(cat osx_volname)"
mkdir -p unsigned-app-${i}
cp osx_volname unsigned-app-${i}/
cp contrib/macdeploy/detached-sig-apply.sh unsigned-app-${i}
cp contrib/macdeploy/detached-sig-create.sh unsigned-app-${i}
cp ${BASEPREFIX}/${i}/native/bin/dmg ${BASEPREFIX}/${i}/native/bin/genisoimage unsigned-app-${i}
cp ${BASEPREFIX}/${i}/native/bin/dmg unsigned-app-${i}
cp ${BASEPREFIX}/${i}/native/bin/${i}-codesign_allocate unsigned-app-${i}/codesign_allocate
cp ${BASEPREFIX}/${i}/native/bin/${i}-pagestuff unsigned-app-${i}/pagestuff
mv dist unsigned-app-${i}
@ -145,8 +145,7 @@ script: |
find . | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz
popd
make deploy
${WRAP_DIR}/dmg dmg "${OSX_VOLNAME}.dmg" ${OUTDIR}/${DISTNAME}-osx-unsigned.dmg
make deploy OSX_DMG="${OUTDIR}/${DISTNAME}-osx-unsigned.dmg"
cd installed
find . -name "lib*.la" -delete

14
contrib/macdeploy/README.md
View file

@ -92,19 +92,15 @@ created using these tools. The build process has been designed to avoid includin
SDK's files in Gitian's outputs. All interim tarballs are fully deterministic and may be freely
redistributed.
`genisoimage` is used to create the initial DMG. It is not deterministic as-is, so it has been
patched. A system `genisoimage` will work fine, but it will not be deterministic because
the file-order will change between invocations. The patch can be seen here: [cdrkit-deterministic.patch](https://github.com/bitcoin/bitcoin/blob/master/depends/patches/native_cdrkit/cdrkit-deterministic.patch).
No effort was made to fix this cleanly, so it likely leaks memory badly, however it's only used for
a single invocation, so that's no real concern.
[`xorrisofs`](https://www.gnu.org/software/xorriso/) is used to create the DMG.
`genisoimage` cannot compress DMGs, so afterwards, the DMG tool from the
`libdmg-hfsplus` project is used to compress it. There are several bugs in this tool and its
maintainer has seemingly abandoned the project.
`xorrisofs` cannot compress DMGs, so afterwards, the DMG tool from the
`libdmg-hfsplus` project is used to compress it. There are several bugs in this
tool and its maintainer has seemingly abandoned the project.
The DMG tool has the ability to create DMGs from scratch as well, but this functionality is
broken. Only the compression feature is currently used. Ideally, the creation could be fixed
and `genisoimage` would no longer be necessary.
and `xorrisofs` would no longer be necessary.
Background images and other features can be added to DMG files by inserting a
`.DS_Store` during creation.

28
depends/packages/native_cdrkit.mk
View file

@ -1,28 +0,0 @@
package=native_cdrkit
$(package)_version=1.1.11
$(package)_download_path=https://distro.ibiblio.org/fatdog/source/600/c
$(package)_file_name=cdrkit-$($(package)_version).tar.bz2
$(package)_sha256_hash=b50d64c214a65b1a79afe3a964c691931a4233e2ba605d793eb85d0ac3652564
$(package)_patches=cdrkit-deterministic.patch
define $(package)_preprocess_cmds
patch -p1 < $($(package)_patch_dir)/cdrkit-deterministic.patch
endef
# Starting with 10.1, GCC defaults to -fno-common, resulting in linking errors.
# Pass -fcommon to retain the legacy behaviour.
define $(package)_config_cmds
$($(package)_cmake) -DCMAKE_C_FLAGS="$$($(1)_cflags) -fcommon"
endef
define $(package)_build_cmds
$(MAKE) genisoimage
endef
define $(package)_stage_cmds
$(MAKE) DESTDIR=$($(package)_staging_dir) -C genisoimage install
endef
define $(package)_postprocess_cmds
rm bin/isovfy bin/isoinfo bin/isodump bin/isodebug bin/devdump
endef

2
depends/packages/packages.mk
View file

@ -25,5 +25,5 @@ darwin_native_packages = native_ds_store native_mac_alias
$(host_arch)_$(host_os)_native_packages += native_b2
ifneq ($(build_os),darwin)
darwin_native_packages += native_cctools native_cdrkit native_libdmg-hfsplus
darwin_native_packages += native_cctools native_libdmg-hfsplus
endif

86
depends/patches/native_cdrkit/cdrkit-deterministic.patch
View file

@ -1,86 +0,0 @@
--- cdrkit-1.1.11.old/genisoimage/tree.c 2008-10-21 19:57:47.000000000 -0400
+++ cdrkit-1.1.11/genisoimage/tree.c 2013-12-06 00:23:18.489622668 -0500
@@ -1139,8 +1139,9 @@
scan_directory_tree(struct directory *this_dir, char *path,
struct directory_entry *de)
{
- DIR *current_dir;
+ int current_file;
char whole_path[PATH_MAX];
+ struct dirent **d_list;
struct dirent *d_entry;
struct directory *parent;
int dflag;
@@ -1164,7 +1165,8 @@
this_dir->dir_flags |= DIR_WAS_SCANNED;
errno = 0; /* Paranoia */
- current_dir = opendir(path);
+ //current_dir = opendir(path);
+ current_file = scandir(path, &d_list, NULL, alphasort);
d_entry = NULL;
/*
@@ -1173,12 +1175,12 @@
*/
old_path = path;
- if (current_dir) {
+ if (current_file >= 0) {
errno = 0;
- d_entry = readdir(current_dir);
+ d_entry = d_list[0];
}
- if (!current_dir || !d_entry) {
+ if (current_file < 0 || !d_entry) {
int ret = 1;
#ifdef USE_LIBSCHILY
@@ -1191,8 +1193,8 @@
de->isorec.flags[0] &= ~ISO_DIRECTORY;
ret = 0;
}
- if (current_dir)
- closedir(current_dir);
+ if(d_list)
+ free(d_list);
return (ret);
}
#ifdef ABORT_DEEP_ISO_ONLY
@@ -1208,7 +1210,7 @@
errmsgno(EX_BAD, "use Rock Ridge extensions via -R or -r,\n");
errmsgno(EX_BAD, "or allow deep ISO9660 directory nesting via -D.\n");
}
- closedir(current_dir);
+ free(d_list);
return (1);
}
#endif
@@ -1250,13 +1252,13 @@
* The first time through, skip this, since we already asked
* for the first entry when we opened the directory.
*/
- if (dflag)
- d_entry = readdir(current_dir);
+ if (dflag && current_file >= 0)
+ d_entry = d_list[current_file];
dflag++;
- if (!d_entry)
+ if (current_file < 0)
break;
-
+ current_file--;
/* OK, got a valid entry */
/* If we do not want all files, then pitch the backups. */
@@ -1348,7 +1350,7 @@
insert_file_entry(this_dir, whole_path, d_entry->d_name);
#endif /* APPLE_HYB */
}
- closedir(current_dir);
+ free(d_list);
#ifdef APPLE_HYB
/*