From ca18aea5c4975ace4e307be96c74641d203fa389 Mon Sep 17 00:00:00 2001 From: Ava Chow Date: Tue, 2 Jan 2024 16:34:50 -0500 Subject: [PATCH 01/16] Add AutoFile::seek and tell It's useful to be able to seek to a specific position in a file. Allow AutoFile to seek by using fseek. It's also useful to be able to get the current position in a file. Allow AutoFile to tell by using ftell. --- src/streams.cpp | 22 ++++++++++++++++++++++ src/streams.h | 3 +++ 2 files changed, 25 insertions(+) diff --git a/src/streams.cpp b/src/streams.cpp index 6921dad677..cdd36a86fe 100644 --- a/src/streams.cpp +++ b/src/streams.cpp @@ -21,6 +21,28 @@ std::size_t AutoFile::detail_fread(Span dst) } } +void AutoFile::seek(int64_t offset, int origin) +{ + if (IsNull()) { + throw std::ios_base::failure("AutoFile::seek: file handle is nullptr"); + } + if (std::fseek(m_file, offset, origin) != 0) { + throw std::ios_base::failure(feof() ? "AutoFile::seek: end of file" : "AutoFile::seek: fseek failed"); + } +} + +int64_t AutoFile::tell() +{ + if (IsNull()) { + throw std::ios_base::failure("AutoFile::tell: file handle is nullptr"); + } + int64_t r{std::ftell(m_file)}; + if (r < 0) { + throw std::ios_base::failure("AutoFile::tell: ftell failed"); + } + return r; +} + void AutoFile::read(Span dst) { if (detail_fread(dst) != dst.size()) { diff --git a/src/streams.h b/src/streams.h index bc04a2babd..57fc600646 100644 --- a/src/streams.h +++ b/src/streams.h @@ -435,6 +435,9 @@ public: /** Implementation detail, only used internally. */ std::size_t detail_fread(Span dst); + void seek(int64_t offset, int origin); + int64_t tell(); + // // Stream subset // From 756ff9b478484b17c4a6e65c171c2e4fecb21ad4 Mon Sep 17 00:00:00 2001 From: Ava Chow Date: Tue, 2 Jan 2024 16:35:07 -0500 Subject: [PATCH 02/16] wallet: add dummy BerkeleyRODatabase and BerkeleyROBatch classes BerkeleyRODatabase and BerkeleyROBatch will be used to access a BDB file without the use of BDB. For now, these are dummy classes. --- src/Makefile.am | 2 + src/wallet/migrate.cpp | 37 ++++++++++++++ src/wallet/migrate.h | 108 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 147 insertions(+) create mode 100644 src/wallet/migrate.cpp create mode 100644 src/wallet/migrate.h diff --git a/src/Makefile.am b/src/Makefile.am index 11b8f6e4c3..0db9812f93 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -347,6 +347,7 @@ BITCOIN_CORE_H = \ wallet/feebumper.h \ wallet/fees.h \ wallet/load.h \ + wallet/migrate.h \ wallet/receive.h \ wallet/rpc/util.h \ wallet/rpc/wallet.h \ @@ -507,6 +508,7 @@ libbitcoin_wallet_a_SOURCES = \ wallet/fees.cpp \ wallet/interfaces.cpp \ wallet/load.cpp \ + wallet/migrate.cpp \ wallet/receive.cpp \ wallet/rpc/addresses.cpp \ wallet/rpc/backup.cpp \ diff --git a/src/wallet/migrate.cpp b/src/wallet/migrate.cpp new file mode 100644 index 0000000000..e567cb0f51 --- /dev/null +++ b/src/wallet/migrate.cpp @@ -0,0 +1,37 @@ +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +namespace wallet { + +void BerkeleyRODatabase::Open() +{ +} + +std::unique_ptr BerkeleyRODatabase::MakeBatch(bool flush_on_close) +{ + return std::make_unique(*this); +} + +bool BerkeleyRODatabase::Backup(const std::string& dest) const +{ + return false; +} + +bool BerkeleyROBatch::ReadKey(DataStream&& key, DataStream& value) +{ + return false; +} + +bool BerkeleyROBatch::HasKey(DataStream&& key) +{ + return false; +} + +DatabaseCursor::Status BerkeleyROCursor::Next(DataStream& ssKey, DataStream& ssValue) +{ + return DatabaseCursor::Status::FAIL; +} + +} // namespace wallet diff --git a/src/wallet/migrate.h b/src/wallet/migrate.h new file mode 100644 index 0000000000..de7082e8a0 --- /dev/null +++ b/src/wallet/migrate.h @@ -0,0 +1,108 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_WALLET_MIGRATE_H +#define BITCOIN_WALLET_MIGRATE_H + +#include + +#include + +namespace wallet { +/** + * A class representing a BerkeleyDB file from which we can only read records. + * This is used only for migration of legacy to descriptor wallets + */ +class BerkeleyRODatabase : public WalletDatabase +{ +private: + const fs::path m_filepath; + +public: + /** Create DB handle */ + BerkeleyRODatabase(const fs::path& filepath, bool open = true) : WalletDatabase(), m_filepath(filepath) + { + if (open) Open(); + } + ~BerkeleyRODatabase(){}; + + /** Open the database if it is not already opened. */ + void Open() override; + + /** Indicate the a new database user has began using the database. Increments m_refcount */ + void AddRef() override {} + /** Indicate that database user has stopped using the database and that it could be flushed or closed. Decrement m_refcount */ + void RemoveRef() override {} + + /** Rewrite the entire database on disk, with the exception of key pszSkip if non-zero + */ + bool Rewrite(const char* pszSkip = nullptr) override { return false; } + + /** Back up the entire database to a file. + */ + bool Backup(const std::string& strDest) const override; + + /** Make sure all changes are flushed to database file. + */ + void Flush() override {} + /** Flush to the database file and close the database. + * Also close the environment if no other databases are open in it. + */ + void Close() override {} + /* flush the wallet passively (TRY_LOCK) + ideal to be called periodically */ + bool PeriodicFlush() override { return false; } + + void IncrementUpdateCounter() override {} + + void ReloadDbEnv() override {} + + /** Return path to main database file for logs and error messages. */ + std::string Filename() override { return fs::PathToString(m_filepath); } + + std::string Format() override { return "bdb_ro"; } + + /** Make a DatabaseBatch connected to this database */ + std::unique_ptr MakeBatch(bool flush_on_close = true) override; +}; + +class BerkeleyROCursor : public DatabaseCursor +{ +public: + Status Next(DataStream& key, DataStream& value) override; +}; + +/** RAII class that provides access to a BerkeleyRODatabase */ +class BerkeleyROBatch : public DatabaseBatch +{ +private: + const BerkeleyRODatabase& m_database; + + bool ReadKey(DataStream&& key, DataStream& value) override; + // WriteKey returns true since various automatic upgrades for older wallets will expect writing to not fail. + // It is okay for this batch type to not actually write anything as those automatic upgrades will occur again after migration. + bool WriteKey(DataStream&& key, DataStream&& value, bool overwrite = true) override { return true; } + bool EraseKey(DataStream&& key) override { return false; } + bool HasKey(DataStream&& key) override; + bool ErasePrefix(Span prefix) override { return false; } + +public: + explicit BerkeleyROBatch(const BerkeleyRODatabase& database) : m_database(database) {} + ~BerkeleyROBatch() {} + + BerkeleyROBatch(const BerkeleyROBatch&) = delete; + BerkeleyROBatch& operator=(const BerkeleyROBatch&) = delete; + + void Flush() override {} + void Close() override {} + + std::unique_ptr GetNewCursor() override { return std::make_unique(); } + std::unique_ptr GetNewPrefixCursor(Span prefix) override { return std::make_unique(); } + bool TxnBegin() override { return false; } + bool TxnCommit() override { return false; } + bool TxnAbort() override { return false; } +}; +} // namespace wallet + +#endif // BITCOIN_WALLET_MIGRATE_H From 0c8e72847603540bb29b8b8aeb80fa3f2e3a2c9a Mon Sep 17 00:00:00 2001 From: Ava Chow Date: Tue, 2 Jan 2024 16:35:22 -0500 Subject: [PATCH 03/16] wallet: implement BerkeleyROBatch Implement ReadKey and HasKey of BerkeleyROBatch, and Next of BerkeleyROCursor. Also adds the containers for records to BerkeleyRODatabase so that BerkeleyROBatch will be able to access the records. --- src/wallet/db.cpp | 3 +++ src/wallet/db.h | 6 ++++++ src/wallet/migrate.cpp | 31 ++++++++++++++++++++++++++++--- src/wallet/migrate.h | 17 +++++++++++++++-- src/wallet/test/util.cpp | 5 ----- 5 files changed, 52 insertions(+), 10 deletions(-) diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index ea06767e9b..a5a5f8ec6f 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -16,6 +16,9 @@ #include namespace wallet { +bool operator<(BytePrefix a, Span b) { return a.prefix < b.subspan(0, std::min(a.prefix.size(), b.size())); } +bool operator<(Span a, BytePrefix b) { return a.subspan(0, std::min(a.size(), b.prefix.size())) < b.prefix; } + std::vector ListDatabases(const fs::path& wallet_dir) { std::vector paths; diff --git a/src/wallet/db.h b/src/wallet/db.h index 084fcadc24..648adff5fe 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -20,6 +20,12 @@ class ArgsManager; struct bilingual_str; namespace wallet { +// BytePrefix compares equality with other byte spans that begin with the same prefix. +struct BytePrefix { + Span prefix; +}; +bool operator<(BytePrefix a, Span b); +bool operator<(Span a, BytePrefix b); class DatabaseCursor { diff --git a/src/wallet/migrate.cpp b/src/wallet/migrate.cpp index e567cb0f51..f922d99a17 100644 --- a/src/wallet/migrate.cpp +++ b/src/wallet/migrate.cpp @@ -21,17 +21,42 @@ bool BerkeleyRODatabase::Backup(const std::string& dest) const bool BerkeleyROBatch::ReadKey(DataStream&& key, DataStream& value) { - return false; + SerializeData key_data{key.begin(), key.end()}; + const auto it{m_database.m_records.find(key_data)}; + if (it == m_database.m_records.end()) { + return false; + } + auto val = it->second; + value.clear(); + value.write(Span(val)); + return true; } bool BerkeleyROBatch::HasKey(DataStream&& key) { - return false; + SerializeData key_data{key.begin(), key.end()}; + return m_database.m_records.count(key_data) > 0; +} + +BerkeleyROCursor::BerkeleyROCursor(const BerkeleyRODatabase& database, Span prefix) + : m_database(database) +{ + std::tie(m_cursor, m_cursor_end) = m_database.m_records.equal_range(BytePrefix{prefix}); } DatabaseCursor::Status BerkeleyROCursor::Next(DataStream& ssKey, DataStream& ssValue) { - return DatabaseCursor::Status::FAIL; + if (m_cursor == m_cursor_end) { + return DatabaseCursor::Status::DONE; + } + ssKey.write(Span(m_cursor->first)); + ssValue.write(Span(m_cursor->second)); + m_cursor++; + return DatabaseCursor::Status::MORE; } +std::unique_ptr BerkeleyROBatch::GetNewPrefixCursor(Span prefix) +{ + return std::make_unique(m_database, prefix); +} } // namespace wallet diff --git a/src/wallet/migrate.h b/src/wallet/migrate.h index de7082e8a0..a3b0d78d02 100644 --- a/src/wallet/migrate.h +++ b/src/wallet/migrate.h @@ -10,6 +10,9 @@ #include namespace wallet { + +using BerkeleyROData = std::map>; + /** * A class representing a BerkeleyDB file from which we can only read records. * This is used only for migration of legacy to descriptor wallets @@ -27,6 +30,8 @@ public: } ~BerkeleyRODatabase(){}; + BerkeleyROData m_records; + /** Open the database if it is not already opened. */ void Open() override; @@ -69,7 +74,15 @@ public: class BerkeleyROCursor : public DatabaseCursor { +private: + const BerkeleyRODatabase& m_database; + BerkeleyROData::const_iterator m_cursor; + BerkeleyROData::const_iterator m_cursor_end; + public: + explicit BerkeleyROCursor(const BerkeleyRODatabase& database, Span prefix = {}); + ~BerkeleyROCursor() {} + Status Next(DataStream& key, DataStream& value) override; }; @@ -97,8 +110,8 @@ public: void Flush() override {} void Close() override {} - std::unique_ptr GetNewCursor() override { return std::make_unique(); } - std::unique_ptr GetNewPrefixCursor(Span prefix) override { return std::make_unique(); } + std::unique_ptr GetNewCursor() override { return std::make_unique(m_database); } + std::unique_ptr GetNewPrefixCursor(Span prefix) override; bool TxnBegin() override { return false; } bool TxnCommit() override { return false; } bool TxnAbort() override { return false; } diff --git a/src/wallet/test/util.cpp b/src/wallet/test/util.cpp index 49d206f409..b21a9a601d 100644 --- a/src/wallet/test/util.cpp +++ b/src/wallet/test/util.cpp @@ -93,11 +93,6 @@ CTxDestination getNewDestination(CWallet& w, OutputType output_type) return *Assert(w.GetNewDestination(output_type, "")); } -// BytePrefix compares equality with other byte spans that begin with the same prefix. -struct BytePrefix { Span prefix; }; -bool operator<(BytePrefix a, Span b) { return a.prefix < b.subspan(0, std::min(a.prefix.size(), b.size())); } -bool operator<(Span a, BytePrefix b) { return a.subspan(0, std::min(a.size(), b.prefix.size())) < b.prefix; } - MockableCursor::MockableCursor(const MockableData& records, bool pass, Span prefix) { m_pass = pass; From ecba23097955dad7208baa687fc405c846aee794 Mon Sep 17 00:00:00 2001 From: Ava Chow Date: Tue, 2 Jan 2024 16:35:30 -0500 Subject: [PATCH 04/16] wallet: implement BerkeleyRODatabase::Backup --- src/wallet/migrate.cpp | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/wallet/migrate.cpp b/src/wallet/migrate.cpp index f922d99a17..4c37dfee6f 100644 --- a/src/wallet/migrate.cpp +++ b/src/wallet/migrate.cpp @@ -1,6 +1,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include #include namespace wallet { @@ -16,7 +17,25 @@ std::unique_ptr BerkeleyRODatabase::MakeBatch(bool flush_on_close bool BerkeleyRODatabase::Backup(const std::string& dest) const { - return false; + fs::path src(m_filepath); + fs::path dst(fs::PathFromString(dest)); + + if (fs::is_directory(dst)) { + dst = BDBDataFile(dst); + } + try { + if (fs::exists(dst) && fs::equivalent(src, dst)) { + LogPrintf("cannot backup to wallet source file %s\n", fs::PathToString(dst)); + return false; + } + + fs::copy_file(src, dst, fs::copy_options::overwrite_existing); + LogPrintf("copied %s to %s\n", fs::PathToString(m_filepath), fs::PathToString(dst)); + return true; + } catch (const fs::filesystem_error& e) { + LogPrintf("error copying %s to %s - %s\n", fs::PathToString(m_filepath), fs::PathToString(dst), fsbridge::get_filesystem_error_message(e)); + return false; + } } bool BerkeleyROBatch::ReadKey(DataStream&& key, DataStream& value) From cdd61c9cc108df8e13f4e3891ff2c96355b3ee38 Mon Sep 17 00:00:00 2001 From: Ava Chow Date: Tue, 2 Jan 2024 16:35:37 -0500 Subject: [PATCH 05/16] wallet: implement independent BDB deserializer in BerkeleyRODatabase BerkeleyRODatabase is intended for use after BDB is removed, so it needs to be able to read all of the records from a BDB file. Thus an independent deserializer for BDB data files is implemented in it. This deserializer is targeted towards the data files that Bitcoin Core creates so it does not fully support all of BDB's features (e.g. other database types, encryption, etc.). --- src/wallet/migrate.cpp | 607 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 607 insertions(+) diff --git a/src/wallet/migrate.cpp b/src/wallet/migrate.cpp index 4c37dfee6f..c2b4e22f6e 100644 --- a/src/wallet/migrate.cpp +++ b/src/wallet/migrate.cpp @@ -1,13 +1,620 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include // For ReadBE32 #include +#include #include +#include +#include + namespace wallet { +// Magic bytes in both endianness's +constexpr uint32_t BTREE_MAGIC = 0x00053162; // If the file endianness matches our system, we see this magic +constexpr uint32_t BTREE_MAGIC_OE = 0x62310500; // If the file endianness is the other one, we will see this magic + +// Subdatabase name +static const std::vector SUBDATABASE_NAME = {std::byte{'m'}, std::byte{'a'}, std::byte{'i'}, std::byte{'n'}}; + +enum class PageType : uint8_t { + /* + * BDB has several page types, most of which we do not use + * They are listed here for completeness, but commented out + * to avoid opening something unintended. + INVALID = 0, // Invalid page type + DUPLICATE = 1, // Duplicate. Deprecated and no longer used + HASH_UNSORTED = 2, // Hash pages. Deprecated. + RECNO_INTERNAL = 4, // Recno internal + RECNO_LEAF = 6, // Recno leaf + HASH_META = 8, // Hash metadata + QUEUE_META = 10, // Queue Metadata + QUEUE_DATA = 11, // Queue Data + DUPLICATE_LEAF = 12, // Off-page duplicate leaf + HASH_SORTED = 13, // Sorted hash page + */ + BTREE_INTERNAL = 3, // BTree internal + BTREE_LEAF = 5, // BTree leaf + OVERFLOW_DATA = 7, // Overflow + BTREE_META = 9, // BTree metadata +}; + +enum class RecordType : uint8_t { + KEYDATA = 1, + // DUPLICATE = 2, Unused as our databases do not support duplicate records + OVERFLOW_DATA = 3, + DELETE = 0x80, // Indicate this record is deleted. This is OR'd with the real type. +}; + +enum class BTreeFlags : uint32_t { + /* + * BTree databases have feature flags, but we do not use them except for + * subdatabases. The unused flags are included for completeness, but commented out + * to avoid accidental use. + DUP = 1, // Duplicates + RECNO = 2, // Recno tree + RECNUM = 4, // BTree: Maintain record counts + FIXEDLEN = 8, // Recno: fixed length records + RENUMBER = 0x10, // Recno: renumber on insert/delete + DUPSORT = 0x40, // Duplicates are sorted + COMPRESS = 0x80, // Compressed + */ + SUBDB = 0x20, // Subdatabases +}; + +/** Berkeley DB BTree metadata page layout */ +class MetaPage +{ +public: + uint32_t lsn_file; // Log Sequence Number file + uint32_t lsn_offset; // Log Sequence Number offset + uint32_t page_num; // Current page number + uint32_t magic; // Magic number + uint32_t version; // Version + uint32_t pagesize; // Page size + uint8_t encrypt_algo; // Encryption algorithm + PageType type; // Page type + uint8_t metaflags; // Meta-only flags + uint8_t unused1; // Unused + uint32_t free_list; // Free list page number + uint32_t last_page; // Page number of last page in db + uint32_t partitions; // Number of partitions + uint32_t key_count; // Cached key count + uint32_t record_count; // Cached record count + BTreeFlags flags; // Flags + std::array uid; // 20 byte unique file ID + uint32_t unused2; // Unused + uint32_t minkey; // Minimum key + uint32_t re_len; // Recno: fixed length record length + uint32_t re_pad; // Recno: fixed length record pad + uint32_t root; // Root page number + char unused3[368]; // 92 * 4 bytes of unused space + uint32_t crypto_magic; // Crypto magic number + char trash[12]; // 3 * 4 bytes of trash space + unsigned char iv[20]; // Crypto IV + unsigned char chksum[16]; // Checksum + + uint32_t expected_page_num; + + MetaPage(uint32_t expected_page_num) : expected_page_num(expected_page_num) {} + MetaPage() = delete; + + template + void Unserialize(Stream& s) + { + s >> lsn_file; + s >> lsn_offset; + s >> page_num; + s >> magic; + s >> version; + s >> pagesize; + s >> encrypt_algo; + + uint8_t uint8_type; + s >> uint8_type; + type = static_cast(uint8_type); + + s >> metaflags; + s >> unused1; + s >> free_list; + s >> last_page; + s >> partitions; + s >> key_count; + s >> record_count; + + uint32_t uint32_flags; + s >> uint32_flags; + flags = static_cast(uint32_flags); + + s >> uid; + s >> unused2; + s >> minkey; + s >> re_len; + s >> re_pad; + s >> root; + s >> unused3; + s >> crypto_magic; + s >> trash; + s >> iv; + s >> chksum; + + // Page number must match + if (page_num != expected_page_num) { + throw std::runtime_error("Meta page number mismatch"); + } + + // Check magic + if (magic != BTREE_MAGIC) { + throw std::runtime_error("Not a BDB file"); + } + + // Only version 9 is supported + if (version != 9) { + throw std::runtime_error("Unsupported BDB data file version number"); + } + + // Page size must be 512 <= pagesize <= 64k, and be a power of 2 + if (pagesize < 512 || pagesize > 65536 || (pagesize & (pagesize - 1)) != 0) { + throw std::runtime_error("Bad page size"); + } + + // Page type must be the btree type + if (type != PageType::BTREE_META) { + throw std::runtime_error("Unexpected page type, should be 9 (BTree Metadata)"); + } + + // Only supported meta-flag is subdatabase + if (flags != BTreeFlags::SUBDB) { + throw std::runtime_error("Unexpected database flags, should only be 0x20 (subdatabases)"); + } + } +}; + +/** General class for records in a BDB BTree database. Contains common fields. */ +class RecordHeader +{ +public: + uint16_t len; // Key/data item length + RecordType type; // Page type (BDB has this include a DELETE FLAG that we track separately) + bool deleted; // Whether the DELETE flag was set on type + + static constexpr size_t SIZE = 3; // The record header is 3 bytes + + template + void Unserialize(Stream& s) + { + s >> len; + + uint8_t uint8_type; + s >> uint8_type; + type = static_cast(uint8_type & ~static_cast(RecordType::DELETE)); + deleted = uint8_type & static_cast(RecordType::DELETE); + } +}; + +/** Class for data in the record directly */ +class DataRecord +{ +public: + DataRecord(const RecordHeader& header) : m_header(header) {} + DataRecord() = delete; + + RecordHeader m_header; + + std::vector data; // Variable length key/data item + + template + void Unserialize(Stream& s) + { + data.resize(m_header.len); + s.read(AsWritableBytes(Span(data.data(), data.size()))); + } +}; + +/** Class for records representing internal nodes of the BTree. */ +class InternalRecord +{ +public: + InternalRecord(const RecordHeader& header) : m_header(header) {} + InternalRecord() = delete; + + RecordHeader m_header; + + uint8_t unused; // Padding, unused + uint32_t page_num; // Page number of referenced page + uint32_t records; // Subtree record count + std::vector data; // Variable length key item + + static constexpr size_t FIXED_SIZE = 9; // Size of fixed data is 9 bytes + + template + void Unserialize(Stream& s) + { + s >> unused; + s >> page_num; + s >> records; + + data.resize(m_header.len); + s.read(AsWritableBytes(Span(data.data(), data.size()))); + } +}; + +/** Class for records representing overflow records of the BTree. + * Overflow records point to a page which contains the data in the record. + * Those pages may point to further pages with the rest of the data if it does not fit + * in one page */ +class OverflowRecord +{ +public: + OverflowRecord(const RecordHeader& header) : m_header(header) {} + OverflowRecord() = delete; + + RecordHeader m_header; + + uint8_t unused2; // Padding, unused + uint32_t page_number; // Page number where data begins + uint32_t item_len; // Total length of item + + static constexpr size_t SIZE = 9; // Overflow record is always 9 bytes + + template + void Unserialize(Stream& s) + { + s >> unused2; + s >> page_number; + s >> item_len; + } +}; + +/** A generic data page in the database. Contains fields common to all data pages. */ +class PageHeader +{ +public: + uint32_t lsn_file; // Log Sequence Number file + uint32_t lsn_offset; // Log Sequence Number offset + uint32_t page_num; // Current page number + uint32_t prev_page; // Previous page number + uint32_t next_page; // Next page number + uint16_t entries; // Number of items on the page + uint16_t hf_offset; // High free byte page offset + uint8_t level; // Btree page level + PageType type; // Page type + + static constexpr int64_t SIZE = 26; // The header is 26 bytes + + uint32_t expected_page_num; + + PageHeader(uint32_t page_num) : expected_page_num(page_num) {} + PageHeader() = delete; + + template + void Unserialize(Stream& s) + { + s >> lsn_file; + s >> lsn_offset; + s >> page_num; + s >> prev_page; + s >> next_page; + s >> entries; + s >> hf_offset; + s >> level; + + uint8_t uint8_type; + s >> uint8_type; + type = static_cast(uint8_type); + + if (expected_page_num != page_num) { + throw std::runtime_error("Page number mismatch"); + } + if ((type != PageType::OVERFLOW_DATA && level < 1) || (type == PageType::OVERFLOW_DATA && level != 0)) { + throw std::runtime_error("Bad btree level"); + } + } +}; + +/** A page of records in the database */ +class RecordsPage +{ +public: + RecordsPage(const PageHeader& header) : m_header(header) {} + RecordsPage() = delete; + + PageHeader m_header; + + std::vector indexes; + std::vector> records; + + template + void Unserialize(Stream& s) + { + // Current position within the page + int64_t pos = PageHeader::SIZE; + + // Get the items + for (uint32_t i = 0; i < m_header.entries; ++i) { + // Get the index + uint16_t index; + s >> index; + indexes.push_back(index); + pos += sizeof(uint16_t); + + // Go to the offset from the index + int64_t to_jump = index - pos; + if (to_jump < 0) { + throw std::runtime_error("Data record position not in page"); + } + s.ignore(to_jump); + + // Read the record + RecordHeader rec_hdr; + s >> rec_hdr; + to_jump += RecordHeader::SIZE; + + switch (rec_hdr.type) { + case RecordType::KEYDATA: { + DataRecord record(rec_hdr); + s >> record; + records.emplace_back(record); + to_jump += rec_hdr.len; + break; + } + case RecordType::OVERFLOW_DATA: { + OverflowRecord record(rec_hdr); + s >> record; + records.emplace_back(record); + to_jump += OverflowRecord::SIZE; + break; + } + default: + throw std::runtime_error("Unknown record type in records page"); + } + + // Go back to the indexes + s.seek(-to_jump, SEEK_CUR); + } + } +}; + +/** A page containing overflow data */ +class OverflowPage +{ +public: + OverflowPage(const PageHeader& header) : m_header(header) {} + OverflowPage() = delete; + + PageHeader m_header; + + // BDB overloads some page fields to store overflow page data + // hf_offset contains the length of the overflow data stored on this page + // entries contains a reference count for references to this item + + // The overflow data itself. Begins immediately following header + std::vector data; + + template + void Unserialize(Stream& s) + { + data.resize(m_header.hf_offset); + s.read(AsWritableBytes(Span(data.data(), data.size()))); + } +}; + +/** A page of records in the database */ +class InternalPage +{ +public: + InternalPage(const PageHeader& header) : m_header(header) {} + InternalPage() = delete; + + PageHeader m_header; + + std::vector indexes; + std::vector records; + + template + void Unserialize(Stream& s) + { + // Current position within the page + int64_t pos = PageHeader::SIZE; + + // Get the items + for (uint32_t i = 0; i < m_header.entries; ++i) { + // Get the index + uint16_t index; + s >> index; + indexes.push_back(index); + pos += sizeof(uint16_t); + + // Go to the offset from the index + int64_t to_jump = index - pos; + if (to_jump < 0) { + throw std::runtime_error("Internal record position not in page"); + } + s.ignore(to_jump); + + // Read the record + RecordHeader rec_hdr; + s >> rec_hdr; + to_jump += RecordHeader::SIZE; + + if (rec_hdr.type != RecordType::KEYDATA) { + throw std::runtime_error("Unknown record type in internal page"); + } + InternalRecord record(rec_hdr); + s >> record; + records.emplace_back(record); + to_jump += InternalRecord::FIXED_SIZE + rec_hdr.len; + + // Go back to the indexes + s.seek(-to_jump, SEEK_CUR); + } + } +}; + +static void SeekToPage(AutoFile& s, uint32_t page_num, uint32_t page_size) +{ + int64_t pos = int64_t{page_num} * page_size; + s.seek(pos, SEEK_SET); +} void BerkeleyRODatabase::Open() { + // Open the file + FILE* file = fsbridge::fopen(m_filepath, "rb"); + AutoFile db_file(file); + if (db_file.IsNull()) { + throw std::runtime_error("BerkeleyRODatabase: Failed to open database file"); + } + + uint32_t page_size = 4096; // Default page size + + // Read the outer metapage + // Expected page number is 0 + MetaPage outer_meta(0); + db_file >> outer_meta; + page_size = outer_meta.pagesize; + + // Verify the size of the file is a multiple of the page size + db_file.seek(0, SEEK_END); + int64_t size = db_file.tell(); + + // Since BDB stores everything in a page, the file size should be a multiple of the page size; + // However, BDB doesn't actually check that this is the case, and enforcing this check results + // in us rejecting a database that BDB would not, so this check needs to be excluded. + // This is left commented out as a reminder to not accidentally implement this in the future. + // if (size % page_size != 0) { + // throw std::runtime_error("File size is not a multiple of page size"); + // } + + // Check the last page number + uint32_t expected_last_page = (size / page_size) - 1; + if (outer_meta.last_page != expected_last_page) { + throw std::runtime_error("Last page number could not fit in file"); + } + + // Make sure encryption is disabled + if (outer_meta.encrypt_algo != 0) { + throw std::runtime_error("BDB builtin encryption is not supported"); + } + + // Read the root page + SeekToPage(db_file, outer_meta.root, page_size); + PageHeader header(outer_meta.root); + db_file >> header; + if (header.type != PageType::BTREE_LEAF) { + throw std::runtime_error("Unexpected outer database root page type"); + } + if (header.entries != 2) { + throw std::runtime_error("Unexpected number of entries in outer database root page"); + } + RecordsPage page(header); + db_file >> page; + + // First record should be the string "main" + if (!std::holds_alternative(page.records.at(0)) || std::get(page.records.at(0)).data != SUBDATABASE_NAME) { + throw std::runtime_error("Subdatabase has an unexpected name"); + } + // Check length of page number for subdatabase location + if (!std::holds_alternative(page.records.at(1)) || std::get(page.records.at(1)).m_header.len != 4) { + throw std::runtime_error("Subdatabase page number has unexpected length"); + } + + // Read subdatabase page number + // It is written as a big endian 32 bit number + uint32_t main_db_page = ReadBE32(UCharCast(std::get(page.records.at(1)).data.data())); + + // The main database is in a page that doesn't exist + if (main_db_page > outer_meta.last_page) { + throw std::runtime_error("Page number is greater than database last page"); + } + + // Read the inner metapage + SeekToPage(db_file, main_db_page, page_size); + MetaPage inner_meta(main_db_page); + db_file >> inner_meta; + + if (inner_meta.pagesize != page_size) { + throw std::runtime_error("Unexpected page size"); + } + + if (inner_meta.last_page > outer_meta.last_page) { + throw std::runtime_error("Subdatabase last page is greater than database last page"); + } + + // Make sure encryption is disabled + if (inner_meta.encrypt_algo != 0) { + throw std::runtime_error("BDB builtin encryption is not supported"); + } + + // Do a DFS through the BTree, starting at root + std::vector pages{inner_meta.root}; + while (pages.size() > 0) { + uint32_t curr_page = pages.back(); + // It turns out BDB completely ignores this last_page field and doesn't actually update it to the correct + // last page. While we should be checking this, we can't. + // This is left commented out as a reminder to not accidentally implement this in the future. + // if (curr_page > inner_meta.last_page) { + // throw std::runtime_error("Page number is greater than subdatabase last page"); + // } + pages.pop_back(); + SeekToPage(db_file, curr_page, page_size); + PageHeader header(curr_page); + db_file >> header; + switch (header.type) { + case PageType::BTREE_INTERNAL: { + InternalPage int_page(header); + db_file >> int_page; + for (const InternalRecord& rec : int_page.records) { + if (rec.m_header.deleted) continue; + pages.push_back(rec.page_num); + } + break; + } + case PageType::BTREE_LEAF: { + RecordsPage rec_page(header); + db_file >> rec_page; + if (rec_page.records.size() % 2 != 0) { + // BDB stores key value pairs in consecutive records, thus an odd number of records is unexpected + throw std::runtime_error("Records page has odd number of records"); + } + bool is_key = true; + std::vector key; + for (const std::variant& rec : rec_page.records) { + std::vector data; + if (const DataRecord* drec = std::get_if(&rec)) { + if (drec->m_header.deleted) continue; + data = drec->data; + } else if (const OverflowRecord* orec = std::get_if(&rec)) { + if (orec->m_header.deleted) continue; + uint32_t next_page = orec->page_number; + while (next_page != 0) { + SeekToPage(db_file, next_page, page_size); + PageHeader opage_header(next_page); + db_file >> opage_header; + if (opage_header.type != PageType::OVERFLOW_DATA) { + throw std::runtime_error("Bad overflow record page type"); + } + OverflowPage opage(opage_header); + db_file >> opage; + data.insert(data.end(), opage.data.begin(), opage.data.end()); + next_page = opage_header.next_page; + } + } + + if (is_key) { + key = data; + } else { + m_records.emplace(SerializeData{key.begin(), key.end()}, SerializeData{data.begin(), data.end()}); + key.clear(); + } + is_key = !is_key; + } + break; + } + default: + throw std::runtime_error("Unexpected page type"); + } + } } std::unique_ptr BerkeleyRODatabase::MakeBatch(bool flush_on_close) From 6e50bee67d1d58aecd8a0ce8b7c3f5a7979365f5 Mon Sep 17 00:00:00 2001 From: Ava Chow Date: Tue, 2 Jan 2024 16:35:50 -0500 Subject: [PATCH 06/16] Implement handling of other endianness in BerkeleyRODatabase --- src/wallet/migrate.cpp | 75 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 69 insertions(+), 6 deletions(-) diff --git a/src/wallet/migrate.cpp b/src/wallet/migrate.cpp index c2b4e22f6e..44e8b26be5 100644 --- a/src/wallet/migrate.cpp +++ b/src/wallet/migrate.cpp @@ -1,6 +1,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include #include // For ReadBE32 #include #include @@ -94,6 +95,7 @@ public: unsigned char iv[20]; // Crypto IV unsigned char chksum[16]; // Checksum + bool other_endian; uint32_t expected_page_num; MetaPage(uint32_t expected_page_num) : expected_page_num(expected_page_num) {} @@ -110,6 +112,8 @@ public: s >> pagesize; s >> encrypt_algo; + other_endian = magic == BTREE_MAGIC_OE; + uint8_t uint8_type; s >> uint8_type; type = static_cast(uint8_type); @@ -124,6 +128,9 @@ public: uint32_t uint32_flags; s >> uint32_flags; + if (other_endian) { + uint32_flags = internal_bswap_32(uint32_flags); + } flags = static_cast(uint32_flags); s >> uid; @@ -138,6 +145,26 @@ public: s >> iv; s >> chksum; + if (other_endian) { + lsn_file = internal_bswap_32(lsn_file); + lsn_offset = internal_bswap_32(lsn_offset); + page_num = internal_bswap_32(page_num); + magic = internal_bswap_32(magic); + version = internal_bswap_32(version); + pagesize = internal_bswap_32(pagesize); + free_list = internal_bswap_32(free_list); + last_page = internal_bswap_32(last_page); + partitions = internal_bswap_32(partitions); + key_count = internal_bswap_32(key_count); + record_count = internal_bswap_32(record_count); + unused2 = internal_bswap_32(unused2); + minkey = internal_bswap_32(minkey); + re_len = internal_bswap_32(re_len); + re_pad = internal_bswap_32(re_pad); + root = internal_bswap_32(root); + crypto_magic = internal_bswap_32(crypto_magic); + } + // Page number must match if (page_num != expected_page_num) { throw std::runtime_error("Meta page number mismatch"); @@ -180,6 +207,11 @@ public: static constexpr size_t SIZE = 3; // The record header is 3 bytes + bool other_endian; + + RecordHeader(bool other_endian) : other_endian(other_endian) {} + RecordHeader() = delete; + template void Unserialize(Stream& s) { @@ -189,6 +221,10 @@ public: s >> uint8_type; type = static_cast(uint8_type & ~static_cast(RecordType::DELETE)); deleted = uint8_type & static_cast(RecordType::DELETE); + + if (other_endian) { + len = internal_bswap_16(len); + } } }; @@ -236,6 +272,11 @@ public: data.resize(m_header.len); s.read(AsWritableBytes(Span(data.data(), data.size()))); + + if (m_header.other_endian) { + page_num = internal_bswap_32(page_num); + records = internal_bswap_32(records); + } } }; @@ -263,6 +304,11 @@ public: s >> unused2; s >> page_number; s >> item_len; + + if (m_header.other_endian) { + page_number = internal_bswap_32(page_number); + item_len = internal_bswap_32(item_len); + } } }; @@ -283,8 +329,9 @@ public: static constexpr int64_t SIZE = 26; // The header is 26 bytes uint32_t expected_page_num; + bool other_endian; - PageHeader(uint32_t page_num) : expected_page_num(page_num) {} + PageHeader(uint32_t page_num, bool other_endian) : expected_page_num(page_num), other_endian(other_endian) {} PageHeader() = delete; template @@ -303,6 +350,16 @@ public: s >> uint8_type; type = static_cast(uint8_type); + if (other_endian) { + lsn_file = internal_bswap_32(lsn_file); + lsn_offset = internal_bswap_32(lsn_offset); + page_num = internal_bswap_32(page_num); + prev_page = internal_bswap_32(prev_page); + next_page = internal_bswap_32(next_page); + entries = internal_bswap_16(entries); + hf_offset = internal_bswap_16(hf_offset); + } + if (expected_page_num != page_num) { throw std::runtime_error("Page number mismatch"); } @@ -335,6 +392,9 @@ public: // Get the index uint16_t index; s >> index; + if (m_header.other_endian) { + index = internal_bswap_16(index); + } indexes.push_back(index); pos += sizeof(uint16_t); @@ -346,7 +406,7 @@ public: s.ignore(to_jump); // Read the record - RecordHeader rec_hdr; + RecordHeader rec_hdr(m_header.other_endian); s >> rec_hdr; to_jump += RecordHeader::SIZE; @@ -422,6 +482,9 @@ public: // Get the index uint16_t index; s >> index; + if (m_header.other_endian) { + index = internal_bswap_16(index); + } indexes.push_back(index); pos += sizeof(uint16_t); @@ -433,7 +496,7 @@ public: s.ignore(to_jump); // Read the record - RecordHeader rec_hdr; + RecordHeader rec_hdr(m_header.other_endian); s >> rec_hdr; to_jump += RecordHeader::SIZE; @@ -499,7 +562,7 @@ void BerkeleyRODatabase::Open() // Read the root page SeekToPage(db_file, outer_meta.root, page_size); - PageHeader header(outer_meta.root); + PageHeader header(outer_meta.root, outer_meta.other_endian); db_file >> header; if (header.type != PageType::BTREE_LEAF) { throw std::runtime_error("Unexpected outer database root page type"); @@ -558,7 +621,7 @@ void BerkeleyRODatabase::Open() // } pages.pop_back(); SeekToPage(db_file, curr_page, page_size); - PageHeader header(curr_page); + PageHeader header(curr_page, inner_meta.other_endian); db_file >> header; switch (header.type) { case PageType::BTREE_INTERNAL: { @@ -589,7 +652,7 @@ void BerkeleyRODatabase::Open() uint32_t next_page = orec->page_number; while (next_page != 0) { SeekToPage(db_file, next_page, page_size); - PageHeader opage_header(next_page); + PageHeader opage_header(next_page, inner_meta.other_endian); db_file >> opage_header; if (opage_header.type != PageType::OVERFLOW_DATA) { throw std::runtime_error("Bad overflow record page type"); From dd57713f6ede3d46e97ee7df87c10001b0bf4c3d Mon Sep 17 00:00:00 2001 From: Ava Chow Date: Tue, 2 Jan 2024 16:36:06 -0500 Subject: [PATCH 07/16] Add MakeBerkeleyRODatabase Implements MakeBerkeleyRODatabase and adds DatabaseFormat::BERKELEY_RO so that MakeDatabase can use BerkeleyRO as the backend database. --- src/wallet/db.h | 1 + src/wallet/migrate.cpp | 15 +++++++++++++++ src/wallet/migrate.h | 3 +++ src/wallet/walletdb.cpp | 10 ++++++++++ 4 files changed, 29 insertions(+) diff --git a/src/wallet/db.h b/src/wallet/db.h index 648adff5fe..5751bba2e9 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -183,6 +183,7 @@ public: enum class DatabaseFormat { BERKELEY, SQLITE, + BERKELEY_RO, }; struct DatabaseOptions { diff --git a/src/wallet/migrate.cpp b/src/wallet/migrate.cpp index 44e8b26be5..895cee9e14 100644 --- a/src/wallet/migrate.cpp +++ b/src/wallet/migrate.cpp @@ -5,6 +5,7 @@ #include // For ReadBE32 #include #include +#include #include #include @@ -748,4 +749,18 @@ std::unique_ptr BerkeleyROBatch::GetNewPrefixCursor(Span(m_database, prefix); } + +std::unique_ptr MakeBerkeleyRODatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error) +{ + fs::path data_file = BDBDataFile(path); + try { + std::unique_ptr db = std::make_unique(data_file); + status = DatabaseStatus::SUCCESS; + return db; + } catch (const std::runtime_error& e) { + error.original = e.what(); + status = DatabaseStatus::FAILED_LOAD; + return nullptr; + } +} } // namespace wallet diff --git a/src/wallet/migrate.h b/src/wallet/migrate.h index a3b0d78d02..e4826450af 100644 --- a/src/wallet/migrate.h +++ b/src/wallet/migrate.h @@ -116,6 +116,9 @@ public: bool TxnCommit() override { return false; } bool TxnAbort() override { return false; } }; + +//! Return object giving access to Berkeley Read Only database at specified path. +std::unique_ptr MakeBerkeleyRODatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error); } // namespace wallet #endif // BITCOIN_WALLET_MIGRATE_H diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index b1ce7ee4e7..ddd90939bf 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -23,6 +23,7 @@ #ifdef USE_BDB #include #endif +#include #ifdef USE_SQLITE #include #endif @@ -1389,6 +1390,11 @@ std::unique_ptr MakeDatabase(const fs::path& path, const Databas return nullptr; } + // If BERKELEY was the format, then change the format from BERKELEY to BERKELEY_RO + if (format && options.require_format && format == DatabaseFormat::BERKELEY && options.require_format == DatabaseFormat::BERKELEY_RO) { + format = DatabaseFormat::BERKELEY_RO; + } + // A db already exists so format is set, but options also specifies the format, so make sure they agree if (format && options.require_format && format != options.require_format) { error = Untranslated(strprintf("Failed to load database path '%s'. Data is not in required format.", fs::PathToString(path))); @@ -1422,6 +1428,10 @@ std::unique_ptr MakeDatabase(const fs::path& path, const Databas } } + if (format == DatabaseFormat::BERKELEY_RO) { + return MakeBerkeleyRODatabase(path, options, status, error); + } + #ifdef USE_BDB if constexpr (true) { return MakeBerkeleyDatabase(path, options, status, error); From 70cfbfdadf16d3b115309c6938f07ef5b96c7cc1 Mon Sep 17 00:00:00 2001 From: Ava Chow Date: Tue, 2 Jan 2024 16:36:17 -0500 Subject: [PATCH 08/16] wallettool: Optionally use BERKELEY_RO as format when dumping BDB wallets In order to ease the transition to not having BDB, make the dump tool use DatabaseFormmat::BERKELEY_RO when -withinternalbdb is set. --- src/bitcoin-wallet.cpp | 1 + src/wallet/dump.cpp | 8 +++++++- src/wallet/wallettool.cpp | 5 +++++ test/functional/test_runner.py | 1 + test/functional/tool_wallet.py | 3 +++ 5 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index d5dfbbec27..d1714f3e31 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -44,6 +44,7 @@ static void SetupWalletToolArgs(ArgsManager& argsman) argsman.AddArg("-legacy", "Create legacy wallet. Only for 'create'", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-format=", "The format of the wallet file to create. Either \"bdb\" or \"sqlite\". Only used with 'createfromdump'", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-printtoconsole", "Send trace/debug info to console (default: 1 when no -debug is true, 0 otherwise).", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-withinternalbdb", "Use the internal Berkeley DB parser when dumping a Berkeley DB wallet file (default: false)", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); argsman.AddCommand("info", "Get wallet info"); argsman.AddCommand("create", "Create new wallet file"); diff --git a/src/wallet/dump.cpp b/src/wallet/dump.cpp index 7a36910dc1..970830754d 100644 --- a/src/wallet/dump.cpp +++ b/src/wallet/dump.cpp @@ -60,7 +60,13 @@ bool DumpWallet(const ArgsManager& args, WalletDatabase& db, bilingual_str& erro hasher << Span{line}; // Write out the file format - line = strprintf("%s,%s\n", "format", db.Format()); + std::string format = db.Format(); + // BDB files that are opened using BerkeleyRODatabase have it's format as "bdb_ro" + // We want to override that format back to "bdb" + if (format == "bdb_ro") { + format = "bdb"; + } + line = strprintf("%s,%s\n", "format", format); dump_file.write(line.data(), line.size()); hasher << Span{line}; diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index cda344ab19..56868272bb 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -194,6 +194,11 @@ bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command) ReadDatabaseArgs(args, options); options.require_existing = true; DatabaseStatus status; + + if (args.GetBoolArg("-withinternalbdb", false) && IsBDBFile(BDBDataFile(path))) { + options.require_format = DatabaseFormat::BERKELEY_RO; + } + bilingual_str error; std::unique_ptr database = MakeDatabase(path, options, status, error); if (!database) { diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 3f6e47d410..89e4aa7055 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -192,6 +192,7 @@ BASE_SCRIPTS = [ 'mempool_resurrect.py', 'wallet_txn_doublespend.py --mineblock', 'tool_wallet.py --legacy-wallet', + 'tool_wallet.py --legacy-wallet --bdbro', 'tool_wallet.py --descriptors', 'tool_signet_miner.py --legacy-wallet', 'tool_signet_miner.py --descriptors', diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index fc042bca66..704f1fe9e8 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -21,6 +21,7 @@ from test_framework.util import ( class ToolWalletTest(BitcoinTestFramework): def add_options(self, parser): self.add_wallet_options(parser) + parser.add_argument("--bdbro", action="store_true", help="Use the BerkeleyRO internal parser when dumping a Berkeley DB wallet file") def set_test_params(self): self.num_nodes = 1 @@ -35,6 +36,8 @@ class ToolWalletTest(BitcoinTestFramework): default_args = ['-datadir={}'.format(self.nodes[0].datadir_path), '-chain=%s' % self.chain] if not self.options.descriptors and 'create' in args: default_args.append('-legacy') + if "dump" in args and self.options.bdbro: + default_args.append("-withinternalbdb") return subprocess.Popen([self.options.bitcoinwallet] + default_args + list(args), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) From 3568dce9e93295674cdf5458c5bdf93ff01fd0a2 Mon Sep 17 00:00:00 2001 From: Ava Chow Date: Tue, 2 Jan 2024 16:36:23 -0500 Subject: [PATCH 09/16] tests: Add BerkeleyRO to db prefix tests --- src/wallet/test/db_tests.cpp | 40 ++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/src/wallet/test/db_tests.cpp b/src/wallet/test/db_tests.cpp index f783424df8..280bb6c429 100644 --- a/src/wallet/test/db_tests.cpp +++ b/src/wallet/test/db_tests.cpp @@ -18,6 +18,7 @@ #ifdef USE_SQLITE #include #endif +#include #include #include // for WALLET_FLAG_DESCRIPTORS @@ -134,6 +135,8 @@ static std::vector> TestDatabases(const fs::path bilingual_str error; #ifdef USE_BDB dbs.emplace_back(MakeBerkeleyDatabase(path_root / "bdb", options, status, error)); + // Needs BDB to make the DB to read + dbs.emplace_back(std::make_unique(BDBDataFile(path_root / "bdb"), /*open=*/false)); #endif #ifdef USE_SQLITE dbs.emplace_back(MakeSQLiteDatabase(path_root / "sqlite", options, status, error)); @@ -148,11 +151,16 @@ BOOST_AUTO_TEST_CASE(db_cursor_prefix_range_test) for (const auto& database : TestDatabases(m_path_root)) { std::vector prefixes = {"", "FIRST", "SECOND", "P\xfe\xff", "P\xff\x01", "\xff\xff"}; - // Write elements to it std::unique_ptr handler = Assert(database)->MakeBatch(); - for (unsigned int i = 0; i < 10; i++) { - for (const auto& prefix : prefixes) { - BOOST_CHECK(handler->Write(std::make_pair(prefix, i), i)); + if (dynamic_cast(database.get())) { + // For BerkeleyRO, open the file now. This must happen after BDB has written to the file + database->Open(); + } else { + // Write elements to it if not berkeleyro + for (unsigned int i = 0; i < 10; i++) { + for (const auto& prefix : prefixes) { + BOOST_CHECK(handler->Write(std::make_pair(prefix, i), i)); + } } } @@ -180,6 +188,8 @@ BOOST_AUTO_TEST_CASE(db_cursor_prefix_range_test) // Let's now read it once more, it should return DONE BOOST_CHECK(cursor->Next(key, value) == DatabaseCursor::Status::DONE); } + handler.reset(); + database->Close(); } } @@ -199,13 +209,23 @@ BOOST_AUTO_TEST_CASE(db_cursor_prefix_byte_test) ffs{StringData("\xff\xffsuffix"), StringData("ffs")}; for (const auto& database : TestDatabases(m_path_root)) { std::unique_ptr batch = database->MakeBatch(); - for (const auto& [k, v] : {e, p, ps, f, fs, ff, ffs}) { - batch->Write(Span{k}, Span{v}); + + if (dynamic_cast(database.get())) { + // For BerkeleyRO, open the file now. This must happen after BDB has written to the file + database->Open(); + } else { + // Write elements to it if not berkeleyro + for (const auto& [k, v] : {e, p, ps, f, fs, ff, ffs}) { + batch->Write(Span{k}, Span{v}); + } } + CheckPrefix(*batch, StringBytes(""), {e, p, ps, f, fs, ff, ffs}); CheckPrefix(*batch, StringBytes("prefix"), {p, ps}); CheckPrefix(*batch, StringBytes("\xff"), {f, fs, ff, ffs}); CheckPrefix(*batch, StringBytes("\xff\xff"), {ff, ffs}); + batch.reset(); + database->Close(); } } @@ -215,6 +235,10 @@ BOOST_AUTO_TEST_CASE(db_availability_after_write_error) // To simulate the behavior, record overwrites are disallowed, and the test verifies // that the database remains active after failing to store an existing record. for (const auto& database : TestDatabases(m_path_root)) { + if (dynamic_cast(database.get())) { + // Skip this test if BerkeleyRO + continue; + } // Write original record std::unique_ptr batch = database->MakeBatch(); std::string key = "key"; @@ -243,6 +267,10 @@ BOOST_AUTO_TEST_CASE(erase_prefix) auto make_key = [](std::string type, std::string id) { return std::make_pair(type, id); }; for (const auto& database : TestDatabases(m_path_root)) { + if (dynamic_cast(database.get())) { + // Skip this test if BerkeleyRO + continue; + } std::unique_ptr batch = database->MakeBatch(); // Write two entries with the same key type prefix, a third one with a different prefix From 4d7a3ae78e55f25868979f1bd920857a4aecb825 Mon Sep 17 00:00:00 2001 From: TheCharlatan Date: Wed, 21 Jun 2023 00:14:56 +0200 Subject: [PATCH 10/16] Berkeley RO Database fuzz test --- src/Makefile.test.include | 3 +- src/wallet/test/fuzz/wallet_bdb_parser.cpp | 133 +++++++++++++++++++++ 2 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 src/wallet/test/fuzz/wallet_bdb_parser.cpp diff --git a/src/Makefile.test.include b/src/Makefile.test.include index cf88a02b95..74252b10ce 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -203,7 +203,8 @@ FUZZ_WALLET_SRC = \ wallet/test/fuzz/coincontrol.cpp \ wallet/test/fuzz/coinselection.cpp \ wallet/test/fuzz/fees.cpp \ - wallet/test/fuzz/parse_iso8601.cpp + wallet/test/fuzz/parse_iso8601.cpp \ + wallet/test/fuzz/wallet_bdb_parser.cpp if USE_SQLITE FUZZ_WALLET_SRC += \ diff --git a/src/wallet/test/fuzz/wallet_bdb_parser.cpp b/src/wallet/test/fuzz/wallet_bdb_parser.cpp new file mode 100644 index 0000000000..24ef75f791 --- /dev/null +++ b/src/wallet/test/fuzz/wallet_bdb_parser.cpp @@ -0,0 +1,133 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include // IWYU pragma: keep +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +using wallet::DatabaseOptions; +using wallet::DatabaseStatus; + +namespace { +TestingSetup* g_setup; +} // namespace + +void initialize_wallet_bdb_parser() +{ + static auto testing_setup = MakeNoLogFileContext(); + g_setup = testing_setup.get(); +} + +FUZZ_TARGET(wallet_bdb_parser, .init = initialize_wallet_bdb_parser) +{ + const auto wallet_path = g_setup->m_args.GetDataDirNet() / "fuzzed_wallet.dat"; + + { + AutoFile outfile{fsbridge::fopen(wallet_path, "wb")}; + outfile << Span{buffer}; + } + + const DatabaseOptions options{}; + DatabaseStatus status; + bilingual_str error; + + fs::path bdb_ro_dumpfile{g_setup->m_args.GetDataDirNet() / "fuzzed_dumpfile_bdb_ro.dump"}; + if (fs::exists(bdb_ro_dumpfile)) { // Writing into an existing dump file will throw an exception + remove(bdb_ro_dumpfile); + } + g_setup->m_args.ForceSetArg("-dumpfile", fs::PathToString(bdb_ro_dumpfile)); + +#ifdef USE_BDB + bool bdb_ro_err = false; + bool bdb_ro_pgno_err = false; +#endif + auto db{MakeBerkeleyRODatabase(wallet_path, options, status, error)}; + if (db) { + assert(DumpWallet(g_setup->m_args, *db, error)); + } else { +#ifdef USE_BDB + bdb_ro_err = true; +#endif + if (error.original == "AutoFile::ignore: end of file: iostream error" || + error.original == "AutoFile::read: end of file: iostream error" || + error.original == "Not a BDB file" || + error.original == "Unsupported BDB data file version number" || + error.original == "Unexpected page type, should be 9 (BTree Metadata)" || + error.original == "Unexpected database flags, should only be 0x20 (subdatabases)" || + error.original == "Unexpected outer database root page type" || + error.original == "Unexpected number of entries in outer database root page" || + error.original == "Subdatabase has an unexpected name" || + error.original == "Subdatabase page number has unexpected length" || + error.original == "Unexpected inner database page type" || + error.original == "Unknown record type in records page" || + error.original == "Unknown record type in internal page" || + error.original == "Unexpected page size" || + error.original == "Unexpected page type" || + error.original == "Page number mismatch" || + error.original == "Bad btree level" || + error.original == "Bad page size" || + error.original == "File size is not a multiple of page size" || + error.original == "Meta page number mismatch") { + // Do nothing + } else if (error.original == "Subdatabase last page is greater than database last page" || + error.original == "Page number is greater than database last page" || + error.original == "Page number is greater than subdatabase last page" || + error.original == "Last page number could not fit in file") { +#ifdef USE_BDB + bdb_ro_pgno_err = true; +#endif + } else { + throw std::runtime_error(error.original); + } + } + +#ifdef USE_BDB + // Try opening with BDB + fs::path bdb_dumpfile{g_setup->m_args.GetDataDirNet() / "fuzzed_dumpfile_bdb.dump"}; + if (fs::exists(bdb_dumpfile)) { // Writing into an existing dump file will throw an exception + remove(bdb_dumpfile); + } + g_setup->m_args.ForceSetArg("-dumpfile", fs::PathToString(bdb_dumpfile)); + + try { + auto db{MakeBerkeleyDatabase(wallet_path, options, status, error)}; + if (bdb_ro_err && !db) { + return; + } + assert(db); + if (bdb_ro_pgno_err) { + // BerkeleyRO will throw on opening for errors involving bad page numbers, but BDB does not. + // Ignore those. + return; + } + assert(!bdb_ro_err); + assert(DumpWallet(g_setup->m_args, *db, error)); + } catch (const std::runtime_error& e) { + if (bdb_ro_err) return; + throw e; + } + + // Make sure the dumpfiles match + if (fs::exists(bdb_ro_dumpfile) && fs::exists(bdb_dumpfile)) { + std::ifstream bdb_ro_dump(bdb_ro_dumpfile, std::ios_base::binary | std::ios_base::in); + std::ifstream bdb_dump(bdb_dumpfile, std::ios_base::binary | std::ios_base::in); + assert(std::equal( + std::istreambuf_iterator(bdb_ro_dump.rdbuf()), + std::istreambuf_iterator(), + std::istreambuf_iterator(bdb_dump.rdbuf()))); + } +#endif +} From d9878903fb34939dee8e1462f079acc68110253d Mon Sep 17 00:00:00 2001 From: Ava Chow Date: Mon, 15 Apr 2024 16:47:14 -0400 Subject: [PATCH 11/16] Error if LSNs are not reset --- src/wallet/migrate.cpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/wallet/migrate.cpp b/src/wallet/migrate.cpp index 895cee9e14..09254a76ad 100644 --- a/src/wallet/migrate.cpp +++ b/src/wallet/migrate.cpp @@ -561,6 +561,24 @@ void BerkeleyRODatabase::Open() throw std::runtime_error("BDB builtin encryption is not supported"); } + // Check all Log Sequence Numbers (LSN) point to file 0 and offset 1 which indicates that the LSNs were + // reset and that the log files are not necessary to get all of the data in the database. + for (uint32_t i = 0; i < outer_meta.last_page; ++i) { + // The LSN is composed of 2 32-bit ints, the first is a file id, the second an offset + // It will always be the first 8 bytes of a page, so we deserialize it directly for every page + uint32_t file; + uint32_t offset; + SeekToPage(db_file, i, page_size); + db_file >> file >> offset; + if (outer_meta.other_endian) { + file = internal_bswap_32(file); + offset = internal_bswap_32(offset); + } + if (file != 0 || offset != 1) { + throw std::runtime_error("LSNs are not reset, this database is not completely flushed. Please reopen then close the database with a version that has BDB support"); + } + } + // Read the root page SeekToPage(db_file, outer_meta.root, page_size); PageHeader header(outer_meta.root, outer_meta.other_endian); From 6ace3e953f0864bd7818f040c59a1bc70aa47512 Mon Sep 17 00:00:00 2001 From: Ava Chow Date: Mon, 22 Apr 2024 15:21:58 -0400 Subject: [PATCH 12/16] bdb: Be able to make byteswapped databases Byteswapped databases make it easier to test opening and deserializing other endian databases. --- src/wallet/bdb.cpp | 16 +++++++++++++++- src/wallet/bdb.h | 3 +++ src/wallet/db.h | 1 + src/wallet/dump.cpp | 2 ++ 4 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/wallet/bdb.cpp b/src/wallet/bdb.cpp index 38cca32f80..d82d8d4513 100644 --- a/src/wallet/bdb.cpp +++ b/src/wallet/bdb.cpp @@ -65,6 +65,8 @@ RecursiveMutex cs_db; std::map> g_dbenvs GUARDED_BY(cs_db); //!< Map from directory name to db environment. } // namespace +static constexpr auto REVERSE_BYTE_ORDER{std::endian::native == std::endian::little ? 4321 : 1234}; + bool WalletDatabaseFileId::operator==(const WalletDatabaseFileId& rhs) const { return memcmp(value, &rhs.value, sizeof(value)) == 0; @@ -300,7 +302,11 @@ static Span SpanFromDbt(const SafeDbt& dbt) } BerkeleyDatabase::BerkeleyDatabase(std::shared_ptr env, fs::path filename, const DatabaseOptions& options) : - WalletDatabase(), env(std::move(env)), m_filename(std::move(filename)), m_max_log_mb(options.max_log_mb) + WalletDatabase(), + env(std::move(env)), + m_byteswap(options.require_format == DatabaseFormat::BERKELEY_SWAP), + m_filename(std::move(filename)), + m_max_log_mb(options.max_log_mb) { auto inserted = this->env->m_databases.emplace(m_filename, std::ref(*this)); assert(inserted.second); @@ -389,6 +395,10 @@ void BerkeleyDatabase::Open() } } + if (m_byteswap) { + pdb_temp->set_lorder(REVERSE_BYTE_ORDER); + } + ret = pdb_temp->open(nullptr, // Txn pointer fMockDb ? nullptr : strFile.c_str(), // Filename fMockDb ? strFile.c_str() : "main", // Logical db name @@ -521,6 +531,10 @@ bool BerkeleyDatabase::Rewrite(const char* pszSkip) BerkeleyBatch db(*this, true); std::unique_ptr pdbCopy = std::make_unique(env->dbenv.get(), 0); + if (m_byteswap) { + pdbCopy->set_lorder(REVERSE_BYTE_ORDER); + } + int ret = pdbCopy->open(nullptr, // Txn pointer strFileRes.c_str(), // Filename "main", // Logical db name diff --git a/src/wallet/bdb.h b/src/wallet/bdb.h index 630630ebe0..af0c78f0d9 100644 --- a/src/wallet/bdb.h +++ b/src/wallet/bdb.h @@ -147,6 +147,9 @@ public: /** Database pointer. This is initialized lazily and reset during flushes, so it can be null. */ std::unique_ptr m_db; + // Whether to byteswap + bool m_byteswap; + fs::path m_filename; int64_t m_max_log_mb; diff --git a/src/wallet/db.h b/src/wallet/db.h index 5751bba2e9..b45076d10c 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -184,6 +184,7 @@ enum class DatabaseFormat { BERKELEY, SQLITE, BERKELEY_RO, + BERKELEY_SWAP, }; struct DatabaseOptions { diff --git a/src/wallet/dump.cpp b/src/wallet/dump.cpp index 970830754d..db2756e0ca 100644 --- a/src/wallet/dump.cpp +++ b/src/wallet/dump.cpp @@ -186,6 +186,8 @@ bool CreateFromDump(const ArgsManager& args, const std::string& name, const fs:: data_format = DatabaseFormat::BERKELEY; } else if (file_format == "sqlite") { data_format = DatabaseFormat::SQLITE; + } else if (file_format == "bdb_swap") { + data_format = DatabaseFormat::BERKELEY_SWAP; } else { error = strprintf(_("Unknown wallet file format \"%s\" provided. Please provide one of \"bdb\" or \"sqlite\"."), file_format); return false; From fd7b16e391ed320e35255157a28be14c947ef30a Mon Sep 17 00:00:00 2001 From: Ava Chow Date: Mon, 22 Apr 2024 15:39:07 -0400 Subject: [PATCH 13/16] test: Test dumps of other endian BDB files --- test/functional/tool_wallet.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index 704f1fe9e8..c08f96bbdc 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -454,6 +454,25 @@ class ToolWalletTest(BitcoinTestFramework): ''') self.assert_tool_output(expected_output, "-wallet=conflicts", "info") + def test_dump_endianness(self): + self.log.info("Testing dumps of the same contents with different BDB endianness") + + self.start_node(0) + self.nodes[0].createwallet("endian") + self.stop_node(0) + + wallet_dump = self.nodes[0].datadir_path / "endian.dump" + self.assert_tool_output("The dumpfile may contain private keys. To ensure the safety of your Bitcoin, do not share the dumpfile.\n", "-wallet=endian", f"-dumpfile={wallet_dump}", "dump") + expected_dump = self.read_dump(wallet_dump) + + self.do_tool_createfromdump("native_endian", "endian.dump", "bdb") + native_dump = self.read_dump(self.nodes[0].datadir_path / "rt-native_endian.dump") + self.assert_dump(expected_dump, native_dump) + + self.do_tool_createfromdump("other_endian", "endian.dump", "bdb_swap") + other_dump = self.read_dump(self.nodes[0].datadir_path / "rt-other_endian.dump") + self.assert_dump(expected_dump, other_dump) + def run_test(self): self.wallet_path = self.nodes[0].wallets_path / self.default_wallet_name / self.wallet_data_filename self.test_invalid_tool_commands_and_args() @@ -465,6 +484,7 @@ class ToolWalletTest(BitcoinTestFramework): if not self.options.descriptors: # Salvage is a legacy wallet only thing self.test_salvage() + self.test_dump_endianness() self.test_dump_createfromdump() self.test_chainless_conflicts() From c1984f128284589423b7e0cc06c9a3b23a242d95 Mon Sep 17 00:00:00 2001 From: Ava Chow Date: Mon, 22 Apr 2024 16:06:29 -0400 Subject: [PATCH 14/16] test: Test dumping dbs with overflow pages --- test/functional/tool_wallet.py | 38 ++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index c08f96bbdc..92da687b02 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -14,6 +14,7 @@ from collections import OrderedDict from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, + assert_greater_than, sha256sum_file, ) @@ -473,6 +474,42 @@ class ToolWalletTest(BitcoinTestFramework): other_dump = self.read_dump(self.nodes[0].datadir_path / "rt-other_endian.dump") self.assert_dump(expected_dump, other_dump) + def test_dump_very_large_records(self): + self.log.info("Test that wallets with large records are successfully dumped") + + self.start_node(0) + self.nodes[0].createwallet("bigrecords") + wallet = self.nodes[0].get_wallet_rpc("bigrecords") + + # Both BDB and sqlite have maximum page sizes of 65536 bytes, with defaults of 4096 + # When a record exceeds some size threshold, both BDB and SQLite will store the data + # in one or more overflow pages. We want to make sure that our tooling can dump such + # records, even when they span multiple pages. To make a large record, we just need + # to make a very big transaction. + self.generate(self.nodes[0], 101) + def_wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + outputs = {} + for i in range(500): + outputs[wallet.getnewaddress(address_type="p2sh-segwit")] = 0.01 + def_wallet.sendmany(amounts=outputs) + self.generate(self.nodes[0], 1) + send_res = wallet.sendall([def_wallet.getnewaddress()]) + self.generate(self.nodes[0], 1) + assert_equal(send_res["complete"], True) + tx = wallet.gettransaction(txid=send_res["txid"], verbose=True) + assert_greater_than(tx["decoded"]["size"], 70000) + + self.stop_node(0) + + wallet_dump = self.nodes[0].datadir_path / "bigrecords.dump" + self.assert_tool_output("The dumpfile may contain private keys. To ensure the safety of your Bitcoin, do not share the dumpfile.\n", "-wallet=bigrecords", f"-dumpfile={wallet_dump}", "dump") + dump = self.read_dump(wallet_dump) + for k,v in dump.items(): + if tx["hex"] in v: + break + else: + assert False, "Big transaction was not found in wallet dump" + def run_test(self): self.wallet_path = self.nodes[0].wallets_path / self.default_wallet_name / self.wallet_data_filename self.test_invalid_tool_commands_and_args() @@ -487,6 +524,7 @@ class ToolWalletTest(BitcoinTestFramework): self.test_dump_endianness() self.test_dump_createfromdump() self.test_chainless_conflicts() + self.test_dump_very_large_records() if __name__ == '__main__': ToolWalletTest().main() From 0b753156ce60c29efb2386954ba7555ad8f642f5 Mon Sep 17 00:00:00 2001 From: Ava Chow Date: Mon, 22 Apr 2024 16:42:44 -0400 Subject: [PATCH 15/16] test: Test bdb_ro dump of wallet without reset LSNs --- test/functional/test_framework/test_node.py | 5 +-- test/functional/tool_wallet.py | 37 +++++++++++++++++++-- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index 67e0be5280..534f25e535 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -419,8 +419,9 @@ class TestNode(): return True def wait_until_stopped(self, *, timeout=BITCOIND_PROC_WAIT_TIMEOUT, expect_error=False, **kwargs): - expected_ret_code = 1 if expect_error else 0 # Whether node shutdown return EXIT_FAILURE or EXIT_SUCCESS - self.wait_until(lambda: self.is_node_stopped(expected_ret_code=expected_ret_code, **kwargs), timeout=timeout) + if "expected_ret_code" not in kwargs: + kwargs["expected_ret_code"] = 1 if expect_error else 0 # Whether node shutdown return EXIT_FAILURE or EXIT_SUCCESS + self.wait_until(lambda: self.is_node_stopped(**kwargs), timeout=timeout) def replace_in_config(self, replacements): """ diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index 92da687b02..51239831cc 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -5,6 +5,7 @@ """Test bitcoin-wallet.""" import os +import platform import stat import subprocess import textwrap @@ -45,9 +46,13 @@ class ToolWalletTest(BitcoinTestFramework): def assert_raises_tool_error(self, error, *args): p = self.bitcoin_wallet_process(*args) stdout, stderr = p.communicate() - assert_equal(p.poll(), 1) assert_equal(stdout, '') - assert_equal(stderr.strip(), error) + if isinstance(error, tuple): + assert_equal(p.poll(), error[0]) + assert error[1] in stderr.strip() + else: + assert_equal(p.poll(), 1) + assert error in stderr.strip() def assert_tool_output(self, output, *args): p = self.bitcoin_wallet_process(*args) @@ -510,6 +515,33 @@ class ToolWalletTest(BitcoinTestFramework): else: assert False, "Big transaction was not found in wallet dump" + def test_dump_unclean_lsns(self): + if not self.options.bdbro: + return + self.log.info("Test that a legacy wallet that has not been compacted is not dumped by bdbro") + + self.start_node(0, extra_args=["-flushwallet=0"]) + self.nodes[0].createwallet("unclean_lsn") + wallet = self.nodes[0].get_wallet_rpc("unclean_lsn") + # First unload and load normally to make sure everything is written + wallet.unloadwallet() + self.nodes[0].loadwallet("unclean_lsn") + # Next cause a bunch of writes by filling the keypool + wallet.keypoolrefill(wallet.getwalletinfo()["keypoolsize"] + 100) + # Lastly kill bitcoind so that the LSNs don't get reset + self.nodes[0].process.kill() + self.nodes[0].wait_until_stopped(expected_ret_code=1 if platform.system() == "Windows" else -9) + assert self.nodes[0].is_node_stopped() + + wallet_dump = self.nodes[0].datadir_path / "unclean_lsn.dump" + self.assert_raises_tool_error("LSNs are not reset, this database is not completely flushed. Please reopen then close the database with a version that has BDB support", "-wallet=unclean_lsn", f"-dumpfile={wallet_dump}", "dump") + + # File can be dumped after reload it normally + self.start_node(0) + self.nodes[0].loadwallet("unclean_lsn") + self.stop_node(0) + self.assert_tool_output("The dumpfile may contain private keys. To ensure the safety of your Bitcoin, do not share the dumpfile.\n", "-wallet=unclean_lsn", f"-dumpfile={wallet_dump}", "dump") + def run_test(self): self.wallet_path = self.nodes[0].wallets_path / self.default_wallet_name / self.wallet_data_filename self.test_invalid_tool_commands_and_args() @@ -522,6 +554,7 @@ class ToolWalletTest(BitcoinTestFramework): # Salvage is a legacy wallet only thing self.test_salvage() self.test_dump_endianness() + self.test_dump_unclean_lsns() self.test_dump_createfromdump() self.test_chainless_conflicts() self.test_dump_very_large_records() From d51fbab4b32d56765e8faab6ad01245fb259b0ca Mon Sep 17 00:00:00 2001 From: Ava Chow Date: Mon, 22 Apr 2024 17:08:18 -0400 Subject: [PATCH 16/16] wallet, test: Be able to always swap BDB endianness --- src/dummywallet.cpp | 1 + src/wallet/init.cpp | 3 ++- src/wallet/wallet.cpp | 5 +++++ test/functional/test_runner.py | 1 + test/functional/tool_wallet.py | 3 +++ 5 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/dummywallet.cpp b/src/dummywallet.cpp index 9160ec19e6..42282c32d1 100644 --- a/src/dummywallet.cpp +++ b/src/dummywallet.cpp @@ -53,6 +53,7 @@ void DummyWalletInit::AddWalletOptions(ArgsManager& argsman) const "-walletrejectlongchains", "-walletcrosschain", "-unsafesqlitesync", + "-swapbdbendian", }); } diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index f151fad740..cbdce88810 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -87,8 +87,9 @@ void WalletInit::AddWalletOptions(ArgsManager& argsman) const argsman.AddArg("-dblogsize=", strprintf("Flush wallet database activity from memory to disk log every megabytes (default: %u)", DatabaseOptions().max_log_mb), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); argsman.AddArg("-flushwallet", strprintf("Run a thread to flush wallet periodically (default: %u)", DEFAULT_FLUSHWALLET), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); argsman.AddArg("-privdb", strprintf("Sets the DB_PRIVATE flag in the wallet db environment (default: %u)", !DatabaseOptions().use_shared_memory), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); + argsman.AddArg("-swapbdbendian", "Swaps the internal endianness of BDB wallet databases (default: false)", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); #else - argsman.AddHiddenArgs({"-dblogsize", "-flushwallet", "-privdb"}); + argsman.AddHiddenArgs({"-dblogsize", "-flushwallet", "-privdb", "-swapbdbendian"}); #endif #ifdef USE_SQLITE diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 96c4397504..5a8df5790e 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -375,7 +375,12 @@ std::shared_ptr CreateWallet(WalletContext& context, const std::string& uint64_t wallet_creation_flags = options.create_flags; const SecureString& passphrase = options.create_passphrase; + ArgsManager& args = *Assert(context.args); + if (wallet_creation_flags & WALLET_FLAG_DESCRIPTORS) options.require_format = DatabaseFormat::SQLITE; + else if (args.GetBoolArg("-swapbdbendian", false)) { + options.require_format = DatabaseFormat::BERKELEY_SWAP; + } // Indicate that the wallet is actually supposed to be blank and not just blank to make it encrypted bool create_blank = (wallet_creation_flags & WALLET_FLAG_BLANK_WALLET); diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 89e4aa7055..0a28cc9394 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -193,6 +193,7 @@ BASE_SCRIPTS = [ 'wallet_txn_doublespend.py --mineblock', 'tool_wallet.py --legacy-wallet', 'tool_wallet.py --legacy-wallet --bdbro', + 'tool_wallet.py --legacy-wallet --bdbro --swap-bdb-endian', 'tool_wallet.py --descriptors', 'tool_signet_miner.py --legacy-wallet', 'tool_signet_miner.py --descriptors', diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index 51239831cc..dcf74f6075 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -24,11 +24,14 @@ class ToolWalletTest(BitcoinTestFramework): def add_options(self, parser): self.add_wallet_options(parser) parser.add_argument("--bdbro", action="store_true", help="Use the BerkeleyRO internal parser when dumping a Berkeley DB wallet file") + parser.add_argument("--swap-bdb-endian", action="store_true",help="When making Legacy BDB wallets, always make then byte swapped internally") def set_test_params(self): self.num_nodes = 1 self.setup_clean_chain = True self.rpc_timeout = 120 + if self.options.swap_bdb_endian: + self.extra_args = [["-swapbdbendian"]] def skip_test_if_missing_module(self): self.skip_if_no_wallet()