foster/bitcoin-bitcoin-core
0
0
Fork 0
mirror of https://github.com/bitcoin/bitcoin.git synced 2025-02-02 09:46:52 -05:00
Code Issues Activity

Merge bitcoin/bitcoin#26606: wallet: Implement independent BDB parser

Browse source 
d51fbab4b3 wallet, test: Be able to always swap BDB endianness (Ava Chow)
0b753156ce test: Test bdb_ro dump of wallet without reset LSNs (Ava Chow)
c1984f1282 test: Test dumping dbs with overflow pages (Ava Chow)
fd7b16e391 test: Test dumps of other endian BDB files (Ava Chow)
6ace3e953f bdb: Be able to make byteswapped databases (Ava Chow)
d9878903fb Error if LSNs are not reset (Ava Chow)
4d7a3ae78e Berkeley RO Database fuzz test (TheCharlatan)
3568dce9e9 tests: Add BerkeleyRO to db prefix tests (Ava Chow)
70cfbfdadf wallettool: Optionally use BERKELEY_RO as format when dumping BDB wallets (Ava Chow)
dd57713f6e Add MakeBerkeleyRODatabase (Ava Chow)
6e50bee67d Implement handling of other endianness in BerkeleyRODatabase (Ava Chow)
cdd61c9cc1 wallet: implement independent BDB deserializer in BerkeleyRODatabase (Ava Chow)
ecba230979 wallet: implement BerkeleyRODatabase::Backup (Ava Chow)
0c8e728476 wallet: implement BerkeleyROBatch (Ava Chow)
756ff9b478 wallet: add dummy BerkeleyRODatabase and BerkeleyROBatch classes (Ava Chow)
ca18aea5c4 Add AutoFile::seek and tell (Ava Chow)

Pull request description:

  Split from #26596

  This PR adds `BerkeleyRODatabase` which is an independent implementation of a BDB file parser. It provides read only access to a BDB file, and can therefore be used as a read only database backend for wallets. This will be used for dumping legacy wallet records and migrating legacy wallets without the need for BDB itself.

  Wallettool's `dump` command is changed to use `BerkeleyRODatabase` instead of `BerkeleyDatabase` (and `CWallet` itself) to demonstrate that this parser works and to test it against the existing wallettool functional tests.

ACKs for top commit:
  josibake:
    reACK d51fbab4b3
  TheCharlatan:
    Re-ACK d51fbab4b3
  furszy:
    reACK d51fbab4b3
  laanwj:
    re-ACK d51fbab4b3
  theStack:
    ACK d51fbab4b3

Tree-SHA512: 1e7b97edf223b2974eed2e9eac1179fc82bb6359e0a66b7d2a0c8b9fa515eae9ea036f1edf7c76cdab2e75ad994962b134b41056ccfbc33b8d54f0859e86657b
This commit is contained in:
merge-script 2024-05-21 10:05:09 +01:00
commit 5acdc2b97d
No known key found for this signature in database
GPG key ID: 2EEB9F5CC09526C1
23 changed files with 1270 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/Makefile.am
View file

@ -348,6 +348,7 @@ BITCOIN_CORE_H = \
wallet/feebumper.h \
wallet/fees.h \
wallet/load.h \
wallet/migrate.h \
wallet/receive.h \
wallet/rpc/util.h \
wallet/rpc/wallet.h \
@ -508,6 +509,7 @@ libbitcoin_wallet_a_SOURCES = \
wallet/fees.cpp \
wallet/interfaces.cpp \
wallet/load.cpp \
wallet/migrate.cpp \
wallet/receive.cpp \
wallet/rpc/addresses.cpp \
wallet/rpc/backup.cpp \

3
src/Makefile.test.include
View file

@ -204,7 +204,8 @@ FUZZ_WALLET_SRC = \
wallet/test/fuzz/coincontrol.cpp \
wallet/test/fuzz/coinselection.cpp \
wallet/test/fuzz/fees.cpp \
wallet/test/fuzz/parse_iso8601.cpp
wallet/test/fuzz/parse_iso8601.cpp \
wallet/test/fuzz/wallet_bdb_parser.cpp
if USE_SQLITE
FUZZ_WALLET_SRC += \

1
src/bitcoin-wallet.cpp
View file

@ -40,6 +40,7 @@ static void SetupWalletToolArgs(ArgsManager& argsman)
argsman.AddArg("-legacy", "Create legacy wallet. Only for 'create'", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
argsman.AddArg("-format=<format>", "The format of the wallet file to create. Either \"bdb\" or \"sqlite\". Only used with 'createfromdump'", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
argsman.AddArg("-printtoconsole", "Send trace/debug info to console (default: 1 when no -debug is true, 0 otherwise).", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST);
argsman.AddArg("-withinternalbdb", "Use the internal Berkeley DB parser when dumping a Berkeley DB wallet file (default: false)", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST);
argsman.AddCommand("info", "Get wallet info");
argsman.AddCommand("create", "Create new wallet file");

1
src/dummywallet.cpp
View file

@ -53,6 +53,7 @@ void DummyWalletInit::AddWalletOptions(ArgsManager& argsman) const
"-walletrejectlongchains",
"-walletcrosschain",
"-unsafesqlitesync",
"-swapbdbendian",
});
}

22
src/streams.cpp
View file

@ -21,6 +21,28 @@ std::size_t AutoFile::detail_fread(Span<std::byte> dst)
}
}
void AutoFile::seek(int64_t offset, int origin)
{
if (IsNull()) {
throw std::ios_base::failure("AutoFile::seek: file handle is nullptr");
}
if (std::fseek(m_file, offset, origin) != 0) {
throw std::ios_base::failure(feof() ? "AutoFile::seek: end of file" : "AutoFile::seek: fseek failed");
}
}
int64_t AutoFile::tell()
{
if (IsNull()) {
throw std::ios_base::failure("AutoFile::tell: file handle is nullptr");
}
int64_t r{std::ftell(m_file)};
if (r < 0) {
throw std::ios_base::failure("AutoFile::tell: ftell failed");
}
return r;
}
void AutoFile::read(Span<std::byte> dst)
{
if (detail_fread(dst) != dst.size()) {

3
src/streams.h
View file

@ -435,6 +435,9 @@ public:
/** Implementation detail, only used internally. */
std::size_t detail_fread(Span<std::byte> dst);
void seek(int64_t offset, int origin);
int64_t tell();
//
// Stream subset
//

16
src/wallet/bdb.cpp
View file

@ -65,6 +65,8 @@ RecursiveMutex cs_db;
std::map<std::string, std::weak_ptr<BerkeleyEnvironment>> g_dbenvs GUARDED_BY(cs_db); //!< Map from directory name to db environment.
} // namespace
static constexpr auto REVERSE_BYTE_ORDER{std::endian::native == std::endian::little ? 4321 : 1234};
bool WalletDatabaseFileId::operator==(const WalletDatabaseFileId& rhs) const
{
return memcmp(value, &rhs.value, sizeof(value)) == 0;
@ -300,7 +302,11 @@ static Span<const std::byte> SpanFromDbt(const SafeDbt& dbt)
}
BerkeleyDatabase::BerkeleyDatabase(std::shared_ptr<BerkeleyEnvironment> env, fs::path filename, const DatabaseOptions& options) :
WalletDatabase(), env(std::move(env)), m_filename(std::move(filename)), m_max_log_mb(options.max_log_mb)
WalletDatabase(),
env(std::move(env)),
m_byteswap(options.require_format == DatabaseFormat::BERKELEY_SWAP),
m_filename(std::move(filename)),
m_max_log_mb(options.max_log_mb)
{
auto inserted = this->env->m_databases.emplace(m_filename, std::ref(*this));
assert(inserted.second);
@ -389,6 +395,10 @@ void BerkeleyDatabase::Open()
}
}
if (m_byteswap) {
pdb_temp->set_lorder(REVERSE_BYTE_ORDER);
}
ret = pdb_temp->open(nullptr, // Txn pointer
fMockDb ? nullptr : strFile.c_str(), // Filename
fMockDb ? strFile.c_str() : "main", // Logical db name
@ -521,6 +531,10 @@ bool BerkeleyDatabase::Rewrite(const char* pszSkip)
BerkeleyBatch db(*this, true);
std::unique_ptr<Db> pdbCopy = std::make_unique<Db>(env->dbenv.get(), 0);
if (m_byteswap) {
pdbCopy->set_lorder(REVERSE_BYTE_ORDER);
}
int ret = pdbCopy->open(nullptr, // Txn pointer
strFileRes.c_str(), // Filename
"main", // Logical db name

3
src/wallet/bdb.h
View file

@ -147,6 +147,9 @@ public:
/** Database pointer. This is initialized lazily and reset during flushes, so it can be null. */
std::unique_ptr<Db> m_db;
// Whether to byteswap
bool m_byteswap;
fs::path m_filename;
int64_t m_max_log_mb;

3
src/wallet/db.cpp
View file

@ -16,6 +16,9 @@
#include <vector>
namespace wallet {
bool operator<(BytePrefix a, Span<const std::byte> b) { return a.prefix < b.subspan(0, std::min(a.prefix.size(), b.size())); }
bool operator<(Span<const std::byte> a, BytePrefix b) { return a.subspan(0, std::min(a.size(), b.prefix.size())) < b.prefix; }
std::vector<fs::path> ListDatabases(const fs::path& wallet_dir)
{
std::vector<fs::path> paths;

8
src/wallet/db.h
View file

@ -20,6 +20,12 @@ class ArgsManager;
struct bilingual_str;
namespace wallet {
// BytePrefix compares equality with other byte spans that begin with the same prefix.
struct BytePrefix {
Span<const std::byte> prefix;
};
bool operator<(BytePrefix a, Span<const std::byte> b);
bool operator<(Span<const std::byte> a, BytePrefix b);
class DatabaseCursor
{
@ -177,6 +183,8 @@ public:
enum class DatabaseFormat {
BERKELEY,
SQLITE,
BERKELEY_RO,
BERKELEY_SWAP,
};
struct DatabaseOptions {

10
src/wallet/dump.cpp
View file

@ -60,7 +60,13 @@ bool DumpWallet(const ArgsManager& args, WalletDatabase& db, bilingual_str& erro
hasher << Span{line};
// Write out the file format
line = strprintf("%s,%s\n", "format", db.Format());
std::string format = db.Format();
// BDB files that are opened using BerkeleyRODatabase have it's format as "bdb_ro"
// We want to override that format back to "bdb"
if (format == "bdb_ro") {
format = "bdb";
}
line = strprintf("%s,%s\n", "format", format);
dump_file.write(line.data(), line.size());
hasher << Span{line};
@ -180,6 +186,8 @@ bool CreateFromDump(const ArgsManager& args, const std::string& name, const fs::
data_format = DatabaseFormat::BERKELEY;
} else if (file_format == "sqlite") {
data_format = DatabaseFormat::SQLITE;
} else if (file_format == "bdb_swap") {
data_format = DatabaseFormat::BERKELEY_SWAP;
} else {
error = strprintf(_("Unknown wallet file format \"%s\" provided. Please provide one of \"bdb\" or \"sqlite\"."), file_format);
return false;

3
src/wallet/init.cpp
View file

@ -85,8 +85,9 @@ void WalletInit::AddWalletOptions(ArgsManager& argsman) const
argsman.AddArg("-dblogsize=<n>", strprintf("Flush wallet database activity from memory to disk log every <n> megabytes (default: %u)", DatabaseOptions().max_log_mb), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST);
argsman.AddArg("-flushwallet", strprintf("Run a thread to flush wallet periodically (default: %u)", DEFAULT_FLUSHWALLET), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST);
argsman.AddArg("-privdb", strprintf("Sets the DB_PRIVATE flag in the wallet db environment (default: %u)", !DatabaseOptions().use_shared_memory), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST);
argsman.AddArg("-swapbdbendian", "Swaps the internal endianness of BDB wallet databases (default: false)", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST);
#else
argsman.AddHiddenArgs({"-dblogsize", "-flushwallet", "-privdb"});
argsman.AddHiddenArgs({"-dblogsize", "-flushwallet", "-privdb", "-swapbdbendian"});
#endif
#ifdef USE_SQLITE

784
src/wallet/migrate.cpp Normal file
View file

@ -0,0 +1,784 @@
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include <compat/byteswap.h>
#include <crypto/common.h> // For ReadBE32
#include <logging.h>
#include <streams.h>
#include <util/translation.h>
#include <wallet/migrate.h>
#include <optional>
#include <variant>
namespace wallet {
// Magic bytes in both endianness's
constexpr uint32_t BTREE_MAGIC = 0x00053162; // If the file endianness matches our system, we see this magic
constexpr uint32_t BTREE_MAGIC_OE = 0x62310500; // If the file endianness is the other one, we will see this magic
// Subdatabase name
static const std::vector<std::byte> SUBDATABASE_NAME = {std::byte{'m'}, std::byte{'a'}, std::byte{'i'}, std::byte{'n'}};
enum class PageType : uint8_t {
/*
* BDB has several page types, most of which we do not use
* They are listed here for completeness, but commented out
* to avoid opening something unintended.
INVALID = 0, // Invalid page type
DUPLICATE = 1, // Duplicate. Deprecated and no longer used
HASH_UNSORTED = 2, // Hash pages. Deprecated.
RECNO_INTERNAL = 4, // Recno internal
RECNO_LEAF = 6, // Recno leaf
HASH_META = 8, // Hash metadata
QUEUE_META = 10, // Queue Metadata
QUEUE_DATA = 11, // Queue Data
DUPLICATE_LEAF = 12, // Off-page duplicate leaf
HASH_SORTED = 13, // Sorted hash page
*/
BTREE_INTERNAL = 3, // BTree internal
BTREE_LEAF = 5, // BTree leaf
OVERFLOW_DATA = 7, // Overflow
BTREE_META = 9, // BTree metadata
};
enum class RecordType : uint8_t {
KEYDATA = 1,
// DUPLICATE = 2, Unused as our databases do not support duplicate records
OVERFLOW_DATA = 3,
DELETE = 0x80, // Indicate this record is deleted. This is OR'd with the real type.
};
enum class BTreeFlags : uint32_t {
/*
* BTree databases have feature flags, but we do not use them except for
* subdatabases. The unused flags are included for completeness, but commented out
* to avoid accidental use.
DUP = 1, // Duplicates
RECNO = 2, // Recno tree
RECNUM = 4, // BTree: Maintain record counts
FIXEDLEN = 8, // Recno: fixed length records
RENUMBER = 0x10, // Recno: renumber on insert/delete
DUPSORT = 0x40, // Duplicates are sorted
COMPRESS = 0x80, // Compressed
*/
SUBDB = 0x20, // Subdatabases
};
/** Berkeley DB BTree metadata page layout */
class MetaPage
{
public:
uint32_t lsn_file; // Log Sequence Number file
uint32_t lsn_offset; // Log Sequence Number offset
uint32_t page_num; // Current page number
uint32_t magic; // Magic number
uint32_t version; // Version
uint32_t pagesize; // Page size
uint8_t encrypt_algo; // Encryption algorithm
PageType type; // Page type
uint8_t metaflags; // Meta-only flags
uint8_t unused1; // Unused
uint32_t free_list; // Free list page number
uint32_t last_page; // Page number of last page in db
uint32_t partitions; // Number of partitions
uint32_t key_count; // Cached key count
uint32_t record_count; // Cached record count
BTreeFlags flags; // Flags
std::array<std::byte, 20> uid; // 20 byte unique file ID
uint32_t unused2; // Unused
uint32_t minkey; // Minimum key
uint32_t re_len; // Recno: fixed length record length
uint32_t re_pad; // Recno: fixed length record pad
uint32_t root; // Root page number
char unused3[368]; // 92 * 4 bytes of unused space
uint32_t crypto_magic; // Crypto magic number
char trash[12]; // 3 * 4 bytes of trash space
unsigned char iv[20]; // Crypto IV
unsigned char chksum[16]; // Checksum
bool other_endian;
uint32_t expected_page_num;
MetaPage(uint32_t expected_page_num) : expected_page_num(expected_page_num) {}
MetaPage() = delete;
template <typename Stream>
void Unserialize(Stream& s)
{
s >> lsn_file;
s >> lsn_offset;
s >> page_num;
s >> magic;
s >> version;
s >> pagesize;
s >> encrypt_algo;
other_endian = magic == BTREE_MAGIC_OE;
uint8_t uint8_type;
s >> uint8_type;
type = static_cast<PageType>(uint8_type);
s >> metaflags;
s >> unused1;
s >> free_list;
s >> last_page;
s >> partitions;
s >> key_count;
s >> record_count;
uint32_t uint32_flags;
s >> uint32_flags;
if (other_endian) {
uint32_flags = internal_bswap_32(uint32_flags);
}
flags = static_cast<BTreeFlags>(uint32_flags);
s >> uid;
s >> unused2;
s >> minkey;
s >> re_len;
s >> re_pad;
s >> root;
s >> unused3;
s >> crypto_magic;
s >> trash;
s >> iv;
s >> chksum;
if (other_endian) {
lsn_file = internal_bswap_32(lsn_file);
lsn_offset = internal_bswap_32(lsn_offset);
page_num = internal_bswap_32(page_num);
magic = internal_bswap_32(magic);
version = internal_bswap_32(version);
pagesize = internal_bswap_32(pagesize);
free_list = internal_bswap_32(free_list);
last_page = internal_bswap_32(last_page);
partitions = internal_bswap_32(partitions);
key_count = internal_bswap_32(key_count);
record_count = internal_bswap_32(record_count);
unused2 = internal_bswap_32(unused2);
minkey = internal_bswap_32(minkey);
re_len = internal_bswap_32(re_len);
re_pad = internal_bswap_32(re_pad);
root = internal_bswap_32(root);
crypto_magic = internal_bswap_32(crypto_magic);
}
// Page number must match
if (page_num != expected_page_num) {
throw std::runtime_error("Meta page number mismatch");
}
// Check magic
if (magic != BTREE_MAGIC) {
throw std::runtime_error("Not a BDB file");
}
// Only version 9 is supported
if (version != 9) {
throw std::runtime_error("Unsupported BDB data file version number");
}
// Page size must be 512 <= pagesize <= 64k, and be a power of 2
if (pagesize < 512 || pagesize > 65536 || (pagesize & (pagesize - 1)) != 0) {
throw std::runtime_error("Bad page size");
}
// Page type must be the btree type
if (type != PageType::BTREE_META) {
throw std::runtime_error("Unexpected page type, should be 9 (BTree Metadata)");
}
// Only supported meta-flag is subdatabase
if (flags != BTreeFlags::SUBDB) {
throw std::runtime_error("Unexpected database flags, should only be 0x20 (subdatabases)");
}
}
};
/** General class for records in a BDB BTree database. Contains common fields. */
class RecordHeader
{
public:
uint16_t len; // Key/data item length
RecordType type; // Page type (BDB has this include a DELETE FLAG that we track separately)
bool deleted; // Whether the DELETE flag was set on type
static constexpr size_t SIZE = 3; // The record header is 3 bytes
bool other_endian;
RecordHeader(bool other_endian) : other_endian(other_endian) {}
RecordHeader() = delete;
template <typename Stream>
void Unserialize(Stream& s)
{
s >> len;
uint8_t uint8_type;
s >> uint8_type;
type = static_cast<RecordType>(uint8_type & ~static_cast<uint8_t>(RecordType::DELETE));
deleted = uint8_type & static_cast<uint8_t>(RecordType::DELETE);
if (other_endian) {
len = internal_bswap_16(len);
}
}
};
/** Class for data in the record directly */
class DataRecord
{
public:
DataRecord(const RecordHeader& header) : m_header(header) {}
DataRecord() = delete;
RecordHeader m_header;
std::vector<std::byte> data; // Variable length key/data item
template <typename Stream>
void Unserialize(Stream& s)
{
data.resize(m_header.len);
s.read(AsWritableBytes(Span(data.data(), data.size())));
}
};
/** Class for records representing internal nodes of the BTree. */
class InternalRecord
{
public:
InternalRecord(const RecordHeader& header) : m_header(header) {}
InternalRecord() = delete;
RecordHeader m_header;
uint8_t unused; // Padding, unused
uint32_t page_num; // Page number of referenced page
uint32_t records; // Subtree record count
std::vector<std::byte> data; // Variable length key item
static constexpr size_t FIXED_SIZE = 9; // Size of fixed data is 9 bytes
template <typename Stream>
void Unserialize(Stream& s)
{
s >> unused;
s >> page_num;
s >> records;
data.resize(m_header.len);
s.read(AsWritableBytes(Span(data.data(), data.size())));
if (m_header.other_endian) {
page_num = internal_bswap_32(page_num);
records = internal_bswap_32(records);
}
}
};
/** Class for records representing overflow records of the BTree.
* Overflow records point to a page which contains the data in the record.
* Those pages may point to further pages with the rest of the data if it does not fit
* in one page */
class OverflowRecord
{
public:
OverflowRecord(const RecordHeader& header) : m_header(header) {}
OverflowRecord() = delete;
RecordHeader m_header;
uint8_t unused2; // Padding, unused
uint32_t page_number; // Page number where data begins
uint32_t item_len; // Total length of item
static constexpr size_t SIZE = 9; // Overflow record is always 9 bytes
template <typename Stream>
void Unserialize(Stream& s)
{
s >> unused2;
s >> page_number;
s >> item_len;
if (m_header.other_endian) {
page_number = internal_bswap_32(page_number);
item_len = internal_bswap_32(item_len);
}
}
};
/** A generic data page in the database. Contains fields common to all data pages. */
class PageHeader
{
public:
uint32_t lsn_file; // Log Sequence Number file
uint32_t lsn_offset; // Log Sequence Number offset
uint32_t page_num; // Current page number
uint32_t prev_page; // Previous page number
uint32_t next_page; // Next page number
uint16_t entries; // Number of items on the page
uint16_t hf_offset; // High free byte page offset
uint8_t level; // Btree page level
PageType type; // Page type
static constexpr int64_t SIZE = 26; // The header is 26 bytes
uint32_t expected_page_num;
bool other_endian;
PageHeader(uint32_t page_num, bool other_endian) : expected_page_num(page_num), other_endian(other_endian) {}
PageHeader() = delete;
template <typename Stream>
void Unserialize(Stream& s)
{
s >> lsn_file;
s >> lsn_offset;
s >> page_num;
s >> prev_page;
s >> next_page;
s >> entries;
s >> hf_offset;
s >> level;
uint8_t uint8_type;
s >> uint8_type;
type = static_cast<PageType>(uint8_type);
if (other_endian) {
lsn_file = internal_bswap_32(lsn_file);
lsn_offset = internal_bswap_32(lsn_offset);
page_num = internal_bswap_32(page_num);
prev_page = internal_bswap_32(prev_page);
next_page = internal_bswap_32(next_page);
entries = internal_bswap_16(entries);
hf_offset = internal_bswap_16(hf_offset);
}
if (expected_page_num != page_num) {
throw std::runtime_error("Page number mismatch");
}
if ((type != PageType::OVERFLOW_DATA && level < 1) || (type == PageType::OVERFLOW_DATA && level != 0)) {
throw std::runtime_error("Bad btree level");
}
}
};
/** A page of records in the database */
class RecordsPage
{
public:
RecordsPage(const PageHeader& header) : m_header(header) {}
RecordsPage() = delete;
PageHeader m_header;
std::vector<uint16_t> indexes;
std::vector<std::variant<DataRecord, OverflowRecord>> records;
template <typename Stream>
void Unserialize(Stream& s)
{
// Current position within the page
int64_t pos = PageHeader::SIZE;
// Get the items
for (uint32_t i = 0; i < m_header.entries; ++i) {
// Get the index
uint16_t index;
s >> index;
if (m_header.other_endian) {
index = internal_bswap_16(index);
}
indexes.push_back(index);
pos += sizeof(uint16_t);
// Go to the offset from the index
int64_t to_jump = index - pos;
if (to_jump < 0) {
throw std::runtime_error("Data record position not in page");
}
s.ignore(to_jump);
// Read the record
RecordHeader rec_hdr(m_header.other_endian);
s >> rec_hdr;
to_jump += RecordHeader::SIZE;
switch (rec_hdr.type) {
case RecordType::KEYDATA: {
DataRecord record(rec_hdr);
s >> record;
records.emplace_back(record);
to_jump += rec_hdr.len;
break;
}
case RecordType::OVERFLOW_DATA: {
OverflowRecord record(rec_hdr);
s >> record;
records.emplace_back(record);
to_jump += OverflowRecord::SIZE;
break;
}
default:
throw std::runtime_error("Unknown record type in records page");
}
// Go back to the indexes
s.seek(-to_jump, SEEK_CUR);
}
}
};
/** A page containing overflow data */
class OverflowPage
{
public:
OverflowPage(const PageHeader& header) : m_header(header) {}
OverflowPage() = delete;
PageHeader m_header;
// BDB overloads some page fields to store overflow page data
// hf_offset contains the length of the overflow data stored on this page
// entries contains a reference count for references to this item
// The overflow data itself. Begins immediately following header
std::vector<std::byte> data;
template <typename Stream>
void Unserialize(Stream& s)
{
data.resize(m_header.hf_offset);
s.read(AsWritableBytes(Span(data.data(), data.size())));
}
};
/** A page of records in the database */
class InternalPage
{
public:
InternalPage(const PageHeader& header) : m_header(header) {}
InternalPage() = delete;
PageHeader m_header;
std::vector<uint16_t> indexes;
std::vector<InternalRecord> records;
template <typename Stream>
void Unserialize(Stream& s)
{
// Current position within the page
int64_t pos = PageHeader::SIZE;
// Get the items
for (uint32_t i = 0; i < m_header.entries; ++i) {
// Get the index
uint16_t index;
s >> index;
if (m_header.other_endian) {
index = internal_bswap_16(index);
}
indexes.push_back(index);
pos += sizeof(uint16_t);
// Go to the offset from the index
int64_t to_jump = index - pos;
if (to_jump < 0) {
throw std::runtime_error("Internal record position not in page");
}
s.ignore(to_jump);
// Read the record
RecordHeader rec_hdr(m_header.other_endian);
s >> rec_hdr;
to_jump += RecordHeader::SIZE;
if (rec_hdr.type != RecordType::KEYDATA) {
throw std::runtime_error("Unknown record type in internal page");
}
InternalRecord record(rec_hdr);
s >> record;
records.emplace_back(record);
to_jump += InternalRecord::FIXED_SIZE + rec_hdr.len;
// Go back to the indexes
s.seek(-to_jump, SEEK_CUR);
}
}
};
static void SeekToPage(AutoFile& s, uint32_t page_num, uint32_t page_size)
{
int64_t pos = int64_t{page_num} * page_size;
s.seek(pos, SEEK_SET);
}
void BerkeleyRODatabase::Open()
{
// Open the file
FILE* file = fsbridge::fopen(m_filepath, "rb");
AutoFile db_file(file);
if (db_file.IsNull()) {
throw std::runtime_error("BerkeleyRODatabase: Failed to open database file");
}
uint32_t page_size = 4096; // Default page size
// Read the outer metapage
// Expected page number is 0
MetaPage outer_meta(0);
db_file >> outer_meta;
page_size = outer_meta.pagesize;
// Verify the size of the file is a multiple of the page size
db_file.seek(0, SEEK_END);
int64_t size = db_file.tell();
// Since BDB stores everything in a page, the file size should be a multiple of the page size;
// However, BDB doesn't actually check that this is the case, and enforcing this check results
// in us rejecting a database that BDB would not, so this check needs to be excluded.
// This is left commented out as a reminder to not accidentally implement this in the future.
// if (size % page_size != 0) {
// throw std::runtime_error("File size is not a multiple of page size");
// }
// Check the last page number
uint32_t expected_last_page = (size / page_size) - 1;
if (outer_meta.last_page != expected_last_page) {
throw std::runtime_error("Last page number could not fit in file");
}
// Make sure encryption is disabled
if (outer_meta.encrypt_algo != 0) {
throw std::runtime_error("BDB builtin encryption is not supported");
}
// Check all Log Sequence Numbers (LSN) point to file 0 and offset 1 which indicates that the LSNs were
// reset and that the log files are not necessary to get all of the data in the database.
for (uint32_t i = 0; i < outer_meta.last_page; ++i) {
// The LSN is composed of 2 32-bit ints, the first is a file id, the second an offset
// It will always be the first 8 bytes of a page, so we deserialize it directly for every page
uint32_t file;
uint32_t offset;
SeekToPage(db_file, i, page_size);
db_file >> file >> offset;
if (outer_meta.other_endian) {
file = internal_bswap_32(file);
offset = internal_bswap_32(offset);
}
if (file != 0 || offset != 1) {
throw std::runtime_error("LSNs are not reset, this database is not completely flushed. Please reopen then close the database with a version that has BDB support");
}
}
// Read the root page
SeekToPage(db_file, outer_meta.root, page_size);
PageHeader header(outer_meta.root, outer_meta.other_endian);
db_file >> header;
if (header.type != PageType::BTREE_LEAF) {
throw std::runtime_error("Unexpected outer database root page type");
}
if (header.entries != 2) {
throw std::runtime_error("Unexpected number of entries in outer database root page");
}
RecordsPage page(header);
db_file >> page;
// First record should be the string "main"
if (!std::holds_alternative<DataRecord>(page.records.at(0)) || std::get<DataRecord>(page.records.at(0)).data != SUBDATABASE_NAME) {
throw std::runtime_error("Subdatabase has an unexpected name");
}
// Check length of page number for subdatabase location
if (!std::holds_alternative<DataRecord>(page.records.at(1)) || std::get<DataRecord>(page.records.at(1)).m_header.len != 4) {
throw std::runtime_error("Subdatabase page number has unexpected length");
}
// Read subdatabase page number
// It is written as a big endian 32 bit number
uint32_t main_db_page = ReadBE32(UCharCast(std::get<DataRecord>(page.records.at(1)).data.data()));
// The main database is in a page that doesn't exist
if (main_db_page > outer_meta.last_page) {
throw std::runtime_error("Page number is greater than database last page");
}
// Read the inner metapage
SeekToPage(db_file, main_db_page, page_size);
MetaPage inner_meta(main_db_page);
db_file >> inner_meta;
if (inner_meta.pagesize != page_size) {
throw std::runtime_error("Unexpected page size");
}
if (inner_meta.last_page > outer_meta.last_page) {
throw std::runtime_error("Subdatabase last page is greater than database last page");
}
// Make sure encryption is disabled
if (inner_meta.encrypt_algo != 0) {
throw std::runtime_error("BDB builtin encryption is not supported");
}
// Do a DFS through the BTree, starting at root
std::vector<uint32_t> pages{inner_meta.root};
while (pages.size() > 0) {
uint32_t curr_page = pages.back();
// It turns out BDB completely ignores this last_page field and doesn't actually update it to the correct
// last page. While we should be checking this, we can't.
// This is left commented out as a reminder to not accidentally implement this in the future.
// if (curr_page > inner_meta.last_page) {
// throw std::runtime_error("Page number is greater than subdatabase last page");
// }
pages.pop_back();
SeekToPage(db_file, curr_page, page_size);
PageHeader header(curr_page, inner_meta.other_endian);
db_file >> header;
switch (header.type) {
case PageType::BTREE_INTERNAL: {
InternalPage int_page(header);
db_file >> int_page;
for (const InternalRecord& rec : int_page.records) {
if (rec.m_header.deleted) continue;
pages.push_back(rec.page_num);
}
break;
}
case PageType::BTREE_LEAF: {
RecordsPage rec_page(header);
db_file >> rec_page;
if (rec_page.records.size() % 2 != 0) {
// BDB stores key value pairs in consecutive records, thus an odd number of records is unexpected
throw std::runtime_error("Records page has odd number of records");
}
bool is_key = true;
std::vector<std::byte> key;
for (const std::variant<DataRecord, OverflowRecord>& rec : rec_page.records) {
std::vector<std::byte> data;
if (const DataRecord* drec = std::get_if<DataRecord>(&rec)) {
if (drec->m_header.deleted) continue;
data = drec->data;
} else if (const OverflowRecord* orec = std::get_if<OverflowRecord>(&rec)) {
if (orec->m_header.deleted) continue;
uint32_t next_page = orec->page_number;
while (next_page != 0) {
SeekToPage(db_file, next_page, page_size);
PageHeader opage_header(next_page, inner_meta.other_endian);
db_file >> opage_header;
if (opage_header.type != PageType::OVERFLOW_DATA) {
throw std::runtime_error("Bad overflow record page type");
}
OverflowPage opage(opage_header);
db_file >> opage;
data.insert(data.end(), opage.data.begin(), opage.data.end());
next_page = opage_header.next_page;
}
}
if (is_key) {
key = data;
} else {
m_records.emplace(SerializeData{key.begin(), key.end()}, SerializeData{data.begin(), data.end()});
key.clear();
}
is_key = !is_key;
}
break;
}
default:
throw std::runtime_error("Unexpected page type");
}
}
}
std::unique_ptr<DatabaseBatch> BerkeleyRODatabase::MakeBatch(bool flush_on_close)
{
return std::make_unique<BerkeleyROBatch>(*this);
}
bool BerkeleyRODatabase::Backup(const std::string& dest) const
{
fs::path src(m_filepath);
fs::path dst(fs::PathFromString(dest));
if (fs::is_directory(dst)) {
dst = BDBDataFile(dst);
}
try {
if (fs::exists(dst) && fs::equivalent(src, dst)) {
LogPrintf("cannot backup to wallet source file %s\n", fs::PathToString(dst));
return false;
}
fs::copy_file(src, dst, fs::copy_options::overwrite_existing);
LogPrintf("copied %s to %s\n", fs::PathToString(m_filepath), fs::PathToString(dst));
return true;
} catch (const fs::filesystem_error& e) {
LogPrintf("error copying %s to %s - %s\n", fs::PathToString(m_filepath), fs::PathToString(dst), fsbridge::get_filesystem_error_message(e));
return false;
}
}
bool BerkeleyROBatch::ReadKey(DataStream&& key, DataStream& value)
{
SerializeData key_data{key.begin(), key.end()};
const auto it{m_database.m_records.find(key_data)};
if (it == m_database.m_records.end()) {
return false;
}
auto val = it->second;
value.clear();
value.write(Span(val));
return true;
}
bool BerkeleyROBatch::HasKey(DataStream&& key)
{
SerializeData key_data{key.begin(), key.end()};
return m_database.m_records.count(key_data) > 0;
}
BerkeleyROCursor::BerkeleyROCursor(const BerkeleyRODatabase& database, Span<const std::byte> prefix)
: m_database(database)
{
std::tie(m_cursor, m_cursor_end) = m_database.m_records.equal_range(BytePrefix{prefix});
}
DatabaseCursor::Status BerkeleyROCursor::Next(DataStream& ssKey, DataStream& ssValue)
{
if (m_cursor == m_cursor_end) {
return DatabaseCursor::Status::DONE;
}
ssKey.write(Span(m_cursor->first));
ssValue.write(Span(m_cursor->second));
m_cursor++;
return DatabaseCursor::Status::MORE;
}
std::unique_ptr<DatabaseCursor> BerkeleyROBatch::GetNewPrefixCursor(Span<const std::byte> prefix)
{
return std::make_unique<BerkeleyROCursor>(m_database, prefix);
}
std::unique_ptr<BerkeleyRODatabase> MakeBerkeleyRODatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error)
{
fs::path data_file = BDBDataFile(path);
try {
std::unique_ptr<BerkeleyRODatabase> db = std::make_unique<BerkeleyRODatabase>(data_file);
status = DatabaseStatus::SUCCESS;
return db;
} catch (const std::runtime_error& e) {
error.original = e.what();
status = DatabaseStatus::FAILED_LOAD;
return nullptr;
}
}
} // namespace wallet

124
src/wallet/migrate.h Normal file
View file

@ -0,0 +1,124 @@
// Copyright (c) 2021 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#ifndef BITCOIN_WALLET_MIGRATE_H
#define BITCOIN_WALLET_MIGRATE_H
#include <wallet/db.h>
#include <optional>
namespace wallet {
using BerkeleyROData = std::map<SerializeData, SerializeData, std::less<>>;
/**
* A class representing a BerkeleyDB file from which we can only read records.
* This is used only for migration of legacy to descriptor wallets
*/
class BerkeleyRODatabase : public WalletDatabase
{
private:
const fs::path m_filepath;
public:
/** Create DB handle */
BerkeleyRODatabase(const fs::path& filepath, bool open = true) : WalletDatabase(), m_filepath(filepath)
{
if (open) Open();
}
~BerkeleyRODatabase(){};
BerkeleyROData m_records;
/** Open the database if it is not already opened. */
void Open() override;
/** Indicate the a new database user has began using the database. Increments m_refcount */
void AddRef() override {}
/** Indicate that database user has stopped using the database and that it could be flushed or closed. Decrement m_refcount */
void RemoveRef() override {}
/** Rewrite the entire database on disk, with the exception of key pszSkip if non-zero
*/
bool Rewrite(const char* pszSkip = nullptr) override { return false; }
/** Back up the entire database to a file.
*/
bool Backup(const std::string& strDest) const override;
/** Make sure all changes are flushed to database file.
*/
void Flush() override {}
/** Flush to the database file and close the database.
* Also close the environment if no other databases are open in it.
*/
void Close() override {}
/* flush the wallet passively (TRY_LOCK)
ideal to be called periodically */
bool PeriodicFlush() override { return false; }
void IncrementUpdateCounter() override {}
void ReloadDbEnv() override {}
/** Return path to main database file for logs and error messages. */
std::string Filename() override { return fs::PathToString(m_filepath); }
std::string Format() override { return "bdb_ro"; }
/** Make a DatabaseBatch connected to this database */
std::unique_ptr<DatabaseBatch> MakeBatch(bool flush_on_close = true) override;
};
class BerkeleyROCursor : public DatabaseCursor
{
private:
const BerkeleyRODatabase& m_database;
BerkeleyROData::const_iterator m_cursor;
BerkeleyROData::const_iterator m_cursor_end;
public:
explicit BerkeleyROCursor(const BerkeleyRODatabase& database, Span<const std::byte> prefix = {});
~BerkeleyROCursor() {}
Status Next(DataStream& key, DataStream& value) override;
};
/** RAII class that provides access to a BerkeleyRODatabase */
class BerkeleyROBatch : public DatabaseBatch
{
private:
const BerkeleyRODatabase& m_database;
bool ReadKey(DataStream&& key, DataStream& value) override;
// WriteKey returns true since various automatic upgrades for older wallets will expect writing to not fail.
// It is okay for this batch type to not actually write anything as those automatic upgrades will occur again after migration.
bool WriteKey(DataStream&& key, DataStream&& value, bool overwrite = true) override { return true; }
bool EraseKey(DataStream&& key) override { return false; }
bool HasKey(DataStream&& key) override;
bool ErasePrefix(Span<const std::byte> prefix) override { return false; }
public:
explicit BerkeleyROBatch(const BerkeleyRODatabase& database) : m_database(database) {}
~BerkeleyROBatch() {}
BerkeleyROBatch(const BerkeleyROBatch&) = delete;
BerkeleyROBatch& operator=(const BerkeleyROBatch&) = delete;
void Flush() override {}
void Close() override {}
std::unique_ptr<DatabaseCursor> GetNewCursor() override { return std::make_unique<BerkeleyROCursor>(m_database); }
std::unique_ptr<DatabaseCursor> GetNewPrefixCursor(Span<const std::byte> prefix) override;
bool TxnBegin() override { return false; }
bool TxnCommit() override { return false; }
bool TxnAbort() override { return false; }
};
//! Return object giving access to Berkeley Read Only database at specified path.
std::unique_ptr<BerkeleyRODatabase> MakeBerkeleyRODatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error);
} // namespace wallet
#endif // BITCOIN_WALLET_MIGRATE_H

30
src/wallet/test/db_tests.cpp
View file

@ -16,6 +16,7 @@
#ifdef USE_SQLITE
#include <wallet/sqlite.h>
#endif
#include <wallet/migrate.h>
#include <wallet/test/util.h>
#include <wallet/walletutil.h> // for WALLET_FLAG_DESCRIPTORS
@ -132,6 +133,8 @@ static std::vector<std::unique_ptr<WalletDatabase>> TestDatabases(const fs::path
bilingual_str error;
#ifdef USE_BDB
dbs.emplace_back(MakeBerkeleyDatabase(path_root / "bdb", options, status, error));
// Needs BDB to make the DB to read
dbs.emplace_back(std::make_unique<BerkeleyRODatabase>(BDBDataFile(path_root / "bdb"), /*open=*/false));
#endif
#ifdef USE_SQLITE
dbs.emplace_back(MakeSQLiteDatabase(path_root / "sqlite", options, status, error));
@ -146,13 +149,18 @@ BOOST_AUTO_TEST_CASE(db_cursor_prefix_range_test)
for (const auto& database : TestDatabases(m_path_root)) {
std::vector<std::string> prefixes = {"", "FIRST", "SECOND", "P\xfe\xff", "P\xff\x01", "\xff\xff"};
// Write elements to it
std::unique_ptr<DatabaseBatch> handler = Assert(database)->MakeBatch();
if (dynamic_cast<BerkeleyRODatabase*>(database.get())) {
// For BerkeleyRO, open the file now. This must happen after BDB has written to the file
database->Open();
} else {
// Write elements to it if not berkeleyro
for (unsigned int i = 0; i < 10; i++) {
for (const auto& prefix : prefixes) {
BOOST_CHECK(handler->Write(std::make_pair(prefix, i), i));
}
}
}
// Now read all the items by prefix and verify that each element gets parsed correctly
for (const auto& prefix : prefixes) {
@ -178,6 +186,8 @@ BOOST_AUTO_TEST_CASE(db_cursor_prefix_range_test)
// Let's now read it once more, it should return DONE
BOOST_CHECK(cursor->Next(key, value) == DatabaseCursor::Status::DONE);
}
handler.reset();
database->Close();
}
}
@ -197,13 +207,23 @@ BOOST_AUTO_TEST_CASE(db_cursor_prefix_byte_test)
ffs{StringData("\xff\xffsuffix"), StringData("ffs")};
for (const auto& database : TestDatabases(m_path_root)) {
std::unique_ptr<DatabaseBatch> batch = database->MakeBatch();
if (dynamic_cast<BerkeleyRODatabase*>(database.get())) {
// For BerkeleyRO, open the file now. This must happen after BDB has written to the file
database->Open();
} else {
// Write elements to it if not berkeleyro
for (const auto& [k, v] : {e, p, ps, f, fs, ff, ffs}) {
batch->Write(Span{k}, Span{v});
}
}
CheckPrefix(*batch, StringBytes(""), {e, p, ps, f, fs, ff, ffs});
CheckPrefix(*batch, StringBytes("prefix"), {p, ps});
CheckPrefix(*batch, StringBytes("\xff"), {f, fs, ff, ffs});
CheckPrefix(*batch, StringBytes("\xff\xff"), {ff, ffs});
batch.reset();
database->Close();
}
}
@ -213,6 +233,10 @@ BOOST_AUTO_TEST_CASE(db_availability_after_write_error)
// To simulate the behavior, record overwrites are disallowed, and the test verifies
// that the database remains active after failing to store an existing record.
for (const auto& database : TestDatabases(m_path_root)) {
if (dynamic_cast<BerkeleyRODatabase*>(database.get())) {
// Skip this test if BerkeleyRO
continue;
}
// Write original record
std::unique_ptr<DatabaseBatch> batch = database->MakeBatch();
std::string key = "key";
@ -241,6 +265,10 @@ BOOST_AUTO_TEST_CASE(erase_prefix)
auto make_key = [](std::string type, std::string id) { return std::make_pair(type, id); };
for (const auto& database : TestDatabases(m_path_root)) {
if (dynamic_cast<BerkeleyRODatabase*>(database.get())) {
// Skip this test if BerkeleyRO
continue;
}
std::unique_ptr<DatabaseBatch> batch = database->MakeBatch();
// Write two entries with the same key type prefix, a third one with a different prefix

133
src/wallet/test/fuzz/wallet_bdb_parser.cpp Normal file
View file

@ -0,0 +1,133 @@
// Copyright (c) 2023 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include <config/bitcoin-config.h> // IWYU pragma: keep
#include <test/fuzz/FuzzedDataProvider.h>
#include <test/fuzz/fuzz.h>
#include <test/fuzz/util.h>
#include <test/util/setup_common.h>
#include <util/fs.h>
#include <util/time.h>
#include <util/translation.h>
#include <wallet/bdb.h>
#include <wallet/db.h>
#include <wallet/dump.h>
#include <wallet/migrate.h>
#include <fstream>
#include <iostream>
using wallet::DatabaseOptions;
using wallet::DatabaseStatus;
namespace {
TestingSetup* g_setup;
} // namespace
void initialize_wallet_bdb_parser()
{
static auto testing_setup = MakeNoLogFileContext<TestingSetup>();
g_setup = testing_setup.get();
}
FUZZ_TARGET(wallet_bdb_parser, .init = initialize_wallet_bdb_parser)
{
const auto wallet_path = g_setup->m_args.GetDataDirNet() / "fuzzed_wallet.dat";
{
AutoFile outfile{fsbridge::fopen(wallet_path, "wb")};
outfile << Span{buffer};
}
const DatabaseOptions options{};
DatabaseStatus status;
bilingual_str error;
fs::path bdb_ro_dumpfile{g_setup->m_args.GetDataDirNet() / "fuzzed_dumpfile_bdb_ro.dump"};
if (fs::exists(bdb_ro_dumpfile)) { // Writing into an existing dump file will throw an exception
remove(bdb_ro_dumpfile);
}
g_setup->m_args.ForceSetArg("-dumpfile", fs::PathToString(bdb_ro_dumpfile));
#ifdef USE_BDB
bool bdb_ro_err = false;
bool bdb_ro_pgno_err = false;
#endif
auto db{MakeBerkeleyRODatabase(wallet_path, options, status, error)};
if (db) {
assert(DumpWallet(g_setup->m_args, *db, error));
} else {
#ifdef USE_BDB
bdb_ro_err = true;
#endif
if (error.original == "AutoFile::ignore: end of file: iostream error" ||
error.original == "AutoFile::read: end of file: iostream error" ||
error.original == "Not a BDB file" ||
error.original == "Unsupported BDB data file version number" ||
error.original == "Unexpected page type, should be 9 (BTree Metadata)" ||
error.original == "Unexpected database flags, should only be 0x20 (subdatabases)" ||
error.original == "Unexpected outer database root page type" ||
error.original == "Unexpected number of entries in outer database root page" ||
error.original == "Subdatabase has an unexpected name" ||
error.original == "Subdatabase page number has unexpected length" ||
error.original == "Unexpected inner database page type" ||
error.original == "Unknown record type in records page" ||
error.original == "Unknown record type in internal page" ||
error.original == "Unexpected page size" ||
error.original == "Unexpected page type" ||
error.original == "Page number mismatch" ||
error.original == "Bad btree level" ||
error.original == "Bad page size" ||
error.original == "File size is not a multiple of page size" ||
error.original == "Meta page number mismatch") {
// Do nothing
} else if (error.original == "Subdatabase last page is greater than database last page" ||
error.original == "Page number is greater than database last page" ||
error.original == "Page number is greater than subdatabase last page" ||
error.original == "Last page number could not fit in file") {
#ifdef USE_BDB
bdb_ro_pgno_err = true;
#endif
} else {
throw std::runtime_error(error.original);
}
}
#ifdef USE_BDB
// Try opening with BDB
fs::path bdb_dumpfile{g_setup->m_args.GetDataDirNet() / "fuzzed_dumpfile_bdb.dump"};
if (fs::exists(bdb_dumpfile)) { // Writing into an existing dump file will throw an exception
remove(bdb_dumpfile);
}
g_setup->m_args.ForceSetArg("-dumpfile", fs::PathToString(bdb_dumpfile));
try {
auto db{MakeBerkeleyDatabase(wallet_path, options, status, error)};
if (bdb_ro_err && !db) {
return;
}
assert(db);
if (bdb_ro_pgno_err) {
// BerkeleyRO will throw on opening for errors involving bad page numbers, but BDB does not.
// Ignore those.
return;
}
assert(!bdb_ro_err);
assert(DumpWallet(g_setup->m_args, *db, error));
} catch (const std::runtime_error& e) {
if (bdb_ro_err) return;
throw e;
}
// Make sure the dumpfiles match
if (fs::exists(bdb_ro_dumpfile) && fs::exists(bdb_dumpfile)) {
std::ifstream bdb_ro_dump(bdb_ro_dumpfile, std::ios_base::binary | std::ios_base::in);
std::ifstream bdb_dump(bdb_dumpfile, std::ios_base::binary | std::ios_base::in);
assert(std::equal(
std::istreambuf_iterator<char>(bdb_ro_dump.rdbuf()),
std::istreambuf_iterator<char>(),
std::istreambuf_iterator<char>(bdb_dump.rdbuf())));
}
#endif
}

5
src/wallet/test/util.cpp
View file

@ -93,11 +93,6 @@ CTxDestination getNewDestination(CWallet& w, OutputType output_type)
return *Assert(w.GetNewDestination(output_type, ""));
}
// BytePrefix compares equality with other byte spans that begin with the same prefix.
struct BytePrefix { Span<const std::byte> prefix; };
bool operator<(BytePrefix a, Span<const std::byte> b) { return a.prefix < b.subspan(0, std::min(a.prefix.size(), b.size())); }
bool operator<(Span<const std::byte> a, BytePrefix b) { return a.subspan(0, std::min(a.size(), b.prefix.size())) < b.prefix; }
MockableCursor::MockableCursor(const MockableData& records, bool pass, Span<const std::byte> prefix)
{
m_pass = pass;

5
src/wallet/wallet.cpp
View file

@ -373,7 +373,12 @@ std::shared_ptr<CWallet> CreateWallet(WalletContext& context, const std::string&
uint64_t wallet_creation_flags = options.create_flags;
const SecureString& passphrase = options.create_passphrase;
ArgsManager& args = *Assert(context.args);
if (wallet_creation_flags & WALLET_FLAG_DESCRIPTORS) options.require_format = DatabaseFormat::SQLITE;
else if (args.GetBoolArg("-swapbdbendian", false)) {
options.require_format = DatabaseFormat::BERKELEY_SWAP;
}
// Indicate that the wallet is actually supposed to be blank and not just blank to make it encrypted
bool create_blank = (wallet_creation_flags & WALLET_FLAG_BLANK_WALLET);

10
src/wallet/walletdb.cpp
View file

@ -21,6 +21,7 @@
#ifdef USE_BDB
#include <wallet/bdb.h>
#endif
#include <wallet/migrate.h>
#ifdef USE_SQLITE
#include <wallet/sqlite.h>
#endif
@ -1387,6 +1388,11 @@ std::unique_ptr<WalletDatabase> MakeDatabase(const fs::path& path, const Databas
return nullptr;
}
// If BERKELEY was the format, then change the format from BERKELEY to BERKELEY_RO
if (format && options.require_format && format == DatabaseFormat::BERKELEY && options.require_format == DatabaseFormat::BERKELEY_RO) {
format = DatabaseFormat::BERKELEY_RO;
}
// A db already exists so format is set, but options also specifies the format, so make sure they agree
if (format && options.require_format && format != options.require_format) {
error = Untranslated(strprintf("Failed to load database path '%s'. Data is not in required format.", fs::PathToString(path)));
@ -1420,6 +1426,10 @@ std::unique_ptr<WalletDatabase> MakeDatabase(const fs::path& path, const Databas
}
}
if (format == DatabaseFormat::BERKELEY_RO) {
return MakeBerkeleyRODatabase(path, options, status, error);
}
#ifdef USE_BDB
if constexpr (true) {
return MakeBerkeleyDatabase(path, options, status, error);

5
src/wallet/wallettool.cpp
View file

@ -192,6 +192,11 @@ bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command)
ReadDatabaseArgs(args, options);
options.require_existing = true;
DatabaseStatus status;
if (args.GetBoolArg("-withinternalbdb", false) && IsBDBFile(BDBDataFile(path))) {
options.require_format = DatabaseFormat::BERKELEY_RO;
}
bilingual_str error;
std::unique_ptr<WalletDatabase> database = MakeDatabase(path, options, status, error);
if (!database) {

5
test/functional/test_framework/test_node.py
View file

@ -419,8 +419,9 @@ class TestNode():
return True
def wait_until_stopped(self, *, timeout=BITCOIND_PROC_WAIT_TIMEOUT, expect_error=False, **kwargs):
expected_ret_code = 1 if expect_error else 0 # Whether node shutdown return EXIT_FAILURE or EXIT_SUCCESS
self.wait_until(lambda: self.is_node_stopped(expected_ret_code=expected_ret_code, **kwargs), timeout=timeout)
if "expected_ret_code" not in kwargs:
kwargs["expected_ret_code"] = 1 if expect_error else 0 # Whether node shutdown return EXIT_FAILURE or EXIT_SUCCESS
self.wait_until(lambda: self.is_node_stopped(**kwargs), timeout=timeout)
def replace_in_config(self, replacements):
"""

2
test/functional/test_runner.py
View file

@ -183,6 +183,8 @@ BASE_SCRIPTS = [
'mempool_resurrect.py',
'wallet_txn_doublespend.py --mineblock',
'tool_wallet.py --legacy-wallet',
'tool_wallet.py --legacy-wallet --bdbro',
'tool_wallet.py --legacy-wallet --bdbro --swap-bdb-endian',
'tool_wallet.py --descriptors',
'tool_signet_miner.py --legacy-wallet',
'tool_signet_miner.py --descriptors',

101
test/functional/tool_wallet.py
View file

@ -5,6 +5,7 @@
"""Test bitcoin-wallet."""
import os
import platform
import stat
import subprocess
import textwrap
@ -14,6 +15,7 @@ from collections import OrderedDict
from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import (
assert_equal,
assert_greater_than,
sha256sum_file,
)
@ -21,11 +23,15 @@ from test_framework.util import (
class ToolWalletTest(BitcoinTestFramework):
def add_options(self, parser):
self.add_wallet_options(parser)
parser.add_argument("--bdbro", action="store_true", help="Use the BerkeleyRO internal parser when dumping a Berkeley DB wallet file")
parser.add_argument("--swap-bdb-endian", action="store_true",help="When making Legacy BDB wallets, always make then byte swapped internally")
def set_test_params(self):
self.num_nodes = 1
self.setup_clean_chain = True
self.rpc_timeout = 120
if self.options.swap_bdb_endian:
self.extra_args = [["-swapbdbendian"]]
def skip_test_if_missing_module(self):
self.skip_if_no_wallet()
@ -35,15 +41,21 @@ class ToolWalletTest(BitcoinTestFramework):
default_args = ['-datadir={}'.format(self.nodes[0].datadir_path), '-chain=%s' % self.chain]
if not self.options.descriptors and 'create' in args:
default_args.append('-legacy')
if "dump" in args and self.options.bdbro:
default_args.append("-withinternalbdb")
return subprocess.Popen([self.options.bitcoinwallet] + default_args + list(args), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
def assert_raises_tool_error(self, error, *args):
p = self.bitcoin_wallet_process(*args)
stdout, stderr = p.communicate()
assert_equal(p.poll(), 1)
assert_equal(stdout, '')
assert_equal(stderr.strip(), error)
if isinstance(error, tuple):
assert_equal(p.poll(), error[0])
assert error[1] in stderr.strip()
else:
assert_equal(p.poll(), 1)
assert error in stderr.strip()
def assert_tool_output(self, output, *args):
p = self.bitcoin_wallet_process(*args)
@ -451,6 +463,88 @@ class ToolWalletTest(BitcoinTestFramework):
''')
self.assert_tool_output(expected_output, "-wallet=conflicts", "info")
def test_dump_endianness(self):
self.log.info("Testing dumps of the same contents with different BDB endianness")
self.start_node(0)
self.nodes[0].createwallet("endian")
self.stop_node(0)
wallet_dump = self.nodes[0].datadir_path / "endian.dump"
self.assert_tool_output("The dumpfile may contain private keys. To ensure the safety of your Bitcoin, do not share the dumpfile.\n", "-wallet=endian", f"-dumpfile={wallet_dump}", "dump")
expected_dump = self.read_dump(wallet_dump)
self.do_tool_createfromdump("native_endian", "endian.dump", "bdb")
native_dump = self.read_dump(self.nodes[0].datadir_path / "rt-native_endian.dump")
self.assert_dump(expected_dump, native_dump)
self.do_tool_createfromdump("other_endian", "endian.dump", "bdb_swap")
other_dump = self.read_dump(self.nodes[0].datadir_path / "rt-other_endian.dump")
self.assert_dump(expected_dump, other_dump)
def test_dump_very_large_records(self):
self.log.info("Test that wallets with large records are successfully dumped")
self.start_node(0)
self.nodes[0].createwallet("bigrecords")
wallet = self.nodes[0].get_wallet_rpc("bigrecords")
# Both BDB and sqlite have maximum page sizes of 65536 bytes, with defaults of 4096
# When a record exceeds some size threshold, both BDB and SQLite will store the data
# in one or more overflow pages. We want to make sure that our tooling can dump such
# records, even when they span multiple pages. To make a large record, we just need
# to make a very big transaction.
self.generate(self.nodes[0], 101)
def_wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name)
outputs = {}
for i in range(500):
outputs[wallet.getnewaddress(address_type="p2sh-segwit")] = 0.01
def_wallet.sendmany(amounts=outputs)
self.generate(self.nodes[0], 1)
send_res = wallet.sendall([def_wallet.getnewaddress()])
self.generate(self.nodes[0], 1)
assert_equal(send_res["complete"], True)
tx = wallet.gettransaction(txid=send_res["txid"], verbose=True)
assert_greater_than(tx["decoded"]["size"], 70000)
self.stop_node(0)
wallet_dump = self.nodes[0].datadir_path / "bigrecords.dump"
self.assert_tool_output("The dumpfile may contain private keys. To ensure the safety of your Bitcoin, do not share the dumpfile.\n", "-wallet=bigrecords", f"-dumpfile={wallet_dump}", "dump")
dump = self.read_dump(wallet_dump)
for k,v in dump.items():
if tx["hex"] in v:
break
else:
assert False, "Big transaction was not found in wallet dump"
def test_dump_unclean_lsns(self):
if not self.options.bdbro:
return
self.log.info("Test that a legacy wallet that has not been compacted is not dumped by bdbro")
self.start_node(0, extra_args=["-flushwallet=0"])
self.nodes[0].createwallet("unclean_lsn")
wallet = self.nodes[0].get_wallet_rpc("unclean_lsn")
# First unload and load normally to make sure everything is written
wallet.unloadwallet()
self.nodes[0].loadwallet("unclean_lsn")
# Next cause a bunch of writes by filling the keypool
wallet.keypoolrefill(wallet.getwalletinfo()["keypoolsize"] + 100)
# Lastly kill bitcoind so that the LSNs don't get reset
self.nodes[0].process.kill()
self.nodes[0].wait_until_stopped(expected_ret_code=1 if platform.system() == "Windows" else -9)
assert self.nodes[0].is_node_stopped()
wallet_dump = self.nodes[0].datadir_path / "unclean_lsn.dump"
self.assert_raises_tool_error("LSNs are not reset, this database is not completely flushed. Please reopen then close the database with a version that has BDB support", "-wallet=unclean_lsn", f"-dumpfile={wallet_dump}", "dump")
# File can be dumped after reload it normally
self.start_node(0)
self.nodes[0].loadwallet("unclean_lsn")
self.stop_node(0)
self.assert_tool_output("The dumpfile may contain private keys. To ensure the safety of your Bitcoin, do not share the dumpfile.\n", "-wallet=unclean_lsn", f"-dumpfile={wallet_dump}", "dump")
def run_test(self):
self.wallet_path = self.nodes[0].wallets_path / self.default_wallet_name / self.wallet_data_filename
self.test_invalid_tool_commands_and_args()
@ -462,8 +556,11 @@ class ToolWalletTest(BitcoinTestFramework):
if not self.options.descriptors:
# Salvage is a legacy wallet only thing
self.test_salvage()
self.test_dump_endianness()
self.test_dump_unclean_lsns()
self.test_dump_createfromdump()
self.test_chainless_conflicts()
self.test_dump_very_large_records()
if __name__ == '__main__':
ToolWalletTest().main()