diff --git a/src/dummywallet.cpp b/src/dummywallet.cpp index 8d2dcd0279..4543f098a1 100644 --- a/src/dummywallet.cpp +++ b/src/dummywallet.cpp @@ -40,7 +40,6 @@ void DummyWalletInit::AddWalletOptions(ArgsManager& argsman) const "-salvagewallet", "-spendzeroconfchange", "-txconfirmtarget=", - "-upgradewallet", "-wallet=", "-walletbroadcast", "-walletdir=", diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index 188289b010..d2e1be6402 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -438,12 +438,12 @@ bool LegacyScriptPubKeyMan::CanGetAddresses(bool internal) const return keypool_has_keys; } -bool LegacyScriptPubKeyMan::Upgrade(int prev_version, bilingual_str& error) +bool LegacyScriptPubKeyMan::Upgrade(int prev_version, int new_version, bilingual_str& error) { LOCK(cs_KeyStore); bool hd_upgrade = false; bool split_upgrade = false; - if (m_storage.CanSupportFeature(FEATURE_HD) && !IsHDEnabled()) { + if (IsFeatureSupported(new_version, FEATURE_HD) && !IsHDEnabled()) { WalletLogPrintf("Upgrading wallet to HD\n"); m_storage.SetMinVersion(FEATURE_HD); @@ -453,10 +453,17 @@ bool LegacyScriptPubKeyMan::Upgrade(int prev_version, bilingual_str& error) hd_upgrade = true; } // Upgrade to HD chain split if necessary - if (m_storage.CanSupportFeature(FEATURE_HD_SPLIT)) { + if (IsFeatureSupported(new_version, FEATURE_HD_SPLIT)) { WalletLogPrintf("Upgrading wallet to use HD chain split\n"); m_storage.SetMinVersion(FEATURE_PRE_SPLIT_KEYPOOL); split_upgrade = FEATURE_HD_SPLIT > prev_version; + // Upgrade the HDChain + if (m_hd_chain.nVersion < CHDChain::VERSION_HD_CHAIN_SPLIT) { + m_hd_chain.nVersion = CHDChain::VERSION_HD_CHAIN_SPLIT; + if (!WalletBatch(m_storage.GetDatabase()).WriteHDChain(m_hd_chain)) { + throw std::runtime_error(std::string(__func__) + ": writing chain failed"); + } + } } // Mark all keys currently in the keypool as pre-split if (split_upgrade) { diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index 63c10b7a0d..3bf8f78120 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -37,7 +37,7 @@ public: virtual bool IsWalletFlagSet(uint64_t) const = 0; virtual void UnsetBlankWalletFlag(WalletBatch&) = 0; virtual bool CanSupportFeature(enum WalletFeature) const = 0; - virtual void SetMinVersion(enum WalletFeature, WalletBatch* = nullptr, bool = false) = 0; + virtual void SetMinVersion(enum WalletFeature, WalletBatch* = nullptr) = 0; virtual const CKeyingMaterial& GetEncryptionKey() const = 0; virtual bool HasEncryptionKeys() const = 0; virtual bool IsLocked() const = 0; @@ -206,7 +206,7 @@ public: virtual bool CanGetAddresses(bool internal = false) const { return false; } /** Upgrades the wallet to the specified version */ - virtual bool Upgrade(int prev_version, bilingual_str& error) { return false; } + virtual bool Upgrade(int prev_version, int new_version, bilingual_str& error) { return false; } virtual bool HavePrivateKeys() const { return false; } @@ -371,7 +371,7 @@ public: bool SetupGeneration(bool force = false) override; - bool Upgrade(int prev_version, bilingual_str& error) override; + bool Upgrade(int prev_version, int new_version, bilingual_str& error) override; bool HavePrivateKeys() const override; diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index e1f34fbcf9..d414555511 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -436,21 +436,13 @@ void CWallet::chainStateFlushed(const CBlockLocator& loc) batch.WriteBestBlock(loc); } -void CWallet::SetMinVersion(enum WalletFeature nVersion, WalletBatch* batch_in, bool fExplicit) +void CWallet::SetMinVersion(enum WalletFeature nVersion, WalletBatch* batch_in) { LOCK(cs_wallet); if (nWalletVersion >= nVersion) return; - - // when doing an explicit upgrade, if we pass the max version permitted, upgrade all the way - if (fExplicit && nVersion > nWalletMaxVersion) - nVersion = FEATURE_LATEST; - nWalletVersion = nVersion; - if (nVersion > nWalletMaxVersion) - nWalletMaxVersion = nVersion; - { WalletBatch* batch = batch_in ? batch_in : new WalletBatch(*database); if (nWalletVersion > 40000) @@ -460,18 +452,6 @@ void CWallet::SetMinVersion(enum WalletFeature nVersion, WalletBatch* batch_in, } } -bool CWallet::SetMaxVersion(int nVersion) -{ - LOCK(cs_wallet); - // cannot downgrade below current version - if (nWalletVersion > nVersion) - return false; - - nWalletMaxVersion = nVersion; - - return true; -} - std::set CWallet::GetConflicts(const uint256& txid) const { std::set result; @@ -656,7 +636,7 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) } // Encryption was introduced in version 0.4.0 - SetMinVersion(FEATURE_WALLETCRYPT, encrypted_batch, true); + SetMinVersion(FEATURE_WALLETCRYPT, encrypted_batch); if (!encrypted_batch->TxnCommit()) { delete encrypted_batch; @@ -4125,33 +4105,31 @@ const CAddressBookData* CWallet::FindAddressBookEntry(const CTxDestination& dest bool CWallet::UpgradeWallet(int version, bilingual_str& error, std::vector& warnings) { int prev_version = GetVersion(); - int nMaxVersion = version; - if (nMaxVersion == 0) // the -upgradewallet without argument case - { + if (version == 0) { WalletLogPrintf("Performing wallet upgrade to %i\n", FEATURE_LATEST); - nMaxVersion = FEATURE_LATEST; - SetMinVersion(FEATURE_LATEST); // permanently upgrade the wallet immediately + version = FEATURE_LATEST; } else { - WalletLogPrintf("Allowing wallet upgrade up to %i\n", nMaxVersion); + WalletLogPrintf("Allowing wallet upgrade up to %i\n", version); } - if (nMaxVersion < GetVersion()) + if (version < prev_version) { error = _("Cannot downgrade wallet"); return false; } - SetMaxVersion(nMaxVersion); LOCK(cs_wallet); // Do not upgrade versions to any version between HD_SPLIT and FEATURE_PRE_SPLIT_KEYPOOL unless already supporting HD_SPLIT - int max_version = GetVersion(); - if (!CanSupportFeature(FEATURE_HD_SPLIT) && max_version >= FEATURE_HD_SPLIT && max_version < FEATURE_PRE_SPLIT_KEYPOOL) { + if (!CanSupportFeature(FEATURE_HD_SPLIT) && version >= FEATURE_HD_SPLIT && version < FEATURE_PRE_SPLIT_KEYPOOL) { error = _("Cannot upgrade a non HD split wallet without upgrading to support pre split keypool. Please use version 169900 or no version specified."); return false; } + // Permanently upgrade to the version + SetMinVersion(GetClosestWalletFeature(version)); + for (auto spk_man : GetActiveScriptPubKeyMans()) { - if (!spk_man->Upgrade(prev_version, error)) { + if (!spk_man->Upgrade(prev_version, version, error)) { return false; } } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 00e0e3c84d..0934213fc7 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -636,9 +636,6 @@ private: //! the current wallet version: clients below this version are not able to load the wallet int nWalletVersion GUARDED_BY(cs_wallet){FEATURE_BASE}; - //! the maximum wallet format version: memory-only variable that specifies to what version this wallet may be upgraded - int nWalletMaxVersion GUARDED_BY(cs_wallet) = FEATURE_BASE; - int64_t nNextResend = 0; bool fBroadcastTransactions = false; // Local time that the tip block was received. Used to schedule wallet rebroadcasts. @@ -800,8 +797,8 @@ public: const CWalletTx* GetWalletTx(const uint256& hash) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool IsTrusted(const CWalletTx& wtx, std::set& trusted_parents) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - //! check whether we are allowed to upgrade (or already support) to the named feature - bool CanSupportFeature(enum WalletFeature wf) const override EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { AssertLockHeld(cs_wallet); return nWalletMaxVersion >= wf; } + //! check whether we support the named feature + bool CanSupportFeature(enum WalletFeature wf) const override EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { AssertLockHeld(cs_wallet); return IsFeatureSupported(nWalletVersion, wf); } /** * populate vCoins with vector of available COutputs. @@ -853,7 +850,7 @@ public: //! Upgrade stored CKeyMetadata objects to store key origin info as KeyOriginInfo void UpgradeKeyMetadata() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - bool LoadMinVersion(int nVersion) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { AssertLockHeld(cs_wallet); nWalletVersion = nVersion; nWalletMaxVersion = std::max(nWalletMaxVersion, nVersion); return true; } + bool LoadMinVersion(int nVersion) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { AssertLockHeld(cs_wallet); nWalletVersion = nVersion; return true; } /** * Adds a destination data tuple to the store, and saves it to disk @@ -1076,11 +1073,8 @@ public: unsigned int GetKeyPoolSize() const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - //! signify that a particular wallet feature is now used. this may change nWalletVersion and nWalletMaxVersion if those are lower - void SetMinVersion(enum WalletFeature, WalletBatch* batch_in = nullptr, bool fExplicit = false) override; - - //! change which version we're allowed to upgrade to (note that this does not immediately imply upgrading to that format) - bool SetMaxVersion(int nVersion); + //! signify that a particular wallet feature is now used. + void SetMinVersion(enum WalletFeature, WalletBatch* batch_in = nullptr) override; //! get the current wallet format (the oldest client version guaranteed to understand this wallet) int GetVersion() const { LOCK(cs_wallet); return nWalletVersion; } diff --git a/src/wallet/walletutil.cpp b/src/wallet/walletutil.cpp index 6563c45134..702293e6c7 100644 --- a/src/wallet/walletutil.cpp +++ b/src/wallet/walletutil.cpp @@ -79,3 +79,21 @@ std::vector ListWalletDir() return paths; } + +bool IsFeatureSupported(int wallet_version, int feature_version) +{ + return wallet_version >= feature_version; +} + +WalletFeature GetClosestWalletFeature(int version) +{ + if (version >= FEATURE_LATEST) return FEATURE_LATEST; + if (version >= FEATURE_PRE_SPLIT_KEYPOOL) return FEATURE_PRE_SPLIT_KEYPOOL; + if (version >= FEATURE_NO_DEFAULT_KEY) return FEATURE_NO_DEFAULT_KEY; + if (version >= FEATURE_HD_SPLIT) return FEATURE_HD_SPLIT; + if (version >= FEATURE_HD) return FEATURE_HD; + if (version >= FEATURE_COMPRPUBKEY) return FEATURE_COMPRPUBKEY; + if (version >= FEATURE_WALLETCRYPT) return FEATURE_WALLETCRYPT; + if (version >= FEATURE_BASE) return FEATURE_BASE; + return static_cast(0); +} diff --git a/src/wallet/walletutil.h b/src/wallet/walletutil.h index afdcb2e18a..27521abd81 100644 --- a/src/wallet/walletutil.h +++ b/src/wallet/walletutil.h @@ -29,7 +29,8 @@ enum WalletFeature FEATURE_LATEST = FEATURE_PRE_SPLIT_KEYPOOL }; - +bool IsFeatureSupported(int wallet_version, int feature_version); +WalletFeature GetClosestWalletFeature(int version); enum WalletFlags : uint64_t { // wallet flags in the upper section (> 1 << 31) will lead to not opening the wallet if flag is unknown diff --git a/test/functional/data/wallets/high_minversion/.walletlock b/test/functional/data/wallets/high_minversion/.walletlock deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/functional/data/wallets/high_minversion/GENERATE.md b/test/functional/data/wallets/high_minversion/GENERATE.md deleted file mode 100644 index e55c4557ca..0000000000 --- a/test/functional/data/wallets/high_minversion/GENERATE.md +++ /dev/null @@ -1,8 +0,0 @@ -The wallet has been created by starting Bitcoin Core with the options -`-regtest -datadir=/tmp -nowallet -walletdir=$(pwd)/test/functional/data/wallets/`. - -In the source code, `WalletFeature::FEATURE_LATEST` has been modified to be large, so that the minversion is too high -for a current build of the wallet. - -The wallet has then been created with the RPC `createwallet high_minversion true true`, so that a blank wallet with -private keys disabled is created. diff --git a/test/functional/data/wallets/high_minversion/db.log b/test/functional/data/wallets/high_minversion/db.log deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/functional/data/wallets/high_minversion/wallet.dat b/test/functional/data/wallets/high_minversion/wallet.dat deleted file mode 100644 index 99ab809263..0000000000 Binary files a/test/functional/data/wallets/high_minversion/wallet.dat and /dev/null differ diff --git a/test/functional/test_framework/bdb.py b/test/functional/test_framework/bdb.py new file mode 100644 index 0000000000..9de358aa0a --- /dev/null +++ b/test/functional/test_framework/bdb.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +Utilities for working directly with the wallet's BDB database file + +This is specific to the configuration of BDB used in this project: + - pagesize: 4096 bytes + - Outer database contains single subdatabase named 'main' + - btree + - btree leaf pages + +Each key-value pair is two entries in a btree leaf. The first is the key, the one that follows +is the value. And so on. Note that the entry data is itself not in the correct order. Instead +entry offsets are stored in the correct order and those offsets are needed to then retrieve +the data itself. + +Page format can be found in BDB source code dbinc/db_page.h +This only implements the deserialization of btree metadata pages and normal btree pages. Overflow +pages are not implemented but may be needed in the future if dealing with wallets with large +transactions. + +`db_dump -da wallet.dat` is useful to see the data in a wallet.dat BDB file +""" + +import binascii +import struct + +# Important constants +PAGESIZE = 4096 +OUTER_META_PAGE = 0 +INNER_META_PAGE = 2 + +# Page type values +BTREE_INTERNAL = 3 +BTREE_LEAF = 5 +BTREE_META = 9 + +# Some magic numbers for sanity checking +BTREE_MAGIC = 0x053162 +DB_VERSION = 9 + +# Deserializes a leaf page into a dict. +# Btree internal pages have the same header, for those, return None. +# For the btree leaf pages, deserialize them and put all the data into a dict +def dump_leaf_page(data): + page_info = {} + page_header = data[0:26] + _, pgno, prev_pgno, next_pgno, entries, hf_offset, level, pg_type = struct.unpack('QIIIHHBB', page_header) + page_info['pgno'] = pgno + page_info['prev_pgno'] = prev_pgno + page_info['next_pgno'] = next_pgno + page_info['entries'] = entries + page_info['hf_offset'] = hf_offset + page_info['level'] = level + page_info['pg_type'] = pg_type + page_info['entry_offsets'] = struct.unpack('{}H'.format(entries), data[26:26 + entries * 2]) + page_info['entries'] = [] + + if pg_type == BTREE_INTERNAL: + # Skip internal pages. These are the internal nodes of the btree and don't contain anything relevant to us + return None + + assert pg_type == BTREE_LEAF, 'A non-btree leaf page has been encountered while dumping leaves' + + for i in range(0, entries): + offset = page_info['entry_offsets'][i] + entry = {'offset': offset} + page_data_header = data[offset:offset + 3] + e_len, pg_type = struct.unpack('HB', page_data_header) + entry['len'] = e_len + entry['pg_type'] = pg_type + entry['data'] = data[offset + 3:offset + 3 + e_len] + page_info['entries'].append(entry) + + return page_info + +# Deserializes a btree metadata page into a dict. +# Does a simple sanity check on the magic value, type, and version +def dump_meta_page(page): + # metadata page + # general metadata + metadata = {} + meta_page = page[0:72] + _, pgno, magic, version, pagesize, encrypt_alg, pg_type, metaflags, _, free, last_pgno, nparts, key_count, record_count, flags, uid = struct.unpack('QIIIIBBBBIIIIII20s', meta_page) + metadata['pgno'] = pgno + metadata['magic'] = magic + metadata['version'] = version + metadata['pagesize'] = pagesize + metadata['encrypt_alg'] = encrypt_alg + metadata['pg_type'] = pg_type + metadata['metaflags'] = metaflags + metadata['free'] = free + metadata['last_pgno'] = last_pgno + metadata['nparts'] = nparts + metadata['key_count'] = key_count + metadata['record_count'] = record_count + metadata['flags'] = flags + metadata['uid'] = binascii.hexlify(uid) + + assert magic == BTREE_MAGIC, 'bdb magic does not match bdb btree magic' + assert pg_type == BTREE_META, 'Metadata page is not a btree metadata page' + assert version == DB_VERSION, 'Database too new' + + # btree metadata + btree_meta_page = page[72:512] + _, minkey, re_len, re_pad, root, _, crypto_magic, _, iv, chksum = struct.unpack('IIIII368sI12s16s20s', btree_meta_page) + metadata['minkey'] = minkey + metadata['re_len'] = re_len + metadata['re_pad'] = re_pad + metadata['root'] = root + metadata['crypto_magic'] = crypto_magic + metadata['iv'] = binascii.hexlify(iv) + metadata['chksum'] = binascii.hexlify(chksum) + return metadata + +# Given the dict from dump_leaf_page, get the key-value pairs and put them into a dict +def extract_kv_pairs(page_data): + out = {} + last_key = None + for i, entry in enumerate(page_data['entries']): + # By virtue of these all being pairs, even number entries are keys, and odd are values + if i % 2 == 0: + out[entry['data']] = b'' + last_key = entry['data'] + else: + out[last_key] = entry['data'] + return out + +# Extract the key-value pairs of the BDB file given in filename +def dump_bdb_kv(filename): + # Read in the BDB file and start deserializing it + pages = [] + with open(filename, 'rb') as f: + data = f.read(PAGESIZE) + while len(data) > 0: + pages.append(data) + data = f.read(PAGESIZE) + + # Sanity check the meta pages + dump_meta_page(pages[OUTER_META_PAGE]) + dump_meta_page(pages[INNER_META_PAGE]) + + # Fetch the kv pairs from the leaf pages + kv = {} + for i in range(3, len(pages)): + info = dump_leaf_page(pages[i]) + if info is not None: + info_kv = extract_kv_pairs(info) + kv = {**kv, **info_kv} + return kv diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 3356f1ab10..62ff5c6e33 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -8,6 +8,7 @@ from base64 import b64encode from binascii import unhexlify from decimal import Decimal, ROUND_DOWN from subprocess import CalledProcessError +import hashlib import inspect import json import logging @@ -260,6 +261,14 @@ def wait_until_helper(predicate, *, attempts=float('inf'), timeout=float('inf'), raise AssertionError("Predicate {} not true after {} seconds".format(predicate_source, timeout)) raise RuntimeError('Unreachable') +def sha256sum_file(filename): + h = hashlib.sha256() + with open(filename, 'rb') as f: + d = f.read(4096) + while len(d) > 0: + h.update(d) + d = f.read(4096) + return h.digest() # RPC/P2P connection constants and functions ############################################ diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index cf55b28afb..df16ec741f 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -171,6 +171,9 @@ class MultiWalletTest(BitcoinTestFramework): open(not_a_dir, 'a', encoding="utf8").close() self.nodes[0].assert_start_raises_init_error(['-walletdir=' + not_a_dir], 'Error: Specified -walletdir "' + not_a_dir + '" is not a directory') + self.log.info("Do not allow -upgradewallet with multiwallet") + self.nodes[0].assert_start_raises_init_error(['-upgradewallet'], "Error: Error parsing command line arguments: Invalid parameter -upgradewallet") + # if wallets/ doesn't exist, datadir should be the default wallet dir wallet_dir2 = data_dir('walletdir') os.rename(wallet_dir(), wallet_dir2) diff --git a/test/functional/wallet_upgradewallet.py b/test/functional/wallet_upgradewallet.py index 15d9b109c5..8ab4b3f76c 100755 --- a/test/functional/wallet_upgradewallet.py +++ b/test/functional/wallet_upgradewallet.py @@ -13,23 +13,47 @@ Only v0.15.2 and v0.16.3 are required by this test. The others are used in featu import os import shutil +import struct +from io import BytesIO + +from test_framework.bdb import dump_bdb_kv +from test_framework.messages import deser_compact_size, deser_string from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_greater_than, assert_is_hex_string, + assert_raises_rpc_error, + sha256sum_file, ) +UPGRADED_KEYMETA_VERSION = 12 + +def deser_keymeta(f): + ver, create_time = struct.unpack('