tests: Test specific upgradewallet scenarios and that upgrades work

test/functional/wallet_upgradewallet.py

@@ -13,23 +13,47 @@ Only v0.15.2 and v0.16.3 are required by this test. The others are used in featu
 
 import os
 import shutil
+import struct
 
+from io import BytesIO
+
+from test_framework.bdb import dump_bdb_kv
+from test_framework.messages import deser_compact_size, deser_string
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import (
     assert_equal,
     assert_greater_than,
     assert_is_hex_string,
+    assert_raises_rpc_error,
+    sha256sum_file,
 )
 
 
+UPGRADED_KEYMETA_VERSION = 12
+
+def deser_keymeta(f):
+    ver, create_time = struct.unpack('<Iq', f.read(12))
+    kp_str = deser_string(f)
+    seed_id = f.read(20)
+    fpr = f.read(4)
+    path_len = 0
+    path = []
+    has_key_orig = False
+    if ver == UPGRADED_KEYMETA_VERSION:
+        path_len = deser_compact_size(f)
+        for i in range(0, path_len):
+            path.append(struct.unpack('<I', f.read(4))[0])
+        has_key_orig = bool(f.read(1))
+    return ver, create_time, kp_str, seed_id, fpr, path_len, path, has_key_orig
+
 class UpgradeWalletTest(BitcoinTestFramework):
     def set_test_params(self):
         self.setup_clean_chain = True
         self.num_nodes = 3
         self.extra_args = [
-            ["-addresstype=bech32"], # current wallet version
+            ["-addresstype=bech32", "-keypool=2"], # current wallet version
-            ["-usehd=1"],            # v0.16.3 wallet
+            ["-usehd=1", "-keypool=2"],            # v0.16.3 wallet
-            ["-usehd=0"]             # v0.15.2 wallet
+            ["-usehd=0", "-keypool=2"]             # v0.15.2 wallet
         ]
         self.wallet_names = [self.default_wallet_name, None, None]
 
@@ -87,11 +111,19 @@ class UpgradeWalletTest(BitcoinTestFramework):
 
         self.log.info("Test upgradewallet RPC...")
         # Prepare for copying of the older wallet
-        node_master_wallet_dir = os.path.join(node_master.datadir, "regtest/wallets")
+        node_master_wallet_dir = os.path.join(node_master.datadir, "regtest/wallets", self.default_wallet_name)
+        node_master_wallet = os.path.join(node_master_wallet_dir, self.default_wallet_name, self.wallet_data_filename)
         v16_3_wallet       = os.path.join(v16_3_node.datadir, "regtest/wallets/wallet.dat")
         v15_2_wallet       = os.path.join(v15_2_node.datadir, "regtest/wallet.dat")
+        split_hd_wallet    = os.path.join(v15_2_node.datadir, "regtest/splithd")
         self.stop_nodes()
 
+        # Make split hd wallet
+        self.start_node(2, ['-usehd=1', '-keypool=2', '-wallet=splithd'])
+        self.stop_node(2)
+
+        def copy_v16():
+            node_master.get_wallet_rpc(self.default_wallet_name).unloadwallet()
             # Copy the 0.16.3 wallet to the last Bitcoin Core version and open it:
             shutil.rmtree(node_master_wallet_dir)
             os.mkdir(node_master_wallet_dir)
@@ -99,10 +131,33 @@ class UpgradeWalletTest(BitcoinTestFramework):
                 v16_3_wallet,
                 node_master_wallet_dir
             )
-        self.restart_node(0, ['-nowallet'])
+            node_master.loadwallet(self.default_wallet_name)
-        node_master.loadwallet('')
 
-        wallet = node_master.get_wallet_rpc('')
+        def copy_non_hd():
+            node_master.get_wallet_rpc(self.default_wallet_name).unloadwallet()
+            # Copy the 0.15.2 non hd wallet to the last Bitcoin Core version and open it:
+            shutil.rmtree(node_master_wallet_dir)
+            os.mkdir(node_master_wallet_dir)
+            shutil.copy(
+                v15_2_wallet,
+                node_master_wallet_dir
+            )
+            node_master.loadwallet(self.default_wallet_name)
+
+        def copy_split_hd():
+            node_master.get_wallet_rpc(self.default_wallet_name).unloadwallet()
+            # Copy the 0.15.2 split hd wallet to the last Bitcoin Core version and open it:
+            shutil.rmtree(node_master_wallet_dir)
+            os.mkdir(node_master_wallet_dir)
+            shutil.copy(
+                split_hd_wallet,
+                os.path.join(node_master_wallet_dir, 'wallet.dat')
+            )
+            node_master.loadwallet(self.default_wallet_name)
+
+        self.restart_node(0)
+        copy_v16()
+        wallet = node_master.get_wallet_rpc(self.default_wallet_name)
         old_version = wallet.getwalletinfo()["walletversion"]
 
         # calling upgradewallet without version arguments
@@ -114,18 +169,8 @@ class UpgradeWalletTest(BitcoinTestFramework):
         # wallet should still contain the same balance
         assert_equal(wallet.getbalance(), v16_3_balance)
 
-        self.stop_node(0)
+        copy_non_hd()
-        # Copy the 0.15.2 wallet to the last Bitcoin Core version and open it:
+        wallet = node_master.get_wallet_rpc(self.default_wallet_name)
-        shutil.rmtree(node_master_wallet_dir)
-        os.mkdir(node_master_wallet_dir)
-        shutil.copy(
-            v15_2_wallet,
-            node_master_wallet_dir
-        )
-        self.restart_node(0, ['-nowallet'])
-        node_master.loadwallet('')
-
-        wallet = node_master.get_wallet_rpc('')
         # should have no master key hash before conversion
         assert_equal('hdseedid' in wallet.getwalletinfo(), False)
         # calling upgradewallet with explicit version number
@@ -137,5 +182,165 @@ class UpgradeWalletTest(BitcoinTestFramework):
         # after conversion master key hash should be present
         assert_is_hex_string(wallet.getwalletinfo()['hdseedid'])
 
+        self.log.info('Intermediary versions don\'t effect anything')
+        copy_non_hd()
+        # Wallet starts with 60000
+        assert_equal(60000, wallet.getwalletinfo()['walletversion'])
+        wallet.unloadwallet()
+        before_checksum = sha256sum_file(node_master_wallet)
+        node_master.loadwallet('')
+        # Can "upgrade" to 129999 which should have no effect on the wallet
+        wallet.upgradewallet(129999)
+        assert_equal(60000, wallet.getwalletinfo()['walletversion'])
+        wallet.unloadwallet()
+        assert_equal(before_checksum, sha256sum_file(node_master_wallet))
+        node_master.loadwallet('')
+
+        self.log.info('Wallets cannot be downgraded')
+        copy_non_hd()
+        assert_raises_rpc_error(-4, 'Cannot downgrade wallet', wallet.upgradewallet, 40000)
+        wallet.unloadwallet()
+        assert_equal(before_checksum, sha256sum_file(node_master_wallet))
+        node_master.loadwallet('')
+
+        self.log.info('Can upgrade to HD')
+        # Inspect the old wallet and make sure there is no hdchain
+        orig_kvs = dump_bdb_kv(node_master_wallet)
+        assert b'\x07hdchain' not in orig_kvs
+        # Upgrade to HD, no split
+        wallet.upgradewallet(130000)
+        assert_equal(130000, wallet.getwalletinfo()['walletversion'])
+        # Check that there is now a hd chain and it is version 1, no internal chain counter
+        new_kvs = dump_bdb_kv(node_master_wallet)
+        assert b'\x07hdchain' in new_kvs
+        hd_chain = new_kvs[b'\x07hdchain']
+        assert_equal(28, len(hd_chain))
+        hd_chain_version, external_counter, seed_id = struct.unpack('<iI20s', hd_chain)
+        assert_equal(1, hd_chain_version)
+        seed_id = bytearray(seed_id)
+        seed_id.reverse()
+        old_kvs = new_kvs
+        # First 2 keys should still be non-HD
+        for i in range(0, 2):
+            info = wallet.getaddressinfo(wallet.getnewaddress())
+            assert 'hdkeypath' not in info
+            assert 'hdseedid' not in info
+        # Next key should be HD
+        info = wallet.getaddressinfo(wallet.getnewaddress())
+        assert_equal(seed_id.hex(), info['hdseedid'])
+        assert_equal('m/0\'/0\'/0\'', info['hdkeypath'])
+        prev_seed_id = info['hdseedid']
+        # Change key should be the same keypool
+        info = wallet.getaddressinfo(wallet.getrawchangeaddress())
+        assert_equal(prev_seed_id, info['hdseedid'])
+        assert_equal('m/0\'/0\'/1\'', info['hdkeypath'])
+
+        self.log.info('Cannot upgrade to HD Split, needs Pre Split Keypool')
+        assert_raises_rpc_error(-4, 'Cannot upgrade a non HD split wallet without upgrading to support pre split keypool', wallet.upgradewallet, 139900)
+        assert_equal(130000, wallet.getwalletinfo()['walletversion'])
+        assert_raises_rpc_error(-4, 'Cannot upgrade a non HD split wallet without upgrading to support pre split keypool', wallet.upgradewallet, 159900)
+        assert_equal(130000, wallet.getwalletinfo()['walletversion'])
+        assert_raises_rpc_error(-4, 'Cannot upgrade a non HD split wallet without upgrading to support pre split keypool', wallet.upgradewallet, 169899)
+        assert_equal(130000, wallet.getwalletinfo()['walletversion'])
+
+        self.log.info('Upgrade HD to HD chain split')
+        wallet.upgradewallet(169900)
+        assert_equal(169900, wallet.getwalletinfo()['walletversion'])
+        # Check that the hdchain updated correctly
+        new_kvs = dump_bdb_kv(node_master_wallet)
+        hd_chain = new_kvs[b'\x07hdchain']
+        assert_equal(32, len(hd_chain))
+        hd_chain_version, external_counter, seed_id, internal_counter = struct.unpack('<iI20sI', hd_chain)
+        assert_equal(2, hd_chain_version)
+        assert_equal(0, internal_counter)
+        seed_id = bytearray(seed_id)
+        seed_id.reverse()
+        assert_equal(seed_id.hex(), prev_seed_id)
+        # Next change address is the same keypool
+        info = wallet.getaddressinfo(wallet.getrawchangeaddress())
+        assert_equal(prev_seed_id, info['hdseedid'])
+        assert_equal('m/0\'/0\'/2\'', info['hdkeypath'])
+        # Next change address is the new keypool
+        info = wallet.getaddressinfo(wallet.getrawchangeaddress())
+        assert_equal(prev_seed_id, info['hdseedid'])
+        assert_equal('m/0\'/1\'/0\'', info['hdkeypath'])
+        # External addresses use the same keypool
+        info = wallet.getaddressinfo(wallet.getnewaddress())
+        assert_equal(prev_seed_id, info['hdseedid'])
+        assert_equal('m/0\'/0\'/3\'', info['hdkeypath'])
+
+        self.log.info('Upgrade non-HD to HD chain split')
+        copy_non_hd()
+        wallet.upgradewallet(169900)
+        assert_equal(169900, wallet.getwalletinfo()['walletversion'])
+        # Check that the hdchain updated correctly
+        new_kvs = dump_bdb_kv(node_master_wallet)
+        hd_chain = new_kvs[b'\x07hdchain']
+        assert_equal(32, len(hd_chain))
+        hd_chain_version, external_counter, seed_id, internal_counter = struct.unpack('<iI20sI', hd_chain)
+        assert_equal(2, hd_chain_version)
+        assert_equal(2, internal_counter)
+        # Drain the keypool by fetching one external key and one change key. Should still be the same keypool
+        info = wallet.getaddressinfo(wallet.getnewaddress())
+        assert 'hdseedid' not in info
+        assert 'hdkeypath' not in info
+        info = wallet.getaddressinfo(wallet.getrawchangeaddress())
+        assert 'hdseedid' not in info
+        assert 'hdkeypath' not in info
+        # The next addresses are HD and should be on different HD chains
+        info = wallet.getaddressinfo(wallet.getnewaddress())
+        ext_id = info['hdseedid']
+        assert_equal('m/0\'/0\'/0\'', info['hdkeypath'])
+        info = wallet.getaddressinfo(wallet.getrawchangeaddress())
+        assert_equal(ext_id, info['hdseedid'])
+        assert_equal('m/0\'/1\'/0\'', info['hdkeypath'])
+
+        self.log.info('KeyMetadata should upgrade when loading into master')
+        copy_v16()
+        old_kvs = dump_bdb_kv(v16_3_wallet)
+        new_kvs = dump_bdb_kv(node_master_wallet)
+        for k, old_v in old_kvs.items():
+            if k.startswith(b'\x07keymeta'):
+                new_ver, new_create_time, new_kp_str, new_seed_id, new_fpr, new_path_len, new_path, new_has_key_orig = deser_keymeta(BytesIO(new_kvs[k]))
+                old_ver, old_create_time, old_kp_str, old_seed_id, old_fpr, old_path_len, old_path, old_has_key_orig = deser_keymeta(BytesIO(old_v))
+                assert_equal(10, old_ver)
+                if old_kp_str == b"": # imported things that don't have keymeta (i.e. imported coinbase privkeys) won't be upgraded
+                    assert_equal(new_kvs[k], old_v)
+                    continue
+                assert_equal(12, new_ver)
+                assert_equal(new_create_time, old_create_time)
+                assert_equal(new_kp_str, old_kp_str)
+                assert_equal(new_seed_id, old_seed_id)
+                assert_equal(0, old_path_len)
+                assert_equal(new_path_len, len(new_path))
+                assert_equal([], old_path)
+                assert_equal(False, old_has_key_orig)
+                assert_equal(True, new_has_key_orig)
+
+                # Check that the path is right
+                built_path = []
+                for s in new_kp_str.decode().split('/')[1:]:
+                    h = 0
+                    if s[-1] == '\'':
+                        s = s[:-1]
+                        h = 0x80000000
+                    p = int(s) | h
+                    built_path.append(p)
+                assert_equal(new_path, built_path)
+
+        self.log.info('Upgrading to NO_DEFAULT_KEY should not remove the defaultkey')
+        copy_split_hd()
+        # Check the wallet has a default key initially
+        old_kvs = dump_bdb_kv(node_master_wallet)
+        defaultkey = old_kvs[b'\x0adefaultkey']
+        # Upgrade the wallet. Should still have the same default key
+        wallet.upgradewallet(159900)
+        new_kvs = dump_bdb_kv(node_master_wallet)
+        up_defaultkey = new_kvs[b'\x0adefaultkey']
+        assert_equal(defaultkey, up_defaultkey)
+        # 0.16.3 doesn't have a default key
+        v16_3_kvs = dump_bdb_kv(v16_3_wallet)
+        assert b'\x0adefaultkey' not in v16_3_kvs
+
 if __name__ == '__main__':
     UpgradeWalletTest().main()