From faf3cd659a72473a1aa73c4367a145f4ec64f146 Mon Sep 17 00:00:00 2001
From: MarcoFalke <*~=`'#}+{/-|&$^_@721217.xyz>
Date: Wed, 7 Feb 2024 11:30:21 +0100
Subject: [PATCH 1/4] test: Normalize struct.pack format

* Add () around some int values
* Remove b-prefix from strings

This is needed for the scripted diff to work.
---
 test/functional/feature_addrman.py       | 10 +++++-----
 test/functional/test_framework/script.py |  6 +++---
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/test/functional/feature_addrman.py b/test/functional/feature_addrman.py
index 95d33d62ea..c17b697ec1 100755
--- a/test/functional/feature_addrman.py
+++ b/test/functional/feature_addrman.py
@@ -29,14 +29,14 @@ def serialize_addrman(
     INCOMPATIBILITY_BASE = 32
     r = MAGIC_BYTES[net_magic]
     r += struct.pack("B", format)
-    r += struct.pack("B", INCOMPATIBILITY_BASE + lowest_compatible)
+    r += struct.pack("B", (INCOMPATIBILITY_BASE + lowest_compatible))
     r += ser_uint256(bucket_key)
-    r += struct.pack("<i", len_new or len(new))
-    r += struct.pack("<i", len_tried or len(tried))
+    r += struct.pack("<i", (len_new or len(new)))
+    r += struct.pack("<i", (len_tried or len(tried)))
     ADDRMAN_NEW_BUCKET_COUNT = 1 << 10
-    r += struct.pack("<i", ADDRMAN_NEW_BUCKET_COUNT ^ (1 << 30))
+    r += struct.pack("<i", (ADDRMAN_NEW_BUCKET_COUNT ^ (1 << 30)))
     for _ in range(ADDRMAN_NEW_BUCKET_COUNT):
-        r += struct.pack("<i", 0)
+        r += struct.pack("<i", (0))
     checksum = hash256(r)
     r += mock_checksum or checksum
     return r
diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py
index 7b19d31e17..ee3c3a0efd 100644
--- a/test/functional/test_framework/script.py
+++ b/test/functional/test_framework/script.py
@@ -58,9 +58,9 @@ class CScriptOp(int):
         elif len(d) <= 0xff:
             return b'\x4c' + bytes([len(d)]) + d  # OP_PUSHDATA1
         elif len(d) <= 0xffff:
-            return b'\x4d' + struct.pack(b'<H', len(d)) + d  # OP_PUSHDATA2
+            return b'\x4d' + struct.pack('<H', len(d)) + d  # OP_PUSHDATA2
         elif len(d) <= 0xffffffff:
-            return b'\x4e' + struct.pack(b'<I', len(d)) + d  # OP_PUSHDATA4
+            return b'\x4e' + struct.pack('<I', len(d)) + d  # OP_PUSHDATA4
         else:
             raise ValueError("Data too long to encode in a PUSHDATA op")
 
@@ -670,7 +670,7 @@ def LegacySignatureMsg(script, txTo, inIdx, hashtype):
         txtmp.vin.append(tmp)
 
     s = txtmp.serialize_without_witness()
-    s += struct.pack(b"<I", hashtype)
+    s += struct.pack("<I", hashtype)
 
     return (s, None)
 

From faf2a975ad46799d075e3a70c699da0d8182aab9 Mon Sep 17 00:00:00 2001
From: MarcoFalke <*~=`'#}+{/-|&$^_@721217.xyz>
Date: Wed, 7 Feb 2024 11:20:41 +0100
Subject: [PATCH 2/4] test: Use int.to_bytes over struct packing

This is done in prepration for the scripted diff, which can not deal
with those lines.
---
 test/functional/p2p_invalid_messages.py  | 3 +--
 test/functional/test_framework/script.py | 4 ++--
 test/functional/wallet_balance.py        | 5 ++---
 3 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py
index 40a69936bc..adcbb4fd05 100755
--- a/test/functional/p2p_invalid_messages.py
+++ b/test/functional/p2p_invalid_messages.py
@@ -5,7 +5,6 @@
 """Test node responses to invalid network messages."""
 
 import random
-import struct
 import time
 
 from test_framework.messages import (
@@ -233,7 +232,7 @@ class InvalidMessagesTest(BitcoinTestFramework):
                 '208d'))     # port
 
     def test_addrv2_unrecognized_network(self):
-        now_hex = struct.pack('<I', int(time.time())).hex()
+        now_hex = int(time.time()).to_bytes(4, "little").hex()
         self.test_addrv2('unrecognized network',
             [
                 'received: addrv2 (25 bytes)',
diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py
index ee3c3a0efd..5d6f65b83d 100644
--- a/test/functional/test_framework/script.py
+++ b/test/functional/test_framework/script.py
@@ -800,13 +800,13 @@ def BIP341_sha_prevouts(txTo):
     return sha256(b"".join(i.prevout.serialize() for i in txTo.vin))
 
 def BIP341_sha_amounts(spent_utxos):
-    return sha256(b"".join(struct.pack("<q", u.nValue) for u in spent_utxos))
+    return sha256(b"".join(u.nValue.to_bytes(8, "little", signed=True) for u in spent_utxos))
 
 def BIP341_sha_scriptpubkeys(spent_utxos):
     return sha256(b"".join(ser_string(u.scriptPubKey) for u in spent_utxos))
 
 def BIP341_sha_sequences(txTo):
-    return sha256(b"".join(struct.pack("<I", i.nSequence) for i in txTo.vin))
+    return sha256(b"".join(i.nSequence.to_bytes(4, "little") for i in txTo.vin))
 
 def BIP341_sha_outputs(txTo):
     return sha256(b"".join(o.serialize() for o in txTo.vout))
diff --git a/test/functional/wallet_balance.py b/test/functional/wallet_balance.py
index c322ae52c1..2c85773bf3 100755
--- a/test/functional/wallet_balance.py
+++ b/test/functional/wallet_balance.py
@@ -4,7 +4,6 @@
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Test the wallet balance RPC methods."""
 from decimal import Decimal
-import struct
 
 from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE as ADDRESS_WATCHONLY
 from test_framework.blocktools import COINBASE_MATURITY
@@ -266,8 +265,8 @@ class WalletTest(BitcoinTestFramework):
         tx_orig = self.nodes[0].gettransaction(txid)['hex']
         # Increase fee by 1 coin
         tx_replace = tx_orig.replace(
-            struct.pack("<q", 99 * 10**8).hex(),
-            struct.pack("<q", 98 * 10**8).hex(),
+            (99 * 10**8).to_bytes(8, "little", signed=True).hex(),
+            (98 * 10**8).to_bytes(8, "little", signed=True).hex(),
         )
         tx_replace = self.nodes[0].signrawtransactionwithwallet(tx_replace)['hex']
         # Total balance is given by the sum of outputs of the tx

From fa826db477a981b48bff53021f9695a5f6682dc0 Mon Sep 17 00:00:00 2001
From: MarcoFalke <*~=`'#}+{/-|&$^_@721217.xyz>
Date: Wed, 7 Feb 2024 11:26:05 +0100
Subject: [PATCH 3/4] scripted-diff: test: Use int.to_bytes over struct packing

-BEGIN VERIFY SCRIPT-
 sed -i --regexp-extended 's!struct.pack\(.<?B., (.*)\)!\1.to_bytes(1, "little")!g'             $( git grep -l struct.pack )
 sed -i --regexp-extended 's!struct.pack\(.<I., (.*)\)!\1.to_bytes(4, "little")!g'              $( git grep -l struct.pack )
 sed -i --regexp-extended 's!struct.pack\(.<H., (.*)\)!\1.to_bytes(2, "little")!g'              $( git grep -l struct.pack )
 sed -i --regexp-extended 's!struct.pack\(.<i., (.*)\)!\1.to_bytes(4, "little", signed=True)!g' $( git grep -l struct.pack )
 sed -i --regexp-extended 's!struct.pack\(.<q., (.*)\)!\1.to_bytes(8, "little", signed=True)!g' $( git grep -l struct.pack )
 sed -i --regexp-extended 's!struct.pack\(.>H., (.*)\)!\1.to_bytes(2, "big")!g'                 $( git grep -l struct.pack )
-END VERIFY SCRIPT-
---
 contrib/seeds/generate-seeds.py          |  6 ++---
 contrib/signet/miner                     |  4 ++--
 test/functional/feature_addrman.py       | 12 +++++-----
 test/functional/p2p_segwit.py            | 12 +++++-----
 test/functional/test_framework/p2p.py    |  2 +-
 test/functional/test_framework/script.py | 28 ++++++++++++------------
 6 files changed, 32 insertions(+), 32 deletions(-)

diff --git a/contrib/seeds/generate-seeds.py b/contrib/seeds/generate-seeds.py
index e921757802..c84be1055d 100755
--- a/contrib/seeds/generate-seeds.py
+++ b/contrib/seeds/generate-seeds.py
@@ -115,7 +115,7 @@ def parse_spec(s):
 def ser_compact_size(l):
     r = b""
     if l < 253:
-        r = struct.pack("B", l)
+        r = l.to_bytes(1, "little")
     elif l < 0x10000:
         r = struct.pack("<BH", 253, l)
     elif l < 0x100000000:
@@ -129,10 +129,10 @@ def bip155_serialize(spec):
     Serialize (networkID, addr, port) tuple to BIP155 binary format.
     '''
     r = b""
-    r += struct.pack('B', spec[0].value)
+    r += spec[0].value.to_bytes(1, "little")
     r += ser_compact_size(len(spec[1]))
     r += spec[1]
-    r += struct.pack('>H', spec[2])
+    r += spec[2].to_bytes(2, "big")
     return r
 
 def process_nodes(g, f, structname):
diff --git a/contrib/signet/miner b/contrib/signet/miner
index e5daf9f993..7b7b3feb39 100755
--- a/contrib/signet/miner
+++ b/contrib/signet/miner
@@ -52,10 +52,10 @@ def signet_txs(block, challenge):
     mroot = block.get_merkle_root(hashes)
 
     sd = b""
-    sd += struct.pack("<i", block.nVersion)
+    sd += block.nVersion.to_bytes(4, "little", signed=True)
     sd += ser_uint256(block.hashPrevBlock)
     sd += ser_uint256(mroot)
-    sd += struct.pack("<I", block.nTime)
+    sd += block.nTime.to_bytes(4, "little")
 
     to_spend = CTransaction()
     to_spend.nVersion = 0
diff --git a/test/functional/feature_addrman.py b/test/functional/feature_addrman.py
index c17b697ec1..f422673f6b 100755
--- a/test/functional/feature_addrman.py
+++ b/test/functional/feature_addrman.py
@@ -28,15 +28,15 @@ def serialize_addrman(
     tried = []
     INCOMPATIBILITY_BASE = 32
     r = MAGIC_BYTES[net_magic]
-    r += struct.pack("B", format)
-    r += struct.pack("B", (INCOMPATIBILITY_BASE + lowest_compatible))
+    r += format.to_bytes(1, "little")
+    r += (INCOMPATIBILITY_BASE + lowest_compatible).to_bytes(1, "little")
     r += ser_uint256(bucket_key)
-    r += struct.pack("<i", (len_new or len(new)))
-    r += struct.pack("<i", (len_tried or len(tried)))
+    r += (len_new or len(new)).to_bytes(4, "little", signed=True)
+    r += (len_tried or len(tried)).to_bytes(4, "little", signed=True)
     ADDRMAN_NEW_BUCKET_COUNT = 1 << 10
-    r += struct.pack("<i", (ADDRMAN_NEW_BUCKET_COUNT ^ (1 << 30)))
+    r += (ADDRMAN_NEW_BUCKET_COUNT ^ (1 << 30)).to_bytes(4, "little", signed=True)
     for _ in range(ADDRMAN_NEW_BUCKET_COUNT):
-        r += struct.pack("<i", (0))
+        r += (0).to_bytes(4, "little", signed=True)
     checksum = hash256(r)
     r += mock_checksum or checksum
     return r
diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py
index 45bbd7f1c3..03a19be722 100755
--- a/test/functional/p2p_segwit.py
+++ b/test/functional/p2p_segwit.py
@@ -1165,16 +1165,16 @@ class SegWitTest(BitcoinTestFramework):
                 if not self.wit.is_null():
                     flags |= 1
                 r = b""
-                r += struct.pack("<i", self.nVersion)
+                r += self.nVersion.to_bytes(4, "little", signed=True)
                 if flags:
                     dummy = []
                     r += ser_vector(dummy)
-                    r += struct.pack("<B", flags)
+                    r += flags.to_bytes(1, "little")
                 r += ser_vector(self.vin)
                 r += ser_vector(self.vout)
                 if flags & 1:
                     r += self.wit.serialize()
-                r += struct.pack("<I", self.nLockTime)
+                r += self.nLockTime.to_bytes(4, "little")
                 return r
 
         tx2 = BrokenCTransaction()
@@ -1976,11 +1976,11 @@ class SegWitTest(BitcoinTestFramework):
         def serialize_with_bogus_witness(tx):
             flags = 3
             r = b""
-            r += struct.pack("<i", tx.nVersion)
+            r += tx.nVersion.to_bytes(4, "little", signed=True)
             if flags:
                 dummy = []
                 r += ser_vector(dummy)
-                r += struct.pack("<B", flags)
+                r += flags.to_bytes(1, "little")
             r += ser_vector(tx.vin)
             r += ser_vector(tx.vout)
             if flags & 1:
@@ -1990,7 +1990,7 @@ class SegWitTest(BitcoinTestFramework):
                     for _ in range(len(tx.wit.vtxinwit), len(tx.vin)):
                         tx.wit.vtxinwit.append(CTxInWitness())
                 r += tx.wit.serialize()
-            r += struct.pack("<I", tx.nLockTime)
+            r += tx.nLockTime.to_bytes(4, "little")
             return r
 
         class msg_bogus_tx(msg_tx):
diff --git a/test/functional/test_framework/p2p.py b/test/functional/test_framework/p2p.py
index 00bd1e4017..4b846df94a 100755
--- a/test/functional/test_framework/p2p.py
+++ b/test/functional/test_framework/p2p.py
@@ -411,7 +411,7 @@ class P2PConnection(asyncio.Protocol):
             tmsg = self.magic_bytes
             tmsg += msgtype
             tmsg += b"\x00" * (12 - len(msgtype))
-            tmsg += struct.pack("<I", len(data))
+            tmsg += len(data).to_bytes(4, "little")
             th = sha256(data)
             h = sha256(th)
             tmsg += h[:4]
diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py
index 5d6f65b83d..4bf598ca15 100644
--- a/test/functional/test_framework/script.py
+++ b/test/functional/test_framework/script.py
@@ -58,9 +58,9 @@ class CScriptOp(int):
         elif len(d) <= 0xff:
             return b'\x4c' + bytes([len(d)]) + d  # OP_PUSHDATA1
         elif len(d) <= 0xffff:
-            return b'\x4d' + struct.pack('<H', len(d)) + d  # OP_PUSHDATA2
+            return b'\x4d' + len(d).to_bytes(2, "little") + d  # OP_PUSHDATA2
         elif len(d) <= 0xffffffff:
-            return b'\x4e' + struct.pack('<I', len(d)) + d  # OP_PUSHDATA4
+            return b'\x4e' + len(d).to_bytes(4, "little") + d  # OP_PUSHDATA4
         else:
             raise ValueError("Data too long to encode in a PUSHDATA op")
 
@@ -670,7 +670,7 @@ def LegacySignatureMsg(script, txTo, inIdx, hashtype):
         txtmp.vin.append(tmp)
 
     s = txtmp.serialize_without_witness()
-    s += struct.pack("<I", hashtype)
+    s += hashtype.to_bytes(4, "little")
 
     return (s, None)
 
@@ -726,7 +726,7 @@ def SegwitV0SignatureMsg(script, txTo, inIdx, hashtype, amount):
     if (not (hashtype & SIGHASH_ANYONECANPAY) and (hashtype & 0x1f) != SIGHASH_SINGLE and (hashtype & 0x1f) != SIGHASH_NONE):
         serialize_sequence = bytes()
         for i in txTo.vin:
-            serialize_sequence += struct.pack("<I", i.nSequence)
+            serialize_sequence += i.nSequence.to_bytes(4, "little")
         hashSequence = uint256_from_str(hash256(serialize_sequence))
 
     if ((hashtype & 0x1f) != SIGHASH_SINGLE and (hashtype & 0x1f) != SIGHASH_NONE):
@@ -739,16 +739,16 @@ def SegwitV0SignatureMsg(script, txTo, inIdx, hashtype, amount):
         hashOutputs = uint256_from_str(hash256(serialize_outputs))
 
     ss = bytes()
-    ss += struct.pack("<i", txTo.nVersion)
+    ss += txTo.nVersion.to_bytes(4, "little", signed=True)
     ss += ser_uint256(hashPrevouts)
     ss += ser_uint256(hashSequence)
     ss += txTo.vin[inIdx].prevout.serialize()
     ss += ser_string(script)
-    ss += struct.pack("<q", amount)
-    ss += struct.pack("<I", txTo.vin[inIdx].nSequence)
+    ss += amount.to_bytes(8, "little", signed=True)
+    ss += txTo.vin[inIdx].nSequence.to_bytes(4, "little")
     ss += ser_uint256(hashOutputs)
     ss += txTo.nLockTime.to_bytes(4, "little")
-    ss += struct.pack("<I", hashtype)
+    ss += hashtype.to_bytes(4, "little")
     return ss
 
 def SegwitV0SignatureHash(*args, **kwargs):
@@ -818,8 +818,8 @@ def TaprootSignatureMsg(txTo, spent_utxos, hash_type, input_index = 0, scriptpat
     in_type = hash_type & SIGHASH_ANYONECANPAY
     spk = spent_utxos[input_index].scriptPubKey
     ss = bytes([0, hash_type]) # epoch, hash_type
-    ss += struct.pack("<i", txTo.nVersion)
-    ss += struct.pack("<I", txTo.nLockTime)
+    ss += txTo.nVersion.to_bytes(4, "little", signed=True)
+    ss += txTo.nLockTime.to_bytes(4, "little")
     if in_type != SIGHASH_ANYONECANPAY:
         ss += BIP341_sha_prevouts(txTo)
         ss += BIP341_sha_amounts(spent_utxos)
@@ -835,11 +835,11 @@ def TaprootSignatureMsg(txTo, spent_utxos, hash_type, input_index = 0, scriptpat
     ss += bytes([spend_type])
     if in_type == SIGHASH_ANYONECANPAY:
         ss += txTo.vin[input_index].prevout.serialize()
-        ss += struct.pack("<q", spent_utxos[input_index].nValue)
+        ss += spent_utxos[input_index].nValue.to_bytes(8, "little", signed=True)
         ss += ser_string(spk)
-        ss += struct.pack("<I", txTo.vin[input_index].nSequence)
+        ss += txTo.vin[input_index].nSequence.to_bytes(4, "little")
     else:
-        ss += struct.pack("<I", input_index)
+        ss += input_index.to_bytes(4, "little")
     if (spend_type & 1):
         ss += sha256(ser_string(annex))
     if out_type == SIGHASH_SINGLE:
@@ -850,7 +850,7 @@ def TaprootSignatureMsg(txTo, spent_utxos, hash_type, input_index = 0, scriptpat
     if (scriptpath):
         ss += TaggedHash("TapLeaf", bytes([leaf_ver]) + ser_string(script))
         ss += bytes([0])
-        ss += struct.pack("<i", codeseparator_pos)
+        ss += codeseparator_pos.to_bytes(4, "little", signed=True)
     assert len(ss) ==  175 - (in_type == SIGHASH_ANYONECANPAY) * 49 - (out_type != SIGHASH_ALL and out_type != SIGHASH_SINGLE) * 32 + (annex is not None) * 32 + scriptpath * 37
     return ss
 

From fa52e13ee81fcc7543890dbd6986fcb55168583f Mon Sep 17 00:00:00 2001
From: MarcoFalke <*~=`'#}+{/-|&$^_@721217.xyz>
Date: Wed, 7 Feb 2024 13:19:37 +0100
Subject: [PATCH 4/4] test: Remove struct.pack from almost all places

---
 contrib/seeds/generate-seeds.py          | 7 +++----
 test/functional/feature_addrman.py       | 1 -
 test/functional/feature_block.py         | 3 +--
 test/functional/p2p_segwit.py            | 1 -
 test/functional/test_framework/script.py | 1 -
 5 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/contrib/seeds/generate-seeds.py b/contrib/seeds/generate-seeds.py
index c84be1055d..f67e7b0f4c 100755
--- a/contrib/seeds/generate-seeds.py
+++ b/contrib/seeds/generate-seeds.py
@@ -29,7 +29,6 @@ These should be pasted into `src/chainparamsseeds.h`.
 
 from base64 import b32decode
 from enum import Enum
-import struct
 import sys
 import os
 import re
@@ -117,11 +116,11 @@ def ser_compact_size(l):
     if l < 253:
         r = l.to_bytes(1, "little")
     elif l < 0x10000:
-        r = struct.pack("<BH", 253, l)
+        r = (253).to_bytes(1, "little") + l.to_bytes(2, "little")
     elif l < 0x100000000:
-        r = struct.pack("<BI", 254, l)
+        r = (254).to_bytes(1, "little") + l.to_bytes(4, "little")
     else:
-        r = struct.pack("<BQ", 255, l)
+        r = (255).to_bytes(1, "little") + l.to_bytes(8, "little")
     return r
 
 def bip155_serialize(spec):
diff --git a/test/functional/feature_addrman.py b/test/functional/feature_addrman.py
index f422673f6b..2efad70900 100755
--- a/test/functional/feature_addrman.py
+++ b/test/functional/feature_addrman.py
@@ -6,7 +6,6 @@
 
 import os
 import re
-import struct
 
 from test_framework.messages import ser_uint256, hash256, MAGIC_BYTES
 from test_framework.netutil import ADDRMAN_NEW_BUCKET_COUNT, ADDRMAN_TRIED_BUCKET_COUNT, ADDRMAN_BUCKET_SIZE
diff --git a/test/functional/feature_block.py b/test/functional/feature_block.py
index 8a95975184..932f37a083 100755
--- a/test/functional/feature_block.py
+++ b/test/functional/feature_block.py
@@ -4,7 +4,6 @@
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Test block processing."""
 import copy
-import struct
 import time
 
 from test_framework.blocktools import (
@@ -67,7 +66,7 @@ class CBrokenBlock(CBlock):
     def serialize(self, with_witness=False):
         r = b""
         r += super(CBlock, self).serialize()
-        r += struct.pack("<BQ", 255, len(self.vtx))
+        r += (255).to_bytes(1, "little") + len(self.vtx).to_bytes(8, "little")
         for tx in self.vtx:
             if with_witness:
                 r += tx.serialize_with_witness()
diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py
index 03a19be722..b63c2c7a8d 100755
--- a/test/functional/p2p_segwit.py
+++ b/test/functional/p2p_segwit.py
@@ -5,7 +5,6 @@
 """Test segwit transactions and blocks on P2P network."""
 from decimal import Decimal
 import random
-import struct
 import time
 
 from test_framework.blocktools import (
diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py
index 4bf598ca15..ab3dc2ffb1 100644
--- a/test/functional/test_framework/script.py
+++ b/test/functional/test_framework/script.py
@@ -8,7 +8,6 @@ This file is modified from python-bitcoinlib.
 """
 
 from collections import namedtuple
-import struct
 import unittest
 
 from .key import TaggedHash, tweak_add_pubkey, compute_xonly_pubkey