foster/bitcoin-bitcoin-core
0
0
Fork 0
mirror of https://github.com/bitcoin/bitcoin.git synced 2025-02-08 10:31:50 -05:00
Code Issues Activity

Merge bitcoin/bitcoin#29401: test: Remove struct.pack from almost all places

Browse source 
fa52e13ee8 test: Remove struct.pack from almost all places (MarcoFalke)
fa826db477 scripted-diff: test: Use int.to_bytes over struct packing (MarcoFalke)
faf2a975ad test: Use int.to_bytes over struct packing (MarcoFalke)
faf3cd659a test: Normalize struct.pack format (MarcoFalke)

Pull request description:

  `struct.pack` has many issues:

  * The format string consists of characters that may be confusing and may need to be looked up in the documentation, as opposed to using easy to understand self-documenting code.

  This lead to many test bugs, which weren't hit, which is fine, but still confusing. Ref: https://github.com/bitcoin/bitcoin/pull/29400, https://github.com/bitcoin/bitcoin/pull/29399, https://github.com/bitcoin/bitcoin/pull/29363, fa3886b7c6, ...

  Fix all issues by using the built-in `int` helpers `to_bytes` via a scripted diff.

  Review notes:

  * For `struct.pack` and `int.to_bytes` the error behavior is the same, although the error messages are not identical.
  * Two `struct.pack` remain. One for float serialization in a C++ code comment, and one for native serialization.

ACKs for top commit:
  achow101:
    ACK fa52e13ee8
  rkrux:
    tACK [fa52e13](fa52e13ee8)
  theStack:
    Code-review ACK fa52e13ee8

Tree-SHA512: ee80d935b68ae43d1654b047e84ceb80abbd20306df35cca848b3f7874634b518560ddcbc7e714e2e7a19241e153dee64556dc4701287ae811e26e4f8c57fe3e
This commit is contained in:
Ava Chow 2024-06-06 19:18:55 -04:00
commit 4a020ca443
No known key found for this signature in database
GPG key ID: 17565732E08E5E41
9 changed files with 41 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
contrib/seeds/generate-seeds.py
View file

@ -29,7 +29,6 @@ These should be pasted into `src/chainparamsseeds.h`.
from base64 import b32decode
from enum import Enum
import struct
import sys
import os
import re
@ -115,13 +114,13 @@ def parse_spec(s):
def ser_compact_size(l):
r = b""
if l < 253:
r = struct.pack("B", l)
r = l.to_bytes(1, "little")
elif l < 0x10000:
r = struct.pack("<BH", 253, l)
r = (253).to_bytes(1, "little") + l.to_bytes(2, "little")
elif l < 0x100000000:
r = struct.pack("<BI", 254, l)
r = (254).to_bytes(1, "little") + l.to_bytes(4, "little")
else:
r = struct.pack("<BQ", 255, l)
r = (255).to_bytes(1, "little") + l.to_bytes(8, "little")
return r
def bip155_serialize(spec):
@ -129,10 +128,10 @@ def bip155_serialize(spec):
Serialize (networkID, addr, port) tuple to BIP155 binary format.
'''
r = b""
r += struct.pack('B', spec[0].value)
r += spec[0].value.to_bytes(1, "little")
r += ser_compact_size(len(spec[1]))
r += spec[1]
r += struct.pack('>H', spec[2])
r += spec[2].to_bytes(2, "big")
return r
def process_nodes(g, f, structname):

4
contrib/signet/miner
View file

@ -52,10 +52,10 @@ def signet_txs(block, challenge):
mroot = block.get_merkle_root(hashes)
sd = b""
sd += struct.pack("<i", block.nVersion)
sd += block.nVersion.to_bytes(4, "little", signed=True)
sd += ser_uint256(block.hashPrevBlock)
sd += ser_uint256(mroot)
sd += struct.pack("<I", block.nTime)
sd += block.nTime.to_bytes(4, "little")
to_spend = CTransaction()
to_spend.nVersion = 0

13
test/functional/feature_addrman.py
View file

@ -6,7 +6,6 @@
import os
import re
import struct
from test_framework.messages import ser_uint256, hash256, MAGIC_BYTES
from test_framework.netutil import ADDRMAN_NEW_BUCKET_COUNT, ADDRMAN_TRIED_BUCKET_COUNT, ADDRMAN_BUCKET_SIZE
@ -28,15 +27,15 @@ def serialize_addrman(
tried = []
INCOMPATIBILITY_BASE = 32
r = MAGIC_BYTES[net_magic]
r += struct.pack("B", format)
r += struct.pack("B", INCOMPATIBILITY_BASE + lowest_compatible)
r += format.to_bytes(1, "little")
r += (INCOMPATIBILITY_BASE + lowest_compatible).to_bytes(1, "little")
r += ser_uint256(bucket_key)
r += struct.pack("<i", len_new or len(new))
r += struct.pack("<i", len_tried or len(tried))
r += (len_new or len(new)).to_bytes(4, "little", signed=True)
r += (len_tried or len(tried)).to_bytes(4, "little", signed=True)
ADDRMAN_NEW_BUCKET_COUNT = 1 << 10
r += struct.pack("<i", ADDRMAN_NEW_BUCKET_COUNT ^ (1 << 30))
r += (ADDRMAN_NEW_BUCKET_COUNT ^ (1 << 30)).to_bytes(4, "little", signed=True)
for _ in range(ADDRMAN_NEW_BUCKET_COUNT):
r += struct.pack("<i", 0)
r += (0).to_bytes(4, "little", signed=True)
checksum = hash256(r)
r += mock_checksum or checksum
return r

3
test/functional/feature_block.py
View file

@ -4,7 +4,6 @@
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test block processing."""
import copy
import struct
import time
from test_framework.blocktools import (
@ -67,7 +66,7 @@ class CBrokenBlock(CBlock):
def serialize(self, with_witness=False):
r = b""
r += super(CBlock, self).serialize()
r += struct.pack("<BQ", 255, len(self.vtx))
r += (255).to_bytes(1, "little") + len(self.vtx).to_bytes(8, "little")
for tx in self.vtx:
if with_witness:
r += tx.serialize_with_witness()

3
test/functional/p2p_invalid_messages.py
View file

@ -5,7 +5,6 @@
"""Test node responses to invalid network messages."""
import random
import struct
import time
from test_framework.messages import (
@ -233,7 +232,7 @@ class InvalidMessagesTest(BitcoinTestFramework):
'208d')) # port
def test_addrv2_unrecognized_network(self):
now_hex = struct.pack('<I', int(time.time())).hex()
now_hex = int(time.time()).to_bytes(4, "little").hex()
self.test_addrv2('unrecognized network',
[
'received: addrv2 (25 bytes)',

13
test/functional/p2p_segwit.py
View file

@ -5,7 +5,6 @@
"""Test segwit transactions and blocks on P2P network."""
from decimal import Decimal
import random
import struct
import time
from test_framework.blocktools import (
@ -1165,16 +1164,16 @@ class SegWitTest(BitcoinTestFramework):
if not self.wit.is_null():
flags |= 1
r = b""
r += struct.pack("<i", self.nVersion)
r += self.nVersion.to_bytes(4, "little", signed=True)
if flags:
dummy = []
r += ser_vector(dummy)
r += struct.pack("<B", flags)
r += flags.to_bytes(1, "little")
r += ser_vector(self.vin)
r += ser_vector(self.vout)
if flags & 1:
r += self.wit.serialize()
r += struct.pack("<I", self.nLockTime)
r += self.nLockTime.to_bytes(4, "little")
return r
tx2 = BrokenCTransaction()
@ -1976,11 +1975,11 @@ class SegWitTest(BitcoinTestFramework):
def serialize_with_bogus_witness(tx):
flags = 3
r = b""
r += struct.pack("<i", tx.nVersion)
r += tx.nVersion.to_bytes(4, "little", signed=True)
if flags:
dummy = []
r += ser_vector(dummy)
r += struct.pack("<B", flags)
r += flags.to_bytes(1, "little")
r += ser_vector(tx.vin)
r += ser_vector(tx.vout)
if flags & 1:
@ -1990,7 +1989,7 @@ class SegWitTest(BitcoinTestFramework):
for _ in range(len(tx.wit.vtxinwit), len(tx.vin)):
tx.wit.vtxinwit.append(CTxInWitness())
r += tx.wit.serialize()
r += struct.pack("<I", tx.nLockTime)
r += tx.nLockTime.to_bytes(4, "little")
return r
class msg_bogus_tx(msg_tx):

2
test/functional/test_framework/p2p.py
View file

@ -411,7 +411,7 @@ class P2PConnection(asyncio.Protocol):
tmsg = self.magic_bytes
tmsg += msgtype
tmsg += b"\x00" * (12 - len(msgtype))
tmsg += struct.pack("<I", len(data))
tmsg += len(data).to_bytes(4, "little")
th = sha256(data)
h = sha256(th)
tmsg += h[:4]

33
test/functional/test_framework/script.py
View file

@ -8,7 +8,6 @@ This file is modified from python-bitcoinlib.
"""
from collections import namedtuple
import struct
import unittest
from .key import TaggedHash, tweak_add_pubkey, compute_xonly_pubkey
@ -58,9 +57,9 @@ class CScriptOp(int):
elif len(d) <= 0xff:
return b'\x4c' + bytes([len(d)]) + d # OP_PUSHDATA1
elif len(d) <= 0xffff:
return b'\x4d' + struct.pack(b'<H', len(d)) + d # OP_PUSHDATA2
return b'\x4d' + len(d).to_bytes(2, "little") + d # OP_PUSHDATA2
elif len(d) <= 0xffffffff:
return b'\x4e' + struct.pack(b'<I', len(d)) + d # OP_PUSHDATA4
return b'\x4e' + len(d).to_bytes(4, "little") + d # OP_PUSHDATA4
else:
raise ValueError("Data too long to encode in a PUSHDATA op")
@ -670,7 +669,7 @@ def LegacySignatureMsg(script, txTo, inIdx, hashtype):
txtmp.vin.append(tmp)
s = txtmp.serialize_without_witness()
s += struct.pack(b"<I", hashtype)
s += hashtype.to_bytes(4, "little")
return (s, None)
@ -726,7 +725,7 @@ def SegwitV0SignatureMsg(script, txTo, inIdx, hashtype, amount):
if (not (hashtype & SIGHASH_ANYONECANPAY) and (hashtype & 0x1f) != SIGHASH_SINGLE and (hashtype & 0x1f) != SIGHASH_NONE):
serialize_sequence = bytes()
for i in txTo.vin:
serialize_sequence += struct.pack("<I", i.nSequence)
serialize_sequence += i.nSequence.to_bytes(4, "little")
hashSequence = uint256_from_str(hash256(serialize_sequence))
if ((hashtype & 0x1f) != SIGHASH_SINGLE and (hashtype & 0x1f) != SIGHASH_NONE):
@ -739,16 +738,16 @@ def SegwitV0SignatureMsg(script, txTo, inIdx, hashtype, amount):
hashOutputs = uint256_from_str(hash256(serialize_outputs))
ss = bytes()
ss += struct.pack("<i", txTo.nVersion)
ss += txTo.nVersion.to_bytes(4, "little", signed=True)
ss += ser_uint256(hashPrevouts)
ss += ser_uint256(hashSequence)
ss += txTo.vin[inIdx].prevout.serialize()
ss += ser_string(script)
ss += struct.pack("<q", amount)
ss += struct.pack("<I", txTo.vin[inIdx].nSequence)
ss += amount.to_bytes(8, "little", signed=True)
ss += txTo.vin[inIdx].nSequence.to_bytes(4, "little")
ss += ser_uint256(hashOutputs)
ss += txTo.nLockTime.to_bytes(4, "little")
ss += struct.pack("<I", hashtype)
ss += hashtype.to_bytes(4, "little")
return ss
def SegwitV0SignatureHash(*args, **kwargs):
@ -800,13 +799,13 @@ def BIP341_sha_prevouts(txTo):
return sha256(b"".join(i.prevout.serialize() for i in txTo.vin))
def BIP341_sha_amounts(spent_utxos):
return sha256(b"".join(struct.pack("<q", u.nValue) for u in spent_utxos))
return sha256(b"".join(u.nValue.to_bytes(8, "little", signed=True) for u in spent_utxos))
def BIP341_sha_scriptpubkeys(spent_utxos):
return sha256(b"".join(ser_string(u.scriptPubKey) for u in spent_utxos))
def BIP341_sha_sequences(txTo):
return sha256(b"".join(struct.pack("<I", i.nSequence) for i in txTo.vin))
return sha256(b"".join(i.nSequence.to_bytes(4, "little") for i in txTo.vin))
def BIP341_sha_outputs(txTo):
return sha256(b"".join(o.serialize() for o in txTo.vout))
@ -818,8 +817,8 @@ def TaprootSignatureMsg(txTo, spent_utxos, hash_type, input_index = 0, scriptpat
in_type = hash_type & SIGHASH_ANYONECANPAY
spk = spent_utxos[input_index].scriptPubKey
ss = bytes([0, hash_type]) # epoch, hash_type
ss += struct.pack("<i", txTo.nVersion)
ss += struct.pack("<I", txTo.nLockTime)
ss += txTo.nVersion.to_bytes(4, "little", signed=True)
ss += txTo.nLockTime.to_bytes(4, "little")
if in_type != SIGHASH_ANYONECANPAY:
ss += BIP341_sha_prevouts(txTo)
ss += BIP341_sha_amounts(spent_utxos)
@ -835,11 +834,11 @@ def TaprootSignatureMsg(txTo, spent_utxos, hash_type, input_index = 0, scriptpat
ss += bytes([spend_type])
if in_type == SIGHASH_ANYONECANPAY:
ss += txTo.vin[input_index].prevout.serialize()
ss += struct.pack("<q", spent_utxos[input_index].nValue)
ss += spent_utxos[input_index].nValue.to_bytes(8, "little", signed=True)
ss += ser_string(spk)
ss += struct.pack("<I", txTo.vin[input_index].nSequence)
ss += txTo.vin[input_index].nSequence.to_bytes(4, "little")
else:
ss += struct.pack("<I", input_index)
ss += input_index.to_bytes(4, "little")
if (spend_type & 1):
ss += sha256(ser_string(annex))
if out_type == SIGHASH_SINGLE:
@ -850,7 +849,7 @@ def TaprootSignatureMsg(txTo, spent_utxos, hash_type, input_index = 0, scriptpat
if (scriptpath):
ss += TaggedHash("TapLeaf", bytes([leaf_ver]) + ser_string(script))
ss += bytes([0])
ss += struct.pack("<i", codeseparator_pos)
ss += codeseparator_pos.to_bytes(4, "little", signed=True)
assert len(ss) == 175 - (in_type == SIGHASH_ANYONECANPAY) * 49 - (out_type != SIGHASH_ALL and out_type != SIGHASH_SINGLE) * 32 + (annex is not None) * 32 + scriptpath * 37
return ss

5
test/functional/wallet_balance.py
View file

@ -4,7 +4,6 @@
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test the wallet balance RPC methods."""
from decimal import Decimal
import struct
from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE as ADDRESS_WATCHONLY
from test_framework.blocktools import COINBASE_MATURITY
@ -266,8 +265,8 @@ class WalletTest(BitcoinTestFramework):
tx_orig = self.nodes[0].gettransaction(txid)['hex']
# Increase fee by 1 coin
tx_replace = tx_orig.replace(
struct.pack("<q", 99 * 10**8).hex(),
struct.pack("<q", 98 * 10**8).hex(),
(99 * 10**8).to_bytes(8, "little", signed=True).hex(),
(98 * 10**8).to_bytes(8, "little", signed=True).hex(),
)
tx_replace = self.nodes[0].signrawtransactionwithwallet(tx_replace)['hex']
# Total balance is given by the sum of outputs of the tx