Merge bitcoin/bitcoin#26341: test: add BIP158 false-positive element check in rpc_scanblocks.py

fa54d3011e test: check for false-positives in rpc_scanblocks.py (Sebastian Falbesoner)
3bca6cd61a test: add compact block filter (BIP158) helper routines (Sebastian Falbesoner)
25ee74dd11 test: add SipHash implementation for generic data in Python (Sebastian Falbesoner)

Pull request description:

  This PR adds a fixed false-positive element check to the functional test rpc_scanblocks.py by using a pre-calculated scriptPubKey that collides with the regtest genesis block's coinbase output. Note that determining a BIP158 false-positive at runtime would also be possible, but take too long (we'd need to create and check ~800k output scripts on average, which took at least 2 minutes on average on my machine).

  The introduced check is related to issue #26322 and more concretely inspired by PR #26325 which introduces an "accurate" mode that filters out these false-positives. The introduced cryptography routines (siphash for generic data) and helpers (BIP158 ranged hash calculation, relevant scriptPubKey per block determination) could potentially also be useful for more tests in the future that involve compact block filters.

ACKs for top commit:
  achow101:
    ACK fa54d3011e

Tree-SHA512: c6af50864146028228d197fb022ba2ff24d1ef48dc7d171bccfb21e62dd50ac80db5fae0c53f5d205edabd48b3493c7aa2040f628a223e68df086ec2243e5a93

test/functional/rpc_scanblocks.py

@@ -3,6 +3,10 @@
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Test the scanblocks RPC call."""
+from test_framework.blockfilter import (
+    bip158_basic_element_hash,
+    bip158_relevant_scriptpubkeys,
+)
 from test_framework.messages import COIN
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import (
@@ -71,6 +75,28 @@ class ScanblocksTest(BitcoinTestFramework):
         assert(blockhash in node.scanblocks(
             "start", [{"desc": f"pkh({parent_key}/*)", "range": [0, 100]}], height)['relevant_blocks'])
 
+        # check that false-positives are included in the result now; note that
+        # finding a false-positive at runtime would take too long, hence we simply
+        # use a pre-calculated one that collides with the regtest genesis block's
+        # coinbase output and verify that their BIP158 ranged hashes match
+        genesis_blockhash = node.getblockhash(0)
+        genesis_spks = bip158_relevant_scriptpubkeys(node, genesis_blockhash)
+        assert_equal(len(genesis_spks), 1)
+        genesis_coinbase_spk = list(genesis_spks)[0]
+        false_positive_spk = bytes.fromhex("001400000000000000000000000000000000000cadcb")
+
+        genesis_coinbase_hash = bip158_basic_element_hash(genesis_coinbase_spk, 1, genesis_blockhash)
+        false_positive_hash = bip158_basic_element_hash(false_positive_spk, 1, genesis_blockhash)
+        assert_equal(genesis_coinbase_hash, false_positive_hash)
+
+        assert(genesis_blockhash in node.scanblocks(
+            "start", [{"desc": f"raw({genesis_coinbase_spk.hex()})"}], 0, 0)['relevant_blocks'])
+        assert(genesis_blockhash in node.scanblocks(
+            "start", [{"desc": f"raw({false_positive_spk.hex()})"}], 0, 0)['relevant_blocks'])
+
+        # TODO: after an "accurate" mode for scanblocks is implemented (e.g. PR #26325)
+        # check here that it filters out the false-positive
+
         # test node with disabled blockfilterindex
         assert_raises_rpc_error(-1, "Index is not enabled for filtertype basic",
                                 self.nodes[1].scanblocks, "start", [f"addr({addr_1})"])

test/functional/test_framework/blockfilter.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+# Copyright (c) 2022 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Helper routines relevant for compact block filters (BIP158).
+"""
+from .siphash import siphash
+
+
+def bip158_basic_element_hash(script_pub_key, N, block_hash):
+    """ Calculates the ranged hash of a filter element as defined in BIP158:
+
+    'The first step in the filter construction is hashing the variable-sized
+    raw items in the set to the range [0, F), where F = N * M.'
+
+    'The items are first passed through the pseudorandom function SipHash, which takes a
+    128-bit key k and a variable-sized byte vector and produces a uniformly random 64-bit
+    output. Implementations of this BIP MUST use the SipHash parameters c = 2 and d = 4.'
+
+    'The parameter k MUST be set to the first 16 bytes of the hash (in standard
+    little-endian representation) of the block for which the filter is constructed. This
+    ensures the key is deterministic while still varying from block to block.'
+    """
+    M = 784931
+    block_hash_bytes = bytes.fromhex(block_hash)[::-1]
+    k0 = int.from_bytes(block_hash_bytes[0:8], 'little')
+    k1 = int.from_bytes(block_hash_bytes[8:16], 'little')
+    return (siphash(k0, k1, script_pub_key) * (N * M)) >> 64
+
+
+def bip158_relevant_scriptpubkeys(node, block_hash):
+    """ Determines the basic filter relvant scriptPubKeys as defined in BIP158:
+
+    'A basic filter MUST contain exactly the following items for each transaction in a block:
+       - The previous output script (the script being spent) for each input, except for
+         the coinbase transaction.
+       - The scriptPubKey of each output, aside from all OP_RETURN output scripts.'
+    """
+    spks = set()
+    for tx in node.getblock(blockhash=block_hash, verbosity=3)['tx']:
+        # gather prevout scripts
+        for i in tx['vin']:
+            if 'prevout' in i:
+                spks.add(bytes.fromhex(i['prevout']['scriptPubKey']['hex']))
+        # gather output scripts
+        for o in tx['vout']:
+            if o['scriptPubKey']['type'] != 'nulldata':
+                spks.add(bytes.fromhex(o['scriptPubKey']['hex']))
+    return spks

test/functional/test_framework/siphash.py

@@ -1,15 +1,17 @@
 #!/usr/bin/env python3
-# Copyright (c) 2016-2018 The Bitcoin Core developers
+# Copyright (c) 2016-2022 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
-"""Specialized SipHash-2-4 implementations.
+"""SipHash-2-4 implementation.
 
-This implements SipHash-2-4 for 256-bit integers.
+This implements SipHash-2-4. For convenience, an interface taking 256-bit
+integers is provided in addition to the one accepting generic data.
 """
 
 def rotl64(n, b):
     return n >> (64 - b) | (n & ((1 << (64 - b)) - 1)) << b
 
+
 def siphash_round(v0, v1, v2, v3):
     v0 = (v0 + v1) & ((1 << 64) - 1)
     v1 = rotl64(v1, 13)
@@ -27,37 +29,37 @@ def siphash_round(v0, v1, v2, v3):
     v2 = rotl64(v2, 32)
     return (v0, v1, v2, v3)
 
-def siphash256(k0, k1, h):
+
-    n0 = h & ((1 << 64) - 1)
+def siphash(k0, k1, data):
-    n1 = (h >> 64) & ((1 << 64) - 1)
+    assert(type(data) == bytes)
-    n2 = (h >> 128) & ((1 << 64) - 1)
-    n3 = (h >> 192) & ((1 << 64) - 1)
     v0 = 0x736f6d6570736575 ^ k0
     v1 = 0x646f72616e646f6d ^ k1
     v2 = 0x6c7967656e657261 ^ k0
-    v3 = 0x7465646279746573 ^ k1 ^ n0
+    v3 = 0x7465646279746573 ^ k1
+    c = 0
+    t = 0
+    for d in data:
+        t |= d << (8 * (c % 8))
+        c = (c + 1) & 0xff
+        if (c & 7) == 0:
+            v3 ^= t
+            v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+            v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
+            v0 ^= t
+            t = 0
+    t = t | (c << 56)
+    v3 ^= t
     v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
     v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
-    v0 ^= n0
+    v0 ^= t
-    v3 ^= n1
+    v2 ^= 0xff
-    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
-    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
-    v0 ^= n1
-    v3 ^= n2
-    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
-    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
-    v0 ^= n2
-    v3 ^= n3
-    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
-    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
-    v0 ^= n3
-    v3 ^= 0x2000000000000000
-    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
-    v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
-    v0 ^= 0x2000000000000000
-    v2 ^= 0xFF
     v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
     v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
     v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
     v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3)
     return v0 ^ v1 ^ v2 ^ v3
+
+
+def siphash256(k0, k1, num):
+    assert(type(num) == int)
+    return siphash(k0, k1, num.to_bytes(32, 'little'))