Merge #17957: Serialization improvements step 3 (compression.h)

4de934b9b5 Convert compression.h to new serialization framework (Pieter Wuille)
ca34c5cba5 Add FORMATTER_METHODS, similar to SERIALIZE_METHODS, but for formatters (Pieter Wuille)

Pull request description:

  This is the next piece of the puzzle from #10785. It includes:
  * The `FORMATTER_METHODS` macro, similar to `SERIALIZE_METHODS`, for defining a formatter with a unified serialization/deserialization implementation.
  * Updating `compression.h` to consist of 3 formatters, rather than old-style wrappers (`ScriptCompression`, `AmountCompression`, `TxOutCompression`).

ACKs for top commit:
  laanwj:
    code review ACK 4de934b9b5
  ryanofsky:
    Code review ACK 4de934b9b5. Only change since last review is removing REF usages

Tree-SHA512: d52ca21eb1ce87d9bc3c90d00c905bd4fada522759aaa144c02a58b4d738d5e8647c0558b8ce393c707f6e3c4d20bf93781a2dcc1e1dcbd276d9b5ffd0e02cd6

src/coins.h

@@ -61,7 +61,7 @@ public:
         assert(!IsSpent());
         uint32_t code = nHeight * 2 + fCoinBase;
         ::Serialize(s, VARINT(code));
-        ::Serialize(s, CTxOutCompressor(REF(out)));
+        ::Serialize(s, Using<TxOutCompression>(out));
     }
 
     template<typename Stream>
@@ -70,7 +70,7 @@ public:
         ::Unserialize(s, VARINT(code));
         nHeight = code >> 1;
         fCoinBase = code & 1;
-        ::Unserialize(s, CTxOutCompressor(out));
+        ::Unserialize(s, Using<TxOutCompression>(out));
     }
 
     bool IsSpent() const {

src/compressor.h

@@ -11,10 +11,6 @@
 #include <serialize.h>
 #include <span.h>
 
-class CKeyID;
-class CPubKey;
-class CScriptID;
-
 bool CompressScript(const CScript& script, std::vector<unsigned char> &out);
 unsigned int GetSpecialScriptSize(unsigned int nSize);
 bool DecompressScript(CScript& script, unsigned int nSize, const std::vector<unsigned char> &out);
@@ -33,9 +29,8 @@ uint64_t DecompressAmount(uint64_t nAmount);
  *  Other scripts up to 121 bytes require 1 byte + script length. Above
  *  that, scripts up to 16505 bytes require 2 bytes + script length.
  */
-class CScriptCompressor
+struct ScriptCompression
 {
-private:
     /**
      * make this static for now (there are only 6 special scripts defined)
      * this can potentially be extended together with a new nVersion for
@@ -44,12 +39,8 @@ private:
      */
     static const unsigned int nSpecialScripts = 6;
 
-    CScript &script;
-public:
-    explicit CScriptCompressor(CScript &scriptIn) : script(scriptIn) { }
-
     template<typename Stream>
-    void Serialize(Stream &s) const {
+    void Ser(Stream &s, const CScript& script) {
         std::vector<unsigned char> compr;
         if (CompressScript(script, compr)) {
             s << MakeSpan(compr);
@@ -61,7 +52,7 @@ public:
     }
 
     template<typename Stream>
-    void Unserialize(Stream &s) {
+    void Unser(Stream &s, CScript& script) {
         unsigned int nSize = 0;
         s >> VARINT(nSize);
         if (nSize < nSpecialScripts) {
@@ -82,30 +73,24 @@ public:
     }
 };
 
-/** wrapper for CTxOut that provides a more compact serialization */
+struct AmountCompression
-class CTxOutCompressor
 {
-private:
+    template<typename Stream, typename I> void Ser(Stream& s, I val)
-    CTxOut &txout;
+    {
-
+        s << VARINT(CompressAmount(val));
-public:
+    }
-    explicit CTxOutCompressor(CTxOut &txoutIn) : txout(txoutIn) { }
+    template<typename Stream, typename I> void Unser(Stream& s, I& val)
-
+    {
-    ADD_SERIALIZE_METHODS;
+        uint64_t v;
-
+        s >> VARINT(v);
-    template <typename Stream, typename Operation>
+        val = DecompressAmount(v);
-    inline void SerializationOp(Stream& s, Operation ser_action) {
-        if (!ser_action.ForRead()) {
-            uint64_t nVal = CompressAmount(txout.nValue);
-            READWRITE(VARINT(nVal));
-        } else {
-            uint64_t nVal = 0;
-            READWRITE(VARINT(nVal));
-            txout.nValue = DecompressAmount(nVal);
-        }
-        CScriptCompressor cscript(REF(txout.scriptPubKey));
-        READWRITE(cscript);
     }
 };
 
+/** wrapper for CTxOut that provides a more compact serialization */
+struct TxOutCompression
+{
+    FORMATTER_METHODS(CTxOut, obj) { READWRITE(Using<AmountCompression>(obj.nValue), Using<ScriptCompression>(obj.scriptPubKey)); }
+};
+
 #endif // BITCOIN_COMPRESSOR_H

src/serialize.h

@@ -199,6 +199,30 @@ template<typename X> const X& ReadWriteAsHelper(const X& x) { return x; }
         SerializationOp(s, CSerActionUnserialize());                  \
     }
 
+/**
+ * Implement the Ser and Unser methods needed for implementing a formatter (see Using below).
+ *
+ * Both Ser and Unser are delegated to a single static method SerializationOps, which is polymorphic
+ * in the serialized/deserialized type (allowing it to be const when serializing, and non-const when
+ * deserializing).
+ *
+ * Example use:
+ *   struct FooFormatter {
+ *     FORMATTER_METHODS(Class, obj) { READWRITE(obj.val1, VARINT(obj.val2)); }
+ *   }
+ *   would define a class FooFormatter that defines a serialization of Class objects consisting
+ *   of serializing its val1 member using the default serialization, and its val2 member using
+ *   VARINT serialization. That FooFormatter can then be used in statements like
+ *   READWRITE(Using<FooFormatter>(obj.bla)).
+ */
+#define FORMATTER_METHODS(cls, obj) \
+    template<typename Stream> \
+    static void Ser(Stream& s, const cls& obj) { SerializationOps(obj, s, CSerActionSerialize()); } \
+    template<typename Stream> \
+    static void Unser(Stream& s, cls& obj) { SerializationOps(obj, s, CSerActionUnserialize()); } \
+    template<typename Stream, typename Type, typename Operation> \
+    static inline void SerializationOps(Type& obj, Stream& s, Operation ser_action) \
+
 /**
  * Implement the Serialize and Unserialize methods by delegating to a single templated
  * static method that takes the to-be-(de)serialized object as a parameter. This approach
@@ -211,17 +235,15 @@ template<typename X> const X& ReadWriteAsHelper(const X& x) { return x; }
     void Serialize(Stream& s) const                                                 \
     {                                                                               \
         static_assert(std::is_same<const cls&, decltype(*this)>::value, "Serialize type mismatch"); \
-        SerializationOps(*this, s, CSerActionSerialize());                          \
+        Ser(s, *this);                                                              \
     }                                                                               \
     template<typename Stream>                                                       \
     void Unserialize(Stream& s)                                                     \
     {                                                                               \
         static_assert(std::is_same<cls&, decltype(*this)>::value, "Unserialize type mismatch"); \
-        SerializationOps(*this, s, CSerActionUnserialize());                        \
+        Unser(s, *this);                                                            \
     }                                                                               \
-    template<typename Stream, typename Type, typename Operation>                    \
+    FORMATTER_METHODS(cls, obj)
-    static inline void SerializationOps(Type& obj, Stream& s, Operation ser_action) \
-
 
 #ifndef CHAR_EQUALS_INT8
 template<typename Stream> inline void Serialize(Stream& s, char a    ) { ser_writedata8(s, a); } // TODO Get rid of bare char

src/test/fuzz/deserialize.cpp

@@ -206,7 +206,7 @@ void test_one_input(const std::vector<uint8_t>& buffer)
         DeserializeFromFuzzingInput(buffer, dbi);
 #elif TXOUTCOMPRESSOR_DESERIALIZE
         CTxOut to;
-        CTxOutCompressor toc(to);
+        auto toc = Using<TxOutCompression>(to);
         DeserializeFromFuzzingInput(buffer, toc);
 #elif BLOCKTRANSACTIONS_DESERIALIZE
         BlockTransactions bt;

src/txdb.cpp

@@ -336,7 +336,7 @@ public:
         vout.assign(vAvail.size(), CTxOut());
         for (unsigned int i = 0; i < vAvail.size(); i++) {
             if (vAvail[i])
-                ::Unserialize(s, CTxOutCompressor(vout[i]));
+                ::Unserialize(s, Using<TxOutCompression>(vout[i]));
         }
         // coinbase height
         ::Unserialize(s, VARINT(nHeight, VarIntMode::NONNEGATIVE_SIGNED));

src/undo.h

@@ -32,7 +32,7 @@ public:
             // Required to maintain compatibility with older undo format.
             ::Serialize(s, (unsigned char)0);
         }
-        ::Serialize(s, CTxOutCompressor(REF(txout->out)));
+        ::Serialize(s, Using<TxOutCompression>(REF(txout->out)));
     }
 
     explicit TxInUndoSerializer(const Coin* coin) : txout(coin) {}
@@ -56,7 +56,7 @@ public:
             unsigned int nVersionDummy;
             ::Unserialize(s, VARINT(nVersionDummy));
         }
-        ::Unserialize(s, CTxOutCompressor(REF(txout->out)));
+        ::Unserialize(s, Using<TxOutCompression>(REF(txout->out)));
     }
 
     explicit TxInUndoDeserializer(Coin* coin) : txout(coin) {}