backport: merge bitcoin#27653, #24748, #29352, #29372, #29460, #29358, #29511, #29390, #29431, bitcoin-core/gui#788 (BIP324 backports: part 4)

src/net.cpp

@@ -1015,7 +1015,7 @@ namespace {
  * Only message types that are actually implemented in this codebase need to be listed, as other
  * messages get ignored anyway - whether we know how to decode them or not.
  */
-const std::array<std::string, 33> V2_MESSAGE_IDS = {
+const std::array<std::string, 33> V2_BITCOIN_IDS = {
     "", // 12 bytes follow encoding the message type like in V1
     NetMsgType::ADDR,
     NetMsgType::BLOCK,
@@ -1052,15 +1052,102 @@ const std::array<std::string, 33> V2_MESSAGE_IDS = {
     ""
 };
 
+/** List of short messages allocated in Dash's reserved namespace, in order.
+ *
+ * Slots should not be reused unless the switchover has already been done
+ * by a protocol upgrade, the old message is no longer supported by the client
+ * and a new slot wasn't already allotted for the message.
+ */
+const std::array<std::string, 40> V2_DASH_IDS = {
+    NetMsgType::SPORK,
+    NetMsgType::GETSPORKS,
+    NetMsgType::SENDDSQUEUE,
+    NetMsgType::DSACCEPT,
+    NetMsgType::DSVIN,
+    NetMsgType::DSFINALTX,
+    NetMsgType::DSSIGNFINALTX,
+    NetMsgType::DSCOMPLETE,
+    NetMsgType::DSSTATUSUPDATE,
+    NetMsgType::DSTX,
+    NetMsgType::DSQUEUE,
+    NetMsgType::SYNCSTATUSCOUNT,
+    NetMsgType::MNGOVERNANCESYNC,
+    NetMsgType::MNGOVERNANCEOBJECT,
+    NetMsgType::MNGOVERNANCEOBJECTVOTE,
+    NetMsgType::GETMNLISTDIFF,
+    NetMsgType::MNLISTDIFF,
+    NetMsgType::QSENDRECSIGS,
+    NetMsgType::QFCOMMITMENT,
+    NetMsgType::QCONTRIB,
+    NetMsgType::QCOMPLAINT,
+    NetMsgType::QJUSTIFICATION,
+    NetMsgType::QPCOMMITMENT,
+    NetMsgType::QWATCH,
+    NetMsgType::QSIGSESANN,
+    NetMsgType::QSIGSHARESINV,
+    NetMsgType::QGETSIGSHARES,
+    NetMsgType::QBSIGSHARES,
+    NetMsgType::QSIGREC,
+    NetMsgType::QSIGSHARE,
+    NetMsgType::QGETDATA,
+    NetMsgType::QDATA,
+    NetMsgType::CLSIG,
+    NetMsgType::ISDLOCK,
+    NetMsgType::MNAUTH,
+    NetMsgType::GETHEADERS2,
+    NetMsgType::SENDHEADERS2,
+    NetMsgType::HEADERS2,
+    NetMsgType::GETQUORUMROTATIONINFO,
+    NetMsgType::QUORUMROTATIONINFO
+};
+
+/** A complete set of short IDs
+ *
+ * Bitcoin takes up short IDs upto 128 (lower half) while Dash can take
+ * up short IDs between 128 and 256 (upper half) most of the array will
+ * have entries that correspond to nothing.
+ *
+ * To distinguish between entries that are *meant* to correspond to
+ * nothing versus empty space, use IsValidV2ShortID()
+ */
+constexpr std::array<std::string_view, 256> V2ShortIDs() {
+    static_assert(std::size(V2_BITCOIN_IDS) <= 128);
+    static_assert(std::size(V2_DASH_IDS) <= 128);
+
+    std::array<std::string_view, 256> ret{};
+    for (size_t idx{0}; idx < std::size(ret); idx++) {
+        if (idx < 128 && idx < std::size(V2_BITCOIN_IDS)) {
+            ret[idx] = V2_BITCOIN_IDS[idx];
+        } else if (idx >= 128 && idx - 128 < std::size(V2_DASH_IDS)) {
+            ret[idx] = V2_DASH_IDS[idx - 128];
+        } else {
+            ret[idx] = "";
+        }
+    }
+
+    return ret;
+}
+
+bool IsValidV2ShortID(uint8_t first_byte) {
+    // Since we have filled the namespace of short IDs, we have to preserve
+    // the expected behaviour of coming up short when going beyond Bitcoin's
+    // and Dash's *used* slots. We do this by checking if the byte is within
+    // the range where a valid message is expected to reside.
+    return first_byte < std::size(V2_BITCOIN_IDS) ||
+           (first_byte >= 128 && static_cast<uint8_t>(first_byte - 128) < std::size(V2_DASH_IDS));
+}
+
 class V2MessageMap
 {
     std::unordered_map<std::string, uint8_t> m_map;
 
 public:
     V2MessageMap() noexcept
     {
-        for (size_t i = 1; i < std::size(V2_MESSAGE_IDS); ++i) {
-            m_map.emplace(V2_MESSAGE_IDS[i], i);
+        for (size_t i = 1; i < std::size(V2ShortIDs()); ++i) {
+            if (IsValidV2ShortID(i)) {
+                m_map.emplace(V2ShortIDs()[i], i);
+            }
         }
     }
 
@@ -1524,9 +1611,9 @@ std::optional<std::string> V2Transport::GetMessageType(Span<const uint8_t>& cont
 
     if (first_byte != 0) {
         // Short (1 byte) encoding.
-        if (first_byte < std::size(V2_MESSAGE_IDS)) {
+        if (IsValidV2ShortID(first_byte)) {
             // Valid short message id.
-            return V2_MESSAGE_IDS[first_byte];
+            return std::string{V2ShortIDs()[first_byte]};
         } else {
             // Unknown short message id.
             return std::nullopt;
@@ -2044,7 +2131,7 @@ void CConnman::CreateNodeFromAcceptedSocket(std::unique_ptr<Sock>&& sock,
     RandAddEvent((uint32_t)id);
 }
 
-bool CConnman::AddConnection(const std::string& address, ConnectionType conn_type)
+bool CConnman::AddConnection(const std::string& address, ConnectionType conn_type, bool use_v2transport = false)
 {
     AssertLockNotHeld(m_unused_i2p_sessions_mutex);
     std::optional<int> max_connections;
@@ -2077,7 +2164,7 @@ bool CConnman::AddConnection(const std::string& address, ConnectionType conn_typ
     CSemaphoreGrant grant(*semOutbound, true);
     if (!grant) return false;
 
-    OpenNetworkConnection(CAddress(), false, std::move(grant), address.c_str(), conn_type, /*use_v2transport=*/false);
+    OpenNetworkConnection(CAddress(), false, std::move(grant), address.c_str(), conn_type, /*use_v2transport=*/use_v2transport);
     return true;
 }
 
@@ -2314,7 +2401,11 @@ bool CConnman::InactivityCheck(const CNode& node) const
     }
 
     if (!node.fSuccessfullyConnected) {
-        LogPrint(BCLog::NET, "version handshake timeout peer=%d\n", node.GetId());
+        if (node.m_transport->GetInfo().transport_type == TransportProtocolType::DETECTING) {
+            LogPrint(BCLog::NET, "V2 handshake timeout peer=%d\n", node.GetId());
+        } else {
+            LogPrint(BCLog::NET, "version handshake timeout peer=%d\n", node.GetId());
+        }
         return true;
     }
 
@@ -3696,7 +3787,7 @@ void CConnman::ThreadOpenMasternodeConnections(CDeterministicMNManager& dmnman,
 
         mn_metaman.GetMetaInfo(connectToDmn->proTxHash)->SetLastOutboundAttempt(nANow);
 
-        OpenMasternodeConnection(CAddress(connectToDmn->pdmnState->addr, NODE_NETWORK), isProbe);
+        OpenMasternodeConnection(CAddress(connectToDmn->pdmnState->addr, NODE_NETWORK), /*use_v2transport=*/GetLocalServices() & NODE_P2P_V2, isProbe);
         // should be in the list now if connection was opened
         bool connected = ForNode(connectToDmn->pdmnState->addr, CConnman::AllNodes, [&](CNode* pnode) {
             if (pnode->fDisconnect) {
@@ -3806,9 +3897,9 @@ void CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFai
     }
 }
 
-void CConnman::OpenMasternodeConnection(const CAddress &addrConnect, MasternodeProbeConn probe) {
+void CConnman::OpenMasternodeConnection(const CAddress &addrConnect, bool use_v2transport, MasternodeProbeConn probe) {
     OpenNetworkConnection(addrConnect, false, {}, /*strDest=*/nullptr, ConnectionType::OUTBOUND_FULL_RELAY,
-                          /*use_v2transport=*/false, MasternodeConn::IsConnection, probe);
+                          use_v2transport, MasternodeConn::IsConnection, probe);
 }
 
 Mutex NetEventsInterface::g_msgproc_mutex;

src/net.h

@@ -1283,7 +1283,7 @@ friend class CNode;
                                MasternodeConn masternode_connection = MasternodeConn::IsNotConnection,
                                MasternodeProbeConn masternode_probe_connection = MasternodeProbeConn::IsNotConnection)
         EXCLUSIVE_LOCKS_REQUIRED(!m_unused_i2p_sessions_mutex, !mutexMsgProc);
-    void OpenMasternodeConnection(const CAddress& addrConnect, MasternodeProbeConn probe = MasternodeProbeConn::IsConnection)
+    void OpenMasternodeConnection(const CAddress& addrConnect, bool use_v2transport, MasternodeProbeConn probe = MasternodeProbeConn::IsConnection)
         EXCLUSIVE_LOCKS_REQUIRED(!m_unused_i2p_sessions_mutex, !mutexMsgProc);
     bool CheckIncomingNonce(uint64_t nonce);
 
@@ -1476,13 +1476,14 @@ friend class CNode;
      * @param[in]   address     Address of node to try connecting to
      * @param[in]   conn_type   ConnectionType::OUTBOUND, ConnectionType::BLOCK_RELAY,
      *                          ConnectionType::ADDR_FETCH or ConnectionType::FEELER
+     * @param[in]   use_v2transport  Set to true if node attempts to connect using BIP 324 v2 transport protocol.
      * @return      bool        Returns false if there are no available
      *                          slots for this connection:
      *                          - conn_type not a supported ConnectionType
      *                          - Max total outbound connection capacity filled
      *                          - Max connection capacity for type is filled
      */
-    bool AddConnection(const std::string& address, ConnectionType conn_type)
+    bool AddConnection(const std::string& address, ConnectionType conn_type, bool use_v2transport)
         EXCLUSIVE_LOCKS_REQUIRED(!m_unused_i2p_sessions_mutex, !mutexMsgProc);
 
     bool AddPendingMasternode(const uint256& proTxHash);

src/protocol.cpp

@@ -166,7 +166,10 @@ const static std::string allNetMessageTypes[] = {
     NetMsgType::MNAUTH,
     NetMsgType::GETHEADERS2,
     NetMsgType::SENDHEADERS2,
-    NetMsgType::HEADERS2};
+    NetMsgType::HEADERS2,
+    NetMsgType::GETQUORUMROTATIONINFO,
+    NetMsgType::QUORUMROTATIONINFO
+};
 const static std::vector<std::string> allNetMessageTypesVec(std::begin(allNetMessageTypes), std::end(allNetMessageTypes));
 
 /** Message types that are not allowed by blocks-relay-only policy.
@@ -184,6 +187,7 @@ const static std::string netMessageTypesViolateBlocksOnly[] = {
     NetMsgType::DSSTATUSUPDATE,
     NetMsgType::DSTX,
     NetMsgType::DSVIN,
+    NetMsgType::GETQUORUMROTATIONINFO,
     NetMsgType::QBSIGSHARES,
     NetMsgType::QCOMPLAINT,
     NetMsgType::QCONTRIB,
@@ -197,6 +201,7 @@ const static std::string netMessageTypesViolateBlocksOnly[] = {
     NetMsgType::QSIGSESANN,
     NetMsgType::QSIGSHARE,
     NetMsgType::QSIGSHARESINV,
+    NetMsgType::QUORUMROTATIONINFO,
     NetMsgType::QWATCH,
     NetMsgType::TX,
 };

src/qt/forms/debugwindow.ui

@@ -1005,7 +1005,7 @@
                <item row="3" column="0">
                 <widget class="QLabel" name="peerSessionIdLabel">
                  <property name="toolTip">
-                  <string>The BIP324 session ID string in hex, if any.</string>
+                  <string>The BIP324 session ID string in hex.</string>
                  </property>
                  <property name="text">
                   <string>Session ID</string>

src/rpc/client.cpp

@@ -233,6 +233,7 @@ static const CRPCConvertParam vRPCConvertParams[] =
     { "sendmsgtopeer", 0, "peer_id" },
     { "stop", 0, "wait" },
     { "addnode", 2, "v2transport" },
+    { "addconnection", 2, "v2transport" },
     { "verifychainlock", 2, "blockHeight" },
     { "verifyislock", 3, "maxHeight" },
     { "submitchainlock", 2, "blockHeight" },

src/rpc/masternode.cpp

@@ -38,6 +38,7 @@ static RPCHelpMan masternode_connect()
         "Connect to given masternode\n",
         {
             {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The address of the masternode to connect"},
+            {"v2transport", RPCArg::Type::BOOL, RPCArg::Default{false}, "Attempt to connect using BIP324 v2 transport protocol"},
         },
         RPCResults{},
         RPCExamples{""},
@@ -50,12 +51,19 @@ static RPCHelpMan masternode_connect()
         throw JSONRPCError(RPC_INTERNAL_ERROR, strprintf("Incorrect masternode address %s", strAddress));
     }
 
+    bool use_v2transport = !request.params[1].isNull() && ParseBoolV(request.params[1], "v2transport");
+
     const NodeContext& node = EnsureAnyNodeContext(request.context);
     CConnman& connman = EnsureConnman(node);
 
-    connman.OpenMasternodeConnection(CAddress(addr.value(), NODE_NETWORK));
-    if (!connman.IsConnected(CAddress(addr.value(), NODE_NETWORK), CConnman::AllNodes))
+    if (use_v2transport && !(connman.GetLocalServices() & NODE_P2P_V2)) {
+        throw JSONRPCError(RPC_INVALID_PARAMETER, "Error: Adding v2transport connections requires -v2transport init flag to be set.");
+    }
+
+    connman.OpenMasternodeConnection(CAddress(addr.value(), NODE_NETWORK), use_v2transport);
+    if (!connman.IsConnected(CAddress(addr.value(), NODE_NETWORK), CConnman::AllNodes)) {
         throw JSONRPCError(RPC_INTERNAL_ERROR, strprintf("Couldn't connect to masternode %s", strAddress));
+    }
 
     return "successfully connected";
 },

src/rpc/net.cpp

@@ -371,6 +371,7 @@ static RPCHelpMan addconnection()
         {
             {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The IP address and port to attempt connecting to."},
             {"connection_type", RPCArg::Type::STR, RPCArg::Optional::NO, "Type of connection to open (\"outbound-full-relay\", \"block-relay-only\", \"addr-fetch\" or \"feeler\")."},
+            {"v2transport", RPCArg::Type::BOOL, RPCArg::Default{false}, "Attempt to connect using BIP324 v2 transport protocol"},
         },
         RPCResult{
             RPCResult::Type::OBJ, "", "",
@@ -379,8 +380,8 @@ static RPCHelpMan addconnection()
                 { RPCResult::Type::STR, "connection_type", "Type of connection opened." },
             }},
         RPCExamples{
-            HelpExampleCli("addconnection", "\"192.168.0.6:8333\" \"outbound-full-relay\"")
-            + HelpExampleRpc("addconnection", "\"192.168.0.6:8333\" \"outbound-full-relay\"")
+            HelpExampleCli("addconnection", "\"192.168.0.6:8333\" \"outbound-full-relay\" true")
+            + HelpExampleRpc("addconnection", "\"192.168.0.6:8333\" \"outbound-full-relay\" true")
         },
         [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue
 {
@@ -403,11 +404,16 @@ static RPCHelpMan addconnection()
     } else {
         throw JSONRPCError(RPC_INVALID_PARAMETER, self.ToString());
     }
+    bool use_v2transport = !request.params[2].isNull() && request.params[2].get_bool();
 
     NodeContext& node = EnsureAnyNodeContext(request.context);
     CConnman& connman = EnsureConnman(node);
 
-    const bool success = connman.AddConnection(address, conn_type);
+    if (use_v2transport && !(connman.GetLocalServices() & NODE_P2P_V2)) {
+        throw JSONRPCError(RPC_INVALID_PARAMETER, "Error: Adding v2transport connections requires -v2transport init flag to be set.");
+    }
+
+    const bool success = connman.AddConnection(address, conn_type, use_v2transport);
     if (!success) {
         throw JSONRPCError(RPC_CLIENT_NODE_CAPACITY_REACHED, "Error: Already at capacity for specified connection type.");
     }

src/test/net_tests.cpp

@@ -1537,7 +1537,13 @@ BOOST_AUTO_TEST_CASE(v2transport_test)
         tester.CompareSessionIDs();
         auto msg_data_1 = g_insecure_rand_ctx.randbytes<uint8_t>(MAX_PROTOCOL_MESSAGE_LENGTH); // test that receiving max size payload works
         auto msg_data_2 = g_insecure_rand_ctx.randbytes<uint8_t>(MAX_PROTOCOL_MESSAGE_LENGTH); // test that sending max size payload works
-        tester.SendMessage(uint8_t(InsecureRandRange(223) + 33), {}); // unknown short id
+        tester.SendMessage([]() {
+            if (g_insecure_rand_ctx.randbool()) {
+                return static_cast<uint8_t>(InsecureRandRange(95) + 33); // Bitcoin's range
+            } else {
+                return static_cast<uint8_t>(InsecureRandRange(88) + 40 + 128); // Dash's range
+            }
+        }(), {}); // unknown short id
         tester.SendMessage(uint8_t(2), msg_data_1); // "block" short id
         tester.AddMessage("blocktxn", msg_data_2); // schedule blocktxn to be sent to us
         ret = tester.Interact();

test/functional/feature_addrman.py

@@ -8,8 +8,7 @@
 import re
 import struct
 
-from test_framework.messages import ser_uint256, hash256
-from test_framework.p2p import MAGIC_BYTES
+from test_framework.messages import ser_uint256, hash256, MAGIC_BYTES
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.test_node import ErrorMatch
 from test_framework.util import assert_equal

test/functional/feature_block.py

@@ -1277,6 +1277,10 @@ def run_test(self):
         b89a = self.update_block("89a", [tx])
         self.send_blocks([b89a], success=False, reject_reason='bad-txns-inputs-missingorspent', reconnect=True)
 
+        # Don't use v2transport for the large reorg, which is too slow with the unoptimized python ChaCha20 implementation
+        if self.options.v2transport:
+            self.nodes[0].disconnect_p2ps()
+            self.helper_peer = self.nodes[0].add_p2p_connection(P2PDataStore(), supports_v2_p2p=False)
         self.log.info("Test a re-org of ~2 days' worth of blocks (1088 blocks)")
 
         self.move_tip(88)

test/functional/feature_maxuploadtarget.py

@@ -68,7 +68,8 @@ def run_test(self):
         p2p_conns = []
 
         for _ in range(3):
-            p2p_conns.append(self.nodes[0].add_p2p_connection(TestP2PConn()))
+            # Don't use v2transport in this test (too slow with the unoptimized python ChaCha20 implementation)
+            p2p_conns.append(self.nodes[0].add_p2p_connection(TestP2PConn(), supports_v2_p2p=False))
 
         # Now mine a big block
         mine_large_block(self, self.nodes[0], self.utxo_cache)
@@ -150,7 +151,7 @@ def run_test(self):
         self.restart_node(0, ["-whitelist=download@127.0.0.1", "-maxuploadtarget=1", "-blockmaxsize=999000", "-mocktime="+str(current_mocktime)])
 
         # Reconnect to self.nodes[0]
-        peer = self.nodes[0].add_p2p_connection(TestP2PConn())
+        peer = self.nodes[0].add_p2p_connection(TestP2PConn(), supports_v2_p2p=False)
 
         #retrieve 20 blocks which should be enough to break the 1MB limit
         getdata_request.inv = [CInv(MSG_BLOCK, big_new_block)]

test/functional/feature_reindex.py

@@ -12,7 +12,7 @@
 
 import os
 from test_framework.test_framework import BitcoinTestFramework
-from test_framework.p2p import MAGIC_BYTES
+from test_framework.messages import MAGIC_BYTES
 from test_framework.util import assert_equal
 
 

test/functional/p2p_ibd_stalling.py

@@ -83,7 +83,8 @@ def run_test(self):
 
         # Need to wait until 1023 blocks are received - the magic total bytes number is a workaround in lack of an rpc
         # returning the number of downloaded (but not connected) blocks.
-        self.wait_until(lambda: self.total_bytes_recv_for_blocks() == 172761)
+        bytes_recv = 172761 if not self.options.v2transport else 169692
+        self.wait_until(lambda: self.total_bytes_recv_for_blocks() == bytes_recv)
 
         self.all_sync_send_with_ping(peers)
         # If there was a peer marked for stalling, it would get disconnected