backport: bitcoin#26094, #26325, #26499, #26649, #26965, #27033, #27066, #27093, #34185, partial #35070, bitcon-core/gui#704

ci/dash/lint-tidy.sh

@@ -105,6 +105,7 @@ iwyu_tool.py \
   "src/node/chainstate.cpp" \
   "src/node/chainstatemanager_args.cpp" \
   "src/node/mempool_args.cpp" \
+  "src/node/utxo_snapshot.cpp" \
   "src/node/validation_cache_args.cpp" \
   "src/node/minisketchwrapper.cpp" \
   "src/policy/feerate.cpp" \

ci/test/00_setup_env.sh

@@ -8,7 +8,6 @@ export LC_ALL=C.UTF-8
 
 # The root dir.
 # The ci system copies this folder.
-# This is where the depends build is done.
 BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd )
 export BASE_ROOT_DIR
 
@@ -62,18 +61,20 @@ export CCACHE_MAXSIZE=${CCACHE_MAXSIZE:-100M}
 export CCACHE_TEMPDIR=${CCACHE_TEMPDIR:-/tmp/.ccache-temp}
 export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1}
 # The cache dir.
-# This folder exists on the ci host and ci guest. Changes are propagated back and forth.
+# This folder exists only on the ci guest, and on the ci host as a volume.
 export CCACHE_DIR=${CCACHE_DIR:-$CACHE_DIR/ccache}
 # ctcache (clang-tidy cache) configuration
 export CTCACHE_DIR=${CTCACHE_DIR:-$CACHE_DIR/ctcache}
 export CTCACHE_MAXSIZE_MB=${CTCACHE_MAXSIZE_MB:-50}
 # The depends dir.
-# This folder exists on the ci host and ci guest. Changes are propagated back and forth.
+# This folder exists only on the ci guest, and on the ci host as a volume.
 export DEPENDS_DIR=${DEPENDS_DIR:-$BASE_ROOT_DIR/depends}
 # Folder where the build result is put (bin and lib).
 export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out/$HOST}
 # Folder where the build is done (dist and out-of-tree build).
 export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_SCRATCH_DIR/build-ci}
+# The folder for previous release binaries.
+# This folder exists only on the ci guest, and on the ci host as a volume.
 export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/releases/$HOST}
 export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}
 export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps}

src/hash.h

@@ -167,6 +167,39 @@ class CHashWriter : public HashWriter
 };
 
 /** Reads data from an underlying stream, while hashing the read data. */
+template <typename Source>
+class HashVerifier : public HashWriter
+{
+private:
+    Source& m_source;
+
+public:
+    explicit HashVerifier(Source& source LIFETIMEBOUND) : m_source{source} {}
+
+    void read(Span<std::byte> dst)
+    {
+        m_source.read(dst);
+        this->write(dst);
+    }
+
+    void ignore(size_t num_bytes)
+    {
+        std::byte data[1024];
+        while (num_bytes > 0) {
+            size_t now = std::min<size_t>(num_bytes, 1024);
+            read({data, now});
+            num_bytes -= now;
+        }
+    }
+
+    template <typename T>
+    HashVerifier<Source>& operator>>(T&& obj)
+    {
+        ::Unserialize(*this, obj);
+        return *this;
+    }
+};
+
 template<typename Source>
 class CHashVerifier : public CHashWriter
 {

src/node/blockstorage.cpp

@@ -384,7 +384,7 @@ bool BlockManager::LoadBlockIndexDB()
     }
     for (std::set<int>::iterator it = setBlkDataFiles.begin(); it != setBlkDataFiles.end(); it++) {
         FlatFilePos pos(*it, 0);
-        if (CAutoFile(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION).IsNull()) {
+        if (AutoFile{OpenBlockFile(pos, true)}.IsNull()) {
             return false;
         }
     }
@@ -509,14 +509,14 @@ CBlockFileInfo* BlockManager::GetBlockFileInfo(size_t n)
 static bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart)
 {
     // Open history file to append
-    CAutoFile fileout(OpenUndoFile(pos), SER_DISK, CLIENT_VERSION);
+    AutoFile fileout{OpenUndoFile(pos)};
     if (fileout.IsNull()) {
         LogError("%s: OpenUndoFile failed\n", __func__);
         return false;
     }
 
     // Write index header
-    unsigned int nSize = GetSerializeSize(blockundo, fileout.GetVersion());
+    unsigned int nSize = GetSerializeSize(blockundo, CLIENT_VERSION);
     fileout << messageStart << nSize;
 
     // Write undo data
@@ -547,15 +547,15 @@ bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex)
     }
 
     // Open history file to read
-    CAutoFile filein(OpenUndoFile(pos, true), SER_DISK, CLIENT_VERSION);
+    AutoFile filein{OpenUndoFile(pos, true)};
     if (filein.IsNull()) {
         LogError("%s: OpenUndoFile failed\n", __func__);
         return false;
     }
 
     // Read block
     uint256 hashChecksum;
-    CHashVerifier<CAutoFile> verifier(&filein); // We need a CHashVerifier as reserializing may lose data
+    HashVerifier verifier{filein}; // Use HashVerifier as reserializing may lose data, c.f. commit d342424301013ec47dc146a4beb49d5c9319d80a
     try {
         verifier << pindex->pprev->GetBlockHash();
         verifier >> blockundo;
@@ -748,14 +748,14 @@ bool BlockManager::FindUndoPos(BlockValidationState& state, int nFile, FlatFileP
 static bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart)
 {
     // Open history file to append
-    CAutoFile fileout(OpenBlockFile(pos), SER_DISK, CLIENT_VERSION);
+    AutoFile fileout{OpenBlockFile(pos)};
     if (fileout.IsNull()) {
         LogError("WriteBlockToDisk: OpenBlockFile failed\n");
         return false;
     }
 
     // Write index header
-    unsigned int nSize = GetSerializeSize(block, fileout.GetVersion());
+    unsigned int nSize = GetSerializeSize(block, CLIENT_VERSION);
     fileout << messageStart << nSize;
 
     // Write block
@@ -807,7 +807,7 @@ std::optional<uint256> ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos,
     block.SetNull();
 
     // Open history file to read
-    CAutoFile filein(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION);
+    AutoFile filein{OpenBlockFile(pos, true)};
     if (filein.IsNull()) {
         LogError("ReadBlockFromDisk: OpenBlockFile failed for %s\n", pos.ToString());
         return std::nullopt;

src/node/blockstorage.h

@@ -168,7 +168,6 @@ class BlockManager
 
     /**
      * All pairs A->B, where A (or one of its ancestors) misses transactions, but B has transactions.
-     * Pruned nodes may have entries where B is missing data.
      */
     std::multimap<CBlockIndex*, CBlockIndex*> m_blocks_unlinked;
 

src/node/utxo_snapshot.cpp

@@ -6,14 +6,19 @@
 
 #include <logging.h>
 #include <streams.h>
+#include <sync.h>
+#include <tinyformat.h>
+#include <txdb.h>
 #include <uint256.h>
 #include <util/fs.h>
 #include <util/fs_helpers.h>
 #include <util/system.h>
 #include <validation.h>
 
+#include <cassert>
 #include <cstdio>
 #include <optional>
+#include <string>
 
 namespace node {
 

src/node/utxo_snapshot.h

@@ -7,11 +7,16 @@
 #define BITCOIN_NODE_UTXO_SNAPSHOT_H
 
 #include <serialize.h>
-#include <uint256.h>
+#include <kernel/cs_main.h>
 #include <util/fs.h>
-#include <validation.h>
+#include <sync.h>
+#include <uint256.h>
 
+#include <cstdint>
 #include <optional>
+#include <string_view>
+
+class Chainstate;
 
 namespace node {
 //! Metadata describing a serialized version of a UTXO set from which an

src/qt/overviewpage.cpp

@@ -343,6 +343,16 @@ void OverviewPage::setWalletModel(WalletModel *model)
     }
 }
 
+// Only show the most recent m_num_items rows
+void OverviewPage::LimitTransactionRows()
+{
+    if (filter && ui->listTransactions && ui->listTransactions->model() && filter.get() == ui->listTransactions->model()) {
+        for (int i = 0; i < filter->rowCount(); ++i) {
+            ui->listTransactions->setRowHidden(i, i >= m_num_items);
+        }
+    }
+}
+
 void OverviewPage::updateDisplayUnit()
 {
     if (walletModel && walletModel->getOptionsModel()) {
@@ -761,13 +771,19 @@ void OverviewPage::SetupTransactionList(int nNumItems)
         filter->setShowInactive(false);
         filter->sort(TransactionTableModel::Date, Qt::DescendingOrder);
         ui->listTransactions->setModel(filter.get());
+
+        connect(filter.get(), &TransactionFilterProxy::rowsInserted, this, &OverviewPage::LimitTransactionRows);
+        connect(filter.get(), &TransactionFilterProxy::rowsRemoved, this, &OverviewPage::LimitTransactionRows);
+        connect(filter.get(), &TransactionFilterProxy::rowsMoved, this, &OverviewPage::LimitTransactionRows);
+        connect(filter.get(), &TransactionFilterProxy::modelReset, this, &OverviewPage::LimitTransactionRows);
     }
 
-    if (filter->rowCount() == nNumItems) {
+    if (m_num_items == nNumItems) {
         return;
     }
+    m_num_items = nNumItems;
+    LimitTransactionRows();
 
-    filter->setLimit(nNumItems);
     ui->listTransactions->setMinimumHeight(nNumItems * ITEM_HEIGHT);
 }
 

src/qt/overviewpage.h

@@ -63,11 +63,14 @@ public Q_SLOTS:
 
     TxViewDelegate *txdelegate;
     std::unique_ptr<TransactionFilterProxy> filter;
+    //! Number of most recent transactions the overview list keeps visible
+    int m_num_items{0};
 
     void SetupTransactionList(int nNumItems);
     void DisableCoinJoinCompletely();
 
 private Q_SLOTS:
+    void LimitTransactionRows();
     void toggleCoinJoin();
     void updateDisplayUnit();
     void updateCoinJoinProgress();

src/qt/transactionfilterproxy.cpp

@@ -88,27 +88,8 @@ void TransactionFilterProxy::setWatchOnlyFilter(WatchOnlyFilter filter)
     invalidateFilter();
 }
 
-void TransactionFilterProxy::setLimit(int limit)
-{
-    Q_EMIT layoutAboutToBeChanged();
-    this->limitRows = limit;
-    Q_EMIT layoutChanged();
-}
-
 void TransactionFilterProxy::setShowInactive(bool _showInactive)
 {
     this->showInactive = _showInactive;
     invalidateFilter();
 }
-
-int TransactionFilterProxy::rowCount(const QModelIndex &parent) const
-{
-    if(limitRows != -1)
-    {
-        return std::min(QSortFilterProxyModel::rowCount(parent), limitRows);
-    }
-    else
-    {
-        return QSortFilterProxyModel::rowCount(parent);
-    }
-}

src/qt/transactionfilterproxy.h

@@ -59,14 +59,9 @@ class TransactionFilterProxy : public QSortFilterProxyModel
     void setMinAmount(const CAmount& minimum);
     void setWatchOnlyFilter(WatchOnlyFilter filter);
 
-    /** Set maximum number of rows returned, -1 if unlimited. */
-    void setLimit(int limit);
-
     /** Set whether to show conflicted transactions. */
     void setShowInactive(bool showInactive);
 
-    int rowCount(const QModelIndex &parent = QModelIndex()) const override;
-
 protected:
     bool filterAcceptsRow(int source_row, const QModelIndex & source_parent) const override;
 
@@ -77,7 +72,6 @@ class TransactionFilterProxy : public QSortFilterProxyModel
     quint32 typeFilter;
     WatchOnlyFilter watchOnlyFilter{WatchOnlyFilter_All};
     CAmount minAmount{0};
-    int limitRows{-1};
     bool showInactive{true};
 };
 

src/rpc/blockchain.cpp

@@ -28,6 +28,7 @@
 #include <net.h>
 #include <net_processing.h>
 #include <node/context.h>
+#include <node/transaction.h>
 #include <node/utxo_snapshot.h>
 #include <merkleblock.h>
 #include <primitives/transaction.h>
@@ -57,6 +58,7 @@
 #include <evo/evodb.h>
 #include <evo/mnhftx.h>
 #include <evo/specialtx.h>
+#include <evo/specialtx_filter.h>
 #include <instantsend/instantsend.h>
 
 #include <stdint.h>
@@ -2833,17 +2835,52 @@ class BlockFiltersScanReserver
     }
 };
 
+static bool CheckBlockFilterMatches(BlockManager& blockman, const CBlockIndex& blockindex, const GCSFilter::ElementSet& needles)
+{
+    const CBlock block{GetBlockChecked(blockman, &blockindex)};
+    const CBlockUndo block_undo{GetUndoChecked(blockman, &blockindex)};
+
+    // Check if any of the outputs match the scriptPubKey
+    for (const auto& tx : block.vtx) {
+        if (std::any_of(tx->vout.cbegin(), tx->vout.cend(), [&](const auto& txout) {
+                return needles.count(std::vector<unsigned char>(txout.scriptPubKey.begin(), txout.scriptPubKey.end())) != 0;
+            })) {
+            return true;
+        }
+        bool special_tx_match{false};
+        ExtractSpecialTxFilterElements(*tx, [&](Span<const unsigned char> element) {
+            special_tx_match = special_tx_match || needles.count(std::vector<unsigned char>(element.begin(), element.end())) != 0;
+        });
+        if (special_tx_match) return true;
+    }
+    // Check if any of the inputs match the scriptPubKey
+    for (const auto& txundo : block_undo.vtxundo) {
+        if (std::any_of(txundo.vprevout.cbegin(), txundo.vprevout.cend(), [&](const auto& coin) {
+                return needles.count(std::vector<unsigned char>(coin.out.scriptPubKey.begin(), coin.out.scriptPubKey.end())) != 0;
+            })) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
 static RPCHelpMan scanblocks()
 {
     return RPCHelpMan{"scanblocks",
-        "\nReturn relevant blockhashes for given descriptors.\n"
+        "\nReturn relevant blockhashes for given descriptors (requires blockfilterindex).\n"
         "This call may take several minutes. Make sure to use no RPC timeout (bitcoin-cli -rpcclienttimeout=0)",
         {
             scan_action_arg_desc,
             scan_objects_arg_desc,
             RPCArg{"start_height", RPCArg::Type::NUM, RPCArg::Default{0}, "Height to start to scan from"},
             RPCArg{"stop_height", RPCArg::Type::NUM, RPCArg::DefaultHint{"chain tip"}, "Height to stop to scan"},
-            RPCArg{"filtertype", RPCArg::Type::STR, RPCArg::Default{BlockFilterTypeName(BlockFilterType::BASIC_FILTER)}, "The type name of the filter"}
+            RPCArg{"filtertype", RPCArg::Type::STR, RPCArg::Default{BlockFilterTypeName(BlockFilterType::BASIC_FILTER)}, "The type name of the filter"},
+            RPCArg{"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "",
+                {
+                    {"filter_false_positives", RPCArg::Type::BOOL, RPCArg::Default{false}, "Filter false positives (slower and may fail on pruned nodes). Otherwise they may occur at a rate of 1/M"},
+                },
+                RPCArgOptions{.oneline_description="\"options\""}},
         },
         {
             scan_result_status_none,
@@ -2903,6 +2940,9 @@ static RPCHelpMan scanblocks()
             throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Unknown filtertype");
         }
 
+        UniValue options{request.params[5].isNull() ? UniValue::VOBJ : request.params[5]};
+        bool filter_false_positives{options.exists("filter_false_positives") ? options["filter_false_positives"].get_bool() : false};
+
         BlockFilterIndex* index = GetBlockFilterIndex(filtertype);
         if (!index) {
             throw JSONRPCError(RPC_MISC_ERROR, "Index is not enabled for filtertype " + filtertype_name);
@@ -2973,6 +3013,15 @@ static RPCHelpMan scanblocks()
                     for (const BlockFilter& filter : filters) {
                         // compare the elements-set with each filter
                         if (filter.GetFilter().MatchAny(needle_set)) {
+                            if (filter_false_positives) {
+                                // Double check the filter matches by scanning the block
+                                const CBlockIndex& blockindex = *CHECK_NONFATAL(WITH_LOCK(cs_main, return chainman.m_blockman.LookupBlockIndex(filter.GetBlockHash())));
+
+                                if (!CheckBlockFilterMatches(chainman.m_blockman, blockindex, needle_set)) {
+                                    continue;
+                                }
+                            }
+
                             blocks.push_back(filter.GetBlockHash().GetHex());
                             LogPrint(BCLog::RPC, "scanblocks: found match in %s\n", filter.GetBlockHash().GetHex());
                         }