Merge master commit 100cc3816c59357df488b17972aa5e2846ead831 into overlay V2

ci-build.sh

@@ -204,17 +204,24 @@ if [ $DISABLE_POSTGRES != '--disable-postgres' ] ; then
     fi
 fi
 
-export ALL_VERSIONS=0
-export NUM_PARTITIONS=1
-export RUN_PARTITIONS=0
+# Run tests at the current protocol version only. run-selftest-* adds
+# --all-versions whenever ALL_VERSIONS is non-empty (even "0"), so it must be
+# unset here, not set to 0.
+unset ALL_VERSIONS
+export NUM_PARTITIONS=$((NPROCS*2))
+export RUN_PARTITIONS
 export RND_SEED=$(($(date +%s) / 86400))  # Convert to days since epoch
 echo "Using RND_SEED: $RND_SEED"
 ulimit -n 65536
 export INTERACTIVE=0
-
-export TEST_SPEC='[overlay-ipc],[loadgen],[overlay-ipc-large]'
-export SKIP_SOROBAN_TESTS=true
 export STELLAR_OVERLAY_BINARY="${SRC_DIR}/build-${CC}-${PROTOCOL}/stellar-overlay"
+
+# Everything a plain `make check` runs (the default spec, first alternative;
+# this also runs the Soroban host tests via check-sorobans), plus the Rust
+# overlay IPC tests, which are hidden ([.]) so they don't run by default.
+# Benchmark-style tests ([acceptance]: the loadgen apply-load benchmarks and
+# the multi-node TPS stress tests) are deliberately not run in CI.
+export TEST_SPEC='~[acceptance]~[.],[overlay-ipc]'
 time make check
 
 # echo Running fixed check-test-tx-meta tests

docs/apply-load-benchmark-sac.cfg

@@ -7,6 +7,11 @@
 APPLY_LOAD_MODE="benchmark"
 APPLY_LOAD_MODEL_TX="sac"
 
+# Which timing path to use: "apply" times only transaction application, while
+# "txset-validation-and-apply" also simulates a non-leader receiving and
+# validating a tx set before applying it. Tx-set creation is not measured.
+APPLY_LOAD_TIMING_PHASES = "apply"
+
 # Whether to time the write part of the apply stage. This can be
 # disabled to get less noisy results for non-write related changes,
 # but should be enabled to get more comprehensive e2e numbers.
@@ -62,4 +67,4 @@ NODE_SEED="SDQVDISRYN2JXBS7ICL7QJAEKB3HWBJFP2QECXG7GZICAHBK4UNJCWK2 self"
 
 [QUORUM_SET]
 THRESHOLD_PERCENT=100
-VALIDATORS=["$self"]
+VALIDATORS=["$self"]

docs/apply-load-benchmark-token.cfg

@@ -7,6 +7,11 @@
 APPLY_LOAD_MODE="benchmark"
 APPLY_LOAD_MODEL_TX="custom_token"
 
+# Which timing path to use: "apply" times only transaction application, while
+# "txset-validation-and-apply" also simulates a non-leader receiving and
+# validating a tx set before applying it. Tx-set creation is not measured.
+APPLY_LOAD_TIMING_PHASES = "apply"
+
 # Whether to time the write part of the apply stage. This can be
 # disabled to get less noisy results for non-write related changes,
 # but should be enabled to get more comprehensive e2e numbers.

docs/apply-load-for-meta.cfg

@@ -9,7 +9,12 @@
 # Select the apply-load mode.
 APPLY_LOAD_MODE="ledger-limits"
 
-# Custom meta path - if not set it will be written to a temp directory and 
+# Which timing path to use: "apply" times only transaction application, while
+# "txset-validation-and-apply" also simulates a non-leader receiving and
+# validating a tx set before applying it. Tx-set creation is not measured.
+APPLY_LOAD_TIMING_PHASES = "apply"
+
+# Custom meta path - if not set it will be written to a temp directory and
 # cleaned up after running the benchmark
 METADATA_OUTPUT_STREAM='meta.xdr'
 

docs/apply-load-ledger-limits.cfg

@@ -7,6 +7,11 @@
 # Select the apply-load mode.
 APPLY_LOAD_MODE="ledger-limits"
 
+# Which timing path to use: "apply" times only transaction application, while
+# "txset-validation-and-apply" also simulates a non-leader receiving and
+# validating a tx set before applying it. Tx-set creation is not measured.
+APPLY_LOAD_TIMING_PHASES = "apply"
+
 # Medida metrics (histograms in particular) in apply path cause severe and
 # non-deterministic performance degradation. While this has to be addressed
 # eventually, it is useful to disable these when optimizing anything besides

docs/apply-load-max-sac-tps.cfg

@@ -7,6 +7,10 @@
 # Select the apply-load mode.
 APPLY_LOAD_MODE="max-sac-tps"
 
+# Which timing path to use. The max-sac-tps search targets apply-only close
+# time, so it only supports "apply".
+APPLY_LOAD_TIMING_PHASES = "apply"
+
 # Whether to time the write part of the apply stage. This can be
 # disabled to get less noisy results for non-write related changes,
 # but should be enabled to get more comprehensive e2e numbers.

docs/metrics.md

@@ -62,17 +62,14 @@ crypto.verify.miss                        | meter     | number of signature cach
 crypto.verify.total                       | meter     | sum of both hits and misses
 crypto.verify.tx-valid-hit                | meter     | signature cache hits that occurred while validating transactions (outside of background signature validation)
 crypto.verify.tx-valid-total              | meter     | sum of both hits and misses during transaction validation (outside of background signature validation)
+herder.txset.validate                     | timer     | time spent turning a received tx set into an applicable tx set and validating it on a validity-cache miss
 herder.pending[-soroban]-txs.age0         | counter   | number of gen0 pending transactions
 herder.pending[-soroban]-txs.age1         | counter   | number of gen1 pending transactions
 herder.pending[-soroban]-txs.age2         | counter   | number of gen2 pending transactions
 herder.pending[-soroban]-txs.age3         | counter   | number of gen3 pending transactions
 herder.pending[-soroban]-txs.banned       | counter   | number of transactions that got banned
-herder.pending[-soroban]-txs.sum          | counter   | sum of time (milliseconds) for transactions to be included in a ledger
-herder.pending[-soroban]-txs.count        | counter   | number of transactions to be included in a ledger
-herder.pending[-soroban]-txs.max          | counter   | largest time (milliseconds) for a transaction to be included in a ledger since last metrics call
-herder.pending[-soroban]-txs.self-sum     | counter   | sum of time (milliseconds) for transactions submitted from this node to be included in a ledger
-herder.pending[-soroban]-txs.self-count   | counter   | number of transactions submitted from this node to be included in a ledger
-herder.pending[-soroban]-txs.self-max     | counter   | largest time (milliseconds) for a transaction submitted from this node to be included in a ledger since last metrics call
+herder.pending[-soroban]-txs.delay        | timer     | time for transactions to be included in a ledger
+herder.pending[-soroban]-txs.self-delay   | timer     | time for transactions submitted from this node to be included in a ledger
 herder.pending[-soroban]-txs.evicted-due-to-low-fee-count   | counter   | Count of transactions evicted by higher fee txs when queue is near its capacity.
 herder.pending[-soroban]-txs.evicted-due-to-age-count   | counter   | Count of transactions that had low fee for too long and have not been included into several ledgers in a row.
 herder.pending[-soroban]-txs.not-included-due-to-low-fee-count   | counter   | Count of transactions that were not included into queue because it is at capacity and the fee is too low to replace other txs.
@@ -168,10 +165,7 @@ overlay.outbound.attempt                  | meter     | outbound connection atte
 overlay.outbound.cancel                   | meter     | outbound connection cancelled
 overlay.outbound.drop                     | meter     | outbound connection dropped
 overlay.outbound.establish                | meter     | outbound connection established (added to pending)
-overlay.recv.<X>                          | timer     | received message <X> (except transaction)
-overlay.recv-transaction.sum              | counter   | sum of time (microseconds) to receive transaction message
-overlay.recv-transaction.count            | counter   | number of transaction messages received
-overlay.recv-transaction.max              | counter   | maximum time (microseconds) to receive transaction message since last metrics call
+overlay.recv.<X>                          | timer     | received message <X>
 overlay.send.<X>                          | meter     | sent message <X>
 overlay.timeout.idle                      | meter     | idle peer timeout
 overlay.timeout.straggler                 | meter     | straggler peer timeout

docs/software/commands.md

@@ -20,9 +20,11 @@ Common options can be placed at any place in the command line.
 Command options can only by placed after command.
 
 * **apply-load**: Benchmarks Soroban transaction application time using
-    synthetic transactions. The benchmark is isolated to mostly just executing
-    the transactions and thus it omits a lot of the supporting mechanisms
-    (such as overlay, SCP, mempool etc). This command will generate enough
+    synthetic transactions. By default the benchmark is isolated to mostly just
+    executing the transactions and thus it omits a lot of the supporting
+    mechanisms (such as overlay, SCP, mempool etc). It may also measure tx-set
+    validation and consensus processing via `APPLY_LOAD_TIMING_PHASES` (see
+    below). This command will generate enough
     transactions to fill up a synthetic transaction queue (it's just a list of
     transactions with the same limits as the real queue), and then create a
     transaction set off of that to apply. This can also be used to record the
@@ -36,6 +38,21 @@ Command options can only by placed after command.
       consisting only of fast SAC transfer.
     - `APPLY_LOAD_MODE="benchmark"`: benchmarks a fixed-size ledger of model
       transactions. Use `APPLY_LOAD_MODEL_TX` to select the model transaction.
+  * `APPLY_LOAD_TIMING_PHASES` selects one of two timing paths:
+    - `"apply"`: the default apply-only benchmark. Its close helper still calls
+      `checkValid`, but that happens before the recorded ledger-close timer and
+      leaves the caches warm, as consensus validation would on a live node.
+    - `"txset-validation-and-apply"`: simulates a non-leader receiving the tx
+      set over the wire, validating it through local consensus (with the node
+      as its own single-validator quorum), and then applying it. It reports
+      validation, ledger close, and end-to-end time in addition to the
+      apply-only output. It does not simulate network transport, peer fetching,
+      or multi-node timing. Leader-side tx-set creation and signing happen
+      before the measured span. The signature verification cache is cleared
+      before validation, then retained so apply sees the warm cache produced by
+      validation.
+    `"txset-validation-and-apply"` is not supported with
+    `APPLY_LOAD_MODE="max-sac-tps"`; that search targets apply-only close time.
   * Load generation is configured in the Core config file. The relevant settings
     all begin with `APPLY_LOAD_`. See full example configurations with
     per-setting documentation in the `docs` directory

docs/stellar-core_example.cfg

@@ -544,6 +544,8 @@ STATE_SNAPSHOT_INVARIANT_LEDGER_FREQUENCY=300
 # MANUAL_CLOSE (true or false) defaults to false
 # Mode for testing. Ledger will only close when stellar-core gets
 #  the `manualclose` command
+# NOTE: setting this to true is not supported with the Rust overlay and
+#  will fail config validation
 MANUAL_CLOSE=false
 
 

src/database/test/DatabaseTests.cpp

@@ -471,10 +471,7 @@ TEST_CASE("Database splitting migration works correctly", "[db]")
             return count > 0;
         };
 
-        // Insert test data into all tables that should be migrated
-        execSQL("INSERT INTO peers (ip, port, nextattempt, numfailures, type) "
-                "VALUES ('127.0.0.1', 11625, '2024-01-01 00:00:00', 0, 1)",
-                db.getSession());
+        // Insert test data into all tables that should be migrated.
         execSQL("INSERT INTO ban (nodeid) VALUES "
                 "('GAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAWHF')",
                 db.getSession());
@@ -512,7 +509,6 @@ TEST_CASE("Database splitting migration works correctly", "[db]")
                 db.getSession());
 
         // Verify data exists in main before migration
-        REQUIRE(countRows("peers", db.getSession()) == 1);
         REQUIRE(countRows("ban", db.getSession()) == 1);
         REQUIRE(countRows("scphistory", db.getSession()) == 1);
         REQUIRE(countRows("scpquorums", db.getSession()) == 1);
@@ -543,37 +539,22 @@ TEST_CASE("Database splitting migration works correctly", "[db]")
         REQUIRE_FALSE(tableExists("storestate", db.getMiscSession()));
 
         // Verify all misc tables are dropped from main
-        std::vector<std::string> migratedTables = {"peers",      "ban",
-                                                   "scphistory", "scpquorums",
-                                                   "quoruminfo", "slotstate"};
+        std::vector<std::string> migratedTables = {
+            "ban", "scphistory", "scpquorums", "quoruminfo", "slotstate"};
         for (auto const& table : migratedTables)
         {
             REQUIRE_FALSE(tableExists(table, db.getSession()));
         }
 
         // Verify data was migrated to misc DB
         // Note: slotstate has 2 rows (test data + miscdatabaseschema)
-        REQUIRE(countRows("peers", db.getMiscSession()) == 1);
         REQUIRE(countRows("ban", db.getMiscSession()) == 1);
         REQUIRE(countRows("scphistory", db.getMiscSession()) == 1);
         REQUIRE(countRows("scpquorums", db.getMiscSession()) == 1);
         REQUIRE(countRows("quoruminfo", db.getMiscSession()) == 1);
         REQUIRE(countRows("slotstate", db.getMiscSession()) == 2);
 
         // Verify specific data values in misc DB
-        {
-            std::string ip;
-            int port = 0;
-            auto prep = db.getPreparedStatement("SELECT ip, port FROM peers",
-                                                db.getMiscSession());
-            auto& st = prep.statement();
-            st.exchange(soci::into(ip));
-            st.exchange(soci::into(port));
-            st.define_and_bind();
-            st.execute(true);
-            REQUIRE(ip == "127.0.0.1");
-            REQUIRE(port == 11625);
-        }
         {
             std::string state;
             auto prep = db.getPreparedStatement(

src/herder/HerderImpl.cpp

@@ -17,6 +17,7 @@
 #include "herder/RustQuorumCheckerAdaptor.h"
 #include "herder/TxSetFrame.h"
 #include "herder/TxSetUtils.h"
+#include "ledger/ImmutableLedgerView.h"
 #include "ledger/LedgerManager.h"
 #include "ledger/LedgerTxnImpl.h"
 #include "ledger/P23HotArchiveBug.h"
@@ -352,7 +353,8 @@ HerderImpl::processExternalized(uint64 slotIndex, StellarValue const& value,
             }
         }
 #ifdef BUILD_TESTS
-        mApp.getLoadGenerator().cleanupAccounts(txFramesList);
+        mApp.getLoadGenerator().cleanupAccounts(
+            static_cast<uint32_t>(slotIndex), txFramesList);
 #endif
     }
     mApp.getOverlayManager().notifyTxSetExternalized(value.txSetHash, txHashes);
@@ -969,8 +971,24 @@ HerderImpl::getSCPStateForPeer(uint32 ledgerSeq)
     std::vector<SCPEnvelope> envelopes;
     auto maxSlots = Herder::LEDGER_VALIDITY_BRACKET;
 
-    // Collect up to MAX_SLOTS_TO_SEND slots worth of envelopes
+    // SCP keeps the sequential recent slots plus the most recent checkpoint
+    // slot, which may be far behind them. Checkpoint messages are almost
+    // always outside MAXIMUM_LEDGER_CLOSETIME_DRIFT and are only accepted
+    // once the receiving node knows the network state from the recent
+    // messages, so if the oldest slot is such a detached checkpoint, send it
+    // *after* everything else.
+    std::vector<uint64> slots;
     getSCP().processSlotsAscendingFrom(ledgerSeq, [&](uint64 seq) {
+        slots.push_back(seq);
+        return true;
+    });
+    std::optional<uint64> detachedCheckpoint;
+    if (slots.size() >= 2 && slots[1] > slots[0] + 1)
+    {
+        detachedCheckpoint = slots[0];
+    }
+
+    auto appendSlot = [&](uint64 seq) {
         bool slotHadData = false;
         getSCP().processCurrentState(
             seq,
@@ -980,12 +998,31 @@ HerderImpl::getSCPStateForPeer(uint32 ledgerSeq)
                 return true; // continue
             },
             false);
-        if (slotHadData)
+        return slotHadData;
+    };
+
+    // Collect up to maxSlots slots worth of envelopes
+    for (auto seq : slots)
+    {
+        if (detachedCheckpoint && seq == *detachedCheckpoint)
+        {
+            continue; // appended below
+        }
+        if (appendSlot(seq))
         {
             --maxSlots;
         }
-        return maxSlots != 0;
-    });
+        if (maxSlots == 0)
+        {
+            break;
+        }
+    }
+    // After sending the most recent slots, if we have a detached checkpoint,
+    // send it at the end
+    if (detachedCheckpoint)
+    {
+        appendSlot(*detachedCheckpoint);
+    }
 
     return envelopes;
 }
@@ -1636,6 +1673,32 @@ HerderImpl::triggerNextLedger(uint32_t ledgerSeqToTrigger,
             classicTxs.push_back(txFrame);
         }
     }
+    // The mempool is fee-ordered and sequence-number-oblivious, so it can
+    // hand us several transactions from one source account (e.g. a chained
+    // pair). A tx set may only contain one tx per source account, so keep
+    // the lowest sequence number per account and let the others wait for a
+    // later ledger.
+    auto onePerSourceAccount = [](TxFrameList& txs) {
+        std::unordered_map<AccountID, size_t> firstBySource;
+        TxFrameList kept;
+        for (auto const& tx : txs)
+        {
+            auto [it, inserted] =
+                firstBySource.emplace(tx->getSourceID(), kept.size());
+            if (inserted)
+            {
+                kept.push_back(tx);
+            }
+            else if (tx->getSeqNum() < kept[it->second]->getSeqNum())
+            {
+                kept[it->second] = tx;
+            }
+        }
+        txs = std::move(kept);
+    };
+    onePerSourceAccount(classicTxs);
+    onePerSourceAccount(sorobanTxs);
+
     txPhases.emplace_back(std::move(classicTxs));
     if (supportsSoroban)
     {
@@ -1651,6 +1714,44 @@ HerderImpl::triggerNextLedger(uint32_t ledgerSeqToTrigger,
     CLOG_INFO(Herder, "Proposed TX set has {} transactions",
               proposedSet->sizeTxTotal());
 
+    // The mempool does no stateful validation, so it would keep handing us the
+    // transactions that just failed validation (stale sequence number, can't
+    // pay fee, expired, ...) on every nomination, crowding out valid ones.
+    // Drop them, except for transactions with a *future* sequence number:
+    // those are chained behind a pending transaction from the same account
+    // and become valid once it applies.
+    std::vector<Hash> invalidTxHashes;
+    if (!invalidTxPhases.empty())
+    {
+        CheckValidLedgerViewWrapper ledgerView(mApp);
+        for (auto const& phase : invalidTxPhases)
+        {
+            for (auto const& tx : phase)
+            {
+                auto acc = ledgerView.getAccount(tx->getSourceID());
+                if (acc &&
+                    tx->getSeqNum() > acc.current().data.account().seqNum + 1)
+                {
+                    continue;
+                }
+                CLOG_DEBUG(Herder,
+                           "Dropping invalid tx {} from mempool: seq {} "
+                           "(account seq {})",
+                           hexAbbrev(tx->getFullHash()), tx->getSeqNum(),
+                           acc ? acc.current().data.account().seqNum : -1);
+                invalidTxHashes.push_back(tx->getFullHash());
+            }
+        }
+    }
+    if (!invalidTxHashes.empty())
+    {
+        CLOG_DEBUG(Herder,
+                   "Removing {} transactions that failed tx set validation "
+                   "from the mempool",
+                   invalidTxHashes.size());
+        overlayMgr.removeTransactions(invalidTxHashes);
+    }
+
     if (!applicableProposedSet)
     {
         releaseAssert(!mApp.getConfig().FORCE_SCP);