Merge v25.2.0 and v25.2.1 into master (#5190)

Author: marta-lokhova

.gitmodules

@@ -46,6 +46,7 @@
 [submodule "src/rust/soroban/p25"]
 	path = src/rust/soroban/p25
 	url = https://github.com/stellar/rs-soroban-env.git
+	branch = internal/v25.2.0
 [submodule "src/rust/soroban/p26"]
 	path = src/rust/soroban/p26
 	url = https://github.com/stellar/rs-soroban-env.git

docs/software/commands.md

@@ -252,6 +252,28 @@ Most commands return their results in JSON format.
 * **bans**
   List current active bans
 
+* **banaccounts**
+  Manages the persistent list of banned accounts. Banned accounts are stored in
+  the database and survive restarts. Any transaction whose source account,
+  operation source account, fee-bump fee source, or (for Soroban transactions)
+  write footprint account entry matches a banned address will be rejected from
+  the transaction queue.
+  * `banaccounts`<br>
+    Lists the currently banned account addresses as a JSON array.<br>
+  * `banaccounts?accountids=G_ADDRESS1,G_ADDRESS2,...`<br>
+    Adds the specified addresses to the persistent ban list. Existing bans are
+    preserved (additive).<br>
+
+  Note: The `FILTERED_G_ADDRESSES` configuration option is deprecated. Any
+  addresses configured there will be automatically migrated to the persistent
+  ban list on startup.
+
+* **unbanaccounts**
+  * `unbanaccounts`<br>
+    Clears all banned accounts.<br>
+  * `unbanaccounts?accountids=G_ADDRESS1,G_ADDRESS2,...`<br>
+    Removes the specified addresses from the persistent ban list.<br>
+
 * **checkdb**
   Triggers the instance to perform a background check of the database's state.
 
@@ -337,6 +359,12 @@ Most commands return their results in JSON format.
         The network is under high load and the fee is too low.
     * "FILTERED" - transaction rejected because it contains an operation type that Stellar Core filters out. See Stellar Core configuration `EXCLUDE_TRANSACTIONS_CONTAINING_OPERATION_TYPE` for more details.
 
+  Optional parameters:
+    * `force=true` - bypasses banned account filtering (see `banaccounts`),
+      allowing the transaction into the mempool even if its source account or
+      fee source is on the ban list. Other filtering (operation type, Soroban
+      key filtering) still applies. Example: `tx?blob=Base64&force=true`
+
 * **upgrades**
   * `upgrades?mode=get`<br>
     Retrieves the currently configured upgrade settings.<br>

src/catchup/ApplyCheckpointWork.cpp

@@ -360,10 +360,19 @@ ApplyCheckpointWork::onRun()
         openInputFiles();
     }
 
-    auto lcd = getNextLedgerCloseData();
-    if (!lcd)
+    std::shared_ptr<LedgerCloseData> lcd;
+    try
     {
-        return State::WORK_RUNNING;
+        lcd = getNextLedgerCloseData();
+        if (!lcd)
+        {
+            return State::WORK_RUNNING;
+        }
+    }
+    catch (std::runtime_error const& e)
+    {
+        CLOG_ERROR(History, "ApplyCheckpointWork failed: {}", e.what());
+        return State::WORK_FAILURE;
     }
 
     auto applyLedger = std::make_shared<ApplyLedgerWork>(mApp, *lcd);

src/catchup/DownloadApplyTxsWork.cpp

@@ -24,7 +24,7 @@ DownloadApplyTxsWork::DownloadApplyTxsWork(
     Application& app, TmpDir const& downloadDir, LedgerRange const& range,
     LedgerHeaderHistoryEntry& lastApplied, bool waitForPublish,
     std::shared_ptr<HistoryArchive> archive)
-    : BatchWork(app, "download-apply-ledgers")
+    : BatchWork(app, "download-apply-ledgers", BasicWork::RETRY_A_FEW)
     , mRange(range)
     , mDownloadDir(downloadDir)
     , mLastApplied(lastApplied)
@@ -223,7 +223,7 @@ void
 DownloadApplyTxsWork::resetIter()
 {
     mCheckpointToQueue = HistoryManager::checkpointContainingLedger(
-        mRange.mFirst, mApp.getConfig());
+        mApp.getLedgerManager().getLastClosedLedgerNum() + 1, mApp.getConfig());
     mLastYieldedWork.reset();
     mLastApplied = mApp.getLedgerManager().getLastClosedLedgerHeader();
 }

src/catchup/VerifyLedgerChainWork.cpp

@@ -200,6 +200,13 @@ VerifyLedgerChainWork::verifyHistoryOfSingleCheckpoint()
         {
             return HistoryManager::VERIFY_STATUS_ERR_BAD_LEDGER_VERSION;
         }
+        catch (std::runtime_error const& e)
+        {
+            CLOG_ERROR(History,
+                       "Failed when verifying ledger chain with error {}",
+                       e.what());
+            return HistoryManager::VERIFY_STATUS_ERR_CORRUPT_HEADER;
+        }
 
         if (curr.header.ledgerVersion >
             mApp.getConfig().LEDGER_PROTOCOL_VERSION)
@@ -546,6 +553,12 @@ VerifyLedgerChainWork::onRun()
         CLOG_ERROR(History, "{}", POSSIBLY_CORRUPTED_HISTORY);
         mApp.getLedgerApplyManager().ledgerChainsVerificationFailed();
         return BasicWork::State::WORK_FAILURE;
+    case HistoryManager::VERIFY_STATUS_ERR_CORRUPT_HEADER:
+        CLOG_ERROR(History, "Catchup material failed verification - "
+                            "corrupted header, propagating failure");
+        CLOG_ERROR(History, "{}", POSSIBLY_CORRUPTED_HISTORY);
+        mApp.getLedgerApplyManager().ledgerChainsVerificationFailed();
+        return BasicWork::State::WORK_FAILURE;
     default:
         releaseAssert(false);
         throw std::runtime_error("unexpected VerifyLedgerChainWork state");

src/database/Database.cpp

@@ -7,6 +7,7 @@
 #include "database/DatabaseConnectionString.h"
 #include "database/DatabaseTypeSpecificOperation.h"
 #include "main/Application.h"
+#include "main/BannedAccountsPersistor.h"
 #include "main/Config.h"
 #include "overlay/StellarXDR.h"
 #include "util/Decoder.h"
@@ -301,15 +302,20 @@ Database::applyMiscSchemaUpgrade(unsigned long vers)
         BanManager::maybeDropAndCreateNew(mMiscSession);
         // Copy contents from the main DB.
         populateMiscDatabase();
+        tx.commit();
+        // Detach the source database (attached by populateMiscDatabase)
+        // _after_ commit to avoid "database is locked errors". If schema
+        // version is already the most recent, DETACH is a no-op.
+        getRawMiscSession() << "DETACH DATABASE source_db";
+        return;
+    case 2:
+        // Add banned accounts table for persistent account filtering.
+        BannedAccountsPersistor::maybeDropAndCreateNew(mMiscSession.session());
         break;
     default:
         throw std::runtime_error("Unknown DB schema version");
     }
     tx.commit();
-
-    // Detach the source database _after_ commit to avoid "database is locked
-    // errors". If schema version is already the most recent, DETACH is a no-op.
-    getRawMiscSession() << "DETACH DATABASE source_db";
 }
 
 void
@@ -396,8 +402,18 @@ Database::applySchemaUpgrade(unsigned long vers)
         }
         break;
     case 27:
+        // Add banned accounts table for persistent account filtering.
+        // For SQLite-on-disk this is handled by misc schema upgrade v2;
+        // for Postgres and in-memory SQLite, create in the main DB.
+        if (!canUseMiscDB())
+        {
+            BannedAccountsPersistor::maybeDropAndCreateNew(getRawSession());
+        }
+        break;
+    case 28:
         migrateLedgerHeadersToStoreState(*this);
         break;
+
     default:
         throw std::runtime_error("Unknown DB schema version");
     }

src/database/Database.h

@@ -29,11 +29,11 @@ class Application;
 
 // smallest schema version supported
 static constexpr unsigned long MIN_SCHEMA_VERSION = 25;
-static constexpr unsigned long SCHEMA_VERSION = 27;
+static constexpr unsigned long SCHEMA_VERSION = 28;
 static constexpr unsigned long FIRST_MAIN_VERSION_WITH_MISC = 26;
 // Misc schema version 0 means no misc table exists yet
 static constexpr unsigned long MIN_MISC_SCHEMA_VERSION = 0;
-static constexpr unsigned long MISC_SCHEMA_VERSION = 1;
+static constexpr unsigned long MISC_SCHEMA_VERSION = 2;
 
 /**
  * Helper class for borrowing a SOCI prepared statement handle into a local

src/database/test/DatabaseTests.cpp

@@ -5,12 +5,14 @@
 #include "util/asio.h"
 #include "crypto/Hex.h"
 #include "crypto/KeyUtils.h"
+#include "crypto/SecretKey.h"
 #include "database/Database.h"
 #include "ledger/LedgerHeaderUtils.h"
 #include "ledger/LedgerTxn.h"
 #include "ledger/test/LedgerTestUtils.h"
 #include "lib/util/stdrandom.h"
 #include "main/Application.h"
+#include "main/BannedAccountsPersistor.h"
 #include "main/Config.h"
 #include "main/PersistentState.h"
 #include "overlay/BanManager.h"
@@ -26,6 +28,7 @@
 #include <algorithm>
 #include <optional>
 #include <random>
+#include <set>
 
 using namespace stellar;
 
@@ -671,3 +674,127 @@ TEST_CASE("ledgerheaders migration works correctly", "[db]")
         checkMigration(headerEncoded);
     }
 }
+
+#ifdef USE_POSTGRES
+TEST_CASE("schema parity across DB backends", "[db][schematest]")
+{
+    // This test verifies that after initialization, persistent SQLite (with
+    // main + misc DB split) and PostgreSQL end up with the exact same set of
+    // tables and the same row counts. It catches bugs where a table or schema
+    // upgrade is applied for one backend but not the other.
+
+    // Helper: get sorted table names from a SQLite session.
+    auto getSqliteTables = [](Database& db, SessionWrapper& session) {
+        std::set<std::string> tables;
+        std::string name;
+        auto prep = db.getPreparedStatement(
+            "SELECT name FROM sqlite_master WHERE type='table' "
+            "AND name NOT LIKE 'sqlite_%' ORDER BY name",
+            session);
+        auto& st = prep.statement();
+        st.exchange(soci::into(name));
+        st.define_and_bind();
+        st.execute(false);
+        while (st.fetch())
+        {
+            tables.insert(name);
+        }
+        return tables;
+    };
+
+    // Helper: count rows in a table.
+    auto countRows = [](soci::session& sess, std::string const& table) {
+        int count = 0;
+        soci::statement st = (sess.prepare << "SELECT COUNT(*) FROM " + table,
+                              soci::into(count));
+        st.execute(true);
+        return count;
+    };
+
+    // ---- Build the SQLite persistent reference (main + misc split) ----
+    TmpDir tmpDir("schema-parity-test");
+    Config cfg1 = getTestConfig(0, Config::TESTDB_BUCKET_DB_PERSISTENT);
+    cfg1.DATABASE = SecretValue{"sqlite3://" + tmpDir.getName() + "/test.db"};
+    // Use non-empty FILTERED_G_ADDRESSES to test migration as well
+    cfg1.FILTERED_G_ADDRESSES = {
+        "GBO7VUL2TOKPWFAWKATIW7K3QYA7WQ63VDY5CAE6AFUUX6BHZBOC2WXC",
+        "GATDQL767ZM2JQTBEG4BQ5WKOQNGAGWZDUN4GYT2UINPEU3RT2UAMVZH"};
+
+    VirtualClock clock1;
+    Application::pointer app1 = createTestApplication(clock1, cfg1);
+    auto& db1 = app1->getDatabase();
+
+    REQUIRE(db1.canUseMiscDB());
+    REQUIRE(db1.getMainDBSchemaVersion() == SCHEMA_VERSION);
+    REQUIRE(db1.getMiscDBSchemaVersion() == MISC_SCHEMA_VERSION);
+
+    // Union of main + misc tables is the full set
+    auto mainTables = getSqliteTables(db1, db1.getSession());
+    auto miscTables = getSqliteTables(db1, db1.getMiscSession());
+
+    // Main and misc must not overlap
+    for (auto const& t : mainTables)
+    {
+        INFO("Table in both main and misc: " << t);
+        REQUIRE(miscTables.count(t) == 0);
+    }
+
+    std::set<std::string> allSqliteTables = mainTables;
+    allSqliteTables.insert(miscTables.begin(), miscTables.end());
+
+    // ---- PostgreSQL: compare tables and row counts ----
+    Config cfg2 = getTestConfig(1, Config::TESTDB_POSTGRESQL);
+    cfg2.FILTERED_G_ADDRESSES = {
+        "GBO7VUL2TOKPWFAWKATIW7K3QYA7WQ63VDY5CAE6AFUUX6BHZBOC2WXC",
+        "GATDQL767ZM2JQTBEG4BQ5WKOQNGAGWZDUN4GYT2UINPEU3RT2UAMVZH"};
+
+    VirtualClock clock2;
+    Application::pointer app2 = createTestApplication(clock2, cfg2);
+    auto& db2 = app2->getDatabase();
+
+    REQUIRE_FALSE(db2.canUseMiscDB());
+    REQUIRE(db2.getMainDBSchemaVersion() == SCHEMA_VERSION);
+
+    // Get Postgres table names
+    std::set<std::string> pgTables;
+    {
+        std::string name;
+        soci::statement st =
+            (db2.getRawSession().prepare << "SELECT tablename FROM pg_tables "
+                                            "WHERE schemaname = 'public' "
+                                            "ORDER BY tablename",
+             soci::into(name));
+        st.execute(false);
+        while (st.fetch())
+        {
+            pgTables.insert(name);
+        }
+    }
+
+    // Must have the exact same tables
+    CHECK(pgTables == allSqliteTables);
+
+    // Verify every table has the same row count across both backends.
+    // slotstate has an extra row in SQLite misc DB for the misc schema
+    // version, which doesn't exist in Postgres (it uses storestate).
+    for (auto const& table : allSqliteTables)
+    {
+        // For SQLite, query the right session (main or misc)
+        auto& sqliteSess = miscTables.count(table) ? db1.getRawMiscSession()
+                                                   : db1.getRawSession();
+        int sqliteRows = countRows(sqliteSess, table);
+        int pgRows = countRows(db2.getRawSession(), table);
+
+        INFO("Table: " << table);
+        if (table == "slotstate")
+        {
+            // SQLite misc DB has one extra row for miscdatabaseschema
+            CHECK(sqliteRows == pgRows + 1);
+        }
+        else
+        {
+            CHECK(sqliteRows == pgRows);
+        }
+    }
+}
+#endif

src/herder/Herder.h

@@ -139,10 +139,11 @@ class Herder
     // generator, and therefore can skip certain expensive validity checks
     virtual TransactionQueue::AddResult
     recvTransaction(TransactionFrameBasePtr tx, bool submittedFromSelf,
-                    bool isLoadgenTx = false) = 0;
+                    bool force = false, bool isLoadgenTx = false) = 0;
 #else
     virtual TransactionQueue::AddResult
-    recvTransaction(TransactionFrameBasePtr tx, bool submittedFromSelf) = 0;
+    recvTransaction(TransactionFrameBasePtr tx, bool submittedFromSelf,
+                    bool force = false) = 0;
 #endif
     virtual void peerDoesntHave(stellar::MessageType type,
                                 uint256 const& itemID, Peer::pointer peer) = 0;
@@ -213,6 +214,9 @@ class Herder
     // gets the upgrades that are scheduled by this node
     virtual std::string getUpgradesJson() = 0;
 
+    // Override the filtered accounts at runtime using pre-parsed AccountIDs.
+    virtual void setFilteredAccounts(std::set<AccountID> const& accounts) = 0;
+
     virtual void forceSCPStateIntoSyncWithLastClosedLedger() = 0;
 
     // helper function to craft an SCPValue