Database_test.cpp

#include <test/jtx.h>
#include <test/jtx/CheckMessageLogs.h>
#include <test/jtx/envconfig.h>
#include <test/nodestore/TestBase.h>
#include <test/unit_test/SuiteJournal.h>
 
#include <xrpld/core/DatabaseCon.h>
 
#include <xrpl/beast/utility/temp_dir.h>
#include <xrpl/nodestore/DummyScheduler.h>
#include <xrpl/nodestore/Manager.h>
 
namespace ripple {
 
namespace NodeStore {
 
class Database_test : public TestBase
{
    test::SuiteJournal journal_;
 
public:
    Database_test() : journal_("Database_test", *this)
    {
    }
 
    void
    testConfig()
    {
        testcase("Config");
 
        using namespace ripple::test;
        using namespace ripple::test::jtx;
 
        auto const integrityWarning =
            "reducing the data integrity guarantees from the "
            "default [sqlite] behavior is not recommended for "
            "nodes storing large amounts of history, because of the "
            "difficulty inherent in rebuilding corrupted data.";
        {
            // defaults
            Env env(*this);
 
            auto const s = setup_DatabaseCon(env.app().config());
 
            if (BEAST_EXPECT(s.globalPragma->size() == 3))
            {
                BEAST_EXPECT(
                    s.globalPragma->at(0) == "PRAGMA journal_mode=wal;");
                BEAST_EXPECT(
                    s.globalPragma->at(1) == "PRAGMA synchronous=normal;");
                BEAST_EXPECT(
                    s.globalPragma->at(2) == "PRAGMA temp_store=file;");
            }
        }
        {
            // High safety level
            DatabaseCon::Setup::globalPragma.reset();
 
            bool found = false;
            Env env = [&]() {
                auto p = test::jtx::envconfig();
                {
                    auto& section = p->section("sqlite");
                    section.set("safety_level", "high");
                }
                p->LEDGER_HISTORY = 100'000'000;
 
                return Env(
                    *this,
                    std::move(p),
                    std::make_unique<CheckMessageLogs>(
                        integrityWarning, &found),
                    beast::severities::kWarning);
            }();
 
            BEAST_EXPECT(!found);
            auto const s = setup_DatabaseCon(env.app().config());
            if (BEAST_EXPECT(s.globalPragma->size() == 3))
            {
                BEAST_EXPECT(
                    s.globalPragma->at(0) == "PRAGMA journal_mode=wal;");
                BEAST_EXPECT(
                    s.globalPragma->at(1) == "PRAGMA synchronous=normal;");
                BEAST_EXPECT(
                    s.globalPragma->at(2) == "PRAGMA temp_store=file;");
            }
        }
        {
            // Low safety level
            DatabaseCon::Setup::globalPragma.reset();
 
            bool found = false;
            Env env = [&]() {
                auto p = test::jtx::envconfig();
                {
                    auto& section = p->section("sqlite");
                    section.set("safety_level", "low");
                }
                p->LEDGER_HISTORY = 100'000'000;
 
                return Env(
                    *this,
                    std::move(p),
                    std::make_unique<CheckMessageLogs>(
                        integrityWarning, &found),
                    beast::severities::kWarning);
            }();
 
            BEAST_EXPECT(found);
            auto const s = setup_DatabaseCon(env.app().config());
            if (BEAST_EXPECT(s.globalPragma->size() == 3))
            {
                BEAST_EXPECT(
                    s.globalPragma->at(0) == "PRAGMA journal_mode=memory;");
                BEAST_EXPECT(
                    s.globalPragma->at(1) == "PRAGMA synchronous=off;");
                BEAST_EXPECT(
                    s.globalPragma->at(2) == "PRAGMA temp_store=memory;");
            }
        }
        {
            // Override individual settings
            DatabaseCon::Setup::globalPragma.reset();
 
            bool found = false;
            Env env = [&]() {
                auto p = test::jtx::envconfig();
                {
                    auto& section = p->section("sqlite");
                    section.set("journal_mode", "off");
                    section.set("synchronous", "extra");
                    section.set("temp_store", "default");
                }
 
                return Env(
                    *this,
                    std::move(p),
                    std::make_unique<CheckMessageLogs>(
                        integrityWarning, &found),
                    beast::severities::kWarning);
            }();
 
            // No warning, even though higher risk settings were used because
            // LEDGER_HISTORY is small
            BEAST_EXPECT(!found);
            auto const s = setup_DatabaseCon(env.app().config());
            if (BEAST_EXPECT(s.globalPragma->size() == 3))
            {
                BEAST_EXPECT(
                    s.globalPragma->at(0) == "PRAGMA journal_mode=off;");
                BEAST_EXPECT(
                    s.globalPragma->at(1) == "PRAGMA synchronous=extra;");
                BEAST_EXPECT(
                    s.globalPragma->at(2) == "PRAGMA temp_store=default;");
            }
        }
        {
            // Override individual settings with large history
            DatabaseCon::Setup::globalPragma.reset();
 
            bool found = false;
            Env env = [&]() {
                auto p = test::jtx::envconfig();
                {
                    auto& section = p->section("sqlite");
                    section.set("journal_mode", "off");
                    section.set("synchronous", "extra");
                    section.set("temp_store", "default");
                }
                p->LEDGER_HISTORY = 50'000'000;
 
                return Env(
                    *this,
                    std::move(p),
                    std::make_unique<CheckMessageLogs>(
                        integrityWarning, &found),
                    beast::severities::kWarning);
            }();
 
            // No warning, even though higher risk settings were used because
            // LEDGER_HISTORY is small
            BEAST_EXPECT(found);
            auto const s = setup_DatabaseCon(env.app().config());
            if (BEAST_EXPECT(s.globalPragma->size() == 3))
            {
                BEAST_EXPECT(
                    s.globalPragma->at(0) == "PRAGMA journal_mode=off;");
                BEAST_EXPECT(
                    s.globalPragma->at(1) == "PRAGMA synchronous=extra;");
                BEAST_EXPECT(
                    s.globalPragma->at(2) == "PRAGMA temp_store=default;");
            }
        }
        {
            // Error: Mix safety_level and individual settings
            DatabaseCon::Setup::globalPragma.reset();
            auto const expected =
                "Failed to initialize SQL databases: "
                "Configuration file may not define both \"safety_level\" and "
                "\"journal_mode\"";
            bool found = false;
 
            auto p = test::jtx::envconfig();
            {
                auto& section = p->section("sqlite");
                section.set("safety_level", "low");
                section.set("journal_mode", "off");
                section.set("synchronous", "extra");
                section.set("temp_store", "default");
            }
 
            try
            {
                Env env(
                    *this,
                    std::move(p),
                    std::make_unique<CheckMessageLogs>(expected, &found),
                    beast::severities::kWarning);
                fail();
            }
            catch (...)
            {
                BEAST_EXPECT(found);
            }
        }
        {
            // Error: Mix safety_level and one setting (gotta catch 'em all)
            DatabaseCon::Setup::globalPragma.reset();
            auto const expected =
                "Failed to initialize SQL databases: Configuration file may "
                "not define both \"safety_level\" and \"journal_mode\"";
            bool found = false;
 
            auto p = test::jtx::envconfig();
            {
                auto& section = p->section("sqlite");
                section.set("safety_level", "high");
                section.set("journal_mode", "off");
            }
 
            try
            {
                Env env(
                    *this,
                    std::move(p),
                    std::make_unique<CheckMessageLogs>(expected, &found),
                    beast::severities::kWarning);
                fail();
            }
            catch (...)
            {
                BEAST_EXPECT(found);
            }
        }
        {
            // Error: Mix safety_level and one setting (gotta catch 'em all)
            DatabaseCon::Setup::globalPragma.reset();
            auto const expected =
                "Failed to initialize SQL databases: Configuration file may "
                "not define both \"safety_level\" and \"synchronous\"";
            bool found = false;
 
            auto p = test::jtx::envconfig();
            {
                auto& section = p->section("sqlite");
                section.set("safety_level", "low");
                section.set("synchronous", "extra");
            }
 
            try
            {
                Env env(
                    *this,
                    std::move(p),
                    std::make_unique<CheckMessageLogs>(expected, &found),
                    beast::severities::kWarning);
                fail();
            }
            catch (...)
            {
                BEAST_EXPECT(found);
            }
        }
        {
            // Error: Mix safety_level and one setting (gotta catch 'em all)
            DatabaseCon::Setup::globalPragma.reset();
            auto const expected =
                "Failed to initialize SQL databases: Configuration file may "
                "not define both \"safety_level\" and \"temp_store\"";
            bool found = false;
 
            auto p = test::jtx::envconfig();
            {
                auto& section = p->section("sqlite");
                section.set("safety_level", "high");
                section.set("temp_store", "default");
            }
 
            try
            {
                Env env(
                    *this,
                    std::move(p),
                    std::make_unique<CheckMessageLogs>(expected, &found),
                    beast::severities::kWarning);
                fail();
            }
            catch (...)
            {
                BEAST_EXPECT(found);
            }
        }
        {
            // Error: Invalid value
            DatabaseCon::Setup::globalPragma.reset();
            auto const expected =
                "Failed to initialize SQL databases: Invalid safety_level "
                "value: slow";
            bool found = false;
 
            auto p = test::jtx::envconfig();
            {
                auto& section = p->section("sqlite");
                section.set("safety_level", "slow");
            }
 
            try
            {
                Env env(
                    *this,
                    std::move(p),
                    std::make_unique<CheckMessageLogs>(expected, &found),
                    beast::severities::kWarning);
                fail();
            }
            catch (...)
            {
                BEAST_EXPECT(found);
            }
        }
        {
            // Error: Invalid value
            DatabaseCon::Setup::globalPragma.reset();
            auto const expected =
                "Failed to initialize SQL databases: Invalid journal_mode "
                "value: fast";
            bool found = false;
 
            auto p = test::jtx::envconfig();
            {
                auto& section = p->section("sqlite");
                section.set("journal_mode", "fast");
            }
 
            try
            {
                Env env(
                    *this,
                    std::move(p),
                    std::make_unique<CheckMessageLogs>(expected, &found),
                    beast::severities::kWarning);
                fail();
            }
            catch (...)
            {
                BEAST_EXPECT(found);
            }
        }
        {
            // Error: Invalid value
            DatabaseCon::Setup::globalPragma.reset();
            auto const expected =
                "Failed to initialize SQL databases: Invalid synchronous "
                "value: instant";
            bool found = false;
 
            auto p = test::jtx::envconfig();
            {
                auto& section = p->section("sqlite");
                section.set("synchronous", "instant");
            }
 
            try
            {
                Env env(
                    *this,
                    std::move(p),
                    std::make_unique<CheckMessageLogs>(expected, &found),
                    beast::severities::kWarning);
                fail();
            }
            catch (...)
            {
                BEAST_EXPECT(found);
            }
        }
        {
            // Error: Invalid value
            DatabaseCon::Setup::globalPragma.reset();
            auto const expected =
                "Failed to initialize SQL databases: Invalid temp_store "
                "value: network";
            bool found = false;
 
            auto p = test::jtx::envconfig();
            {
                auto& section = p->section("sqlite");
                section.set("temp_store", "network");
            }
 
            try
            {
                Env env(
                    *this,
                    std::move(p),
                    std::make_unique<CheckMessageLogs>(expected, &found),
                    beast::severities::kWarning);
                fail();
            }
            catch (...)
            {
                BEAST_EXPECT(found);
            }
        }
        {
            // N/A: Default values
            Env env(*this);
            auto const s = setup_DatabaseCon(env.app().config());
            if (BEAST_EXPECT(s.txPragma.size() == 4))
            {
                BEAST_EXPECT(s.txPragma.at(0) == "PRAGMA page_size=4096;");
                BEAST_EXPECT(
                    s.txPragma.at(1) == "PRAGMA journal_size_limit=1582080;");
                BEAST_EXPECT(
                    s.txPragma.at(2) == "PRAGMA max_page_count=4294967294;");
                BEAST_EXPECT(
                    s.txPragma.at(3) == "PRAGMA mmap_size=17179869184;");
            }
        }
        {
            // Success: Valid values
            Env env = [&]() {
                auto p = test::jtx::envconfig();
                {
                    auto& section = p->section("sqlite");
                    section.set("page_size", "512");
                    section.set("journal_size_limit", "2582080");
                }
                return Env(*this, std::move(p));
            }();
            auto const s = setup_DatabaseCon(env.app().config());
            if (BEAST_EXPECT(s.txPragma.size() == 4))
            {
                BEAST_EXPECT(s.txPragma.at(0) == "PRAGMA page_size=512;");
                BEAST_EXPECT(
                    s.txPragma.at(1) == "PRAGMA journal_size_limit=2582080;");
                BEAST_EXPECT(
                    s.txPragma.at(2) == "PRAGMA max_page_count=4294967294;");
                BEAST_EXPECT(
                    s.txPragma.at(3) == "PRAGMA mmap_size=17179869184;");
            }
        }
        {
            // Error: Invalid values
            auto const expected =
                "Invalid page_size. Must be between 512 and 65536.";
            bool found = false;
            auto p = test::jtx::envconfig();
            {
                auto& section = p->section("sqlite");
                section.set("page_size", "256");
            }
            try
            {
                Env env(
                    *this,
                    std::move(p),
                    std::make_unique<CheckMessageLogs>(expected, &found),
                    beast::severities::kWarning);
                fail();
            }
            catch (...)
            {
                BEAST_EXPECT(found);
            }
        }
        {
            // Error: Invalid values
            auto const expected =
                "Invalid page_size. Must be between 512 and 65536.";
            bool found = false;
            auto p = test::jtx::envconfig();
            {
                auto& section = p->section("sqlite");
                section.set("page_size", "131072");
            }
            try
            {
                Env env(
                    *this,
                    std::move(p),
                    std::make_unique<CheckMessageLogs>(expected, &found),
                    beast::severities::kWarning);
                fail();
            }
            catch (...)
            {
                BEAST_EXPECT(found);
            }
        }
        {
            // Error: Invalid values
            auto const expected = "Invalid page_size. Must be a power of 2.";
            bool found = false;
            auto p = test::jtx::envconfig();
            {
                auto& section = p->section("sqlite");
                section.set("page_size", "513");
            }
            try
            {
                Env env(
                    *this,
                    std::move(p),
                    std::make_unique<CheckMessageLogs>(expected, &found),
                    beast::severities::kWarning);
                fail();
            }
            catch (...)
            {
                BEAST_EXPECT(found);
            }
        }
    }
 
    //--------------------------------------------------------------------------
 
    void
    testImport(
        std::string const& destBackendType,
        std::string const& srcBackendType,
        std::int64_t seedValue)
    {
        DummyScheduler scheduler;
 
        beast::temp_dir node_db;
        Section srcParams;
        srcParams.set("type", srcBackendType);
        srcParams.set("path", node_db.path());
 
        // Create a batch
        auto batch = createPredictableBatch(numObjectsToTest, seedValue);
 
        // Write to source db
        {
            std::unique_ptr<Database> src = Manager::instance().make_Database(
                megabytes(4), scheduler, 2, srcParams, journal_);
            storeBatch(*src, batch);
        }
 
        Batch copy;
 
        {
            // Re-open the db
            std::unique_ptr<Database> src = Manager::instance().make_Database(
                megabytes(4), scheduler, 2, srcParams, journal_);
 
            // Set up the destination database
            beast::temp_dir dest_db;
            Section destParams;
            destParams.set("type", destBackendType);
            destParams.set("path", dest_db.path());
 
            std::unique_ptr<Database> dest = Manager::instance().make_Database(
                megabytes(4), scheduler, 2, destParams, journal_);
 
            testcase(
                "import into '" + destBackendType + "' from '" +
                srcBackendType + "'");
 
            // Do the import
            dest->importDatabase(*src);
 
            // Get the results of the import
            fetchCopyOfBatch(*dest, &copy, batch);
        }
 
        // Canonicalize the source and destination batches
        std::sort(batch.begin(), batch.end(), LessThan{});
        std::sort(copy.begin(), copy.end(), LessThan{});
        BEAST_EXPECT(areBatchesEqual(batch, copy));
    }
 
    //--------------------------------------------------------------------------
 
    void
    testNodeStore(
        std::string const& type,
        bool const testPersistence,
        std::int64_t const seedValue,
        int numObjsToTest = 2000)
    {
        DummyScheduler scheduler;
 
        std::string s = "NodeStore backend '" + type + "'";
 
        testcase(s);
 
        beast::temp_dir node_db;
        Section nodeParams;
        nodeParams.set("type", type);
        nodeParams.set("path", node_db.path());
 
        beast::xor_shift_engine rng(seedValue);
 
        // Create a batch
        auto batch = createPredictableBatch(numObjsToTest, rng());
 
        {
            // Open the database
            std::unique_ptr<Database> db = Manager::instance().make_Database(
                megabytes(4), scheduler, 2, nodeParams, journal_);
 
            // Write the batch
            storeBatch(*db, batch);
 
            {
                // Read it back in
                Batch copy;
                fetchCopyOfBatch(*db, &copy, batch);
                BEAST_EXPECT(areBatchesEqual(batch, copy));
            }
 
            {
                // Reorder and read the copy again
                std::shuffle(batch.begin(), batch.end(), rng);
                Batch copy;
                fetchCopyOfBatch(*db, &copy, batch);
                BEAST_EXPECT(areBatchesEqual(batch, copy));
            }
        }
 
        if (testPersistence)
        {
            // Re-open the database without the ephemeral DB
            std::unique_ptr<Database> db = Manager::instance().make_Database(
                megabytes(4), scheduler, 2, nodeParams, journal_);
 
            // Read it back in
            Batch copy;
            fetchCopyOfBatch(*db, &copy, batch);
 
            // Canonicalize the source and destination batches
            std::sort(batch.begin(), batch.end(), LessThan{});
            std::sort(copy.begin(), copy.end(), LessThan{});
            BEAST_EXPECT(areBatchesEqual(batch, copy));
        }
 
        if (type == "memory")
        {
            // Verify default earliest ledger sequence
            {
                std::unique_ptr<Database> db =
                    Manager::instance().make_Database(
                        megabytes(4), scheduler, 2, nodeParams, journal_);
                BEAST_EXPECT(
                    db->earliestLedgerSeq() == XRP_LEDGER_EARLIEST_SEQ);
            }
 
            // Set an invalid earliest ledger sequence
            try
            {
                nodeParams.set("earliest_seq", "0");
                std::unique_ptr<Database> db =
                    Manager::instance().make_Database(
                        megabytes(4), scheduler, 2, nodeParams, journal_);
            }
            catch (std::runtime_error const& e)
            {
                BEAST_EXPECT(
                    std::strcmp(e.what(), "Invalid earliest_seq") == 0);
            }
 
            {
                // Set a valid earliest ledger sequence
                nodeParams.set("earliest_seq", "1");
                std::unique_ptr<Database> db =
                    Manager::instance().make_Database(
                        megabytes(4), scheduler, 2, nodeParams, journal_);
 
                // Verify database uses the earliest ledger sequence setting
                BEAST_EXPECT(db->earliestLedgerSeq() == 1);
            }
 
            // Create another database that attempts to set the value again
            try
            {
                // Set to default earliest ledger sequence
                nodeParams.set(
                    "earliest_seq", std::to_string(XRP_LEDGER_EARLIEST_SEQ));
                std::unique_ptr<Database> db2 =
                    Manager::instance().make_Database(
                        megabytes(4), scheduler, 2, nodeParams, journal_);
            }
            catch (std::runtime_error const& e)
            {
                BEAST_EXPECT(
                    std::strcmp(e.what(), "earliest_seq set more than once") ==
                    0);
            }
        }
    }
 
    //--------------------------------------------------------------------------
 
    void
    run() override
    {
        std::int64_t const seedValue = 50;
 
        testConfig();
 
        testNodeStore("memory", false, seedValue);
 
        // Persistent backend tests
        {
            testNodeStore("nudb", true, seedValue);
 
#if XRPL_ROCKSDB_AVAILABLE
            testNodeStore("rocksdb", true, seedValue);
#endif
        }
 
        // Import tests
        {
            testImport("nudb", "nudb", seedValue);
 
#if XRPL_ROCKSDB_AVAILABLE
            testImport("rocksdb", "rocksdb", seedValue);
#endif
 
#if XRPL_ENABLE_SQLITE_BACKEND_TESTS
            testImport("sqlite", "sqlite", seedValue);
#endif
        }
    }
};
 
BEAST_DEFINE_TESTSUITE(Database, nodestore, ripple);
 
}  // namespace NodeStore
}  // namespace ripple