rippled
Loading...
Searching...
No Matches
Database_test.cpp
1#include <test/jtx.h>
2#include <test/jtx/CheckMessageLogs.h>
3#include <test/jtx/envconfig.h>
4#include <test/nodestore/TestBase.h>
5#include <test/unit_test/SuiteJournal.h>
6
7#include <xrpl/beast/utility/temp_dir.h>
8#include <xrpl/nodestore/DummyScheduler.h>
9#include <xrpl/nodestore/Manager.h>
10#include <xrpl/rdb/DatabaseCon.h>
11
12namespace xrpl {
13
14namespace NodeStore {
15
16class Database_test : public TestBase
17{
19
20public:
21 Database_test() : journal_("Database_test", *this)
22 {
23 }
24
25 void
27 {
28 testcase("Config");
29
30 using namespace xrpl::test;
31 using namespace xrpl::test::jtx;
32
33 auto const integrityWarning =
34 "reducing the data integrity guarantees from the "
35 "default [sqlite] behavior is not recommended for "
36 "nodes storing large amounts of history, because of the "
37 "difficulty inherent in rebuilding corrupted data.";
38 {
39 // defaults
40 Env env(*this);
41
42 auto const s = setup_DatabaseCon(env.app().config());
43
44 if (BEAST_EXPECT(s.globalPragma->size() == 3))
45 {
46 BEAST_EXPECT(s.globalPragma->at(0) == "PRAGMA journal_mode=wal;");
47 BEAST_EXPECT(s.globalPragma->at(1) == "PRAGMA synchronous=normal;");
48 BEAST_EXPECT(s.globalPragma->at(2) == "PRAGMA temp_store=file;");
49 }
50 }
51 {
52 // High safety level
54
55 bool found = false;
56 Env env = [&]() {
57 auto p = test::jtx::envconfig();
58 {
59 auto& section = p->section("sqlite");
60 section.set("safety_level", "high");
61 }
62 p->LEDGER_HISTORY = 100'000'000;
63
64 return Env(
65 *this,
66 std::move(p),
67 std::make_unique<CheckMessageLogs>(integrityWarning, &found),
69 }();
70
71 BEAST_EXPECT(!found);
72 auto const s = setup_DatabaseCon(env.app().config());
73 if (BEAST_EXPECT(s.globalPragma->size() == 3))
74 {
75 BEAST_EXPECT(s.globalPragma->at(0) == "PRAGMA journal_mode=wal;");
76 BEAST_EXPECT(s.globalPragma->at(1) == "PRAGMA synchronous=normal;");
77 BEAST_EXPECT(s.globalPragma->at(2) == "PRAGMA temp_store=file;");
78 }
79 }
80 {
81 // Low safety level
83
84 bool found = false;
85 Env env = [&]() {
86 auto p = test::jtx::envconfig();
87 {
88 auto& section = p->section("sqlite");
89 section.set("safety_level", "low");
90 }
91 p->LEDGER_HISTORY = 100'000'000;
92
93 return Env(
94 *this,
95 std::move(p),
96 std::make_unique<CheckMessageLogs>(integrityWarning, &found),
98 }();
99
100 BEAST_EXPECT(found);
101 auto const s = setup_DatabaseCon(env.app().config());
102 if (BEAST_EXPECT(s.globalPragma->size() == 3))
103 {
104 BEAST_EXPECT(s.globalPragma->at(0) == "PRAGMA journal_mode=memory;");
105 BEAST_EXPECT(s.globalPragma->at(1) == "PRAGMA synchronous=off;");
106 BEAST_EXPECT(s.globalPragma->at(2) == "PRAGMA temp_store=memory;");
107 }
108 }
109 {
110 // Override individual settings
112
113 bool found = false;
114 Env env = [&]() {
115 auto p = test::jtx::envconfig();
116 {
117 auto& section = p->section("sqlite");
118 section.set("journal_mode", "off");
119 section.set("synchronous", "extra");
120 section.set("temp_store", "default");
121 }
122
123 return Env(
124 *this,
125 std::move(p),
126 std::make_unique<CheckMessageLogs>(integrityWarning, &found),
128 }();
129
130 // No warning, even though higher risk settings were used because
131 // LEDGER_HISTORY is small
132 BEAST_EXPECT(!found);
133 auto const s = setup_DatabaseCon(env.app().config());
134 if (BEAST_EXPECT(s.globalPragma->size() == 3))
135 {
136 BEAST_EXPECT(s.globalPragma->at(0) == "PRAGMA journal_mode=off;");
137 BEAST_EXPECT(s.globalPragma->at(1) == "PRAGMA synchronous=extra;");
138 BEAST_EXPECT(s.globalPragma->at(2) == "PRAGMA temp_store=default;");
139 }
140 }
141 {
142 // Override individual settings with large history
144
145 bool found = false;
146 Env env = [&]() {
147 auto p = test::jtx::envconfig();
148 {
149 auto& section = p->section("sqlite");
150 section.set("journal_mode", "off");
151 section.set("synchronous", "extra");
152 section.set("temp_store", "default");
153 }
154 p->LEDGER_HISTORY = 50'000'000;
155
156 return Env(
157 *this,
158 std::move(p),
159 std::make_unique<CheckMessageLogs>(integrityWarning, &found),
161 }();
162
163 // No warning, even though higher risk settings were used because
164 // LEDGER_HISTORY is small
165 BEAST_EXPECT(found);
166 auto const s = setup_DatabaseCon(env.app().config());
167 if (BEAST_EXPECT(s.globalPragma->size() == 3))
168 {
169 BEAST_EXPECT(s.globalPragma->at(0) == "PRAGMA journal_mode=off;");
170 BEAST_EXPECT(s.globalPragma->at(1) == "PRAGMA synchronous=extra;");
171 BEAST_EXPECT(s.globalPragma->at(2) == "PRAGMA temp_store=default;");
172 }
173 }
174 {
175 // Error: Mix safety_level and individual settings
177 auto const expected =
178 "Failed to initialize SQL databases: "
179 "Configuration file may not define both \"safety_level\" and "
180 "\"journal_mode\"";
181 bool found = false;
182
183 auto p = test::jtx::envconfig();
184 {
185 auto& section = p->section("sqlite");
186 section.set("safety_level", "low");
187 section.set("journal_mode", "off");
188 section.set("synchronous", "extra");
189 section.set("temp_store", "default");
190 }
191
192 try
193 {
194 Env env(
195 *this,
196 std::move(p),
197 std::make_unique<CheckMessageLogs>(expected, &found),
199 fail();
200 }
201 catch (...)
202 {
203 BEAST_EXPECT(found);
204 }
205 }
206 {
207 // Error: Mix safety_level and one setting (gotta catch 'em all)
209 auto const expected =
210 "Failed to initialize SQL databases: Configuration file may "
211 "not define both \"safety_level\" and \"journal_mode\"";
212 bool found = false;
213
214 auto p = test::jtx::envconfig();
215 {
216 auto& section = p->section("sqlite");
217 section.set("safety_level", "high");
218 section.set("journal_mode", "off");
219 }
220
221 try
222 {
223 Env env(
224 *this,
225 std::move(p),
226 std::make_unique<CheckMessageLogs>(expected, &found),
228 fail();
229 }
230 catch (...)
231 {
232 BEAST_EXPECT(found);
233 }
234 }
235 {
236 // Error: Mix safety_level and one setting (gotta catch 'em all)
238 auto const expected =
239 "Failed to initialize SQL databases: Configuration file may "
240 "not define both \"safety_level\" and \"synchronous\"";
241 bool found = false;
242
243 auto p = test::jtx::envconfig();
244 {
245 auto& section = p->section("sqlite");
246 section.set("safety_level", "low");
247 section.set("synchronous", "extra");
248 }
249
250 try
251 {
252 Env env(
253 *this,
254 std::move(p),
255 std::make_unique<CheckMessageLogs>(expected, &found),
257 fail();
258 }
259 catch (...)
260 {
261 BEAST_EXPECT(found);
262 }
263 }
264 {
265 // Error: Mix safety_level and one setting (gotta catch 'em all)
267 auto const expected =
268 "Failed to initialize SQL databases: Configuration file may "
269 "not define both \"safety_level\" and \"temp_store\"";
270 bool found = false;
271
272 auto p = test::jtx::envconfig();
273 {
274 auto& section = p->section("sqlite");
275 section.set("safety_level", "high");
276 section.set("temp_store", "default");
277 }
278
279 try
280 {
281 Env env(
282 *this,
283 std::move(p),
284 std::make_unique<CheckMessageLogs>(expected, &found),
286 fail();
287 }
288 catch (...)
289 {
290 BEAST_EXPECT(found);
291 }
292 }
293 {
294 // Error: Invalid value
296 auto const expected =
297 "Failed to initialize SQL databases: Invalid safety_level "
298 "value: slow";
299 bool found = false;
300
301 auto p = test::jtx::envconfig();
302 {
303 auto& section = p->section("sqlite");
304 section.set("safety_level", "slow");
305 }
306
307 try
308 {
309 Env env(
310 *this,
311 std::move(p),
312 std::make_unique<CheckMessageLogs>(expected, &found),
314 fail();
315 }
316 catch (...)
317 {
318 BEAST_EXPECT(found);
319 }
320 }
321 {
322 // Error: Invalid value
324 auto const expected =
325 "Failed to initialize SQL databases: Invalid journal_mode "
326 "value: fast";
327 bool found = false;
328
329 auto p = test::jtx::envconfig();
330 {
331 auto& section = p->section("sqlite");
332 section.set("journal_mode", "fast");
333 }
334
335 try
336 {
337 Env env(
338 *this,
339 std::move(p),
340 std::make_unique<CheckMessageLogs>(expected, &found),
342 fail();
343 }
344 catch (...)
345 {
346 BEAST_EXPECT(found);
347 }
348 }
349 {
350 // Error: Invalid value
352 auto const expected =
353 "Failed to initialize SQL databases: Invalid synchronous "
354 "value: instant";
355 bool found = false;
356
357 auto p = test::jtx::envconfig();
358 {
359 auto& section = p->section("sqlite");
360 section.set("synchronous", "instant");
361 }
362
363 try
364 {
365 Env env(
366 *this,
367 std::move(p),
368 std::make_unique<CheckMessageLogs>(expected, &found),
370 fail();
371 }
372 catch (...)
373 {
374 BEAST_EXPECT(found);
375 }
376 }
377 {
378 // Error: Invalid value
380 auto const expected =
381 "Failed to initialize SQL databases: Invalid temp_store "
382 "value: network";
383 bool found = false;
384
385 auto p = test::jtx::envconfig();
386 {
387 auto& section = p->section("sqlite");
388 section.set("temp_store", "network");
389 }
390
391 try
392 {
393 Env env(
394 *this,
395 std::move(p),
396 std::make_unique<CheckMessageLogs>(expected, &found),
398 fail();
399 }
400 catch (...)
401 {
402 BEAST_EXPECT(found);
403 }
404 }
405 {
406 // N/A: Default values
407 Env env(*this);
408 auto const s = setup_DatabaseCon(env.app().config());
409 if (BEAST_EXPECT(s.txPragma.size() == 4))
410 {
411 BEAST_EXPECT(s.txPragma.at(0) == "PRAGMA page_size=4096;");
412 BEAST_EXPECT(s.txPragma.at(1) == "PRAGMA journal_size_limit=1582080;");
413 BEAST_EXPECT(s.txPragma.at(2) == "PRAGMA max_page_count=4294967294;");
414 BEAST_EXPECT(s.txPragma.at(3) == "PRAGMA mmap_size=17179869184;");
415 }
416 }
417 {
418 // Success: Valid values
419 Env env = [&]() {
420 auto p = test::jtx::envconfig();
421 {
422 auto& section = p->section("sqlite");
423 section.set("page_size", "512");
424 section.set("journal_size_limit", "2582080");
425 }
426 return Env(*this, std::move(p));
427 }();
428 auto const s = setup_DatabaseCon(env.app().config());
429 if (BEAST_EXPECT(s.txPragma.size() == 4))
430 {
431 BEAST_EXPECT(s.txPragma.at(0) == "PRAGMA page_size=512;");
432 BEAST_EXPECT(s.txPragma.at(1) == "PRAGMA journal_size_limit=2582080;");
433 BEAST_EXPECT(s.txPragma.at(2) == "PRAGMA max_page_count=4294967294;");
434 BEAST_EXPECT(s.txPragma.at(3) == "PRAGMA mmap_size=17179869184;");
435 }
436 }
437 {
438 // Error: Invalid values
439 auto const expected = "Invalid page_size. Must be between 512 and 65536.";
440 bool found = false;
441 auto p = test::jtx::envconfig();
442 {
443 auto& section = p->section("sqlite");
444 section.set("page_size", "256");
445 }
446 try
447 {
448 Env env(
449 *this,
450 std::move(p),
451 std::make_unique<CheckMessageLogs>(expected, &found),
453 fail();
454 }
455 catch (...)
456 {
457 BEAST_EXPECT(found);
458 }
459 }
460 {
461 // Error: Invalid values
462 auto const expected = "Invalid page_size. Must be between 512 and 65536.";
463 bool found = false;
464 auto p = test::jtx::envconfig();
465 {
466 auto& section = p->section("sqlite");
467 section.set("page_size", "131072");
468 }
469 try
470 {
471 Env env(
472 *this,
473 std::move(p),
474 std::make_unique<CheckMessageLogs>(expected, &found),
476 fail();
477 }
478 catch (...)
479 {
480 BEAST_EXPECT(found);
481 }
482 }
483 {
484 // Error: Invalid values
485 auto const expected = "Invalid page_size. Must be a power of 2.";
486 bool found = false;
487 auto p = test::jtx::envconfig();
488 {
489 auto& section = p->section("sqlite");
490 section.set("page_size", "513");
491 }
492 try
493 {
494 Env env(
495 *this,
496 std::move(p),
497 std::make_unique<CheckMessageLogs>(expected, &found),
499 fail();
500 }
501 catch (...)
502 {
503 BEAST_EXPECT(found);
504 }
505 }
506 }
507
508 //--------------------------------------------------------------------------
509
510 void
511 testImport(std::string const& destBackendType, std::string const& srcBackendType, std::int64_t seedValue)
512 {
513 DummyScheduler scheduler;
514
515 beast::temp_dir node_db;
516 Section srcParams;
517 srcParams.set("type", srcBackendType);
518 srcParams.set("path", node_db.path());
519
520 // Create a batch
522
523 // Write to source db
524 {
526 Manager::instance().make_Database(megabytes(4), scheduler, 2, srcParams, journal_);
527 storeBatch(*src, batch);
528 }
529
530 Batch copy;
531
532 {
533 // Re-open the db
535 Manager::instance().make_Database(megabytes(4), scheduler, 2, srcParams, journal_);
536
537 // Set up the destination database
538 beast::temp_dir dest_db;
539 Section destParams;
540 destParams.set("type", destBackendType);
541 destParams.set("path", dest_db.path());
542
544 Manager::instance().make_Database(megabytes(4), scheduler, 2, destParams, journal_);
545
546 testcase("import into '" + destBackendType + "' from '" + srcBackendType + "'");
547
548 // Do the import
549 dest->importDatabase(*src);
550
551 // Get the results of the import
552 fetchCopyOfBatch(*dest, &copy, batch);
553 }
554
555 // Canonicalize the source and destination batches
556 std::sort(batch.begin(), batch.end(), LessThan{});
557 std::sort(copy.begin(), copy.end(), LessThan{});
558 BEAST_EXPECT(areBatchesEqual(batch, copy));
559 }
560
561 //--------------------------------------------------------------------------
562
563 void
565 std::string const& type,
566 bool const testPersistence,
567 std::int64_t const seedValue,
568 int numObjsToTest = 2000)
569 {
570 DummyScheduler scheduler;
571
572 std::string s = "NodeStore backend '" + type + "'";
573
574 testcase(s);
575
576 beast::temp_dir node_db;
577 Section nodeParams;
578 nodeParams.set("type", type);
579 nodeParams.set("path", node_db.path());
580
581 beast::xor_shift_engine rng(seedValue);
582
583 // Create a batch
584 auto batch = createPredictableBatch(numObjsToTest, rng());
585
586 {
587 // Open the database
589 Manager::instance().make_Database(megabytes(4), scheduler, 2, nodeParams, journal_);
590
591 // Write the batch
592 storeBatch(*db, batch);
593
594 {
595 // Read it back in
596 Batch copy;
597 fetchCopyOfBatch(*db, &copy, batch);
598 BEAST_EXPECT(areBatchesEqual(batch, copy));
599 }
600
601 {
602 // Reorder and read the copy again
603 std::shuffle(batch.begin(), batch.end(), rng);
604 Batch copy;
605 fetchCopyOfBatch(*db, &copy, batch);
606 BEAST_EXPECT(areBatchesEqual(batch, copy));
607 }
608 }
609
610 if (testPersistence)
611 {
612 // Re-open the database without the ephemeral DB
614 Manager::instance().make_Database(megabytes(4), scheduler, 2, nodeParams, journal_);
615
616 // Read it back in
617 Batch copy;
618 fetchCopyOfBatch(*db, &copy, batch);
619
620 // Canonicalize the source and destination batches
621 std::sort(batch.begin(), batch.end(), LessThan{});
622 std::sort(copy.begin(), copy.end(), LessThan{});
623 BEAST_EXPECT(areBatchesEqual(batch, copy));
624 }
625
626 if (type == "memory")
627 {
628 // Verify default earliest ledger sequence
629 {
631 Manager::instance().make_Database(megabytes(4), scheduler, 2, nodeParams, journal_);
632 BEAST_EXPECT(db->earliestLedgerSeq() == XRP_LEDGER_EARLIEST_SEQ);
633 }
634
635 // Set an invalid earliest ledger sequence
636 try
637 {
638 nodeParams.set("earliest_seq", "0");
640 Manager::instance().make_Database(megabytes(4), scheduler, 2, nodeParams, journal_);
641 }
642 catch (std::runtime_error const& e)
643 {
644 BEAST_EXPECT(std::strcmp(e.what(), "Invalid earliest_seq") == 0);
645 }
646
647 {
648 // Set a valid earliest ledger sequence
649 nodeParams.set("earliest_seq", "1");
651 Manager::instance().make_Database(megabytes(4), scheduler, 2, nodeParams, journal_);
652
653 // Verify database uses the earliest ledger sequence setting
654 BEAST_EXPECT(db->earliestLedgerSeq() == 1);
655 }
656
657 // Create another database that attempts to set the value again
658 try
659 {
660 // Set to default earliest ledger sequence
661 nodeParams.set("earliest_seq", std::to_string(XRP_LEDGER_EARLIEST_SEQ));
663 Manager::instance().make_Database(megabytes(4), scheduler, 2, nodeParams, journal_);
664 }
665 catch (std::runtime_error const& e)
666 {
667 BEAST_EXPECT(std::strcmp(e.what(), "earliest_seq set more than once") == 0);
668 }
669 }
670 }
671
672 //--------------------------------------------------------------------------
673
674 void
675 run() override
676 {
677 std::int64_t const seedValue = 50;
678
679 testConfig();
680
681 testNodeStore("memory", false, seedValue);
682
683 // Persistent backend tests
684 {
685 testNodeStore("nudb", true, seedValue);
686
687#if XRPL_ROCKSDB_AVAILABLE
688 testNodeStore("rocksdb", true, seedValue);
689#endif
690 }
691
692 // Import tests
693 {
694 testImport("nudb", "nudb", seedValue);
695
696#if XRPL_ROCKSDB_AVAILABLE
697 testImport("rocksdb", "rocksdb", seedValue);
698#endif
699
700#if XRPL_ENABLE_SQLITE_BACKEND_TESTS
701 testImport("sqlite", "sqlite", seedValue);
702#endif
703 }
704 }
705};
706
707BEAST_DEFINE_TESTSUITE(Database, nodestore, xrpl);
708
709} // namespace NodeStore
710} // namespace xrpl
RAII temporary directory.
Definition temp_dir.h:15
std::string path() const
Get the native path for the temporary directory.
Definition temp_dir.h:47
testcase_t testcase
Memberspace for declaring test cases.
Definition suite.h:147
void fail(String const &reason, char const *file, int line)
Record a failure.
Definition suite.h:516
virtual Config & config()=0
void run() override
Runs the suite.
void testImport(std::string const &destBackendType, std::string const &srcBackendType, std::int64_t seedValue)
void testNodeStore(std::string const &type, bool const testPersistence, std::int64_t const seedValue, int numObjsToTest=2000)
Persistency layer for NodeObject.
Definition Database.h:31
Simple NodeStore Scheduler that just performs the tasks synchronously.
static Manager & instance()
Returns the instance of the manager singleton.
virtual std::unique_ptr< Database > make_Database(std::size_t burstSize, Scheduler &scheduler, int readThreads, Section const &backendParameters, beast::Journal journal)=0
Construct a NodeStore database.
static bool areBatchesEqual(Batch const &lhs, Batch const &rhs)
Definition TestBase.h:96
void fetchCopyOfBatch(Backend &backend, Batch *pCopy, Batch const &batch)
Definition TestBase.h:131
static int const numObjectsToTest
Definition TestBase.h:52
void storeBatch(Backend &backend, Batch const &batch)
Definition TestBase.h:121
static Batch createPredictableBatch(int numObjects, std::uint64_t seed)
Definition TestBase.h:57
Holds a collection of configuration values.
Definition BasicConfig.h:24
void set(std::string const &key, std::string const &value)
Set a key/value pair.
A transaction testing environment.
Definition Env.h:119
Application & app()
Definition Env.h:251
T is_same_v
std::unique_ptr< Config > envconfig()
creates and initializes a default configuration for jtx::Env
Definition envconfig.h:34
Use hash_* containers for keys that do not need a cryptographically secure hashing algorithm.
Definition algorithm.h:5
constexpr auto megabytes(T value) noexcept
DatabaseCon::Setup setup_DatabaseCon(Config const &c, std::optional< beast::Journal > j=std::nullopt)
Definition Config.cpp:1043
static constexpr std::uint32_t XRP_LEDGER_EARLIEST_SEQ
The XRP ledger network's earliest allowed sequence.
T shuffle(T... args)
T sort(T... args)
T strcmp(T... args)
static std::unique_ptr< std::vector< std::string > const > globalPragma
Definition DatabaseCon.h:87
Binary function that satisfies the strict-weak-ordering requirement.
Definition TestBase.h:27
T to_string(T... args)
T what(T... args)