rippled
Loading...
Searching...
No Matches
Database_test.cpp
1#include <test/jtx.h>
2#include <test/jtx/CheckMessageLogs.h>
3#include <test/jtx/envconfig.h>
4#include <test/nodestore/TestBase.h>
5#include <test/unit_test/SuiteJournal.h>
6
7#include <xrpl/beast/utility/temp_dir.h>
8#include <xrpl/nodestore/DummyScheduler.h>
9#include <xrpl/nodestore/Manager.h>
10#include <xrpl/rdb/DatabaseCon.h>
11
12namespace xrpl {
13
14namespace NodeStore {
15
16class Database_test : public TestBase
17{
19
20public:
21 Database_test() : journal_("Database_test", *this)
22 {
23 }
24
25 void
27 {
28 testcase("Config");
29
30 using namespace xrpl::test;
31 using namespace xrpl::test::jtx;
32
33 auto const integrityWarning =
34 "reducing the data integrity guarantees from the "
35 "default [sqlite] behavior is not recommended for "
36 "nodes storing large amounts of history, because of the "
37 "difficulty inherent in rebuilding corrupted data.";
38 {
39 // defaults
40 Env env(*this);
41
42 auto const s = setup_DatabaseCon(env.app().config());
43
44 if (BEAST_EXPECT(s.globalPragma->size() == 3))
45 {
46 BEAST_EXPECT(s.globalPragma->at(0) == "PRAGMA journal_mode=wal;");
47 BEAST_EXPECT(s.globalPragma->at(1) == "PRAGMA synchronous=normal;");
48 BEAST_EXPECT(s.globalPragma->at(2) == "PRAGMA temp_store=file;");
49 }
50 }
51 {
52 // High safety level
54
55 bool found = false;
56 Env env = [&]() {
57 auto p = test::jtx::envconfig();
58 {
59 auto& section = p->section("sqlite");
60 section.set("safety_level", "high");
61 }
62 p->LEDGER_HISTORY = 100'000'000;
63
64 return Env(
65 *this,
66 std::move(p),
67 std::make_unique<CheckMessageLogs>(integrityWarning, &found),
69 }();
70
71 BEAST_EXPECT(!found);
72 auto const s = setup_DatabaseCon(env.app().config());
73 if (BEAST_EXPECT(s.globalPragma->size() == 3))
74 {
75 BEAST_EXPECT(s.globalPragma->at(0) == "PRAGMA journal_mode=wal;");
76 BEAST_EXPECT(s.globalPragma->at(1) == "PRAGMA synchronous=normal;");
77 BEAST_EXPECT(s.globalPragma->at(2) == "PRAGMA temp_store=file;");
78 }
79 }
80 {
81 // Low safety level
83
84 bool found = false;
85 Env env = [&]() {
86 auto p = test::jtx::envconfig();
87 {
88 auto& section = p->section("sqlite");
89 section.set("safety_level", "low");
90 }
91 p->LEDGER_HISTORY = 100'000'000;
92
93 return Env(
94 *this,
95 std::move(p),
96 std::make_unique<CheckMessageLogs>(integrityWarning, &found),
98 }();
99
100 BEAST_EXPECT(found);
101 auto const s = setup_DatabaseCon(env.app().config());
102 if (BEAST_EXPECT(s.globalPragma->size() == 3))
103 {
104 BEAST_EXPECT(s.globalPragma->at(0) == "PRAGMA journal_mode=memory;");
105 BEAST_EXPECT(s.globalPragma->at(1) == "PRAGMA synchronous=off;");
106 BEAST_EXPECT(s.globalPragma->at(2) == "PRAGMA temp_store=memory;");
107 }
108 }
109 {
110 // Override individual settings
112
113 bool found = false;
114 Env env = [&]() {
115 auto p = test::jtx::envconfig();
116 {
117 auto& section = p->section("sqlite");
118 section.set("journal_mode", "off");
119 section.set("synchronous", "extra");
120 section.set("temp_store", "default");
121 }
122
123 return Env(
124 *this,
125 std::move(p),
126 std::make_unique<CheckMessageLogs>(integrityWarning, &found),
128 }();
129
130 // No warning, even though higher risk settings were used because
131 // LEDGER_HISTORY is small
132 BEAST_EXPECT(!found);
133 auto const s = setup_DatabaseCon(env.app().config());
134 if (BEAST_EXPECT(s.globalPragma->size() == 3))
135 {
136 BEAST_EXPECT(s.globalPragma->at(0) == "PRAGMA journal_mode=off;");
137 BEAST_EXPECT(s.globalPragma->at(1) == "PRAGMA synchronous=extra;");
138 BEAST_EXPECT(s.globalPragma->at(2) == "PRAGMA temp_store=default;");
139 }
140 }
141 {
142 // Override individual settings with large history
144
145 bool found = false;
146 Env env = [&]() {
147 auto p = test::jtx::envconfig();
148 {
149 auto& section = p->section("sqlite");
150 section.set("journal_mode", "off");
151 section.set("synchronous", "extra");
152 section.set("temp_store", "default");
153 }
154 p->LEDGER_HISTORY = 50'000'000;
155
156 return Env(
157 *this,
158 std::move(p),
159 std::make_unique<CheckMessageLogs>(integrityWarning, &found),
161 }();
162
163 // No warning, even though higher risk settings were used because
164 // LEDGER_HISTORY is small
165 BEAST_EXPECT(found);
166 auto const s = setup_DatabaseCon(env.app().config());
167 if (BEAST_EXPECT(s.globalPragma->size() == 3))
168 {
169 BEAST_EXPECT(s.globalPragma->at(0) == "PRAGMA journal_mode=off;");
170 BEAST_EXPECT(s.globalPragma->at(1) == "PRAGMA synchronous=extra;");
171 BEAST_EXPECT(s.globalPragma->at(2) == "PRAGMA temp_store=default;");
172 }
173 }
174 {
175 // Error: Mix safety_level and individual settings
177 auto const expected =
178 "Failed to initialize SQL databases: "
179 "Configuration file may not define both \"safety_level\" and "
180 "\"journal_mode\"";
181 bool found = false;
182
183 auto p = test::jtx::envconfig();
184 {
185 auto& section = p->section("sqlite");
186 section.set("safety_level", "low");
187 section.set("journal_mode", "off");
188 section.set("synchronous", "extra");
189 section.set("temp_store", "default");
190 }
191
192 try
193 {
194 Env const env(
195 *this,
196 std::move(p),
197 std::make_unique<CheckMessageLogs>(expected, &found),
199 fail();
200 }
201 catch (...)
202 {
203 BEAST_EXPECT(found);
204 }
205 }
206 {
207 // Error: Mix safety_level and one setting (gotta catch 'em all)
209 auto const expected =
210 "Failed to initialize SQL databases: Configuration file may "
211 "not define both \"safety_level\" and \"journal_mode\"";
212 bool found = false;
213
214 auto p = test::jtx::envconfig();
215 {
216 auto& section = p->section("sqlite");
217 section.set("safety_level", "high");
218 section.set("journal_mode", "off");
219 }
220
221 try
222 {
223 Env const env(
224 *this,
225 std::move(p),
226 std::make_unique<CheckMessageLogs>(expected, &found),
228 fail();
229 }
230 catch (...)
231 {
232 BEAST_EXPECT(found);
233 }
234 }
235 {
236 // Error: Mix safety_level and one setting (gotta catch 'em all)
238 auto const expected =
239 "Failed to initialize SQL databases: Configuration file may "
240 "not define both \"safety_level\" and \"synchronous\"";
241 bool found = false;
242
243 auto p = test::jtx::envconfig();
244 {
245 auto& section = p->section("sqlite");
246 section.set("safety_level", "low");
247 section.set("synchronous", "extra");
248 }
249
250 try
251 {
252 Env const env(
253 *this,
254 std::move(p),
255 std::make_unique<CheckMessageLogs>(expected, &found),
257 fail();
258 }
259 catch (...)
260 {
261 BEAST_EXPECT(found);
262 }
263 }
264 {
265 // Error: Mix safety_level and one setting (gotta catch 'em all)
267 auto const expected =
268 "Failed to initialize SQL databases: Configuration file may "
269 "not define both \"safety_level\" and \"temp_store\"";
270 bool found = false;
271
272 auto p = test::jtx::envconfig();
273 {
274 auto& section = p->section("sqlite");
275 section.set("safety_level", "high");
276 section.set("temp_store", "default");
277 }
278
279 try
280 {
281 Env const env(
282 *this,
283 std::move(p),
284 std::make_unique<CheckMessageLogs>(expected, &found),
286 fail();
287 }
288 catch (...)
289 {
290 BEAST_EXPECT(found);
291 }
292 }
293 {
294 // Error: Invalid value
296 auto const expected =
297 "Failed to initialize SQL databases: Invalid safety_level "
298 "value: slow";
299 bool found = false;
300
301 auto p = test::jtx::envconfig();
302 {
303 auto& section = p->section("sqlite");
304 section.set("safety_level", "slow");
305 }
306
307 try
308 {
309 Env const env(
310 *this,
311 std::move(p),
312 std::make_unique<CheckMessageLogs>(expected, &found),
314 fail();
315 }
316 catch (...)
317 {
318 BEAST_EXPECT(found);
319 }
320 }
321 {
322 // Error: Invalid value
324 auto const expected =
325 "Failed to initialize SQL databases: Invalid journal_mode "
326 "value: fast";
327 bool found = false;
328
329 auto p = test::jtx::envconfig();
330 {
331 auto& section = p->section("sqlite");
332 section.set("journal_mode", "fast");
333 }
334
335 try
336 {
337 Env const env(
338 *this,
339 std::move(p),
340 std::make_unique<CheckMessageLogs>(expected, &found),
342 fail();
343 }
344 catch (...)
345 {
346 BEAST_EXPECT(found);
347 }
348 }
349 {
350 // Error: Invalid value
352 auto const expected =
353 "Failed to initialize SQL databases: Invalid synchronous "
354 "value: instant";
355 bool found = false;
356
357 auto p = test::jtx::envconfig();
358 {
359 auto& section = p->section("sqlite");
360 section.set("synchronous", "instant");
361 }
362
363 try
364 {
365 Env const env(
366 *this,
367 std::move(p),
368 std::make_unique<CheckMessageLogs>(expected, &found),
370 fail();
371 }
372 catch (...)
373 {
374 BEAST_EXPECT(found);
375 }
376 }
377 {
378 // Error: Invalid value
380 auto const expected =
381 "Failed to initialize SQL databases: Invalid temp_store "
382 "value: network";
383 bool found = false;
384
385 auto p = test::jtx::envconfig();
386 {
387 auto& section = p->section("sqlite");
388 section.set("temp_store", "network");
389 }
390
391 try
392 {
393 Env const env(
394 *this,
395 std::move(p),
396 std::make_unique<CheckMessageLogs>(expected, &found),
398 fail();
399 }
400 catch (...)
401 {
402 BEAST_EXPECT(found);
403 }
404 }
405 {
406 // N/A: Default values
407 Env env(*this);
408 auto const s = setup_DatabaseCon(env.app().config());
409 if (BEAST_EXPECT(s.txPragma.size() == 4))
410 {
411 BEAST_EXPECT(s.txPragma.at(0) == "PRAGMA page_size=4096;");
412 BEAST_EXPECT(s.txPragma.at(1) == "PRAGMA journal_size_limit=1582080;");
413 BEAST_EXPECT(s.txPragma.at(2) == "PRAGMA max_page_count=4294967294;");
414 BEAST_EXPECT(s.txPragma.at(3) == "PRAGMA mmap_size=17179869184;");
415 }
416 }
417 {
418 // Success: Valid values
419 Env env = [&]() {
420 auto p = test::jtx::envconfig();
421 {
422 auto& section = p->section("sqlite");
423 section.set("page_size", "512");
424 section.set("journal_size_limit", "2582080");
425 }
426 return Env(*this, std::move(p));
427 }();
428 auto const s = setup_DatabaseCon(env.app().config());
429 if (BEAST_EXPECT(s.txPragma.size() == 4))
430 {
431 BEAST_EXPECT(s.txPragma.at(0) == "PRAGMA page_size=512;");
432 BEAST_EXPECT(s.txPragma.at(1) == "PRAGMA journal_size_limit=2582080;");
433 BEAST_EXPECT(s.txPragma.at(2) == "PRAGMA max_page_count=4294967294;");
434 BEAST_EXPECT(s.txPragma.at(3) == "PRAGMA mmap_size=17179869184;");
435 }
436 }
437 {
438 // Error: Invalid values
439 auto const expected = "Invalid page_size. Must be between 512 and 65536.";
440 bool found = false;
441 auto p = test::jtx::envconfig();
442 {
443 auto& section = p->section("sqlite");
444 section.set("page_size", "256");
445 }
446 try
447 {
448 Env const env(
449 *this,
450 std::move(p),
451 std::make_unique<CheckMessageLogs>(expected, &found),
453 fail();
454 }
455 catch (...)
456 {
457 BEAST_EXPECT(found);
458 }
459 }
460 {
461 // Error: Invalid values
462 auto const expected = "Invalid page_size. Must be between 512 and 65536.";
463 bool found = false;
464 auto p = test::jtx::envconfig();
465 {
466 auto& section = p->section("sqlite");
467 section.set("page_size", "131072");
468 }
469 try
470 {
471 Env const env(
472 *this,
473 std::move(p),
474 std::make_unique<CheckMessageLogs>(expected, &found),
476 fail();
477 }
478 catch (...)
479 {
480 BEAST_EXPECT(found);
481 }
482 }
483 {
484 // Error: Invalid values
485 auto const expected = "Invalid page_size. Must be a power of 2.";
486 bool found = false;
487 auto p = test::jtx::envconfig();
488 {
489 auto& section = p->section("sqlite");
490 section.set("page_size", "513");
491 }
492 try
493 {
494 Env const env(
495 *this,
496 std::move(p),
497 std::make_unique<CheckMessageLogs>(expected, &found),
499 fail();
500 }
501 catch (...)
502 {
503 BEAST_EXPECT(found);
504 }
505 }
506 }
507
508 //--------------------------------------------------------------------------
509
510 void
512 std::string const& destBackendType,
513 std::string const& srcBackendType,
514 std::int64_t seedValue)
515 {
516 DummyScheduler scheduler;
517
518 beast::temp_dir const node_db;
519 Section srcParams;
520 srcParams.set("type", srcBackendType);
521 srcParams.set("path", node_db.path());
522
523 // Create a batch
525
526 // Write to source db
527 {
529 Manager::instance().make_Database(megabytes(4), scheduler, 2, srcParams, journal_);
530 storeBatch(*src, batch);
531 }
532
533 Batch copy;
534
535 {
536 // Re-open the db
538 Manager::instance().make_Database(megabytes(4), scheduler, 2, srcParams, journal_);
539
540 // Set up the destination database
541 beast::temp_dir const dest_db;
542 Section destParams;
543 destParams.set("type", destBackendType);
544 destParams.set("path", dest_db.path());
545
547 Manager::instance().make_Database(megabytes(4), scheduler, 2, destParams, journal_);
548
549 testcase("import into '" + destBackendType + "' from '" + srcBackendType + "'");
550
551 // Do the import
552 dest->importDatabase(*src);
553
554 // Get the results of the import
555 fetchCopyOfBatch(*dest, &copy, batch);
556 }
557
558 // Canonicalize the source and destination batches
559 std::sort(batch.begin(), batch.end(), LessThan{});
560 std::sort(copy.begin(), copy.end(), LessThan{});
561 BEAST_EXPECT(areBatchesEqual(batch, copy));
562 }
563
564 //--------------------------------------------------------------------------
565
566 void
568 std::string const& type,
569 bool const testPersistence,
570 std::int64_t const seedValue,
571 int numObjsToTest = 2000)
572 {
573 DummyScheduler scheduler;
574
575 std::string const s = "NodeStore backend '" + type + "'";
576
577 testcase(s);
578
579 beast::temp_dir const node_db;
580 Section nodeParams;
581 nodeParams.set("type", type);
582 nodeParams.set("path", node_db.path());
583
584 beast::xor_shift_engine rng(seedValue);
585
586 // Create a batch
587 auto batch = createPredictableBatch(numObjsToTest, rng());
588
589 {
590 // Open the database
592 Manager::instance().make_Database(megabytes(4), scheduler, 2, nodeParams, journal_);
593
594 // Write the batch
595 storeBatch(*db, batch);
596
597 {
598 // Read it back in
599 Batch copy;
600 fetchCopyOfBatch(*db, &copy, batch);
601 BEAST_EXPECT(areBatchesEqual(batch, copy));
602 }
603
604 {
605 // Reorder and read the copy again
606 std::shuffle(batch.begin(), batch.end(), rng);
607 Batch copy;
608 fetchCopyOfBatch(*db, &copy, batch);
609 BEAST_EXPECT(areBatchesEqual(batch, copy));
610 }
611 }
612
613 if (testPersistence)
614 {
615 // Re-open the database without the ephemeral DB
617 Manager::instance().make_Database(megabytes(4), scheduler, 2, nodeParams, journal_);
618
619 // Read it back in
620 Batch copy;
621 fetchCopyOfBatch(*db, &copy, batch);
622
623 // Canonicalize the source and destination batches
624 std::sort(batch.begin(), batch.end(), LessThan{});
625 std::sort(copy.begin(), copy.end(), LessThan{});
626 BEAST_EXPECT(areBatchesEqual(batch, copy));
627 }
628
629 if (type == "memory")
630 {
631 // Verify default earliest ledger sequence
632 {
634 megabytes(4), scheduler, 2, nodeParams, journal_);
635 BEAST_EXPECT(db->earliestLedgerSeq() == XRP_LEDGER_EARLIEST_SEQ);
636 }
637
638 // Set an invalid earliest ledger sequence
639 try
640 {
641 nodeParams.set("earliest_seq", "0");
643 megabytes(4), scheduler, 2, nodeParams, journal_);
644 }
645 catch (std::runtime_error const& e)
646 {
647 BEAST_EXPECT(std::strcmp(e.what(), "Invalid earliest_seq") == 0);
648 }
649
650 {
651 // Set a valid earliest ledger sequence
652 nodeParams.set("earliest_seq", "1");
654 megabytes(4), scheduler, 2, nodeParams, journal_);
655
656 // Verify database uses the earliest ledger sequence setting
657 BEAST_EXPECT(db->earliestLedgerSeq() == 1);
658 }
659
660 // Create another database that attempts to set the value again
661 try
662 {
663 // Set to default earliest ledger sequence
664 nodeParams.set("earliest_seq", std::to_string(XRP_LEDGER_EARLIEST_SEQ));
666 megabytes(4), scheduler, 2, nodeParams, journal_);
667 }
668 catch (std::runtime_error const& e)
669 {
670 BEAST_EXPECT(std::strcmp(e.what(), "earliest_seq set more than once") == 0);
671 }
672 }
673 }
674
675 //--------------------------------------------------------------------------
676
677 void
678 run() override
679 {
680 std::int64_t const seedValue = 50;
681
682 testConfig();
683
684 testNodeStore("memory", false, seedValue);
685
686 // Persistent backend tests
687 {
688 testNodeStore("nudb", true, seedValue);
689
690#if XRPL_ROCKSDB_AVAILABLE
691 testNodeStore("rocksdb", true, seedValue);
692#endif
693 }
694
695 // Import tests
696 {
697 testImport("nudb", "nudb", seedValue);
698
699#if XRPL_ROCKSDB_AVAILABLE
700 testImport("rocksdb", "rocksdb", seedValue);
701#endif
702
703#if XRPL_ENABLE_SQLITE_BACKEND_TESTS
704 testImport("sqlite", "sqlite", seedValue);
705#endif
706 }
707 }
708};
709
710BEAST_DEFINE_TESTSUITE(Database, nodestore, xrpl);
711
712} // namespace NodeStore
713} // namespace xrpl
RAII temporary directory.
Definition temp_dir.h:15
std::string path() const
Get the native path for the temporary directory.
Definition temp_dir.h:47
testcase_t testcase
Memberspace for declaring test cases.
Definition suite.h:150
void fail(String const &reason, char const *file, int line)
Record a failure.
Definition suite.h:519
virtual Config & config()=0
void run() override
Runs the suite.
void testImport(std::string const &destBackendType, std::string const &srcBackendType, std::int64_t seedValue)
void testNodeStore(std::string const &type, bool const testPersistence, std::int64_t const seedValue, int numObjsToTest=2000)
Persistency layer for NodeObject.
Definition Database.h:31
Simple NodeStore Scheduler that just performs the tasks synchronously.
static Manager & instance()
Returns the instance of the manager singleton.
virtual std::unique_ptr< Database > make_Database(std::size_t burstSize, Scheduler &scheduler, int readThreads, Section const &backendParameters, beast::Journal journal)=0
Construct a NodeStore database.
static bool areBatchesEqual(Batch const &lhs, Batch const &rhs)
Definition TestBase.h:97
void fetchCopyOfBatch(Backend &backend, Batch *pCopy, Batch const &batch)
Definition TestBase.h:132
static int const numObjectsToTest
Definition TestBase.h:53
void storeBatch(Backend &backend, Batch const &batch)
Definition TestBase.h:122
static Batch createPredictableBatch(int numObjects, std::uint64_t seed)
Definition TestBase.h:58
Holds a collection of configuration values.
Definition BasicConfig.h:24
void set(std::string const &key, std::string const &value)
Set a key/value pair.
A transaction testing environment.
Definition Env.h:122
Application & app()
Definition Env.h:259
T is_same_v
std::unique_ptr< Config > envconfig()
creates and initializes a default configuration for jtx::Env
Definition envconfig.h:34
Use hash_* containers for keys that do not need a cryptographically secure hashing algorithm.
Definition algorithm.h:5
constexpr auto megabytes(T value) noexcept
DatabaseCon::Setup setup_DatabaseCon(Config const &c, std::optional< beast::Journal > j=std::nullopt)
Definition Config.cpp:1179
static constexpr std::uint32_t XRP_LEDGER_EARLIEST_SEQ
The XRP ledger network's earliest allowed sequence.
T shuffle(T... args)
T sort(T... args)
T strcmp(T... args)
static std::unique_ptr< std::vector< std::string > const > globalPragma
Definition DatabaseCon.h:90
Binary function that satisfies the strict-weak-ordering requirement.
Definition TestBase.h:27
T to_string(T... args)
T what(T... args)