xrpld
Loading...
Searching...
No Matches
SHAMapStore_test.cpp
1#include <test/jtx/Account.h>
2#include <test/jtx/Env.h>
3#include <test/jtx/amount.h>
4#include <test/jtx/envconfig.h>
5#include <test/jtx/noop.h>
6
7#include <xrpld/app/ledger/LedgerMaster.h>
8#include <xrpld/app/main/Application.h>
9#include <xrpld/app/main/NodeStoreScheduler.h>
10#include <xrpld/app/misc/SHAMapStore.h>
11#include <xrpld/app/rdb/backend/SQLiteDatabase.h>
12#include <xrpld/core/Config.h>
13
14#include <xrpl/basics/ByteUtilities.h>
15#include <xrpl/basics/Log.h>
16#include <xrpl/basics/base_uint.h>
17#include <xrpl/beast/unit_test/suite.h>
18#include <xrpl/beast/utility/Journal.h>
19#include <xrpl/config/BasicConfig.h>
20#include <xrpl/config/Constants.h>
21#include <xrpl/core/JobQueue.h>
22#include <xrpl/json/json_value.h>
23#include <xrpl/nodestore/Backend.h>
24#include <xrpl/nodestore/Manager.h>
25#include <xrpl/nodestore/detail/DatabaseRotatingImp.h>
26#include <xrpl/protocol/ErrorCodes.h>
27#include <xrpl/protocol/LedgerHeader.h>
28#include <xrpl/protocol/Protocol.h>
29#include <xrpl/protocol/XRPAmount.h>
30#include <xrpl/protocol/jss.h>
31#include <xrpl/server/NetworkOPs.h>
32
33#include <algorithm>
34#include <atomic>
35#include <chrono>
36#include <condition_variable>
37#include <cstddef>
38#include <cstdint>
39#include <exception>
40#include <filesystem>
41#include <functional>
42#include <limits>
43#include <map>
44#include <memory>
45#include <mutex>
46#include <optional>
47#include <ostream>
48#include <sstream>
49#include <string>
50#include <thread>
51#include <utility>
52#include <vector>
53
54namespace xrpl::test {
55
57{
58 static constexpr int kDeleteInterval = 8;
59
60 // Mirrors SHAMapStoreImp::kMinimumDeletionIntervalSa, the floor that
61 // online_delete is held to in standalone mode. Note that the
62 // max_waiting_ledgers floor is derived from this minimum rather than from
63 // the configured interval -- SHAMapStoreImp.cpp computes it as
64 // minInterval / 4 -- so both constants below stay put if kDeleteInterval is
65 // ever raised.
66 static constexpr int kMinDeleteInterval = 8;
67 static constexpr int kMinWaitingLedgers = kMinDeleteInterval / 4;
68 static_assert(kDeleteInterval >= kMinDeleteInterval);
69
70 // The two wait durations healthWait() can choose from, spelled as they
71 // appear in its log message. onlineDelete() below sets
72 // recovery_wait_seconds to 1, so the full wait is 1000ms and the shortened
73 // wait is a tenth of that. Note that "Waiting 1000ms" does not contain
74 // "Waiting 100ms", so the two are distinguishable by substring.
75 static constexpr char const* kFullWait = "Waiting 1000ms for node to stabilize";
76 static constexpr char const* kShortWait = "Waiting 100ms for node to stabilize";
77
78 // Distinctive fragments of the other messages the tests below key off. Each
79 // is unique among everything SHAMapStoreImp logs, so a substring match
80 // identifies the message unambiguously.
81 //
82 // kRotating is logged once run() has committed to a rotation, immediately
83 // after the health check that gates it, and kFinished once a rotation has
84 // run to completion. kExpired is logged by healthWait() when the circuit
85 // breaker trips.
86 static constexpr char const* kRotating = "rotating";
87 static constexpr char const* kFinished = "finished rotation";
88 static constexpr char const* kExpired = "unable to make progress";
89
90 // A Logs implementation that records every message the store's own
91 // partition emits, keeping each message's severity alongside its text, and
92 // lets a test block until a given message has appeared.
93 //
94 // Severity is recorded because healthWait() picks the severity and the wait
95 // duration together, so the pair identifies which of its three logging
96 // branches ran: warn at the full wait when the server is unhealthy for a
97 // reason that is not expected to resolve on its own, trace at a tenth of
98 // the wait when the only missing ledger is the one currently being built,
99 // and info at the full wait otherwise. Matching on the pair is what lets
100 // the tests below assert which branch was taken instead of merely that some
101 // wait happened.
102 //
103 // waitFor() exists because run() logs on entry to a rotation, which is the
104 // only signal a test has that the store has passed the health check gating
105 // the rotation and is now inside it. Several of the branches under test are
106 // only reachable from there, and no other handle on the store exposes it.
107 class StoreLogs : public Logs
108 {
112
114 {
116
117 public:
119 : beast::Journal::Sink(threshold, false), owner_(owner)
120 {
121 }
122
123 // Env::AppBundle calls Logs::threshold() after the Application is
124 // built, which would otherwise raise this sink above Trace and
125 // discard the messages the buildingIndex branch logs.
126 void
128 {
129 }
130
131 void
132 write(beast::Severity level, std::string const& text) override
133 {
134 {
135 std::scoped_lock const lock(owner_.mutex_);
136 owner_.messages_.emplace_back(level, text);
137 }
138 owner_.cond_.notify_all();
139 }
140
141 void
142 writeAlways(beast::Severity level, std::string const& text) override
143 {
144 write(level, text);
145 }
146 };
147
148 // Caller must hold mutex_. A nullopt severity matches any severity.
149 [[nodiscard]] std::size_t
151 {
152 return std::count_if(messages_.begin(), messages_.end(), [&](auto const& message) {
153 return (!severity || message.first == *severity) &&
154 message.second.find(text) != std::string::npos;
155 });
156 }
157
158 public:
160 {
161 }
162
163 // Only the store's own partition is logged at Trace; everything else
164 // is silenced, so that enabling trace for this one branch does not pay
165 // for formatting every trace message in the server.
167 makeSink(std::string const& partition, beast::Severity) override
168 {
170 partition == "SHAMapStore" ? beast::Severity::Trace : beast::Severity::Disabled,
171 *this);
172 }
173
174 // How many recorded messages were logged at `severity` and contain
175 // `text`.
176 [[nodiscard]] std::size_t
177 count(beast::Severity severity, std::string const& text) const
178 {
179 std::scoped_lock const lock(mutex_);
180 return countLocked(severity, text);
181 }
182
183 // How many recorded messages contain `text`, at any severity.
184 [[nodiscard]] std::size_t
185 count(std::string const& text) const
186 {
187 std::scoped_lock const lock(mutex_);
188 return countLocked(std::nullopt, text);
189 }
190
191 // Blocks until `text` has been logged at least `expected` times at
192 // `severity` -- or at any severity, if that is nullopt -- or until the
193 // timeout expires. Returns whether it got there.
194 //
195 // Waiting rather than sleeping-then-counting matters for the branches
196 // that are only reached after a rotation has started: the store gets
197 // there when it gets there, so a fixed sleep has to be sized for the
198 // slowest plausible machine, whereas this returns as soon as the
199 // message appears.
200 [[nodiscard]] bool
203 std::string const& text,
205 std::size_t expected = 1)
206 {
208 return cond_.wait_for(
209 lock, timeout, [&] { return countLocked(severity, text) >= expected; });
210 }
211
212 // As above, at any severity.
213 [[nodiscard]] bool
215 std::string const& text,
217 std::size_t expected = 1)
218 {
219 return waitFor(std::nullopt, text, timeout, expected);
220 }
221 };
222
223 static auto
225 {
226 cfg = jtx::onlineDelete(std::move(cfg), kDeleteInterval);
228 return cfg;
229 }
230
231 // online delete tuned so that a rotation, once it has started, spends a
232 // long time in clearPrior() before reaching the first health check inside
233 // the rotation body.
234 //
235 // clearSql() sleeps back_off_milliseconds at the top of every iteration and
236 // advances by delete_batch rows per iteration, so a delete_batch of 1 costs
237 // one sleep per ledger removed, for each of the three tables it is called
238 // on. That is what gives parkMidRotation() below a window measured in
239 // seconds rather than in microseconds. delete_batch is therefore the actual
240 // lever; back_off_milliseconds is set to the value SHAMapStoreImp already
241 // defaults to, and is spelled out only so the arithmetic above can be
242 // checked against the config rather than against the implementation.
243 //
244 // max_waiting_ledgers is pinned to its floor so that tripping the circuit
245 // breaker takes the fewest possible ledger closes.
246 static auto
248 {
249 cfg = onlineDelete(std::move(cfg));
250 auto& section = cfg->section(Sections::kNodeDatabase);
251 section.set(Keys::kDeleteBatch, "1");
252 section.set(Keys::kBackOffMilliseconds, "100");
254 return cfg;
255 }
256
257 static auto
259 {
260 cfg = onlineDelete(std::move(cfg));
261 cfg->section(Sections::kNodeDatabase).set(Keys::kAdvisoryDelete, "1");
262 return cfg;
263 }
264
265 static bool
266 goodLedger(jtx::Env& env, json::Value const& json, std::string ledgerID, bool checkDB = false)
267 {
268 auto const good = json.isMember(jss::result) && !rpc::containsError(json[jss::result]) &&
269 json[jss::result][jss::ledger][jss::ledger_index] == ledgerID;
270 if (!good || !checkDB)
271 return good;
272
273 auto const seq = json[jss::result][jss::ledger_index].asUInt();
274
277 if (!outInfo)
278 return false;
279 LedgerHeader const& info = outInfo.value();
280
281 std::string const outHash = to_string(info.hash);
282 LedgerIndex const outSeq = info.seq;
283 std::string const outParentHash = to_string(info.parentHash);
284 std::string const outDrops = to_string(info.drops);
285 std::uint64_t const outCloseTime = info.closeTime.time_since_epoch().count();
286 std::uint64_t const outParentCloseTime = info.parentCloseTime.time_since_epoch().count();
287 std::uint64_t const outCloseTimeResolution = info.closeTimeResolution.count();
288 std::uint64_t const outCloseFlags = info.closeFlags;
289 std::string const outAccountHash = to_string(info.accountHash);
290 std::string const outTxHash = to_string(info.txHash);
291
292 auto const& ledger = json[jss::result][jss::ledger];
293 return outHash == ledger[jss::ledger_hash].asString() && outSeq == seq &&
294 outParentHash == ledger[jss::parent_hash].asString() &&
295 outDrops == ledger[jss::total_coins].asString() &&
296 outCloseTime == ledger[jss::close_time].asUInt() &&
297 outParentCloseTime == ledger[jss::parent_close_time].asUInt() &&
298 outCloseTimeResolution == ledger[jss::close_time_resolution].asUInt() &&
299 outCloseFlags == ledger[jss::close_flags].asUInt() &&
300 outAccountHash == ledger[jss::account_hash].asString() &&
301 outTxHash == ledger[jss::transaction_hash].asString();
302 }
303
304 static bool
306 std::string const& token,
307 ErrorCodeI error,
308 std::string const& message)
309 {
310 return json.isMember(jss::result) && json[jss::result][jss::error] == token &&
311 json[jss::result][jss::error_code] == error &&
312 json[jss::result][jss::error_message] == message;
313 }
314
317 {
318 BEAST_EXPECT(
319 json.isMember(jss::result) && json[jss::result].isMember(jss::ledger) &&
320 json[jss::result][jss::ledger].isMember(jss::ledger_hash) &&
321 json[jss::result][jss::ledger][jss::ledger_hash].isString());
322 return json[jss::result][jss::ledger][jss::ledger_hash].asString();
323 }
324
325 void
326 ledgerCheck(jtx::Env& env, int const rows, int const first)
327 {
328 auto const [actualRows, actualFirst, actualLast] =
330
331 BEAST_EXPECT(actualRows == rows);
332 BEAST_EXPECT(actualFirst == first);
333 BEAST_EXPECT(actualLast == first + rows - 1);
334 }
335
336 void
337 transactionCheck(jtx::Env& env, int const rows)
338 {
339 BEAST_EXPECT(env.app().getRelationalDatabase().getTransactionCount() == rows);
340 }
341
342 void
343 accountTransactionCheck(jtx::Env& env, int const rows)
344 {
345 BEAST_EXPECT(env.app().getRelationalDatabase().getAccountTransactionCount() == rows);
346 }
347
348 // Wait until the SHAMapStore has finished processing the ledger that the
349 // preceding env.close() produced.
350 //
351 // env.close() returns as soon as the ledger_accept RPC returns, but the
352 // validated ledger path -- LedgerMaster::setValidLedger() ->
353 // SHAMapStore::onLedgerClosed() -- runs on a job queue thread. Without
354 // draining the job queue first, the store may not have been handed the
355 // ledger at all, in which case rendezvous() observes working_ == false and
356 // returns immediately, before any work has been done.
357 [[nodiscard]] static bool
359 {
360 // Drain the job queue first, so that onLedgerClosed() has run and
361 // working_ is set. Then use the store's timeout overload, so a store
362 // that never finishes fails this test instead of blocking on it.
363 //
364 // Only the second wait is bounded: JobQueue::rendezvous() has no
365 // timeout overload, so a job that never completes hangs here. That is
366 // pre-existing -- ~AppBundle waits on it the same way for every jtx
367 // test -- but it does mean this helper is not hang-proof end to end.
368 env.app().getJobQueue().rendezvous();
370 }
371
372 // Bring the SHAMapStore to the point where it has been handed a validated
373 // ledger and initialized lastRotated, and report how many extra ledgers had
374 // to be closed to get it there (normally none). Returns std::nullopt if
375 // syncStore() itself failed.
376 //
377 // syncStore() alone does not guarantee that, because
378 // SHAMapStoreImp::run()'s loop does not use the notification and the
379 // working_ flag safely:
380 //
381 // * onLedgerClosed() notifies cond_ whether or not run()'s thread is
382 // parked on it, and run() waits on cond_ without a predicate, so a
383 // notification that lands while the thread is still starting up --
384 // before it first reaches that wait -- is lost.
385 // * run() clears working_ at the top of its loop without checking
386 // whether newLedger_ is still set, so rendezvous() can report the
387 // store idle with a validated ledger queued.
388 //
389 // Either way the store ends up parked with work pending, and only another
390 // notification gets it moving again. In a standalone test nothing else
391 // closes ledgers, so that has to come from here: this closes a ledger
392 // rather than polling getLastRotated(), because polling would just time
393 // out. onLedgerClosed() keeps only the most recent ledger in newLedger_,
394 // so the ledger the store picks up -- and therefore lastRotated -- is a
395 // timing detail, which is why the callers derive their expectations from
396 // the value they observe instead of assuming one.
397 //
398 // run() is deliberately left as it is. In production the only effect is
399 // latency: the trigger is validatedSeq >= lastRotated + deleteInterval, so
400 // a lost notification delays rotation to the next validated ledger and
401 // nothing is skipped or accumulated -- starting at 513 instead of 512 does
402 // not matter. Two consequences do follow from leaving it in place, and both
403 // hold today: nothing in production decides anything from working_ or
404 // rendezvous() (rendezvous() has no production callers at all), and a node
405 // whose ledgers only advance on demand -- standalone, driven by
406 // ledger_accept -- can sit on a queued ledger until something closes the
407 // next one, which is exactly the situation this helper is working around.
408 //
409 // So this helper is permanent rather than a stopgap. Working around the
410 // race must not make it invisible, so every extra close is logged. That
411 // keeps how often it is actually hit observable in the unit test output --
412 // which is the only signal left once these testcases stop flaking on it.
413 [[nodiscard]] std::optional<int>
414 initializeStore(jtx::Env& env, int const maxExtraCloses = 3)
415 {
416 auto& store = env.app().getSHAMapStore();
417
418 for (int extraCloses = 0;; ++extraCloses)
419 {
420 if (!syncStore(env))
421 return std::nullopt;
422 if (store.getLastRotated() != 0 || extraCloses == maxExtraCloses)
423 {
424 if (extraCloses != 0)
425 {
426 log << "initializeStore: the store needed " << extraCloses
427 << " extra ledger close(s) to pick up a validated ledger. "
428 "SHAMapStoreImp::run() dropped the notification for the "
429 "first one; see the comment on initializeStore()."
430 << std::endl;
431 }
432 return extraCloses;
433 }
434 env.close();
435 }
436 }
437
438 int
440 {
441 using namespace std::chrono_literals;
442
443 auto& store = env.app().getSHAMapStore();
444
445 int ledgerSeq = 3;
446 BEAST_EXPECT(syncStore(env));
447 BEAST_EXPECT(!store.getLastRotated());
448
449 env.close();
450 BEAST_EXPECT(syncStore(env));
451
452 auto const ledger = env.rpc("ledger", "validated");
453 BEAST_EXPECT(goodLedger(env, ledger, std::to_string(ledgerSeq++)));
454
455 BEAST_EXPECT(store.getLastRotated() == ledgerSeq - 1);
456 return ledgerSeq;
457 }
458
459 // Construct an Env whose config has online_delete enabled and is then
460 // mutated by `tweak`, and report how SHAMapStoreImp's constructor judged
461 // it: the message of the exception it threw, or std::nullopt if the Env was
462 // constructed successfully.
463 //
464 // SHAMapStoreImp is built from ApplicationImp's member initializer list, so
465 // a configuration it rejects surfaces as an exception thrown out of the Env
466 // constructor rather than as a failure at some later point.
467 //
468 // Note that ~AppBundle does not run when the Env constructor throws, so the
469 // global debug log sink it installed -- which holds a reference to this
470 // suite -- would outlive the suite. The catch below clears it, so callers
471 // are free to end on a configuration that is rejected.
474 {
475 using namespace test::jtx;
476
477 try
478 {
479 Env const env{
480 *this,
481 envconfig([&tweak](std::unique_ptr<Config> cfg) {
482 cfg = onlineDelete(std::move(cfg));
483 tweak(*cfg);
484 return cfg;
485 }),
486 nullptr,
488 return std::nullopt;
489 }
490 // Deliberately broader than the std::runtime_error that
491 // SHAMapStoreImp throws: an unexpected exception type then shows up as
492 // a message mismatch naming the actual failure, rather than escaping
493 // this testcase.
494 catch (std::exception const& e)
495 {
496 // ~AppBundle did not run, so drop the sink it installed by hand
497 // rather than leaving a reference to this suite live in a global.
498 setDebugLogSink(nullptr);
499 return std::string{e.what()};
500 }
501 }
502
503 void
504 expectConfigRejected(std::string const& expected, std::function<void(Config&)> const& tweak)
505 {
506 auto const result = storeConfigResult(tweak);
507 BEAST_EXPECTS(result == expected, result.value_or("<accepted>"));
508 }
509
510 void
512 {
513 auto const result = storeConfigResult(tweak);
514 BEAST_EXPECTS(!result, result.value_or(""));
515 }
516
517 // The state parkInHealthWait() leaves behind.
518 struct Parked
519 {
520 // Value of getLastRotated() before the rotation attempt began. The
521 // store must still report this for as long as it stays parked.
523 // The validated ledger the store is waiting on, and the sequence
524 // getLastRotated() will report once the rotation finally completes.
526 // Sequence removed from LedgerMaster to create the gap, or 0 if
527 // `createGap` was false.
529 };
530
531 // Drive the store to the point where it is parked inside healthWait(),
532 // unable to proceed with a rotation: close ledgers until one more close
533 // would make the store due to rotate, optionally remove the newest ledger
534 // from LedgerMaster so that the attempt sees a gap in the range, close the
535 // triggering ledger, and then set the operating mode to `modeAfterClose`.
536 //
537 // Once parked, the store stays parked indefinitely. Its wait loop reruns
538 // every recovery_wait_seconds and exits only when the server looks healthy,
539 // when it is stopped, or when the validated ledger index reaches the
540 // circuit breaker -- and that index only advances when this test closes
541 // another ledger. So a caller can establish any server state it likes,
542 // hold it, and be sure the store observes it. That is what makes the tests
543 // below state machines rather than races.
544 //
545 // Returns std::nullopt if the setup did not reach a parked store, having
546 // already reported the failure.
548 parkInHealthWait(jtx::Env& env, bool createGap, OperatingMode modeAfterClose)
549 {
550 using namespace std::chrono_literals;
551 using namespace test::jtx;
552
553 auto& lm = env.app().getLedgerMaster();
554 auto& store = env.app().getSHAMapStore();
555 auto& netOPs = env.app().getOPs();
556
557 env.fund(XRP(1000), Account("alice"));
558 env.close();
559 if (!BEAST_EXPECT(initializeStore(env).has_value()))
560 return std::nullopt;
561
562 Parked parked;
563 // The store adopts the first validated ledger it sees as lastRotated,
564 // and which one that is depends on timing, so read it rather than
565 // assuming a value.
566 parked.lastRotated = store.getLastRotated();
567 if (!BEAST_EXPECT(parked.lastRotated))
568 return std::nullopt;
569
570 // Close ledgers until the next close is the one that makes
571 // validatedSeq reach lastRotated + deleteInterval.
572 LedgerIndex maxSeq = env.closed()->header().seq;
573 while (maxSeq + 1 < parked.lastRotated + kDeleteInterval)
574 {
575 env.close();
576 ++maxSeq;
577 if (!BEAST_EXPECT(syncStore(env)))
578 return std::nullopt;
579 if (!BEAST_EXPECTS(
580 store.getLastRotated() == parked.lastRotated,
581 std::to_string(store.getLastRotated())))
582 return std::nullopt;
583 }
584
585 // Drop out of FULL before touching LedgerMaster's internals, matching
586 // testLedgerGaps. This also keeps the store from rotating on the
587 // triggering close before the caller has set the state it wants
588 // observed.
589 netOPs.setMode(OperatingMode::CONNECTED);
590
591 // The gap goes one below the sequence that the close further down makes
592 // validated, never at that sequence itself: healthWait() derives
593 // buildingIndex from numMissing == 1 && !haveLedger(index), so a gap at
594 // the validated index reads as "that ledger is about to be built" and
595 // takes the short trace wait, whereas a gap below it reads as a
596 // genuinely incomplete range and takes the full wait. Nothing refills
597 // the gap, so the wait loop ends only when the circuit breaker trips or
598 // stop() intervenes.
599 if (createGap)
600 {
601 std::size_t iterations = 30;
602 while (!lm.haveLedger(maxSeq) && --iterations > 0)
603 {
605 }
606 if (!BEAST_EXPECTS(lm.haveLedger(maxSeq), std::to_string(maxSeq)))
607 return std::nullopt;
608
609 // Give the server a moment to finish any internal work on the
610 // ledger about to be removed, as testLedgerGaps does.
612
613 lm.clearLedger(maxSeq);
614 if (!BEAST_EXPECT(!lm.haveLedger(maxSeq)))
615 return std::nullopt;
616 parked.gap = maxSeq;
617 }
618
619 // This close makes the store due to rotate.
620 env.close();
621 ++maxSeq;
622 parked.validated = maxSeq;
623 netOPs.setMode(modeAfterClose);
624
625 // Drain the job queue so that onLedgerClosed() has handed the ledger to
626 // the store. Without this, working_ may still be false from the
627 // previous cycle and rendezvous() would report "done" before the store
628 // has even looked at this ledger.
629 env.app().getJobQueue().rendezvous();
630
631 if (!BEAST_EXPECT(!store.rendezvous(1s)))
632 return std::nullopt;
633 if (!BEAST_EXPECTS(
634 store.getLastRotated() == parked.lastRotated,
635 std::to_string(store.getLastRotated())))
636 return std::nullopt;
637
638 return parked;
639 }
640
641 // Drive the store to the point where it has passed the health check that
642 // gates a rotation and is inside the rotation body, then make the server
643 // unhealthy so that the next health check in there parks it.
644 //
645 // The gap cannot be created up front the way parkInHealthWait() does it,
646 // because the gating check would see it and refuse to start the rotation at
647 // all -- which is what testLedgerGaps() exercises. So this waits for the
648 // message run() logs immediately after that check, which is the store
649 // publishing that it is committed to the rotation, and creates the gap then.
650 //
651 // The margin that makes that safe is clearPrior(), which runs between the
652 // log and the first health check inside the rotation. Under
653 // slowOnlineDelete() it works through three tables one sequence at a time,
654 // sleeping back_off_milliseconds before each, and checks health after every
655 // one of those sleeps. So the store spends on the order of a second per
656 // table repeatedly asking whether it is healthy, against the microseconds
657 // this function needs to clear a ledger once waitFor() has returned.
658 //
659 // The gap has to be the validated ledger itself. healthWait() counts missing
660 // ledgers over the range from lastGoodValidatedLedger_ to the validated
661 // index, and run() sets the former to the latter just before starting the
662 // rotation, so for the duration of the rotation that range begins as a
663 // single sequence and grows only as the caller closes more ledgers.
664 //
665 // Which health check inside the rotation ends up observing the gap is not
666 // pinned down, and does not need to be: whichever one it is returns the same
667 // answer, clearPrior() gives up, and run() reaches its first switch on
668 // healthWait() with the condition still in force. Every assertion below
669 // holds for any of them.
670 //
671 // Returns std::nullopt if the setup did not reach a parked store, having
672 // already reported the failure.
675 {
676 using namespace std::chrono_literals;
677 using namespace test::jtx;
678
679 auto& lm = env.app().getLedgerMaster();
680 auto& store = env.app().getSHAMapStore();
681
682 auto const alice = Account("alice");
683 env.fund(XRP(1000), alice);
684 env.close();
685 if (!BEAST_EXPECT(initializeStore(env).has_value()))
686 return std::nullopt;
687
688 LedgerIndex maxSeq = env.closed()->header().seq;
689 // Close one ledger, carrying a transaction so that the sequence has rows
690 // in all three of the tables clearSql() works through.
691 auto const closeOne = [&] -> bool {
692 env(noop(alice));
693 env.close();
694 ++maxSeq;
695 return BEAST_EXPECT(syncStore(env));
696 };
697
698 // Let one rotation complete before setting up the one to be parked. The
699 // window this helper depends on only exists once the tables hold a full
700 // delete interval of rows: on the very first rotation there is at most
701 // one sequence to remove, so clearSql() sleeps once or not at all.
702 LedgerIndex const firstRotated = store.getLastRotated();
703 if (!BEAST_EXPECT(firstRotated))
704 return std::nullopt;
705 while (store.getLastRotated() == firstRotated)
706 {
707 if (!closeOne())
708 return std::nullopt;
709 // The rotation is due once maxSeq reaches firstRotated +
710 // kDeleteInterval. Allow one close beyond that before giving up,
711 // rather than closing ledgers forever.
712 if (!BEAST_EXPECTS(maxSeq <= firstRotated + kDeleteInterval, std::to_string(maxSeq)))
713 return std::nullopt;
714 }
715
716 Parked parked;
717 parked.lastRotated = store.getLastRotated();
718
719 // Close ledgers until the next close is the one that makes the store due
720 // to rotate again.
721 while (maxSeq + 1 < parked.lastRotated + kDeleteInterval)
722 {
723 if (!closeOne())
724 return std::nullopt;
725 if (!BEAST_EXPECTS(
726 store.getLastRotated() == parked.lastRotated,
727 std::to_string(store.getLastRotated())))
728 return std::nullopt;
729 }
730
731 // One rotation has already been logged, so wait for the next one rather
732 // than for the first.
733 auto const rotationsBefore = log.count(kRotating);
734
735 // This close makes the store due to rotate. Deliberately do not drain
736 // the store here: the point is to interrupt it partway through.
737 env(noop(alice));
738 env.close();
739 ++maxSeq;
740 parked.validated = maxSeq;
741
742 // Draining the job queue, on the other hand, is required. In standalone
743 // mode switchLCL() inserts the closed ledger into LedgerMaster's
744 // complete range and then posts an advance job, and publishing the
745 // ledger from that job inserts it a second time. Clearing the ledger
746 // between those two inserts does not leave a lasting gap: publication
747 // puts it straight back, the rotation's health checks see a healthy
748 // node, and the rotation runs to completion. Waiting for the queue to
749 // drain closes that window, because publication is what advances
750 // pubLedger_ -- once it has happened, the ledger is never published, and
751 // so never inserted, again.
752 //
753 // This waits only for the job queue, not for the store, whose thread is
754 // its own and is the thing being interrupted here.
755 env.app().getJobQueue().rendezvous();
756
757 // Publishing the ledger is what advances pubLedger_, so this is the
758 // observable confirmation that the window above has closed. Asserting it
759 // here means that if anything ever reopens it, this setup step says so
760 // directly instead of the tests below failing for reasons that look
761 // nothing like the cause.
762 auto const published = lm.getPublishedLedger();
763 if (!BEAST_EXPECTS(
764 published && published->header().seq >= parked.validated,
765 std::to_string(published ? published->header().seq : 0)))
766 return std::nullopt;
767
768 if (!BEAST_EXPECT(log.waitFor(kRotating, 10s, rotationsBefore + 1)))
769 return std::nullopt;
770
771 // The store is now inside clearPrior(). Remove the validated ledger so
772 // that every health check from here on reports a gap.
773 if (!BEAST_EXPECTS(lm.haveLedger(parked.validated), std::to_string(parked.validated)))
774 return std::nullopt;
775 lm.clearLedger(parked.validated);
776 parked.gap = parked.validated;
777 if (!BEAST_EXPECT(!lm.haveLedger(parked.gap)))
778 return std::nullopt;
779
780 // The rotation must now be stuck. Wait longer than
781 // recovery_wait_seconds, so that this is a settled state rather than a
782 // store that has yet to reach its next health check.
783 if (!BEAST_EXPECT(!store.rendezvous(1500ms)))
784 return std::nullopt;
785 if (!BEAST_EXPECTS(
786 store.getLastRotated() == parked.lastRotated,
787 std::to_string(store.getLastRotated())))
788 return std::nullopt;
789 // The rotation started, has not finished, and has not yet given up.
790 if (!BEAST_EXPECTS(
791 log.count(kRotating) == rotationsBefore + 1, std::to_string(log.count(kRotating))))
792 return std::nullopt;
793 if (!BEAST_EXPECTS(log.count(kFinished) == 1, std::to_string(log.count(kFinished))))
794 return std::nullopt;
795 if (!BEAST_EXPECTS(log.count(kExpired) == 0, std::to_string(log.count(kExpired))))
796 return std::nullopt;
797
798 return parked;
799 }
800
801public:
802 // Cover the [node_db] validation that SHAMapStoreImp performs when it is
803 // constructed. The rejected cases stop inside SHAMapStoreImp's constructor,
804 // so they cost only a partial Application construction; the accepted ones
805 // start a full node and immediately tear it down.
806 void
808 {
809 testcase("config validation");
810
811 // online_delete below the standalone minimum. ledger_history is still
812 // kDeleteInterval here, so it is too large for this online_delete as
813 // well; the assertion pins which of the two errors wins.
815 "online_delete must be at least " + std::to_string(kMinDeleteInterval),
816 [](Config& cfg) {
819 });
820
821 // ledger_history above online_delete asks the node to retain more
822 // history than online delete is allowed to keep.
824 "online_delete must not be less than ledger_history (currently " +
826 [](Config& cfg) { cfg.ledgerHistory = kDeleteInterval + 1; });
827
828 // recovery_wait_seconds is the interval at which online delete rechecks
829 // the node's health while it waits for missing ledgers to arrive, so a
830 // zero wait would turn that into a spin.
831 expectConfigRejected("recovery_wait_seconds must be at least 1 second", [](Config& cfg) {
833 });
834
835 // max_waiting_ledgers is the circuit breaker that eventually lets
836 // online delete stop waiting, so it has a floor rather than being
837 // free-form.
838 auto const tooFewWaiting =
839 "max_waiting_ledgers must be at least " + std::to_string(kMinWaitingLedgers);
840 expectConfigRejected(tooFewWaiting, [](Config& cfg) {
843 });
844 // 0 is not a magic "never give up" value, just a value below the floor.
845 expectConfigRejected(tooFewWaiting, [](Config& cfg) {
847 });
848
849 // The floor itself is accepted, and so is a value far above
850 // online_delete: there is no upper bound.
854 });
858 });
859
860 // All of the above is gated on online_delete being enabled. With it
861 // turned off, the same values are ignored rather than rejected.
863 auto& section = cfg.section(Sections::kNodeDatabase);
864 section.set(Keys::kOnlineDelete, "0");
865 section.set(Keys::kMaxWaitingLedgers, "0");
866 section.set(Keys::kRecoveryWaitSeconds, "0");
867 });
868 }
869
870 void
872 {
873 using namespace std::chrono_literals;
874
875 testcase("clearPrior");
876 using namespace jtx;
877
878 Env env(*this, envconfig(onlineDelete));
879
880 auto& store = env.app().getSHAMapStore();
881 env.fund(XRP(10000), noripple("alice"));
882
883 ledgerCheck(env, 1, 2);
884 transactionCheck(env, 0);
886
888
889 auto ledgerTmp = env.rpc("ledger", "0");
890 BEAST_EXPECT(bad(ledgerTmp, "lgrNotFound", RpcLgrNotFound, "ledgerNotFound"));
891
892 ledgers.emplace(1, env.rpc("ledger", "1"));
893 BEAST_EXPECT(goodLedger(env, ledgers[1], "1"));
894
895 ledgers.emplace(2, env.rpc("ledger", "2"));
896 BEAST_EXPECT(goodLedger(env, ledgers[2], "2"));
897
898 ledgerTmp = env.rpc("ledger", "current");
899 BEAST_EXPECT(goodLedger(env, ledgerTmp, "3"));
900
901 ledgerTmp = env.rpc("ledger", "4");
902 BEAST_EXPECT(bad(ledgerTmp, "lgrNotFound", RpcLgrNotFound, "ledgerNotFound"));
903
904 ledgerTmp = env.rpc("ledger", "100");
905 BEAST_EXPECT(bad(ledgerTmp, "lgrNotFound", RpcLgrNotFound, "ledgerNotFound"));
906
907 auto const firstSeq = waitForReady(env);
908 auto lastRotated = firstSeq - 1;
909
910 for (auto i = firstSeq + 1; i < kDeleteInterval + firstSeq; ++i)
911 {
912 env.fund(XRP(10000), noripple("test" + std::to_string(i)));
913 env.close();
914
915 ledgerTmp = env.rpc("ledger", "current");
916 BEAST_EXPECT(goodLedger(env, ledgerTmp, std::to_string(i)));
917 }
918 BEAST_EXPECT(store.getLastRotated() == lastRotated);
919
920 for (auto i = 3; i < kDeleteInterval + lastRotated; ++i)
921 {
922 ledgers.emplace(i, env.rpc("ledger", std::to_string(i)));
923 BEAST_EXPECT(
924 goodLedger(env, ledgers[i], std::to_string(i), true) &&
925 !getHash(ledgers[i]).empty());
926 }
927
928 ledgerCheck(env, kDeleteInterval + 1, 2);
931
932 {
933 // Closing one more ledger triggers a rotate
934 env.close();
935
936 auto const ledger = env.rpc("ledger", "current");
937 BEAST_EXPECT(goodLedger(env, ledger, std::to_string(kDeleteInterval + 4)));
938 }
939
940 BEAST_EXPECT(syncStore(env));
941
942 BEAST_EXPECT(store.getLastRotated() == kDeleteInterval + 3);
943 lastRotated = store.getLastRotated();
944 BEAST_EXPECT(lastRotated == 11);
945
946 // That took care of the fake hashes
947 ledgerCheck(env, kDeleteInterval + 1, 3);
950
951 // The last iteration of this loop should trigger a rotate
952 for (auto i = lastRotated - 1; i < lastRotated + kDeleteInterval - 1; ++i)
953 {
954 env.close();
955
956 ledgerTmp = env.rpc("ledger", "current");
957 BEAST_EXPECT(goodLedger(env, ledgerTmp, std::to_string(i + 3)));
958
959 ledgers.emplace(i, env.rpc("ledger", std::to_string(i)));
960 BEAST_EXPECT(
961 store.getLastRotated() == lastRotated || i == lastRotated + kDeleteInterval - 2);
962 BEAST_EXPECT(
963 goodLedger(env, ledgers[i], std::to_string(i), true) &&
964 !getHash(ledgers[i]).empty());
965 }
966
967 BEAST_EXPECT(syncStore(env));
968
969 BEAST_EXPECT(store.getLastRotated() == kDeleteInterval + lastRotated);
970
971 ledgerCheck(env, kDeleteInterval + 1, lastRotated);
972 transactionCheck(env, 0);
974 }
975
976 void
978 {
979 testcase("automatic online_delete");
980 using namespace jtx;
981 using namespace std::chrono_literals;
982
983 Env env(*this, envconfig(onlineDelete));
984 auto& store = env.app().getSHAMapStore();
985
986 auto ledgerSeq = waitForReady(env);
987 auto lastRotated = ledgerSeq - 1;
988 BEAST_EXPECT(store.getLastRotated() == lastRotated);
989 BEAST_EXPECT(lastRotated != 2);
990
991 // Because advisory_delete is unset,
992 // "can_delete" is disabled.
993 auto const canDelete = env.rpc("can_delete");
994 BEAST_EXPECT(bad(canDelete, "notEnabled", RpcNotEnabled, "Not enabled in configuration."));
995
996 // Close ledgers without triggering a rotate
997 for (; ledgerSeq < lastRotated + kDeleteInterval; ++ledgerSeq)
998 {
999 env.close();
1000
1001 auto const ledger = env.rpc("ledger", "validated");
1002 BEAST_EXPECT(goodLedger(env, ledger, std::to_string(ledgerSeq), true));
1003 }
1004
1005 BEAST_EXPECT(syncStore(env));
1006
1007 // The database will always have back to ledger 2,
1008 // regardless of lastRotated.
1009 ledgerCheck(env, ledgerSeq - 2, 2);
1010 BEAST_EXPECT(lastRotated == store.getLastRotated());
1011
1012 {
1013 // Closing one more ledger triggers a rotate
1014 env.close();
1015
1016 auto const ledger = env.rpc("ledger", "validated");
1017 BEAST_EXPECT(goodLedger(env, ledger, std::to_string(ledgerSeq++), true));
1018 }
1019
1020 BEAST_EXPECT(syncStore(env));
1021
1022 ledgerCheck(env, ledgerSeq - lastRotated, lastRotated);
1023 BEAST_EXPECT(lastRotated != store.getLastRotated());
1024
1025 lastRotated = store.getLastRotated();
1026
1027 // Close enough ledgers to trigger another rotate
1028 for (; ledgerSeq < lastRotated + kDeleteInterval + 1; ++ledgerSeq)
1029 {
1030 env.close();
1031
1032 auto const ledger = env.rpc("ledger", "validated");
1033 BEAST_EXPECT(goodLedger(env, ledger, std::to_string(ledgerSeq), true));
1034 }
1035
1036 BEAST_EXPECT(syncStore(env));
1037
1038 ledgerCheck(env, kDeleteInterval + 1, lastRotated);
1039 BEAST_EXPECT(lastRotated != store.getLastRotated());
1040 }
1041
1042 void
1044 {
1045 testcase("online_delete with advisory_delete");
1046 using namespace jtx;
1047 using namespace std::chrono_literals;
1048
1049 // Same config with advisory_delete enabled
1050 Env env(*this, envconfig(advisoryDelete));
1051 auto& store = env.app().getSHAMapStore();
1052
1053 auto ledgerSeq = waitForReady(env);
1054 auto lastRotated = ledgerSeq - 1;
1055 BEAST_EXPECT(store.getLastRotated() == lastRotated);
1056 BEAST_EXPECT(lastRotated != 2);
1057
1058 auto canDelete = env.rpc("can_delete");
1059 BEAST_EXPECT(!rpc::containsError(canDelete[jss::result]));
1060 BEAST_EXPECT(canDelete[jss::result][jss::can_delete] == 0);
1061
1062 canDelete = env.rpc("can_delete", "never");
1063 BEAST_EXPECT(!rpc::containsError(canDelete[jss::result]));
1064 BEAST_EXPECT(canDelete[jss::result][jss::can_delete] == 0);
1065
1066 auto const firstBatch = kDeleteInterval + ledgerSeq;
1067 for (; ledgerSeq < firstBatch; ++ledgerSeq)
1068 {
1069 env.close();
1070
1071 auto const ledger = env.rpc("ledger", "validated");
1072 BEAST_EXPECT(goodLedger(env, ledger, std::to_string(ledgerSeq), true));
1073 }
1074
1075 BEAST_EXPECT(syncStore(env));
1076
1077 ledgerCheck(env, ledgerSeq - 2, 2);
1078 BEAST_EXPECT(lastRotated == store.getLastRotated());
1079
1080 // This does not kick off a cleanup
1081 canDelete = env.rpc("can_delete", std::to_string(ledgerSeq + (kDeleteInterval / 2)));
1082 BEAST_EXPECT(!rpc::containsError(canDelete[jss::result]));
1083 BEAST_EXPECT(canDelete[jss::result][jss::can_delete] == ledgerSeq + (kDeleteInterval / 2));
1084
1085 BEAST_EXPECT(syncStore(env));
1086
1087 ledgerCheck(env, ledgerSeq - 2, 2);
1088 BEAST_EXPECT(store.getLastRotated() == lastRotated);
1089
1090 {
1091 // This kicks off a cleanup, but it stays small.
1092 env.close();
1093
1094 auto const ledger = env.rpc("ledger", "validated");
1095 BEAST_EXPECT(goodLedger(env, ledger, std::to_string(ledgerSeq++), true));
1096 }
1097
1098 BEAST_EXPECT(syncStore(env));
1099
1100 ledgerCheck(env, ledgerSeq - lastRotated, lastRotated);
1101
1102 BEAST_EXPECT(store.getLastRotated() == ledgerSeq - 1);
1103 lastRotated = ledgerSeq - 1;
1104
1105 for (; ledgerSeq < lastRotated + kDeleteInterval; ++ledgerSeq)
1106 {
1107 // No cleanups in this loop.
1108 env.close();
1109
1110 auto const ledger = env.rpc("ledger", "validated");
1111 BEAST_EXPECT(goodLedger(env, ledger, std::to_string(ledgerSeq), true));
1112 }
1113
1114 BEAST_EXPECT(syncStore(env));
1115
1116 BEAST_EXPECT(store.getLastRotated() == lastRotated);
1117
1118 {
1119 // This kicks off another cleanup.
1120 env.close();
1121
1122 auto const ledger = env.rpc("ledger", "validated");
1123 BEAST_EXPECT(goodLedger(env, ledger, std::to_string(ledgerSeq++), true));
1124 }
1125
1126 BEAST_EXPECT(syncStore(env));
1127
1128 ledgerCheck(env, ledgerSeq - firstBatch, firstBatch);
1129
1130 BEAST_EXPECT(store.getLastRotated() == ledgerSeq - 1);
1131 lastRotated = ledgerSeq - 1;
1132
1133 // This does not kick off a cleanup
1134 canDelete = env.rpc("can_delete", "always");
1135 BEAST_EXPECT(!rpc::containsError(canDelete[jss::result]));
1136 BEAST_EXPECT(
1137 canDelete[jss::result][jss::can_delete] == std::numeric_limits<unsigned int>::max());
1138
1139 for (; ledgerSeq < lastRotated + kDeleteInterval; ++ledgerSeq)
1140 {
1141 // No cleanups in this loop.
1142 env.close();
1143
1144 auto const ledger = env.rpc("ledger", "validated");
1145 BEAST_EXPECT(goodLedger(env, ledger, std::to_string(ledgerSeq), true));
1146 }
1147
1148 BEAST_EXPECT(syncStore(env));
1149
1150 BEAST_EXPECT(store.getLastRotated() == lastRotated);
1151
1152 {
1153 // This kicks off another cleanup.
1154 env.close();
1155
1156 auto const ledger = env.rpc("ledger", "validated");
1157 BEAST_EXPECT(goodLedger(env, ledger, std::to_string(ledgerSeq++), true));
1158 }
1159
1160 BEAST_EXPECT(syncStore(env));
1161
1162 ledgerCheck(env, ledgerSeq - lastRotated, lastRotated);
1163
1164 BEAST_EXPECT(store.getLastRotated() == ledgerSeq - 1);
1165 lastRotated = ledgerSeq - 1;
1166
1167 // This does not kick off a cleanup
1168 canDelete = env.rpc("can_delete", "now");
1169 BEAST_EXPECT(!rpc::containsError(canDelete[jss::result]));
1170 BEAST_EXPECT(canDelete[jss::result][jss::can_delete] == ledgerSeq - 1);
1171
1172 for (; ledgerSeq < lastRotated + kDeleteInterval; ++ledgerSeq)
1173 {
1174 // No cleanups in this loop.
1175 env.close();
1176
1177 auto const ledger = env.rpc("ledger", "validated");
1178 BEAST_EXPECT(goodLedger(env, ledger, std::to_string(ledgerSeq), true));
1179 }
1180
1181 BEAST_EXPECT(syncStore(env));
1182
1183 BEAST_EXPECT(store.getLastRotated() == lastRotated);
1184
1185 {
1186 // This kicks off another cleanup.
1187 env.close();
1188
1189 auto const ledger = env.rpc("ledger", "validated");
1190 BEAST_EXPECT(goodLedger(env, ledger, std::to_string(ledgerSeq++), true));
1191 }
1192
1193 BEAST_EXPECT(syncStore(env));
1194
1195 ledgerCheck(env, ledgerSeq - lastRotated, lastRotated);
1196
1197 BEAST_EXPECT(store.getLastRotated() == ledgerSeq - 1);
1198 lastRotated = ledgerSeq - 1;
1199 }
1200
1203 {
1205 std::filesystem::path newPath;
1206
1207 if (!BEAST_EXPECT(path.size()))
1208 return {};
1209 newPath = path;
1210 section.set(Keys::kPath, newPath.string());
1211
1212 auto backend{
1214 section,
1215 megabytes(env.app().config().getValueFor(SizedItem::BurstSize, std::nullopt)),
1216 scheduler,
1217 env.app().getJournal("NodeStoreTest")),
1218 };
1219 backend->open();
1220 return backend;
1221 }
1222
1223 void
1225 {
1226 // The only purpose of this test is to ensure that if something that
1227 // should never happen happens, we don't get a deadlock.
1228 testcase("rotate with lock contention");
1229
1230 using namespace jtx;
1231 Env env(*this, envconfig(onlineDelete));
1232
1234 // Create NodeStore with two backends to allow online deletion of data.
1235 // Normally, SHAMapStoreImp handles all these details.
1236 auto nscfg = env.app().config().section(Sections::kNodeDatabase);
1237
1238 // Provide default values.
1239 if (!nscfg.exists(Keys::kCacheSize))
1240 {
1241 nscfg.set(
1244 env.app().config().getValueFor(SizedItem::TreeCacheSize, std::nullopt)));
1245 }
1246
1247 if (!nscfg.exists(Keys::kCacheAge))
1248 {
1249 nscfg.set(
1252 env.app().config().getValueFor(SizedItem::TreeCacheAge, std::nullopt)));
1253 }
1254
1255 NodeStoreScheduler scheduler(env.app().getJobQueue());
1256
1257 std::string const writableDb = "write";
1258 std::string const archiveDb = "archive";
1259 auto writableBackend = makeBackendRotating(env, scheduler, writableDb);
1260 auto archiveBackend = makeBackendRotating(env, scheduler, archiveDb);
1261
1262 static constexpr int kReadThreads = 4;
1264 scheduler,
1265 kReadThreads,
1266 std::move(writableBackend),
1267 std::move(archiveBackend),
1268 nscfg,
1269 env.app().getJournal("NodeStoreTest"));
1270
1272 // Check basic functionality
1273 using namespace std::chrono_literals;
1274 std::atomic<int> threadNum = 0;
1275
1276 {
1277 auto newBackend = makeBackendRotating(env, scheduler, std::to_string(++threadNum));
1278
1279 auto const cb = [&](std::string const& writableName, std::string const& archiveName) {
1280 BEAST_EXPECT(writableName == "1");
1281 BEAST_EXPECT(archiveName == "write");
1282 // Ensure that dbr functions can be called from within the
1283 // callback
1284 BEAST_EXPECT(dbr->getName() == "1");
1285 };
1286
1287 dbr->rotate(std::move(newBackend), cb);
1288 }
1289 BEAST_EXPECT(threadNum == 1);
1290 BEAST_EXPECT(dbr->getName() == "1");
1291
1293 // Do something stupid. Try to re-enter rotate from inside the callback.
1294 {
1295 auto const cb = [&](std::string const& writableName, std::string const& archiveName) {
1296 BEAST_EXPECT(writableName == "3");
1297 BEAST_EXPECT(archiveName == "2");
1298 // Ensure that dbr functions can be called from within the
1299 // callback
1300 BEAST_EXPECT(dbr->getName() == "3");
1301 };
1302 auto const cbReentrant = [&](std::string const& writableName,
1303 std::string const& archiveName) {
1304 BEAST_EXPECT(writableName == "2");
1305 BEAST_EXPECT(archiveName == "1");
1306 auto newBackend = makeBackendRotating(env, scheduler, std::to_string(++threadNum));
1307 // Reminder: doing this is stupid and should never happen
1308 dbr->rotate(std::move(newBackend), cb);
1309 };
1310 auto newBackend = makeBackendRotating(env, scheduler, std::to_string(++threadNum));
1311 dbr->rotate(std::move(newBackend), cbReentrant);
1312 }
1313
1314 BEAST_EXPECT(threadNum == 3);
1315 BEAST_EXPECT(dbr->getName() == "3");
1316 }
1317
1318 void
1320 {
1321 // Note that this test is intentionally very similar to
1322 // LedgerMaster_test::testCompleteLedgerRange, but has a different
1323 // focus.
1324
1325 testcase("Wait for ledger gaps to fill in");
1326
1327 using namespace test::jtx;
1328
1329 Env env{*this, envconfig(onlineDelete)};
1330
1331 auto failureMessage = [&](char const* label, auto expected, auto actual) {
1333 ss << label << ": Expected: " << expected << ", Got: " << actual;
1334 return ss.str();
1335 };
1336
1337 auto const alice = Account("alice");
1338 env.fund(XRP(1000), alice);
1339 env.close();
1340
1341 auto& lm = env.app().getLedgerMaster();
1342 LedgerIndex minSeq = 2;
1343 auto& store = env.app().getSHAMapStore();
1344 auto& netOPs = env.app().getOPs();
1345 // Which of the existing complete ledgers the store initializes
1346 // lastRotated from is a timing detail, so everything below derives from
1347 // the observed value rather than assuming a particular one. Spinning
1348 // until it equals a hard-coded value never terminates when a different
1349 // one legitimately wins.
1350 //
1351 // The range check and the initializeStore() one both end the testcase
1352 // rather than merely reporting, because lastRotated is the only value
1353 // from the store that enters minSeq. A lastRotated of 0 -- the value
1354 // getLastRotated() reports until the store has been handed a validated
1355 // ledger -- makes minSeq 0 below, and the minSeq - 1 and minSeq - 2
1356 // ranges then underflow to first > last, which aborts a Debug build
1357 // inside missingFromCompleteLedgerRange().
1358 auto const extraCloses = initializeStore(env);
1359 if (!BEAST_EXPECT(extraCloses.has_value()))
1360 return;
1361 LedgerIndex maxSeq = env.closed()->header().seq;
1362 LedgerIndex lastRotated = store.getLastRotated();
1363 if (!BEAST_EXPECTS(
1364 lastRotated >= minSeq && lastRotated <= maxSeq, std::to_string(lastRotated)))
1365 return;
1366 // The BEAST_EXPECT above already returned if this is nullopt, but that
1367 // is invisible to clang-tidy's optional model.
1368 // NOLINTNEXTLINE(bugprone-unchecked-optional-access)
1369 BEAST_EXPECTS(maxSeq == 3 + *extraCloses, std::to_string(maxSeq));
1370 std::stringstream initialRange;
1371 initialRange << minSeq << "-" << maxSeq;
1372 BEAST_EXPECTS(lm.getCompleteLedgers() == initialRange.str(), lm.getCompleteLedgers());
1373 BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 0);
1374 // The inner range is empty unless initializeStore() had to close extra
1375 // ledgers, and missingFromCompleteLedgerRange() treats first > last as a
1376 // precondition violation that aborts a Debug build via UNREACHABLE, so
1377 // only check it when it is well formed.
1378 if (minSeq + 1 <= maxSeq - 1)
1379 {
1380 BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 1, maxSeq - 1) == 0);
1381 }
1382 BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 1, maxSeq + 1) == 2);
1383 BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 2, maxSeq - 2) == 2);
1384 BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 2, maxSeq + 2) == 2);
1385
1386 auto expectedRange =
1387 [](LedgerIndex minSeq, std::vector<LedgerIndex> const& deleteSeqs, LedgerIndex maxSeq) {
1388 std::stringstream expectedRange;
1389 expectedRange << minSeq;
1390 auto lastDelete = minSeq - 1;
1391 for (auto const deleteSeq : deleteSeqs)
1392 {
1393 if (deleteSeq <= lastDelete)
1394 continue;
1395 expectedRange << "-" << (deleteSeq - 1);
1396 if (deleteSeq + 1 <= maxSeq)
1397 expectedRange << "," << (deleteSeq + 1);
1398 lastDelete = deleteSeq;
1399 }
1400 if (lastDelete + 1 < maxSeq)
1401 {
1402 expectedRange << "-" << maxSeq;
1403 }
1404 return expectedRange.str();
1405 };
1406
1407 auto const deleteLedgerSeq =
1408 [&lm, &store, &netOPs, &minSeq, &lastRotated, &expectedRange, &failureMessage, this](
1409 Env& env,
1410 LedgerIndex& maxSeq,
1411 std::vector<LedgerIndex>& deleteSeqs) -> LedgerIndex {
1412 using namespace std::chrono_literals;
1413
1414 // The next ledger will trigger a rotation. Delete the
1415 // current ledger from LedgerMaster.
1416
1417 netOPs.setMode(OperatingMode::CONNECTED);
1418
1419 LedgerIndex const deleteSeq = maxSeq;
1420 std::size_t iterations = 30;
1421 while (!lm.haveLedger(deleteSeq) && --iterations > 0)
1422 {
1424 }
1425 // Even the slowest machines should be able to finalize deleteSeq within 10
1426 // loops (100ms). If this test ever actually fails feel free to lower this
1427 // cutoff. The intent of this test is to flag if the loop takes a very long
1428 // time, but still allow the rest of this function to finish.
1429 BEAST_EXPECTS(iterations > 20, std::to_string(iterations));
1430 if (!BEAST_EXPECT(lm.haveLedger(deleteSeq)))
1431 return 0;
1432
1433 // This test may be timing sensitive, because it's messing with server internals in ways
1434 // that they can't be messed with normally. Sleep a little bit to give the server time
1435 // to finish any internal work before we delete the ledger.
1437
1438 lm.clearLedger(deleteSeq);
1439 deleteSeqs.push_back(deleteSeq);
1440 if (!BEAST_EXPECT(!lm.haveLedger(deleteSeq)))
1441 return 0;
1442
1443 BEAST_EXPECTS(
1444 lm.getCompleteLedgers() == expectedRange(minSeq, deleteSeqs, maxSeq),
1445 failureMessage(
1446 "Complete ledgers",
1447 expectedRange(minSeq, deleteSeqs, maxSeq),
1448 lm.getCompleteLedgers()));
1449 BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == deleteSeqs.size());
1450
1451 if (!BEAST_EXPECT(!lm.haveLedger(deleteSeq)))
1452 return 0;
1453 // Close another ledger, which will trigger a rotation, but the
1454 // rotation will be stuck until the missing ledger is filled in.
1455 env.close();
1456 // Do not call rendezvous() here without a timeout; it will block until the missing
1457 // ledger is backfilled. That will not happen automatically. It's a manual step that
1458 // is done later in this test.
1459 ++maxSeq;
1460
1461 if (!BEAST_EXPECT(!lm.haveLedger(deleteSeq)))
1462 return 0;
1463 netOPs.setMode(OperatingMode::FULL);
1464
1465 if (!BEAST_EXPECT(!lm.haveLedger(deleteSeq)))
1466 return 0;
1467 BEAST_EXPECT(!store.rendezvous(10ms));
1468 BEAST_EXPECT(netOPs.getOperatingMode() == OperatingMode::FULL);
1469
1470 // Nothing has changed
1471 BEAST_EXPECTS(
1472 store.getLastRotated() == lastRotated,
1473 failureMessage("lastRotated", lastRotated, store.getLastRotated()));
1474 BEAST_EXPECTS(
1475 lm.getCompleteLedgers() == expectedRange(minSeq, deleteSeqs, maxSeq),
1476 failureMessage(
1477 "Complete ledgers",
1478 expectedRange(minSeq, deleteSeqs, maxSeq),
1479 lm.getCompleteLedgers()));
1480
1481 return deleteSeq;
1482 };
1483
1484 std::vector<LedgerIndex> deleteSeqs;
1485
1486 // Close enough ledgers to rotate a few times
1487 while (maxSeq < 40)
1488 {
1489 for (int t = 0; t < 3; ++t)
1490 {
1491 env(noop(alice));
1492 }
1493 env.close();
1494 BEAST_EXPECT(syncStore(env));
1495
1496 ++maxSeq;
1497
1498 if (maxSeq + 1 == lastRotated + kDeleteInterval)
1499 {
1500 using namespace std::chrono_literals;
1501
1502 {
1503 // Trigger the circuit breaker in SHAMapStoreImp::healthWait() to ensure it
1504 // doesn't block forever.
1505 LedgerIndex const deleteSeq = deleteLedgerSeq(env, maxSeq, deleteSeqs);
1506 if (!BEAST_EXPECT(deleteSeq > 0))
1507 return;
1508 if (!BEAST_EXPECT(!lm.haveLedger(deleteSeq)))
1509 return;
1510
1511 // Close 7 more ledgers, waiting a little bit in between to
1512 // simulate the ledger making progress while online delete waits
1513 // for the missing ledger to be filled in.
1514 // After the 7th ledger, the circuit breaker will trigger and abort the attempt.
1515 while (maxSeq < lastRotated + (kDeleteInterval * 2) - 2)
1516 {
1517 env.close();
1518 ++maxSeq;
1519 // Nothing has changed
1520 BEAST_EXPECTS(
1521 store.getLastRotated() == lastRotated,
1522 failureMessage("lastRotated", lastRotated, store.getLastRotated()));
1523 BEAST_EXPECTS(
1524 lm.getCompleteLedgers() == expectedRange(minSeq, deleteSeqs, maxSeq),
1525 failureMessage(
1526 "Complete Ledgers",
1527 expectedRange(minSeq, deleteSeqs, maxSeq),
1528 lm.getCompleteLedgers()));
1529 // The Store is "stuck" in healthWait() and won't finish the run() loop
1530 // until it's backfilled
1531 if (!BEAST_EXPECT(!lm.haveLedger(deleteSeq)))
1532 return;
1533 }
1534
1535 // Close one more ledger, which will NOT trigger the circuit breaker. Wait for
1536 // the full 1 second recovery wait timeout to ensure the circuit breaker is not
1537 // triggered.
1538 env.close();
1539 ++maxSeq;
1540 // The Store is "stuck" in healthWait() and won't finish the run() loop
1541 // until it's backfilled
1542 BEAST_EXPECT(!store.rendezvous(1s));
1543
1544 // Close one more ledger, which will trigger the circuit breaker and abort the
1545 // attempt to rotate.
1546 env.close();
1547 ++maxSeq;
1548 // Nothing has changed
1549 BEAST_EXPECTS(
1550 store.getLastRotated() == lastRotated,
1551 failureMessage("lastRotated", lastRotated, store.getLastRotated()));
1552 BEAST_EXPECTS(
1553 lm.getCompleteLedgers() == expectedRange(minSeq, deleteSeqs, maxSeq),
1554 failureMessage(
1555 "Complete Ledgers",
1556 expectedRange(minSeq, deleteSeqs, maxSeq),
1557 lm.getCompleteLedgers()));
1558
1559 // The circuit breaker has been triggered.
1560 BEAST_EXPECT(syncStore(env));
1561 }
1562 {
1563 // Recover before the circuit breaker triggers, so the test can continue.
1564 LedgerIndex const deleteSeq = deleteLedgerSeq(env, maxSeq, deleteSeqs);
1565 if (!BEAST_EXPECT(deleteSeq > 0))
1566 return;
1567 if (!BEAST_EXPECT(!lm.haveLedger(deleteSeq)))
1568 return;
1569
1570 // Close 5 more ledgers, waiting a little bit in between to
1571 // simulate the ledger making progress while online delete waits
1572 // for the missing ledger to be filled in.
1573 // This ensures the healthWait check has time to run and
1574 // detect the gap.
1575 for (int l = 0; l < 5; ++l)
1576 {
1577 env.close();
1578 ++maxSeq;
1579 // Nothing has changed
1580 BEAST_EXPECTS(
1581 store.getLastRotated() == lastRotated,
1582 failureMessage("lastRotated", lastRotated, store.getLastRotated()));
1583 BEAST_EXPECTS(
1584 lm.getCompleteLedgers() == expectedRange(minSeq, deleteSeqs, maxSeq),
1585 failureMessage(
1586 "Complete Ledgers",
1587 expectedRange(minSeq, deleteSeqs, maxSeq),
1588 lm.getCompleteLedgers()));
1589 if (!BEAST_EXPECT(!lm.haveLedger(deleteSeq)))
1590 return;
1591 }
1592
1593 // The Store is "stuck" in healthWait() and won't finish the run() loop
1594 // until it's backfilled
1595 // Wait for the full 1 second recovery wait timeout to ensure the circuit
1596 // breaker is not triggered, and this isn't some other timing fluke.
1597 BEAST_EXPECT(!store.rendezvous(1s));
1598
1599 // Put the missing ledger back in LedgerMaster
1600 lm.setLedgerRangePresent(deleteSeq, deleteSeq);
1601 BEAST_EXPECT(deleteSeqs.back() == deleteSeq);
1602 deleteSeqs.pop_back();
1603
1604 // Wait for the rotation to finish
1605 BEAST_EXPECT(syncStore(env));
1606
1607 minSeq = lastRotated;
1608 while (deleteSeqs.front() < minSeq)
1609 {
1610 deleteSeqs.erase(deleteSeqs.begin());
1611 }
1612 lastRotated = deleteSeq + 1;
1613 }
1614 }
1615 BEAST_EXPECT(maxSeq != lastRotated + kDeleteInterval);
1616 BEAST_EXPECTS(
1617 env.closed()->header().seq == maxSeq,
1618 failureMessage("maxSeq", maxSeq, env.closed()->header().seq));
1619 BEAST_EXPECTS(
1620 store.getLastRotated() == lastRotated,
1621 failureMessage("lastRotated", lastRotated, store.getLastRotated()));
1622 {
1623 auto const expected = expectedRange(minSeq, deleteSeqs, maxSeq);
1624 BEAST_EXPECTS(
1625 lm.getCompleteLedgers() == expected,
1626 failureMessage("CompleteLedgers", expected, lm.getCompleteLedgers()));
1627 }
1628 BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == deleteSeqs.size());
1629 // missingFromCompleteLedgerRange() treats first > last as a
1630 // precondition violation and aborts a Debug build via UNREACHABLE.
1631 // The range can only collapse if this test's model of minSeq /
1632 // maxSeq has desynced from the store, so report that as a failure
1633 // instead of taking down the whole unit test job.
1634 if (minSeq + 1 <= maxSeq - 1)
1635 {
1636 BEAST_EXPECT(
1637 lm.missingFromCompleteLedgerRange(minSeq + 1, maxSeq - 1) == deleteSeqs.size());
1638 }
1639 else
1640 {
1641 BEAST_EXPECTS(false, failureMessage("range collapsed", minSeq, maxSeq));
1642 }
1643 BEAST_EXPECT(
1644 lm.missingFromCompleteLedgerRange(minSeq - 1, maxSeq + 1) == deleteSeqs.size() + 2);
1645 BEAST_EXPECT(
1646 lm.missingFromCompleteLedgerRange(minSeq - 2, maxSeq - 2) == deleteSeqs.size() + 2);
1647 BEAST_EXPECT(
1648 lm.missingFromCompleteLedgerRange(minSeq + 2, maxSeq + 2) == deleteSeqs.size() + 2);
1649 }
1650 }
1651
1652 // Cover the branches of SHAMapStoreImp::healthWait() that decide whether
1653 // the server is healthy enough to rotate, and how loudly to complain while
1654 // it is not. testLedgerGaps() covers the case where a gap holds the
1655 // rotation back until the circuit breaker trips; these cover the rest of
1656 // the decision table.
1657 void
1659 {
1660 testcase("healthWait server state");
1661
1662 using namespace std::chrono_literals;
1663 using namespace test::jtx;
1664
1665 auto logs = std::make_unique<StoreLogs>();
1666 // Not `auto const*`: waitFor() blocks, so it is not const.
1667 auto* const log = logs.get();
1668 Env env{*this, envconfig(onlineDelete), std::move(logs), beast::Severity::Trace};
1669
1670 auto& store = env.app().getSHAMapStore();
1671 auto& netOPs = env.app().getOPs();
1672
1673 // No gap: the only thing holding the store back is the operating mode.
1674 auto const parked = parkInHealthWait(env, false, OperatingMode::CONNECTED);
1675 if (!parked)
1676 return;
1677
1678 // Hold the non-FULL mode until the store has logged that it is waiting
1679 // on it. With no gap, a fresh validated ledger and a mode that is not
1680 // DISCONNECTED, the only check left that can report unhealthy is
1681 // "mode != FULL", and a mode that is not FULL is not expected to fix
1682 // itself, so the wait is logged at warn, for the full duration.
1683 //
1684 // Waiting for the message rather than sleeping past it is what keeps
1685 // this from depending on how quickly the store gets around to sampling.
1686 // The store cannot leave the wait loop while the mode stays put -- the
1687 // validated ledger index does not advance, so the circuit breaker is
1688 // never reached -- so the rendezvous() below is not racing it.
1689 BEAST_EXPECT(log->waitFor(beast::Severity::Warning, kFullWait, 10s));
1690 BEAST_EXPECT(!store.rendezvous(10ms));
1691 BEAST_EXPECT(netOPs.getOperatingMode() != OperatingMode::FULL);
1692 BEAST_EXPECT(netOPs.getOperatingMode() != OperatingMode::DISCONNECTED);
1693 BEAST_EXPECTS(
1694 store.getLastRotated() == parked->lastRotated, std::to_string(store.getLastRotated()));
1695
1696 // Now make the mode FULL but the validated ledger stale. Advancing the
1697 // clock without closing a ledger ages the validated ledger past
1698 // age_threshold_seconds, which defaults to 60. The mode check can no
1699 // longer be the reason the store is unhealthy, so the age check is.
1700 //
1701 // This one does sleep: what is being asserted is that the store did not
1702 // rotate, and 1500ms is long enough for it to have re-sampled the server
1703 // at least once -- the full wait is 1000ms -- so the age check, not a
1704 // stale sample of the old mode, is what held it back.
1705 auto const closeTime = env.now();
1706 env.timeKeeper().set(closeTime + 2min);
1707 netOPs.setMode(OperatingMode::FULL);
1708 BEAST_EXPECT(netOPs.getOperatingMode() == OperatingMode::FULL);
1709 BEAST_EXPECT(!store.rendezvous(1500ms));
1710 BEAST_EXPECTS(
1711 store.getLastRotated() == parked->lastRotated, std::to_string(store.getLastRotated()));
1712
1713 // Restore the clock. Nothing is wrong any more, so the rotation that
1714 // has been waiting all along runs to completion.
1715 env.timeKeeper().set(closeTime);
1716 BEAST_EXPECT(syncStore(env));
1717 BEAST_EXPECTS(
1718 store.getLastRotated() == parked->validated, std::to_string(store.getLastRotated()));
1719 }
1720
1721 void
1723 {
1724 testcase("healthWait gap wait levels");
1725
1726 using namespace std::chrono_literals;
1727 using namespace test::jtx;
1728
1729 auto logs = std::make_unique<StoreLogs>();
1730 // Not `auto const*`: waitFor() blocks, so it is not const.
1731 auto* const log = logs.get();
1732 Env env{*this, envconfig(onlineDelete), std::move(logs), beast::Severity::Trace};
1733
1734 auto& lm = env.app().getLedgerMaster();
1735 auto& store = env.app().getSHAMapStore();
1736
1737 auto const parked = parkInHealthWait(env, true, OperatingMode::FULL);
1738 if (!parked)
1739 return;
1740
1741 // The missing ledger is an older one; the validated ledger itself is
1742 // present. The store has no reason to think the gap will close on its
1743 // own, so it waits the full duration and says so at info -- not warn,
1744 // because the server is otherwise healthy and has not been waiting long
1745 // enough to have fallen behind.
1746 BEAST_EXPECT(lm.haveLedger(parked->validated));
1747 BEAST_EXPECT(!lm.haveLedger(parked->gap));
1748 BEAST_EXPECT(log->waitFor(beast::Severity::Info, kFullWait, 10s));
1749 // Nothing so far should have looked like a ledger being built.
1750 BEAST_EXPECT(log->count(beast::Severity::Trace, kShortWait) == 0);
1751 // Nothing fills the gap in, so the store is still in the wait loop.
1752 BEAST_EXPECT(!store.rendezvous(10ms));
1753
1754 // Move the gap onto the validated ledger itself. That is the one case
1755 // the store treats as transient -- the ledger is expected to be built
1756 // shortly -- so it drops to trace and waits a tenth as long. Asserting
1757 // that the shortened wait appears only after this swap is what pins the
1758 // branch to the buildingIndex condition, rather than to anything
1759 // incidental about a store that happens to be waiting.
1760 lm.setLedgerRangePresent(parked->gap, parked->gap);
1761 lm.clearLedger(parked->validated);
1762 BEAST_EXPECT(lm.haveLedger(parked->gap));
1763 BEAST_EXPECT(!lm.haveLedger(parked->validated));
1764 BEAST_EXPECT(log->waitFor(beast::Severity::Trace, kShortWait, 10s));
1765 BEAST_EXPECT(!store.rendezvous(10ms));
1766 BEAST_EXPECTS(
1767 store.getLastRotated() == parked->lastRotated, std::to_string(store.getLastRotated()));
1768
1769 // Fill it in and the rotation completes.
1770 lm.setLedgerRangePresent(parked->validated, parked->validated);
1771 BEAST_EXPECT(syncStore(env));
1772 BEAST_EXPECTS(
1773 store.getLastRotated() == parked->validated, std::to_string(store.getLastRotated()));
1774 }
1775
1776 void
1778 {
1779 testcase("healthWait disconnected");
1780
1781 using namespace std::chrono_literals;
1782 using namespace test::jtx;
1783
1784 Env env{*this, envconfig(onlineDelete)};
1785
1786 auto const& lm = env.app().getLedgerMaster();
1787 auto& store = env.app().getSHAMapStore();
1788 auto& netOPs = env.app().getOPs();
1789
1790 auto const parked = parkInHealthWait(env, true, OperatingMode::FULL);
1791 if (!parked)
1792 return;
1793
1794 // While the server is FULL, the gap holds the rotation back.
1795 BEAST_EXPECT(!store.rendezvous(1500ms));
1796 BEAST_EXPECTS(
1797 store.getLastRotated() == parked->lastRotated, std::to_string(store.getLastRotated()));
1798
1799 // A disconnected server is not doing any ledger I/O, so the gap cannot
1800 // have been caused by its own activity and will not close until it has
1801 // peers again. The store deliberately takes advantage of that to get as
1802 // much rotation done as possible: this is the one case where a gap does
1803 // not hold online delete back at all.
1804 netOPs.setMode(OperatingMode::DISCONNECTED);
1805 BEAST_EXPECT(netOPs.getOperatingMode() == OperatingMode::DISCONNECTED);
1806 BEAST_EXPECT(syncStore(env));
1807 BEAST_EXPECTS(
1808 store.getLastRotated() == parked->validated, std::to_string(store.getLastRotated()));
1809 // The rotation ran with the gap still present -- nothing filled it in.
1810 BEAST_EXPECT(!lm.haveLedger(parked->gap));
1811 }
1812
1813 void
1815 {
1816 testcase("healthWait stop");
1817
1818 using namespace test::jtx;
1819
1820 Env env{*this, envconfig(onlineDelete)};
1821
1822 auto& store = env.app().getSHAMapStore();
1823
1824 auto const parked = parkInHealthWait(env, true, OperatingMode::FULL);
1825 if (!parked)
1826 return;
1827
1828 // Stopping the store has to break it out of the wait loop, which it
1829 // would otherwise never leave: the gap is never filled in and the
1830 // validated ledger index never advances to reach the circuit breaker.
1831 //
1832 // stop() joins the store's thread, so its return is the
1833 // synchronisation point here. rendezvous() afterwards is only a
1834 // cross-check, and cannot block: the store is parked in the health
1835 // check that gates a rotation, so Stopping there merely leaves
1836 // readyToRotate false, and run() falls through to the top of its loop,
1837 // where it clears working_ and notifies before returning on stop_.
1838 store.stop();
1839 BEAST_EXPECT(store.rendezvous());
1840 BEAST_EXPECTS(
1841 store.getLastRotated() == parked->lastRotated, std::to_string(store.getLastRotated()));
1842 }
1843
1844 // The two tests below cover the health check that run() performs between the
1845 // stages of a rotation it has already committed to, which is a different
1846 // decision from the one that gates the rotation in the first place: giving
1847 // up here means abandoning work in progress. run() makes it at four points
1848 // -- after clearing prior ledgers, after copying the validated ledger, after
1849 // freshening the caches, and after clearing them -- with the same three-way
1850 // switch each time, and parkMidRotation() parks the store at the first of
1851 // them.
1852 void
1854 {
1855 testcase("healthWait circuit breaker mid-rotation");
1856
1857 using namespace test::jtx;
1858
1859 auto logs = std::make_unique<StoreLogs>();
1860 auto* const log = logs.get();
1861 Env env{*this, envconfig(slowOnlineDelete), std::move(logs), beast::Severity::Trace};
1862
1863 auto const& lm = env.app().getLedgerMaster();
1864 auto& store = env.app().getSHAMapStore();
1865
1866 auto const parked = parkMidRotation(env, *log);
1867 if (!parked)
1868 return;
1869
1870 // Advance the validated ledger index past the circuit breaker. The store
1871 // has had no successful health check since the gap appeared, so once the
1872 // index has moved max_waiting_ledgers on from the last one that did
1873 // succeed, it abandons the rotation instead of waiting for the gap
1874 // forever. Nothing here fills the gap in.
1875 for (int i = 0; i < kMinWaitingLedgers; ++i)
1876 {
1877 env.close();
1878 BEAST_EXPECT(!lm.haveLedger(parked->gap));
1879 }
1880
1881 // Abandoning the rotation returns the store to waiting for work, so it
1882 // reports itself idle -- but with lastRotated left where it started,
1883 // unlike the completed rotation parkMidRotation() drove first.
1884 BEAST_EXPECT(syncStore(env));
1885 BEAST_EXPECTS(
1886 store.getLastRotated() == parked->lastRotated, std::to_string(store.getLastRotated()));
1887 BEAST_EXPECT(log->count(kExpired) > 0);
1888 BEAST_EXPECTS(log->count(kFinished) == 1, std::to_string(log->count(kFinished)));
1889 BEAST_EXPECT(!lm.haveLedger(parked->gap));
1890 }
1891
1892 void
1894 {
1895 testcase("healthWait stop mid-rotation");
1896
1897 using namespace test::jtx;
1898
1899 auto logs = std::make_unique<StoreLogs>();
1900 auto* const log = logs.get();
1901 Env env{*this, envconfig(slowOnlineDelete), std::move(logs), beast::Severity::Trace};
1902
1903 auto& store = env.app().getSHAMapStore();
1904
1905 auto const parked = parkMidRotation(env, *log);
1906 if (!parked)
1907 return;
1908
1909 // Stopping has to break the store out of the rotation, which it would
1910 // otherwise never leave: the gap is never filled in and the validated
1911 // ledger index never advances to reach the circuit breaker. Note that
1912 // being stopped outranks being healthy -- the health check reports it
1913 // even when nothing is wrong with the server -- so this does not depend
1914 // on the store still being parked when stop() lands.
1915 //
1916 // stop() joins the store's thread, so its return is the synchronisation
1917 // point. Deliberately do not call the untimed rendezvous() afterwards:
1918 // run() returns without clearing working_, so it would block forever.
1919 store.stop();
1920 BEAST_EXPECTS(
1921 store.getLastRotated() == parked->lastRotated, std::to_string(store.getLastRotated()));
1922 // The rotation was abandoned rather than completed, and the circuit
1923 // breaker was not what abandoned it.
1924 BEAST_EXPECTS(log->count(kFinished) == 1, std::to_string(log->count(kFinished)));
1925 BEAST_EXPECTS(log->count(kExpired) == 0, std::to_string(log->count(kExpired)));
1926 }
1927
1928 void
1944};
1945
1946// VFALCO This test fails because of thread asynchronous issues
1948
1949} // namespace xrpl::test
Abstraction for the underlying message destination.
Definition Journal.h:59
A testsuite class.
Definition suite.h:52
LogOs< char > log
Logging output stream.
Definition suite.h:150
TestcaseT testcase
Memberspace for declaring test cases.
Definition suite.h:155
Represents a JSON value.
Definition json_value.h:121
virtual Config & config()=0
Section & section(std::string const &name)
Returns the section with the given name.
std::uint32_t ledgerHistory
int getValueFor(SizedItem item, std::optional< std::size_t > node=std::nullopt) const
Retrieve the default value for the item at the specified node size.
void rendezvous()
Block until no jobs running.
Definition JobQueue.cpp:243
Logs(beast::Severity level)
Definition Log.cpp:112
A node_store::Scheduler which uses the JobQueue.
virtual std::size_t getAccountTransactionCount()=0
getAccountTransactionCount Returns the number of account transactions.
virtual std::size_t getTransactionCount()=0
getTransactionCount Returns the number of transactions.
virtual CountMinMax getLedgerCountMinMax()=0
getLedgerCountMinMax Returns the minimum ledger sequence, maximum ledger sequence and total number of...
virtual std::optional< LedgerHeader > getLedgerInfoByIndex(LedgerIndex ledgerSeq)=0
getLedgerInfoByIndex Returns a ledger by its sequence.
class to create database, launch online delete thread, and related SQLite database
Definition SHAMapStore.h:25
virtual bool rendezvous(std::optional< std::chrono::milliseconds > const &timeout={}) const =0
Holds a collection of configuration values.
Definition BasicConfig.h:28
void set(std::string const &key, std::string const &value)
Set a key/value pair.
virtual JobQueue & getJobQueue()=0
virtual RelationalDatabase & getRelationalDatabase()=0
virtual beast::Journal getJournal(std::string const &name)=0
virtual NetworkOPs & getOPs()=0
virtual SHAMapStore & getSHAMapStore()=0
virtual LedgerMaster & getLedgerMaster()=0
virtual std::unique_ptr< Backend > makeBackend(Section const &parameters, std::size_t burstSize, Scheduler &scheduler, beast::Journal journal)=0
Create a backend.
static Manager & instance()
Returns the instance of the manager singleton.
void write(beast::Severity level, std::string const &text) override
Write text to the sink at the specified severity.
Sink(beast::Severity threshold, StoreLogs &owner)
void threshold(beast::Severity) override
Set the minimum severity this sink will report.
void writeAlways(beast::Severity level, std::string const &text) override
Bypass filter and write text to the sink at the specified severity.
std::size_t countLocked(std::optional< beast::Severity > severity, std::string const &text) const
bool waitFor(std::string const &text, std::chrono::milliseconds timeout, std::size_t expected=1)
std::unique_ptr< beast::Journal::Sink > makeSink(std::string const &partition, beast::Severity) override
bool waitFor(std::optional< beast::Severity > severity, std::string const &text, std::chrono::milliseconds timeout, std::size_t expected=1)
std::vector< std::pair< beast::Severity, std::string > > messages_
std::size_t count(std::string const &text) const
std::size_t count(beast::Severity severity, std::string const &text) const
static auto onlineDelete(std::unique_ptr< Config > cfg)
static auto advisoryDelete(std::unique_ptr< Config > cfg)
std::optional< Parked > parkInHealthWait(jtx::Env &env, bool createGap, OperatingMode modeAfterClose)
std::optional< int > initializeStore(jtx::Env &env, int const maxExtraCloses=3)
static constexpr char const * kShortWait
static constexpr int kMinDeleteInterval
static constexpr int kDeleteInterval
std::unique_ptr< node_store::Backend > makeBackendRotating(jtx::Env &env, NodeStoreScheduler &scheduler, std::string path)
static constexpr char const * kExpired
static bool bad(json::Value const &json, std::string const &token, ErrorCodeI error, std::string const &message)
void ledgerCheck(jtx::Env &env, int const rows, int const first)
void expectConfigAccepted(std::function< void(Config &)> const &tweak)
void accountTransactionCheck(jtx::Env &env, int const rows)
void run() override
Runs the suite.
void expectConfigRejected(std::string const &expected, std::function< void(Config &)> const &tweak)
static constexpr int kMinWaitingLedgers
static auto slowOnlineDelete(std::unique_ptr< Config > cfg)
static bool syncStore(jtx::Env &env)
std::optional< std::string > storeConfigResult(std::function< void(Config &)> const &tweak)
void transactionCheck(jtx::Env &env, int const rows)
static constexpr char const * kFullWait
std::string getHash(json::Value const &json)
std::optional< Parked > parkMidRotation(jtx::Env &env, StoreLogs &log)
static constexpr char const * kRotating
static constexpr char const * kFinished
static bool goodLedger(jtx::Env &env, json::Value const &json, std::string ledgerID, bool checkDB=false)
A transaction testing environment.
Definition Env.h:161
Application & app()
Definition Env.h:300
bool close(NetClock::time_point closeTime, std::optional< std::chrono::milliseconds > consensusDelay=std::nullopt)
Close and advance the ledger.
Definition Env.cpp:133
std::shared_ptr< ReadView const > closed()
Returns the last closed ledger.
Definition Env.cpp:127
void fund(bool setDefaultRipple, STAmount const &amount, Account const &account)
Definition Env.cpp:323
json::Value rpc(unsigned apiVersion, std::unordered_map< std::string, std::string > const &headers, std::string const &cmd, Args &&... args)
Execute an RPC command.
Definition Env.h:1058
ManualTimeKeeper & timeKeeper()
Definition Env.h:313
NetClock::time_point now()
Returns the current network time.
Definition Env.h:326
T count_if(T... args)
T emplace(T... args)
T endl(T... args)
T make_unique(T... args)
T max(T... args)
Severity
Severity level / threshold of a Journal message.
Definition Journal.h:16
JSON (JavaScript Object Notation).
Definition json_errors.h:5
bool containsError(json::Value const &json)
Returns true if the json contains an rpc error specification.
XrpT const XRP
Converts to XRP Issue or STAmount.
Definition amount.cpp:92
json::Value noop(Account const &account)
The null transaction.
Definition noop.h:14
std::array< Account, 1+sizeof...(Args)> noripple(Account const &account, Args const &... args)
Designate accounts as no-ripple in Env::fund.
Definition Env.h:86
std::unique_ptr< Config > envconfig()
creates and initializes a default configuration for jtx::Env
Definition envconfig.h:38
std::unique_ptr< Config > onlineDelete(std::unique_ptr< Config > cfg, std::uint32_t deleteInterval=8)
adjust config to enable online_delete
Definition envconfig.cpp:67
BEAST_DEFINE_TESTSUITE(AMMClawback, app, xrpl)
Use hash_* containers for keys that do not need a cryptographically secure hashing algorithm.
Definition algorithm.h:5
ErrorCodeI
Definition ErrorCodes.h:23
@ RpcLgrNotFound
Definition ErrorCodes.h:55
@ RpcNotEnabled
Definition ErrorCodes.h:42
std::uint32_t LedgerIndex
A ledger index.
Definition Protocol.h:409
std::unique_ptr< beast::Journal::Sink > setDebugLogSink(std::unique_ptr< beast::Journal::Sink > sink)
Set the sink for the debug journal.
Definition Log.cpp:393
std::string to_string(BaseUInt< Bits, Tag > const &a)
Definition base_uint.h:658
constexpr auto megabytes(T value) noexcept
OperatingMode
Specifies the mode under which the server believes it's operating.
Definition NetworkOPs.h:60
@ DISCONNECTED
not ready to process requests
Definition NetworkOPs.h:61
@ CONNECTED
convinced we are talking to the network
Definition NetworkOPs.h:62
@ FULL
we have the ledger and can even validate
Definition NetworkOPs.h:65
T sleep_for(T... args)
T str(T... args)
static constexpr auto kMaxWaitingLedgers
Definition Constants.h:131
static constexpr auto kBackOffMilliseconds
Definition Constants.h:92
static constexpr auto kAdvisoryDelete
Definition Constants.h:89
static constexpr auto kCacheSize
Definition Constants.h:100
static constexpr auto kDeleteBatch
Definition Constants.h:105
static constexpr auto kRecoveryWaitSeconds
Definition Constants.h:152
static constexpr auto kPath
Definition Constants.h:147
static constexpr auto kCacheAge
Definition Constants.h:98
static constexpr auto kOnlineDelete
Definition Constants.h:140
Information about the notional ledger backing the view.
NetClock::time_point parentCloseTime
NetClock::duration closeTimeResolution
NetClock::time_point closeTime
static constexpr auto kNodeDatabase
Definition Constants.h:32
T time_since_epoch(T... args)
T to_string(T... args)
T value(T... args)
T what(T... args)