Clio  develop
The XRP Ledger API server.
Loading...
Searching...
No Matches
BackendInterface.hpp
1//------------------------------------------------------------------------------
2/*
3 This file is part of clio: https://github.com/XRPLF/clio
4 Copyright (c) 2022, the clio developers.
5
6 Permission to use, copy, modify, and distribute this software for any
7 purpose with or without fee is hereby granted, provided that the above
8 copyright notice and this permission notice appear in all copies.
9
10 THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17*/
18//==============================================================================
19
20#pragma once
21
22#include "data/DBHelpers.hpp"
23#include "data/LedgerCacheInterface.hpp"
24#include "data/Types.hpp"
25#include "etl/CorruptionDetector.hpp"
26#include "util/Spawn.hpp"
27#include "util/log/Logger.hpp"
28
29#include <boost/asio/executor_work_guard.hpp>
30#include <boost/asio/io_context.hpp>
31#include <boost/asio/spawn.hpp>
32#include <boost/json.hpp>
33#include <boost/json/object.hpp>
34#include <boost/utility/result_of.hpp>
35#include <boost/uuid/uuid.hpp>
36#include <xrpl/basics/base_uint.h>
37#include <xrpl/protocol/AccountID.h>
38#include <xrpl/protocol/Fees.h>
39#include <xrpl/protocol/LedgerHeader.h>
40
41#include <chrono>
42#include <cstddef>
43#include <cstdint>
44#include <exception>
45#include <functional>
46#include <optional>
47#include <shared_mutex>
48#include <string>
49#include <thread>
50#include <type_traits>
51#include <utility>
52#include <vector>
53
54namespace data {
55
59class DatabaseTimeout : public std::exception {
60public:
64 char const*
65 what() const throw() override
66 {
67 return "Database read timed out. Please retry the request";
68 }
69};
70
71static constexpr std::size_t kDEFAULT_WAIT_BETWEEN_RETRY = 500;
80template <typename FnType>
81auto
82retryOnTimeout(FnType func, size_t waitMs = kDEFAULT_WAIT_BETWEEN_RETRY)
83{
84 static util::Logger const log{"Backend"}; // NOLINT(readability-identifier-naming)
85
86 while (true) {
87 try {
88 return func();
89 } catch (DatabaseTimeout const&) {
90 LOG(log.error()) << "Database request timed out. Sleeping and retrying ... ";
91 std::this_thread::sleep_for(std::chrono::milliseconds(waitMs));
92 }
93 }
94}
95
103template <typename FnType>
104auto
105synchronous(FnType&& func)
106{
107 boost::asio::io_context ctx;
108
109 using R = typename boost::result_of<FnType(boost::asio::yield_context)>::type;
110 if constexpr (!std::is_same_v<R, void>) {
111 R res;
112 util::spawn(ctx, [_ = boost::asio::make_work_guard(ctx), &func, &res](auto yield) { res = func(yield); });
113
114 ctx.run();
115 return res;
116 } else {
117 util::spawn(ctx, [_ = boost::asio::make_work_guard(ctx), &func](auto yield) { func(yield); });
118 ctx.run();
119 }
120}
121
129template <typename FnType>
130auto
132{
133 return retryOnTimeout([&]() { return synchronous(func); });
134}
135
140protected:
141 util::Logger log_{"Backend"};
142 mutable std::shared_mutex rngMtx_;
143 std::optional<LedgerRange> range_;
144 std::reference_wrapper<LedgerCacheInterface> cache_;
145 std::optional<etl::CorruptionDetector> corruptionDetector_;
146
147public:
156 virtual ~BackendInterface() = default;
157
158 // TODO https://github.com/XRPLF/clio/issues/1956: Remove this hack once old ETL is removed.
159 // Cache should not be exposed thru BackendInterface
160
165 cache() const
166 {
167 return cache_;
168 }
169
175 {
176 return cache_;
177 }
178
184 void
186 {
187 corruptionDetector_ = std::move(detector);
188 }
189
197 virtual std::optional<ripple::LedgerHeader>
198 fetchLedgerBySequence(std::uint32_t sequence, boost::asio::yield_context yield) const = 0;
199
207 virtual std::optional<ripple::LedgerHeader>
208 fetchLedgerByHash(ripple::uint256 const& hash, boost::asio::yield_context yield) const = 0;
209
216 virtual std::optional<std::uint32_t>
217 fetchLatestLedgerSequence(boost::asio::yield_context yield) const = 0;
218
224 std::optional<LedgerRange>
225 fetchLedgerRange() const;
226
236 virtual std::vector<ripple::uint256>
238 std::uint32_t number,
239 std::uint32_t pageSize,
240 std::uint32_t seq,
241 boost::asio::yield_context yield
242 ) const = 0;
243
249 void
250 updateRange(uint32_t newMax);
251
259 void
260 setRange(uint32_t min, uint32_t max, bool force = false);
261
269 std::optional<ripple::Fees>
270 fetchFees(std::uint32_t seq, boost::asio::yield_context yield) const;
271
279 virtual std::optional<TransactionAndMetadata>
280 fetchTransaction(ripple::uint256 const& hash, boost::asio::yield_context yield) const = 0;
281
289 virtual std::vector<TransactionAndMetadata>
290 fetchTransactions(std::vector<ripple::uint256> const& hashes, boost::asio::yield_context yield) const = 0;
291
304 ripple::AccountID const& account,
305 std::uint32_t limit,
306 bool forward,
307 std::optional<TransactionsCursor> const& txnCursor,
308 boost::asio::yield_context yield
309 ) const = 0;
310
318 virtual std::vector<TransactionAndMetadata>
319 fetchAllTransactionsInLedger(std::uint32_t ledgerSequence, boost::asio::yield_context yield) const = 0;
320
328 virtual std::vector<ripple::uint256>
329 fetchAllTransactionHashesInLedger(std::uint32_t ledgerSequence, boost::asio::yield_context yield) const = 0;
330
339 virtual std::optional<NFT>
340 fetchNFT(ripple::uint256 const& tokenID, std::uint32_t ledgerSequence, boost::asio::yield_context yield) const = 0;
341
354 ripple::uint256 const& tokenID,
355 std::uint32_t limit,
356 bool forward,
357 std::optional<TransactionsCursor> const& cursorIn,
358 boost::asio::yield_context yield
359 ) const = 0;
360
372 virtual NFTsAndCursor
374 ripple::AccountID const& issuer,
375 std::optional<std::uint32_t> const& taxon,
376 std::uint32_t ledgerSequence,
377 std::uint32_t limit,
378 std::optional<ripple::uint256> const& cursorIn,
379 boost::asio::yield_context yield
380 ) const = 0;
381
392 virtual MPTHoldersAndCursor
394 ripple::uint192 const& mptID,
395 std::uint32_t const limit,
396 std::optional<ripple::AccountID> const& cursorIn,
397 std::uint32_t const ledgerSequence,
398 boost::asio::yield_context yield
399 ) const = 0;
400
412 std::optional<Blob>
413 fetchLedgerObject(ripple::uint256 const& key, std::uint32_t sequence, boost::asio::yield_context yield) const;
414
425 std::optional<std::uint32_t>
426 fetchLedgerObjectSeq(ripple::uint256 const& key, std::uint32_t sequence, boost::asio::yield_context yield) const;
427
439 std::vector<Blob>
441 std::vector<ripple::uint256> const& keys,
442 std::uint32_t sequence,
443 boost::asio::yield_context yield
444 ) const;
445
454 virtual std::optional<Blob>
455 doFetchLedgerObject(ripple::uint256 const& key, std::uint32_t sequence, boost::asio::yield_context yield) const = 0;
456
465 virtual std::optional<std::uint32_t>
467 ripple::uint256 const& key,
468 std::uint32_t sequence,
469 boost::asio::yield_context yield
470 ) const = 0;
471
480 virtual std::vector<Blob>
482 std::vector<ripple::uint256> const& keys,
483 std::uint32_t sequence,
484 boost::asio::yield_context yield
485 ) const = 0;
486
494 virtual std::vector<LedgerObject>
495 fetchLedgerDiff(std::uint32_t ledgerSequence, boost::asio::yield_context yield) const = 0;
496
509 std::optional<ripple::uint256> const& cursor,
510 std::uint32_t ledgerSequence,
511 std::uint32_t limit,
512 bool outOfOrder,
513 boost::asio::yield_context yield
514 );
515
524 std::optional<LedgerObject>
525 fetchSuccessorObject(ripple::uint256 key, std::uint32_t ledgerSequence, boost::asio::yield_context yield) const;
526
538 std::optional<ripple::uint256>
539 fetchSuccessorKey(ripple::uint256 key, std::uint32_t ledgerSequence, boost::asio::yield_context yield) const;
540
549 virtual std::optional<ripple::uint256>
550 doFetchSuccessorKey(ripple::uint256 key, std::uint32_t ledgerSequence, boost::asio::yield_context yield) const = 0;
551
563 ripple::uint256 const& book,
564 std::uint32_t ledgerSequence,
565 std::uint32_t limit,
566 boost::asio::yield_context yield
567 ) const;
568
576 virtual std::optional<std::string>
577 fetchMigratorStatus(std::string const& migratorName, boost::asio::yield_context yield) const = 0;
578
581 std::expected<std::vector<std::pair<boost::uuids::uuid, std::string>>, std::string>;
582
589 [[nodiscard]] virtual ClioNodesDataFetchResult
590 fetchClioNodesData(boost::asio::yield_context yield) const = 0;
591
599 std::optional<LedgerRange>
600 hardFetchLedgerRange() const;
601
608 virtual std::optional<LedgerRange>
609 hardFetchLedgerRange(boost::asio::yield_context yield) const = 0;
610
616 std::optional<LedgerRange>
618
625 virtual void
626 writeLedger(ripple::LedgerHeader const& ledgerHeader, std::string&& blob) = 0;
627
635 virtual void
636 writeLedgerObject(std::string&& key, std::uint32_t seq, std::string&& blob);
637
647 virtual void
649 std::string&& hash,
650 std::uint32_t seq,
651 std::uint32_t date,
652 std::string&& transaction,
653 std::string&& metadata
654 ) = 0;
655
661 virtual void
662 writeNFTs(std::vector<NFTsData> const& data) = 0;
663
669 virtual void
670 writeAccountTransactions(std::vector<AccountTransactionsData> data) = 0;
671
677 virtual void
679
685 virtual void
686 writeNFTTransactions(std::vector<NFTTransactionsData> const& data) = 0;
687
693 virtual void
694 writeMPTHolders(std::vector<MPTHolderData> const& data) = 0;
695
703 virtual void
704 writeSuccessor(std::string&& key, std::uint32_t seq, std::string&& successor) = 0;
705
712 virtual void
713 writeNodeMessage(boost::uuids::uuid const& uuid, std::string message) = 0;
714
720 virtual void
721 startWrites() const = 0;
722
731 bool
732 finishWrites(std::uint32_t ledgerSequence);
733
737 virtual void
739
746 virtual void
747 writeMigratorStatus(std::string const& migratorName, std::string const& status) = 0;
748
752 virtual bool
753 isTooBusy() const = 0;
754
758 virtual boost::json::object
759 stats() const = 0;
760
761private:
769 virtual void
770 doWriteLedgerObject(std::string&& key, std::uint32_t seq, std::string&& blob) = 0;
771
777 virtual bool
778 doFinishWrites() = 0;
779};
780
781} // namespace data
782using BackendInterface = data::BackendInterface;
The interface to the database used by Clio.
Definition BackendInterface.hpp:139
virtual std::vector< TransactionAndMetadata > fetchAllTransactionsInLedger(std::uint32_t ledgerSequence, boost::asio::yield_context yield) const =0
Fetches all transactions from a specific ledger.
virtual void writeNodeMessage(boost::uuids::uuid const &uuid, std::string message)=0
Write a node message. Used by ClusterCommunicationService.
std::optional< ripple::Fees > fetchFees(std::uint32_t seq, boost::asio::yield_context yield) const
Fetch the fees from a specific ledger sequence.
Definition BackendInterface.cpp:358
virtual std::vector< ripple::uint256 > fetchAccountRoots(std::uint32_t number, std::uint32_t pageSize, std::uint32_t seq, boost::asio::yield_context yield) const =0
Fetch the specified number of account root object indexes by page, the accounts need to exist for seq...
virtual bool isTooBusy() const =0
virtual ClioNodesDataFetchResult fetchClioNodesData(boost::asio::yield_context yield) const =0
Fetches the data of all nodes in the cluster.
virtual TransactionsAndCursor fetchAccountTransactions(ripple::AccountID const &account, std::uint32_t limit, bool forward, std::optional< TransactionsCursor > const &txnCursor, boost::asio::yield_context yield) const =0
Fetches all transactions for a specific account.
virtual void writeNFTs(std::vector< NFTsData > const &data)=0
Writes NFTs to the database.
std::vector< Blob > fetchLedgerObjects(std::vector< ripple::uint256 > const &keys, std::uint32_t sequence, boost::asio::yield_context yield) const
Fetches all ledger objects by their keys.
Definition BackendInterface.cpp:114
virtual std::optional< TransactionAndMetadata > fetchTransaction(ripple::uint256 const &hash, boost::asio::yield_context yield) const =0
Fetches a specific transaction.
void setRange(uint32_t min, uint32_t max, bool force=false)
Sets the range of sequences that are stored in the DB.
Definition BackendInterface.cpp:291
LedgerCacheInterface & cache()
Definition BackendInterface.hpp:174
std::optional< LedgerObject > fetchSuccessorObject(ripple::uint256 key, std::uint32_t ledgerSequence, boost::asio::yield_context yield) const
Fetches the successor object.
Definition BackendInterface.cpp:164
virtual std::vector< Blob > doFetchLedgerObjects(std::vector< ripple::uint256 > const &keys, std::uint32_t sequence, boost::asio::yield_context yield) const =0
The database-specific implementation for fetching ledger objects.
virtual void startWrites() const =0
Starts a write transaction with the DB. No-op for cassandra.
virtual std::vector< ripple::uint256 > fetchAllTransactionHashesInLedger(std::uint32_t ledgerSequence, boost::asio::yield_context yield) const =0
Fetches all transaction hashes from a specific ledger.
std::optional< std::uint32_t > fetchLedgerObjectSeq(ripple::uint256 const &key, std::uint32_t sequence, boost::asio::yield_context yield) const
Fetches a specific ledger object sequence.
Definition BackendInterface.cpp:101
BookOffersPage fetchBookOffers(ripple::uint256 const &book, std::uint32_t ledgerSequence, std::uint32_t limit, boost::asio::yield_context yield) const
Fetches book offers.
Definition BackendInterface.cpp:182
virtual std::optional< LedgerRange > hardFetchLedgerRange(boost::asio::yield_context yield) const =0
Fetches the ledger range from DB.
virtual void writeLedgerObject(std::string &&key, std::uint32_t seq, std::string &&blob)
Writes a new ledger object.
Definition BackendInterface.cpp:65
virtual void writeSuccessor(std::string &&key, std::uint32_t seq, std::string &&successor)=0
Write a new successor.
virtual std::optional< std::string > fetchMigratorStatus(std::string const &migratorName, boost::asio::yield_context yield) const =0
Fetches the status of migrator by name.
virtual void waitForWritesToFinish()=0
Wait for all pending writes to finish.
std::optional< LedgerRange > hardFetchLedgerRange() const
Synchronously fetches the ledger range from DB.
Definition BackendInterface.cpp:256
BackendInterface(LedgerCacheInterface &cache)
Construct a new backend interface instance.
Definition BackendInterface.hpp:153
virtual std::vector< TransactionAndMetadata > fetchTransactions(std::vector< ripple::uint256 > const &hashes, boost::asio::yield_context yield) const =0
Fetches multiple transactions.
virtual void writeAccountTransaction(AccountTransactionsData record)=0
Write a new account transaction.
virtual NFTsAndCursor fetchNFTsByIssuer(ripple::AccountID const &issuer, std::optional< std::uint32_t > const &taxon, std::uint32_t ledgerSequence, std::uint32_t limit, std::optional< ripple::uint256 > const &cursorIn, boost::asio::yield_context yield) const =0
Fetches all NFTs issued by a given address.
virtual std::optional< ripple::LedgerHeader > fetchLedgerBySequence(std::uint32_t sequence, boost::asio::yield_context yield) const =0
Fetches a specific ledger by sequence number.
virtual std::optional< Blob > doFetchLedgerObject(ripple::uint256 const &key, std::uint32_t sequence, boost::asio::yield_context yield) const =0
The database-specific implementation for fetching a ledger object.
std::optional< ripple::uint256 > fetchSuccessorKey(ripple::uint256 key, std::uint32_t ledgerSequence, boost::asio::yield_context yield) const
Fetches the successor key.
Definition BackendInterface.cpp:148
std::optional< LedgerRange > hardFetchLedgerRangeNoThrow() const
Fetches the ledger range from DB retrying until no DatabaseTimeout is thrown.
Definition BackendInterface.cpp:72
virtual void writeNFTTransactions(std::vector< NFTTransactionsData > const &data)=0
Write NFTs transactions.
std::optional< LedgerRange > fetchLedgerRange() const
Fetch the current ledger range.
Definition BackendInterface.cpp:262
virtual void writeTransaction(std::string &&hash, std::uint32_t seq, std::uint32_t date, std::string &&transaction, std::string &&metadata)=0
Writes a new transaction.
bool finishWrites(std::uint32_t ledgerSequence)
Tells database we finished writing all data for a specific ledger.
Definition BackendInterface.cpp:54
virtual boost::json::object stats() const =0
void setCorruptionDetector(etl::CorruptionDetector detector)
Sets the corruption detector.
Definition BackendInterface.hpp:185
virtual TransactionsAndCursor fetchNFTTransactions(ripple::uint256 const &tokenID, std::uint32_t limit, bool forward, std::optional< TransactionsCursor > const &cursorIn, boost::asio::yield_context yield) const =0
Fetches all transactions for a specific NFT.
virtual std::optional< std::uint32_t > fetchLatestLedgerSequence(boost::asio::yield_context yield) const =0
Fetches the latest ledger sequence.
virtual std::optional< NFT > fetchNFT(ripple::uint256 const &tokenID, std::uint32_t ledgerSequence, boost::asio::yield_context yield) const =0
Fetches a specific NFT.
virtual void writeAccountTransactions(std::vector< AccountTransactionsData > data)=0
Write a new set of account transactions.
std::expected< std::vector< std::pair< boost::uuids::uuid, std::string > >, std::string > ClioNodesDataFetchResult
Return type for fetchClioNodesData() method.
Definition BackendInterface.hpp:580
virtual void writeMigratorStatus(std::string const &migratorName, std::string const &status)=0
Mark the migration status of a migrator as Migrated in the database.
virtual void writeMPTHolders(std::vector< MPTHolderData > const &data)=0
Write accounts that started holding onto a MPT.
virtual std::vector< LedgerObject > fetchLedgerDiff(std::uint32_t ledgerSequence, boost::asio::yield_context yield) const =0
Returns the difference between ledgers.
std::optional< Blob > fetchLedgerObject(ripple::uint256 const &key, std::uint32_t sequence, boost::asio::yield_context yield) const
Fetches a specific ledger object.
Definition BackendInterface.cpp:79
LedgerPage fetchLedgerPage(std::optional< ripple::uint256 > const &cursor, std::uint32_t ledgerSequence, std::uint32_t limit, bool outOfOrder, boost::asio::yield_context yield)
Fetches a page of ledger objects, ordered by key/index.
Definition BackendInterface.cpp:304
virtual std::optional< ripple::uint256 > doFetchSuccessorKey(ripple::uint256 key, std::uint32_t ledgerSequence, boost::asio::yield_context yield) const =0
Database-specific implementation of fetching the successor key.
virtual std::optional< std::uint32_t > doFetchLedgerObjectSeq(ripple::uint256 const &key, std::uint32_t sequence, boost::asio::yield_context yield) const =0
The database-specific implementation for fetching a ledger object sequence.
void updateRange(uint32_t newMax)
Updates the range of sequences that are stored in the DB.
Definition BackendInterface.cpp:269
virtual void writeLedger(ripple::LedgerHeader const &ledgerHeader, std::string &&blob)=0
Writes to a specific ledger.
virtual MPTHoldersAndCursor fetchMPTHolders(ripple::uint192 const &mptID, std::uint32_t const limit, std::optional< ripple::AccountID > const &cursorIn, std::uint32_t const ledgerSequence, boost::asio::yield_context yield) const =0
Fetches all holders' balances for a MPTIssuanceID.
LedgerCacheInterface const & cache() const
Definition BackendInterface.hpp:165
virtual std::optional< ripple::LedgerHeader > fetchLedgerByHash(ripple::uint256 const &hash, boost::asio::yield_context yield) const =0
Fetches a specific ledger by hash.
Represents a database timeout error.
Definition BackendInterface.hpp:59
char const * what() const override
Definition BackendInterface.hpp:65
Cache for an entire ledger.
Definition LedgerCacheInterface.hpp:38
A helper to notify Clio operator about a corruption in the DB.
Definition CorruptionDetector.hpp:35
A simple thread-safe logger for the channel specified in the constructor.
Definition Logger.hpp:94
Pump error(SourceLocationType const &loc=CURRENT_SRC_LOCATION) const
Interface for logging at Severity::ERR severity.
Definition Logger.cpp:473
This namespace implements the data access layer and related components.
Definition AmendmentCenter.cpp:70
auto synchronous(FnType &&func)
Synchronously executes the given function object inside a coroutine.
Definition BackendInterface.hpp:105
auto synchronousAndRetryOnTimeout(FnType &&func)
Synchronously execute the given function object and retry until no DatabaseTimeout is thrown.
Definition BackendInterface.hpp:131
auto retryOnTimeout(FnType func, size_t waitMs=kDEFAULT_WAIT_BETWEEN_RETRY)
A helper function that catches DatabaseTimeout exceptions and retries indefinitely.
Definition BackendInterface.hpp:82
void spawn(Ctx &&ctx, F &&func)
Spawns a coroutine using boost::asio::spawn
Definition Spawn.hpp:69
Struct used to keep track of what to write to account_transactions/account_tx tables.
Definition DBHelpers.hpp:45
Represents a page of book offer objects.
Definition Types.hpp:60
Represents a page of LedgerObjects.
Definition Types.hpp:52
Represents an array of MPTokens.
Definition Types.hpp:239
Represents a bundle of NFTs with a cursor to the next page.
Definition Types.hpp:231
Represests a bundle of transactions with metadata and a cursor to the next page.
Definition Types.hpp:164