BackendInterface.hpp

#pragma once
 
#include "data/DBHelpers.hpp"
#include "data/LedgerCacheInterface.hpp"
#include "data/Types.hpp"
#include "etl/CorruptionDetector.hpp"
#include "util/Spawn.hpp"
#include "util/log/Logger.hpp"
 
#include <boost/asio/executor_work_guard.hpp>
#include <boost/asio/io_context.hpp>
#include <boost/asio/spawn.hpp>
#include <boost/json.hpp>
#include <boost/json/object.hpp>
#include <boost/utility/result_of.hpp>
#include <boost/uuid/uuid.hpp>
#include <xrpl/basics/base_uint.h>
#include <xrpl/protocol/AccountID.h>
#include <xrpl/protocol/Fees.h>
#include <xrpl/protocol/LedgerHeader.h>
 
#include <chrono>
#include <cstddef>
#include <cstdint>
#include <exception>
#include <functional>
#include <optional>
#include <shared_mutex>
#include <string>
#include <thread>
#include <type_traits>
#include <utility>
#include <vector>
 
namespace data {

class DatabaseTimeout : public std::exception {
public:
    char const*
    what() const throw() override
    {
        return "Database read timed out. Please retry the request";
    }
};
 
static constexpr std::size_t kDEFAULT_WAIT_BETWEEN_RETRY = 500;
template <typename FnType>
auto
retryOnTimeout(FnType func, size_t waitMs = kDEFAULT_WAIT_BETWEEN_RETRY)
{
    static util::Logger const log{"Backend"};  // NOLINT(readability-identifier-naming)
 
    while (true) {
        try {
            return func();
        } catch (DatabaseTimeout const&) {
            LOG(log.error()) << "Database request timed out. Sleeping and retrying ... ";
            std::this_thread::sleep_for(std::chrono::milliseconds(waitMs));
        }
    }
}

template <typename FnType>
auto
synchronous(FnType&& func)
{
    boost::asio::io_context ctx;
 
    using R = typename boost::result_of<FnType(boost::asio::yield_context)>::type;
    if constexpr (!std::is_same_v<R, void>) {
        R res;
        util::spawn(ctx, [_ = boost::asio::make_work_guard(ctx), &func, &res](auto yield) {
            res = func(yield);
        });
 
        ctx.run();
        return res;
    } else {
        util::spawn(ctx, [_ = boost::asio::make_work_guard(ctx), &func](auto yield) {
            func(yield);
        });
        ctx.run();
    }
}

template <typename FnType>
auto
synchronousAndRetryOnTimeout(FnType&& func)
{
    return retryOnTimeout([&]() { return synchronous(func); });
}

class BackendInterface {
protected:
    util::Logger log_{"Backend"};
    mutable std::shared_mutex rngMtx_;
    std::optional<LedgerRange> range_;
    std::reference_wrapper<LedgerCacheInterface> cache_;
    std::optional<etl::CorruptionDetector> corruptionDetector_;
 
public:
    BackendInterface(LedgerCacheInterface& cache) : cache_{cache}
    {
    }
    virtual ~BackendInterface() = default;
 
    // TODO https://github.com/XRPLF/clio/issues/1956: Remove this hack once old ETL is removed.
    // Cache should not be exposed thru BackendInterface

    LedgerCacheInterface const&
    cache() const
    {
        return cache_;
    }

    LedgerCacheInterface&
    cache()
    {
        return cache_;
    }

    void
    setCorruptionDetector(etl::CorruptionDetector detector)
    {
        corruptionDetector_ = std::move(detector);
    }

    virtual std::optional<ripple::LedgerHeader>
    fetchLedgerBySequence(std::uint32_t sequence, boost::asio::yield_context yield) const = 0;

    virtual std::optional<ripple::LedgerHeader>
    fetchLedgerByHash(ripple::uint256 const& hash, boost::asio::yield_context yield) const = 0;

    virtual std::optional<std::uint32_t>
    fetchLatestLedgerSequence(boost::asio::yield_context yield) const = 0;

    std::optional<LedgerRange>
    fetchLedgerRange() const;

    virtual std::vector<ripple::uint256>
    fetchAccountRoots(
        std::uint32_t number,
        std::uint32_t pageSize,
        std::uint32_t seq,
        boost::asio::yield_context yield
    ) const = 0;

    void
    updateRange(uint32_t newMax);

    void
    forceUpdateRange(uint32_t newMax);

    void
    setRange(uint32_t min, uint32_t max, bool force = false);

    std::optional<ripple::Fees>
    fetchFees(std::uint32_t seq, boost::asio::yield_context yield) const;

    virtual std::optional<TransactionAndMetadata>
    fetchTransaction(ripple::uint256 const& hash, boost::asio::yield_context yield) const = 0;

    virtual std::vector<TransactionAndMetadata>
    fetchTransactions(
        std::vector<ripple::uint256> const& hashes,
        boost::asio::yield_context yield
    ) const = 0;

    virtual TransactionsAndCursor
    fetchAccountTransactions(
        ripple::AccountID const& account,
        std::uint32_t limit,
        bool forward,
        std::optional<TransactionsCursor> const& txnCursor,
        boost::asio::yield_context yield
    ) const = 0;

    virtual std::vector<TransactionAndMetadata>
    fetchAllTransactionsInLedger(
        std::uint32_t ledgerSequence,
        boost::asio::yield_context yield
    ) const = 0;

    virtual std::vector<ripple::uint256>
    fetchAllTransactionHashesInLedger(
        std::uint32_t ledgerSequence,
        boost::asio::yield_context yield
    ) const = 0;

    virtual std::optional<NFT>
    fetchNFT(
        ripple::uint256 const& tokenID,
        std::uint32_t ledgerSequence,
        boost::asio::yield_context yield
    ) const = 0;

    virtual TransactionsAndCursor
    fetchNFTTransactions(
        ripple::uint256 const& tokenID,
        std::uint32_t limit,
        bool forward,
        std::optional<TransactionsCursor> const& cursorIn,
        boost::asio::yield_context yield
    ) const = 0;

    virtual NFTsAndCursor
    fetchNFTsByIssuer(
        ripple::AccountID const& issuer,
        std::optional<std::uint32_t> const& taxon,
        std::uint32_t ledgerSequence,
        std::uint32_t limit,
        std::optional<ripple::uint256> const& cursorIn,
        boost::asio::yield_context yield
    ) const = 0;

    virtual MPTHoldersAndCursor
    fetchMPTHolders(
        ripple::uint192 const& mptID,
        std::uint32_t const limit,
        std::optional<ripple::AccountID> const& cursorIn,
        std::uint32_t const ledgerSequence,
        boost::asio::yield_context yield
    ) const = 0;

    std::optional<Blob>
    fetchLedgerObject(
        ripple::uint256 const& key,
        std::uint32_t sequence,
        boost::asio::yield_context yield
    ) const;

    std::optional<std::uint32_t>
    fetchLedgerObjectSeq(
        ripple::uint256 const& key,
        std::uint32_t sequence,
        boost::asio::yield_context yield
    ) const;

    std::vector<Blob>
    fetchLedgerObjects(
        std::vector<ripple::uint256> const& keys,
        std::uint32_t sequence,
        boost::asio::yield_context yield
    ) const;

    virtual std::optional<Blob>
    doFetchLedgerObject(
        ripple::uint256 const& key,
        std::uint32_t sequence,
        boost::asio::yield_context yield
    ) const = 0;

    virtual std::optional<std::uint32_t>
    doFetchLedgerObjectSeq(
        ripple::uint256 const& key,
        std::uint32_t sequence,
        boost::asio::yield_context yield
    ) const = 0;

    virtual std::vector<Blob>
    doFetchLedgerObjects(
        std::vector<ripple::uint256> const& keys,
        std::uint32_t sequence,
        boost::asio::yield_context yield
    ) const = 0;

    virtual std::vector<LedgerObject>
    fetchLedgerDiff(std::uint32_t ledgerSequence, boost::asio::yield_context yield) const = 0;

    LedgerPage
    fetchLedgerPage(
        std::optional<ripple::uint256> const& cursor,
        std::uint32_t ledgerSequence,
        std::uint32_t limit,
        bool outOfOrder,
        boost::asio::yield_context yield
    );

    std::optional<LedgerObject>
    fetchSuccessorObject(
        ripple::uint256 key,
        std::uint32_t ledgerSequence,
        boost::asio::yield_context yield
    ) const;

    std::optional<ripple::uint256>
    fetchSuccessorKey(
        ripple::uint256 key,
        std::uint32_t ledgerSequence,
        boost::asio::yield_context yield
    ) const;

    virtual std::optional<ripple::uint256>
    doFetchSuccessorKey(
        ripple::uint256 key,
        std::uint32_t ledgerSequence,
        boost::asio::yield_context yield
    ) const = 0;

    BookOffersPage
    fetchBookOffers(
        ripple::uint256 const& book,
        std::uint32_t ledgerSequence,
        std::uint32_t limit,
        boost::asio::yield_context yield
    ) const;

    virtual std::optional<std::string>
    fetchMigratorStatus(
        std::string const& migratorName,
        boost::asio::yield_context yield
    ) const = 0;

    using ClioNodesDataFetchResult =
        std::expected<std::vector<std::pair<boost::uuids::uuid, std::string>>, std::string>;

    [[nodiscard]] virtual ClioNodesDataFetchResult
    fetchClioNodesData(boost::asio::yield_context yield) const = 0;

    std::optional<LedgerRange>
    hardFetchLedgerRange() const;

    virtual std::optional<LedgerRange>
    hardFetchLedgerRange(boost::asio::yield_context yield) const = 0;

    std::optional<LedgerRange>
    hardFetchLedgerRangeNoThrow() const;

    virtual void
    writeLedger(ripple::LedgerHeader const& ledgerHeader, std::string&& blob) = 0;

    virtual void
    writeLedgerObject(std::string&& key, std::uint32_t seq, std::string&& blob);

    virtual void
    writeTransaction(
        std::string&& hash,
        std::uint32_t seq,
        std::uint32_t date,
        std::string&& transaction,
        std::string&& metadata
    ) = 0;

    virtual void
    writeNFTs(std::vector<NFTsData> const& data) = 0;

    virtual void
    writeAccountTransactions(std::vector<AccountTransactionsData> data) = 0;

    virtual void
    writeAccountTransaction(AccountTransactionsData record) = 0;

    virtual void
    writeNFTTransactions(std::vector<NFTTransactionsData> const& data) = 0;

    virtual void
    writeMPTHolders(std::vector<MPTHolderData> const& data) = 0;

    virtual void
    writeSuccessor(std::string&& key, std::uint32_t seq, std::string&& successor) = 0;

    virtual void
    writeNodeMessage(boost::uuids::uuid const& uuid, std::string message) = 0;

    virtual void
    startWrites() const = 0;

    bool
    finishWrites(std::uint32_t ledgerSequence);

    virtual void
    waitForWritesToFinish() = 0;

    virtual void
    writeMigratorStatus(std::string const& migratorName, std::string const& status) = 0;

    virtual bool
    isTooBusy() const = 0;

    virtual boost::json::object
    stats() const = 0;
 
private:
    virtual void
    doWriteLedgerObject(std::string&& key, std::uint32_t seq, std::string&& blob) = 0;

    virtual bool
    doFinishWrites() = 0;
 
    void
    updateRangeImpl(uint32_t newMax);
};
 
}  // namespace data
using BackendInterface = data::BackendInterface;