1#include <xrpld/app/ledger/InboundLedger.h>
3#include <xrpld/app/ledger/AccountStateSF.h>
4#include <xrpld/app/ledger/InboundLedgers.h>
5#include <xrpld/app/ledger/LedgerMaster.h>
6#include <xrpld/app/ledger/LedgerNodeHelpers.h>
7#include <xrpld/app/ledger/TransactionStateSF.h>
8#include <xrpld/app/ledger/detail/TimeoutCounter.h>
9#include <xrpld/app/main/Application.h>
10#include <xrpld/overlay/Message.h>
11#include <xrpld/overlay/Overlay.h>
12#include <xrpld/overlay/PeerSet.h>
14#include <xrpl/basics/Blob.h>
15#include <xrpl/basics/Log.h>
16#include <xrpl/basics/Slice.h>
17#include <xrpl/basics/base_uint.h>
18#include <xrpl/beast/utility/instrumentation.h>
19#include <xrpl/core/Job.h>
20#include <xrpl/core/JobQueue.h>
21#include <xrpl/json/json_value.h>
22#include <xrpl/ledger/entries/FeeSettingsEntry.h>
23#include <xrpl/nodestore/Database.h>
24#include <xrpl/nodestore/NodeObject.h>
25#include <xrpl/protocol/HashPrefix.h>
26#include <xrpl/protocol/Indexes.h>
27#include <xrpl/protocol/LedgerHeader.h>
28#include <xrpl/protocol/Rules.h>
29#include <xrpl/protocol/Serializer.h>
30#include <xrpl/protocol/SystemParameters.h>
31#include <xrpl/protocol/jss.h>
32#include <xrpl/resource/Fees.h>
33#include <xrpl/shamap/SHAMapNodeID.h>
34#include <xrpl/shamap/SHAMapSyncFilter.h>
36#include <boost/iterator/function_output_iterator.hpp>
79 {.jobType =
JtLedgerData, .jobName =
"InboundLedger", .jobLimit = 5},
80 app.getJournal(
"InboundLedger"))
84 , peerSet_(std::move(peerSet))
86 JLOG(journal_.trace()) <<
"Acquiring ledger " << hash_;
111 JLOG(
journal_.debug()) <<
"Acquiring ledger we already have in "
112 <<
" local store. " <<
hash_;
115 "xrpl::InboundLedger::init : valid ledger fees");
127 UNREACHABLE(
"xrpl::InboundLedger::init : map is invalid");
145 auto const& peerIds =
peerSet_->getPeerIds();
147 peerIds, [
this](
auto id) {
return (
app_.getOverlay().findPeerByShortID(
id) !=
nullptr); });
156 if ((seq != 0) && (
seq_ == 0))
192 if (entry.second->
type() == protocol::liAS_NODE)
193 app_.getInboundLedgers().gotStaleData(entry.second);
220 for (
auto const& n : mn)
247 auto const makeLedger = [&,
this](
Blob const& data) {
248 JLOG(
journal_.trace()) <<
"Ledger header found in fetch pack";
249 Rules const rules{
app_.config().features};
270 JLOG(
journal_.trace()) <<
"Ledger header found in local store";
272 makeLedger(nodeObject->getData());
277 auto& dstDB{
ledger_->stateMap().family().db()};
280 Blob blob{nodeObject->getData()};
287 auto data =
app_.getLedgerMaster().getFetchPack(
hash_);
291 JLOG(
journal_.trace()) <<
"Ledger header found in fetch pack";
298 ledger_->stateMap().family().db().store(
311 if (
ledger_->header().txHash.isZero())
313 JLOG(
journal_.trace()) <<
"No TXNs to fetch";
323 JLOG(
journal_.trace()) <<
"Had full txn map locally";
333 if (
ledger_->stateMap().fetchRoot(
SHAMapHash{ledger_->header().accountHash}, &filter))
337 JLOG(
journal_.trace()) <<
"Had full AS map locally";
347 "xrpl::InboundLedger::tryDB : valid ledger fees");
352 JLOG(
journal_.warn()) <<
"Ledger " <<
hash_ <<
" found locally is invalid";
357 JLOG(
journal_.debug()) <<
"Had everything locally";
398 JLOG(
journal_.debug()) <<
"No progress(" << pc <<
") for ledger " <<
hash_;
420 [
this](
auto peer) {
return peer->hasLedger(
hash_,
seq_); },
450 "xrpl::InboundLedger::done : valid ledger fees");
458 JLOG(
journal_.warn()) <<
"Acquired ledger " <<
hash_ <<
" is invalid";
469 app_.getInboundLedgers().onLedgerFetched();
484 XRPL_ASSERT(
complete_ ||
failed_,
"xrpl::InboundLedger::done : complete or failed");
488 if (self->complete_ && !self->failed_)
490 self->app_.getLedgerMaster().checkAccept(self->getLedger());
491 self->app_.getLedgerMaster().tryAdvance();
495 self->app_.getInboundLedgers().logFailure(self->hash_, self->seq_);
515 if (
auto const stream =
journal_.debug())
518 ss <<
"Trigger acquiring ledger " <<
hash_;
520 ss <<
" from " << peer;
544 protocol::TMGetLedger tmGL;
545 tmGL.set_ledgerhash(
hash_.begin(),
hash_.size());
550 tmGL.set_querytype(protocol::qtINDIRECT);
558 protocol::TMGetObjectByHash tmBH;
559 bool typeSet =
false;
560 tmBH.set_query(
true);
561 tmBH.set_ledgerhash(
hash_.begin(),
hash_.size());
562 for (
auto const& p : need)
564 JLOG(
journal_.debug()) <<
"Want: " << p.second;
568 tmBH.set_type(p.first);
572 if (p.first == tmBH.type())
574 protocol::TMIndexedObject* io = tmBH.add_objects();
575 io->set_hash(p.second.begin(), p.second.size());
577 io->set_ledgerseq(
seq_);
582 auto const& peerIds =
peerSet_->getPeerIds();
584 if (
auto const p =
app_.getOverlay().findPeerByShortID(
id))
593 JLOG(
journal_.info()) <<
"getNeededHashes says acquire is complete";
606 tmGL.set_itype(protocol::liBASE);
608 tmGL.set_ledgerseq(
seq_);
609 JLOG(
journal_.trace()) <<
"Sending header request to "
610 << (peer ?
"selected peer" :
"all peers");
616 tmGL.set_ledgerseq(
ledger_->header().seq);
621 tmGL.set_querydepth(0);
623 else if (peer && peer->isHighLatency())
626 tmGL.set_querydepth(2);
630 tmGL.set_querydepth(1);
639 "xrpl::InboundLedger::trigger : non-null ledger to read state "
642 if (!
ledger_->stateMap().isValid())
646 else if (
ledger_->stateMap().getHash().isZero())
649 tmGL.set_itype(protocol::liAS_NODE);
652 <<
"Sending AS root request to " << (peer ?
"selected peer" :
"all peers");
670 if (!
ledger_->stateMap().isValid())
688 tmGL.set_itype(protocol::liAS_NODE);
689 for (
auto const&
id : nodes)
691 *tmGL.add_nodeids() =
id.first.getRawString();
694 JLOG(
journal_.trace()) <<
"Sending AS node request (" << nodes.size()
695 <<
") to " << (peer ?
"selected peer" :
"all peers");
700 JLOG(
journal_.trace()) <<
"All AS nodes filtered";
710 "xrpl::InboundLedger::trigger : non-null ledger to read "
711 "transactions from");
713 if (!
ledger_->txMap().isValid())
717 else if (
ledger_->txMap().getHash().isZero())
720 tmGL.set_itype(protocol::liTX_NODE);
723 <<
"Sending TX root request to " << (peer ?
"selected peer" :
"all peers");
735 if (!
ledger_->txMap().isValid())
753 tmGL.set_itype(protocol::liTX_NODE);
754 for (
auto const& n : nodes)
756 *tmGL.add_nodeids() = n.first.getRawString();
758 JLOG(
journal_.trace()) <<
"Sending TX node request (" << nodes.size() <<
") to "
759 << (peer ?
"selected peer" :
"all peers");
764 JLOG(
journal_.trace()) <<
"All TX nodes filtered";
787 nodes, [
this](
auto const& item) {
return recentNodes_.count(item.second) == 0; });
792 if (dup.begin() == nodes.begin())
794 JLOG(
journal_.trace()) <<
"filterNodes: all duplicates";
804 JLOG(
journal_.trace()) <<
"filterNodes: pruning duplicates";
806 nodes.erase(dup.begin(), dup.end());
811 if (nodes.size() > limit)
814 for (
auto const& n : nodes)
821 bool const zero =
ledger_->header().accountHash.isZero();
822 SOMETIMES(zero,
"xrpl::InboundLedger::failOnZeroAccountHash : zero account hash");
826 JLOG(
journal_.warn()) <<
"We are acquiring a ledger with a zero account hash: " <<
hash_;
843 JLOG(
journal_.trace()) <<
"got header acquiring ledger " <<
hash_;
848 auto* f = &
app_.getNodeFamily();
849 Rules const rules{
app_.config().features};
853 JLOG(
journal_.warn()) <<
"Acquire hash mismatch: " <<
ledger_->header().hash
872 s.
addRaw(data.data(), data.size());
875 if (
ledger_->header().txHash.isZero())
878 ledger_->txMap().setSynching();
879 ledger_->stateMap().setSynching();
891 protocol::TMLedgerData
const& packet,
896 JLOG(
journal_.warn()) <<
"Missing ledger header";
900 if (packet.type() == protocol::liTX_NODE)
914 auto [map, rootHash, filter] =
916 if (packet.type() == protocol::liTX_NODE)
922 ledger_->txMap().family().db(),
app_.getLedgerMaster())};
928 ledger_->stateMap().family().db(),
app_.getLedgerMaster())};
933 auto const f = filter.get();
935 for (
auto const& ledgerNode : packet.nodes())
937 auto treeNode =
getTreeNode(ledgerNode.nodedata());
941 <<
"Got invalid node data for ledger " <<
hash_ <<
" from peer " << peer->id();
951 <<
"Got invalid node id for ledger " <<
hash_ <<
" from peer " << peer->id();
957 auto const result = nodeID->isRoot()
958 ? map.addRootNode(rootHash, std::move(treeNode), f)
959 : map.addKnownNode(*nodeID, std::move(treeNode), f);
962 if (result.isInvalid())
964 JLOG(
journal_.warn()) <<
"Got invalid node " << *nodeID <<
" for ledger " <<
hash_
965 <<
" from peer " << peer->id();
974 JLOG(
journal_.error()) <<
"Could not process node for ledger " <<
hash_ <<
" from peer "
975 << peer->id() <<
": " << e.
what();
980 if (!map.isSynching())
982 if (packet.type() == protocol::liTX_NODE)
1015 UNREACHABLE(
"xrpl::InboundLedger::takeAsRootNode : no ledger header");
1023 JLOG(
journal_.warn()) <<
"Got invalid AS root node data for ledger " <<
hash_;
1029 auto const result =
ledger_->stateMap().addRootNode(
1051 UNREACHABLE(
"xrpl::InboundLedger::takeTxRootNode : no ledger header");
1059 JLOG(
journal_.warn()) <<
"Got invalid TX root node data for ledger " <<
hash_;
1065 auto const result =
ledger_->txMap().addRootNode(
1087 ret.
emplace_back(protocol::TMGetObjectByHash::otSTATE_NODE, h);
1096 ret.
emplace_back(protocol::TMGetObjectByHash::otTRANSACTION_NODE, h);
1138 if (packet.type() == protocol::liBASE)
1140 if (packet.nodes().empty())
1142 JLOG(
journal_.warn()) << peer->id() <<
": empty header data";
1157 JLOG(
journal_.warn()) <<
"Got invalid header data";
1172 if (!
haveState_ && (packet.nodes().size() > 1) &&
1175 JLOG(
journal_.warn()) <<
"Included AS root invalid for ledger " <<
hash_
1176 <<
" from peer " << peer->id();
1187 JLOG(
journal_.warn()) <<
"Included TX root invalid for ledger " <<
hash_
1188 <<
" from peer " << peer->id();
1198 JLOG(
journal_.warn()) <<
"Included AS/TX root invalid for ledger " <<
hash_
1199 <<
" from peer " << peer->id() <<
": " << ex.
what();
1200 using namespace std::string_literals;
1212 if ((packet.type() == protocol::liTX_NODE) || (packet.type() == protocol::liAS_NODE))
1214 if (packet.nodes().empty())
1216 JLOG(
journal_.info()) << peer->id() <<
": response with no nodes";
1226 JLOG(
journal_.debug()) <<
"Ledger "
1227 << ((packet.type() == protocol::liTX_NODE) ?
"TX" :
"AS")
1228 <<
" node stats: " << san.
get();
1261 auto const i =
counts.find(peer);
1264 counts.emplace(std::move(peer), dataCount);
1267 i->second =
std::max(i->second, dataCount);
1278 while (i !=
counts.end())
1280 if (i->second < thresh)
1300 auto const outFunc = [&f](
auto&& v) { f(v.first); };
1312 counts.begin(),
counts.end(), boost::make_function_output_iterator(outFunc), n, rng);
1351 for (
auto const& entry : data)
1353 if (
auto peer = entry.first.lock())
1355 int const count =
processData(peer, *entry.second);
1356 dataCounts.
update(std::move(peer), count);
1379 ret[jss::complete] =
true;
1382 ret[jss::failed] =
true;
1385 ret[jss::peers] =
static_cast<int>(
peerSet_->getPeerIds().size());
1404 ret[jss::needed_state_hashes] = hv;
1414 ret[jss::needed_transaction_hashes] = hv;
T back_inserter(T... args)
Value & append(Value const &value)
Append value to array at the end.
InboundLedger(Application &app, UInt256 const &hash, std::uint32_t seq, Reason reason, ClockType &clock, std::unique_ptr< PeerSet > peerSet, std::chrono::milliseconds retryInterval=kRetryInterval)
json::Value getJson(int)
Return a json::ValueType::Object.
void tryDB(node_store::Database &srcDB)
std::weak_ptr< TimeoutCounter > pmDowncast() override
Return a weak pointer to this.
void trigger(std::shared_ptr< Peer > const &peer, TriggerReason reason)
Ask for more nodes, or judge what has been collected.
void runData()
Process pending TMLedgerData Query the a random sample of the 'best' peers.
void filterNodes(std::vector< std::pair< SHAMapNodeID, UInt256 > > &nodes, TriggerReason reason)
std::size_t getPeerCount() const
bool failOnZeroAccountHash()
Fail the acquisition when the header's account hash is zero.
std::set< UInt256 > recentNodes_
void receiveNode(std::shared_ptr< Peer > const &peer, protocol::TMLedgerData const &packet, SHAMapAddNode &san)
Process node data received from a peer Call with a lock.
std::vector< UInt256 > neededStateHashes(int max, SHAMapSyncFilter const *filter) const
int processData(std::shared_ptr< Peer > peer, protocol::TMLedgerData const &data)
Process one TMLedgerData Returns the number of useful nodes.
bool takeHeader(std::string_view data)
Build the ledger from a header a peer supplied.
bool takeAsRootNode(std::string_view data, SHAMapAddNode &san)
Process AS root node received from a peer Call with a lock.
std::shared_ptr< Ledger > ledger_
std::vector< std::pair< std::weak_ptr< Peer >, std::shared_ptr< protocol::TMLedgerData > > > receivedData_
std::mutex receivedDataLock_
std::unique_ptr< PeerSet > peerSet_
void addPeers()
Add more peers to the set, if possible.
bool takeTxRootNode(std::string_view data, SHAMapAddNode &san)
Process AS root node received from a peer Call with a lock.
void onTimer(bool progress, ScopedLockType &sl) override
Called with a lock by the PeerSet when the timer expires.
void init(ScopedLockType &collectionLock)
void update(std::uint32_t seq)
void done()
Settle the acquisition and signal whatever is waiting on it.
bool gotData(std::weak_ptr< Peer >, std::shared_ptr< protocol::TMLedgerData > const &)
Stash a TMLedgerData received from a peer for later processing Returns 'true' if we need to dispatch.
std::vector< NeededHashT > getNeededHashes()
beast::AbstractClock< std::chrono::steady_clock > ClockType
std::vector< UInt256 > neededTxHashes(int max, SHAMapSyncFilter const *filter) const
~InboundLedger() override
Rules controlling protocol behavior.
void incDuplicate()
Record one node that was not needed: the map already held it, or the map or its acquisition was no lo...
void incUseful()
Record one node that produced a good result.
std::string get() const
Render the tally as a log line.
void incInvalid()
Record one node that was rejected.
Identifies a node inside a SHAMap.
std::string getRawString() const
std::vector< std::pair< SHAMapNodeID, UInt256 > > getMissingNodes(int maxNodes, SHAMapSyncFilter const *filter)
Check for nodes in the SHAMap not available.
SHAMapHash getHash() const
int addRaw(Blob const &vector)
std::recursive_mutex mtx_
std::unique_lock< std::recursive_mutex > ScopedLockType
UInt256 const hash_
The hash of the object (in practice, always a ledger) we are trying to fetch.
void queueJob(ScopedLockType &)
Queue a job to call invokeOnTimer().
TimeoutCounter(Application &app, UInt256 const &targetHash, std::chrono::milliseconds timeoutInterval, QueueJobParameter &&jobParameter, beast::Journal journal)
bool progress_
Whether forward progress has been made.
Persistency layer for NodeObject.
std::shared_ptr< NodeObject > fetchNodeObject(UInt256 const &hash, std::uint32_t ledgerSeq=0, FetchType fetchType=FetchType::Synchronous, bool duplicate=false)
Fetch a node object.
T emplace_back(T... args)
@ Array
array value (ordered list)
@ Object
object value (collection of name/value pairs).
Charge const kFeeMalformedRequest
Schedule of fees charged for imposing load on the server.
Charge const kFeeInvalidData
Use hash_* containers for keys that do not need a cryptographically secure hashing algorithm.
static constexpr auto kReqNodesReply
Number root(Number f, unsigned d)
std::optional< SHAMapNodeID > getSHAMapNodeID(protocol::TMLedgerNode const &ledgerNode, SHAMapTreeNode const &treeNode)
Extracts or reconstructs the SHAMapNodeID from a ledger node proto message.
LedgerHeader deserializeHeader(Slice data, bool hasHash=false)
Deserialize a ledger header from a byte array.
std::string to_string(BaseUInt< Bits, Tag > const &a)
static constexpr std::uint32_t kXrpLedgerEarliestFees
The XRP Ledger mainnet's earliest ledger with a FeeSettings object.
SHAMapTreeNodePtr getTreeNode(std::string_view data)
Deserializes a SHAMapTreeNode from wire format data.
static constexpr auto kMissingNodesFind
Slice makeSlice(std::array< T, N > const &a)
static std::vector< UInt256 > neededHashes(UInt256 const &root, SHAMap &map, int max, SHAMapSyncFilter const *filter)
static constexpr auto kLedgerBecomeAggressiveThreshold
static constexpr auto kPeerCountAdd
static constexpr auto kLedgerTimeoutRetriesMax
static constexpr auto kPeerCountStart
@ LedgerMaster
ledger master data for signing
FeeSettingsEntry< ReadView > FeeSettingsEntryR
std::vector< unsigned char > Blob
Storage for linear binary data.
static constexpr auto kReqNodes
LedgerHeader deserializePrefixedHeader(Slice data, bool hasHash=false)
Deserialize a ledger header (prefixed with 4 bytes) from a byte array.
T shared_from_this(T... args)
T stable_partition(T... args)
std::unordered_map< std::shared_ptr< Peer >, int > counts
void update(std::shared_ptr< Peer > &&peer, int dataCount)
void sampleN(std::size_t n, F &&f)