This namespace contains everything to do with the ETL and ETL sources. More...

Classes
class	CacheLoader
	Cache loading interface. More...
struct	CacheLoaderSettings
	Settings for the cache loader. More...
class	CorruptionDetector
	A helper to notify Clio operator about a corruption in the DB. More...
class	ETLService
	This class is responsible for continuously extracting data from a p2p node, and writing that data to the databases. More...
struct	ETLServiceTag
	A tag class to help identify ETLService in templated code. More...
struct	ETLState
	This class is responsible for fetching and storing the state of the ETL information, such as the network id. More...
struct	LedgerFetcherInterface
	An interface for LedgerFetcher. More...
class	LoadBalancer
	This class is used to manage connections to transaction processing processes. More...
struct	LoadBalancerTag
	A tag class to help identify LoadBalancer in templated code. More...
class	NetworkValidatedLedgers
	This datastructure is used to keep track of the sequence of the most recent ledger validated by the network. More...
class	NetworkValidatedLedgersInterface
	An interface for NetworkValidatedLedgers. More...
class	SourceBase
	Provides an implementation of a ETL source. More...
struct	SystemState
	Represents the state of the ETL subsystem. More...
class	ThreadSafeQueue
	Generic thread-safe queue with a max capacity. More...

Concepts
concept	SomeETLService
concept	SomeLoadBalancer

Typedefs
using	SourcePtr = std::unique_ptr<SourceBase>
using	SourceFactory

Functions
CacheLoaderSettings	makeCacheLoaderSettings (util::config::ClioConfigDefinition const &config)
	Create a CacheLoaderSettings object from a Config object.
std::vector< ripple::uint256 >	getMarkers (size_t numMarkers)
	Partitions the uint256 keyspace into numMarkers partitions, each of equal size.
ETLState	tag_invoke (boost::json::value_to_tag< ETLState >, boost::json::value const &jv)
	Parse a boost::json::value into a ETLState.
std::optional< MPTHolderData >	getMPTokenAuthorize (ripple::TxMeta const &txMeta)
	Get the MPToken created from a transaction.
std::optional< MPTHolderData >	getMPTHolderFromTx (ripple::TxMeta const &txMeta, ripple::STTx const &sttx)
	Pull MPT data from TX via ETLService.
std::optional< MPTHolderData >	getMPTHolderFromObj (std::string const &key, std::string const &blob)
	Pull MPT data from ledger object via loadInitialLedger.
std::pair< std::vector< NFTTransactionsData >, std::optional< NFTsData > >	getNftokenModifyData (ripple::TxMeta const &txMeta, ripple::STTx const &sttx)
	Get the NFT URI change data from a NFToken Modify transaction.
std::pair< std::vector< NFTTransactionsData >, std::optional< NFTsData > >	getNFTokenMintData (ripple::TxMeta const &txMeta, ripple::STTx const &sttx)
	Get the NFT Token mint data from a transaction.
std::pair< std::vector< NFTTransactionsData >, std::optional< NFTsData > >	getNFTokenBurnData (ripple::TxMeta const &txMeta, ripple::STTx const &sttx)
	Get the NFT Token burn data from a transaction.
std::pair< std::vector< NFTTransactionsData >, std::optional< NFTsData > >	getNFTokenAcceptOfferData (ripple::TxMeta const &txMeta, ripple::STTx const &sttx)
	Get the NFT Token accept offer data from a transaction.
std::pair< std::vector< NFTTransactionsData >, std::optional< NFTsData > >	getNFTokenCancelOfferData (ripple::TxMeta const &txMeta, ripple::STTx const &sttx)
	Get the NFT Token cancel offer data from a transaction.
std::pair< std::vector< NFTTransactionsData >, std::optional< NFTsData > >	getNFTokenCreateOfferData (ripple::TxMeta const &txMeta, ripple::STTx const &sttx)
	Get the NFT Token create offer data from a transaction.
std::pair< std::vector< NFTTransactionsData >, std::optional< NFTsData > >	getNFTDataFromTx (ripple::TxMeta const &txMeta, ripple::STTx const &sttx)
	Pull NFT data from TX via ETLService.
std::vector< NFTsData >	getNFTDataFromObj (std::uint32_t seq, std::string const &key, std::string const &blob)
	Pull NFT data from ledger object via loadInitialLedger.
std::vector< NFTsData >	getUniqueNFTsDatas (std::vector< NFTsData > const &nfts)
	Get the unique NFTs data from a vector of NFTsData happening in the same ledger. For example, if a NFT has both accept offer and burn happening in the same ledger,we only keep the final state of the NFT.
SourcePtr	makeSource (util::config::ObjectView const &config, boost::asio::io_context &ioc, std::shared_ptr< BackendInterface > backend, std::shared_ptr< feed::SubscriptionManagerInterface > subscriptions, std::shared_ptr< NetworkValidatedLedgersInterface > validatedLedgers, std::chrono::steady_clock::duration forwardingTimeout, SourceBase::OnConnectHook onConnect, SourceBase::OnDisconnectHook onDisconnect, SourceBase::OnLedgerClosedHook onLedgerClosed)
	Create a source.

Detailed Description

This namespace contains everything to do with the ETL and ETL sources.

Typedef Documentation

◆ SourceFactory

using etl::SourceFactory

Initial value:

 std::function<SourcePtr(
    util::config::ObjectView const& config,
    boost::asio::io_context& ioc,
    std::shared_ptr<BackendInterface> backend,
    std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
    std::shared_ptr<NetworkValidatedLedgersInterface> validatedLedgers,
    std::chrono::steady_clock::duration forwardingTimeout,
    SourceBase::OnConnectHook onConnect,
    SourceBase::OnDisconnectHook onDisconnect,
    SourceBase::OnLedgerClosedHook onLedgerClosed
)>

Function Documentation

◆ getMarkers()

std::vector< ripple::uint256 > etl::getMarkers ( size_t numMarkers )

Partitions the uint256 keyspace into numMarkers partitions, each of equal size.

Parameters

numMarkers Total markers to partition for

Returns: The markers

◆ getMPTHolderFromObj()

std::optional< MPTHolderData > etl::getMPTHolderFromObj	(	std::string const &	key,
		std::string const &	blob )

Pull MPT data from ledger object via loadInitialLedger.

Parameters

key	The owner key
blob	Object data as blob

Returns: The MPTIssuanceID and holder pair as a optional

◆ getMPTHolderFromTx()

std::optional< MPTHolderData > etl::getMPTHolderFromTx	(	ripple::TxMeta const &	txMeta,
		ripple::STTx const &	sttx )

Pull MPT data from TX via ETLService.

Parameters

txMeta	Transaction metadata
sttx	The transaction

Returns: The MPTIssuanceID and holder pair as a optional

◆ getMPTokenAuthorize()

std::optional< MPTHolderData > etl::getMPTokenAuthorize ( ripple::TxMeta const & txMeta )

Get the MPToken created from a transaction.

Parameters

txMeta Transaction metadata

Returns: MPT and holder account pair

◆ getNFTDataFromObj()

std::vector< NFTsData > etl::getNFTDataFromObj	(	std::uint32_t	seq,
		std::string const &	key,
		std::string const &	blob )

Pull NFT data from ledger object via loadInitialLedger.

Parameters

seq	The ledger sequence to pull for
key	The owner key
blob	Object data as blob

Returns: The NFT data as a vector

◆ getNFTDataFromTx()

std::pair< std::vector< NFTTransactionsData >, std::optional< NFTsData > > etl::getNFTDataFromTx	(	ripple::TxMeta const &	txMeta,
		ripple::STTx const &	sttx )

Pull NFT data from TX via ETLService.

Parameters

txMeta	Transaction metadata
sttx	The transaction

Returns: NFT transactions data as a pair of transactions and optional NFTsData

◆ getNFTokenAcceptOfferData()

std::pair< std::vector< NFTTransactionsData >, std::optional< NFTsData > > etl::getNFTokenAcceptOfferData	(	ripple::TxMeta const &	txMeta,
		ripple::STTx const &	sttx )

Get the NFT Token accept offer data from a transaction.

Parameters

txMeta	Transaction metadata
sttx	The transaction

Returns: NFT Token accept offer data as a pair of transactions and optional NFTsData

◆ getNFTokenBurnData()

std::pair< std::vector< NFTTransactionsData >, std::optional< NFTsData > > etl::getNFTokenBurnData	(	ripple::TxMeta const &	txMeta,
		ripple::STTx const &	sttx )

Get the NFT Token burn data from a transaction.

Parameters

txMeta	Transaction metadata
sttx	The transaction

Returns: NFT Token burn data as a pair of transactions and optional NFTsData

◆ getNFTokenCancelOfferData()

std::pair< std::vector< NFTTransactionsData >, std::optional< NFTsData > > etl::getNFTokenCancelOfferData	(	ripple::TxMeta const &	txMeta,
		ripple::STTx const &	sttx )

Get the NFT Token cancel offer data from a transaction.

Parameters

txMeta	Transaction metadata
sttx	The transaction

Returns: NFT Token cancel offer data as a pair of transactions and optional NFTsData

◆ getNFTokenCreateOfferData()

std::pair< std::vector< NFTTransactionsData >, std::optional< NFTsData > > etl::getNFTokenCreateOfferData	(	ripple::TxMeta const &	txMeta,
		ripple::STTx const &	sttx )

Get the NFT Token create offer data from a transaction.

Parameters

txMeta	Transaction metadata
sttx	The transaction

Returns: NFT Token create offer data as a pair of transactions and optional NFTsData

◆ getNFTokenMintData()

std::pair< std::vector< NFTTransactionsData >, std::optional< NFTsData > > etl::getNFTokenMintData	(	ripple::TxMeta const &	txMeta,
		ripple::STTx const &	sttx )

Get the NFT Token mint data from a transaction.

Parameters

txMeta	Transaction metadata
sttx	The transaction

Returns: NFT Token mint data as a pair of transactions and optional NFTsData

◆ getNftokenModifyData()

std::pair< std::vector< NFTTransactionsData >, std::optional< NFTsData > > etl::getNftokenModifyData	(	ripple::TxMeta const &	txMeta,
		ripple::STTx const &	sttx )

Get the NFT URI change data from a NFToken Modify transaction.

Parameters

txMeta	Transaction metadata
sttx	The transaction

Returns: NFT URI change data as a pair of transactions and optional NFTsData

◆ getUniqueNFTsDatas()

std::vector< NFTsData > etl::getUniqueNFTsDatas ( std::vector< NFTsData > const & nfts )

Get the unique NFTs data from a vector of NFTsData happening in the same ledger. For example, if a NFT has both accept offer and burn happening in the same ledger,we only keep the final state of the NFT.

Parameters

nfts	The NFTs data to filter, happening in the same ledger

Returns: The unique NFTs data

◆ makeCacheLoaderSettings()

CacheLoaderSettings etl::makeCacheLoaderSettings ( util::config::ClioConfigDefinition const & config )

nodiscard

Create a CacheLoaderSettings object from a Config object.

Parameters

config The configuration object

Returns: The CacheLoaderSettings object

◆ makeSource()

SourcePtr etl::makeSource	(	util::config::ObjectView const &	config,
		boost::asio::io_context &	ioc,
		std::shared_ptr< BackendInterface >	backend,
		std::shared_ptr< feed::SubscriptionManagerInterface >	subscriptions,
		std::shared_ptr< NetworkValidatedLedgersInterface >	validatedLedgers,
		std::chrono::steady_clock::duration	forwardingTimeout,
		SourceBase::OnConnectHook	onConnect,
		SourceBase::OnDisconnectHook	onDisconnect,
		SourceBase::OnLedgerClosedHook	onLedgerClosed )

Create a source.

Parameters

config	The configuration to use
ioc	The io_context to run on
backend	BackendInterface implementation
subscriptions	Subscription manager
validatedLedgers	The network validated ledgers data structure
forwardingTimeout	The timeout for forwarding to rippled
onConnect	The hook to call on connect
onDisconnect	The hook to call on disconnect
onLedgerClosed	The hook to call on ledger closed. This is called when a ledger is closed and the source is set as forwarding.

Returns: The created source

◆ tag_invoke()

ETLState etl::tag_invoke	(	boost::json::value_to_tag< ETLState >	,
		boost::json::value const &	jv )

Parse a boost::json::value into a ETLState.

Parameters

jv	The json value to convert

Returns: The ETLState

Classes

Concepts

Typedefs

Functions

Detailed Description

Typedef Documentation

◆ SourceFactory

Function Documentation

◆ getMarkers()

◆ getMPTHolderFromObj()

◆ getMPTHolderFromTx()

◆ getMPTokenAuthorize()

◆ getNFTDataFromObj()

◆ getNFTDataFromTx()

◆ getNFTokenAcceptOfferData()

◆ getNFTokenBurnData()

◆ getNFTokenCancelOfferData()

◆ getNFTokenCreateOfferData()

◆ getNFTokenMintData()

◆ getNftokenModifyData()

◆ getUniqueNFTsDatas()

◆ makeCacheLoaderSettings()

◆ makeSource()

◆ tag_invoke()