ValidatorList.h

#pragma once
 
#include <xrpld/core/TimeKeeper.h>
#include <xrpld/overlay/Message.h>
 
#include <xrpl/basics/Log.h>
#include <xrpl/basics/UnorderedContainers.h>
#include <xrpl/crypto/csprng.h>
#include <xrpl/json/json_value.h>
#include <xrpl/protocol/PublicKey.h>
#include <xrpl/server/Manifest.h>
 
#include <boost/thread/shared_mutex.hpp>
 
#include <mutex>
#include <shared_mutex>
 
namespace protocol {
class TMValidatorList;
class TMValidatorListCollection;
}  // namespace protocol
 
namespace xrpl {
 
class Overlay;
class HashRouter;
class Message;
class NetworkOPs;
class Peer;
class STValidation;
 
/* Entries in this enum are ordered by "desirability".
   The "better" dispositions have lower values than the
   "worse" dispositions */
enum class ListDisposition {
    Accepted = 0,

    Expired,

    Pending,

    SameSequence,

    KnownSequence,

    Stale,

    Untrusted,

    UnsupportedVersion,

    Invalid
};
 
/* Entries in this enum are ordered by "desirability".
   The "better" dispositions have lower values than the
   "worse" dispositions */
enum class PublisherStatus {
    // Publisher has provided a valid file
    Available = 0,
 
    // Current list is expired without replacement
    Expired,
 
    // No file seen yet
    Unavailable,
 
    // Publisher has revoked their manifest key
    Revoked,
 
};
 
std::string
to_string(ListDisposition disposition);

struct TrustChanges
{
    explicit TrustChanges() = default;
 
    hash_set<NodeID> added;
    hash_set<NodeID> removed;
};

struct ValidatorBlobInfo
{
    // base-64 encoded JSON containing the validator list.
    std::string blob;
    // hex-encoded signature of the blob using the publisher's signing key
    std::string signature;
    // base-64 or hex-encoded manifest containing the publisher's master and
    // signing public keys
    std::optional<std::string> manifest;
};

class ValidatorList
{
    struct PublisherList
    {
        explicit PublisherList() = default;
 
        std::vector<PublicKey> list;
        std::vector<std::string> manifests;
        std::size_t sequence{};
        TimeKeeper::time_point validFrom;
        TimeKeeper::time_point validUntil;
        std::string siteUri;
        // base-64 encoded JSON containing the validator list.
        std::string rawBlob;
        // hex-encoded signature of the blob using the publisher's signing key
        std::string rawSignature;
        // base-64 or hex-encoded manifest containing the publisher's master and
        // signing public keys
        std::optional<std::string> rawManifest;
        uint256 hash;
    };
 
    struct PublisherListCollection
    {
        PublisherStatus status = PublisherStatus::Unavailable;
        /*
        The `current` VL is the one which
         1. Has the largest sequence number that
         2. Has ever been effective (the effective date is absent or in the
            past).
        If this VL has expired, all VLs with previous sequence numbers
        will also be considered expired, and thus there will be no valid VL
        until one with a larger sequence number becomes effective. This is to
        prevent allowing old VLs to reactivate.
        */
        PublisherList current;
        /*
        The `remaining` list holds any relevant VLs which have a larger sequence
        number than current. By definition they will all have an effective date
        in the future. Relevancy will be determined by sorting the VLs by
        sequence number, then iterating over the list and removing any VLs for
        which the following VL (ignoring gaps) has the same or earlier effective
        date.
        */
        std::map<std::size_t, PublisherList> remaining;
        std::optional<std::size_t> maxSequence;
        // The hash of the full set if sent in a single message
        uint256 fullHash;
        std::string rawManifest;
        std::uint32_t rawVersion = 0;
    };
 
    ManifestCache& validatorManifests_;
    ManifestCache& publisherManifests_;
    TimeKeeper& timeKeeper_;
    boost::filesystem::path const dataPath_;
    beast::Journal const j_;
    std::shared_mutex mutable mutex_;
    using scoped_lock = std::scoped_lock<decltype(mutex_)>;
    using shared_lock = std::shared_lock<decltype(mutex_)>;
 
    std::atomic<std::size_t> quorum_;
    std::optional<std::size_t> minimumQuorum_;
 
    // Published lists stored by publisher master public key
    hash_map<PublicKey, PublisherListCollection> publisherLists_;
 
    // Listed master public keys with the number of lists they appear on
    hash_map<PublicKey, std::size_t> keyListings_;
 
    // The current list of trusted master keys
    hash_set<PublicKey> trustedMasterKeys_;
 
    // Minimum number of lists on which a trusted validator must appear on
    std::size_t listThreshold_{1};
 
    // The current list of trusted signing keys. For those validators using
    // a manifest, the signing key is the ephemeral key. For the ones using
    // a seed, the signing key is the same as the master key.
    hash_set<PublicKey> trustedSigningKeys_;
 
    std::optional<PublicKey> localPubKey_;
 
    // The below variable contains the Publisher list specified in the local
    // config file under the title of [validators]. This list is not associated
    // with the masterKey of any publisher.
 
    // Apropos PublisherListCollection fields, localPublisherList does not
    // have any "remaining" manifests. It is assumed to be perennially
    // "available". The "validUntil" field is set to the highest possible
    // value of the field, hence this list is always valid.
    PublisherList localPublisherList_;
 
    // The master public keys of the current negative UNL
    hash_set<PublicKey> negativeUNL_;
 
    // Currently supported versions of publisher list format
    static constexpr std::uint32_t kSupportedListVersions[]{1, 2};
    // In the initial release, to prevent potential abuse and attacks, any VL
    // collection with more than 5 entries will be considered malformed.
    static constexpr std::size_t kMaxSupportedBlobs = 5;
    // Prefix of the file name used to store cache files.
    static std::string const kFilePrefix;
 
public:
    ValidatorList(
        ManifestCache& validatorManifests,
        ManifestCache& publisherManifests,
        TimeKeeper& timeKeeper,
        std::string const& databasePath,
        beast::Journal j,
        std::optional<std::size_t> minimumQuorum = std::nullopt);
    ~ValidatorList() = default;

    struct PublisherListStats
    {
        explicit PublisherListStats() = default;
        explicit PublisherListStats(ListDisposition d);
        PublisherListStats(ListDisposition d, PublicKey key, PublisherStatus stat, std::size_t seq);
 
        [[nodiscard]] ListDisposition
        bestDisposition() const;
        [[nodiscard]] ListDisposition
        worstDisposition() const;
        void
        mergeDispositions(PublisherListStats const& src);
 
        // Tracks the dispositions of each processed list and how many times it
        // occurred
        std::map<ListDisposition, std::size_t> dispositions;
        std::optional<PublicKey> publisherKey;
        PublisherStatus status = PublisherStatus::Unavailable;
        std::size_t sequence = 0;
    };
 
    struct MessageWithHash
    {
        explicit MessageWithHash() = default;
        explicit MessageWithHash(
            std::shared_ptr<Message> const& message,
            uint256 hash,
            std::size_t num);
        std::shared_ptr<Message> message;
        uint256 hash;
        std::size_t numVLs = 0;
    };

    bool
    load(
        std::optional<PublicKey> const& localSigningKey,
        std::vector<std::string> const& configKeys,
        std::vector<std::string> const& publisherKeys,
        std::optional<std::size_t> listThreshold = {});

    static std::vector<ValidatorBlobInfo>
    parseBlobs(std::uint32_t version, json::Value const& body);
 
    static std::vector<ValidatorBlobInfo>
    parseBlobs(protocol::TMValidatorList const& body);
 
    static std::vector<ValidatorBlobInfo>
    parseBlobs(protocol::TMValidatorListCollection const& body);
 
    static void
    sendValidatorList(
        Peer& peer,
        std::uint64_t peerSequence,
        PublicKey const& publisherKey,
        std::size_t maxSequence,
        std::uint32_t rawVersion,
        std::string const& rawManifest,
        std::map<std::size_t, ValidatorBlobInfo> const& blobInfos,
        HashRouter& hashRouter,
        beast::Journal j);
 
    [[nodiscard]] static std::pair<std::size_t, std::size_t>
    buildValidatorListMessages(
        std::size_t messageVersion,
        std::uint64_t peerSequence,
        std::size_t maxSequence,
        std::uint32_t rawVersion,
        std::string const& rawManifest,
        std::map<std::size_t, ValidatorBlobInfo> const& blobInfos,
        std::vector<MessageWithHash>& messages,
        std::size_t maxSize = kMaximumMessageSize);

    PublisherListStats
    applyListsAndBroadcast(
        std::string const& manifest,
        std::uint32_t version,
        std::vector<ValidatorBlobInfo> const& blobs,
        std::string siteUri,
        uint256 const& hash,
        Overlay& overlay,
        HashRouter& hashRouter,
        NetworkOPs& networkOPs);

    PublisherListStats
    applyLists(
        std::string const& manifest,
        std::uint32_t version,
        std::vector<ValidatorBlobInfo> const& blobs,
        std::string siteUri,
        std::optional<uint256> const& hash = {});
 
    /* Attempt to read previously stored list files. Expected to only be
       called when loading from URL fails.
 
       @return A list of valid file:// URLs, if any.
 
       @par Thread Safety
 
       May be called concurrently
    */
    std::vector<std::string>
    loadLists();

    TrustChanges
    updateTrusted(
        hash_set<NodeID> const& seenValidators,
        NetClock::time_point closeTime,
        NetworkOPs& ops,
        Overlay& overlay,
        HashRouter& hashRouter);

    std::size_t
    quorum() const
    {
        return quorum_;
    }

    bool
    trusted(PublicKey const& identity) const;

    bool
    listed(PublicKey const& identity) const;

    std::optional<PublicKey>
    getTrustedKey(PublicKey const& identity) const;

    std::optional<PublicKey>
    getListedKey(PublicKey const& identity) const;

    bool
    trustedPublisher(PublicKey const& identity) const;

    std::optional<PublicKey>
    localPublicKey() const;

    void
    forEachListed(std::function<void(PublicKey const&, bool)> func) const;

    void
    forEachAvailable(
        std::function<void(
            std::string const& manifest,
            std::uint32_t version,
            std::map<std::size_t, ValidatorBlobInfo> const& blobInfos,
            PublicKey const& pubKey,
            std::size_t maxSequence,
            uint256 const& hash)> func) const;

    std::optional<json::Value>
    getAvailable(std::string_view pubKey, std::optional<std::uint32_t> forceVersion = {});

    std::size_t
    count() const;

    std::optional<TimeKeeper::time_point>
    expires() const;

    json::Value
    getJson() const;
 
    using QuorumKeys = std::pair<std::size_t const, hash_set<PublicKey>>;
    QuorumKeys
    getQuorumKeys() const
    {
        shared_lock const readLock{mutex_};
        return {quorum_, trustedSigningKeys_};
    }

    hash_set<PublicKey>
    getTrustedMasterKeys() const;

    std::size_t
    getListThreshold() const;

    hash_set<PublicKey>
    getNegativeUNL() const;

    void
    setNegativeUNL(hash_set<PublicKey> const& negUnl);

    std::vector<std::shared_ptr<STValidation>>
    negativeUNLFilter(std::vector<std::shared_ptr<STValidation>>&& validations) const;
 
private:
    std::size_t
    count(shared_lock const&) const;

    bool
    trusted(shared_lock const&, PublicKey const& identity) const;

    std::optional<PublicKey>
    getTrustedKey(shared_lock const&, PublicKey const& identity) const;

    std::optional<TimeKeeper::time_point>
    expires(shared_lock const&) const;

    PublisherListStats
    applyList(
        std::string const& globalManifest,
        std::optional<std::string> const& localManifest,
        std::string const& blob,
        std::string const& signature,
        std::uint32_t version,
        std::string siteUri,
        std::optional<uint256> const& hash,
        scoped_lock const&);
 
    // This function updates the keyListings_ counts for all the trusted
    // master keys
    void
    updatePublisherList(
        PublicKey const& pubKey,
        PublisherList const& current,
        std::vector<PublicKey> const& oldList,
        scoped_lock const&);
 
    static void
    buildBlobInfos(
        std::map<std::size_t, ValidatorBlobInfo>& blobInfos,
        PublisherListCollection const& lists);
 
    static std::map<std::size_t, ValidatorBlobInfo>
    buildBlobInfos(PublisherListCollection const& lists);
 
    static void
    broadcastBlobs(
        PublicKey const& publisherKey,
        PublisherListCollection const& lists,
        std::size_t maxSequence,
        uint256 const& hash,
        Overlay& overlay,
        HashRouter& hashRouter,
        beast::Journal j);
 
    static void
    sendValidatorList(
        Peer& peer,
        std::uint64_t peerSequence,
        PublicKey const& publisherKey,
        std::size_t maxSequence,
        std::uint32_t rawVersion,
        std::string const& rawManifest,
        std::map<std::size_t, ValidatorBlobInfo> const& blobInfos,
        std::vector<MessageWithHash>& messages,
        HashRouter& hashRouter,
        beast::Journal j);

    boost::filesystem::path
    getCacheFileName(scoped_lock const&, PublicKey const& pubKey) const;

    static json::Value
    buildFileData(
        std::string const& pubKey,
        PublisherListCollection const& pubCollection,
        beast::Journal j);

    static json::Value
    buildFileData(
        std::string const& pubKey,
        PublisherListCollection const& pubCollection,
        std::optional<std::uint32_t> forceVersion,
        beast::Journal j);
 
    template <class Hasher>
    friend void
    hash_append(Hasher& h, PublisherListCollection pl)
    {
        using beast::hash_append;
        hash_append(h, pl.rawManifest, buildBlobInfos(pl), pl.rawVersion);
    }

    void
    cacheValidatorFile(scoped_lock const& lock, PublicKey const& pubKey) const;

    std::pair<ListDisposition, std::optional<PublicKey>>
    verify(
        scoped_lock const&,
        json::Value& list,
        Manifest manifest,
        std::string const& blob,
        std::string const& signature);

    bool
    removePublisherList(scoped_lock const&, PublicKey const& publisherKey, PublisherStatus reason);

    std::size_t
    calculateQuorum(std::size_t unlSize, std::size_t effectiveUnlSize, std::size_t seenSize);
};
 
// hashing helpers
template <class Hasher>
void
hash_append(Hasher& h, ValidatorBlobInfo const& blobInfo)
{
    using beast::hash_append;
    hash_append(h, blobInfo.blob, blobInfo.signature);
    if (blobInfo.manifest)
    {
        hash_append(h, *blobInfo.manifest);
    }
}
 
template <class Hasher>
void
hash_append(Hasher& h, std::vector<ValidatorBlobInfo> const& blobs)
{
    for (auto const& item : blobs)
        hash_append(h, item);
}
 
template <class Hasher>
void
hash_append(Hasher& h, std::map<std::size_t, ValidatorBlobInfo> const& blobs)
{
    for (auto const& [_, item] : blobs)
    {
        (void)_;
        hash_append(h, item);
    }
}
 
}  // namespace xrpl
 
namespace protocol {
 
template <class Hasher>
void
hash_append(Hasher& h, TMValidatorList const& msg)
{
    using beast::hash_append;
    hash_append(h, msg.manifest(), msg.blob(), msg.signature(), msg.version());
}
 
template <class Hasher>
void
hash_append(Hasher& h, TMValidatorListCollection const& msg)
{
    using beast::hash_append;
    hash_append(h, msg.manifest(), xrpl::ValidatorList::parseBlobs(msg), msg.version());
}
 
}  // namespace protocol