xrpld
Loading...
Searching...
No Matches
compression_test.cpp
1#include <test/jtx/Account.h>
2#include <test/jtx/Env.h>
3#include <test/jtx/WSClient.h>
4#include <test/jtx/amount.h>
5#include <test/jtx/envconfig.h>
6#include <test/jtx/pay.h>
7
8#include <xrpld/core/Config.h>
9#include <xrpld/overlay/Compression.h>
10#include <xrpld/overlay/Message.h>
11#include <xrpld/overlay/detail/Handshake.h>
12#include <xrpld/overlay/detail/ProtocolMessage.h>
13#include <xrpld/overlay/detail/ZeroCopyStream.h>
14
15#include <xrpl/basics/Log.h>
16#include <xrpl/basics/Slice.h>
17#include <xrpl/basics/StringUtilities.h>
18#include <xrpl/basics/base_uint.h>
19#include <xrpl/basics/chrono.h>
20#include <xrpl/basics/strHex.h>
21#include <xrpl/beast/net/IPAddress.h>
22#include <xrpl/beast/unit_test/suite.h>
23#include <xrpl/beast/utility/Journal.h>
24#include <xrpl/json/json_value.h>
25#include <xrpl/protocol/HashPrefix.h>
26#include <xrpl/protocol/KeyType.h>
27#include <xrpl/protocol/LedgerHeader.h>
28#include <xrpl/protocol/SField.h>
29#include <xrpl/protocol/STObject.h>
30#include <xrpl/protocol/SecretKey.h>
31#include <xrpl/protocol/Seed.h>
32#include <xrpl/protocol/Serializer.h>
33#include <xrpl/protocol/Sign.h>
34#include <xrpl/protocol/digest.h>
35#include <xrpl/protocol/jss.h>
36#include <xrpl/shamap/SHAMapNodeID.h>
37
38#include <boost/asio/buffer.hpp>
39#include <boost/asio/ip/address.hpp>
40#include <boost/beast/core/multi_buffer.hpp>
41#include <boost/system/detail/error_code.hpp>
42
43#include <xrpl.pb.h>
44
45#include <algorithm>
46#include <cstdint>
47#include <memory>
48#include <sstream>
49#include <string>
50#include <vector>
51
52namespace xrpl::test {
53
54using namespace xrpl::test;
55using namespace xrpl::test::jtx;
56
57static UInt256
72
74{
77
78public:
79 compression_test() = default;
80
81 template <typename T>
82 void
83 doTest(std::shared_ptr<T> proto, protocol::MessageType mt, uint16_t nbuffers, std::string msg)
84 {
85 testcase("Compress/Decompress: " + msg);
86
87 Message m(*proto, mt);
88
89 auto& buffer = m.getBuffer(Compressed::On);
90
91 boost::beast::multi_buffer buffers;
92
93 // simulate multi-buffer
94 auto const sz = buffer.size() / nbuffers;
95 for (int i = 0; i < nbuffers; i++)
96 {
97 auto const start = buffer.begin() + sz * i;
98 auto const end = i < nbuffers - 1 ? (buffer.begin() + sz * (i + 1)) : buffer.end();
99 std::vector<std::uint8_t> slice(start, end);
100 buffers.commit(
101 boost::asio::buffer_copy(
102 buffers.prepare(slice.size()), boost::asio::buffer(slice)));
103 }
104
105 boost::system::error_code ec;
106 auto header = xrpl::detail::parseMessageHeader(ec, buffers.data(), buffer.size());
107
108 BEAST_EXPECT(header);
109
110 if (!header || header->algorithm == Algorithm::None)
111 return;
112
113 std::vector<std::uint8_t> decompressed;
114 decompressed.resize(header->uncompressedSize);
115
116 BEAST_EXPECT(header->payloadWireSize == buffer.size() - header->headerSize);
117
118 ZeroCopyInputStream stream(buffers.data());
119 stream.Skip(header->headerSize);
120
121 auto const decompressedSize = xrpl::compression::decompress(
122 stream, header->payloadWireSize, decompressed.data(), header->uncompressedSize);
123 BEAST_EXPECT(decompressedSize == header->uncompressedSize);
124 auto const proto1 = std::make_shared<T>();
125
126 BEAST_EXPECT(proto1->ParseFromArray(decompressed.data(), decompressedSize));
127 auto uncompressed = m.getBuffer(Compressed::Off);
128 BEAST_EXPECT(
130 uncompressed.begin() + xrpl::compression::kHeaderBytes,
131 uncompressed.end(),
132 decompressed.begin()));
133 }
134
137 {
139 manifests->mutable_list()->Reserve(n);
140 for (int i = 0; i < n; i++)
141 {
142 auto master = randomKeyPair(KeyType::Ed25519);
143 auto signing = randomKeyPair(KeyType::Ed25519);
145 st[sfSequence] = i;
146 st[sfPublicKey] = std::get<0>(master);
147 st[sfSigningPubKey] = std::get<0>(signing);
148 st[sfDomain] =
149 makeSlice(std::string("example") + std::to_string(i) + std::string(".com"));
150 sign(
151 st, HashPrefix::Manifest, KeyType::Ed25519, std::get<1>(master), sfMasterSignature);
152 sign(st, HashPrefix::Manifest, KeyType::Ed25519, std::get<1>(signing));
153 Serializer s;
154 st.add(s);
155 auto* manifest = manifests->add_list();
156 manifest->set_stobject(s.data(), s.size());
157 }
158 return manifests;
159 }
160
163 {
165 endpoints->mutable_endpoints_v2()->Reserve(n);
166 for (int i = 0; i < n; i++)
167 {
168 auto ep = endpoints->add_endpoints_v2();
169 ep->set_endpoint(std::string("10.0.1.") + std::to_string(i));
170 ep->set_hops(i);
171 }
172 endpoints->set_version(2);
173
174 return endpoints;
175 }
176
179 {
180 Env env(*this, envconfig());
181 int const fund = 10000;
182 auto const alice = Account("alice");
183 auto const bob = Account("bob");
184 env.fund(XRP(fund), "alice", "bob");
185 env.trust(bob["USD"](fund), alice);
186 env.close();
187
188 auto const toBinary = [this](std::string const& text) {
189 auto blob = strUnHex(text);
190 BEAST_EXPECT(blob);
191 return std::string{reinterpret_cast<char const*>(blob->data()), blob->size()};
192 };
193
194 std::string usdTxBlob;
195 auto wsc = makeWSClient(env.app().config());
196 {
197 json::Value requestUSD;
198 requestUSD[jss::secret] = toBase58(generateSeed("bob"));
199 requestUSD[jss::tx_json] = pay("bob", "alice", bob["USD"](fund / 2));
200 json::Value replyUSD = wsc->invoke("sign", requestUSD);
201
202 usdTxBlob = toBinary(replyUSD[jss::result][jss::tx_blob].asString());
203 }
204
206 transaction->set_rawtransaction(usdTxBlob);
207 transaction->set_status(protocol::tsNEW);
208 transaction->set_deferred(true);
209
210 return transaction;
211 }
212
215 {
217 getLedger->set_itype(protocol::liTS_CANDIDATE);
218 getLedger->set_ltype(protocol::TMLedgerType::ltACCEPTED);
219 UInt256 const hash(xrpl::sha512Half(123456789));
220 getLedger->set_ledgerhash(hash.begin(), hash.size());
221 getLedger->set_ledgerseq(123456789);
222 xrpl::SHAMapNodeID const sha(64, hash);
223 getLedger->add_nodeids(sha.getRawString());
224 getLedger->set_requestcookie(123456789);
225 getLedger->set_querytype(protocol::qtINDIRECT);
226 getLedger->set_querydepth(3);
227 return getLedger;
228 }
229
231 buildLedgerData(uint32_t n, Logs& logs)
232 {
234 UInt256 const hash(xrpl::sha512Half(12356789));
235 ledgerData->set_ledgerhash(hash.data(), hash.size());
236 ledgerData->set_ledgerseq(123456789);
237 ledgerData->set_type(protocol::TMLedgerInfoType::liAS_NODE);
238 ledgerData->set_requestcookie(123456789);
239 ledgerData->set_error(protocol::TMReplyError::reNO_LEDGER);
240 ledgerData->mutable_nodes()->Reserve(n);
241 UInt256 parentHash(0);
242
243 NetClock::duration const resolution{10};
244 NetClock::time_point ct{resolution};
245
246 for (int i = 0; i < n; i++)
247 {
248 LedgerHeader info;
249 info.seq = i;
250 info.parentCloseTime = ct;
251 info.hash = xrpl::sha512Half(i);
252 info.txHash = xrpl::sha512Half(i + 1);
253 info.accountHash = xrpl::sha512Half(i + 2);
254 info.parentHash = parentHash;
255 info.drops = XRPAmount(10);
256 info.closeTimeResolution = resolution;
257 info.closeTime = ct;
258 ct += resolution;
259 parentHash = ledgerHash(info);
260 Serializer nData;
261 xrpl::addRaw(info, nData);
262 ledgerData->add_nodes()->set_nodedata(nData.getDataPtr(), nData.getLength());
263 }
264
265 return ledgerData;
266 }
267
270 {
272
273 getObject->set_type(
274 protocol::TMGetObjectByHash_ObjectType::TMGetObjectByHash_ObjectType_otTRANSACTION);
275 getObject->set_query(true);
276 UInt256 hash(xrpl::sha512Half(123456789));
277 getObject->set_ledgerhash(hash.data(), hash.size());
278 for (int i = 0; i < 100; i++)
279 {
280 UInt256 hash(xrpl::sha512Half(i));
281 auto object = getObject->add_objects();
282 object->set_hash(hash.data(), hash.size());
283 object->set_data("");
284 object->set_ledgerseq(i);
285 }
286 return getObject;
287 }
288
291 {
293
294 auto master = randomKeyPair(KeyType::Ed25519);
295 auto signing = randomKeyPair(KeyType::Ed25519);
297 st[sfSequence] = 0;
298 st[sfPublicKey] = std::get<0>(master);
299 st[sfSigningPubKey] = std::get<0>(signing);
300 st[sfDomain] = makeSlice(std::string("example.com"));
301 sign(st, HashPrefix::Manifest, KeyType::Ed25519, std::get<1>(master), sfMasterSignature);
302 sign(st, HashPrefix::Manifest, KeyType::Ed25519, std::get<1>(signing));
303 Serializer s;
304 st.add(s);
305 list->set_manifest(s.data(), s.size());
306 list->set_version(4);
307 STObject const signature(sfSignature);
308 xrpl::sign(st, HashPrefix::Manifest, KeyType::Ed25519, std::get<1>(signing));
309 Serializer s1;
310 st.add(s1);
311 auto& blob = *list->add_blobs();
312 blob.set_signature(s1.data(), s1.size());
313 blob.set_blob(strHex(s.slice()));
314 return list;
315 }
316
317 void
319 {
320 auto const thresh = beast::Severity::Info;
321 auto logs = std::make_unique<Logs>(thresh);
322
323 protocol::TMManifests const manifests;
324 protocol::TMEndpoints const endpoints;
325 protocol::TMTransaction const transaction;
326 protocol::TMGetLedger const getLedger;
327 protocol::TMLedgerData const ledgerData;
328 protocol::TMGetObjectByHash const getObject;
329 protocol::TMValidatorListCollection const validatorListCollection;
330
331 // 4.5KB
332 doTest(buildManifests(20), protocol::mtMANIFESTS, 4, "TMManifests20");
333 // 22KB
334 doTest(buildManifests(100), protocol::mtMANIFESTS, 4, "TMManifests100");
335 // 131B
336 doTest(buildEndpoints(10), protocol::mtENDPOINTS, 4, "TMEndpoints10");
337 // 1.3KB
338 doTest(buildEndpoints(100), protocol::mtENDPOINTS, 4, "TMEndpoints100");
339 // 242B
340 doTest(buildTransaction(*logs), protocol::mtTRANSACTION, 1, "TMTransaction");
341 // 87B
342 doTest(buildGetLedger(), protocol::mtGET_LEDGER, 1, "TMGetLedger");
343 // 61KB
344 doTest(buildLedgerData(500, *logs), protocol::mtLEDGER_DATA, 10, "TMLedgerData500");
345 // 122 KB
346 doTest(buildLedgerData(1000, *logs), protocol::mtLEDGER_DATA, 20, "TMLedgerData1000");
347 // 1.2MB
348 doTest(buildLedgerData(10000, *logs), protocol::mtLEDGER_DATA, 50, "TMLedgerData10000");
349 // 12MB
350 doTest(buildLedgerData(100000, *logs), protocol::mtLEDGER_DATA, 100, "TMLedgerData100000");
351 // 61MB
352 doTest(buildLedgerData(500000, *logs), protocol::mtLEDGER_DATA, 100, "TMLedgerData500000");
353 // 7.7KB
354 doTest(buildGetObjectByHash(), protocol::mtGET_OBJECTS, 4, "TMGetObjectByHash");
355 doTest(
357 protocol::mtVALIDATOR_LIST_COLLECTION,
358 4,
359 "TMValidatorListCollection");
360 }
361
362 void
364 {
365 testcase("Handshake");
366 auto getEnv = [&](bool enable) {
367 Config c;
369 str << "[reduce_relay]\n"
370 << "vp_base_squelch_enable=1\n"
371 << "[compression]\n"
372 << enable << "\n";
373 c.loadFromString(str.str());
374 auto env = std::make_shared<jtx::Env>(*this);
375 env->app().config().compression = c.compression;
376 env->app().config().vpReduceRelayBaseSquelchEnable = c.vpReduceRelayBaseSquelchEnable;
377 return env;
378 };
379 auto const handshake = [&](int outboundEnable, int inboundEnable) {
380 beast::ip::Address const addr = boost::asio::ip::make_address("172.1.1.100");
381
382 auto env = getEnv(outboundEnable);
383 auto request = xrpl::makeRequest(
384 true,
385 env->app().config().compression,
386 false,
387 env->app().config().txReduceRelayEnable,
388 env->app().config().vpReduceRelayBaseSquelchEnable);
389 HttpRequestType httpRequest;
390 httpRequest.version(request.version());
391 httpRequest.base() = request.base();
392 // feature enabled on the peer's connection only if both sides are
393 // enabled
394 auto const peerEnabled = inboundEnable && outboundEnable;
395 // inbound is enabled if the request's header has the feature
396 // enabled and the peer's configuration is enabled
397 auto const inboundEnabled =
398 peerFeatureEnabled(httpRequest, kFeatureCompr, "lz4", inboundEnable);
399 BEAST_EXPECT(!(peerEnabled ^ inboundEnabled));
400
401 env.reset();
402 env = getEnv(inboundEnable);
403 auto const httpResp = xrpl::makeResponse(
404 true, httpRequest, addr, addr, UInt256{1}, 1, {1, 0}, env->app());
405 // outbound is enabled if the response's header has the feature
406 // enabled and the peer's configuration is enabled
407 auto const outboundEnabled =
408 peerFeatureEnabled(httpResp, kFeatureCompr, "lz4", outboundEnable);
409 BEAST_EXPECT(!(peerEnabled ^ outboundEnabled));
410 };
411 handshake(1, 1);
412 handshake(1, 0);
413 handshake(0, 1);
414 handshake(0, 0);
415 }
416
417 void
418 run() override
419 {
420 testProtocol();
422 }
423};
424
426
427} // namespace xrpl::test
T begin(T... args)
A testsuite class.
Definition suite.h:52
TestcaseT testcase
Memberspace for declaring test cases.
Definition suite.h:155
Represents a JSON value.
Definition json_value.h:121
virtual Config & config()=0
pointer data()
Definition base_uint.h:117
iterator begin()
Definition base_uint.h:128
static constexpr std::size_t size()
Definition base_uint.h:549
void loadFromString(std::string const &fileContents)
Load the config from the contents of the string.
Manages partitions for logging.
Definition Log.h:23
std::vector< uint8_t > const & getBuffer(Compressed tryCompressed)
Retrieve the packed message data.
Definition Message.cpp:201
std::chrono::time_point< NetClock > time_point
Definition chrono.h:48
std::chrono::duration< rep, period > duration
Definition chrono.h:47
Identifies a node inside a SHAMap.
void add(Serializer &s) const override
Definition STObject.cpp:123
void const * getDataPtr() const
Definition Serializer.h:294
int getLength() const
Definition Serializer.h:304
Slice slice() const noexcept
Definition Serializer.h:141
std::size_t size() const noexcept
Definition Serializer.h:147
void const * data() const noexcept
Definition Serializer.h:153
constexpr value_type drops() const
Returns the number of drops.
Definition XRPAmount.h:163
Implements ZeroCopyInputStream around a buffer sequence.
static std::shared_ptr< protocol::TMEndpoints > buildEndpoints(int n)
void doTest(std::shared_ptr< T > proto, protocol::MessageType mt, uint16_t nbuffers, std::string msg)
compression::Compressed Compressed
static std::shared_ptr< protocol::TMLedgerData > buildLedgerData(uint32_t n, Logs &logs)
std::shared_ptr< protocol::TMTransaction > buildTransaction(Logs &logs)
static std::shared_ptr< protocol::TMValidatorListCollection > buildValidatorListCollection()
compression::Algorithm Algorithm
static std::shared_ptr< protocol::TMGetLedger > buildGetLedger()
void run() override
Runs the suite.
static std::shared_ptr< protocol::TMGetObjectByHash > buildGetObjectByHash()
static std::shared_ptr< protocol::TMManifests > buildManifests(int n)
A transaction testing environment.
Definition Env.h:161
Application & app()
Definition Env.h:300
bool close(NetClock::time_point closeTime, std::optional< std::chrono::milliseconds > consensusDelay=std::nullopt)
Close and advance the ledger.
Definition Env.cpp:133
void fund(bool setDefaultRipple, STAmount const &amount, Account const &account)
Definition Env.cpp:323
void trust(STAmount const &amount, Account const &account)
Establish trust lines.
Definition Env.cpp:354
T data(T... args)
T equal(T... args)
T make_shared(T... args)
T make_unique(T... args)
boost::asio::ip::address Address
Definition IPAddress.h:20
constexpr std::size_t kHeaderBytes
Definition Compression.h:12
std::size_t decompress(InputStream &in, std::size_t inSize, std::uint8_t *decompressed, std::size_t decompressedSize, Algorithm algorithm=Algorithm::LZ4)
Decompress input stream.
Definition Compression.h:32
std::optional< MessageHeader > parseMessageHeader(boost::system::error_code &ec, BufferSequence const &bufs, std::size_t size)
Parse a message header.
json::Value pay(AccountID const &account, AccountID const &to, AnyAmount amount)
Create a payment.
Definition pay.cpp:14
std::vector< STAmount > fund(jtx::Env &env, jtx::Account const &gw, std::vector< jtx::Account > const &accounts, std::vector< STAmount > const &amts, Fund how)
Definition AMMTest.cpp:34
XrpT const XRP
Converts to XRP Issue or STAmount.
Definition amount.cpp:92
std::unique_ptr< Config > envconfig()
creates and initializes a default configuration for jtx::Env
Definition envconfig.h:38
void sign(json::Value const &jv, Account const &account, json::Value &sigObject, HashPrefix prefix)
Sign automatically into a specific Json field of the jv object.
Definition utility.cpp:52
BEAST_DEFINE_TESTSUITE_MANUAL(AMMCalc, app, xrpl)
static UInt256 ledgerHash(LedgerHeader const &info)
constexpr XRPAmount
Convert XRP to drops (integral types).
Definition TxTest.h:53
std::unique_ptr< WSClient > makeWSClient(Config const &cfg, bool v2, unsigned rpcVersion, std::unordered_map< std::string, std::string > const &headers)
Returns a client operating through WebSockets/S.
Definition WSClient.cpp:373
Use hash_* containers for keys that do not need a cryptographically secure hashing algorithm.
Definition algorithm.h:5
std::pair< PublicKey, SecretKey > randomKeyPair(KeyType type)
Create a key pair using secure random numbers.
std::string strHex(FwdIt begin, FwdIt end)
Definition strHex.h:13
HttpResponseType makeResponse(bool crawlPublic, HttpRequestType const &req, beast::ip::Address publicIp, beast::ip::Address remoteIp, UInt256 const &sharedValue, std::optional< std::uint32_t > networkID, ProtocolVersion protocol, Application &app)
Make http response.
SField const sfGeneric
std::string toBase58(AccountID const &v)
Convert AccountID to base58 checked string.
Definition AccountID.cpp:95
auto makeRequest(bool crawlPublic, bool comprEnabled, bool ledgerReplayEnabled, bool txReduceRelayEnabled, bool vpReduceRelayEnabled) -> RequestType
Make outbound http request.
Seed generateSeed(std::string const &passPhrase)
Generate a seed deterministically.
Definition Seed.cpp:58
Slice makeSlice(std::array< T, N > const &a)
Definition Slice.h:228
BaseUInt< 256 > UInt256
Definition base_uint.h:581
std::optional< Blob > strUnHex(std::size_t strSize, Iterator begin, Iterator end)
void addRaw(LedgerHeader const &, Serializer &, bool includeHash=false)
boost::beast::http::request< boost::beast::http::dynamic_body > HttpRequestType
Definition Handoff.h:12
@ LedgerMaster
ledger master data for signing
Definition HashPrefix.h:59
@ Manifest
Manifest.
Definition HashPrefix.h:84
Buffer sign(PublicKey const &pk, SecretKey const &sk, Slice const &message)
Generate a signature for a message.
bool peerFeatureEnabled(Headers const &request, std::string const &feature, std::string value, bool config)
Check if a feature should be enabled for a peer.
Definition Handshake.h:182
static constexpr char kFeatureCompr[]
Definition Handshake.h:125
Sha512HalfHasher::result_type sha512Half(Args const &... args)
Returns the SHA512-Half of a series of objects.
Definition digest.h:215
T resize(T... args)
T size(T... args)
T str(T... args)
Information about the notional ledger backing the view.
NetClock::time_point parentCloseTime
NetClock::duration closeTimeResolution
NetClock::time_point closeTime
T time_since_epoch(T... args)
T to_string(T... args)