deepseek-ai / deepseek-ai/3FS

How to config rpc parameters for benchmark?

Open
#297 0 comments 2 reactions 0 assignees View on GitHub
Dominant language
C++
Stars
10.2k
Forks
1.1k
PR merge metrics
No merged PRs in 30d

Description

I wrote a simple test case to benchmark the 3FS RPC (rdma), but found that it couldn't fully utilize the available bandwidth.
The test was conducted in a 200Gb/s environment, but the throughput is not expected. I suspect that some misconfiguration is causing this result.
Could someone please share how to properly configure the RPC server and client for benchmarking purposes?

benchmark code in test_net.cpp:
```cpp
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include

#include "common/net/Client.h"
#include "common/net/Listener.h"
#include "common/net/Server.h"
#include "common/net/ib/IBDevice.h"
#include "common/net/sync/Client.h"
#include "common/serde/ClientContext.h"
#include "common/utils/Address.h"
#include "common/utils/Coroutine.h"
#include "tests/GtestHelpers.h"
#include "tests/common/net/Echo.h"
#include "tests/common/net/ib/SetupIB.h"
#include

DEFINE_uint32(req_len, 13, "request data len");
DEFINE_uint32(par, 10, "parallel");
DEFINE_uint32(port, 0, "port");
DEFINE_string(host, "10.0.0.42:9004", "host");
DEFINE_uint32(tcp, 1, "enable tcp, otherwise rdma");
DEFINE_uint32(dur, 20, "test duration");

using namespace hf3fs;
using namespace hf3fs::net;
using namespace hf3fs::net::test;

std::atomic g_count = 0;
ylt::metric::summary_t g_latency{"Latency(us) of rpc call", "help",
std::vector{0.5, 0.9, 0.95, 0.99},
std::chrono::seconds{60}};

class EchoServiceImpl : public serde::ServiceWrapper {
public:
CoTryTask echo(serde::CallContext &ctx, const EchoReq &req) {
EchoRsp rsp;
rsp.val = "hello for test";
co_return rsp;
}

CoTryTask hello(serde::CallContext &ctx, const HelloReq &req) {
HelloRsp rsp;
rsp.val = "Hello, " + req.val;
rsp.idx = ++idx_;
co_return rsp;
}

CoTryTask fail(serde::CallContext &ctx, const HelloReq &) {
fmt::print("Request from {}\n", ctx.transport()->describe());
co_return makeError(RPCCode::kInvalidMessageType, "failed");
}

private:
uint32_t idx_ = 0;
};

int main(int argc, char **argv) {
folly::Init init(&argc, &argv);
if(FLAGS_tcp == 0) {
static IBConfig config;
config.set_allow_unknown_zone(true);
config.set_device_filter({"mlx5_bond_0"});
auto ib = IBManager::start(config);
std::cout << "is empty " << IBDevice::all().empty() << "\n";
XLOGF_IF(FATAL, ib.hasError(), "IBManager start failed, result {}", ib.error());
assert(IBDevice::all().empty());
}

if (FLAGS_port > 0) {
hf3fs::net::Server::Config serverConfig;
if(FLAGS_tcp == 1) {
serverConfig.groups(0).set_network_type(Address::TCP);
}else {
serverConfig.groups(0).set_network_type(Address::RDMA);
}
serverConfig.groups(0).listener().set_listen_port(FLAGS_port);
serverConfig.groups(0).io_worker().transport_pool().set_max_connections(2000);
serverConfig.groups(0).io_worker().set_num_event_loop(std::thread::hardware_concurrency());
serverConfig.groups(0).io_worker().set_read_write_tcp_in_event_thread(true);
serverConfig.groups(0).io_worker().ibsocket().set_buf_size(2 * 1024 * 1024);
serverConfig.thread_pool().set_num_io_threads(std::thread::hardware_concurrency());
serverConfig.thread_pool().set_num_proc_threads(std::thread::hardware_concurrency());
// serverConfig.thread_pool().set_num_bg_threads(std::thread::hardware_concurrency());
std::cout << "start server, port " << FLAGS_port << ", enable tcp " << FLAGS_tcp << "\n";
hf3fs::net::Server server{serverConfig};
server.addSerdeService(std::make_unique());
server.setup();
server.start();

std::string wait_quit;
std::cin >> wait_quit;
}
std::cout << "start client, remote addr " << FLAGS_host << ", concurrency " << FLAGS_par << ", req_len "
<< FLAGS_req_len << "\n";
hf3fs::net::Client::Config clientConfig{};
clientConfig.io_worker().transport_pool().set_max_connections(std::thread::hardware_concurrency());
clientConfig.io_worker().set_read_write_tcp_in_event_thread(true);
clientConfig.io_worker().set_num_event_loop(std::thread::hardware_concurrency());
clientConfig.thread_pool().set_num_connect_threads(std::thread::hardware_concurrency());
clientConfig.thread_pool().set_num_io_threads(std::thread::hardware_concurrency());
clientConfig.thread_pool().set_num_proc_threads(std::thread::hardware_concurrency());
clientConfig.set_default_timeout(5_s);
clientConfig.io_worker().ibsocket().set_buf_size(2*1024*1024);
hf3fs::net::Client client{clientConfig};
Address addr = Address::fromString(FLAGS_host, Address::RDMA);
if(FLAGS_tcp == 1) {
addr = Address::fromString(FLAGS_host, Address::TCP);
}

auto ctx = client.serdeCtx(addr);
client.start();

EchoReq req;
req.val = std::string(FLAGS_req_len, 'A');
auto task = [&]() -> CoTask {
Echo<> client;
while (true) {
auto start = std::chrono::steady_clock::now();
auto result = co_await client.echo(ctx, req);
if (result.hasValue()) {
g_count += FLAGS_req_len;
auto now = std::chrono::steady_clock::now();
g_latency.observe(std::chrono::duration_cast(now - start).count());
}
}
};

std::cout << std::fixed << std::setprecision(2);
g_count = 0;
std::thread thd([] {
size_t total = 0;
for (uint32_t i = 0; i < FLAGS_dur; i++) {
auto start = std::chrono::system_clock::now();
std::this_thread::sleep_for(std::chrono::seconds{1});
auto c = g_count.exchange(0);
total += c;
auto end = std::chrono::system_clock::now();
auto dur = (end - start) / std::chrono::milliseconds(1);
double val = (8.0 * c * 1000) / (1000'000'000ll * dur);
std::cout << "qps " << c / FLAGS_req_len << ", Throughput:" << val << " Gb/s\n";
}
std::cout << "# Benchmark result \n";
double val = (8.0 * total) / (1000'000'000ll * FLAGS_dur);
std::cout << "avg qps " << total / (FLAGS_req_len * FLAGS_dur) << " and throughput:" << val << " Gb/s in duration "
<< FLAGS_dur << "\n";
std::string str_rate;
g_latency.serialize(str_rate);
std::cout << str_rate << "\n";
});

folly::CPUThreadPoolExecutor exec(std::thread::hardware_concurrency());
for (size_t i = 0; i < FLAGS_par; i++) {
task().scheduleOn(folly::Executor::getKeepAliveToken(exec)).start();
}
exec.join();
return 0;
}
```

rdma benchmark command line:
```
./test_net -port 9004 -tcp 0
```
```
./test_net -host server_ip:9004 -req_len 8388608 -par 16 -dur 15
```
some test results:

![img](https://github.com/user-attachments/assets/2c0e2696-5bbb-4868-b9f3-3c25eb7ca0df)

When sending 8M+ data to test, the throughout become lower and lower, and the cpu is very high.

I'm eager to hear your suggestions.

Contributor guide

No contributing guide indexed for this repository

Research direction

Start by reviewing the benchmark code in test_net.cpp, especially the Server::Config and Client::Config settings, then run the two provided RDMA commands to reproduce the throughput and CPU behavior. Compare results across request sizes and concurrency levels. The issue does not define a specific code change or completion criterion, so a maintainer would need to clarify the expected fix.

Written by the indexing model from the issue text.

Assessment

Tech stack
cpp
Domain
networking, performance
Issue type
Bug
Difficulty
4/5
Estimated time
3-5 days
Activity status
Stale
Clarity
Needs clarification
Newbie friendliness
25/100

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.