Create Target Error

您好，我这边创建存储目标报错，不确定该如何解决，信息如下（一台存储运行了两个存储实例）：
```shell
root@3fs:/opt/3fs/etc/chanis# ibdev2netdev
mlx5_0 port 1 ==> ibs9 (Down)
mlx5_1 port 1 ==> ibs21 (Up)
root@3fs:/opt/3fs/etc/chanis# ibstatus
Infiniband device 'mlx5_0' port 1 status:
default gid: fe80:0000:0000:0000:58a2:e103:00be:4cd2
base lid: 0xffff
sm lid: 0x0
state: 1: DOWN
phys state: 3: Disabled
rate: 10 Gb/sec (4X SDR)
link_layer: InfiniBand

Infiniband device 'mlx5_1' port 1 status:
default gid: fe80:0000:0000:0000:58a2:e103:00be:4cee
base lid: 0xb
sm lid: 0x1
state: 4: ACTIVE
phys state: 5: LinkUp
rate: 200 Gb/sec (4X HDR)
link_layer: InfiniBand

```
```shell
root@3fs:/opt/3fs/etc/chanis# /opt/3fs/bin/admin_cli -cfg /opt/3fs/etc/admin_cli.toml --config.mgmtd_client.mgmtd_server_addresses '["IPoIB://10.252.0.100:8000"]' "list-nodes"
Id Type Status Hostname Pid Tags LastHeartbeatTime ConfigVersion ReleaseVersion
1 MGMTD PRIMARY_MGMTD 3fs 11904 [] N/A 1(UPTODATE) 250228-dev-1-999999-3a30c53b
100 META HEARTBEAT_CONNECTED 3fs 5862 [] 2025-03-17 18:28:12 1(UPTODATE) 250228-dev-1-999999-3a30c53b
10001 STORAGE HEARTBEAT_CONNECTED 3fs 10883 [] 2025-03-17 18:28:19 0(UPTODATE) 250228-dev-1-999999-3a30c53b
10002 STORAGE HEARTBEAT_CONNECTED 3fs 10392 [] 2025-03-17 18:28:17 0(UPTODATE) 250228-dev-1-999999-3a30c53b
```
```shell
root@3fs:/opt/3fs/etc/chanis# ss -lntp
State Recv-Q Send-Q Local Address:Port Peer Address:Port Process
LISTEN 0 128 0.0.0.0:22 0.0.0.0:* users:(("sshd",pid=3151,fd=3))
LISTEN 0 4096 10.252.0.100:8100 0.0.0.0:* users:(("storage_main",pid=10883,fd=59))
LISTEN 0 4096 10.252.0.100:8000 0.0.0.0:* users:(("mgmtd_main",pid=11904,fd=45))
LISTEN 0 4096 10.252.0.100:8001 0.0.0.0:* users:(("meta_main",pid=5862,fd=52))
LISTEN 0 4096 10.252.0.100:9200 0.0.0.0:* users:(("storage_main",pid=10392,fd=60))
LISTEN 0 4096 10.252.0.100:9100 0.0.0.0:* users:(("storage_main",pid=10883,fd=60))
LISTEN 0 4096 10.252.0.100:9000 0.0.0.0:* users:(("mgmtd_main",pid=11904,fd=46))
LISTEN 0 4096 10.252.0.100:9001 0.0.0.0:* users:(("meta_main",pid=5862,fd=53))
LISTEN 0 4096 10.252.0.100:8200 0.0.0.0:* users:(("storage_main",pid=10392,fd=59))
LISTEN 0 4096 10.252.0.100:10000 0.0.0.0:* users:(("monitor_collect",pid=5192,fd=29))
LISTEN 0 128 127.0.0.1:4500 0.0.0.0:* users:(("fdbserver",pid=5049,fd=17))
LISTEN 0 128 127.0.0.1:6012 0.0.0.0:* users:(("sshd",pid=12422,fd=9))
LISTEN 0 128 127.0.0.1:6010 0.0.0.0:* users:(("sshd",pid=4832,fd=9))
LISTEN 0 128 127.0.0.1:6011 0.0.0.0:* users:(("sshd",pid=5483,fd=9))
LISTEN 0 4096 127.0.0.1:8123 0.0.0.0:* users:(("clickhouse-serv",pid=3694,fd=230))
LISTEN 0 4096 127.0.0.1:9009 0.0.0.0:* users:(("clickhouse-serv",pid=3694,fd=82))
LISTEN 0 4096 127.0.0.1:9004 0.0.0.0:* users:(("clickhouse-serv",pid=3694,fd=221))
LISTEN 0 4096 127.0.0.1:9005 0.0.0.0:* users:(("clickhouse-serv",pid=3694,fd=189))
LISTEN 0 4096 127.0.0.1:9000 0.0.0.0:* users:(("clickhouse-serv",pid=3694,fd=103))
LISTEN 0 4096 127.0.0.53%lo:53 0.0.0.0:* users:(("systemd-resolve",pid=2842,fd=14))
LISTEN 0 128 [::]:22 [::]:* users:(("sshd",pid=3151,fd=4))
LISTEN 0 4096 [::1]:9009 [::]:* users:(("clickhouse-serv",pid=3694,fd=81))
LISTEN 0 4096 [::1]:9000 [::]:* users:(("clickhouse-serv",pid=3694,fd=215))
LISTEN 0 4096 [::1]:9005 [::]:* users:(("clickhouse-serv",pid=3694,fd=87))
LISTEN 0 4096 [::1]:9004 [::]:* users:(("clickhouse-serv",pid=3694,fd=219))
LISTEN 0 128 [::1]:6011 [::]:* users:(("sshd",pid=5483,fd=7))
LISTEN 0 128 [::1]:6010 [::]:* users:(("sshd",pid=4832,fd=7))
LISTEN 0 128 [::1]:6012 [::]:* users:(("sshd",pid=12422,fd=7))
LISTEN 0 4096 [::1]:8123 [::]:* users:(("clickhouse-serv",pid=3694,fd=208))
root@3fs:/opt/3fs/etc/chanis# ps aux | grep main | grep -v grep
root 5192 0.3 0.0 4312200 313484 ? Ssl 14:16 0:54 /opt/3fs/bin/monitor_collector_main --cfg /opt/3fs/etc/monitor_collector_main.toml
root 5862 1.6 0.0 1219256 497092 ? Ssl 14:30 4:05 /opt/3fs/bin/meta_main --launcher_cfg /opt/3fs/etc/meta_main_launcher.toml --app-cfg /opt/3fs/etc/meta_main_app.toml
root 10392 1.1 0.9 16624852 10069960 ? Ssl 17:38 0:37 /opt/3fs/bin/storage_main --cfg /opt/3fs/etc/storage_main_2.toml --launcher_cfg /opt/3fs/etc/storage_main_launcher_2.toml --app-cfg /opt/3fs/etc/storage_main_app_2.toml
root 10883 1.0 0.9 16624864 10068784 ? Ssl 17:38 0:36 /opt/3fs/bin/storage_main --cfg /opt/3fs/etc/storage_main_1.toml --launcher_cfg /opt/3fs/etc/storage_main_launcher_1.toml --app-cfg /opt/3fs/etc/storage_main_app_1.toml
root 11904 0.2 0.0 2732124 254416 ? Ssl 17:47 0:07 /opt/3fs/bin/mgmtd_main --launcher_cfg /opt/3fs/etc/mgmtd_main_launcher.toml --app-cfg /opt/3fs/etc/mgmtd_main_app.toml
```
```shell
root@3fs:/opt/3fs/etc/chanis# python3 ~/3fs/deploy/data_placement/src/model/data_placement.py \
-ql -relax -type CR --num_nodes 2 --replication_factor 2 --min_targets_per_disk 6

root@3fs:/opt/3fs/etc/chanis# python3 ~/3fs/deploy/data_placement/src/setup/gen_chain_table.py \
--chain_table_type CR --node_id_begin 10001 --node_id_end 10002 \
--num_disks_per_node 12 --num_targets_per_disk 6 \
--target_id_prefix 1 --chain_id_prefix 9 \
--incidence_matrix_path output/DataPlacementModel-v_2-b_6-r_6-k_2-λ_6-lb_1-ub_0/incidence_matrix.pickle \
```
```shell
root@3fs:/opt/3fs/etc/chanis# /opt/3fs/bin/admin_cli --cfg /opt/3fs/etc/admin_cli.toml --config.mgmtd_client.mgmtd_server_addresses '["IPoIB://10.252.0.100:8000"]' --config.user_info.token $(<"/opt/3fs/etc/token.txt") < output/create_target_cmd.txt
Encounter error: 7016(StorageClient::NoRDMAInterface)
./src/client/cli/admin/CreateTarget.cc:69, '_result' has error
Encounter error: 7016(StorageClient::NoRDMAInterface)
./src/client/cli/admin/CreateTarget.cc:69, '_result' has error
Encounter error: 7016(StorageClient::NoRDMAInterface)
./src/client/cli/admin/CreateTarget.cc:69, '_result' has error
Encounter error: 7016(StorageClient::NoRDMAInterface)
./src/client/cli/admin/CreateTarget.cc:69, '_result' has error
Encounter error: 7016(StorageClient::NoRDMAInterface)
./src/client/cli/admin/CreateTarget.cc:69, '_result' has error
Encounter error: 7016(StorageClient::NoRDMAInterface)
````
```shell
root@3fs:/opt/3fs/etc/chanis# tail -f /var/log/3fs/cli.log
[2025-03-18T15:14:26.759932141+08:00 admin_cli: 6688 IBDevice.cc:169 INFO] ibdev2netdev: mlx5_0 port 1 ==> ibs9 (Down)
[2025-03-18T15:14:26.759932141+08:00 admin_cli: 6688 IBDevice.cc:169 INFO] mlx5_1 port 1 ==> ibs21 (Up)
[2025-03-18T15:14:26.759957380+08:00 admin_cli: 6688 IBDevice.cc:186 INFO] ibdev2netdev parsed: mlx5_0 => ibs9
[2025-03-18T15:14:26.759961879+08:00 admin_cli: 6688 IBDevice.cc:186 INFO] ibdev2netdev parsed: mlx5_1 => ibs21
[2025-03-18T15:14:26.760320094+08:00 admin_cli: 6688 IfAddrs.h:56 DEBUG] Skip ifaddr of lo, ifa_addr 0x55f5e0a94038 or ifa_netmask 0x0 missing.
[2025-03-18T15:14:26.760327789+08:00 admin_cli: 6688 IfAddrs.h:56 DEBUG] Skip ifaddr of ens29f0, ifa_addr 0x55f5e0a940f0 or ifa_netmask 0x0 missing.
[2025-03-18T15:14:26.760330720+08:00 admin_cli: 6688 IfAddrs.h:56 DEBUG] Skip ifaddr of ens29f1, ifa_addr 0x55f5e0a941a8 or ifa_netmask 0x0 missing.
[2025-03-18T15:14:26.760333786+08:00 admin_cli: 6688 IfAddrs.h:56 DEBUG] Skip ifaddr of ibs9, ifa_addr 0x55f5e0a94260 or ifa_netmask 0x0 missing.
[2025-03-18T15:14:26.760336542+08:00 admin_cli: 6688 IfAddrs.h:56 DEBUG] Skip ifaddr of ibs21, ifa_addr 0x55f5e0a94318 or ifa_netmask 0x0 missing.
[2025-03-18T15:14:26.760368510+08:00 admin_cli: 6688 IfAddrs.h:102 INFO] Get ifaddr of ens29f0, addr 10.20.1.100/24, subnet 10.20.1.0/24, up true
[2025-03-18T15:14:26.760381540+08:00 admin_cli: 6688 IfAddrs.h:102 INFO] Get ifaddr of ibs21, addr 10.252.0.100/24, subnet 10.252.0.0/24, up true
[2025-03-18T15:14:26.760387419+08:00 admin_cli: 6688 IfAddrs.h:64 DEBUG] Skip ifaddr of lo, address family 17 10 != AF_INET
[2025-03-18T15:14:26.760390428+08:00 admin_cli: 6688 IfAddrs.h:64 DEBUG] Skip ifaddr of ens29f0, address family 17 10 != AF_INET
[2025-03-18T15:14:26.760392967+08:00 admin_cli: 6688 IfAddrs.h:64 DEBUG] Skip ifaddr of ibs21, address family 17 10 != AF_INET
[2025-03-18T15:14:26.784423002+08:00 admin_cli: 6688 IBDevice.cc:341 WARNING] IBDevice mlx5_0 port 1 is not active, state IBV_PORT_DOWN, skip true
[2025-03-18T15:14:26.784444366+08:00 admin_cli: 6688 IBDevice.cc:253 INFO] IBDevice skip mlx5_0 because it doesn't have available ports.
[2025-03-18T15:14:26.791648840+08:00 admin_cli: 6688 IBDevice.cc:396 INFO] IBDevice mlx5_1:1 IP addrs 10.252.0.100/24
[2025-03-18T15:14:26.791670166+08:00 admin_cli: 6688 IBDevice.cc:441 CRITICAL] IBDevice mlx5_1:1 can't set zone by IP, fallback to UNKNOWN
[2025-03-18T15:14:26.791699653+08:00 admin_cli: 6688 IBDevice.cc:367 INFO] IBDevice mlx5_1 add active port 1, linklayer INFINIBAND, addrs 10.252.0.100/24, zones UNKNOWN, RoCE v2 GID std::nullopt
[2025-03-18T15:14:26.791711503+08:00 admin_cli: 6688 IBDevice.cc:256 INFO] IBDevice add mlx5_1, id 0, 1 available ports
[2025-03-18T15:14:26.795899673+08:00 admin_cli: 6841 EventLoop.cc:116 INFO] EventLoop::loop() started.
[2025-03-18T15:14:26.796282841+08:00 admin_cli: 6688 ConstructLog.h:11 INFO] Construct serde::Services
[2025-03-18T15:14:26.802451755+08:00 admin_cli: 6688 ConstructLog.h:11 INFO] Construct net::ThreadPoolGroup
[2025-03-18T15:14:26.819462744+08:00 admin_cli: 6688 ConstructLog.h:11 INFO] Construct CPUExecutorGroup
[2025-03-18T15:14:26.826626197+08:00 admin_cli: 6688 ConstructLog.h:11 INFO] Construct CPUExecutorGroup
[2025-03-18T15:14:26.833333679+08:00 admin_cli: 6688 ConstructLog.h:11 INFO] Construct CPUExecutorGroup
[2025-03-18T15:14:26.846321588+08:00 admin_cli: 6688 Semaphore.h:27 INFO] usableTokens: 64, maxTokens: 4096, availableTokens: 64
[2025-03-18T15:14:26.849195489+08:00 CliEL0: 6850 EventLoop.cc:116 INFO] EventLoop::loop() started.
[2025-03-18T15:14:26.849349904+08:00 admin_cli: 6688 MgmtdClient.cc:222 INFO] MgmtdClient: started
[2025-03-18T15:14:26.849390536+08:00 CliBG0: 6846 BackgroundRunner.cc:83 INFO] BackgroundRunner: AutoRefresh start
[2025-03-18T15:14:26.849390847+08:00 CliBG1: 6847 MgmtdClient.cc:388 DEBUG] MgmtdClient: receive RefreshItem force=false
[2025-03-18T15:14:26.849417264+08:00 CliBG0: 6846 BackgroundRunner.cc:85 DEBUG] BackgroundRunner: AutoRefresh round 1 start
[2025-03-18T15:14:26.849425843+08:00 CliBG1: 6847 MgmtdClient.cc:390 INFO] [MgmtdClientOp RefreshRoutingInfo force=false No.1] start execution. from:
[2025-03-18T15:14:26.849461268+08:00 CliBG1: 6847 MgmtdClient.cc:424 INFO] MgmtdClient: start probe for connecting ...
[2025-03-18T15:14:26.849467162+08:00 CliBG1: 6847 MgmtdClient.cc:542 INFO] MgmtdClient: start probe from unknown address IPoIB://10.252.0.100:8000
[2025-03-18T15:14:26.849474653+08:00 CliBG1: 6847 MgmtdClient.cc:504 INFO] MgmtdClient: probePrimary NodeId(0) via IPoIB://10.252.0.100:8000 ...
[2025-03-18T15:14:26.854035858+08:00 CliProc1: 6843 Processor.h:153 DEBUG] receive response 217:1
[2025-03-18T15:14:26.854078131+08:00 CliBG1: 6847 MgmtdClient.cc:585 INFO] MgmtdClient: found new mgmtd NodeId(1) IPoIB://10.252.0.100:8000
[2025-03-18T15:14:26.854088716+08:00 CliBG1: 6847 MgmtdClient.cc:532 INFO] MgmtdClient: probePrimary succeeded at NodeId(1)
[2025-03-18T15:14:26.854107307+08:00 CliBG1: 6847 MgmtdClient.cc:428 INFO] MgmtdClient: found new primary NodeId(1)
[2025-03-18T15:14:26.854111879+08:00 CliBG1: 6847 MgmtdClient.cc:181 DEBUG] MgmtdClient: execute RefreshRoutingInfo via NodeId(1) IPoIB://10.252.0.100:8000
[2025-03-18T15:14:26.854339996+08:00 CliProc1: 6843 Processor.h:153 DEBUG] receive response 217:5
[2025-03-18T15:14:26.854367940+08:00 CliBG1: 6847 MgmtdClient.cc:183 INFO] MgmtdClient: RefreshRoutingInfo finished
[2025-03-18T15:14:26.854373013+08:00 CliBG1: 6847 MgmtdClient.cc:612 INFO] MgmtdClient: get new routing info version 8
[2025-03-18T15:14:26.854466809+08:00 CliBG1: 6847 MgmtdClient.cc:614 DEBUG] MgmtdClient: get new routing info {"routingInfoVersion":8,"bootstrapping":false,"nodes":{"1":{"app":{"nodeId":1,"hostname":"3fs","pid":3184,"serviceGroups":[{"services":["Mgmtd"],"endpoints":["IPoIB://10.252.0.100:8000"]},{"services":["Core"],"endpoints":["IPoIB://10.252.0.100:9000"]}],"releaseVersion":{"majorVersion":1,"minorVersion":0,"patchVersion":0,"commitHashShort":976274747,"buildTimeInSeconds":1074721076543489,"buildPipelineId":999999},"podname":""},"type":"MGMTD","status":"PRIMARY_MGMTD","lastHeartbeatTs":"1970-01-01 08:00:00","tags":[],"configVersion":1,"configStatus":"NORMAL"},"10002":{"app":{"nodeId":10002,"hostname":"3fs","pid":3195,"serviceGroups":[{"services":["StorageSerde"],"endpoints":["IPoIB://10.252.0.100:8200"]},{"services":["Core"],"endpoints":["IPoIB://10.252.0.100:9200"]}],"releaseVersion":{"majorVersion":1,"minorVersion":0,"patchVersion":0,"commitHashShort":976274747,"buildTimeInSeconds":1074721076543489,"buildPipelineId":999999},"podname":""},"type":"STORAGE","status":"HEARTBEAT_CONNECTED","lastHeartbeatTs":"2025-03-18 15:14:25","tags":[],"configVersion":0,"configStatus":"NORMAL"},"10001":{"app":{"nodeId":10001,"hostname":"3fs","pid":3193,"serviceGroups":[{"services":["StorageSerde"],"endpoints":["IPoIB://10.252.0.100:8100"]},{"services":["Core"],"endpoints":["IPoIB://10.252.0.100:9100"]}],"releaseVersion":{"majorVersion":1,"minorVersion":0,"patchVersion":0,"commitHashShort":976274747,"buildTimeInSeconds":1074721076543489,"buildPipelineId":999999},"podname":""},"type":"STORAGE","status":"HEARTBEAT_CONNECTED","lastHeartbeatTs":"2025-03-18 15:14:25","tags":[],"configVersion":0,"configStatus":"NORMAL"},"100":{"app":{"nodeId":100,"hostname":"3fs","pid":3182,"serviceGroups":[{"services":["MetaSerde"],"endpoints":["IPoIB://10.252.0.100:8001"]},{"services":["Core"],"endpoints":["IPoIB://10.252.0.100:9001"]}],"releaseVersion":{"majorVersion":1,"minorVersion":0,"patchVersion":0,"commitHashShort":976274747,"buildTimeInSeconds":1074721076543489,"buildPipelineId":999999},"podname":""},"type":"META","status":"HEARTBEAT_CONNECTED","lastHeartbeatTs":"2025-03-18 15:14:20","tags":[],"configVersion":1,"configStatus":"NORMAL"}},"chainTables":{},"chains":{},"targets":{}}
[2025-03-18T15:14:26.854491720+08:00 CliBG1: 6847 MgmtdClient.cc:390 INFO] [MgmtdClientOp RefreshRoutingInfo force=false No.1] succeeded. latency: 0ns
[2025-03-18T15:14:26.854500361+08:00 CliBG1: 6847 MgmtdClient.cc:388 DEBUG] MgmtdClient: receive RefreshItem force=false
[2025-03-18T15:14:26.854503921+08:00 CliBG1: 6847 MgmtdClient.cc:390 INFO] [MgmtdClientOp RefreshRoutingInfo force=false No.2] start execution. from:
[2025-03-18T15:14:26.854507789+08:00 CliBG1: 6847 MgmtdClient.cc:181 DEBUG] MgmtdClient: execute RefreshRoutingInfo via NodeId(1) IPoIB://10.252.0.100:8000
[2025-03-18T15:14:26.854750907+08:00 CliProc1: 6843 Processor.h:153 DEBUG] receive response 217:5
[2025-03-18T15:14:26.854763360+08:00 CliBG1: 6847 MgmtdClient.cc:183 INFO] MgmtdClient: RefreshRoutingInfo finished
[2025-03-18T15:14:26.854767617+08:00 CliBG1: 6847 MgmtdClient.cc:614 DEBUG] MgmtdClient: get new routing info null
[2025-03-18T15:14:26.854770464+08:00 CliBG1: 6847 MgmtdClient.cc:390 INFO] [MgmtdClientOp RefreshRoutingInfo force=false No.2] succeeded. latency: 0ns
[2025-03-18T15:14:26.854775127+08:00 CliBG1: 6847 MgmtdClient.cc:388 DEBUG] MgmtdClient: receive RefreshItem force=true
[2025-03-18T15:14:26.854777298+08:00 CliBG1: 6847 MgmtdClient.cc:390 INFO] [MgmtdClientOp RefreshRoutingInfo force=true No.3] start execution. from:
[2025-03-18T15:14:26.854780068+08:00 CliBG1: 6847 MgmtdClient.cc:181 DEBUG] MgmtdClient: execute RefreshRoutingInfo via NodeId(1) IPoIB://10.252.0.100:8000
[2025-03-18T15:14:26.854787093+08:00 CliBG0: 6846 MgmtdClient.cc:286 DEBUG] MgmtdClient: AutoRefresh finished
[2025-03-18T15:14:26.854792989+08:00 CliBG0: 6846 BackgroundRunner.cc:102 DEBUG] BackgroundRunner: AutoRefresh wait 994634us for next round
[2025-03-18T15:14:26.854935269+08:00 CliProc1: 6843 Processor.h:153 DEBUG] receive response 217:5
[2025-03-18T15:14:26.854954060+08:00 CliBG1: 6847 MgmtdClient.cc:183 INFO] MgmtdClient: RefreshRoutingInfo finished
[2025-03-18T15:14:26.854958163+08:00 CliBG1: 6847 MgmtdClient.cc:612 INFO] MgmtdClient: get new routing info version 8
[2025-03-18T15:14:26.855009275+08:00 CliBG1: 6847 MgmtdClient.cc:614 DEBUG] MgmtdClient: get new routing info {"routingInfoVersion":8,"bootstrapping":false,"nodes":{"1":{"app":{"nodeId":1,"hostname":"3fs","pid":3184,"serviceGroups":[{"services":["Mgmtd"],"endpoints":["IPoIB://10.252.0.100:8000"]},{"services":["Core"],"endpoints":["IPoIB://10.252.0.100:9000"]}],"releaseVersion":{"majorVersion":1,"minorVersion":0,"patchVersion":0,"commitHashShort":976274747,"buildTimeInSeconds":1074721076543489,"buildPipelineId":999999},"podname":""},"type":"MGMTD","status":"PRIMARY_MGMTD","lastHeartbeatTs":"1970-01-01 08:00:00","tags":[],"configVersion":1,"configStatus":"NORMAL"},"10002":{"app":{"nodeId":10002,"hostname":"3fs","pid":3195,"serviceGroups":[{"services":["StorageSerde"],"endpoints":["IPoIB://10.252.0.100:8200"]},{"services":["Core"],"endpoints":["IPoIB://10.252.0.100:9200"]}],"releaseVersion":{"majorVersion":1,"minorVersion":0,"patchVersion":0,"commitHashShort":976274747,"buildTimeInSeconds":1074721076543489,"buildPipelineId":999999},"podname":""},"type":"STORAGE","status":"HEARTBEAT_CONNECTED","lastHeartbeatTs":"2025-03-18 15:14:25","tags":[],"configVersion":0,"configStatus":"NORMAL"},"10001":{"app":{"nodeId":10001,"hostname":"3fs","pid":3193,"serviceGroups":[{"services":["StorageSerde"],"endpoints":["IPoIB://10.252.0.100:8100"]},{"services":["Core"],"endpoints":["IPoIB://10.252.0.100:9100"]}],"releaseVersion":{"majorVersion":1,"minorVersion":0,"patchVersion":0,"commitHashShort":976274747,"buildTimeInSeconds":1074721076543489,"buildPipelineId":999999},"podname":""},"type":"STORAGE","status":"HEARTBEAT_CONNECTED","lastHeartbeatTs":"2025-03-18 15:14:25","tags":[],"configVersion":0,"configStatus":"NORMAL"},"100":{"app":{"nodeId":100,"hostname":"3fs","pid":3182,"serviceGroups":[{"services":["MetaSerde"],"endpoints":["IPoIB://10.252.0.100:8001"]},{"services":["Core"],"endpoints":["IPoIB://10.252.0.100:9001"]}],"releaseVersion":{"majorVersion":1,"minorVersion":0,"patchVersion":0,"commitHashShort":976274747,"buildTimeInSeconds":1074721076543489,"buildPipelineId":999999},"podname":""},"type":"META","status":"HEARTBEAT_CONNECTED","lastHeartbeatTs":"2025-03-18 15:14:20","tags":[],"configVersion":1,"configStatus":"NORMAL"}},"chainTables":{},"chains":{},"targets":{}}
[2025-03-18T15:14:26.855027155+08:00 CliBG1: 6847 MgmtdClient.cc:390 INFO] [MgmtdClientOp RefreshRoutingInfo force=true No.3] succeeded. latency: 0ns
[2025-03-18T15:14:26.855121762+08:00 admin_cli: 6688 ConstructLog.h:11 INFO] Construct serde::Services
[2025-03-18T15:14:26.859301654+08:00 admin_cli: 6688 ConstructLog.h:11 INFO] Construct net::ThreadPoolGroup
[2025-03-18T15:14:26.871052794+08:00 admin_cli: 6688 ConstructLog.h:11 INFO] Construct CPUExecutorGroup
[2025-03-18T15:14:26.875968622+08:00 admin_cli: 6688 ConstructLog.h:11 INFO] Construct CPUExecutorGroup
[2025-03-18T15:14:26.880473076+08:00 admin_cli: 6688 ConstructLog.h:11 INFO] Construct CPUExecutorGroup
[2025-03-18T15:14:26.889591888+08:00 admin_cli: 6688 ConstructLog.h:11 INFO] Construct serde::Services
[2025-03-18T15:14:26.892815054+08:00 admin_cli: 6688 ConstructLog.h:11 INFO] Construct net::ThreadPoolGroup
[2025-03-18T15:14:26.901399945+08:00 admin_cli: 6688 ConstructLog.h:11 INFO] Construct CPUExecutorGroup
[2025-03-18T15:14:26.905348087+08:00 admin_cli: 6688 ConstructLog.h:11 INFO] Construct CPUExecutorGroup
[2025-03-18T15:14:26.909146307+08:00 admin_cli: 6688 ConstructLog.h:11 INFO] Construct CPUExecutorGroup
[2025-03-18T15:14:26.917114299+08:00 admin_cli: 6688 Semaphore.h:27 INFO] usableTokens: 32, maxTokens: 4096, availableTokens: 32
[2025-03-18T15:14:26.917201375+08:00 admin_cli: 6688 Semaphore.h:27 INFO] usableTokens: 32, maxTokens: 4096, availableTokens: 32
[2025-03-18T15:14:26.917279229+08:00 admin_cli: 6688 Semaphore.h:27 INFO] usableTokens: 32, maxTokens: 4096, availableTokens: 32
[2025-03-18T15:14:26.917352990+08:00 admin_cli: 6688 Semaphore.h:27 INFO] usableTokens: 32, maxTokens: 4096, availableTokens: 32
[2025-03-18T15:14:26.917429152+08:00 admin_cli: 6688 Semaphore.h:27 INFO] usableTokens: 32, maxTokens: 4096, availableTokens: 32
[2025-03-18T15:14:26.917449246+08:00 admin_cli: 6688 StorageClientImpl.cc:1413 INFO] Starting storage client ClientId(7448ee7d-f585-42b9-974d-f29dd8709b46)@
[2025-03-18T15:14:26.917526887+08:00 admin_cli: 6688 Semaphore.h:27 INFO] usableTokens: 64, maxTokens: 4096, availableTokens: 64
[2025-03-18T15:14:26.919360877+08:00 admin_cli: 6688 StorageClientImpl.cc:1442 INFO] Storage client ClientId(7448ee7d-f585-42b9-974d-f29dd8709b46)@ is getting the first routing info
[2025-03-18T15:14:26.919375825+08:00 admin_cli: 6688 StorageClientImpl.cc:1503 INFO] First routing info obtained: RoutingInfoVersion(8)
[2025-03-18T15:14:26.919386568+08:00 admin_cli: 6688 StorageClientImpl.cc:1461 INFO] Storage client ClientId(7448ee7d-f585-42b9-974d-f29dd8709b46)@ started
[2025-03-18T15:14:26.919374119+08:00 StorageMsgrEL0: 6869 EventLoop.cc:116 INFO] EventLoop::loop() started.
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] No RDMA interface found on node: {
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] "app": {
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] "hostname": "3fs",
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] "nodeId": 10001,
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] "pid": 3193,
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] "podname": "",
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] "releaseVersion": {
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] "buildPipelineId": 999999,
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] "buildTimeInSeconds": 1074721076543489,
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] "commitHashShort": 976274747,
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] "majorVersion": 1,
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] "minorVersion": 0,
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] "patchVersion": 0
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] },
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] "serviceGroups": [
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] {
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] "endpoints": [
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] "IPoIB://10.252.0.100:8100"
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] ],
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] "services": [
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] "StorageSerde"
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] ]
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] },
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] {
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] "endpoints": [
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] "IPoIB://10.252.0.100:9100"
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] ],
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] "services": [
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] "Core"
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] ]
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] }
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] ]
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] },
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] "configStatus": "NORMAL",
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] "configVersion": 0,
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] "lastHeartbeatTs": "2025-03-18 15:14:25",
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] "status": "HEARTBEAT_CONNECTED",
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] "tags": [],
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] "type": "STORAGE"
[2025-03-18T15:14:26.919495974+08:00 admin_cli: 6688 StorageClientImpl.cc:885 DEBUG] }
[2025-03-18T15:14:26.995973761+08:00 admin_cli: 6841 IBSocket.cc:1219 DEBUG] IBSocketManager handleEvents running
[2025-03-18T15:14:27.182961505+08:00 Collector: 6691 Recorder.cc:242 DEBUG] Last 30s storage_client.overall_latency: count:1 mean:84us min:84us max:84us p50:84us p90:84us p95:84us p99:84us
[2025-03-18T15:14:27.183097748+08:00 Collector: 6691 Recorder.cc:98 DEBUG] Last 30s common_write_has_msg_suspend: 4
[2025-03-18T15:14:27.183256908+08:00 Collector: 6691 Recorder.cc:175 DEBUG] Last 30s storage_client.inflight_requests: count:2 mean:0.5 min:0.0 max:1.0 p50:0.5 p90:1.0 p95:1.0 p99:1.0
[2025-03-18T15:14:27.183423256+08:00 Collector: 6691 Recorder.cc:242 DEBUG] Last 30s storage_client.request_latency: count:1 mean:69us min:69us max:69us p50:69us p90:69us p95:69us p99:69us
[2025-03-18T15:14:27.183435122+08:00 Collector: 6691 Recorder.cc:98 DEBUG] Last 30s MgmtdClient.op.total: 3
[2025-03-18T15:14:27.183462710+08:00 Collector: 6691 Recorder.cc:175 DEBUG] Last 30s storage_client.num_pending_ops: count:2 mean:0.5 min:0.0 max:1.0 p50:0.5 p90:1.0 p95:1.0 p99:1.0
[2025-03-18T15:14:27.183483475+08:00 Collector: 6691 Recorder.cc:98 DEBUG] Last 30s common_read_available_suspend: 4
[2025-03-18T15:14:27.183578923+08:00 Collector: 6691 Recorder.cc:175 DEBUG] Last 30s storage_client.concurrent_user_calls: count:2 mean:0.5 min:0.0 max:1.0 p50:0.5 p90:1.0 p95:1.0 p99:1.0
[2025-03-18T15:14:27.195957916+08:00 admin_cli: 6841 IBSocket.cc:1219 DEBUG] IBSocketManager handleEvents running
[2025-03-18T15:14:27.395965540+08:00 admin_cli: 6841 IBSocket.cc:1219 DEBUG] IBSocketManager handleEvents running
[2025-03-18T15:14:27.595955707+08:00 admin_cli: 6841 IBSocket.cc:1219 DEBUG] IBSocketManager handleEvents running
[2025-03-18T15:14:27.795951553+08:00 admin_cli: 6841 IBSocket.cc:1219 DEBUG] IBSocketManager handleEvents running
[2025-03-18T15:14:27.854424074+08:00 CliBG0: 6846 BackgroundRunner.cc:85 DEBUG] BackgroundRunner: AutoRefresh round 2 start
[2025-03-18T15:14:27.854483755+08:00 CliBG1: 6847 MgmtdClient.cc:388 DEBUG] MgmtdClient: receive RefreshItem force=false
[2025-03-18T15:14:27.854504528+08:00 CliBG1: 6847 MgmtdClient.cc:390 INFO] [MgmtdClientOp RefreshRoutingInfo force=false No.4] start execution. from:
[2025-03-18T15:14:27.854518076+08:00 CliBG1: 6847 MgmtdClient.cc:181 DEBUG] MgmtdClient: execute RefreshRoutingInfo via NodeId(1) IPoIB://10.252.0.100:8000
[2025-03-18T15:14:27.854923886+08:00 CliProc1: 6843 Processor.h:153 DEBUG] receive response 217:5
[2025-03-18T15:14:27.854977583+08:00 CliBG1: 6847 MgmtdClient.cc:183 INFO] MgmtdClient: RefreshRoutingInfo finished
[2025-03-18T15:14:27.854989581+08:00 CliBG1: 6847 MgmtdClient.cc:614 DEBUG] MgmtdClient: get new routing info null
[2025-03-18T15:14:27.854998552+08:00 CliBG1: 6847 MgmtdClient.cc:390 INFO] [MgmtdClientOp RefreshRoutingInfo force=false No.4] succeeded. latency: 0ns
[2025-03-18T15:14:27.855027394+08:00 CliBG0: 6846 MgmtdClient.cc:286 DEBUG] MgmtdClient: AutoRefresh finished
[2025-03-18T15:14:27.855037970+08:00 CliBG0: 6846 BackgroundRunner.cc:102 DEBUG] BackgroundRunner: AutoRefresh wait 999408us for next round
[2025-03-18T15:14:27.995961943+08:00 admin_cli: 6841 IBSocket.cc:1219 DEBUG] IBSocketManager handleEvents running
[2025-03-18T15:14:28.195958598+08:00 admin_cli: 6841 IBSocket.cc:1219 DEBUG] IBSocketManager handleEvents running
[2025-03-18T15:14:28.395958134+08:00 admin_cli: 6841 IBSocket.cc:1219 DEBUG] IBSocketManager handleEvents running
[2025-03-18T15:14:28.595951080+08:00 admin_cli: 6841 IBSocket.cc:1219 DEBUG] IBSocketManager handleEvents running
[2025-03-18T15:14:28.795958291+08:00 admin_cli: 6841 IBSocket.cc:1219 DEBUG] IBSocketManager handleEvents running
[2025-03-18T15:14:28.858331609+08:00 CliBG0: 6846 BackgroundRunner.cc:85 DEBUG] BackgroundRunner: AutoRefresh round 3 start
[2025-03-18T15:14:28.858374925+08:00 CliBG1: 6847 MgmtdClient.cc:388 DEBUG] MgmtdClient: receive RefreshItem force=false
[2025-03-18T15:14:28.858390595+08:00 CliBG1: 6847 MgmtdClient.cc:390 INFO] [MgmtdClientOp RefreshRoutingInfo force=false No.5] start execution. from:
[2025-03-18T15:14:28.858402573+08:00 CliBG1: 6847 MgmtdClient.cc:181 DEBUG] MgmtdClient: execute RefreshRoutingInfo via NodeId(1) IPoIB://10.252.0.100:8000
[2025-03-18T15:14:28.858688553+08:00 CliProc1: 6843 Processor.h:153 DEBUG] receive response 217:5
[2025-03-18T15:14:28.858722740+08:00 CliBG1: 6847 MgmtdClient.cc:183 INFO] MgmtdClient: RefreshRoutingInfo finished
[2025-03-18T15:14:28.858729050+08:00 CliBG1: 6847 MgmtdClient.cc:614 DEBUG] MgmtdClient: get new routing info null
[2025-03-18T15:14:28.858734308+08:00 CliBG1: 6847 MgmtdClient.cc:390 INFO] [MgmtdClientOp RefreshRoutingInfo force=false No.5] succeeded. latency: 0ns
[2025-03-18T15:14:28.858750470+08:00 CliBG0: 6846 MgmtdClient.cc:286 DEBUG] MgmtdClient: AutoRefresh finished
[2025-03-18T15:14:28.858758512+08:00 CliBG0: 6846 BackgroundRunner.cc:102 DEBUG] BackgroundRunner: AutoRefresh wait 999588us for next round
[2025-03-18T15:14:28.995960401+08:00 admin_cli: 6841 IBSocket.cc:1219 DEBUG] IBSocketManager handleEvents running
[2025-03-18T15:14:29.195954909+08:00 admin_cli: 6841 IBSocket.cc:1219 DEBUG] IBSocketManager handleEvents running
[2025-03-18T15:14:29.395955418+08:00 admin_cli: 6841 IBSocket.cc:1219 DEBUG] IBSocketManager handleEvents running
[2025-03-18T15:14:29.595955690+08:00 admin_cli: 6841 IBSocket.cc:1219 DEBUG] IBSocketManager handleEvents running
[2025-03-18T15:14:29.795951278+08:00 admin_cli: 6841 IBSocket.cc:1219 DEBUG] IBSocketManager handleEvents running
[2025-03-18T15:14:29.862068822+08:00 CliBG0: 6846 BackgroundRunner.cc:85 DEBUG] BackgroundRunner: AutoRefresh round 4 start
[2025-03-18T15:14:29.862112995+08:00 CliBG1: 6847 MgmtdClient.cc:388 DEBUG] MgmtdClient: receive RefreshItem force=false
[2025-03-18T15:14:29.862130656+08:00 CliBG1: 6847 MgmtdClient.cc:390 INFO] [MgmtdClientOp RefreshRoutingInfo force=false No.6] start execution. from:
[2025-03-18T15:14:29.862141592+08:00 CliBG1: 6847 MgmtdClient.cc:181 DEBUG] MgmtdClient: execute RefreshRoutingInfo via NodeId(1) IPoIB://10.252.0.100:8000
[2025-03-18T15:14:29.862483003+08:00 CliProc1: 6843 Processor.h:153 DEBUG] receive response 217:5
[2025-03-18T15:14:29.862521572+08:00 CliBG1: 6847 MgmtdClient.cc:183 INFO] MgmtdClient: RefreshRoutingInfo finished
[2025-03-18T15:14:29.862531766+08:00 CliBG1: 6847 MgmtdClient.cc:614 DEBUG] MgmtdClient: get new routing info null
[2025-03-18T15:14:29.862539220+08:00 CliBG1: 6847 MgmtdClient.cc:390 INFO] [MgmtdClientOp RefreshRoutingInfo force=false No.6] succeeded. latency: 0ns
[2025-03-18T15:14:29.862561166+08:00 CliBG0: 6846 MgmtdClient.cc:286 DEBUG] MgmtdClient: AutoRefresh finished
[2025-03-18T15:14:29.862570431+08:00 CliBG0: 6846 BackgroundRunner.cc:102 DEBUG] BackgroundRunner: AutoRefresh wait 999514us for next round
[2025-03-18T15:14:29.995957519+08:00 admin_cli: 6841 IBSocket.cc:1219 DEBUG] IBSocketManager handleEvents running
[2025-03-18T15:14:30.195951013+08:00 admin_cli: 6841 IBSocket.cc:1219 DEBUG] IBSocketManager handleEvents running
[2025-03-18T15:14:30.395957326+08:00 admin_cli: 6841 IBSocket.cc:1219 DEBUG] IBSocketManager handleEvents running
[2025-03-18T15:14:30.595955623+08:00 admin_cli: 6841 IBSocket.cc:1219 DEBUG] IBSocketManager handleEvents running
[2025-03-18T15:14:30.795963244+08:00 admin_cli: 6841 IBSocket.cc:1219 DEBUG] IBSocketManager handleEvents running
[2025-03-18T15:14:30.866648538+08:00 CliBG0: 6846 BackgroundRunner.cc:85 DEBUG] BackgroundRunner: AutoRefresh round 5 start
[2025-03-18T15:14:30.866689260+08:00 CliBG1: 6847 MgmtdClient.cc:388 DEBUG] MgmtdClient: receive RefreshItem force=false
[2025-03-18T15:14:30.866711058+08:00 CliBG1: 6847 MgmtdClient.cc:390 INFO] [MgmtdClientOp RefreshRoutingInfo force=false No.7] start execution. from:
[2025-03-18T15:14:30.866722978+08:00 CliBG1: 6847 MgmtdClient.cc:181 DEBUG] MgmtdClient: execute RefreshRoutingInfo via NodeId(1) IPoIB://10.252.0.100:8000
[2025-03-18T15:14:30.867007851+08:00 CliProc1: 6843 Processor.h:153 DEBUG] receive response 217:5
[2025-03-18T15:14:30.867047734+08:00 CliBG1: 6847 MgmtdClient.cc:183 INFO] MgmtdClient: RefreshRoutingInfo finished
[2025-03-18T15:14:30.867058536+08:00 CliBG1: 6847 MgmtdClient.cc:614 DEBUG] MgmtdClient: get new routing info null
[2025-03-18T15:14:30.867066264+08:00 CliBG1: 6847 MgmtdClient.cc:390 INFO] [MgmtdClientOp RefreshRoutingInfo force=false No.7] succeeded. latency: 0ns
[2025-03-18T15:14:30.867087915+08:00 CliBG0: 6846 MgmtdClient.cc:286 DEBUG] MgmtdClient: AutoRefresh finished
[2025-03-18T15:14:30.867097480+08:00 CliBG0: 6846 BackgroundRunner.cc:102 DEBUG] BackgroundRunner: AutoRefresh wait 999564us for next round
[2025-03-18T15:14:30.995958810+08:00 admin_cli: 6841 IBSocket.cc:1219 DEBUG] IBSocketManager handleEvents running
[2025-03-18T15:14:31.195955061+08:00 admin_cli: 6841 IBSocket.cc:1219 DEBUG] IBSocketManager handleEvents running
```

Start with src/client/cli/admin/CreateTarget.cc:69 and trace the reported StorageClient::NoRDMAInterface error through IBDevice.cc and IfAddrs.h, using the provided admin_cli create-target command and cli.log output. Reproduce the failure with the two storage instances and document a confirmed cause and fix or diagnostic, with successful target creation as the completion check.

Written by the indexing model from the issue text.