ceph / ceph/ceph-nvmeof

get_io_stats fails to display IO statistics.

Open
#601 3 comments 0 reactions 0 assignees View on GitHub
Dominant language
Python
Stars
132
Forks
69
Avg merge
1d 7h
Merged PRs (30d)
47

Description

Get IO stats displays the IO metrics correctly when quried with active connected Gateway IP, else many metrics are "0" like bytes_written from other gateways.

**Steps-to-follow:**
- Deploy service with gateways (GW1 : LB-grp-id: 1, GW2: LB-grp-id: 2)
- Configure 2 subsystems
- Add both gateway listeners on each subsystem.
- Setup host, Add namespaces like below
- Subsystem-1 Namespaces tagged with LB-grp-id-1
- Subsystem-2 Namespaces tagged with LB-grp-id-2
- Start IO.
- Get IO stats for subsystem-1 namespaces from from GW2, then we can observe "bytes_written" will be "0"

```
root@ceph-sunilkumar-00-7v6gmf-node6 ~]# podman run --quiet --rm quay.io/barakda1/nvmeof-cli:1.2.1 --format json --server-address 10.0.208.84 --server-port 5500 get_subsystems
Get subsystems:
{
"subsystems": [
{
"nqn": "nqn.2016-06.io.spdk:cnode1",
"subtype": "NVMe",
"listen_addresses": [
{
"trtype": "TCP",
"adrfam": "IPv4",
"traddr": "10.0.208.84",
"trsvcid": "4420",
"transport": "TCP"
}
],
"allow_any_host": true,
"serial_number": "Ceph67140833472324",
"model_number": "Ceph bdev Controller",
"max_namespaces": 32,
"min_cntlid": 2041,
"max_cntlid": 4080,
"namespaces": [
{
"nsid": 1,
"name": "bdev_e42aafab-8f9b-4911-b326-4308cc962112",
"bdev_name": "bdev_e42aafab-8f9b-4911-b326-4308cc962112",
"nguid": "E42AAFAB8F9B4911B3264308CC962112",
"uuid": "e42aafab-8f9b-4911-b326-4308cc962112",
"anagrpid": 2,
"nonce": "10.0.208.84:0/3145406409"
},
{
"nsid": 2,
"name": "bdev_b0f9bf47-925a-4bcb-8e93-3749a98d6e2f",
"bdev_name": "bdev_b0f9bf47-925a-4bcb-8e93-3749a98d6e2f",
"nguid": "B0F9BF47925A4BCB8E933749A98D6E2F",
"uuid": "b0f9bf47-925a-4bcb-8e93-3749a98d6e2f",
"anagrpid": 2,
"nonce": "10.0.208.84:0/3145406409"
}
],
"hosts": []
},
{
"nqn": "nqn.2016-06.io.spdk:cnode2",
"subtype": "NVMe",
"listen_addresses": [
{
"trtype": "TCP",
"adrfam": "IPv4",
"traddr": "10.0.208.84",
"trsvcid": "4420",
"transport": "TCP"
}
],
"allow_any_host": true,
"serial_number": "Ceph5822363922143",
"model_number": "Ceph bdev Controller",
"max_namespaces": 32,
"min_cntlid": 2041,
"max_cntlid": 4080,
"namespaces": [
{
"nsid": 1,
"name": "bdev_e7721ff0-5d0e-4c9c-b703-6553aafc945a",
"bdev_name": "bdev_e7721ff0-5d0e-4c9c-b703-6553aafc945a",
"nguid": "E7721FF05D0E4C9CB7036553AAFC945A",
"uuid": "e7721ff0-5d0e-4c9c-b703-6553aafc945a",
"anagrpid": 1,
"nonce": "10.0.208.84:0/3359539278"
},
{
"nsid": 2,
"name": "bdev_35c56671-2204-42a9-9913-f51f77b65f60",
"bdev_name": "bdev_35c56671-2204-42a9-9913-f51f77b65f60",
"nguid": "35C56671220442A99913F51F77B65F60",
"uuid": "35c56671-2204-42a9-9913-f51f77b65f60",
"anagrpid": 1,
"nonce": "10.0.208.84:0/3359539278"
}
],
"hosts": []
}
]
}
```

**Listeners are added**
```
[root@ceph-sunilkumar-00-7v6gmf-node6 ~]# podman run --quiet --rm quay.io/barakda1/nvmeof-cli:1.2.1 --format json --server-address 10.0.208.84 --server-port 5500 listener list -n nqn.2016-06.io.spdk:cnode2
{
"error_message": "Success",
"listeners": [
{
"host_name": "ceph-sunilkumar-00-7v6gmf-node6",
"trtype": "TCP",
"traddr": "10.0.208.84",
"trsvcid": 4420,
"adrfam": "ipv4"
},
{
"host_name": "ceph-sunilkumar-00-7v6gmf-node7",
"trtype": "TCP",
"traddr": "10.0.209.23",
"trsvcid": 4420,
"adrfam": "ipv4"
}
],
"status": 0
}
[root@ceph-sunilkumar-00-7v6gmf-node6 ~]# podman run --quiet --rm quay.io/barakda1/nvmeof-cli:1.2.1 --format json --server-address 10.0.208.84 --server-port 5500 listener list -n nqn.2016-06.io.spdk:cnode1
{
"error_message": "Success",
"listeners": [
{
"host_name": "ceph-sunilkumar-00-7v6gmf-node6",
"trtype": "TCP",
"traddr": "10.0.208.84",
"trsvcid": 4420,
"adrfam": "ipv4"
},
{
"host_name": "ceph-sunilkumar-00-7v6gmf-node7",
"trtype": "TCP",
"traddr": "10.0.209.23",
"trsvcid": 4420,
"adrfam": "ipv4"
}
],
"status": 0
}

```

**Get IO stats**
```
[root@ceph-sunilkumar-00-7v6gmf-node6 ~]# podman run --quiet --rm quay.io/barakda1/nvmeof-cli:1.2.1 --format json --server-address 10.0.208.84 --server-port 5500 namespace get_io_stats --subsystem nqn.2016-06.io.spdk:cnode1 --nsid 1
{
"error_message": "Success",
"subsystem_nqn": "nqn.2016-06.io.spdk:cnode1",
"nsid": 1,
"uuid": "e42aafab-8f9b-4911-b326-4308cc962112",
"bdev_name": "bdev_e42aafab-8f9b-4911-b326-4308cc962112",
"tick_rate": "2290000000",
"ticks": "2621647473989225",
"bytes_read": "2183168",
"num_read_ops": "102",
"bytes_written": "113770496",
"num_write_ops": "868",
"read_latency_ticks": "127828488",
"max_read_latency_ticks": "5130384",
"min_read_latency_ticks": "570238",
"write_latency_ticks": "238402928",
"max_write_latency_ticks": "65301792",
"min_write_latency_ticks": "79444",
"status": 0,
"bytes_unmapped": "0",
"num_unmap_ops": "0",
"unmap_latency_ticks": "0",
"max_unmap_latency_ticks": "0",
"min_unmap_latency_ticks": "0",
"copy_latency_ticks": "0",
"max_copy_latency_ticks": "0",
"min_copy_latency_ticks": "0",
"io_error": []
}

[root@ceph-sunilkumar-00-7v6gmf-node6 ~]# podman run --quiet --rm quay.io/barakda1/nvmeof-cli:1.2.1 --format json --server-address 10.0.209.23 --server-port 5500 namespace get_io_stats --subsystem nqn.2016-06.io.spdk:cnode1 --nsid 1
{
"error_message": "Success",
"subsystem_nqn": "nqn.2016-06.io.spdk:cnode1",
"nsid": 1,
"uuid": "e42aafab-8f9b-4911-b326-4308cc962112",
"bdev_name": "bdev_e42aafab-8f9b-4911-b326-4308cc962112",
"tick_rate": "2190000000",
"ticks": "2507761566245463",
"bytes_read": "36864",
"num_read_ops": "2",
"read_latency_ticks": "8553886",
"max_read_latency_ticks": "4332226",
"min_read_latency_ticks": "4221660",
"status": 0,
"bytes_written": "0",
"num_write_ops": "0",
"bytes_unmapped": "0",
"num_unmap_ops": "0",
"write_latency_ticks": "0",
"max_write_latency_ticks": "0",
"min_write_latency_ticks": "0",
"unmap_latency_ticks": "0",
"max_unmap_latency_ticks": "0",
"min_unmap_latency_ticks": "0",
"copy_latency_ticks": "0",
"max_copy_latency_ticks": "0",
"min_copy_latency_ticks": "0",
"io_error": []
}
```

Contributor guide

Open the contributing guide

Research direction

Start by reproducing the two-gateway setup and running the namespace get_io_stats commands shown in the issue against each gateway. Trace the get_io_stats entry point and compare how metrics are gathered for namespaces served by another gateway; done means the response reports the expected IO statistics consistently regardless of the queried gateway.

Written by the indexing model from the issue text.

Assessment

Tech stack
python
Domain
backend, distributed-systems
Issue type
Bug
Difficulty
4/5
Estimated time
3-5 days
Activity status
Stale
Clarity
Mostly clear
Newbie friendliness
35/100

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.