pd panic when inject pd leader io delay 1s last for 5mins

## Bug Report

### What did you do?
1、run workload
2、add index for one table
3、inject pdleader io delay 1s last for 5mins

### What did you expect to see?
no panic

### What did you see instead?
pd panic

{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.396963998+08:00 stderr F \t/go/pkg/mod/go.etcd.io/etcd@v0.5.0-alpha.5.0.20220915004622-85b640cee793/etcdserver/server.go:1072 +0x25 fp=0xc00228ff10 sp=0xc00228fee8 pc=0x1833325","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.396947043+08:00 stderr F go.etcd.io/etcd/etcdserver.(*EtcdServer).run.func8({0xc00228ff90, 0xc00228ff58})","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.396923179+08:00 stderr F \t/go/pkg/mod/go.etcd.io/etcd@v0.5.0-alpha.5.0.20220915004622-85b640cee793/etcdserver/server.go:1136 +0x65 fp=0xc00228fee8 sp=0xc00228f678 pc=0x1834085","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.396894581+08:00 stderr F go.etcd.io/etcd/etcdserver.(*EtcdServer).applyAll(0xc00002e600, 0xc0008b6000, 0xc005358800)","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.396878649+08:00 stderr F \t/go/pkg/mod/go.etcd.io/etcd@v0.5.0-alpha.5.0.20220915004622-85b640cee793/etcdserver/server.go:1412 +0x305 fp=0xc00228f678 sp=0xc00228f570 pc=0x1836b45","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.396859039+08:00 stderr F go.etcd.io/etcd/etcdserver.(*EtcdServer).applyEntries(0xc00002e600, 0xc0008b6000, 0x0?)","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.396836692+08:00 stderr F \t/go/pkg/mod/go.etcd.io/etcd@v0.5.0-alpha.5.0.20220915004622-85b640cee793/etcdserver/server.go:2170 +0x432 fp=0xc00228f570 sp=0xc00228f428 pc=0x183e472","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.396813345+08:00 stderr F go.etcd.io/etcd/etcdserver.(*EtcdServer).apply(0xc00002e600, {0xc010bacf28, 0x1, 0x28?}, 0xc00657f000?)","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.396779284+08:00 stderr F \t/go/pkg/mod/go.etcd.io/etcd@v0.5.0-alpha.5.0.20220915004622-85b640cee793/etcdserver/server.go:2254 +0x1b3 fp=0xc00228f428 sp=0xc00228f180 pc=0x183e6b3","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.396763801+08:00 stderr F go.etcd.io/etcd/etcdserver.(*EtcdServer).applyEntryNormal(0xc00002e600, 0xc00228f518)","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.39674498+08:00 stderr F \t/go/pkg/mod/go.etcd.io/etcd@v0.5.0-alpha.5.0.20220915004622-85b640cee793/etcdserver/apply_auth.go:60 +0x1c2 fp=0xc00228f180 sp=0xc00228f120 pc=0x1811922","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.39672267+08:00 stderr F go.etcd.io/etcd/etcdserver.(*authApplierV3).Apply(0xc00097e050, 0xc000224a80)","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.396705852+08:00 stderr F \t:1 +0x2a fp=0xc00228f120 sp=0xc00228f100 pc=0x1853eca","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.39668579+08:00 stderr F go.etcd.io/etcd/etcdserver.(*quotaApplierV3).Apply(0xc00228f170?, 0x1809231?)","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.396671018+08:00 stderr F \t/go/pkg/mod/go.etcd.io/etcd@v0.5.0-alpha.5.0.20220915004622-85b640cee793/etcdserver/apply.go:129 +0xd37 fp=0xc00228f100 sp=0xc00228f058 pc=0x180a857","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.396652354+08:00 stderr F go.etcd.io/etcd/etcdserver.(*applierV3backend).Apply(0xc0006f60c0, 0xc000224a80)","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.396634157+08:00 stderr F \t/go/pkg/mod/go.etcd.io/etcd@v0.5.0-alpha.5.0.20220915004622-85b640cee793/etcdserver/apply_auth.go:168 +0x69 fp=0xc00228f058 sp=0xc00228f020 pc=0x1812309","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.396608874+08:00 stderr F go.etcd.io/etcd/etcdserver.(*authApplierV3).Txn(0xc00097e050, 0xc000224a80?)","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.396592299+08:00 stderr F \t/go/pkg/mod/go.etcd.io/etcd@v0.5.0-alpha.5.0.20220915004622-85b640cee793/etcdserver/apply.go:855 +0x5a fp=0xc00228f020 sp=0xc00228efe8 pc=0x18108ba","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.39657107+08:00 stderr F go.etcd.io/etcd/etcdserver.(*quotaApplierV3).Txn(0xc000112060, 0xc00041e500?)","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.396559145+08:00 stderr F \t/go/pkg/mod/go.etcd.io/etcd@v0.5.0-alpha.5.0.20220915004622-85b640cee793/etcdserver/apply.go:359 +0x10d fp=0xc00228efe8 sp=0xc00228eef0 pc=0x180cbed","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.396538763+08:00 stderr F go.etcd.io/etcd/etcdserver.(*applierV3backend).Txn(0xc0006f60c0, 0x29a6d60?)","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.396519444+08:00 stderr F \t/go/pkg/mod/go.etcd.io/etcd@v0.5.0-alpha.5.0.20220915004622-85b640cee793/etcdserver/apply.go:426 +0xdb fp=0xc00228eef0 sp=0xc00228ee58 pc=0x180d5db","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.396498511+08:00 stderr F go.etcd.io/etcd/etcdserver.compareToPath({0x7ff6983aab70, 0xc004ab3f60}, 0xc0096c51d0)","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.396482278+08:00 stderr F \t/go/pkg/mod/go.etcd.io/etcd@v0.5.0-alpha.5.0.20220915004622-85b640cee793/etcdserver/apply.go:441","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.396472259+08:00 stderr F go.etcd.io/etcd/etcdserver.applyCompares(...)","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.39646739+08:00 stderr F \t/go/pkg/mod/go.etcd.io/etcd@v0.5.0-alpha.5.0.20220915004622-85b640cee793/etcdserver/apply.go:456 +0x91 fp=0xc00228ee58 sp=0xc00228ed88 pc=0x180d811","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.396445963+08:00 stderr F go.etcd.io/etcd/etcdserver.applyCompare({0x7ff6983aab70?, 0xc004ab3f60?}, 0xc0096c5220)","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.396422167+08:00 stderr F \t:1 +0x64 fp=0xc00228ed88 sp=0xc00228ed28 pc=0x1724144","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.39639799+08:00 stderr F go.etcd.io/etcd/mvcc.(*txnReadWrite).Range(0x203002?, {0xc0084d5000?, 0xc0098e9740?, 0x0?}, {0x0?, 0x0?, 0x200?}, {0x0, 0x0, 0x0})","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.396329105+08:00 stderr F \t/go/pkg/mod/go.etcd.io/etcd@v0.5.0-alpha.5.0.20220915004622-85b640cee793/mvcc/metrics_txn.go:37 +0x53 fp=0xc00228ed28 sp=0xc00228ecc8 pc=0x171c7b3","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.396305542+08:00 stderr F go.etcd.io/etcd/mvcc.(*metricsTxnWrite).Range(0xc00006b800?, {0xc0084d5000?, 0x8?, 0xc00144aae0?}, {0x0?, 0xdd1fc7?, 0x9?}, {0x0, 0x0, 0x0})","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.396239913+08:00 stderr F \t:1 +0x64 fp=0xc00228ecc8 sp=0xc00228ec68 pc=0x1724144","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.396222176+08:00 stderr F go.etcd.io/etcd/mvcc.(*txnReadWrite).Range(0x4905d00?, {0xc0084d5000?, 0xdd1c7f?, 0x7ff6c04e7f18?}, {0x0?, 0x6000?, 0x0?}, {0x0, 0x0, 0x0})","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.39615672+08:00 stderr F \t/go/pkg/mod/go.etcd.io/etcd@v0.5.0-alpha.5.0.20220915004622-85b640cee793/mvcc/kvstore_txn.go:51 +0x49 fp=0xc00228ec68 sp=0xc00228ec00 pc=0x17184a9","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.396132508+08:00 stderr F go.etcd.io/etcd/mvcc.(*storeTxnRead).Range(0x0?, {0xc0084d5000?, 0xc00228ecd0?, 0x203002?}, {0x0?, 0x203002?, 0xc0098e96e0?}, {0x0, 0x0, 0x0})","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.396073601+08:00 stderr F \t/go/pkg/mod/go.etcd.io/etcd@v0.5.0-alpha.5.0.20220915004622-85b640cee793/mvcc/kvstore_txn.go:147 +0x617 fp=0xc00228ec00 sp=0xc00228e968 pc=0x17190f7","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.396050432+08:00 stderr F go.etcd.io/etcd/mvcc.(*storeTxnRead).rangeKeys(0xc0098e9710, {0xc0084d5000?, 0x6000100000003?, 0xc002489a00?}, {0x0?, 0xbf4?, 0x0?}, 0x16b14, {0x0, 0x0, ...})","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.39596116+08:00 stderr F \t/go/pkg/mod/go.etcd.io/etcd@v0.5.0-alpha.5.0.20220915004622-85b640cee793/mvcc/backend/read_tx.go:208 +0x3d5 fp=0xc00228e968 sp=0xc00228e838 pc=0x16df015","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.395946079+08:00 stderr F go.etcd.io/etcd/mvcc/backend.(*concurrentReadTx).UnsafeRange(0xc0098e96e0, {0x482c11e, 0x3, 0x3}, {0xc00a3de0f0, 0x11, 0x12}, {0x0, 0x0, 0x0}, ...)","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.395893317+08:00 stderr F \t/go/pkg/mod/go.etcd.io/etcd@v0.5.0-alpha.5.0.20220915004622-85b640cee793/mvcc/backend/batch_tx.go:156 +0xf6 fp=0xc00228e838 sp=0xc00228e708 pc=0x16dba96","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.395860571+08:00 stderr F go.etcd.io/etcd/mvcc/backend.unsafeRange(0xc00052ec00?, {0xc00a3de0f0?, 0xc001993d39?, 0x3?}, {0x0?, 0x0?, 0x203002?}, 0xc00a3de0f0?)","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.395808876+08:00 stderr F \t/go/pkg/mod/go.etcd.io/bbolt@v1.3.6/cursor.go:118 +0x27 fp=0xc00228e708 sp=0xc00228e6d8 pc=0x16c3b67","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.39577665+08:00 stderr F go.etcd.io/bbolt.(*Cursor).Seek(0xc00228e938, {0xc00a3de0f0?, 0x36af6c0?, 0xc00ae2fd40?})","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.395749938+08:00 stderr F \t/go/pkg/mod/go.etcd.io/bbolt@v1.3.6/cursor.go:159 +0x48 fp=0xc00228e6d8 sp=0xc00228e680 pc=0x16c3d68","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.395733839+08:00 stderr F go.etcd.io/bbolt.(*Cursor).seek(0xc00228e938, {0xc00a3de0f0?, 0xc00228e7d0?, 0x16dfe0f?})","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.395701436+08:00 stderr F \t/go/pkg/mod/go.etcd.io/bbolt@v1.3.6/cursor.go:248 +0x54 fp=0xc00228e680 sp=0xc00228e5e8 pc=0x16c4334","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.395677557+08:00 stderr F go.etcd.io/bbolt.(*Cursor).search(0xc00228e938, {0xc00a3de0f0, 0x11, 0x12}, 0x1704177?)","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.395631432+08:00 stderr F \t/go/pkg/mod/go.etcd.io/bbolt@v1.3.6/bucket.go:726 +0xc8 fp=0xc00228e5e8 sp=0xc00228e598 pc=0x16c3448","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.395616063+08:00 stderr F go.etcd.io/bbolt.(*Bucket).pageNode(0x10eac60, 0xf7)","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.39561306+08:00 stderr F \t/go/pkg/mod/go.etcd.io/bbolt@v1.3.6/tx.go:617","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.395610589+08:00 stderr F go.etcd.io/bbolt.(*Tx).page(...)","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.395608243+08:00 stderr F \t/go/pkg/mod/go.etcd.io/bbolt@v1.3.6/db.go:933","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.395605462+08:00 stderr F go.etcd.io/bbolt.(*DB).page(...)","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.395602567+08:00 stderr F \t/usr/local/go/src/runtime/signal_unix.go:846 +0x2c5 fp=0xc00228e598 sp=0xc00228e548 pc=0xe15865","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.395597947+08:00 stderr F runtime.sigpanic()","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.395539852+08:00 stderr F runtime.throw({0x2a9c76f?, 0x36c6aa0?})","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.395517695+08:00 stderr F goroutine 197 [running]:","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.395513674+08:00 stderr F ","container":"pd","pod":"tc-pd-0"}
{"namespace":"endless-ha-test-add-index-tps-3540115-1-849","log":"2023-10-25T20:58:41.395482167+08:00 stderr F [signal SIGSEGV: segmentation violation code=0x1 addr=0x7ff433e00000 pc=0x16c3448]","container":"pd","pod":"tc-pd-0"}

### What version of PD are you using (`pd-server -V`)?
./pd-server -V
Release Version: v6.5.0-nightly
Edition: Community
Git Commit Hash: e0ab17c4c684cdffd87b20d0d8486bbc81a60b1d
Git Branch: heads/refs/tags/v6.5.0-nightly
UTC Build Time: 2023-10-18 10:21:05
2023-10-25T20:57:28.512+0800