Sentry: panic.go:770: runtime error: invalid memory address or nil pointer dereference (1) attached stack trace -- stack trace: | runtime.gopanic | GOROOT/src/runtime/panic.go:770 | runtime.p...

This issue was auto filed by Sentry. It represents a crash or reported error on a live cluster with telemetry enabled.

Sentry Link: [https://cockroach-labs.sentry.io/issues/7221482763/?referrer=webhooks_plugin](https://cockroach-labs.sentry.io/issues/7221482763/?referrer=webhooks_plugin)

Panic Message:

```
panic.go:770: runtime error: invalid memory address or nil pointer dereference
(1) attached stack trace
-- stack trace:
| runtime.gopanic
| GOROOT/src/runtime/panic.go:770
| runtime.panicmem
| GOROOT/src/runtime/panic.go:261
| runtime.sigpanic
| GOROOT/src/runtime/signal_unix.go:881
| cmpbody
| src/internal/bytealg/compare_amd64.s:100
| bytes.Compare
| GOROOT/src/bytes/bytes.go:27
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.keyBound.compare
| bazel-out/k8-opt/bin/pkg/kv/kvserver/spanlatch/latch_interval_btree.go:68
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*node).adjustUpperBoundOnRemoval
| bazel-out/k8-opt/bin/pkg/kv/kvserver/spanlatch/latch_interval_btree.go:657
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*node).remove
| bazel-out/k8-opt/bin/pkg/kv/kvserver/spanlatch/latch_interval_btree.go:461
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*node).remove
| bazel-out/k8-opt/bin/pkg/kv/kvserver/spanlatch/latch_interval_btree.go:478
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*btree).Delete
| bazel-out/k8-opt/bin/pkg/kv/kvserver/spanlatch/latch_interval_btree.go:722
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).remove
| pkg/kv/kvserver/spanlatch/manager.go:669
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Release
| pkg/kv/kvserver/spanlatch/manager.go:639
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Release
| pkg/kv/kvserver/concurrency/latch_manager.go:65
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).FinishReq
| pkg/kv/kvserver/concurrency/concurrency_manager.go:448
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*endCmds).done
| pkg/kv/kvserver/replica_send.go:1356
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*ProposalData).finishApplication
| pkg/kv/kvserver/replica_proposal.go:254
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicatedCmd).AckOutcomeAndFinish
| pkg/kv/kvserver/replica_application_cmd.go:151
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.forEachAppliedCmdIter
| pkg/kv/kvserver/apply/cmd.go:263
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).applyOneBatch
| pkg/kv/kvserver/apply/task.go:296
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).ApplyCommittedEntries
| pkg/kv/kvserver/apply/task.go:246
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked
| pkg/kv/kvserver/replica_raft.go:1269
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady
| pkg/kv/kvserver/replica_raft.go:846
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady
| pkg/kv/kvserver/store_raft.go:682
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker
| pkg/kv/kvserver/scheduler.go:419
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2
| pkg/kv/kvserver/scheduler.go:319
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2
| pkg/util/stop/stopper.go:498
| runtime.goexit
| src/runtime/asm_amd64.s:1695
Wraps: (2) runtime error: invalid memory address or nil pointer dereference
Error types: (1) *withstack.withStack (2) runtime.errorString
-- report composition:
runtime.errorString
panic.go:770: *withstack.withStack (top exception)
```

Stacktrace (expand for inline code snippets):

[src/runtime/asm_amd64.s#L1694-L1696](src/runtime/asm_amd64.s#L1694-L1696)
[pkg/util/stop/stopper.go#L497-L499](pkg/util/stop/stopper.go#L497-L499)
[pkg/kv/kvserver/scheduler.go#L318-L320](pkg/kv/kvserver/scheduler.go#L318-L320)
[pkg/kv/kvserver/scheduler.go#L418-L420](pkg/kv/kvserver/scheduler.go#L418-L420)
[pkg/kv/kvserver/store_raft.go#L681-L683](pkg/kv/kvserver/store_raft.go#L681-L683)
[pkg/kv/kvserver/replica_raft.go#L845-L847](pkg/kv/kvserver/replica_raft.go#L845-L847)
[pkg/kv/kvserver/replica_raft.go#L1268-L1270](pkg/kv/kvserver/replica_raft.go#L1268-L1270)
[pkg/kv/kvserver/apply/task.go#L245-L247](pkg/kv/kvserver/apply/task.go#L245-L247)
[pkg/kv/kvserver/apply/task.go#L295-L297](pkg/kv/kvserver/apply/task.go#L295-L297)
[pkg/kv/kvserver/apply/cmd.go#L262-L264](pkg/kv/kvserver/apply/cmd.go#L262-L264)
[pkg/kv/kvserver/replica_application_cmd.go#L150-L152](pkg/kv/kvserver/replica_application_cmd.go#L150-L152)
[pkg/kv/kvserver/replica_proposal.go#L253-L255](pkg/kv/kvserver/replica_proposal.go#L253-L255)
[pkg/kv/kvserver/replica_send.go#L1355-L1357](pkg/kv/kvserver/replica_send.go#L1355-L1357)
[pkg/kv/kvserver/concurrency/concurrency_manager.go#L447-L449](pkg/kv/kvserver/concurrency/concurrency_manager.go#L447-L449)
[pkg/kv/kvserver/concurrency/latch_manager.go#L64-L66](pkg/kv/kvserver/concurrency/latch_manager.go#L64-L66)
[pkg/kv/kvserver/spanlatch/manager.go#L638-L640](pkg/kv/kvserver/spanlatch/manager.go#L638-L640)
[pkg/kv/kvserver/spanlatch/manager.go#L668-L670](pkg/kv/kvserver/spanlatch/manager.go#L668-L670)
[bazel-out/k8-opt/bin/pkg/kv/kvserver/spanlatch/latch_interval_btree.go#L721-L723](bazel-out/k8-opt/bin/pkg/kv/kvserver/spanlatch/latch_interval_btree.go#L721-L723)
[bazel-out/k8-opt/bin/pkg/kv/kvserver/spanlatch/latch_interval_btree.go#L477-L479](bazel-out/k8-opt/bin/pkg/kv/kvserver/spanlatch/latch_interval_btree.go#L477-L479)
[bazel-out/k8-opt/bin/pkg/kv/kvserver/spanlatch/latch_interval_btree.go#L460-L462](bazel-out/k8-opt/bin/pkg/kv/kvserver/spanlatch/latch_interval_btree.go#L460-L462)
[bazel-out/k8-opt/bin/pkg/kv/kvserver/spanlatch/latch_interval_btree.go#L656-L658](bazel-out/k8-opt/bin/pkg/kv/kvserver/spanlatch/latch_interval_btree.go#L656-L658)
[bazel-out/k8-opt/bin/pkg/kv/kvserver/spanlatch/latch_interval_btree.go#L67-L69](bazel-out/k8-opt/bin/pkg/kv/kvserver/spanlatch/latch_interval_btree.go#L67-L69)
[GOROOT/src/bytes/bytes.go#L26-L28](GOROOT/src/bytes/bytes.go#L26-L28)
[src/internal/bytealg/compare_amd64.s#L99-L101](src/internal/bytealg/compare_amd64.s#L99-L101)
[GOROOT/src/runtime/signal_unix.go#L880-L882](GOROOT/src/runtime/signal_unix.go#L880-L882)
[GOROOT/src/runtime/panic.go#L260-L262](GOROOT/src/runtime/panic.go#L260-L262)
[GOROOT/src/runtime/panic.go#L769-L771](GOROOT/src/runtime/panic.go#L769-L771)

```
src/runtime/asm_amd64.s in runtime.goexit at line 1695
pkg/util/stop/stopper.go in pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2 at line 498
pkg/kv/kvserver/scheduler.go in pkg/kv/kvserver.(*raftScheduler).Start.func2 at line 319
pkg/kv/kvserver/scheduler.go in pkg/kv/kvserver.(*raftSchedulerShard).worker at line 419
pkg/kv/kvserver/store_raft.go in pkg/kv/kvserver.(*Store).processReady at line 682
pkg/kv/kvserver/replica_raft.go in pkg/kv/kvserver.(*Replica).handleRaftReady at line 846
pkg/kv/kvserver/replica_raft.go in pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked at line 1269
pkg/kv/kvserver/apply/task.go in pkg/kv/kvserver/apply.(*Task).ApplyCommittedEntries at line 246
pkg/kv/kvserver/apply/task.go in pkg/kv/kvserver/apply.(*Task).applyOneBatch at line 296
pkg/kv/kvserver/apply/cmd.go in pkg/kv/kvserver/apply.forEachAppliedCmdIter at line 263
pkg/kv/kvserver/replica_application_cmd.go in pkg/kv/kvserver.(*replicatedCmd).AckOutcomeAndFinish at line 151
pkg/kv/kvserver/replica_proposal.go in pkg/kv/kvserver.(*ProposalData).finishApplication at line 254
pkg/kv/kvserver/replica_send.go in pkg/kv/kvserver.(*endCmds).done at line 1356
pkg/kv/kvserver/concurrency/concurrency_manager.go in pkg/kv/kvserver/concurrency.(*managerImpl).FinishReq at line 448
pkg/kv/kvserver/concurrency/latch_manager.go in pkg/kv/kvserver/concurrency.(*latchManagerImpl).Release at line 65
pkg/kv/kvserver/spanlatch/manager.go in pkg/kv/kvserver/spanlatch.(*Manager).Release at line 639
pkg/kv/kvserver/spanlatch/manager.go in pkg/kv/kvserver/spanlatch.(*Manager).remove at line 669
bazel-out/k8-opt/bin/pkg/kv/kvserver/spanlatch/latch_interval_btree.go in pkg/kv/kvserver/spanlatch.(*btree).Delete at line 722
bazel-out/k8-opt/bin/pkg/kv/kvserver/spanlatch/latch_interval_btree.go in pkg/kv/kvserver/spanlatch.(*node).remove at line 478
bazel-out/k8-opt/bin/pkg/kv/kvserver/spanlatch/latch_interval_btree.go in pkg/kv/kvserver/spanlatch.(*node).remove at line 461
bazel-out/k8-opt/bin/pkg/kv/kvserver/spanlatch/latch_interval_btree.go in pkg/kv/kvserver/spanlatch.(*node).adjustUpperBoundOnRemoval at line 657
bazel-out/k8-opt/bin/pkg/kv/kvserver/spanlatch/latch_interval_btree.go in pkg/kv/kvserver/spanlatch.keyBound.compare at line 68
GOROOT/src/bytes/bytes.go in bytes.Compare at line 27
src/internal/bytealg/compare_amd64.s in .cmpbody at line 100
GOROOT/src/runtime/signal_unix.go in runtime.sigpanic at line 881
GOROOT/src/runtime/panic.go in runtime.panicmem at line 261
GOROOT/src/runtime/panic.go in runtime.gopanic at line 770
```

### Tags

| Tag | Value |
| --- | --- |
| Command | start-single-node |
| Environment | v24.3.3 |
| Go Version | go1.22.8 X:nocoverageredesign |
| Platform | linux amd64 |
| Distribution | CCL |
| Cockroach Release | v24.3.3 |
| Cockroach SHA | 343b4202e553d58a76a36c60018588615bd4c30a |
| # of CPUs | 24 |
| # of Goroutines | 1327 |

Jira issue: CRDB-59118