foyer 0.14.1 panics on cache read

Nobody has claimed this yet.

On commit 96243fd4625d269e14b3207f274a6abe0162e055, a call to the query service in `chromadb/test/property/test_add.py` occasionally flakes. The query service logs a panic on these flakes:

```
2025-03-20T03:11:46.295961Z ERROR A panic occurred, panic.payload: "called `Result::unwrap()` on an `Err` value: JoinError::Panic(Id(16043), \"range end index 11086 out of range for slice of length 10918\", ...)", panic.location: "/usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/foyer-storage-0.14.1/src/store.rs:161:61", panic.backtrace: 0: chroma_tracing::init_tracer::init_panic_tracing_hook::{{closure}}
at ./chroma/rust/tracing/src/init_tracer.rs:155:55
1: as core::ops::function::Fn>::call
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/alloc/src/boxed.rs:2084:9
2: std::panicking::rust_panic_with_hook
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:808:13
3: std::panicking::begin_panic_handler::{{closure}}
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:674:13
4: std::sys::backtrace::__rust_end_short_backtrace
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/sys/backtrace.rs:168:18
5: rust_begin_unwind
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:665:5
6: core::panicking::panic_fmt
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/panicking.rs:74:14
7: core::result::unwrap_failed
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/result.rs:1679:5
8: core::result::Result::unwrap
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/result.rs:1102:23
9: foyer_storage::store::Store::load::{{closure}}
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/foyer-storage-0.14.1/src/store.rs:161:15
10: as core::future::future::Future>::poll
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/fastrace-0.7.4/src/future.rs:124:19
11: foyer::hybrid::cache::HybridCache::get::{{closure}}
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/foyer-0.14.1/src/hybrid/cache.rs:283:14
12: as chroma_cache::Cache>::get::{{closure}}
at ./chroma/rust/cache/src/foyer.rs:457:39
13: as core::future::future::Future>::poll
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/future/future.rs:123:9
14: chroma_blockstore::arrow::provider::BlockManager::get::{{closure}}
at ./chroma/rust/blockstore/src/arrow/provider.rs:323:46
15: chroma_blockstore::arrow::blockfile::ArrowBlockfileReader::get_block::{{closure}}
at ./chroma/rust/blockstore/src/arrow/blockfile.rs:386:65
16: chroma_blockstore::arrow::blockfile::ArrowBlockfileReader::get::{{closure}}
at ./chroma/rust/blockstore/src/arrow/blockfile.rs:462:53
17: chroma_blockstore::types::reader::BlockfileReader::get::{{closure}}
at ./chroma/rust/blockstore/src/types/reader.rs:37:86
18: chroma_segment::blockfile_record::RecordSegmentReader::get_data_for_offset_id::{{closure}}
at ./chroma/rust/segment/src/blockfile_record.rs:799:44
19: >::run::{{closure}}
at ./chroma/rust/worker/src/execution/operators/projection.rs:153:30
20: as core::future::future::Future>::poll
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/future/future.rs:123:9
21: >::run::{{closure}}
at ./chroma/rust/worker/src/execution/operators/knn_projection.rs:97:61
22: as core::future::future::Future>::poll
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/future/future.rs:123:9
23: as core::future::future::Future>::poll
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/panic/unwind_safe.rs:297:9
24: as core::future::future::Future>::poll::{{closure}}
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.31/src/future/future/catch_unwind.rs:37:42
25: as core::ops::function::FnOnce<()>>::call_once
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/panic/unwind_safe.rs:272:9
26: std::panicking::try::do_call
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:557:40
27: __rust_try
28: std::panicking::try
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:521:19
29: std::panic::catch_unwind
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panic.rs:350:14
30: as core::future::future::Future>::poll
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.31/src/future/future/catch_unwind.rs:37:9
31: as chroma_system::execution::operator::TaskWrapper>::run::{{closure}}
at ./chroma/rust/system/src/execution/operator.rs:146:14
32: as core::future::future::Future>::poll
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/future/future.rs:123:9
33: as core::future::future::Future>::poll
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9
34: >>::handle::{{closure}}
at ./chroma/rust/system/src/execution/worker_thread.rs:62:43
35: as core::future::future::Future>::poll
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/future/future.rs:123:9
36: as core::future::future::Future>::poll
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/panic/unwind_safe.rs:297:9
37: as core::future::future::Future>::poll::{{closure}}
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.31/src/future/future/catch_unwind.rs:37:42
38: as core::ops::function::FnOnce<()>>::call_once
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/panic/unwind_safe.rs:272:9
39: std::panicking::try::do_call
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:557:40
40: __rust_try
41: std::panicking::try
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:521:19
42: std::panic::catch_unwind
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panic.rs:350:14
43: as core::future::future::Future>::poll
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.31/src/future/future/catch_unwind.rs:37:9
44: >::Result>> as chroma_system::wrapped_message::HandleableMessage>::handle_and_reply::{{closure}}
at ./chroma/rust/system/src/wrapped_message.rs:81:18
45: as core::future::future::Future>::poll
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/future/future.rs:123:9
46: chroma_system::wrapped_message::WrappedMessage::handle::{{closure}}
at ./chroma/rust/system/src/wrapped_message.rs:55:61
47: as core::future::future::Future>::poll
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9
48: chroma_system::executor::ComponentExecutor::run::{{closure}}
at ./chroma/rust/system/src/executor.rs:98:64
49: chroma_system::system::System::start_component::{{closure}}::{{closure}}
at ./chroma/rust/system/src/system.rs:67:63
50: as core::future::future::Future>::poll
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/future/future.rs:123:9
51: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}}
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.1/src/runtime/scheduler/current_thread/mod.rs:729:57
52: tokio::runtime::coop::with_budget
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.1/src/runtime/coop.rs:107:5
--
63: tokio::runtime::scheduler::current_thread::CoreGuard::enter
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.1/src/runtime/scheduler/current_thread/mod.rs:807:27
64: tokio::runtime::scheduler::current_thread::CoreGuard::block_on
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.1/src/runtime/scheduler/current_thread/mod.rs:716:19
65: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}}
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.1/src/runtime/scheduler/current_thread/mod.rs:196:28
66: tokio::runtime::context::runtime::enter_runtime
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.1/src/runtime/context/runtime.rs:65:16
67: tokio::runtime::scheduler::current_thread::CurrentThread::block_on
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.1/src/runtime/scheduler/current_thread/mod.rs:184:9
68: tokio::runtime::runtime::Runtime::block_on_inner
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.1/src/runtime/runtime.rs:368:47
69: tokio::runtime::runtime::Runtime::block_on
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.1/src/runtime/runtime.rs:342:13
70: chroma_system::system::System::start_component::{{closure}}
at ./chroma/rust/system/src/system.rs:67:21
71: std::sys::backtrace::__rust_begin_short_backtrace
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/sys/backtrace.rs:152:18
72: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/thread/mod.rs:538:17
73: as core::ops::function::FnOnce<()>>::call_once
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/panic/unwind_safe.rs:272:9
74: std::panicking::try::do_call
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:557:40
75: __rust_try
76: std::panicking::try
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:521:19
77: std::panic::catch_unwind
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panic.rs:350:14
78: std::thread::Builder::spawn_unchecked_::{{closure}}
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/thread/mod.rs:537:30
79: core::ops::function::FnOnce::call_once{{vtable.shim}}
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/ops/function.rs:250:5
80: as core::ops::function::FnOnce>::call_once
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/alloc/src/boxed.rs:2070:9
81: as core::ops::function::FnOnce>::call_once
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/alloc/src/boxed.rs:2070:9
82: std::sys::pal::unix::thread::Thread::new::thread_start
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/sys/pal/unix/thread.rs:108:17
```

We believe this is because we manually insert to both the disk and in-memory cache on cache writes.

Start with chromadb/test/property/test_add.py and reproduce the intermittent query failure, then trace the cache read through chroma/rust/cache/src/foyer.rs and the blockstore frames in chroma/rust/blockstore/src/arrow/provider.rs and blockfile.rs. Compare the panic at foyer-storage-0.14.1/src/store.rs:161 with the failing path; done means the query test no longer panics during cache reads.

Written by the indexing model from the issue text.