Nexus - steno panics on dogfood

During this week's dogfood update, @qdzlug pulled back two corefiles from Nexus panics (core.oxz_nexus_20b100d0-84c3-4119-aa9b-0c632b0b6a3a.nexus.11208.1751312404 and core.oxz_nexus_65a11c18-7f59-41ac-b9e7-680627f996e7.nexus.15708.1751605144, both in /staff/core/dogfood-2025-07-10/) that has have nearly identical backtraces:

fffff5ffed7fdf50 libc.so.1`_lwp_kill+0xa()
fffff5ffed7fdf80 libc.so.1`raise+0x22(6)
fffff5ffed7fdfd0 libc.so.1`abort+0x58()
fffff5ffed7fdfe0 ~panic_abort::__rust_start_panic::abort::h349954c3ed15731c+8()
fffff5ffed7fdff0 ~__rustc::__rust_start_panic+8()
fffff5ffed7fe050 __rustc::rust_panic+0xd()
fffff5ffed7fe110 std::panicking::rust_panic_with_hook::hb99087f77e1043a2+0x22f()
fffff5ffed7fe150 std::panicking::begin_panic_handler::{{closure}}::hdc6b5764a1eb26a4+0x65()
fffff5ffed7fe160 ~std::sys::backtrace::__rust_end_short_backtrace::h2fdfec1ded489945+8()
fffff5ffed7fe190 ~__rustc::rust_begin_unwind+0x1b()
fffff5ffed7fe1c0 ~core::panicking::panic_fmt::h84330e1efd4e6522+0x1e()
fffff5ffed7fe210 ~core::panicking::panic::h7668cc856f831b5d+0x3a()
fffff5ffed7fe2b0 steno::saga_exec::SagaExecLiveState::node_exec_state::h4f393b5a65faf092+0x540()
fffff5ffed7fe360 <steno::saga_exec::SagaNode<steno::saga_exec::SgnsUndone> as steno::saga_exec::SagaNodeRest<UserType>>::propagate::h2b0fa5d8b901ba17+0x28f()
fffff5ffed7fe410 <steno::saga_exec::SagaNode<steno::saga_exec::SgnsUndone> as steno::saga_exec::SagaNodeRest<UserType>>::propagate::h2b0fa5d8b901ba17+0x2e1()
fffff5ffed7fe4c0 <steno::saga_exec::SagaNode<steno::saga_exec::SgnsUndone> as steno::saga_exec::SagaNodeRest<UserType>>::propagate::h2b0fa5d8b901ba17+0x2e1()
fffff5ffed7fe570 <steno::saga_exec::SagaNode<steno::saga_exec::SgnsUndone> as steno::saga_exec::SagaNodeRest<UserType>>::propagate::h2b0fa5d8b901ba17+0x2e1()
fffff5ffed7fe620 <steno::saga_exec::SagaNode<steno::saga_exec::SgnsUndone> as steno::saga_exec::SagaNodeRest<UserType>>::propagate::h2b0fa5d8b901ba17+0x2e1()
fffff5ffed7fe6d0 <steno::saga_exec::SagaNode<steno::saga_exec::SgnsUndone> as steno::saga_exec::SagaNodeRest<UserType>>::propagate::h2b0fa5d8b901ba17+0x2e1()
fffff5ffed7fe780 <steno::saga_exec::SagaNode<steno::saga_exec::SgnsUndone> as steno::saga_exec::SagaNodeRest<UserType>>::propagate::h2b0fa5d8b901ba17+0x2e1()
fffff5ffed7fe830 <steno::saga_exec::SagaNode<steno::saga_exec::SgnsUndone> as steno::saga_exec::SagaNodeRest<UserType>>::propagate::h2b0fa5d8b901ba17+0x2e1()
fffff5ffed7fe8d0 <steno::saga_exec::SagaNode<steno::saga_exec::SgnsFailed> as steno::saga_exec::SagaNodeRest<UserType>>::propagate::haca492a25121ce43+0x270()
fffff5ffed7fee70 steno::saga_exec::SagaExecutor<UserType>::run::{{closure}}::hc9931081a98796da+0x1339()
fffff5ffed7ff100 steno::sec::Sec::do_saga_start::{{closure}}::hf8250a0d23521acc+0x287()
fffff5ffed7ff2e0 <futures_util::stream::futures_unordered::FuturesUnordered<Fut> as futures_core::stream::Stream>::poll_next::h8f25452c9755a660+0x1de()
fffff5ffed7ff470 <core::future::poll_fn::PollFn<F> as core::future::future::Future>::poll::hf264ee5c177df6db+0x2a5()
fffff5ffed7ff9c0 steno::sec::sec::_$u7b$$u7b$closure$u7d$$u7d$::h9b401b5a4a5915bf +0x38a()
fffff5ffed7ffa30 tokio::runtime::task::core::Core<T,S>::poll::ha69702807e22b828+0x3b()
fffff5ffed7ffa90 tokio::runtime::task::raw::poll::h2307297c6641e8b0+0x5d()
fffff5ffed7ffae0 tokio::runtime::scheduler::multi_thread::worker::Context::run_task::h24302e62d59f4ced+0x125()
fffff5ffed7ffba0 tokio::runtime::scheduler::multi_thread::worker::Context::run::h79f5fa907bcd79ff+0xe9e()
fffff5ffed7ffc00 tokio::runtime::context::scoped::Scoped<T>::set::ha0efc89e8e219599+0x2a()
fffff5ffed7ffcc0 tokio::runtime::context::runtime::enter_runtime::h133adcc598ce03d0+0xb1()
fffff5ffed7ffd00 tokio::runtime::scheduler::multi_thread::worker::run::hc8a2b5840fd64dd5+0xa0()
fffff5ffed7ffd60 tokio::runtime::task::core::Core<T,S>::poll::h749eb27040cd6fa2+0x70()
fffff5ffed7ffdb0 tokio::runtime::task::harness::Harness<T,S>::poll::h1869188eab05bd6b+0x8a()
fffff5ffed7ffe60 tokio::runtime::blocking::pool::Inner::run::hf10abaf5b6f4145f+0xe4()
fffff5ffed7ffec0 std::sys::backtrace::__rust_begin_short_backtrace::hb37c0e407a4a0246+0xc3()
fffff5ffed7fff60 core::ops::function::FnOnce::call_once{{vtable.shim}}::hdc4ea6ccc8db15d0+0xa8()
fffff5ffed7fffb0 std::sys::pal::unix::thread::Thread::new::thread_start::h8d13b9564d02f57c+0x2b()
fffff5ffed7fffe0 libc.so.1`_thrp_setup+0x77(fffff5ffed9e0240)
fffff5ffed7ffff0 libc.so.1`_lwp_start()

fffff5ffd5bfd590 libc.so.1`_lwp_kill+0xa()
fffff5ffd5bfd5c0 libc.so.1`raise+0x22(6)
fffff5ffd5bfd610 libc.so.1`abort+0x58()
fffff5ffd5bfd620 ~panic_abort::__rust_start_panic::abort::h6b7c627c7df281d0+8()
fffff5ffd5bfd630 ~__rustc::__rust_start_panic+8()
fffff5ffd5bfd690 __rustc::rust_panic+0xd()
fffff5ffd5bfd750 std::panicking::rust_panic_with_hook::hb8223ccd7c528a7f+0x22f()
fffff5ffd5bfd790 std::panicking::begin_panic_handler::{{closure}}::hcec802fa388f0165+0x65()
fffff5ffd5bfd7a0 ~std::sys::backtrace::__rust_end_short_backtrace::hd75f19ce2a7de6c4+8()
fffff5ffd5bfd7d0 ~__rustc::rust_begin_unwind+0x1b()
fffff5ffd5bfd800 ~core::panicking::panic_fmt::hbf9bff9fdba4d13d+0x1e()
fffff5ffd5bfd850 ~core::panicking::panic::h8dd2218ac6d40be7+0x3a()
fffff5ffd5bfd900 steno::saga_exec::SagaExecLiveState::node_exec_state::h1e61b4e80623fd3c+0x559()
fffff5ffd5bfd9b0 <steno::saga_exec::SagaNode<steno::saga_exec::SgnsUndone> as steno::saga_exec::SagaNodeRest<UserType>>::propagate::hec9f48e28896360e+0x28f()
fffff5ffd5bfda60 <steno::saga_exec::SagaNode<steno::saga_exec::SgnsUndone> as steno::saga_exec::SagaNodeRest<UserType>>::propagate::hec9f48e28896360e+0x2e1()
fffff5ffd5bfdb10 <steno::saga_exec::SagaNode<steno::saga_exec::SgnsUndone> as steno::saga_exec::SagaNodeRest<UserType>>::propagate::hec9f48e28896360e+0x2e1()
fffff5ffd5bfdbc0 <steno::saga_exec::SagaNode<steno::saga_exec::SgnsUndone> as steno::saga_exec::SagaNodeRest<UserType>>::propagate::hec9f48e28896360e+0x2e1()
fffff5ffd5bfdc70 <steno::saga_exec::SagaNode<steno::saga_exec::SgnsUndone> as steno::saga_exec::SagaNodeRest<UserType>>::propagate::hec9f48e28896360e+0x2e1()
fffff5ffd5bfdd20 <steno::saga_exec::SagaNode<steno::saga_exec::SgnsUndone> as steno::saga_exec::SagaNodeRest<UserType>>::propagate::hec9f48e28896360e+0x2e1()
fffff5ffd5bfddd0 <steno::saga_exec::SagaNode<steno::saga_exec::SgnsUndone> as steno::saga_exec::SagaNodeRest<UserType>>::propagate::hec9f48e28896360e+0x2e1()
fffff5ffd5bfde80 <steno::saga_exec::SagaNode<steno::saga_exec::SgnsUndone> as steno::saga_exec::SagaNodeRest<UserType>>::propagate::hec9f48e28896360e+0x2e1()
fffff5ffd5bfdf20 <steno::saga_exec::SagaNode<steno::saga_exec::SgnsFailed> as steno::saga_exec::SagaNodeRest<UserType>>::propagate::h3352a1ba00cb4735+0x270()
fffff5ffd5bfee30 steno::saga_exec::SagaExecutor<UserType>::run::{{closure}}::h9673695f112331f2+0x16d5()
fffff5ffd5bff0c0 steno::sec::Sec::do_saga_start::{{closure}}::hb7b9be75638ebb04+0x287()
fffff5ffd5bff2a0 <futures_util::stream::futures_unordered::FuturesUnordered<Fut> as futures_core::stream::Stream>::poll_next::h405dae231821e0b6+0x1de()
fffff5ffd5bff4b0 <core::future::poll_fn::PollFn<F> as core::future::future::Future>::poll::h27cdfa2fd73cba1e+0x109()
fffff5ffd5bffa00 steno::sec::sec::_$u7b$$u7b$closure$u7d$$u7d$::h18221bab138ce620 +0x399()
fffff5ffd5bffa70 tokio::runtime::task::core::Core<T,S>::poll::h86dfff40daf3b8cd+0x3b()
fffff5ffd5bffad0 tokio::runtime::task::raw::poll::h57de1fcd624e35af+0x5d()
fffff5ffd5bffb20 tokio::runtime::scheduler::multi_thread::worker::Context::run_task::h220372af2b69c236+0x125()
fffff5ffd5bffbb0 tokio::runtime::scheduler::multi_thread::worker::Context::run::ha6137785420c341f+0xc0e()
fffff5ffd5bffc10 tokio::runtime::context::scoped::Scoped<T>::set::h404d11d03fae2798+0x2a()
fffff5ffd5bffcd0 tokio::runtime::context::runtime::enter_runtime::hbea92029ca951a9a+0xb1()
fffff5ffd5bffd10 tokio::runtime::scheduler::multi_thread::worker::run::h85e66819c406cdd0+0xa0()
fffff5ffd5bffd70 tokio::runtime::task::core::Core<T,S>::poll::hd8de6c001086b642+0x70()
fffff5ffd5bffdc0 tokio::runtime::task::harness::Harness<T,S>::poll::hd41585bd4a0d94e4+0x8a()
fffff5ffd5bffe70 tokio::runtime::blocking::pool::Inner::run::hfccadbb87964bbce+0xe4()
fffff5ffd5bffec0 std::sys::backtrace::__rust_begin_short_backtrace::h0d8835be7d5c7d5b+0x72()
fffff5ffd5bfff60 core::ops::function::FnOnce::call_once{{vtable.shim}}::h2c58797cb5db6f94+0x97()
fffff5ffd5bfffb0 std::sys::pal::unix::thread::Thread::new::thread_start::hb8b719afb1b31961+0x2b()
fffff5ffd5bfffe0 libc.so.1`_thrp_setup+0x77(fffff5ffee8ffa40)
fffff5ffd5bffff0 libc.so.1`_lwp_start()

The logfiles from the Nexuses that contain these panics are oxz_nexus_20b100d0-84c3-4119-aa9b-0c632b0b6a3a.log.1751313601 and oxz_nexus_65a11c18-7f59-41ac-b9e7-680627f996e7.log.1751605201 in the same directory. (Note that oxz_nexus_65a11c18-7f59-41ac-b9e7-680627f996e7.log.1751605201 contains two panics, about five minutes apart; the first is filed as #8579. I don't know whether they're related in any way.)

In both cases, the panic message is:

thread 'tokio-runtime-worker-0' panicked at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/steno-0.4.1/src/saga_exec.rs:1587:13:
assertion failed: self.node_errors.contains_key(&node_id)
note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace

and in both cases, shortly before the panic there are logs about undo actions:

04:59:03.740Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): saga log event
    new_state = N008 undo_started
    sec_id = 65a11c18-7f59-41ac-b9e7-680627f996e7
04:59:03.740Z DEBG 65a11c18-7f59-41ac-b9e7-680627f996e7 (ServerContext): recording saga event
    event_type = UndoStarted
    node_id = 8
    saga_id = 8cd1e4a7-f288-45cd-83c2-1783eac6ddbf

19:40:03.624Z DEBG 20b100d0-84c3-4119-aa9b-0c632b0b6a3a (ServerContext): saga log event
    new_state = N004 undo_started
    sec_id = 20b100d0-84c3-4119-aa9b-0c632b0b6a3a
19:40:03.624Z DEBG 20b100d0-84c3-4119-aa9b-0c632b0b6a3a (ServerContext): recording saga event
    event_type = UndoStarted
    node_id = 4
    saga_id = 6bdba216-7969-4ab6-a10c-afdc855a8f29

Start at steno-0.4.1/src/saga_exec.rs:1587 and correlate the node_errors assertion with the UndoStarted entries in the two Nexus logfiles. Use the referenced corefiles and backtraces to trace the saga propagation path; done means the reported undo sequence no longer panics and its failure is handled correctly.