Heap corruption(?) when consuming program uses jemalloc

Hello!

Repro instructions:

$ git clone https://github.com/rustls/rustls --branch=jbp-bench-more-providers tmp-rustls-bench
$ cd tmp-rustls-bench
$ cp -r ~/symcrypt-release/symcrypt .
$ LD_LIBRARY_PATH=symcrypt/lib RUSTFLAGS="-Lsymcrypt/lib" cargo run -p rustls-bench --features symcrypt
( .. build output ..)
     Running `target/debug/rustls-bench`
thread '<unnamed>' panicked at /home/jbp/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tikv-jemallocator-0.6.0/src/lib.rs:126:9:
assertion failed: !ptr.is_null()
note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
thread '<unnamed>' panicked at /home/jbp/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tikv-jemallocator-0.6.0/src/lib.rs:126:9:
assertion failed: !ptr.is_null()
stack backtrace:
   0:     0x5d744e1da72a - std::backtrace_rs::backtrace::libunwind::trace::h5a5b8284f2d0c266
                               at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/../../backtrace/src/backtrace/libunwind.rs:116:5
   1:     0x5d744e1da72a - std::backtrace_rs::backtrace::trace_unsynchronized::h76d4f1c9b0b875e3
                               at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
   2:     0x5d744e1da72a - std::sys::backtrace::_print_fmt::hc4546b8364a537c6
                               at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/sys/backtrace.rs:66:9
   3:     0x5d744e1da72a - <std::sys::backtrace::BacktraceLock::print::DisplayBacktrace as core::fmt::Display>::fmt::h5b6bd5631a6d1f6b
                               at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/sys/backtrace.rs:39:26
   4:     0x5d744e1fccd3 - core::fmt::rt::Argument::fmt::h270f6602a2b96f62
                               at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/fmt/rt.rs:177:76
   5:     0x5d744e1fccd3 - core::fmt::write::h7550c97b06c86515
                               at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/fmt/mod.rs:1186:21
   6:     0x5d744e1d8113 - std::io::Write::write_fmt::h7b09c64fe0be9c84
                               at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/io/mod.rs:1839:15
   7:     0x5d744e1da572 - std::sys::backtrace::BacktraceLock::print::h2395ccd2c84ba3aa
                               at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/sys/backtrace.rs:42:9
   8:     0x5d744e1db36c - std::panicking::default_hook::{{closure}}::he19d4c7230e07961
                               at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:268:22
   9:     0x5d744e1db1b2 - std::panicking::default_hook::hf614597d3c67bbdb
                               at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:295:9
  10:     0x5d744e1db947 - std::panicking::rust_panic_with_hook::h8942133a8b252070
                               at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:801:13
  11:     0x5d744e1db7a6 - std::panicking::begin_panic_handler::{{closure}}::hb5f5963570096b29
                               at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:667:13
  12:     0x5d744e1dac09 - std::sys::backtrace::__rust_end_short_backtrace::h6208cedc1922feda
                               at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/sys/backtrace.rs:170:18
  13:     0x5d744e1db46c - rust_begin_unwind
                               at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:665:5
  14:     0x5d744db1ba40 - core::panicking::panic_fmt::h0c3082644d1bf418
                               at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/panicking.rs:74:14
  15:     0x5d744db1bacc - core::panicking::panic::h957f98c65a3b3074
                               at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/panicking.rs:148:5
  16:     0x5d744db5892f - <tikv_jemallocator::Jemalloc as core::alloc::global::GlobalAlloc>::dealloc::hedc842ac093257b2
                               at /home/jbp/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tikv-jemallocator-0.6.0/src/lib.rs:83:9
  17:     0x5d744db36a13 - __rust_dealloc
                               at /home/jbp/src/rustls-bench/rustls-bench/src/main.rs:1602:16
  18:     0x5d744de8daad - alloc::alloc::dealloc::h382052ba811bccea
                               at /home/jbp/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/alloc.rs:119:14
  19:     0x5d744de8daad - <alloc::alloc::Global as core::alloc::Allocator>::deallocate::h0e3de5955721d44a
                               at /home/jbp/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/alloc.rs:269:22
  20:     0x5d744de8e1f8 - <alloc::boxed::Box<T,A> as core::ops::drop::Drop>::drop::hb95a9f0504083e56
                               at /home/jbp/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:1679:17
  21:     0x5d744de8bb19 - core::ptr::drop_in_place<alloc::boxed::Box<symcrypt_sys::symcrypt_bindings::_SYMCRYPT_SHA256_STATE>>::h275e3d8726dd3c06
                               at /home/jbp/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:574:1
  22:     0x5d744de8bbab - core::ptr::drop_in_place<core::pin::Pin<alloc::boxed::Box<symcrypt_sys::symcrypt_bindings::_SYMCRYPT_SHA256_STATE>>>::h6213d6f3aa2a278a
                               at /home/jbp/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:574:1
  23:     0x5d744de8bdb4 - core::ptr::drop_in_place<symcrypt::hash::Sha256State>::hea2c8f483a270cb8
                               at /home/jbp/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:574:1
  24:     0x5d744de8189b - core::ptr::drop_in_place<rustls_symcrypt::hash::Sha256Context>::h34a3b3d1a5e76692
                               at /home/jbp/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:574:1
  25:     0x5d744df104a4 - core::ptr::drop_in_place<alloc::boxed::Box<dyn rustls::crypto::hash::Context>>::h11cb0cc48ecef68f
                               at /home/jbp/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:574:1
  26:     0x5d744df0958b - core::ptr::drop_in_place<rustls::hash_hs::HandshakeHash>::h924ceceab6b7b931
                               at /home/jbp/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:574:1
  27:     0x5d744deec31b - <rustls::client::tls12::ExpectServerDone as rustls::common_state::State<rustls::client::client_conn::ClientConnectionData>>::handle::h268060d63d2b78d0
                               at /home/jbp/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rustls-0.23.19/src/client/tls12.rs:1024:5
  28:     0x5d744dee86c8 - <rustls::client::tls12::ExpectServerDoneOrCertReq as rustls::common_state::State<rustls::client::client_conn::ClientConnectionData>>::handle::h82ac08874e007170
                               at /home/jbp/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rustls-0.23.19/src/client/tls12.rs:670:13
  29:     0x5d744dedadd4 - rustls::common_state::CommonState::process_main_protocol::hfbd882bf55f60164
                               at /home/jbp/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rustls-0.23.19/src/common_state.rs:217:15
  30:     0x5d744df5cec2 - rustls::conn::ConnectionCore<Data>::process_msg::h323574f022cdfe12
                               at /home/jbp/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rustls-0.23.19/src/conn.rs:1125:9
  31:     0x5d744df5a359 - rustls::conn::ConnectionCore<Data>::process_new_packets::he912692da86d2078
                               at /home/jbp/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rustls-0.23.19/src/conn.rs:856:19
  32:     0x5d744df58b2b - rustls::conn::ConnectionCommon<Data>::process_new_packets::hc14c7f9893089aef
                               at /home/jbp/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rustls-0.23.19/src/conn.rs:397:9
  33:     0x5d744db43ab9 - rustls_bench::transfer::h54738ba9126fcc9d
                               at /home/jbp/src/rustls-bench/rustls-bench/src/main.rs:1470:21
  34:     0x5d744db30c3c - rustls_bench::do_handshake_step::h6c26a9996bd3eac0
                               at /home/jbp/src/rustls-bench/rustls-bench/src/main.rs:1406:9
  35:     0x5d744db30c96 - rustls_bench::do_handshake::h5ffc98985a5a024e
                               at /home/jbp/src/rustls-bench/rustls-bench/src/main.rs:1418:11
  36:     0x5d744db29895 - rustls_bench::bench_bulk_buffered::h19c14b65729d90d7
                               at /home/jbp/src/rustls-bench/rustls-bench/src/main.rs:577:5
  37:     0x5d744db41d6b - rustls_bench::bench_bulk::{{closure}}::h82f8a20a70b1200d
                               at /home/jbp/src/rustls-bench/rustls-bench/src/main.rs:542:17
  38:     0x5d744db3e644 - rustls_bench::multithreaded::{{closure}}::{{closure}}::{{closure}}::h6e05c33aecf612ba
                               at /home/jbp/src/rustls-bench/rustls-bench/src/main.rs:451:28
  39:     0x5d744db44e09 - std::sys::backtrace::__rust_begin_short_backtrace::h0481bca1bbf0f628
                               at /home/jbp/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys/backtrace.rs:154:18
  40:     0x5d744db5198c - std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}::hd6ecfca71a53afba
                               at /home/jbp/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:538:17
  41:     0x5d744db499cf - <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once::hfaab9a6831003b02
                               at /home/jbp/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panic/unwind_safe.rs:272:9
  42:     0x5d744db3ac81 - std::panicking::try::do_call::h80913bac6ea894e6
                               at /home/jbp/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:557:40
  43:     0x5d744db51a6b - __rust_try
  44:     0x5d744db4fbfc - std::panicking::try::h09b62b4bdf00bede
                               at /home/jbp/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:520:19
  45:     0x5d744db4fbfc - std::panic::catch_unwind::h0b834a2e4b41b51a
                               at /home/jbp/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panic.rs:358:14
  46:     0x5d744db4fbfc - std::thread::Builder::spawn_unchecked_::{{closure}}::h1d9acaf841d73ec4
                               at /home/jbp/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:537:30
  47:     0x5d744db1cbbe - core::ops::function::FnOnce::call_once{{vtable.shim}}::h25b3feb814444bd0
                               at /home/jbp/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5
  48:     0x5d744e1dd77b - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::hf75717d9f28faebf
                               at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/alloc/src/boxed.rs:2454:9
  49:     0x5d744e1dd77b - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::h7bd883a5f3c5f3c1
                               at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/alloc/src/boxed.rs:2454:9
  50:     0x5d744e1dd77b - std::sys::pal::unix::thread::Thread::new::thread_start::hcc78f3943333fa94
                               at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/sys/pal/unix/thread.rs:105:17
  51:     0x7e6b47a9ca94 - start_thread
                               at ./nptl/pthread_create.c:447:8
  52:     0x7e6b47b29c3c - clone3
                               at ./misc/../sysdeps/unix/sysv/linux/x86_64/clone3.S:78
  53:                0x0 - <unknown>
thread '<unnamed>' panicked at core/src/panicking.rs:229:5:
panic in a destructor during cleanup
thread caused non-unwinding panic. aborting.
Aborted (core dumped)

or in release mode:

$ LD_LIBRARY_PATH=symcrypt/lib RUSTFLAGS="-Lsymcrypt/lib" cargo run -p rustls-bench --release --features symcrypt
( .. build output ..)
     Running `target/release/rustls-bench`
memory allocation of 128 bytes failed
Aborted (core dumped)

However, if I stop using jemalloc, things start working:

$ git diff
diff --git a/rustls-bench/src/main.rs b/rustls-bench/src/main.rs
index bb45b014..2d3d3b93 100644
--- a/rustls-bench/src/main.rs
+++ b/rustls-bench/src/main.rs
@@ -1596,7 +1596,3 @@ static ALL_BENCHMARKS: &[BenchmarkParam] = &[
         &rustls::version::TLS13,
     ),
 ];
-
-#[cfg(not(target_env = "msvc"))]
-#[global_allocator]
-static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;

At a guess, something is mixing allocators -- eg, something inside libsymcrypt.so calls malloc(), and then passes the pointer back and it gets free()d by rust. That works if they are the same allocator, but not if they are different. However, I didn't find the exact place where this happens so this is just a theory.