[Start/Stop API] Non-det CRASH for multi-threaded application: ASSERT(thread_takeover_records != NULL) on non-main thread

**Describe the bug**

The bug occurs when using the start/stop API on a multi-threaded application, when the main thread delimits several regions to be traced, starting with `dr_app_start()` and ending with `dr_app_stop()`, while another thread is running in the background.
The application appears to crash on `dr_app_stop()` with the message: `Received SIGSEGV at pc 0x0000000071077a55 in thread `.
The error is non-deterministic and may not occur when the number of start/stop regions is small.
The error has been tested with the drmemtrace tool and the sample opcode_count client, but also seems to happen without any client.

When testing with the Debug version of DynamoRIO (see To Reproduce), the console displays something like:
```
and

**To Reproduce**
Steps to reproduce the behavior:
1. Pointer to a minimized application (ideally the source code for it and instructions on which toolchain it was built with).

We can test that behaviour on a pthread code doing dummy computation.
We can change the values of `NUM_THREADS` and `NUM_START_STOP_REGIONS` to see that the lower they are, the less likely DynamoRIO will crash:
```c
#include "dr_api.h"
#include
#include
#include
#include

#define NUM_THREADS 2
#define NUM_LOOPS 1000
#define NUM_START_STOP_REGIONS 100
#define N 10000

static void init(int n, double *a, double *b) {
for (int i = 0; i < n; i++) {
a[i] = 0.23 + (double)i;
b[i] = 1.67 + (double)i;
}
}

static void triad(int n, double *a, double *b, double *c) {
for (int i = 0; i < n; i++) {
c[i] = a[i] * 1.2 + b[i];
}
}

static void *thread_routine(void *argument) {
int tid = *((int *)argument);
printf("Hello from thread %d!\n", tid);

for (int i = 0; i < NUM_LOOPS; i++) {
double *a = malloc(sizeof(double) * N);
double *b = malloc(sizeof(double) * N);
double *c = malloc(sizeof(double) * N);

init(N, a, b);
triad(N, a, b, c);

free(a);
free(b);
free(c);
}
printf("Thread %d done!\n", tid);
return NULL;
}

int main(int argc, char **argv) {
dr_app_setup();
const int n_threads_to_create = NUM_THREADS - 1;
pthread_t threads[n_threads_to_create];
int thread_args[n_threads_to_create];

/* create all threads */
for (int i = 0; i < n_threads_to_create; ++i) {
thread_args[i] = i + 1;
printf("In main: creating thread %d\n", i + 1);
int rc = pthread_create(&threads[i], NULL, thread_routine,
(void *)&thread_args[i]);
assert(0 == rc);
}

for (int i = 0; i < NUM_START_STOP_REGIONS; i++) {
double *a = malloc(sizeof(double) * N);
double *b = malloc(sizeof(double) * N);
double *c = malloc(sizeof(double) * N);

init(N, a, b);

printf("Will dr_app_start()\n");
dr_app_start();
printf("dr_app_start() done\n");

triad(N, a, b, c);

printf("Will dr_app_stop()\n");
dr_app_stop();
printf("dr_app_stop() done\n");

free(a);
free(b);
free(c);
}
printf("Thread 0 done!\n");
/* wait for all threads to complete */
for (int i = 0; i < n_threads_to_create; ++i) {
int rc = pthread_join(threads[i], NULL);
assert(0 == rc);
}

dr_app_cleanup();

printf("In main: EXITING\n");

return EXIT_SUCCESS;
}

```

The CMakeLists.txt
```sh
cmake_minimum_required(VERSION 3.10)

project(DynamoRIO_samples)

set(output_dir "${PROJECT_BINARY_DIR}/bin")
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${output_dir}")
set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}")
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${output_dir}")

find_package(DynamoRIO)
if(NOT DynamoRIO_FOUND)
message(FATAL_ERROR "DynamoRIO package required to build")
endif()

# ##############################################################################

# As we'll be calling configure_DynamoRIO_{client,standalone} from within a
# function scope, we must set the global vars ahead of time:
configure_dynamorio_global(OFF ON)

# set(CMAKE_SKIP_BUILD_RPATH FALSE)

find_package(Threads REQUIRED)

# Example for using the start/stop API
add_executable(dr_hello_pthread dr_hello_pthread.c)
configure_dynamorio_standalone(dr_hello_pthread)
target_link_libraries(dr_hello_pthread Threads::Threads)

add_custom_command(
TARGET dr_hello_pthread
POST_BUILD
COMMAND ${CMAKE_COMMAND} ARGS -E echo "Usage:\n"
COMMAND ${CMAKE_COMMAND} ARGS -E echo " export LD_LIBRARY_PATH=\"\$\{DYNAMORIO_DIR\}/lib64/release/:\$\{LD_LIBRARY_PATH\}\""
COMMAND ${CMAKE_COMMAND} ARGS -E echo " export DYNAMORIO_OPTIONS=\"-client_lib \'\$\{DYNAMORIO_DIR\}/tools/lib64/release/libdrmemtrace.so;;-offline\' -code_api\""
COMMAND ${CMAKE_COMMAND} ARGS -E echo " ./bin/dr_hello_pthread"
VERBATIM)
```

2. Precise command line for running the application.

Build the example:

```sh
export DYNAMORIO_DIR=
export DynamoRIO_DIR=${DYNAMORIO_DIR}/cmake/
mkdir build ; cd build
cmake ..
make
```

Run the example

```sh
export DR_BUILD="release" # or "debug"
export LD_LIBRARY_PATH="${DYNAMORIO_DIR}/lib64/${DR_BUILD}/:${LD_LIBRARY_PATH}"
export DYNAMORIO_OPTIONS="-client_lib '${DYNAMORIO_DIR}/tools/lib64/${DR_BUILD}/libdrmemtrace.so;;-offline' -code_api"

./bin/dr_hello_pthread
```

3. Exact output or incorrect behavior.

Please also answer these questions:
- What happens when you run without any client?
Same kind of crash.

- What happens when you run with debug build ("-debug" flag to drrun/drconfig/drinject)?
See below

GDB callstack:

```c
Thread 2 "dr_hello_pthrea" hit Breakpoint 1, d_r_internal_error (file=0x713cd8d8 "dynamorio/core/unix/os.c", line=11166,
expr=0x713d2e18 "thread_takeover_records != NULL") at dynamorio/core/utils.c:154
154 if (!IS_STRING_OPTION_EMPTY(ignore_assert_list)) {
(gdb) bt full
#0 d_r_internal_error (file=0x713cd8d8 "dynamorio/core/unix/os.c", line=11166, expr=0x713d2e18 "thread_takeover_records != NULL")
at dynamorio/core/utils.c:154
No locals.
#1 0x00000000712fff8b in os_thread_signal_taken_over () at dynamorio/core/unix/os.c:11166
mytid = 48012
event = 0x0
i = 32765
#2 0x00000000713001e6 in os_thread_take_over (mc=0x7ffdf7d76fa0, sigset=0x7ffdf7d77ae8) at dynamorio/core/unix/os.c:11217
dcontext = 0x7ffdf7c58080
dc_mc = 0x7ffdf7c58080
__FUNCTION__ = "os_thread_take_over"
#3 0x00000000713116e8 in sig_take_over (uc=0x7ffdf7d779c0) at dynamorio/core/unix/signal.c:5617
mc = {{xdi = 10000, rdi = 10000}, {xsi = 140728629988208, rsi = 140728629988208}, {xbp = 140728753634944, rbp = 140728753634944}, {xsp = 140728753634944,
rsp = 140728753634944}, {xbx = 140728753638620, rbx = 140728753638620}, {xdx = 31680, rdx = 31680}, {xcx = 140728629985328, rcx = 140728629985328}, {xax = 3961,
rax = 3961}, r8 = 140728629985280, r9 = 245760, r10 = 1, r11 = 3, r12 = 140728753637056, r13 = 18446744073709551496, r14 = 0, r15 = 140737488344688, {
xflags = 647, rflags = 647}, {xip = 0x5555555552d3 "H\215", ,
pc = 0x5555555552d3 "H\215", , rip = 0x5555555552d3 "H\215", },
padding = {} , opmask = {140728761546912, 1899087584, 21337962672, 140728761547504, 140728753638620, 140728753637056, 18446744073709551496,
140728761171648}}
#4 0x000000007131ca0b in handle_suspend_signal (dcontext=0x7ffdf7c58080, siginfo=0x7ffdf7d77af0, ucxt=0x7ffdf7d779c0, frame=0x7ffdf7d779b8)
at dynamorio/core/unix/signal.c:8753
ostd = 0x7ffdf7d5d048
prevmask = {sig = {140728761547024}}
sc_full = {sc = 0x7ffdf7d779e8, fp_simd_state = 0x10000007147a240}
prior_whereami = DR_WHERE_APP
unblocked_sigs = false
#5 0x0000000071312c57 in main_signal_handler_C (sig=4, siginfo=0x7ffdf7d77af0, ucxt=0x7ffdf7d779c0, xsp=0x7ffdf7d779b8 "\036\256,q")
at dynamorio/core/unix/signal.c:6143
frame = 0x7ffdf7d779b8
sc = 0x7ffdf7d779e8
tr = 0x7ffdf7d1bec0
level = 2
local = false
dcontext = 0x7ffdf7c58080
#6 0x00000000712cae1e in xfer_to_new_libdr () at dynamorio/core/arch/x86/x86.asm:1203
No locals.
#7 0x0000000000000007 in ?? ()
No symbol table info available.
#8 0x0000000000000000 in ?? ()
No symbol table info available.

(gdb) thread apply 1 bt full
Thread 1 (LWP 48011 "dr_hello_pthrea"):
#0 syscall_ready () at dynamorio/core/drlibc/drlibc_x86.asm:187
No locals.
#1 0x0000000000000206 in ?? ()
No symbol table info available.
#2 0x0000000071331a3e in ksynch_wait (futex=0x7ffdf7d5d0e4, mustbe=0, timeout_ms=5000) at dynamorio/core/unix/ksynch_linux.c:125
timeout = {tv_sec = 5, tv_nsec = 0}
res = 140728761438436
#3 0x00000000712eb65e in os_thread_suspend (tr=0x7ffdf7d1bec0) at dynamorio/core/unix/os.c:3925
ostd = 0x7ffdf7d5d048
#4 0x00000000711ef0eb in synch_with_thread (id=48012, block=false, hold_initexit_lock=true, caller_state=THREAD_SYNCH_NONE, desired_state=THREAD_SYNCH_SUSPENDED_VALID_MCONTEXT_OR_NO_XFER, flags=2) at dynamorio/core/synch.c:1006
my_id = 48011
loop_count = 1
expect_exiting = 0
my_tr = 0x7ffdf7c45060
trec = 0x7ffdf7d1bec0
dcontext = 0x7ffdf7c8b980
mc = {}
res = THREAD_SYNCH_RESULT_NOT_SAFE
first_loop = false
actually_suspended = true
max_loops = 10000
#5 0x00000000711f0ba3 in synch_with_all_threads (desired_synch_state=THREAD_SYNCH_SUSPENDED_VALID_MCONTEXT_OR_NO_XFER, threads_out=0x7ffdf7cb2eb0, num_threads_out=0x7ffdf7cb2eac, cur_state=THREAD_SYNCH_NO_LOCKS_NO_XFER, flags=2) at dynamorio/core/synch.c:1414
threads_are_stale = false
threads = 0x7ffdf7c432c8
num_threads = 2
thread_ids_temp = 0x7ffdf7d26540
num_threads_temp = 2
i = 1
j = 1
expect_self_exiting = 0
synch_array = 0x7ffdf7ce4fb0
synch_array_temp = 0x7ffdf7ce4fb0
SYNCH_WITH_ALL_NEW = SYNCH_WITH_ALL_NEW
SYNCH_WITH_ALL_NOTIFIED = SYNCH_WITH_ALL_NOTIFIED
SYNCH_WITH_ALL_SYNCHED = SYNCH_WITH_ALL_SYNCHED
all_synched = true
my_id = 48011
loop_count = 0
tr = 0x7ffdf7c45060
dcontext = 0x7ffdf7c8b980
flags_one = 2
synch_res = 0
max_loops = 10000
finished_non_client_threads = false
#6 0x00000000711f2997 in send_all_other_threads_native () at dynamorio/core/synch.c:1889
threads = 0x27aff7cb2ed0
my_dcontext = 0x7ffdf7c8b980
i = 32765
num_threads = 32765
waslinking = true
desired_state = THREAD_SYNCH_SUSPENDED_VALID_MCONTEXT_OR_NO_XFER
__FUNCTION__ = "send_all_other_threads_native"
#7 0x00000000710d8944 in handle_special_tag (dcontext=0x7ffdf7c8b980) at dynamorio/core/dispatch.c:603
No locals.
#8 0x00000000710d63d1 in d_r_dispatch (dcontext=0x7ffdf7c8b980) at dynamorio/core/dispatch.c:171
targetf = 0x7ffdf7c997b8
coarse_f = {tag = 0xabababababababab , flags = 2880154539, size = 43947, prefix_size = 171 '\253', fcache_extra = 171 '\253', start_pc = 0xabababababababab , in_xlate = {incoming_stubs = 0xabababababababab, translation_info = 0xabababababababab}, next_vmarea = 0xabababababababab, prev_vmarea = 0xabababababababab, also = {also_vmarea = 0xabababababababab, flushtime = 2880154539}, id = -1414812757}
#9 0x00005555155faf0d in ?? ()
No symbol table info available.
#10 0x0000000000000000 in ?? ()
No symbol table info available.

```

**Expected behavior**

The application should run properly until the end and create a `drmemtrace.dr_hello_pthread.34394.3684.dir` containing the memory trace made with drcachesim, or any correct results according to the client/tool used.

The expected output should be:

```
In main: creating thread 1
Hello from thread 1!
Will dr_app_start()
dr_app_start() done
Will dr_app_stop()
dr_app_stop() done
Will dr_app_start()
dr_app_start() done
Will dr_app_stop()
dr_app_stop() done
(...)
Thread 1 done! (or after "Thread 0 done!")
(...)
Will dr_app_stop()
dr_app_stop() done
Thread 0 done!
In main: EXITING
```

**Screenshots or Pasted Text**

As already said, this is a non-deterministic behaviour, but the error should look like this:

```
In main: creating thread 1
Hello from thread 1!
Will dr_app_start()
dr_app_start() done
Will dr_app_stop()
dr_app_stop() done
Will dr_app_start()
dr_app_start() done
Will dr_app_stop()
dr_app_stop() done
Will dr_app_start()
dr_app_start() done
Will dr_app_stop()
dr_app_stop() done
Will dr_app_start()
dr_app_start() done
Will dr_app_stop()

```

As for with debug DynamoRIO, like this:

```

<(1+x) Handling our fault in a TRY at 0x00000000712cb2db>
In main: creating thread 1
Hello from thread 1!
Will dr_app_start()

dr_app_start() done
Will dr_app_stop()
dr_app_stop() done
Will dr_app_start()

dr_app_start() done
Will dr_app_stop()
dr_app_stop() done
Will dr_app_start()

dr_app_start() done
Will dr_app_stop()
dr_app_stop() done
Will dr_app_start()

dr_app_start() done
Will dr_app_stop()

```

**Versions**

- What version of DynamoRIO are you using?
cronbuild-11.90.20363. I also tried the 11.3 release before.

- Does the latest build from https://github.com/DynamoRIO/dynamorio/releases solve the problem?
No.

- What operating system version are you running on? ("Windows 10" is *not* sufficient: give the release number.)
x86_64 Linux Ubuntu 24.04.1

- Is your application 32-bit or 64-bit?
64-bit

**Additional context**

At first, the idea was to use start/stop API for CUDA applications, and CUDA launches threads in the background we cannot control.
That's why the reproducible test does dummy computation: it is to mimic a background thread which has an unknown behaviour.