thread safety issues in unicodeobject.c

Bug report

Bug description:

Running a TSAN analyser exposed some issues in unicodeobject.c

it->it_index++ is accessed and incremented raw in unicode_ascii_iter_next.

This can result in a crash with the debug build with a small test harness of multiple threads iterating on the same string.

import os
import sys
import threading


ROUNDS = int(sys.argv[1]) if len(sys.argv) > 1 else 4000
LENGTH = 2000
NTHREADS = 8
CHAR = "A"  # the ONLY character that a correct iterator may ever return
# NOEXHAUST>0: each thread stops after that many items, so the iterator is
# never drained to the end -> isolates the it_index race from the it_seq
# double-DECREF-on-exhaustion bug.
NOEXHAUST = int(os.environ.get("NOEXHAUST", "0"))


def drain(it, out, anomalies, barrier, cap):
    barrier.wait()
    n = 0
    while True:
        try:
            c = next(it)
        except StopIteration:
            return
        except Exception as e:  # a corrupted object can raise on use
            anomalies.append(("exception", repr(e)))
            return
        # The corruption checks -- impossible for a correct implementation:
        if type(c) is not str or len(c) != 1 or c != CHAR:
            anomalies.append(("bad-char", repr(c)))
        out.append(1)
        n += 1
        if cap and n >= cap:  # stop BEFORE exhaustion to isolate the it_seq bug
            return


def main():
    total_seen = 0
    total_expected = 0
    dup_skip_rounds = 0
    anomalies = []

    for r in range(ROUNDS):
        s = CHAR * LENGTH
        it = iter(s)
        barrier = threading.Barrier(NTHREADS)
        outs = [[] for _ in range(NTHREADS)]
        threads = [
            threading.Thread(target=drain,
                             args=(it, outs[i], anomalies, barrier, NOEXHAUST))
            for i in range(NTHREADS)
        ]
        for t in threads:
            t.start()
        for t in threads:
            t.join()

        seen = sum(len(o) for o in outs)
        total_seen += seen
        total_expected += LENGTH
        if seen != LENGTH:
            dup_skip_rounds += 1

        if anomalies:
            print(f"[round {r}] CORRUPTION: {anomalies[:5]}")
            break

    print("=" * 60)
    print(f"rounds run            : {r + 1}")
    print(f"chars expected total  : {total_expected}")
    print(f"chars actually seen   : {total_seen}  "
          f"(delta {total_seen - total_expected:+d} = dup/skip)")
    print(f"rounds with dup/skip  : {dup_skip_rounds}")
    print(f"OOB corruptions found : {len(anomalies)}")
    if anomalies:
        print(f"  samples: {anomalies[:10]}")
        print("--> proves the out-of-bounds read: a char not in the source.")
    else:
        print("--> no OOB char observed this run (still UB; TSan flags it "
              "deterministically).")
    return 1 if anomalies else 0


if __name__ == "__main__":
    sys.exit(main())

https://github.com/python/cpython/blob/main/Objects/unicodeobject.c#L14979-L14983

With the small test harness above

Stack (most recent call first):
File "/Users/johng/repos/cpython/str_race_impact.py", line 68 in main
File "/Users/johng/repos/cpython/str_race_impact.py", line 109 in
[1] 59403 abort PYTHON_GIL=0 ./python.exe str_race_impact.py

CPython versions tested on:

CPython main branch

Operating systems tested on:

macOS

Linked PRs

• gh-155873

Bắt đầu tại Objects/unicodeobject.c quanh các dòng 14979-14983 và tái hiện báo cáo bằng harness đa luồng str_race_impact.py được cung cấp trong bản build debug hoặc TSAN. Được xem là hoàn thành khi iterator chuỗi dùng chung không còn xuất hiện race đã báo cáo, crash, ký tự bị trùng hoặc bị bỏ qua, hay giá trị bị hỏng; gh-155873 đã được liên kết trong issue.