thread safety issues in unicodeobject.c

Bug report

Bug description:

Running a TSAN analyser exposed some issues in unicodeobject.c

it->it_index++ is accessed and incremented raw in unicode_ascii_iter_next.

This can result in a crash with the debug build with a small test harness of multiple threads iterating on the same string.

import os
import sys
import threading


ROUNDS = int(sys.argv[1]) if len(sys.argv) > 1 else 4000
LENGTH = 2000
NTHREADS = 8
CHAR = "A"  # the ONLY character that a correct iterator may ever return
# NOEXHAUST>0: each thread stops after that many items, so the iterator is
# never drained to the end -> isolates the it_index race from the it_seq
# double-DECREF-on-exhaustion bug.
NOEXHAUST = int(os.environ.get("NOEXHAUST", "0"))


def drain(it, out, anomalies, barrier, cap):
    barrier.wait()
    n = 0
    while True:
        try:
            c = next(it)
        except StopIteration:
            return
        except Exception as e:  # a corrupted object can raise on use
            anomalies.append(("exception", repr(e)))
            return
        # The corruption checks -- impossible for a correct implementation:
        if type(c) is not str or len(c) != 1 or c != CHAR:
            anomalies.append(("bad-char", repr(c)))
        out.append(1)
        n += 1
        if cap and n >= cap:  # stop BEFORE exhaustion to isolate the it_seq bug
            return


def main():
    total_seen = 0
    total_expected = 0
    dup_skip_rounds = 0
    anomalies = []

    for r in range(ROUNDS):
        s = CHAR * LENGTH
        it = iter(s)
        barrier = threading.Barrier(NTHREADS)
        outs = [[] for _ in range(NTHREADS)]
        threads = [
            threading.Thread(target=drain,
                             args=(it, outs[i], anomalies, barrier, NOEXHAUST))
            for i in range(NTHREADS)
        ]
        for t in threads:
            t.start()
        for t in threads:
            t.join()

        seen = sum(len(o) for o in outs)
        total_seen += seen
        total_expected += LENGTH
        if seen != LENGTH:
            dup_skip_rounds += 1

        if anomalies:
            print(f"[round {r}] CORRUPTION: {anomalies[:5]}")
            break

    print("=" * 60)
    print(f"rounds run            : {r + 1}")
    print(f"chars expected total  : {total_expected}")
    print(f"chars actually seen   : {total_seen}  "
          f"(delta {total_seen - total_expected:+d} = dup/skip)")
    print(f"rounds with dup/skip  : {dup_skip_rounds}")
    print(f"OOB corruptions found : {len(anomalies)}")
    if anomalies:
        print(f"  samples: {anomalies[:10]}")
        print("--> proves the out-of-bounds read: a char not in the source.")
    else:
        print("--> no OOB char observed this run (still UB; TSan flags it "
              "deterministically).")
    return 1 if anomalies else 0


if __name__ == "__main__":
    sys.exit(main())

https://github.com/python/cpython/blob/main/Objects/unicodeobject.c#L14979-L14983

With the small test harness above

Stack (most recent call first):
File "/Users/johng/repos/cpython/str_race_impact.py", line 68 in main
File "/Users/johng/repos/cpython/str_race_impact.py", line 109 in
[1] 59403 abort PYTHON_GIL=0 ./python.exe str_race_impact.py

CPython versions tested on:

CPython main branch

Operating systems tested on:

macOS

Linked PRs

• gh-155873

Start in Objects/unicodeobject.c around lines 14979-14983 and reproduce the report with the supplied str_race_impact.py multi-threaded harness under the debug or TSAN build. Done means the shared-string iterator no longer exhibits the reported race, crashes, duplicate or skipped characters, or corrupt values; gh-155873 is already linked in the issue.