thread safety issues in unicodeobject.c

Bug report

Bug description:

Running a TSAN analyser exposed some issues in unicodeobject.c

it->it_index++ is accessed and incremented raw in unicode_ascii_iter_next.

This can result in a crash with the debug build with a small test harness of multiple threads iterating on the same string.

import os
import sys
import threading


ROUNDS = int(sys.argv[1]) if len(sys.argv) > 1 else 4000
LENGTH = 2000
NTHREADS = 8
CHAR = "A"  # the ONLY character that a correct iterator may ever return
# NOEXHAUST>0: each thread stops after that many items, so the iterator is
# never drained to the end -> isolates the it_index race from the it_seq
# double-DECREF-on-exhaustion bug.
NOEXHAUST = int(os.environ.get("NOEXHAUST", "0"))


def drain(it, out, anomalies, barrier, cap):
    barrier.wait()
    n = 0
    while True:
        try:
            c = next(it)
        except StopIteration:
            return
        except Exception as e:  # a corrupted object can raise on use
            anomalies.append(("exception", repr(e)))
            return
        # The corruption checks -- impossible for a correct implementation:
        if type(c) is not str or len(c) != 1 or c != CHAR:
            anomalies.append(("bad-char", repr(c)))
        out.append(1)
        n += 1
        if cap and n >= cap:  # stop BEFORE exhaustion to isolate the it_seq bug
            return


def main():
    total_seen = 0
    total_expected = 0
    dup_skip_rounds = 0
    anomalies = []

    for r in range(ROUNDS):
        s = CHAR * LENGTH
        it = iter(s)
        barrier = threading.Barrier(NTHREADS)
        outs = [[] for _ in range(NTHREADS)]
        threads = [
            threading.Thread(target=drain,
                             args=(it, outs[i], anomalies, barrier, NOEXHAUST))
            for i in range(NTHREADS)
        ]
        for t in threads:
            t.start()
        for t in threads:
            t.join()

        seen = sum(len(o) for o in outs)
        total_seen += seen
        total_expected += LENGTH
        if seen != LENGTH:
            dup_skip_rounds += 1

        if anomalies:
            print(f"[round {r}] CORRUPTION: {anomalies[:5]}")
            break

    print("=" * 60)
    print(f"rounds run            : {r + 1}")
    print(f"chars expected total  : {total_expected}")
    print(f"chars actually seen   : {total_seen}  "
          f"(delta {total_seen - total_expected:+d} = dup/skip)")
    print(f"rounds with dup/skip  : {dup_skip_rounds}")
    print(f"OOB corruptions found : {len(anomalies)}")
    if anomalies:
        print(f"  samples: {anomalies[:10]}")
        print("--> proves the out-of-bounds read: a char not in the source.")
    else:
        print("--> no OOB char observed this run (still UB; TSan flags it "
              "deterministically).")
    return 1 if anomalies else 0


if __name__ == "__main__":
    sys.exit(main())

https://github.com/python/cpython/blob/main/Objects/unicodeobject.c#L14979-L14983

With the small test harness above

Stack (most recent call first):
File "/Users/johng/repos/cpython/str_race_impact.py", line 68 in main
File "/Users/johng/repos/cpython/str_race_impact.py", line 109 in
[1] 59403 abort PYTHON_GIL=0 ./python.exe str_race_impact.py

CPython versions tested on:

CPython main branch

Operating systems tested on:

macOS

Linked PRs

• gh-155873

Commencez dans Objects/unicodeobject.c autour des lignes 14979-14983 et reproduisez le rapport avec le harness multithread str_race_impact.py fourni sous la build debug ou TSAN. C’est terminé lorsque l’itérateur de chaînes partagées ne présente plus la race signalée, de crashs, de caractères dupliqués ou ignorés, ni de valeurs corrompues ; gh-155873 est déjà lié dans l’issue.