Assertion error in _PyPegen_raise_error_known_location() when running PyRun_SimpleFileExFlags()

Script reproducing the bug:

import _testcapi
import ctypes.util

filename = ctypes.util.find_library('c')
libc = ctypes.cdll.LoadLibrary(filename)
pythonapi = ctypes.pythonapi
FILE_p = ctypes.c_void_p

fopen = libc.fopen
fopen.argtypes = (ctypes.c_char_p, ctypes.c_char_p)
fopen.restype = FILE_p

fclose = libc.fclose
fclose.argtypes = (FILE_p,)

PyRun_SimpleFileExFlags = pythonapi.PyRun_SimpleFileExFlags
PyRun_SimpleFileExFlags.argtypes = (FILE_p, ctypes.c_char_p, ctypes.c_int, ctypes.c_void_p)
PyRun_SimpleFileExFlags.restype = ctypes.c_int

def main():
    filename = 'demo.py'
    with open(filename, 'w') as fp:
        fp.write('import sys = 123')

    fp = fopen(filename.encode(), b"r")

    closeit = 0
    res = PyRun_SimpleFileExFlags(fp, b'abc', closeit, None)
    print(res)

    fclose(fp)

main()

I discovered the bug when writing C code. I converted it to Python to make it easier to reproduce.

gdb traceback:

python: Parser/pegen_errors.c:314: _PyPegen_raise_error_known_location: Assertion `p->tok->fp == NULL || p->tok->fp == stdin || p->tok->done == E_EOF' failed.

Program received signal SIGABRT, Aborted.
0x00007ffff7cffdcc in __pthread_kill_implementation () from /lib64/libc.so.6

(gdb) where
#0  0x00007ffff7cffdcc in __pthread_kill_implementation () from /lib64/libc.so.6
#1  0x00007ffff7ca4f8e in raise () from /lib64/libc.so.6
#2  0x00007ffff7c8c7b3 in abort () from /lib64/libc.so.6
#3  0x00007ffff7c8d804 in __libc_message_impl.cold () from /lib64/libc.so.6
#4  0x00007ffff7c9ce45 in __assert_fail () from /lib64/libc.so.6
#5  0x000000000040440d in _PyPegen_raise_error_known_location (p=0x7fffe9a979f0, 
    errtype=<type at remote 0x9e7740>, lineno=1, col_offset=12, end_lineno=1, end_col_offset=13, 
    errmsg=0x73b0ab "invalid syntax", va=0x7fffffffcaa8) at Parser/pegen_errors.c:314
#6  0x00000000004045b9 in RAISE_ERROR_KNOWN_LOCATION (p=p@entry=0x7fffe9a979f0, errtype=<optimized out>, 
    lineno=<optimized out>, col_offset=<optimized out>, end_lineno=<optimized out>, 
    end_col_offset=<optimized out>, errmsg=0x73b0ab "invalid syntax") at Parser/pegen.h:196
#7  0x0000000000404c90 in _Pypegen_set_syntax_error (p=p@entry=0x7fffe9a979f0, 
    last_token=last_token@entry=0x7fffe96d0ef0) at Parser/pegen_errors.c:404
#8  0x0000000000403d21 in _PyPegen_run_parser (p=p@entry=0x7fffe9a979f0) at Parser/pegen.c:965
#9  0x0000000000403ec1 in _PyPegen_run_parser_from_file_pointer (fp=fp@entry=0xaf55b0, 
    start_rule=start_rule@entry=257, filename_ob=filename_ob@entry='abc', enc=enc@entry=0x0, ps1=ps1@entry=0x0, 
    ps2=ps2@entry=0x0, flags=0x0, errcode=0x0, interactive_src=0x0, arena=0x7fffe96cefe0) at Parser/pegen.c:1032
#10 0x0000000000479815 in _PyParser_ASTFromFile (fp=fp@entry=0xaf55b0, filename_ob=filename_ob@entry='abc', 
    enc=enc@entry=0x0, mode=mode@entry=257, ps1=ps1@entry=0x0, ps2=ps2@entry=0x0, flags=0x0, errcode=0x0, 
    arena=0x7fffe96cefe0) at Parser/peg_api.c:27
#11 0x000000000066f7c7 in pyrun_file (fp=fp@entry=0xaf55b0, filename=filename@entry='abc', start=start@entry=257, 
    globals=globals@entry={'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': <SourceFileLoader(name='__main__', path='abc') at remote 0x7fffe96c5b10>, '__spec__': None, '__builtins__': <module at remote 0x7ffff7c385f0>, '__file__': '/home/vstinner/python/main/bug.py', '_testcapi': <module at remote 0x7fffe9bc3410>, 'ctypes': <module at remote 0x7fffe9a10ef0>, 'filename': 'libc.so.6', 'libc': <CDLL(_FuncPtr=<_ctypes.PyCFuncPtrType(__module__='ctypes', __firstlineno__=410, _flags_=1, _restype_=<_ctypes.PyCSimpleType(__module__='ctypes', __firstlineno__=186, _type_='i', __static_attributes__=(), __dict__=<getset_descriptor at remote 0x7ffff7c57f50>, __weakref__=<getset_descriptor at remote 0x7ffff7c553d0>, __doc__=None, __ctype_be__=<_ctypes.PyCSimpleType(__module__='ctypes', __firstlineno__=186, _type_='i', __static_attributes__=(...), __dict__=<getset_descriptor at remote 0x7ffff7c57f50>, __weakref__=<getset_descriptor at remote 0x7ffff7c553d0>, __doc__=None, __ctype_le__=<...>, __ctype_be__=<...>) at remote 0...(truncated), 
    locals=locals@entry={'__name__': '__main__', '__doc__': None, '__package__': None, '__loader__': <SourceFileLoader(name='__main__', path='abc') at remote 0x7fffe96c5b10>, '__spec__': None, '__builtins__': <module at remote 0x7ffff7c385f0>, '__file__': '/home/vstinner/python/main/bug.py', '_testcapi': <module at remote 0x7fffe9bc3410>, 'ctypes': <module at remote 0x7fffe9a10ef0>, 'filename': 'libc.so.6', 'libc': <CDLL(_FuncPtr=<_ctypes.PyCFuncPtrType(__module__='ctypes', __firstlineno__=410, _flags_=1, _restype_=<_ctypes.PyCSimpleType(__module__='ctypes', __firstlineno__=186, _type_='i', __static_attributes__=(), __dict__=<getset_descriptor at remote 0x7ffff7c57f50>, __weakref__=<getset_descriptor at remote 0x7ffff7c553d0>, __doc__=None, __ctype_be__=<_ctypes.PyCSimpleType(__module__='ctypes', __firstlineno__=186, _type_='i', __static_attributes__=(...), __dict__=<getset_descriptor at remote 0x7ffff7c57f50>, __weakref__=<getset_descriptor at remote 0x7ffff7c553d0>, __doc__=None, __ctype_le__=<...>, __ctype_be__=<...>) at remote 0...(truncated), closeit=closeit@entry=0, flags=0x0) at Python/pythonrun.c:1287
#12 0x0000000000670f32 in _PyRun_SimpleFileObject (fp=fp@entry=0xaf55b0, filename=filename@entry='abc', 
    closeit=closeit@entry=0, flags=flags@entry=0x0) at Python/pythonrun.c:518
#13 0x0000000000671017 in PyRun_SimpleFileExFlags (fp=0xaf55b0, filename=<optimized out>, closeit=0, flags=0x0)

(gdb) frame 5
#5  0x000000000040440d in _PyPegen_raise_error_known_location (...)

(gdb) p p->tok->fp
$5 = (FILE *) 0xaf55b0
(gdb) p p->tok->done   // 10 == E_OK
$6 = 10

Linked PRs

• gh-153833

Bắt đầu với reproducer trong issue và lần theo assertion bị lỗi từ Parser/pegen_errors.c qua Parser/pegen.c, Python/pythonrun.c và PyRun_SimpleFileExFlags. Xác nhận hành vi đối với đầu vào có cú pháp không hợp lệ và đảm bảo reproducer không còn abort trong khi vẫn báo cáo lỗi cú pháp bình thường.