Grammar lexer

NB : I don't need this bug fixed. It is just funny so I report it.

Reading the example from this package being a nice citizen of internet doing its stackoverflow like routine coding follwing https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/examples/prompts/regular-language.py

I stumbled on a funny bug, which I suspect comes from the regexp used ... to parse the regexp.

https://github.com/prompt-toolkit/python-prompt-toolkit/blob/3cec4c97a98f7777bdaeb14c7a11ae214df1fe0d/src/prompt_toolkit/contrib/regular_languages/regex_parser.py#L150

Here is the code

from prompt_toolkit.contrib.regular_languages.lexer import GrammarLexer
from prompt_toolkit.lexers import SimpleLexer
from prompt_toolkit.contrib.regular_languages.compiler import compile


grammar = compile(r'''(?P<STR>("[^"\\]*(?:\\[\S\s][^"\\]*)*"|'[^'\\]*(?:\\[\S\s][^'\\]*)*'):([a-z]|_)?(?:[a-z0-9_]*))|(?P<NUM>[+-]?((?=\d*[.eE])(?=\.?\d)\d*\.?\d*(?:[eE][+-]?\d+)?|\d+):([a-z]|_)?(?:[a-z0-9_]*))|(?P<OP>MUL|DIV|ADD|SUB|>NUM|>STR|CAT|XOR|OR|AND|CMP|IN|NOT|TAG|IFT|MATCH|ROT|DUP|TOP|DROP|SWAP|OVER|NOP|GET|LEN|EJOIN|EDICT|EVAL|<<TERM>>)|(?P<VAR>\$\S+)''')

lexer = GrammarLexer(
        grammar,
        lexers={
            'STR': SimpleLexer('class:var'),
            'NUM':   SimpleLexer('class:var'),
            'OP':   SimpleLexer('class:operator'),
            'VAR':   SimpleLexer('class:class'),
            })

Actual result

  File "/home/jul/src/confined/iconf.py", line 25, in <module>
    grammar = compile(r'''(?P<STR>("[^"\\]*(?:\\[\S\s][^"\\]*)*"|'[^'\\]*(?:\\[\S\s][^'\\]*)*'):([a-z]|_)?(?:[a-z0-9_]*))|(?P<NUM>[+-]?((?=\d*[.eE])(?=\.?\d)\d*\.?\d*(?:[eE][+-]?\d+)?|\d+):([a-z]|_)?(?:[a-z0-9_]*))|(?P<OP>MUL|DIV|ADD|SUB|>NUM|>STR|CAT|XOR|OR|AND|CMP|IN|NOT|TAG|IFT|MATCH|ROT|DUP|TOP|DROP|SWAP|OVER|NOP|GET|LEN|EJOIN|EDICT|EVAL|<<TERM>>)|(?P<VAR>\$\S+)''')
  File "/home/jul/venv/p3/lib/python3.9/site-packages/prompt_toolkit/contrib/regular_languages/compiler.py", line 557, in compile
    parse_regex(tokenize_regex(expression)),
  File "/home/jul/venv/p3/lib/python3.9/site-packages/prompt_toolkit/contrib/regular_languages/regex_parser.py", line 276, in parse_regex
    result = _parse()
  File "/home/jul/venv/p3/lib/python3.9/site-packages/prompt_toolkit/contrib/regular_languages/regex_parser.py", line 223, in _parse
    variable = Variable(_parse(), varname=t[4:-1])
  File "/home/jul/venv/p3/lib/python3.9/site-packages/prompt_toolkit/contrib/regular_languages/regex_parser.py", line 248, in _parse
    result.append(_parse())
  File "/home/jul/venv/p3/lib/python3.9/site-packages/prompt_toolkit/contrib/regular_languages/regex_parser.py", line 248, in _parse
    result.append(_parse())
  File "/home/jul/venv/p3/lib/python3.9/site-packages/prompt_toolkit/contrib/regular_languages/regex_parser.py", line 236, in _parse
    raise Exception("Nothing to repeat." + repr(tokens))
Exception: Nothing to repeat.[')', ')', '+', '\\S', '\\$', '(?P<VAR>', '|', ')', '>', '>', 'M', 'R', 'E', 'T', '<', '<', '|', 'L', 'A', 'V', 'E', '|', 'T', 'C', 'I', 'D', 'E', '|', 'N', 'I', 'O', 'J', 'E', '|', 'N', 'E', 'L', '|', 'T', 'E', 'G', '|', 'P', 'O', 'N', '|', 'R', 'E', 'V', 'O', '|', 'P', 'A', 'W', 'S', '|', 'P', 'O', 'R', 'D', '|', 'P', 'O', 'T', '|', 'P', 'U', 'D', '|', 'T', 'O', 'R', '|', 'H', 'C', 'T', 'A', 'M', '|', 'T', 'F', 'I', '|', 'G', 'A', 'T', '|', 'T', 'O', 'N', '|', 'N', 'I', '|', 'P', 'M', 'C', '|', 'D', 'N', 'A', '|', 'R', 'O', '|', 'R', 'O', 'X', '|', 'T', 'A', 'C', '|', 'R', 'T', 'S', '>', '|', 'M', 'U', 'N', '>', '|', 'B', 'U', 'S', '|', 'D', 'D', 'A', '|', 'V', 'I', 'D', '|', 'L', 'U', 'M', '(?P<OP>', '|', ')', ')', '*', '[a-z0-9_]', ':', '?', '(', '?', ')', '_', '|', '[a-z]', '(', ':', ')', '+', '\\d', '|', '?', ')', '+', '\\d', '?', '[+-]', '[eE]', ':', '?', '(', '*', '\\d', '?', '\\.', '*', '\\d', ')', '\\d', '?', '\\.', '=', '?', '(', ')', '[.eE]', '*', '\\d', '=', '?', '(', '(', '?', '[+-]', '(?P<NUM>', '|', ')', ')', '*', '[a-z0-9_]', ':', '?', '(', '?', ')', '_', '|', '[a-z]', '(', ':', ')', "'", '*', ')', '*', "[^'\\\\]", '[\\S\\s]', '\\\\', ':', '?', '(', '*', "[^'\\\\]", "'", '|', '"', '*', ')', '*', '[^"\\\\]', '[\\S\\s]', '\\\\', ':']

Start with examples/prompts/regular-language.py and run the supplied reproducer. Then inspect src/prompt_toolkit/contrib/regular_languages/regex_parser.py around line 150 and the compile entry point in compiler.py to trace the reported parsing failure. Done means determining whether the behavior should be fixed or documented, with the result captured in a regression test or a clear explanation.