PyQt5 Cannot Test for ToolTip in Headless Mode on Windows

I use GitLab for CI and need to test my PyQt5 GUI in headless mode with pytest-qt (I use python 3.8 on Windows 10). To that end, I can run in headless mode by setting the environment variable QT_QPA_PLATFORM to "offscreen" in my pyproject.toml:

[tool.pytest.ini_options]
env = [
    "D:QT_QPA_PLATFORM=offscreen"
]

and the following test passes when run in windowed mode, but the tooltip test fails in headless mode (regardless of whether I use qtbot.waitUntil or a simple qtbot.wait). How can I make this pass in headless mode?:

tests/test_view.py

def test_tooltip_messages(app: MainApp, qtbot: QtBot) -> None:
    """Test for correct tooltip message when toolbar items are hovered.

    For example, when the user hovers over the 'New' button, the button tooltip should
    read 'New Project'.

    Args:
        app (MainApp): (fixture) Qt main application
        qtbot (QtBot): (fixture) Bot that imitates user interaction
    """
    # Arrange
    window = app.view

    statusbar = window.statusbar
    toolbar = window.toolbar
    new_action = window.new_action
    new_button = toolbar.widgetForAction(new_action)

    # By default, QWidgets only receive mouse move events when at least one mouse button
    # is pressed while the mouse is being moved. Using ``setMouseTracking`` to ``True``
    # should enable all events, but this does not seem to work in headless mode. Use
    # ``mouseMove`` and ``mousePress`` instead.
    #
    # See: https://doc.qt.io/qtforpython-5/PySide2/QtWidgets/QWidget.html#PySide2.QtWidgets.PySide2.QtWidgets.QWidget.setMouseTracking  # pylint: disable=line-too-long
    # -----
    # toolbar.setMouseTracking(True)
    # new_button.setMouseTracking(True)

    new_rect = toolbar.actionGeometry(new_action)
    tooltip = QtWidgets.QToolTip

    # Assert - Precondition
    assert statusbar.currentMessage() == ''
    assert tooltip.text() == ''

    # Act
    qtbot.wait(10)  # In non-headless mode, give time for previous test to finish
    qtbot.mouseMove(new_button)
    qtbot.mousePress(new_button, QtCore.Qt.LeftButton)

    # Assert - Postcondition
    def check_status():
        assert statusbar.currentMessage() == 'Create a new project...'

    def check_tooltip():
        assert tooltip.text() == 'New Project'

    qtbot.waitUntil(check_status)
    qtbot.waitUntil(check_tooltip)

Here is the remaining code for the MRE:

tests/conftest.py

from typing import Generator, Union, Sequence

import pytest
from pytestqt.qtbot import QtBot
from qtpy import QtCore

from myproj.main import MainApp

# Register plugins to use in testing
pyteset_plugins: Union[str, Sequence[str]] = [
    'pytestqt.qtbot',
]

@pytest.fixture(autouse=True)
def clear_settings() -> Generator[None, None, None]:
    """Fixture to clear ``Qt`` settings."""
    yield
    QtCore.QSettings().clear()


@pytest.fixture(name='app')
def fixture_app(qtbot: QtBot) -> Generator[MainApp, None, None]:
    """``pytest`` fixture for ``Qt``.

    Args:
        qtbot (QtBot): pytest fixture for Qt

    Yields:
        Generator[QtBot, None, None]: Generator that yields QtBot fixtures
    """
    # Setup
    root = MainApp()
    root.show()
    qtbot.addWidget(root.view)

    # Run
    yield root

    # Teardown
    # None

myproj/main.py

from pyvistaqt import MainWindow
from qtpy import QtCore, QtGui, QtWidgets
from qtpy.QtCore import Qt
from qtpy.QtWidgets import QApplication

import resources


class View(MainWindow):
    def __init__(
        self,
        controller: 'MainApp',
    ) -> None:
        """Display GUI main window.

        Args:
            controller (): The application controller, in the model-view-controller (MVC)
                framework sense
        """
        super().__init__()
        self.controller = controller

        self.setWindowTitle('My Project')

        self.container = QtWidgets.QFrame()

        self.layout_ = QtWidgets.QVBoxLayout()
        self.layout_.setSpacing(0)
        self.layout_.setContentsMargins(0, 0, 0, 0)

        self.container.setLayout(self.layout_)
        self.setCentralWidget(self.container)

        self._create_actions()
        self._create_menubar()
        self._create_toolbar()
        self._create_statusbar()

    def _create_actions(self) -> None:
        """Create QAction items for menu- and toolbar."""
        self.new_action = QtWidgets.QAction(
            QtGui.QIcon(resources.NEW_ICO),
            '&New Project...',
            self,
        )
        self.new_action.setShortcut('Ctrl+N')
        self.new_action.setStatusTip('Create a new project...')

    def _create_menubar(self) -> None:
        """Create the main menubar."""
        self.menubar = self.menuBar()
        self.file_menu = self.menubar.addMenu('&File')
        self.file_menu.addAction(self.new_action)

    def _create_toolbar(self) -> None:
        """Create the main toolbar."""
        self.toolbar = QtWidgets.QToolBar('Main Toolbar')
        self.toolbar.setIconSize(QtCore.QSize(24, 24))
        self.addToolBar(self.toolbar)
        self.toolbar.addAction(self.new_action)

    def _create_statusbar(self) -> None:
        """Create the main status bar."""
        self.statusbar = QtWidgets.QStatusBar(self)
        self.setStatusBar(self.statusbar)


class MainApp:
    def __init__(self) -> None:
        """GUI controller."""
        self.view = View(controller=self)

    def show(self) -> None:
        """Display the main window."""
        self.view.showMaximized()


if __name__ == '__main__':
    app = QApplication([])
    app.setStyle('fusion')
    app.setAttribute(Qt.AA_DontShowIconsInMenus, True)

    root = MainApp()
    root.show()

    app.exec_()

Results

Here is the error message I receive in headless mode:

PS> pytest
===================================================================================================================== test session starts ====================================================================================================================== 
platform win32 -- Python 3.8.10, pytest-7.1.2, pluggy-1.0.0
PyQt5 5.15.6 -- Qt runtime 5.15.2 -- Qt compiled 5.15.2
Using --randomly-seed=1234
rootdir: C:\Users\hendra11\Code\myproj, configfile: pyproject.toml, testpaths: tests
plugins: hypothesis-6.46.7, cov-3.0.0, doctestplus-0.12.0, env-0.6.2, forked-1.4.0, memprof-0.2.0, qt-4.0.2, randomly-3.12.0, xdist-2.5.0, typeguard-2.13.3
collected 5 items
run-last-failure: no previously failed tests, not deselecting items.

tests\docs_tests\test_index_page.py .                                                                                                                                                                                                                     [ 20%]
tests\test_view.py .F..                                                                                                                                                                                                                                   [100%]

=========================================================================================================================== FAILURES ===========================================================================================================================
____________________________________________________________________________________________________________________ test_tooltip_messages _____________________________________________________________________________________________________________________

self = <pytestqt.qtbot.QtBot object at 0x0000026DB0EFC070>, callback = <function test_tooltip_messages.<locals>.check_tooltip at 0x0000026DB0F16280>

    def waitUntil(self, callback, *, timeout=5000):
        """
        .. versionadded:: 2.0

        Wait in a busy loop, calling the given callback periodically until timeout is reached.    

        ``callback()`` should raise ``AssertionError`` to indicate that the desired condition     
        has not yet been reached, or just return ``None`` when it does. Useful to ``assert`` until
        some condition is satisfied:

        .. code-block:: python

            def view_updated():
                assert view_model.count() > 10


            qtbot.waitUntil(view_updated)

        Another possibility is for ``callback()`` to return ``True`` when the desired condition   
        is met, ``False`` otherwise. Useful specially with ``lambda`` for terser code, but keep   
        in mind that the error message in those cases is usually not very useful because it is    
        not using an ``assert`` expression.

        .. code-block:: python

            qtbot.waitUntil(lambda: view_model.count() > 10)

        Note that this usage only accepts returning actual ``True`` and ``False`` values,
        so returning an empty list to express "falseness" raises a ``ValueError``.

        :param callback: callable that will be called periodically.
        :param timeout: timeout value in ms.
        :raises ValueError: if the return value from the callback is anything other than ``None``,
            ``True`` or ``False``.

        .. note:: This method is also available as ``wait_until`` (pep-8 alias)
        """
        __tracebackhide__ = True
        import time

        start = time.time()

        def timed_out():
            elapsed = time.time() - start
            elapsed_ms = elapsed * 1000
            return elapsed_ms > timeout

        timeout_msg = f"waitUntil timed out in {timeout} milliseconds"

        while True:
            try:
>               result = callback()

.venv\lib\site-packages\pytestqt\qtbot.py:510:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

    def check_tooltip():
>       assert tooltip.text() == 'New Project'
E       AssertionError: assert '' == 'New Project'
E         - New Project

tests\test_view.py:138: AssertionError

The above exception was the direct cause of the following exception:

app = MainApp(), qtbot = <pytestqt.qtbot.QtBot object at 0x0000026DB0EFC070>

    def test_tooltip_messages(app: MainApp, qtbot: QtBot) -> None:
        """Test for correct tooltip message when toolbar items are hovered.

        For example, when the user hovers over the 'New' button, the button tooltip should
        read 'New Project'.

        Args:
            app (MainApp): (fixture) Qt main application
            qtbot (QtBot): (fixture) Bot that imitates user interaction
        """
        # Arrange
        window = app.view

        toolbar = window.toolbar
        new_action = window.new_action

        new_button = toolbar.widgetForAction(new_action)

        # By default, QWidgets only receive mouse move events when at least one mouse button
        # is pressed while the mouse is being moved. Using ``setMouseTracking`` to ``True``
        # should enable all events, but this does not seem to work in headless mode. Use
        # ``mouseMove`` and ``mousePress`` instead.
        #
        # See: https://doc.qt.io/qtforpython-5/PySide2/QtWidgets/QWidget.html#PySide2.QtWidgets.PySide2.QtWidgets.QWidget.setMouseTracking  # pylint: disable=line-too-long
        # -----
        # toolbar.setMouseTracking(True)
        # new_button.setMouseTracking(True)

        qtbot.addWidget(toolbar)
        qtbot.addWidget(new_button)

        tooltip = QtWidgets.QToolTip

        def check_tooltip():
            assert tooltip.text() == 'New Project'

        # Assert - Precondition
        assert tooltip.text() == ''

        # Act
        qtbot.wait(10)  # In non-headless mode, give time for previous test to finish
        qtbot.mouseMove(new_button)
        qtbot.mousePress(new_button, QtCore.Qt.LeftButton)

        # Assert - Postcondition
>       qtbot.waitUntil(check_tooltip)
E       pytestqt.exceptions.TimeoutError: waitUntil timed out in 5000 milliseconds

tests\test_view.py:154: TimeoutError
--------------------------------------------------------------------------------------------------------------------- Captured Qt messages --------------------------------------------------------------------------------------------------------------------- 
QtWarningMsg: This plugin does not support propagateSizeHints()
================================================================================================================= memory consumption estimates ================================================================================================================= 
tests/test_view.py::test_tooltip_messages               - 732.0 KB
tests/test_view.py::test_statusbar_messages             - 464.0 KB
docs_tests::test_index_page.py::test_index[index.html]  - 140.0 KB
tests/test_view.py::test_window_appears                 - 4.0 KB

---------- coverage: platform win32, python 3.8.10-final-0 -----------
Name                          Stmts   Miss Branch BrPart  Cover   Missing
-------------------------------------------------------------------------
docs\source\conf.py              33      0      2      0   100%
myproj\__init__.py                  3      0      0      0   100%
myproj\main.py                     11      1      0      0    91%   15
myproj\model.py                     3      0      2      0   100%
myproj\view.py                     50      4      6      1    88%   14, 94-96
myproj\widgets\__init__.py          1      0      0      0   100%
myproj\widgets\project.py           2      0      2      0   100%
resources\__init__.py             1      0      0      0   100%
resources\icons\__init__.py       4      0      0      0   100%
-------------------------------------------------------------------------
TOTAL                           108      5     12      1    93%
Coverage HTML written to dir logs/coverage/html
Coverage XML written to file logs/coverage/coverage.xml

=================================================================================================================== short test summary info ==================================================================================================================== 
FAILED tests/test_view.py::test_tooltip_messages - pytestqt.exceptions.TimeoutError: waitUntil timed out in 5000 milliseconds
================================================================================================================= 1 failed, 4 passed in 6.73s ==================================================================================================================

Update

I also run a similar test for my menubar items and found if I didn't run this that the test does pass in headless mode. I wonder if there is something wrong with my fixtures...

def test_menubar_statusbar_messages(app: MainApp, qtbot: QtBot) -> None:
    """Test for correct status bar message when a menu item is hovered.

    For example, when the user clicks 'File' in the menubar and hovers over 'New', the
    statusbar tooltip should read 'Create a new project...'.

    Args:
        app (MainApp): (fixture) Qt main application
        qtbot (QtBot): (fixture) Bot that imitates user interaction
    """
    # Arrange
    window = app.view

    menubar = window.menubar
    statusbar = window.statusbar
    file_menu = window.file_menu
    new_action = window.new_action

    file_rect = menubar.actionGeometry(file_menu.menuAction())
    new_rect = file_menu.actionGeometry(new_action)

    # Act
    qtbot.mouseMove(menubar, file_rect.center())
    qtbot.mouseClick(menubar, qt_api.QtCore.Qt.LeftButton, pos=file_rect.center())
    qtbot.mouseMove(file_menu, new_rect.center())
    qtbot.mousePress(file_menu, qt_api.QtCore.Qt.LeftButton, pos=new_rect.center())

    # Assert
    def check_status():
        assert statusbar.currentMessage() == 'Create a new project...'

    qtbot.waitUntil(check_status)

Update

If I attempt to use setMouseTracking(True) and only use mouseMove, I get these warning messages (otherwise, I just get the This plugin does not support propagateSizeHints() warning):

QtWarningMsg: QFontDatabase: Cannot find font directory %USERPROFILE%/code/myproj/.venv/lib/site-packages/PyQt5/Qt5/lib/fonts.
Note that Qt no longer ships fonts. Deploy some (from https://dejavu-fonts.github.io/ for example) or switch to fontconfig.
QtWarningMsg: QFontDatabase: Cannot find font directory %USERPROFILE%/code/myproj/.venv/lib/site-packages/PyQt5/Qt5/lib/fonts.
Note that Qt no longer ships fonts. Deploy some (from https://dejavu-fonts.github.io/ for example) or switch to fontconfig.
QtWarningMsg: QFontDatabase: Cannot find font directory %USERPROFILE%/code/myproj/.venv/lib/site-packages/PyQt5/Qt5/lib/fonts.
Note that Qt no longer ships fonts. Deploy some (from https://dejavu-fonts.github.io/ for example) or switch to fontconfig.
QtWarningMsg: QFontDatabase: Cannot find font directory %USERPROFILE%/code/myproj/.venv/lib/site-packages/PyQt5/Qt5/lib/fonts.
Note that Qt no longer ships fonts. Deploy some (from https://dejavu-fonts.github.io/ for example) or switch to fontconfig.
QtWarningMsg: QFontDatabase: Cannot find font directory %USERPROFILE%/code/myproj/.venv/lib/site-packages/PyQt5/Qt5/lib/fonts.
Note that Qt no longer ships fonts. Deploy some (from https://dejavu-fonts.github.io/ for example) or switch to fontconfig.
QtWarningMsg: This plugin does not support propagateSizeHints()
QtWarningMsg: This plugin does not support raise()
QtWarningMsg: This plugin does not support grabbing the keyboard
QtWarningMsg: QFont::setPointSizeF: Point size <= 0 (-0.720000), must be greater than 0

Also, I seem to get a segfault:

tests\docs_tests\test_index_page.py .                                                                                                                                                                                                                     [ 20%]
tests\test_view.py Windows fatal exception: code 0x8001010d

Current thread 0x00001de4 (most recent call first):
  File "%USERPROFILE%\Code\myproj\.venv\lib\site-packages\pytestqt\plugin.py", line 182 in _process_events     
  File "%USERPROFILE%\Code\myproj\.venv\lib\site-packages\pytestqt\plugin.py", line 142 in pytest_runtest_setup
  File "%USERPROFILE%\code\myproj\.venv\lib\site-packages\pluggy\_callers.py", line 55 in _multicall
  File "%USERPROFILE%\code\myproj\.venv\lib\site-packages\pluggy\_manager.py", line 80 in _hookexec
  File "%USERPROFILE%\code\myproj\.venv\lib\site-packages\pluggy\_hooks.py", line 265 in __call__
  File "%USERPROFILE%\code\myproj\.venv\lib\site-packages\_pytest\runner.py", line 259 in <lambda>
  File "%USERPROFILE%\code\myproj\.venv\lib\site-packages\_pytest\runner.py", line 338 in from_call        
  File "%USERPROFILE%\code\myproj\.venv\lib\site-packages\_pytest\runner.py", line 258 in call_runtest_hook
  File "%USERPROFILE%\code\myproj\.venv\lib\site-packages\_pytest\runner.py", line 219 in call_and_report
  File "%USERPROFILE%\code\myproj\.venv\lib\site-packages\_pytest\runner.py", line 124 in runtestprotocol
  File "%USERPROFILE%\code\myproj\.venv\lib\site-packages\_pytest\runner.py", line 111 in pytest_runtest_protocol
  File "%USERPROFILE%\code\myproj\.venv\lib\site-packages\pluggy\_callers.py", line 39 in _multicall
  File "%USERPROFILE%\code\myproj\.venv\lib\site-packages\pluggy\_manager.py", line 80 in _hookexec
  File "%USERPROFILE%\code\myproj\.venv\lib\site-packages\pluggy\_hooks.py", line 265 in __call__
  File "%USERPROFILE%\code\myproj\.venv\lib\site-packages\_pytest\main.py", line 347 in pytest_runtestloop
  File "%USERPROFILE%\code\myproj\.venv\lib\site-packages\pluggy\_callers.py", line 39 in _multicall
  File "%USERPROFILE%\code\myproj\.venv\lib\site-packages\pluggy\_manager.py", line 80 in _hookexec
  File "%USERPROFILE%\code\myproj\.venv\lib\site-packages\pluggy\_hooks.py", line 265 in __call__
  File "%USERPROFILE%\code\myproj\.venv\lib\site-packages\_pytest\main.py", line 322 in _main
  File "%USERPROFILE%\code\myproj\.venv\lib\site-packages\_pytest\main.py", line 268 in wrap_session
  File "%USERPROFILE%\code\myproj\.venv\lib\site-packages\_pytest\main.py", line 315 in pytest_cmdline_main
  File "%USERPROFILE%\code\myproj\.venv\lib\site-packages\pluggy\_callers.py", line 39 in _multicall
  File "%USERPROFILE%\code\myproj\.venv\lib\site-packages\pluggy\_manager.py", line 80 in _hookexec
  File "%USERPROFILE%\code\myproj\.venv\lib\site-packages\pluggy\_hooks.py", line 265 in __call__
  File "%USERPROFILE%\code\myproj\.venv\lib\site-packages\_pytest\config\__init__.py", line 164 in main
  File "%USERPROFILE%\code\myproj\.venv\lib\site-packages\_pytest\config\__init__.py", line 187 in console_main
  File "%USERPROFILE%\Code\myproj\.venv\Scripts\pytest.exe\__main__.py", line 7 in <module>
  File "C:\Program Files\Python38\lib\runpy.py", line 87 in _run_code
  File "C:\Program Files\Python38\lib\runpy.py", line 194 in _run_module_as_main
...F

ASIDE:

(I typically suppress these messages with -p no:faulthandler to [tool.pytest.ini_options]; see pytest - Windows fatal exception: code 0x8001010):

This is an effect of a change introduced with pytest 5.0.0. From the release notes:

#5440: The faulthandler standard library module is now enabled by default to help users diagnose crashes in C modules.

This functionality was provided by integrating the external pytest-faulthandler plugin into the core, so users should remove that plugin from their requirements if used.

For more information see the docs: https://docs.pytest.org/en/stable/usage.html#fault-handler

Additional

Further information here on the QtWarningMsg I receive that This plugin does not support propogateSizeHints().

Start by running the failing test in tests/test_view.py with QT_QPA_PLATFORM=offscreen and inspect the widget and action setup in myproj/main.py. Trace how pytest-qt's QtBot mouse events interact with QToolTip in headless mode, using the existing tooltip assertion as the acceptance check. Done means the test observes “New Project” without requiring windowed mode.