dask / dask/distributed

test_pid_file failure

Open
#3,734 0 comments 0 reactions 0 assignees View on GitHub
flaky test
Dominant language
Python
Stars
1.7k
Forks
778
Avg merge
2h 50m
Merged PRs (30d)
3

Description

I've started seeing `distributed/cli/tests/test_dask_scheduler.py::test_pid_file` sporadically fail in CI builds (xref https://travis-ci.org/github/dask/distributed/jobs/677794876#L1468-L1549)

Traceback:

```
________________________________ test_pid_file _________________________________

loop =

def test_pid_file(loop):

def check_pidfile(proc, pidfile):

start = time()

while not os.path.exists(pidfile):

sleep(0.01)

assert time() < start + 5

text = False

start = time()

while not text:

sleep(0.01)

assert time() < start + 5

with open(pidfile) as f:

text = f.read()

pid = int(text)

if sys.platform.startswith("win"):

# On Windows, `dask-XXX` invokes the dask-XXX.exe

# shim, but the PID is written out by the child Python process

assert pid

else:

assert proc.pid == pid

with tmpfile() as s:

with popen(["dask-scheduler", "--pid-file", s, "--no-dashboard"]) as sched:

> check_pidfile(sched, s)

distributed/cli/tests/test_dask_scheduler.py:207:

_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

../../../miniconda/envs/dask-distributed/lib/python3.6/contextlib.py:88: in __exit__

next(self.gen)

distributed/utils_test.py:1041: in popen

terminate_process(proc)

distributed/utils_test.py:1005: in terminate_process

proc.wait(10)

_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = , timeout = 10

endtime = 190.384243671

def wait(self, timeout=None, endtime=None):

"""Wait for child process to terminate. Returns returncode

attribute."""

if self.returncode is not None:

return self.returncode

if endtime is not None:

warnings.warn(

"'endtime' argument is deprecated; use 'timeout3m'.",

DeprecationWarning,

stacklevel=2)

if endtime is not None or timeout is not None:

if endtime is None:

endtime = _time() + timeout

elif timeout is None:

timeout = self._remaining_time(endtime)

if endtime is not None:

# Enter a busy loop if we have a timeout. This busy loop was

# cribbed from Lib/threading.py in Thread.wait() at r71065.

delay = 0.0005 # 500 us -> initial delay of 1 ms

while True:

if self._waitpid_lock.acquire(False):

try:

if self.returncode is not None:

break # Another thread waited.

(pid, sts) = self._try_wait(os.WNOHANG)

assert pid == self.pid or pid == 0

if pid == self.pid:

self._handle_exitstatus(sts)

break

finally:

self._waitpid_lock.release()

remaining = self._remaining_time(endtime)

if remaining <= 0:

> raise TimeoutExpired(self.args, timeout)

E subprocess.TimeoutExpired: Command '['/home/travis/miniconda/envs/dask-distributed/bin/dask-scheduler', '--pid-file', '/tmp/tmp3afymwkx.', '--no-dashboard']' timed out after 10 seconds

../../../miniconda/envs/dask-distributed/lib/python3.6/subprocess.py:1469: TimeoutExpired
```

Contributor guide

Open the contributing guide

Assessment

This issue has not been assessed yet.

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.