test_pid_file failure
- Dominant language
- Python
- Stars
- 1.7k
- Forks
- 778
- Avg merge
- 2h 50m
- Merged PRs (30d)
- 3
Description
I've started seeing `distributed/cli/tests/test_dask_scheduler.py::test_pid_file` sporadically fail in CI builds (xref https://travis-ci.org/github/dask/distributed/jobs/677794876#L1468-L1549)
Traceback:
```
________________________________ test_pid_file _________________________________
loop =
def test_pid_file(loop):
def check_pidfile(proc, pidfile):
start = time()
while not os.path.exists(pidfile):
sleep(0.01)
assert time() < start + 5
text = False
start = time()
while not text:
sleep(0.01)
assert time() < start + 5
with open(pidfile) as f:
text = f.read()
pid = int(text)
if sys.platform.startswith("win"):
# On Windows, `dask-XXX` invokes the dask-XXX.exe
# shim, but the PID is written out by the child Python process
assert pid
else:
assert proc.pid == pid
with tmpfile() as s:
with popen(["dask-scheduler", "--pid-file", s, "--no-dashboard"]) as sched:
> check_pidfile(sched, s)
distributed/cli/tests/test_dask_scheduler.py:207:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../miniconda/envs/dask-distributed/lib/python3.6/contextlib.py:88: in __exit__
next(self.gen)
distributed/utils_test.py:1041: in popen
terminate_process(proc)
distributed/utils_test.py:1005: in terminate_process
proc.wait(10)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = , timeout = 10
endtime = 190.384243671
def wait(self, timeout=None, endtime=None):
"""Wait for child process to terminate. Returns returncode
attribute."""
if self.returncode is not None:
return self.returncode
if endtime is not None:
warnings.warn(
"'endtime' argument is deprecated; use 'timeout3m'.",
DeprecationWarning,
stacklevel=2)
if endtime is not None or timeout is not None:
if endtime is None:
endtime = _time() + timeout
elif timeout is None:
timeout = self._remaining_time(endtime)
if endtime is not None:
# Enter a busy loop if we have a timeout. This busy loop was
# cribbed from Lib/threading.py in Thread.wait() at r71065.
delay = 0.0005 # 500 us -> initial delay of 1 ms
while True:
if self._waitpid_lock.acquire(False):
try:
if self.returncode is not None:
break # Another thread waited.
(pid, sts) = self._try_wait(os.WNOHANG)
assert pid == self.pid or pid == 0
if pid == self.pid:
self._handle_exitstatus(sts)
break
finally:
self._waitpid_lock.release()
remaining = self._remaining_time(endtime)
if remaining <= 0:
> raise TimeoutExpired(self.args, timeout)
E subprocess.TimeoutExpired: Command '['/home/travis/miniconda/envs/dask-distributed/bin/dask-scheduler', '--pid-file', '/tmp/tmp3afymwkx.', '--no-dashboard']' timed out after 10 seconds
../../../miniconda/envs/dask-distributed/lib/python3.6/subprocess.py:1469: TimeoutExpired
```
Contributor guide
Assessment
This issue has not been assessed yet.