Clean up output from client when master dies/goes offline
- Dominant language
- Python
- Stars
- 184
- Forks
- 41
- PR merge metrics
- No merged PRs in 30d
Description
Here is an example of what currently gets output when the master is killed while the client is polling a build:
```
Build is running. (Build id: 1)
Polling build status url: http://jharrington:43000/v1/build/1
0 of 4 subjobs are complete (0.0%).
1 of 4 subjobs are complete (25.0%).
2 of 4 subjobs are complete (50.0%).
Unhandled exception handler caught exception.
Traceback (most recent call last):
File "/Users/jharrington/.virtualenvs/clusterrunner/lib/python3.4/site-packages/requests/packages/urllib3/connectionpool.py", line 493, in urlopen
body=body, headers=headers)
File "/Users/jharrington/.virtualenvs/clusterrunner/lib/python3.4/site-packages/requests/packages/urllib3/connectionpool.py", line 291, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/local/Cellar/python3/3.4.2/Frameworks/Python.framework/Versions/3.4/lib/python3.4/http/client.py", line 1090, in request
self._send_request(method, url, body, headers)
File "/usr/local/Cellar/python3/3.4.2/Frameworks/Python.framework/Versions/3.4/lib/python3.4/http/client.py", line 1128, in _send_request
self.endheaders(body)
File "/usr/local/Cellar/python3/3.4.2/Frameworks/Python.framework/Versions/3.4/lib/python3.4/http/client.py", line 1086, in endheaders
self._send_output(message_body)
File "/usr/local/Cellar/python3/3.4.2/Frameworks/Python.framework/Versions/3.4/lib/python3.4/http/client.py", line 924, in _send_output
self.send(msg)
File "/usr/local/Cellar/python3/3.4.2/Frameworks/Python.framework/Versions/3.4/lib/python3.4/http/client.py", line 859, in send
self.connect()
File "/Users/jharrington/.virtualenvs/clusterrunner/lib/python3.4/site-packages/requests/packages/urllib3/connection.py", line 106, in connect
conn = self._new_conn()
File "/Users/jharrington/.virtualenvs/clusterrunner/lib/python3.4/site-packages/requests/packages/urllib3/connection.py", line 90, in _new_conn
(self.host, self.port), self.timeout, *extra_args)
File "/usr/local/Cellar/python3/3.4.2/Frameworks/Python.framework/Versions/3.4/lib/python3.4/socket.py", line 509, in create_connection
raise err
File "/usr/local/Cellar/python3/3.4.2/Frameworks/Python.framework/Versions/3.4/lib/python3.4/socket.py", line 500, in create_connection
sock.connect(sa)
ConnectionRefusedError: [Errno 61] Connection refused
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/jharrington/.virtualenvs/clusterrunner/lib/python3.4/site-packages/requests/adapters.py", line 327, in send
timeout=timeout
File "/Users/jharrington/.virtualenvs/clusterrunner/lib/python3.4/site-packages/requests/packages/urllib3/connectionpool.py", line 543, in urlopen
raise MaxRetryError(self, url, e)
requests.packages.urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='jharrington', port=43000): Max retries exceeded with url: /v1/build/1 (Caused by : [Errno 61] Connection refused)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "./main.py", line 265, in main
subcommand_class().run(**parsed_args)
File "/Users/jharrington/code/ClusterRunner/app/subcommands/build_subcommand.py", line 51, in run
if not runner.run():
File "/Users/jharrington/code/ClusterRunner/app/client/build_runner.py", line 58, in run
result = self._block_until_finished()
File "/Users/jharrington/code/ClusterRunner/app/client/build_runner.py", line 106, in _block_until_finished
response = self._network.get(build_status_url)
File "/Users/jharrington/code/ClusterRunner/app/util/network.py", line 39, in get
return self._request('GET', *args, **kwargs)
File "/Users/jharrington/code/ClusterRunner/app/util/network.py", line 124, in _request
resp = self._session.request(method, url, data=data_to_send, *args, **kwargs)
File "/Users/jharrington/.virtualenvs/clusterrunner/lib/python3.4/site-packages/requests/sessions.py", line 456, in request
resp = self.send(prep, **send_kwargs)
File "/Users/jharrington/.virtualenvs/clusterrunner/lib/python3.4/site-packages/requests/sessions.py", line 559, in send
r = adapter.send(request, **kwargs)
File "/Users/jharrington/.virtualenvs/clusterrunner/lib/python3.4/site-packages/requests/adapters.py", line 375, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPConnectionPool(host='jharrington', port=43000): Max retries exceeded with url: /v1/build/1 (Caused by : [Errno 61] Connection refused)
Executing teardown callback: >
Cancelling build 1
Call to Network.put raised ConnectionError("HTTPConnectionPool(host='jharrington', port=43000): Max retries exceeded with url: /v1/build/1 (Caused by : [Errno 61] Connection refused)"). Retrying in 0.1 seconds.
Call to Network.put raised ConnectionError("HTTPConnectionPool(host='jharrington', port=43000): Max retries exceeded with url: /v1/build/1 (Caused by : [Errno 61] Connection refused)"). Retrying in 0.2 seconds.
Call to Network.put raised ConnectionError("HTTPConnectionPool(host='jharrington', port=43000): Max retries exceeded with url: /v1/build/1 (Caused by : [Errno 61] Connection refused)"). Retrying in 0.4 seconds.
Call to Network.put raised ConnectionError("HTTPConnectionPool(host='jharrington', port=43000): Max retries exceeded with url: /v1/build/1 (Caused by : [Errno 61] Connection refused)"). Retrying in 0.8 seconds.
Call to Network.put raised ConnectionError("HTTPConnectionPool(host='jharrington', port=43000): Max retries exceeded with url: /v1/build/1 (Caused by : [Errno 61] Connection refused)"). Retrying in 1.6 seconds.
^CSIGINT signal received. Triggering teardown.
Exception raised by teardown callback >
Traceback (most recent call last):
File "/Users/jharrington/.virtualenvs/clusterrunner/lib/python3.4/site-packages/requests/packages/urllib3/connectionpool.py", line 493, in urlopen
body=body, headers=headers)
File "/Users/jharrington/.virtualenvs/clusterrunner/lib/python3.4/site-packages/requests/packages/urllib3/connectionpool.py", line 291, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/local/Cellar/python3/3.4.2/Frameworks/Python.framework/Versions/3.4/lib/python3.4/http/client.py", line 1090, in request
self._send_request(method, url, body, headers)
File "/usr/local/Cellar/python3/3.4.2/Frameworks/Python.framework/Versions/3.4/lib/python3.4/http/client.py", line 1128, in _send_request
self.endheaders(body)
File "/usr/local/Cellar/python3/3.4.2/Frameworks/Python.framework/Versions/3.4/lib/python3.4/http/client.py", line 1086, in endheaders
self._send_output(message_body)
File "/usr/local/Cellar/python3/3.4.2/Frameworks/Python.framework/Versions/3.4/lib/python3.4/http/client.py", line 924, in _send_output
self.send(msg)
File "/usr/local/Cellar/python3/3.4.2/Frameworks/Python.framework/Versions/3.4/lib/python3.4/http/client.py", line 859, in send
self.connect()
File "/Users/jharrington/.virtualenvs/clusterrunner/lib/python3.4/site-packages/requests/packages/urllib3/connection.py", line 106, in connect
conn = self._new_conn()
File "/Users/jharrington/.virtualenvs/clusterrunner/lib/python3.4/site-packages/requests/packages/urllib3/connection.py", line 90, in _new_conn
(self.host, self.port), self.timeout, *extra_args)
File "/usr/local/Cellar/python3/3.4.2/Frameworks/Python.framework/Versions/3.4/lib/python3.4/socket.py", line 509, in create_connection
raise err
File "/usr/local/Cellar/python3/3.4.2/Frameworks/Python.framework/Versions/3.4/lib/python3.4/socket.py", line 500, in create_connection
sock.connect(sa)
ConnectionRefusedError: [Errno 61] Connection refused
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/jharrington/.virtualenvs/clusterrunner/lib/python3.4/site-packages/requests/adapters.py", line 327, in send
timeout=timeout
File "/Users/jharrington/.virtualenvs/clusterrunner/lib/python3.4/site-packages/requests/packages/urllib3/connectionpool.py", line 543, in urlopen
raise MaxRetryError(self, url, e)
requests.packages.urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='jharrington', port=43000): Max retries exceeded with url: /v1/build/1 (Caused by : [Errno 61] Connection refused)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "./main.py", line 265, in main
subcommand_class().run(**parsed_args)
File "/Users/jharrington/code/ClusterRunner/app/subcommands/build_subcommand.py", line 51, in run
if not runner.run():
File "/Users/jharrington/code/ClusterRunner/app/client/build_runner.py", line 58, in run
result = self._block_until_finished()
File "/Users/jharrington/code/ClusterRunner/app/client/build_runner.py", line 106, in _block_until_finished
response = self._network.get(build_status_url)
File "/Users/jharrington/code/ClusterRunner/app/util/network.py", line 39, in get
return self._request('GET', *args, **kwargs)
File "/Users/jharrington/code/ClusterRunner/app/util/network.py", line 124, in _request
resp = self._session.request(method, url, data=data_to_send, *args, **kwargs)
File "/Users/jharrington/.virtualenvs/clusterrunner/lib/python3.4/site-packages/requests/sessions.py", line 456, in request
resp = self.send(prep, **send_kwargs)
File "/Users/jharrington/.virtualenvs/clusterrunner/lib/python3.4/site-packages/requests/sessions.py", line 559, in send
r = adapter.send(request, **kwargs)
File "/Users/jharrington/.virtualenvs/clusterrunner/lib/python3.4/site-packages/requests/adapters.py", line 375, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPConnectionPool(host='jharrington', port=43000): Max retries exceeded with url: /v1/build/1 (Caused by : [Errno 61] Connection refused)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/jharrington/.virtualenvs/clusterrunner/lib/python3.4/site-packages/requests/packages/urllib3/connectionpool.py", line 493, in urlopen
body=body, headers=headers)
File "/Users/jharrington/.virtualenvs/clusterrunner/lib/python3.4/site-packages/requests/packages/urllib3/connectionpool.py", line 291, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/local/Cellar/python3/3.4.2/Frameworks/Python.framework/Versions/3.4/lib/python3.4/http/client.py", line 1090, in request
self._send_request(method, url, body, headers)
File "/usr/local/Cellar/python3/3.4.2/Frameworks/Python.framework/Versions/3.4/lib/python3.4/http/client.py", line 1128, in _send_request
self.endheaders(body)
File "/usr/local/Cellar/python3/3.4.2/Frameworks/Python.framework/Versions/3.4/lib/python3.4/http/client.py", line 1086, in endheaders
self._send_output(message_body)
File "/usr/local/Cellar/python3/3.4.2/Frameworks/Python.framework/Versions/3.4/lib/python3.4/http/client.py", line 924, in _send_output
self.send(msg)
File "/usr/local/Cellar/python3/3.4.2/Frameworks/Python.framework/Versions/3.4/lib/python3.4/http/client.py", line 859, in send
self.connect()
File "/Users/jharrington/.virtualenvs/clusterrunner/lib/python3.4/site-packages/requests/packages/urllib3/connection.py", line 106, in connect
conn = self._new_conn()
File "/Users/jharrington/.virtualenvs/clusterrunner/lib/python3.4/site-packages/requests/packages/urllib3/connection.py", line 90, in _new_conn
(self.host, self.port), self.timeout, *extra_args)
File "/usr/local/Cellar/python3/3.4.2/Frameworks/Python.framework/Versions/3.4/lib/python3.4/socket.py", line 509, in create_connection
raise err
File "/usr/local/Cellar/python3/3.4.2/Frameworks/Python.framework/Versions/3.4/lib/python3.4/socket.py", line 500, in create_connection
sock.connect(sa)
ConnectionRefusedError: [Errno 61] Connection refused
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/jharrington/.virtualenvs/clusterrunner/lib/python3.4/site-packages/requests/adapters.py", line 327, in send
timeout=timeout
File "/Users/jharrington/.virtualenvs/clusterrunner/lib/python3.4/site-packages/requests/packages/urllib3/connectionpool.py", line 543, in urlopen
raise MaxRetryError(self, url, e)
requests.packages.urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='jharrington', port=43000): Max retries exceeded with url: /v1/build/1 (Caused by : [Errno 61] Connection refused)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/jharrington/code/ClusterRunner/app/util/decorators.py", line 38, in function_with_retries
return function(*args, **kwargs)
File "/Users/jharrington/code/ClusterRunner/app/util/network.py", line 73, in put
return self._request('PUT', *args, **kwargs)
File "/Users/jharrington/code/ClusterRunner/app/util/network.py", line 124, in _request
resp = self._session.request(method, url, data=data_to_send, *args, **kwargs)
File "/Users/jharrington/.virtualenvs/clusterrunner/lib/python3.4/site-packages/requests/sessions.py", line 456, in request
resp = self.send(prep, **send_kwargs)
File "/Users/jharrington/.virtualenvs/clusterrunner/lib/python3.4/site-packages/requests/sessions.py", line 559, in send
r = adapter.send(request, **kwargs)
File "/Users/jharrington/.virtualenvs/clusterrunner/lib/python3.4/site-packages/requests/adapters.py", line 375, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPConnectionPool(host='jharrington', port=43000): Max retries exceeded with url: /v1/build/1 (Caused by : [Errno 61] Connection refused)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/jharrington/code/ClusterRunner/app/util/unhandled_exception_handler.py", line 146, in __exit__
callback(*args, **kwargs)
File "/Users/jharrington/code/ClusterRunner/app/client/build_runner.py", line 74, in _cancel_build
self._cluster_master_api_client.cancel_build(self._build_id)
File "/Users/jharrington/code/ClusterRunner/app/client/cluster_api_client.py", line 58, in cancel_build
error_on_failure=True
File "/Users/jharrington/code/ClusterRunner/app/util/network.py", line 85, in put_with_digest
error_on_failure=error_on_failure)
File "/Users/jharrington/code/ClusterRunner/app/util/decorators.py", line 44, in function_with_retries
time.sleep(delay)
File "/Users/jharrington/code/ClusterRunner/app/util/unhandled_exception_handler.py", line 83, in _application_teardown_signal_handler
raise AppTeardown
app.util.unhandled_exception_handler.AppTeardown
```
Contributor guide
No contributing guide indexed for this repository
Assessment
This issue has not been assessed yet.