[exception] [communication] mqtt connection timeout error
- Dominant language
- Python
- Stars
- 4.1k
- Forks
- 765
- PR merge metrics
- No merged PRs in 30d
Description
File "C:\Users\chaoy\anaconda3\envs\fedml\lib\site-packages\fedml\core\distributed\communication\mqtt\mqtt_manager.py", line 134, in on_message
passthrough_listener(msg)
File "C:\Users\chaoy\anaconda3\envs\fedml\lib\site-packages\fedml\core\distributed\communication\mqtt_s3\mqtt_s3_multi_clients_comm_manager.py", line 268, in _on_message
self._on_message_impl(msg)
File "C:\Users\chaoy\anaconda3\envs\fedml\lib\site-packages\fedml\core\distributed\communication\mqtt_s3\mqtt_s3_multi_clients_comm_manager.py", line 265, in _on_message_impl
self._notify(payload_obj)
File "C:\Users\chaoy\anaconda3\envs\fedml\lib\site-packages\fedml\core\distributed\communication\mqtt_s3\mqtt_s3_multi_clients_comm_manager.py", line 217, in _notify
observer.receive_message(msg_type, msg_params)
File "C:\Users\chaoy\anaconda3\envs\fedml\lib\site-packages\fedml\core\distributed\fedml_comm_manager.py", line 45, in receive_message
handler_callback_func(msg_params)
File "C:\Users\chaoy\anaconda3\envs\fedml\lib\site-packages\fedml\cross_silo\client\fedml_client_master_manager.py", line 78, in handle_message_init
self.__train()
File "C:\Users\chaoy\anaconda3\envs\fedml\lib\site-packages\fedml\cross_silo\client\fedml_client_master_manager.py", line 150, in __train
mlops.event("train", event_started=False, event_value=str(self.round_idx))
File "C:\Users\chaoy\anaconda3\envs\fedml\lib\site-packages\fedml\mlops\__init__.py", line 13, in event
mlops.event(event_name, event_started, event_value, event_edge_id)
File "C:\Users\chaoy\anaconda3\envs\fedml\lib\site-packages\fedml\core\mlops\__init__.py", line 151, in event
MLOpsStore.mlops_event.log_event_ended(event_name, event_value, event_edge_id)
File "C:\Users\chaoy\anaconda3\envs\fedml\lib\site-packages\fedml\core\mlops\mlops_profiler_event.py", line 106, in log_event_ended
self.com_manager.send_message_json(event_topic, event_msg_str)
File "C:\Users\chaoy\anaconda3\envs\fedml\lib\site-packages\fedml\core\distributed\communication\mqtt\mqtt_manager.py", line 110, in send_message_json
auth={'username': self.user, 'password': self.pwd})
File "C:\Users\chaoy\anaconda3\envs\fedml\lib\site-packages\paho\mqtt\publish.py", line 241, in single
protocol, transport, proxy_args)
File "C:\Users\chaoy\anaconda3\envs\fedml\lib\site-packages\paho\mqtt\publish.py", line 176, in multiple
client.connect(hostname, port, keepalive)
File "C:\Users\chaoy\anaconda3\envs\fedml\lib\site-packages\paho\mqtt\client.py", line 914, in connect
return self.reconnect()
File "C:\Users\chaoy\anaconda3\envs\fedml\lib\site-packages\paho\mqtt\client.py", line 1044, in reconnect
sock = self._create_socket_connection()
File "C:\Users\chaoy\anaconda3\envs\fedml\lib\site-packages\paho\mqtt\client.py", line 3685, in _create_socket_connection
return socket.create_connection(addr, timeout=self._connect_timeout, source_address=source)
File "C:\Users\chaoy\anaconda3\envs\fedml\lib\socket.py", line 728, in create_connection
raise err
File "C:\Users\chaoy\anaconda3\envs\fedml\lib\socket.py", line 716, in create_connection
sock.connect(sa)
socket.timeout: timed out
[FedML-Client @device-id-11226] [Tue, 14 Feb 2023 21:08:41] [INFO] [__init__.py:307:log_training_failed_status] log training inner status FAILED
log training inner status FAILED
[FedML-Client @device-id-11226] [Tue, 14 Feb 2023 21:08:41] [INFO] [mlops_metrics.py:158:common_broadcast_client_training_status] report_client_training_status. message_json = {"edge_id": 11226, "run_id": "5177", "status": "FAILED"}
report_client_training_status. message_json = {"edge_id": 11226, "run_id": "5177", "status": "FAILED"}
Contributor guide
Assessment
This issue has not been assessed yet.