eclipse-iceoryx / eclipse-iceoryx/iceoryx2
Notification Resource Exhaustion in iceoryx2 0.5.0
- Dominant language
- Rust
- Stars
- 2.5k
- Forks
- 185
- Avg merge
- 1d 23h
- Merged PRs (30d)
- 47
Description
Description
When publishing notifications at very high frequency, iceoryx2 v0.5.0 experiences notification resource exhaustion, resulting in "FailedToDeliverSignal" errors.
Steps to Reproduce
1. Modify iceoryx2-0.5.0/examples/cxx/event_based_communication/src/publisher.cpp and change the cycle time to nanosecond level:constexpr iox::units::Duration CYCLE_TIME = iox::units::Duration::fromNanoseconds(1);
2. Run subscriber and publisher:
./example_cxx_event_based_communication_subscriber
./example_cxx_event_based_communication_publisher
Expected Behavior
The system should run without any warn or error.
Actual Behavior
The system allows unlimited notification sending until resources are exhausted, then fails with errors:
[W] Notifier { listener_connections: ListenerConnections {
connections: [UnsafeCell { .. }, UnsafeCell { .. }, Uns
afeCell { .. }, UnsafeCell { .. }, UnsafeCell { .. }, U
nsafeCell { .. }, UnsafeCell { .. }, UnsafeCell { .. },
UnsafeCell { .. }, UnsafeCell { .. }, UnsafeCell { ..
}, UnsafeCell { .. }, UnsafeCell { .. }, UnsafeCell { .
. }, UnsafeCell { .. }, UnsafeCell { .. }], service_sta
te: ServiceState { static_config: StaticConfig { servic
e_id: ServiceId(RestrictedFileName { value: FixedSizeBy
teString<64> { len: 40, data: "b61bd15e8c3ea16146985e96
0906a8e125156a73" } }), service_name: ServiceName { val
ue: "My/Funk/ServiceName" }, attributes: AttributeSet([
]), messaging_pattern: Event(StaticConfig { max_notifie
rs: 16, max_listeners: 16, max_nodes: 36, event_id_max_
value: 4294967295, notifier_created_event: None, notifi
er_dropped_event: None, notifier_dead_event: None }) },
shared_node: SharedNode { id: NodeId(UniqueSystemId {
value: 1960422421266913882581929908, pid: 2002868, crea
tion_time: Time { clock_type: Realtime, seconds: 174523
4363, nanoseconds: 106274712 } }), details: NodeDetails
{ executable: FileName { value: FixedSizeByteString<25
5> { len: 47, data: "example_cxx_event_based_communicat
ion_publisher" } }, name: NodeName { value: "" }, confi
g: Config { global: Global { root_path_unix: Path { val
ue: FixedSizeByteString<255> { len: 14, data: "/tmp/ice
oryx2/" } }, root_path_windows: Path { value: FixedSize
ByteString<255> { len: 17, data: "c:\Temp\iceoryx2\" }
}, prefix: FileName { value: FixedSizeByteString<255> {
len: 5, data: "iox2_" } }, service: Service { director
y: Path { value: FixedSizeByteString<255> { len: 8, dat
a: "services" } }, publisher_data_segment_suffix: FileN
ame { value: FixedSizeByteString<255> { len: 15, data:
".publisher_data" } }, static_config_storage_suffix: Fi
leName { value: FixedSizeByteString<255> { len: 8, data
: ".service" } }, dynamic_config_storage_suffix: FileNa
me { value: FixedSizeByteString<255> { len: 8, data: ".
dynamic" } }, creation_timeout: 500ms, connection_suffi
x: FileName { value: FixedSizeByteString<255> { len: 11
, data: ".connection" } }, event_connection_suffix: Fil
eName { value: FixedSizeByteString<255> { len: 6, data:
".event" } } }, node: Node { directory: Path { value:
FixedSizeByteString<255> { len: 5, data: "nodes" } }, m
onitor_suffix: FileName { value: FixedSizeByteString<25
5> { len: 13, data: ".node_monitor" } }, static_config_
suffix: FileName { value: FixedSizeByteString<255> { le
n: 8, data: ".details" } }, service_tag_suffix: FileNam
e { value: FixedSizeByteString<255> { len: 12, data: ".
service_tag" } }, cleanup_dead_nodes_on_creation: true,
cleanup_dead_nodes_on_destruction: true } }, defaults:
Defaults { publish_subscribe: PublishSubscribe { max_s
ubscribers: 8, max_publishers: 2, max_nodes: 20, subscr
iber_max_buffer_size: 2, subscriber_max_borrowed_sample
s: 2, publisher_max_loaned_samples: 2, publisher_histor
y_size: 1, enable_safe_overflow: true, unable_to_delive
r_strategy: Block, subscriber_expired_connection_buffer
: 128 }, event: Event { max_listeners: 16, max_notifier
s: 16, max_nodes: 36, event_id_max_value: 4294967295, n
otifier_created_event: None, notifier_dropped_event: No
ne, notifier_dead_event: None } } } }, monitoring_token
: UnsafeCell { .. }, registered_services: RegisteredSer
vices { data: Mutex { data: {ServiceId(RestrictedFileNa
me { value: FixedSizeByteString<64> { len: 40, data: "4
eacadf2695a3f4b2eb95485759246ce1a2aa906" } }): (Contain
erHandle { index: 1, container_id: 2 }, 1), ServiceId(R
estrictedFileName { value: FixedSizeByteString<64> { le
n: 40, data: "b61bd15e8c3ea16146985e960906a8e125156a73"
} }): (ContainerHandle { index: 1, container_id: 5 },
1)}, poisoned: false, .. } }, signal_handling_mode: Han
dleTerminationRequests, _details_storage: Storage { nam
e: FileName { value: FixedSizeByteString<255> { len: 4,
data: "node" } }, config: Configuration { path: Path {
value: FixedSizeByteString<255> { len: 48, data: "/tmp
/iceoryx2/nodes/1960422421266913882581929908" } }, suff
ix: FileName { value: FixedSizeByteString<255> { len: 8
, data: ".details" } }, prefix: FileName { value: Fixed
SizeByteString<255> { len: 5, data: "iox2_" } } }, has_
ownership: false, file: File { path: Some(FilePath { va
lue: FixedSizeByteString<255> { len: 66, data: "/tmp/ic
eoryx2/nodes/1960422421266913882581929908/iox2_node.det
ails" } }), file_descriptor: FileDescriptor { value: 3,
is_owned: true }, has_ownership: false }, len: 1162 }
}, dynamic_storage: Storage { shm: SharedMemory { name:
FileName { value: FixedSizeByteString<255> { len: 94,
data: "iox2_305ad9523c6b202364d581359ec3d2c5743e42e7_b6
1bd15e8c3ea16146985e960906a8e125156a73.dynamic" } }, si
ze: 2723, base_address: 0x781e9dbd7000, has_ownership:
false, file_descriptor: FileDescriptor { value: 11, is_
owned: true }, memory_lock: None }, name: FileName { va
lue: FixedSizeByteString<255> { len: 40, data: "b61bd15
e8c3ea16146985e960906a8e125156a73" } }, _phantom_data:
PhantomData }, static_storage: Storage { name: FileName { va
lue: FixedSizeByteString<255> { len: 40, data: "b61bd15
e8c3ea16146985e960906a8e125156a73" } }, config: Configu
ration { path: Path { value: FixedSizeByteString<255> {
len: 22, data: "/tmp/iceoryx2/services" } }, suffix: F
ileName { value: FixedSizeByteString<255> { len: 8, dat
a: ".service" } }, prefix: FileName { value: FixedSizeB
yteString<255> { len: 5, data: "iox2_" } } }, has_owner
ship: false, file: File { path: Some(FilePath { value:
FixedSizeByteString<255> { len: 76, data: "/tmp/iceoryx
2/services/iox2_b61bd15e8c3ea16146985e960906a8e125156a7
3.service" } }), file_descriptor: FileDescriptor { valu
e: 8, is_owned: true }, has_ownership: false }, len: 24
3 } } }, listener_list_state: UnsafeCell { .. }, defaul
t_event_id: TriggerId(0), event_id_max_value: 429496729
5, dynamic_notifier_handle: Some(ContainerHandle { inde
x: 1, container_id: 4 }), notifier_id: UniqueNotifierId
(UniqueSystemId { value: 81199473864090521487349288884,
pid: 2002868, creation_time: Time { clock_type: Realti
me, seconds: 1745234363, nanoseconds: 106865002 } }), o
n_drop_notification: None }
| Unable to send notification via connection Connection
| { notifier: Notifier { sender: UnixDatagramSender {
| socket: UnixDatagramSocket { name: FilePath { value:
| FixedSizeByteString<255> { len: 62, data: "/tmp/iceor
| yx2/iox2_2002860_81767375862099321876681494444.event"
| } }, is_non_blocking: true, file_descriptor: FileDes
| criptor { value: 6, is_owned: true } } }, name: FileN
| ame { value: FixedSizeByteString<255> { len: 37, data
| : "2002860_81767375862099321876681494444" } } }, list
| ener_id: UniqueListenerId(UniqueSystemId { value: 817
| 67375862099321876681494444, pid: 2002860, creation_ti
| me: Time { clock_type: Realtime, seconds: 1745234362,
| nanoseconds: 137651031 } }) } due to FailedToDeliver
| Signal.
Contributor guide
Research direction
Start by reproducing the issue using iceoryx2-0.5.0/examples/cxx/event_based_communication/src/publisher.cpp with a 1-nanosecond cycle time, then run the subscriber and publisher commands from the report. Trace the notification delivery path around the reported FailedToDeliverSignal error. Done means sustained high-frequency publishing no longer exhausts notification resources or emits the reported warning/error.
Written by the indexing model from the issue text.
Assessment
- Tech stack
- rust
- Domain
- distributed-systems
- Issue type
- Bug
- Difficulty
- 4/5
- Estimated time
- 3-5 days
- Activity status
- Stale
- Clarity
- Mostly clear
- Newbie friendliness
- 35/100