galaxyproject / galaxyproject/pulsar
Status update publisher dies on oversize message
- Dominant language
- Python
- Stars
- 46
- Forks
- 62
- Avg merge
- 3d 9h
- Merged PRs (30d)
- 15
Description
```pytb
Exception in thread consume-setup-amqp://main_pulsar:********@mq.galaxyproject.org:5671//main_pulsar?ssl=1:
}{"message": "ok"}{"message": "ok"}{"message": "ok"}{"message": "ok"}{"message": "ok"}{"message": "ok"}{"message": "ok"}{"message": "ok"}{"message": "ok"}{"message": "ok"}{"message": "ok"}{"message": "ok"}{"message": "ok"}{"message": "ok"}{"message": "ok"}{"message": "ok"}>
File "/usr/lib64/python3.11/threading.py", line 1045, in _bootstrap_inner
self.run()
File "/srv/pulsar/main/venv/lib64/python3.11/site-packages/sentry_sdk/integrations/threading.py", line 99, in run
return _run_old_run_func()
^^^^^^^^^^^^^^^^^^^
File "/srv/pulsar/main/venv/lib64/python3.11/site-packages/sentry_sdk/integrations/threading.py", line 94, in _run_old_run_func
reraise(*_capture_exception())
File "/srv/pulsar/main/venv/lib64/python3.11/site-packages/sentry_sdk/utils.py", line 1640, in reraise
raise value
File "/srv/pulsar/main/venv/lib64/python3.11/site-packages/sentry_sdk/integrations/threading.py", line 92, in _run_old_run_func
return old_run_func(self, *a, **kw)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib64/python3.11/threading.py", line 982, in run
self._target(*self._args, **self._kwargs)
File "/srv/pulsar/main/venv/lib64/python3.11/site-packages/pulsar/messaging/bind_amqp.py", line 53, in drain
__drain(name, queue_state, pulsar_exchange, callback)
File "/srv/pulsar/main/venv/lib64/python3.11/site-packages/pulsar/messaging/bind_amqp.py", line 101, in __drain
pulsar_exchange.consume(name, callback=callback, check=queue_state)
File "/srv/pulsar/main/venv/lib64/python3.11/site-packages/pulsar/client/amqp_exchange.py", line 146, in consume
connection.drain_events(timeout=self.__timeout)
File "/srv/pulsar/main/venv/lib64/python3.11/site-packages/kombu/connection.py", line 341, in drain_events
return self.transport.drain_events(self.connection, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/srv/pulsar/main/venv/lib64/python3.11/site-packages/kombu/transport/pyamqp.py", line 171, in drain_events
return connection.drain_events(**kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/srv/pulsar/main/venv/lib64/python3.11/site-packages/amqp/connection.py", line 526, in drain_events
while not self.blocking_read(timeout):
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/srv/pulsar/main/venv/lib64/python3.11/site-packages/amqp/connection.py", line 532, in blocking_read
return self.on_inbound_frame(frame)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/srv/pulsar/main/venv/lib64/python3.11/site-packages/amqp/method_framing.py", line 77, in on_frame
callback(channel, msg.frame_method, msg.frame_args, msg)
File "/srv/pulsar/main/venv/lib64/python3.11/site-packages/amqp/connection.py", line 538, in on_inbound_method
return self.channels[channel_id].dispatch_method(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/srv/pulsar/main/venv/lib64/python3.11/site-packages/amqp/abstract_channel.py", line 156, in dispatch_method
listener(*args)
File "/srv/pulsar/main/venv/lib64/python3.11/site-packages/amqp/channel.py", line 1629, in _on_basic_deliver
fun(msg)
File "/srv/pulsar/main/venv/lib64/python3.11/site-packages/kombu/messaging.py", line 656, in _receive_callback
return on_m(message) if on_m else self.receive(decoded, message)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/srv/pulsar/main/venv/lib64/python3.11/site-packages/kombu/messaging.py", line 622, in receive
[callback(body, message) for callback in callbacks]
File "/srv/pulsar/main/venv/lib64/python3.11/site-packages/kombu/messaging.py", line 622, in
[callback(body, message) for callback in callbacks]
^^^^^^^^^^^^^^^^^^^^^^^
File "/srv/pulsar/main/venv/lib64/python3.11/site-packages/pulsar/client/amqp_exchange.py", line 164, in __ack_callback
self.publish(ack_queue, response)
File "/srv/pulsar/main/venv/lib64/python3.11/site-packages/pulsar/client/amqp_exchange.py", line 236, in publish
producer.publish(
File "/srv/pulsar/main/venv/lib64/python3.11/site-packages/kombu/messaging.py", line 186, in publish
return _publish(
^^^^^^^^^
File "/srv/pulsar/main/venv/lib64/python3.11/site-packages/kombu/connection.py", line 556, in _ensured
return fun(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^
File "/srv/pulsar/main/venv/lib64/python3.11/site-packages/kombu/messaging.py", line 208, in _publish
return channel.basic_publish(
^^^^^^^^^^^^^^^^^^^^^^
File "/srv/pulsar/main/venv/lib64/python3.11/site-packages/amqp/channel.py", line 1791, in _basic_publish
self.connection.drain_events(timeout=0)
File "/srv/pulsar/main/venv/lib64/python3.11/site-packages/amqp/connection.py", line 526, in drain_events
while not self.blocking_read(timeout):
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/srv/pulsar/main/venv/lib64/python3.11/site-packages/amqp/connection.py", line 532, in blocking_read
return self.on_inbound_frame(frame)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/srv/pulsar/main/venv/lib64/python3.11/site-packages/amqp/method_framing.py", line 53, in on_frame
callback(channel, method_sig, buf, None)
File "/srv/pulsar/main/venv/lib64/python3.11/site-packages/amqp/connection.py", line 538, in on_inbound_method
return self.channels[channel_id].dispatch_method(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/srv/pulsar/main/venv/lib64/python3.11/site-packages/amqp/abstract_channel.py", line 156, in dispatch_method
listener(*args)
File "/srv/pulsar/main/venv/lib64/python3.11/site-packages/amqp/channel.py", line 293, in _on_close
raise error_for_code(
amqp.exceptions.PreconditionFailed: Basic.publish: (406) PRECONDITION_FAILED - message size 31486805 is larger than configured max size 16777216
```
Contributor guide
Research direction
Start in pulsar/client/amqp_exchange.py, especially __ack_callback and publish, then trace the failing call through Kombu and AMQP using the reported stack trace. Reproduce the status update with a message larger than the configured 16777216-byte limit. Done means the publisher no longer dies when this oversize message is encountered, with behavior verified against the failure case.
Written by the indexing model from the issue text.
Assessment
- Tech stack
- python, rabbitmq
- Domain
- backend, distributed-systems
- Issue type
- Bug
- Difficulty
- 4/5
- Estimated time
- 3-5 days
- Activity status
- Quiet
- Clarity
- Needs clarification
- Newbie friendliness
- 38/100