Consumers keep pulling duplicate data
- Dominant language
- Java
- Stars
- 15.3k
- Forks
- 3.8k
- Avg merge
- 1d 14h
- Merged PRs (30d)
- 160
Description
**Describe the bug**
A clear and concise description of what the bug is.
Consumers keep pulling duplicate data。
pulsar-admin topics stats bms_calculate_dto-1
{
"msgRateIn" : 1.0000113916131006,
"msgThroughputIn" : 944.3440908133047,
"msgRateOut" : 100.0011371412641,
"msgThroughputOut" : 167396.90351761904,
"bytesInCounter" : 180801601,
"msgInCounter" : 20044,
"bytesOutCounter" : 2484808859,
"msgOutCounter" : 1462501,
"averageMsgSize" : 944.3333333333334,
"msgChunkPublished" : false,
"storageSize" : 180801601,
"backlogSize" : 180801601,
"offloadedStorageSize" : 0,
"publishers" : [ {
"msgRateIn" : 1.0000113916131006,
"msgThroughputIn" : 944.3440908133047,
"averageMsgSize" : 944.0,
"chunkedMessageRate" : 0.0,
"producerId" : 175,
"metadata" : { },
"producerName" : "pulsar-cluster-zk-4-1187",
"connectedSince" : "2022-05-25T16:57:07.314+08:00",
"clientVersion" : "2.7.4",
"address" : "/127.0.0.1:45074"
} ],
"subscriptions" : {
"my-subscription" : {
"msgRateOut" : 0.0,
"msgThroughputOut" : 0.0,
"bytesOutCounter" : 45717523,
"msgOutCounter" : 5500,
"msgRateRedeliver" : 0.0,
"chuckedMessageRate" : 0,
"msgBacklog" : 12502,
"backlogSize" : 0,
"msgBacklogNoDelayed" : 12502,
"blockedSubscriptionOnUnackedMsgs" : false,
"msgDelayed" : 0,
"unackedMessages" : 0,
"type" : "Failover",
"activeConsumerName" : "",
"msgRateExpired" : 0.0,
"totalMsgExpired" : 0,
"lastExpireTimestamp" : 0,
"lastConsumedFlowTimestamp" : 1653483827529,
"lastConsumedTimestamp" : 1653483827539,
"lastAckedTimestamp" : 1653483994951,
"lastMarkDeleteAdvancedTimestamp" : 1653483994951,
"consumers" : [ {
"msgRateOut" : 0.0,
"msgThroughputOut" : 0.0,
"bytesOutCounter" : 45717523,
"msgOutCounter" : 5500,
"msgRateRedeliver" : 0.0,
"chuckedMessageRate" : 0.0,
"consumerName" : "",
"availablePermits" : 0,
"unackedMessages" : 0,
"avgMessagesPerEntry" : 6,
"blockedConsumerOnUnackedMsgs" : false,
"lastAckedTimestamp" : 1653483994951,
"lastConsumedTimestamp" : 1653483827539,
"metadata" : { },
"connectedSince" : "2022-05-25T21:01:30.854+08:00",
"clientVersion" : "2.10.0",
"address" : "/127.0.0.1:60064"
} ],
"isDurable" : true,
"isReplicated" : false,
"consumersAfterMarkDeletePosition" : { },
"nonContiguousDeletedMessagesRanges" : 0,
"nonContiguousDeletedMessagesRangesSerializedSize" : 18
},
"__compaction" : {
"msgRateOut" : 100.0011371412641,
"msgThroughputOut" : 167396.90351761904,
"bytesOutCounter" : 2439091336,
"msgOutCounter" : 1457001,
"msgRateRedeliver" : 0.0,
"chuckedMessageRate" : 0,
"msgBacklog" : 20044,
"backlogSize" : 0,
"msgBacklogNoDelayed" : 20044,
"blockedSubscriptionOnUnackedMsgs" : false,
"msgDelayed" : 0,
"unackedMessages" : 0,
"type" : "Exclusive",
"activeConsumerName" : "7f3e7",
"msgRateExpired" : 0.0,
"totalMsgExpired" : 0,
"lastExpireTimestamp" : 0,
"lastConsumedFlowTimestamp" : 1653483989659,
"lastConsumedTimestamp" : 1653483989671,
"lastAckedTimestamp" : 0,
"lastMarkDeleteAdvancedTimestamp" : 0,
"consumers" : [ {
"msgRateOut" : 100.0011371412641,
"msgThroughputOut" : 167396.90351761904,
"bytesOutCounter" : 2439091336,
"msgOutCounter" : 1457001,
"msgRateRedeliver" : 0.0,
"chuckedMessageRate" : 0.0,
"consumerName" : "7f3e7",
"availablePermits" : -1,
"unackedMessages" : 0,
"avgMessagesPerEntry" : 6,
"blockedConsumerOnUnackedMsgs" : false,
"lastAckedTimestamp" : 0,
"lastConsumedTimestamp" : 1653483989671,
"metadata" : { },
"connectedSince" : "2022-05-25T17:14:48.541+08:00",
"clientVersion" : "2.7.4",
"address" : "/127.0.0.1:52964"
} ],
"isDurable" : true,
"isReplicated" : false,
"consumersAfterMarkDeletePosition" : { },
"nonContiguousDeletedMessagesRanges" : 0,
"nonContiguousDeletedMessagesRangesSerializedSize" : 0
}
},
"replication" : { },
"deduplicationStatus" : "Disabled",
"nonContiguousDeletedMessagesRanges" : 0,
"nonContiguousDeletedMessagesRangesSerializedSize" : 18
}
client:python 2.10.0
server: 2.7.4
**To Reproduce**
Steps to reproduce the behavior:
1. set consumerparameter is_read_compacted=true
2. start comsumer,then the problem
**Expected behavior**
Consumption goes on all the time, not duplicate data
**Screenshots**
If applicable, add screenshots to help explain your problem.
**Desktop (please complete the following information):**
- OS:
**Additional context**
Add any other context about the problem here.
Contributor guide
Research direction
Start by reproducing the issue with the Python client 2.10.0 against the server 2.7.4, using is_read_compacted=true as described. Compare consumer output with pulsar-admin topics stats for the subscription, and consider the issue done when consumption continues without duplicate data.
Written by the indexing model from the issue text.
Assessment
- Tech stack
- java, python
- Domain
- distributed-systems
- Issue type
- Bug
- Difficulty
- 4/5
- Estimated time
- 3-5 days
- Activity status
- Stale
- Clarity
- Mostly clear
- Newbie friendliness
- 35/100