apache / apache/pulsar

Consumers keep pulling duplicate data

Open
#15,781 1 comment 0 reactions 0 assignees View on GitHub
Stale type/bug
Dominant language
Java
Stars
15.3k
Forks
3.8k
Avg merge
1d 14h
Merged PRs (30d)
160

Description

**Describe the bug**
A clear and concise description of what the bug is.

Consumers keep pulling duplicate data。

pulsar-admin topics stats bms_calculate_dto-1
{
"msgRateIn" : 1.0000113916131006,
"msgThroughputIn" : 944.3440908133047,
"msgRateOut" : 100.0011371412641,
"msgThroughputOut" : 167396.90351761904,
"bytesInCounter" : 180801601,
"msgInCounter" : 20044,
"bytesOutCounter" : 2484808859,
"msgOutCounter" : 1462501,
"averageMsgSize" : 944.3333333333334,
"msgChunkPublished" : false,
"storageSize" : 180801601,
"backlogSize" : 180801601,
"offloadedStorageSize" : 0,
"publishers" : [ {
"msgRateIn" : 1.0000113916131006,
"msgThroughputIn" : 944.3440908133047,
"averageMsgSize" : 944.0,
"chunkedMessageRate" : 0.0,
"producerId" : 175,
"metadata" : { },
"producerName" : "pulsar-cluster-zk-4-1187",
"connectedSince" : "2022-05-25T16:57:07.314+08:00",
"clientVersion" : "2.7.4",
"address" : "/127.0.0.1:45074"
} ],
"subscriptions" : {
"my-subscription" : {
"msgRateOut" : 0.0,
"msgThroughputOut" : 0.0,
"bytesOutCounter" : 45717523,
"msgOutCounter" : 5500,
"msgRateRedeliver" : 0.0,
"chuckedMessageRate" : 0,
"msgBacklog" : 12502,
"backlogSize" : 0,
"msgBacklogNoDelayed" : 12502,
"blockedSubscriptionOnUnackedMsgs" : false,
"msgDelayed" : 0,
"unackedMessages" : 0,
"type" : "Failover",
"activeConsumerName" : "",
"msgRateExpired" : 0.0,
"totalMsgExpired" : 0,
"lastExpireTimestamp" : 0,
"lastConsumedFlowTimestamp" : 1653483827529,
"lastConsumedTimestamp" : 1653483827539,
"lastAckedTimestamp" : 1653483994951,
"lastMarkDeleteAdvancedTimestamp" : 1653483994951,
"consumers" : [ {
"msgRateOut" : 0.0,
"msgThroughputOut" : 0.0,
"bytesOutCounter" : 45717523,
"msgOutCounter" : 5500,
"msgRateRedeliver" : 0.0,
"chuckedMessageRate" : 0.0,
"consumerName" : "",
"availablePermits" : 0,
"unackedMessages" : 0,
"avgMessagesPerEntry" : 6,
"blockedConsumerOnUnackedMsgs" : false,
"lastAckedTimestamp" : 1653483994951,
"lastConsumedTimestamp" : 1653483827539,
"metadata" : { },
"connectedSince" : "2022-05-25T21:01:30.854+08:00",
"clientVersion" : "2.10.0",
"address" : "/127.0.0.1:60064"
} ],
"isDurable" : true,
"isReplicated" : false,
"consumersAfterMarkDeletePosition" : { },
"nonContiguousDeletedMessagesRanges" : 0,
"nonContiguousDeletedMessagesRangesSerializedSize" : 18
},
"__compaction" : {
"msgRateOut" : 100.0011371412641,
"msgThroughputOut" : 167396.90351761904,
"bytesOutCounter" : 2439091336,
"msgOutCounter" : 1457001,
"msgRateRedeliver" : 0.0,
"chuckedMessageRate" : 0,
"msgBacklog" : 20044,
"backlogSize" : 0,
"msgBacklogNoDelayed" : 20044,
"blockedSubscriptionOnUnackedMsgs" : false,
"msgDelayed" : 0,
"unackedMessages" : 0,
"type" : "Exclusive",
"activeConsumerName" : "7f3e7",
"msgRateExpired" : 0.0,
"totalMsgExpired" : 0,
"lastExpireTimestamp" : 0,
"lastConsumedFlowTimestamp" : 1653483989659,
"lastConsumedTimestamp" : 1653483989671,
"lastAckedTimestamp" : 0,
"lastMarkDeleteAdvancedTimestamp" : 0,
"consumers" : [ {
"msgRateOut" : 100.0011371412641,
"msgThroughputOut" : 167396.90351761904,
"bytesOutCounter" : 2439091336,
"msgOutCounter" : 1457001,
"msgRateRedeliver" : 0.0,
"chuckedMessageRate" : 0.0,
"consumerName" : "7f3e7",
"availablePermits" : -1,
"unackedMessages" : 0,
"avgMessagesPerEntry" : 6,
"blockedConsumerOnUnackedMsgs" : false,
"lastAckedTimestamp" : 0,
"lastConsumedTimestamp" : 1653483989671,
"metadata" : { },
"connectedSince" : "2022-05-25T17:14:48.541+08:00",
"clientVersion" : "2.7.4",
"address" : "/127.0.0.1:52964"
} ],
"isDurable" : true,
"isReplicated" : false,
"consumersAfterMarkDeletePosition" : { },
"nonContiguousDeletedMessagesRanges" : 0,
"nonContiguousDeletedMessagesRangesSerializedSize" : 0
}
},
"replication" : { },
"deduplicationStatus" : "Disabled",
"nonContiguousDeletedMessagesRanges" : 0,
"nonContiguousDeletedMessagesRangesSerializedSize" : 18
}

client:python 2.10.0
server: 2.7.4

**To Reproduce**
Steps to reproduce the behavior:
1. set consumerparameter is_read_compacted=true
2. start comsumer,then the problem

**Expected behavior**
Consumption goes on all the time, not duplicate data

**Screenshots**
If applicable, add screenshots to help explain your problem.

**Desktop (please complete the following information):**
- OS:

**Additional context**
Add any other context about the problem here.

Contributor guide

Open the contributing guide

Research direction

Start by reproducing the issue with the Python client 2.10.0 against the server 2.7.4, using is_read_compacted=true as described. Compare consumer output with pulsar-admin topics stats for the subscription, and consider the issue done when consumption continues without duplicate data.

Written by the indexing model from the issue text.

Assessment

Tech stack
java, python
Domain
distributed-systems
Issue type
Bug
Difficulty
4/5
Estimated time
3-5 days
Activity status
Stale
Clarity
Mostly clear
Newbie friendliness
35/100

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.