awslabs / awslabs/amazon-kinesis-client-nodejs
processRecords with async operations?
- Dominant language
- JavaScript
- Stars
- 304
- Forks
- 198
- Avg merge
- 36m
- Merged PRs (30d)
- 2
Description
Decided to use the new basic consumer sample as a template for my current processor..
So on this project, we have some asynchronous tasks to do for each kinesis message, we also want to checkpoint after each message. Was wondering if you could take a peek at my processRecords impl to see if it looks solid:
fyi to decouple a lot of our processing and retry logic from the consumer we pass a `recordProcessingStrategyCallback` to the consumer.. @sahilpalvia
```
'use strict';
var util = require('util');
module.exports = function strategizedKinesisConsumer(logger, recordProcessingStrategyCallback) {
var shardId;
var logger = logger;
var recordProcessingStrategyCallback = recordProcessingStrategyCallback;
return {
initialize: function(initializeInput, completeCallback) {
shardId = initializeInput.shardId;
logger.info('strategizedKinesisConsumer', {customText: `New KCL consumer initializing with config: ${JSON.stringify(initializeInput)}`});
completeCallback();
},
processRecords: function(processRecordsInput, completeCallback) {
if (!processRecordsInput || !processRecordsInput.records) {
completeCallback();
return;
}
var records = processRecordsInput.records;
return new Promise( async (resolve, reject) => {
for (var i = 0 ; i < records.length ; ++i) {
let record = records[i];
let data = new Buffer(record.data, 'base64').toString();
let dataObj = JSON.parse(data);
let sequenceNumber = record.sequenceNumber;
let partitionKey = record.partitionKey;
logger.info(util.format('ShardID: %s, Record: %s, SeqenceNumber: %s, PartitionKey:%s', shardId, data, sequenceNumber, partitionKey));
//call logic
logger.info('event', {eventId: dataObj.eventId, correlationId: dataObj.correlationId, customText: 'Delegating message to blackbox strategy for processing'});
await recordProcessingStrategyCallback(dataObj);
logger.info('event', {eventId: dataObj.eventId, correlationId: dataObj.correlationId, customText: '...back from blackbox strategy'});
if (!sequenceNumber) {
completeCallback();
return;
}
const checkpoint = util.promisify(processRecordsInput.checkpointer.checkpoint).bind(processRecordsInput.checkpointer);
logger.info('event', {eventId: dataObj.eventId, correlationId: dataObj.correlationId, customText: 'Checkpointing message...'});
await checkpoint(sequenceNumber);
logger.info('event', {eventId: dataObj.eventId, correlationId: dataObj.correlationId, customText: '..message checkpointed'});
}
completeCallback()
resolve()
});
},
leaseLost: function(leaseLostInput, completeCallback) {
...
},
shardEnded: function(shardEndedInput, completeCallback) {
..
},
shutdownRequested: function(shutdownRequestedInput, completeCallback) {
..
}
};
}
```
Contributor guide
Research direction
Start with the supplied processRecords implementation and compare it with the new basic consumer sample; inspect recordProcessingStrategyCallback, completeCallback, and the checkpointer flow. Confirm the intended behavior for asynchronous per-record processing and checkpointing after each message, then document or test the accepted completion and error behavior.
Written by the indexing model from the issue text.
Assessment
- Tech stack
- javascript, node.js
- Domain
- backend, stream-processing
- Issue type
- Bug
- Difficulty
- 4/5
- Estimated time
- 3-5 days
- Activity status
- Stale
- Clarity
- Needs clarification
- Newbie friendliness
- 25/100