awslabs / awslabs/amazon-kinesis-client-nodejs

processRecords with async operations?

Open
#66 1 comment 0 reactions 0 assignees View on GitHub
Dominant language
JavaScript
Stars
304
Forks
198
Avg merge
36m
Merged PRs (30d)
2

Description

Decided to use the new basic consumer sample as a template for my current processor..

So on this project, we have some asynchronous tasks to do for each kinesis message, we also want to checkpoint after each message. Was wondering if you could take a peek at my processRecords impl to see if it looks solid:

fyi to decouple a lot of our processing and retry logic from the consumer we pass a `recordProcessingStrategyCallback` to the consumer.. @sahilpalvia

```
'use strict';

var util = require('util');

module.exports = function strategizedKinesisConsumer(logger, recordProcessingStrategyCallback) {

var shardId;
var logger = logger;
var recordProcessingStrategyCallback = recordProcessingStrategyCallback;

return {

initialize: function(initializeInput, completeCallback) {
shardId = initializeInput.shardId;

logger.info('strategizedKinesisConsumer', {customText: `New KCL consumer initializing with config: ${JSON.stringify(initializeInput)}`});

completeCallback();
},

processRecords: function(processRecordsInput, completeCallback) {

if (!processRecordsInput || !processRecordsInput.records) {
completeCallback();
return;
}

var records = processRecordsInput.records;

return new Promise( async (resolve, reject) => {

for (var i = 0 ; i < records.length ; ++i) {
let record = records[i];
let data = new Buffer(record.data, 'base64').toString();
let dataObj = JSON.parse(data);
let sequenceNumber = record.sequenceNumber;
let partitionKey = record.partitionKey;

logger.info(util.format('ShardID: %s, Record: %s, SeqenceNumber: %s, PartitionKey:%s', shardId, data, sequenceNumber, partitionKey));

//call logic
logger.info('event', {eventId: dataObj.eventId, correlationId: dataObj.correlationId, customText: 'Delegating message to blackbox strategy for processing'});
await recordProcessingStrategyCallback(dataObj);
logger.info('event', {eventId: dataObj.eventId, correlationId: dataObj.correlationId, customText: '...back from blackbox strategy'});

if (!sequenceNumber) {
completeCallback();
return;
}

const checkpoint = util.promisify(processRecordsInput.checkpointer.checkpoint).bind(processRecordsInput.checkpointer);

logger.info('event', {eventId: dataObj.eventId, correlationId: dataObj.correlationId, customText: 'Checkpointing message...'});
await checkpoint(sequenceNumber);
logger.info('event', {eventId: dataObj.eventId, correlationId: dataObj.correlationId, customText: '..message checkpointed'});


}

completeCallback()

resolve()

});
},

leaseLost: function(leaseLostInput, completeCallback) {
...
},

shardEnded: function(shardEndedInput, completeCallback) {
..
},

shutdownRequested: function(shutdownRequestedInput, completeCallback) {
..
}
};
}
```

Contributor guide

Open the contributing guide

Research direction

Start with the supplied processRecords implementation and compare it with the new basic consumer sample; inspect recordProcessingStrategyCallback, completeCallback, and the checkpointer flow. Confirm the intended behavior for asynchronous per-record processing and checkpointing after each message, then document or test the accepted completion and error behavior.

Written by the indexing model from the issue text.

Assessment

Tech stack
javascript, node.js
Domain
backend, stream-processing
Issue type
Bug
Difficulty
4/5
Estimated time
3-5 days
Activity status
Stale
Clarity
Needs clarification
Newbie friendliness
25/100

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.