awslabs / awslabs/aws-lambda-redshift-loader
addAdditionalClusterEndpoint.js error during DynamoDB update
- Dominant language
- JavaScript
- Stars
- 595
- Forks
- 161
- PR merge metrics
- No merged PRs in 30d
Description
I ran addAdditionalClusterEndpoint.js and got couple of issues.
```javascript
Enter the Region for the Configuration > ****
Enter the Configuration S3 Bucket & Prefix > ****
Enter the Cluster Endpoint > ****
Enter the Cluster Port > ****
Does your cluster use SSL (Y/N) > ****
Enter the Database Name > ****
Enter the Table to be Loaded > ****
Should the Table be Truncated before Load? (Y/N) > N
Enter the Database Username > ****
Enter the Database Password > ****
/Workspace/aws-lambda-redshift-loader/node_modules/aws-sdk/lib/request.js:31
throw err;
^
ReferenceError: dynamoConfig is not defined
at Response. (/Workspace/aws-lambda-redshift-loader/common.js:183:34)
at Request. (/Workspace/aws-lambda-redshift-loader/node_modules/aws-sdk/lib/request.js:360:18)
at Request.callListeners (/Workspace/aws-lambda-redshift-loader/node_modules/aws-sdk/lib/sequential_executor.js:105:20)
at Request.emit (/Workspace/aws-lambda-redshift-loader/node_modules/aws-sdk/lib/sequential_executor.js:77:10)
at Request.emit (/Workspace/aws-lambda-redshift-loader/node_modules/aws-sdk/lib/request.js:673:14)
at Request.transition (/Workspace/aws-lambda-redshift-loader/node_modules/aws-sdk/lib/request.js:22:10)
at AcceptorStateMachine.runTo (/Workspace/aws-lambda-redshift-loader/node_modules/aws-sdk/lib/state_machine.js:14:12)
at /Workspace/aws-lambda-redshift-loader/node_modules/aws-sdk/lib/state_machine.js:26:10
at Request. (/Workspace/aws-lambda-redshift-loader/node_modules/aws-sdk/lib/request.js:38:9)
at Request. (/Workspace/aws-lambda-redshift-loader/node_modules/aws-sdk/lib/request.js:675:12)
```
I removed dynamoConfig from common.js and rerun the script.
```javascript
{ ValidationException: ExpressionAttributeValues contains invalid value: The parameter cannot be converted to a numeric value: 2017-10-19 16:41:43 for key :updateTime
at Request.extractError (/Workspace/aws-lambda-redshift-loader/node_modules/aws-sdk/lib/protocol/json.js:48:27)
at Request.callListeners (/Workspace/aws-lambda-redshift-loader/node_modules/aws-sdk/lib/sequential_executor.js:105:20)
at Request.emit (/Workspace/aws-lambda-redshift-loader/node_modules/aws-sdk/lib/sequential_executor.js:77:10)
at Request.emit (/Workspace/aws-lambda-redshift-loader/node_modules/aws-sdk/lib/request.js:673:14)
at Request.transition (/Workspace/aws-lambda-redshift-loader/node_modules/aws-sdk/lib/request.js:22:10)
at AcceptorStateMachine.runTo (/Workspace/aws-lambda-redshift-loader/node_modules/aws-sdk/lib/state_machine.js:14:12)
at /Workspace/aws-lambda-redshift-loader/node_modules/aws-sdk/lib/state_machine.js:26:10
at Request. (/Workspace/aws-lambda-redshift-loader/node_modules/aws-sdk/lib/request.js:38:9)
at Request. (/Workspace/aws-lambda-redshift-loader/node_modules/aws-sdk/lib/request.js:675:12)
at Request.callListeners (/Workspace/aws-lambda-redshift-loader/node_modules/aws-sdk/lib/sequential_executor.js:115:18)
message: 'ExpressionAttributeValues contains invalid value: The parameter cannot be converted to a numeric value: 2017-10-19 16:41:43 for key :updateTime',
code: 'ValidationException',
time: 2017-10-19T16:42:00.627Z,
requestId: 'HRP8T25D9NHQ6IU2F4IUR1JRUFVV4KQNSO5AEMVJF66Q9ASUAAJG',
statusCode: 400,
retryable: false,
retryDelay: 34.08456752183452 }
```
I checked LambdaRedshiftBatchLoadConfig table and found it doesn't have lastUpdate column so I removed it from UpdateExpression and it worked.
```diff
diff --git a/addAdditionalClusterEndpoint.js b/addAdditionalClusterEndpoint.js
index 1aa43d8..09833af 100644
--- a/addAdditionalClusterEndpoint.js
+++ b/addAdditionalClusterEndpoint.js
@@ -33,12 +33,9 @@ var updateRequest = {
s3Prefix : undefined
},
TableName : configTable,
- UpdateExpression : "SET loadClusters = list_append(loadClusters, :newLoadCluster),lastUpdate = :updateTime",
+ UpdateExpression : "SET loadClusters = list_append(loadClusters, :newLoadCluster)",
ExpressionAttributeValues : {
":newLoadCluster" : null,
- ":updateTime" : {
- N : '' + common.readableTime(common.now())
- }
}
};
@@ -225,4 +222,4 @@ qs.push(last);
// call the first function in the function list, to invoke the callback
// reference chain
-async.waterfall(qs);
\ No newline at end of file
+async.waterfall(qs);
diff --git a/common.js b/common.js
index 79d9c4d..54d7d70 100644
--- a/common.js
+++ b/common.js
@@ -180,7 +180,6 @@ exports.retryableUpdate = function(dynamoDB, updateRequest, callback) {
// being created
setTimeout(asyncCallback, 1000);
} else {
- console.log(JSON.stringify(dynamoConfig));
console.log(err);
asyncCallback(err);
}
@@ -219,7 +218,6 @@ exports.retryablePut = function(dynamoDB, putRequest, callback) {
// being created
setTimeout(asyncCallback, 1000);
} else {
- console.log(JSON.stringify(dynamoConfig));
console.log(err);
asyncCallback(err);
```
Contributor guide
Assessment
This issue has not been assessed yet.