elastic / elastic/integrations
[AWS]: Firehose/Cloudfront ingest pipeline failed
- Dominant language
- Handlebars
- Stars
- 333
- Forks
- 647
- Avg merge
- 3d 4h
- Merged PRs (30d)
- 209
Description
### Integration Name
AWS [aws]
### Dataset Name
aws.cloudfront_logs
### Integration Version
4.3.0
### Agent Version
firehose
### Agent Output Type
elasticsearch
### Elasticsearch Version
serverless
### OS Version and Architecture
serverless
### Software/API Version
_No response_
### Error Message
Processor 'csv' failed with message 'Illegal character inside unquoted field at 488'
### Event Original
_No response_
### What did you do?
- Installed AWS integration
- Installed Amazon Data Firehose integration
- Set up delivery streams for cloudfront according to https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudfront_distribution#with-v2-logging-to-data-firehose
```hcl
resource "aws_cloudfront_distribution" "example" {
# other config
}
resource "aws_kinesis_firehose_delivery_stream" "cloudfront_logs" {
destination = "http_endpoint"
request_configuration {
content_encoding = "GZIP"
}
# other config
}
resource "aws_cloudwatch_log_delivery_source" "example" {
region = "us-east-1"
name = "cloudfront-logs-source"
log_type = "ACCESS_LOGS"
resource_arn = aws_cloudfront_distribution.example.arn
}
resource "aws_cloudwatch_log_delivery_destination" "example" {
region = "us-east-1"
name = "firehose-destination"
output_format = "raw"
delivery_destination_configuration {
destination_resource_arn = aws_kinesis_firehose_delivery_stream.cloudfront_logs.arn
}
}
resource "aws_cloudwatch_log_delivery" "example" {
region = "us-east-1"
delivery_source_name = aws_cloudwatch_log_delivery_source.example.name
delivery_destination_arn = aws_cloudwatch_log_delivery_destination.example.arn
}
````
### What did you see?
```json
{
"_index": ".ds-logs-aws.cloudfront_logs-default-2025.10.17-000001",
"_id": "f3a167455f",
"_version": 1,
"_source": {
"@timestamp": "2025-10-17T21:02:18.537Z",
"aws": {
"firehose": {
"arn": "",
"request_id": ""
},
"kinesis": {
"name": "cloudfront-logs-prod",
"type": "deliverystream"
}
},
"cloud": {
"account": {
"id": ""
},
"provider": "aws",
"region": "us-east-1"
},
"data_stream": {
"dataset": "aws.cloudfront_logs",
"namespace": "default",
"type": "logs"
},
"ecs": {
"version": "8.11.0"
},
"error": {
"message": [
"Processor 'csv' failed with message 'Illegal character inside unquoted field at 603'"
]
},
"event": {
"category": [
"web"
],
"dataset": "aws.cloudfront_logs",
"ingested": "2025-10-17T21:02:19Z",
"kind": "pipeline_error",
"type": [
"access"
]
}
},
"fields": {
"cloud.region": [
"us-east-1"
],
"event.category": [
"web"
],
"aws.firehose.arn": [
""
],
"data_stream.namespace": [
"default"
],
"aws.kinesis.type": [
"deliverystream"
],
"data_stream.type": [
"logs"
],
"aws.firehose.request_id": [
""
],
"cloud.provider": [
"aws"
],
"event.ingested": [
"2025-10-17T21:02:19.000Z"
],
"@timestamp": [
"2025-10-17T21:02:18.537Z"
],
"event.module": [
"aws"
],
"aws.kinesis.name.text": [
"cloudfront-logs-prod"
],
"cloud.account.id": [
""
],
"ecs.version": [
"8.11.0"
],
"error.message": [
"Processor 'csv' failed with message 'Illegal character inside unquoted field at 603'"
],
"data_stream.dataset": [
"aws.cloudfront_logs"
],
"event.type": [
"access"
],
"aws.kinesis.name": [
"cloudfront-logs-prod"
],
"event.kind": [
"pipeline_error"
],
"event.dataset": [
"aws.cloudfront_logs"
]
}
}
```
### What did you expect to see?
Processor 'csv' failed with message 'Illegal character inside unquoted field at 488'
### Anything else?
_No response_
Contributor guide
Assessment
This issue has not been assessed yet.