[AWS for Fluent Bit] [request]: multiline parser to newline/separate between concatenated partial logs

@PettitWesley
hi here iam trying to use multiline parser and trying to merge logs which are related to same pod
below is my log formate before using multiline parser i can view logs in cloudwatch in below formate
----------
{
"log": "2023-04-28T09:42:39.72883634Z stderr F [2023/04/28 09:42:39] [ info] [input:tail:tail.2] multiline core started",
"kubernetes": {
"pod_name": "6cf9856b8-5jffb",
"namespace_name": "",
"pod_id": "",
"host": "",
"container_name": "resetter",
"docker_id": "",
"container_hash": "",
"container_image": ""
}
}
{
"log": "2023-04-28T09:42:39.729908443Z stderr F [2023/04/28 09:42:39] [ info] [input:systemd:systemd.3] seek_cursor=s=14fca72a59e8a;i=f38... OK",
"kubernetes": {
"pod_name": "6cf9856b8-5jffb",
"namespace_name": "",
"pod_id": "",
"host": "",
"container_name": "resetter",
"docker_id": "",
"container_hash": "",
"container_image": ""
}
}
----------
here i have modified my config by adding filter multiline to merge logs
configmap:
[SERVICE]
Flush 2
Log_Level info
Daemon Off
Parsers_File parsers.conf
Parsers_File custom_parsers.conf
HTTP_Server On
HTTP_Listen 0.0.0.0
HTTP_Port 2020

[INPUT]
Name tail
Tag application.*
Exclude_Path /var/log/containers/cloudwatch-agent*, /var/log/containers/fluent-bit*, /var/log/containers/aws-node*, /var/log/containers/kube-proxy*
Path /var/log/containers/*.log
Docker_Mode On
Docker_Mode_Flush 5
Docker_Mode_Parser container_firstline
#Parser docker
multiline.parser docker
DB /var/fluent-bit/state/flb_container.db
Mem_Buf_Limit 50MB
Skip_Long_Lines On
Refresh_Interval 10
Rotate_Wait 30
Read_from_Head Off

[INPUT]
Name tail
Tag application.*
Path /var/log/containers/fluent-bit*
#Parser docker
multiline.parser docker
DB /var/fluent-bit/state/flb_log.db
Mem_Buf_Limit 5MB
Skip_Long_Lines On
Refresh_Interval 10
Read_from_Head Off

[INPUT]
Name tail
Tag application.*
Path /var/log/containers/cloudwatch-agent*
Docker_Mode On
Docker_Mode_Flush 5
Docker_Mode_Parser cwagent_firstline
#Parser docker
multiline.parser docker
DB /var/fluent-bit/state/flb_cwagent.db
rotate_wait 15
Mem_Buf_Limit 15MB
Skip_Long_Lines On
Refresh_Interval 25
#Read_from_Head true

[FILTER]
Name kubernetes
Match application.*
Kube_URL https://kubernetes.default.svc:443
Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token
Kube_Tag_Prefix application.var.log.containers.
Merge_Log On
Merge_Log_Trim Off
Merge_Log_Key log
K8S-Logging.Parser On
K8S-Logging.Exclude Off
Labels Off
Annotations Off

[FILTER]
name multiline
match application.*
multiline.key_content log
multiline.parser docker

[OUTPUT]
Name cloudwatch_logs
Match application.*
region ${REGION}
log_group_name /aws/containerinsights/${CLUSTER_NAME}/application
log_stream_prefix ${APP_POD_NAMESPACE}-
auto_create_group true
extra_user_agent container-insights

custom_parsers.conf: |
[PARSER]
Name docker
Format json
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%LZ

[PARSER]
Name cwagent_firstline
Format regex
Regex (?(?<="log":")\d{4}[\/-]\d{1,2}[\/-]\d{1,2}[ T]\d{2}:\d{2}:\d{2}(?!\.).?)(?(?<="stream":").?)".(?\d{4}-\d{1,2}-\d{1,2}T\d{2}:\d{2}:\d{2}\.\w*).*(?=})
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%LZ

---------- logs got merged but here i want to append /n between log to log ,how i can do it please can you help for this @PettitWesley

After using multiline
{
"log": "2023-04-27T10:13:27.942291886Z stdout F Kafka Mirror Maker consumer configuration:2023-04-27T10:13:27.950593993Z stdout F # Bootstrap servers2023-04-27T10:13:27.950616934Z stdout F bootstrap.servers=2023-04-27T10:13:27.950621654Z stdout F # Consumer group2023-04-27T10:13:27.950625174Z stdout F group.id=mirrormaker2023-04-27T10:13:27.950628244Z stdout F # Provided configuration2023-04-27T10:13:27.950631164Z stdout F auto.commit.interval.ms=30002023-04-27T10:13:27.950633984Z stdout F auto.offset.reset=latest2023-04-27T10:13:27.950636494Z stdout F enable.auto.commit=true2023-04-27T10:13:27.950638424Z stdout F isolation.level=read_committed2023-04-27T10:13:27.950640334Z stdout F 2023-04-27T10:13:27.950642154Z stdout F 2023-04-27T10:13:27.950644074Z stdout F security.protocol=PLAINTEXT2023-04-27T10:13:27.950645995Z stdout F 2023-04-27T10:13:27.950647745Z stdout F 2023-04-27T10:13:27.950649485Z stdout F 2023-04-27T10:13:27.950730916Z stdout F 2023-04-27T10:13:27.950740377Z stdout F Kafka Mirror Maker producer configuration:2023-04-27T10:13:27.957949969Z stdout F # Bootstrap servers2023-04-27T10:13:27.95796353Z stdout F bootstrap.servers=""2023-04-27T10:13:27.95796602Z stdout F # Provided configuration",
"kubernetes": {
"pod_name": "5df8d48f89-6ngsw",
"namespace_name": "realtime",
"pod_id": "",
"host": "",
"container_name": "mirror-maker",
"docker_id": "",
"container_hash": "",
"container_image": ""
}
}