Layer 7 related metadata available with Textual dump but partially missing in direct Kafka dump and almost missing in IPFIX format also

Add all Layer 7 related metadata available with Textual dump but partially missing in direct Kafka dump and almost missing in IPFIX format. In IPFIX format though they are sending 3/5 parameters as per template but only application id is filled, others are with no value

For Example DNS_QUERY, DNS_QUERY_TYPE, DNS_RET_CODE parameters

====================================
|| Cento Logs - Textual Dump ||

#FIRST_SWITCHED|LAST_SWITCHED|VLAN_ID|IN_SRC_MAC|IN_DST_MAC|IP_PROTOCOL_VERSION|PROTOCOL|IP_SRC_ADDR|L4_SRC_PORT|IP_DST_ADDR|L4_DST_PORT|DIRECTION|INPUT_SNMP|OUTPUT_SNMP|SRC_PKTS|SRC_BYTES|DST_PKTS|DST_BYTES|TCP_FLAGS|SRC_TOS|DST_TOS|CLIENT_NW_LATENCY_MS|SERVER_NW_LATENCY_MS|L7_PROTO|L7_PROTO_NAME|DNS_QUERY|DNS_QUERY_TYPE|DNS_RET_CODE|HTTP_HOST|HTTP_URL|HTTP_RET_CODE
1556023172.304410|1556023268.606157|0|00:23:9C:F5:30:52|D4:04:FF:2E:B4:17|4|6|117.225.44.236|50076|47.252.102.220|443|R2R|1|2|10|711|0|0|24|0|0|0|0|91|SSL||0|0||
1556023158.566344|1556023278.626888|0|00:23:9C:F5:36:1F|84:C1:C1:12:40:50|4|6|13.107.42.11|443|223.190.91.7|57187|R2R|1|2|20|9217|12|558|24|0|0|0|0|91|SSL||0|0||
1556023163.287940|1556023272.309336|0|00:23:9C:F5:35:D0|84:C1:C1:12:40:2F|4|6|49.201.46.25|28363|157.240.16.16|443|R2R|1|2|56|12037|0|0|24|0|0|0|0|119|SSL.Facebook||0|0|lithium.facebook.com|
1556023173.518779|1556023293.446617|0|84:C1:C1:12:47:F9|00:23:9C:F5:36:F2|4|6|119.28.243.122|17500|110.172.135.254|44437|R2R|1|2|48|4555|0|0|24|104|0|0|0|0|Unknown||0|0||
1556023171.594695|1556023287.517919|0|00:23:9C:F5:30:29|84:C1:C1:12:47:F6|4|17|117.239.224.140|389|51.15.144.131|36439|R2R|1|2|28|36400|0|0|0|0|0|0|0|112|LDAP||0|0||
1556023159.606407|1556023278.556043|0|00:23:9C:F5:36:9E|D4:04:FF:2E:BB:F9|4|6|13.228.85.158|80|150.242.63.249|47195|R2R|1|2|8|462|0|0|24|0|0|0|0|0|Unknown||0|0||
1556023177.142562|1556023274.599517|0|D4:04:FF:2E:BC:22|00:23:9C:F5:32:C2|4|17|59.144.144.100|53|182.68.215.225|2285|R2R|1|2|5|275|0|0|0|0|0|0|0|5|DNS|localhost|0|REFUSED|||
1556023176.776758|1556023290.867930|0|D4:04:FF:2E:BB:F9|00:23:9C:F5:36:9E|4|17|173.66.100.16|33543|182.74.79.227|40081|R2R|1|2|17|8867|0|0|0|0|0|0|0|0|Unknown||0|0||
1556023168.585437|1556023283.937525|0|00:23:9C:F5:34:D5|D4:04:FF:2E:BC:27|4|17|103.93.209.72|65208|117.222.204.73|1025|R2R|1|2|29|3594|0|0|0|0|0|0|0|0|Unknown||0|0||
1556023177.617739|1556023295.787524|0|84:C1:C1:12:40:50|00:23:9C:F5:36:1F|4|17|197.0.45.98|61491|103.81.215.53|33926|R2R|1|2|293|323196|0|0|0|0|0|0|0|37|BitTorrent||0|0||
1556023174.476213|1556023288.646248|0|00:23:9C:F5:30:52|D4:04:FF:2E:B4:17|4|6|117.234.63.81|44120|178.152.82.25|49967|R2R|1|2|19|1266|0|0|26|0|0|0|0|0|Unknown||0|0||
1556023175.102641|1556023293.828305|0|00:23:9C:F5:30:CE|D4:04:FF:2E:BB:EF|4|6|182.68.18.66|39850|182.79.144.141|443|R2R|1|2|510|27746|1207|1781991|26|0|0|15|0|91|SSL||0|0|r2---sn-ci5gup-qxay.gvt1.com|
1556023176.181269|1556023293.647783|0|84:C1:C1:12:48:2A|00:23:9C:F5:36:48|4|6|220.243.208.116|443|223.185.205.115|50731|R2R|1|2|25|1150|0|0|17|0|0|0|0|91|SSL||0|0||
1556023175.317150|1556023287.384505|0|84:C1:C1:12:40:17|00:23:9C:F5:35:00|4|17|173.66.100.16|50554|182.74.79.227|6600|R2R|1|2|24|11552|0|0|0|0|0|0|0|0|Unknown||0|0||
1556023163.648935|1556023283.650788|0|D4:04:FF:2E:BC:22|00:23:9C:F5:32:C2|4|17|172.217.26.205|443|182.64.167.170|51542|R2R|1|2|9|450|0|0|0|0|0|0|0|126|QUIC.Google||0|0||
1556023168.779856|1556023283.607061|0|84:C1:C1:12:40:17|00:23:9C:F5:35:00|4|17|172.217.160.238|443|182.64.126.72|45780|R2R|1|2|15|935|0|0|0|0|0|0|0|126|QUIC.Google||0|0||

====================================
|| Cento Logs for Kafka type Export ||

{"8":"208.80.127.2","12":"202.56.215.65","10":0,"14":0,"2":1,"1":77,"24":0,"23":0,"22":1554201392,"21":1554201392,"7":53,"11":4540,"6":0,"4":17,"5":0,"57590":5,"57677":"www.frbhjvazapgo.com"}
{"8":"117.220.2.131","12":"85.25.103.30","10":0,"14":0,"2":7,"1":372,"24":0,"23":0,"22":1554201350,"21":1554201392,"7":56106,"11":80,"6":19,"4":6,"5":0,"57590":7}
{"8":"123.136.203.90","12":"40.68.212.154","10":0,"14":0,"2":20,"1":3040,"24":0,"23":0,"22":1554201390,"21":1554201392,"7":60353,"11":3478,"6":0,"4":17,"5":0,"57590":78}
{"8":"106.207.162.1","12":"202.56.215.41","10":0,"14":0,"2":1,"1":62,"24":0,"23":0,"22":1554201392,"21":1554201392,"7":15201,"11":53,"6":0,"4":17,"5":104,"57590":119,"57677":"www.facebook.com"}
{"8":"106.207.241.117","12":"202.56.215.41","10":0,"14":0,"2":1,"1":70,"24":0,"23":0,"22":1554201392,"21":1554201392,"7":8420,"11":53,"6":0,"4":17,"5":104,"57590":239,"57677":"people-pa.googleapis.com"}
{"8":"106.207.215.83","12":"202.56.215.41","10":0,"14":0,"2":1,"1":64,"24":0,"23":0,"22":1554201392,"21":1554201392,"7":10642,"11":53,"6":0,"4":17,"5":104,"57590":5,"57677":"ap.appcloudbox.net"}
{"8":"106.219.255.48","12":"23.57.232.76","10":0,"14":0,"2":2,"1":230,"24":0,"23":0,"22":1554201392,"21":1554201392,"7":39525,"11":443,"6":24,"4":6,"5":104,"57590":91}
{"8":"103.212.128.172","12":"51.38.8.84","10":0,"14":0,"2":4,"1":186,"24":0,"23":0,"22":1554201383,"21":1554201392,"7":39149,"11":52569,"6":22,"4":6,"5":0,"57590":0}
{"8":"208.100.26.228","12":"49.249.147.87","10":0,"14":0,"2":1,"1":46,"24":0,"23":0,"22":1554201392,"21":1554201392,"7":59277,"11":587,"6":2,"4":6,"5":0,"57590":29}
{"8":"106.207.195.3","12":"202.56.215.41","10":0,"14":0,"2":1,"1":72,"24":0,"23":0,"22":1554201392,"21":1554201392,"7":12356,"11":53,"6":0,"4":17,"5":104,"57590":228,"57677":"android.clients.google.com"}
{"8":"117.252.70.57","12":"84.201.146.69","10":0,"14":0,"2":5,"1":749,"24":0,"23":0,"22":1554201391,"21":1554201392,"7":8080,"11":63185,"6":91,"4":6,"5":0,"57590":175}
{"8":"223.185.53.39","12":"202.56.215.41","10":0,"14":0,"2":1,"1":78,"24":0,"23":0,"22":1554201392,"21":1554201392,"7":17631,"11":53,"6":0,"4":17,"5":104,"57590":124,"57677":"r1---sn-h557sns7.googlevideo.com"}
{"8":"182.79.251.161","12":"110.225.82.100","10":0,"14":0,"2":34,"1":1916,"24":0,"23":0,"22":1554201062,"21":1554201392,"7":443,"11":25286,"6":0,"4":17,"5":0,"57590":188}
{"8":"27.60.54.111","12":"202.56.215.41","10":0,"14":0,"2":1,"1":59,"24":0,"23":0,"22":1554201392,"21":1554201392,"7":3860,"11":53,"6":0,"4":17,"5":104,"57590":124,"57677":"s.youtube.com"}
{"8":"117.234.106.117","12":"47.74.184.229","10":0,"14":0,"2":12,"1":1198,"24":0,"23":0,"22":1554201390,"21":1554201392,"7":60322,"11":443,"6":26,"4":6,"5":0,"57590":91,"57659":"api.cloudmobi.net"}
{"8":"223.185.24.82","12":"202.56.215.41","10":0,"14":0,"2":1,"1":68,"24":0,"23":0,"22":1554201392,"21":1554201392,"7":20071,"11":53,"6":0,"4":17,"5":104,"57590":5,"57677":"2.android.pool.ntp.org"}
{"8":"185.153.197.231","12":"116.119.135.234","10":0,"14":0,"2":31,"1":1426,"24":0,"23":0,"22":1554201392,"21":1554201392,"7":53802,"11":3325,"6":2,"4":6,"5":0,"57590":0}
{"8":"14.102.80.137","12":"27.8.255.253","10":0,"14":0,"2":1,"1":42,"24":0,"23":0,"22":1554201392,"21":1554201392,"7":24103,"11":23,"6":2,"4":6,"5":0,"57590":77}
{"8":"223.229.243.74","12":"202.56.215.54","10":0,"14":0,"2":1,"1":61,"24":0,"23":0,"22":1554201392,"21":1554201392,"7":64169,"11":53,"6":0,"4":17,"5":0,"57590":5,"57677":"m30.fcsyzii.com"}
{"8":"172.217.167.10","12":"223.225.189.189","10":0,"14":0,"2":18,"1":21458,"24":0,"23":0,"22":1554201392,"21":1554201392,"7":443,"11":62901,"6":24,"4":6,"5":0,"57590":91,"57659":""}
{"8":"106.207.193.230","12":"202.56.215.41","10":0,"14":0,"2":1,"1":80,"24":0,"23":0,"22":1554201392,"21":1554201392,"7":12400,"11":53,"6":0,"4":17,"5":104,"57590":5,"57677":"presence-grpc-noneu.truecaller.com"}
{"8":"172.217.31.14","12":"223.189.91.228","10":0,"14":0,"2":3,"1":1495,"24":0,"23":0,"22":1554201392,"21":1554201392,"7":443,"11":15933,"6":0,"4":17,"5":0,"57590":188}

=================================================
|| Cento Logs for IPFIX type Export collected by ipfixcol and converted into JSON ||

{"@type": "ipfix.entry", "ipfix.58": 0, "ipfix.sourceMacAddress": "d4:04:ff:2e:bc:22", "ipfix.postDestinationMacAddress": "00:23:9c:f5:32:c2", "ipfix.4": 17, "ipfix.8": "106.215.131.85", "ipfix.7": 6153, "ipfix.12": "202.56.215.41", "ipfix.11": 53, "ipfix.10": 1, "ipfix.14": 2, "ipfix.2": 1, "ipfix.1": 61, "ipfix.152": 1554206414178, "ipfix.153": 1554206414178, "ipfix.6": "......", "ipfix.5": 0, "ipfix.postIpClassOfService": 104, "ipfix.cento_118": 211, "ipfix.cento_123": "", "ipfix.cento_124": "", "ipfix.IPFIXReverseIEPEN_PKTS": 0, "ipfix.IPFIXReverseIEPEN_BYTES": 0}

{"@type": "ipfix.entry", "ipfix.58": 0, "ipfix.sourceMacAddress": "d4:04:ff:2e:bb:ef", "ipfix.postDestinationMacAddress": "00:23:9c:f5:30:ce", "ipfix.4": 17, "ipfix.8": "106.215.213.69", "ipfix.7": 60467, "ipfix.12": "202.56.215.41", "ipfix.11": 53, "ipfix.10": 1, "ipfix.14": 2, "ipfix.2": 1, "ipfix.1": 81, "ipfix.152": 1554206414169, "ipfix.153": 1554206414169, "ipfix.6": "......", "ipfix.5": 104, "ipfix.postIpClassOfService": 0, "ipfix.cento_118": 124, "ipfix.cento_123": "", "ipfix.cento_124": "", "ipfix.IPFIXReverseIEPEN_PKTS": 0, "ipfix.IPFIXReverseIEPEN_BYTES": 0}

{"@type": "ipfix.entry", "ipfix.58": 0, "ipfix.sourceMacAddress": "00:23:9c:f5:30:ce", "ipfix.postDestinationMacAddress": "d4:04:ff:2e:bb:ef", "ipfix.4": 6, "ipfix.8": "49.201.201.232", "ipfix.7": 63067, "ipfix.12": "8.37.239.4", "ipfix.11": 443, "ipfix.10": 1, "ipfix.14": 2, "ipfix.2": 8, "ipfix.1": 1284, "ipfix.152": 1554206407018, "ipfix.153": 1554206414755, "ipfix.6": ".APR.F", "ipfix.5": 0, "ipfix.postIpClassOfService": 0, "ipfix.cento_118": 91, "ipfix.cento_123": "", "ipfix.cento_124": "", "ipfix.IPFIXReverseIEPEN_PKTS": 0, "ipfix.IPFIXReverseIEPEN_BYTES": 0}

{"@type": "ipfix.entry", "ipfix.58": 0, "ipfix.sourceMacAddress": "00:23:9c:f5:30:ce", "ipfix.postDestinationMacAddress": "d4:04:ff:2e:bb:ef", "ipfix.4": 6, "ipfix.8": "14.102.8.38", "ipfix.7": 51955, "ipfix.12": "152.195.35.250", "ipfix.11": 443, "ipfix.10": 1, "ipfix.14": 2, "ipfix.2": 14, "ipfix.1": 2732, "ipfix.152": 1554206413037, "ipfix.153": 1554206414436, "ipfix.6": ".AP..F", "ipfix.5": 0, "ipfix.postIpClassOfService": 0, "ipfix.cento_118": 91, "ipfix.cento_123": "", "ipfix.cento_124": "", "ipfix.IPFIXReverseIEPEN_PKTS": 0, "ipfix.IPFIXReverseIEPEN_BYTES": 0}

{"@type": "ipfix.entry", "ipfix.58": 0, "ipfix.sourceMacAddress": "84:c1:c1:12:40:17", "ipfix.postDestinationMacAddress": "00:23:9c:f5:35:00", "ipfix.4": 17, "ipfix.8": "23.61.199.131", "ipfix.7": 53, "ipfix.12": "202.56.215.65", "ipfix.11": 35712, "ipfix.10": 1, "ipfix.14": 2, "ipfix.2": 1, "ipfix.1": 195, "ipfix.152": 1554206414105, "ipfix.153": 1554206414105, "ipfix.6": "......", "ipfix.5": 0, "ipfix.postIpClassOfService": 0, "ipfix.cento_118": 5, "ipfix.cento_123": "", "ipfix.cento_124": "", "ipfix.IPFIXReverseIEPEN_PKTS": 0, "ipfix.IPFIXReverseIEPEN_BYTES": 0}

{"@type": "ipfix.entry", "ipfix.58": 0, "ipfix.sourceMacAddress": "d4:04:ff:2e:bb:ef", "ipfix.postDestinationMacAddress": "00:23:9c:f5:30:ce", "ipfix.4": 17, "ipfix.8": "106.215.169.230", "ipfix.7": 64206, "ipfix.12": "202.56.215.41", "ipfix.11": 53, "ipfix.10": 1, "ipfix.14": 2, "ipfix.2": 1, "ipfix.1": 71, "ipfix.152": 1554206414548, "ipfix.153": 1554206414548, "ipfix.6": "......", "ipfix.5": 104, "ipfix.postIpClassOfService": 0, "ipfix.cento_118": 126, "ipfix.cento_123": "", "ipfix.cento_124": "", "ipfix.IPFIXReverseIEPEN_PKTS": 0, "ipfix.IPFIXReverseIEPEN_BYTES": 0}

{"@type": "ipfix.entry", "ipfix.58": 0, "ipfix.sourceMacAddress": "84:c1:c1:12:40:17", "ipfix.postDestinationMacAddress": "00:23:9c:f5:35:00", "ipfix.4": 17, "ipfix.8": "106.207.134.52", "ipfix.7": 9841, "ipfix.12": "202.56.215.41", "ipfix.11": 53, "ipfix.10": 1, "ipfix.14": 2, "ipfix.2": 1, "ipfix.1": 60, "ipfix.152": 1554206414275, "ipfix.153": 1554206414275, "ipfix.6": "......", "ipfix.5": 104, "ipfix.postIpClassOfService": 0, "ipfix.cento_118": 5, "ipfix.cento_123": "", "ipfix.cento_124": "", "ipfix.IPFIXReverseIEPEN_PKTS": 0, "ipfix.IPFIXReverseIEPEN_BYTES": 0}

No source files or tests are named. Start by tracing the textual, direct Kafka, and IPFIX export entry points and comparing their Layer 7 field definitions, including DNS_QUERY, DNS_QUERY_TYPE, and DNS_RET_CODE. Done means the Layer 7 metadata shown by the textual dump is consistently represented and populated in the Kafka and IPFIX outputs.