OpenTelemetry log export is missing TCP logs
- Dominant language
- Go
- Stars
- 30.1k
- Forks
- 4.6k
- Avg merge
- 2d 6h
- Merged PRs (30d)
- 43
Description
#### Overview of the Issue
OTEL integration was setup by following the instructions in https://developer.hashicorp.com/consul/docs/envoy-extension/otel and https://discuss.hashicorp.com/t/deploy-opentelemetry-collector-in-service-mesh/60888. The expectation was that all the access logs would be captured in the otel collector. However, on debugging found that consul is adding the envoy.extensions.access_loggers.open_telemetry.v3.OpenTelemetryAccessLogConfig to only envoy.filters.network.http_connection_manager. As a result only the HTTP based requests are collected at OTEL.
#### Consul config dump (snippet)
```
"filters": [
{
"name": "envoy.filters.network.http_connection_manager",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
"stat_prefix": "upstream.otel-collector-opentelemetry-collector.default.default.dc1",
"route_config": {
"name": "otel-collector-opentelemetry-collector",
"virtual_hosts": [
{
"name": "otel-collector-opentelemetry-collector.default.default.dc1",
"domains": [
"*"
],
"routes": [
{
"match": {
"prefix": "/"
},
"route": {
"cluster": "otel-collector-opentelemetry-collector.default.dc1.internal.68f75898-209a-0c49-92fe-a775b96445e3.consul"
}
}
]
}
]
},
"http_filters": [
{
"name": "envoy.filters.http.grpc_stats",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.filters.http.grpc_stats.v3.FilterConfig",
"stats_for_all_methods": true
}
},
{
"name": "envoy.filters.http.grpc_http1_bridge",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.filters.http.grpc_http1_bridge.v3.Config"
}
},
{
"name": "envoy.filters.http.router",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"
}
}
],
"tracing": {
"random_sampling": {
}
},
"http2_protocol_options": {
},
"access_log": [
{
"name": "Consul Listener Filter Log",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog",
"log_format": {
"json_format": {
"user_agent": "%REQ(USER-AGENT)%",
"upstream_cluster": "%UPSTREAM_CLUSTER%",
"requested_server_name": "%REQUESTED_SERVER_NAME%",
"start_time": "%START_TIME%",
"method": "%REQ(:METHOD)%",
"bytes_sent": "%BYTES_SENT%",
"route_name": "%ROUTE_NAME%",
"response_code_details": "%RESPONSE_CODE_DETAILS%",
"duration": "%DURATION%",
"x_forwarded_for": "%REQ(X-FORWARDED-FOR)%",
"authority": "%REQ(:AUTHORITY)%",
"upstream_host": "%UPSTREAM_HOST%",
"upstream_local_address": "%UPSTREAM_LOCAL_ADDRESS%",
"downstream_local_address": "%DOWNSTREAM_LOCAL_ADDRESS%",
"protocol": "%PROTOCOL%",
"response_flags": "%RESPONSE_FLAGS%",
"upstream_service_time": "%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%",
"request_id": "%REQ(X-REQUEST-ID)%",
"downstream_remote_address": "%DOWNSTREAM_REMOTE_ADDRESS%",
"upstream_transport_failure_reason": "%UPSTREAM_TRANSPORT_FAILURE_REASON%",
"path": "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%",
"response_code": "%RESPONSE_CODE%",
"connection_termination_details": "%CONNECTION_TERMINATION_DETAILS%",
"bytes_received": "%BYTES_RECEIVED%"
}
}
}
},
{
"name": "envoy.access_loggers.open_telemetry",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.access_loggers.open_telemetry.v3.OpenTelemetryAccessLogConfig",
"common_config": {
"log_name": "outbound",
"grpc_service": {
"envoy_grpc": {
"cluster_name": "otel-collector-opentelemetry-collector.default.dc1.internal.68f75898-209a-0c49-92fe-a775b96445e3.consul"
}
},
"buffer_size_bytes": 0,
"transport_api_version": "V3"
},
"body": {
"string_value": "[%START_TIME%] \"%REQ(:METHOD)%\" \"%PROTOCOL%\" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% \"%UPSTREAM_HOST%\""
},
"attributes": {
},
"resource_attributes": {
}
}
}
],
"upgrade_configs": [
{
"upgrade_type": "websocket"
}
]
}
}
]
},
{
"filter_chain_match": {
"prefix_ranges": [
{
"address_prefix": "10.0.183.226",
"prefix_len": 32
},
{
"address_prefix": "240.0.0.10",
"prefix_len": 32
}
]
},
"filters": [
{
"name": "envoy.filters.network.tcp_proxy",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy",
"stat_prefix": "upstream.yelb-db.default.default.dc1",
"cluster": "yelb-db.default.dc1.internal.68f75898-209a-0c49-92fe-a775b96445e3.consul",
"access_log": [
{
"name": "Consul Listener Filter Log",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog",
"log_format": {
"json_format": {
"route_name": "%ROUTE_NAME%",
"protocol": "%PROTOCOL%",
"response_code_details": "%RESPONSE_CODE_DETAILS%",
"duration": "%DURATION%",
"request_id": "%REQ(X-REQUEST-ID)%",
"authority": "%REQ(:AUTHORITY)%",
"upstream_local_address": "%UPSTREAM_LOCAL_ADDRESS%",
"downstream_remote_address": "%DOWNSTREAM_REMOTE_ADDRESS%",
"path": "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%",
"response_flags": "%RESPONSE_FLAGS%",
"bytes_received": "%BYTES_RECEIVED%",
"upstream_service_time": "%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%",
"x_forwarded_for": "%REQ(X-FORWARDED-FOR)%",
"user_agent": "%REQ(USER-AGENT)%",
"upstream_host": "%UPSTREAM_HOST%",
"requested_server_name": "%REQUESTED_SERVER_NAME%",
"method": "%REQ(:METHOD)%",
"connection_termination_details": "%CONNECTION_TERMINATION_DETAILS%",
"bytes_sent": "%BYTES_SENT%",
"upstream_cluster": "%UPSTREAM_CLUSTER%",
"downstream_local_address": "%DOWNSTREAM_LOCAL_ADDRESS%",
"upstream_transport_failure_reason": "%UPSTREAM_TRANSPORT_FAILURE_REASON%",
"start_time": "%START_TIME%",
"response_code": "%RESPONSE_CODE%"
}
}
}
}
]
}
}
]
},
{
```
#### Reproduction Steps
Consul is injected on a yelb demo application in yelb namespace. Otel collector is running in otel-collector namespace. Both the namespaces are enrolled into consul service mesh. The traffic flows is as follows
Internet ---(http)---> [permissing mutualTlsMode] yelb-ui -- (http) --> yelb appserver --(tcp)-> yelb-db
Only the yelb-ui to yelb-appserver access logs are exported, the access logs of yelb appserver to yelb-db are not exported to the otel collector
**proxy defaults:**
```
apiVersion: consul.hashicorp.com/v1alpha1
kind: ProxyDefaults
metadata:
name: global
spec:
envoyExtensions:
- name: builtin/otel-access-logging
arguments:
proxyType: connect-proxy
config:
grpcService:
target:
service:
name: otel-collector-opentelemetry-collector
namespace: otel-collector
config:
protocol: "http"
accessLogs:
enabled: true
```
**service defaults:**
```
apiVersion: consul.hashicorp.com/v1alpha1
kind: ServiceDefaults
metadata:
name: otel-collector-opentelemetry-collector
namespace: default
spec:
protocol: grpc
apiVersion: consul.hashicorp.com/v1alpha1
kind: ServiceDefaults
metadata:
name: redis-server
namespace: default
spec:
protocol: tcp
apiVersion: consul.hashicorp.com/v1alpha1
kind: ServiceDefaults
metadata:
name: yelb-appserver
namespace: default
spec:
protocol: http
apiVersion: consul.hashicorp.com/v1alpha1
kind: ServiceDefaults
metadata:
name: yelb-db
namespace: default
spec:
protocol: tcp
apiVersion: consul.hashicorp.com/v1alpha1
kind: ServiceDefaults
metadata:
name: yelb-ui
namespace: default
spec:
mutualTLSMode: permissive
protocol: http
```
Contributor guide
Assessment
This issue has not been assessed yet.