Burrow failed to connect kafka cluster (kafka: client has run out of available brokers to talk to)

Nobody has claimed this yet.

Hi,

We have issue with burrow, which fails to connect kafka cluster, with below error.

Kafka.log
-----------

We see below info in server.log file.

[2022-07-26 15:01:24,823] INFO [SocketServer brokerId=1] Failed authentication with /xx.xx.xx.xx (SSL handshake failed) (org.apache.kafka.common.network.Selector)

Burrow.log
-------------
{"level":"info","ts":1657739261.2620974,"msg":"configuring","type":"coordinator","name":"httpserver"}
{"level":"info","ts":1657739261.2879257,"msg":"configuring","type":"coordinator","name":"notifier"}
{"level":"info","ts":1657739261.287961,"msg":"configuring","type":"coordinator","name":"cluster"}
{"level":"info","ts":1657739261.2879865,"msg":"configuring","type":"module","coordinator":"cluster","class":"kafka","name":"kafka"}
{"level":"info","ts":1657739261.2911863,"msg":"configuring","type":"coordinator","name":"consumer"}
{"level":"info","ts":1657739261.2912247,"msg":"configuring","type":"module","coordinator":"consumer","class":"kafka","name":"burrow"}
{"level":"info","ts":1657739261.2921846,"msg":"starting","type":"coordinator","name":"zookeeper"}
{"level":"info","ts":1657739261.3041592,"msg":"Connected to xx.xx.xx.xx:xxxx","type":"coordinator","name":"zookeeper"}
{"level":"info","ts":1657739261.321232,"msg":"Authenticated: id=144154910105734385, timeout=6000","type":"coordinator","name":"zookeeper"}
{"level":"info","ts":1657739261.3213053,"msg":"Re-submitting `0` credentials after reconnect","type":"coordinator","name":"zookeeper"}
{"level":"info","ts":1657739261.3518999,"msg":"starting","type":"coordinator","name":"storage"}
{"level":"info","ts":1657739261.351937,"msg":"starting","type":"module","coordinator":"storage","class":"inmemory","name":"default"}
{"level":"info","ts":1657739261.3519754,"msg":"starting","type":"coordinator","name":"evaluator"}
{"level":"info","ts":1657739261.3519819,"msg":"starting","type":"module","coordinator":"evaluator","class":"caching","name":"default"}
{"level":"info","ts":1657739261.3519928,"msg":"starting","type":"coordinator","name":"httpserver"}
{"level":"info","ts":1657739261.352134,"msg":"started listener","type":"coordinator","name":"httpserver","listener":"[::]:9100"}
{"level":"info","ts":1657739261.3521514,"msg":"starting","type":"coordinator","name":"notifier"}
{"level":"info","ts":1657739261.3521652,"msg":"starting","type":"coordinator","name":"cluster"}
{"level":"info","ts":1657739261.3521693,"msg":"starting","type":"module","coordinator":"cluster","class":"kafka","name":"kafka"}
{"level":"info","ts":1657739261.4988637,"msg":"starting evaluations","type":"coordinator","name":"notifier"}
{"level":"error","ts":1657739263.573983,"msg":"failed to start client","type":"module","coordinator":"cluster","class":"kafka","name":"kafka","error":"kafka: client has run out of available brokers to talk to (Is your cluster reachable?)"}
{"level":"info","ts":1657739263.5740325,"msg":"stopping","type":"coordinator","name":"notifier"}
{"level":"info","ts":1657739263.5740402,"msg":"shutdown","type":"coordinator","name":"httpserver"}
{"level":"info","ts":1657739263.5740795,"msg":"stopping","type":"coordinator","name":"evaluator"}
{"level":"info","ts":1657739263.5740893,"msg":"stopping","type":"module","coordinator":"evaluator","class":"caching","name":"default"}
{"level":"info","ts":1657739263.5740964,"msg":"stopping","type":"coordinator","name":"storage"}
{"level":"info","ts":1657739263.574107,"msg":"stopping","type":"module","coordinator":"storage","class":"inmemory","name":"default"}
{"level":"info","ts":1657739263.5741801,"msg":"stopping","type":"coordinator","name":"zookeeper"}
{"level":"info","ts":1657739263.5933099,"msg":"Recv loop terminated: err=EOF","type":"coordinator","name":"zookeeper"}
{"level":"info","ts":1657739263.5933375,"msg":"Send loop terminated: err=","type":"coordinator","name":"zookeeper"}

Burrow.toml
--------------
[general]
pidfile="burrow.pid"
stdout-logfile="burrow.out"

[logging]
filename="burrow.log"
level="info"
maxsize=100
maxbackups=5
maxage=7
use-localtime=true
use-compression=true

[zookeeper]
servers=[ "host:port","host:port","host:port" ]
timeout=6
root-path="/burrow/"

[client-profile.tlsonly]
client-id=""
kafka-version="0.10.2.1"
tls="tlsprofile"

[tls.tlsprofile]
certfile=""
keyfile=""
cafile=""
noverify=true

[cluster.kafka]
class-name="kafka"
servers=[ "host:port","host:port","host:port" ]
client-profile="tlsonly"
topic-refresh=120
offset-refresh=30

[consumer.burrow]
class-name="kafka"
cluster="kafka"
servers=[ "host:port","host:port","host:port" ]
client-profile="tlsonly"
group-blacklist="^(console-consumer-|python-kafka-consumer-|quick-).*$"
group-whitelist=""

[httpserver.burrowListener]
address=":9100"
tls="tlsprofile"

Any debugging help would be appriciated.

Regards,
Swapnil

Start with the Burrow.toml client-profile and cluster settings, then compare them with the Kafka server.log and burrow.log entries showing the SSL handshake and broker-availability failures. Done means identifying and documenting a reproducible configuration or connectivity cause; the payload does not name a source file or test.

Written by the indexing model from the issue text.