Weirdness with lan peering in 0.10

### Checklist

- [X] This is a bug report, not a question. Ask questions on [discuss.ipfs.io](https://discuss.ipfs.io).
- [X] I have searched on the [issue tracker](https://github.com/ipfs/go-ipfs/issues?q=is%3Aissue) for my bug.
- [X] I am running the latest [go-ipfs version](https://dist.ipfs.io/#go-ipfs) or have an issue updating.

### Installation method

ipfs-desktop

### Version

```Text
go-ipfs version: 0.10.0
Repo version: 11
System version: amd64/darwin
Golang version: go1.16.8
```

### Config

```json
{
"API": {
"HTTPHeaders": {}
},
"Addresses": {
"API": "/ip4/127.0.0.1/tcp/5001",
"Announce": [],
"Gateway": "/ip4/127.0.0.1/tcp/8080",
"NoAnnounce": [],
"Swarm": [
"/ip4/0.0.0.0/tcp/4001",
"/ip6/::/tcp/4001",
"/ip4/0.0.0.0/udp/4001/quic",
"/ip6/::/udp/4001/quic"
]
},
"AutoNAT": {},
"Bootstrap": [
"/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN",
"/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa",
"/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb",
"/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt",
"/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ",
"/ip4/104.131.131.82/udp/4001/quic/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ"
],
"DNS": {
"Resolvers": null
},
"Datastore": {
"BloomFilterSize": 0,
"GCPeriod": "1h",
"HashOnRead": false,
"Spec": {
"mounts": [
{
"child": {
"path": "blocks",
"shardFunc": "/repo/flatfs/shard/v1/next-to-last/2",
"sync": true,
"type": "flatfs"
},
"mountpoint": "/blocks",
"prefix": "flatfs.datastore",
"type": "measure"
},
{
"child": {
"compression": "none",
"path": "datastore",
"type": "levelds"
},
"mountpoint": "/",
"prefix": "leveldb.datastore",
"type": "measure"
}
],
"type": "mount"
},
"StorageGCWatermark": 90,
"StorageMax": "10GB"
},
"Discovery": {
"MDNS": {
"Enabled": true,
"Interval": 10
}
},
"Experimental": {
"AcceleratedDHTClient": false,
"FilestoreEnabled": false,
"GraphsyncEnabled": false,
"Libp2pStreamMounting": false,
"P2pHttpProxy": false,
"ShardingEnabled": false,
"StrategicProviding": false,
"UrlstoreEnabled": false
},
"Gateway": {
"APICommands": [],
"HTTPHeaders": {
"Access-Control-Allow-Headers": [
"X-Requested-With",
"Range",
"User-Agent"
],
"Access-Control-Allow-Methods": [
"GET"
],
"Access-Control-Allow-Origin": [
"*"
]
},
"NoDNSLink": false,
"NoFetch": false,
"PathPrefixes": [],
"PublicGateways": null,
"RootRedirect": "",
"Writable": false
},
"Identity": {
"PeerID": "12D3KooW...T"
},
"Internal": {},
"Ipns": {
"RecordLifetime": "",
"RepublishPeriod": "",
"ResolveCacheSize": 128
},
"Migration": {
"DownloadSources": null,
"Keep": ""
},
"Mounts": {
"FuseAllowOther": false,
"IPFS": "/ipfs",
"IPNS": "/ipns"
},
"Peering": {
"Peers": [
{
"Addrs": [],
"ID": "12D3KooW...V"
}
]
},
"Pinning": {},
"Plugins": {
"Plugins": null
},
"Provider": {
"Strategy": ""
},
"Pubsub": {
"DisableSigning": false,
"Router": ""
},
"Reprovider": {
"Interval": "12h",
"Strategy": "all"
},
"Routing": {
"Type": "dhtclient"
},
"Swarm": {
"AddrFilters": null,
"ConnMgr": {
"GracePeriod": "60s",
"HighWater": 40,
"LowWater": 20,
"Type": "basic"
},
"DisableBandwidthMetrics": false,
"DisableNatPortMap": false,
"EnableAutoRelay": false,
"EnableRelayHop": false,
"Transports": {
"Multiplexers": {},
"Network": {},
"Security": {}
}
}
}
```

### Description

I have a peering agreement between two nodes on my LAN (by peerID only). When using previous versions, this resulted in one connection between the two. With 0.10, this results in many connections, with duplicate connections using the same addresses and even, at times, duplicate connections in both directions. I assume this is not intentional. Here is an example dump (peerID redacted):

```
> ipfs swarm peers --direction | grep ...T
/ip4/192.168.2.11/tcp/4001/p2p/12D3KooW...T inbound
/ip4/192.168.2.11/tcp/4001/p2p/12D3KooW...T inbound
/ip4/192.168.2.11/udp/4001/quic/p2p/12D3KooW...T inbound
/ip4/192.168.2.11/udp/4001/quic/p2p/12D3KooW...T inbound
/ip6/fd2c:c3b0:dca4:1:14a4:f52b:ec32:3ff7/tcp/4001/p2p/12D3KooW...T inbound
```

If it matters at all, this node is running dhtclient, the other is running dht (and is an active server). The dump is from the server node's point of view.