google / google/gvisor

A veth create that fails on the first end leaks the pair's goroutine

Open
#14,514 0 comments 0 reactions 0 assignees View on GitHub
Dominant language
Go
Stars
19.3k
Forks
2k
Avg merge
3d 5h
Merged PRs (30d)
264

Description

veth.NewPair starts a goroutine that only ends when the pair closes, and the first-end failure path (stack.go:484) returns without closing it. A create with no IFLA_IFNAME reaches it, since newVeth picks `veth` (stack.go:478) and that can collide, which is #14518.

Each mode makes twenty creates. The live veth.NewPair.func1 count afterwards was the same in three runs:

| mode | goroutines | links left |
| --- | --- | --- |
| fails on the first end | 20 | [lo] |
| fails on the peer end | 0 | [lo] |
| succeeds | 0 | [lo] |
| refused before the pair exists | 0 | [lo] |

Proposed fix in #14515.

reproducer

```go
// Run as root, one mode per sandbox:
//
// sudo runsc --network=none --ignore-cgroups --platform=systrap do $PWD/vethleak leak 20
// sudo runsc --network=none --ignore-cgroups --platform=systrap do $PWD/vethleak ok 20
// sudo runsc --network=none --ignore-cgroups --platform=systrap do $PWD/vethleak peerfail 20
// sudo runsc --network=none --ignore-cgroups --platform=systrap do $PWD/vethleak noop 20
//
// Every mode ends with only "lo" in the link list, then prints READY and sleeps so
// the caller can run "runsc debug --stacks" against the sandbox.
package main

import (
"encoding/binary"
"fmt"
"os"
"strconv"
"strings"
"syscall"
"time"
)

const (
rtmNewLink = 16
rtmDelLink = 17
rtmGetLink = 18

iflaIfname = 3
iflaLinkInfo = 18

iflaInfoKind = 1
iflaInfoData = 2

vethInfoPeer = 1
)

type builder struct {
b []byte
}

func (bd *builder) attr(typ uint16, payload []byte) {
hdr := make([]byte, 4)
binary.LittleEndian.PutUint16(hdr[0:], uint16(4+len(payload)))
binary.LittleEndian.PutUint16(hdr[2:], typ)
bd.b = append(bd.b, hdr...)
bd.b = append(bd.b, payload...)
for len(bd.b)%4 != 0 {
bd.b = append(bd.b, 0)
}
}

func (bd *builder) str(typ uint16, s string) {
bd.attr(typ, append([]byte(s), 0))
}

type conn struct {
fd int
seq uint32
}

func dial() (*conn, error) {
fd, err := syscall.Socket(syscall.AF_NETLINK, syscall.SOCK_RAW, syscall.NETLINK_ROUTE)
if err != nil {
return nil, err
}
err = syscall.Bind(fd, &syscall.SockaddrNetlink{Family: syscall.AF_NETLINK})
if err != nil {
syscall.Close(fd)
return nil, err
}
return &conn{fd: fd}, nil
}

func (c *conn) send(msgType uint16, flags uint16, body []byte) (uint32, error) {
c.seq++
msg := make([]byte, 16)
binary.LittleEndian.PutUint32(msg[0:], uint32(16+len(body)))
binary.LittleEndian.PutUint16(msg[4:], msgType)
binary.LittleEndian.PutUint16(msg[6:], flags)
binary.LittleEndian.PutUint32(msg[8:], c.seq)
msg = append(msg, body...)
err := syscall.Sendto(c.fd, msg, 0, &syscall.SockaddrNetlink{Family: syscall.AF_NETLINK})
if err != nil {
return 0, err
}
return c.seq, nil
}

func (c *conn) readAck(seq uint32) error {
buf := make([]byte, 65536)
for {
n, _, err := syscall.Recvfrom(c.fd, buf, 0)
if err != nil {
return err
}
for p := buf[:n]; len(p) >= 16; {
l := int(binary.LittleEndian.Uint32(p[0:]))
if l < 16 || l > len(p) {
return fmt.Errorf("short netlink message: len %d of %d", l, len(p))
}
typ := binary.LittleEndian.Uint16(p[4:])
s := binary.LittleEndian.Uint32(p[8:])
if typ == syscall.NLMSG_ERROR && s == seq {
code := int32(binary.LittleEndian.Uint32(p[16:]))
if code == 0 {
return nil
}
return syscall.Errno(-code)
}
p = p[(l+3)&^3:]
}
}
}

func (c *conn) request(msgType uint16, flags uint16, body []byte) error {
seq, err := c.send(msgType, flags, body)
if err != nil {
return err
}
return c.readAck(seq)
}

func ifinfomsg() []byte {
return make([]byte, 16)
}

func vethBody(name, peerName string) []byte {
peer := builder{}
peer.b = append(peer.b, ifinfomsg()...)
if peerName != "" {
peer.str(iflaIfname, peerName)
}

data := builder{}
data.attr(vethInfoPeer, peer.b)

info := builder{}
info.str(iflaInfoKind, "veth")
info.attr(iflaInfoData, data.b)

msg := builder{}
msg.b = append(msg.b, ifinfomsg()...)
if name != "" {
msg.str(iflaIfname, name)
}
msg.attr(iflaLinkInfo, info.b)
return msg.b
}

func unsupportedKindBody(name string) []byte {
info := builder{}
info.str(iflaInfoKind, "dummy")

msg := builder{}
msg.b = append(msg.b, ifinfomsg()...)
msg.str(iflaIfname, name)
msg.attr(iflaLinkInfo, info.b)
return msg.b
}

func (c *conn) addVeth(name, peerName string) error {
flags := uint16(syscall.NLM_F_REQUEST | syscall.NLM_F_ACK | syscall.NLM_F_CREATE | syscall.NLM_F_EXCL)
return c.request(rtmNewLink, flags, vethBody(name, peerName))
}

func (c *conn) addUnsupported(name string) error {
flags := uint16(syscall.NLM_F_REQUEST | syscall.NLM_F_ACK | syscall.NLM_F_CREATE | syscall.NLM_F_EXCL)
return c.request(rtmNewLink, flags, unsupportedKindBody(name))
}

func (c *conn) delLink(name string) error {
msg := builder{}
msg.b = append(msg.b, ifinfomsg()...)
msg.str(iflaIfname, name)
return c.request(rtmDelLink, syscall.NLM_F_REQUEST|syscall.NLM_F_ACK, msg.b)
}

func (c *conn) links() ([]string, error) {
flags := uint16(syscall.NLM_F_REQUEST | syscall.NLM_F_DUMP)
seq, err := c.send(rtmGetLink, flags, ifinfomsg())
if err != nil {
return nil, err
}

var names []string
buf := make([]byte, 65536)
for {
n, _, err := syscall.Recvfrom(c.fd, buf, 0)
if err != nil {
return nil, err
}
for p := buf[:n]; len(p) >= 16; {
l := int(binary.LittleEndian.Uint32(p[0:]))
if l < 16 || l > len(p) {
return nil, fmt.Errorf("short netlink message: len %d of %d", l, len(p))
}
typ := binary.LittleEndian.Uint16(p[4:])
s := binary.LittleEndian.Uint32(p[8:])
if s == seq {
switch typ {
case syscall.NLMSG_DONE:
return names, nil

case syscall.NLMSG_ERROR:
code := int32(binary.LittleEndian.Uint32(p[16:]))
if code != 0 {
return nil, syscall.Errno(-code)
}
return names, nil

case rtmNewLink:
name, ok := ifnameOf(p[16:l])
if ok {
names = append(names, name)
}
}
}
p = p[(l+3)&^3:]
}
}
}

func ifnameOf(body []byte) (string, bool) {
if len(body) < 16 {
return "", false
}
for p := body[16:]; len(p) >= 4; {
l := int(binary.LittleEndian.Uint16(p[0:]))
typ := binary.LittleEndian.Uint16(p[2:])
if l < 4 || l > len(p) {
return "", false
}
if typ == iflaIfname {
return strings.TrimRight(string(p[4:l]), "\x00"), true
}
p = p[(l+3)&^3:]
}
return "", false
}

func (c *conn) nextNICID() (int, error) {
err := c.addVeth("" /* name */, "" /* peerName */)
if err != nil {
return 0, fmt.Errorf("probe create: %w", err)
}
names, err := c.links()
if err != nil {
return 0, fmt.Errorf("probe list: %w", err)
}
first := 0
for _, name := range names {
if !strings.HasPrefix(name, "veth") {
continue
}
id, err := strconv.Atoi(strings.TrimPrefix(name, "veth"))
if err != nil {
continue
}
if first == 0 || id < first {
first = id
}
}
if first == 0 {
return 0, fmt.Errorf("probe pair not found in %v", names)
}
err = c.delLink(fmt.Sprintf("veth%d", first))
if err != nil {
return 0, fmt.Errorf("probe delete: %w", err)
}
return first + 2, nil
}

func runLeak(c *conn, n int) error {
base, err := c.nextNICID()
if err != nil {
return err
}

collide := base + 2*n
for i := 0; i < n; i++ {
err = c.addVeth(fmt.Sprintf("veth%d", collide+2*i), fmt.Sprintf("anchor%d", i))
if err != nil {
return fmt.Errorf("anchor %d: %w", i, err)
}
}
for i := 0; i < n; i++ {
err = c.addVeth("" /* name */, fmt.Sprintf("unused%d", i))
if err != syscall.EEXIST {
return fmt.Errorf("create %d: want EEXIST, got %v", i, err)
}
}
for i := 0; i < n; i++ {
err = c.delLink(fmt.Sprintf("veth%d", collide+2*i))
if err != nil {
return fmt.Errorf("anchor %d delete: %w", i, err)
}
}
fmt.Printf("%d creates refused with EEXIST on the first end\n", n)
return nil
}

func runOK(c *conn, n int) error {
_, err := c.nextNICID()
if err != nil {
return err
}

for i := 0; i < n; i++ {
err = c.addVeth(fmt.Sprintf("good%d", i), fmt.Sprintf("goodpeer%d", i))
if err != nil {
return fmt.Errorf("create %d: %w", i, err)
}
}
for i := 0; i < n; i++ {
err = c.delLink(fmt.Sprintf("good%d", i))
if err != nil {
return fmt.Errorf("delete %d: %w", i, err)
}
}
fmt.Printf("%d pairs created and deleted\n", n)
return nil
}

func runPeerFail(c *conn, n int) error {
_, err := c.nextNICID()
if err != nil {
return err
}

err = c.addVeth("taken", "takenpeer")
if err != nil {
return fmt.Errorf("anchor: %w", err)
}
for i := 0; i < n; i++ {
err = c.addVeth(fmt.Sprintf("second%d", i), "taken")
if err != syscall.EEXIST {
return fmt.Errorf("create %d: want EEXIST, got %v", i, err)
}
}
err = c.delLink("taken")
if err != nil {
return fmt.Errorf("anchor delete: %w", err)
}
fmt.Printf("%d creates refused with EEXIST on the peer end\n", n)
return nil
}

func runNoop(c *conn, n int) error {
_, err := c.nextNICID()
if err != nil {
return err
}

for i := 0; i < n; i++ {
err = c.addUnsupported(fmt.Sprintf("dummy%d", i))
if err != syscall.EOPNOTSUPP {
return fmt.Errorf("create %d: want EOPNOTSUPP, got %v", i, err)
}
}
fmt.Printf("%d creates refused before the pair is built\n", n)
return nil
}

func main() {
mode := "leak"
n := 20
if len(os.Args) > 1 {
mode = os.Args[1]
}
if len(os.Args) > 2 {
v, err := strconv.Atoi(os.Args[2])
if err != nil {
fmt.Printf("bad count %q: %v\n", os.Args[2], err)
os.Exit(1)
}
n = v
}

c, err := dial()
if err != nil {
fmt.Printf("netlink: %v\n", err)
os.Exit(1)
}

switch mode {
case "leak":
err = runLeak(c, n)

case "ok":
err = runOK(c, n)

case "peerfail":
err = runPeerFail(c, n)

case "noop":
err = runNoop(c, n)

default:
err = fmt.Errorf("unknown mode %q", mode)
}
if err != nil {
fmt.Printf("%s: %v\n", mode, err)
os.Exit(1)
}

names, err := c.links()
if err != nil {
fmt.Printf("list: %v\n", err)
os.Exit(1)
}
fmt.Printf("links left: %v\n", names)
fmt.Printf("READY %s %d\n", mode, n)
os.Stdout.Sync()
time.Sleep(120 * time.Second)
}
```

Contributor guide

Open the contributing guide

Research direction

Read stack.go around lines 478 and 484, focusing on veth.NewPair and the first-end failure path. Run the supplied leak reproducer as root and compare its goroutine and link counts with the peerfail, ok, and noop modes; done means first-end failures leave no veth.NewPair.func1 goroutines and only [lo] remains.

Written by the indexing model from the issue text.

Assessment

Tech stack
go, linux
Domain
networking
Issue type
Bug
Difficulty
3/5
Estimated time
1-2 days
Activity status
Stale
Clarity
Mostly clear
Newbie friendliness
35/100

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.