iovisor / iovisor/bcc

verifier failure for a xdp code computing udp checksum

Open
#2,463 18 comments 0 reactions 0 assignees View on GitHub
Dominant language
C
Stars
22.7k
Forks
4.1k
Avg merge
10d 4h
Merged PRs (30d)
3

Description

I filed this issue based on some old discussions in iovisor-dev mailing list.
https://lists.iovisor.org/g/iovisor-dev/topic/30285987

I finally got some time to explore the solution for this and created a standalone example.
Just to recap, the following is a simplified program.
```
-bash-4.4$ cat xdp_simple.c
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include

#include "bpf_helpers.h"
#include "bpf_endian.h"

/* 0x3FFF mask to check for fragment offset field */
#define IP_FRAGMENTED 65343

// MAC address
typedef unsigned char mac[6];

// Real Server structure (MAC address + IP address)
struct server {
__be32 ipAddr;
unsigned char macAddr[ETH_ALEN];
};

// packet structure to log load balancing
struct packet {
unsigned char dmac[ETH_ALEN];
unsigned char smac[ETH_ALEN];
__be32 daddr;
__be32 saddr;
};

struct {
__uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
__uint(key_size, sizeof(int));
__uint(value_size, sizeof(int));
} events SEC(".maps");

// A map which contains port to redirect
struct {
__uint(type, BPF_MAP_TYPE_HASH);
__uint(max_entries, 10);
__type(key, __be16);
__type(value, int);
} ports SEC(".maps");

// A map which contains real server
struct {
__uint(type, BPF_MAP_TYPE_HASH);
__uint(max_entries, 10);
__type(key, int);
__type(value, struct server);
} realServers SEC(".maps");
// Virtual IP is accessible via the '0x5' constant

SEC("xdpclient")
int xdp_prog(struct xdp_md *ctx) {
void *data_end = (void *)(long)ctx->data_end;
void *data = (void *)(long)ctx->data;

struct ethhdr * eth = data;
if (eth + 1 > data_end)
return XDP_DROP;

if (eth->h_proto != bpf_htons(ETH_P_IP)){
return XDP_PASS;
}

struct iphdr *iph;
iph = eth + 1;
if (iph + 1 > data_end)
return XDP_DROP;

if (iph->ihl < 5)
return XDP_DROP;
if (iph->ihl != 5)
return XDP_PASS;

if (iph->protocol != IPPROTO_UDP) {
return XDP_PASS;
}

struct udphdr *udp;
udp = iph + 1;
if (udp + 1 > data_end)
return XDP_DROP;
__u16 udp_len = bpf_ntohs(udp->len);
if (udp_len < 8 || udp_len >= 512)
return XDP_DROP;
udp_len &= 0x1ff;
if ((void *) udp + udp_len > data_end)
return XDP_DROP;

// Update UDP checksum
__u64 cs = 0;
udp->check = 0;
cs = bpf_csum_diff(0, 0, udp, udp_len, cs);
udp->check = cs;

// Log packet after
struct packet pkt = {};
memcpy(&pkt, data, sizeof(pkt)); // crappy
pkt.daddr = iph->daddr;
pkt.saddr = iph->saddr;
bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, &pkt,sizeof(pkt));

return XDP_TX;
}

char _license[] SEC("license") = "GPL";
```

The compilation command line looks like
```
clang -target bpf -O2 -c -I/home/yhs/work/linux/tools/include/uapi -I. -g xdp_simple.c
```
where the above include path pointing to linux repo for two helper header files.

Using bpftool (linux/tools/bpf/bpftool) to load and can reproduce the verifier issue:
```
./bpftool -d prog load ./xdp_simple.o /sys/fs/bpf/xdp_example type xdp
```
```
45: R0_w=inv1 R1_w=inv0 R2_w=pkt(id=1,off=34,r=34,umax_value=511,var_off=(0x0; 0x1ff)) R3=pkt(id=0,off=34,r=42,imm=0) R4_w=invP(id=0,umax_value=511,var_off=(0x0; 0x1ff)
) R6=ctx(id=0,off=0,imm=0) R7=pkt(id=0,off=0,r=42,imm=0) R10=fp0
; udp->check = 0;
45: (6b) *(u16 *)(r7 +40) = r1
46: R0_w=inv1 R1_w=inv0 R2_w=pkt(id=1,off=34,r=34,umax_value=511,var_off=(0x0; 0x1ff)) R3=pkt(id=0,off=34,r=42,imm=0) R4_w=invP(id=0,umax_value=511,var_off=(0x0; 0x1ff)
) R6=ctx(id=0,off=0,imm=0) R7=pkt(id=0,off=0,r=42,imm=0) R10=fp0
; cs = bpf_csum_diff(0, 0, udp, udp_len, cs);
46: (b7) r1 = 0
47: R0_w=inv1 R1_w=inv0 R2_w=pkt(id=1,off=34,r=34,umax_value=511,var_off=(0x0; 0x1ff)) R3=pkt(id=0,off=34,r=42,imm=0) R4_w=invP(id=0,umax_value=511,var_off=(0x0; 0x1ff)
) R6=ctx(id=0,off=0,imm=0) R7=pkt(id=0,off=0,r=42,imm=0) R10=fp0
47: (b7) r2 = 0
48: R0_w=inv1 R1_w=inv0 R2_w=inv0 R3=pkt(id=0,off=34,r=42,imm=0) R4_w=invP(id=0,umax_value=511,var_off=(0x0; 0x1ff)) R6=ctx(id=0,off=0,imm=0) R7=pkt(id=0,off=0,r=42,imm
=0) R10=fp0
48: (b7) r5 = 0
49: R0_w=inv1 R1_w=inv0 R2_w=inv0 R3=pkt(id=0,off=34,r=42,imm=0) R4_w=invP(id=0,umax_value=511,var_off=(0x0; 0x1ff)) R5_w=inv0 R6=ctx(id=0,off=0,imm=0) R7=pkt(id=0,off=
0,r=42,imm=0) R10=fp0
49: (85) call bpf_csum_diff#28
last_idx 49 first_idx 37
regs=4 stack=0 before 48: (b7) r5 = 0
regs=4 stack=0 before 47: (b7) r2 = 0
invalid access to packet, off=34 size=511, R3(id=0,off=34,r=42)
R3 offset is outside of the packet
```

Contributor guide

No contributing guide indexed for this repository

Research direction

Reproduce the failure with xdp_simple.c using the shown clang command and bpftool prog load invocation. Start with the verifier trace around bpf_csum_diff, udp_len, and packet bounds; done means the reported verifier failure is explained and the appropriate behavior is established.

Written by the indexing model from the issue text.

Assessment

Tech stack
c, linux
Domain
networking, operating-systems
Issue type
Bug
Difficulty
4/5
Estimated time
3-5 days
Activity status
Stale
Clarity
Needs clarification
Newbie friendliness
35/100

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.