verifier failure for a xdp code computing udp checksum
- Dominant language
- C
- Stars
- 22.7k
- Forks
- 4.1k
- Avg merge
- 10d 4h
- Merged PRs (30d)
- 3
Description
I filed this issue based on some old discussions in iovisor-dev mailing list.
https://lists.iovisor.org/g/iovisor-dev/topic/30285987
I finally got some time to explore the solution for this and created a standalone example.
Just to recap, the following is a simplified program.
```
-bash-4.4$ cat xdp_simple.c
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include "bpf_helpers.h"
#include "bpf_endian.h"
/* 0x3FFF mask to check for fragment offset field */
#define IP_FRAGMENTED 65343
// MAC address
typedef unsigned char mac[6];
// Real Server structure (MAC address + IP address)
struct server {
__be32 ipAddr;
unsigned char macAddr[ETH_ALEN];
};
// packet structure to log load balancing
struct packet {
unsigned char dmac[ETH_ALEN];
unsigned char smac[ETH_ALEN];
__be32 daddr;
__be32 saddr;
};
struct {
__uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
__uint(key_size, sizeof(int));
__uint(value_size, sizeof(int));
} events SEC(".maps");
// A map which contains port to redirect
struct {
__uint(type, BPF_MAP_TYPE_HASH);
__uint(max_entries, 10);
__type(key, __be16);
__type(value, int);
} ports SEC(".maps");
// A map which contains real server
struct {
__uint(type, BPF_MAP_TYPE_HASH);
__uint(max_entries, 10);
__type(key, int);
__type(value, struct server);
} realServers SEC(".maps");
// Virtual IP is accessible via the '0x5' constant
SEC("xdpclient")
int xdp_prog(struct xdp_md *ctx) {
void *data_end = (void *)(long)ctx->data_end;
void *data = (void *)(long)ctx->data;
struct ethhdr * eth = data;
if (eth + 1 > data_end)
return XDP_DROP;
if (eth->h_proto != bpf_htons(ETH_P_IP)){
return XDP_PASS;
}
struct iphdr *iph;
iph = eth + 1;
if (iph + 1 > data_end)
return XDP_DROP;
if (iph->ihl < 5)
return XDP_DROP;
if (iph->ihl != 5)
return XDP_PASS;
if (iph->protocol != IPPROTO_UDP) {
return XDP_PASS;
}
struct udphdr *udp;
udp = iph + 1;
if (udp + 1 > data_end)
return XDP_DROP;
__u16 udp_len = bpf_ntohs(udp->len);
if (udp_len < 8 || udp_len >= 512)
return XDP_DROP;
udp_len &= 0x1ff;
if ((void *) udp + udp_len > data_end)
return XDP_DROP;
// Update UDP checksum
__u64 cs = 0;
udp->check = 0;
cs = bpf_csum_diff(0, 0, udp, udp_len, cs);
udp->check = cs;
// Log packet after
struct packet pkt = {};
memcpy(&pkt, data, sizeof(pkt)); // crappy
pkt.daddr = iph->daddr;
pkt.saddr = iph->saddr;
bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, &pkt,sizeof(pkt));
return XDP_TX;
}
char _license[] SEC("license") = "GPL";
```
The compilation command line looks like
```
clang -target bpf -O2 -c -I/home/yhs/work/linux/tools/include/uapi -I. -g xdp_simple.c
```
where the above include path pointing to linux repo for two helper header files.
Using bpftool (linux/tools/bpf/bpftool) to load and can reproduce the verifier issue:
```
./bpftool -d prog load ./xdp_simple.o /sys/fs/bpf/xdp_example type xdp
```
```
45: R0_w=inv1 R1_w=inv0 R2_w=pkt(id=1,off=34,r=34,umax_value=511,var_off=(0x0; 0x1ff)) R3=pkt(id=0,off=34,r=42,imm=0) R4_w=invP(id=0,umax_value=511,var_off=(0x0; 0x1ff)
) R6=ctx(id=0,off=0,imm=0) R7=pkt(id=0,off=0,r=42,imm=0) R10=fp0
; udp->check = 0;
45: (6b) *(u16 *)(r7 +40) = r1
46: R0_w=inv1 R1_w=inv0 R2_w=pkt(id=1,off=34,r=34,umax_value=511,var_off=(0x0; 0x1ff)) R3=pkt(id=0,off=34,r=42,imm=0) R4_w=invP(id=0,umax_value=511,var_off=(0x0; 0x1ff)
) R6=ctx(id=0,off=0,imm=0) R7=pkt(id=0,off=0,r=42,imm=0) R10=fp0
; cs = bpf_csum_diff(0, 0, udp, udp_len, cs);
46: (b7) r1 = 0
47: R0_w=inv1 R1_w=inv0 R2_w=pkt(id=1,off=34,r=34,umax_value=511,var_off=(0x0; 0x1ff)) R3=pkt(id=0,off=34,r=42,imm=0) R4_w=invP(id=0,umax_value=511,var_off=(0x0; 0x1ff)
) R6=ctx(id=0,off=0,imm=0) R7=pkt(id=0,off=0,r=42,imm=0) R10=fp0
47: (b7) r2 = 0
48: R0_w=inv1 R1_w=inv0 R2_w=inv0 R3=pkt(id=0,off=34,r=42,imm=0) R4_w=invP(id=0,umax_value=511,var_off=(0x0; 0x1ff)) R6=ctx(id=0,off=0,imm=0) R7=pkt(id=0,off=0,r=42,imm
=0) R10=fp0
48: (b7) r5 = 0
49: R0_w=inv1 R1_w=inv0 R2_w=inv0 R3=pkt(id=0,off=34,r=42,imm=0) R4_w=invP(id=0,umax_value=511,var_off=(0x0; 0x1ff)) R5_w=inv0 R6=ctx(id=0,off=0,imm=0) R7=pkt(id=0,off=
0,r=42,imm=0) R10=fp0
49: (85) call bpf_csum_diff#28
last_idx 49 first_idx 37
regs=4 stack=0 before 48: (b7) r5 = 0
regs=4 stack=0 before 47: (b7) r2 = 0
invalid access to packet, off=34 size=511, R3(id=0,off=34,r=42)
R3 offset is outside of the packet
```
Contributor guide
No contributing guide indexed for this repository
Research direction
Reproduce the failure with xdp_simple.c using the shown clang command and bpftool prog load invocation. Start with the verifier trace around bpf_csum_diff, udp_len, and packet bounds; done means the reported verifier failure is explained and the appropriate behavior is established.
Written by the indexing model from the issue text.
Assessment
- Tech stack
- c, linux
- Domain
- networking, operating-systems
- Issue type
- Bug
- Difficulty
- 4/5
- Estimated time
- 3-5 days
- Activity status
- Stale
- Clarity
- Needs clarification
- Newbie friendliness
- 35/100