Need help in XDP program failing to load with error "R4 min value is negative, either use unsigned or 'var &= const'"

Question

I have written a XDP program that looks at the incoming TCP packets.

Basically I'm exchanging the destination IPv4 address to another server on the same network

Observation: If I put a fixed value in the function instead of the tcp_len variable the problem goes away, or if I add the following check:

if(tcp_len > 20){
    return XDP_PASS;
}

the error also disappears

I need to fix this problem, I believe the error is related to my function to calculate the checksum or some other detail that I'm missing

Following is the code:

// SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
// Copyright (c) 2018 Netronome Systems, Inc.
#define BPF_NO_GLOBAL_DATA

#define MAX_PACKET_OFF 0xffff

#include <stdbool.h>
#include <stddef.h>
#include <string.h>
#include <linux/bpf.h>
#include <linux/icmp.h>
#include <linux/if_ether.h>
#include <linux/if_vlan.h>
#include <linux/in.h>
#include <linux/ip.h>
#include <linux/tcp.h>
#include <linux/udp.h>
#include <linux/ipv6.h>

#include "bpf_endian.h"
#include "bpf_helpers.h"
#include "jhash.h"

#include <stdint.h>

#define htons(x) ((__be16)___constant_swab16((x)))
#define htonl(x) ((__be32)___constant_swab32((x)))

__attribute__((__always_inline__))
static inline __u16 csum_fold_helper(__u64 csum) {
    int i;
#pragma unroll
    for (i = 0; i < 4; i++) {
        if (csum >> 16)
            csum = (csum & 0xffff) + (csum >> 16);
    }
    return ~csum;
}

__attribute__((__always_inline__))
static inline void ipv4_csum(void* data_start, int data_size, __u64* csum) {
    *csum = bpf_csum_diff(0, 0, data_start, data_size, *csum);
    *csum = csum_fold_helper(*csum);
}

__attribute__((__always_inline__))
static inline void ipv4_l4_csum(void* data_start, __u32 data_size,
    __u64* csum, struct iphdr* iph) {
    __u32 tmp = 0;
    *csum = bpf_csum_diff(0, 0, &iph->saddr, sizeof(__be32), *csum);
    *csum = bpf_csum_diff(0, 0, &iph->daddr, sizeof(__be32), *csum);
    tmp = __builtin_bswap32((__u32)(iph->protocol));
    *csum = bpf_csum_diff(0, 0, &tmp, sizeof(__u32), *csum);
    tmp = __builtin_bswap32((__u32)(data_size));
    *csum = bpf_csum_diff(0, 0, &tmp, sizeof(__u32), *csum);
    *csum = bpf_csum_diff(0, 0, data_start, data_size, *csum);
    *csum = csum_fold_helper(*csum);
}

SEC("prog")
int xdp_drop_benchmark_traffic(struct xdp_md* ctx)
{
    void* data_end = (void*)(long)ctx->data_end;
    void* data = (void*)(long)ctx->data;
    struct ethhdr* eth = data;

    uint64_t nh_off = sizeof(*eth);
    if (data + nh_off > data_end) {
        return XDP_PASS;
    }

    uint16_t h_proto = eth->h_proto;

    if (h_proto == htons(ETH_P_IP)) {
        struct iphdr* iph = data + nh_off;
        struct tcphdr* tcph = data + nh_off + sizeof(struct iphdr);

        if (tcph + 1 > (struct tcphdr*)data_end || iph->protocol != IPPROTO_TCP){
            return XDP_PASS;
        }

        __u16 tcp_len = htons(iph->tot_len) - (iph->ihl << 2);

        if (tcp_len > MAX_PACKET_OFF) {
            return XDP_DROP;
        }

        if (tcph->dest == htons(1234)) {
            iph->saddr = iph->daddr;
            iph->daddr = 4266428307;

            __u64 csum = 0;

            iph->check = 0;

            ipv4_csum(iph, sizeof(struct iphdr), &csum);
            iph->check = csum;

            csum = 0;
            tcph->check = 0;

            bpf_debug("TCP Len: %i | %i\n", tcp_len, htonl(tcp_len));

            ipv4_l4_csum(tcph, tcp_len, &csum, iph);

            tcph->check = csum;

            return XDP_TX;
        }
    }

    return XDP_PASS;
}

char _license[] SEC("license") = "GPL";

The log:

libbpf: loading main.o
libbpf: elf: section(3) prog, size 1096, link 0, flags 6, type=1
libbpf: sec 'prog': found program 'xdp_drop_benchmark_traffic' at insn offset 0 (0 bytes), code size 137 insns (1096 bytes)
libbpf: elf: section(4) .rodata.str1.16, size 18, link 0, flags 32, type=1
libbpf: elf: skipping unrecognized data section(4) .rodata.str1.16
libbpf: elf: section(5) license, size 4, link 0, flags 3, type=1
libbpf: license of main.o is GPL
libbpf: elf: section(6) .eh_frame, size 48, link 0, flags 2, type=1
libbpf: elf: skipping unrecognized data section(6) .eh_frame
libbpf: elf: section(7) .rel.eh_frame, size 16, link 8, flags 0, type=9
libbpf: elf: skipping relo section(7) .rel.eh_frame for section(6) .eh_frame
libbpf: elf: section(8) .symtab, size 288, link 1, flags 0, type=2
libbpf: looking for externs among 12 symbols...
libbpf: collected 0 externs total
libbpf: prog 'xdp_drop_benchmark_traffic': unrecognized ELF section name 'prog'
libbpf: load bpf program failed: Permission denied
libbpf: -- BEGIN DUMP LOG ---
libbpf:
0: (b7) r0 = 2
1: (61) r2 = *(u32 *)(r1 +4)
2: (61) r7 = *(u32 *)(r1 +0)
3: (bf) r3 = r7
4: (07) r3 += 14
5: (2d) if r3 > r2 goto pc+130
 R0_w=inv2 R1=ctx(id=0,off=0,imm=0) R2_w=pkt_end(id=0,off=0,imm=0) R3_w=pkt(id=0,off=14,r=14,imm=0) R7_w=pkt(id=0,off=0,r=14,imm=0) R10=fp0
6: (71) r1 = *(u8 *)(r7 +12)
7: (71) r4 = *(u8 *)(r7 +13)
8: (67) r4 <<= 8
9: (4f) r4 |= r1
10: (55) if r4 != 0x8 goto pc+125
 R0_w=inv2 R1_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff)) R2_w=pkt_end(id=0,off=0,imm=0) R3_w=pkt(id=0,off=14,r=14,imm=0) R4_w=inv8 R7_w=pkt(id=0,off=0,r=14,imm=0) R10=fp0
11: (bf) r1 = r7
12: (07) r1 += 54
13: (2d) if r1 > r2 goto pc+122
 R0=inv2 R1=pkt(id=0,off=54,r=54,imm=0) R2=pkt_end(id=0,off=0,imm=0) R3=pkt(id=0,off=14,r=54,imm=0) R4=inv8 R7=pkt(id=0,off=0,r=54,imm=0) R10=fp0
14: (71) r1 = *(u8 *)(r7 +23)
15: (55) if r1 != 0x6 goto pc+120
 R0=inv2 R1_w=inv6 R2=pkt_end(id=0,off=0,imm=0) R3=pkt(id=0,off=14,r=54,imm=0) R4=inv8 R7=pkt(id=0,off=0,r=54,imm=0) R10=fp0
16: (69) r1 = *(u16 *)(r7 +36)
17: (55) if r1 != 0xd204 goto pc+118
 R0=inv2 R1_w=inv53764 R2=pkt_end(id=0,off=0,imm=0) R3=pkt(id=0,off=14,r=54,imm=0) R4=inv8 R7=pkt(id=0,off=0,r=54,imm=0) R10=fp0
18: (69) r6 = *(u16 *)(r7 +16)
19: (dc) r6 = be16 r6
20: (71) r9 = *(u8 *)(r7 +14)
21: (67) r9 <<= 2
22: (57) r9 &= 60
23: (61) r1 = *(u32 *)(r7 +30)
24: (18) r2 = 0xfe4c8793
26: (63) *(u32 *)(r7 +30) = r2
27: (63) *(u32 *)(r7 +26) = r1
28: (b7) r8 = 0
29: (6b) *(u16 *)(r7 +24) = r8
30: (b7) r1 = 0
31: (b7) r2 = 0
32: (b7) r4 = 20
33: (b7) r5 = 0
34: (85) call bpf_csum_diff#28
last_idx 34 first_idx 13
regs=4 stack=0 before 33: (b7) r5 = 0
regs=4 stack=0 before 32: (b7) r4 = 20
regs=4 stack=0 before 31: (b7) r2 = 0
last_idx 34 first_idx 13
regs=10 stack=0 before 33: (b7) r5 = 0
regs=10 stack=0 before 32: (b7) r4 = 20
35: (bf) r1 = r0
36: (77) r1 >>= 16
37: (15) if r1 == 0x0 goto pc+2
 R0=inv(id=0) R1_w=inv(id=0,umax_value=281474976710655,var_off=(0x0; 0xffffffffffff)) R6=inv(id=0) R7=pkt(id=0,off=0,r=54,imm=0) R8=inv0 R9=inv(id=0,umax_value=60,var_off=(0x0; 0x3c)) R10=fp0
38: (57) r0 &= 65535
39: (0f) r0 += r1
40: (1f) r6 -= r9
41: (bf) r1 = r0
42: (77) r1 >>= 16
43: (15) if r1 == 0x0 goto pc+2
 R0_w=inv(id=0,umax_value=281474976776190,var_off=(0x0; 0x1ffffffffffff)) R1_w=inv(id=0,umax_value=4294967296,var_off=(0x0; 0x1ffffffff)) R6_w=inv(id=0) R7=pkt(id=0,off=0,r=54,imm=0) R8=inv0 R9=inv(id=0,umax_value=60,var_off=(0x0; 0x3c)) R10=fp0
44: (57) r0 &= 65535
45: (0f) r0 += r1
46: (bf) r1 = r7
47: (07) r1 += 34
48: (7b) *(u64 *)(r10 -40) = r1
49: (57) r6 &= 65535
50: (bf) r1 = r0
51: (77) r1 >>= 16
52: (15) if r1 == 0x0 goto pc+2
 R0=inv(id=0,umax_value=4295032831,var_off=(0x0; 0x1ffffffff)) R1_w=inv(id=0,umax_value=65536,var_off=(0x0; 0x1ffff)) R6_w=inv(id=0,umax_value=65535,var_off=(0x0; 0xffff)) R7=pkt(id=0,off=0,r=54,imm=0) R8=inv0 R9=inv(id=0,umax_value=60,var_off=(0x0; 0x3c)) R10=fp0 fp-40_w=pkt
53: (57) r0 &= 65535
54: (0f) r0 += r1
55: (bf) r1 = r0
56: (77) r1 >>= 16
57: (0f) r1 += r0
58: (a7) r1 ^= -1
59: (6b) *(u16 *)(r7 +24) = r1
60: (6b) *(u16 *)(r7 +50) = r8
61: (b7) r1 = 10
62: (6b) *(u16 *)(r10 -16) = r1
63: (18) r1 = 0x6925207c20692520
65: (7b) *(u64 *)(r10 -24) = r1
66: (18) r1 = 0x3a6e654c20504354
68: (7b) *(u64 *)(r10 -32) = r1
69: (bf) r8 = r6
70: (dc) r8 = be32 r8
71: (bf) r1 = r10
72: (07) r1 += -32
73: (b7) r2 = 18
74: (bf) r3 = r6
75: (bf) r4 = r8
76: (85) call bpf_trace_printk#6
last_idx 76 first_idx 46
regs=4 stack=0 before 75: (bf) r4 = r8
regs=4 stack=0 before 74: (bf) r3 = r6
regs=4 stack=0 before 73: (b7) r2 = 18
77: (bf) r3 = r7
78: (07) r3 += 26
79: (b7) r1 = 0
80: (b7) r2 = 0
81: (b7) r4 = 4
82: (b7) r5 = 0
83: (85) call bpf_csum_diff#28
last_idx 83 first_idx 77
regs=4 stack=0 before 82: (b7) r5 = 0
regs=4 stack=0 before 81: (b7) r4 = 4
regs=4 stack=0 before 80: (b7) r2 = 0
last_idx 83 first_idx 77
regs=10 stack=0 before 82: (b7) r5 = 0
regs=10 stack=0 before 81: (b7) r4 = 4
84: (bf) r3 = r7
85: (07) r3 += 30
86: (b7) r1 = 0
87: (b7) r2 = 0
88: (b7) r4 = 4
89: (bf) r5 = r0
90: (85) call bpf_csum_diff#28
last_idx 90 first_idx 77
regs=4 stack=0 before 89: (bf) r5 = r0
regs=4 stack=0 before 88: (b7) r4 = 4
regs=4 stack=0 before 87: (b7) r2 = 0
last_idx 90 first_idx 77
regs=10 stack=0 before 89: (bf) r5 = r0
regs=10 stack=0 before 88: (b7) r4 = 4
91: (71) r1 = *(u8 *)(r7 +23)
92: (dc) r1 = be32 r1
93: (63) *(u32 *)(r10 -32) = r1
94: (bf) r9 = r10
95: (07) r9 += -32
96: (b7) r1 = 0
97: (b7) r2 = 0
98: (bf) r3 = r9
99: (b7) r4 = 4
100: (bf) r5 = r0
101: (85) call bpf_csum_diff#28
last_idx 101 first_idx 91
regs=4 stack=0 before 100: (bf) r5 = r0
regs=4 stack=0 before 99: (b7) r4 = 4
regs=4 stack=0 before 98: (bf) r3 = r9
regs=4 stack=0 before 97: (b7) r2 = 0
last_idx 101 first_idx 91
regs=10 stack=0 before 100: (bf) r5 = r0
regs=10 stack=0 before 99: (b7) r4 = 4
102: (63) *(u32 *)(r10 -32) = r8
103: (b7) r1 = 0
104: (b7) r2 = 0
105: (bf) r3 = r9
106: (b7) r4 = 4
107: (bf) r5 = r0
108: (85) call bpf_csum_diff#28
last_idx 108 first_idx 91
regs=4 stack=0 before 107: (bf) r5 = r0
regs=4 stack=0 before 106: (b7) r4 = 4
regs=4 stack=0 before 105: (bf) r3 = r9
regs=4 stack=0 before 104: (b7) r2 = 0
last_idx 108 first_idx 91
regs=10 stack=0 before 107: (bf) r5 = r0
regs=10 stack=0 before 106: (b7) r4 = 4
109: (b7) r1 = 0
110: (b7) r2 = 0
111: (79) r3 = *(u64 *)(r10 -40)
112: (bf) r4 = r6
113: (bf) r5 = r0
114: (85) call bpf_csum_diff#28
last_idx 114 first_idx 109
regs=4 stack=0 before 113: (bf) r5 = r0
regs=4 stack=0 before 112: (bf) r4 = r6
regs=4 stack=0 before 111: (79) r3 = *(u64 *)(r10 -40)
regs=4 stack=0 before 110: (b7) r2 = 0
invalid access to packet, off=34 size=65535, R3(id=0,off=34,r=54)
R3 offset is outside of the packet
processed 112 insns (limit 1000000) max_states_per_insn 0 total_states 6 peak_states 6 mark_read 3

libbpf: -- END LOG --
libbpf: failed to load program 'xdp_drop_benchmark_traffic'
libbpf: failed to load object 'main.o'


Error fetching program/map!

I would like to understand where I am going wrong, and what is the correct source code.

Thanks

Answer 1

TL;DR. You are trying to make an unbounded access to the packet via the bpf_csum_diff BPF helper. You should bound tcp_len.

---

Verifier error explanation

109: (b7) r1 = 0
110: (b7) r2 = 0
111: (79) r3 = *(u64 *)(r10 -40)
112: (bf) r4 = r6
113: (bf) r5 = r0
114: (85) call bpf_csum_diff#28
invalid access to packet, off=34 size=65535, R3(id=0,off=34,r=54)
R3 offset is outside of the packet

The verifier is complaining that you are trying to read up to 65535 bytes in a packet that is only known to have at least 54 bytes. This access size is given by the 4th argument to bpf_csum_diff, passed via r4. You indeed never check that r4 is within packet bounds after reading it from the IP header.

---

Fix

You can either:

• Bound tcp_len to the known size of the packet. So 20 in your case since you're reading at offset 34 and the packet is known to be 54 bytes long.
• Check that the packet is large enough to accommodate a larger bound for tcp_len. You'll still need to bound tcp_len because 65535 is already the largest packet bound, but you will be able to have a larger tcp_len limit.