Run cilium/ebpf program on old kernel machine

[y5a9f2y]

First, I write a program on a machine with kernel version 5.10.134 and remove all the features that on supported by 5.x kernel, then I want to run(from the binary or compile a new binary on old kernel machine) this program on the old kernel machine, but I found it is not work. the source code is

//go:build ignore

#define BPF_NO_GLOBAL_DATA

#include "vmlinux.h"
#include "bpf_endian.h"
#include "bpf_helpers.h"
#include "bpf_tracing.h"
#include "bpf_core_read.h"

struct event {
    unsigned short sport;
    unsigned short dport;
    unsigned saddr; 
    unsigned daddr;
    unsigned sz;
};

const struct event *unused __attribute__((unused)) = 0;

char LICENSE[] SEC("license") = "Dual BSD/GPL";

struct {
	__uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
	__uint(key_size, sizeof(int));
	__uint(value_size, sizeof(int));
} pb SEC(".maps");

struct {
	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
	__uint(max_entries, 1);
	__type(key, int);
	__type(value, struct event);
} heap SEC(".maps");

#define member_address(source_struct, source_member)            \
    ({                                                          \
        void* __ret;                                            \
        __ret = (void*) (((char*)source_struct) + offsetof(typeof(*source_struct), source_member)); \
        __ret;                                                  \
    })

#define member_read(destination, source_struct, source_member)  \
  do{                                                           \
    bpf_probe_read(                                             \
      destination,                                              \
      sizeof(source_struct->source_member),                     \
      member_address(source_struct, source_member)              \
    );                                                          \
  } while(0)

#define pt_regs_parm1(x) ((x)->di)
#define pt_regs_parm3(x) ((x)->dx)

static inline int ip_output_rcv_kprobe_helper(struct pt_regs *ctx, struct sk_buff *skb) {
    
    struct event *e = 0;
    int zero = 0;
    e = bpf_map_lookup_elem(&heap, &zero);
    if(!e) {
        return 0;
    }

    char *head;
    u16 network_header;
    u16 transport_header;
    char *l3_header;
    u8 ip_version;
    u8 l4_header_offset;
    u8 l4_protocal;
    char *l4_header;

    member_read(&head, skb, head);
    member_read(&network_header, skb, network_header);
    member_read(&transport_header, skb, transport_header);
    member_read(&e->sz, skb, len);
    l3_header = head + network_header;
    bpf_probe_read(&ip_version, sizeof(u8), l3_header);
    ip_version = ip_version >> 4 & 0xf;

    if(ip_version == 4) {
        struct iphdr ip_header;
        bpf_probe_read(&ip_header, sizeof(ip_header), l3_header);
        bpf_probe_read(&e->saddr, sizeof(ip_header.saddr), (char *)&ip_header+offsetof(struct iphdr, saddr));
        bpf_probe_read(&e->daddr, sizeof(ip_header.daddr), (char *)&ip_header+offsetof(struct iphdr, daddr));
        l4_header_offset = ip_header.ihl * 4;
        l4_protocal = ip_header.protocol;    
    } else {
        return 0;
    }

    l4_header = l3_header + l4_header_offset;
    struct udphdr udp_header;
    struct tcphdr tcp_header;
    if(l4_protocal == 17) {
        bpf_probe_read(&udp_header, sizeof(udp_header), l4_header);
        e->sport = udp_header.source;
        e->dport = udp_header.dest;
    } else if(l4_protocal == 6) {
        bpf_probe_read(&tcp_header, sizeof(tcp_header), l4_header);
        e->sport = tcp_header.source;
        e->dport = tcp_header.dest;
    } else {
        return 0;
    }

    bpf_perf_event_output(ctx, &pb, BPF_F_CURRENT_CPU, e, sizeof(*e));
    return 0;
}

SEC("kprobe/ip_output")
int kprobe_ip_output(struct pt_regs *ctx/*, struct net *net, struct sock *sk, struct sk_buff *skb*/) {
    struct sk_buff *skb = (struct sk_buff *)pt_regs_parm3(ctx);
    return ip_output_rcv_kprobe_helper(ctx, skb);
}

SEC("kprobe/ip_rcv")
int kprobe_ip_rcv(struct pt_regs *ctx/*struct sk_buff *skb, struct net_device *dev, struct packet_type *pt, struct net_device *orig_dev*/) {
    struct sk_buff *skb = (struct sk_buff *)pt_regs_parm1(ctx);
    return ip_output_rcv_kprobe_helper(ctx, skb);
}

First the error information is:
loading objects error: field KprobeIpOutput: program kprobe_ip_output: apply CO-RE relocations: load kernel spec: no BTF found for kernel version 4.19.91-27.5.al7.x86_64: not supported

In order to explore running this program with a btf file, I download the btf file from https://mirrors.openanolis.cn/coolbpf/btf/x86_64/ and place the file under /boot，then the error information changes to
loading objects error: field KprobeIpOutput: program kprobe_ip_output: apply CO-RE relocations: load kernel spec: can't read symbols: no symbol section
It seems that the library can not work with the hacked btf file.

So I wonder how to run the ebpf program without the advanced feature in kernel 5.x on an old kernel machine.

[lmb]

Are you using @main version of the library? You might need 78e76c9

[y5a9f2y]

I actually use the @main version

[root@iZ8vb2hazjjfefepdsuojaZ cmd]# go list -m -json github.com/cilium/ebpf
{
        "Path": "github.com/cilium/ebpf",
        "Version": "v0.12.4-0.20231115103209-e1bf3cc9f28c",
        "Time": "2023-11-15T10:32:09Z",
        "Dir": "/root/go/pkg/mod/github.com/cilium/ebpf@v0.12.4-0.20231115103209-e1bf3cc9f28c",
        "GoMod": "/root/go/pkg/mod/cache/download/github.com/cilium/ebpf/@v/v0.12.4-0.20231115103209-e1bf3cc9f28c.mod",
        "GoVersion": "1.20"
}

[Voyager-Zhang]

How did you solve this problem later?

[brycekahle]

That error means the vmlinux file you downloaded doesn't have a symbol table section.

This may be a bug, because it shouldn't need that section for just loading the kernel BTF. I think it is reusing the same function that is used to read eBPF object files.

[dirkvdmerwe]

I agree...libbpf accepts the same program with the same vmlinux (with BTF section only, no debug symbol section).

[shy-Xu]

I meet the same problem on 4.9 kernel, then I change the btf file and the problem is solved. maybe you need double check your btf file