diff --git a/include/linux/bpf.h b/include/linux/bpf.h index af02d7c55b14093e96d0b626f1173272f73678c0..4e4fa080b001ada2336a65e72985256ddac7490e 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -381,6 +381,11 @@ enum bpf_cgroup_storage_type { #define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX +/* The longest tracepoint has 12 args. + * See include/trace/bpf_probe.h + */ +#define MAX_BPF_FUNC_ARGS 12 + struct bpf_prog_stats { u64 cnt; u64 nsecs; @@ -400,6 +405,11 @@ struct bpf_prog_aux { u32 attach_btf_id; /* in-kernel BTF type id to attach to */ bool verifier_zext; /* Zero extensions has been inserted by verifier. */ bool offload_requested; + bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */ + /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */ + const struct btf_type *attach_func_proto; + /* function name for valid attach_btf_id */ + const char *attach_func_name; struct bpf_prog **func; void *jit_data; /* JIT specific data. arch dependent */ struct latch_tree_node ksym_tnode; @@ -1023,31 +1033,6 @@ static inline int sock_map_get_from_fd(const union bpf_attr *attr, } #endif -#if defined(CONFIG_XDP_SOCKETS) -struct xdp_sock; -struct xdp_sock *__xsk_map_lookup_elem(struct bpf_map *map, u32 key); -int __xsk_map_redirect(struct bpf_map *map, struct xdp_buff *xdp, - struct xdp_sock *xs); -void __xsk_map_flush(struct bpf_map *map); -#else -struct xdp_sock; -static inline struct xdp_sock *__xsk_map_lookup_elem(struct bpf_map *map, - u32 key) -{ - return NULL; -} - -static inline int __xsk_map_redirect(struct bpf_map *map, struct xdp_buff *xdp, - struct xdp_sock *xs) -{ - return -EOPNOTSUPP; -} - -static inline void __xsk_map_flush(struct bpf_map *map) -{ -} -#endif - #if defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) void bpf_sk_reuseport_detach(struct sock *sk); int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key, diff --git a/include/linux/bpf_types.h b/include/linux/bpf_types.h index 36a9c2325176b91c3f948bb917ed13ce07000d39..de14872b01ba0cdfbe7bf8c2dc92f39a5d90021f 100644 --- a/include/linux/bpf_types.h +++ b/include/linux/bpf_types.h @@ -26,6 +26,7 @@ BPF_PROG_TYPE(BPF_PROG_TYPE_TRACEPOINT, tracepoint) BPF_PROG_TYPE(BPF_PROG_TYPE_PERF_EVENT, perf_event) BPF_PROG_TYPE(BPF_PROG_TYPE_RAW_TRACEPOINT, raw_tracepoint) BPF_PROG_TYPE(BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE, raw_tracepoint_writable) +BPF_PROG_TYPE(BPF_PROG_TYPE_TRACING, tracing) #endif #ifdef CONFIG_CGROUP_BPF BPF_PROG_TYPE(BPF_PROG_TYPE_CGROUP_DEVICE, cg_dev) diff --git a/include/linux/btf.h b/include/linux/btf.h index bed89d6ae56873059a6da0b6403a033db667f3c7..8a25dd571b7b7c9d8ddcfb1d10a45c9e4562e02a 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -5,6 +5,7 @@ #define _LINUX_BTF_H 1 #include +#include struct btf; struct btf_member; @@ -55,6 +56,36 @@ bool btf_type_is_void(const struct btf_type *t); const struct btf_type *btf_type_skip_modifiers(const struct btf *btf, u32 id, u32 *res_id); +static inline bool btf_type_is_ptr(const struct btf_type *t) +{ + return BTF_INFO_KIND(t->info) == BTF_KIND_PTR; +} + +static inline bool btf_type_is_int(const struct btf_type *t) +{ + return BTF_INFO_KIND(t->info) == BTF_KIND_INT; +} + +static inline bool btf_type_is_enum(const struct btf_type *t) +{ + return BTF_INFO_KIND(t->info) == BTF_KIND_ENUM; +} + +static inline bool btf_type_is_typedef(const struct btf_type *t) +{ + return BTF_INFO_KIND(t->info) == BTF_KIND_TYPEDEF; +} + +static inline bool btf_type_is_func(const struct btf_type *t) +{ + return BTF_INFO_KIND(t->info) == BTF_KIND_FUNC; +} + +static inline bool btf_type_is_func_proto(const struct btf_type *t) +{ + return BTF_INFO_KIND(t->info) == BTF_KIND_FUNC_PROTO; +} + #ifdef CONFIG_BPF_SYSCALL const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id); const char *btf_name_by_offset(const struct btf *btf, u32 offset); diff --git a/include/linux/filter.h b/include/linux/filter.h index 66027f3c54cc76c548bba8d9e3f550a695e0f06e..878e1614a379f85bd5c95fce8a355873c44d1aec 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1144,7 +1144,6 @@ static inline void bpf_get_prog_name(const struct bpf_prog *prog, char *sym) #endif /* CONFIG_BPF_JIT */ -void bpf_prog_kallsyms_del_subprogs(struct bpf_prog *fp); void bpf_prog_kallsyms_del_all(struct bpf_prog *fp); #define BPF_ANC BIT(15) diff --git a/include/net/xdp_sock.h b/include/net/xdp_sock.h index 5d2f626aa4df856bf65f7d112cb25606da98e478..b31d745ff7d77c6f84bf5f7c9a88559ccfd7eca2 100644 --- a/include/net/xdp_sock.h +++ b/include/net/xdp_sock.h @@ -50,7 +50,14 @@ struct xdp_umem { /* Nodes are linked in the struct xdp_sock map_list field, and used to * track which maps a certain socket reside in. */ -struct xsk_map; + +struct xsk_map { + struct bpf_map map; + struct list_head __percpu *flush_list; + spinlock_t lock; /* Synchronize map updates */ + struct xdp_sock *xsk_map[]; +}; + struct xsk_map_node { struct list_head node; struct xsk_map *map; @@ -83,8 +90,6 @@ struct xdp_sock { struct xdp_buff; #ifdef CONFIG_XDP_SOCKETS int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp); -int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp); -void xsk_flush(struct xdp_sock *xs); bool xsk_is_setup_for_bpf_map(struct xdp_sock *xs); /* Used from netdev driver */ u64 *xsk_umem_peek_addr(struct xdp_umem *umem, u64 *addr); @@ -102,6 +107,23 @@ int xsk_map_inc(struct xsk_map *map); void xsk_map_put(struct xsk_map *map); struct xdp_umem *xdp_get_umem_from_qid(struct net_device *dev, u16 queue_id); +int __xsk_map_redirect(struct bpf_map *map, struct xdp_buff *xdp, + struct xdp_sock *xs); +void __xsk_map_flush(struct bpf_map *map); + +static inline struct xdp_sock *__xsk_map_lookup_elem(struct bpf_map *map, + u32 key) +{ + struct xsk_map *m = container_of(map, struct xsk_map, map); + struct xdp_sock *xs; + + if (key >= map->max_entries) + return NULL; + + xs = READ_ONCE(m->xsk_map[key]); + return xs; +} + static inline char *xdp_umem_get_data(struct xdp_umem *umem, u64 addr) { return umem->pages[addr >> PAGE_SHIFT].addr + (addr & (PAGE_SIZE - 1)); @@ -123,15 +145,6 @@ static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) return -ENOTSUPP; } -static inline int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) -{ - return -ENOTSUPP; -} - -static inline void xsk_flush(struct xdp_sock *xs) -{ -} - static inline bool xsk_is_setup_for_bpf_map(struct xdp_sock *xs) { return false; @@ -188,6 +201,23 @@ static inline struct page **xsk_umem_pgs_delay_unpin(struct xdp_umem *umem, u64 static inline void xsk_umem_unpin_pages(struct page **pgs, u64 npgs) { } + +static inline int __xsk_map_redirect(struct bpf_map *map, struct xdp_buff *xdp, + struct xdp_sock *xs) +{ + return -EOPNOTSUPP; +} + +static inline void __xsk_map_flush(struct bpf_map *map) +{ +} + +static inline struct xdp_sock *__xsk_map_lookup_elem(struct bpf_map *map, + u32 key) +{ + return NULL; +} + #endif /* CONFIG_XDP_SOCKETS */ #endif /* _LINUX_XDP_SOCK_H */ diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index d20b4778285894baf224e5689a73d52fcb57f27d..8a56b1163d5d5a378ee678ff744ea927439aaa48 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -173,6 +173,7 @@ enum bpf_prog_type { BPF_PROG_TYPE_CGROUP_SYSCTL, BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE, BPF_PROG_TYPE_CGROUP_SOCKOPT, + BPF_PROG_TYPE_TRACING, }; enum bpf_attach_type { @@ -199,6 +200,7 @@ enum bpf_attach_type { BPF_CGROUP_UDP6_RECVMSG, BPF_CGROUP_GETSOCKOPT, BPF_CGROUP_SETSOCKOPT, + BPF_TRACE_RAW_TP, __MAX_BPF_ATTACH_TYPE }; diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 070714553d29a62d46507e07693266f2e60dd717..8b31fe9b789f74f27df7741fe7cdf9232865772f 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -336,16 +336,6 @@ static bool btf_type_is_fwd(const struct btf_type *t) return BTF_INFO_KIND(t->info) == BTF_KIND_FWD; } -static bool btf_type_is_func(const struct btf_type *t) -{ - return BTF_INFO_KIND(t->info) == BTF_KIND_FUNC; -} - -static bool btf_type_is_func_proto(const struct btf_type *t) -{ - return BTF_INFO_KIND(t->info) == BTF_KIND_FUNC_PROTO; -} - static bool btf_type_nosize(const struct btf_type *t) { return btf_type_is_void(t) || btf_type_is_fwd(t) || @@ -377,16 +367,6 @@ static bool btf_type_is_array(const struct btf_type *t) return BTF_INFO_KIND(t->info) == BTF_KIND_ARRAY; } -static bool btf_type_is_ptr(const struct btf_type *t) -{ - return BTF_INFO_KIND(t->info) == BTF_KIND_PTR; -} - -static bool btf_type_is_int(const struct btf_type *t) -{ - return BTF_INFO_KIND(t->info) == BTF_KIND_INT; -} - static inline bool btf_type_is_small_int(const struct btf_type *t) { return btf_type_is_int(t) && t->size <= sizeof(u64); @@ -3466,54 +3446,27 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, const struct bpf_prog *prog, struct bpf_insn_access_aux *info) { + const struct btf_type *t = prog->aux->attach_func_proto; + const char *tname = prog->aux->attach_func_name; struct bpf_verifier_log *log = info->log; - u32 btf_id = prog->aux->attach_btf_id; const struct btf_param *args; - const struct btf_type *t; - const char prefix[] = "btf_trace_"; - const char *tname; u32 nr_args, arg; - if (!btf_id) - return true; - - if (IS_ERR(btf_vmlinux)) { - bpf_log(log, "btf_vmlinux is malformed\n"); - return false; - } - - t = btf_type_by_id(btf_vmlinux, btf_id); - if (!t || BTF_INFO_KIND(t->info) != BTF_KIND_TYPEDEF) { - bpf_log(log, "btf_id is invalid\n"); - return false; - } - - tname = __btf_name_by_offset(btf_vmlinux, t->name_off); - if (strncmp(prefix, tname, sizeof(prefix) - 1)) { - bpf_log(log, "btf_id points to wrong type name %s\n", tname); - return false; - } - tname += sizeof(prefix) - 1; - - t = btf_type_by_id(btf_vmlinux, t->type); - if (!btf_type_is_ptr(t)) - return false; - t = btf_type_by_id(btf_vmlinux, t->type); - if (!btf_type_is_func_proto(t)) - return false; - if (off % 8) { - bpf_log(log, "raw_tp '%s' offset %d is not multiple of 8\n", + bpf_log(log, "func '%s' offset %d is not multiple of 8\n", tname, off); return false; } arg = off / 8; args = (const struct btf_param *)(t + 1); - /* skip first 'void *__data' argument in btf_trace_##name typedef */ - args++; - nr_args = btf_type_vlen(t) - 1; + nr_args = btf_type_vlen(t); + if (prog->aux->attach_btf_trace) { + /* skip first 'void *__data' argument in btf_trace_##name typedef */ + args++; + nr_args--; + } if (arg >= nr_args) { - bpf_log(log, "raw_tp '%s' doesn't have %d-th argument\n", + bpf_log(log, "func '%s' doesn't have %d-th argument\n", tname, arg); return false; } @@ -3527,7 +3480,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, return true; if (!btf_type_is_ptr(t)) { bpf_log(log, - "raw_tp '%s' arg%d '%s' has type %s. Only pointer access is allowed\n", + "func '%s' arg%d '%s' has type %s. Only pointer access is allowed\n", tname, arg, __btf_name_by_offset(btf_vmlinux, t->name_off), btf_kind_str[BTF_INFO_KIND(t->info)]); @@ -3550,11 +3503,11 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, t = btf_type_by_id(btf_vmlinux, t->type); if (!btf_type_is_struct(t)) { bpf_log(log, - "raw_tp '%s' arg%d type %s is not a struct\n", + "func '%s' arg%d type %s is not a struct\n", tname, arg, btf_kind_str[BTF_INFO_KIND(t->info)]); return false; } - bpf_log(log, "raw_tp '%s' arg%d has btf_id %d type %s '%s'\n", + bpf_log(log, "func '%s' arg%d has btf_id %d type %s '%s'\n", tname, arg, info->btf_id, btf_kind_str[BTF_INFO_KIND(t->info)], __btf_name_by_offset(btf_vmlinux, t->name_off)); return true; diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 817d3b3c0663f53e1380a49364e12952b9ff8b3f..c235fc9f43c31276a373229187c528e8b540185e 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -1341,12 +1341,12 @@ static bool sysctl_is_valid_access(int off, int size, enum bpf_access_type type, return false; switch (off) { - case offsetof(struct bpf_sysctl, write): + case bpf_ctx_range(struct bpf_sysctl, write): if (type != BPF_READ) return false; bpf_ctx_record_field_size(info, size_default); return bpf_ctx_narrow_access_ok(off, size, size_default); - case offsetof(struct bpf_sysctl, file_pos): + case bpf_ctx_range(struct bpf_sysctl, file_pos): if (type == BPF_READ) { bpf_ctx_record_field_size(info, size_default); return bpf_ctx_narrow_access_ok(off, size, size_default); diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index bde9c891d0565192917c857e0d3397f9bf7f5af6..e4e1962aed6009e786a579ce73dee5835fdde11d 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -506,7 +506,7 @@ int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt) return WARN_ON_ONCE(bpf_adj_branches(prog, off, off + cnt, off, false)); } -void bpf_prog_kallsyms_del_subprogs(struct bpf_prog *fp) +static void bpf_prog_kallsyms_del_subprogs(struct bpf_prog *fp) { int i; diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 99a9d929a6390e3e805eaa5af17819bfe7d65f8a..41b23a16ee0af15598a9e5d4f683a182ed1ed140 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1343,24 +1343,32 @@ static void __bpf_prog_put_rcu(struct rcu_head *rcu) { struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu); + kvfree(aux->func_info); free_used_maps(aux); bpf_prog_uncharge_memlock(aux->prog); security_bpf_prog_free(aux); bpf_prog_free(aux->prog); } +static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) +{ + bpf_prog_kallsyms_del_all(prog); + btf_put(prog->aux->btf); + bpf_prog_free_linfo(prog); + + if (deferred) + call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); + else + __bpf_prog_put_rcu(&prog->aux->rcu); +} + static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock) { if (atomic_dec_and_test(&prog->aux->refcnt)) { perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_UNLOAD, 0); /* bpf_prog_free_id() must be called first */ bpf_prog_free_id(prog, do_idr_lock); - bpf_prog_kallsyms_del_all(prog); - btf_put(prog->aux->btf); - kvfree(prog->aux->func_info); - bpf_prog_free_linfo(prog); - - call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); + __bpf_prog_put_noref(prog, true); } } @@ -1587,7 +1595,7 @@ bpf_prog_load_check_attach(enum bpf_prog_type prog_type, u32 btf_id) { switch (prog_type) { - case BPF_PROG_TYPE_RAW_TRACEPOINT: + case BPF_PROG_TYPE_TRACING: if (btf_id > BTF_MAX_TYPE) return -EINVAL; break; @@ -1772,11 +1780,12 @@ static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr) return err; free_used_maps: - bpf_prog_free_linfo(prog); - kvfree(prog->aux->func_info); - btf_put(prog->aux->btf); - bpf_prog_kallsyms_del_subprogs(prog); - free_used_maps(prog->aux); + /* In case we have subprogs, we need to wait for a grace + * period before we can tear down JIT memory since symbols + * are already exposed under kallsyms. + */ + __bpf_prog_put_noref(prog, prog->aux->func_cnt); + return err; free_prog: bpf_prog_uncharge_memlock(prog); free_prog_sec: @@ -1849,13 +1858,13 @@ static int bpf_raw_tracepoint_open(const union bpf_attr *attr) return PTR_ERR(prog); if (prog->type != BPF_PROG_TYPE_RAW_TRACEPOINT && + prog->type != BPF_PROG_TYPE_TRACING && prog->type != BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE) { err = -EINVAL; goto out_put_prog; } - if (prog->type == BPF_PROG_TYPE_RAW_TRACEPOINT && - prog->aux->attach_btf_id) { + if (prog->type == BPF_PROG_TYPE_TRACING) { if (attr->raw_tracepoint.name) { /* raw_tp name should not be specified in raw_tp * programs that were verified via in-kernel BTF info @@ -1864,9 +1873,7 @@ static int bpf_raw_tracepoint_open(const union bpf_attr *attr) goto out_put_prog; } /* raw_tp name is taken from type name instead */ - tp_name = kernel_type_name(prog->aux->attach_btf_id); - /* skip the prefix */ - tp_name += sizeof("btf_trace_") - 1; + tp_name = prog->aux->attach_func_name; } else { if (strncpy_from_user(buf, u64_to_user_ptr(attr->raw_tracepoint.name), diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 0e7ec9d69ee7f66fffe3750347f8a0762c684a61..b6f0d846b7fb7fe8d6f2ddb7816db98c06d4fb30 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6473,6 +6473,11 @@ static int check_return_code(struct bpf_verifier_env *env) case BPF_PROG_TYPE_CGROUP_SYSCTL: case BPF_PROG_TYPE_CGROUP_SOCKOPT: break; + case BPF_PROG_TYPE_RAW_TRACEPOINT: + if (!env->prog->aux->attach_btf_id) + return 0; + range = tnum_const(0); + break; default: return 0; } @@ -9574,6 +9579,66 @@ static void print_verification_stats(struct bpf_verifier_env *env) env->peak_states, env->longest_mark_read_walk); } +static int check_attach_btf_id(struct bpf_verifier_env *env) +{ + struct bpf_prog *prog = env->prog; + u32 btf_id = prog->aux->attach_btf_id; + const char prefix[] = "btf_trace_"; + const struct btf_type *t; + const char *tname; + + if (prog->type != BPF_PROG_TYPE_TRACING) + return 0; + + if (!btf_id) { + verbose(env, "Tracing programs must provide btf_id\n"); + return -EINVAL; + } + t = btf_type_by_id(btf_vmlinux, btf_id); + if (!t) { + verbose(env, "attach_btf_id %u is invalid\n", btf_id); + return -EINVAL; + } + tname = btf_name_by_offset(btf_vmlinux, t->name_off); + if (!tname) { + verbose(env, "attach_btf_id %u doesn't have a name\n", btf_id); + return -EINVAL; + } + + switch (prog->expected_attach_type) { + case BPF_TRACE_RAW_TP: + if (!btf_type_is_typedef(t)) { + verbose(env, "attach_btf_id %u is not a typedef\n", + btf_id); + return -EINVAL; + } + if (strncmp(prefix, tname, sizeof(prefix) - 1)) { + verbose(env, "attach_btf_id %u points to wrong type name %s\n", + btf_id, tname); + return -EINVAL; + } + tname += sizeof(prefix) - 1; + t = btf_type_by_id(btf_vmlinux, t->type); + if (!btf_type_is_ptr(t)) + /* should never happen in valid vmlinux build */ + return -EINVAL; + t = btf_type_by_id(btf_vmlinux, t->type); + if (!btf_type_is_func_proto(t)) + /* should never happen in valid vmlinux build */ + return -EINVAL; + + /* remember two read only pointers that are valid for + * the life time of the kernel + */ + prog->aux->attach_func_name = tname; + prog->aux->attach_func_proto = t; + prog->aux->attach_btf_trace = true; + return 0; + default: + return -EINVAL; + } +} + int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, union bpf_attr __user *uattr) { @@ -9637,9 +9702,13 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, /* Either gcc or pahole or kernel are broken. */ verbose(env, "in-kernel BTF is malformed\n"); ret = PTR_ERR(btf_vmlinux); - goto err_unlock; + goto skip_full_check; } + ret = check_attach_btf_id(env); + if (ret) + goto skip_full_check; + env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT); if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) env->strict_alignment = true; diff --git a/kernel/bpf/xskmap.c b/kernel/bpf/xskmap.c index aa9bde8ccea67a67f4c620af7064f67abe4c5950..6a2f6f6dacd4fd0d762d4c05ce5255ebd223fd37 100644 --- a/kernel/bpf/xskmap.c +++ b/kernel/bpf/xskmap.c @@ -9,13 +9,6 @@ #include #include -struct xsk_map { - struct bpf_map map; - struct xdp_sock **xsk_map; - struct list_head __percpu *flush_list; - spinlock_t lock; /* Synchronize map updates */ -}; - int xsk_map_inc(struct xsk_map *map) { struct bpf_map *m = &map->map; @@ -80,9 +73,10 @@ static void xsk_map_sock_delete(struct xdp_sock *xs, static struct bpf_map *xsk_map_alloc(union bpf_attr *attr) { + struct bpf_map_memory mem; + int cpu, err, numa_node; struct xsk_map *m; - int cpu, err; - u64 cost; + u64 cost, size; if (!capable(CAP_NET_ADMIN)) return ERR_PTR(-EPERM); @@ -92,44 +86,35 @@ static struct bpf_map *xsk_map_alloc(union bpf_attr *attr) attr->map_flags & ~(BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY)) return ERR_PTR(-EINVAL); - m = kzalloc(sizeof(*m), GFP_USER); - if (!m) + numa_node = bpf_map_attr_numa_node(attr); + size = struct_size(m, xsk_map, attr->max_entries); + cost = size + array_size(sizeof(*m->flush_list), num_possible_cpus()); + + err = bpf_map_charge_init(&mem, cost); + if (err < 0) + return ERR_PTR(err); + + m = bpf_map_area_alloc(size, numa_node); + if (!m) { + bpf_map_charge_finish(&mem); return ERR_PTR(-ENOMEM); + } bpf_map_init_from_attr(&m->map, attr); + bpf_map_charge_move(&m->map.memory, &mem); spin_lock_init(&m->lock); - cost = (u64)m->map.max_entries * sizeof(struct xdp_sock *); - cost += sizeof(struct list_head) * num_possible_cpus(); - - /* Notice returns -EPERM on if map size is larger than memlock limit */ - err = bpf_map_charge_init(&m->map.memory, cost); - if (err) - goto free_m; - - err = -ENOMEM; - m->flush_list = alloc_percpu(struct list_head); - if (!m->flush_list) - goto free_charge; + if (!m->flush_list) { + bpf_map_charge_finish(&m->map.memory); + bpf_map_area_free(m); + return ERR_PTR(-ENOMEM); + } for_each_possible_cpu(cpu) INIT_LIST_HEAD(per_cpu_ptr(m->flush_list, cpu)); - m->xsk_map = bpf_map_area_alloc(m->map.max_entries * - sizeof(struct xdp_sock *), - m->map.numa_node); - if (!m->xsk_map) - goto free_percpu; return &m->map; - -free_percpu: - free_percpu(m->flush_list); -free_charge: - bpf_map_charge_finish(&m->map.memory); -free_m: - kfree(m); - return ERR_PTR(err); } static void xsk_map_free(struct bpf_map *map) @@ -139,8 +124,7 @@ static void xsk_map_free(struct bpf_map *map) bpf_clear_redirect_map(map); synchronize_net(); free_percpu(m->flush_list); - bpf_map_area_free(m->xsk_map); - kfree(m); + bpf_map_area_free(m); } static int xsk_map_get_next_key(struct bpf_map *map, void *key, void *next_key) @@ -160,45 +144,20 @@ static int xsk_map_get_next_key(struct bpf_map *map, void *key, void *next_key) return 0; } -struct xdp_sock *__xsk_map_lookup_elem(struct bpf_map *map, u32 key) -{ - struct xsk_map *m = container_of(map, struct xsk_map, map); - struct xdp_sock *xs; - - if (key >= map->max_entries) - return NULL; - - xs = READ_ONCE(m->xsk_map[key]); - return xs; -} - -int __xsk_map_redirect(struct bpf_map *map, struct xdp_buff *xdp, - struct xdp_sock *xs) -{ - struct xsk_map *m = container_of(map, struct xsk_map, map); - struct list_head *flush_list = this_cpu_ptr(m->flush_list); - int err; - - err = xsk_rcv(xs, xdp); - if (err) - return err; - - if (!xs->flush_node.prev) - list_add(&xs->flush_node, flush_list); - - return 0; -} - -void __xsk_map_flush(struct bpf_map *map) +static u32 xsk_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) { - struct xsk_map *m = container_of(map, struct xsk_map, map); - struct list_head *flush_list = this_cpu_ptr(m->flush_list); - struct xdp_sock *xs, *tmp; - - list_for_each_entry_safe(xs, tmp, flush_list, flush_node) { - xsk_flush(xs); - __list_del_clearprev(&xs->flush_node); - } + const int ret = BPF_REG_0, mp = BPF_REG_1, index = BPF_REG_2; + struct bpf_insn *insn = insn_buf; + + *insn++ = BPF_LDX_MEM(BPF_W, ret, index, 0); + *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 5); + *insn++ = BPF_ALU64_IMM(BPF_LSH, ret, ilog2(sizeof(struct xsk_sock *))); + *insn++ = BPF_ALU64_IMM(BPF_ADD, mp, offsetof(struct xsk_map, xsk_map)); + *insn++ = BPF_ALU64_REG(BPF_ADD, ret, mp); + *insn++ = BPF_LDX_MEM(BPF_SIZEOF(struct xsk_sock *), ret, ret, 0); + *insn++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); + *insn++ = BPF_MOV64_IMM(ret, 0); + return insn - insn_buf; } static void *xsk_map_lookup_elem(struct bpf_map *map, void *key) @@ -308,6 +267,7 @@ const struct bpf_map_ops xsk_map_ops = { .map_free = xsk_map_free, .map_get_next_key = xsk_map_get_next_key, .map_lookup_elem = xsk_map_lookup_elem, + .map_gen_lookup = xsk_map_gen_lookup, .map_lookup_elem_sys_only = xsk_map_lookup_elem_sys_only, .map_update_elem = xsk_map_update_elem, .map_delete_elem = xsk_map_delete_elem, diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 2f7fa79203cdeb7fad85bc593ae3c6a845fed1ac..07c05fe6ea8068f7abca8e8aba20750a6a7f9105 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -1040,10 +1040,6 @@ raw_tp_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) switch (func_id) { case BPF_FUNC_perf_event_output: return &bpf_perf_event_output_proto_raw_tp; -#ifdef CONFIG_NET - case BPF_FUNC_skb_output: - return &bpf_skb_output_proto; -#endif case BPF_FUNC_get_stackid: return &bpf_get_stackid_proto_raw_tp; case BPF_FUNC_get_stack: @@ -1053,13 +1049,39 @@ raw_tp_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) } } +static const struct bpf_func_proto * +tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) +{ + switch (func_id) { +#ifdef CONFIG_NET + case BPF_FUNC_skb_output: + return &bpf_skb_output_proto; +#endif + default: + return raw_tp_prog_func_proto(func_id, prog); + } +} + static bool raw_tp_prog_is_valid_access(int off, int size, enum bpf_access_type type, const struct bpf_prog *prog, struct bpf_insn_access_aux *info) { - /* largest tracepoint in the kernel has 12 args */ - if (off < 0 || off >= sizeof(__u64) * 12) + if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) + return false; + if (type != BPF_READ) + return false; + if (off % size != 0) + return false; + return true; +} + +static bool tracing_prog_is_valid_access(int off, int size, + enum bpf_access_type type, + const struct bpf_prog *prog, + struct bpf_insn_access_aux *info) +{ + if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) return false; if (type != BPF_READ) return false; @@ -1076,6 +1098,14 @@ const struct bpf_verifier_ops raw_tracepoint_verifier_ops = { const struct bpf_prog_ops raw_tracepoint_prog_ops = { }; +const struct bpf_verifier_ops tracing_verifier_ops = { + .get_func_proto = tracing_prog_func_proto, + .is_valid_access = tracing_prog_is_valid_access, +}; + +const struct bpf_prog_ops tracing_prog_ops = { +}; + static bool raw_tp_writable_prog_is_valid_access(int off, int size, enum bpf_access_type type, const struct bpf_prog *prog, diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c index 502e0493fd8bb522a9ff98f1c0c64468ab677e44..f7969d1e9d6362bd232c024ad4a01c9c59666d75 100644 --- a/net/xdp/xsk.c +++ b/net/xdp/xsk.c @@ -125,7 +125,7 @@ static int __xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) return err; } -int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) +static int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) { u32 len; @@ -138,7 +138,7 @@ int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) __xsk_rcv_zc(xs, xdp, len) : __xsk_rcv(xs, xdp, len); } -void xsk_flush(struct xdp_sock *xs) +static void xsk_flush(struct xdp_sock *xs) { xskq_produce_flush_desc(xs->rx); xs->sk.sk_data_ready(&xs->sk); @@ -175,6 +175,35 @@ int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) return err; } +int __xsk_map_redirect(struct bpf_map *map, struct xdp_buff *xdp, + struct xdp_sock *xs) +{ + struct xsk_map *m = container_of(map, struct xsk_map, map); + struct list_head *flush_list = this_cpu_ptr(m->flush_list); + int err; + + err = xsk_rcv(xs, xdp); + if (err) + return err; + + if (!xs->flush_node.prev) + list_add(&xs->flush_node, flush_list); + + return 0; +} + +void __xsk_map_flush(struct bpf_map *map) +{ + struct xsk_map *m = container_of(map, struct xsk_map, map); + struct list_head *flush_list = this_cpu_ptr(m->flush_list); + struct xdp_sock *xs, *tmp; + + list_for_each_entry_safe(xs, tmp, flush_list, flush_node) { + xsk_flush(xs); + __list_del_clearprev(&xs->flush_node); + } +} + void xsk_umem_complete_tx(struct xdp_umem *umem, u32 nb_entries) { xskq_produce_flush_addr_n(umem->cq, nb_entries); diff --git a/tools/bpf/bpftool/btf.c b/tools/bpf/bpftool/btf.c index ff2cff8c2f038db4033f59956d8e23a013e48d51..964f58ff44cbb633e1882715145e188b7f102b29 100644 --- a/tools/bpf/bpftool/btf.c +++ b/tools/bpf/bpftool/btf.c @@ -12,6 +12,9 @@ #include #include #include +#include +#include +#include #include "btf.h" #include "json_writer.h" @@ -388,6 +391,54 @@ static int dump_btf_c(const struct btf *btf, return err; } +static struct btf *btf__parse_raw(const char *file) +{ + struct btf *btf; + struct stat st; + __u8 *buf; + FILE *f; + + if (stat(file, &st)) + return NULL; + + f = fopen(file, "rb"); + if (!f) + return NULL; + + buf = malloc(st.st_size); + if (!buf) { + btf = ERR_PTR(-ENOMEM); + goto exit_close; + } + + if ((size_t) st.st_size != fread(buf, 1, st.st_size, f)) { + btf = ERR_PTR(-EINVAL); + goto exit_free; + } + + btf = btf__new(buf, st.st_size); + +exit_free: + free(buf); +exit_close: + fclose(f); + return btf; +} + +static bool is_btf_raw(const char *file) +{ + __u16 magic = 0; + int fd; + + fd = open(file, O_RDONLY); + if (fd < 0) + return false; + + read(fd, &magic, sizeof(magic)); + close(fd); + return magic == BTF_MAGIC; +} + static int do_dump(int argc, char **argv) { struct btf *btf = NULL; @@ -465,7 +516,11 @@ static int do_dump(int argc, char **argv) } NEXT_ARG(); } else if (is_prefix(src, "file")) { - btf = btf__parse_elf(*argv, NULL); + if (is_btf_raw(*argv)) + btf = btf__parse_raw(*argv); + else + btf = btf__parse_elf(*argv, NULL); + if (IS_ERR(btf)) { err = PTR_ERR(btf); btf = NULL; diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c index 27da96a797ab6d4a8c96770fae71900a7cd7df39..4535c863d2cd9136ef542bd50883bd51f3fc5bba 100644 --- a/tools/bpf/bpftool/prog.c +++ b/tools/bpf/bpftool/prog.c @@ -1091,8 +1091,11 @@ static int do_run(int argc, char **argv) static int load_with_options(int argc, char **argv, bool first_prog_only) { - struct bpf_object_load_attr load_attr = { 0 }; enum bpf_prog_type common_prog_type = BPF_PROG_TYPE_UNSPEC; + DECLARE_LIBBPF_OPTS(bpf_object_open_opts, open_opts, + .relaxed_maps = relaxed_maps, + ); + struct bpf_object_load_attr load_attr = { 0 }; enum bpf_attach_type expected_attach_type; struct map_replace *map_replace = NULL; struct bpf_program *prog = NULL, *pos; @@ -1106,9 +1109,6 @@ static int load_with_options(int argc, char **argv, bool first_prog_only) const char *file; int idx, err; - LIBBPF_OPTS(bpf_object_open_opts, open_opts, - .relaxed_maps = relaxed_maps, - ); if (!REQ_ARGS(2)) return -1; diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index e599d2818cfe981b02e823e9fb6d73c917dc60f7..9e4db9d972f4798a9136448a97eba012614ccd0b 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -173,6 +173,7 @@ enum bpf_prog_type { BPF_PROG_TYPE_CGROUP_SYSCTL, BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE, BPF_PROG_TYPE_CGROUP_SOCKOPT, + BPF_PROG_TYPE_TRACING, }; enum bpf_attach_type { @@ -199,6 +200,7 @@ enum bpf_attach_type { BPF_CGROUP_UDP6_RECVMSG, BPF_CGROUP_GETSOCKOPT, BPF_CGROUP_SETSOCKOPT, + BPF_TRACE_RAW_TP, __MAX_BPF_ATTACH_TYPE }; diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c index a954faf1032a649aac58741d79c491a1f02bb1a9..42788b20e2fd5fe9f4f13b5c9769878fd996cb31 100644 --- a/tools/lib/bpf/bpf.c +++ b/tools/lib/bpf/bpf.c @@ -228,9 +228,10 @@ int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr, memset(&attr, 0, sizeof(attr)); attr.prog_type = load_attr->prog_type; attr.expected_attach_type = load_attr->expected_attach_type; - if (attr.prog_type == BPF_PROG_TYPE_RAW_TRACEPOINT) - /* expected_attach_type is ignored for tracing progs */ - attr.attach_btf_id = attr.expected_attach_type; + if (attr.prog_type == BPF_PROG_TYPE_TRACING) + attr.attach_btf_id = load_attr->attach_btf_id; + else + attr.prog_ifindex = load_attr->prog_ifindex; attr.insn_cnt = (__u32)load_attr->insns_cnt; attr.insns = ptr_to_u64(load_attr->insns); attr.license = ptr_to_u64(load_attr->license); @@ -245,7 +246,6 @@ int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr, } attr.kern_version = load_attr->kern_version; - attr.prog_ifindex = load_attr->prog_ifindex; attr.prog_btf_fd = load_attr->prog_btf_fd; attr.func_info_rec_size = load_attr->func_info_rec_size; attr.func_info_cnt = load_attr->func_info_cnt; diff --git a/tools/lib/bpf/bpf.h b/tools/lib/bpf/bpf.h index 0db01334740f8d1961a54bb28d8cb7f11d8ea9f6..1c53bc5b4b3c70e5c2aaa71d20f212a609605787 100644 --- a/tools/lib/bpf/bpf.h +++ b/tools/lib/bpf/bpf.h @@ -78,7 +78,10 @@ struct bpf_load_program_attr { size_t insns_cnt; const char *license; __u32 kern_version; - __u32 prog_ifindex; + union { + __u32 prog_ifindex; + __u32 attach_btf_id; + }; __u32 prog_btf_fd; __u32 func_info_rec_size; const void *func_info; diff --git a/tools/lib/bpf/bpf_core_read.h b/tools/lib/bpf/bpf_core_read.h index 3e2c8341c2b9eaffdc4a79443f0852c107349cca..fae33e0314bb1b677d3ef273db3db9e656881ee4 100644 --- a/tools/lib/bpf/bpf_core_read.h +++ b/tools/lib/bpf/bpf_core_read.h @@ -12,9 +12,81 @@ */ enum bpf_field_info_kind { BPF_FIELD_BYTE_OFFSET = 0, /* field byte offset */ + BPF_FIELD_BYTE_SIZE = 1, BPF_FIELD_EXISTS = 2, /* field existence in target kernel */ + BPF_FIELD_SIGNED = 3, + BPF_FIELD_LSHIFT_U64 = 4, + BPF_FIELD_RSHIFT_U64 = 5, }; +#define __CORE_RELO(src, field, info) \ + __builtin_preserve_field_info((src)->field, BPF_FIELD_##info) + +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define __CORE_BITFIELD_PROBE_READ(dst, src, fld) \ + bpf_probe_read((void *)dst, \ + __CORE_RELO(src, fld, BYTE_SIZE), \ + (const void *)src + __CORE_RELO(src, fld, BYTE_OFFSET)) +#else +/* semantics of LSHIFT_64 assumes loading values into low-ordered bytes, so + * for big-endian we need to adjust destination pointer accordingly, based on + * field byte size + */ +#define __CORE_BITFIELD_PROBE_READ(dst, src, fld) \ + bpf_probe_read((void *)dst + (8 - __CORE_RELO(src, fld, BYTE_SIZE)), \ + __CORE_RELO(src, fld, BYTE_SIZE), \ + (const void *)src + __CORE_RELO(src, fld, BYTE_OFFSET)) +#endif + +/* + * Extract bitfield, identified by src->field, and put its value into u64 + * *res. All this is done in relocatable manner, so bitfield changes such as + * signedness, bit size, offset changes, this will be handled automatically. + * This version of macro is using bpf_probe_read() to read underlying integer + * storage. Macro functions as an expression and its return type is + * bpf_probe_read()'s return value: 0, on success, <0 on error. + */ +#define BPF_CORE_READ_BITFIELD_PROBED(src, field, res) ({ \ + unsigned long long val; \ + \ + *res = 0; \ + val = __CORE_BITFIELD_PROBE_READ(res, src, field); \ + if (!val) { \ + *res <<= __CORE_RELO(src, field, LSHIFT_U64); \ + val = __CORE_RELO(src, field, RSHIFT_U64); \ + if (__CORE_RELO(src, field, SIGNED)) \ + *res = ((long long)*res) >> val; \ + else \ + *res = ((unsigned long long)*res) >> val; \ + val = 0; \ + } \ + val; \ +}) + +/* + * Extract bitfield, identified by src->field, and return its value as u64. + * This version of macro is using direct memory reads and should be used from + * BPF program types that support such functionality (e.g., typed raw + * tracepoints). + */ +#define BPF_CORE_READ_BITFIELD(s, field) ({ \ + const void *p = (const void *)s + __CORE_RELO(s, field, BYTE_OFFSET); \ + unsigned long long val; \ + \ + switch (__CORE_RELO(s, field, BYTE_SIZE)) { \ + case 1: val = *(const unsigned char *)p; \ + case 2: val = *(const unsigned short *)p; \ + case 4: val = *(const unsigned int *)p; \ + case 8: val = *(const unsigned long long *)p; \ + } \ + val <<= __CORE_RELO(s, field, LSHIFT_U64); \ + if (__CORE_RELO(s, field, SIGNED)) \ + val = ((long long)val) >> __CORE_RELO(s, field, RSHIFT_U64); \ + else \ + val = val >> __CORE_RELO(s, field, RSHIFT_U64); \ + val; \ +}) + /* * Convenience macro to check that field actually exists in target kernel's. * Returns: @@ -24,6 +96,13 @@ enum bpf_field_info_kind { #define bpf_core_field_exists(field) \ __builtin_preserve_field_info(field, BPF_FIELD_EXISTS) +/* + * Convenience macro to get byte size of a field. Works for integers, + * struct/unions, pointers, arrays, and enums. + */ +#define bpf_core_field_size(field) \ + __builtin_preserve_field_info(field, BPF_FIELD_BYTE_SIZE) + /* * bpf_core_read() abstracts away bpf_probe_read() call and captures offset * relocation for source address using __builtin_preserve_access_index() diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c index 306d35082db0bd3836604721eeaab9aeef58bce5..86a1847e4a9fc56f156daf152308e5e2c66fa481 100644 --- a/tools/lib/bpf/btf.c +++ b/tools/lib/bpf/btf.c @@ -389,14 +389,14 @@ struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext) GElf_Ehdr ehdr; if (elf_version(EV_CURRENT) == EV_NONE) { - pr_warning("failed to init libelf for %s\n", path); + pr_warn("failed to init libelf for %s\n", path); return ERR_PTR(-LIBBPF_ERRNO__LIBELF); } fd = open(path, O_RDONLY); if (fd < 0) { err = -errno; - pr_warning("failed to open %s: %s\n", path, strerror(errno)); + pr_warn("failed to open %s: %s\n", path, strerror(errno)); return ERR_PTR(err); } @@ -404,19 +404,19 @@ struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext) elf = elf_begin(fd, ELF_C_READ, NULL); if (!elf) { - pr_warning("failed to open %s as ELF file\n", path); + pr_warn("failed to open %s as ELF file\n", path); goto done; } if (!gelf_getehdr(elf, &ehdr)) { - pr_warning("failed to get EHDR from %s\n", path); + pr_warn("failed to get EHDR from %s\n", path); goto done; } if (!btf_check_endianness(&ehdr)) { - pr_warning("non-native ELF endianness is not supported\n"); + pr_warn("non-native ELF endianness is not supported\n"); goto done; } if (!elf_rawdata(elf_getscn(elf, ehdr.e_shstrndx), NULL)) { - pr_warning("failed to get e_shstrndx from %s\n", path); + pr_warn("failed to get e_shstrndx from %s\n", path); goto done; } @@ -426,29 +426,29 @@ struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext) idx++; if (gelf_getshdr(scn, &sh) != &sh) { - pr_warning("failed to get section(%d) header from %s\n", - idx, path); + pr_warn("failed to get section(%d) header from %s\n", + idx, path); goto done; } name = elf_strptr(elf, ehdr.e_shstrndx, sh.sh_name); if (!name) { - pr_warning("failed to get section(%d) name from %s\n", - idx, path); + pr_warn("failed to get section(%d) name from %s\n", + idx, path); goto done; } if (strcmp(name, BTF_ELF_SEC) == 0) { btf_data = elf_getdata(scn, 0); if (!btf_data) { - pr_warning("failed to get section(%d, %s) data from %s\n", - idx, name, path); + pr_warn("failed to get section(%d, %s) data from %s\n", + idx, name, path); goto done; } continue; } else if (btf_ext && strcmp(name, BTF_EXT_ELF_SEC) == 0) { btf_ext_data = elf_getdata(scn, 0); if (!btf_ext_data) { - pr_warning("failed to get section(%d, %s) data from %s\n", - idx, name, path); + pr_warn("failed to get section(%d, %s) data from %s\n", + idx, name, path); goto done; } continue; @@ -599,9 +599,9 @@ int btf__load(struct btf *btf) log_buf, log_buf_size, false); if (btf->fd < 0) { err = -errno; - pr_warning("Error loading BTF: %s(%d)\n", strerror(errno), errno); + pr_warn("Error loading BTF: %s(%d)\n", strerror(errno), errno); if (*log_buf) - pr_warning("%s\n", log_buf); + pr_warn("%s\n", log_buf); goto done; } @@ -706,8 +706,8 @@ int btf__get_map_kv_tids(const struct btf *btf, const char *map_name, if (snprintf(container_name, max_name, "____btf_map_%s", map_name) == max_name) { - pr_warning("map:%s length of '____btf_map_%s' is too long\n", - map_name, map_name); + pr_warn("map:%s length of '____btf_map_%s' is too long\n", + map_name, map_name); return -EINVAL; } @@ -720,14 +720,14 @@ int btf__get_map_kv_tids(const struct btf *btf, const char *map_name, container_type = btf__type_by_id(btf, container_id); if (!container_type) { - pr_warning("map:%s cannot find BTF type for container_id:%u\n", - map_name, container_id); + pr_warn("map:%s cannot find BTF type for container_id:%u\n", + map_name, container_id); return -EINVAL; } if (!btf_is_struct(container_type) || btf_vlen(container_type) < 2) { - pr_warning("map:%s container_name:%s is an invalid container struct\n", - map_name, container_name); + pr_warn("map:%s container_name:%s is an invalid container struct\n", + map_name, container_name); return -EINVAL; } @@ -736,25 +736,25 @@ int btf__get_map_kv_tids(const struct btf *btf, const char *map_name, key_size = btf__resolve_size(btf, key->type); if (key_size < 0) { - pr_warning("map:%s invalid BTF key_type_size\n", map_name); + pr_warn("map:%s invalid BTF key_type_size\n", map_name); return key_size; } if (expected_key_size != key_size) { - pr_warning("map:%s btf_key_type_size:%u != map_def_key_size:%u\n", - map_name, (__u32)key_size, expected_key_size); + pr_warn("map:%s btf_key_type_size:%u != map_def_key_size:%u\n", + map_name, (__u32)key_size, expected_key_size); return -EINVAL; } value_size = btf__resolve_size(btf, value->type); if (value_size < 0) { - pr_warning("map:%s invalid BTF value_type_size\n", map_name); + pr_warn("map:%s invalid BTF value_type_size\n", map_name); return value_size; } if (expected_value_size != value_size) { - pr_warning("map:%s btf_value_type_size:%u != map_def_value_size:%u\n", - map_name, (__u32)value_size, expected_value_size); + pr_warn("map:%s btf_value_type_size:%u != map_def_value_size:%u\n", + map_name, (__u32)value_size, expected_value_size); return -EINVAL; } diff --git a/tools/lib/bpf/btf_dump.c b/tools/lib/bpf/btf_dump.c index c79f031d85431c113d514efc08ac7ab6dd515833..3b7262d6d0ea5ef48eb475771685917a1094d381 100644 --- a/tools/lib/bpf/btf_dump.c +++ b/tools/lib/bpf/btf_dump.c @@ -428,7 +428,7 @@ static int btf_dump_order_type(struct btf_dump *d, __u32 id, bool through_ptr) /* type loop, but resolvable through fwd declaration */ if (btf_is_composite(t) && through_ptr && t->name_off != 0) return 0; - pr_warning("unsatisfiable type cycle, id:[%u]\n", id); + pr_warn("unsatisfiable type cycle, id:[%u]\n", id); return -ELOOP; } @@ -636,8 +636,8 @@ static void btf_dump_emit_type(struct btf_dump *d, __u32 id, __u32 cont_id) if (id == cont_id) return; if (t->name_off == 0) { - pr_warning("anonymous struct/union loop, id:[%u]\n", - id); + pr_warn("anonymous struct/union loop, id:[%u]\n", + id); return; } btf_dump_emit_struct_fwd(d, id, t); @@ -782,7 +782,7 @@ static int btf_align_of(const struct btf *btf, __u32 id) return align; } default: - pr_warning("unsupported BTF_KIND:%u\n", btf_kind(t)); + pr_warn("unsupported BTF_KIND:%u\n", btf_kind(t)); return 1; } } @@ -1067,7 +1067,7 @@ static void btf_dump_emit_type_decl(struct btf_dump *d, __u32 id, * chain, restore stack, emit warning, and try to * proceed nevertheless */ - pr_warning("not enough memory for decl stack:%d", err); + pr_warn("not enough memory for decl stack:%d", err); d->decl_stack_cnt = stack_start; return; } @@ -1096,8 +1096,8 @@ static void btf_dump_emit_type_decl(struct btf_dump *d, __u32 id, case BTF_KIND_TYPEDEF: goto done; default: - pr_warning("unexpected type in decl chain, kind:%u, id:[%u]\n", - btf_kind(t), id); + pr_warn("unexpected type in decl chain, kind:%u, id:[%u]\n", + btf_kind(t), id); goto done; } } @@ -1338,8 +1338,8 @@ static void btf_dump_emit_type_chain(struct btf_dump *d, return; } default: - pr_warning("unexpected type in decl chain, kind:%u, id:[%u]\n", - kind, id); + pr_warn("unexpected type in decl chain, kind:%u, id:[%u]\n", + kind, id); return; } diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 1330f7e0961cfcd672279c77bb85bc1f7466d78e..326d85f7956eb620c91f3fc306e93362c96fc70e 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -190,6 +190,7 @@ struct bpf_program { bpf_program_clear_priv_t clear_priv; enum bpf_attach_type expected_attach_type; + __u32 attach_btf_id; void *func_info; __u32 func_info_rec_size; __u32 func_info_cnt; @@ -314,8 +315,8 @@ void bpf_program__unload(struct bpf_program *prog) for (i = 0; i < prog->instances.nr; i++) zclose(prog->instances.fds[i]); } else if (prog->instances.nr != -1) { - pr_warning("Internal error: instances.nr is %d\n", - prog->instances.nr); + pr_warn("Internal error: instances.nr is %d\n", + prog->instances.nr); } prog->instances.nr = -1; @@ -366,8 +367,8 @@ bpf_program__init(void *data, size_t size, char *section_name, int idx, const size_t bpf_insn_sz = sizeof(struct bpf_insn); if (size == 0 || size % bpf_insn_sz) { - pr_warning("corrupted section '%s', size: %zu\n", - section_name, size); + pr_warn("corrupted section '%s', size: %zu\n", + section_name, size); return -EINVAL; } @@ -375,22 +376,22 @@ bpf_program__init(void *data, size_t size, char *section_name, int idx, prog->section_name = strdup(section_name); if (!prog->section_name) { - pr_warning("failed to alloc name for prog under section(%d) %s\n", - idx, section_name); + pr_warn("failed to alloc name for prog under section(%d) %s\n", + idx, section_name); goto errout; } prog->pin_name = __bpf_program__pin_name(prog); if (!prog->pin_name) { - pr_warning("failed to alloc pin name for prog under section(%d) %s\n", - idx, section_name); + pr_warn("failed to alloc pin name for prog under section(%d) %s\n", + idx, section_name); goto errout; } prog->insns = malloc(size); if (!prog->insns) { - pr_warning("failed to alloc insns for prog under section %s\n", - section_name); + pr_warn("failed to alloc insns for prog under section %s\n", + section_name); goto errout; } prog->insns_cnt = size / bpf_insn_sz; @@ -428,8 +429,8 @@ bpf_object__add_program(struct bpf_object *obj, void *data, size_t size, * is still valid, so don't need special treat for * bpf_close_object(). */ - pr_warning("failed to alloc a new program under section '%s'\n", - section_name); + pr_warn("failed to alloc a new program under section '%s'\n", + section_name); bpf_program__exit(&prog); return -ENOMEM; } @@ -469,8 +470,8 @@ bpf_object__init_prog_names(struct bpf_object *obj) obj->efile.strtabidx, sym.st_name); if (!name) { - pr_warning("failed to get sym name string for prog %s\n", - prog->section_name); + pr_warn("failed to get sym name string for prog %s\n", + prog->section_name); return -LIBBPF_ERRNO__LIBELF; } } @@ -479,15 +480,15 @@ bpf_object__init_prog_names(struct bpf_object *obj) name = ".text"; if (!name) { - pr_warning("failed to find sym for prog %s\n", - prog->section_name); + pr_warn("failed to find sym for prog %s\n", + prog->section_name); return -EINVAL; } prog->name = strdup(name); if (!prog->name) { - pr_warning("failed to allocate memory for prog sym %s\n", - name); + pr_warn("failed to allocate memory for prog sym %s\n", + name); return -ENOMEM; } } @@ -516,7 +517,7 @@ static struct bpf_object *bpf_object__new(const char *path, obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); if (!obj) { - pr_warning("alloc memory failed for %s\n", path); + pr_warn("alloc memory failed for %s\n", path); return ERR_PTR(-ENOMEM); } @@ -583,7 +584,7 @@ static int bpf_object__elf_init(struct bpf_object *obj) GElf_Ehdr *ep; if (obj_elf_valid(obj)) { - pr_warning("elf init: internal error\n"); + pr_warn("elf init: internal error\n"); return -LIBBPF_ERRNO__LIBELF; } @@ -601,7 +602,7 @@ static int bpf_object__elf_init(struct bpf_object *obj) err = -errno; cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); - pr_warning("failed to open %s: %s\n", obj->path, cp); + pr_warn("failed to open %s: %s\n", obj->path, cp); return err; } @@ -610,13 +611,13 @@ static int bpf_object__elf_init(struct bpf_object *obj) } if (!obj->efile.elf) { - pr_warning("failed to open %s as ELF file\n", obj->path); + pr_warn("failed to open %s as ELF file\n", obj->path); err = -LIBBPF_ERRNO__LIBELF; goto errout; } if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { - pr_warning("failed to get EHDR from %s\n", obj->path); + pr_warn("failed to get EHDR from %s\n", obj->path); err = -LIBBPF_ERRNO__FORMAT; goto errout; } @@ -625,7 +626,7 @@ static int bpf_object__elf_init(struct bpf_object *obj) /* Old LLVM set e_machine to EM_NONE */ if (ep->e_type != ET_REL || (ep->e_machine && ep->e_machine != EM_BPF)) { - pr_warning("%s is not an eBPF object file\n", obj->path); + pr_warn("%s is not an eBPF object file\n", obj->path); err = -LIBBPF_ERRNO__FORMAT; goto errout; } @@ -647,7 +648,7 @@ static int bpf_object__check_endianness(struct bpf_object *obj) #else # error "Unrecognized __BYTE_ORDER__" #endif - pr_warning("endianness mismatch.\n"); + pr_warn("endianness mismatch.\n"); return -LIBBPF_ERRNO__ENDIAN; } @@ -665,7 +666,7 @@ bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size) __u32 kver; if (size != sizeof(kver)) { - pr_warning("invalid kver section in %s\n", obj->path); + pr_warn("invalid kver section in %s\n", obj->path); return -LIBBPF_ERRNO__FORMAT; } memcpy(&kver, data, sizeof(kver)); @@ -707,15 +708,15 @@ static int bpf_object_search_section_size(const struct bpf_object *obj, idx++; if (gelf_getshdr(scn, &sh) != &sh) { - pr_warning("failed to get section(%d) header from %s\n", - idx, obj->path); + pr_warn("failed to get section(%d) header from %s\n", + idx, obj->path); return -EIO; } sec_name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); if (!sec_name) { - pr_warning("failed to get section(%d) name from %s\n", - idx, obj->path); + pr_warn("failed to get section(%d) name from %s\n", + idx, obj->path); return -EIO; } @@ -724,8 +725,8 @@ static int bpf_object_search_section_size(const struct bpf_object *obj, data = elf_getdata(scn, 0); if (!data) { - pr_warning("failed to get section(%d) data from %s(%s)\n", - idx, name, obj->path); + pr_warn("failed to get section(%d) data from %s(%s)\n", + idx, name, obj->path); return -EIO; } @@ -785,8 +786,8 @@ int bpf_object__variable_offset(const struct bpf_object *obj, const char *name, sname = elf_strptr(obj->efile.elf, obj->efile.strtabidx, sym.st_name); if (!sname) { - pr_warning("failed to get sym name string for var %s\n", - name); + pr_warn("failed to get sym name string for var %s\n", + name); return -EIO; } if (strcmp(name, sname) == 0) { @@ -810,7 +811,7 @@ static struct bpf_map *bpf_object__add_map(struct bpf_object *obj) new_cap = max((size_t)4, obj->maps_cap * 3 / 2); new_maps = realloc(obj->maps, new_cap * sizeof(*obj->maps)); if (!new_maps) { - pr_warning("alloc maps for object failed\n"); + pr_warn("alloc maps for object failed\n"); return ERR_PTR(-ENOMEM); } @@ -851,7 +852,7 @@ bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, libbpf_type_to_btf_name[type]); map->name = strdup(map_name); if (!map->name) { - pr_warning("failed to alloc map name\n"); + pr_warn("failed to alloc map name\n"); return -ENOMEM; } pr_debug("map '%s' (global data): at sec_idx %d, offset %zu.\n", @@ -867,7 +868,7 @@ bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, *data_buff = malloc(data->d_size); if (!*data_buff) { zfree(&map->name); - pr_warning("failed to alloc map content buffer\n"); + pr_warn("failed to alloc map content buffer\n"); return -ENOMEM; } memcpy(*data_buff, data->d_buf, data->d_size); @@ -929,8 +930,8 @@ static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict) if (scn) data = elf_getdata(scn, NULL); if (!scn || !data) { - pr_warning("failed to get Elf_Data from map section %d\n", - obj->efile.maps_shndx); + pr_warn("failed to get Elf_Data from map section %d\n", + obj->efile.maps_shndx); return -EINVAL; } @@ -956,7 +957,7 @@ static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict) obj->path, nr_maps, data->d_size); if (!data->d_size || nr_maps == 0 || (data->d_size % nr_maps) != 0) { - pr_warning("unable to determine map definition size " + pr_warn("unable to determine map definition size " "section %s, %d maps in %zd bytes\n", obj->path, nr_maps, data->d_size); return -EINVAL; @@ -982,8 +983,8 @@ static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict) map_name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, sym.st_name); if (!map_name) { - pr_warning("failed to get map #%d name sym string for obj %s\n", - i, obj->path); + pr_warn("failed to get map #%d name sym string for obj %s\n", + i, obj->path); return -LIBBPF_ERRNO__FORMAT; } @@ -993,14 +994,14 @@ static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict) pr_debug("map '%s' (legacy): at sec_idx %d, offset %zu.\n", map_name, map->sec_idx, map->sec_offset); if (sym.st_value + map_def_sz > data->d_size) { - pr_warning("corrupted maps section in %s: last map \"%s\" too small\n", - obj->path, map_name); + pr_warn("corrupted maps section in %s: last map \"%s\" too small\n", + obj->path, map_name); return -EINVAL; } map->name = strdup(map_name); if (!map->name) { - pr_warning("failed to alloc map name\n"); + pr_warn("failed to alloc map name\n"); return -ENOMEM; } pr_debug("map %d is \"%s\"\n", i, map->name); @@ -1024,10 +1025,10 @@ static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict) for (b = ((char *)def) + sizeof(struct bpf_map_def); b < ((char *)def) + map_def_sz; b++) { if (*b != 0) { - pr_warning("maps section in %s: \"%s\" " - "has unrecognized, non-zero " - "options\n", - obj->path, map_name); + pr_warn("maps section in %s: \"%s\" " + "has unrecognized, non-zero " + "options\n", + obj->path, map_name); if (strict) return -EINVAL; } @@ -1071,20 +1072,20 @@ static bool get_map_field_int(const char *map_name, const struct btf *btf, const struct btf_type *arr_t; if (!btf_is_ptr(t)) { - pr_warning("map '%s': attr '%s': expected PTR, got %u.\n", - map_name, name, btf_kind(t)); + pr_warn("map '%s': attr '%s': expected PTR, got %u.\n", + map_name, name, btf_kind(t)); return false; } arr_t = btf__type_by_id(btf, t->type); if (!arr_t) { - pr_warning("map '%s': attr '%s': type [%u] not found.\n", - map_name, name, t->type); + pr_warn("map '%s': attr '%s': type [%u] not found.\n", + map_name, name, t->type); return false; } if (!btf_is_array(arr_t)) { - pr_warning("map '%s': attr '%s': expected ARRAY, got %u.\n", - map_name, name, btf_kind(arr_t)); + pr_warn("map '%s': attr '%s': expected ARRAY, got %u.\n", + map_name, name, btf_kind(arr_t)); return false; } arr_info = btf_array(arr_t); @@ -1112,33 +1113,33 @@ static int bpf_object__init_user_btf_map(struct bpf_object *obj, vlen = btf_vlen(var); if (map_name == NULL || map_name[0] == '\0') { - pr_warning("map #%d: empty name.\n", var_idx); + pr_warn("map #%d: empty name.\n", var_idx); return -EINVAL; } if ((__u64)vi->offset + vi->size > data->d_size) { - pr_warning("map '%s' BTF data is corrupted.\n", map_name); + pr_warn("map '%s' BTF data is corrupted.\n", map_name); return -EINVAL; } if (!btf_is_var(var)) { - pr_warning("map '%s': unexpected var kind %u.\n", - map_name, btf_kind(var)); + pr_warn("map '%s': unexpected var kind %u.\n", + map_name, btf_kind(var)); return -EINVAL; } if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED && var_extra->linkage != BTF_VAR_STATIC) { - pr_warning("map '%s': unsupported var linkage %u.\n", - map_name, var_extra->linkage); + pr_warn("map '%s': unsupported var linkage %u.\n", + map_name, var_extra->linkage); return -EOPNOTSUPP; } def = skip_mods_and_typedefs(obj->btf, var->type, NULL); if (!btf_is_struct(def)) { - pr_warning("map '%s': unexpected def kind %u.\n", - map_name, btf_kind(var)); + pr_warn("map '%s': unexpected def kind %u.\n", + map_name, btf_kind(var)); return -EINVAL; } if (def->size > vi->size) { - pr_warning("map '%s': invalid def size.\n", map_name); + pr_warn("map '%s': invalid def size.\n", map_name); return -EINVAL; } @@ -1147,7 +1148,7 @@ static int bpf_object__init_user_btf_map(struct bpf_object *obj, return PTR_ERR(map); map->name = strdup(map_name); if (!map->name) { - pr_warning("map '%s': failed to alloc map name.\n", map_name); + pr_warn("map '%s': failed to alloc map name.\n", map_name); return -ENOMEM; } map->libbpf_type = LIBBPF_MAP_UNSPEC; @@ -1163,8 +1164,7 @@ static int bpf_object__init_user_btf_map(struct bpf_object *obj, const char *name = btf__name_by_offset(obj->btf, m->name_off); if (!name) { - pr_warning("map '%s': invalid field #%d.\n", - map_name, i); + pr_warn("map '%s': invalid field #%d.\n", map_name, i); return -EINVAL; } if (strcmp(name, "type") == 0) { @@ -1194,8 +1194,8 @@ static int bpf_object__init_user_btf_map(struct bpf_object *obj, pr_debug("map '%s': found key_size = %u.\n", map_name, sz); if (map->def.key_size && map->def.key_size != sz) { - pr_warning("map '%s': conflicting key size %u != %u.\n", - map_name, map->def.key_size, sz); + pr_warn("map '%s': conflicting key size %u != %u.\n", + map_name, map->def.key_size, sz); return -EINVAL; } map->def.key_size = sz; @@ -1204,26 +1204,26 @@ static int bpf_object__init_user_btf_map(struct bpf_object *obj, t = btf__type_by_id(obj->btf, m->type); if (!t) { - pr_warning("map '%s': key type [%d] not found.\n", - map_name, m->type); + pr_warn("map '%s': key type [%d] not found.\n", + map_name, m->type); return -EINVAL; } if (!btf_is_ptr(t)) { - pr_warning("map '%s': key spec is not PTR: %u.\n", - map_name, btf_kind(t)); + pr_warn("map '%s': key spec is not PTR: %u.\n", + map_name, btf_kind(t)); return -EINVAL; } sz = btf__resolve_size(obj->btf, t->type); if (sz < 0) { - pr_warning("map '%s': can't determine key size for type [%u]: %lld.\n", - map_name, t->type, sz); + pr_warn("map '%s': can't determine key size for type [%u]: %lld.\n", + map_name, t->type, sz); return sz; } pr_debug("map '%s': found key [%u], sz = %lld.\n", map_name, t->type, sz); if (map->def.key_size && map->def.key_size != sz) { - pr_warning("map '%s': conflicting key size %u != %lld.\n", - map_name, map->def.key_size, sz); + pr_warn("map '%s': conflicting key size %u != %lld.\n", + map_name, map->def.key_size, sz); return -EINVAL; } map->def.key_size = sz; @@ -1237,8 +1237,8 @@ static int bpf_object__init_user_btf_map(struct bpf_object *obj, pr_debug("map '%s': found value_size = %u.\n", map_name, sz); if (map->def.value_size && map->def.value_size != sz) { - pr_warning("map '%s': conflicting value size %u != %u.\n", - map_name, map->def.value_size, sz); + pr_warn("map '%s': conflicting value size %u != %u.\n", + map_name, map->def.value_size, sz); return -EINVAL; } map->def.value_size = sz; @@ -1247,34 +1247,34 @@ static int bpf_object__init_user_btf_map(struct bpf_object *obj, t = btf__type_by_id(obj->btf, m->type); if (!t) { - pr_warning("map '%s': value type [%d] not found.\n", - map_name, m->type); + pr_warn("map '%s': value type [%d] not found.\n", + map_name, m->type); return -EINVAL; } if (!btf_is_ptr(t)) { - pr_warning("map '%s': value spec is not PTR: %u.\n", - map_name, btf_kind(t)); + pr_warn("map '%s': value spec is not PTR: %u.\n", + map_name, btf_kind(t)); return -EINVAL; } sz = btf__resolve_size(obj->btf, t->type); if (sz < 0) { - pr_warning("map '%s': can't determine value size for type [%u]: %lld.\n", - map_name, t->type, sz); + pr_warn("map '%s': can't determine value size for type [%u]: %lld.\n", + map_name, t->type, sz); return sz; } pr_debug("map '%s': found value [%u], sz = %lld.\n", map_name, t->type, sz); if (map->def.value_size && map->def.value_size != sz) { - pr_warning("map '%s': conflicting value size %u != %lld.\n", - map_name, map->def.value_size, sz); + pr_warn("map '%s': conflicting value size %u != %lld.\n", + map_name, map->def.value_size, sz); return -EINVAL; } map->def.value_size = sz; map->btf_value_type_id = t->type; } else { if (strict) { - pr_warning("map '%s': unknown field '%s'.\n", - map_name, name); + pr_warn("map '%s': unknown field '%s'.\n", + map_name, name); return -ENOTSUP; } pr_debug("map '%s': ignoring unknown field '%s'.\n", @@ -1283,7 +1283,7 @@ static int bpf_object__init_user_btf_map(struct bpf_object *obj, } if (map->def.type == BPF_MAP_TYPE_UNSPEC) { - pr_warning("map '%s': map type isn't specified.\n", map_name); + pr_warn("map '%s': map type isn't specified.\n", map_name); return -EINVAL; } @@ -1306,8 +1306,8 @@ static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict) if (scn) data = elf_getdata(scn, NULL); if (!scn || !data) { - pr_warning("failed to get Elf_Data from map section %d (%s)\n", - obj->efile.maps_shndx, MAPS_ELF_SEC); + pr_warn("failed to get Elf_Data from map section %d (%s)\n", + obj->efile.maps_shndx, MAPS_ELF_SEC); return -EINVAL; } @@ -1324,7 +1324,7 @@ static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict) } if (!sec) { - pr_warning("DATASEC '%s' not found.\n", MAPS_ELF_SEC); + pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC); return -ENOENT; } @@ -1477,14 +1477,13 @@ static int bpf_object__init_btf(struct bpf_object *obj, if (btf_data) { obj->btf = btf__new(btf_data->d_buf, btf_data->d_size); if (IS_ERR(obj->btf)) { - pr_warning("Error loading ELF section %s: %d.\n", - BTF_ELF_SEC, err); + pr_warn("Error loading ELF section %s: %d.\n", + BTF_ELF_SEC, err); goto out; } err = btf__finalize_data(obj, obj->btf); if (err) { - pr_warning("Error finalizing %s: %d.\n", - BTF_ELF_SEC, err); + pr_warn("Error finalizing %s: %d.\n", BTF_ELF_SEC, err); goto out; } } @@ -1497,8 +1496,8 @@ static int bpf_object__init_btf(struct bpf_object *obj, obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size); if (IS_ERR(obj->btf_ext)) { - pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n", - BTF_EXT_ELF_SEC, PTR_ERR(obj->btf_ext)); + pr_warn("Error loading ELF section %s: %ld. Ignored and continue.\n", + BTF_EXT_ELF_SEC, PTR_ERR(obj->btf_ext)); obj->btf_ext = NULL; goto out; } @@ -1514,7 +1513,7 @@ static int bpf_object__init_btf(struct bpf_object *obj, obj->btf = NULL; } if (btf_required && !obj->btf) { - pr_warning("BTF is required, but is missing or corrupted.\n"); + pr_warn("BTF is required, but is missing or corrupted.\n"); return err == 0 ? -ENOENT : err; } return 0; @@ -1532,8 +1531,8 @@ static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj) err = btf__load(obj->btf); if (err) { - pr_warning("Error loading %s into kernel: %d.\n", - BTF_ELF_SEC, err); + pr_warn("Error loading %s into kernel: %d.\n", + BTF_ELF_SEC, err); btf__free(obj->btf); obj->btf = NULL; /* btf_ext can't exist without btf, so free it as well */ @@ -1559,7 +1558,7 @@ static int bpf_object__elf_collect(struct bpf_object *obj, bool relaxed_maps) /* Elf is corrupted/truncated, avoid calling elf_strptr. */ if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) { - pr_warning("failed to get e_shstrndx from %s\n", obj->path); + pr_warn("failed to get e_shstrndx from %s\n", obj->path); return -LIBBPF_ERRNO__FORMAT; } @@ -1570,22 +1569,22 @@ static int bpf_object__elf_collect(struct bpf_object *obj, bool relaxed_maps) idx++; if (gelf_getshdr(scn, &sh) != &sh) { - pr_warning("failed to get section(%d) header from %s\n", - idx, obj->path); + pr_warn("failed to get section(%d) header from %s\n", + idx, obj->path); return -LIBBPF_ERRNO__FORMAT; } name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); if (!name) { - pr_warning("failed to get section(%d) name from %s\n", - idx, obj->path); + pr_warn("failed to get section(%d) name from %s\n", + idx, obj->path); return -LIBBPF_ERRNO__FORMAT; } data = elf_getdata(scn, 0); if (!data) { - pr_warning("failed to get section(%d) data from %s(%s)\n", - idx, name, obj->path); + pr_warn("failed to get section(%d) data from %s(%s)\n", + idx, name, obj->path); return -LIBBPF_ERRNO__FORMAT; } pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n", @@ -1615,8 +1614,8 @@ static int bpf_object__elf_collect(struct bpf_object *obj, bool relaxed_maps) btf_ext_data = data; } else if (sh.sh_type == SHT_SYMTAB) { if (obj->efile.symbols) { - pr_warning("bpf: multiple SYMTAB in %s\n", - obj->path); + pr_warn("bpf: multiple SYMTAB in %s\n", + obj->path); return -LIBBPF_ERRNO__FORMAT; } obj->efile.symbols = data; @@ -1632,8 +1631,8 @@ static int bpf_object__elf_collect(struct bpf_object *obj, bool relaxed_maps) char *cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); - pr_warning("failed to alloc program %s (%s): %s", - name, obj->path, cp); + pr_warn("failed to alloc program %s (%s): %s", + name, obj->path, cp); return err; } } else if (strcmp(name, ".data") == 0) { @@ -1660,7 +1659,7 @@ static int bpf_object__elf_collect(struct bpf_object *obj, bool relaxed_maps) reloc = reallocarray(reloc, nr_reloc + 1, sizeof(*obj->efile.reloc)); if (!reloc) { - pr_warning("realloc failed\n"); + pr_warn("realloc failed\n"); return -ENOMEM; } @@ -1677,8 +1676,8 @@ static int bpf_object__elf_collect(struct bpf_object *obj, bool relaxed_maps) } } - if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) { - pr_warning("Corrupted ELF file: index of strtab invalid\n"); + if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) { + pr_warn("Corrupted ELF file: index of strtab invalid\n"); return -LIBBPF_ERRNO__FORMAT; } err = bpf_object__init_btf(obj, btf_data, btf_ext_data); @@ -1768,7 +1767,7 @@ bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels); if (!prog->reloc_desc) { - pr_warning("failed to alloc memory in relocation\n"); + pr_warn("failed to alloc memory in relocation\n"); return -ENOMEM; } prog->nr_reloc = nrels; @@ -1784,13 +1783,13 @@ bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, GElf_Rel rel; if (!gelf_getrel(data, i, &rel)) { - pr_warning("relocation: failed to get %d reloc\n", i); + pr_warn("relocation: failed to get %d reloc\n", i); return -LIBBPF_ERRNO__FORMAT; } if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) { - pr_warning("relocation: symbol %"PRIx64" not found\n", - GELF_R_SYM(rel.r_info)); + pr_warn("relocation: symbol %"PRIx64" not found\n", + GELF_R_SYM(rel.r_info)); return -LIBBPF_ERRNO__FORMAT; } @@ -1807,20 +1806,20 @@ bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, insn_idx, shdr_idx); if (shdr_idx >= SHN_LORESERVE) { - pr_warning("relocation: not yet supported relo for non-static global \'%s\' variable in special section (0x%x) found in insns[%d].code 0x%x\n", - name, shdr_idx, insn_idx, - insns[insn_idx].code); + pr_warn("relocation: not yet supported relo for non-static global \'%s\' variable in special section (0x%x) found in insns[%d].code 0x%x\n", + name, shdr_idx, insn_idx, + insns[insn_idx].code); return -LIBBPF_ERRNO__RELOC; } if (!bpf_object__relo_in_known_section(obj, shdr_idx)) { - pr_warning("Program '%s' contains unrecognized relo data pointing to section %u\n", - prog->section_name, shdr_idx); + pr_warn("Program '%s' contains unrecognized relo data pointing to section %u\n", + prog->section_name, shdr_idx); return -LIBBPF_ERRNO__RELOC; } if (insns[insn_idx].code == (BPF_JMP | BPF_CALL)) { if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) { - pr_warning("incorrect bpf_call opcode\n"); + pr_warn("incorrect bpf_call opcode\n"); return -LIBBPF_ERRNO__RELOC; } prog->reloc_desc[i].type = RELO_CALL; @@ -1831,8 +1830,8 @@ bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, } if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) { - pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n", - insn_idx, insns[insn_idx].code); + pr_warn("bpf: relocation: invalid relo for insns[%d].code 0x%x\n", + insn_idx, insns[insn_idx].code); return -LIBBPF_ERRNO__RELOC; } @@ -1841,13 +1840,13 @@ bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx); if (type != LIBBPF_MAP_UNSPEC) { if (GELF_ST_BIND(sym.st_info) == STB_GLOBAL) { - pr_warning("bpf: relocation: not yet supported relo for non-static global \'%s\' variable found in insns[%d].code 0x%x\n", - name, insn_idx, insns[insn_idx].code); + pr_warn("bpf: relocation: not yet supported relo for non-static global \'%s\' variable found in insns[%d].code 0x%x\n", + name, insn_idx, insns[insn_idx].code); return -LIBBPF_ERRNO__RELOC; } if (!obj->caps.global_data) { - pr_warning("bpf: relocation: kernel does not support global \'%s\' variable access in insns[%d]\n", - name, insn_idx); + pr_warn("bpf: relocation: kernel does not support global \'%s\' variable access in insns[%d]\n", + name, insn_idx); return -LIBBPF_ERRNO__RELOC; } } @@ -1868,8 +1867,8 @@ bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, } if (map_idx >= nr_maps) { - pr_warning("bpf relocation: map_idx %d larger than %d\n", - (int)map_idx, (int)nr_maps - 1); + pr_warn("bpf relocation: map_idx %d larger than %d\n", + (int)map_idx, (int)nr_maps - 1); return -LIBBPF_ERRNO__RELOC; } @@ -1996,8 +1995,8 @@ bpf_object__probe_name(struct bpf_object *obj) ret = bpf_load_program_xattr(&attr, NULL, 0); if (ret < 0) { cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); - pr_warning("Error in %s():%s(%d). Couldn't load basic 'r0 = 0' BPF program.\n", - __func__, cp, errno); + pr_warn("Error in %s():%s(%d). Couldn't load basic 'r0 = 0' BPF program.\n", + __func__, cp, errno); return -errno; } close(ret); @@ -2037,8 +2036,8 @@ bpf_object__probe_global_data(struct bpf_object *obj) map = bpf_create_map_xattr(&map_attr); if (map < 0) { cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); - pr_warning("Error in %s():%s(%d). Couldn't create simple array map.\n", - __func__, cp, errno); + pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n", + __func__, cp, errno); return -errno; } @@ -2180,8 +2179,8 @@ bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) err = bpf_map_freeze(map->fd); if (err) { cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); - pr_warning("Error freezing map(%s) as read-only: %s\n", - map->name, cp); + pr_warn("Error freezing map(%s) as read-only: %s\n", + map->name, cp); err = 0; } } @@ -2220,8 +2219,8 @@ bpf_object__create_maps(struct bpf_object *obj) if (!nr_cpus) nr_cpus = libbpf_num_possible_cpus(); if (nr_cpus < 0) { - pr_warning("failed to determine number of system CPUs: %d\n", - nr_cpus); + pr_warn("failed to determine number of system CPUs: %d\n", + nr_cpus); err = nr_cpus; goto err_out; } @@ -2249,8 +2248,8 @@ bpf_object__create_maps(struct bpf_object *obj) create_attr.btf_value_type_id)) { err = -errno; cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); - pr_warning("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n", - map->name, cp, err); + pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n", + map->name, cp, err); create_attr.btf_fd = 0; create_attr.btf_key_type_id = 0; create_attr.btf_value_type_id = 0; @@ -2265,8 +2264,8 @@ bpf_object__create_maps(struct bpf_object *obj) err = -errno; err_out: cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); - pr_warning("failed to create map (name: '%s'): %s(%d)\n", - map->name, cp, err); + pr_warn("failed to create map (name: '%s'): %s(%d)\n", + map->name, cp, err); for (j = 0; j < i; j++) zclose(obj->maps[j].fd); return err; @@ -2291,8 +2290,8 @@ check_btf_ext_reloc_err(struct bpf_program *prog, int err, void *btf_prog_info, const char *info_name) { if (err != -ENOENT) { - pr_warning("Error in loading %s for sec %s.\n", - info_name, prog->section_name); + pr_warn("Error in loading %s for sec %s.\n", + info_name, prog->section_name); return err; } @@ -2303,14 +2302,14 @@ check_btf_ext_reloc_err(struct bpf_program *prog, int err, * Some info has already been found but has problem * in the last btf_ext reloc. Must have to error out. */ - pr_warning("Error in relocating %s for sec %s.\n", - info_name, prog->section_name); + pr_warn("Error in relocating %s for sec %s.\n", + info_name, prog->section_name); return err; } /* Have problem loading the very first info. Ignore the rest. */ - pr_warning("Cannot find %s for main program sec %s. Ignore all %s.\n", - info_name, prog->section_name, info_name); + pr_warn("Cannot find %s for main program sec %s. Ignore all %s.\n", + info_name, prog->section_name, info_name); return 0; } @@ -2374,8 +2373,8 @@ struct bpf_core_spec { int raw_spec[BPF_CORE_SPEC_MAX_LEN]; /* raw spec length */ int raw_len; - /* field byte offset represented by spec */ - __u32 offset; + /* field bit offset represented by spec */ + __u32 bit_offset; }; static bool str_is_empty(const char *s) @@ -2386,8 +2385,8 @@ static bool str_is_empty(const char *s) /* * Turn bpf_field_reloc into a low- and high-level spec representation, * validating correctness along the way, as well as calculating resulting - * field offset (in bytes), specified by accessor string. Low-level spec - * captures every single level of nestedness, including traversing anonymous + * field bit offset, specified by accessor string. Low-level spec captures + * every single level of nestedness, including traversing anonymous * struct/union members. High-level one only captures semantically meaningful * "turning points": named fields and array indicies. * E.g., for this case: @@ -2459,7 +2458,7 @@ static int bpf_core_spec_parse(const struct btf *btf, sz = btf__resolve_size(btf, id); if (sz < 0) return sz; - spec->offset = access_idx * sz; + spec->bit_offset = access_idx * sz * 8; for (i = 1; i < spec->raw_len; i++) { t = skip_mods_and_typedefs(btf, id, &id); @@ -2470,17 +2469,13 @@ static int bpf_core_spec_parse(const struct btf *btf, if (btf_is_composite(t)) { const struct btf_member *m; - __u32 offset; + __u32 bit_offset; if (access_idx >= btf_vlen(t)) return -EINVAL; - if (btf_member_bitfield_size(t, access_idx)) - return -EINVAL; - offset = btf_member_bit_offset(t, access_idx); - if (offset % 8) - return -EINVAL; - spec->offset += offset / 8; + bit_offset = btf_member_bit_offset(t, access_idx); + spec->bit_offset += bit_offset; m = btf_members(t) + access_idx; if (m->name_off) { @@ -2509,10 +2504,10 @@ static int bpf_core_spec_parse(const struct btf *btf, sz = btf__resolve_size(btf, id); if (sz < 0) return sz; - spec->offset += access_idx * sz; + spec->bit_offset += access_idx * sz * 8; } else { - pr_warning("relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %d\n", - type_id, spec_str, i, id, btf_kind(t)); + pr_warn("relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %d\n", + type_id, spec_str, i, id, btf_kind(t)); return -EINVAL; } } @@ -2616,12 +2611,14 @@ static struct ids_vec *bpf_core_find_cands(const struct btf *local_btf, } /* Check two types for compatibility, skipping const/volatile/restrict and - * typedefs, to ensure we are relocating offset to the compatible entities: + * typedefs, to ensure we are relocating compatible entities: * - any two STRUCTs/UNIONs are compatible and can be mixed; - * - any two FWDs are compatible; + * - any two FWDs are compatible, if their names match (modulo flavor suffix); * - any two PTRs are always compatible; + * - for ENUMs, names should be the same (ignoring flavor suffix) or at + * least one of enums should be anonymous; * - for ENUMs, check sizes, names are ignored; - * - for INT, size and bitness should match, signedness is ignored; + * - for INT, size and signedness are ignored; * - for ARRAY, dimensionality is ignored, element types are checked for * compatibility recursively; * - everything else shouldn't be ever a target of relocation. @@ -2647,23 +2644,36 @@ static int bpf_core_fields_are_compat(const struct btf *local_btf, return 0; switch (btf_kind(local_type)) { - case BTF_KIND_FWD: case BTF_KIND_PTR: return 1; - case BTF_KIND_ENUM: - return local_type->size == targ_type->size; + case BTF_KIND_FWD: + case BTF_KIND_ENUM: { + const char *local_name, *targ_name; + size_t local_len, targ_len; + + local_name = btf__name_by_offset(local_btf, + local_type->name_off); + targ_name = btf__name_by_offset(targ_btf, targ_type->name_off); + local_len = bpf_core_essential_name_len(local_name); + targ_len = bpf_core_essential_name_len(targ_name); + /* one of them is anonymous or both w/ same flavor-less names */ + return local_len == 0 || targ_len == 0 || + (local_len == targ_len && + strncmp(local_name, targ_name, local_len) == 0); + } case BTF_KIND_INT: + /* just reject deprecated bitfield-like integers; all other + * integers are by default compatible between each other + */ return btf_int_offset(local_type) == 0 && - btf_int_offset(targ_type) == 0 && - local_type->size == targ_type->size && - btf_int_bits(local_type) == btf_int_bits(targ_type); + btf_int_offset(targ_type) == 0; case BTF_KIND_ARRAY: local_id = btf_array(local_type)->type; targ_id = btf_array(targ_type)->type; goto recur; default: - pr_warning("unexpected kind %d relocated, local [%d], target [%d]\n", - btf_kind(local_type), local_id, targ_id); + pr_warn("unexpected kind %d relocated, local [%d], target [%d]\n", + btf_kind(local_type), local_id, targ_id); return 0; } } @@ -2672,7 +2682,7 @@ static int bpf_core_fields_are_compat(const struct btf *local_btf, * Given single high-level named field accessor in local type, find * corresponding high-level accessor for a target type. Along the way, * maintain low-level spec for target as well. Also keep updating target - * offset. + * bit offset. * * Searching is performed through recursive exhaustive enumeration of all * fields of a struct/union. If there are any anonymous (embedded) @@ -2711,21 +2721,16 @@ static int bpf_core_match_member(const struct btf *local_btf, n = btf_vlen(targ_type); m = btf_members(targ_type); for (i = 0; i < n; i++, m++) { - __u32 offset; + __u32 bit_offset; - /* bitfield relocations not supported */ - if (btf_member_bitfield_size(targ_type, i)) - continue; - offset = btf_member_bit_offset(targ_type, i); - if (offset % 8) - continue; + bit_offset = btf_member_bit_offset(targ_type, i); /* too deep struct/union/array nesting */ if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN) return -E2BIG; /* speculate this member will be the good one */ - spec->offset += offset / 8; + spec->bit_offset += bit_offset; spec->raw_spec[spec->raw_len++] = i; targ_name = btf__name_by_offset(targ_btf, m->name_off); @@ -2754,7 +2759,7 @@ static int bpf_core_match_member(const struct btf *local_btf, return found; } /* member turned out not to be what we looked for */ - spec->offset -= offset / 8; + spec->bit_offset -= bit_offset; spec->raw_len--; } @@ -2763,7 +2768,7 @@ static int bpf_core_match_member(const struct btf *local_btf, /* * Try to match local spec to a target type and, if successful, produce full - * target spec (high-level, low-level + offset). + * target spec (high-level, low-level + bit offset). */ static int bpf_core_spec_match(struct bpf_core_spec *local_spec, const struct btf *targ_btf, __u32 targ_id, @@ -2826,13 +2831,120 @@ static int bpf_core_spec_match(struct bpf_core_spec *local_spec, sz = btf__resolve_size(targ_btf, targ_id); if (sz < 0) return sz; - targ_spec->offset += local_acc->idx * sz; + targ_spec->bit_offset += local_acc->idx * sz * 8; } } return 1; } +static int bpf_core_calc_field_relo(const struct bpf_program *prog, + const struct bpf_field_reloc *relo, + const struct bpf_core_spec *spec, + __u32 *val, bool *validate) +{ + const struct bpf_core_accessor *acc = &spec->spec[spec->len - 1]; + const struct btf_type *t = btf__type_by_id(spec->btf, acc->type_id); + __u32 byte_off, byte_sz, bit_off, bit_sz; + const struct btf_member *m; + const struct btf_type *mt; + bool bitfield; + __s64 sz; + + /* a[n] accessor needs special handling */ + if (!acc->name) { + if (relo->kind == BPF_FIELD_BYTE_OFFSET) { + *val = spec->bit_offset / 8; + } else if (relo->kind == BPF_FIELD_BYTE_SIZE) { + sz = btf__resolve_size(spec->btf, acc->type_id); + if (sz < 0) + return -EINVAL; + *val = sz; + } else { + pr_warn("prog '%s': relo %d at insn #%d can't be applied to array access\n", + bpf_program__title(prog, false), + relo->kind, relo->insn_off / 8); + return -EINVAL; + } + if (validate) + *validate = true; + return 0; + } + + m = btf_members(t) + acc->idx; + mt = skip_mods_and_typedefs(spec->btf, m->type, NULL); + bit_off = spec->bit_offset; + bit_sz = btf_member_bitfield_size(t, acc->idx); + + bitfield = bit_sz > 0; + if (bitfield) { + byte_sz = mt->size; + byte_off = bit_off / 8 / byte_sz * byte_sz; + /* figure out smallest int size necessary for bitfield load */ + while (bit_off + bit_sz - byte_off * 8 > byte_sz * 8) { + if (byte_sz >= 8) { + /* bitfield can't be read with 64-bit read */ + pr_warn("prog '%s': relo %d at insn #%d can't be satisfied for bitfield\n", + bpf_program__title(prog, false), + relo->kind, relo->insn_off / 8); + return -E2BIG; + } + byte_sz *= 2; + byte_off = bit_off / 8 / byte_sz * byte_sz; + } + } else { + sz = btf__resolve_size(spec->btf, m->type); + if (sz < 0) + return -EINVAL; + byte_sz = sz; + byte_off = spec->bit_offset / 8; + bit_sz = byte_sz * 8; + } + + /* for bitfields, all the relocatable aspects are ambiguous and we + * might disagree with compiler, so turn off validation of expected + * value, except for signedness + */ + if (validate) + *validate = !bitfield; + + switch (relo->kind) { + case BPF_FIELD_BYTE_OFFSET: + *val = byte_off; + break; + case BPF_FIELD_BYTE_SIZE: + *val = byte_sz; + break; + case BPF_FIELD_SIGNED: + /* enums will be assumed unsigned */ + *val = btf_is_enum(mt) || + (btf_int_encoding(mt) & BTF_INT_SIGNED); + if (validate) + *validate = true; /* signedness is never ambiguous */ + break; + case BPF_FIELD_LSHIFT_U64: +#if __BYTE_ORDER == __LITTLE_ENDIAN + *val = 64 - (bit_off + bit_sz - byte_off * 8); +#else + *val = (8 - byte_sz) * 8 + (bit_off - byte_off * 8); +#endif + break; + case BPF_FIELD_RSHIFT_U64: + *val = 64 - bit_sz; + if (validate) + *validate = true; /* right shift is never ambiguous */ + break; + case BPF_FIELD_EXISTS: + default: + pr_warn("prog '%s': unknown relo %d at insn #%d\n", + bpf_program__title(prog, false), + relo->kind, relo->insn_off / 8); + return -EINVAL; + } + + return 0; +} + /* * Patch relocatable BPF instruction. * @@ -2852,36 +2964,31 @@ static int bpf_core_reloc_insn(struct bpf_program *prog, const struct bpf_core_spec *local_spec, const struct bpf_core_spec *targ_spec) { + bool failed = false, validate = true; __u32 orig_val, new_val; struct bpf_insn *insn; - int insn_idx; + int insn_idx, err; __u8 class; if (relo->insn_off % sizeof(struct bpf_insn)) return -EINVAL; insn_idx = relo->insn_off / sizeof(struct bpf_insn); - switch (relo->kind) { - case BPF_FIELD_BYTE_OFFSET: - orig_val = local_spec->offset; - if (targ_spec) { - new_val = targ_spec->offset; - } else { - pr_warning("prog '%s': patching insn #%d w/ failed reloc, imm %d -> %d\n", - bpf_program__title(prog, false), insn_idx, - orig_val, -1); - new_val = (__u32)-1; - } - break; - case BPF_FIELD_EXISTS: + if (relo->kind == BPF_FIELD_EXISTS) { orig_val = 1; /* can't generate EXISTS relo w/o local field */ new_val = targ_spec ? 1 : 0; - break; - default: - pr_warning("prog '%s': unknown relo %d at insn #%d'\n", - bpf_program__title(prog, false), - relo->kind, insn_idx); - return -EINVAL; + } else if (!targ_spec) { + failed = true; + new_val = (__u32)-1; + } else { + err = bpf_core_calc_field_relo(prog, relo, local_spec, + &orig_val, &validate); + if (err) + return err; + err = bpf_core_calc_field_relo(prog, relo, targ_spec, + &new_val, NULL); + if (err) + return err; } insn = &prog->insns[insn_idx]; @@ -2890,17 +2997,22 @@ static int bpf_core_reloc_insn(struct bpf_program *prog, if (class == BPF_ALU || class == BPF_ALU64) { if (BPF_SRC(insn->code) != BPF_K) return -EINVAL; - if (insn->imm != orig_val) + if (!failed && validate && insn->imm != orig_val) { + pr_warn("prog '%s': unexpected insn #%d value: got %u, exp %u -> %u\n", + bpf_program__title(prog, false), insn_idx, + insn->imm, orig_val, new_val); return -EINVAL; + } + orig_val = insn->imm; insn->imm = new_val; - pr_debug("prog '%s': patched insn #%d (ALU/ALU64) imm %d -> %d\n", - bpf_program__title(prog, false), - insn_idx, orig_val, new_val); + pr_debug("prog '%s': patched insn #%d (ALU/ALU64)%s imm %u -> %u\n", + bpf_program__title(prog, false), insn_idx, + failed ? " w/ failed reloc" : "", orig_val, new_val); } else { - pr_warning("prog '%s': trying to relocate unrecognized insn #%d, code:%x, src:%x, dst:%x, off:%x, imm:%x\n", - bpf_program__title(prog, false), - insn_idx, insn->code, insn->src_reg, insn->dst_reg, - insn->off, insn->imm); + pr_warn("prog '%s': trying to relocate unrecognized insn #%d, code:%x, src:%x, dst:%x, off:%x, imm:%x\n", + bpf_program__title(prog, false), + insn_idx, insn->code, insn->src_reg, insn->dst_reg, + insn->off, insn->imm); return -EINVAL; } @@ -2989,7 +3101,7 @@ static struct btf *bpf_core_find_kernel_btf(void) return btf; } - pr_warning("failed to find valid kernel BTF\n"); + pr_warn("failed to find valid kernel BTF\n"); return ERR_PTR(-ESRCH); } @@ -3013,7 +3125,8 @@ static void bpf_core_dump_spec(int level, const struct bpf_core_spec *spec) libbpf_print(level, "%d%s", spec->raw_spec[i], i == spec->raw_len - 1 ? " => " : ":"); - libbpf_print(level, "%u @ &x", spec->offset); + libbpf_print(level, "%u.%u @ &x", + spec->bit_offset / 8, spec->bit_offset % 8); for (i = 0; i < spec->len; i++) { if (spec->spec[i].name) @@ -3121,22 +3234,23 @@ static int bpf_core_reloc_field(struct bpf_program *prog, err = bpf_core_spec_parse(local_btf, local_id, spec_str, &local_spec); if (err) { - pr_warning("prog '%s': relo #%d: parsing [%d] %s + %s failed: %d\n", - prog_name, relo_idx, local_id, local_name, spec_str, - err); + pr_warn("prog '%s': relo #%d: parsing [%d] %s + %s failed: %d\n", + prog_name, relo_idx, local_id, local_name, spec_str, + err); return -EINVAL; } - pr_debug("prog '%s': relo #%d: spec is ", prog_name, relo_idx); + pr_debug("prog '%s': relo #%d: kind %d, spec is ", prog_name, relo_idx, + relo->kind); bpf_core_dump_spec(LIBBPF_DEBUG, &local_spec); libbpf_print(LIBBPF_DEBUG, "\n"); if (!hashmap__find(cand_cache, type_key, (void **)&cand_ids)) { cand_ids = bpf_core_find_cands(local_btf, local_id, targ_btf); if (IS_ERR(cand_ids)) { - pr_warning("prog '%s': relo #%d: target candidate search failed for [%d] %s: %ld", - prog_name, relo_idx, local_id, local_name, - PTR_ERR(cand_ids)); + pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s: %ld", + prog_name, relo_idx, local_id, local_name, + PTR_ERR(cand_ids)); return PTR_ERR(cand_ids); } err = hashmap__set(cand_cache, type_key, cand_ids, NULL, NULL); @@ -3158,8 +3272,8 @@ static int bpf_core_reloc_field(struct bpf_program *prog, bpf_core_dump_spec(LIBBPF_DEBUG, &cand_spec); libbpf_print(LIBBPF_DEBUG, ": %d\n", err); if (err < 0) { - pr_warning("prog '%s': relo #%d: matching error: %d\n", - prog_name, relo_idx, err); + pr_warn("prog '%s': relo #%d: matching error: %d\n", + prog_name, relo_idx, err); return err; } if (err == 0) @@ -3167,13 +3281,13 @@ static int bpf_core_reloc_field(struct bpf_program *prog, if (j == 0) { targ_spec = cand_spec; - } else if (cand_spec.offset != targ_spec.offset) { + } else if (cand_spec.bit_offset != targ_spec.bit_offset) { /* if there are many candidates, they should all - * resolve to the same offset + * resolve to the same bit offset */ - pr_warning("prog '%s': relo #%d: offset ambiguity: %u != %u\n", - prog_name, relo_idx, cand_spec.offset, - targ_spec.offset); + pr_warn("prog '%s': relo #%d: offset ambiguity: %u != %u\n", + prog_name, relo_idx, cand_spec.bit_offset, + targ_spec.bit_offset); return -EINVAL; } @@ -3192,8 +3306,8 @@ static int bpf_core_reloc_field(struct bpf_program *prog, if (j == 0 && !prog->obj->relaxed_core_relocs && relo->kind != BPF_FIELD_EXISTS) { - pr_warning("prog '%s': relo #%d: no matching targets found for [%d] %s + %s\n", - prog_name, relo_idx, local_id, local_name, spec_str); + pr_warn("prog '%s': relo #%d: no matching targets found for [%d] %s + %s\n", + prog_name, relo_idx, local_id, local_name, spec_str); return -ESRCH; } @@ -3201,8 +3315,8 @@ static int bpf_core_reloc_field(struct bpf_program *prog, err = bpf_core_reloc_insn(prog, relo, &local_spec, j ? &targ_spec : NULL); if (err) { - pr_warning("prog '%s': relo #%d: failed to patch insn at offset %d: %d\n", - prog_name, relo_idx, relo->insn_off, err); + pr_warn("prog '%s': relo #%d: failed to patch insn at offset %d: %d\n", + prog_name, relo_idx, relo->insn_off, err); return -EINVAL; } @@ -3227,8 +3341,7 @@ bpf_core_reloc_fields(struct bpf_object *obj, const char *targ_btf_path) else targ_btf = bpf_core_find_kernel_btf(); if (IS_ERR(targ_btf)) { - pr_warning("failed to get target BTF: %ld\n", - PTR_ERR(targ_btf)); + pr_warn("failed to get target BTF: %ld\n", PTR_ERR(targ_btf)); return PTR_ERR(targ_btf); } @@ -3253,7 +3366,7 @@ bpf_core_reloc_fields(struct bpf_object *obj, const char *targ_btf_path) } } if (!prog) { - pr_warning("failed to find program '%s' for CO-RE offset relocation\n", + pr_warn("failed to find program '%s' for CO-RE offset relocation\n", sec_name); err = -EINVAL; goto out; @@ -3266,8 +3379,8 @@ bpf_core_reloc_fields(struct bpf_object *obj, const char *targ_btf_path) err = bpf_core_reloc_field(prog, rec, i, obj->btf, targ_btf, cand_cache); if (err) { - pr_warning("prog '%s': relo #%d: failed to relocate: %d\n", - sec_name, i, err); + pr_warn("prog '%s': relo #%d: failed to relocate: %d\n", + sec_name, i, err); goto out; } } @@ -3308,7 +3421,7 @@ bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj, return -LIBBPF_ERRNO__RELOC; if (prog->idx == obj->efile.text_shndx) { - pr_warning("relo in .text insn %d into off %d\n", + pr_warn("relo in .text insn %d into off %d\n", relo->insn_idx, relo->text_off); return -LIBBPF_ERRNO__RELOC; } @@ -3316,13 +3429,13 @@ bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj, if (prog->main_prog_cnt == 0) { text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx); if (!text) { - pr_warning("no .text section found yet relo into text exist\n"); + pr_warn("no .text section found yet relo into text exist\n"); return -LIBBPF_ERRNO__RELOC; } new_cnt = prog->insns_cnt + text->insns_cnt; new_insn = reallocarray(prog->insns, new_cnt, sizeof(*insn)); if (!new_insn) { - pr_warning("oom in prog realloc\n"); + pr_warn("oom in prog realloc\n"); return -ENOMEM; } prog->insns = new_insn; @@ -3377,8 +3490,8 @@ bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj) map_idx = prog->reloc_desc[i].map_idx; if (insn_idx + 1 >= (int)prog->insns_cnt) { - pr_warning("relocation out of range: '%s'\n", - prog->section_name); + pr_warn("relocation out of range: '%s'\n", + prog->section_name); return -LIBBPF_ERRNO__RELOC; } @@ -3412,8 +3525,8 @@ bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path) if (obj->btf_ext) { err = bpf_object__relocate_core(obj, targ_btf_path); if (err) { - pr_warning("failed to perform CO-RE relocations: %d\n", - err); + pr_warn("failed to perform CO-RE relocations: %d\n", + err); return err; } } @@ -3422,8 +3535,7 @@ bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path) err = bpf_program__relocate(prog, obj); if (err) { - pr_warning("failed to relocate '%s'\n", - prog->section_name); + pr_warn("failed to relocate '%s'\n", prog->section_name); return err; } } @@ -3435,7 +3547,7 @@ static int bpf_object__collect_reloc(struct bpf_object *obj) int i, err; if (!obj_elf_valid(obj)) { - pr_warning("Internal error: elf object is closed\n"); + pr_warn("Internal error: elf object is closed\n"); return -LIBBPF_ERRNO__INTERNAL; } @@ -3446,13 +3558,13 @@ static int bpf_object__collect_reloc(struct bpf_object *obj) struct bpf_program *prog; if (shdr->sh_type != SHT_REL) { - pr_warning("internal error at %d\n", __LINE__); + pr_warn("internal error at %d\n", __LINE__); return -LIBBPF_ERRNO__INTERNAL; } prog = bpf_object__find_prog_by_idx(obj, idx); if (!prog) { - pr_warning("relocation failed: no section(%d)\n", idx); + pr_warn("relocation failed: no section(%d)\n", idx); return -LIBBPF_ERRNO__RELOC; } @@ -3500,11 +3612,12 @@ load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, load_attr.line_info_cnt = prog->line_info_cnt; load_attr.log_level = prog->log_level; load_attr.prog_flags = prog->prog_flags; + load_attr.attach_btf_id = prog->attach_btf_id; retry_load: log_buf = malloc(log_buf_size); if (!log_buf) - pr_warning("Alloc log buffer for bpf loader error, continue without log\n"); + pr_warn("Alloc log buffer for bpf loader error, continue without log\n"); ret = bpf_load_program_xattr(&load_attr, log_buf, log_buf_size); @@ -3523,16 +3636,16 @@ load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, } ret = -LIBBPF_ERRNO__LOAD; cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); - pr_warning("load bpf program failed: %s\n", cp); + pr_warn("load bpf program failed: %s\n", cp); if (log_buf && log_buf[0] != '\0') { ret = -LIBBPF_ERRNO__VERIFY; - pr_warning("-- BEGIN DUMP LOG ---\n"); - pr_warning("\n%s\n", log_buf); - pr_warning("-- END LOG --\n"); + pr_warn("-- BEGIN DUMP LOG ---\n"); + pr_warn("\n%s\n", log_buf); + pr_warn("-- END LOG --\n"); } else if (load_attr.insns_cnt >= BPF_MAXINSNS) { - pr_warning("Program too large (%zu insns), at most %d insns\n", - load_attr.insns_cnt, BPF_MAXINSNS); + pr_warn("Program too large (%zu insns), at most %d insns\n", + load_attr.insns_cnt, BPF_MAXINSNS); ret = -LIBBPF_ERRNO__PROG2BIG; } else { /* Wrong program type? */ @@ -3566,14 +3679,14 @@ bpf_program__load(struct bpf_program *prog, if (prog->instances.nr < 0 || !prog->instances.fds) { if (prog->preprocessor) { - pr_warning("Internal error: can't load program '%s'\n", - prog->section_name); + pr_warn("Internal error: can't load program '%s'\n", + prog->section_name); return -LIBBPF_ERRNO__INTERNAL; } prog->instances.fds = malloc(sizeof(int)); if (!prog->instances.fds) { - pr_warning("Not enough memory for BPF fds\n"); + pr_warn("Not enough memory for BPF fds\n"); return -ENOMEM; } prog->instances.nr = 1; @@ -3582,8 +3695,8 @@ bpf_program__load(struct bpf_program *prog, if (!prog->preprocessor) { if (prog->instances.nr != 1) { - pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n", - prog->section_name, prog->instances.nr); + pr_warn("Program '%s' is inconsistent: nr(%d) != 1\n", + prog->section_name, prog->instances.nr); } err = load_program(prog, prog->insns, prog->insns_cnt, license, kern_version, &fd); @@ -3600,8 +3713,8 @@ bpf_program__load(struct bpf_program *prog, err = preprocessor(prog, i, prog->insns, prog->insns_cnt, &result); if (err) { - pr_warning("Preprocessing the %dth instance of program '%s' failed\n", - i, prog->section_name); + pr_warn("Preprocessing the %dth instance of program '%s' failed\n", + i, prog->section_name); goto out; } @@ -3619,8 +3732,8 @@ bpf_program__load(struct bpf_program *prog, license, kern_version, &fd); if (err) { - pr_warning("Loading the %dth instance of program '%s' failed\n", - i, prog->section_name); + pr_warn("Loading the %dth instance of program '%s' failed\n", + i, prog->section_name); goto out; } @@ -3630,8 +3743,7 @@ bpf_program__load(struct bpf_program *prog, } out: if (err) - pr_warning("failed to load program '%s'\n", - prog->section_name); + pr_warn("failed to load program '%s'\n", prog->section_name); zfree(&prog->insns); prog->insns_cnt = 0; return err; @@ -3662,10 +3774,13 @@ bpf_object__load_progs(struct bpf_object *obj, int log_level) return 0; } +static int libbpf_attach_btf_id_by_name(const char *name, __u32 *btf_id); + static struct bpf_object * __bpf_object__open(const char *path, const void *obj_buf, size_t obj_buf_sz, struct bpf_object_open_opts *opts) { + struct bpf_program *prog; struct bpf_object *obj; const char *obj_name; char tmp_name[64]; @@ -3673,8 +3788,8 @@ __bpf_object__open(const char *path, const void *obj_buf, size_t obj_buf_sz, int err; if (elf_version(EV_CURRENT) == EV_NONE) { - pr_warning("failed to init libelf for %s\n", - path ? : "(mem buf)"); + pr_warn("failed to init libelf for %s\n", + path ? : "(mem buf)"); return ERR_PTR(-LIBBPF_ERRNO__LIBELF); } @@ -3705,8 +3820,31 @@ __bpf_object__open(const char *path, const void *obj_buf, size_t obj_buf_sz, CHECK_ERR(bpf_object__probe_caps(obj), err, out); CHECK_ERR(bpf_object__elf_collect(obj, relaxed_maps), err, out); CHECK_ERR(bpf_object__collect_reloc(obj), err, out); - bpf_object__elf_finish(obj); + + bpf_object__for_each_program(prog, obj) { + enum bpf_prog_type prog_type; + enum bpf_attach_type attach_type; + __u32 btf_id; + + err = libbpf_prog_type_by_name(prog->section_name, &prog_type, + &attach_type); + if (err == -ESRCH) + /* couldn't guess, but user might manually specify */ + continue; + if (err) + goto out; + + bpf_program__set_type(prog, prog_type); + bpf_program__set_expected_attach_type(prog, attach_type); + if (prog_type == BPF_PROG_TYPE_TRACING) { + err = libbpf_attach_btf_id_by_name(prog->section_name, &btf_id); + if (err) + goto out; + prog->attach_btf_id = btf_id; + } + } + return obj; out: bpf_object__close(obj); @@ -3716,7 +3854,7 @@ __bpf_object__open(const char *path, const void *obj_buf, size_t obj_buf_sz, static struct bpf_object * __bpf_object__open_xattr(struct bpf_object_open_attr *attr, int flags) { - LIBBPF_OPTS(bpf_object_open_opts, opts, + DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts, .relaxed_maps = flags & MAPS_RELAX_COMPAT, ); @@ -3768,7 +3906,7 @@ struct bpf_object * bpf_object__open_buffer(const void *obj_buf, size_t obj_buf_sz, const char *name) { - LIBBPF_OPTS(bpf_object_open_opts, opts, + DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts, .object_name = name, /* wrong default, but backwards-compatible */ .relaxed_maps = true, @@ -3809,7 +3947,7 @@ int bpf_object__load_xattr(struct bpf_object_load_attr *attr) return -EINVAL; if (obj->loaded) { - pr_warning("object should not be loaded twice\n"); + pr_warn("object should not be loaded twice\n"); return -EINVAL; } @@ -3822,7 +3960,7 @@ int bpf_object__load_xattr(struct bpf_object_load_attr *attr) return 0; out: bpf_object__unload(obj); - pr_warning("failed to load object '%s'\n", obj->path); + pr_warn("failed to load object '%s'\n", obj->path); return err; } @@ -3852,13 +3990,13 @@ static int check_path(const char *path) dir = dirname(dname); if (statfs(dir, &st_fs)) { cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); - pr_warning("failed to statfs %s: %s\n", dir, cp); + pr_warn("failed to statfs %s: %s\n", dir, cp); err = -errno; } free(dname); if (!err && st_fs.f_type != BPF_FS_MAGIC) { - pr_warning("specified path %s is not on BPF FS\n", path); + pr_warn("specified path %s is not on BPF FS\n", path); err = -EINVAL; } @@ -3876,19 +4014,19 @@ int bpf_program__pin_instance(struct bpf_program *prog, const char *path, return err; if (prog == NULL) { - pr_warning("invalid program pointer\n"); + pr_warn("invalid program pointer\n"); return -EINVAL; } if (instance < 0 || instance >= prog->instances.nr) { - pr_warning("invalid prog instance %d of prog %s (max %d)\n", - instance, prog->section_name, prog->instances.nr); + pr_warn("invalid prog instance %d of prog %s (max %d)\n", + instance, prog->section_name, prog->instances.nr); return -EINVAL; } if (bpf_obj_pin(prog->instances.fds[instance], path)) { cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); - pr_warning("failed to pin program: %s\n", cp); + pr_warn("failed to pin program: %s\n", cp); return -errno; } pr_debug("pinned program '%s'\n", path); @@ -3906,13 +4044,13 @@ int bpf_program__unpin_instance(struct bpf_program *prog, const char *path, return err; if (prog == NULL) { - pr_warning("invalid program pointer\n"); + pr_warn("invalid program pointer\n"); return -EINVAL; } if (instance < 0 || instance >= prog->instances.nr) { - pr_warning("invalid prog instance %d of prog %s (max %d)\n", - instance, prog->section_name, prog->instances.nr); + pr_warn("invalid prog instance %d of prog %s (max %d)\n", + instance, prog->section_name, prog->instances.nr); return -EINVAL; } @@ -3934,7 +4072,7 @@ static int make_dir(const char *path) if (err) { cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); - pr_warning("failed to mkdir %s: %s\n", path, cp); + pr_warn("failed to mkdir %s: %s\n", path, cp); } return err; } @@ -3948,12 +4086,12 @@ int bpf_program__pin(struct bpf_program *prog, const char *path) return err; if (prog == NULL) { - pr_warning("invalid program pointer\n"); + pr_warn("invalid program pointer\n"); return -EINVAL; } if (prog->instances.nr <= 0) { - pr_warning("no instances of prog %s to pin\n", + pr_warn("no instances of prog %s to pin\n", prog->section_name); return -EINVAL; } @@ -4015,12 +4153,12 @@ int bpf_program__unpin(struct bpf_program *prog, const char *path) return err; if (prog == NULL) { - pr_warning("invalid program pointer\n"); + pr_warn("invalid program pointer\n"); return -EINVAL; } if (prog->instances.nr <= 0) { - pr_warning("no instances of prog %s to pin\n", + pr_warn("no instances of prog %s to pin\n", prog->section_name); return -EINVAL; } @@ -4062,13 +4200,13 @@ int bpf_map__pin(struct bpf_map *map, const char *path) return err; if (map == NULL) { - pr_warning("invalid map pointer\n"); + pr_warn("invalid map pointer\n"); return -EINVAL; } if (bpf_obj_pin(map->fd, path)) { cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); - pr_warning("failed to pin map: %s\n", cp); + pr_warn("failed to pin map: %s\n", cp); return -errno; } @@ -4086,7 +4224,7 @@ int bpf_map__unpin(struct bpf_map *map, const char *path) return err; if (map == NULL) { - pr_warning("invalid map pointer\n"); + pr_warn("invalid map pointer\n"); return -EINVAL; } @@ -4107,7 +4245,7 @@ int bpf_object__pin_maps(struct bpf_object *obj, const char *path) return -ENOENT; if (!obj->loaded) { - pr_warning("object not yet loaded; load it first\n"); + pr_warn("object not yet loaded; load it first\n"); return -ENOENT; } @@ -4190,7 +4328,7 @@ int bpf_object__pin_programs(struct bpf_object *obj, const char *path) return -ENOENT; if (!obj->loaded) { - pr_warning("object not yet loaded; load it first\n"); + pr_warn("object not yet loaded; load it first\n"); return -ENOENT; } @@ -4391,7 +4529,7 @@ __bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj, &obj->programs[nr_programs - 1]; if (p->obj != obj) { - pr_warning("error: program handler doesn't match object\n"); + pr_warn("error: program handler doesn't match object\n"); return NULL; } @@ -4454,7 +4592,7 @@ const char *bpf_program__title(const struct bpf_program *prog, bool needs_copy) if (needs_copy) { title = strdup(title); if (!title) { - pr_warning("failed to strdup program title\n"); + pr_warn("failed to strdup program title\n"); return ERR_PTR(-ENOMEM); } } @@ -4476,13 +4614,13 @@ int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, return -EINVAL; if (prog->instances.nr > 0 || prog->instances.fds) { - pr_warning("Can't set pre-processor after loading\n"); + pr_warn("Can't set pre-processor after loading\n"); return -EINVAL; } instances_fds = malloc(sizeof(int) * nr_instances); if (!instances_fds) { - pr_warning("alloc memory failed for fds\n"); + pr_warn("alloc memory failed for fds\n"); return -ENOMEM; } @@ -4503,21 +4641,26 @@ int bpf_program__nth_fd(const struct bpf_program *prog, int n) return -EINVAL; if (n >= prog->instances.nr || n < 0) { - pr_warning("Can't get the %dth fd from program %s: only %d instances\n", - n, prog->section_name, prog->instances.nr); + pr_warn("Can't get the %dth fd from program %s: only %d instances\n", + n, prog->section_name, prog->instances.nr); return -EINVAL; } fd = prog->instances.fds[n]; if (fd < 0) { - pr_warning("%dth instance of program '%s' is invalid\n", - n, prog->section_name); + pr_warn("%dth instance of program '%s' is invalid\n", + n, prog->section_name); return -ENOENT; } return fd; } +enum bpf_prog_type bpf_program__get_type(struct bpf_program *prog) +{ + return prog->type; +} + void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) { prog->type = type; @@ -4551,6 +4694,13 @@ BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT); BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT); BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP); BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT); +BPF_PROG_TYPE_FNS(tracing, BPF_PROG_TYPE_TRACING); + +enum bpf_attach_type +bpf_program__get_expected_attach_type(struct bpf_program *prog) +{ + return prog->expected_attach_type; +} void bpf_program__set_expected_attach_type(struct bpf_program *prog, enum bpf_attach_type type) @@ -4573,7 +4723,8 @@ void bpf_program__set_expected_attach_type(struct bpf_program *prog, BPF_PROG_SEC_IMPL(string, ptype, eatype, 1, 0, eatype) /* Programs that use BTF to identify attach point */ -#define BPF_PROG_BTF(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 1, 0) +#define BPF_PROG_BTF(string, ptype, eatype) \ + BPF_PROG_SEC_IMPL(string, ptype, eatype, 0, 1, 0) /* Programs that can be attached but attach type can't be identified by section * name. Kept for backward compatibility. @@ -4591,12 +4742,17 @@ static const struct { } section_names[] = { BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER), BPF_PROG_SEC("kprobe/", BPF_PROG_TYPE_KPROBE), + BPF_PROG_SEC("uprobe/", BPF_PROG_TYPE_KPROBE), BPF_PROG_SEC("kretprobe/", BPF_PROG_TYPE_KPROBE), + BPF_PROG_SEC("uretprobe/", BPF_PROG_TYPE_KPROBE), BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS), BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT), BPF_PROG_SEC("tracepoint/", BPF_PROG_TYPE_TRACEPOINT), + BPF_PROG_SEC("tp/", BPF_PROG_TYPE_TRACEPOINT), BPF_PROG_SEC("raw_tracepoint/", BPF_PROG_TYPE_RAW_TRACEPOINT), - BPF_PROG_BTF("tp_btf/", BPF_PROG_TYPE_RAW_TRACEPOINT), + BPF_PROG_SEC("raw_tp/", BPF_PROG_TYPE_RAW_TRACEPOINT), + BPF_PROG_BTF("tp_btf/", BPF_PROG_TYPE_TRACING, + BPF_TRACE_RAW_TP), BPF_PROG_SEC("xdp", BPF_PROG_TYPE_XDP), BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT), BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN), @@ -4701,37 +4857,56 @@ int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, continue; *prog_type = section_names[i].prog_type; *expected_attach_type = section_names[i].expected_attach_type; - if (section_names[i].is_attach_btf) { - struct btf *btf = bpf_core_find_kernel_btf(); - char raw_tp_btf_name[128] = "btf_trace_"; - char *dst = raw_tp_btf_name + sizeof("btf_trace_") - 1; - int ret; - - if (IS_ERR(btf)) { - pr_warning("vmlinux BTF is not found\n"); - return -EINVAL; - } - /* prepend "btf_trace_" prefix per kernel convention */ - strncat(dst, name + section_names[i].len, - sizeof(raw_tp_btf_name) - sizeof("btf_trace_")); - ret = btf__find_by_name(btf, raw_tp_btf_name); - btf__free(btf); - if (ret <= 0) { - pr_warning("%s is not found in vmlinux BTF\n", dst); - return -EINVAL; - } - *expected_attach_type = ret; - } return 0; } - pr_warning("failed to guess program type based on ELF section name '%s'\n", name); + pr_warn("failed to guess program type based on ELF section name '%s'\n", name); type_names = libbpf_get_type_names(false); if (type_names != NULL) { pr_info("supported section(type) names are:%s\n", type_names); free(type_names); } - return -EINVAL; + return -ESRCH; +} + +#define BTF_PREFIX "btf_trace_" +static int libbpf_attach_btf_id_by_name(const char *name, __u32 *btf_id) +{ + struct btf *btf = bpf_core_find_kernel_btf(); + char raw_tp_btf_name[128] = BTF_PREFIX; + char *dst = raw_tp_btf_name + sizeof(BTF_PREFIX) - 1; + int ret, i, err = -EINVAL; + + if (IS_ERR(btf)) { + pr_warn("vmlinux BTF is not found\n"); + return -EINVAL; + } + + if (!name) + goto out; + + for (i = 0; i < ARRAY_SIZE(section_names); i++) { + if (!section_names[i].is_attach_btf) + continue; + if (strncmp(name, section_names[i].sec, section_names[i].len)) + continue; + /* prepend "btf_trace_" prefix per kernel convention */ + strncat(dst, name + section_names[i].len, + sizeof(raw_tp_btf_name) - sizeof(BTF_PREFIX)); + ret = btf__find_by_name(btf, raw_tp_btf_name); + if (ret <= 0) { + pr_warn("%s is not found in vmlinux BTF\n", dst); + goto out; + } + *btf_id = ret; + err = 0; + goto out; + } + pr_warn("failed to identify btf_id based on ELF section name '%s'\n", name); + err = -ESRCH; +out: + btf__free(btf); + return err; } int libbpf_attach_type_by_name(const char *name, @@ -4751,7 +4926,7 @@ int libbpf_attach_type_by_name(const char *name, *attach_type = section_names[i].attach_type; return 0; } - pr_warning("failed to guess attach type based on ELF section name '%s'\n", name); + pr_warn("failed to guess attach type based on ELF section name '%s'\n", name); type_names = libbpf_get_type_names(true); if (type_names != NULL) { pr_info("attachable section(type) names are:%s\n", type_names); @@ -4761,15 +4936,6 @@ int libbpf_attach_type_by_name(const char *name, return -EINVAL; } -static int -bpf_program__identify_section(struct bpf_program *prog, - enum bpf_prog_type *prog_type, - enum bpf_attach_type *expected_attach_type) -{ - return libbpf_prog_type_by_name(prog->section_name, prog_type, - expected_attach_type); -} - int bpf_map__fd(const struct bpf_map *map) { return map ? map->fd : -EINVAL; @@ -4834,11 +5000,11 @@ void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex) int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd) { if (!bpf_map_type__is_map_in_map(map->def.type)) { - pr_warning("error: unsupported map type\n"); + pr_warn("error: unsupported map type\n"); return -EINVAL; } if (map->inner_map_fd != -1) { - pr_warning("error: inner_map_fd already specified\n"); + pr_warn("error: inner_map_fd already specified\n"); return -EINVAL; } map->inner_map_fd = fd; @@ -4858,8 +5024,8 @@ __bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i) e = obj->maps + obj->nr_maps; if ((m < s) || (m >= e)) { - pr_warning("error in %s: map handler doesn't belong to object\n", - __func__); + pr_warn("error in %s: map handler doesn't belong to object\n", + __func__); return NULL; } @@ -4939,8 +5105,6 @@ int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, { struct bpf_object_open_attr open_attr = {}; struct bpf_program *prog, *first_prog = NULL; - enum bpf_attach_type expected_attach_type; - enum bpf_prog_type prog_type; struct bpf_object *obj; struct bpf_map *map; int err; @@ -4958,26 +5122,27 @@ int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, return -ENOENT; bpf_object__for_each_program(prog, obj) { + enum bpf_attach_type attach_type = attr->expected_attach_type; /* - * If type is not specified, try to guess it based on - * section name. + * to preserve backwards compatibility, bpf_prog_load treats + * attr->prog_type, if specified, as an override to whatever + * bpf_object__open guessed */ - prog_type = attr->prog_type; - prog->prog_ifindex = attr->ifindex; - expected_attach_type = attr->expected_attach_type; - if (prog_type == BPF_PROG_TYPE_UNSPEC) { - err = bpf_program__identify_section(prog, &prog_type, - &expected_attach_type); - if (err < 0) { - bpf_object__close(obj); - return -EINVAL; - } + if (attr->prog_type != BPF_PROG_TYPE_UNSPEC) { + bpf_program__set_type(prog, attr->prog_type); + bpf_program__set_expected_attach_type(prog, + attach_type); + } + if (bpf_program__get_type(prog) == BPF_PROG_TYPE_UNSPEC) { + /* + * we haven't guessed from section name and user + * didn't provide a fallback type, too bad... + */ + bpf_object__close(obj); + return -EINVAL; } - bpf_program__set_type(prog, prog_type); - bpf_program__set_expected_attach_type(prog, - expected_attach_type); - + prog->prog_ifindex = attr->ifindex; prog->log_level = attr->log_level; prog->prog_flags = attr->prog_flags; if (!bpf_program__is_function_storage(prog, obj) && !first_prog) @@ -4990,7 +5155,7 @@ int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, } if (!first_prog) { - pr_warning("object file doesn't contain bpf program\n"); + pr_warn("object file doesn't contain bpf program\n"); bpf_object__close(obj); return -ENOENT; } @@ -5049,14 +5214,14 @@ struct bpf_link *bpf_program__attach_perf_event(struct bpf_program *prog, int prog_fd, err; if (pfd < 0) { - pr_warning("program '%s': invalid perf event FD %d\n", - bpf_program__title(prog, false), pfd); + pr_warn("program '%s': invalid perf event FD %d\n", + bpf_program__title(prog, false), pfd); return ERR_PTR(-EINVAL); } prog_fd = bpf_program__fd(prog); if (prog_fd < 0) { - pr_warning("program '%s': can't attach BPF program w/o FD (did you load it?)\n", - bpf_program__title(prog, false)); + pr_warn("program '%s': can't attach BPF program w/o FD (did you load it?)\n", + bpf_program__title(prog, false)); return ERR_PTR(-EINVAL); } @@ -5069,16 +5234,16 @@ struct bpf_link *bpf_program__attach_perf_event(struct bpf_program *prog, if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) { err = -errno; free(link); - pr_warning("program '%s': failed to attach to pfd %d: %s\n", - bpf_program__title(prog, false), pfd, + pr_warn("program '%s': failed to attach to pfd %d: %s\n", + bpf_program__title(prog, false), pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); return ERR_PTR(err); } if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) { err = -errno; free(link); - pr_warning("program '%s': failed to enable pfd %d: %s\n", - bpf_program__title(prog, false), pfd, + pr_warn("program '%s': failed to enable pfd %d: %s\n", + bpf_program__title(prog, false), pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); return ERR_PTR(err); } @@ -5153,9 +5318,9 @@ static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, type = uprobe ? determine_uprobe_perf_type() : determine_kprobe_perf_type(); if (type < 0) { - pr_warning("failed to determine %s perf type: %s\n", - uprobe ? "uprobe" : "kprobe", - libbpf_strerror_r(type, errmsg, sizeof(errmsg))); + pr_warn("failed to determine %s perf type: %s\n", + uprobe ? "uprobe" : "kprobe", + libbpf_strerror_r(type, errmsg, sizeof(errmsg))); return type; } if (retprobe) { @@ -5163,10 +5328,9 @@ static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, : determine_kprobe_retprobe_bit(); if (bit < 0) { - pr_warning("failed to determine %s retprobe bit: %s\n", - uprobe ? "uprobe" : "kprobe", - libbpf_strerror_r(bit, errmsg, - sizeof(errmsg))); + pr_warn("failed to determine %s retprobe bit: %s\n", + uprobe ? "uprobe" : "kprobe", + libbpf_strerror_r(bit, errmsg, sizeof(errmsg))); return bit; } attr.config |= 1 << bit; @@ -5183,9 +5347,9 @@ static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); if (pfd < 0) { err = -errno; - pr_warning("%s perf_event_open() failed: %s\n", - uprobe ? "uprobe" : "kprobe", - libbpf_strerror_r(err, errmsg, sizeof(errmsg))); + pr_warn("%s perf_event_open() failed: %s\n", + uprobe ? "uprobe" : "kprobe", + libbpf_strerror_r(err, errmsg, sizeof(errmsg))); return err; } return pfd; @@ -5202,20 +5366,20 @@ struct bpf_link *bpf_program__attach_kprobe(struct bpf_program *prog, pfd = perf_event_open_probe(false /* uprobe */, retprobe, func_name, 0 /* offset */, -1 /* pid */); if (pfd < 0) { - pr_warning("program '%s': failed to create %s '%s' perf event: %s\n", - bpf_program__title(prog, false), - retprobe ? "kretprobe" : "kprobe", func_name, - libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); + pr_warn("program '%s': failed to create %s '%s' perf event: %s\n", + bpf_program__title(prog, false), + retprobe ? "kretprobe" : "kprobe", func_name, + libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); return ERR_PTR(pfd); } link = bpf_program__attach_perf_event(prog, pfd); if (IS_ERR(link)) { close(pfd); err = PTR_ERR(link); - pr_warning("program '%s': failed to attach to %s '%s': %s\n", - bpf_program__title(prog, false), - retprobe ? "kretprobe" : "kprobe", func_name, - libbpf_strerror_r(err, errmsg, sizeof(errmsg))); + pr_warn("program '%s': failed to attach to %s '%s': %s\n", + bpf_program__title(prog, false), + retprobe ? "kretprobe" : "kprobe", func_name, + libbpf_strerror_r(err, errmsg, sizeof(errmsg))); return link; } return link; @@ -5233,22 +5397,22 @@ struct bpf_link *bpf_program__attach_uprobe(struct bpf_program *prog, pfd = perf_event_open_probe(true /* uprobe */, retprobe, binary_path, func_offset, pid); if (pfd < 0) { - pr_warning("program '%s': failed to create %s '%s:0x%zx' perf event: %s\n", - bpf_program__title(prog, false), - retprobe ? "uretprobe" : "uprobe", - binary_path, func_offset, - libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); + pr_warn("program '%s': failed to create %s '%s:0x%zx' perf event: %s\n", + bpf_program__title(prog, false), + retprobe ? "uretprobe" : "uprobe", + binary_path, func_offset, + libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); return ERR_PTR(pfd); } link = bpf_program__attach_perf_event(prog, pfd); if (IS_ERR(link)) { close(pfd); err = PTR_ERR(link); - pr_warning("program '%s': failed to attach to %s '%s:0x%zx': %s\n", - bpf_program__title(prog, false), - retprobe ? "uretprobe" : "uprobe", - binary_path, func_offset, - libbpf_strerror_r(err, errmsg, sizeof(errmsg))); + pr_warn("program '%s': failed to attach to %s '%s:0x%zx': %s\n", + bpf_program__title(prog, false), + retprobe ? "uretprobe" : "uprobe", + binary_path, func_offset, + libbpf_strerror_r(err, errmsg, sizeof(errmsg))); return link; } return link; @@ -5282,9 +5446,9 @@ static int perf_event_open_tracepoint(const char *tp_category, tp_id = determine_tracepoint_id(tp_category, tp_name); if (tp_id < 0) { - pr_warning("failed to determine tracepoint '%s/%s' perf event ID: %s\n", - tp_category, tp_name, - libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg))); + pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n", + tp_category, tp_name, + libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg))); return tp_id; } @@ -5296,9 +5460,9 @@ static int perf_event_open_tracepoint(const char *tp_category, -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); if (pfd < 0) { err = -errno; - pr_warning("tracepoint '%s/%s' perf_event_open() failed: %s\n", - tp_category, tp_name, - libbpf_strerror_r(err, errmsg, sizeof(errmsg))); + pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n", + tp_category, tp_name, + libbpf_strerror_r(err, errmsg, sizeof(errmsg))); return err; } return pfd; @@ -5314,20 +5478,20 @@ struct bpf_link *bpf_program__attach_tracepoint(struct bpf_program *prog, pfd = perf_event_open_tracepoint(tp_category, tp_name); if (pfd < 0) { - pr_warning("program '%s': failed to create tracepoint '%s/%s' perf event: %s\n", - bpf_program__title(prog, false), - tp_category, tp_name, - libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); + pr_warn("program '%s': failed to create tracepoint '%s/%s' perf event: %s\n", + bpf_program__title(prog, false), + tp_category, tp_name, + libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); return ERR_PTR(pfd); } link = bpf_program__attach_perf_event(prog, pfd); if (IS_ERR(link)) { close(pfd); err = PTR_ERR(link); - pr_warning("program '%s': failed to attach to tracepoint '%s/%s': %s\n", - bpf_program__title(prog, false), - tp_category, tp_name, - libbpf_strerror_r(err, errmsg, sizeof(errmsg))); + pr_warn("program '%s': failed to attach to tracepoint '%s/%s': %s\n", + bpf_program__title(prog, false), + tp_category, tp_name, + libbpf_strerror_r(err, errmsg, sizeof(errmsg))); return link; } return link; @@ -5349,8 +5513,8 @@ struct bpf_link *bpf_program__attach_raw_tracepoint(struct bpf_program *prog, prog_fd = bpf_program__fd(prog); if (prog_fd < 0) { - pr_warning("program '%s': can't attach before loaded\n", - bpf_program__title(prog, false)); + pr_warn("program '%s': can't attach before loaded\n", + bpf_program__title(prog, false)); return ERR_PTR(-EINVAL); } @@ -5363,9 +5527,9 @@ struct bpf_link *bpf_program__attach_raw_tracepoint(struct bpf_program *prog, if (pfd < 0) { pfd = -errno; free(link); - pr_warning("program '%s': failed to attach to raw tracepoint '%s': %s\n", - bpf_program__title(prog, false), tp_name, - libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); + pr_warn("program '%s': failed to attach to raw tracepoint '%s': %s\n", + bpf_program__title(prog, false), tp_name, + libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); return ERR_PTR(pfd); } link->fd = pfd; @@ -5467,7 +5631,7 @@ static void perf_buffer__free_cpu_buf(struct perf_buffer *pb, return; if (cpu_buf->base && munmap(cpu_buf->base, pb->mmap_size + pb->page_size)) - pr_warning("failed to munmap cpu_buf #%d\n", cpu_buf->cpu); + pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu); if (cpu_buf->fd >= 0) { ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0); close(cpu_buf->fd); @@ -5520,8 +5684,8 @@ perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr, -1, PERF_FLAG_FD_CLOEXEC); if (cpu_buf->fd < 0) { err = -errno; - pr_warning("failed to open perf buffer event on cpu #%d: %s\n", - cpu, libbpf_strerror_r(err, msg, sizeof(msg))); + pr_warn("failed to open perf buffer event on cpu #%d: %s\n", + cpu, libbpf_strerror_r(err, msg, sizeof(msg))); goto error; } @@ -5531,15 +5695,15 @@ perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr, if (cpu_buf->base == MAP_FAILED) { cpu_buf->base = NULL; err = -errno; - pr_warning("failed to mmap perf buffer on cpu #%d: %s\n", - cpu, libbpf_strerror_r(err, msg, sizeof(msg))); + pr_warn("failed to mmap perf buffer on cpu #%d: %s\n", + cpu, libbpf_strerror_r(err, msg, sizeof(msg))); goto error; } if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) { err = -errno; - pr_warning("failed to enable perf buffer event on cpu #%d: %s\n", - cpu, libbpf_strerror_r(err, msg, sizeof(msg))); + pr_warn("failed to enable perf buffer event on cpu #%d: %s\n", + cpu, libbpf_strerror_r(err, msg, sizeof(msg))); goto error; } @@ -5601,7 +5765,7 @@ static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, int err, i, j, n; if (page_cnt & (page_cnt - 1)) { - pr_warning("page count should be power of two, but is %zu\n", + pr_warn("page count should be power of two, but is %zu\n", page_cnt); return ERR_PTR(-EINVAL); } @@ -5610,14 +5774,14 @@ static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, err = bpf_obj_get_info_by_fd(map_fd, &map, &map_info_len); if (err) { err = -errno; - pr_warning("failed to get map info for map FD %d: %s\n", - map_fd, libbpf_strerror_r(err, msg, sizeof(msg))); + pr_warn("failed to get map info for map FD %d: %s\n", + map_fd, libbpf_strerror_r(err, msg, sizeof(msg))); return ERR_PTR(err); } if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) { - pr_warning("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n", - map.name); + pr_warn("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n", + map.name); return ERR_PTR(-EINVAL); } @@ -5637,8 +5801,8 @@ static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC); if (pb->epoll_fd < 0) { err = -errno; - pr_warning("failed to create epoll instance: %s\n", - libbpf_strerror_r(err, msg, sizeof(msg))); + pr_warn("failed to create epoll instance: %s\n", + libbpf_strerror_r(err, msg, sizeof(msg))); goto error; } @@ -5657,19 +5821,19 @@ static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events)); if (!pb->events) { err = -ENOMEM; - pr_warning("failed to allocate events: out of memory\n"); + pr_warn("failed to allocate events: out of memory\n"); goto error; } pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs)); if (!pb->cpu_bufs) { err = -ENOMEM; - pr_warning("failed to allocate buffers: out of memory\n"); + pr_warn("failed to allocate buffers: out of memory\n"); goto error; } err = parse_cpu_mask_file(online_cpus_file, &online, &n); if (err) { - pr_warning("failed to get online CPU mask: %d\n", err); + pr_warn("failed to get online CPU mask: %d\n", err); goto error; } @@ -5698,9 +5862,9 @@ static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, &cpu_buf->fd, 0); if (err) { err = -errno; - pr_warning("failed to set cpu #%d, key %d -> perf FD %d: %s\n", - cpu, map_key, cpu_buf->fd, - libbpf_strerror_r(err, msg, sizeof(msg))); + pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n", + cpu, map_key, cpu_buf->fd, + libbpf_strerror_r(err, msg, sizeof(msg))); goto error; } @@ -5709,7 +5873,7 @@ static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd, &pb->events[j]) < 0) { err = -errno; - pr_warning("failed to epoll_ctl cpu #%d perf FD %d: %s\n", + pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n", cpu, cpu_buf->fd, libbpf_strerror_r(err, msg, sizeof(msg))); goto error; @@ -5768,7 +5932,7 @@ perf_buffer__process_record(struct perf_event_header *e, void *ctx) break; } default: - pr_warning("unknown perf sample type %d\n", e->type); + pr_warn("unknown perf sample type %d\n", e->type); return LIBBPF_PERF_EVENT_ERROR; } return LIBBPF_PERF_EVENT_CONT; @@ -5798,7 +5962,7 @@ int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms) err = perf_buffer__process_records(pb, cpu_buf); if (err) { - pr_warning("error while processing records: %d\n", err); + pr_warn("error while processing records: %d\n", err); return err; } } @@ -5995,13 +6159,13 @@ bpf_program__get_prog_info_linear(int fd, __u64 arrays) v2 = bpf_prog_info_read_offset_u32(&info_linear->info, desc->count_offset); if (v1 != v2) - pr_warning("%s: mismatch in element count\n", __func__); + pr_warn("%s: mismatch in element count\n", __func__); v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset); v2 = bpf_prog_info_read_offset_u32(&info_linear->info, desc->size_offset); if (v1 != v2) - pr_warning("%s: mismatch in rec size\n", __func__); + pr_warn("%s: mismatch in rec size\n", __func__); } /* step 7: update info_len and data_len */ @@ -6067,14 +6231,14 @@ int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz) } n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len); if (n <= 0 || n > 2) { - pr_warning("Failed to get CPU range %s: %d\n", s, n); + pr_warn("Failed to get CPU range %s: %d\n", s, n); err = -EINVAL; goto cleanup; } else if (n == 1) { end = start; } if (start < 0 || start > end) { - pr_warning("Invalid CPU range [%d,%d] in %s\n", + pr_warn("Invalid CPU range [%d,%d] in %s\n", start, end, s); err = -EINVAL; goto cleanup; @@ -6091,7 +6255,7 @@ int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz) s += len; } if (!*mask_sz) { - pr_warning("Empty CPU range\n"); + pr_warn("Empty CPU range\n"); return -EINVAL; } return 0; @@ -6109,18 +6273,18 @@ int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz) fd = open(fcpu, O_RDONLY); if (fd < 0) { err = -errno; - pr_warning("Failed to open cpu mask file %s: %d\n", fcpu, err); + pr_warn("Failed to open cpu mask file %s: %d\n", fcpu, err); return err; } len = read(fd, buf, sizeof(buf)); close(fd); if (len <= 0) { err = len ? -errno : -EINVAL; - pr_warning("Failed to read cpu mask from %s: %d\n", fcpu, err); + pr_warn("Failed to read cpu mask from %s: %d\n", fcpu, err); return err; } if (len >= sizeof(buf)) { - pr_warning("CPU mask is too big in file %s\n", fcpu); + pr_warn("CPU mask is too big in file %s\n", fcpu); return -E2BIG; } buf[len] = '\0'; diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h index 53ce212764e01582cbed7e2df5418f1e7495c0be..2b126ee5e173cdd0205eb49274d205d4cbfce0af 100644 --- a/tools/lib/bpf/libbpf.h +++ b/tools/lib/bpf/libbpf.h @@ -75,14 +75,19 @@ struct bpf_object_open_attr { * have all the padding bytes initialized to zero. It's not guaranteed though, * when copying literal, that compiler won't copy garbage in literal's padding * bytes, but that's the best way I've found and it seems to work in practice. + * + * Macro declares opts struct of given type and name, zero-initializes, + * including any extra padding, it with memset() and then assigns initial + * values provided by users in struct initializer-syntax as varargs. */ -#define LIBBPF_OPTS(TYPE, NAME, ...) \ - struct TYPE NAME; \ - memset(&NAME, 0, sizeof(struct TYPE)); \ - NAME = (struct TYPE) { \ - .sz = sizeof(struct TYPE), \ - __VA_ARGS__ \ - } +#define DECLARE_LIBBPF_OPTS(TYPE, NAME, ...) \ + struct TYPE NAME = ({ \ + memset(&NAME, 0, sizeof(struct TYPE)); \ + (struct TYPE) { \ + .sz = sizeof(struct TYPE), \ + __VA_ARGS__ \ + }; \ + }) struct bpf_object_open_opts { /* size of this struct, for forward/backward compatiblity */ @@ -302,8 +307,14 @@ LIBBPF_API int bpf_program__set_sched_cls(struct bpf_program *prog); LIBBPF_API int bpf_program__set_sched_act(struct bpf_program *prog); LIBBPF_API int bpf_program__set_xdp(struct bpf_program *prog); LIBBPF_API int bpf_program__set_perf_event(struct bpf_program *prog); +LIBBPF_API int bpf_program__set_tracing(struct bpf_program *prog); + +LIBBPF_API enum bpf_prog_type bpf_program__get_type(struct bpf_program *prog); LIBBPF_API void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type); + +LIBBPF_API enum bpf_attach_type +bpf_program__get_expected_attach_type(struct bpf_program *prog); LIBBPF_API void bpf_program__set_expected_attach_type(struct bpf_program *prog, enum bpf_attach_type type); @@ -316,6 +327,7 @@ LIBBPF_API bool bpf_program__is_sched_cls(const struct bpf_program *prog); LIBBPF_API bool bpf_program__is_sched_act(const struct bpf_program *prog); LIBBPF_API bool bpf_program__is_xdp(const struct bpf_program *prog); LIBBPF_API bool bpf_program__is_perf_event(const struct bpf_program *prog); +LIBBPF_API bool bpf_program__is_tracing(const struct bpf_program *prog); /* * No need for __attribute__((packed)), all members of 'bpf_map_def' diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map index 04b1c754b03fafc574d66c6faf6be68a2190e5b9..7627c68c572a5e89f8607227aa76d7997191b35d 100644 --- a/tools/lib/bpf/libbpf.map +++ b/tools/lib/bpf/libbpf.map @@ -194,4 +194,8 @@ LIBBPF_0.0.6 { global: bpf_object__open_file; bpf_object__open_mem; + bpf_program__get_expected_attach_type; + bpf_program__get_type; + bpf_program__is_tracing; + bpf_program__set_tracing; } LIBBPF_0.0.5; diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h index eb163554125da3061e30366c6100409cbd015078..996a8e6d6aa6d4139b1182208080383b711fa77c 100644 --- a/tools/lib/bpf/libbpf_internal.h +++ b/tools/lib/bpf/libbpf_internal.h @@ -43,7 +43,7 @@ do { \ libbpf_print(level, "libbpf: " fmt, ##__VA_ARGS__); \ } while (0) -#define pr_warning(fmt, ...) __pr(LIBBPF_WARN, fmt, ##__VA_ARGS__) +#define pr_warn(fmt, ...) __pr(LIBBPF_WARN, fmt, ##__VA_ARGS__) #define pr_info(fmt, ...) __pr(LIBBPF_INFO, fmt, ##__VA_ARGS__) #define pr_debug(fmt, ...) __pr(LIBBPF_DEBUG, fmt, ##__VA_ARGS__) @@ -52,7 +52,7 @@ static inline bool libbpf_validate_opts(const char *opts, const char *type_name) { if (user_sz < sizeof(size_t)) { - pr_warning("%s size (%zu) is too small\n", type_name, user_sz); + pr_warn("%s size (%zu) is too small\n", type_name, user_sz); return false; } if (user_sz > opts_sz) { @@ -60,7 +60,7 @@ static inline bool libbpf_validate_opts(const char *opts, for (i = opts_sz; i < user_sz; i++) { if (opts[i]) { - pr_warning("%s has non-zero extra bytes\n", + pr_warn("%s has non-zero extra bytes\n", type_name); return false; } @@ -144,7 +144,11 @@ struct bpf_line_info_min { */ enum bpf_field_info_kind { BPF_FIELD_BYTE_OFFSET = 0, /* field byte offset */ + BPF_FIELD_BYTE_SIZE = 1, BPF_FIELD_EXISTS = 2, /* field existence in target kernel */ + BPF_FIELD_SIGNED = 3, + BPF_FIELD_LSHIFT_U64 = 4, + BPF_FIELD_RSHIFT_U64 = 5, }; /* The minimum bpf_field_reloc checked by the loader diff --git a/tools/lib/bpf/libbpf_probes.c b/tools/lib/bpf/libbpf_probes.c index 84a0a0ff6671405104cf635f9a7ab62f4aedf6a9..22b9510c0b5e49c1b154c0a11f2d0f1e1fbe30a3 100644 --- a/tools/lib/bpf/libbpf_probes.c +++ b/tools/lib/bpf/libbpf_probes.c @@ -104,6 +104,7 @@ probe_load(enum bpf_prog_type prog_type, const struct bpf_insn *insns, case BPF_PROG_TYPE_SK_REUSEPORT: case BPF_PROG_TYPE_FLOW_DISSECTOR: case BPF_PROG_TYPE_CGROUP_SYSCTL: + case BPF_PROG_TYPE_TRACING: default: break; } diff --git a/tools/lib/bpf/xsk.c b/tools/lib/bpf/xsk.c index 39f4bd5c16998c5c0d00b5e4cce078487a0e4bdb..988f2b9280f9d9b08709dc7370dec23d68f8e271 100644 --- a/tools/lib/bpf/xsk.c +++ b/tools/lib/bpf/xsk.c @@ -303,7 +303,7 @@ static int xsk_load_xdp_prog(struct xsk_socket *xsk) "LGPL-2.1 or BSD-2-Clause", 0, log_buf, log_buf_size); if (prog_fd < 0) { - pr_warning("BPF log buffer:\n%s", log_buf); + pr_warn("BPF log buffer:\n%s", log_buf); return prog_fd; } @@ -494,7 +494,7 @@ int xsk_socket__create(struct xsk_socket **xsk_ptr, const char *ifname, return -EFAULT; if (umem->refcount) { - pr_warning("Error: shared umems not supported by libbpf.\n"); + pr_warn("Error: shared umems not supported by libbpf.\n"); return -EBUSY; } diff --git a/tools/testing/selftests/bpf/progs/btf__core_reloc_ints___err_wrong_sz_16.c b/tools/testing/selftests/bpf/progs/btf__core_reloc_arrays___err_wrong_val_type.c similarity index 34% rename from tools/testing/selftests/bpf/progs/btf__core_reloc_ints___err_wrong_sz_16.c rename to tools/testing/selftests/bpf/progs/btf__core_reloc_arrays___err_wrong_val_type.c index 823bac13d641f2029458ebda0271dbfdaf39426b..f5a7c832d0f21415bbaec144d03fd59aca0284cb 100644 --- a/tools/testing/selftests/bpf/progs/btf__core_reloc_ints___err_wrong_sz_16.c +++ b/tools/testing/selftests/bpf/progs/btf__core_reloc_arrays___err_wrong_val_type.c @@ -1,3 +1,3 @@ #include "core_reloc_types.h" -void f(struct core_reloc_ints___err_wrong_sz_16 x) {} +void f(struct core_reloc_arrays___err_wrong_val_type x) {} diff --git a/tools/testing/selftests/bpf/progs/btf__core_reloc_arrays___err_wrong_val_type2.c b/tools/testing/selftests/bpf/progs/btf__core_reloc_arrays___err_wrong_val_type2.c deleted file mode 100644 index 3af74b837c4d069f2beb23cabf20e7a6ada0b786..0000000000000000000000000000000000000000 --- a/tools/testing/selftests/bpf/progs/btf__core_reloc_arrays___err_wrong_val_type2.c +++ /dev/null @@ -1,3 +0,0 @@ -#include "core_reloc_types.h" - -void f(struct core_reloc_arrays___err_wrong_val_type2 x) {} diff --git a/tools/testing/selftests/bpf/progs/btf__core_reloc_ints___err_bitfield.c b/tools/testing/selftests/bpf/progs/btf__core_reloc_bitfields.c similarity index 37% rename from tools/testing/selftests/bpf/progs/btf__core_reloc_ints___err_bitfield.c rename to tools/testing/selftests/bpf/progs/btf__core_reloc_bitfields.c index 50369e8320a0f73d7ab3d9978dddca357973b23f..cff6f1836cc515e7fa88d606f696b924c855832b 100644 --- a/tools/testing/selftests/bpf/progs/btf__core_reloc_ints___err_bitfield.c +++ b/tools/testing/selftests/bpf/progs/btf__core_reloc_bitfields.c @@ -1,3 +1,3 @@ #include "core_reloc_types.h" -void f(struct core_reloc_ints___err_bitfield x) {} +void f(struct core_reloc_bitfields x) {} diff --git a/tools/testing/selftests/bpf/progs/btf__core_reloc_ints___err_wrong_sz_32.c b/tools/testing/selftests/bpf/progs/btf__core_reloc_bitfields___bit_sz_change.c similarity index 35% rename from tools/testing/selftests/bpf/progs/btf__core_reloc_ints___err_wrong_sz_32.c rename to tools/testing/selftests/bpf/progs/btf__core_reloc_bitfields___bit_sz_change.c index b44f3be18535655d024b62e9bed3c91c96dd622b..a1cd157d5451bd6b0b919859083dc33871690158 100644 --- a/tools/testing/selftests/bpf/progs/btf__core_reloc_ints___err_wrong_sz_32.c +++ b/tools/testing/selftests/bpf/progs/btf__core_reloc_bitfields___bit_sz_change.c @@ -1,3 +1,3 @@ #include "core_reloc_types.h" -void f(struct core_reloc_ints___err_wrong_sz_32 x) {} +void f(struct core_reloc_bitfields___bit_sz_change x) {} diff --git a/tools/testing/selftests/bpf/progs/btf__core_reloc_bitfields___bitfield_vs_int.c b/tools/testing/selftests/bpf/progs/btf__core_reloc_bitfields___bitfield_vs_int.c new file mode 100644 index 0000000000000000000000000000000000000000..3f2c7b07c456e636f92adbf887b86f9b30305076 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/btf__core_reloc_bitfields___bitfield_vs_int.c @@ -0,0 +1,3 @@ +#include "core_reloc_types.h" + +void f(struct core_reloc_bitfields___bitfield_vs_int x) {} diff --git a/tools/testing/selftests/bpf/progs/btf__core_reloc_arrays___err_wrong_val_type1.c b/tools/testing/selftests/bpf/progs/btf__core_reloc_bitfields___err_too_big_bitfield.c similarity index 32% rename from tools/testing/selftests/bpf/progs/btf__core_reloc_arrays___err_wrong_val_type1.c rename to tools/testing/selftests/bpf/progs/btf__core_reloc_bitfields___err_too_big_bitfield.c index 795a5b729176c3cba016c5dd9dcb771e95bbe6d3..f9746d6be399fd325185052548201474b7f48437 100644 --- a/tools/testing/selftests/bpf/progs/btf__core_reloc_arrays___err_wrong_val_type1.c +++ b/tools/testing/selftests/bpf/progs/btf__core_reloc_bitfields___err_too_big_bitfield.c @@ -1,3 +1,3 @@ #include "core_reloc_types.h" -void f(struct core_reloc_arrays___err_wrong_val_type1 x) {} +void f(struct core_reloc_bitfields___err_too_big_bitfield x) {} diff --git a/tools/testing/selftests/bpf/progs/btf__core_reloc_bitfields___just_big_enough.c b/tools/testing/selftests/bpf/progs/btf__core_reloc_bitfields___just_big_enough.c new file mode 100644 index 0000000000000000000000000000000000000000..e7c75a6953dd4e2f2d952eddc7416df943d96f73 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/btf__core_reloc_bitfields___just_big_enough.c @@ -0,0 +1,3 @@ +#include "core_reloc_types.h" + +void f(struct core_reloc_bitfields___just_big_enough x) {} diff --git a/tools/testing/selftests/bpf/progs/btf__core_reloc_ints___err_wrong_sz_8.c b/tools/testing/selftests/bpf/progs/btf__core_reloc_size.c similarity index 36% rename from tools/testing/selftests/bpf/progs/btf__core_reloc_ints___err_wrong_sz_8.c rename to tools/testing/selftests/bpf/progs/btf__core_reloc_size.c index 9f11ef5f6e883f0ef2a2caa6f49747968aa68073..3c80903da5a4e738bd5ec4e90d95af36ba60ff1a 100644 --- a/tools/testing/selftests/bpf/progs/btf__core_reloc_ints___err_wrong_sz_8.c +++ b/tools/testing/selftests/bpf/progs/btf__core_reloc_size.c @@ -1,3 +1,3 @@ #include "core_reloc_types.h" -void f(struct core_reloc_ints___err_wrong_sz_8 x) {} +void f(struct core_reloc_size x) {} diff --git a/tools/testing/selftests/bpf/progs/btf__core_reloc_ints___err_wrong_sz_64.c b/tools/testing/selftests/bpf/progs/btf__core_reloc_size___diff_sz.c similarity index 36% rename from tools/testing/selftests/bpf/progs/btf__core_reloc_ints___err_wrong_sz_64.c rename to tools/testing/selftests/bpf/progs/btf__core_reloc_size___diff_sz.c index 9a3dd2099c0f108a088e06bb7a259a4181d70fce..6dbd14436b5266f049def89c204f998648edc0bf 100644 --- a/tools/testing/selftests/bpf/progs/btf__core_reloc_ints___err_wrong_sz_64.c +++ b/tools/testing/selftests/bpf/progs/btf__core_reloc_size___diff_sz.c @@ -1,3 +1,3 @@ #include "core_reloc_types.h" -void f(struct core_reloc_ints___err_wrong_sz_64 x) {} +void f(struct core_reloc_size___diff_sz x) {} diff --git a/tools/testing/selftests/bpf/progs/core_reloc_types.h b/tools/testing/selftests/bpf/progs/core_reloc_types.h index ad763ec0ba8f18ccf65179414411f9720f936a5a..9311489e14b2b4169e9f4a08c465e216500e7fba 100644 --- a/tools/testing/selftests/bpf/progs/core_reloc_types.h +++ b/tools/testing/selftests/bpf/progs/core_reloc_types.h @@ -6,7 +6,7 @@ struct core_reloc_kernel_output { int valid[10]; - char comm[16]; + char comm[sizeof("test_progs")]; int comm_len; }; @@ -386,14 +386,7 @@ struct core_reloc_arrays___err_non_array { struct core_reloc_arrays_substruct d[1][2]; }; -struct core_reloc_arrays___err_wrong_val_type1 { - char a[5]; /* char instead of int */ - char b[2][3][4]; - struct core_reloc_arrays_substruct c[3]; - struct core_reloc_arrays_substruct d[1][2]; -}; - -struct core_reloc_arrays___err_wrong_val_type2 { +struct core_reloc_arrays___err_wrong_val_type { int a[5]; char b[2][3][4]; int c[3]; /* value is not a struct */ @@ -589,67 +582,6 @@ struct core_reloc_ints___bool { int64_t s64_field; }; -struct core_reloc_ints___err_bitfield { - uint8_t u8_field; - int8_t s8_field; - uint16_t u16_field; - int16_t s16_field; - uint32_t u32_field: 32; /* bitfields are not supported */ - int32_t s32_field; - uint64_t u64_field; - int64_t s64_field; -}; - -struct core_reloc_ints___err_wrong_sz_8 { - uint16_t u8_field; /* not 8-bit anymore */ - int16_t s8_field; /* not 8-bit anymore */ - - uint16_t u16_field; - int16_t s16_field; - uint32_t u32_field; - int32_t s32_field; - uint64_t u64_field; - int64_t s64_field; -}; - -struct core_reloc_ints___err_wrong_sz_16 { - uint8_t u8_field; - int8_t s8_field; - - uint32_t u16_field; /* not 16-bit anymore */ - int32_t s16_field; /* not 16-bit anymore */ - - uint32_t u32_field; - int32_t s32_field; - uint64_t u64_field; - int64_t s64_field; -}; - -struct core_reloc_ints___err_wrong_sz_32 { - uint8_t u8_field; - int8_t s8_field; - uint16_t u16_field; - int16_t s16_field; - - uint64_t u32_field; /* not 32-bit anymore */ - int64_t s32_field; /* not 32-bit anymore */ - - uint64_t u64_field; - int64_t s64_field; -}; - -struct core_reloc_ints___err_wrong_sz_64 { - uint8_t u8_field; - int8_t s8_field; - uint16_t u16_field; - int16_t s16_field; - uint32_t u32_field; - int32_t s32_field; - - uint32_t u64_field; /* not 64-bit anymore */ - int32_t s64_field; /* not 64-bit anymore */ -}; - /* * MISC */ @@ -730,3 +662,106 @@ struct core_reloc_existence___err_wrong_arr_value_type { struct core_reloc_existence___err_wrong_struct_type { int s; }; + +/* + * BITFIELDS + */ +/* bitfield read results, all as plain integers */ +struct core_reloc_bitfields_output { + int64_t ub1; + int64_t ub2; + int64_t ub7; + int64_t sb4; + int64_t sb20; + int64_t u32; + int64_t s32; +}; + +struct core_reloc_bitfields { + /* unsigned bitfields */ + uint8_t ub1: 1; + uint8_t ub2: 2; + uint32_t ub7: 7; + /* signed bitfields */ + int8_t sb4: 4; + int32_t sb20: 20; + /* non-bitfields */ + uint32_t u32; + int32_t s32; +}; + +/* different bit sizes (both up and down) */ +struct core_reloc_bitfields___bit_sz_change { + /* unsigned bitfields */ + uint16_t ub1: 3; /* 1 -> 3 */ + uint32_t ub2: 20; /* 2 -> 20 */ + uint8_t ub7: 1; /* 7 -> 1 */ + /* signed bitfields */ + int8_t sb4: 1; /* 4 -> 1 */ + int32_t sb20: 30; /* 20 -> 30 */ + /* non-bitfields */ + uint16_t u32; /* 32 -> 16 */ + int64_t s32; /* 32 -> 64 */ +}; + +/* turn bitfield into non-bitfield and vice versa */ +struct core_reloc_bitfields___bitfield_vs_int { + uint64_t ub1; /* 3 -> 64 non-bitfield */ + uint8_t ub2; /* 20 -> 8 non-bitfield */ + int64_t ub7; /* 7 -> 64 non-bitfield signed */ + int64_t sb4; /* 4 -> 64 non-bitfield signed */ + uint64_t sb20; /* 20 -> 16 non-bitfield unsigned */ + int32_t u32: 20; /* 32 non-bitfield -> 20 bitfield */ + uint64_t s32: 60; /* 32 non-bitfield -> 60 bitfield */ +}; + +struct core_reloc_bitfields___just_big_enough { + uint64_t ub1: 4; + uint64_t ub2: 60; /* packed tightly */ + uint32_t ub7; + uint32_t sb4; + uint32_t sb20; + uint32_t u32; + uint32_t s32; +} __attribute__((packed)) ; + +struct core_reloc_bitfields___err_too_big_bitfield { + uint64_t ub1: 4; + uint64_t ub2: 61; /* packed tightly */ + uint32_t ub7; + uint32_t sb4; + uint32_t sb20; + uint32_t u32; + uint32_t s32; +} __attribute__((packed)) ; + +/* + * SIZE + */ +struct core_reloc_size_output { + int int_sz; + int struct_sz; + int union_sz; + int arr_sz; + int arr_elem_sz; + int ptr_sz; + int enum_sz; +}; + +struct core_reloc_size { + int int_field; + struct { int x; } struct_field; + union { int x; } union_field; + int arr_field[4]; + void *ptr_field; + enum { VALUE = 123 } enum_field; +}; + +struct core_reloc_size___diff_sz { + uint64_t int_field; + struct { int x; int y; int z; } struct_field; + union { int x; char bla[123]; } union_field; + char arr_field[10]; + void *ptr_field; + enum { OTHER_VALUE = 0xFFFFFFFFFFFFFFFF } enum_field; +}; diff --git a/tools/testing/selftests/bpf/progs/test_core_reloc_bitfields_direct.c b/tools/testing/selftests/bpf/progs/test_core_reloc_bitfields_direct.c new file mode 100644 index 0000000000000000000000000000000000000000..ce116698f5af34c6ba35e83effeec40ae50c0588 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_core_reloc_bitfields_direct.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2019 Facebook + +#include +#include +#include "bpf_helpers.h" +#include "bpf_core_read.h" + +char _license[] SEC("license") = "GPL"; + +static volatile struct data { + char in[256]; + char out[256]; +} data; + +struct core_reloc_bitfields { + /* unsigned bitfields */ + uint8_t ub1: 1; + uint8_t ub2: 2; + uint32_t ub7: 7; + /* signed bitfields */ + int8_t sb4: 4; + int32_t sb20: 20; + /* non-bitfields */ + uint32_t u32; + int32_t s32; +}; + +/* bitfield read results, all as plain integers */ +struct core_reloc_bitfields_output { + int64_t ub1; + int64_t ub2; + int64_t ub7; + int64_t sb4; + int64_t sb20; + int64_t u32; + int64_t s32; +}; + +struct pt_regs; + +struct trace_sys_enter { + struct pt_regs *regs; + long id; +}; + +SEC("tp_btf/sys_enter") +int test_core_bitfields_direct(void *ctx) +{ + struct core_reloc_bitfields *in = (void *)&data.in; + struct core_reloc_bitfields_output *out = (void *)&data.out; + + out->ub1 = BPF_CORE_READ_BITFIELD(in, ub1); + out->ub2 = BPF_CORE_READ_BITFIELD(in, ub2); + out->ub7 = BPF_CORE_READ_BITFIELD(in, ub7); + out->sb4 = BPF_CORE_READ_BITFIELD(in, sb4); + out->sb20 = BPF_CORE_READ_BITFIELD(in, sb20); + out->u32 = BPF_CORE_READ_BITFIELD(in, u32); + out->s32 = BPF_CORE_READ_BITFIELD(in, s32); + + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/test_core_reloc_bitfields_probed.c b/tools/testing/selftests/bpf/progs/test_core_reloc_bitfields_probed.c new file mode 100644 index 0000000000000000000000000000000000000000..4e4133e96c37f2bf259ce26e1b0d3356fda14119 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_core_reloc_bitfields_probed.c @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2019 Facebook + +#include +#include +#include "bpf_helpers.h" +#include "bpf_core_read.h" + +char _license[] SEC("license") = "GPL"; + +static volatile struct data { + char in[256]; + char out[256]; +} data; + +struct core_reloc_bitfields { + /* unsigned bitfields */ + uint8_t ub1: 1; + uint8_t ub2: 2; + uint32_t ub7: 7; + /* signed bitfields */ + int8_t sb4: 4; + int32_t sb20: 20; + /* non-bitfields */ + uint32_t u32; + int32_t s32; +}; + +/* bitfield read results, all as plain integers */ +struct core_reloc_bitfields_output { + int64_t ub1; + int64_t ub2; + int64_t ub7; + int64_t sb4; + int64_t sb20; + int64_t u32; + int64_t s32; +}; + +#define TRANSFER_BITFIELD(in, out, field) \ + if (BPF_CORE_READ_BITFIELD_PROBED(in, field, &res)) \ + return 1; \ + out->field = res + +SEC("raw_tracepoint/sys_enter") +int test_core_bitfields(void *ctx) +{ + struct core_reloc_bitfields *in = (void *)&data.in; + struct core_reloc_bitfields_output *out = (void *)&data.out; + uint64_t res; + + TRANSFER_BITFIELD(in, out, ub1); + TRANSFER_BITFIELD(in, out, ub2); + TRANSFER_BITFIELD(in, out, ub7); + TRANSFER_BITFIELD(in, out, sb4); + TRANSFER_BITFIELD(in, out, sb20); + TRANSFER_BITFIELD(in, out, u32); + TRANSFER_BITFIELD(in, out, s32); + + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/test_core_reloc_kernel.c b/tools/testing/selftests/bpf/progs/test_core_reloc_kernel.c index ab0ea09cc11d48b90c06b9d4bc20c73e54e28e1f..13b75f68f01dc1cacbd5d74884b405c465a297ac 100644 --- a/tools/testing/selftests/bpf/progs/test_core_reloc_kernel.c +++ b/tools/testing/selftests/bpf/progs/test_core_reloc_kernel.c @@ -15,7 +15,8 @@ static volatile struct data { struct core_reloc_kernel_output { int valid[10]; - char comm[16]; + /* we have test_progs[-flavor], so cut flavor part */ + char comm[sizeof("test_progs")]; int comm_len; }; diff --git a/tools/testing/selftests/bpf/progs/test_core_reloc_size.c b/tools/testing/selftests/bpf/progs/test_core_reloc_size.c new file mode 100644 index 0000000000000000000000000000000000000000..e7fbd145a211764b62f5bdb6858ecb2cdb268171 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_core_reloc_size.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2019 Facebook + +#include +#include +#include "bpf_helpers.h" +#include "bpf_core_read.h" + +char _license[] SEC("license") = "GPL"; + +static volatile struct data { + char in[256]; + char out[256]; +} data; + +struct core_reloc_size_output { + int int_sz; + int struct_sz; + int union_sz; + int arr_sz; + int arr_elem_sz; + int ptr_sz; + int enum_sz; +}; + +struct core_reloc_size { + int int_field; + struct { int x; } struct_field; + union { int x; } union_field; + int arr_field[4]; + void *ptr_field; + enum { VALUE = 123 } enum_field; +}; + +SEC("raw_tracepoint/sys_enter") +int test_core_size(void *ctx) +{ + struct core_reloc_size *in = (void *)&data.in; + struct core_reloc_size_output *out = (void *)&data.out; + + out->int_sz = bpf_core_field_size(in->int_field); + out->struct_sz = bpf_core_field_size(in->struct_field); + out->union_sz = bpf_core_field_size(in->union_field); + out->arr_sz = bpf_core_field_size(in->arr_field); + out->arr_elem_sz = bpf_core_field_size(in->arr_field[0]); + out->ptr_sz = bpf_core_field_size(in->ptr_field); + out->enum_sz = bpf_core_field_size(in->enum_field); + + return 0; +} diff --git a/tools/testing/selftests/bpf/test_maps.c b/tools/testing/selftests/bpf/test_maps.c index 4ff1dbf949fb5d64321f370ca21324667069eaef..b5f92f5c11329f28844574f458c0245829b48071 100644 --- a/tools/testing/selftests/bpf/test_maps.c +++ b/tools/testing/selftests/bpf/test_maps.c @@ -1162,7 +1162,6 @@ static void test_sockmap(int tasks, void *data) #define MAPINMAP_PROG "./test_map_in_map.o" static void test_map_in_map(void) { - struct bpf_program *prog; struct bpf_object *obj; struct bpf_map *map; int mim_fd, fd, err; @@ -1199,9 +1198,6 @@ static void test_map_in_map(void) goto out_map_in_map; } - bpf_object__for_each_program(prog, obj) { - bpf_program__set_xdp(prog); - } bpf_object__load(obj); map = bpf_object__find_map_by_name(obj, "mim_array"); diff --git a/tools/testing/selftests/bpf/test_progs.c b/tools/testing/selftests/bpf/test_progs.c index 24c67c64b36e96d1de1d6b09f2f5378f72c58f6e..716dc0a3add12324408c166aab53476126046004 100644 --- a/tools/testing/selftests/bpf/test_progs.c +++ b/tools/testing/selftests/bpf/test_progs.c @@ -3492,21 +3492,82 @@ static void test_perf_buffer(void) .fails = true, \ } -#define EXISTENCE_DATA(struct_name) STRUCT_TO_CHAR_PTR(struct_name) { \ - .a = 42, \ -} - #define EXISTENCE_CASE_COMMON(name) \ .case_name = #name, \ .bpf_obj_file = "test_core_reloc_existence.o", \ .btf_src_file = "btf__core_reloc_" #name ".o", \ - .relaxed_core_relocs = true \ + .relaxed_core_relocs = true #define EXISTENCE_ERR_CASE(name) { \ EXISTENCE_CASE_COMMON(name), \ .fails = true, \ } +#define BITFIELDS_CASE_COMMON(objfile, test_name_prefix, name) \ + .case_name = test_name_prefix#name, \ + .bpf_obj_file = objfile, \ + .btf_src_file = "btf__core_reloc_" #name ".o" + +#define BITFIELDS_CASE(name, ...) { \ + BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_probed.o", \ + "direct:", name), \ + .input = STRUCT_TO_CHAR_PTR(core_reloc_##name) __VA_ARGS__, \ + .input_len = sizeof(struct core_reloc_##name), \ + .output = STRUCT_TO_CHAR_PTR(core_reloc_bitfields_output) \ + __VA_ARGS__, \ + .output_len = sizeof(struct core_reloc_bitfields_output), \ +}, { \ + BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_direct.o", \ + "probed:", name), \ + .input = STRUCT_TO_CHAR_PTR(core_reloc_##name) __VA_ARGS__, \ + .input_len = sizeof(struct core_reloc_##name), \ + .output = STRUCT_TO_CHAR_PTR(core_reloc_bitfields_output) \ + __VA_ARGS__, \ + .output_len = sizeof(struct core_reloc_bitfields_output), \ + .direct_raw_tp = true, \ +} + + +#define BITFIELDS_ERR_CASE(name) { \ + BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_probed.o", \ + "probed:", name), \ + .fails = true, \ +}, { \ + BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_direct.o", \ + "direct:", name), \ + .direct_raw_tp = true, \ + .fails = true, \ +} + +#define SIZE_CASE_COMMON(name) \ + .case_name = #name, \ + .bpf_obj_file = "test_core_reloc_size.o", \ + .btf_src_file = "btf__core_reloc_" #name ".o", \ + .relaxed_core_relocs = true + +#define SIZE_OUTPUT_DATA(type) \ + STRUCT_TO_CHAR_PTR(core_reloc_size_output) { \ + .int_sz = sizeof(((type *)0)->int_field), \ + .struct_sz = sizeof(((type *)0)->struct_field), \ + .union_sz = sizeof(((type *)0)->union_field), \ + .arr_sz = sizeof(((type *)0)->arr_field), \ + .arr_elem_sz = sizeof(((type *)0)->arr_field[0]), \ + .ptr_sz = sizeof(((type *)0)->ptr_field), \ + .enum_sz = sizeof(((type *)0)->enum_field), \ + } + +#define SIZE_CASE(name) { \ + SIZE_CASE_COMMON(name), \ + .input_len = 0, \ + .output = SIZE_OUTPUT_DATA(struct core_reloc_##name), \ + .output_len = sizeof(struct core_reloc_size_output), \ +} + +#define SIZE_ERR_CASE(name) { \ + SIZE_CASE_COMMON(name), \ + .fails = true, \ +} + struct core_reloc_test_case { const char *case_name; const char *bpf_obj_file; @@ -3517,6 +3578,7 @@ struct core_reloc_test_case { int output_len; bool fails; bool relaxed_core_relocs; + bool direct_raw_tp; }; static struct core_reloc_test_case test_cases[] = { @@ -3529,8 +3591,8 @@ static struct core_reloc_test_case test_cases[] = { .input_len = 0, .output = STRUCT_TO_CHAR_PTR(core_reloc_kernel_output) { .valid = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, }, - .comm = "test_progs\0\0\0\0\0", - .comm_len = 11, + .comm = "test_progs", + .comm_len = sizeof("test_progs"), }, .output_len = sizeof(struct core_reloc_kernel_output), }, @@ -3593,12 +3655,6 @@ static struct core_reloc_test_case test_cases[] = { INTS_CASE(ints___bool), INTS_CASE(ints___reverse_sign), - INTS_ERR_CASE(ints___err_bitfield), - INTS_ERR_CASE(ints___err_wrong_sz_8), - INTS_ERR_CASE(ints___err_wrong_sz_16), - INTS_ERR_CASE(ints___err_wrong_sz_32), - INTS_ERR_CASE(ints___err_wrong_sz_64), - /* validate edge cases of capturing relocations */ { .case_name = "misc", @@ -3670,6 +3726,44 @@ static struct core_reloc_test_case test_cases[] = { EXISTENCE_ERR_CASE(existence__err_arr_kind), EXISTENCE_ERR_CASE(existence__err_arr_value_type), EXISTENCE_ERR_CASE(existence__err_struct_type), + + /* bitfield relocation checks */ + BITFIELDS_CASE(bitfields, { + .ub1 = 1, + .ub2 = 2, + .ub7 = 96, + .sb4 = -7, + .sb20 = -0x76543, + .u32 = 0x80000000, + .s32 = -0x76543210, + }), + BITFIELDS_CASE(bitfields___bit_sz_change, { + .ub1 = 6, + .ub2 = 0xABCDE, + .ub7 = 1, + .sb4 = -1, + .sb20 = -0x17654321, + .u32 = 0xBEEF, + .s32 = -0x3FEDCBA987654321, + }), + BITFIELDS_CASE(bitfields___bitfield_vs_int, { + .ub1 = 0xFEDCBA9876543210, + .ub2 = 0xA6, + .ub7 = -0x7EDCBA987654321, + .sb4 = -0x6123456789ABCDE, + .sb20 = 0xD00D, + .u32 = -0x76543, + .s32 = 0x0ADEADBEEFBADB0B, + }), + BITFIELDS_CASE(bitfields___just_big_enough, { + .ub1 = 0xF, + .ub2 = 0x0812345678FEDCBA, + }), + BITFIELDS_ERR_CASE(bitfields___err_too_big_bitfield), + + /* size relocation checks */ + SIZE_CASE(size), + SIZE_CASE(size___diff_sz), }; struct data { @@ -3679,9 +3773,9 @@ struct data { static void test_core_reloc(void) { - const char *probe_name = "raw_tracepoint/sys_enter"; struct bpf_object_load_attr load_attr = {}; struct core_reloc_test_case *test_case; + const char *tp_name, *probe_name; int err, duration = 0, i, equal; struct bpf_link *link = NULL; struct bpf_map *data_map; @@ -3693,7 +3787,7 @@ static void test_core_reloc(void) for (i = 0; i < ARRAY_SIZE(test_cases); i++) { test_case = &test_cases[i]; - LIBBPF_OPTS(bpf_object_open_opts, opts, + DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts, .relaxed_core_relocs = test_case->relaxed_core_relocs, ); @@ -3703,11 +3797,19 @@ static void test_core_reloc(void) test_case->bpf_obj_file, PTR_ERR(obj))) continue; + /* for typed raw tracepoints, NULL should be specified */ + if (test_case->direct_raw_tp) { + probe_name = "tp_btf/sys_enter"; + tp_name = NULL; + } else { + probe_name = "raw_tracepoint/sys_enter"; + tp_name = "sys_enter"; + } + prog = bpf_object__find_program_by_title(obj, probe_name); if (CHECK(!prog, "find_probe", "prog '%s' not found\n", probe_name)) goto cleanup; - bpf_program__set_type(prog, BPF_PROG_TYPE_RAW_TRACEPOINT); load_attr.obj = obj; load_attr.log_level = 0; @@ -3724,7 +3826,7 @@ static void test_core_reloc(void) goto cleanup; } - link = bpf_program__attach_raw_tracepoint(prog, "sys_enter"); + link = bpf_program__attach_raw_tracepoint(prog, tp_name); if (CHECK(IS_ERR(link), "attach_raw_tp", "err %ld\n", PTR_ERR(link))) goto cleanup; diff --git a/tools/testing/selftests/bpf/test_sysctl.c b/tools/testing/selftests/bpf/test_sysctl.c index a320e3844b17f18afda20da8724af83ca32f9ae2..7c6e5b173f3342802b30419fdb3e2548788624a0 100644 --- a/tools/testing/selftests/bpf/test_sysctl.c +++ b/tools/testing/selftests/bpf/test_sysctl.c @@ -161,9 +161,14 @@ static struct sysctl_test tests[] = { .descr = "ctx:file_pos sysctl:read read ok narrow", .insns = { /* If (file_pos == X) */ +#if __BYTE_ORDER == __LITTLE_ENDIAN BPF_LDX_MEM(BPF_B, BPF_REG_7, BPF_REG_1, offsetof(struct bpf_sysctl, file_pos)), - BPF_JMP_IMM(BPF_JNE, BPF_REG_7, 0, 2), +#else + BPF_LDX_MEM(BPF_B, BPF_REG_7, BPF_REG_1, + offsetof(struct bpf_sysctl, file_pos) + 3), +#endif + BPF_JMP_IMM(BPF_JNE, BPF_REG_7, 4, 2), /* return ALLOW; */ BPF_MOV64_IMM(BPF_REG_0, 1), @@ -176,6 +181,7 @@ static struct sysctl_test tests[] = { .attach_type = BPF_CGROUP_SYSCTL, .sysctl = "kernel/ostype", .open_flags = O_RDONLY, + .seek = 4, .result = SUCCESS, }, {