Enable syscall tracing via ftrace. This also enables eBPF programs of type BPF_PROG_TYPE_TRACEPOINT to hook syscall traces and execute eBPF code.

Signed-off-by: Zachary Leaf zachary.leaf@arm.com --- arch/arm64/configs/morello_transitional_pcuabi_defconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/arch/arm64/configs/morello_transitional_pcuabi_defconfig b/arch/arm64/configs/morello_transitional_pcuabi_defconfig index 479371a69515..24e97ba15d9b 100644 --- a/arch/arm64/configs/morello_transitional_pcuabi_defconfig +++ b/arch/arm64/configs/morello_transitional_pcuabi_defconfig @@ -160,6 +160,6 @@ CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_FS=y # CONFIG_SCHED_DEBUG is not set -# CONFIG_FTRACE is not set +CONFIG_FTRACE_SYSCALLS=y CONFIG_CORESIGHT=y CONFIG_MEMTEST=y

Similar to ("arch: add compat helpers specific to 32-bit"), add 64 bit variants to determine if we're in a COMPAT64 syscall.

Signed-off-by: Zachary Leaf zachary.leaf@arm.com --- arch/arm64/include/asm/compat.h | 5 +++++ include/linux/compat.h | 6 ++++++ 2 files changed, 11 insertions(+)

diff --git a/arch/arm64/include/asm/compat.h b/arch/arm64/include/asm/compat.h index d673b829b30c..c293bda48df1 100644 --- a/arch/arm64/include/asm/compat.h +++ b/arch/arm64/include/asm/compat.h @@ -112,6 +112,11 @@ static inline int is_compat32_task(void) return IS_ENABLED(CONFIG_COMPAT32) && test_thread_flag(TIF_32BIT); }

+static inline bool is_compat64_task(void) +{ + return IS_ENABLED(CONFIG_COMPAT64) && test_thread_flag(TIF_64BIT_COMPAT); +} + static inline int is_compat_task(void) { return (IS_ENABLED(CONFIG_COMPAT32) && test_thread_flag(TIF_32BIT)) || diff --git a/include/linux/compat.h b/include/linux/compat.h index 68fd4787048f..7a87857776dc 100644 --- a/include/linux/compat.h +++ b/include/linux/compat.h @@ -1023,16 +1023,22 @@ static inline bool in_compat_syscall(void) { return is_compat_task(); } #ifndef in_compat32_syscall static inline bool in_compat32_syscall(void) { return is_compat32_task(); } #endif +#ifndef in_compat64_syscall +static inline bool in_compat64_syscall(void) { return is_compat64_task(); } +#endif

#else /* !CONFIG_COMPAT */

#define is_compat_task() (0) #define is_compat32_task() (0) +#define is_compat64_task() (0) /* Ensure no one redefines in_compat_syscall() under !CONFIG_COMPAT */ #define in_compat_syscall in_compat_syscall static inline bool in_compat_syscall(void) { return false; } #define in_compat32_syscall in_compat32_syscall static inline bool in_compat32_syscall(void) { return false; } +#define in_compat64_syscall in_compat64_syscall +static inline bool in_compat64_syscall(void) { return false; }

#endif /* CONFIG_COMPAT */

The bpf syscall does not require a compat handler for 32-bit compat. This is achieved by using u64 instead of pointer types in the bpf_attr union used to pass arguments to the syscall. This means that in a system where pointers are 32-bit, the struct/union layouts and offsets are the same as in a 64-bit arch, since the u64 field is split into two u32 fields/registers.

This greatly simplifies 32-bit compat at the small cost of requiring casting pointers passed in through the uAPI to u64 (generally via ptr_to_u64() helper functions).

This poses a problem in architectures where user pointers are longer than 64b such as Morello/PCuABI where pointers are represented as 129b capabilities. In order to extend the bpf syscall interface to accept capabilities and still retain compatibility with the existing 64/32b ABI, 64-bit compat handling and appropriate conversions must be added to handle the different union/struct sizes caused by this pointer size mis-match.

Before extending the number of bits in union bpf_attr to accept capabilities, set the groundwork for handling compat64. When in_compat64_syscall(), take the compat64 sized bpf_attr and convert it to what will be the new native offsets.

Inbound conversion is handled upfront to minimise impact on existing code and reduce overall diff size. After dispatch_bpf the majority of code can remain unchanged. The cases where conversion back out to userspace is required are handled in subsequent commits.

Signed-off-by: Zachary Leaf zachary.leaf@arm.com --- include/linux/bpf.h | 1 + include/linux/bpf_compat.h | 300 +++++++++++++++++++++++++++++++++++++ kernel/bpf/syscall.c | 259 +++++++++++++++++++++++++++++++- 3 files changed, 553 insertions(+), 7 deletions(-) create mode 100644 include/linux/bpf_compat.h

diff --git a/include/linux/bpf.h b/include/linux/bpf.h index e53ceee1df37..6c8594a0f883 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -29,6 +29,7 @@ #include <linux/rcupdate_trace.h> #include <linux/static_call.h> #include <linux/memcontrol.h> +#include <linux/bpf_compat.h>

struct bpf_verifier_env; struct bpf_verifier_log; diff --git a/include/linux/bpf_compat.h b/include/linux/bpf_compat.h new file mode 100644 index 000000000000..85e0198bede7 --- /dev/null +++ b/include/linux/bpf_compat.h @@ -0,0 +1,300 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2023 Arm Ltd */ + +union compat_bpf_attr { + struct { /* anonymous struct used by BPF_MAP_CREATE command */ + __u32 map_type; /* one of enum bpf_map_type */ + __u32 key_size; /* size of key in bytes */ + __u32 value_size; /* size of value in bytes */ + __u32 max_entries; /* max number of entries in a map */ + __u32 map_flags; /* BPF_MAP_CREATE related + * flags defined above. + */ + __u32 inner_map_fd; /* fd pointing to the inner map */ + __u32 numa_node; /* numa node (effective only if + * BPF_F_NUMA_NODE is set). + */ + char map_name[BPF_OBJ_NAME_LEN]; + __u32 map_ifindex; /* ifindex of netdev to create on */ + __u32 btf_fd; /* fd pointing to a BTF type data */ + __u32 btf_key_type_id; /* BTF type_id of the key */ + __u32 btf_value_type_id; /* BTF type_id of the value */ + __u32 btf_vmlinux_value_type_id;/* BTF type_id of a kernel- + * struct stored as the + * map value + */ + /* Any per-map-type extra fields + * + * BPF_MAP_TYPE_BLOOM_FILTER - the lowest 4 bits indicate the + * number of hash functions (if 0, the bloom filter will default + * to using 5 hash functions). + */ + __u64 map_extra; + }; + + struct { /* anonymous struct used by BPF_MAP_*_ELEM commands */ + __u32 map_fd; + __aligned_u64 key; + union { + __aligned_u64 value; + __aligned_u64 next_key; + }; + __u64 flags; + }; + + struct { /* struct used by BPF_MAP_*_BATCH commands */ + __aligned_u64 in_batch; /* start batch, + * NULL to start from beginning + */ + __aligned_u64 out_batch; /* output: next start batch */ + __aligned_u64 keys; + __aligned_u64 values; + __u32 count; /* input/output: + * input: # of key/value + * elements + * output: # of filled elements + */ + __u32 map_fd; + __u64 elem_flags; + __u64 flags; + } batch; + + struct { /* anonymous struct used by BPF_PROG_LOAD command */ + __u32 prog_type; /* one of enum bpf_prog_type */ + __u32 insn_cnt; + __aligned_u64 insns; + __aligned_u64 license; + __u32 log_level; /* verbosity level of verifier */ + __u32 log_size; /* size of user buffer */ + __aligned_u64 log_buf; /* user supplied buffer */ + __u32 kern_version; /* not used */ + __u32 prog_flags; + char prog_name[BPF_OBJ_NAME_LEN]; + __u32 prog_ifindex; /* ifindex of netdev to prep for */ + /* For some prog types expected attach type must be known at + * load time to verify attach type specific parts of prog + * (context accesses, allowed helpers, etc). + */ + __u32 expected_attach_type; + __u32 prog_btf_fd; /* fd pointing to BTF type data */ + __u32 func_info_rec_size; /* userspace bpf_func_info size */ + __aligned_u64 func_info; /* func info */ + __u32 func_info_cnt; /* number of bpf_func_info records */ + __u32 line_info_rec_size; /* userspace bpf_line_info size */ + __aligned_u64 line_info; /* line info */ + __u32 line_info_cnt; /* number of bpf_line_info records */ + __u32 attach_btf_id; /* in-kernel BTF type id to attach to */ + union { + /* valid prog_fd to attach to bpf prog */ + __u32 attach_prog_fd; + /* or valid module BTF object fd or 0 to attach to vmlinux */ + __u32 attach_btf_obj_fd; + }; + __u32 core_relo_cnt; /* number of bpf_core_relo */ + __aligned_u64 fd_array; /* array of FDs */ + __aligned_u64 core_relos; + __u32 core_relo_rec_size; /* sizeof(struct bpf_core_relo) */ + /* output: actual total log contents size (including termintaing zero). + * It could be both larger than original log_size (if log was + * truncated), or smaller (if log buffer wasn't filled completely). + */ + __u32 log_true_size; + }; + + struct { /* anonymous struct used by BPF_OBJ_* commands */ + __aligned_u64 pathname; + __u32 bpf_fd; + __u32 file_flags; + }; + + struct { /* anonymous struct used by BPF_PROG_ATTACH/DETACH commands */ + __u32 target_fd; /* container object to attach to */ + __u32 attach_bpf_fd; /* eBPF program to attach */ + __u32 attach_type; + __u32 attach_flags; + __u32 replace_bpf_fd; /* previously attached eBPF + * program to replace if + * BPF_F_REPLACE is used + */ + }; + + struct { /* anonymous struct used by BPF_PROG_TEST_RUN command */ + __u32 prog_fd; + __u32 retval; + __u32 data_size_in; /* input: len of data_in */ + __u32 data_size_out; /* input/output: len of data_out + * returns ENOSPC if data_out + * is too small. + */ + __aligned_u64 data_in; + __aligned_u64 data_out; + __u32 repeat; + __u32 duration; + __u32 ctx_size_in; /* input: len of ctx_in */ + __u32 ctx_size_out; /* input/output: len of ctx_out + * returns ENOSPC if ctx_out + * is too small. + */ + __aligned_u64 ctx_in; + __aligned_u64 ctx_out; + __u32 flags; + __u32 cpu; + __u32 batch_size; + } test; + + struct { /* anonymous struct used by BPF_*_GET_*_ID */ + union { + __u32 start_id; + __u32 prog_id; + __u32 map_id; + __u32 btf_id; + __u32 link_id; + }; + __u32 next_id; + __u32 open_flags; + }; + + struct { /* anonymous struct used by BPF_OBJ_GET_INFO_BY_FD */ + __u32 bpf_fd; + __u32 info_len; + __aligned_u64 info; + } info; + + struct { /* anonymous struct used by BPF_PROG_QUERY command */ + __u32 target_fd; /* container object to query */ + __u32 attach_type; + __u32 query_flags; + __u32 attach_flags; + __aligned_u64 prog_ids; + __u32 prog_cnt; + /* output: per-program attach_flags. + * not allowed to be set during effective query. + */ + __aligned_u64 prog_attach_flags; + } query; + + struct { /* anonymous struct used by BPF_RAW_TRACEPOINT_OPEN command */ + __u64 name; + __u32 prog_fd; + } raw_tracepoint; + + struct { /* anonymous struct for BPF_BTF_LOAD */ + __aligned_u64 btf; + __aligned_u64 btf_log_buf; + __u32 btf_size; + __u32 btf_log_size; + __u32 btf_log_level; + /* output: actual total log contents size (including termintaing zero). + * It could be both larger than original log_size (if log was + * truncated), or smaller (if log buffer wasn't filled completely). + */ + __u32 btf_log_true_size; + }; + + struct { + __u32 pid; /* input: pid */ + __u32 fd; /* input: fd */ + __u32 flags; /* input: flags */ + __u32 buf_len; /* input/output: buf len */ + __aligned_u64 buf; /* input/output: + * tp_name for tracepoint + * symbol for kprobe + * filename for uprobe + */ + __u32 prog_id; /* output: prod_id */ + __u32 fd_type; /* output: BPF_FD_TYPE_* */ + __u64 probe_offset; /* output: probe_offset */ + __u64 probe_addr; /* output: probe_addr */ + } task_fd_query; + + struct { /* struct used by BPF_LINK_CREATE command */ + union { + __u32 prog_fd; /* eBPF program to attach */ + __u32 map_fd; /* struct_ops to attach */ + }; + union { + __u32 target_fd; /* object to attach to */ + __u32 target_ifindex; /* target ifindex */ + }; + __u32 attach_type; /* attach type */ + __u32 flags; /* extra flags */ + union { + __u32 target_btf_id; /* btf_id of target to attach to */ + struct { + __aligned_u64 iter_info; /* extra bpf_iter_link_info */ + __u32 iter_info_len; /* iter_info length */ + }; + struct { + /* black box user-provided value passed through + * to BPF program at the execution time and + * accessible through bpf_get_attach_cookie() BPF helper + */ + __u64 bpf_cookie; + } perf_event; + struct { + __u32 flags; + __u32 cnt; + __aligned_u64 syms; + __aligned_u64 addrs; + __aligned_u64 cookies; + } kprobe_multi; + struct { + /* this is overlaid with the target_btf_id above. */ + __u32 target_btf_id; + /* black box user-provided value passed through + * to BPF program at the execution time and + * accessible through bpf_get_attach_cookie() BPF helper + */ + __u64 cookie; + } tracing; + struct { + __u32 pf; + __u32 hooknum; + __s32 priority; + __u32 flags; + } netfilter; + }; + } link_create; + + struct { /* struct used by BPF_LINK_UPDATE command */ + __u32 link_fd; /* link fd */ + union { + /* new program fd to update link with */ + __u32 new_prog_fd; + /* new struct_ops map fd to update link with */ + __u32 new_map_fd; + }; + __u32 flags; /* extra flags */ + union { + /* expected link's program fd; is specified only if + * BPF_F_REPLACE flag is set in flags. + */ + __u32 old_prog_fd; + /* expected link's map fd; is specified only + * if BPF_F_REPLACE flag is set. + */ + __u32 old_map_fd; + }; + } link_update; + + struct { + __u32 link_fd; + } link_detach; + + struct { /* struct used by BPF_ENABLE_STATS command */ + __u32 type; + } enable_stats; + + struct { /* struct used by BPF_ITER_CREATE command */ + __u32 link_fd; + __u32 flags; + } iter_create; + + struct { /* struct used by BPF_PROG_BIND_MAP command */ + __u32 prog_fd; + __u32 map_fd; + __u32 flags; /* extra flags */ + } prog_bind_map; + +} __attribute__((aligned(8))); + diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index f1c8733f76b8..1b8188884593 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -5015,6 +5015,257 @@ static int bpf_prog_bind_map(union bpf_attr *attr) return ret; }

+static void convert_compat_bpf_attr(union bpf_attr *dest, + const union compat_bpf_attr *cattr, int cmd) +{ + switch (cmd) { + case BPF_MAP_CREATE: + copy_field(dest, cattr, map_type); + copy_field(dest, cattr, key_size); + copy_field(dest, cattr, value_size); + copy_field(dest, cattr, max_entries); + copy_field(dest, cattr, map_flags); + copy_field(dest, cattr, inner_map_fd); + copy_field(dest, cattr, numa_node); + strncpy(dest->map_name, cattr->map_name, BPF_OBJ_NAME_LEN); + copy_field(dest, cattr, map_ifindex); + copy_field(dest, cattr, btf_fd); + copy_field(dest, cattr, btf_key_type_id); + copy_field(dest, cattr, btf_value_type_id); + copy_field(dest, cattr, btf_vmlinux_value_type_id); + copy_field(dest, cattr, map_extra); + break; + case BPF_MAP_LOOKUP_ELEM: + case BPF_MAP_UPDATE_ELEM: + case BPF_MAP_DELETE_ELEM: + case BPF_MAP_LOOKUP_AND_DELETE_ELEM: + copy_field(dest, cattr, map_fd); + copy_field(dest, cattr, key); + copy_field(dest, cattr, value); + /* u64 next_key is in a union with u64 value */ + copy_field(dest, cattr, flags); + break; + case BPF_MAP_LOOKUP_BATCH: + case BPF_MAP_LOOKUP_AND_DELETE_BATCH: + case BPF_MAP_UPDATE_BATCH: + case BPF_MAP_DELETE_BATCH: + copy_field(dest, cattr, batch.in_batch); + copy_field(dest, cattr, batch.out_batch); + copy_field(dest, cattr, batch.keys); + copy_field(dest, cattr, batch.values); + copy_field(dest, cattr, batch.count); + copy_field(dest, cattr, batch.map_fd); + copy_field(dest, cattr, batch.elem_flags); + copy_field(dest, cattr, batch.flags); + break; + case BPF_PROG_LOAD: + copy_field(dest, cattr, prog_type); + copy_field(dest, cattr, insn_cnt); + copy_field(dest, cattr, insns); + copy_field(dest, cattr, license); + copy_field(dest, cattr, log_level); + copy_field(dest, cattr, log_size); + copy_field(dest, cattr, log_buf); + copy_field(dest, cattr, kern_version); + copy_field(dest, cattr, prog_flags); + strncpy(dest->prog_name, cattr->prog_name, BPF_OBJ_NAME_LEN); + copy_field(dest, cattr, prog_ifindex); + copy_field(dest, cattr, expected_attach_type); + copy_field(dest, cattr, prog_btf_fd); + copy_field(dest, cattr, func_info_rec_size); + copy_field(dest, cattr, func_info); + copy_field(dest, cattr, func_info_cnt); + copy_field(dest, cattr, line_info_rec_size); + copy_field(dest, cattr, line_info); + copy_field(dest, cattr, line_info_cnt); + copy_field(dest, cattr, attach_btf_id); + copy_field(dest, cattr, attach_prog_fd); + /* u32 attach_btf_obj_fd is in a union with u32 attach_prog_fd */ + copy_field(dest, cattr, core_relo_cnt); + copy_field(dest, cattr, fd_array); + copy_field(dest, cattr, core_relos); + copy_field(dest, cattr, core_relo_rec_size); + copy_field(dest, cattr, log_true_size); + break; + case BPF_OBJ_PIN: + case BPF_OBJ_GET: + copy_field(dest, cattr, pathname); + copy_field(dest, cattr, bpf_fd); + copy_field(dest, cattr, file_flags); + break; + case BPF_PROG_ATTACH: + case BPF_PROG_DETACH: + copy_field(dest, cattr, target_fd); + copy_field(dest, cattr, attach_bpf_fd); + copy_field(dest, cattr, attach_type); + copy_field(dest, cattr, attach_flags); + copy_field(dest, cattr, replace_bpf_fd); + break; + case BPF_PROG_RUN: /* same as BPF_PROG_TEST_RUN */ + copy_field(dest, cattr, test.prog_fd); + copy_field(dest, cattr, test.retval); + copy_field(dest, cattr, test.data_size_in); + copy_field(dest, cattr, test.data_size_out); + copy_field(dest, cattr, test.data_in); + copy_field(dest, cattr, test.data_out); + copy_field(dest, cattr, test.repeat); + copy_field(dest, cattr, test.duration); + copy_field(dest, cattr, test.ctx_size_in); + copy_field(dest, cattr, test.ctx_size_out); + copy_field(dest, cattr, test.ctx_in); + copy_field(dest, cattr, test.ctx_out); + copy_field(dest, cattr, test.flags); + copy_field(dest, cattr, test.cpu); + copy_field(dest, cattr, test.batch_size); + break; + case BPF_PROG_GET_NEXT_ID: + case BPF_MAP_GET_NEXT_ID: + case BPF_PROG_GET_FD_BY_ID: + case BPF_MAP_GET_FD_BY_ID: + case BPF_BTF_GET_FD_BY_ID: + case BPF_BTF_GET_NEXT_ID: + case BPF_LINK_GET_FD_BY_ID: + case BPF_LINK_GET_NEXT_ID: + /* + * u32 prog_id, map_id, btf_id + link_id are in a union with + * u32 start_id + */ + copy_field(dest, cattr, start_id); + copy_field(dest, cattr, next_id); + copy_field(dest, cattr, open_flags); + break; + case BPF_OBJ_GET_INFO_BY_FD: + copy_field(dest, cattr, info.bpf_fd); + copy_field(dest, cattr, info.info_len); + copy_field(dest, cattr, info.info); + break; + case BPF_PROG_QUERY: + copy_field(dest, cattr, query.target_fd); + copy_field(dest, cattr, query.attach_type); + copy_field(dest, cattr, query.query_flags); + copy_field(dest, cattr, query.attach_flags); + copy_field(dest, cattr, query.prog_ids); + copy_field(dest, cattr, query.prog_cnt); + copy_field(dest, cattr, query.prog_attach_flags); + break; + case BPF_RAW_TRACEPOINT_OPEN: + copy_field(dest, cattr, raw_tracepoint.name); + copy_field(dest, cattr, raw_tracepoint.prog_fd); + break; + case BPF_BTF_LOAD: + copy_field(dest, cattr, btf); + copy_field(dest, cattr, btf_log_buf); + copy_field(dest, cattr, btf_size); + copy_field(dest, cattr, btf_log_size); + copy_field(dest, cattr, btf_log_level); + copy_field(dest, cattr, btf_log_true_size); + break; + case BPF_TASK_FD_QUERY: + copy_field(dest, cattr, task_fd_query.pid); + copy_field(dest, cattr, task_fd_query.fd); + copy_field(dest, cattr, task_fd_query.flags); + copy_field(dest, cattr, task_fd_query.buf_len); + copy_field(dest, cattr, task_fd_query.buf); + copy_field(dest, cattr, task_fd_query.prog_id); + copy_field(dest, cattr, task_fd_query.fd_type); + copy_field(dest, cattr, task_fd_query.probe_offset); + copy_field(dest, cattr, task_fd_query.probe_addr); + break; + case BPF_LINK_CREATE: + copy_field(dest, cattr, link_create.prog_fd); + copy_field(dest, cattr, link_create.target_fd); + /* u32 target_ifindex is in a union with u32 target_fd */ + copy_field(dest, cattr, link_create.attach_type); + copy_field(dest, cattr, link_create.flags); + + /* + * identify the union members that require conversion (i.e. with + * pointers) by attach_type, otherwise just memcpy the lot + */ + if (cattr->link_create.attach_type == BPF_TRACE_ITER) { + /* + * iter_info is a user pointer to union + * bpf_iter_link_info however since this union + * contains no pointers, the size/offsets are + * the same for compat64/purecap; hence no + * conversion needed + */ + copy_field(dest, cattr, link_create.iter_info); + copy_field(dest, cattr, link_create.iter_info_len); + break; + } + + /* kprobe_multi is used in bpf_kprobe_multi_link_attach() */ + if (cattr->link_create.attach_type == BPF_TRACE_KPROBE_MULTI) { + copy_field(dest, cattr, link_create.kprobe_multi.flags); + copy_field(dest, cattr, link_create.kprobe_multi.cnt); + copy_field(dest, cattr, link_create.kprobe_multi.syms); + copy_field(dest, cattr, link_create.kprobe_multi.addrs); + copy_field(dest, cattr, link_create.kprobe_multi.cookies); + break; + } + + /* remaining union members only contain fixed size integers */ + memcpy((u8 *)dest+offsetof(union bpf_attr, + link_create.netfilter.pf), + (u8 *)cattr+offsetof(union compat_bpf_attr, + link_create.netfilter.pf), + offsetofend(union bpf_attr, link_create.netfilter.flags) - + offsetof(union bpf_attr, link_create.netfilter.pf)); + + break; + case BPF_LINK_UPDATE: + copy_field(dest, cattr, link_update.link_fd); + copy_field(dest, cattr, link_update.new_prog_fd); + copy_field(dest, cattr, link_update.flags); + copy_field(dest, cattr, link_update.old_prog_fd); + break; + case BPF_LINK_DETACH: + copy_field(dest, cattr, link_detach.link_fd); + break; + case BPF_ENABLE_STATS: + copy_field(dest, cattr, enable_stats.type); + break; + case BPF_ITER_CREATE: + copy_field(dest, cattr, iter_create.link_fd); + copy_field(dest, cattr, iter_create.flags); + break; + case BPF_PROG_BIND_MAP: + copy_field(dest, cattr, prog_bind_map.prog_fd); + copy_field(dest, cattr, prog_bind_map.map_fd); + copy_field(dest, cattr, prog_bind_map.flags); + break; + }; +} + +static int copy_bpf_attr_from_user(union bpf_attr *attr, int cmd, + bpfptr_t uattr, unsigned int *size) +{ + union compat_bpf_attr cattr; + size_t attr_size = in_compat64_syscall() ? sizeof(union compat_bpf_attr) + : sizeof(union bpf_attr); + int err; + + err = bpf_check_uarg_tail_zero(uattr, attr_size, *size); + if (err) + return err; + *size = min_t(u32, *size, attr_size); + + /* copy attributes from user space, may be less than sizeof(bpf_attr) */ + memset(attr, 0, sizeof(*attr)); + if (in_compat64_syscall()) { + memset(&cattr, 0, sizeof(cattr)); + if (copy_from_bpfptr(&cattr, uattr, *size) != 0) + return -EFAULT; + convert_compat_bpf_attr(attr, &cattr, cmd); + } else { + if (copy_from_bpfptr(attr, uattr, *size) != 0) + return -EFAULT; + } + + return 0; +} + static int __sys_bpf(int cmd, bpfptr_t uattr, unsigned int size) { union bpf_attr attr; @@ -5035,15 +5286,9 @@ static int __sys_bpf(int cmd, bpfptr_t uattr, unsigned int size) (cmd == BPF_MAP_CREATE || cmd == BPF_PROG_LOAD)) return -EPERM;

- err = bpf_check_uarg_tail_zero(uattr, sizeof(attr), size); + err = copy_bpf_attr_from_user(&attr, cmd, uattr, &size); if (err) return err; - size = min_t(u32, size, sizeof(attr)); - - /* copy attributes from user space, may be less than sizeof(bpf_attr) */ - memset(&attr, 0, sizeof(attr)); - if (copy_from_bpfptr(&attr, uattr, size) != 0) - return -EFAULT;

err = security_bpf(cmd, &attr, size); if (err < 0)

The patch "bpf: compat64: add handler and convert bpf_attr in" handled the case where a compat64 version of union bpf_attr is passed into the kernel via the bpf syscall. It resolved the differences in size and member offsets by copying the compat version into a native version of the struct to be used in the kernel.

When writing back out to the original __user *bpf_attr union, we must reverse the conversion to correctly write to the appropriate offsets.

The "technically correct" approach would probably be to replace all functions that have a union bpf_attr __user* parameter with void __user* (at least where we pass uattr.user in dispatch_bpf() and propagate this down). This forces us to check the type and cast before we use uattr, since the type is technically unknown. This avoids any potential mis-use by not checking for compat64 before trying to access a compat_bpf_attr ptr with native union offsets. The downside of this approach is we would end up touching many function signatures as uattr propagates down the call stack (currently 73).

The minimal diff approach used here is to handle only specifically where we write directly to uattr with macros to select the correct struct type (compat or native). This greatly reduces the diff size at the cost of having compat_bpf_attr unions passed around as union bpf_attr and needing to remember to cast back where required.

The instances where we write out to uattr were found by grepping 'copy_to_user(&uattr', 'put_user(.*uattr' and 'copy_to_bpfptr' so the list may or may not be complete. Luckily this doesn't appear to happen that often and usage seems to be consistent.

We can also replace all instances of copy_to_user() here with put_user() which somewhat simplifies things. Similarly since we're only copying individual members with copy_to_bpfptr_offset(), bpfptr_put_uattr() usage is simplified by not having to specify the size.

Note: conversion out is only required where we're writing out to the original userspace union bpf_attr i.e. uattr. There are many instances where we receive a __user ptr from userspace via bpf_attr, e.g. a ptr to some userspace buffer. This pointer is copied into a native struct (it may be converted from a compat layout), and we can then *write directly to pointer target* without requiring any additional conversion out. copy_to_user() or put_user() calls where the target is a __user ptr (not a ptr to the __user union bpf_attr) therefore do not require any additional handling, since the offsets and struct layouts are irrelevant.

Signed-off-by: Zachary Leaf zachary.leaf@arm.com --- drivers/media/rc/bpf-lirc.c | 4 ++-- include/linux/bpf.h | 14 ++++++++++++++ include/linux/bpfptr.h | 9 +++++++++ kernel/bpf/btf.c | 5 ++--- kernel/bpf/cgroup.c | 4 ++-- kernel/bpf/hashtab.c | 4 ++-- kernel/bpf/net_namespace.c | 4 ++-- kernel/bpf/syscall.c | 22 +++++++++++----------- kernel/bpf/verifier.c | 17 +++++------------ net/bpf/bpf_dummy_struct_ops.c | 2 +- net/bpf/test_run.c | 15 +++++++-------- net/core/sock_map.c | 4 ++-- 12 files changed, 59 insertions(+), 45 deletions(-)

diff --git a/drivers/media/rc/bpf-lirc.c b/drivers/media/rc/bpf-lirc.c index fe17c7f98e81..7f713422049d 100644 --- a/drivers/media/rc/bpf-lirc.c +++ b/drivers/media/rc/bpf-lirc.c @@ -319,12 +319,12 @@ int lirc_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr) progs = lirc_rcu_dereference(rcdev->raw->progs); cnt = progs ? bpf_prog_array_length(progs) : 0;

- if (copy_to_user(&uattr->query.prog_cnt, &cnt, sizeof(cnt))) { + if (bpf_put_uattr(cnt, uattr, query.prog_cnt)) { ret = -EFAULT; goto unlock; }

- if (copy_to_user(&uattr->query.attach_flags, &flags, sizeof(flags))) { + if (bpf_put_uattr(flags, uattr, query.attach_flags)) { ret = -EFAULT; goto unlock; } diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 6c8594a0f883..8bc99afd0fc0 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -29,6 +29,7 @@ #include <linux/rcupdate_trace.h> #include <linux/static_call.h> #include <linux/memcontrol.h> +#include <linux/compat.h> #include <linux/bpf_compat.h>

struct bpf_verifier_env; @@ -59,6 +60,19 @@ extern struct kobject *btf_kobj; extern struct bpf_mem_alloc bpf_global_ma; extern bool bpf_global_ma_set;

+#define __bpf_put_uattr(x, uattr, to_field) \ + (put_user(x, &((uattr)->to_field))) + +#define bpf_put_uattr(x, uattr, to_field) \ + (in_compat64_syscall() ? \ + __bpf_put_uattr(x, (union compat_bpf_attr __user *)uattr, to_field) : \ + __bpf_put_uattr(x, (union bpf_attr __user *)uattr, to_field)) + +#define bpf_field_exists(uattr_size, field) \ + (in_compat64_syscall() ? \ + (uattr_size >= offsetofend(union compat_bpf_attr, field)) : \ + (uattr_size >= offsetofend(union bpf_attr, field))) + typedef u64 (*bpf_callback_t)(u64, u64, u64, u64, u64); typedef int (*bpf_iter_init_seq_priv_t)(void *private_data, struct bpf_iter_aux_info *aux); diff --git a/include/linux/bpfptr.h b/include/linux/bpfptr.h index 79b2f78eec1a..f798677e1b8e 100644 --- a/include/linux/bpfptr.h +++ b/include/linux/bpfptr.h @@ -3,11 +3,20 @@ #ifndef _LINUX_BPFPTR_H #define _LINUX_BPFPTR_H

+#include <linux/compat.h> #include <linux/mm.h> #include <linux/sockptr.h>

typedef sockptr_t bpfptr_t;

+#define __bpfptr_put_uattr(type, x, uattr, to_field) \ + (copy_to_bpfptr_offset(uattr, offsetof(type, to_field), &x, sizeof(x))) + +#define bpfptr_put_uattr(x, uattr, to_field) \ + (in_compat64_syscall() ? \ + __bpfptr_put_uattr(union compat_bpf_attr, x, uattr, to_field) : \ + __bpfptr_put_uattr(union bpf_attr, x, uattr, to_field)) + static inline bool bpfptr_is_kernel(bpfptr_t bpfptr) { return bpfptr.is_kernel; diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 72b32b7cd9cd..061a8356abf5 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -5454,9 +5454,8 @@ static int finalize_log(struct bpf_verifier_log *log, bpfptr_t uattr, u32 uattr_

err = bpf_vlog_finalize(log, &log_true_size);

- if (uattr_size >= offsetofend(union bpf_attr, btf_log_true_size) && - copy_to_bpfptr_offset(uattr, offsetof(union bpf_attr, btf_log_true_size), - &log_true_size, sizeof(log_true_size))) + if (bpf_field_exists(uattr_size, btf_log_true_size) && + bpfptr_put_uattr(log_true_size, uattr, btf_log_true_size)) err = -EFAULT;

return err; diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 517b6a5928cc..44bbd2309ce5 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -1061,9 +1061,9 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,

/* always output uattr->query.attach_flags as 0 during effective query */ flags = effective_query ? 0 : flags; - if (copy_to_user(&uattr->query.attach_flags, &flags, sizeof(flags))) + if (bpf_put_uattr(flags, uattr, query.attach_flags)) return -EFAULT; - if (copy_to_user(&uattr->query.prog_cnt, &total_cnt, sizeof(total_cnt))) + if (bpf_put_uattr(total_cnt, uattr, query.prog_cnt)) return -EFAULT; if (attr->query.prog_cnt == 0 || !prog_ids || !total_cnt) /* return early if user requested only program count + flags */ diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 9901efee4339..522760031688 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -1693,7 +1693,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map, if (!max_count) return 0;

- if (put_user(0, &uattr->batch.count)) + if (bpf_put_uattr(0, uattr, batch.count)) return -EFAULT;

batch = 0; @@ -1875,7 +1875,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map, /* copy # of entries and next batch */ ubatch = u64_to_user_ptr(attr->batch.out_batch); if (copy_to_user(ubatch, &batch, sizeof(batch)) || - put_user(total, &uattr->batch.count)) + bpf_put_uattr(total, uattr, batch.count)) ret = -EFAULT;

out: diff --git a/kernel/bpf/net_namespace.c b/kernel/bpf/net_namespace.c index 868cc2c43899..466c7b1acb61 100644 --- a/kernel/bpf/net_namespace.c +++ b/kernel/bpf/net_namespace.c @@ -257,9 +257,9 @@ static int __netns_bpf_prog_query(const union bpf_attr *attr, if (run_array) prog_cnt = bpf_prog_array_length(run_array);

- if (copy_to_user(&uattr->query.attach_flags, &flags, sizeof(flags))) + if (bpf_put_uattr(flags, uattr, query.attach_flags)) return -EFAULT; - if (copy_to_user(&uattr->query.prog_cnt, &prog_cnt, sizeof(prog_cnt))) + if (bpf_put_uattr(prog_cnt, uattr, query.prog_cnt)) return -EFAULT; if (!attr->query.prog_cnt || !prog_ids || !prog_cnt) return 0; diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 1b8188884593..d2a768cd6ecb 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1649,7 +1649,7 @@ int generic_map_delete_batch(struct bpf_map *map, break; cond_resched(); } - if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp))) + if (bpf_put_uattr(cp, uattr, batch.count)) err = -EFAULT;

kvfree(key); @@ -1707,7 +1707,7 @@ int generic_map_update_batch(struct bpf_map *map, struct file *map_file, cond_resched(); }

- if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp))) + if (bpf_put_uattr(cp, uattr, batch.count)) err = -EFAULT;

kvfree(value); @@ -1742,7 +1742,7 @@ int generic_map_lookup_batch(struct bpf_map *map, if (!max_count) return 0;

- if (put_user(0, &uattr->batch.count)) + if (bpf_put_uattr(0, uattr, batch.count)) return -EFAULT;

buf_prevkey = kvmalloc(map->key_size, GFP_USER | __GFP_NOWARN); @@ -1807,8 +1807,8 @@ int generic_map_lookup_batch(struct bpf_map *map, if (err == -EFAULT) goto free_buf;

- if ((copy_to_user(&uattr->batch.count, &cp, sizeof(cp)) || - (cp && copy_to_user(uobatch, prev_key, map->key_size)))) + if (bpf_put_uattr(cp, uattr, batch.count) || + (cp && copy_to_user(uobatch, prev_key, map->key_size))) err = -EFAULT;

free_buf: @@ -3716,7 +3716,7 @@ static int bpf_obj_get_next_id(const union bpf_attr *attr, spin_unlock_bh(lock);

if (!err) - err = put_user(next_id, &uattr->next_id); + err = bpf_put_uattr(next_id, uattr, next_id);

return err; } @@ -4408,7 +4408,7 @@ static int bpf_task_fd_query_copy(const union bpf_attr *attr, u32 len = buf ? strlen(buf) : 0, input_len; int err = 0;

- if (put_user(len, &uattr->task_fd_query.buf_len)) + if (bpf_put_uattr(len, uattr, task_fd_query.buf_len)) return -EFAULT; input_len = attr->task_fd_query.buf_len; if (input_len && ubuf) { @@ -4436,10 +4436,10 @@ static int bpf_task_fd_query_copy(const union bpf_attr *attr, } }

- if (put_user(prog_id, &uattr->task_fd_query.prog_id) || - put_user(fd_type, &uattr->task_fd_query.fd_type) || - put_user(probe_offset, &uattr->task_fd_query.probe_offset) || - put_user(probe_addr, &uattr->task_fd_query.probe_addr)) + if (bpf_put_uattr(prog_id, uattr, task_fd_query.prog_id) || + bpf_put_uattr(fd_type, uattr, task_fd_query.fd_type) || + bpf_put_uattr(probe_offset, uattr, task_fd_query.probe_offset) || + bpf_put_uattr(probe_addr, uattr, task_fd_query.probe_addr)) return -EFAULT;

return err; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index cf5f230360f5..fdec45bbbc7e 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -14253,9 +14253,7 @@ static int check_btf_func(struct bpf_verifier_env *env, /* set the size kernel expects so loader can zero * out the rest of the record. */ - if (copy_to_bpfptr_offset(uattr, - offsetof(union bpf_attr, func_info_rec_size), - &min_size, sizeof(min_size))) + if (bpfptr_put_uattr(min_size, uattr, func_info_rec_size)) ret = -EFAULT; } goto err_free; @@ -14387,9 +14385,7 @@ static int check_btf_line(struct bpf_verifier_env *env, if (err) { if (err == -E2BIG) { verbose(env, "nonzero tailing record in line_info"); - if (copy_to_bpfptr_offset(uattr, - offsetof(union bpf_attr, line_info_rec_size), - &expected_size, sizeof(expected_size))) + if (bpfptr_put_uattr(expected_size, uattr, line_info_rec_size)) err = -EFAULT; } goto err_free; @@ -14510,9 +14506,7 @@ static int check_core_relo(struct bpf_verifier_env *env, if (err) { if (err == -E2BIG) { verbose(env, "nonzero tailing record in core_relo"); - if (copy_to_bpfptr_offset(uattr, - offsetof(union bpf_attr, core_relo_rec_size), - &expected_size, sizeof(expected_size))) + if (bpfptr_put_uattr(expected_size, uattr, core_relo_rec_size)) err = -EFAULT; } break; @@ -18957,9 +18951,8 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3 if (err) ret = err;

- if (uattr_size >= offsetofend(union bpf_attr, log_true_size) && - copy_to_bpfptr_offset(uattr, offsetof(union bpf_attr, log_true_size), - &log_true_size, sizeof(log_true_size))) { + if (bpf_field_exists(uattr_size, log_true_size) && + bpfptr_put_uattr(log_true_size, uattr, log_true_size)) { ret = -EFAULT; goto err_release_maps; } diff --git a/net/bpf/bpf_dummy_struct_ops.c b/net/bpf/bpf_dummy_struct_ops.c index 5918d1b32e19..c7bd67d75dac 100644 --- a/net/bpf/bpf_dummy_struct_ops.c +++ b/net/bpf/bpf_dummy_struct_ops.c @@ -130,7 +130,7 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, err = dummy_ops_copy_args(args); if (err) goto out; - if (put_user(prog_ret, &uattr->test.retval)) + if (bpf_put_uattr(prog_ret, uattr, test.retval)) err = -EFAULT; out: kfree(args); diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c index e79e3a415ca9..8ffff57e13a0 100644 --- a/net/bpf/test_run.c +++ b/net/bpf/test_run.c @@ -485,11 +485,11 @@ static int bpf_test_finish(const union bpf_attr *kattr, } }

- if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size))) + if (bpf_put_uattr(size, uattr, test.data_size_out)) goto out; - if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) + if (bpf_put_uattr(retval, uattr, test.retval)) goto out; - if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration))) + if (bpf_put_uattr(duration, uattr, test.duration)) goto out; if (err != -ENOSPC) err = 0; @@ -871,7 +871,7 @@ int bpf_prog_test_run_tracing(struct bpf_prog *prog, }

retval = ((u32)side_effect << 16) | ret; - if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) + if (bpf_put_uattr(retval, uattr, test.retval)) goto out;

err = 0; @@ -946,8 +946,7 @@ int bpf_prog_test_run_raw_tp(struct bpf_prog *prog, } put_cpu();

- if (!err && - copy_to_user(&uattr->test.retval, &info.retval, sizeof(u32))) + if (!err && bpf_put_uattr(info.retval, uattr, test.retval)) err = -EFAULT;

kfree(info.ctx); @@ -1003,7 +1002,7 @@ static int bpf_ctx_finish(const union bpf_attr *kattr,

if (copy_to_user(data_out, data, copy_size)) goto out; - if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size))) + if (bpf_put_uattr(size, uattr, test.ctx_size_out)) goto out; if (err != -ENOSPC) err = 0; @@ -1681,7 +1680,7 @@ int bpf_prog_test_run_syscall(struct bpf_prog *prog, retval = bpf_prog_run_pin_on_cpu(prog, ctx); rcu_read_unlock_trace();

- if (copy_to_user(&uattr->test.retval, &retval, sizeof(u32))) { + if (bpf_put_uattr(retval, uattr, test.retval)) { err = -EFAULT; goto out; } diff --git a/net/core/sock_map.c b/net/core/sock_map.c index 00afb66cd095..782e9f7799ae 100644 --- a/net/core/sock_map.c +++ b/net/core/sock_map.c @@ -1547,9 +1547,9 @@ int sock_map_bpf_prog_query(const union bpf_attr *attr, end: rcu_read_unlock();

- if (copy_to_user(&uattr->query.attach_flags, &flags, sizeof(flags)) || + if (bpf_put_uattr(flags, uattr, query.attach_flags) || (id != 0 && copy_to_user(prog_ids, &id, sizeof(u32))) || - copy_to_user(&uattr->query.prog_cnt, &prog_cnt, sizeof(prog_cnt))) + bpf_put_uattr(prog_cnt, uattr, query.prog_cnt)) ret = -EFAULT;

fdput(f);

union bpf_attr member info.info is a __user *ptr to either: struct bpf_btf_info struct bpf_prog_info struct bpf_map_info struct bpf_link_info

These info structs are passed in from userspace and used to store information returned from calls to BPF_OBJ_GET_INFO_BY_FD.

While inbound conversion for all other members of union bpf_attr is handled upfront, this same approach is not possible here for two main reasons:

1. Due to the extra complexity of a further userspace struct that is one of 4 types. These types themselves can be a union/multiplex of even more additional options and types. It is not straight forward or possible in many cases to determine which elements of the union are active upfront, and each has different native/compat64 layouts/offsets (except bpf_map_info - see below).

2. Due to the nature of this subcommand, the main purpose is to copy information back out to userspace, there is more work to do to convert the struct back out to compat64 compatible offsets compared to other subcommands.

Instead of an upfront conversion, convert where each is used at the end site both in/out.

Note: since struct bpf_map_info contains no pointers, the layout is the same in native/compat64. No conversion or additional handling is therefore required.

Signed-off-by: Zachary Leaf zachary.leaf@arm.com --- include/linux/bpf_compat.h | 112 ++++++++++++++ kernel/bpf/btf.c | 94 ++++++++++-- kernel/bpf/syscall.c | 302 ++++++++++++++++++++++++++++++++++--- 3 files changed, 473 insertions(+), 35 deletions(-)

diff --git a/include/linux/bpf_compat.h b/include/linux/bpf_compat.h index 85e0198bede7..710815417a27 100644 --- a/include/linux/bpf_compat.h +++ b/include/linux/bpf_compat.h @@ -298,3 +298,115 @@ union compat_bpf_attr {

} __attribute__((aligned(8)));

+struct compat_bpf_prog_info { + __u32 type; + __u32 id; + __u8 tag[BPF_TAG_SIZE]; + __u32 jited_prog_len; + __u32 xlated_prog_len; + __aligned_u64 jited_prog_insns; + __aligned_u64 xlated_prog_insns; + __u64 load_time; /* ns since boottime */ + __u32 created_by_uid; + __u32 nr_map_ids; + __aligned_u64 map_ids; + char name[BPF_OBJ_NAME_LEN]; + __u32 ifindex; + __u32 gpl_compatible:1; + __u32:31; /* alignment pad */ + __u64 netns_dev; + __u64 netns_ino; + __u32 nr_jited_ksyms; + __u32 nr_jited_func_lens; + __aligned_u64 jited_ksyms; + __aligned_u64 jited_func_lens; + __u32 btf_id; + __u32 func_info_rec_size; + __aligned_u64 func_info; + __u32 nr_func_info; + __u32 nr_line_info; + __aligned_u64 line_info; + __aligned_u64 jited_line_info; + __u32 nr_jited_line_info; + __u32 line_info_rec_size; + __u32 jited_line_info_rec_size; + __u32 nr_prog_tags; + __aligned_u64 prog_tags; + __u64 run_time_ns; + __u64 run_cnt; + __u64 recursion_misses; + __u32 verified_insns; + __u32 attach_btf_obj_id; + __u32 attach_btf_id; +} __attribute__((aligned(8))); + +struct compat_bpf_btf_info { + __aligned_u64 btf; + __u32 btf_size; + __u32 id; + __aligned_u64 name; + __u32 name_len; + __u32 kernel_btf; +} __attribute__((aligned(8))); + +struct compat_bpf_link_info { + __u32 type; + __u32 id; + __u32 prog_id; + union { + struct { + __aligned_u64 tp_name; /* in/out: tp_name buffer ptr */ + __u32 tp_name_len; /* in/out: tp_name buffer len */ + } raw_tracepoint; + struct { + __u32 attach_type; + __u32 target_obj_id; /* prog_id for PROG_EXT, otherwise btf object id */ + __u32 target_btf_id; /* BTF type id inside the object */ + } tracing; + struct { + __u64 cgroup_id; + __u32 attach_type; + } cgroup; + struct { + __aligned_u64 target_name; /* in/out: target_name buffer ptr */ + __u32 target_name_len; /* in/out: target_name buffer len */ + + /* If the iter specific field is 32 bits, it can be put + * in the first or second union. Otherwise it should be + * put in the second union. + */ + union { + struct { + __u32 map_id; + } map; + }; + union { + struct { + __u64 cgroup_id; + __u32 order; + } cgroup; + struct { + __u32 tid; + __u32 pid; + } task; + }; + } iter; + struct { + __u32 netns_ino; + __u32 attach_type; + } netns; + struct { + __u32 ifindex; + } xdp; + struct { + __u32 map_id; + } struct_ops; + struct { + __u32 pf; + __u32 hooknum; + __s32 priority; + __u32 flags; + } netfilter; + }; +} __attribute__((aligned(8))); + diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index e7dac3c07f08..58421266a5b8 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7214,31 +7214,96 @@ struct btf *btf_get_by_fd(int fd) return btf; }

+static void convert_compat_btf_info_in(struct bpf_btf_info *dest, + const struct compat_bpf_btf_info *cinfo) +{ + copy_field(dest, cinfo, btf); + copy_field(dest, cinfo, btf_size); + copy_field(dest, cinfo, id); + copy_field(dest, cinfo, name); + copy_field(dest, cinfo, name_len); + copy_field(dest, cinfo, kernel_btf); +} + +static void convert_compat_btf_info_out(struct compat_bpf_btf_info *dest, + const struct bpf_btf_info *info) +{ + copy_field(dest, info, btf); + copy_field(dest, info, btf_size); + copy_field(dest, info, id); + copy_field(dest, info, name); + copy_field(dest, info, name_len); + copy_field(dest, info, kernel_btf); +} + +static int copy_bpf_btf_info_from_user(const union bpf_attr *attr, + struct bpf_btf_info *info, + u32 *info_len) +{ + struct compat_bpf_btf_info cinfo; + int err; + size_t info_size = in_compat64_syscall() ? sizeof(struct compat_bpf_btf_info) + : sizeof(struct bpf_btf_info); + void __user *uinfo = u64_to_user_ptr(attr->info.info); + + *info_len = attr->info.info_len; + err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), + info_size, *info_len); + if (err) + return err; + *info_len = min_t(u32, info_size, *info_len); + + memset(info, 0, sizeof(*info)); + if (in_compat64_syscall()) { + memset(&cinfo, 0, sizeof(cinfo)); + if (copy_from_user(&cinfo, uinfo, *info_len)) + return -EFAULT; + convert_compat_btf_info_in(info, &cinfo); + } else { + if (copy_from_user(info, uinfo, *info_len)) + return -EFAULT; + } + + return 0; +} + +static int copy_bpf_btf_info_to_user(const union bpf_attr *attr, + union bpf_attr __user *uattr, + struct bpf_btf_info *info, + u32 *info_len) +{ + struct compat_bpf_btf_info cinfo; + void *src_info = in_compat64_syscall() ? (struct bpf_btf_info *)&cinfo + : info; + void __user *uinfo = u64_to_user_ptr(attr->info.info); + + if (in_compat64_syscall()) { + memset(&cinfo, 0, sizeof(cinfo)); + convert_compat_btf_info_out(&cinfo, info); + } + + if (copy_to_user(uinfo, src_info, *info_len) || + bpf_put_uattr(*info_len, uattr, info.info_len)) + return -EFAULT; + + return 0; +} + int btf_get_info_by_fd(const struct btf *btf, const union bpf_attr *attr, union bpf_attr __user *uattr) { - struct bpf_btf_info __user *uinfo; struct bpf_btf_info info; - u32 info_copy, btf_copy; + u32 btf_copy; void __user *ubtf; char __user *uname; u32 uinfo_len, uname_len, name_len; - int ret = 0; - - uinfo = u64_to_user_ptr(attr->info.info); - uinfo_len = attr->info.info_len; + int ret;

- ret = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(*uinfo), - uinfo_len); + ret = copy_bpf_btf_info_from_user(attr, &info, &uinfo_len); if (ret) return ret;

- info_copy = min_t(u32, uinfo_len, sizeof(info)); - memset(&info, 0, sizeof(info)); - if (copy_from_user(&info, uinfo, info_copy)) - return -EFAULT; - info.id = btf->id; ubtf = u64_to_user_ptr(info.btf); btf_copy = min_t(u32, btf->data_size, info.btf_size); @@ -7272,8 +7337,7 @@ int btf_get_info_by_fd(const struct btf *btf, } }

- if (copy_to_user(uinfo, &info, info_copy) || - put_user(info_copy, &uattr->info.info_len)) + if (copy_bpf_btf_info_to_user(attr, uattr, &info, &uinfo_len)) return -EFAULT;

return ret; diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 8a052c4bb11c..bad24f30e53f 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -3950,28 +3950,163 @@ static int set_info_rec_size(struct bpf_prog_info *info) return 0; }

+static void +convert_compat_prog_info_in(struct bpf_prog_info *dest, + const struct compat_bpf_prog_info *cinfo) +{ + copy_field(dest, cinfo, type); + copy_field(dest, cinfo, id); + memcpy(dest->tag, cinfo->tag, BPF_TAG_SIZE); + copy_field(dest, cinfo, jited_prog_len); + copy_field(dest, cinfo, xlated_prog_len); + copy_field(dest, cinfo, jited_prog_insns); + copy_field(dest, cinfo, xlated_prog_insns); + copy_field(dest, cinfo, load_time); + copy_field(dest, cinfo, created_by_uid); + copy_field(dest, cinfo, nr_map_ids); + copy_field(dest, cinfo, map_ids); + strncpy((char *)dest->name, (char *)cinfo->name, BPF_OBJ_NAME_LEN); + copy_field(dest, cinfo, ifindex); + copy_field(dest, cinfo, gpl_compatible); + copy_field(dest, cinfo, netns_dev); + copy_field(dest, cinfo, netns_ino); + copy_field(dest, cinfo, nr_jited_ksyms); + copy_field(dest, cinfo, nr_jited_func_lens); + copy_field(dest, cinfo, jited_ksyms); + copy_field(dest, cinfo, jited_func_lens); + copy_field(dest, cinfo, btf_id); + copy_field(dest, cinfo, func_info_rec_size); + copy_field(dest, cinfo, func_info); + copy_field(dest, cinfo, nr_func_info); + copy_field(dest, cinfo, nr_line_info); + copy_field(dest, cinfo, line_info); + copy_field(dest, cinfo, jited_line_info); + copy_field(dest, cinfo, nr_jited_line_info); + copy_field(dest, cinfo, line_info_rec_size); + copy_field(dest, cinfo, jited_line_info_rec_size); + copy_field(dest, cinfo, nr_prog_tags); + copy_field(dest, cinfo, prog_tags); + copy_field(dest, cinfo, run_time_ns); + copy_field(dest, cinfo, run_cnt); + copy_field(dest, cinfo, recursion_misses); + copy_field(dest, cinfo, verified_insns); + copy_field(dest, cinfo, attach_btf_obj_id); + copy_field(dest, cinfo, attach_btf_id); +} + +static void +convert_compat_prog_info_out(struct compat_bpf_prog_info *dest, + const struct bpf_prog_info *info) +{ + copy_field(dest, info, type); + copy_field(dest, info, id); + memcpy(dest->tag, info->tag, BPF_TAG_SIZE); + copy_field(dest, info, jited_prog_len); + copy_field(dest, info, xlated_prog_len); + copy_field(dest, info, jited_prog_insns); + copy_field(dest, info, xlated_prog_insns); + copy_field(dest, info, load_time); + copy_field(dest, info, created_by_uid); + copy_field(dest, info, nr_map_ids); + copy_field(dest, info, map_ids); + strncpy((char *)dest->name, (char *)info->name, BPF_OBJ_NAME_LEN); + copy_field(dest, info, ifindex); + copy_field(dest, info, gpl_compatible); + copy_field(dest, info, netns_dev); + copy_field(dest, info, netns_ino); + copy_field(dest, info, nr_jited_ksyms); + copy_field(dest, info, nr_jited_func_lens); + copy_field(dest, info, jited_ksyms); + copy_field(dest, info, jited_func_lens); + copy_field(dest, info, btf_id); + copy_field(dest, info, func_info_rec_size); + copy_field(dest, info, func_info); + copy_field(dest, info, nr_func_info); + copy_field(dest, info, nr_line_info); + copy_field(dest, info, line_info); + copy_field(dest, info, jited_line_info); + copy_field(dest, info, nr_jited_line_info); + copy_field(dest, info, line_info_rec_size); + copy_field(dest, info, jited_line_info_rec_size); + copy_field(dest, info, nr_prog_tags); + copy_field(dest, info, prog_tags); + copy_field(dest, info, run_time_ns); + copy_field(dest, info, run_cnt); + copy_field(dest, info, recursion_misses); + copy_field(dest, info, verified_insns); + copy_field(dest, info, attach_btf_obj_id); + copy_field(dest, info, attach_btf_id); +} + +static int copy_bpf_prog_info_from_user(const union bpf_attr *attr, + struct bpf_prog_info *info, + u32 *info_len) +{ + struct compat_bpf_prog_info cinfo; + int err; + size_t info_size = in_compat64_syscall() ? sizeof(struct compat_bpf_prog_info) + : sizeof(struct bpf_prog_info); + void __user *uinfo = u64_to_user_ptr(attr->info.info); + + *info_len = attr->info.info_len; + err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), + info_size, *info_len); + if (err) + return err; + *info_len = min_t(u32, info_size, *info_len); + + memset(info, 0, sizeof(*info)); + if (in_compat64_syscall()) { + memset(&cinfo, 0, sizeof(cinfo)); + if (copy_from_user(&cinfo, uinfo, *info_len)) + return -EFAULT; + convert_compat_prog_info_in(info, &cinfo); + } else { + if (copy_from_user(info, uinfo, *info_len)) + return -EFAULT; + } + + return 0; +} + +static int copy_bpf_prog_info_to_user(const union bpf_attr *attr, + union bpf_attr __user *uattr, + struct bpf_prog_info *info, + u32 *info_len) +{ + struct compat_bpf_prog_info cinfo; + void *src_info = in_compat64_syscall() ? (struct bpf_prog_info *)&cinfo + : info; + void __user *uinfo = (void __user *)attr->info.info; + + if (in_compat64_syscall()) { + memset(&cinfo, 0, sizeof(cinfo)); + convert_compat_prog_info_out(&cinfo, info); + } + + if (copy_to_user(uinfo, src_info, *info_len) || + bpf_put_uattr(*info_len, uattr, info.info_len)) + return -EFAULT; + + return 0; +} + static int bpf_prog_get_info_by_fd(struct file *file, struct bpf_prog *prog, const union bpf_attr *attr, union bpf_attr __user *uattr) { - struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info); struct btf *attach_btf = bpf_prog_get_target_btf(prog); struct bpf_prog_info info; - u32 info_len = attr->info.info_len; + u32 info_len; struct bpf_prog_kstats stats; char __user *uinsns; u32 ulen; int err;

- err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len); + err = copy_bpf_prog_info_from_user(attr, &info, &info_len); if (err) return err; - info_len = min_t(u32, sizeof(info), info_len); - - memset(&info, 0, sizeof(info)); - if (copy_from_user(&info, uinfo, info_len)) - return -EFAULT;

info.type = prog->type; info.id = prog->aux->id; @@ -4232,8 +4367,7 @@ static int bpf_prog_get_info_by_fd(struct file *file, }

done: - if (copy_to_user(uinfo, &info, info_len) || - put_user(info_len, &uattr->info.info_len)) + if (copy_bpf_prog_info_to_user(attr, uattr, &info, &info_len)) return -EFAULT;

return 0; @@ -4292,24 +4426,153 @@ static int bpf_btf_get_info_by_fd(struct file *file, return btf_get_info_by_fd(btf, attr, uattr); }

+static void +convert_compat_link_info_in(struct bpf_link_info *dest, + const struct compat_bpf_link_info *cinfo, + enum bpf_link_type type) +{ + /* + * Only a few fields in bpf_link_info are used as input, the + * rest do not need conversion in + */ + switch (type) { + case BPF_LINK_TYPE_RAW_TRACEPOINT: + copy_field(dest, cinfo, raw_tracepoint.tp_name); + copy_field(dest, cinfo, raw_tracepoint.tp_name_len); + return; + case BPF_LINK_TYPE_ITER: + copy_field(dest, cinfo, iter.target_name); + copy_field(dest, cinfo, iter.target_name_len); + return; + default: + return; + } +} + +static void +convert_compat_link_info_out(struct compat_bpf_link_info *dest, + const struct bpf_link_info *info, + enum bpf_link_type type) +{ + copy_field(dest, info, type); + copy_field(dest, info, id); + copy_field(dest, info, prog_id); + + switch (type) { + case BPF_LINK_TYPE_RAW_TRACEPOINT: + copy_field(dest, info, raw_tracepoint.tp_name); + copy_field(dest, info, raw_tracepoint.tp_name_len); + return; + case BPF_LINK_TYPE_TRACING: + copy_field(dest, info, tracing.attach_type); + copy_field(dest, info, tracing.target_obj_id); + copy_field(dest, info, tracing.target_btf_id); + return; + case BPF_LINK_TYPE_CGROUP: + copy_field(dest, info, cgroup.cgroup_id); + copy_field(dest, info, cgroup.attach_type); + return; + case BPF_LINK_TYPE_ITER: + copy_field(dest, info, iter.target_name); + copy_field(dest, info, iter.target_name_len); + /* + * remaining struct is fixed size integers, so identical in + * any 64-bit ABI - since figuring out what members are active + * is non-trivial, memcpy to the end of the struct + */ + memcpy((u8 *)dest+offsetof(struct compat_bpf_link_info, iter.map), + (u8 *)info+offsetof(struct bpf_link_info, iter.map), + offsetofend(struct bpf_link_info, iter.cgroup) - + offsetof(struct bpf_link_info, iter.map)); + return; + case BPF_LINK_TYPE_NETNS: + copy_field(dest, info, netns.netns_ino); + copy_field(dest, info, netns.attach_type); + return; + case BPF_LINK_TYPE_XDP: + copy_field(dest, info, xdp.ifindex); + return; + case BPF_LINK_TYPE_STRUCT_OPS: + copy_field(dest, info, struct_ops.map_id); + return; + case BPF_LINK_TYPE_NETFILTER: + copy_field(dest, info, netfilter.pf); + copy_field(dest, info, netfilter.hooknum); + copy_field(dest, info, netfilter.priority); + copy_field(dest, info, netfilter.flags); + return; + default: + return; + } +} + +static int copy_bpf_link_info_from_user(const union bpf_attr *attr, + struct bpf_link_info *info, + u32 *info_len, + enum bpf_link_type type) +{ + struct compat_bpf_link_info cinfo; + int err; + size_t info_size = in_compat64_syscall() ? sizeof(struct compat_bpf_link_info) + : sizeof(struct bpf_link_info); + void __user *uinfo = u64_to_user_ptr(attr->info.info); + + *info_len = attr->info.info_len; + err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), + info_size, *info_len); + if (err) + return err; + *info_len = min_t(u32, info_size, *info_len); + + memset(info, 0, sizeof(*info)); + if (in_compat64_syscall()) { + memset(&cinfo, 0, sizeof(cinfo)); + if (copy_from_user(&cinfo, uinfo, *info_len)) + return -EFAULT; + convert_compat_link_info_in(info, &cinfo, type); + } else { + if (copy_from_user(info, uinfo, *info_len)) + return -EFAULT; + } + + return 0; +} + +static int copy_bpf_link_info_to_user(const union bpf_attr *attr, + union bpf_attr __user *uattr, + struct bpf_link_info *info, + u32 *info_len, + enum bpf_link_type type) +{ + struct compat_bpf_link_info cinfo; + void *src_info = in_compat64_syscall() ? (struct bpf_link_info *)&cinfo + : info; + void __user *uinfo = (void __user *)attr->info.info; + + if (in_compat64_syscall()) { + memset(&cinfo, 0, sizeof(cinfo)); + convert_compat_link_info_out(&cinfo, info, type); + } + + if (copy_to_user(uinfo, src_info, *info_len) || + bpf_put_uattr(*info_len, uattr, info.info_len)) + return -EFAULT; + + return 0; +} + static int bpf_link_get_info_by_fd(struct file *file, struct bpf_link *link, const union bpf_attr *attr, union bpf_attr __user *uattr) { - struct bpf_link_info __user *uinfo = u64_to_user_ptr(attr->info.info); struct bpf_link_info info; - u32 info_len = attr->info.info_len; + u32 info_len; int err;

- err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len); + err = copy_bpf_link_info_from_user(attr, &info, &info_len, link->type); if (err) return err; - info_len = min_t(u32, sizeof(info), info_len); - - memset(&info, 0, sizeof(info)); - if (copy_from_user(&info, uinfo, info_len)) - return -EFAULT;

info.type = link->type; info.id = link->id; @@ -4322,8 +4585,7 @@ static int bpf_link_get_info_by_fd(struct file *file, return err; }

- if (copy_to_user(uinfo, &info, info_len) || - put_user(info_len, &uattr->info.info_len)) + if (copy_bpf_link_info_to_user(attr, uattr, &info, &info_len, link->type)) return -EFAULT;

return 0;

The CHECK_ATTR macro needs adapting to support union compat_bpf_attr as well as union bpf_attr.

The union bpf_attr passed in from userspace contains many structs of various sizes for each of the bpf syscall's multiplexed options. Since most of the structs do not take up the full memory footprint of the union, we should not have any data past the last struct member for any given option.

The CHECK_ATTR macro checks for non-null data in the gap between the end of the last member and the end of the union. Any non-zero data signals a malformed request from userspace so should be rejected.

Adapt the macro to be able to handle the compat64 offsets based on in_compat64_syscall().

Since compat64 conversion involves copying out individual fields to a new native sized union, we need to check for invalid data before the conversion occurs i.e. in copy_bpf_attr_from_user.

This presents a problem since the existing macro relies on the string XYZ_LAST_FIELD called inside each sub-command handler. However at the __sys_bpf + copy_bpf_attr_from_user stage we have only int cmd to distinguish subcommands as per enum bpf_cmd, and: 1. there is no way to go cleanly from int to enum string/name 2. enum bpf_cmd does not cleanly match 1:1 in all cases for XYZ_LAST_FIELD

We can either: a) rework/refactor XYZ_LAST_FIELD system to encode by int cmd b) add a check_attr function with case statement to call the macro with the correct XYZ_LAST_FIELD argument

The latter is preferred here for being the slightly easier and cleaner approach. Rather than refactor the entire system, abstract the details away in a check_attr() function called from copy_bpf_attr_from_user().

Signed-off-by: Zachary Leaf zachary.leaf@arm.com --- kernel/bpf/syscall.c | 193 +++++++++++++++++++++++-------------------- 1 file changed, 104 insertions(+), 89 deletions(-)

diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index bad24f30e53f..22ba6cc7b0a1 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -973,12 +973,17 @@ int bpf_get_file_flag(int flags) }

/* helper macro to check that unused fields 'union bpf_attr' are zero */ +#define __CHECK_ATTR(CMD, TYPE) \ + (memchr_inv((void *) &(((TYPE *)vattr)->CMD##_LAST_FIELD) + \ + sizeof(((TYPE *)vattr)->CMD##_LAST_FIELD), 0, \ + sizeof(*(TYPE *)vattr) - \ + offsetof(TYPE, CMD##_LAST_FIELD) - \ + sizeof(((TYPE *)vattr)->CMD##_LAST_FIELD)) != NULL) + #define CHECK_ATTR(CMD) \ - memchr_inv((void *) &attr->CMD##_LAST_FIELD + \ - sizeof(attr->CMD##_LAST_FIELD), 0, \ - sizeof(*attr) - \ - offsetof(union bpf_attr, CMD##_LAST_FIELD) - \ - sizeof(attr->CMD##_LAST_FIELD)) != NULL + (in_compat64_syscall() ? \ + __CHECK_ATTR(CMD, union compat_bpf_attr) : \ + __CHECK_ATTR(CMD, union bpf_attr))

/* dst and src must have at least "size" number of bytes. * Return strlen on success and < 0 on error. @@ -1132,10 +1137,6 @@ static int map_create(union bpf_attr *attr) int f_flags; int err;

- err = CHECK_ATTR(BPF_MAP_CREATE); - if (err) - return -EINVAL; - if (attr->btf_vmlinux_value_type_id) { if (attr->map_type != BPF_MAP_TYPE_STRUCT_OPS || attr->btf_key_type_id || attr->btf_value_type_id) @@ -1365,9 +1366,6 @@ static int map_lookup_elem(union bpf_attr *attr) struct fd f; int err;

- if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM)) - return -EINVAL; - if (attr->flags & ~BPF_F_LOCK) return -EINVAL;

@@ -1440,9 +1438,6 @@ static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr) struct fd f; int err;

- if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM)) - return -EINVAL; - f = fdget(ufd); map = __bpf_map_get(f); if (IS_ERR(map)) @@ -1494,9 +1489,6 @@ static int map_delete_elem(union bpf_attr *attr, bpfptr_t uattr) void *key; int err;

- if (CHECK_ATTR(BPF_MAP_DELETE_ELEM)) - return -EINVAL; - f = fdget(ufd); map = __bpf_map_get(f); if (IS_ERR(map)) @@ -1550,9 +1542,6 @@ static int map_get_next_key(union bpf_attr *attr) struct fd f; int err;

- if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY)) - return -EINVAL; - f = fdget(ufd); map = __bpf_map_get(f); if (IS_ERR(map)) @@ -1830,9 +1819,6 @@ static int map_lookup_and_delete_elem(union bpf_attr *attr) struct fd f; int err;

- if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM)) - return -EINVAL; - if (attr->flags & ~BPF_F_LOCK) return -EINVAL;

@@ -1918,9 +1904,6 @@ static int map_freeze(const union bpf_attr *attr) struct bpf_map *map; struct fd f;

- if (CHECK_ATTR(BPF_MAP_FREEZE)) - return -EINVAL; - f = fdget(ufd); map = __bpf_map_get(f); if (IS_ERR(map)) @@ -2508,9 +2491,6 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) char license[128]; bool is_gpl;

- if (CHECK_ATTR(BPF_PROG_LOAD)) - return -EINVAL; - if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT | BPF_F_ANY_ALIGNMENT | BPF_F_TEST_STATE_FREQ | @@ -2705,7 +2685,7 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size)

static int bpf_obj_pin(const union bpf_attr *attr) { - if (CHECK_ATTR(BPF_OBJ) || attr->file_flags != 0) + if (attr->file_flags != 0) return -EINVAL;

return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname)); @@ -2713,8 +2693,7 @@ static int bpf_obj_pin(const union bpf_attr *attr)

static int bpf_obj_get(const union bpf_attr *attr) { - if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0 || - attr->file_flags & ~BPF_OBJ_FLAG_MASK) + if (attr->bpf_fd != 0 || attr->file_flags & ~BPF_OBJ_FLAG_MASK) return -EINVAL;

return bpf_obj_get_user(u64_to_user_ptr(attr->pathname), @@ -3409,9 +3388,6 @@ static int bpf_raw_tracepoint_open(const union bpf_attr *attr) struct bpf_prog *prog; int fd;

- if (CHECK_ATTR(BPF_RAW_TRACEPOINT_OPEN)) - return -EINVAL; - prog = bpf_prog_get(attr->raw_tracepoint.prog_fd); if (IS_ERR(prog)) return PTR_ERR(prog); @@ -3524,9 +3500,6 @@ static int bpf_prog_attach(const union bpf_attr *attr) struct bpf_prog *prog; int ret;

- if (CHECK_ATTR(BPF_PROG_ATTACH)) - return -EINVAL; - if (attr->attach_flags & ~BPF_F_ATTACH_MASK) return -EINVAL;

@@ -3583,9 +3556,6 @@ static int bpf_prog_detach(const union bpf_attr *attr) { enum bpf_prog_type ptype;

- if (CHECK_ATTR(BPF_PROG_DETACH)) - return -EINVAL; - ptype = attach_type_to_prog_type(attr->attach_type);

switch (ptype) { @@ -3617,8 +3587,6 @@ static int bpf_prog_query(const union bpf_attr *attr, { if (!capable(CAP_NET_ADMIN)) return -EPERM; - if (CHECK_ATTR(BPF_PROG_QUERY)) - return -EINVAL; if (attr->query.query_flags & ~BPF_F_QUERY_EFFECTIVE) return -EINVAL;

@@ -3671,9 +3639,6 @@ static int bpf_prog_test_run(const union bpf_attr *attr, struct bpf_prog *prog; int ret = -ENOTSUPP;

- if (CHECK_ATTR(BPF_PROG_TEST_RUN)) - return -EINVAL; - if ((attr->test.ctx_size_in && !attr->test.ctx_in) || (!attr->test.ctx_size_in && attr->test.ctx_in)) return -EINVAL; @@ -3703,7 +3668,7 @@ static int bpf_obj_get_next_id(const union bpf_attr *attr, u32 next_id = attr->start_id; int err = 0;

- if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX) + if (next_id >= INT_MAX) return -EINVAL;

if (!capable(CAP_SYS_ADMIN)) @@ -3784,9 +3749,6 @@ static int bpf_prog_get_fd_by_id(const union bpf_attr *attr) u32 id = attr->prog_id; int fd;

- if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID)) - return -EINVAL; - if (!capable(CAP_SYS_ADMIN)) return -EPERM;

@@ -3810,8 +3772,7 @@ static int bpf_map_get_fd_by_id(const union bpf_attr *attr) int f_flags; int fd;

- if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID) || - attr->open_flags & ~BPF_OBJ_FLAG_MASK) + if (attr->open_flags & ~BPF_OBJ_FLAG_MASK) return -EINVAL;

if (!capable(CAP_SYS_ADMIN)) @@ -4601,9 +4562,6 @@ static int bpf_obj_get_info_by_fd(const union bpf_attr *attr, struct fd f; int err;

- if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD)) - return -EINVAL; - f = fdget(ufd); if (!f.file) return -EBADFD; @@ -4630,9 +4588,6 @@ static int bpf_obj_get_info_by_fd(const union bpf_attr *attr,

static int bpf_btf_load(const union bpf_attr *attr, bpfptr_t uattr, __u32 uattr_size) { - if (CHECK_ATTR(BPF_BTF_LOAD)) - return -EINVAL; - if (!bpf_capable()) return -EPERM;

@@ -4643,9 +4598,6 @@ static int bpf_btf_load(const union bpf_attr *attr, bpfptr_t uattr, __u32 uattr_

static int bpf_btf_get_fd_by_id(const union bpf_attr *attr) { - if (CHECK_ATTR(BPF_BTF_GET_FD_BY_ID)) - return -EINVAL; - if (!capable(CAP_SYS_ADMIN)) return -EPERM;

@@ -4711,9 +4663,6 @@ static int bpf_task_fd_query(const union bpf_attr *attr, struct file *file; int err;

- if (CHECK_ATTR(BPF_TASK_FD_QUERY)) - return -EINVAL; - if (!capable(CAP_SYS_ADMIN)) return -EPERM;

@@ -4795,9 +4744,6 @@ static int bpf_map_do_batch(const union bpf_attr *attr, int err, ufd; struct fd f;

- if (CHECK_ATTR(BPF_MAP_BATCH)) - return -EINVAL; - ufd = attr->batch.map_fd; f = fdget(ufd); map = __bpf_map_get(f); @@ -4836,9 +4782,6 @@ static int link_create(union bpf_attr *attr, bpfptr_t uattr) struct bpf_prog *prog; int ret;

- if (CHECK_ATTR(BPF_LINK_CREATE)) - return -EINVAL; - if (attr->link_create.attach_type == BPF_STRUCT_OPS) return bpf_struct_ops_link_create(attr);

@@ -4987,9 +4930,6 @@ static int link_update(union bpf_attr *attr) u32 flags; int ret;

- if (CHECK_ATTR(BPF_LINK_UPDATE)) - return -EINVAL; - flags = attr->link_update.flags; if (flags & ~BPF_F_REPLACE) return -EINVAL; @@ -5043,9 +4983,6 @@ static int link_detach(union bpf_attr *attr) struct bpf_link *link; int ret;

- if (CHECK_ATTR(BPF_LINK_DETACH)) - return -EINVAL; - link = bpf_link_get_from_fd(attr->link_detach.link_fd); if (IS_ERR(link)) return PTR_ERR(link); @@ -5113,9 +5050,6 @@ static int bpf_link_get_fd_by_id(const union bpf_attr *attr) u32 id = attr->link_id; int fd;

- if (CHECK_ATTR(BPF_LINK_GET_FD_BY_ID)) - return -EINVAL; - if (!capable(CAP_SYS_ADMIN)) return -EPERM;

@@ -5169,9 +5103,6 @@ static int bpf_enable_runtime_stats(void) static int bpf_enable_stats(union bpf_attr *attr) {

- if (CHECK_ATTR(BPF_ENABLE_STATS)) - return -EINVAL; - if (!capable(CAP_SYS_ADMIN)) return -EPERM;

@@ -5191,9 +5122,6 @@ static int bpf_iter_create(union bpf_attr *attr) struct bpf_link *link; int err;

- if (CHECK_ATTR(BPF_ITER_CREATE)) - return -EINVAL; - if (attr->iter_create.flags) return -EINVAL;

@@ -5216,9 +5144,6 @@ static int bpf_prog_bind_map(union bpf_attr *attr) struct bpf_map **used_maps_old, **used_maps_new; int i, ret = 0;

- if (CHECK_ATTR(BPF_PROG_BIND_MAP)) - return -EINVAL; - if (attr->prog_bind_map.flags) return -EINVAL;

@@ -5269,6 +5194,90 @@ static int bpf_prog_bind_map(union bpf_attr *attr) return ret; }

+static int check_attr(enum bpf_cmd cmd, void *vattr) +{ + switch (cmd) { + case BPF_MAP_CREATE: + return CHECK_ATTR(BPF_MAP_CREATE); + case BPF_MAP_LOOKUP_ELEM: + return CHECK_ATTR(BPF_MAP_LOOKUP_ELEM); + case BPF_MAP_UPDATE_ELEM: + return CHECK_ATTR(BPF_MAP_UPDATE_ELEM); + case BPF_MAP_DELETE_ELEM: + return CHECK_ATTR(BPF_MAP_DELETE_ELEM); + case BPF_MAP_GET_NEXT_KEY: + return CHECK_ATTR(BPF_MAP_GET_NEXT_KEY); + case BPF_MAP_FREEZE: + return CHECK_ATTR(BPF_MAP_FREEZE); + case BPF_PROG_LOAD: + return CHECK_ATTR(BPF_PROG_LOAD); + case BPF_OBJ_PIN: + return CHECK_ATTR(BPF_OBJ); + case BPF_OBJ_GET: + return CHECK_ATTR(BPF_OBJ); + case BPF_PROG_ATTACH: + return CHECK_ATTR(BPF_PROG_ATTACH); + case BPF_PROG_DETACH: + return CHECK_ATTR(BPF_PROG_DETACH); + case BPF_PROG_QUERY: + return CHECK_ATTR(BPF_PROG_QUERY); + case BPF_PROG_TEST_RUN: + return CHECK_ATTR(BPF_PROG_TEST_RUN); + case BPF_PROG_GET_NEXT_ID: + return CHECK_ATTR(BPF_OBJ_GET_NEXT_ID); + case BPF_MAP_GET_NEXT_ID: + return CHECK_ATTR(BPF_OBJ_GET_NEXT_ID); + case BPF_BTF_GET_NEXT_ID: + return CHECK_ATTR(BPF_OBJ_GET_NEXT_ID); + case BPF_MAP_LOOKUP_BATCH: + return CHECK_ATTR(BPF_MAP_BATCH); + case BPF_MAP_LOOKUP_AND_DELETE_BATCH: + return CHECK_ATTR(BPF_MAP_BATCH); + case BPF_MAP_UPDATE_BATCH: + return CHECK_ATTR(BPF_MAP_BATCH); + case BPF_MAP_DELETE_BATCH: + return CHECK_ATTR(BPF_MAP_BATCH); + case BPF_PROG_GET_FD_BY_ID: + return CHECK_ATTR(BPF_PROG_GET_FD_BY_ID); + case BPF_MAP_GET_FD_BY_ID: + return CHECK_ATTR(BPF_MAP_GET_FD_BY_ID); + case BPF_OBJ_GET_INFO_BY_FD: + return CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD); + case BPF_RAW_TRACEPOINT_OPEN: + return CHECK_ATTR(BPF_RAW_TRACEPOINT_OPEN); + case BPF_BTF_LOAD: + return CHECK_ATTR(BPF_BTF_LOAD); + case BPF_BTF_GET_FD_BY_ID: + return CHECK_ATTR(BPF_BTF_GET_FD_BY_ID); + case BPF_TASK_FD_QUERY: + return CHECK_ATTR(BPF_TASK_FD_QUERY); + case BPF_MAP_LOOKUP_AND_DELETE_ELEM: + return CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM); + case BPF_LINK_CREATE: + return CHECK_ATTR(BPF_LINK_CREATE); + case BPF_LINK_UPDATE: + return CHECK_ATTR(BPF_LINK_UPDATE); + case BPF_LINK_GET_FD_BY_ID: + return CHECK_ATTR(BPF_LINK_GET_FD_BY_ID); + case BPF_LINK_GET_NEXT_ID: + return CHECK_ATTR(BPF_OBJ_GET_NEXT_ID); + case BPF_ENABLE_STATS: + return CHECK_ATTR(BPF_ENABLE_STATS); + case BPF_ITER_CREATE: + return CHECK_ATTR(BPF_ITER_CREATE); + case BPF_LINK_DETACH: + return CHECK_ATTR(BPF_LINK_DETACH); + case BPF_PROG_BIND_MAP: + return CHECK_ATTR(BPF_PROG_BIND_MAP); + /* + * no default case so we're warned of unhandled enum bpf_cmd + * values by -Wswitch + */ + } + + return 0; +} + static void convert_compat_bpf_attr(union bpf_attr *dest, const union compat_bpf_attr *cattr, int cmd) { @@ -5511,10 +5520,16 @@ static int copy_bpf_attr_from_user(union bpf_attr *attr, int cmd, memset(&cattr, 0, sizeof(cattr)); if (copy_from_bpfptr(&cattr, uattr, *size) != 0) return -EFAULT; + err = check_attr(cmd, &cattr); + if (err) + return -EINVAL; convert_compat_bpf_attr(attr, &cattr, cmd); } else { if (copy_from_bpfptr(attr, uattr, *size) != 0) return -EFAULT; + err = check_attr(cmd, attr); + if (err) + return -EINVAL; }

return 0;

union bpf_attr and structs bpf_xyz_info are used to pass config to the kernel or return info to userspace via the bpf syscall.

In order to avoid needing a separate 32-bit compat handler for the bpf syscall, __user pointers inside these union/structs are stored as __u64 or __aligned_u64.

In Morello, PCuABI a user pointer is a 129-bit capability so __aligned_u64 type is not big enough to hold it. Use type __kernel_aligned_uintptr_t instead, which is big enough on the affected architectures while remaining __aligned_64 on others.

Use copy_{to,from}_user_with_ptr variants where blocks of memory containing pointers are being copied to preserve capabilities.

Signed-off-by: Zachary Leaf zachary.leaf@arm.com --- drivers/media/rc/bpf-lirc.c | 2 +- include/linux/bpf_compat.h | 3 + include/linux/bpfptr.h | 6 +- include/uapi/linux/bpf.h | 94 +++++++++---------- kernel/bpf/bpf_iter.c | 2 +- kernel/bpf/btf.c | 18 ++-- kernel/bpf/cgroup.c | 5 +- kernel/bpf/hashtab.c | 8 +- kernel/bpf/net_namespace.c | 2 +- kernel/bpf/offload.c | 2 +- kernel/bpf/syscall.c | 161 +++++++++++++++++---------------- kernel/bpf/verifier.c | 3 +- kernel/trace/bpf_trace.c | 6 +- net/bpf/bpf_dummy_struct_ops.c | 6 +- net/bpf/test_run.c | 16 ++-- net/core/sock_map.c | 2 +- 16 files changed, 173 insertions(+), 163 deletions(-)

diff --git a/drivers/media/rc/bpf-lirc.c b/drivers/media/rc/bpf-lirc.c index 7f713422049d..96861d247be0 100644 --- a/drivers/media/rc/bpf-lirc.c +++ b/drivers/media/rc/bpf-lirc.c @@ -294,7 +294,7 @@ int lirc_prog_detach(const union bpf_attr *attr)

int lirc_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr) { - __u32 __user *prog_ids = u64_to_user_ptr(attr->query.prog_ids); + __u32 __user *prog_ids = (__u32 __user *)attr->query.prog_ids; struct bpf_prog_array *progs; struct rc_dev *rcdev; u32 cnt, flags = 0; diff --git a/include/linux/bpf_compat.h b/include/linux/bpf_compat.h index 710815417a27..257c895945c3 100644 --- a/include/linux/bpf_compat.h +++ b/include/linux/bpf_compat.h @@ -1,6 +1,9 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (C) 2023 Arm Ltd */

+#define bpf_compat_ptr_field(dest, src, field) \ + ((dest)->field = (__kernel_aligned_uintptr_t)compat_ptr((src)->field)) + union compat_bpf_attr { struct { /* anonymous struct used by BPF_MAP_CREATE command */ __u32 map_type; /* one of enum bpf_map_type */ diff --git a/include/linux/bpfptr.h b/include/linux/bpfptr.h index ff923e4bb713..3d0e1604ac94 100644 --- a/include/linux/bpfptr.h +++ b/include/linux/bpfptr.h @@ -32,12 +32,12 @@ static inline bpfptr_t USER_BPFPTR(void __user *p) return (bpfptr_t) { .user = p }; }

-static inline bpfptr_t make_bpfptr(u64 addr, bool is_kernel) +static inline bpfptr_t make_bpfptr(__kernel_uintptr_t ptr, bool is_kernel) { if (is_kernel) - return KERNEL_BPFPTR((void*) (uintptr_t) addr); + return KERNEL_BPFPTR((void *)(uintptr_t)ptr); else - return USER_BPFPTR(u64_to_user_ptr(addr)); + return USER_BPFPTR((void __user *)ptr); }

static inline bool bpfptr_is_null(bpfptr_t bpfptr) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index c994ff5b157c..a536d8f8805e 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -1350,21 +1350,21 @@ union bpf_attr {

struct { /* anonymous struct used by BPF_MAP_*_ELEM commands */ __u32 map_fd; - __aligned_u64 key; + __kernel_aligned_uintptr_t key; union { - __aligned_u64 value; - __aligned_u64 next_key; + __kernel_aligned_uintptr_t value; + __kernel_aligned_uintptr_t next_key; }; __u64 flags; };

struct { /* struct used by BPF_MAP_*_BATCH commands */ - __aligned_u64 in_batch; /* start batch, - * NULL to start from beginning - */ - __aligned_u64 out_batch; /* output: next start batch */ - __aligned_u64 keys; - __aligned_u64 values; + /* start batch, NULL to start from beginning */ + __kernel_aligned_uintptr_t in_batch; + /* output: next start batch */ + __kernel_aligned_uintptr_t out_batch; + __kernel_aligned_uintptr_t keys; + __kernel_aligned_uintptr_t values; __u32 count; /* input/output: * input: # of key/value * elements @@ -1378,11 +1378,11 @@ union bpf_attr { struct { /* anonymous struct used by BPF_PROG_LOAD command */ __u32 prog_type; /* one of enum bpf_prog_type */ __u32 insn_cnt; - __aligned_u64 insns; - __aligned_u64 license; + __kernel_aligned_uintptr_t insns; + __kernel_aligned_uintptr_t license; __u32 log_level; /* verbosity level of verifier */ __u32 log_size; /* size of user buffer */ - __aligned_u64 log_buf; /* user supplied buffer */ + __kernel_aligned_uintptr_t log_buf; /* user supplied buffer */ __u32 kern_version; /* not used */ __u32 prog_flags; char prog_name[BPF_OBJ_NAME_LEN]; @@ -1394,10 +1394,10 @@ union bpf_attr { __u32 expected_attach_type; __u32 prog_btf_fd; /* fd pointing to BTF type data */ __u32 func_info_rec_size; /* userspace bpf_func_info size */ - __aligned_u64 func_info; /* func info */ + __kernel_aligned_uintptr_t func_info; /* func info */ __u32 func_info_cnt; /* number of bpf_func_info records */ __u32 line_info_rec_size; /* userspace bpf_line_info size */ - __aligned_u64 line_info; /* line info */ + __kernel_aligned_uintptr_t line_info; /* line info */ __u32 line_info_cnt; /* number of bpf_line_info records */ __u32 attach_btf_id; /* in-kernel BTF type id to attach to */ union { @@ -1407,8 +1407,8 @@ union bpf_attr { __u32 attach_btf_obj_fd; }; __u32 core_relo_cnt; /* number of bpf_core_relo */ - __aligned_u64 fd_array; /* array of FDs */ - __aligned_u64 core_relos; + __kernel_aligned_uintptr_t fd_array; /* array of FDs */ + __kernel_aligned_uintptr_t core_relos; __u32 core_relo_rec_size; /* sizeof(struct bpf_core_relo) */ /* output: actual total log contents size (including termintaing zero). * It could be both larger than original log_size (if log was @@ -1418,7 +1418,7 @@ union bpf_attr { };

struct { /* anonymous struct used by BPF_OBJ_* commands */ - __aligned_u64 pathname; + __kernel_aligned_uintptr_t pathname; __u32 bpf_fd; __u32 file_flags; }; @@ -1442,8 +1442,8 @@ union bpf_attr { * returns ENOSPC if data_out * is too small. */ - __aligned_u64 data_in; - __aligned_u64 data_out; + __kernel_aligned_uintptr_t data_in; + __kernel_aligned_uintptr_t data_out; __u32 repeat; __u32 duration; __u32 ctx_size_in; /* input: len of ctx_in */ @@ -1451,8 +1451,8 @@ union bpf_attr { * returns ENOSPC if ctx_out * is too small. */ - __aligned_u64 ctx_in; - __aligned_u64 ctx_out; + __kernel_aligned_uintptr_t ctx_in; + __kernel_aligned_uintptr_t ctx_out; __u32 flags; __u32 cpu; __u32 batch_size; @@ -1473,7 +1473,7 @@ union bpf_attr { struct { /* anonymous struct used by BPF_OBJ_GET_INFO_BY_FD */ __u32 bpf_fd; __u32 info_len; - __aligned_u64 info; + __kernel_aligned_uintptr_t info; } info;

struct { /* anonymous struct used by BPF_PROG_QUERY command */ @@ -1481,22 +1481,22 @@ union bpf_attr { __u32 attach_type; __u32 query_flags; __u32 attach_flags; - __aligned_u64 prog_ids; + __kernel_aligned_uintptr_t prog_ids; __u32 prog_cnt; /* output: per-program attach_flags. * not allowed to be set during effective query. */ - __aligned_u64 prog_attach_flags; + __kernel_aligned_uintptr_t prog_attach_flags; } query;

struct { /* anonymous struct used by BPF_RAW_TRACEPOINT_OPEN command */ - __u64 name; + __kernel_aligned_uintptr_t name; __u32 prog_fd; } raw_tracepoint;

struct { /* anonymous struct for BPF_BTF_LOAD */ - __aligned_u64 btf; - __aligned_u64 btf_log_buf; + __kernel_aligned_uintptr_t btf; + __kernel_aligned_uintptr_t btf_log_buf; __u32 btf_size; __u32 btf_log_size; __u32 btf_log_level; @@ -1512,7 +1512,7 @@ union bpf_attr { __u32 fd; /* input: fd */ __u32 flags; /* input: flags */ __u32 buf_len; /* input/output: buf len */ - __aligned_u64 buf; /* input/output: + __kernel_aligned_uintptr_t buf; /* input/output: * tp_name for tracepoint * symbol for kprobe * filename for uprobe @@ -1537,8 +1537,10 @@ union bpf_attr { union { __u32 target_btf_id; /* btf_id of target to attach to */ struct { - __aligned_u64 iter_info; /* extra bpf_iter_link_info */ - __u32 iter_info_len; /* iter_info length */ + /* extra bpf_iter_link_info */ + __kernel_aligned_uintptr_t iter_info; + /* iter_info length */ + __u32 iter_info_len; }; struct { /* black box user-provided value passed through @@ -1550,9 +1552,9 @@ union bpf_attr { struct { __u32 flags; __u32 cnt; - __aligned_u64 syms; - __aligned_u64 addrs; - __aligned_u64 cookies; + __kernel_aligned_uintptr_t syms; + __kernel_aligned_uintptr_t addrs; + __kernel_aligned_uintptr_t cookies; } kprobe_multi; struct { /* this is overlaid with the target_btf_id above. */ @@ -6303,12 +6305,12 @@ struct bpf_prog_info { __u8 tag[BPF_TAG_SIZE]; __u32 jited_prog_len; __u32 xlated_prog_len; - __aligned_u64 jited_prog_insns; - __aligned_u64 xlated_prog_insns; + __kernel_aligned_uintptr_t jited_prog_insns; + __kernel_aligned_uintptr_t xlated_prog_insns; __u64 load_time; /* ns since boottime */ __u32 created_by_uid; __u32 nr_map_ids; - __aligned_u64 map_ids; + __kernel_aligned_uintptr_t map_ids; char name[BPF_OBJ_NAME_LEN]; __u32 ifindex; __u32 gpl_compatible:1; @@ -6317,20 +6319,20 @@ struct bpf_prog_info { __u64 netns_ino; __u32 nr_jited_ksyms; __u32 nr_jited_func_lens; - __aligned_u64 jited_ksyms; - __aligned_u64 jited_func_lens; + __kernel_aligned_uintptr_t jited_ksyms; + __kernel_aligned_uintptr_t jited_func_lens; __u32 btf_id; __u32 func_info_rec_size; - __aligned_u64 func_info; + __kernel_aligned_uintptr_t func_info; __u32 nr_func_info; __u32 nr_line_info; - __aligned_u64 line_info; - __aligned_u64 jited_line_info; + __kernel_aligned_uintptr_t line_info; + __kernel_aligned_uintptr_t jited_line_info; __u32 nr_jited_line_info; __u32 line_info_rec_size; __u32 jited_line_info_rec_size; __u32 nr_prog_tags; - __aligned_u64 prog_tags; + __kernel_aligned_uintptr_t prog_tags; __u64 run_time_ns; __u64 run_cnt; __u64 recursion_misses; @@ -6359,10 +6361,10 @@ struct bpf_map_info { } __attribute__((aligned(8)));

struct bpf_btf_info { - __aligned_u64 btf; + __kernel_aligned_uintptr_t btf; __u32 btf_size; __u32 id; - __aligned_u64 name; + __kernel_aligned_uintptr_t name; __u32 name_len; __u32 kernel_btf; } __attribute__((aligned(8))); @@ -6373,7 +6375,7 @@ struct bpf_link_info { __u32 prog_id; union { struct { - __aligned_u64 tp_name; /* in/out: tp_name buffer ptr */ + __kernel_aligned_uintptr_t tp_name; /* in/out: tp_name buffer ptr */ __u32 tp_name_len; /* in/out: tp_name buffer len */ } raw_tracepoint; struct { @@ -6386,7 +6388,7 @@ struct bpf_link_info { __u32 attach_type; } cgroup; struct { - __aligned_u64 target_name; /* in/out: target_name buffer ptr */ + __kernel_aligned_uintptr_t target_name; /* in/out: target_name buffer ptr */ __u32 target_name_len; /* in/out: target_name buffer len */

/* If the iter specific field is 32 bits, it can be put diff --git a/kernel/bpf/bpf_iter.c b/kernel/bpf/bpf_iter.c index 96856f130cbf..669efe46a5a6 100644 --- a/kernel/bpf/bpf_iter.c +++ b/kernel/bpf/bpf_iter.c @@ -456,7 +456,7 @@ static int bpf_iter_link_fill_link_info(const struct bpf_link *link, { struct bpf_iter_link *iter_link = container_of(link, struct bpf_iter_link, link); - char __user *ubuf = u64_to_user_ptr(info->iter.target_name); + char __user *ubuf = (char __user *)info->iter.target_name; bpf_iter_fill_link_info_t fill_link_info; u32 ulen = info->iter.target_name_len; const char *target_name; diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 58421266a5b8..20d7e31231f3 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -5464,7 +5464,7 @@ static int finalize_log(struct bpf_verifier_log *log, bpfptr_t uattr, u32 uattr_ static struct btf *btf_parse(const union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) { bpfptr_t btf_data = make_bpfptr(attr->btf, uattr.is_kernel); - char __user *log_ubuf = u64_to_user_ptr(attr->btf_log_buf); + char __user *log_ubuf = (char __user *)attr->btf_log_buf; struct btf_struct_metas *struct_meta_tab; struct btf_verifier_env *env = NULL; struct btf *btf = NULL; @@ -7217,10 +7217,10 @@ struct btf *btf_get_by_fd(int fd) static void convert_compat_btf_info_in(struct bpf_btf_info *dest, const struct compat_bpf_btf_info *cinfo) { - copy_field(dest, cinfo, btf); + bpf_compat_ptr_field(dest, cinfo, btf); copy_field(dest, cinfo, btf_size); copy_field(dest, cinfo, id); - copy_field(dest, cinfo, name); + bpf_compat_ptr_field(dest, cinfo, name); copy_field(dest, cinfo, name_len); copy_field(dest, cinfo, kernel_btf); } @@ -7244,7 +7244,7 @@ static int copy_bpf_btf_info_from_user(const union bpf_attr *attr, int err; size_t info_size = in_compat64_syscall() ? sizeof(struct compat_bpf_btf_info) : sizeof(struct bpf_btf_info); - void __user *uinfo = u64_to_user_ptr(attr->info.info); + void __user *uinfo = (void __user *)attr->info.info;

*info_len = attr->info.info_len; err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), @@ -7260,7 +7260,7 @@ static int copy_bpf_btf_info_from_user(const union bpf_attr *attr, return -EFAULT; convert_compat_btf_info_in(info, &cinfo); } else { - if (copy_from_user(info, uinfo, *info_len)) + if (copy_from_user_with_ptr(info, uinfo, *info_len)) return -EFAULT; }

@@ -7275,14 +7275,14 @@ static int copy_bpf_btf_info_to_user(const union bpf_attr *attr, struct compat_bpf_btf_info cinfo; void *src_info = in_compat64_syscall() ? (struct bpf_btf_info *)&cinfo : info; - void __user *uinfo = u64_to_user_ptr(attr->info.info); + void __user *uinfo = (void __user *)attr->info.info;

if (in_compat64_syscall()) { memset(&cinfo, 0, sizeof(cinfo)); convert_compat_btf_info_out(&cinfo, info); }

- if (copy_to_user(uinfo, src_info, *info_len) || + if (bpf_copy_to_user_with_ptr(uinfo, src_info, *info_len) || bpf_put_uattr(*info_len, uattr, info.info_len)) return -EFAULT;

@@ -7305,7 +7305,7 @@ int btf_get_info_by_fd(const struct btf *btf, return ret;

info.id = btf->id; - ubtf = u64_to_user_ptr(info.btf); + ubtf = (void __user *)info.btf; btf_copy = min_t(u32, btf->data_size, info.btf_size); if (copy_to_user(ubtf, btf->data, btf_copy)) return -EFAULT; @@ -7313,7 +7313,7 @@ int btf_get_info_by_fd(const struct btf *btf,

info.kernel_btf = btf->kernel_btf;

- uname = u64_to_user_ptr(info.name); + uname = (char __user *)info.name; uname_len = info.name_len; if (!uname ^ !uname_len) return -EINVAL; diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 44bbd2309ce5..2583a43bd468 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -1019,9 +1019,10 @@ static int cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, union bpf_attr __user *uattr) { - __u32 __user *prog_attach_flags = u64_to_user_ptr(attr->query.prog_attach_flags); + __u32 __user *prog_attach_flags = + (__u32 __user *)attr->query.prog_attach_flags; bool effective_query = attr->query.query_flags & BPF_F_QUERY_EFFECTIVE; - __u32 __user *prog_ids = u64_to_user_ptr(attr->query.prog_ids); + __u32 __user *prog_ids = (__u32 __user *)attr->query.prog_ids; enum bpf_attach_type type = attr->query.attach_type; enum cgroup_bpf_attach_type from_atype, to_atype; enum cgroup_bpf_attach_type atype; diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 522760031688..87c0bed3ad0b 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -1666,9 +1666,9 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map, struct bpf_htab *htab = container_of(map, struct bpf_htab, map); u32 bucket_cnt, total, key_size, value_size, roundup_key_size; void *keys = NULL, *values = NULL, *value, *dst_key, *dst_val; - void __user *uvalues = u64_to_user_ptr(attr->batch.values); - void __user *ukeys = u64_to_user_ptr(attr->batch.keys); - void __user *ubatch = u64_to_user_ptr(attr->batch.in_batch); + void __user *uvalues = (void __user *)attr->batch.values; + void __user *ukeys = (void __user *)attr->batch.keys; + void __user *ubatch = (void __user *)attr->batch.in_batch; u32 batch, max_count, size, bucket_size, map_id; struct htab_elem *node_to_free = NULL; u64 elem_map_flags, map_flags; @@ -1873,7 +1873,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map, goto out;

/* copy # of entries and next batch */ - ubatch = u64_to_user_ptr(attr->batch.out_batch); + ubatch = (void __user *)attr->batch.out_batch; if (copy_to_user(ubatch, &batch, sizeof(batch)) || bpf_put_uattr(total, uattr, batch.count)) ret = -EFAULT; diff --git a/kernel/bpf/net_namespace.c b/kernel/bpf/net_namespace.c index 466c7b1acb61..48597768ed22 100644 --- a/kernel/bpf/net_namespace.c +++ b/kernel/bpf/net_namespace.c @@ -248,7 +248,7 @@ static int __netns_bpf_prog_query(const union bpf_attr *attr, struct net *net, enum netns_bpf_attach_type type) { - __u32 __user *prog_ids = u64_to_user_ptr(attr->query.prog_ids); + __u32 __user *prog_ids = (__u32 __user *)attr->query.prog_ids; struct bpf_prog_array *run_array; u32 prog_cnt = 0, flags = 0;

diff --git a/kernel/bpf/offload.c b/kernel/bpf/offload.c index 8a26cd8814c1..a6c0d674fdec 100644 --- a/kernel/bpf/offload.c +++ b/kernel/bpf/offload.c @@ -479,7 +479,7 @@ int bpf_prog_offload_info_fill(struct bpf_prog_info *info, ulen = info->jited_prog_len; info->jited_prog_len = aux->offload->jited_len; if (info->jited_prog_len && ulen) { - uinsns = u64_to_user_ptr(info->jited_prog_insns); + uinsns = (char __user *)info->jited_prog_insns; ulen = min_t(u32, info->jited_prog_len, ulen); if (copy_to_user(uinsns, aux->offload->jited_image, ulen)) { up_read(&bpf_devs_lock); diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 22ba6cc7b0a1..a13dc3e07926 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1357,8 +1357,8 @@ static void *___bpf_copy_key(bpfptr_t ukey, u64 key_size)

static int map_lookup_elem(union bpf_attr *attr) { - void __user *ukey = u64_to_user_ptr(attr->key); - void __user *uvalue = u64_to_user_ptr(attr->value); + void __user *ukey = (void __user *)attr->key; + void __user *uvalue = (void __user *)attr->value; int ufd = attr->map_fd; struct bpf_map *map; void *key, *value; @@ -1534,8 +1534,8 @@ static int map_delete_elem(union bpf_attr *attr, bpfptr_t uattr)

static int map_get_next_key(union bpf_attr *attr) { - void __user *ukey = u64_to_user_ptr(attr->key); - void __user *unext_key = u64_to_user_ptr(attr->next_key); + void __user *ukey = (void __user *)attr->key; + void __user *unext_key = (void __user *)attr->next_key; int ufd = attr->map_fd; struct bpf_map *map; void *key, *next_key; @@ -1597,7 +1597,7 @@ int generic_map_delete_batch(struct bpf_map *map, const union bpf_attr *attr, union bpf_attr __user *uattr) { - void __user *keys = u64_to_user_ptr(attr->batch.keys); + void __user *keys = (void __user *)attr->batch.keys; u32 cp, max_count; int err = 0; void *key; @@ -1651,8 +1651,8 @@ int generic_map_update_batch(struct bpf_map *map, struct file *map_file, const union bpf_attr *attr, union bpf_attr __user *uattr) { - void __user *values = u64_to_user_ptr(attr->batch.values); - void __user *keys = u64_to_user_ptr(attr->batch.keys); + void __user *values = (void __user *)attr->batch.values; + void __user *keys = (void __user *)attr->batch.keys; u32 value_size, cp, max_count; void *key, *value; int err = 0; @@ -1710,10 +1710,10 @@ int generic_map_lookup_batch(struct bpf_map *map, const union bpf_attr *attr, union bpf_attr __user *uattr) { - void __user *uobatch = u64_to_user_ptr(attr->batch.out_batch); - void __user *ubatch = u64_to_user_ptr(attr->batch.in_batch); - void __user *values = u64_to_user_ptr(attr->batch.values); - void __user *keys = u64_to_user_ptr(attr->batch.keys); + void __user *uobatch = (void __user *)attr->batch.out_batch; + void __user *ubatch = (void __user *)attr->batch.in_batch; + void __user *values = (void __user *)attr->batch.values; + void __user *keys = (void __user *)attr->batch.keys; void *buf, *buf_prevkey, *prev_key, *key, *value; int err, retry = MAP_LOOKUP_RETRIES; u32 value_size, cp, max_count; @@ -1810,8 +1810,8 @@ int generic_map_lookup_batch(struct bpf_map *map,

static int map_lookup_and_delete_elem(union bpf_attr *attr) { - void __user *ukey = u64_to_user_ptr(attr->key); - void __user *uvalue = u64_to_user_ptr(attr->value); + void __user *ukey = (void __user *)attr->key; + void __user *uvalue = (void __user *)attr->value; int ufd = attr->map_fd; struct bpf_map *map; void *key, *value; @@ -2688,7 +2688,7 @@ static int bpf_obj_pin(const union bpf_attr *attr) if (attr->file_flags != 0) return -EINVAL;

- return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname)); + return bpf_obj_pin_user(attr->bpf_fd, (void __user *)attr->pathname); }

static int bpf_obj_get(const union bpf_attr *attr) @@ -2696,7 +2696,7 @@ static int bpf_obj_get(const union bpf_attr *attr) if (attr->bpf_fd != 0 || attr->file_flags & ~BPF_OBJ_FLAG_MASK) return -EINVAL;

- return bpf_obj_get_user(u64_to_user_ptr(attr->pathname), + return bpf_obj_get_user((void __user *)attr->pathname, attr->file_flags); }

@@ -3199,7 +3199,7 @@ static int bpf_raw_tp_link_fill_link_info(const struct bpf_link *link, { struct bpf_raw_tp_link *raw_tp_link = container_of(link, struct bpf_raw_tp_link, link); - char __user *ubuf = u64_to_user_ptr(info->raw_tracepoint.tp_name); + char __user *ubuf = (char __user *)info->raw_tracepoint.tp_name; const char *tp_name = raw_tp_link->btp->tp->name; u32 ulen = info->raw_tracepoint.tp_name_len; size_t tp_len = strlen(tp_name); @@ -3392,7 +3392,8 @@ static int bpf_raw_tracepoint_open(const union bpf_attr *attr) if (IS_ERR(prog)) return PTR_ERR(prog);

- fd = bpf_raw_tp_link_attach(prog, u64_to_user_ptr(attr->raw_tracepoint.name)); + fd = bpf_raw_tp_link_attach(prog, + (const char __user *)attr->raw_tracepoint.name); if (fd < 0) bpf_prog_put(prog); return fd; @@ -3920,12 +3921,12 @@ convert_compat_prog_info_in(struct bpf_prog_info *dest, memcpy(dest->tag, cinfo->tag, BPF_TAG_SIZE); copy_field(dest, cinfo, jited_prog_len); copy_field(dest, cinfo, xlated_prog_len); - copy_field(dest, cinfo, jited_prog_insns); - copy_field(dest, cinfo, xlated_prog_insns); + bpf_compat_ptr_field(dest, cinfo, jited_prog_insns); + bpf_compat_ptr_field(dest, cinfo, xlated_prog_insns); copy_field(dest, cinfo, load_time); copy_field(dest, cinfo, created_by_uid); copy_field(dest, cinfo, nr_map_ids); - copy_field(dest, cinfo, map_ids); + bpf_compat_ptr_field(dest, cinfo, map_ids); strncpy((char *)dest->name, (char *)cinfo->name, BPF_OBJ_NAME_LEN); copy_field(dest, cinfo, ifindex); copy_field(dest, cinfo, gpl_compatible); @@ -3933,20 +3934,20 @@ convert_compat_prog_info_in(struct bpf_prog_info *dest, copy_field(dest, cinfo, netns_ino); copy_field(dest, cinfo, nr_jited_ksyms); copy_field(dest, cinfo, nr_jited_func_lens); - copy_field(dest, cinfo, jited_ksyms); - copy_field(dest, cinfo, jited_func_lens); + bpf_compat_ptr_field(dest, cinfo, jited_ksyms); + bpf_compat_ptr_field(dest, cinfo, jited_func_lens); copy_field(dest, cinfo, btf_id); copy_field(dest, cinfo, func_info_rec_size); - copy_field(dest, cinfo, func_info); + bpf_compat_ptr_field(dest, cinfo, func_info); copy_field(dest, cinfo, nr_func_info); copy_field(dest, cinfo, nr_line_info); - copy_field(dest, cinfo, line_info); - copy_field(dest, cinfo, jited_line_info); + bpf_compat_ptr_field(dest, cinfo, line_info); + bpf_compat_ptr_field(dest, cinfo, jited_line_info); copy_field(dest, cinfo, nr_jited_line_info); copy_field(dest, cinfo, line_info_rec_size); copy_field(dest, cinfo, jited_line_info_rec_size); copy_field(dest, cinfo, nr_prog_tags); - copy_field(dest, cinfo, prog_tags); + bpf_compat_ptr_field(dest, cinfo, prog_tags); copy_field(dest, cinfo, run_time_ns); copy_field(dest, cinfo, run_cnt); copy_field(dest, cinfo, recursion_misses); @@ -4007,7 +4008,7 @@ static int copy_bpf_prog_info_from_user(const union bpf_attr *attr, int err; size_t info_size = in_compat64_syscall() ? sizeof(struct compat_bpf_prog_info) : sizeof(struct bpf_prog_info); - void __user *uinfo = u64_to_user_ptr(attr->info.info); + void __user *uinfo = (void __user *)attr->info.info;

*info_len = attr->info.info_len; err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), @@ -4023,7 +4024,7 @@ static int copy_bpf_prog_info_from_user(const union bpf_attr *attr, return -EFAULT; convert_compat_prog_info_in(info, &cinfo); } else { - if (copy_from_user(info, uinfo, *info_len)) + if (copy_from_user_with_ptr(info, uinfo, *info_len)) return -EFAULT; }

@@ -4045,7 +4046,7 @@ static int copy_bpf_prog_info_to_user(const union bpf_attr *attr, convert_compat_prog_info_out(&cinfo, info); }

- if (copy_to_user(uinfo, src_info, *info_len) || + if (bpf_copy_to_user_with_ptr(uinfo, src_info, *info_len) || bpf_put_uattr(*info_len, uattr, info.info_len)) return -EFAULT;

@@ -4084,7 +4085,7 @@ static int bpf_prog_get_info_by_fd(struct file *file, info.nr_map_ids = prog->aux->used_map_cnt; ulen = min_t(u32, info.nr_map_ids, ulen); if (ulen) { - u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids); + u32 __user *user_map_ids = (u32 __user *)info.map_ids; u32 i;

for (i = 0; i < ulen; i++) @@ -4131,7 +4132,7 @@ static int bpf_prog_get_info_by_fd(struct file *file, insns_sanitized = bpf_insn_prepare_dump(prog, file->f_cred); if (!insns_sanitized) return -ENOMEM; - uinsns = u64_to_user_ptr(info.xlated_prog_insns); + uinsns = (char __user *)info.xlated_prog_insns; ulen = min_t(u32, info.xlated_prog_len, ulen); fault = copy_to_user(uinsns, insns_sanitized, ulen); kfree(insns_sanitized); @@ -4163,7 +4164,7 @@ static int bpf_prog_get_info_by_fd(struct file *file,

if (info.jited_prog_len && ulen) { if (bpf_dump_raw_ok(file->f_cred)) { - uinsns = u64_to_user_ptr(info.jited_prog_insns); + uinsns = (char __user *)info.jited_prog_insns; ulen = min_t(u32, info.jited_prog_len, ulen);

/* for multi-function programs, copy the JITed @@ -4206,7 +4207,7 @@ static int bpf_prog_get_info_by_fd(struct file *file, * corresponding to each function */ ulen = min_t(u32, info.nr_jited_ksyms, ulen); - user_ksyms = u64_to_user_ptr(info.jited_ksyms); + user_ksyms = (u64 __user *)info.jited_ksyms; if (prog->aux->func_cnt) { for (i = 0; i < ulen; i++) { ksym_addr = (unsigned long) @@ -4234,7 +4235,7 @@ static int bpf_prog_get_info_by_fd(struct file *file,

/* copy the JITed image lengths for each function */ ulen = min_t(u32, info.nr_jited_func_lens, ulen); - user_lens = u64_to_user_ptr(info.jited_func_lens); + user_lens = (u32 __user *)info.jited_func_lens; if (prog->aux->func_cnt) { for (i = 0; i < ulen; i++) { func_len = @@ -4263,7 +4264,7 @@ static int bpf_prog_get_info_by_fd(struct file *file, if (info.nr_func_info && ulen) { char __user *user_finfo;

- user_finfo = u64_to_user_ptr(info.func_info); + user_finfo = (char __user *)info.func_info; ulen = min_t(u32, info.nr_func_info, ulen); if (copy_to_user(user_finfo, prog->aux->func_info, info.func_info_rec_size * ulen)) @@ -4275,7 +4276,7 @@ static int bpf_prog_get_info_by_fd(struct file *file, if (info.nr_line_info && ulen) { __u8 __user *user_linfo;

- user_linfo = u64_to_user_ptr(info.line_info); + user_linfo = (__u8 __user *)info.line_info; ulen = min_t(u32, info.nr_line_info, ulen); if (copy_to_user(user_linfo, prog->aux->linfo, info.line_info_rec_size * ulen)) @@ -4293,7 +4294,7 @@ static int bpf_prog_get_info_by_fd(struct file *file, __u64 __user *user_linfo; u32 i;

- user_linfo = u64_to_user_ptr(info.jited_line_info); + user_linfo = (__u64 __user *)info.jited_line_info; ulen = min_t(u32, info.nr_jited_line_info, ulen); for (i = 0; i < ulen; i++) { line_addr = (unsigned long)prog->aux->jited_linfo[i]; @@ -4308,20 +4309,20 @@ static int bpf_prog_get_info_by_fd(struct file *file, ulen = info.nr_prog_tags; info.nr_prog_tags = prog->aux->func_cnt ? : 1; if (ulen) { - __u8 __user (*user_prog_tags)[BPF_TAG_SIZE]; + __u8 __user *user_prog_tags; u32 i;

- user_prog_tags = u64_to_user_ptr(info.prog_tags); + user_prog_tags = (__u8 __user *)info.prog_tags; ulen = min_t(u32, info.nr_prog_tags, ulen); if (prog->aux->func_cnt) { for (i = 0; i < ulen; i++) { - if (copy_to_user(user_prog_tags[i], + if (copy_to_user(&user_prog_tags[i], prog->aux->func[i]->tag, BPF_TAG_SIZE)) return -EFAULT; } } else { - if (copy_to_user(user_prog_tags[0], + if (copy_to_user(&user_prog_tags[0], prog->tag, BPF_TAG_SIZE)) return -EFAULT; } @@ -4339,7 +4340,8 @@ static int bpf_map_get_info_by_fd(struct file *file, const union bpf_attr *attr, union bpf_attr __user *uattr) { - struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info); + struct bpf_map_info __user *uinfo = + (struct bpf_map_info __user *)attr->info.info; struct bpf_map_info info; u32 info_len = attr->info.info_len; int err; @@ -4398,11 +4400,11 @@ convert_compat_link_info_in(struct bpf_link_info *dest, */ switch (type) { case BPF_LINK_TYPE_RAW_TRACEPOINT: - copy_field(dest, cinfo, raw_tracepoint.tp_name); + bpf_compat_ptr_field(dest, cinfo, raw_tracepoint.tp_name); copy_field(dest, cinfo, raw_tracepoint.tp_name_len); return; case BPF_LINK_TYPE_ITER: - copy_field(dest, cinfo, iter.target_name); + bpf_compat_ptr_field(dest, cinfo, iter.target_name); copy_field(dest, cinfo, iter.target_name_len); return; default: @@ -4476,7 +4478,7 @@ static int copy_bpf_link_info_from_user(const union bpf_attr *attr, int err; size_t info_size = in_compat64_syscall() ? sizeof(struct compat_bpf_link_info) : sizeof(struct bpf_link_info); - void __user *uinfo = u64_to_user_ptr(attr->info.info); + void __user *uinfo = (void __user *)attr->info.info;

*info_len = attr->info.info_len; err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), @@ -4492,7 +4494,7 @@ static int copy_bpf_link_info_from_user(const union bpf_attr *attr, return -EFAULT; convert_compat_link_info_in(info, &cinfo, type); } else { - if (copy_from_user(info, uinfo, *info_len)) + if (copy_from_user_with_ptr(info, uinfo, *info_len)) return -EFAULT; }

@@ -4515,7 +4517,7 @@ static int copy_bpf_link_info_to_user(const union bpf_attr *attr, convert_compat_link_info_out(&cinfo, info, type); }

- if (copy_to_user(uinfo, src_info, *info_len) || + if (bpf_copy_to_user_with_ptr(uinfo, src_info, *info_len) || bpf_put_uattr(*info_len, uattr, info.info_len)) return -EFAULT;

@@ -4610,7 +4612,7 @@ static int bpf_task_fd_query_copy(const union bpf_attr *attr, const char *buf, u64 probe_offset, u64 probe_addr) { - char __user *ubuf = u64_to_user_ptr(attr->task_fd_query.buf); + char __user *ubuf = (char __user *)attr->task_fd_query.buf; u32 len = buf ? strlen(buf) : 0, input_len; int err = 0;

@@ -5303,8 +5305,8 @@ static void convert_compat_bpf_attr(union bpf_attr *dest, case BPF_MAP_DELETE_ELEM: case BPF_MAP_LOOKUP_AND_DELETE_ELEM: copy_field(dest, cattr, map_fd); - copy_field(dest, cattr, key); - copy_field(dest, cattr, value); + bpf_compat_ptr_field(dest, cattr, key); + bpf_compat_ptr_field(dest, cattr, value); /* u64 next_key is in a union with u64 value */ copy_field(dest, cattr, flags); break; @@ -5312,10 +5314,10 @@ static void convert_compat_bpf_attr(union bpf_attr *dest, case BPF_MAP_LOOKUP_AND_DELETE_BATCH: case BPF_MAP_UPDATE_BATCH: case BPF_MAP_DELETE_BATCH: - copy_field(dest, cattr, batch.in_batch); - copy_field(dest, cattr, batch.out_batch); - copy_field(dest, cattr, batch.keys); - copy_field(dest, cattr, batch.values); + bpf_compat_ptr_field(dest, cattr, batch.in_batch); + bpf_compat_ptr_field(dest, cattr, batch.out_batch); + bpf_compat_ptr_field(dest, cattr, batch.keys); + bpf_compat_ptr_field(dest, cattr, batch.values); copy_field(dest, cattr, batch.count); copy_field(dest, cattr, batch.map_fd); copy_field(dest, cattr, batch.elem_flags); @@ -5324,11 +5326,11 @@ static void convert_compat_bpf_attr(union bpf_attr *dest, case BPF_PROG_LOAD: copy_field(dest, cattr, prog_type); copy_field(dest, cattr, insn_cnt); - copy_field(dest, cattr, insns); - copy_field(dest, cattr, license); + bpf_compat_ptr_field(dest, cattr, insns); + bpf_compat_ptr_field(dest, cattr, license); copy_field(dest, cattr, log_level); copy_field(dest, cattr, log_size); - copy_field(dest, cattr, log_buf); + bpf_compat_ptr_field(dest, cattr, log_buf); copy_field(dest, cattr, kern_version); copy_field(dest, cattr, prog_flags); strncpy(dest->prog_name, cattr->prog_name, BPF_OBJ_NAME_LEN); @@ -5336,23 +5338,23 @@ static void convert_compat_bpf_attr(union bpf_attr *dest, copy_field(dest, cattr, expected_attach_type); copy_field(dest, cattr, prog_btf_fd); copy_field(dest, cattr, func_info_rec_size); - copy_field(dest, cattr, func_info); + bpf_compat_ptr_field(dest, cattr, func_info); copy_field(dest, cattr, func_info_cnt); copy_field(dest, cattr, line_info_rec_size); - copy_field(dest, cattr, line_info); + bpf_compat_ptr_field(dest, cattr, line_info); copy_field(dest, cattr, line_info_cnt); copy_field(dest, cattr, attach_btf_id); copy_field(dest, cattr, attach_prog_fd); /* u32 attach_btf_obj_fd is in a union with u32 attach_prog_fd */ copy_field(dest, cattr, core_relo_cnt); - copy_field(dest, cattr, fd_array); - copy_field(dest, cattr, core_relos); + bpf_compat_ptr_field(dest, cattr, fd_array); + bpf_compat_ptr_field(dest, cattr, core_relos); copy_field(dest, cattr, core_relo_rec_size); copy_field(dest, cattr, log_true_size); break; case BPF_OBJ_PIN: case BPF_OBJ_GET: - copy_field(dest, cattr, pathname); + bpf_compat_ptr_field(dest, cattr, pathname); copy_field(dest, cattr, bpf_fd); copy_field(dest, cattr, file_flags); break; @@ -5369,14 +5371,14 @@ static void convert_compat_bpf_attr(union bpf_attr *dest, copy_field(dest, cattr, test.retval); copy_field(dest, cattr, test.data_size_in); copy_field(dest, cattr, test.data_size_out); - copy_field(dest, cattr, test.data_in); - copy_field(dest, cattr, test.data_out); + bpf_compat_ptr_field(dest, cattr, test.data_in); + bpf_compat_ptr_field(dest, cattr, test.data_out); copy_field(dest, cattr, test.repeat); copy_field(dest, cattr, test.duration); copy_field(dest, cattr, test.ctx_size_in); copy_field(dest, cattr, test.ctx_size_out); - copy_field(dest, cattr, test.ctx_in); - copy_field(dest, cattr, test.ctx_out); + bpf_compat_ptr_field(dest, cattr, test.ctx_in); + bpf_compat_ptr_field(dest, cattr, test.ctx_out); copy_field(dest, cattr, test.flags); copy_field(dest, cattr, test.cpu); copy_field(dest, cattr, test.batch_size); @@ -5400,24 +5402,24 @@ static void convert_compat_bpf_attr(union bpf_attr *dest, case BPF_OBJ_GET_INFO_BY_FD: copy_field(dest, cattr, info.bpf_fd); copy_field(dest, cattr, info.info_len); - copy_field(dest, cattr, info.info); + bpf_compat_ptr_field(dest, cattr, info.info); break; case BPF_PROG_QUERY: copy_field(dest, cattr, query.target_fd); copy_field(dest, cattr, query.attach_type); copy_field(dest, cattr, query.query_flags); copy_field(dest, cattr, query.attach_flags); - copy_field(dest, cattr, query.prog_ids); + bpf_compat_ptr_field(dest, cattr, query.prog_ids); copy_field(dest, cattr, query.prog_cnt); - copy_field(dest, cattr, query.prog_attach_flags); + bpf_compat_ptr_field(dest, cattr, query.prog_attach_flags); break; case BPF_RAW_TRACEPOINT_OPEN: - copy_field(dest, cattr, raw_tracepoint.name); + bpf_compat_ptr_field(dest, cattr, raw_tracepoint.name); copy_field(dest, cattr, raw_tracepoint.prog_fd); break; case BPF_BTF_LOAD: - copy_field(dest, cattr, btf); - copy_field(dest, cattr, btf_log_buf); + bpf_compat_ptr_field(dest, cattr, btf); + bpf_compat_ptr_field(dest, cattr, btf_log_buf); copy_field(dest, cattr, btf_size); copy_field(dest, cattr, btf_log_size); copy_field(dest, cattr, btf_log_level); @@ -5428,7 +5430,7 @@ static void convert_compat_bpf_attr(union bpf_attr *dest, copy_field(dest, cattr, task_fd_query.fd); copy_field(dest, cattr, task_fd_query.flags); copy_field(dest, cattr, task_fd_query.buf_len); - copy_field(dest, cattr, task_fd_query.buf); + bpf_compat_ptr_field(dest, cattr, task_fd_query.buf); copy_field(dest, cattr, task_fd_query.prog_id); copy_field(dest, cattr, task_fd_query.fd_type); copy_field(dest, cattr, task_fd_query.probe_offset); @@ -5453,7 +5455,7 @@ static void convert_compat_bpf_attr(union bpf_attr *dest, * the same for compat64/purecap; hence no * conversion needed */ - copy_field(dest, cattr, link_create.iter_info); + bpf_compat_ptr_field(dest, cattr, link_create.iter_info); copy_field(dest, cattr, link_create.iter_info_len); break; } @@ -5462,9 +5464,12 @@ static void convert_compat_bpf_attr(union bpf_attr *dest, if (cattr->link_create.attach_type == BPF_TRACE_KPROBE_MULTI) { copy_field(dest, cattr, link_create.kprobe_multi.flags); copy_field(dest, cattr, link_create.kprobe_multi.cnt); - copy_field(dest, cattr, link_create.kprobe_multi.syms); - copy_field(dest, cattr, link_create.kprobe_multi.addrs); - copy_field(dest, cattr, link_create.kprobe_multi.cookies); + bpf_compat_ptr_field(dest, cattr, + link_create.kprobe_multi.syms); + bpf_compat_ptr_field(dest, cattr, + link_create.kprobe_multi.addrs); + bpf_compat_ptr_field(dest, cattr, + link_create.kprobe_multi.cookies); break; }

@@ -5525,7 +5530,7 @@ static int copy_bpf_attr_from_user(union bpf_attr *attr, int cmd, return -EINVAL; convert_compat_bpf_attr(attr, &cattr, cmd); } else { - if (copy_from_bpfptr(attr, uattr, *size) != 0) + if (copy_from_bpfptr_with_ptr(attr, uattr, *size) != 0) return -EFAULT; err = check_attr(cmd, attr); if (err) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index fdec45bbbc7e..2881336a3fdb 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -18829,8 +18829,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3 * and supplied buffer to store the verification trace */ ret = bpf_vlog_init(&env->log, attr->log_level, - (char __user *) (unsigned long) attr->log_buf, - attr->log_size); + (char __user *)attr->log_buf, attr->log_size); if (ret) goto err_unlock;

diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 1f4b07da327a..dcaf4c329ee6 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -2797,8 +2797,8 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr if (flags & ~BPF_F_KPROBE_MULTI_RETURN) return -EINVAL;

- uaddrs = u64_to_user_ptr(attr->link_create.kprobe_multi.addrs); - usyms = u64_to_user_ptr(attr->link_create.kprobe_multi.syms); + uaddrs = (void __user *)attr->link_create.kprobe_multi.addrs; + usyms = (void __user *)attr->link_create.kprobe_multi.syms; if (!!uaddrs == !!usyms) return -EINVAL;

@@ -2811,7 +2811,7 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr if (!addrs) return -ENOMEM;

- ucookies = u64_to_user_ptr(attr->link_create.kprobe_multi.cookies); + ucookies = (void __user *)attr->link_create.kprobe_multi.cookies; if (ucookies) { cookies = kvmalloc_array(cnt, sizeof(*addrs), GFP_KERNEL); if (!cookies) { diff --git a/net/bpf/bpf_dummy_struct_ops.c b/net/bpf/bpf_dummy_struct_ops.c index c7bd67d75dac..9c3b0aad72b5 100644 --- a/net/bpf/bpf_dummy_struct_ops.c +++ b/net/bpf/bpf_dummy_struct_ops.c @@ -33,12 +33,12 @@ dummy_ops_init_args(const union bpf_attr *kattr, unsigned int nr) if (!args) return ERR_PTR(-ENOMEM);

- ctx_in = u64_to_user_ptr(kattr->test.ctx_in); + ctx_in = (void __user *)kattr->test.ctx_in; if (copy_from_user(args->args, ctx_in, size_in)) goto out;

/* args[0] is 0 means state argument of test_N will be NULL */ - u_state = u64_to_user_ptr(args->args[0]); + u_state = (void __user *)args->args[0]; if (u_state && copy_from_user(&args->state, u_state, sizeof(args->state))) goto out; @@ -53,7 +53,7 @@ static int dummy_ops_copy_args(struct bpf_dummy_ops_test_args *args) { void __user *u_state;

- u_state = u64_to_user_ptr(args->args[0]); + u_state = (void __user *)args->args[0]; if (u_state && copy_to_user(u_state, &args->state, sizeof(args->state))) return -EFAULT;

diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c index 8ffff57e13a0..f2f1fd3a869f 100644 --- a/net/bpf/test_run.c +++ b/net/bpf/test_run.c @@ -436,7 +436,7 @@ static int bpf_test_finish(const union bpf_attr *kattr, struct skb_shared_info *sinfo, u32 size, u32 retval, u32 duration) { - void __user *data_out = u64_to_user_ptr(kattr->test.data_out); + void __user *data_out = (void __user *)kattr->test.data_out; int err = -EFAULT; u32 copy_size = size;

@@ -813,7 +813,7 @@ BTF_SET8_END(test_sk_check_kfunc_ids) static void *bpf_test_init(const union bpf_attr *kattr, u32 user_size, u32 size, u32 headroom, u32 tailroom) { - void __user *data_in = u64_to_user_ptr(kattr->test.data_in); + void __user *data_in = (void __user *)kattr->test.data_in; void *data;

if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom) @@ -900,7 +900,7 @@ int bpf_prog_test_run_raw_tp(struct bpf_prog *prog, const union bpf_attr *kattr, union bpf_attr __user *uattr) { - void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in); + void __user *ctx_in = (void __user *)kattr->test.ctx_in; __u32 ctx_size_in = kattr->test.ctx_size_in; struct bpf_raw_tp_test_run_info info; int cpu = kattr->test.cpu, err = 0; @@ -955,8 +955,8 @@ int bpf_prog_test_run_raw_tp(struct bpf_prog *prog,

static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size) { - void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in); - void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); + void __user *data_in = (void __user *)kattr->test.ctx_in; + void __user *data_out = (void __user *)kattr->test.ctx_out; u32 size = kattr->test.ctx_size_in; void *data; int err; @@ -988,7 +988,7 @@ static int bpf_ctx_finish(const union bpf_attr *kattr, union bpf_attr __user *uattr, const void *data, u32 size) { - void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); + void __user *data_out = (void __user *)kattr->test.ctx_out; int err = -EFAULT; u32 copy_size = size;

@@ -1395,7 +1395,7 @@ int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, goto free_data;

if (unlikely(kattr->test.data_size_in > size)) { - void __user *data_in = u64_to_user_ptr(kattr->test.data_in); + void __user *data_in = (void __user *)kattr->test.data_in;

while (size < kattr->test.data_size_in) { struct page *page; @@ -1653,7 +1653,7 @@ int bpf_prog_test_run_syscall(struct bpf_prog *prog, const union bpf_attr *kattr, union bpf_attr __user *uattr) { - void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in); + void __user *ctx_in = (void __user *)kattr->test.ctx_in; __u32 ctx_size_in = kattr->test.ctx_size_in; void *ctx = NULL; u32 retval; diff --git a/net/core/sock_map.c b/net/core/sock_map.c index 782e9f7799ae..1c7bc8334aac 100644 --- a/net/core/sock_map.c +++ b/net/core/sock_map.c @@ -1508,7 +1508,7 @@ static int sock_map_prog_update(struct bpf_map *map, struct bpf_prog *prog, int sock_map_bpf_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr) { - __u32 __user *prog_ids = u64_to_user_ptr(attr->query.prog_ids); + __u32 __user *prog_ids = (__u32 __user *)attr->query.prog_ids; u32 prog_cnt = 0, flags = 0, ufd = attr->target_fd; struct bpf_prog **pprog; struct bpf_prog *prog;

The purpose of the bpf_copy_from_user_with_task eBPF helper is to copy user data from the provided task. Since __user *ptrs are only valid for the current task/process, what is actually being passed in here is an address. access_process_vm is then used to access that address given the task specified by the task_struct.

Additionally, this helper is called from an eBPF program, which does not support capabilities. It is therefore unable to pass in a capability as a __user ptr in PCuABI in the first place.

The use case of this helper is to read arbitrary user memory, as used by security or monitoring eBPF programs. There is therefore no requirement to check user capabilities here. Loading of eBPF applications using this helper type is already strictly limited to privileged/root users.

In addition, access_process_vm() is not uaccess in the usual sense, as uaccess is about accessing the current process. This type of access is not currently checked in PCuABI.

Change the inbound type from __user * to ptraddr_t and remove the TODO since no capability checks are required here.

Signed-off-by: Zachary Leaf zachary.leaf@arm.com --- kernel/bpf/helpers.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 1362d7736a93..4910d7b88239 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -670,7 +670,7 @@ const struct bpf_func_proto bpf_copy_from_user_proto = { };

BPF_CALL_5(bpf_copy_from_user_task, void *, dst, u32, size, - const void __user *, user_ptr, struct task_struct *, tsk, u64, flags) + ptraddr_t, addr, struct task_struct *, tsk, u64, flags) { int ret;

@@ -681,8 +681,7 @@ BPF_CALL_5(bpf_copy_from_user_task, void *, dst, u32, size, if (unlikely(!size)) return 0;

- /* TODO [PCuABI] - capability checks for uaccess */ - ret = access_process_vm(tsk, user_ptr_addr(user_ptr), dst, size, 0); + ret = access_process_vm(tsk, addr, dst, size, 0); if (ret == size) return 0;