arm64 JIT'd bpf programs on Morello currently re-use the existing kernel stack in the kernel logical memory map (VMAP_STACK is not available in PCuABI).
Since JIT programs will be running in a compartment with memory accesses limited to the vmalloc region, new stacks for each eBPF program must be created here to allow access and provide isolation of stacks.
Allocate and free new page sized/aligned stacks in the vmalloc region at the same time as the binary image to ensure they last for the lifetime of the program.
Signed-off-by: Zachary Leaf zachary.leaf@arm.com --- arch/arm64/net/bpf_jit_comp.c | 27 +++++++++++++++++++++++++++ include/linux/filter.h | 2 ++ kernel/bpf/core.c | 13 +++++++++++++ 3 files changed, 42 insertions(+)
diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 85810908dc15..36419cdaa710 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -24,6 +24,8 @@
#include "bpf_jit.h"
+#define BPF_STACK_SZ (PAGE_SIZE * 16) + #define TMP_REG_1 (MAX_BPF_JIT_REG + 0) #define TMP_REG_2 (MAX_BPF_JIT_REG + 1) #define TCALL_CNT (MAX_BPF_JIT_REG + 2) @@ -79,6 +81,7 @@ struct jit_ctx { int exentry_idx; __le32 *image; int image_size; + void *stack; u32 stack_size; int fpb_offset; }; @@ -1513,6 +1516,25 @@ struct arm64_jit_data { struct jit_ctx ctx; };
+void *bpf_jit_alloc_stack() +{ + void *sp = __vmalloc_node(BPF_STACK_SZ, PAGE_SIZE, THREADINFO_GFP, + NUMA_NO_NODE, __builtin_return_address(0)); + + if (!sp) { + printk("%s stack allocation failed\n", __func__); + return NULL; + } + printk("%s allocated stack at:%#lx-%lx size:%d\n", __func__, + sp, sp+BPF_STACK_SZ, BPF_STACK_SZ); + if (((uintptr_t)sp & 0xF) != 0) + printk("%s stack is NOT 16B aligned\n", __func__); + if (((uintptr_t)sp & 0xFFF) != 0) + printk("%s stack is NOT 4k page aligned\n", __func__); + + return kasan_reset_tag(sp); +} + struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) { int prog_size, extable_size, extable_align, extable_offset; @@ -1603,6 +1625,11 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) goto out_off; }
+ if (header->stack == NULL) { + goto out_off; + } + ctx.stack = header->stack; + /* 2. Now, the actual pass. */
ctx.image = (__le32 *)image_ptr; diff --git a/include/linux/filter.h b/include/linux/filter.h index a4953fafc8cb..471cef983dd4 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -601,6 +601,7 @@ struct sock_fprog_kern {
struct bpf_binary_header { u32 size; + void *stack; u8 image[] __aligned(BPF_IMAGE_ALIGNMENT); };
@@ -1061,6 +1062,7 @@ bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr, void bpf_jit_binary_free(struct bpf_binary_header *hdr); u64 bpf_jit_alloc_exec_limit(void); void *bpf_jit_alloc_exec(unsigned long size); +void *bpf_jit_alloc_stack(void); void bpf_jit_free_exec(void *addr); void bpf_jit_free(struct bpf_prog *fp); struct bpf_binary_header * diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 510fec53df3f..509a70e6c25f 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -1023,6 +1023,11 @@ void __weak bpf_jit_free_exec(void *addr) module_memfree(addr); }
+void *__weak bpf_jit_alloc_stack() +{ + return NULL; +} + struct bpf_binary_header * bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr, unsigned int alignment, @@ -1061,6 +1066,12 @@ bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr, /* The actual start of the JIT code */ printk("%s JIT loc=%#lx\n", __func__, *image_ptr);
+ /* + * Save the bpf SP in the header; it's the easiest way to ensure the + * memory is free'd at the same time as the image in bpf_jit_binary_free + */ + hdr->stack = bpf_jit_alloc_stack(); + return hdr; }
@@ -1068,6 +1079,8 @@ void bpf_jit_binary_free(struct bpf_binary_header *hdr) { u32 size = hdr->size;
+ if (hdr->stack) + bpf_jit_free_exec(hdr->stack); bpf_jit_free_exec(hdr); bpf_jit_uncharge_modmem(size); }