diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 929123a5431a..f546df886049 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -78,6 +78,7 @@ static const int bpf2a64[] = { struct jit_ctx { const struct bpf_prog *prog; + unsigned long *indirect_targets; int idx; int epilogue_offset; int *offset; @@ -1199,6 +1200,11 @@ static int add_exception_handler(const struct bpf_insn *insn, return 0; } +static bool maybe_indirect_target(int insn_off, unsigned long *targets_bitmap) +{ + return targets_bitmap && test_bit(insn_off, targets_bitmap); +} + /* JITs an eBPF instruction. * Returns: * 0 - successfully JITed an 8-byte eBPF instruction. @@ -1231,6 +1237,9 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, int ret; bool sign_extend; + if (maybe_indirect_target(i, ctx->indirect_targets)) + emit_bti(A64_BTI_J, ctx); + switch (code) { /* dst = src */ case BPF_ALU | BPF_MOV | BPF_X: @@ -2085,6 +2094,16 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) memset(&ctx, 0, sizeof(ctx)); ctx.prog = prog; + if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL) && bpf_prog_has_insn_array(prog)) { + ctx.indirect_targets = kvcalloc(BITS_TO_LONGS(prog->len), sizeof(unsigned long), + GFP_KERNEL); + if (ctx.indirect_targets == NULL) { + prog = orig_prog; + goto out_off; + } + bpf_prog_collect_indirect_targets(prog, ctx.indirect_targets); + } + ctx.offset = kvcalloc(prog->len + 1, sizeof(int), GFP_KERNEL); if (ctx.offset == NULL) { prog = orig_prog; @@ -2248,6 +2267,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) prog->aux->priv_stack_ptr = NULL; } kvfree(ctx.offset); + kvfree(ctx.indirect_targets); out_priv_stack: kfree(jit_data); prog->aux->jit_data = NULL; diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 6498be4c44f8..c7c94dc7f8bb 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -3822,11 +3822,23 @@ void bpf_insn_array_adjust_after_remove(struct bpf_map *map, u32 off, u32 len); #ifdef CONFIG_BPF_SYSCALL void bpf_prog_update_insn_ptrs(struct bpf_prog *prog, u32 *offsets, void *image); +void bpf_prog_collect_indirect_targets(const struct bpf_prog *prog, unsigned long *bitmap); +bool bpf_prog_has_insn_array(const struct bpf_prog *prog); #else static inline void bpf_prog_update_insn_ptrs(struct bpf_prog *prog, u32 *offsets, void *image) { } + +static inline bool bpf_prog_has_insn_array(const struct bpf_prog *prog) +{ + return false; +} + +static inline void +bpf_prog_collect_indirect_targets(const struct bpf_prog *prog, unsigned long *bitmap) +{ +} #endif static inline int bpf_map_check_op_flags(struct bpf_map *map, u64 flags, u64 allowed_flags) diff --git a/kernel/bpf/bpf_insn_array.c b/kernel/bpf/bpf_insn_array.c index c96630cb75bf..20d6c8d8facb 100644 --- a/kernel/bpf/bpf_insn_array.c +++ b/kernel/bpf/bpf_insn_array.c @@ -302,3 +302,46 @@ void bpf_prog_update_insn_ptrs(struct bpf_prog *prog, u32 *offsets, void *image) } } } + +bool bpf_prog_has_insn_array(const struct bpf_prog *prog) +{ + int i; + + for (i = 0; i < prog->aux->used_map_cnt; i++) { + if (is_insn_array(prog->aux->used_maps[i])) + return true; + } + return false; +} + +/* + * This function collects possible indirect jump targets in a BPF program. Since indirect jump + * targets can only be read from instruction arrays, it traverses all instruction arrays used + * by @prog. For each instruction in the arrays, it sets the corresponding bit in @bitmap. + */ +void bpf_prog_collect_indirect_targets(const struct bpf_prog *prog, unsigned long *bitmap) +{ + struct bpf_insn_array *insn_array; + struct bpf_map *map; + u32 xlated_off; + int i, j; + + for (i = 0; i < prog->aux->used_map_cnt; i++) { + map = prog->aux->used_maps[i]; + if (!is_insn_array(map)) + continue; + + insn_array = cast_insn_array(map); + for (j = 0; j < map->max_entries; j++) { + xlated_off = insn_array->values[j].xlated_off; + if (xlated_off == INSN_DELETED) + continue; + if (xlated_off < prog->aux->subprog_start) + continue; + xlated_off -= prog->aux->subprog_start; + if (xlated_off >= prog->len) + continue; + __set_bit(xlated_off, bitmap); + } + } +}