@@ -34,6 +34,9 @@ REALMODE_CFLAGS += -fno-stack-protector
REALMODE_CFLAGS += $(call __cc-option, $(CC), $(REALMODE_CFLAGS), -Wno-address-of-packed-member)
REALMODE_CFLAGS += $(call __cc-option, $(CC), $(REALMODE_CFLAGS), $(cc_stack_align4))
REALMODE_CFLAGS += $(CLANG_FLAGS)
+ifdef CONFIG_DYNAMIC_FTRACE
+ REALMODE_CFLAGS += -DCC_USING_FENTRY
+endif
export REALMODE_CFLAGS
# BITS is used as extension for files which are available in a 32 bit
@@ -54,6 +57,10 @@ KBUILD_CFLAGS += $(call cc-option,-mno-avx,)
# Intel CET isn't enabled in the kernel
KBUILD_CFLAGS += $(call cc-option,-fcf-protection=none)
+ifdef CONFIG_DYNAMIC_FTRACE
+ KBUILD_CFLAGS += -DCC_USING_FENTRY
+endif
+
ifeq ($(CONFIG_X86_32),y)
BITS := 32
UTS_MACHINE := i386
@@ -49,6 +49,10 @@ KBUILD_CFLAGS += $(call as-option,-Wa$(comma)-mrelax-relocations=no)
KBUILD_CFLAGS += -include $(srctree)/include/linux/hidden.h
KBUILD_CFLAGS += $(CLANG_FLAGS)
+ifdef CONFIG_DYNAMIC_FTRACE
+ KBUILD_CFLAGS += -DCC_USING_FENTRY
+endif
+
# sev.c indirectly inludes inat-table.h which is generated during
# compilation and stored in $(objtree). Add the directory to the includes so
# that the compiler finds it even with out-of-tree builds (make O=/some/path).
@@ -15,6 +15,9 @@ cflags-$(CONFIG_X86) += -m$(BITS) -D__KERNEL__ \
$(call cc-disable-warning, gnu) \
-fno-asynchronous-unwind-tables \
$(CLANG_FLAGS)
+ifdef CONFIG_DYNAMIC_FTRACE
+ cflags-$(CONFIG_X86) += -DCC_USING_FENTRY
+endif
# arm64 uses the full KBUILD_CFLAGS so it's necessary to explicitly
# disable the stackleak plugin
@@ -23,6 +23,7 @@
#include <linux/slab.h>
#include <linux/percpu-refcount.h>
#include <linux/bpfptr.h>
+#include <linux/ftrace.h>
struct bpf_verifier_env;
struct bpf_verifier_log;
@@ -703,6 +704,7 @@ struct bpf_trampoline_multi {
struct list_head list;
u32 *ids;
u32 ids_cnt;
+ struct ftrace_ops ops;
int tr_cnt;
struct bpf_trampoline *tr[];
};
@@ -179,11 +179,38 @@ static int is_ftrace_location(void *ip)
return 1;
}
+static int register_ftrace_multi(struct bpf_trampoline *tr, void *new_addr)
+{
+ struct bpf_trampoline_multi *multi;
+
+ multi = container_of(tr, struct bpf_trampoline_multi, main);
+ return register_ftrace_direct_multi(&multi->ops, (long)new_addr);
+}
+
+static int unregister_ftrace_multi(struct bpf_trampoline *tr, void *old_addr)
+{
+ struct bpf_trampoline_multi *multi;
+
+ multi = container_of(tr, struct bpf_trampoline_multi, main);
+ return unregister_ftrace_direct_multi(&multi->ops);
+}
+
+static int modify_ftrace_multi(struct bpf_trampoline *tr, void *new_addr)
+{
+ struct bpf_trampoline_multi *multi;
+
+ multi = container_of(tr, struct bpf_trampoline_multi, main);
+ return modify_ftrace_direct_multi(&multi->ops, (long)new_addr);
+}
+
static int unregister_fentry(struct bpf_trampoline *tr, void *old_addr)
{
void *ip = tr->func.addr;
int ret;
+ if (is_multi_trampoline(tr))
+ return unregister_ftrace_multi(tr, old_addr);
+
if (tr->func.ftrace_managed)
ret = unregister_ftrace_direct((long)ip, (long)old_addr);
else
@@ -199,6 +226,9 @@ static int modify_fentry(struct bpf_trampoline *tr, void *old_addr, void *new_ad
void *ip = tr->func.addr;
int ret;
+ if (is_multi_trampoline(tr))
+ return modify_ftrace_multi(tr, new_addr);
+
if (tr->func.ftrace_managed)
ret = modify_ftrace_direct((long)ip, (long)old_addr, (long)new_addr);
else
@@ -212,6 +242,9 @@ static int register_fentry(struct bpf_trampoline *tr, void *new_addr)
void *ip = tr->func.addr;
int ret;
+ if (is_multi_trampoline(tr))
+ return register_ftrace_multi(tr, new_addr);
+
ret = is_ftrace_location(ip);
if (ret < 0)
return ret;
@@ -703,6 +736,10 @@ struct bpf_trampoline_multi *bpf_trampoline_multi_get(struct bpf_prog *prog,
if (!is_ftrace_location((void *) tgt_info.tgt_addr))
goto out_free;
+ err = ftrace_set_filter_ip(&multi->ops, tgt_info.tgt_addr, 0, 0);
+ if (err)
+ goto out_free;
+
if (nr_args < tgt_info.fmodel.nr_args)
nr_args = tgt_info.fmodel.nr_args;
}
Adding struct ftrace_ops object to bpf_trampoline_multi struct and setting it up with all the requested function addresses. Adding is_multi_trampoline(tr) hooks to installing functions to actually install multiple bpf trampoline via ftrace_ops. I had to add -DCC_USING_FENTRY to several places because arch/x86/include/asm/ftrace.h would fail the compilation if it's not defined. Perhaps there's a better way. Signed-off-by: Jiri Olsa <jolsa@kernel.org> --- arch/x86/Makefile | 7 +++++ arch/x86/boot/compressed/Makefile | 4 +++ drivers/firmware/efi/libstub/Makefile | 3 +++ include/linux/bpf.h | 2 ++ kernel/bpf/trampoline.c | 37 +++++++++++++++++++++++++++ 5 files changed, 53 insertions(+)