Skip to content

Commit 82e6b1e

Browse files
anakryikoborkmann
anakryiko
authored and
borkmann
committed
bpf: Allow to specify user-provided bpf_cookie for BPF perf links
Add ability for users to specify custom u64 value (bpf_cookie) when creating BPF link for perf_event-backed BPF programs (kprobe/uprobe, perf_event, tracepoints). This is useful for cases when the same BPF program is used for attaching and processing invocation of different tracepoints/kprobes/uprobes in a generic fashion, but such that each invocation is distinguished from each other (e.g., BPF program can look up additional information associated with a specific kernel function without having to rely on function IP lookups). This enables new use cases to be implemented simply and efficiently that previously were possible only through code generation (and thus multiple instances of almost identical BPF program) or compilation at runtime (BCC-style) on target hosts (even more expensive resource-wise). For uprobes it is not even possible in some cases to know function IP before hand (e.g., when attaching to shared library without PID filtering, in which case base load address is not known for a library). This is done by storing u64 bpf_cookie in struct bpf_prog_array_item, corresponding to each attached and run BPF program. Given cgroup BPF programs already use two 8-byte pointers for their needs and cgroup BPF programs don't have (yet?) support for bpf_cookie, reuse that space through union of cgroup_storage and new bpf_cookie field. Make it available to kprobe/tracepoint BPF programs through bpf_trace_run_ctx. This is set by BPF_PROG_RUN_ARRAY, used by kprobe/uprobe/tracepoint BPF program execution code, which luckily is now also split from BPF_PROG_RUN_ARRAY_CG. This run context will be utilized by a new BPF helper giving access to this user-provided cookie value from inside a BPF program. Generic perf_event BPF programs will access this value from perf_event itself through passed in BPF program context. Signed-off-by: Andrii Nakryiko <[email protected]> Signed-off-by: Daniel Borkmann <[email protected]> Acked-by: Yonghong Song <[email protected]> Acked-by: Peter Zijlstra (Intel) <[email protected]> Link: https://lore.kernel.org/bpf/[email protected]
1 parent b89fbfb commit 82e6b1e

File tree

10 files changed

+73
-28
lines changed

10 files changed

+73
-28
lines changed

‎drivers/media/rc/bpf-lirc.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ static int lirc_bpf_attach(struct rc_dev *rcdev, struct bpf_prog *prog)
160160
goto unlock;
161161
}
162162

163-
ret = bpf_prog_array_copy(old_array, NULL, prog, &new_array);
163+
ret = bpf_prog_array_copy(old_array, NULL, prog, 0, &new_array);
164164
if (ret < 0)
165165
goto unlock;
166166

@@ -193,7 +193,7 @@ static int lirc_bpf_detach(struct rc_dev *rcdev, struct bpf_prog *prog)
193193
}
194194

195195
old_array = lirc_rcu_dereference(raw->progs);
196-
ret = bpf_prog_array_copy(old_array, prog, NULL, &new_array);
196+
ret = bpf_prog_array_copy(old_array, prog, NULL, 0, &new_array);
197197
/*
198198
* Do not use bpf_prog_array_delete_safe() as we would end up
199199
* with a dummy entry in the array, and the we would free the

‎include/linux/bpf.h

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1114,7 +1114,10 @@ u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size,
11141114
*/
11151115
struct bpf_prog_array_item {
11161116
struct bpf_prog *prog;
1117-
struct bpf_cgroup_storage *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE];
1117+
union {
1118+
struct bpf_cgroup_storage *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE];
1119+
u64 bpf_cookie;
1120+
};
11181121
};
11191122

11201123
struct bpf_prog_array {
@@ -1140,6 +1143,7 @@ int bpf_prog_array_copy_info(struct bpf_prog_array *array,
11401143
int bpf_prog_array_copy(struct bpf_prog_array *old_array,
11411144
struct bpf_prog *exclude_prog,
11421145
struct bpf_prog *include_prog,
1146+
u64 bpf_cookie,
11431147
struct bpf_prog_array **new_array);
11441148

11451149
struct bpf_run_ctx {};
@@ -1149,6 +1153,11 @@ struct bpf_cg_run_ctx {
11491153
const struct bpf_prog_array_item *prog_item;
11501154
};
11511155

1156+
struct bpf_trace_run_ctx {
1157+
struct bpf_run_ctx run_ctx;
1158+
u64 bpf_cookie;
1159+
};
1160+
11521161
static inline struct bpf_run_ctx *bpf_set_run_ctx(struct bpf_run_ctx *new_ctx)
11531162
{
11541163
struct bpf_run_ctx *old_ctx = NULL;
@@ -1239,18 +1248,23 @@ BPF_PROG_RUN_ARRAY(const struct bpf_prog_array __rcu *array_rcu,
12391248
const struct bpf_prog_array_item *item;
12401249
const struct bpf_prog *prog;
12411250
const struct bpf_prog_array *array;
1251+
struct bpf_run_ctx *old_run_ctx;
1252+
struct bpf_trace_run_ctx run_ctx;
12421253
u32 ret = 1;
12431254

12441255
migrate_disable();
12451256
rcu_read_lock();
12461257
array = rcu_dereference(array_rcu);
12471258
if (unlikely(!array))
12481259
goto out;
1260+
old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
12491261
item = &array->items[0];
12501262
while ((prog = READ_ONCE(item->prog))) {
1263+
run_ctx.bpf_cookie = item->bpf_cookie;
12511264
ret &= run_prog(prog, ctx);
12521265
item++;
12531266
}
1267+
bpf_reset_run_ctx(old_run_ctx);
12541268
out:
12551269
rcu_read_unlock();
12561270
migrate_enable();

‎include/linux/perf_event.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -762,6 +762,7 @@ struct perf_event {
762762
#ifdef CONFIG_BPF_SYSCALL
763763
perf_overflow_handler_t orig_overflow_handler;
764764
struct bpf_prog *prog;
765+
u64 bpf_cookie;
765766
#endif
766767

767768
#ifdef CONFIG_EVENT_TRACING

‎include/linux/trace_events.h

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -675,7 +675,7 @@ trace_trigger_soft_disabled(struct trace_event_file *file)
675675

676676
#ifdef CONFIG_BPF_EVENTS
677677
unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx);
678-
int perf_event_attach_bpf_prog(struct perf_event *event, struct bpf_prog *prog);
678+
int perf_event_attach_bpf_prog(struct perf_event *event, struct bpf_prog *prog, u64 bpf_cookie);
679679
void perf_event_detach_bpf_prog(struct perf_event *event);
680680
int perf_event_query_prog_array(struct perf_event *event, void __user *info);
681681
int bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_prog *prog);
@@ -692,7 +692,7 @@ static inline unsigned int trace_call_bpf(struct trace_event_call *call, void *c
692692
}
693693

694694
static inline int
695-
perf_event_attach_bpf_prog(struct perf_event *event, struct bpf_prog *prog)
695+
perf_event_attach_bpf_prog(struct perf_event *event, struct bpf_prog *prog, u64 bpf_cookie)
696696
{
697697
return -EOPNOTSUPP;
698698
}
@@ -803,7 +803,7 @@ extern void ftrace_profile_free_filter(struct perf_event *event);
803803
void perf_trace_buf_update(void *record, u16 type);
804804
void *perf_trace_buf_alloc(int size, struct pt_regs **regs, int *rctxp);
805805

806-
int perf_event_set_bpf_prog(struct perf_event *event, struct bpf_prog *prog);
806+
int perf_event_set_bpf_prog(struct perf_event *event, struct bpf_prog *prog, u64 bpf_cookie);
807807
void perf_event_free_bpf_prog(struct perf_event *event);
808808

809809
void bpf_trace_run1(struct bpf_prog *prog, u64 arg1);

‎include/uapi/linux/bpf.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1448,6 +1448,13 @@ union bpf_attr {
14481448
__aligned_u64 iter_info; /* extra bpf_iter_link_info */
14491449
__u32 iter_info_len; /* iter_info length */
14501450
};
1451+
struct {
1452+
/* black box user-provided value passed through
1453+
* to BPF program at the execution time and
1454+
* accessible through bpf_get_attach_cookie() BPF helper
1455+
*/
1456+
__u64 bpf_cookie;
1457+
} perf_event;
14511458
};
14521459
} link_create;
14531460

‎kernel/bpf/core.c

Lines changed: 18 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2119,13 +2119,13 @@ int bpf_prog_array_update_at(struct bpf_prog_array *array, int index,
21192119
int bpf_prog_array_copy(struct bpf_prog_array *old_array,
21202120
struct bpf_prog *exclude_prog,
21212121
struct bpf_prog *include_prog,
2122+
u64 bpf_cookie,
21222123
struct bpf_prog_array **new_array)
21232124
{
21242125
int new_prog_cnt, carry_prog_cnt = 0;
2125-
struct bpf_prog_array_item *existing;
2126+
struct bpf_prog_array_item *existing, *new;
21262127
struct bpf_prog_array *array;
21272128
bool found_exclude = false;
2128-
int new_prog_idx = 0;
21292129

21302130
/* Figure out how many existing progs we need to carry over to
21312131
* the new array.
@@ -2162,20 +2162,27 @@ int bpf_prog_array_copy(struct bpf_prog_array *old_array,
21622162
array = bpf_prog_array_alloc(new_prog_cnt + 1, GFP_KERNEL);
21632163
if (!array)
21642164
return -ENOMEM;
2165+
new = array->items;
21652166

21662167
/* Fill in the new prog array */
21672168
if (carry_prog_cnt) {
21682169
existing = old_array->items;
2169-
for (; existing->prog; existing++)
2170-
if (existing->prog != exclude_prog &&
2171-
existing->prog != &dummy_bpf_prog.prog) {
2172-
array->items[new_prog_idx++].prog =
2173-
existing->prog;
2174-
}
2170+
for (; existing->prog; existing++) {
2171+
if (existing->prog == exclude_prog ||
2172+
existing->prog == &dummy_bpf_prog.prog)
2173+
continue;
2174+
2175+
new->prog = existing->prog;
2176+
new->bpf_cookie = existing->bpf_cookie;
2177+
new++;
2178+
}
21752179
}
2176-
if (include_prog)
2177-
array->items[new_prog_idx++].prog = include_prog;
2178-
array->items[new_prog_idx].prog = NULL;
2180+
if (include_prog) {
2181+
new->prog = include_prog;
2182+
new->bpf_cookie = bpf_cookie;
2183+
new++;
2184+
}
2185+
new->prog = NULL;
21792186
*new_array = array;
21802187
return 0;
21812188
}

‎kernel/bpf/syscall.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2963,7 +2963,7 @@ static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *pro
29632963
}
29642964

29652965
event = perf_file->private_data;
2966-
err = perf_event_set_bpf_prog(event, prog);
2966+
err = perf_event_set_bpf_prog(event, prog, attr->link_create.perf_event.bpf_cookie);
29672967
if (err) {
29682968
bpf_link_cleanup(&link_primer);
29692969
goto out_put_file;

‎kernel/events/core.c

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5643,7 +5643,7 @@ static long _perf_ioctl(struct perf_event *event, unsigned int cmd, unsigned lon
56435643
if (IS_ERR(prog))
56445644
return PTR_ERR(prog);
56455645

5646-
err = perf_event_set_bpf_prog(event, prog);
5646+
err = perf_event_set_bpf_prog(event, prog, 0);
56475647
if (err) {
56485648
bpf_prog_put(prog);
56495649
return err;
@@ -9936,7 +9936,9 @@ static void bpf_overflow_handler(struct perf_event *event,
99369936
event->orig_overflow_handler(event, data, regs);
99379937
}
99389938

9939-
static int perf_event_set_bpf_handler(struct perf_event *event, struct bpf_prog *prog)
9939+
static int perf_event_set_bpf_handler(struct perf_event *event,
9940+
struct bpf_prog *prog,
9941+
u64 bpf_cookie)
99409942
{
99419943
if (event->overflow_handler_context)
99429944
/* hw breakpoint or kernel counter */
@@ -9966,6 +9968,7 @@ static int perf_event_set_bpf_handler(struct perf_event *event, struct bpf_prog
99669968
}
99679969

99689970
event->prog = prog;
9971+
event->bpf_cookie = bpf_cookie;
99699972
event->orig_overflow_handler = READ_ONCE(event->overflow_handler);
99709973
WRITE_ONCE(event->overflow_handler, bpf_overflow_handler);
99719974
return 0;
@@ -9983,7 +9986,9 @@ static void perf_event_free_bpf_handler(struct perf_event *event)
99839986
bpf_prog_put(prog);
99849987
}
99859988
#else
9986-
static int perf_event_set_bpf_handler(struct perf_event *event, struct bpf_prog *prog)
9989+
static int perf_event_set_bpf_handler(struct perf_event *event,
9990+
struct bpf_prog *prog,
9991+
u64 bpf_cookie)
99879992
{
99889993
return -EOPNOTSUPP;
99899994
}
@@ -10011,12 +10016,13 @@ static inline bool perf_event_is_tracing(struct perf_event *event)
1001110016
return false;
1001210017
}
1001310018

10014-
int perf_event_set_bpf_prog(struct perf_event *event, struct bpf_prog *prog)
10019+
int perf_event_set_bpf_prog(struct perf_event *event, struct bpf_prog *prog,
10020+
u64 bpf_cookie)
1001510021
{
1001610022
bool is_kprobe, is_tracepoint, is_syscall_tp;
1001710023

1001810024
if (!perf_event_is_tracing(event))
10019-
return perf_event_set_bpf_handler(event, prog);
10025+
return perf_event_set_bpf_handler(event, prog, bpf_cookie);
1002010026

1002110027
is_kprobe = event->tp_event->flags & TRACE_EVENT_FL_UKPROBE;
1002210028
is_tracepoint = event->tp_event->flags & TRACE_EVENT_FL_TRACEPOINT;
@@ -10042,7 +10048,7 @@ int perf_event_set_bpf_prog(struct perf_event *event, struct bpf_prog *prog)
1004210048
return -EACCES;
1004310049
}
1004410050

10045-
return perf_event_attach_bpf_prog(event, prog);
10051+
return perf_event_attach_bpf_prog(event, prog, bpf_cookie);
1004610052
}
1004710053

1004810054
void perf_event_free_bpf_prog(struct perf_event *event)
@@ -10064,7 +10070,8 @@ static void perf_event_free_filter(struct perf_event *event)
1006410070
{
1006510071
}
1006610072

10067-
int perf_event_set_bpf_prog(struct perf_event *event, struct bpf_prog *prog)
10073+
int perf_event_set_bpf_prog(struct perf_event *event, struct bpf_prog *prog,
10074+
u64 bpf_cookie)
1006810075
{
1006910076
return -ENOENT;
1007010077
}

‎kernel/trace/bpf_trace.c

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1675,7 +1675,8 @@ static DEFINE_MUTEX(bpf_event_mutex);
16751675
#define BPF_TRACE_MAX_PROGS 64
16761676

16771677
int perf_event_attach_bpf_prog(struct perf_event *event,
1678-
struct bpf_prog *prog)
1678+
struct bpf_prog *prog,
1679+
u64 bpf_cookie)
16791680
{
16801681
struct bpf_prog_array *old_array;
16811682
struct bpf_prog_array *new_array;
@@ -1702,12 +1703,13 @@ int perf_event_attach_bpf_prog(struct perf_event *event,
17021703
goto unlock;
17031704
}
17041705

1705-
ret = bpf_prog_array_copy(old_array, NULL, prog, &new_array);
1706+
ret = bpf_prog_array_copy(old_array, NULL, prog, bpf_cookie, &new_array);
17061707
if (ret < 0)
17071708
goto unlock;
17081709

17091710
/* set the new array to event->tp_event and set event->prog */
17101711
event->prog = prog;
1712+
event->bpf_cookie = bpf_cookie;
17111713
rcu_assign_pointer(event->tp_event->prog_array, new_array);
17121714
bpf_prog_array_free(old_array);
17131715

@@ -1728,7 +1730,7 @@ void perf_event_detach_bpf_prog(struct perf_event *event)
17281730
goto unlock;
17291731

17301732
old_array = bpf_event_rcu_dereference(event->tp_event->prog_array);
1731-
ret = bpf_prog_array_copy(old_array, event->prog, NULL, &new_array);
1733+
ret = bpf_prog_array_copy(old_array, event->prog, NULL, 0, &new_array);
17321734
if (ret == -ENOENT)
17331735
goto unlock;
17341736
if (ret < 0) {

‎tools/include/uapi/linux/bpf.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1448,6 +1448,13 @@ union bpf_attr {
14481448
__aligned_u64 iter_info; /* extra bpf_iter_link_info */
14491449
__u32 iter_info_len; /* iter_info length */
14501450
};
1451+
struct {
1452+
/* black box user-provided value passed through
1453+
* to BPF program at the execution time and
1454+
* accessible through bpf_get_attach_cookie() BPF helper
1455+
*/
1456+
__u64 bpf_cookie;
1457+
} perf_event;
14511458
};
14521459
} link_create;
14531460

0 commit comments

Comments
 (0)