1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* Copyright(c) 2019 Intel Corporation. */ |
3 | |
4 | #include <linux/hash.h> |
5 | #include <linux/bpf.h> |
6 | #include <linux/filter.h> |
7 | #include <linux/static_call.h> |
8 | |
9 | /* The BPF dispatcher is a multiway branch code generator. The |
10 | * dispatcher is a mechanism to avoid the performance penalty of an |
11 | * indirect call, which is expensive when retpolines are enabled. A |
12 | * dispatch client registers a BPF program into the dispatcher, and if |
13 | * there is available room in the dispatcher a direct call to the BPF |
14 | * program will be generated. All calls to the BPF programs called via |
15 | * the dispatcher will then be a direct call, instead of an |
16 | * indirect. The dispatcher hijacks a trampoline function it via the |
17 | * __fentry__ of the trampoline. The trampoline function has the |
18 | * following signature: |
19 | * |
20 | * unsigned int trampoline(const void *ctx, const struct bpf_insn *insnsi, |
21 | * unsigned int (*bpf_func)(const void *, |
22 | * const struct bpf_insn *)); |
23 | */ |
24 | |
25 | static struct bpf_dispatcher_prog *bpf_dispatcher_find_prog( |
26 | struct bpf_dispatcher *d, struct bpf_prog *prog) |
27 | { |
28 | int i; |
29 | |
30 | for (i = 0; i < BPF_DISPATCHER_MAX; i++) { |
31 | if (prog == d->progs[i].prog) |
32 | return &d->progs[i]; |
33 | } |
34 | return NULL; |
35 | } |
36 | |
37 | static struct bpf_dispatcher_prog *bpf_dispatcher_find_free( |
38 | struct bpf_dispatcher *d) |
39 | { |
40 | return bpf_dispatcher_find_prog(d, NULL); |
41 | } |
42 | |
43 | static bool bpf_dispatcher_add_prog(struct bpf_dispatcher *d, |
44 | struct bpf_prog *prog) |
45 | { |
46 | struct bpf_dispatcher_prog *entry; |
47 | |
48 | if (!prog) |
49 | return false; |
50 | |
51 | entry = bpf_dispatcher_find_prog(d, prog); |
52 | if (entry) { |
53 | refcount_inc(r: &entry->users); |
54 | return false; |
55 | } |
56 | |
57 | entry = bpf_dispatcher_find_free(d); |
58 | if (!entry) |
59 | return false; |
60 | |
61 | bpf_prog_inc(prog); |
62 | entry->prog = prog; |
63 | refcount_set(r: &entry->users, n: 1); |
64 | d->num_progs++; |
65 | return true; |
66 | } |
67 | |
68 | static bool bpf_dispatcher_remove_prog(struct bpf_dispatcher *d, |
69 | struct bpf_prog *prog) |
70 | { |
71 | struct bpf_dispatcher_prog *entry; |
72 | |
73 | if (!prog) |
74 | return false; |
75 | |
76 | entry = bpf_dispatcher_find_prog(d, prog); |
77 | if (!entry) |
78 | return false; |
79 | |
80 | if (refcount_dec_and_test(r: &entry->users)) { |
81 | entry->prog = NULL; |
82 | bpf_prog_put(prog); |
83 | d->num_progs--; |
84 | return true; |
85 | } |
86 | return false; |
87 | } |
88 | |
89 | int __weak arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs) |
90 | { |
91 | return -ENOTSUPP; |
92 | } |
93 | |
94 | static int bpf_dispatcher_prepare(struct bpf_dispatcher *d, void *image, void *buf) |
95 | { |
96 | s64 ips[BPF_DISPATCHER_MAX] = {}, *ipsp = &ips[0]; |
97 | int i; |
98 | |
99 | for (i = 0; i < BPF_DISPATCHER_MAX; i++) { |
100 | if (d->progs[i].prog) |
101 | *ipsp++ = (s64)(uintptr_t)d->progs[i].prog->bpf_func; |
102 | } |
103 | return arch_prepare_bpf_dispatcher(image, buf, funcs: &ips[0], num_funcs: d->num_progs); |
104 | } |
105 | |
106 | static void bpf_dispatcher_update(struct bpf_dispatcher *d, int prev_num_progs) |
107 | { |
108 | void *new, *tmp; |
109 | u32 noff = 0; |
110 | |
111 | if (prev_num_progs) |
112 | noff = d->image_off ^ (PAGE_SIZE / 2); |
113 | |
114 | new = d->num_progs ? d->image + noff : NULL; |
115 | tmp = d->num_progs ? d->rw_image + noff : NULL; |
116 | if (new) { |
117 | /* Prepare the dispatcher in d->rw_image. Then use |
118 | * bpf_arch_text_copy to update d->image, which is RO+X. |
119 | */ |
120 | if (bpf_dispatcher_prepare(d, image: new, buf: tmp)) |
121 | return; |
122 | if (IS_ERR(ptr: bpf_arch_text_copy(dst: new, src: tmp, PAGE_SIZE / 2))) |
123 | return; |
124 | } |
125 | |
126 | __BPF_DISPATCHER_UPDATE(d, new ?: (void *)&bpf_dispatcher_nop_func); |
127 | |
128 | /* Make sure all the callers executing the previous/old half of the |
129 | * image leave it, so following update call can modify it safely. |
130 | */ |
131 | synchronize_rcu(); |
132 | |
133 | if (new) |
134 | d->image_off = noff; |
135 | } |
136 | |
137 | void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, |
138 | struct bpf_prog *to) |
139 | { |
140 | bool changed = false; |
141 | int prev_num_progs; |
142 | |
143 | if (from == to) |
144 | return; |
145 | |
146 | mutex_lock(&d->mutex); |
147 | if (!d->image) { |
148 | d->image = bpf_prog_pack_alloc(PAGE_SIZE, bpf_fill_ill_insns: bpf_jit_fill_hole_with_zero); |
149 | if (!d->image) |
150 | goto out; |
151 | d->rw_image = bpf_jit_alloc_exec(PAGE_SIZE); |
152 | if (!d->rw_image) { |
153 | bpf_prog_pack_free(ptr: d->image, PAGE_SIZE); |
154 | d->image = NULL; |
155 | goto out; |
156 | } |
157 | bpf_image_ksym_add(data: d->image, PAGE_SIZE, ksym: &d->ksym); |
158 | } |
159 | |
160 | prev_num_progs = d->num_progs; |
161 | changed |= bpf_dispatcher_remove_prog(d, prog: from); |
162 | changed |= bpf_dispatcher_add_prog(d, prog: to); |
163 | |
164 | if (!changed) |
165 | goto out; |
166 | |
167 | bpf_dispatcher_update(d, prev_num_progs); |
168 | out: |
169 | mutex_unlock(lock: &d->mutex); |
170 | } |
171 | |