1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * eBPF JIT compiler |
4 | * |
5 | * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com> |
6 | * IBM Corporation |
7 | * |
8 | * Based on the powerpc classic BPF JIT compiler by Matt Evans |
9 | */ |
10 | #include <linux/moduleloader.h> |
11 | #include <asm/cacheflush.h> |
12 | #include <asm/asm-compat.h> |
13 | #include <linux/netdevice.h> |
14 | #include <linux/filter.h> |
15 | #include <linux/if_vlan.h> |
16 | #include <linux/kernel.h> |
17 | #include <linux/memory.h> |
18 | #include <linux/bpf.h> |
19 | |
20 | #include <asm/kprobes.h> |
21 | #include <asm/code-patching.h> |
22 | |
23 | #include "bpf_jit.h" |
24 | |
25 | static void bpf_jit_fill_ill_insns(void *area, unsigned int size) |
26 | { |
27 | memset32(s: area, v: BREAKPOINT_INSTRUCTION, n: size / 4); |
28 | } |
29 | |
30 | int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, long exit_addr) |
31 | { |
32 | if (!exit_addr || is_offset_in_branch_range(exit_addr - (ctx->idx * 4))) { |
33 | PPC_JMP(exit_addr); |
34 | } else if (ctx->alt_exit_addr) { |
35 | if (WARN_ON(!is_offset_in_branch_range((long)ctx->alt_exit_addr - (ctx->idx * 4)))) |
36 | return -1; |
37 | PPC_JMP(ctx->alt_exit_addr); |
38 | } else { |
39 | ctx->alt_exit_addr = ctx->idx * 4; |
40 | bpf_jit_build_epilogue(image, ctx); |
41 | } |
42 | |
43 | return 0; |
44 | } |
45 | |
46 | struct powerpc_jit_data { |
47 | /* address of rw header */ |
48 | struct bpf_binary_header *hdr; |
49 | /* address of ro final header */ |
50 | struct bpf_binary_header *fhdr; |
51 | u32 *addrs; |
52 | u8 *fimage; |
53 | u32 proglen; |
54 | struct codegen_context ctx; |
55 | }; |
56 | |
57 | bool bpf_jit_needs_zext(void) |
58 | { |
59 | return true; |
60 | } |
61 | |
62 | struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) |
63 | { |
64 | u32 proglen; |
65 | u32 alloclen; |
66 | u8 *image = NULL; |
67 | u32 *code_base; |
68 | u32 *addrs; |
69 | struct powerpc_jit_data *jit_data; |
70 | struct codegen_context cgctx; |
71 | int pass; |
72 | int flen; |
73 | struct bpf_binary_header *fhdr = NULL; |
74 | struct bpf_binary_header *hdr = NULL; |
75 | struct bpf_prog *org_fp = fp; |
76 | struct bpf_prog *tmp_fp; |
77 | bool bpf_blinded = false; |
78 | bool = false; |
79 | u8 *fimage = NULL; |
80 | u32 *fcode_base; |
81 | u32 extable_len; |
82 | u32 fixup_len; |
83 | |
84 | if (!fp->jit_requested) |
85 | return org_fp; |
86 | |
87 | tmp_fp = bpf_jit_blind_constants(fp: org_fp); |
88 | if (IS_ERR(ptr: tmp_fp)) |
89 | return org_fp; |
90 | |
91 | if (tmp_fp != org_fp) { |
92 | bpf_blinded = true; |
93 | fp = tmp_fp; |
94 | } |
95 | |
96 | jit_data = fp->aux->jit_data; |
97 | if (!jit_data) { |
98 | jit_data = kzalloc(size: sizeof(*jit_data), GFP_KERNEL); |
99 | if (!jit_data) { |
100 | fp = org_fp; |
101 | goto out; |
102 | } |
103 | fp->aux->jit_data = jit_data; |
104 | } |
105 | |
106 | flen = fp->len; |
107 | addrs = jit_data->addrs; |
108 | if (addrs) { |
109 | cgctx = jit_data->ctx; |
110 | /* |
111 | * JIT compiled to a writable location (image/code_base) first. |
112 | * It is then moved to the readonly final location (fimage/fcode_base) |
113 | * using instruction patching. |
114 | */ |
115 | fimage = jit_data->fimage; |
116 | fhdr = jit_data->fhdr; |
117 | proglen = jit_data->proglen; |
118 | hdr = jit_data->hdr; |
119 | image = (void *)hdr + ((void *)fimage - (void *)fhdr); |
120 | extra_pass = true; |
121 | /* During extra pass, ensure index is reset before repopulating extable entries */ |
122 | cgctx.exentry_idx = 0; |
123 | goto skip_init_ctx; |
124 | } |
125 | |
126 | addrs = kcalloc(n: flen + 1, size: sizeof(*addrs), GFP_KERNEL); |
127 | if (addrs == NULL) { |
128 | fp = org_fp; |
129 | goto out_addrs; |
130 | } |
131 | |
132 | memset(&cgctx, 0, sizeof(struct codegen_context)); |
133 | bpf_jit_init_reg_mapping(ctx: &cgctx); |
134 | |
135 | /* Make sure that the stack is quadword aligned. */ |
136 | cgctx.stack_size = round_up(fp->aux->stack_depth, 16); |
137 | |
138 | /* Scouting faux-generate pass 0 */ |
139 | if (bpf_jit_build_body(fp, NULL, NULL, ctx: &cgctx, addrs, pass: 0, extra_pass: false)) { |
140 | /* We hit something illegal or unsupported. */ |
141 | fp = org_fp; |
142 | goto out_addrs; |
143 | } |
144 | |
145 | /* |
146 | * If we have seen a tail call, we need a second pass. |
147 | * This is because bpf_jit_emit_common_epilogue() is called |
148 | * from bpf_jit_emit_tail_call() with a not yet stable ctx->seen. |
149 | * We also need a second pass if we ended up with too large |
150 | * a program so as to ensure BPF_EXIT branches are in range. |
151 | */ |
152 | if (cgctx.seen & SEEN_TAILCALL || !is_offset_in_branch_range((long)cgctx.idx * 4)) { |
153 | cgctx.idx = 0; |
154 | if (bpf_jit_build_body(fp, NULL, NULL, ctx: &cgctx, addrs, pass: 0, extra_pass: false)) { |
155 | fp = org_fp; |
156 | goto out_addrs; |
157 | } |
158 | } |
159 | |
160 | bpf_jit_realloc_regs(ctx: &cgctx); |
161 | /* |
162 | * Pretend to build prologue, given the features we've seen. This will |
163 | * update ctgtx.idx as it pretends to output instructions, then we can |
164 | * calculate total size from idx. |
165 | */ |
166 | bpf_jit_build_prologue(NULL, ctx: &cgctx); |
167 | addrs[fp->len] = cgctx.idx * 4; |
168 | bpf_jit_build_epilogue(NULL, ctx: &cgctx); |
169 | |
170 | fixup_len = fp->aux->num_exentries * BPF_FIXUP_LEN * 4; |
171 | extable_len = fp->aux->num_exentries * sizeof(struct exception_table_entry); |
172 | |
173 | proglen = cgctx.idx * 4; |
174 | alloclen = proglen + FUNCTION_DESCR_SIZE + fixup_len + extable_len; |
175 | |
176 | fhdr = bpf_jit_binary_pack_alloc(proglen: alloclen, ro_image: &fimage, alignment: 4, rw_hdr: &hdr, rw_image: &image, |
177 | bpf_fill_ill_insns: bpf_jit_fill_ill_insns); |
178 | if (!fhdr) { |
179 | fp = org_fp; |
180 | goto out_addrs; |
181 | } |
182 | |
183 | if (extable_len) |
184 | fp->aux->extable = (void *)fimage + FUNCTION_DESCR_SIZE + proglen + fixup_len; |
185 | |
186 | skip_init_ctx: |
187 | code_base = (u32 *)(image + FUNCTION_DESCR_SIZE); |
188 | fcode_base = (u32 *)(fimage + FUNCTION_DESCR_SIZE); |
189 | |
190 | /* Code generation passes 1-2 */ |
191 | for (pass = 1; pass < 3; pass++) { |
192 | /* Now build the prologue, body code & epilogue for real. */ |
193 | cgctx.idx = 0; |
194 | cgctx.alt_exit_addr = 0; |
195 | bpf_jit_build_prologue(image: code_base, ctx: &cgctx); |
196 | if (bpf_jit_build_body(fp, image: code_base, fimage: fcode_base, ctx: &cgctx, addrs, pass, |
197 | extra_pass)) { |
198 | bpf_arch_text_copy(dst: &fhdr->size, src: &hdr->size, len: sizeof(hdr->size)); |
199 | bpf_jit_binary_pack_free(ro_header: fhdr, rw_header: hdr); |
200 | fp = org_fp; |
201 | goto out_addrs; |
202 | } |
203 | bpf_jit_build_epilogue(image: code_base, ctx: &cgctx); |
204 | |
205 | if (bpf_jit_enable > 1) |
206 | pr_info("Pass %d: shrink = %d, seen = 0x%x\n" , pass, |
207 | proglen - (cgctx.idx * 4), cgctx.seen); |
208 | } |
209 | |
210 | if (bpf_jit_enable > 1) |
211 | /* |
212 | * Note that we output the base address of the code_base |
213 | * rather than image, since opcodes are in code_base. |
214 | */ |
215 | bpf_jit_dump(flen, proglen, pass, image: code_base); |
216 | |
217 | #ifdef CONFIG_PPC64_ELF_ABI_V1 |
218 | /* Function descriptor nastiness: Address + TOC */ |
219 | ((u64 *)image)[0] = (u64)fcode_base; |
220 | ((u64 *)image)[1] = local_paca->kernel_toc; |
221 | #endif |
222 | |
223 | fp->bpf_func = (void *)fimage; |
224 | fp->jited = 1; |
225 | fp->jited_len = proglen + FUNCTION_DESCR_SIZE; |
226 | |
227 | if (!fp->is_func || extra_pass) { |
228 | if (bpf_jit_binary_pack_finalize(prog: fp, ro_header: fhdr, rw_header: hdr)) { |
229 | fp = org_fp; |
230 | goto out_addrs; |
231 | } |
232 | bpf_prog_fill_jited_linfo(prog: fp, insn_to_jit_off: addrs); |
233 | out_addrs: |
234 | kfree(objp: addrs); |
235 | kfree(objp: jit_data); |
236 | fp->aux->jit_data = NULL; |
237 | } else { |
238 | jit_data->addrs = addrs; |
239 | jit_data->ctx = cgctx; |
240 | jit_data->proglen = proglen; |
241 | jit_data->fimage = fimage; |
242 | jit_data->fhdr = fhdr; |
243 | jit_data->hdr = hdr; |
244 | } |
245 | |
246 | out: |
247 | if (bpf_blinded) |
248 | bpf_jit_prog_release_other(fp, fp_other: fp == org_fp ? tmp_fp : org_fp); |
249 | |
250 | return fp; |
251 | } |
252 | |
253 | /* |
254 | * The caller should check for (BPF_MODE(code) == BPF_PROBE_MEM) before calling |
255 | * this function, as this only applies to BPF_PROBE_MEM, for now. |
256 | */ |
257 | int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, u32 *fimage, int pass, |
258 | struct codegen_context *ctx, int insn_idx, int jmp_off, |
259 | int dst_reg) |
260 | { |
261 | off_t offset; |
262 | unsigned long pc; |
263 | struct exception_table_entry *ex, *ex_entry; |
264 | u32 *fixup; |
265 | |
266 | /* Populate extable entries only in the last pass */ |
267 | if (pass != 2) |
268 | return 0; |
269 | |
270 | if (!fp->aux->extable || |
271 | WARN_ON_ONCE(ctx->exentry_idx >= fp->aux->num_exentries)) |
272 | return -EINVAL; |
273 | |
274 | /* |
275 | * Program is first written to image before copying to the |
276 | * final location (fimage). Accordingly, update in the image first. |
277 | * As all offsets used are relative, copying as is to the |
278 | * final location should be alright. |
279 | */ |
280 | pc = (unsigned long)&image[insn_idx]; |
281 | ex = (void *)fp->aux->extable - (void *)fimage + (void *)image; |
282 | |
283 | fixup = (void *)ex - |
284 | (fp->aux->num_exentries * BPF_FIXUP_LEN * 4) + |
285 | (ctx->exentry_idx * BPF_FIXUP_LEN * 4); |
286 | |
287 | fixup[0] = PPC_RAW_LI(dst_reg, 0); |
288 | if (IS_ENABLED(CONFIG_PPC32)) |
289 | fixup[1] = PPC_RAW_LI(dst_reg - 1, 0); /* clear higher 32-bit register too */ |
290 | |
291 | fixup[BPF_FIXUP_LEN - 1] = |
292 | PPC_RAW_BRANCH((long)(pc + jmp_off) - (long)&fixup[BPF_FIXUP_LEN - 1]); |
293 | |
294 | ex_entry = &ex[ctx->exentry_idx]; |
295 | |
296 | offset = pc - (long)&ex_entry->insn; |
297 | if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN)) |
298 | return -ERANGE; |
299 | ex_entry->insn = offset; |
300 | |
301 | offset = (long)fixup - (long)&ex_entry->fixup; |
302 | if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN)) |
303 | return -ERANGE; |
304 | ex_entry->fixup = offset; |
305 | |
306 | ctx->exentry_idx++; |
307 | return 0; |
308 | } |
309 | |
310 | void *bpf_arch_text_copy(void *dst, void *src, size_t len) |
311 | { |
312 | int err; |
313 | |
314 | if (WARN_ON_ONCE(core_kernel_text((unsigned long)dst))) |
315 | return ERR_PTR(error: -EINVAL); |
316 | |
317 | mutex_lock(&text_mutex); |
318 | err = patch_instructions(dst, src, len, false); |
319 | mutex_unlock(lock: &text_mutex); |
320 | |
321 | return err ? ERR_PTR(error: err) : dst; |
322 | } |
323 | |
324 | int bpf_arch_text_invalidate(void *dst, size_t len) |
325 | { |
326 | u32 insn = BREAKPOINT_INSTRUCTION; |
327 | int ret; |
328 | |
329 | if (WARN_ON_ONCE(core_kernel_text((unsigned long)dst))) |
330 | return -EINVAL; |
331 | |
332 | mutex_lock(&text_mutex); |
333 | ret = patch_instructions(dst, &insn, len, true); |
334 | mutex_unlock(lock: &text_mutex); |
335 | |
336 | return ret; |
337 | } |
338 | |
339 | void bpf_jit_free(struct bpf_prog *fp) |
340 | { |
341 | if (fp->jited) { |
342 | struct powerpc_jit_data *jit_data = fp->aux->jit_data; |
343 | struct bpf_binary_header *hdr; |
344 | |
345 | /* |
346 | * If we fail the final pass of JIT (from jit_subprogs), |
347 | * the program may not be finalized yet. Call finalize here |
348 | * before freeing it. |
349 | */ |
350 | if (jit_data) { |
351 | bpf_jit_binary_pack_finalize(prog: fp, ro_header: jit_data->fhdr, rw_header: jit_data->hdr); |
352 | kvfree(addr: jit_data->addrs); |
353 | kfree(objp: jit_data); |
354 | } |
355 | hdr = bpf_jit_binary_pack_hdr(fp); |
356 | bpf_jit_binary_pack_free(ro_header: hdr, NULL); |
357 | WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(fp)); |
358 | } |
359 | |
360 | bpf_prog_unlock_free(fp); |
361 | } |
362 | |