1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * eBPF JIT compiler
4 *
5 * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
6 * IBM Corporation
7 *
8 * Based on the powerpc classic BPF JIT compiler by Matt Evans
9 */
10#include <linux/moduleloader.h>
11#include <asm/cacheflush.h>
12#include <asm/asm-compat.h>
13#include <linux/netdevice.h>
14#include <linux/filter.h>
15#include <linux/if_vlan.h>
16#include <linux/kernel.h>
17#include <linux/memory.h>
18#include <linux/bpf.h>
19
20#include <asm/kprobes.h>
21#include <asm/code-patching.h>
22
23#include "bpf_jit.h"
24
25static void bpf_jit_fill_ill_insns(void *area, unsigned int size)
26{
27 memset32(s: area, v: BREAKPOINT_INSTRUCTION, n: size / 4);
28}
29
30int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, long exit_addr)
31{
32 if (!exit_addr || is_offset_in_branch_range(exit_addr - (ctx->idx * 4))) {
33 PPC_JMP(exit_addr);
34 } else if (ctx->alt_exit_addr) {
35 if (WARN_ON(!is_offset_in_branch_range((long)ctx->alt_exit_addr - (ctx->idx * 4))))
36 return -1;
37 PPC_JMP(ctx->alt_exit_addr);
38 } else {
39 ctx->alt_exit_addr = ctx->idx * 4;
40 bpf_jit_build_epilogue(image, ctx);
41 }
42
43 return 0;
44}
45
46struct powerpc_jit_data {
47 /* address of rw header */
48 struct bpf_binary_header *hdr;
49 /* address of ro final header */
50 struct bpf_binary_header *fhdr;
51 u32 *addrs;
52 u8 *fimage;
53 u32 proglen;
54 struct codegen_context ctx;
55};
56
57bool bpf_jit_needs_zext(void)
58{
59 return true;
60}
61
62struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
63{
64 u32 proglen;
65 u32 alloclen;
66 u8 *image = NULL;
67 u32 *code_base;
68 u32 *addrs;
69 struct powerpc_jit_data *jit_data;
70 struct codegen_context cgctx;
71 int pass;
72 int flen;
73 struct bpf_binary_header *fhdr = NULL;
74 struct bpf_binary_header *hdr = NULL;
75 struct bpf_prog *org_fp = fp;
76 struct bpf_prog *tmp_fp;
77 bool bpf_blinded = false;
78 bool extra_pass = false;
79 u8 *fimage = NULL;
80 u32 *fcode_base;
81 u32 extable_len;
82 u32 fixup_len;
83
84 if (!fp->jit_requested)
85 return org_fp;
86
87 tmp_fp = bpf_jit_blind_constants(fp: org_fp);
88 if (IS_ERR(ptr: tmp_fp))
89 return org_fp;
90
91 if (tmp_fp != org_fp) {
92 bpf_blinded = true;
93 fp = tmp_fp;
94 }
95
96 jit_data = fp->aux->jit_data;
97 if (!jit_data) {
98 jit_data = kzalloc(size: sizeof(*jit_data), GFP_KERNEL);
99 if (!jit_data) {
100 fp = org_fp;
101 goto out;
102 }
103 fp->aux->jit_data = jit_data;
104 }
105
106 flen = fp->len;
107 addrs = jit_data->addrs;
108 if (addrs) {
109 cgctx = jit_data->ctx;
110 /*
111 * JIT compiled to a writable location (image/code_base) first.
112 * It is then moved to the readonly final location (fimage/fcode_base)
113 * using instruction patching.
114 */
115 fimage = jit_data->fimage;
116 fhdr = jit_data->fhdr;
117 proglen = jit_data->proglen;
118 hdr = jit_data->hdr;
119 image = (void *)hdr + ((void *)fimage - (void *)fhdr);
120 extra_pass = true;
121 /* During extra pass, ensure index is reset before repopulating extable entries */
122 cgctx.exentry_idx = 0;
123 goto skip_init_ctx;
124 }
125
126 addrs = kcalloc(n: flen + 1, size: sizeof(*addrs), GFP_KERNEL);
127 if (addrs == NULL) {
128 fp = org_fp;
129 goto out_addrs;
130 }
131
132 memset(&cgctx, 0, sizeof(struct codegen_context));
133 bpf_jit_init_reg_mapping(ctx: &cgctx);
134
135 /* Make sure that the stack is quadword aligned. */
136 cgctx.stack_size = round_up(fp->aux->stack_depth, 16);
137
138 /* Scouting faux-generate pass 0 */
139 if (bpf_jit_build_body(fp, NULL, NULL, ctx: &cgctx, addrs, pass: 0, extra_pass: false)) {
140 /* We hit something illegal or unsupported. */
141 fp = org_fp;
142 goto out_addrs;
143 }
144
145 /*
146 * If we have seen a tail call, we need a second pass.
147 * This is because bpf_jit_emit_common_epilogue() is called
148 * from bpf_jit_emit_tail_call() with a not yet stable ctx->seen.
149 * We also need a second pass if we ended up with too large
150 * a program so as to ensure BPF_EXIT branches are in range.
151 */
152 if (cgctx.seen & SEEN_TAILCALL || !is_offset_in_branch_range((long)cgctx.idx * 4)) {
153 cgctx.idx = 0;
154 if (bpf_jit_build_body(fp, NULL, NULL, ctx: &cgctx, addrs, pass: 0, extra_pass: false)) {
155 fp = org_fp;
156 goto out_addrs;
157 }
158 }
159
160 bpf_jit_realloc_regs(ctx: &cgctx);
161 /*
162 * Pretend to build prologue, given the features we've seen. This will
163 * update ctgtx.idx as it pretends to output instructions, then we can
164 * calculate total size from idx.
165 */
166 bpf_jit_build_prologue(NULL, ctx: &cgctx);
167 addrs[fp->len] = cgctx.idx * 4;
168 bpf_jit_build_epilogue(NULL, ctx: &cgctx);
169
170 fixup_len = fp->aux->num_exentries * BPF_FIXUP_LEN * 4;
171 extable_len = fp->aux->num_exentries * sizeof(struct exception_table_entry);
172
173 proglen = cgctx.idx * 4;
174 alloclen = proglen + FUNCTION_DESCR_SIZE + fixup_len + extable_len;
175
176 fhdr = bpf_jit_binary_pack_alloc(proglen: alloclen, ro_image: &fimage, alignment: 4, rw_hdr: &hdr, rw_image: &image,
177 bpf_fill_ill_insns: bpf_jit_fill_ill_insns);
178 if (!fhdr) {
179 fp = org_fp;
180 goto out_addrs;
181 }
182
183 if (extable_len)
184 fp->aux->extable = (void *)fimage + FUNCTION_DESCR_SIZE + proglen + fixup_len;
185
186skip_init_ctx:
187 code_base = (u32 *)(image + FUNCTION_DESCR_SIZE);
188 fcode_base = (u32 *)(fimage + FUNCTION_DESCR_SIZE);
189
190 /* Code generation passes 1-2 */
191 for (pass = 1; pass < 3; pass++) {
192 /* Now build the prologue, body code & epilogue for real. */
193 cgctx.idx = 0;
194 cgctx.alt_exit_addr = 0;
195 bpf_jit_build_prologue(image: code_base, ctx: &cgctx);
196 if (bpf_jit_build_body(fp, image: code_base, fimage: fcode_base, ctx: &cgctx, addrs, pass,
197 extra_pass)) {
198 bpf_arch_text_copy(dst: &fhdr->size, src: &hdr->size, len: sizeof(hdr->size));
199 bpf_jit_binary_pack_free(ro_header: fhdr, rw_header: hdr);
200 fp = org_fp;
201 goto out_addrs;
202 }
203 bpf_jit_build_epilogue(image: code_base, ctx: &cgctx);
204
205 if (bpf_jit_enable > 1)
206 pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass,
207 proglen - (cgctx.idx * 4), cgctx.seen);
208 }
209
210 if (bpf_jit_enable > 1)
211 /*
212 * Note that we output the base address of the code_base
213 * rather than image, since opcodes are in code_base.
214 */
215 bpf_jit_dump(flen, proglen, pass, image: code_base);
216
217#ifdef CONFIG_PPC64_ELF_ABI_V1
218 /* Function descriptor nastiness: Address + TOC */
219 ((u64 *)image)[0] = (u64)fcode_base;
220 ((u64 *)image)[1] = local_paca->kernel_toc;
221#endif
222
223 fp->bpf_func = (void *)fimage;
224 fp->jited = 1;
225 fp->jited_len = proglen + FUNCTION_DESCR_SIZE;
226
227 if (!fp->is_func || extra_pass) {
228 if (bpf_jit_binary_pack_finalize(prog: fp, ro_header: fhdr, rw_header: hdr)) {
229 fp = org_fp;
230 goto out_addrs;
231 }
232 bpf_prog_fill_jited_linfo(prog: fp, insn_to_jit_off: addrs);
233out_addrs:
234 kfree(objp: addrs);
235 kfree(objp: jit_data);
236 fp->aux->jit_data = NULL;
237 } else {
238 jit_data->addrs = addrs;
239 jit_data->ctx = cgctx;
240 jit_data->proglen = proglen;
241 jit_data->fimage = fimage;
242 jit_data->fhdr = fhdr;
243 jit_data->hdr = hdr;
244 }
245
246out:
247 if (bpf_blinded)
248 bpf_jit_prog_release_other(fp, fp_other: fp == org_fp ? tmp_fp : org_fp);
249
250 return fp;
251}
252
253/*
254 * The caller should check for (BPF_MODE(code) == BPF_PROBE_MEM) before calling
255 * this function, as this only applies to BPF_PROBE_MEM, for now.
256 */
257int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, u32 *fimage, int pass,
258 struct codegen_context *ctx, int insn_idx, int jmp_off,
259 int dst_reg)
260{
261 off_t offset;
262 unsigned long pc;
263 struct exception_table_entry *ex, *ex_entry;
264 u32 *fixup;
265
266 /* Populate extable entries only in the last pass */
267 if (pass != 2)
268 return 0;
269
270 if (!fp->aux->extable ||
271 WARN_ON_ONCE(ctx->exentry_idx >= fp->aux->num_exentries))
272 return -EINVAL;
273
274 /*
275 * Program is first written to image before copying to the
276 * final location (fimage). Accordingly, update in the image first.
277 * As all offsets used are relative, copying as is to the
278 * final location should be alright.
279 */
280 pc = (unsigned long)&image[insn_idx];
281 ex = (void *)fp->aux->extable - (void *)fimage + (void *)image;
282
283 fixup = (void *)ex -
284 (fp->aux->num_exentries * BPF_FIXUP_LEN * 4) +
285 (ctx->exentry_idx * BPF_FIXUP_LEN * 4);
286
287 fixup[0] = PPC_RAW_LI(dst_reg, 0);
288 if (IS_ENABLED(CONFIG_PPC32))
289 fixup[1] = PPC_RAW_LI(dst_reg - 1, 0); /* clear higher 32-bit register too */
290
291 fixup[BPF_FIXUP_LEN - 1] =
292 PPC_RAW_BRANCH((long)(pc + jmp_off) - (long)&fixup[BPF_FIXUP_LEN - 1]);
293
294 ex_entry = &ex[ctx->exentry_idx];
295
296 offset = pc - (long)&ex_entry->insn;
297 if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
298 return -ERANGE;
299 ex_entry->insn = offset;
300
301 offset = (long)fixup - (long)&ex_entry->fixup;
302 if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
303 return -ERANGE;
304 ex_entry->fixup = offset;
305
306 ctx->exentry_idx++;
307 return 0;
308}
309
310void *bpf_arch_text_copy(void *dst, void *src, size_t len)
311{
312 int err;
313
314 if (WARN_ON_ONCE(core_kernel_text((unsigned long)dst)))
315 return ERR_PTR(error: -EINVAL);
316
317 mutex_lock(&text_mutex);
318 err = patch_instructions(dst, src, len, false);
319 mutex_unlock(lock: &text_mutex);
320
321 return err ? ERR_PTR(error: err) : dst;
322}
323
324int bpf_arch_text_invalidate(void *dst, size_t len)
325{
326 u32 insn = BREAKPOINT_INSTRUCTION;
327 int ret;
328
329 if (WARN_ON_ONCE(core_kernel_text((unsigned long)dst)))
330 return -EINVAL;
331
332 mutex_lock(&text_mutex);
333 ret = patch_instructions(dst, &insn, len, true);
334 mutex_unlock(lock: &text_mutex);
335
336 return ret;
337}
338
339void bpf_jit_free(struct bpf_prog *fp)
340{
341 if (fp->jited) {
342 struct powerpc_jit_data *jit_data = fp->aux->jit_data;
343 struct bpf_binary_header *hdr;
344
345 /*
346 * If we fail the final pass of JIT (from jit_subprogs),
347 * the program may not be finalized yet. Call finalize here
348 * before freeing it.
349 */
350 if (jit_data) {
351 bpf_jit_binary_pack_finalize(prog: fp, ro_header: jit_data->fhdr, rw_header: jit_data->hdr);
352 kvfree(addr: jit_data->addrs);
353 kfree(objp: jit_data);
354 }
355 hdr = bpf_jit_binary_pack_hdr(fp);
356 bpf_jit_binary_pack_free(ro_header: hdr, NULL);
357 WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(fp));
358 }
359
360 bpf_prog_unlock_free(fp);
361}
362

source code of linux/arch/powerpc/net/bpf_jit_comp.c