1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * BPF JIT compiler for PA-RISC (32-bit) |
4 | * |
5 | * Copyright (c) 2023 Helge Deller <deller@gmx.de> |
6 | * |
7 | * The code is based on the BPF JIT compiler for RV64 by Björn Töpel and |
8 | * the BPF JIT compiler for 32-bit ARM by Shubham Bansal and Mircea Gherzan. |
9 | */ |
10 | |
11 | #include <linux/bpf.h> |
12 | #include <linux/filter.h> |
13 | #include <linux/libgcc.h> |
14 | #include "bpf_jit.h" |
15 | |
16 | /* |
17 | * Stack layout during BPF program execution (note: stack grows up): |
18 | * |
19 | * high |
20 | * HPPA32 sp => +----------+ <= HPPA32 fp |
21 | * | saved sp | |
22 | * | saved rp | |
23 | * | ... | HPPA32 callee-saved registers |
24 | * | curr args| |
25 | * | local var| |
26 | * +----------+ <= (sp - 4 * NR_SAVED_REGISTERS) |
27 | * | lo(R9) | |
28 | * | hi(R9) | |
29 | * | lo(FP) | JIT scratch space for BPF registers |
30 | * | hi(FP) | |
31 | * | ... | |
32 | * +----------+ <= (sp - 4 * NR_SAVED_REGISTERS |
33 | * | | - 4 * BPF_JIT_SCRATCH_REGS) |
34 | * | | |
35 | * | ... | BPF program stack |
36 | * | | |
37 | * | ... | Function call stack |
38 | * | | |
39 | * +----------+ |
40 | * low |
41 | */ |
42 | |
43 | enum { |
44 | /* Stack layout - these are offsets from top of JIT scratch space. */ |
45 | BPF_R8_HI, |
46 | BPF_R8_LO, |
47 | BPF_R9_HI, |
48 | BPF_R9_LO, |
49 | BPF_FP_HI, |
50 | BPF_FP_LO, |
51 | BPF_AX_HI, |
52 | BPF_AX_LO, |
53 | BPF_R0_TEMP_HI, |
54 | BPF_R0_TEMP_LO, |
55 | BPF_JIT_SCRATCH_REGS, |
56 | }; |
57 | |
58 | /* Number of callee-saved registers stored to stack: rp, r3-r18. */ |
59 | #define NR_SAVED_REGISTERS (18 - 3 + 1 + 8) |
60 | |
61 | /* Offset from fp for BPF registers stored on stack. */ |
62 | #define STACK_OFFSET(k) (- (NR_SAVED_REGISTERS + k + 1)) |
63 | #define STACK_ALIGN FRAME_SIZE |
64 | |
65 | #define EXIT_PTR_LOAD(reg) hppa_ldw(-0x08, HPPA_REG_SP, reg) |
66 | #define EXIT_PTR_STORE(reg) hppa_stw(reg, -0x08, HPPA_REG_SP) |
67 | #define EXIT_PTR_JUMP(reg, nop) hppa_bv(HPPA_REG_ZERO, reg, nop) |
68 | |
69 | #define TMP_REG_1 (MAX_BPF_JIT_REG + 0) |
70 | #define TMP_REG_2 (MAX_BPF_JIT_REG + 1) |
71 | #define TMP_REG_R0 (MAX_BPF_JIT_REG + 2) |
72 | |
73 | static const s8 regmap[][2] = { |
74 | /* Return value from in-kernel function, and exit value from eBPF. */ |
75 | [BPF_REG_0] = {HPPA_REG_RET0, HPPA_REG_RET1}, /* HI/LOW */ |
76 | |
77 | /* Arguments from eBPF program to in-kernel function. */ |
78 | [BPF_REG_1] = {HPPA_R(3), HPPA_R(4)}, |
79 | [BPF_REG_2] = {HPPA_R(5), HPPA_R(6)}, |
80 | [BPF_REG_3] = {HPPA_R(7), HPPA_R(8)}, |
81 | [BPF_REG_4] = {HPPA_R(9), HPPA_R(10)}, |
82 | [BPF_REG_5] = {HPPA_R(11), HPPA_R(12)}, |
83 | |
84 | [BPF_REG_6] = {HPPA_R(13), HPPA_R(14)}, |
85 | [BPF_REG_7] = {HPPA_R(15), HPPA_R(16)}, |
86 | /* |
87 | * Callee-saved registers that in-kernel function will preserve. |
88 | * Stored on the stack. |
89 | */ |
90 | [BPF_REG_8] = {STACK_OFFSET(BPF_R8_HI), STACK_OFFSET(BPF_R8_LO)}, |
91 | [BPF_REG_9] = {STACK_OFFSET(BPF_R9_HI), STACK_OFFSET(BPF_R9_LO)}, |
92 | |
93 | /* Read-only frame pointer to access BPF stack. Not needed. */ |
94 | [BPF_REG_FP] = {STACK_OFFSET(BPF_FP_HI), STACK_OFFSET(BPF_FP_LO)}, |
95 | |
96 | /* Temporary register for blinding constants. Stored on the stack. */ |
97 | [BPF_REG_AX] = {STACK_OFFSET(BPF_AX_HI), STACK_OFFSET(BPF_AX_LO)}, |
98 | /* |
99 | * Temporary registers used by the JIT to operate on registers stored |
100 | * on the stack. Save t0 and t1 to be used as temporaries in generated |
101 | * code. |
102 | */ |
103 | [TMP_REG_1] = {HPPA_REG_T3, HPPA_REG_T2}, |
104 | [TMP_REG_2] = {HPPA_REG_T5, HPPA_REG_T4}, |
105 | |
106 | /* temporary space for BPF_R0 during libgcc and millicode calls */ |
107 | [TMP_REG_R0] = {STACK_OFFSET(BPF_R0_TEMP_HI), STACK_OFFSET(BPF_R0_TEMP_LO)}, |
108 | }; |
109 | |
110 | static s8 hi(const s8 *r) |
111 | { |
112 | return r[0]; |
113 | } |
114 | |
115 | static s8 lo(const s8 *r) |
116 | { |
117 | return r[1]; |
118 | } |
119 | |
120 | static void emit_hppa_copy(const s8 rs, const s8 rd, struct hppa_jit_context *ctx) |
121 | { |
122 | REG_SET_SEEN(ctx, rd); |
123 | if (OPTIMIZE_HPPA && (rs == rd)) |
124 | return; |
125 | REG_SET_SEEN(ctx, rs); |
126 | emit(hppa_copy(rs, rd), ctx); |
127 | } |
128 | |
129 | static void emit_hppa_xor(const s8 r1, const s8 r2, const s8 r3, struct hppa_jit_context *ctx) |
130 | { |
131 | REG_SET_SEEN(ctx, r1); |
132 | REG_SET_SEEN(ctx, r2); |
133 | REG_SET_SEEN(ctx, r3); |
134 | if (OPTIMIZE_HPPA && (r1 == r2)) { |
135 | emit(hppa_copy(HPPA_REG_ZERO, r3), ctx); |
136 | } else { |
137 | emit(hppa_xor(r1, r2, r3), ctx); |
138 | } |
139 | } |
140 | |
141 | static void emit_imm(const s8 rd, s32 imm, struct hppa_jit_context *ctx) |
142 | { |
143 | u32 lower = im11(imm); |
144 | |
145 | REG_SET_SEEN(ctx, rd); |
146 | if (OPTIMIZE_HPPA && relative_bits_ok(val: imm, bits: 14)) { |
147 | emit(hppa_ldi(imm, rd), ctx); |
148 | return; |
149 | } |
150 | emit(hppa_ldil(imm, rd), ctx); |
151 | if (OPTIMIZE_HPPA && (lower == 0)) |
152 | return; |
153 | emit(hppa_ldo(lower, rd, rd), ctx); |
154 | } |
155 | |
156 | static void emit_imm32(const s8 *rd, s32 imm, struct hppa_jit_context *ctx) |
157 | { |
158 | /* Emit immediate into lower bits. */ |
159 | REG_SET_SEEN(ctx, lo(rd)); |
160 | emit_imm(rd: lo(r: rd), imm, ctx); |
161 | |
162 | /* Sign-extend into upper bits. */ |
163 | REG_SET_SEEN(ctx, hi(rd)); |
164 | if (imm >= 0) |
165 | emit_hppa_copy(rs: HPPA_REG_ZERO, rd: hi(r: rd), ctx); |
166 | else |
167 | emit(hppa_ldi(-1, hi(rd)), ctx); |
168 | } |
169 | |
170 | static void emit_imm64(const s8 *rd, s32 imm_hi, s32 imm_lo, |
171 | struct hppa_jit_context *ctx) |
172 | { |
173 | emit_imm(rd: hi(r: rd), imm: imm_hi, ctx); |
174 | emit_imm(rd: lo(r: rd), imm: imm_lo, ctx); |
175 | } |
176 | |
177 | static void __build_epilogue(bool is_tail_call, struct hppa_jit_context *ctx) |
178 | { |
179 | const s8 *r0 = regmap[BPF_REG_0]; |
180 | int i; |
181 | |
182 | if (is_tail_call) { |
183 | /* |
184 | * goto *(t0 + 4); |
185 | * Skips first instruction of prologue which initializes tail |
186 | * call counter. Assumes t0 contains address of target program, |
187 | * see emit_bpf_tail_call. |
188 | */ |
189 | emit(hppa_ldo(1 * HPPA_INSN_SIZE, HPPA_REG_T0, HPPA_REG_T0), ctx); |
190 | emit(hppa_bv(HPPA_REG_ZERO, HPPA_REG_T0, EXEC_NEXT_INSTR), ctx); |
191 | /* in delay slot: */ |
192 | emit(hppa_copy(HPPA_REG_TCC, HPPA_REG_TCC_IN_INIT), ctx); |
193 | |
194 | return; |
195 | } |
196 | |
197 | /* load epilogue function pointer and jump to it. */ |
198 | /* exit point is either directly below, or the outest TCC exit function */ |
199 | emit(EXIT_PTR_LOAD(HPPA_REG_RP), ctx); |
200 | emit(EXIT_PTR_JUMP(HPPA_REG_RP, NOP_NEXT_INSTR), ctx); |
201 | |
202 | /* NOTE: we are 32-bit and big-endian, so return lower 32-bit value */ |
203 | emit_hppa_copy(rs: lo(r: r0), rd: HPPA_REG_RET0, ctx); |
204 | |
205 | /* Restore callee-saved registers. */ |
206 | for (i = 3; i <= 18; i++) { |
207 | if (OPTIMIZE_HPPA && !REG_WAS_SEEN(ctx, HPPA_R(i))) |
208 | continue; |
209 | emit(hppa_ldw(-REG_SIZE * (8 + (i-3)), HPPA_REG_SP, HPPA_R(i)), ctx); |
210 | } |
211 | |
212 | /* load original return pointer (stored by outest TCC function) */ |
213 | emit(hppa_ldw(-0x14, HPPA_REG_SP, HPPA_REG_RP), ctx); |
214 | emit(hppa_bv(HPPA_REG_ZERO, HPPA_REG_RP, EXEC_NEXT_INSTR), ctx); |
215 | /* in delay slot: */ |
216 | emit(hppa_ldw(-0x04, HPPA_REG_SP, HPPA_REG_SP), ctx); |
217 | } |
218 | |
219 | static bool is_stacked(s8 reg) |
220 | { |
221 | return reg < 0; |
222 | } |
223 | |
224 | static const s8 *bpf_get_reg64_offset(const s8 *reg, const s8 *tmp, |
225 | u16 offset_sp, struct hppa_jit_context *ctx) |
226 | { |
227 | if (is_stacked(reg: hi(r: reg))) { |
228 | emit(hppa_ldw(REG_SIZE * hi(reg) - offset_sp, HPPA_REG_SP, hi(tmp)), ctx); |
229 | emit(hppa_ldw(REG_SIZE * lo(reg) - offset_sp, HPPA_REG_SP, lo(tmp)), ctx); |
230 | reg = tmp; |
231 | } |
232 | REG_SET_SEEN(ctx, hi(reg)); |
233 | REG_SET_SEEN(ctx, lo(reg)); |
234 | return reg; |
235 | } |
236 | |
237 | static const s8 *bpf_get_reg64(const s8 *reg, const s8 *tmp, |
238 | struct hppa_jit_context *ctx) |
239 | { |
240 | return bpf_get_reg64_offset(reg, tmp, offset_sp: 0, ctx); |
241 | } |
242 | |
243 | static const s8 *bpf_get_reg64_ref(const s8 *reg, const s8 *tmp, |
244 | bool must_load, struct hppa_jit_context *ctx) |
245 | { |
246 | if (!OPTIMIZE_HPPA) |
247 | return bpf_get_reg64(reg, tmp, ctx); |
248 | |
249 | if (is_stacked(reg: hi(r: reg))) { |
250 | if (must_load) |
251 | emit(hppa_ldw(REG_SIZE * hi(reg), HPPA_REG_SP, hi(tmp)), ctx); |
252 | reg = tmp; |
253 | } |
254 | REG_SET_SEEN(ctx, hi(reg)); |
255 | REG_SET_SEEN(ctx, lo(reg)); |
256 | return reg; |
257 | } |
258 | |
259 | |
260 | static void bpf_put_reg64(const s8 *reg, const s8 *src, |
261 | struct hppa_jit_context *ctx) |
262 | { |
263 | if (is_stacked(reg: hi(r: reg))) { |
264 | emit(hppa_stw(hi(src), REG_SIZE * hi(reg), HPPA_REG_SP), ctx); |
265 | emit(hppa_stw(lo(src), REG_SIZE * lo(reg), HPPA_REG_SP), ctx); |
266 | } |
267 | } |
268 | |
269 | static void bpf_save_R0(struct hppa_jit_context *ctx) |
270 | { |
271 | bpf_put_reg64(reg: regmap[TMP_REG_R0], src: regmap[BPF_REG_0], ctx); |
272 | } |
273 | |
274 | static void bpf_restore_R0(struct hppa_jit_context *ctx) |
275 | { |
276 | bpf_get_reg64(reg: regmap[TMP_REG_R0], tmp: regmap[BPF_REG_0], ctx); |
277 | } |
278 | |
279 | |
280 | static const s8 *bpf_get_reg32(const s8 *reg, const s8 *tmp, |
281 | struct hppa_jit_context *ctx) |
282 | { |
283 | if (is_stacked(reg: lo(r: reg))) { |
284 | emit(hppa_ldw(REG_SIZE * lo(reg), HPPA_REG_SP, lo(tmp)), ctx); |
285 | reg = tmp; |
286 | } |
287 | REG_SET_SEEN(ctx, lo(reg)); |
288 | return reg; |
289 | } |
290 | |
291 | static const s8 *bpf_get_reg32_ref(const s8 *reg, const s8 *tmp, |
292 | struct hppa_jit_context *ctx) |
293 | { |
294 | if (!OPTIMIZE_HPPA) |
295 | return bpf_get_reg32(reg, tmp, ctx); |
296 | |
297 | if (is_stacked(reg: hi(r: reg))) { |
298 | reg = tmp; |
299 | } |
300 | REG_SET_SEEN(ctx, lo(reg)); |
301 | return reg; |
302 | } |
303 | |
304 | static void bpf_put_reg32(const s8 *reg, const s8 *src, |
305 | struct hppa_jit_context *ctx) |
306 | { |
307 | if (is_stacked(reg: lo(r: reg))) { |
308 | REG_SET_SEEN(ctx, lo(src)); |
309 | emit(hppa_stw(lo(src), REG_SIZE * lo(reg), HPPA_REG_SP), ctx); |
310 | if (1 && !ctx->prog->aux->verifier_zext) { |
311 | REG_SET_SEEN(ctx, hi(reg)); |
312 | emit(hppa_stw(HPPA_REG_ZERO, REG_SIZE * hi(reg), HPPA_REG_SP), ctx); |
313 | } |
314 | } else if (1 && !ctx->prog->aux->verifier_zext) { |
315 | REG_SET_SEEN(ctx, hi(reg)); |
316 | emit_hppa_copy(rs: HPPA_REG_ZERO, rd: hi(r: reg), ctx); |
317 | } |
318 | } |
319 | |
320 | /* extern hppa millicode functions */ |
321 | extern void $$mulI(void); |
322 | extern void $$divU(void); |
323 | extern void $$remU(void); |
324 | |
325 | static void emit_call_millicode(void *func, const s8 arg0, |
326 | const s8 arg1, u8 opcode, struct hppa_jit_context *ctx) |
327 | { |
328 | u32 func_addr; |
329 | |
330 | emit_hppa_copy(rs: arg0, rd: HPPA_REG_ARG0, ctx); |
331 | emit_hppa_copy(rs: arg1, rd: HPPA_REG_ARG1, ctx); |
332 | |
333 | /* libcgcc overwrites HPPA_REG_RET0/1, save temp. in dest. */ |
334 | if (arg0 != HPPA_REG_RET1) |
335 | bpf_save_R0(ctx); |
336 | |
337 | func_addr = (uintptr_t) dereference_function_descriptor(func); |
338 | emit(hppa_ldil(func_addr, HPPA_REG_R31), ctx); |
339 | /* skip the following be_l instruction if divisor is zero. */ |
340 | if (BPF_OP(opcode) == BPF_DIV || BPF_OP(opcode) == BPF_MOD) { |
341 | if (BPF_OP(opcode) == BPF_DIV) |
342 | emit_hppa_copy(rs: HPPA_REG_ZERO, rd: HPPA_REG_RET1, ctx); |
343 | else |
344 | emit_hppa_copy(rs: HPPA_REG_ARG0, rd: HPPA_REG_RET1, ctx); |
345 | emit(hppa_or_cond(HPPA_REG_ARG1, HPPA_REG_ZERO, 1, 0, HPPA_REG_ZERO), ctx); |
346 | } |
347 | /* Note: millicode functions use r31 as return pointer instead of rp */ |
348 | emit(hppa_be_l(im11(func_addr) >> 2, HPPA_REG_R31, NOP_NEXT_INSTR), ctx); |
349 | emit(hppa_nop(), ctx); /* this nop is needed here for delay slot */ |
350 | |
351 | /* Note: millicode functions return result in RET1, not RET0 */ |
352 | emit_hppa_copy(rs: HPPA_REG_RET1, rd: arg0, ctx); |
353 | |
354 | /* restore HPPA_REG_RET0/1, temp. save in dest. */ |
355 | if (arg0 != HPPA_REG_RET1) |
356 | bpf_restore_R0(ctx); |
357 | } |
358 | |
359 | static void emit_call_libgcc_ll(void *func, const s8 *arg0, |
360 | const s8 *arg1, u8 opcode, struct hppa_jit_context *ctx) |
361 | { |
362 | u32 func_addr; |
363 | |
364 | emit_hppa_copy(rs: lo(r: arg0), rd: HPPA_REG_ARG0, ctx); |
365 | emit_hppa_copy(rs: hi(r: arg0), rd: HPPA_REG_ARG1, ctx); |
366 | emit_hppa_copy(rs: lo(r: arg1), rd: HPPA_REG_ARG2, ctx); |
367 | emit_hppa_copy(rs: hi(r: arg1), rd: HPPA_REG_ARG3, ctx); |
368 | |
369 | /* libcgcc overwrites HPPA_REG_RET0/_RET1, so keep copy of R0 on stack */ |
370 | if (hi(r: arg0) != HPPA_REG_RET0) |
371 | bpf_save_R0(ctx); |
372 | |
373 | /* prepare stack */ |
374 | emit(hppa_ldo(2 * FRAME_SIZE, HPPA_REG_SP, HPPA_REG_SP), ctx); |
375 | |
376 | func_addr = (uintptr_t) dereference_function_descriptor(func); |
377 | emit(hppa_ldil(func_addr, HPPA_REG_R31), ctx); |
378 | /* zero out the following be_l instruction if divisor is 0 (and set default values) */ |
379 | if (BPF_OP(opcode) == BPF_DIV || BPF_OP(opcode) == BPF_MOD) { |
380 | emit_hppa_copy(rs: HPPA_REG_ZERO, rd: HPPA_REG_RET0, ctx); |
381 | if (BPF_OP(opcode) == BPF_DIV) |
382 | emit_hppa_copy(rs: HPPA_REG_ZERO, rd: HPPA_REG_RET1, ctx); |
383 | else |
384 | emit_hppa_copy(rs: HPPA_REG_ARG0, rd: HPPA_REG_RET1, ctx); |
385 | emit(hppa_or_cond(HPPA_REG_ARG2, HPPA_REG_ARG3, 1, 0, HPPA_REG_ZERO), ctx); |
386 | } |
387 | emit(hppa_be_l(im11(func_addr) >> 2, HPPA_REG_R31, EXEC_NEXT_INSTR), ctx); |
388 | emit_hppa_copy(rs: HPPA_REG_R31, rd: HPPA_REG_RP, ctx); |
389 | |
390 | /* restore stack */ |
391 | emit(hppa_ldo(-2 * FRAME_SIZE, HPPA_REG_SP, HPPA_REG_SP), ctx); |
392 | |
393 | emit_hppa_copy(rs: HPPA_REG_RET0, rd: hi(r: arg0), ctx); |
394 | emit_hppa_copy(rs: HPPA_REG_RET1, rd: lo(r: arg0), ctx); |
395 | |
396 | /* restore HPPA_REG_RET0/_RET1 */ |
397 | if (hi(r: arg0) != HPPA_REG_RET0) |
398 | bpf_restore_R0(ctx); |
399 | } |
400 | |
401 | static void emit_jump(s32 paoff, bool force_far, |
402 | struct hppa_jit_context *ctx) |
403 | { |
404 | unsigned long pc, addr; |
405 | |
406 | /* Note: allocate 2 instructions for jumps if force_far is set. */ |
407 | if (relative_bits_ok(val: paoff - HPPA_BRANCH_DISPLACEMENT, bits: 17)) { |
408 | /* use BL,short branch followed by nop() */ |
409 | emit(hppa_bl(paoff - HPPA_BRANCH_DISPLACEMENT, HPPA_REG_ZERO), ctx); |
410 | if (force_far) |
411 | emit(hppa_nop(), ctx); |
412 | return; |
413 | } |
414 | |
415 | pc = (uintptr_t) &ctx->insns[ctx->ninsns]; |
416 | addr = pc + (paoff * HPPA_INSN_SIZE); |
417 | emit(hppa_ldil(addr, HPPA_REG_R31), ctx); |
418 | emit(hppa_be_l(im11(addr) >> 2, HPPA_REG_R31, NOP_NEXT_INSTR), ctx); // be,l,n addr(sr4,r31), %sr0, %r31 |
419 | } |
420 | |
421 | static void emit_alu_i64(const s8 *dst, s32 imm, |
422 | struct hppa_jit_context *ctx, const u8 op) |
423 | { |
424 | const s8 *tmp1 = regmap[TMP_REG_1]; |
425 | const s8 *rd; |
426 | |
427 | if (0 && op == BPF_MOV) |
428 | rd = bpf_get_reg64_ref(reg: dst, tmp: tmp1, must_load: false, ctx); |
429 | else |
430 | rd = bpf_get_reg64(reg: dst, tmp: tmp1, ctx); |
431 | |
432 | /* dst = dst OP imm */ |
433 | switch (op) { |
434 | case BPF_MOV: |
435 | emit_imm32(rd, imm, ctx); |
436 | break; |
437 | case BPF_AND: |
438 | emit_imm(rd: HPPA_REG_T0, imm, ctx); |
439 | emit(hppa_and(lo(rd), HPPA_REG_T0, lo(rd)), ctx); |
440 | if (imm >= 0) |
441 | emit_hppa_copy(rs: HPPA_REG_ZERO, rd: hi(r: rd), ctx); |
442 | break; |
443 | case BPF_OR: |
444 | emit_imm(rd: HPPA_REG_T0, imm, ctx); |
445 | emit(hppa_or(lo(rd), HPPA_REG_T0, lo(rd)), ctx); |
446 | if (imm < 0) |
447 | emit_imm(rd: hi(r: rd), imm: -1, ctx); |
448 | break; |
449 | case BPF_XOR: |
450 | emit_imm(rd: HPPA_REG_T0, imm, ctx); |
451 | emit_hppa_xor(r1: lo(r: rd), r2: HPPA_REG_T0, r3: lo(r: rd), ctx); |
452 | if (imm < 0) { |
453 | emit_imm(rd: HPPA_REG_T0, imm: -1, ctx); |
454 | emit_hppa_xor(r1: hi(r: rd), r2: HPPA_REG_T0, r3: hi(r: rd), ctx); |
455 | } |
456 | break; |
457 | case BPF_LSH: |
458 | if (imm == 0) |
459 | break; |
460 | if (imm > 32) { |
461 | imm -= 32; |
462 | emit(hppa_zdep(lo(rd), imm, imm, hi(rd)), ctx); |
463 | emit_hppa_copy(rs: HPPA_REG_ZERO, rd: lo(r: rd), ctx); |
464 | } else if (imm == 32) { |
465 | emit_hppa_copy(rs: lo(r: rd), rd: hi(r: rd), ctx); |
466 | emit_hppa_copy(rs: HPPA_REG_ZERO, rd: lo(r: rd), ctx); |
467 | } else { |
468 | emit(hppa_shd(hi(rd), lo(rd), 32 - imm, hi(rd)), ctx); |
469 | emit(hppa_zdep(lo(rd), imm, imm, lo(rd)), ctx); |
470 | } |
471 | break; |
472 | case BPF_RSH: |
473 | if (imm == 0) |
474 | break; |
475 | if (imm > 32) { |
476 | imm -= 32; |
477 | emit(hppa_shr(hi(rd), imm, lo(rd)), ctx); |
478 | emit_hppa_copy(rs: HPPA_REG_ZERO, rd: hi(r: rd), ctx); |
479 | } else if (imm == 32) { |
480 | emit_hppa_copy(rs: hi(r: rd), rd: lo(r: rd), ctx); |
481 | emit_hppa_copy(rs: HPPA_REG_ZERO, rd: hi(r: rd), ctx); |
482 | } else { |
483 | emit(hppa_shrpw(hi(rd), lo(rd), imm, lo(rd)), ctx); |
484 | emit(hppa_shr(hi(rd), imm, hi(rd)), ctx); |
485 | } |
486 | break; |
487 | case BPF_ARSH: |
488 | if (imm == 0) |
489 | break; |
490 | if (imm > 32) { |
491 | imm -= 32; |
492 | emit(hppa_extrws(hi(rd), 31 - imm, imm, lo(rd)), ctx); |
493 | emit(hppa_extrws(hi(rd), 0, 31, hi(rd)), ctx); |
494 | } else if (imm == 32) { |
495 | emit_hppa_copy(rs: hi(r: rd), rd: lo(r: rd), ctx); |
496 | emit(hppa_extrws(hi(rd), 0, 31, hi(rd)), ctx); |
497 | } else { |
498 | emit(hppa_shrpw(hi(rd), lo(rd), imm, lo(rd)), ctx); |
499 | emit(hppa_extrws(hi(rd), 31 - imm, imm, hi(rd)), ctx); |
500 | } |
501 | break; |
502 | default: |
503 | WARN_ON(1); |
504 | } |
505 | |
506 | bpf_put_reg64(reg: dst, src: rd, ctx); |
507 | } |
508 | |
509 | static void emit_alu_i32(const s8 *dst, s32 imm, |
510 | struct hppa_jit_context *ctx, const u8 op) |
511 | { |
512 | const s8 *tmp1 = regmap[TMP_REG_1]; |
513 | const s8 *rd = bpf_get_reg32(reg: dst, tmp: tmp1, ctx); |
514 | |
515 | if (op == BPF_MOV) |
516 | rd = bpf_get_reg32_ref(reg: dst, tmp: tmp1, ctx); |
517 | else |
518 | rd = bpf_get_reg32(reg: dst, tmp: tmp1, ctx); |
519 | |
520 | /* dst = dst OP imm */ |
521 | switch (op) { |
522 | case BPF_MOV: |
523 | emit_imm(rd: lo(r: rd), imm, ctx); |
524 | break; |
525 | case BPF_ADD: |
526 | emit_imm(rd: HPPA_REG_T0, imm, ctx); |
527 | emit(hppa_add(lo(rd), HPPA_REG_T0, lo(rd)), ctx); |
528 | break; |
529 | case BPF_SUB: |
530 | emit_imm(rd: HPPA_REG_T0, imm, ctx); |
531 | emit(hppa_sub(lo(rd), HPPA_REG_T0, lo(rd)), ctx); |
532 | break; |
533 | case BPF_AND: |
534 | emit_imm(rd: HPPA_REG_T0, imm, ctx); |
535 | emit(hppa_and(lo(rd), HPPA_REG_T0, lo(rd)), ctx); |
536 | break; |
537 | case BPF_OR: |
538 | emit_imm(rd: HPPA_REG_T0, imm, ctx); |
539 | emit(hppa_or(lo(rd), HPPA_REG_T0, lo(rd)), ctx); |
540 | break; |
541 | case BPF_XOR: |
542 | emit_imm(rd: HPPA_REG_T0, imm, ctx); |
543 | emit_hppa_xor(r1: lo(r: rd), r2: HPPA_REG_T0, r3: lo(r: rd), ctx); |
544 | break; |
545 | case BPF_LSH: |
546 | if (imm != 0) |
547 | emit(hppa_zdep(lo(rd), imm, imm, lo(rd)), ctx); |
548 | break; |
549 | case BPF_RSH: |
550 | if (imm != 0) |
551 | emit(hppa_shr(lo(rd), imm, lo(rd)), ctx); |
552 | break; |
553 | case BPF_ARSH: |
554 | if (imm != 0) |
555 | emit(hppa_extrws(lo(rd), 31 - imm, imm, lo(rd)), ctx); |
556 | break; |
557 | default: |
558 | WARN_ON(1); |
559 | } |
560 | |
561 | bpf_put_reg32(reg: dst, src: rd, ctx); |
562 | } |
563 | |
564 | static void emit_alu_r64(const s8 *dst, const s8 *src, |
565 | struct hppa_jit_context *ctx, const u8 op) |
566 | { |
567 | const s8 *tmp1 = regmap[TMP_REG_1]; |
568 | const s8 *tmp2 = regmap[TMP_REG_2]; |
569 | const s8 *rd; |
570 | const s8 *rs = bpf_get_reg64(reg: src, tmp: tmp2, ctx); |
571 | |
572 | if (op == BPF_MOV) |
573 | rd = bpf_get_reg64_ref(reg: dst, tmp: tmp1, must_load: false, ctx); |
574 | else |
575 | rd = bpf_get_reg64(reg: dst, tmp: tmp1, ctx); |
576 | |
577 | /* dst = dst OP src */ |
578 | switch (op) { |
579 | case BPF_MOV: |
580 | emit_hppa_copy(rs: lo(r: rs), rd: lo(r: rd), ctx); |
581 | emit_hppa_copy(rs: hi(r: rs), rd: hi(r: rd), ctx); |
582 | break; |
583 | case BPF_ADD: |
584 | emit(hppa_add(lo(rd), lo(rs), lo(rd)), ctx); |
585 | emit(hppa_addc(hi(rd), hi(rs), hi(rd)), ctx); |
586 | break; |
587 | case BPF_SUB: |
588 | emit(hppa_sub(lo(rd), lo(rs), lo(rd)), ctx); |
589 | emit(hppa_subb(hi(rd), hi(rs), hi(rd)), ctx); |
590 | break; |
591 | case BPF_AND: |
592 | emit(hppa_and(lo(rd), lo(rs), lo(rd)), ctx); |
593 | emit(hppa_and(hi(rd), hi(rs), hi(rd)), ctx); |
594 | break; |
595 | case BPF_OR: |
596 | emit(hppa_or(lo(rd), lo(rs), lo(rd)), ctx); |
597 | emit(hppa_or(hi(rd), hi(rs), hi(rd)), ctx); |
598 | break; |
599 | case BPF_XOR: |
600 | emit_hppa_xor(r1: lo(r: rd), r2: lo(r: rs), r3: lo(r: rd), ctx); |
601 | emit_hppa_xor(r1: hi(r: rd), r2: hi(r: rs), r3: hi(r: rd), ctx); |
602 | break; |
603 | case BPF_MUL: |
604 | emit_call_libgcc_ll(func: __muldi3, arg0: rd, arg1: rs, opcode: op, ctx); |
605 | break; |
606 | case BPF_DIV: |
607 | emit_call_libgcc_ll(func: &hppa_div64, arg0: rd, arg1: rs, opcode: op, ctx); |
608 | break; |
609 | case BPF_MOD: |
610 | emit_call_libgcc_ll(func: &hppa_div64_rem, arg0: rd, arg1: rs, opcode: op, ctx); |
611 | break; |
612 | case BPF_LSH: |
613 | emit_call_libgcc_ll(func: __ashldi3, arg0: rd, arg1: rs, opcode: op, ctx); |
614 | break; |
615 | case BPF_RSH: |
616 | emit_call_libgcc_ll(func: __lshrdi3, arg0: rd, arg1: rs, opcode: op, ctx); |
617 | break; |
618 | case BPF_ARSH: |
619 | emit_call_libgcc_ll(func: __ashrdi3, arg0: rd, arg1: rs, opcode: op, ctx); |
620 | break; |
621 | case BPF_NEG: |
622 | emit(hppa_sub(HPPA_REG_ZERO, lo(rd), lo(rd)), ctx); |
623 | emit(hppa_subb(HPPA_REG_ZERO, hi(rd), hi(rd)), ctx); |
624 | break; |
625 | default: |
626 | WARN_ON(1); |
627 | } |
628 | |
629 | bpf_put_reg64(reg: dst, src: rd, ctx); |
630 | } |
631 | |
632 | static void emit_alu_r32(const s8 *dst, const s8 *src, |
633 | struct hppa_jit_context *ctx, const u8 op) |
634 | { |
635 | const s8 *tmp1 = regmap[TMP_REG_1]; |
636 | const s8 *tmp2 = regmap[TMP_REG_2]; |
637 | const s8 *rd; |
638 | const s8 *rs = bpf_get_reg32(reg: src, tmp: tmp2, ctx); |
639 | |
640 | if (op == BPF_MOV) |
641 | rd = bpf_get_reg32_ref(reg: dst, tmp: tmp1, ctx); |
642 | else |
643 | rd = bpf_get_reg32(reg: dst, tmp: tmp1, ctx); |
644 | |
645 | /* dst = dst OP src */ |
646 | switch (op) { |
647 | case BPF_MOV: |
648 | emit_hppa_copy(rs: lo(r: rs), rd: lo(r: rd), ctx); |
649 | break; |
650 | case BPF_ADD: |
651 | emit(hppa_add(lo(rd), lo(rs), lo(rd)), ctx); |
652 | break; |
653 | case BPF_SUB: |
654 | emit(hppa_sub(lo(rd), lo(rs), lo(rd)), ctx); |
655 | break; |
656 | case BPF_AND: |
657 | emit(hppa_and(lo(rd), lo(rs), lo(rd)), ctx); |
658 | break; |
659 | case BPF_OR: |
660 | emit(hppa_or(lo(rd), lo(rs), lo(rd)), ctx); |
661 | break; |
662 | case BPF_XOR: |
663 | emit_hppa_xor(r1: lo(r: rd), r2: lo(r: rs), r3: lo(r: rd), ctx); |
664 | break; |
665 | case BPF_MUL: |
666 | emit_call_millicode(func: $$mulI, arg0: lo(r: rd), arg1: lo(r: rs), opcode: op, ctx); |
667 | break; |
668 | case BPF_DIV: |
669 | emit_call_millicode(func: $$divU, arg0: lo(r: rd), arg1: lo(r: rs), opcode: op, ctx); |
670 | break; |
671 | case BPF_MOD: |
672 | emit_call_millicode(func: $$remU, arg0: lo(r: rd), arg1: lo(r: rs), opcode: op, ctx); |
673 | break; |
674 | case BPF_LSH: |
675 | emit(hppa_subi(0x1f, lo(rs), HPPA_REG_T0), ctx); |
676 | emit(hppa_mtsar(HPPA_REG_T0), ctx); |
677 | emit(hppa_depwz_sar(lo(rd), lo(rd)), ctx); |
678 | break; |
679 | case BPF_RSH: |
680 | emit(hppa_mtsar(lo(rs)), ctx); |
681 | emit(hppa_shrpw_sar(lo(rd), lo(rd)), ctx); |
682 | break; |
683 | case BPF_ARSH: /* sign extending arithmetic shift right */ |
684 | // emit(hppa_beq(lo(rs), HPPA_REG_ZERO, 2), ctx); |
685 | emit(hppa_subi(0x1f, lo(rs), HPPA_REG_T0), ctx); |
686 | emit(hppa_mtsar(HPPA_REG_T0), ctx); |
687 | emit(hppa_extrws_sar(lo(rd), lo(rd)), ctx); |
688 | break; |
689 | case BPF_NEG: |
690 | emit(hppa_sub(HPPA_REG_ZERO, lo(rd), lo(rd)), ctx); // sub r0,rd,rd |
691 | break; |
692 | default: |
693 | WARN_ON(1); |
694 | } |
695 | |
696 | bpf_put_reg32(reg: dst, src: rd, ctx); |
697 | } |
698 | |
699 | static int emit_branch_r64(const s8 *src1, const s8 *src2, s32 paoff, |
700 | struct hppa_jit_context *ctx, const u8 op) |
701 | { |
702 | int e, s = ctx->ninsns; |
703 | const s8 *tmp1 = regmap[TMP_REG_1]; |
704 | const s8 *tmp2 = regmap[TMP_REG_2]; |
705 | |
706 | const s8 *rs1 = bpf_get_reg64(reg: src1, tmp: tmp1, ctx); |
707 | const s8 *rs2 = bpf_get_reg64(reg: src2, tmp: tmp2, ctx); |
708 | |
709 | /* |
710 | * NO_JUMP skips over the rest of the instructions and the |
711 | * emit_jump, meaning the BPF branch is not taken. |
712 | * JUMP skips directly to the emit_jump, meaning |
713 | * the BPF branch is taken. |
714 | * |
715 | * The fallthrough case results in the BPF branch being taken. |
716 | */ |
717 | #define NO_JUMP(idx) (2 + (idx) - 1) |
718 | #define JUMP(idx) (0 + (idx) - 1) |
719 | |
720 | switch (op) { |
721 | case BPF_JEQ: |
722 | emit(hppa_bne(hi(rs1), hi(rs2), NO_JUMP(1)), ctx); |
723 | emit(hppa_bne(lo(rs1), lo(rs2), NO_JUMP(0)), ctx); |
724 | break; |
725 | case BPF_JGT: |
726 | emit(hppa_bgtu(hi(rs1), hi(rs2), JUMP(2)), ctx); |
727 | emit(hppa_bltu(hi(rs1), hi(rs2), NO_JUMP(1)), ctx); |
728 | emit(hppa_bleu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx); |
729 | break; |
730 | case BPF_JLT: |
731 | emit(hppa_bltu(hi(rs1), hi(rs2), JUMP(2)), ctx); |
732 | emit(hppa_bgtu(hi(rs1), hi(rs2), NO_JUMP(1)), ctx); |
733 | emit(hppa_bgeu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx); |
734 | break; |
735 | case BPF_JGE: |
736 | emit(hppa_bgtu(hi(rs1), hi(rs2), JUMP(2)), ctx); |
737 | emit(hppa_bltu(hi(rs1), hi(rs2), NO_JUMP(1)), ctx); |
738 | emit(hppa_bltu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx); |
739 | break; |
740 | case BPF_JLE: |
741 | emit(hppa_bltu(hi(rs1), hi(rs2), JUMP(2)), ctx); |
742 | emit(hppa_bgtu(hi(rs1), hi(rs2), NO_JUMP(1)), ctx); |
743 | emit(hppa_bgtu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx); |
744 | break; |
745 | case BPF_JNE: |
746 | emit(hppa_bne(hi(rs1), hi(rs2), JUMP(1)), ctx); |
747 | emit(hppa_beq(lo(rs1), lo(rs2), NO_JUMP(0)), ctx); |
748 | break; |
749 | case BPF_JSGT: |
750 | emit(hppa_bgt(hi(rs1), hi(rs2), JUMP(2)), ctx); |
751 | emit(hppa_blt(hi(rs1), hi(rs2), NO_JUMP(1)), ctx); |
752 | emit(hppa_bleu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx); |
753 | break; |
754 | case BPF_JSLT: |
755 | emit(hppa_blt(hi(rs1), hi(rs2), JUMP(2)), ctx); |
756 | emit(hppa_bgt(hi(rs1), hi(rs2), NO_JUMP(1)), ctx); |
757 | emit(hppa_bgeu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx); |
758 | break; |
759 | case BPF_JSGE: |
760 | emit(hppa_bgt(hi(rs1), hi(rs2), JUMP(2)), ctx); |
761 | emit(hppa_blt(hi(rs1), hi(rs2), NO_JUMP(1)), ctx); |
762 | emit(hppa_bltu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx); |
763 | break; |
764 | case BPF_JSLE: |
765 | emit(hppa_blt(hi(rs1), hi(rs2), JUMP(2)), ctx); |
766 | emit(hppa_bgt(hi(rs1), hi(rs2), NO_JUMP(1)), ctx); |
767 | emit(hppa_bgtu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx); |
768 | break; |
769 | case BPF_JSET: |
770 | emit(hppa_and(hi(rs1), hi(rs2), HPPA_REG_T0), ctx); |
771 | emit(hppa_and(lo(rs1), lo(rs2), HPPA_REG_T1), ctx); |
772 | emit(hppa_bne(HPPA_REG_T0, HPPA_REG_ZERO, JUMP(1)), ctx); |
773 | emit(hppa_beq(HPPA_REG_T1, HPPA_REG_ZERO, NO_JUMP(0)), ctx); |
774 | break; |
775 | default: |
776 | WARN_ON(1); |
777 | } |
778 | |
779 | #undef NO_JUMP |
780 | #undef JUMP |
781 | |
782 | e = ctx->ninsns; |
783 | /* Adjust for extra insns. */ |
784 | paoff -= (e - s); |
785 | emit_jump(paoff, force_far: true, ctx); |
786 | return 0; |
787 | } |
788 | |
789 | static int emit_bcc(u8 op, u8 rd, u8 rs, int paoff, struct hppa_jit_context *ctx) |
790 | { |
791 | int e, s; |
792 | bool far = false; |
793 | int off; |
794 | |
795 | if (op == BPF_JSET) { |
796 | /* |
797 | * BPF_JSET is a special case: it has no inverse so we always |
798 | * treat it as a far branch. |
799 | */ |
800 | emit(hppa_and(rd, rs, HPPA_REG_T0), ctx); |
801 | paoff -= 1; /* reduce offset due to hppa_and() above */ |
802 | rd = HPPA_REG_T0; |
803 | rs = HPPA_REG_ZERO; |
804 | op = BPF_JNE; |
805 | } |
806 | |
807 | s = ctx->ninsns; |
808 | |
809 | if (!relative_bits_ok(val: paoff - HPPA_BRANCH_DISPLACEMENT, bits: 12)) { |
810 | op = invert_bpf_cond(cond: op); |
811 | far = true; |
812 | } |
813 | |
814 | /* |
815 | * For a far branch, the condition is negated and we jump over the |
816 | * branch itself, and the three instructions from emit_jump. |
817 | * For a near branch, just use paoff. |
818 | */ |
819 | off = far ? (HPPA_BRANCH_DISPLACEMENT - 1) : paoff - HPPA_BRANCH_DISPLACEMENT; |
820 | |
821 | switch (op) { |
822 | /* IF (dst COND src) JUMP off */ |
823 | case BPF_JEQ: |
824 | emit(hppa_beq(rd, rs, off), ctx); |
825 | break; |
826 | case BPF_JGT: |
827 | emit(hppa_bgtu(rd, rs, off), ctx); |
828 | break; |
829 | case BPF_JLT: |
830 | emit(hppa_bltu(rd, rs, off), ctx); |
831 | break; |
832 | case BPF_JGE: |
833 | emit(hppa_bgeu(rd, rs, off), ctx); |
834 | break; |
835 | case BPF_JLE: |
836 | emit(hppa_bleu(rd, rs, off), ctx); |
837 | break; |
838 | case BPF_JNE: |
839 | emit(hppa_bne(rd, rs, off), ctx); |
840 | break; |
841 | case BPF_JSGT: |
842 | emit(hppa_bgt(rd, rs, off), ctx); |
843 | break; |
844 | case BPF_JSLT: |
845 | emit(hppa_blt(rd, rs, off), ctx); |
846 | break; |
847 | case BPF_JSGE: |
848 | emit(hppa_bge(rd, rs, off), ctx); |
849 | break; |
850 | case BPF_JSLE: |
851 | emit(hppa_ble(rd, rs, off), ctx); |
852 | break; |
853 | default: |
854 | WARN_ON(1); |
855 | } |
856 | |
857 | if (far) { |
858 | e = ctx->ninsns; |
859 | /* Adjust for extra insns. */ |
860 | paoff -= (e - s); |
861 | emit_jump(paoff, force_far: true, ctx); |
862 | } |
863 | return 0; |
864 | } |
865 | |
866 | static int emit_branch_r32(const s8 *src1, const s8 *src2, s32 paoff, |
867 | struct hppa_jit_context *ctx, const u8 op) |
868 | { |
869 | int e, s = ctx->ninsns; |
870 | const s8 *tmp1 = regmap[TMP_REG_1]; |
871 | const s8 *tmp2 = regmap[TMP_REG_2]; |
872 | |
873 | const s8 *rs1 = bpf_get_reg32(reg: src1, tmp: tmp1, ctx); |
874 | const s8 *rs2 = bpf_get_reg32(reg: src2, tmp: tmp2, ctx); |
875 | |
876 | e = ctx->ninsns; |
877 | /* Adjust for extra insns. */ |
878 | paoff -= (e - s); |
879 | |
880 | if (emit_bcc(op, rd: lo(r: rs1), rs: lo(r: rs2), paoff, ctx)) |
881 | return -1; |
882 | |
883 | return 0; |
884 | } |
885 | |
886 | static void emit_call(bool fixed, u64 addr, struct hppa_jit_context *ctx) |
887 | { |
888 | const s8 *tmp = regmap[TMP_REG_1]; |
889 | const s8 *r0 = regmap[BPF_REG_0]; |
890 | const s8 *reg; |
891 | const int offset_sp = 2 * STACK_ALIGN; |
892 | |
893 | /* prepare stack */ |
894 | emit(hppa_ldo(offset_sp, HPPA_REG_SP, HPPA_REG_SP), ctx); |
895 | |
896 | /* load R1 & R2 in registers, R3-R5 to stack. */ |
897 | reg = bpf_get_reg64_offset(reg: regmap[BPF_REG_5], tmp, offset_sp, ctx); |
898 | emit(hppa_stw(hi(reg), -0x48, HPPA_REG_SP), ctx); |
899 | emit(hppa_stw(lo(reg), -0x44, HPPA_REG_SP), ctx); |
900 | |
901 | reg = bpf_get_reg64_offset(reg: regmap[BPF_REG_4], tmp, offset_sp, ctx); |
902 | emit(hppa_stw(hi(reg), -0x40, HPPA_REG_SP), ctx); |
903 | emit(hppa_stw(lo(reg), -0x3c, HPPA_REG_SP), ctx); |
904 | |
905 | reg = bpf_get_reg64_offset(reg: regmap[BPF_REG_3], tmp, offset_sp, ctx); |
906 | emit(hppa_stw(hi(reg), -0x38, HPPA_REG_SP), ctx); |
907 | emit(hppa_stw(lo(reg), -0x34, HPPA_REG_SP), ctx); |
908 | |
909 | reg = bpf_get_reg64_offset(reg: regmap[BPF_REG_2], tmp, offset_sp, ctx); |
910 | emit_hppa_copy(rs: hi(r: reg), rd: HPPA_REG_ARG3, ctx); |
911 | emit_hppa_copy(rs: lo(r: reg), rd: HPPA_REG_ARG2, ctx); |
912 | |
913 | reg = bpf_get_reg64_offset(reg: regmap[BPF_REG_1], tmp, offset_sp, ctx); |
914 | emit_hppa_copy(rs: hi(r: reg), rd: HPPA_REG_ARG1, ctx); |
915 | emit_hppa_copy(rs: lo(r: reg), rd: HPPA_REG_ARG0, ctx); |
916 | |
917 | /* backup TCC */ |
918 | if (REG_WAS_SEEN(ctx, HPPA_REG_TCC)) |
919 | emit(hppa_copy(HPPA_REG_TCC, HPPA_REG_TCC_SAVED), ctx); |
920 | |
921 | /* |
922 | * Use ldil() to load absolute address. Don't use emit_imm as the |
923 | * number of emitted instructions should not depend on the value of |
924 | * addr. |
925 | */ |
926 | emit(hppa_ldil(addr, HPPA_REG_R31), ctx); |
927 | emit(hppa_be_l(im11(addr) >> 2, HPPA_REG_R31, EXEC_NEXT_INSTR), ctx); |
928 | /* set return address in delay slot */ |
929 | emit_hppa_copy(rs: HPPA_REG_R31, rd: HPPA_REG_RP, ctx); |
930 | |
931 | /* restore TCC */ |
932 | if (REG_WAS_SEEN(ctx, HPPA_REG_TCC)) |
933 | emit(hppa_copy(HPPA_REG_TCC_SAVED, HPPA_REG_TCC), ctx); |
934 | |
935 | /* restore stack */ |
936 | emit(hppa_ldo(-offset_sp, HPPA_REG_SP, HPPA_REG_SP), ctx); |
937 | |
938 | /* set return value. */ |
939 | emit_hppa_copy(rs: HPPA_REG_RET0, rd: hi(r: r0), ctx); |
940 | emit_hppa_copy(rs: HPPA_REG_RET1, rd: lo(r: r0), ctx); |
941 | } |
942 | |
943 | static int emit_bpf_tail_call(int insn, struct hppa_jit_context *ctx) |
944 | { |
945 | /* |
946 | * R1 -> &ctx |
947 | * R2 -> &array |
948 | * R3 -> index |
949 | */ |
950 | int off; |
951 | const s8 *arr_reg = regmap[BPF_REG_2]; |
952 | const s8 *idx_reg = regmap[BPF_REG_3]; |
953 | struct bpf_array bpfa; |
954 | struct bpf_prog bpfp; |
955 | |
956 | /* get address of TCC main exit function for error case into rp */ |
957 | emit(EXIT_PTR_LOAD(HPPA_REG_RP), ctx); |
958 | |
959 | /* max_entries = array->map.max_entries; */ |
960 | off = offsetof(struct bpf_array, map.max_entries); |
961 | BUILD_BUG_ON(sizeof(bpfa.map.max_entries) != 4); |
962 | emit(hppa_ldw(off, lo(arr_reg), HPPA_REG_T1), ctx); |
963 | |
964 | /* |
965 | * if (index >= max_entries) |
966 | * goto out; |
967 | */ |
968 | emit(hppa_bltu(lo(idx_reg), HPPA_REG_T1, 2 - HPPA_BRANCH_DISPLACEMENT), ctx); |
969 | emit(EXIT_PTR_JUMP(HPPA_REG_RP, NOP_NEXT_INSTR), ctx); |
970 | |
971 | /* |
972 | * if (--tcc < 0) |
973 | * goto out; |
974 | */ |
975 | REG_FORCE_SEEN(ctx, HPPA_REG_TCC); |
976 | emit(hppa_ldo(-1, HPPA_REG_TCC, HPPA_REG_TCC), ctx); |
977 | emit(hppa_bge(HPPA_REG_TCC, HPPA_REG_ZERO, 2 - HPPA_BRANCH_DISPLACEMENT), ctx); |
978 | emit(EXIT_PTR_JUMP(HPPA_REG_RP, NOP_NEXT_INSTR), ctx); |
979 | |
980 | /* |
981 | * prog = array->ptrs[index]; |
982 | * if (!prog) |
983 | * goto out; |
984 | */ |
985 | BUILD_BUG_ON(sizeof(bpfa.ptrs[0]) != 4); |
986 | emit(hppa_sh2add(lo(idx_reg), lo(arr_reg), HPPA_REG_T0), ctx); |
987 | off = offsetof(struct bpf_array, ptrs); |
988 | BUILD_BUG_ON(!relative_bits_ok(off, 11)); |
989 | emit(hppa_ldw(off, HPPA_REG_T0, HPPA_REG_T0), ctx); |
990 | emit(hppa_bne(HPPA_REG_T0, HPPA_REG_ZERO, 2 - HPPA_BRANCH_DISPLACEMENT), ctx); |
991 | emit(EXIT_PTR_JUMP(HPPA_REG_RP, NOP_NEXT_INSTR), ctx); |
992 | |
993 | /* |
994 | * tcc = temp_tcc; |
995 | * goto *(prog->bpf_func + 4); |
996 | */ |
997 | off = offsetof(struct bpf_prog, bpf_func); |
998 | BUILD_BUG_ON(!relative_bits_ok(off, 11)); |
999 | BUILD_BUG_ON(sizeof(bpfp.bpf_func) != 4); |
1000 | emit(hppa_ldw(off, HPPA_REG_T0, HPPA_REG_T0), ctx); |
1001 | /* Epilogue jumps to *(t0 + 4). */ |
1002 | __build_epilogue(is_tail_call: true, ctx); |
1003 | return 0; |
1004 | } |
1005 | |
1006 | static int emit_load_r64(const s8 *dst, const s8 *src, s16 off, |
1007 | struct hppa_jit_context *ctx, const u8 size) |
1008 | { |
1009 | const s8 *tmp1 = regmap[TMP_REG_1]; |
1010 | const s8 *tmp2 = regmap[TMP_REG_2]; |
1011 | const s8 *rd = bpf_get_reg64_ref(reg: dst, tmp: tmp1, must_load: ctx->prog->aux->verifier_zext, ctx); |
1012 | const s8 *rs = bpf_get_reg64(reg: src, tmp: tmp2, ctx); |
1013 | s8 srcreg; |
1014 | |
1015 | /* need to calculate address since offset does not fit in 14 bits? */ |
1016 | if (relative_bits_ok(val: off, bits: 14)) |
1017 | srcreg = lo(r: rs); |
1018 | else { |
1019 | /* need to use R1 here, since addil puts result into R1 */ |
1020 | srcreg = HPPA_REG_R1; |
1021 | emit(hppa_addil(off, lo(rs)), ctx); |
1022 | off = im11(off); |
1023 | } |
1024 | |
1025 | /* LDX: dst = *(size *)(src + off) */ |
1026 | switch (size) { |
1027 | case BPF_B: |
1028 | emit(hppa_ldb(off + 0, srcreg, lo(rd)), ctx); |
1029 | if (!ctx->prog->aux->verifier_zext) |
1030 | emit_hppa_copy(rs: HPPA_REG_ZERO, rd: hi(r: rd), ctx); |
1031 | break; |
1032 | case BPF_H: |
1033 | emit(hppa_ldh(off + 0, srcreg, lo(rd)), ctx); |
1034 | if (!ctx->prog->aux->verifier_zext) |
1035 | emit_hppa_copy(rs: HPPA_REG_ZERO, rd: hi(r: rd), ctx); |
1036 | break; |
1037 | case BPF_W: |
1038 | emit(hppa_ldw(off + 0, srcreg, lo(rd)), ctx); |
1039 | if (!ctx->prog->aux->verifier_zext) |
1040 | emit_hppa_copy(rs: HPPA_REG_ZERO, rd: hi(r: rd), ctx); |
1041 | break; |
1042 | case BPF_DW: |
1043 | emit(hppa_ldw(off + 0, srcreg, hi(rd)), ctx); |
1044 | emit(hppa_ldw(off + 4, srcreg, lo(rd)), ctx); |
1045 | break; |
1046 | } |
1047 | |
1048 | bpf_put_reg64(reg: dst, src: rd, ctx); |
1049 | return 0; |
1050 | } |
1051 | |
1052 | static int emit_store_r64(const s8 *dst, const s8 *src, s16 off, |
1053 | struct hppa_jit_context *ctx, const u8 size, |
1054 | const u8 mode) |
1055 | { |
1056 | const s8 *tmp1 = regmap[TMP_REG_1]; |
1057 | const s8 *tmp2 = regmap[TMP_REG_2]; |
1058 | const s8 *rd = bpf_get_reg64(reg: dst, tmp: tmp1, ctx); |
1059 | const s8 *rs = bpf_get_reg64(reg: src, tmp: tmp2, ctx); |
1060 | s8 dstreg; |
1061 | |
1062 | /* need to calculate address since offset does not fit in 14 bits? */ |
1063 | if (relative_bits_ok(val: off, bits: 14)) |
1064 | dstreg = lo(r: rd); |
1065 | else { |
1066 | /* need to use R1 here, since addil puts result into R1 */ |
1067 | dstreg = HPPA_REG_R1; |
1068 | emit(hppa_addil(off, lo(rd)), ctx); |
1069 | off = im11(off); |
1070 | } |
1071 | |
1072 | /* ST: *(size *)(dst + off) = imm */ |
1073 | switch (size) { |
1074 | case BPF_B: |
1075 | emit(hppa_stb(lo(rs), off + 0, dstreg), ctx); |
1076 | break; |
1077 | case BPF_H: |
1078 | emit(hppa_sth(lo(rs), off + 0, dstreg), ctx); |
1079 | break; |
1080 | case BPF_W: |
1081 | emit(hppa_stw(lo(rs), off + 0, dstreg), ctx); |
1082 | break; |
1083 | case BPF_DW: |
1084 | emit(hppa_stw(hi(rs), off + 0, dstreg), ctx); |
1085 | emit(hppa_stw(lo(rs), off + 4, dstreg), ctx); |
1086 | break; |
1087 | } |
1088 | |
1089 | return 0; |
1090 | } |
1091 | |
1092 | static void emit_rev16(const s8 rd, struct hppa_jit_context *ctx) |
1093 | { |
1094 | emit(hppa_extru(rd, 23, 8, HPPA_REG_T1), ctx); |
1095 | emit(hppa_depwz(rd, 23, 8, HPPA_REG_T1), ctx); |
1096 | emit(hppa_extru(HPPA_REG_T1, 31, 16, rd), ctx); |
1097 | } |
1098 | |
1099 | static void emit_rev32(const s8 rs, const s8 rd, struct hppa_jit_context *ctx) |
1100 | { |
1101 | emit(hppa_shrpw(rs, rs, 16, HPPA_REG_T1), ctx); |
1102 | emit(hppa_depwz(HPPA_REG_T1, 15, 8, HPPA_REG_T1), ctx); |
1103 | emit(hppa_shrpw(rs, HPPA_REG_T1, 8, rd), ctx); |
1104 | } |
1105 | |
1106 | static void emit_zext64(const s8 *dst, struct hppa_jit_context *ctx) |
1107 | { |
1108 | const s8 *rd; |
1109 | const s8 *tmp1 = regmap[TMP_REG_1]; |
1110 | |
1111 | rd = bpf_get_reg64(reg: dst, tmp: tmp1, ctx); |
1112 | emit_hppa_copy(rs: HPPA_REG_ZERO, rd: hi(r: rd), ctx); |
1113 | bpf_put_reg64(reg: dst, src: rd, ctx); |
1114 | } |
1115 | |
1116 | int bpf_jit_emit_insn(const struct bpf_insn *insn, struct hppa_jit_context *ctx, |
1117 | bool ) |
1118 | { |
1119 | bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 || |
1120 | BPF_CLASS(insn->code) == BPF_JMP; |
1121 | int s, e, paoff, i = insn - ctx->prog->insnsi; |
1122 | u8 code = insn->code; |
1123 | s16 off = insn->off; |
1124 | s32 imm = insn->imm; |
1125 | |
1126 | const s8 *dst = regmap[insn->dst_reg]; |
1127 | const s8 *src = regmap[insn->src_reg]; |
1128 | const s8 *tmp1 = regmap[TMP_REG_1]; |
1129 | const s8 *tmp2 = regmap[TMP_REG_2]; |
1130 | |
1131 | if (0) printk("CLASS %03d CODE %#02x ALU64:%d BPF_SIZE %#02x " |
1132 | "BPF_CODE %#02x src_reg %d dst_reg %d\n" , |
1133 | BPF_CLASS(code), code, (code & BPF_ALU64) ? 1:0, BPF_SIZE(code), |
1134 | BPF_OP(code), insn->src_reg, insn->dst_reg); |
1135 | |
1136 | switch (code) { |
1137 | /* dst = src */ |
1138 | case BPF_ALU64 | BPF_MOV | BPF_X: |
1139 | |
1140 | case BPF_ALU64 | BPF_ADD | BPF_X: |
1141 | case BPF_ALU64 | BPF_ADD | BPF_K: |
1142 | |
1143 | case BPF_ALU64 | BPF_SUB | BPF_X: |
1144 | case BPF_ALU64 | BPF_SUB | BPF_K: |
1145 | |
1146 | case BPF_ALU64 | BPF_AND | BPF_X: |
1147 | case BPF_ALU64 | BPF_OR | BPF_X: |
1148 | case BPF_ALU64 | BPF_XOR | BPF_X: |
1149 | |
1150 | case BPF_ALU64 | BPF_MUL | BPF_X: |
1151 | case BPF_ALU64 | BPF_MUL | BPF_K: |
1152 | |
1153 | case BPF_ALU64 | BPF_DIV | BPF_X: |
1154 | case BPF_ALU64 | BPF_DIV | BPF_K: |
1155 | |
1156 | case BPF_ALU64 | BPF_MOD | BPF_X: |
1157 | case BPF_ALU64 | BPF_MOD | BPF_K: |
1158 | |
1159 | case BPF_ALU64 | BPF_LSH | BPF_X: |
1160 | case BPF_ALU64 | BPF_RSH | BPF_X: |
1161 | case BPF_ALU64 | BPF_ARSH | BPF_X: |
1162 | if (BPF_SRC(code) == BPF_K) { |
1163 | emit_imm32(rd: tmp2, imm, ctx); |
1164 | src = tmp2; |
1165 | } |
1166 | emit_alu_r64(dst, src, ctx, BPF_OP(code)); |
1167 | break; |
1168 | |
1169 | /* dst = -dst */ |
1170 | case BPF_ALU64 | BPF_NEG: |
1171 | emit_alu_r64(dst, src: tmp2, ctx, BPF_OP(code)); |
1172 | break; |
1173 | |
1174 | case BPF_ALU64 | BPF_MOV | BPF_K: |
1175 | case BPF_ALU64 | BPF_AND | BPF_K: |
1176 | case BPF_ALU64 | BPF_OR | BPF_K: |
1177 | case BPF_ALU64 | BPF_XOR | BPF_K: |
1178 | case BPF_ALU64 | BPF_LSH | BPF_K: |
1179 | case BPF_ALU64 | BPF_RSH | BPF_K: |
1180 | case BPF_ALU64 | BPF_ARSH | BPF_K: |
1181 | emit_alu_i64(dst, imm, ctx, BPF_OP(code)); |
1182 | break; |
1183 | |
1184 | case BPF_ALU | BPF_MOV | BPF_X: |
1185 | if (imm == 1) { |
1186 | /* Special mov32 for zext. */ |
1187 | emit_zext64(dst, ctx); |
1188 | break; |
1189 | } |
1190 | fallthrough; |
1191 | /* dst = dst OP src */ |
1192 | case BPF_ALU | BPF_ADD | BPF_X: |
1193 | case BPF_ALU | BPF_SUB | BPF_X: |
1194 | case BPF_ALU | BPF_AND | BPF_X: |
1195 | case BPF_ALU | BPF_OR | BPF_X: |
1196 | case BPF_ALU | BPF_XOR | BPF_X: |
1197 | |
1198 | case BPF_ALU | BPF_MUL | BPF_X: |
1199 | case BPF_ALU | BPF_MUL | BPF_K: |
1200 | |
1201 | case BPF_ALU | BPF_DIV | BPF_X: |
1202 | case BPF_ALU | BPF_DIV | BPF_K: |
1203 | |
1204 | case BPF_ALU | BPF_MOD | BPF_X: |
1205 | case BPF_ALU | BPF_MOD | BPF_K: |
1206 | |
1207 | case BPF_ALU | BPF_LSH | BPF_X: |
1208 | case BPF_ALU | BPF_RSH | BPF_X: |
1209 | case BPF_ALU | BPF_ARSH | BPF_X: |
1210 | if (BPF_SRC(code) == BPF_K) { |
1211 | emit_imm32(rd: tmp2, imm, ctx); |
1212 | src = tmp2; |
1213 | } |
1214 | emit_alu_r32(dst, src, ctx, BPF_OP(code)); |
1215 | break; |
1216 | |
1217 | /* dst = dst OP imm */ |
1218 | case BPF_ALU | BPF_MOV | BPF_K: |
1219 | case BPF_ALU | BPF_ADD | BPF_K: |
1220 | case BPF_ALU | BPF_SUB | BPF_K: |
1221 | case BPF_ALU | BPF_AND | BPF_K: |
1222 | case BPF_ALU | BPF_OR | BPF_K: |
1223 | case BPF_ALU | BPF_XOR | BPF_K: |
1224 | case BPF_ALU | BPF_LSH | BPF_K: |
1225 | case BPF_ALU | BPF_RSH | BPF_K: |
1226 | case BPF_ALU | BPF_ARSH | BPF_K: |
1227 | /* |
1228 | * mul,div,mod are handled in the BPF_X case. |
1229 | */ |
1230 | emit_alu_i32(dst, imm, ctx, BPF_OP(code)); |
1231 | break; |
1232 | |
1233 | /* dst = -dst */ |
1234 | case BPF_ALU | BPF_NEG: |
1235 | /* |
1236 | * src is ignored---choose tmp2 as a dummy register since it |
1237 | * is not on the stack. |
1238 | */ |
1239 | emit_alu_r32(dst, src: tmp2, ctx, BPF_OP(code)); |
1240 | break; |
1241 | |
1242 | /* dst = BSWAP##imm(dst) */ |
1243 | case BPF_ALU | BPF_END | BPF_FROM_BE: |
1244 | { |
1245 | const s8 *rd = bpf_get_reg64(reg: dst, tmp: tmp1, ctx); |
1246 | |
1247 | switch (imm) { |
1248 | case 16: |
1249 | /* zero-extend 16 bits into 64 bits */ |
1250 | emit(hppa_extru(lo(rd), 31, 16, lo(rd)), ctx); |
1251 | fallthrough; |
1252 | case 32: |
1253 | /* zero-extend 32 bits into 64 bits */ |
1254 | if (!ctx->prog->aux->verifier_zext) |
1255 | emit_hppa_copy(rs: HPPA_REG_ZERO, rd: hi(r: rd), ctx); |
1256 | break; |
1257 | case 64: |
1258 | /* Do nothing. */ |
1259 | break; |
1260 | default: |
1261 | pr_err("bpf-jit: BPF_END imm %d invalid\n" , imm); |
1262 | return -1; |
1263 | } |
1264 | |
1265 | bpf_put_reg64(reg: dst, src: rd, ctx); |
1266 | break; |
1267 | } |
1268 | |
1269 | case BPF_ALU | BPF_END | BPF_FROM_LE: |
1270 | { |
1271 | const s8 *rd = bpf_get_reg64(reg: dst, tmp: tmp1, ctx); |
1272 | |
1273 | switch (imm) { |
1274 | case 16: |
1275 | emit_rev16(rd: lo(r: rd), ctx); |
1276 | if (!ctx->prog->aux->verifier_zext) |
1277 | emit_hppa_copy(rs: HPPA_REG_ZERO, rd: hi(r: rd), ctx); |
1278 | break; |
1279 | case 32: |
1280 | emit_rev32(rs: lo(r: rd), rd: lo(r: rd), ctx); |
1281 | if (!ctx->prog->aux->verifier_zext) |
1282 | emit_hppa_copy(rs: HPPA_REG_ZERO, rd: hi(r: rd), ctx); |
1283 | break; |
1284 | case 64: |
1285 | /* Swap upper and lower halves, then each half. */ |
1286 | emit_hppa_copy(rs: hi(r: rd), rd: HPPA_REG_T0, ctx); |
1287 | emit_rev32(rs: lo(r: rd), rd: hi(r: rd), ctx); |
1288 | emit_rev32(rs: HPPA_REG_T0, rd: lo(r: rd), ctx); |
1289 | break; |
1290 | default: |
1291 | pr_err("bpf-jit: BPF_END imm %d invalid\n" , imm); |
1292 | return -1; |
1293 | } |
1294 | |
1295 | bpf_put_reg64(reg: dst, src: rd, ctx); |
1296 | break; |
1297 | } |
1298 | /* JUMP off */ |
1299 | case BPF_JMP | BPF_JA: |
1300 | paoff = hppa_offset(insn: i, off, ctx); |
1301 | emit_jump(paoff, force_far: false, ctx); |
1302 | break; |
1303 | /* function call */ |
1304 | case BPF_JMP | BPF_CALL: |
1305 | { |
1306 | bool fixed; |
1307 | int ret; |
1308 | u64 addr; |
1309 | |
1310 | ret = bpf_jit_get_func_addr(prog: ctx->prog, insn, extra_pass, func_addr: &addr, |
1311 | func_addr_fixed: &fixed); |
1312 | if (ret < 0) |
1313 | return ret; |
1314 | emit_call(fixed, addr, ctx); |
1315 | break; |
1316 | } |
1317 | /* tail call */ |
1318 | case BPF_JMP | BPF_TAIL_CALL: |
1319 | REG_SET_SEEN_ALL(ctx); |
1320 | if (emit_bpf_tail_call(insn: i, ctx)) |
1321 | return -1; |
1322 | break; |
1323 | /* IF (dst COND imm) JUMP off */ |
1324 | case BPF_JMP | BPF_JEQ | BPF_X: |
1325 | case BPF_JMP | BPF_JEQ | BPF_K: |
1326 | case BPF_JMP32 | BPF_JEQ | BPF_X: |
1327 | case BPF_JMP32 | BPF_JEQ | BPF_K: |
1328 | |
1329 | case BPF_JMP | BPF_JNE | BPF_X: |
1330 | case BPF_JMP | BPF_JNE | BPF_K: |
1331 | case BPF_JMP32 | BPF_JNE | BPF_X: |
1332 | case BPF_JMP32 | BPF_JNE | BPF_K: |
1333 | |
1334 | case BPF_JMP | BPF_JLE | BPF_X: |
1335 | case BPF_JMP | BPF_JLE | BPF_K: |
1336 | case BPF_JMP32 | BPF_JLE | BPF_X: |
1337 | case BPF_JMP32 | BPF_JLE | BPF_K: |
1338 | |
1339 | case BPF_JMP | BPF_JLT | BPF_X: |
1340 | case BPF_JMP | BPF_JLT | BPF_K: |
1341 | case BPF_JMP32 | BPF_JLT | BPF_X: |
1342 | case BPF_JMP32 | BPF_JLT | BPF_K: |
1343 | |
1344 | case BPF_JMP | BPF_JGE | BPF_X: |
1345 | case BPF_JMP | BPF_JGE | BPF_K: |
1346 | case BPF_JMP32 | BPF_JGE | BPF_X: |
1347 | case BPF_JMP32 | BPF_JGE | BPF_K: |
1348 | |
1349 | case BPF_JMP | BPF_JGT | BPF_X: |
1350 | case BPF_JMP | BPF_JGT | BPF_K: |
1351 | case BPF_JMP32 | BPF_JGT | BPF_X: |
1352 | case BPF_JMP32 | BPF_JGT | BPF_K: |
1353 | |
1354 | case BPF_JMP | BPF_JSLE | BPF_X: |
1355 | case BPF_JMP | BPF_JSLE | BPF_K: |
1356 | case BPF_JMP32 | BPF_JSLE | BPF_X: |
1357 | case BPF_JMP32 | BPF_JSLE | BPF_K: |
1358 | |
1359 | case BPF_JMP | BPF_JSLT | BPF_X: |
1360 | case BPF_JMP | BPF_JSLT | BPF_K: |
1361 | case BPF_JMP32 | BPF_JSLT | BPF_X: |
1362 | case BPF_JMP32 | BPF_JSLT | BPF_K: |
1363 | |
1364 | case BPF_JMP | BPF_JSGE | BPF_X: |
1365 | case BPF_JMP | BPF_JSGE | BPF_K: |
1366 | case BPF_JMP32 | BPF_JSGE | BPF_X: |
1367 | case BPF_JMP32 | BPF_JSGE | BPF_K: |
1368 | |
1369 | case BPF_JMP | BPF_JSGT | BPF_X: |
1370 | case BPF_JMP | BPF_JSGT | BPF_K: |
1371 | case BPF_JMP32 | BPF_JSGT | BPF_X: |
1372 | case BPF_JMP32 | BPF_JSGT | BPF_K: |
1373 | |
1374 | case BPF_JMP | BPF_JSET | BPF_X: |
1375 | case BPF_JMP | BPF_JSET | BPF_K: |
1376 | case BPF_JMP32 | BPF_JSET | BPF_X: |
1377 | case BPF_JMP32 | BPF_JSET | BPF_K: |
1378 | paoff = hppa_offset(insn: i, off, ctx); |
1379 | if (BPF_SRC(code) == BPF_K) { |
1380 | s = ctx->ninsns; |
1381 | emit_imm32(rd: tmp2, imm, ctx); |
1382 | src = tmp2; |
1383 | e = ctx->ninsns; |
1384 | paoff -= (e - s); |
1385 | } |
1386 | if (is64) |
1387 | emit_branch_r64(src1: dst, src2: src, paoff, ctx, BPF_OP(code)); |
1388 | else |
1389 | emit_branch_r32(src1: dst, src2: src, paoff, ctx, BPF_OP(code)); |
1390 | break; |
1391 | /* function return */ |
1392 | case BPF_JMP | BPF_EXIT: |
1393 | if (i == ctx->prog->len - 1) |
1394 | break; |
1395 | /* load epilogue function pointer and jump to it. */ |
1396 | emit(EXIT_PTR_LOAD(HPPA_REG_RP), ctx); |
1397 | emit(EXIT_PTR_JUMP(HPPA_REG_RP, NOP_NEXT_INSTR), ctx); |
1398 | break; |
1399 | |
1400 | /* dst = imm64 */ |
1401 | case BPF_LD | BPF_IMM | BPF_DW: |
1402 | { |
1403 | struct bpf_insn insn1 = insn[1]; |
1404 | u32 upper = insn1.imm; |
1405 | u32 lower = imm; |
1406 | const s8 *rd = bpf_get_reg64_ref(reg: dst, tmp: tmp1, must_load: false, ctx); |
1407 | |
1408 | if (0 && bpf_pseudo_func(insn)) { |
1409 | WARN_ON(upper); /* we are 32-bit! */ |
1410 | upper = 0; |
1411 | lower = (uintptr_t) dereference_function_descriptor(lower); |
1412 | } |
1413 | |
1414 | emit_imm64(rd, imm_hi: upper, imm_lo: lower, ctx); |
1415 | bpf_put_reg64(reg: dst, src: rd, ctx); |
1416 | return 1; |
1417 | } |
1418 | |
1419 | /* LDX: dst = *(size *)(src + off) */ |
1420 | case BPF_LDX | BPF_MEM | BPF_B: |
1421 | case BPF_LDX | BPF_MEM | BPF_H: |
1422 | case BPF_LDX | BPF_MEM | BPF_W: |
1423 | case BPF_LDX | BPF_MEM | BPF_DW: |
1424 | if (emit_load_r64(dst, src, off, ctx, BPF_SIZE(code))) |
1425 | return -1; |
1426 | break; |
1427 | |
1428 | /* speculation barrier */ |
1429 | case BPF_ST | BPF_NOSPEC: |
1430 | break; |
1431 | |
1432 | /* ST: *(size *)(dst + off) = imm */ |
1433 | case BPF_ST | BPF_MEM | BPF_B: |
1434 | case BPF_ST | BPF_MEM | BPF_H: |
1435 | case BPF_ST | BPF_MEM | BPF_W: |
1436 | case BPF_ST | BPF_MEM | BPF_DW: |
1437 | |
1438 | case BPF_STX | BPF_MEM | BPF_B: |
1439 | case BPF_STX | BPF_MEM | BPF_H: |
1440 | case BPF_STX | BPF_MEM | BPF_W: |
1441 | case BPF_STX | BPF_MEM | BPF_DW: |
1442 | if (BPF_CLASS(code) == BPF_ST) { |
1443 | emit_imm32(rd: tmp2, imm, ctx); |
1444 | src = tmp2; |
1445 | } |
1446 | |
1447 | if (emit_store_r64(dst, src, off, ctx, BPF_SIZE(code), |
1448 | BPF_MODE(code))) |
1449 | return -1; |
1450 | break; |
1451 | |
1452 | case BPF_STX | BPF_ATOMIC | BPF_W: |
1453 | case BPF_STX | BPF_ATOMIC | BPF_DW: |
1454 | pr_info_once( |
1455 | "bpf-jit: not supported: atomic operation %02x ***\n" , |
1456 | insn->imm); |
1457 | return -EFAULT; |
1458 | |
1459 | default: |
1460 | pr_err("bpf-jit: unknown opcode %02x\n" , code); |
1461 | return -EINVAL; |
1462 | } |
1463 | |
1464 | return 0; |
1465 | } |
1466 | |
1467 | void bpf_jit_build_prologue(struct hppa_jit_context *ctx) |
1468 | { |
1469 | const s8 *tmp = regmap[TMP_REG_1]; |
1470 | const s8 *dst, *reg; |
1471 | int stack_adjust = 0; |
1472 | int i; |
1473 | unsigned long addr; |
1474 | int bpf_stack_adjust; |
1475 | |
1476 | /* |
1477 | * stack on hppa grows up, so if tail calls are used we need to |
1478 | * allocate the maximum stack size |
1479 | */ |
1480 | if (REG_ALL_SEEN(ctx)) |
1481 | bpf_stack_adjust = MAX_BPF_STACK; |
1482 | else |
1483 | bpf_stack_adjust = ctx->prog->aux->stack_depth; |
1484 | bpf_stack_adjust = round_up(bpf_stack_adjust, STACK_ALIGN); |
1485 | |
1486 | /* make space for callee-saved registers. */ |
1487 | stack_adjust += NR_SAVED_REGISTERS * REG_SIZE; |
1488 | /* make space for BPF registers on stack. */ |
1489 | stack_adjust += BPF_JIT_SCRATCH_REGS * REG_SIZE; |
1490 | /* make space for BPF stack. */ |
1491 | stack_adjust += bpf_stack_adjust; |
1492 | /* round up for stack alignment. */ |
1493 | stack_adjust = round_up(stack_adjust, STACK_ALIGN); |
1494 | |
1495 | /* |
1496 | * The first instruction sets the tail-call-counter (TCC) register. |
1497 | * This instruction is skipped by tail calls. |
1498 | * Use a temporary register instead of a caller-saved register initially. |
1499 | */ |
1500 | emit(hppa_ldi(MAX_TAIL_CALL_CNT, HPPA_REG_TCC_IN_INIT), ctx); |
1501 | |
1502 | /* |
1503 | * skip all initializations when called as BPF TAIL call. |
1504 | */ |
1505 | emit(hppa_ldi(MAX_TAIL_CALL_CNT, HPPA_REG_R1), ctx); |
1506 | emit(hppa_bne(HPPA_REG_TCC_IN_INIT, HPPA_REG_R1, ctx->prologue_len - 2 - HPPA_BRANCH_DISPLACEMENT), ctx); |
1507 | |
1508 | /* set up hppa stack frame. */ |
1509 | emit_hppa_copy(rs: HPPA_REG_SP, rd: HPPA_REG_R1, ctx); // copy sp,r1 (=prev_sp) |
1510 | emit(hppa_ldo(stack_adjust, HPPA_REG_SP, HPPA_REG_SP), ctx); // ldo stack_adjust(sp),sp (increase stack) |
1511 | emit(hppa_stw(HPPA_REG_R1, -REG_SIZE, HPPA_REG_SP), ctx); // stw prev_sp,-0x04(sp) |
1512 | emit(hppa_stw(HPPA_REG_RP, -0x14, HPPA_REG_SP), ctx); // stw rp,-0x14(sp) |
1513 | |
1514 | REG_FORCE_SEEN(ctx, HPPA_REG_T0); |
1515 | REG_FORCE_SEEN(ctx, HPPA_REG_T1); |
1516 | REG_FORCE_SEEN(ctx, HPPA_REG_T2); |
1517 | REG_FORCE_SEEN(ctx, HPPA_REG_T3); |
1518 | REG_FORCE_SEEN(ctx, HPPA_REG_T4); |
1519 | REG_FORCE_SEEN(ctx, HPPA_REG_T5); |
1520 | |
1521 | /* save callee-save registers. */ |
1522 | for (i = 3; i <= 18; i++) { |
1523 | if (OPTIMIZE_HPPA && !REG_WAS_SEEN(ctx, HPPA_R(i))) |
1524 | continue; |
1525 | emit(hppa_stw(HPPA_R(i), -REG_SIZE * (8 + (i-3)), HPPA_REG_SP), ctx); // stw ri,-save_area(sp) |
1526 | } |
1527 | |
1528 | /* |
1529 | * now really set the tail call counter (TCC) register. |
1530 | */ |
1531 | if (REG_WAS_SEEN(ctx, HPPA_REG_TCC)) |
1532 | emit(hppa_ldi(MAX_TAIL_CALL_CNT, HPPA_REG_TCC), ctx); |
1533 | |
1534 | /* |
1535 | * save epilogue function pointer for outer TCC call chain. |
1536 | * The main TCC call stores the final RP on stack. |
1537 | */ |
1538 | addr = (uintptr_t) &ctx->insns[ctx->epilogue_offset]; |
1539 | /* skip first two instructions of exit function, which jump to exit */ |
1540 | addr += 2 * HPPA_INSN_SIZE; |
1541 | emit(hppa_ldil(addr, HPPA_REG_T2), ctx); |
1542 | emit(hppa_ldo(im11(addr), HPPA_REG_T2, HPPA_REG_T2), ctx); |
1543 | emit(EXIT_PTR_STORE(HPPA_REG_T2), ctx); |
1544 | |
1545 | /* load R1 & R2 from registers, R3-R5 from stack. */ |
1546 | /* use HPPA_REG_R1 which holds the old stack value */ |
1547 | dst = regmap[BPF_REG_5]; |
1548 | reg = bpf_get_reg64_ref(reg: dst, tmp, must_load: false, ctx); |
1549 | if (REG_WAS_SEEN(ctx, lo(reg)) | REG_WAS_SEEN(ctx, hi(reg))) { |
1550 | if (REG_WAS_SEEN(ctx, hi(reg))) |
1551 | emit(hppa_ldw(-0x48, HPPA_REG_R1, hi(reg)), ctx); |
1552 | if (REG_WAS_SEEN(ctx, lo(reg))) |
1553 | emit(hppa_ldw(-0x44, HPPA_REG_R1, lo(reg)), ctx); |
1554 | bpf_put_reg64(reg: dst, src: tmp, ctx); |
1555 | } |
1556 | |
1557 | dst = regmap[BPF_REG_4]; |
1558 | reg = bpf_get_reg64_ref(reg: dst, tmp, must_load: false, ctx); |
1559 | if (REG_WAS_SEEN(ctx, lo(reg)) | REG_WAS_SEEN(ctx, hi(reg))) { |
1560 | if (REG_WAS_SEEN(ctx, hi(reg))) |
1561 | emit(hppa_ldw(-0x40, HPPA_REG_R1, hi(reg)), ctx); |
1562 | if (REG_WAS_SEEN(ctx, lo(reg))) |
1563 | emit(hppa_ldw(-0x3c, HPPA_REG_R1, lo(reg)), ctx); |
1564 | bpf_put_reg64(reg: dst, src: tmp, ctx); |
1565 | } |
1566 | |
1567 | dst = regmap[BPF_REG_3]; |
1568 | reg = bpf_get_reg64_ref(reg: dst, tmp, must_load: false, ctx); |
1569 | if (REG_WAS_SEEN(ctx, lo(reg)) | REG_WAS_SEEN(ctx, hi(reg))) { |
1570 | if (REG_WAS_SEEN(ctx, hi(reg))) |
1571 | emit(hppa_ldw(-0x38, HPPA_REG_R1, hi(reg)), ctx); |
1572 | if (REG_WAS_SEEN(ctx, lo(reg))) |
1573 | emit(hppa_ldw(-0x34, HPPA_REG_R1, lo(reg)), ctx); |
1574 | bpf_put_reg64(reg: dst, src: tmp, ctx); |
1575 | } |
1576 | |
1577 | dst = regmap[BPF_REG_2]; |
1578 | reg = bpf_get_reg64_ref(reg: dst, tmp, must_load: false, ctx); |
1579 | if (REG_WAS_SEEN(ctx, lo(reg)) | REG_WAS_SEEN(ctx, hi(reg))) { |
1580 | if (REG_WAS_SEEN(ctx, hi(reg))) |
1581 | emit_hppa_copy(rs: HPPA_REG_ARG3, rd: hi(r: reg), ctx); |
1582 | if (REG_WAS_SEEN(ctx, lo(reg))) |
1583 | emit_hppa_copy(rs: HPPA_REG_ARG2, rd: lo(r: reg), ctx); |
1584 | bpf_put_reg64(reg: dst, src: tmp, ctx); |
1585 | } |
1586 | |
1587 | dst = regmap[BPF_REG_1]; |
1588 | reg = bpf_get_reg64_ref(reg: dst, tmp, must_load: false, ctx); |
1589 | if (REG_WAS_SEEN(ctx, lo(reg)) | REG_WAS_SEEN(ctx, hi(reg))) { |
1590 | if (REG_WAS_SEEN(ctx, hi(reg))) |
1591 | emit_hppa_copy(rs: HPPA_REG_ARG1, rd: hi(r: reg), ctx); |
1592 | if (REG_WAS_SEEN(ctx, lo(reg))) |
1593 | emit_hppa_copy(rs: HPPA_REG_ARG0, rd: lo(r: reg), ctx); |
1594 | bpf_put_reg64(reg: dst, src: tmp, ctx); |
1595 | } |
1596 | |
1597 | /* Set up BPF frame pointer. */ |
1598 | dst = regmap[BPF_REG_FP]; |
1599 | reg = bpf_get_reg64_ref(reg: dst, tmp, must_load: false, ctx); |
1600 | if (REG_WAS_SEEN(ctx, lo(reg)) | REG_WAS_SEEN(ctx, hi(reg))) { |
1601 | if (REG_WAS_SEEN(ctx, lo(reg))) |
1602 | emit(hppa_ldo(-REG_SIZE * (NR_SAVED_REGISTERS + BPF_JIT_SCRATCH_REGS), |
1603 | HPPA_REG_SP, lo(reg)), ctx); |
1604 | if (REG_WAS_SEEN(ctx, hi(reg))) |
1605 | emit_hppa_copy(rs: HPPA_REG_ZERO, rd: hi(r: reg), ctx); |
1606 | bpf_put_reg64(reg: dst, src: tmp, ctx); |
1607 | } |
1608 | |
1609 | emit(hppa_nop(), ctx); |
1610 | } |
1611 | |
1612 | void bpf_jit_build_epilogue(struct hppa_jit_context *ctx) |
1613 | { |
1614 | __build_epilogue(is_tail_call: false, ctx); |
1615 | } |
1616 | |