1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
2 | /* |
3 | * bpf_jit.h: BPF JIT compiler for PPC |
4 | * |
5 | * Copyright 2011 Matt Evans <matt@ozlabs.org>, IBM Corporation |
6 | * 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com> |
7 | */ |
8 | #ifndef _BPF_JIT_H |
9 | #define _BPF_JIT_H |
10 | |
11 | #ifndef __ASSEMBLY__ |
12 | |
13 | #include <asm/types.h> |
14 | #include <asm/ppc-opcode.h> |
15 | |
16 | #ifdef CONFIG_PPC64_ELF_ABI_V1 |
17 | #define FUNCTION_DESCR_SIZE 24 |
18 | #else |
19 | #define FUNCTION_DESCR_SIZE 0 |
20 | #endif |
21 | |
22 | #define CTX_NIA(ctx) ((unsigned long)ctx->idx * 4) |
23 | |
24 | #define PLANT_INSTR(d, idx, instr) \ |
25 | do { if (d) { (d)[idx] = instr; } idx++; } while (0) |
26 | #define EMIT(instr) PLANT_INSTR(image, ctx->idx, instr) |
27 | |
28 | /* Long jump; (unconditional 'branch') */ |
29 | #define PPC_JMP(dest) \ |
30 | do { \ |
31 | long offset = (long)(dest) - CTX_NIA(ctx); \ |
32 | if ((dest) != 0 && !is_offset_in_branch_range(offset)) { \ |
33 | pr_err_ratelimited("Branch offset 0x%lx (@%u) out of range\n", offset, ctx->idx); \ |
34 | return -ERANGE; \ |
35 | } \ |
36 | EMIT(PPC_RAW_BRANCH(offset)); \ |
37 | } while (0) |
38 | |
39 | /* "cond" here covers BO:BI fields. */ |
40 | #define PPC_BCC_SHORT(cond, dest) \ |
41 | do { \ |
42 | long offset = (long)(dest) - CTX_NIA(ctx); \ |
43 | if ((dest) != 0 && !is_offset_in_cond_branch_range(offset)) { \ |
44 | pr_err_ratelimited("Conditional branch offset 0x%lx (@%u) out of range\n", offset, ctx->idx); \ |
45 | return -ERANGE; \ |
46 | } \ |
47 | EMIT(PPC_INST_BRANCH_COND | (((cond) & 0x3ff) << 16) | (offset & 0xfffc)); \ |
48 | } while (0) |
49 | |
50 | /* Sign-extended 32-bit immediate load */ |
51 | #define PPC_LI32(d, i) do { \ |
52 | if ((int)(uintptr_t)(i) >= -32768 && \ |
53 | (int)(uintptr_t)(i) < 32768) \ |
54 | EMIT(PPC_RAW_LI(d, i)); \ |
55 | else { \ |
56 | EMIT(PPC_RAW_LIS(d, IMM_H(i))); \ |
57 | if (IMM_L(i)) \ |
58 | EMIT(PPC_RAW_ORI(d, d, IMM_L(i))); \ |
59 | } } while(0) |
60 | |
61 | #ifdef CONFIG_PPC64 |
62 | #define PPC_LI64(d, i) do { \ |
63 | if ((long)(i) >= -2147483648 && \ |
64 | (long)(i) < 2147483648) \ |
65 | PPC_LI32(d, i); \ |
66 | else { \ |
67 | if (!((uintptr_t)(i) & 0xffff800000000000ULL)) \ |
68 | EMIT(PPC_RAW_LI(d, ((uintptr_t)(i) >> 32) & \ |
69 | 0xffff)); \ |
70 | else { \ |
71 | EMIT(PPC_RAW_LIS(d, ((uintptr_t)(i) >> 48))); \ |
72 | if ((uintptr_t)(i) & 0x0000ffff00000000ULL) \ |
73 | EMIT(PPC_RAW_ORI(d, d, \ |
74 | ((uintptr_t)(i) >> 32) & 0xffff)); \ |
75 | } \ |
76 | EMIT(PPC_RAW_SLDI(d, d, 32)); \ |
77 | if ((uintptr_t)(i) & 0x00000000ffff0000ULL) \ |
78 | EMIT(PPC_RAW_ORIS(d, d, \ |
79 | ((uintptr_t)(i) >> 16) & 0xffff)); \ |
80 | if ((uintptr_t)(i) & 0x000000000000ffffULL) \ |
81 | EMIT(PPC_RAW_ORI(d, d, (uintptr_t)(i) & \ |
82 | 0xffff)); \ |
83 | } } while (0) |
84 | #endif |
85 | |
86 | /* |
87 | * The fly in the ointment of code size changing from pass to pass is |
88 | * avoided by padding the short branch case with a NOP. If code size differs |
89 | * with different branch reaches we will have the issue of code moving from |
90 | * one pass to the next and will need a few passes to converge on a stable |
91 | * state. |
92 | */ |
93 | #define PPC_BCC(cond, dest) do { \ |
94 | if (is_offset_in_cond_branch_range((long)(dest) - CTX_NIA(ctx))) { \ |
95 | PPC_BCC_SHORT(cond, dest); \ |
96 | EMIT(PPC_RAW_NOP()); \ |
97 | } else { \ |
98 | /* Flip the 'T or F' bit to invert comparison */ \ |
99 | PPC_BCC_SHORT(cond ^ COND_CMP_TRUE, CTX_NIA(ctx) + 2*4); \ |
100 | PPC_JMP(dest); \ |
101 | } } while(0) |
102 | |
103 | /* To create a branch condition, select a bit of cr0... */ |
104 | #define CR0_LT 0 |
105 | #define CR0_GT 1 |
106 | #define CR0_EQ 2 |
107 | /* ...and modify BO[3] */ |
108 | #define COND_CMP_TRUE 0x100 |
109 | #define COND_CMP_FALSE 0x000 |
110 | /* Together, they make all required comparisons: */ |
111 | #define COND_GT (CR0_GT | COND_CMP_TRUE) |
112 | #define COND_GE (CR0_LT | COND_CMP_FALSE) |
113 | #define COND_EQ (CR0_EQ | COND_CMP_TRUE) |
114 | #define COND_NE (CR0_EQ | COND_CMP_FALSE) |
115 | #define COND_LT (CR0_LT | COND_CMP_TRUE) |
116 | #define COND_LE (CR0_GT | COND_CMP_FALSE) |
117 | |
118 | #define SEEN_FUNC 0x20000000 /* might call external helpers */ |
119 | #define SEEN_TAILCALL 0x40000000 /* uses tail calls */ |
120 | |
121 | struct codegen_context { |
122 | /* |
123 | * This is used to track register usage as well |
124 | * as calls to external helpers. |
125 | * - register usage is tracked with corresponding |
126 | * bits (r3-r31) |
127 | * - rest of the bits can be used to track other |
128 | * things -- for now, we use bits 0 to 2 |
129 | * encoded in SEEN_* macros above |
130 | */ |
131 | unsigned int seen; |
132 | unsigned int idx; |
133 | unsigned int stack_size; |
134 | int b2p[MAX_BPF_JIT_REG + 2]; |
135 | unsigned int exentry_idx; |
136 | unsigned int alt_exit_addr; |
137 | }; |
138 | |
139 | #define bpf_to_ppc(r) (ctx->b2p[r]) |
140 | |
141 | #ifdef CONFIG_PPC32 |
142 | #define BPF_FIXUP_LEN 3 /* Three instructions => 12 bytes */ |
143 | #else |
144 | #define BPF_FIXUP_LEN 2 /* Two instructions => 8 bytes */ |
145 | #endif |
146 | |
147 | static inline bool bpf_is_seen_register(struct codegen_context *ctx, int i) |
148 | { |
149 | return ctx->seen & (1 << (31 - i)); |
150 | } |
151 | |
152 | static inline void bpf_set_seen_register(struct codegen_context *ctx, int i) |
153 | { |
154 | ctx->seen |= 1 << (31 - i); |
155 | } |
156 | |
157 | static inline void bpf_clear_seen_register(struct codegen_context *ctx, int i) |
158 | { |
159 | ctx->seen &= ~(1 << (31 - i)); |
160 | } |
161 | |
162 | void bpf_jit_init_reg_mapping(struct codegen_context *ctx); |
163 | int bpf_jit_emit_func_call_rel(u32 *image, u32 *fimage, struct codegen_context *ctx, u64 func); |
164 | int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct codegen_context *ctx, |
165 | u32 *addrs, int pass, bool ); |
166 | void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx); |
167 | void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx); |
168 | void bpf_jit_realloc_regs(struct codegen_context *ctx); |
169 | int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, long exit_addr); |
170 | |
171 | int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, u32 *fimage, int pass, |
172 | struct codegen_context *ctx, int insn_idx, |
173 | int jmp_off, int dst_reg); |
174 | |
175 | #endif |
176 | |
177 | #endif |
178 | |