1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * jump label x86 support |
4 | * |
5 | * Copyright (C) 2009 Jason Baron <jbaron@redhat.com> |
6 | * |
7 | */ |
8 | #include <linux/jump_label.h> |
9 | #include <linux/memory.h> |
10 | #include <linux/uaccess.h> |
11 | #include <linux/module.h> |
12 | #include <linux/list.h> |
13 | #include <linux/jhash.h> |
14 | #include <linux/cpu.h> |
15 | #include <asm/kprobes.h> |
16 | #include <asm/alternative.h> |
17 | #include <asm/text-patching.h> |
18 | #include <asm/insn.h> |
19 | |
20 | int arch_jump_entry_size(struct jump_entry *entry) |
21 | { |
22 | struct insn insn = {}; |
23 | |
24 | insn_decode_kernel(&insn, (void *)jump_entry_code(entry)); |
25 | BUG_ON(insn.length != 2 && insn.length != 5); |
26 | |
27 | return insn.length; |
28 | } |
29 | |
30 | struct jump_label_patch { |
31 | const void *code; |
32 | int size; |
33 | }; |
34 | |
35 | static struct jump_label_patch |
36 | __jump_label_patch(struct jump_entry *entry, enum jump_label_type type) |
37 | { |
38 | const void *expect, *code, *nop; |
39 | const void *addr, *dest; |
40 | int size; |
41 | |
42 | addr = (void *)jump_entry_code(entry); |
43 | dest = (void *)jump_entry_target(entry); |
44 | |
45 | size = arch_jump_entry_size(entry); |
46 | switch (size) { |
47 | case JMP8_INSN_SIZE: |
48 | code = text_gen_insn(JMP8_INSN_OPCODE, addr, dest); |
49 | nop = x86_nops[size]; |
50 | break; |
51 | |
52 | case JMP32_INSN_SIZE: |
53 | code = text_gen_insn(JMP32_INSN_OPCODE, addr, dest); |
54 | nop = x86_nops[size]; |
55 | break; |
56 | |
57 | default: BUG(); |
58 | } |
59 | |
60 | if (type == JUMP_LABEL_JMP) |
61 | expect = nop; |
62 | else |
63 | expect = code; |
64 | |
65 | if (memcmp(p: addr, q: expect, size)) { |
66 | /* |
67 | * The location is not an op that we were expecting. |
68 | * Something went wrong. Crash the box, as something could be |
69 | * corrupting the kernel. |
70 | */ |
71 | pr_crit("jump_label: Fatal kernel bug, unexpected op at %pS [%p] (%5ph != %5ph)) size:%d type:%d\n" , |
72 | addr, addr, addr, expect, size, type); |
73 | BUG(); |
74 | } |
75 | |
76 | if (type == JUMP_LABEL_NOP) |
77 | code = nop; |
78 | |
79 | return (struct jump_label_patch){.code = code, .size = size}; |
80 | } |
81 | |
82 | static __always_inline void |
83 | __jump_label_transform(struct jump_entry *entry, |
84 | enum jump_label_type type, |
85 | int init) |
86 | { |
87 | const struct jump_label_patch jlp = __jump_label_patch(entry, type); |
88 | |
89 | /* |
90 | * As long as only a single processor is running and the code is still |
91 | * not marked as RO, text_poke_early() can be used; Checking that |
92 | * system_state is SYSTEM_BOOTING guarantees it. It will be set to |
93 | * SYSTEM_SCHEDULING before other cores are awaken and before the |
94 | * code is write-protected. |
95 | * |
96 | * At the time the change is being done, just ignore whether we |
97 | * are doing nop -> jump or jump -> nop transition, and assume |
98 | * always nop being the 'currently valid' instruction |
99 | */ |
100 | if (init || system_state == SYSTEM_BOOTING) { |
101 | text_poke_early(addr: (void *)jump_entry_code(entry), opcode: jlp.code, len: jlp.size); |
102 | return; |
103 | } |
104 | |
105 | text_poke_bp(addr: (void *)jump_entry_code(entry), opcode: jlp.code, len: jlp.size, NULL); |
106 | } |
107 | |
108 | static void __ref jump_label_transform(struct jump_entry *entry, |
109 | enum jump_label_type type, |
110 | int init) |
111 | { |
112 | mutex_lock(&text_mutex); |
113 | __jump_label_transform(entry, type, init); |
114 | mutex_unlock(lock: &text_mutex); |
115 | } |
116 | |
117 | void arch_jump_label_transform(struct jump_entry *entry, |
118 | enum jump_label_type type) |
119 | { |
120 | jump_label_transform(entry, type, init: 0); |
121 | } |
122 | |
123 | bool arch_jump_label_transform_queue(struct jump_entry *entry, |
124 | enum jump_label_type type) |
125 | { |
126 | struct jump_label_patch jlp; |
127 | |
128 | if (system_state == SYSTEM_BOOTING) { |
129 | /* |
130 | * Fallback to the non-batching mode. |
131 | */ |
132 | arch_jump_label_transform(entry, type); |
133 | return true; |
134 | } |
135 | |
136 | mutex_lock(&text_mutex); |
137 | jlp = __jump_label_patch(entry, type); |
138 | text_poke_queue(addr: (void *)jump_entry_code(entry), opcode: jlp.code, len: jlp.size, NULL); |
139 | mutex_unlock(lock: &text_mutex); |
140 | return true; |
141 | } |
142 | |
143 | void arch_jump_label_transform_apply(void) |
144 | { |
145 | mutex_lock(&text_mutex); |
146 | text_poke_finish(); |
147 | mutex_unlock(lock: &text_mutex); |
148 | } |
149 | |