1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * shadow.c - Shadow Variables |
4 | * |
5 | * Copyright (C) 2014 Josh Poimboeuf <jpoimboe@redhat.com> |
6 | * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com> |
7 | * Copyright (C) 2017 Joe Lawrence <joe.lawrence@redhat.com> |
8 | */ |
9 | |
10 | /** |
11 | * DOC: Shadow variable API concurrency notes: |
12 | * |
13 | * The shadow variable API provides a simple relationship between an |
14 | * <obj, id> pair and a pointer value. It is the responsibility of the |
15 | * caller to provide any mutual exclusion required of the shadow data. |
16 | * |
17 | * Once a shadow variable is attached to its parent object via the |
18 | * klp_shadow_*alloc() API calls, it is considered live: any subsequent |
19 | * call to klp_shadow_get() may then return the shadow variable's data |
20 | * pointer. Callers of klp_shadow_*alloc() should prepare shadow data |
21 | * accordingly. |
22 | * |
23 | * The klp_shadow_*alloc() API calls may allocate memory for new shadow |
24 | * variable structures. Their implementation does not call kmalloc |
25 | * inside any spinlocks, but API callers should pass GFP flags according |
26 | * to their specific needs. |
27 | * |
28 | * The klp_shadow_hash is an RCU-enabled hashtable and is safe against |
29 | * concurrent klp_shadow_free() and klp_shadow_get() operations. |
30 | */ |
31 | |
32 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
33 | |
34 | #include <linux/hashtable.h> |
35 | #include <linux/slab.h> |
36 | #include <linux/livepatch.h> |
37 | |
38 | static DEFINE_HASHTABLE(klp_shadow_hash, 12); |
39 | |
40 | /* |
41 | * klp_shadow_lock provides exclusive access to the klp_shadow_hash and |
42 | * the shadow variables it references. |
43 | */ |
44 | static DEFINE_SPINLOCK(klp_shadow_lock); |
45 | |
46 | /** |
47 | * struct klp_shadow - shadow variable structure |
48 | * @node: klp_shadow_hash hash table node |
49 | * @rcu_head: RCU is used to safely free this structure |
50 | * @obj: pointer to parent object |
51 | * @id: data identifier |
52 | * @data: data area |
53 | */ |
54 | struct klp_shadow { |
55 | struct hlist_node node; |
56 | struct rcu_head rcu_head; |
57 | void *obj; |
58 | unsigned long id; |
59 | char data[]; |
60 | }; |
61 | |
62 | /** |
63 | * klp_shadow_match() - verify a shadow variable matches given <obj, id> |
64 | * @shadow: shadow variable to match |
65 | * @obj: pointer to parent object |
66 | * @id: data identifier |
67 | * |
68 | * Return: true if the shadow variable matches. |
69 | */ |
70 | static inline bool klp_shadow_match(struct klp_shadow *shadow, void *obj, |
71 | unsigned long id) |
72 | { |
73 | return shadow->obj == obj && shadow->id == id; |
74 | } |
75 | |
76 | /** |
77 | * klp_shadow_get() - retrieve a shadow variable data pointer |
78 | * @obj: pointer to parent object |
79 | * @id: data identifier |
80 | * |
81 | * Return: the shadow variable data element, NULL on failure. |
82 | */ |
83 | void *klp_shadow_get(void *obj, unsigned long id) |
84 | { |
85 | struct klp_shadow *shadow; |
86 | |
87 | rcu_read_lock(); |
88 | |
89 | hash_for_each_possible_rcu(klp_shadow_hash, shadow, node, |
90 | (unsigned long)obj) { |
91 | |
92 | if (klp_shadow_match(shadow, obj, id)) { |
93 | rcu_read_unlock(); |
94 | return shadow->data; |
95 | } |
96 | } |
97 | |
98 | rcu_read_unlock(); |
99 | |
100 | return NULL; |
101 | } |
102 | EXPORT_SYMBOL_GPL(klp_shadow_get); |
103 | |
104 | static void *__klp_shadow_get_or_alloc(void *obj, unsigned long id, |
105 | size_t size, gfp_t gfp_flags, |
106 | klp_shadow_ctor_t ctor, void *ctor_data, |
107 | bool warn_on_exist) |
108 | { |
109 | struct klp_shadow *new_shadow; |
110 | void *shadow_data; |
111 | unsigned long flags; |
112 | |
113 | /* Check if the shadow variable already exists */ |
114 | shadow_data = klp_shadow_get(obj, id); |
115 | if (shadow_data) |
116 | goto exists; |
117 | |
118 | /* |
119 | * Allocate a new shadow variable. Fill it with zeroes by default. |
120 | * More complex setting can be done by @ctor function. But it is |
121 | * called only when the buffer is really used (under klp_shadow_lock). |
122 | */ |
123 | new_shadow = kzalloc(size: size + sizeof(*new_shadow), flags: gfp_flags); |
124 | if (!new_shadow) |
125 | return NULL; |
126 | |
127 | /* Look for <obj, id> again under the lock */ |
128 | spin_lock_irqsave(&klp_shadow_lock, flags); |
129 | shadow_data = klp_shadow_get(obj, id); |
130 | if (unlikely(shadow_data)) { |
131 | /* |
132 | * Shadow variable was found, throw away speculative |
133 | * allocation. |
134 | */ |
135 | spin_unlock_irqrestore(lock: &klp_shadow_lock, flags); |
136 | kfree(objp: new_shadow); |
137 | goto exists; |
138 | } |
139 | |
140 | new_shadow->obj = obj; |
141 | new_shadow->id = id; |
142 | |
143 | if (ctor) { |
144 | int err; |
145 | |
146 | err = ctor(obj, new_shadow->data, ctor_data); |
147 | if (err) { |
148 | spin_unlock_irqrestore(lock: &klp_shadow_lock, flags); |
149 | kfree(objp: new_shadow); |
150 | pr_err("Failed to construct shadow variable <%p, %lx> (%d)\n" , |
151 | obj, id, err); |
152 | return NULL; |
153 | } |
154 | } |
155 | |
156 | /* No <obj, id> found, so attach the newly allocated one */ |
157 | hash_add_rcu(klp_shadow_hash, &new_shadow->node, |
158 | (unsigned long)new_shadow->obj); |
159 | spin_unlock_irqrestore(lock: &klp_shadow_lock, flags); |
160 | |
161 | return new_shadow->data; |
162 | |
163 | exists: |
164 | if (warn_on_exist) { |
165 | WARN(1, "Duplicate shadow variable <%p, %lx>\n" , obj, id); |
166 | return NULL; |
167 | } |
168 | |
169 | return shadow_data; |
170 | } |
171 | |
172 | /** |
173 | * klp_shadow_alloc() - allocate and add a new shadow variable |
174 | * @obj: pointer to parent object |
175 | * @id: data identifier |
176 | * @size: size of attached data |
177 | * @gfp_flags: GFP mask for allocation |
178 | * @ctor: custom constructor to initialize the shadow data (optional) |
179 | * @ctor_data: pointer to any data needed by @ctor (optional) |
180 | * |
181 | * Allocates @size bytes for new shadow variable data using @gfp_flags. |
182 | * The data are zeroed by default. They are further initialized by @ctor |
183 | * function if it is not NULL. The new shadow variable is then added |
184 | * to the global hashtable. |
185 | * |
186 | * If an existing <obj, id> shadow variable can be found, this routine will |
187 | * issue a WARN, exit early and return NULL. |
188 | * |
189 | * This function guarantees that the constructor function is called only when |
190 | * the variable did not exist before. The cost is that @ctor is called |
191 | * in atomic context under a spin lock. |
192 | * |
193 | * Return: the shadow variable data element, NULL on duplicate or |
194 | * failure. |
195 | */ |
196 | void *klp_shadow_alloc(void *obj, unsigned long id, |
197 | size_t size, gfp_t gfp_flags, |
198 | klp_shadow_ctor_t ctor, void *ctor_data) |
199 | { |
200 | return __klp_shadow_get_or_alloc(obj, id, size, gfp_flags, |
201 | ctor, ctor_data, warn_on_exist: true); |
202 | } |
203 | EXPORT_SYMBOL_GPL(klp_shadow_alloc); |
204 | |
205 | /** |
206 | * klp_shadow_get_or_alloc() - get existing or allocate a new shadow variable |
207 | * @obj: pointer to parent object |
208 | * @id: data identifier |
209 | * @size: size of attached data |
210 | * @gfp_flags: GFP mask for allocation |
211 | * @ctor: custom constructor to initialize the shadow data (optional) |
212 | * @ctor_data: pointer to any data needed by @ctor (optional) |
213 | * |
214 | * Returns a pointer to existing shadow data if an <obj, id> shadow |
215 | * variable is already present. Otherwise, it creates a new shadow |
216 | * variable like klp_shadow_alloc(). |
217 | * |
218 | * This function guarantees that only one shadow variable exists with the given |
219 | * @id for the given @obj. It also guarantees that the constructor function |
220 | * will be called only when the variable did not exist before. The cost is |
221 | * that @ctor is called in atomic context under a spin lock. |
222 | * |
223 | * Return: the shadow variable data element, NULL on failure. |
224 | */ |
225 | void *klp_shadow_get_or_alloc(void *obj, unsigned long id, |
226 | size_t size, gfp_t gfp_flags, |
227 | klp_shadow_ctor_t ctor, void *ctor_data) |
228 | { |
229 | return __klp_shadow_get_or_alloc(obj, id, size, gfp_flags, |
230 | ctor, ctor_data, warn_on_exist: false); |
231 | } |
232 | EXPORT_SYMBOL_GPL(klp_shadow_get_or_alloc); |
233 | |
234 | static void klp_shadow_free_struct(struct klp_shadow *shadow, |
235 | klp_shadow_dtor_t dtor) |
236 | { |
237 | hash_del_rcu(node: &shadow->node); |
238 | if (dtor) |
239 | dtor(shadow->obj, shadow->data); |
240 | kfree_rcu(shadow, rcu_head); |
241 | } |
242 | |
243 | /** |
244 | * klp_shadow_free() - detach and free a <obj, id> shadow variable |
245 | * @obj: pointer to parent object |
246 | * @id: data identifier |
247 | * @dtor: custom callback that can be used to unregister the variable |
248 | * and/or free data that the shadow variable points to (optional) |
249 | * |
250 | * This function releases the memory for this <obj, id> shadow variable |
251 | * instance, callers should stop referencing it accordingly. |
252 | */ |
253 | void klp_shadow_free(void *obj, unsigned long id, klp_shadow_dtor_t dtor) |
254 | { |
255 | struct klp_shadow *shadow; |
256 | unsigned long flags; |
257 | |
258 | spin_lock_irqsave(&klp_shadow_lock, flags); |
259 | |
260 | /* Delete <obj, id> from hash */ |
261 | hash_for_each_possible(klp_shadow_hash, shadow, node, |
262 | (unsigned long)obj) { |
263 | |
264 | if (klp_shadow_match(shadow, obj, id)) { |
265 | klp_shadow_free_struct(shadow, dtor); |
266 | break; |
267 | } |
268 | } |
269 | |
270 | spin_unlock_irqrestore(lock: &klp_shadow_lock, flags); |
271 | } |
272 | EXPORT_SYMBOL_GPL(klp_shadow_free); |
273 | |
274 | /** |
275 | * klp_shadow_free_all() - detach and free all <_, id> shadow variables |
276 | * @id: data identifier |
277 | * @dtor: custom callback that can be used to unregister the variable |
278 | * and/or free data that the shadow variable points to (optional) |
279 | * |
280 | * This function releases the memory for all <_, id> shadow variable |
281 | * instances, callers should stop referencing them accordingly. |
282 | */ |
283 | void klp_shadow_free_all(unsigned long id, klp_shadow_dtor_t dtor) |
284 | { |
285 | struct klp_shadow *shadow; |
286 | unsigned long flags; |
287 | int i; |
288 | |
289 | spin_lock_irqsave(&klp_shadow_lock, flags); |
290 | |
291 | /* Delete all <_, id> from hash */ |
292 | hash_for_each(klp_shadow_hash, i, shadow, node) { |
293 | if (klp_shadow_match(shadow, obj: shadow->obj, id)) |
294 | klp_shadow_free_struct(shadow, dtor); |
295 | } |
296 | |
297 | spin_unlock_irqrestore(lock: &klp_shadow_lock, flags); |
298 | } |
299 | EXPORT_SYMBOL_GPL(klp_shadow_free_all); |
300 | |