1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | // Copyright (C) 2022 Linutronix GmbH, John Ogness |
3 | // Copyright (C) 2022 Intel, Thomas Gleixner |
4 | |
5 | #include <linux/kernel.h> |
6 | #include <linux/console.h> |
7 | #include <linux/delay.h> |
8 | #include <linux/slab.h> |
9 | #include "internal.h" |
10 | /* |
11 | * Printk console printing implementation for consoles which does not depend |
12 | * on the legacy style console_lock mechanism. |
13 | * |
14 | * The state of the console is maintained in the "nbcon_state" atomic |
15 | * variable. |
16 | * |
17 | * The console is locked when: |
18 | * |
19 | * - The 'prio' field contains the priority of the context that owns the |
20 | * console. Only higher priority contexts are allowed to take over the |
21 | * lock. A value of 0 (NBCON_PRIO_NONE) means the console is not locked. |
22 | * |
23 | * - The 'cpu' field denotes on which CPU the console is locked. It is used |
24 | * to prevent busy waiting on the same CPU. Also it informs the lock owner |
25 | * that it has lost the lock in a more complex scenario when the lock was |
26 | * taken over by a higher priority context, released, and taken on another |
27 | * CPU with the same priority as the interrupted owner. |
28 | * |
29 | * The acquire mechanism uses a few more fields: |
30 | * |
31 | * - The 'req_prio' field is used by the handover approach to make the |
32 | * current owner aware that there is a context with a higher priority |
33 | * waiting for the friendly handover. |
34 | * |
35 | * - The 'unsafe' field allows to take over the console in a safe way in the |
36 | * middle of emitting a message. The field is set only when accessing some |
37 | * shared resources or when the console device is manipulated. It can be |
38 | * cleared, for example, after emitting one character when the console |
39 | * device is in a consistent state. |
40 | * |
41 | * - The 'unsafe_takeover' field is set when a hostile takeover took the |
42 | * console in an unsafe state. The console will stay in the unsafe state |
43 | * until re-initialized. |
44 | * |
45 | * The acquire mechanism uses three approaches: |
46 | * |
47 | * 1) Direct acquire when the console is not owned or is owned by a lower |
48 | * priority context and is in a safe state. |
49 | * |
50 | * 2) Friendly handover mechanism uses a request/grant handshake. It is used |
51 | * when the current owner has lower priority and the console is in an |
52 | * unsafe state. |
53 | * |
54 | * The requesting context: |
55 | * |
56 | * a) Sets its priority into the 'req_prio' field. |
57 | * |
58 | * b) Waits (with a timeout) for the owning context to unlock the |
59 | * console. |
60 | * |
61 | * c) Takes the lock and clears the 'req_prio' field. |
62 | * |
63 | * The owning context: |
64 | * |
65 | * a) Observes the 'req_prio' field set on exit from the unsafe |
66 | * console state. |
67 | * |
68 | * b) Gives up console ownership by clearing the 'prio' field. |
69 | * |
70 | * 3) Unsafe hostile takeover allows to take over the lock even when the |
71 | * console is an unsafe state. It is used only in panic() by the final |
72 | * attempt to flush consoles in a try and hope mode. |
73 | * |
74 | * Note that separate record buffers are used in panic(). As a result, |
75 | * the messages can be read and formatted without any risk even after |
76 | * using the hostile takeover in unsafe state. |
77 | * |
78 | * The release function simply clears the 'prio' field. |
79 | * |
80 | * All operations on @console::nbcon_state are atomic cmpxchg based to |
81 | * handle concurrency. |
82 | * |
83 | * The acquire/release functions implement only minimal policies: |
84 | * |
85 | * - Preference for higher priority contexts. |
86 | * - Protection of the panic CPU. |
87 | * |
88 | * All other policy decisions must be made at the call sites: |
89 | * |
90 | * - What is marked as an unsafe section. |
91 | * - Whether to spin-wait if there is already an owner and the console is |
92 | * in an unsafe state. |
93 | * - Whether to attempt an unsafe hostile takeover. |
94 | * |
95 | * The design allows to implement the well known: |
96 | * |
97 | * acquire() |
98 | * output_one_printk_record() |
99 | * release() |
100 | * |
101 | * The output of one printk record might be interrupted with a higher priority |
102 | * context. The new owner is supposed to reprint the entire interrupted record |
103 | * from scratch. |
104 | */ |
105 | |
106 | /** |
107 | * nbcon_state_set - Helper function to set the console state |
108 | * @con: Console to update |
109 | * @new: The new state to write |
110 | * |
111 | * Only to be used when the console is not yet or no longer visible in the |
112 | * system. Otherwise use nbcon_state_try_cmpxchg(). |
113 | */ |
114 | static inline void nbcon_state_set(struct console *con, struct nbcon_state *new) |
115 | { |
116 | atomic_set(v: &ACCESS_PRIVATE(con, nbcon_state), i: new->atom); |
117 | } |
118 | |
119 | /** |
120 | * nbcon_state_read - Helper function to read the console state |
121 | * @con: Console to read |
122 | * @state: The state to store the result |
123 | */ |
124 | static inline void nbcon_state_read(struct console *con, struct nbcon_state *state) |
125 | { |
126 | state->atom = atomic_read(v: &ACCESS_PRIVATE(con, nbcon_state)); |
127 | } |
128 | |
129 | /** |
130 | * nbcon_state_try_cmpxchg() - Helper function for atomic_try_cmpxchg() on console state |
131 | * @con: Console to update |
132 | * @cur: Old/expected state |
133 | * @new: New state |
134 | * |
135 | * Return: True on success. False on fail and @cur is updated. |
136 | */ |
137 | static inline bool nbcon_state_try_cmpxchg(struct console *con, struct nbcon_state *cur, |
138 | struct nbcon_state *new) |
139 | { |
140 | return atomic_try_cmpxchg(v: &ACCESS_PRIVATE(con, nbcon_state), old: &cur->atom, new: new->atom); |
141 | } |
142 | |
143 | /** |
144 | * nbcon_seq_read - Read the current console sequence |
145 | * @con: Console to read the sequence of |
146 | * |
147 | * Return: Sequence number of the next record to print on @con. |
148 | */ |
149 | u64 nbcon_seq_read(struct console *con) |
150 | { |
151 | unsigned long nbcon_seq = atomic_long_read(v: &ACCESS_PRIVATE(con, nbcon_seq)); |
152 | |
153 | return __ulseq_to_u64seq(prb, nbcon_seq); |
154 | } |
155 | |
156 | /** |
157 | * nbcon_seq_force - Force console sequence to a specific value |
158 | * @con: Console to work on |
159 | * @seq: Sequence number value to set |
160 | * |
161 | * Only to be used during init (before registration) or in extreme situations |
162 | * (such as panic with CONSOLE_REPLAY_ALL). |
163 | */ |
164 | void nbcon_seq_force(struct console *con, u64 seq) |
165 | { |
166 | /* |
167 | * If the specified record no longer exists, the oldest available record |
168 | * is chosen. This is especially important on 32bit systems because only |
169 | * the lower 32 bits of the sequence number are stored. The upper 32 bits |
170 | * are derived from the sequence numbers available in the ringbuffer. |
171 | */ |
172 | u64 valid_seq = max_t(u64, seq, prb_first_valid_seq(prb)); |
173 | |
174 | atomic_long_set(v: &ACCESS_PRIVATE(con, nbcon_seq), __u64seq_to_ulseq(valid_seq)); |
175 | |
176 | /* Clear con->seq since nbcon consoles use con->nbcon_seq instead. */ |
177 | con->seq = 0; |
178 | } |
179 | |
180 | /** |
181 | * nbcon_seq_try_update - Try to update the console sequence number |
182 | * @ctxt: Pointer to an acquire context that contains |
183 | * all information about the acquire mode |
184 | * @new_seq: The new sequence number to set |
185 | * |
186 | * @ctxt->seq is updated to the new value of @con::nbcon_seq (expanded to |
187 | * the 64bit value). This could be a different value than @new_seq if |
188 | * nbcon_seq_force() was used or the current context no longer owns the |
189 | * console. In the later case, it will stop printing anyway. |
190 | */ |
191 | static void nbcon_seq_try_update(struct nbcon_context *ctxt, u64 new_seq) |
192 | { |
193 | unsigned long nbcon_seq = __u64seq_to_ulseq(ctxt->seq); |
194 | struct console *con = ctxt->console; |
195 | |
196 | if (atomic_long_try_cmpxchg(v: &ACCESS_PRIVATE(con, nbcon_seq), old: &nbcon_seq, |
197 | __u64seq_to_ulseq(new_seq))) { |
198 | ctxt->seq = new_seq; |
199 | } else { |
200 | ctxt->seq = nbcon_seq_read(con); |
201 | } |
202 | } |
203 | |
204 | /** |
205 | * nbcon_context_try_acquire_direct - Try to acquire directly |
206 | * @ctxt: The context of the caller |
207 | * @cur: The current console state |
208 | * |
209 | * Acquire the console when it is released. Also acquire the console when |
210 | * the current owner has a lower priority and the console is in a safe state. |
211 | * |
212 | * Return: 0 on success. Otherwise, an error code on failure. Also @cur |
213 | * is updated to the latest state when failed to modify it. |
214 | * |
215 | * Errors: |
216 | * |
217 | * -EPERM: A panic is in progress and this is not the panic CPU. |
218 | * Or the current owner or waiter has the same or higher |
219 | * priority. No acquire method can be successful in |
220 | * this case. |
221 | * |
222 | * -EBUSY: The current owner has a lower priority but the console |
223 | * in an unsafe state. The caller should try using |
224 | * the handover acquire method. |
225 | */ |
226 | static int nbcon_context_try_acquire_direct(struct nbcon_context *ctxt, |
227 | struct nbcon_state *cur) |
228 | { |
229 | unsigned int cpu = smp_processor_id(); |
230 | struct console *con = ctxt->console; |
231 | struct nbcon_state new; |
232 | |
233 | do { |
234 | if (other_cpu_in_panic()) |
235 | return -EPERM; |
236 | |
237 | if (ctxt->prio <= cur->prio || ctxt->prio <= cur->req_prio) |
238 | return -EPERM; |
239 | |
240 | if (cur->unsafe) |
241 | return -EBUSY; |
242 | |
243 | /* |
244 | * The console should never be safe for a direct acquire |
245 | * if an unsafe hostile takeover has ever happened. |
246 | */ |
247 | WARN_ON_ONCE(cur->unsafe_takeover); |
248 | |
249 | new.atom = cur->atom; |
250 | new.prio = ctxt->prio; |
251 | new.req_prio = NBCON_PRIO_NONE; |
252 | new.unsafe = cur->unsafe_takeover; |
253 | new.cpu = cpu; |
254 | |
255 | } while (!nbcon_state_try_cmpxchg(con, cur, new: &new)); |
256 | |
257 | return 0; |
258 | } |
259 | |
260 | static bool nbcon_waiter_matches(struct nbcon_state *cur, int expected_prio) |
261 | { |
262 | /* |
263 | * The request context is well defined by the @req_prio because: |
264 | * |
265 | * - Only a context with a higher priority can take over the request. |
266 | * - There are only three priorities. |
267 | * - Only one CPU is allowed to request PANIC priority. |
268 | * - Lower priorities are ignored during panic() until reboot. |
269 | * |
270 | * As a result, the following scenario is *not* possible: |
271 | * |
272 | * 1. Another context with a higher priority directly takes ownership. |
273 | * 2. The higher priority context releases the ownership. |
274 | * 3. A lower priority context takes the ownership. |
275 | * 4. Another context with the same priority as this context |
276 | * creates a request and starts waiting. |
277 | */ |
278 | |
279 | return (cur->req_prio == expected_prio); |
280 | } |
281 | |
282 | /** |
283 | * nbcon_context_try_acquire_requested - Try to acquire after having |
284 | * requested a handover |
285 | * @ctxt: The context of the caller |
286 | * @cur: The current console state |
287 | * |
288 | * This is a helper function for nbcon_context_try_acquire_handover(). |
289 | * It is called when the console is in an unsafe state. The current |
290 | * owner will release the console on exit from the unsafe region. |
291 | * |
292 | * Return: 0 on success and @cur is updated to the new console state. |
293 | * Otherwise an error code on failure. |
294 | * |
295 | * Errors: |
296 | * |
297 | * -EPERM: A panic is in progress and this is not the panic CPU |
298 | * or this context is no longer the waiter. |
299 | * |
300 | * -EBUSY: The console is still locked. The caller should |
301 | * continue waiting. |
302 | * |
303 | * Note: The caller must still remove the request when an error has occurred |
304 | * except when this context is no longer the waiter. |
305 | */ |
306 | static int nbcon_context_try_acquire_requested(struct nbcon_context *ctxt, |
307 | struct nbcon_state *cur) |
308 | { |
309 | unsigned int cpu = smp_processor_id(); |
310 | struct console *con = ctxt->console; |
311 | struct nbcon_state new; |
312 | |
313 | /* Note that the caller must still remove the request! */ |
314 | if (other_cpu_in_panic()) |
315 | return -EPERM; |
316 | |
317 | /* |
318 | * Note that the waiter will also change if there was an unsafe |
319 | * hostile takeover. |
320 | */ |
321 | if (!nbcon_waiter_matches(cur, expected_prio: ctxt->prio)) |
322 | return -EPERM; |
323 | |
324 | /* If still locked, caller should continue waiting. */ |
325 | if (cur->prio != NBCON_PRIO_NONE) |
326 | return -EBUSY; |
327 | |
328 | /* |
329 | * The previous owner should have never released ownership |
330 | * in an unsafe region. |
331 | */ |
332 | WARN_ON_ONCE(cur->unsafe); |
333 | |
334 | new.atom = cur->atom; |
335 | new.prio = ctxt->prio; |
336 | new.req_prio = NBCON_PRIO_NONE; |
337 | new.unsafe = cur->unsafe_takeover; |
338 | new.cpu = cpu; |
339 | |
340 | if (!nbcon_state_try_cmpxchg(con, cur, new: &new)) { |
341 | /* |
342 | * The acquire could fail only when it has been taken |
343 | * over by a higher priority context. |
344 | */ |
345 | WARN_ON_ONCE(nbcon_waiter_matches(cur, ctxt->prio)); |
346 | return -EPERM; |
347 | } |
348 | |
349 | /* Handover success. This context now owns the console. */ |
350 | return 0; |
351 | } |
352 | |
353 | /** |
354 | * nbcon_context_try_acquire_handover - Try to acquire via handover |
355 | * @ctxt: The context of the caller |
356 | * @cur: The current console state |
357 | * |
358 | * The function must be called only when the context has higher priority |
359 | * than the current owner and the console is in an unsafe state. |
360 | * It is the case when nbcon_context_try_acquire_direct() returns -EBUSY. |
361 | * |
362 | * The function sets "req_prio" field to make the current owner aware of |
363 | * the request. Then it waits until the current owner releases the console, |
364 | * or an even higher context takes over the request, or timeout expires. |
365 | * |
366 | * The current owner checks the "req_prio" field on exit from the unsafe |
367 | * region and releases the console. It does not touch the "req_prio" field |
368 | * so that the console stays reserved for the waiter. |
369 | * |
370 | * Return: 0 on success. Otherwise, an error code on failure. Also @cur |
371 | * is updated to the latest state when failed to modify it. |
372 | * |
373 | * Errors: |
374 | * |
375 | * -EPERM: A panic is in progress and this is not the panic CPU. |
376 | * Or a higher priority context has taken over the |
377 | * console or the handover request. |
378 | * |
379 | * -EBUSY: The current owner is on the same CPU so that the hand |
380 | * shake could not work. Or the current owner is not |
381 | * willing to wait (zero timeout). Or the console does |
382 | * not enter the safe state before timeout passed. The |
383 | * caller might still use the unsafe hostile takeover |
384 | * when allowed. |
385 | * |
386 | * -EAGAIN: @cur has changed when creating the handover request. |
387 | * The caller should retry with direct acquire. |
388 | */ |
389 | static int nbcon_context_try_acquire_handover(struct nbcon_context *ctxt, |
390 | struct nbcon_state *cur) |
391 | { |
392 | unsigned int cpu = smp_processor_id(); |
393 | struct console *con = ctxt->console; |
394 | struct nbcon_state new; |
395 | int timeout; |
396 | int request_err = -EBUSY; |
397 | |
398 | /* |
399 | * Check that the handover is called when the direct acquire failed |
400 | * with -EBUSY. |
401 | */ |
402 | WARN_ON_ONCE(ctxt->prio <= cur->prio || ctxt->prio <= cur->req_prio); |
403 | WARN_ON_ONCE(!cur->unsafe); |
404 | |
405 | /* Handover is not possible on the same CPU. */ |
406 | if (cur->cpu == cpu) |
407 | return -EBUSY; |
408 | |
409 | /* |
410 | * Console stays unsafe after an unsafe takeover until re-initialized. |
411 | * Waiting is not going to help in this case. |
412 | */ |
413 | if (cur->unsafe_takeover) |
414 | return -EBUSY; |
415 | |
416 | /* Is the caller willing to wait? */ |
417 | if (ctxt->spinwait_max_us == 0) |
418 | return -EBUSY; |
419 | |
420 | /* |
421 | * Setup a request for the handover. The caller should try to acquire |
422 | * the console directly when the current state has been modified. |
423 | */ |
424 | new.atom = cur->atom; |
425 | new.req_prio = ctxt->prio; |
426 | if (!nbcon_state_try_cmpxchg(con, cur, new: &new)) |
427 | return -EAGAIN; |
428 | |
429 | cur->atom = new.atom; |
430 | |
431 | /* Wait until there is no owner and then acquire the console. */ |
432 | for (timeout = ctxt->spinwait_max_us; timeout >= 0; timeout--) { |
433 | /* On successful acquire, this request is cleared. */ |
434 | request_err = nbcon_context_try_acquire_requested(ctxt, cur); |
435 | if (!request_err) |
436 | return 0; |
437 | |
438 | /* |
439 | * If the acquire should be aborted, it must be ensured |
440 | * that the request is removed before returning to caller. |
441 | */ |
442 | if (request_err == -EPERM) |
443 | break; |
444 | |
445 | udelay(1); |
446 | |
447 | /* Re-read the state because some time has passed. */ |
448 | nbcon_state_read(con, state: cur); |
449 | } |
450 | |
451 | /* Timed out or aborted. Carefully remove handover request. */ |
452 | do { |
453 | /* |
454 | * No need to remove request if there is a new waiter. This |
455 | * can only happen if a higher priority context has taken over |
456 | * the console or the handover request. |
457 | */ |
458 | if (!nbcon_waiter_matches(cur, expected_prio: ctxt->prio)) |
459 | return -EPERM; |
460 | |
461 | /* Unset request for handover. */ |
462 | new.atom = cur->atom; |
463 | new.req_prio = NBCON_PRIO_NONE; |
464 | if (nbcon_state_try_cmpxchg(con, cur, new: &new)) { |
465 | /* |
466 | * Request successfully unset. Report failure of |
467 | * acquiring via handover. |
468 | */ |
469 | cur->atom = new.atom; |
470 | return request_err; |
471 | } |
472 | |
473 | /* |
474 | * Unable to remove request. Try to acquire in case |
475 | * the owner has released the lock. |
476 | */ |
477 | } while (nbcon_context_try_acquire_requested(ctxt, cur)); |
478 | |
479 | /* Lucky timing. The acquire succeeded while removing the request. */ |
480 | return 0; |
481 | } |
482 | |
483 | /** |
484 | * nbcon_context_try_acquire_hostile - Acquire via unsafe hostile takeover |
485 | * @ctxt: The context of the caller |
486 | * @cur: The current console state |
487 | * |
488 | * Acquire the console even in the unsafe state. |
489 | * |
490 | * It can be permitted by setting the 'allow_unsafe_takeover' field only |
491 | * by the final attempt to flush messages in panic(). |
492 | * |
493 | * Return: 0 on success. -EPERM when not allowed by the context. |
494 | */ |
495 | static int nbcon_context_try_acquire_hostile(struct nbcon_context *ctxt, |
496 | struct nbcon_state *cur) |
497 | { |
498 | unsigned int cpu = smp_processor_id(); |
499 | struct console *con = ctxt->console; |
500 | struct nbcon_state new; |
501 | |
502 | if (!ctxt->allow_unsafe_takeover) |
503 | return -EPERM; |
504 | |
505 | /* Ensure caller is allowed to perform unsafe hostile takeovers. */ |
506 | if (WARN_ON_ONCE(ctxt->prio != NBCON_PRIO_PANIC)) |
507 | return -EPERM; |
508 | |
509 | /* |
510 | * Check that try_acquire_direct() and try_acquire_handover() returned |
511 | * -EBUSY in the right situation. |
512 | */ |
513 | WARN_ON_ONCE(ctxt->prio <= cur->prio || ctxt->prio <= cur->req_prio); |
514 | WARN_ON_ONCE(cur->unsafe != true); |
515 | |
516 | do { |
517 | new.atom = cur->atom; |
518 | new.cpu = cpu; |
519 | new.prio = ctxt->prio; |
520 | new.unsafe |= cur->unsafe_takeover; |
521 | new.unsafe_takeover |= cur->unsafe; |
522 | |
523 | } while (!nbcon_state_try_cmpxchg(con, cur, new: &new)); |
524 | |
525 | return 0; |
526 | } |
527 | |
528 | static struct printk_buffers panic_nbcon_pbufs; |
529 | |
530 | /** |
531 | * nbcon_context_try_acquire - Try to acquire nbcon console |
532 | * @ctxt: The context of the caller |
533 | * |
534 | * Return: True if the console was acquired. False otherwise. |
535 | * |
536 | * If the caller allowed an unsafe hostile takeover, on success the |
537 | * caller should check the current console state to see if it is |
538 | * in an unsafe state. Otherwise, on success the caller may assume |
539 | * the console is not in an unsafe state. |
540 | */ |
541 | __maybe_unused |
542 | static bool nbcon_context_try_acquire(struct nbcon_context *ctxt) |
543 | { |
544 | unsigned int cpu = smp_processor_id(); |
545 | struct console *con = ctxt->console; |
546 | struct nbcon_state cur; |
547 | int err; |
548 | |
549 | nbcon_state_read(con, state: &cur); |
550 | try_again: |
551 | err = nbcon_context_try_acquire_direct(ctxt, cur: &cur); |
552 | if (err != -EBUSY) |
553 | goto out; |
554 | |
555 | err = nbcon_context_try_acquire_handover(ctxt, cur: &cur); |
556 | if (err == -EAGAIN) |
557 | goto try_again; |
558 | if (err != -EBUSY) |
559 | goto out; |
560 | |
561 | err = nbcon_context_try_acquire_hostile(ctxt, cur: &cur); |
562 | out: |
563 | if (err) |
564 | return false; |
565 | |
566 | /* Acquire succeeded. */ |
567 | |
568 | /* Assign the appropriate buffer for this context. */ |
569 | if (atomic_read(v: &panic_cpu) == cpu) |
570 | ctxt->pbufs = &panic_nbcon_pbufs; |
571 | else |
572 | ctxt->pbufs = con->pbufs; |
573 | |
574 | /* Set the record sequence for this context to print. */ |
575 | ctxt->seq = nbcon_seq_read(con: ctxt->console); |
576 | |
577 | return true; |
578 | } |
579 | |
580 | static bool nbcon_owner_matches(struct nbcon_state *cur, int expected_cpu, |
581 | int expected_prio) |
582 | { |
583 | /* |
584 | * Since consoles can only be acquired by higher priorities, |
585 | * owning contexts are uniquely identified by @prio. However, |
586 | * since contexts can unexpectedly lose ownership, it is |
587 | * possible that later another owner appears with the same |
588 | * priority. For this reason @cpu is also needed. |
589 | */ |
590 | |
591 | if (cur->prio != expected_prio) |
592 | return false; |
593 | |
594 | if (cur->cpu != expected_cpu) |
595 | return false; |
596 | |
597 | return true; |
598 | } |
599 | |
600 | /** |
601 | * nbcon_context_release - Release the console |
602 | * @ctxt: The nbcon context from nbcon_context_try_acquire() |
603 | */ |
604 | static void nbcon_context_release(struct nbcon_context *ctxt) |
605 | { |
606 | unsigned int cpu = smp_processor_id(); |
607 | struct console *con = ctxt->console; |
608 | struct nbcon_state cur; |
609 | struct nbcon_state new; |
610 | |
611 | nbcon_state_read(con, state: &cur); |
612 | |
613 | do { |
614 | if (!nbcon_owner_matches(cur: &cur, expected_cpu: cpu, expected_prio: ctxt->prio)) |
615 | break; |
616 | |
617 | new.atom = cur.atom; |
618 | new.prio = NBCON_PRIO_NONE; |
619 | |
620 | /* |
621 | * If @unsafe_takeover is set, it is kept set so that |
622 | * the state remains permanently unsafe. |
623 | */ |
624 | new.unsafe |= cur.unsafe_takeover; |
625 | |
626 | } while (!nbcon_state_try_cmpxchg(con, cur: &cur, new: &new)); |
627 | |
628 | ctxt->pbufs = NULL; |
629 | } |
630 | |
631 | /** |
632 | * nbcon_context_can_proceed - Check whether ownership can proceed |
633 | * @ctxt: The nbcon context from nbcon_context_try_acquire() |
634 | * @cur: The current console state |
635 | * |
636 | * Return: True if this context still owns the console. False if |
637 | * ownership was handed over or taken. |
638 | * |
639 | * Must be invoked when entering the unsafe state to make sure that it still |
640 | * owns the lock. Also must be invoked when exiting the unsafe context |
641 | * to eventually free the lock for a higher priority context which asked |
642 | * for the friendly handover. |
643 | * |
644 | * It can be called inside an unsafe section when the console is just |
645 | * temporary in safe state instead of exiting and entering the unsafe |
646 | * state. |
647 | * |
648 | * Also it can be called in the safe context before doing an expensive |
649 | * safe operation. It does not make sense to do the operation when |
650 | * a higher priority context took the lock. |
651 | * |
652 | * When this function returns false then the calling context no longer owns |
653 | * the console and is no longer allowed to go forward. In this case it must |
654 | * back out immediately and carefully. The buffer content is also no longer |
655 | * trusted since it no longer belongs to the calling context. |
656 | */ |
657 | static bool nbcon_context_can_proceed(struct nbcon_context *ctxt, struct nbcon_state *cur) |
658 | { |
659 | unsigned int cpu = smp_processor_id(); |
660 | |
661 | /* Make sure this context still owns the console. */ |
662 | if (!nbcon_owner_matches(cur, expected_cpu: cpu, expected_prio: ctxt->prio)) |
663 | return false; |
664 | |
665 | /* The console owner can proceed if there is no waiter. */ |
666 | if (cur->req_prio == NBCON_PRIO_NONE) |
667 | return true; |
668 | |
669 | /* |
670 | * A console owner within an unsafe region is always allowed to |
671 | * proceed, even if there are waiters. It can perform a handover |
672 | * when exiting the unsafe region. Otherwise the waiter will |
673 | * need to perform an unsafe hostile takeover. |
674 | */ |
675 | if (cur->unsafe) |
676 | return true; |
677 | |
678 | /* Waiters always have higher priorities than owners. */ |
679 | WARN_ON_ONCE(cur->req_prio <= cur->prio); |
680 | |
681 | /* |
682 | * Having a safe point for take over and eventually a few |
683 | * duplicated characters or a full line is way better than a |
684 | * hostile takeover. Post processing can take care of the garbage. |
685 | * Release and hand over. |
686 | */ |
687 | nbcon_context_release(ctxt); |
688 | |
689 | /* |
690 | * It is not clear whether the waiter really took over ownership. The |
691 | * outermost callsite must make the final decision whether console |
692 | * ownership is needed for it to proceed. If yes, it must reacquire |
693 | * ownership (possibly hostile) before carefully proceeding. |
694 | * |
695 | * The calling context no longer owns the console so go back all the |
696 | * way instead of trying to implement reacquire heuristics in tons of |
697 | * places. |
698 | */ |
699 | return false; |
700 | } |
701 | |
702 | /** |
703 | * nbcon_can_proceed - Check whether ownership can proceed |
704 | * @wctxt: The write context that was handed to the write function |
705 | * |
706 | * Return: True if this context still owns the console. False if |
707 | * ownership was handed over or taken. |
708 | * |
709 | * It is used in nbcon_enter_unsafe() to make sure that it still owns the |
710 | * lock. Also it is used in nbcon_exit_unsafe() to eventually free the lock |
711 | * for a higher priority context which asked for the friendly handover. |
712 | * |
713 | * It can be called inside an unsafe section when the console is just |
714 | * temporary in safe state instead of exiting and entering the unsafe state. |
715 | * |
716 | * Also it can be called in the safe context before doing an expensive safe |
717 | * operation. It does not make sense to do the operation when a higher |
718 | * priority context took the lock. |
719 | * |
720 | * When this function returns false then the calling context no longer owns |
721 | * the console and is no longer allowed to go forward. In this case it must |
722 | * back out immediately and carefully. The buffer content is also no longer |
723 | * trusted since it no longer belongs to the calling context. |
724 | */ |
725 | bool nbcon_can_proceed(struct nbcon_write_context *wctxt) |
726 | { |
727 | struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt); |
728 | struct console *con = ctxt->console; |
729 | struct nbcon_state cur; |
730 | |
731 | nbcon_state_read(con, state: &cur); |
732 | |
733 | return nbcon_context_can_proceed(ctxt, cur: &cur); |
734 | } |
735 | EXPORT_SYMBOL_GPL(nbcon_can_proceed); |
736 | |
737 | #define nbcon_context_enter_unsafe(c) __nbcon_context_update_unsafe(c, true) |
738 | #define nbcon_context_exit_unsafe(c) __nbcon_context_update_unsafe(c, false) |
739 | |
740 | /** |
741 | * __nbcon_context_update_unsafe - Update the unsafe bit in @con->nbcon_state |
742 | * @ctxt: The nbcon context from nbcon_context_try_acquire() |
743 | * @unsafe: The new value for the unsafe bit |
744 | * |
745 | * Return: True if the unsafe state was updated and this context still |
746 | * owns the console. Otherwise false if ownership was handed |
747 | * over or taken. |
748 | * |
749 | * This function allows console owners to modify the unsafe status of the |
750 | * console. |
751 | * |
752 | * When this function returns false then the calling context no longer owns |
753 | * the console and is no longer allowed to go forward. In this case it must |
754 | * back out immediately and carefully. The buffer content is also no longer |
755 | * trusted since it no longer belongs to the calling context. |
756 | * |
757 | * Internal helper to avoid duplicated code. |
758 | */ |
759 | static bool __nbcon_context_update_unsafe(struct nbcon_context *ctxt, bool unsafe) |
760 | { |
761 | struct console *con = ctxt->console; |
762 | struct nbcon_state cur; |
763 | struct nbcon_state new; |
764 | |
765 | nbcon_state_read(con, state: &cur); |
766 | |
767 | do { |
768 | /* |
769 | * The unsafe bit must not be cleared if an |
770 | * unsafe hostile takeover has occurred. |
771 | */ |
772 | if (!unsafe && cur.unsafe_takeover) |
773 | goto out; |
774 | |
775 | if (!nbcon_context_can_proceed(ctxt, cur: &cur)) |
776 | return false; |
777 | |
778 | new.atom = cur.atom; |
779 | new.unsafe = unsafe; |
780 | } while (!nbcon_state_try_cmpxchg(con, cur: &cur, new: &new)); |
781 | |
782 | cur.atom = new.atom; |
783 | out: |
784 | return nbcon_context_can_proceed(ctxt, cur: &cur); |
785 | } |
786 | |
787 | /** |
788 | * nbcon_enter_unsafe - Enter an unsafe region in the driver |
789 | * @wctxt: The write context that was handed to the write function |
790 | * |
791 | * Return: True if this context still owns the console. False if |
792 | * ownership was handed over or taken. |
793 | * |
794 | * When this function returns false then the calling context no longer owns |
795 | * the console and is no longer allowed to go forward. In this case it must |
796 | * back out immediately and carefully. The buffer content is also no longer |
797 | * trusted since it no longer belongs to the calling context. |
798 | */ |
799 | bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt) |
800 | { |
801 | struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt); |
802 | |
803 | return nbcon_context_enter_unsafe(ctxt); |
804 | } |
805 | EXPORT_SYMBOL_GPL(nbcon_enter_unsafe); |
806 | |
807 | /** |
808 | * nbcon_exit_unsafe - Exit an unsafe region in the driver |
809 | * @wctxt: The write context that was handed to the write function |
810 | * |
811 | * Return: True if this context still owns the console. False if |
812 | * ownership was handed over or taken. |
813 | * |
814 | * When this function returns false then the calling context no longer owns |
815 | * the console and is no longer allowed to go forward. In this case it must |
816 | * back out immediately and carefully. The buffer content is also no longer |
817 | * trusted since it no longer belongs to the calling context. |
818 | */ |
819 | bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt) |
820 | { |
821 | struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt); |
822 | |
823 | return nbcon_context_exit_unsafe(ctxt); |
824 | } |
825 | EXPORT_SYMBOL_GPL(nbcon_exit_unsafe); |
826 | |
827 | /** |
828 | * nbcon_emit_next_record - Emit a record in the acquired context |
829 | * @wctxt: The write context that will be handed to the write function |
830 | * |
831 | * Return: True if this context still owns the console. False if |
832 | * ownership was handed over or taken. |
833 | * |
834 | * When this function returns false then the calling context no longer owns |
835 | * the console and is no longer allowed to go forward. In this case it must |
836 | * back out immediately and carefully. The buffer content is also no longer |
837 | * trusted since it no longer belongs to the calling context. If the caller |
838 | * wants to do more it must reacquire the console first. |
839 | * |
840 | * When true is returned, @wctxt->ctxt.backlog indicates whether there are |
841 | * still records pending in the ringbuffer, |
842 | */ |
843 | __maybe_unused |
844 | static bool nbcon_emit_next_record(struct nbcon_write_context *wctxt) |
845 | { |
846 | struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt); |
847 | struct console *con = ctxt->console; |
848 | bool is_extended = console_srcu_read_flags(con) & CON_EXTENDED; |
849 | struct printk_message pmsg = { |
850 | .pbufs = ctxt->pbufs, |
851 | }; |
852 | unsigned long con_dropped; |
853 | struct nbcon_state cur; |
854 | unsigned long dropped; |
855 | bool done; |
856 | |
857 | /* |
858 | * The printk buffers are filled within an unsafe section. This |
859 | * prevents NBCON_PRIO_NORMAL and NBCON_PRIO_EMERGENCY from |
860 | * clobbering each other. |
861 | */ |
862 | |
863 | if (!nbcon_context_enter_unsafe(ctxt)) |
864 | return false; |
865 | |
866 | ctxt->backlog = printk_get_next_message(pmsg: &pmsg, seq: ctxt->seq, is_extended, may_supress: true); |
867 | if (!ctxt->backlog) |
868 | return nbcon_context_exit_unsafe(ctxt); |
869 | |
870 | /* |
871 | * @con->dropped is not protected in case of an unsafe hostile |
872 | * takeover. In that situation the update can be racy so |
873 | * annotate it accordingly. |
874 | */ |
875 | con_dropped = data_race(READ_ONCE(con->dropped)); |
876 | |
877 | dropped = con_dropped + pmsg.dropped; |
878 | if (dropped && !is_extended) |
879 | console_prepend_dropped(pmsg: &pmsg, dropped); |
880 | |
881 | if (!nbcon_context_exit_unsafe(ctxt)) |
882 | return false; |
883 | |
884 | /* For skipped records just update seq/dropped in @con. */ |
885 | if (pmsg.outbuf_len == 0) |
886 | goto update_con; |
887 | |
888 | /* Initialize the write context for driver callbacks. */ |
889 | wctxt->outbuf = &pmsg.pbufs->outbuf[0]; |
890 | wctxt->len = pmsg.outbuf_len; |
891 | nbcon_state_read(con, state: &cur); |
892 | wctxt->unsafe_takeover = cur.unsafe_takeover; |
893 | |
894 | if (con->write_atomic) { |
895 | done = con->write_atomic(con, wctxt); |
896 | } else { |
897 | nbcon_context_release(ctxt); |
898 | WARN_ON_ONCE(1); |
899 | done = false; |
900 | } |
901 | |
902 | /* If not done, the emit was aborted. */ |
903 | if (!done) |
904 | return false; |
905 | |
906 | /* |
907 | * Since any dropped message was successfully output, reset the |
908 | * dropped count for the console. |
909 | */ |
910 | dropped = 0; |
911 | update_con: |
912 | /* |
913 | * The dropped count and the sequence number are updated within an |
914 | * unsafe section. This limits update races to the panic context and |
915 | * allows the panic context to win. |
916 | */ |
917 | |
918 | if (!nbcon_context_enter_unsafe(ctxt)) |
919 | return false; |
920 | |
921 | if (dropped != con_dropped) { |
922 | /* Counterpart to the READ_ONCE() above. */ |
923 | WRITE_ONCE(con->dropped, dropped); |
924 | } |
925 | |
926 | nbcon_seq_try_update(ctxt, new_seq: pmsg.seq + 1); |
927 | |
928 | return nbcon_context_exit_unsafe(ctxt); |
929 | } |
930 | |
931 | /** |
932 | * nbcon_alloc - Allocate buffers needed by the nbcon console |
933 | * @con: Console to allocate buffers for |
934 | * |
935 | * Return: True on success. False otherwise and the console cannot |
936 | * be used. |
937 | * |
938 | * This is not part of nbcon_init() because buffer allocation must |
939 | * be performed earlier in the console registration process. |
940 | */ |
941 | bool nbcon_alloc(struct console *con) |
942 | { |
943 | if (con->flags & CON_BOOT) { |
944 | /* |
945 | * Boot console printing is synchronized with legacy console |
946 | * printing, so boot consoles can share the same global printk |
947 | * buffers. |
948 | */ |
949 | con->pbufs = &printk_shared_pbufs; |
950 | } else { |
951 | con->pbufs = kmalloc(size: sizeof(*con->pbufs), GFP_KERNEL); |
952 | if (!con->pbufs) { |
953 | con_printk(KERN_ERR, con, "failed to allocate printing buffer\n" ); |
954 | return false; |
955 | } |
956 | } |
957 | |
958 | return true; |
959 | } |
960 | |
961 | /** |
962 | * nbcon_init - Initialize the nbcon console specific data |
963 | * @con: Console to initialize |
964 | * |
965 | * nbcon_alloc() *must* be called and succeed before this function |
966 | * is called. |
967 | * |
968 | * This function expects that the legacy @con->seq has been set. |
969 | */ |
970 | void nbcon_init(struct console *con) |
971 | { |
972 | struct nbcon_state state = { }; |
973 | |
974 | /* nbcon_alloc() must have been called and successful! */ |
975 | BUG_ON(!con->pbufs); |
976 | |
977 | nbcon_seq_force(con, seq: con->seq); |
978 | nbcon_state_set(con, new: &state); |
979 | } |
980 | |
981 | /** |
982 | * nbcon_free - Free and cleanup the nbcon console specific data |
983 | * @con: Console to free/cleanup nbcon data |
984 | */ |
985 | void nbcon_free(struct console *con) |
986 | { |
987 | struct nbcon_state state = { }; |
988 | |
989 | nbcon_state_set(con, new: &state); |
990 | |
991 | /* Boot consoles share global printk buffers. */ |
992 | if (!(con->flags & CON_BOOT)) |
993 | kfree(objp: con->pbufs); |
994 | |
995 | con->pbufs = NULL; |
996 | } |
997 | |