1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* linux/kernel/printk.c
4	*
5	* Copyright (C) 1991, 1992 Linus Torvalds
6	*
7	* Modified to make sys_syslog() more flexible: added commands to
8	* return the last 4k of kernel messages, regardless of whether
9	* they've been read or not. Added option to suppress kernel printk's
10	* to the console. Added hook for sending the console messages
11	* elsewhere, in preparation for a serial line console (someday).
12	* Ted Ts'o, 2/11/93.
13	* Modified for sysctl support, 1/8/97, Chris Horn.
14	* Fixed SMP synchronization, 08/08/99, Manfred Spraul
15	* manfred@colorfullife.com
16	* Rewrote bits to get rid of console_lock
17	* 01Mar01 Andrew Morton
18	*/
19
20	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
21
22	#include <linux/kernel.h>
23	#include <linux/mm.h>
24	#include <linux/tty.h>
25	#include <linux/tty_driver.h>
26	#include <linux/console.h>
27	#include <linux/init.h>
28	#include <linux/jiffies.h>
29	#include <linux/nmi.h>
30	#include <linux/module.h>
31	#include <linux/moduleparam.h>
32	#include <linux/delay.h>
33	#include <linux/smp.h>
34	#include <linux/security.h>
35	#include <linux/memblock.h>
36	#include <linux/syscalls.h>
37	#include <linux/crash_core.h>
38	#include <linux/ratelimit.h>
39	#include <linux/kmsg_dump.h>
40	#include <linux/syslog.h>
41	#include <linux/cpu.h>
42	#include <linux/rculist.h>
43	#include <linux/poll.h>
44	#include <linux/irq_work.h>
45	#include <linux/ctype.h>
46	#include <linux/uio.h>
47	#include <linux/sched/clock.h>
48	#include <linux/sched/debug.h>
49	#include <linux/sched/task_stack.h>
50
51	#include <linux/uaccess.h>
52	#include <asm/sections.h>
53
54	#include <trace/events/initcall.h>
55	#define CREATE_TRACE_POINTS
56	#include <trace/events/printk.h>
57
58	#include "printk_ringbuffer.h"
59	#include "console_cmdline.h"
60	#include "braille.h"
61	#include "internal.h"
62
63	int console_printk[4] = {
64	CONSOLE_LOGLEVEL_DEFAULT, /* console_loglevel */
65	MESSAGE_LOGLEVEL_DEFAULT, /* default_message_loglevel */
66	CONSOLE_LOGLEVEL_MIN, /* minimum_console_loglevel */
67	CONSOLE_LOGLEVEL_DEFAULT, /* default_console_loglevel */
68	};
69	EXPORT_SYMBOL_GPL(console_printk);
70
71	atomic_t ignore_console_lock_warning __read_mostly = ATOMIC_INIT(0);
72	EXPORT_SYMBOL(ignore_console_lock_warning);
73
74	EXPORT_TRACEPOINT_SYMBOL_GPL(console);
75
76	/*
77	* Low level drivers may need that to know if they can schedule in
78	* their unblank() callback or not. So let's export it.
79	*/
80	int oops_in_progress;
81	EXPORT_SYMBOL(oops_in_progress);
82
83	/*
84	* console_mutex protects console_list updates and console->flags updates.
85	* The flags are synchronized only for consoles that are registered, i.e.
86	* accessible via the console list.
87	*/
88	static DEFINE_MUTEX(console_mutex);
89
90	/*
91	* console_sem protects updates to console->seq
92	* and also provides serialization for console printing.
93	*/
94	static DEFINE_SEMAPHORE(console_sem, 1);
95	HLIST_HEAD(console_list);
96	EXPORT_SYMBOL_GPL(console_list);
97	DEFINE_STATIC_SRCU(console_srcu);
98
99	/*
100	* System may need to suppress printk message under certain
101	* circumstances, like after kernel panic happens.
102	*/
103	int __read_mostly suppress_printk;
104
105	#ifdef CONFIG_LOCKDEP
106	static struct lockdep_map console_lock_dep_map = {
107	.name = "console_lock"
108	};
109
110	void lockdep_assert_console_list_lock_held(void)
111	{
112	lockdep_assert_held(&console_mutex);
113	}
114	EXPORT_SYMBOL(lockdep_assert_console_list_lock_held);
115	#endif
116
117	#ifdef CONFIG_DEBUG_LOCK_ALLOC
118	bool console_srcu_read_lock_is_held(void)
119	{
120	return srcu_read_lock_held(ssp: &console_srcu);
121	}
122	EXPORT_SYMBOL(console_srcu_read_lock_is_held);
123	#endif
124
125	enum devkmsg_log_bits {
126	__DEVKMSG_LOG_BIT_ON = 0,
127	__DEVKMSG_LOG_BIT_OFF,
128	__DEVKMSG_LOG_BIT_LOCK,
129	};
130
131	enum devkmsg_log_masks {
132	DEVKMSG_LOG_MASK_ON = BIT(__DEVKMSG_LOG_BIT_ON),
133	DEVKMSG_LOG_MASK_OFF = BIT(__DEVKMSG_LOG_BIT_OFF),
134	DEVKMSG_LOG_MASK_LOCK = BIT(__DEVKMSG_LOG_BIT_LOCK),
135	};
136
137	/* Keep both the 'on' and 'off' bits clear, i.e. ratelimit by default: */
138	#define DEVKMSG_LOG_MASK_DEFAULT 0
139
140	static unsigned int __read_mostly devkmsg_log = DEVKMSG_LOG_MASK_DEFAULT;
141
142	static int __control_devkmsg(char *str)
143	{
144	size_t len;
145
146	if (!str)
147	return -EINVAL;
148
149	len = str_has_prefix(str, prefix: "on");
150	if (len) {
151	devkmsg_log = DEVKMSG_LOG_MASK_ON;
152	return len;
153	}
154
155	len = str_has_prefix(str, prefix: "off");
156	if (len) {
157	devkmsg_log = DEVKMSG_LOG_MASK_OFF;
158	return len;
159	}
160
161	len = str_has_prefix(str, prefix: "ratelimit");
162	if (len) {
163	devkmsg_log = DEVKMSG_LOG_MASK_DEFAULT;
164	return len;
165	}
166
167	return -EINVAL;
168	}
169
170	static int __init control_devkmsg(char *str)
171	{
172	if (__control_devkmsg(str) < 0) {
173	pr_warn("printk.devkmsg: bad option string '%s'\n", str);
174	return 1;
175	}
176
177	/*
178	* Set sysctl string accordingly:
179	*/
180	if (devkmsg_log == DEVKMSG_LOG_MASK_ON)
181	strcpy(p: devkmsg_log_str, q: "on");
182	else if (devkmsg_log == DEVKMSG_LOG_MASK_OFF)
183	strcpy(p: devkmsg_log_str, q: "off");
184	/* else "ratelimit" which is set by default. */
185
186	/*
187	* Sysctl cannot change it anymore. The kernel command line setting of
188	* this parameter is to force the setting to be permanent throughout the
189	* runtime of the system. This is a precation measure against userspace
190	* trying to be a smarta** and attempting to change it up on us.
191	*/
192	devkmsg_log |= DEVKMSG_LOG_MASK_LOCK;
193
194	return 1;
195	}
196	__setup("printk.devkmsg=", control_devkmsg);
197
198	char devkmsg_log_str[DEVKMSG_STR_MAX_SIZE] = "ratelimit";
199	#if defined(CONFIG_PRINTK) && defined(CONFIG_SYSCTL)
200	int devkmsg_sysctl_set_loglvl(struct ctl_table *table, int write,
201	void *buffer, size_t *lenp, loff_t *ppos)
202	{
203	char old_str[DEVKMSG_STR_MAX_SIZE];
204	unsigned int old;
205	int err;
206
207	if (write) {
208	if (devkmsg_log & DEVKMSG_LOG_MASK_LOCK)
209	return -EINVAL;
210
211	old = devkmsg_log;
212	strncpy(p: old_str, q: devkmsg_log_str, DEVKMSG_STR_MAX_SIZE);
213	}
214
215	err = proc_dostring(table, write, buffer, lenp, ppos);
216	if (err)
217	return err;
218
219	if (write) {
220	err = __control_devkmsg(str: devkmsg_log_str);
221
222	/*
223	* Do not accept an unknown string OR a known string with
224	* trailing crap...
225	*/
226	if (err < 0 || (err + 1 != *lenp)) {
227
228	/* ... and restore old setting. */
229	devkmsg_log = old;
230	strncpy(p: devkmsg_log_str, q: old_str, DEVKMSG_STR_MAX_SIZE);
231
232	return -EINVAL;
233	}
234	}
235
236	return 0;
237	}
238	#endif /* CONFIG_PRINTK && CONFIG_SYSCTL */
239
240	/**
241	* console_list_lock - Lock the console list
242	*
243	* For console list or console->flags updates
244	*/
245	void console_list_lock(void)
246	{
247	/*
248	* In unregister_console() and console_force_preferred_locked(),
249	* synchronize_srcu() is called with the console_list_lock held.
250	* Therefore it is not allowed that the console_list_lock is taken
251	* with the srcu_lock held.
252	*
253	* Detecting if this context is really in the read-side critical
254	* section is only possible if the appropriate debug options are
255	* enabled.
256	*/
257	WARN_ON_ONCE(debug_lockdep_rcu_enabled() &&
258	srcu_read_lock_held(&console_srcu));
259
260	mutex_lock(&console_mutex);
261	}
262	EXPORT_SYMBOL(console_list_lock);
263
264	/**
265	* console_list_unlock - Unlock the console list
266	*
267	* Counterpart to console_list_lock()
268	*/
269	void console_list_unlock(void)
270	{
271	mutex_unlock(lock: &console_mutex);
272	}
273	EXPORT_SYMBOL(console_list_unlock);
274
275	/**
276	* console_srcu_read_lock - Register a new reader for the
277	* SRCU-protected console list
278	*
279	* Use for_each_console_srcu() to iterate the console list
280	*
281	* Context: Any context.
282	* Return: A cookie to pass to console_srcu_read_unlock().
283	*/
284	int console_srcu_read_lock(void)
285	{
286	return srcu_read_lock_nmisafe(ssp: &console_srcu);
287	}
288	EXPORT_SYMBOL(console_srcu_read_lock);
289
290	/**
291	* console_srcu_read_unlock - Unregister an old reader from
292	* the SRCU-protected console list
293	* @cookie: cookie returned from console_srcu_read_lock()
294	*
295	* Counterpart to console_srcu_read_lock()
296	*/
297	void console_srcu_read_unlock(int cookie)
298	{
299	srcu_read_unlock_nmisafe(ssp: &console_srcu, idx: cookie);
300	}
301	EXPORT_SYMBOL(console_srcu_read_unlock);
302
303	/*
304	* Helper macros to handle lockdep when locking/unlocking console_sem. We use
305	* macros instead of functions so that _RET_IP_ contains useful information.
306	*/
307	#define down_console_sem() do { \
308	down(&console_sem);\
309	mutex_acquire(&console_lock_dep_map, 0, 0, _RET_IP_);\
310	} while (0)
311
312	static int __down_trylock_console_sem(unsigned long ip)
313	{
314	int lock_failed;
315	unsigned long flags;
316
317	/*
318	* Here and in __up_console_sem() we need to be in safe mode,
319	* because spindump/WARN/etc from under console ->lock will
320	* deadlock in printk()->down_trylock_console_sem() otherwise.
321	*/
322	printk_safe_enter_irqsave(flags);
323	lock_failed = down_trylock(sem: &console_sem);
324	printk_safe_exit_irqrestore(flags);
325
326	if (lock_failed)
327	return 1;
328	mutex_acquire(&console_lock_dep_map, 0, 1, ip);
329	return 0;
330	}
331	#define down_trylock_console_sem() __down_trylock_console_sem(_RET_IP_)
332
333	static void __up_console_sem(unsigned long ip)
334	{
335	unsigned long flags;
336
337	mutex_release(&console_lock_dep_map, ip);
338
339	printk_safe_enter_irqsave(flags);
340	up(sem: &console_sem);
341	printk_safe_exit_irqrestore(flags);
342	}
343	#define up_console_sem() __up_console_sem(_RET_IP_)
344
345	static bool panic_in_progress(void)
346	{
347	return unlikely(atomic_read(&panic_cpu) != PANIC_CPU_INVALID);
348	}
349
350	/*
351	* This is used for debugging the mess that is the VT code by
352	* keeping track if we have the console semaphore held. It's
353	* definitely not the perfect debug tool (we don't know if _WE_
354	* hold it and are racing, but it helps tracking those weird code
355	* paths in the console code where we end up in places I want
356	* locked without the console semaphore held).
357	*/
358	static int console_locked;
359
360	/*
361	* Array of consoles built from command line options (console=)
362	*/
363
364	#define MAX_CMDLINECONSOLES 8
365
366	static struct console_cmdline console_cmdline[MAX_CMDLINECONSOLES];
367
368	static int preferred_console = -1;
369	int console_set_on_cmdline;
370	EXPORT_SYMBOL(console_set_on_cmdline);
371
372	/* Flag: console code may call schedule() */
373	static int console_may_schedule;
374
375	enum con_msg_format_flags {
376	MSG_FORMAT_DEFAULT = 0,
377	MSG_FORMAT_SYSLOG = (1 << 0),
378	};
379
380	static int console_msg_format = MSG_FORMAT_DEFAULT;
381
382	/*
383	* The printk log buffer consists of a sequenced collection of records, each
384	* containing variable length message text. Every record also contains its
385	* own meta-data (@info).
386	*
387	* Every record meta-data carries the timestamp in microseconds, as well as
388	* the standard userspace syslog level and syslog facility. The usual kernel
389	* messages use LOG_KERN; userspace-injected messages always carry a matching
390	* syslog facility, by default LOG_USER. The origin of every message can be
391	* reliably determined that way.
392	*
393	* The human readable log message of a record is available in @text, the
394	* length of the message text in @text_len. The stored message is not
395	* terminated.
396	*
397	* Optionally, a record can carry a dictionary of properties (key/value
398	* pairs), to provide userspace with a machine-readable message context.
399	*
400	* Examples for well-defined, commonly used property names are:
401	* DEVICE=b12:8 device identifier
402	* b12:8 block dev_t
403	* c127:3 char dev_t
404	* n8 netdev ifindex
405	* +sound:card0 subsystem:devname
406	* SUBSYSTEM=pci driver-core subsystem name
407	*
408	* Valid characters in property names are [a-zA-Z0-9.-_]. Property names
409	* and values are terminated by a '\0' character.
410	*
411	* Example of record values:
412	* record.text_buf = "it's a line" (unterminated)
413	* record.info.seq = 56
414	* record.info.ts_nsec = 36863
415	* record.info.text_len = 11
416	* record.info.facility = 0 (LOG_KERN)
417	* record.info.flags = 0
418	* record.info.level = 3 (LOG_ERR)
419	* record.info.caller_id = 299 (task 299)
420	* record.info.dev_info.subsystem = "pci" (terminated)
421	* record.info.dev_info.device = "+pci:0000:00:01.0" (terminated)
422	*
423	* The 'struct printk_info' buffer must never be directly exported to
424	* userspace, it is a kernel-private implementation detail that might
425	* need to be changed in the future, when the requirements change.
426	*
427	* /dev/kmsg exports the structured data in the following line format:
428	* "<level>,<sequnum>,<timestamp>,<contflag>[,additional_values, ... ];<message text>\n"
429	*
430	* Users of the export format should ignore possible additional values
431	* separated by ',', and find the message after the ';' character.
432	*
433	* The optional key/value pairs are attached as continuation lines starting
434	* with a space character and terminated by a newline. All possible
435	* non-prinatable characters are escaped in the "\xff" notation.
436	*/
437
438	/* syslog_lock protects syslog_* variables and write access to clear_seq. */
439	static DEFINE_MUTEX(syslog_lock);
440
441	#ifdef CONFIG_PRINTK
442	/*
443	* During panic, heavy printk by other CPUs can delay the
444	* panic and risk deadlock on console resources.
445	*/
446	static int __read_mostly suppress_panic_printk;
447
448	DECLARE_WAIT_QUEUE_HEAD(log_wait);
449	/* All 3 protected by @syslog_lock. */
450	/* the next printk record to read by syslog(READ) or /proc/kmsg */
451	static u64 syslog_seq;
452	static size_t syslog_partial;
453	static bool syslog_time;
454
455	struct latched_seq {
456	seqcount_latch_t latch;
457	u64 val[2];
458	};
459
460	/*
461	* The next printk record to read after the last 'clear' command. There are
462	* two copies (updated with seqcount_latch) so that reads can locklessly
463	* access a valid value. Writers are synchronized by @syslog_lock.
464	*/
465	static struct latched_seq clear_seq = {
466	.latch = SEQCNT_LATCH_ZERO(clear_seq.latch),
467	.val[0] = 0,
468	.val[1] = 0,
469	};
470
471	#define LOG_LEVEL(v) ((v) & 0x07)
472	#define LOG_FACILITY(v) ((v) >> 3 & 0xff)
473
474	/* record buffer */
475	#define LOG_ALIGN __alignof__(unsigned long)
476	#define __LOG_BUF_LEN (1 << CONFIG_LOG_BUF_SHIFT)
477	#define LOG_BUF_LEN_MAX (u32)(1 << 31)
478	static char __log_buf[__LOG_BUF_LEN] __aligned(LOG_ALIGN);
479	static char *log_buf = __log_buf;
480	static u32 log_buf_len = __LOG_BUF_LEN;
481
482	/*
483	* Define the average message size. This only affects the number of
484	* descriptors that will be available. Underestimating is better than
485	* overestimating (too many available descriptors is better than not enough).
486	*/
487	#define PRB_AVGBITS 5 /* 32 character average length */
488
489	#if CONFIG_LOG_BUF_SHIFT <= PRB_AVGBITS
490	#error CONFIG_LOG_BUF_SHIFT value too small.
491	#endif
492	_DEFINE_PRINTKRB(printk_rb_static, CONFIG_LOG_BUF_SHIFT - PRB_AVGBITS,
493	PRB_AVGBITS, &__log_buf[0]);
494
495	static struct printk_ringbuffer printk_rb_dynamic;
496
497	struct printk_ringbuffer *prb = &printk_rb_static;
498
499	/*
500	* We cannot access per-CPU data (e.g. per-CPU flush irq_work) before
501	* per_cpu_areas are initialised. This variable is set to true when
502	* it's safe to access per-CPU data.
503	*/
504	static bool __printk_percpu_data_ready __ro_after_init;
505
506	bool printk_percpu_data_ready(void)
507	{
508	return __printk_percpu_data_ready;
509	}
510
511	/* Must be called under syslog_lock. */
512	static void latched_seq_write(struct latched_seq *ls, u64 val)
513	{
514	raw_write_seqcount_latch(s: &ls->latch);
515	ls->val[0] = val;
516	raw_write_seqcount_latch(s: &ls->latch);
517	ls->val[1] = val;
518	}
519
520	/* Can be called from any context. */
521	static u64 latched_seq_read_nolock(struct latched_seq *ls)
522	{
523	unsigned int seq;
524	unsigned int idx;
525	u64 val;
526
527	do {
528	seq = raw_read_seqcount_latch(s: &ls->latch);
529	idx = seq & 0x1;
530	val = ls->val[idx];
531	} while (raw_read_seqcount_latch_retry(s: &ls->latch, start: seq));
532
533	return val;
534	}
535
536	/* Return log buffer address */
537	char *log_buf_addr_get(void)
538	{
539	return log_buf;
540	}
541
542	/* Return log buffer size */
543	u32 log_buf_len_get(void)
544	{
545	return log_buf_len;
546	}
547
548	/*
549	* Define how much of the log buffer we could take at maximum. The value
550	* must be greater than two. Note that only half of the buffer is available
551	* when the index points to the middle.
552	*/
553	#define MAX_LOG_TAKE_PART 4
554	static const char trunc_msg[] = "<truncated>";
555
556	static void truncate_msg(u16 *text_len, u16 *trunc_msg_len)
557	{
558	/*
559	* The message should not take the whole buffer. Otherwise, it might
560	* get removed too soon.
561	*/
562	u32 max_text_len = log_buf_len / MAX_LOG_TAKE_PART;
563
564	if (*text_len > max_text_len)
565	*text_len = max_text_len;
566
567	/* enable the warning message (if there is room) */
568	*trunc_msg_len = strlen(trunc_msg);
569	if (*text_len >= *trunc_msg_len)
570	*text_len -= *trunc_msg_len;
571	else
572	*trunc_msg_len = 0;
573	}
574
575	int dmesg_restrict = IS_ENABLED(CONFIG_SECURITY_DMESG_RESTRICT);
576
577	static int syslog_action_restricted(int type)
578	{
579	if (dmesg_restrict)
580	return 1;
581	/*
582	* Unless restricted, we allow "read all" and "get buffer size"
583	* for everybody.
584	*/
585	return type != SYSLOG_ACTION_READ_ALL &&
586	type != SYSLOG_ACTION_SIZE_BUFFER;
587	}
588
589	static int check_syslog_permissions(int type, int source)
590	{
591	/*
592	* If this is from /proc/kmsg and we've already opened it, then we've
593	* already done the capabilities checks at open time.
594	*/
595	if (source == SYSLOG_FROM_PROC && type != SYSLOG_ACTION_OPEN)
596	goto ok;
597
598	if (syslog_action_restricted(type)) {
599	if (capable(CAP_SYSLOG))
600	goto ok;
601	/*
602	* For historical reasons, accept CAP_SYS_ADMIN too, with
603	* a warning.
604	*/
605	if (capable(CAP_SYS_ADMIN)) {
606	pr_warn_once("%s (%d): Attempt to access syslog with "
607	"CAP_SYS_ADMIN but no CAP_SYSLOG "
608	"(deprecated).\n",
609	current->comm, task_pid_nr(current));
610	goto ok;
611	}
612	return -EPERM;
613	}
614	ok:
615	return security_syslog(type);
616	}
617
618	static void append_char(char **pp, char *e, char c)
619	{
620	if (*pp < e)
621	*(*pp)++ = c;
622	}
623
624	static ssize_t info_print_ext_header(char *buf, size_t size,
625	struct printk_info *info)
626	{
627	u64 ts_usec = info->ts_nsec;
628	char caller[20];
629	#ifdef CONFIG_PRINTK_CALLER
630	u32 id = info->caller_id;
631
632	snprintf(buf: caller, size: sizeof(caller), fmt: ",caller=%c%u",
633	id & 0x80000000 ? 'C' : 'T', id & ~0x80000000);
634	#else
635	caller[0] = '\0';
636	#endif
637
638	do_div(ts_usec, 1000);
639
640	return scnprintf(buf, size, fmt: "%u,%llu,%llu,%c%s;",
641	(info->facility << 3) | info->level, info->seq,
642	ts_usec, info->flags & LOG_CONT ? 'c' : '-', caller);
643	}
644
645	static ssize_t msg_add_ext_text(char *buf, size_t size,
646	const char *text, size_t text_len,
647	unsigned char endc)
648	{
649	char *p = buf, *e = buf + size;
650	size_t i;
651
652	/* escape non-printable characters */
653	for (i = 0; i < text_len; i++) {
654	unsigned char c = text[i];
655
656	if (c < ' ' || c >= 127 || c == '\\')
657	p += scnprintf(buf: p, size: e - p, fmt: "\\x%02x", c);
658	else
659	append_char(pp: &p, e, c);
660	}
661	append_char(pp: &p, e, c: endc);
662
663	return p - buf;
664	}
665
666	static ssize_t msg_add_dict_text(char *buf, size_t size,
667	const char *key, const char *val)
668	{
669	size_t val_len = strlen(val);
670	ssize_t len;
671
672	if (!val_len)
673	return 0;
674
675	len = msg_add_ext_text(buf, size, text: "", text_len: 0, endc: ' '); /* dict prefix */
676	len += msg_add_ext_text(buf: buf + len, size: size - len, text: key, strlen(key), endc: '=');
677	len += msg_add_ext_text(buf: buf + len, size: size - len, text: val, text_len: val_len, endc: '\n');
678
679	return len;
680	}
681
682	static ssize_t msg_print_ext_body(char *buf, size_t size,
683	char *text, size_t text_len,
684	struct dev_printk_info *dev_info)
685	{
686	ssize_t len;
687
688	len = msg_add_ext_text(buf, size, text, text_len, endc: '\n');
689
690	if (!dev_info)
691	goto out;
692
693	len += msg_add_dict_text(buf: buf + len, size: size - len, key: "SUBSYSTEM",
694	val: dev_info->subsystem);
695	len += msg_add_dict_text(buf: buf + len, size: size - len, key: "DEVICE",
696	val: dev_info->device);
697	out:
698	return len;
699	}
700
701	/* /dev/kmsg - userspace message inject/listen interface */
702	struct devkmsg_user {
703	atomic64_t seq;
704	struct ratelimit_state rs;
705	struct mutex lock;
706	struct printk_buffers pbufs;
707	};
708
709	static __printf(3, 4) __cold
710	int devkmsg_emit(int facility, int level, const char *fmt, ...)
711	{
712	va_list args;
713	int r;
714
715	va_start(args, fmt);
716	r = vprintk_emit(facility, level, NULL, fmt, args);
717	va_end(args);
718
719	return r;
720	}
721
722	static ssize_t devkmsg_write(struct kiocb *iocb, struct iov_iter *from)
723	{
724	char *buf, *line;
725	int level = default_message_loglevel;
726	int facility = 1; /* LOG_USER */
727	struct file *file = iocb->ki_filp;
728	struct devkmsg_user *user = file->private_data;
729	size_t len = iov_iter_count(i: from);
730	ssize_t ret = len;
731
732	if (len > PRINTKRB_RECORD_MAX)
733	return -EINVAL;
734
735	/* Ignore when user logging is disabled. */
736	if (devkmsg_log & DEVKMSG_LOG_MASK_OFF)
737	return len;
738
739	/* Ratelimit when not explicitly enabled. */
740	if (!(devkmsg_log & DEVKMSG_LOG_MASK_ON)) {
741	if (!___ratelimit(rs: &user->rs, current->comm))
742	return ret;
743	}
744
745	buf = kmalloc(size: len+1, GFP_KERNEL);
746	if (buf == NULL)
747	return -ENOMEM;
748
749	buf[len] = '\0';
750	if (!copy_from_iter_full(addr: buf, bytes: len, i: from)) {
751	kfree(objp: buf);
752	return -EFAULT;
753	}
754
755	/*
756	* Extract and skip the syslog prefix <[0-9]*>. Coming from userspace
757	* the decimal value represents 32bit, the lower 3 bit are the log
758	* level, the rest are the log facility.
759	*
760	* If no prefix or no userspace facility is specified, we
761	* enforce LOG_USER, to be able to reliably distinguish
762	* kernel-generated messages from userspace-injected ones.
763	*/
764	line = buf;
765	if (line[0] == '<') {
766	char *endp = NULL;
767	unsigned int u;
768
769	u = simple_strtoul(line + 1, &endp, 10);
770	if (endp && endp[0] == '>') {
771	level = LOG_LEVEL(u);
772	if (LOG_FACILITY(u) != 0)
773	facility = LOG_FACILITY(u);
774	endp++;
775	line = endp;
776	}
777	}
778
779	devkmsg_emit(facility, level, fmt: "%s", line);
780	kfree(objp: buf);
781	return ret;
782	}
783
784	static ssize_t devkmsg_read(struct file *file, char __user *buf,
785	size_t count, loff_t *ppos)
786	{
787	struct devkmsg_user *user = file->private_data;
788	char *outbuf = &user->pbufs.outbuf[0];
789	struct printk_message pmsg = {
790	.pbufs = &user->pbufs,
791	};
792	ssize_t ret;
793
794	ret = mutex_lock_interruptible(&user->lock);
795	if (ret)
796	return ret;
797
798	if (!printk_get_next_message(pmsg: &pmsg, seq: atomic64_read(v: &user->seq), is_extended: true, may_supress: false)) {
799	if (file->f_flags & O_NONBLOCK) {
800	ret = -EAGAIN;
801	goto out;
802	}
803
804	/*
805	* Guarantee this task is visible on the waitqueue before
806	* checking the wake condition.
807	*
808	* The full memory barrier within set_current_state() of
809	* prepare_to_wait_event() pairs with the full memory barrier
810	* within wq_has_sleeper().
811	*
812	* This pairs with __wake_up_klogd:A.
813	*/
814	ret = wait_event_interruptible(log_wait,
815	printk_get_next_message(&pmsg, atomic64_read(&user->seq), true,
816	false)); /* LMM(devkmsg_read:A) */
817	if (ret)
818	goto out;
819	}
820
821	if (pmsg.dropped) {
822	/* our last seen message is gone, return error and reset */
823	atomic64_set(v: &user->seq, i: pmsg.seq);
824	ret = -EPIPE;
825	goto out;
826	}
827
828	atomic64_set(v: &user->seq, i: pmsg.seq + 1);
829
830	if (pmsg.outbuf_len > count) {
831	ret = -EINVAL;
832	goto out;
833	}
834
835	if (copy_to_user(to: buf, from: outbuf, n: pmsg.outbuf_len)) {
836	ret = -EFAULT;
837	goto out;
838	}
839	ret = pmsg.outbuf_len;
840	out:
841	mutex_unlock(lock: &user->lock);
842	return ret;
843	}
844
845	/*
846	* Be careful when modifying this function!!!
847	*
848	* Only few operations are supported because the device works only with the
849	* entire variable length messages (records). Non-standard values are
850	* returned in the other cases and has been this way for quite some time.
851	* User space applications might depend on this behavior.
852	*/
853	static loff_t devkmsg_llseek(struct file *file, loff_t offset, int whence)
854	{
855	struct devkmsg_user *user = file->private_data;
856	loff_t ret = 0;
857
858	if (offset)
859	return -ESPIPE;
860
861	switch (whence) {
862	case SEEK_SET:
863	/* the first record */
864	atomic64_set(v: &user->seq, i: prb_first_valid_seq(rb: prb));
865	break;
866	case SEEK_DATA:
867	/*
868	* The first record after the last SYSLOG_ACTION_CLEAR,
869	* like issued by 'dmesg -c'. Reading /dev/kmsg itself
870	* changes no global state, and does not clear anything.
871	*/
872	atomic64_set(v: &user->seq, i: latched_seq_read_nolock(ls: &clear_seq));
873	break;
874	case SEEK_END:
875	/* after the last record */
876	atomic64_set(v: &user->seq, i: prb_next_seq(rb: prb));
877	break;
878	default:
879	ret = -EINVAL;
880	}
881	return ret;
882	}
883
884	static __poll_t devkmsg_poll(struct file *file, poll_table *wait)
885	{
886	struct devkmsg_user *user = file->private_data;
887	struct printk_info info;
888	__poll_t ret = 0;
889
890	poll_wait(filp: file, wait_address: &log_wait, p: wait);
891
892	if (prb_read_valid_info(rb: prb, seq: atomic64_read(v: &user->seq), info: &info, NULL)) {
893	/* return error when data has vanished underneath us */
894	if (info.seq != atomic64_read(v: &user->seq))
895	ret = EPOLLIN|EPOLLRDNORM|EPOLLERR|EPOLLPRI;
896	else
897	ret = EPOLLIN|EPOLLRDNORM;
898	}
899
900	return ret;
901	}
902
903	static int devkmsg_open(struct inode *inode, struct file *file)
904	{
905	struct devkmsg_user *user;
906	int err;
907
908	if (devkmsg_log & DEVKMSG_LOG_MASK_OFF)
909	return -EPERM;
910
911	/* write-only does not need any file context */
912	if ((file->f_flags & O_ACCMODE) != O_WRONLY) {
913	err = check_syslog_permissions(SYSLOG_ACTION_READ_ALL,
914	SYSLOG_FROM_READER);
915	if (err)
916	return err;
917	}
918
919	user = kvmalloc(size: sizeof(struct devkmsg_user), GFP_KERNEL);
920	if (!user)
921	return -ENOMEM;
922
923	ratelimit_default_init(rs: &user->rs);
924	ratelimit_set_flags(rs: &user->rs, RATELIMIT_MSG_ON_RELEASE);
925
926	mutex_init(&user->lock);
927
928	atomic64_set(v: &user->seq, i: prb_first_valid_seq(rb: prb));
929
930	file->private_data = user;
931	return 0;
932	}
933
934	static int devkmsg_release(struct inode *inode, struct file *file)
935	{
936	struct devkmsg_user *user = file->private_data;
937
938	ratelimit_state_exit(rs: &user->rs);
939
940	mutex_destroy(lock: &user->lock);
941	kvfree(addr: user);
942	return 0;
943	}
944
945	const struct file_operations kmsg_fops = {
946	.open = devkmsg_open,
947	.read = devkmsg_read,
948	.write_iter = devkmsg_write,
949	.llseek = devkmsg_llseek,
950	.poll = devkmsg_poll,
951	.release = devkmsg_release,
952	};
953
954	#ifdef CONFIG_CRASH_CORE
955	/*
956	* This appends the listed symbols to /proc/vmcore
957	*
958	* /proc/vmcore is used by various utilities, like crash and makedumpfile to
959	* obtain access to symbols that are otherwise very difficult to locate. These
960	* symbols are specifically used so that utilities can access and extract the
961	* dmesg log from a vmcore file after a crash.
962	*/
963	void log_buf_vmcoreinfo_setup(void)
964	{
965	struct dev_printk_info *dev_info = NULL;
966
967	VMCOREINFO_SYMBOL(prb);
968	VMCOREINFO_SYMBOL(printk_rb_static);
969	VMCOREINFO_SYMBOL(clear_seq);
970
971	/*
972	* Export struct size and field offsets. User space tools can
973	* parse it and detect any changes to structure down the line.
974	*/
975
976	VMCOREINFO_STRUCT_SIZE(printk_ringbuffer);
977	VMCOREINFO_OFFSET(printk_ringbuffer, desc_ring);
978	VMCOREINFO_OFFSET(printk_ringbuffer, text_data_ring);
979	VMCOREINFO_OFFSET(printk_ringbuffer, fail);
980
981	VMCOREINFO_STRUCT_SIZE(prb_desc_ring);
982	VMCOREINFO_OFFSET(prb_desc_ring, count_bits);
983	VMCOREINFO_OFFSET(prb_desc_ring, descs);
984	VMCOREINFO_OFFSET(prb_desc_ring, infos);
985	VMCOREINFO_OFFSET(prb_desc_ring, head_id);
986	VMCOREINFO_OFFSET(prb_desc_ring, tail_id);
987
988	VMCOREINFO_STRUCT_SIZE(prb_desc);
989	VMCOREINFO_OFFSET(prb_desc, state_var);
990	VMCOREINFO_OFFSET(prb_desc, text_blk_lpos);
991
992	VMCOREINFO_STRUCT_SIZE(prb_data_blk_lpos);
993	VMCOREINFO_OFFSET(prb_data_blk_lpos, begin);
994	VMCOREINFO_OFFSET(prb_data_blk_lpos, next);
995
996	VMCOREINFO_STRUCT_SIZE(printk_info);
997	VMCOREINFO_OFFSET(printk_info, seq);
998	VMCOREINFO_OFFSET(printk_info, ts_nsec);
999	VMCOREINFO_OFFSET(printk_info, text_len);
1000	VMCOREINFO_OFFSET(printk_info, caller_id);
1001	VMCOREINFO_OFFSET(printk_info, dev_info);
1002
1003	VMCOREINFO_STRUCT_SIZE(dev_printk_info);
1004	VMCOREINFO_OFFSET(dev_printk_info, subsystem);
1005	VMCOREINFO_LENGTH(printk_info_subsystem, sizeof(dev_info->subsystem));
1006	VMCOREINFO_OFFSET(dev_printk_info, device);
1007	VMCOREINFO_LENGTH(printk_info_device, sizeof(dev_info->device));
1008
1009	VMCOREINFO_STRUCT_SIZE(prb_data_ring);
1010	VMCOREINFO_OFFSET(prb_data_ring, size_bits);
1011	VMCOREINFO_OFFSET(prb_data_ring, data);
1012	VMCOREINFO_OFFSET(prb_data_ring, head_lpos);
1013	VMCOREINFO_OFFSET(prb_data_ring, tail_lpos);
1014
1015	VMCOREINFO_SIZE(atomic_long_t);
1016	VMCOREINFO_TYPE_OFFSET(atomic_long_t, counter);
1017
1018	VMCOREINFO_STRUCT_SIZE(latched_seq);
1019	VMCOREINFO_OFFSET(latched_seq, val);
1020	}
1021	#endif
1022
1023	/* requested log_buf_len from kernel cmdline */
1024	static unsigned long __initdata new_log_buf_len;
1025
1026	/* we practice scaling the ring buffer by powers of 2 */
1027	static void __init log_buf_len_update(u64 size)
1028	{
1029	if (size > (u64)LOG_BUF_LEN_MAX) {
1030	size = (u64)LOG_BUF_LEN_MAX;
1031	pr_err("log_buf over 2G is not supported.\n");
1032	}
1033
1034	if (size)
1035	size = roundup_pow_of_two(size);
1036	if (size > log_buf_len)
1037	new_log_buf_len = (unsigned long)size;
1038	}
1039
1040	/* save requested log_buf_len since it's too early to process it */
1041	static int __init log_buf_len_setup(char *str)
1042	{
1043	u64 size;
1044
1045	if (!str)
1046	return -EINVAL;
1047
1048	size = memparse(ptr: str, retptr: &str);
1049
1050	log_buf_len_update(size);
1051
1052	return 0;
1053	}
1054	early_param("log_buf_len", log_buf_len_setup);
1055
1056	#ifdef CONFIG_SMP
1057	#define __LOG_CPU_MAX_BUF_LEN (1 << CONFIG_LOG_CPU_MAX_BUF_SHIFT)
1058
1059	static void __init log_buf_add_cpu(void)
1060	{
1061	unsigned int cpu_extra;
1062
1063	/*
1064	* archs should set up cpu_possible_bits properly with
1065	* set_cpu_possible() after setup_arch() but just in
1066	* case lets ensure this is valid.
1067	*/
1068	if (num_possible_cpus() == 1)
1069	return;
1070
1071	cpu_extra = (num_possible_cpus() - 1) * __LOG_CPU_MAX_BUF_LEN;
1072
1073	/* by default this will only continue through for large > 64 CPUs */
1074	if (cpu_extra <= __LOG_BUF_LEN / 2)
1075	return;
1076
1077	pr_info("log_buf_len individual max cpu contribution: %d bytes\n",
1078	__LOG_CPU_MAX_BUF_LEN);
1079	pr_info("log_buf_len total cpu_extra contributions: %d bytes\n",
1080	cpu_extra);
1081	pr_info("log_buf_len min size: %d bytes\n", __LOG_BUF_LEN);
1082
1083	log_buf_len_update(size: cpu_extra + __LOG_BUF_LEN);
1084	}
1085	#else /* !CONFIG_SMP */
1086	static inline void log_buf_add_cpu(void) {}
1087	#endif /* CONFIG_SMP */
1088
1089	static void __init set_percpu_data_ready(void)
1090	{
1091	__printk_percpu_data_ready = true;
1092	}
1093
1094	static unsigned int __init add_to_rb(struct printk_ringbuffer *rb,
1095	struct printk_record *r)
1096	{
1097	struct prb_reserved_entry e;
1098	struct printk_record dest_r;
1099
1100	prb_rec_init_wr(r: &dest_r, text_buf_size: r->info->text_len);
1101
1102	if (!prb_reserve(e: &e, rb, r: &dest_r))
1103	return 0;
1104
1105	memcpy(&dest_r.text_buf[0], &r->text_buf[0], r->info->text_len);
1106	dest_r.info->text_len = r->info->text_len;
1107	dest_r.info->facility = r->info->facility;
1108	dest_r.info->level = r->info->level;
1109	dest_r.info->flags = r->info->flags;
1110	dest_r.info->ts_nsec = r->info->ts_nsec;
1111	dest_r.info->caller_id = r->info->caller_id;
1112	memcpy(&dest_r.info->dev_info, &r->info->dev_info, sizeof(dest_r.info->dev_info));
1113
1114	prb_final_commit(e: &e);
1115
1116	return prb_record_text_space(e: &e);
1117	}
1118
1119	static char setup_text_buf[PRINTKRB_RECORD_MAX] __initdata;
1120
1121	void __init setup_log_buf(int early)
1122	{
1123	struct printk_info *new_infos;
1124	unsigned int new_descs_count;
1125	struct prb_desc *new_descs;
1126	struct printk_info info;
1127	struct printk_record r;
1128	unsigned int text_size;
1129	size_t new_descs_size;
1130	size_t new_infos_size;
1131	unsigned long flags;
1132	char *new_log_buf;
1133	unsigned int free;
1134	u64 seq;
1135
1136	/*
1137	* Some archs call setup_log_buf() multiple times - first is very
1138	* early, e.g. from setup_arch(), and second - when percpu_areas
1139	* are initialised.
1140	*/
1141	if (!early)
1142	set_percpu_data_ready();
1143
1144	if (log_buf != __log_buf)
1145	return;
1146
1147	if (!early && !new_log_buf_len)
1148	log_buf_add_cpu();
1149
1150	if (!new_log_buf_len)
1151	return;
1152
1153	new_descs_count = new_log_buf_len >> PRB_AVGBITS;
1154	if (new_descs_count == 0) {
1155	pr_err("new_log_buf_len: %lu too small\n", new_log_buf_len);
1156	return;
1157	}
1158
1159	new_log_buf = memblock_alloc(size: new_log_buf_len, LOG_ALIGN);
1160	if (unlikely(!new_log_buf)) {
1161	pr_err("log_buf_len: %lu text bytes not available\n",
1162	new_log_buf_len);
1163	return;
1164	}
1165
1166	new_descs_size = new_descs_count * sizeof(struct prb_desc);
1167	new_descs = memblock_alloc(size: new_descs_size, LOG_ALIGN);
1168	if (unlikely(!new_descs)) {
1169	pr_err("log_buf_len: %zu desc bytes not available\n",
1170	new_descs_size);
1171	goto err_free_log_buf;
1172	}
1173
1174	new_infos_size = new_descs_count * sizeof(struct printk_info);
1175	new_infos = memblock_alloc(size: new_infos_size, LOG_ALIGN);
1176	if (unlikely(!new_infos)) {
1177	pr_err("log_buf_len: %zu info bytes not available\n",
1178	new_infos_size);
1179	goto err_free_descs;
1180	}
1181
1182	prb_rec_init_rd(r: &r, info: &info, text_buf: &setup_text_buf[0], text_buf_size: sizeof(setup_text_buf));
1183
1184	prb_init(rb: &printk_rb_dynamic,
1185	text_buf: new_log_buf, ilog2(new_log_buf_len),
1186	descs: new_descs, ilog2(new_descs_count),
1187	infos: new_infos);
1188
1189	local_irq_save(flags);
1190
1191	log_buf_len = new_log_buf_len;
1192	log_buf = new_log_buf;
1193	new_log_buf_len = 0;
1194
1195	free = __LOG_BUF_LEN;
1196	prb_for_each_record(0, &printk_rb_static, seq, &r) {
1197	text_size = add_to_rb(rb: &printk_rb_dynamic, r: &r);
1198	if (text_size > free)
1199	free = 0;
1200	else
1201	free -= text_size;
1202	}
1203
1204	prb = &printk_rb_dynamic;
1205
1206	local_irq_restore(flags);
1207
1208	/*
1209	* Copy any remaining messages that might have appeared from
1210	* NMI context after copying but before switching to the
1211	* dynamic buffer.
1212	*/
1213	prb_for_each_record(seq, &printk_rb_static, seq, &r) {
1214	text_size = add_to_rb(rb: &printk_rb_dynamic, r: &r);
1215	if (text_size > free)
1216	free = 0;
1217	else
1218	free -= text_size;
1219	}
1220
1221	if (seq != prb_next_seq(rb: &printk_rb_static)) {
1222	pr_err("dropped %llu messages\n",
1223	prb_next_seq(&printk_rb_static) - seq);
1224	}
1225
1226	pr_info("log_buf_len: %u bytes\n", log_buf_len);
1227	pr_info("early log buf free: %u(%u%%)\n",
1228	free, (free * 100) / __LOG_BUF_LEN);
1229	return;
1230
1231	err_free_descs:
1232	memblock_free(ptr: new_descs, size: new_descs_size);
1233	err_free_log_buf:
1234	memblock_free(ptr: new_log_buf, size: new_log_buf_len);
1235	}
1236
1237	static bool __read_mostly ignore_loglevel;
1238
1239	static int __init ignore_loglevel_setup(char *str)
1240	{
1241	ignore_loglevel = true;
1242	pr_info("debug: ignoring loglevel setting.\n");
1243
1244	return 0;
1245	}
1246
1247	early_param("ignore_loglevel", ignore_loglevel_setup);
1248	module_param(ignore_loglevel, bool, S_IRUGO | S_IWUSR);
1249	MODULE_PARM_DESC(ignore_loglevel,
1250	"ignore loglevel setting (prints all kernel messages to the console)");
1251
1252	static bool suppress_message_printing(int level)
1253	{
1254	return (level >= console_loglevel && !ignore_loglevel);
1255	}
1256
1257	#ifdef CONFIG_BOOT_PRINTK_DELAY
1258
1259	static int boot_delay; /* msecs delay after each printk during bootup */
1260	static unsigned long long loops_per_msec; /* based on boot_delay */
1261
1262	static int __init boot_delay_setup(char *str)
1263	{
1264	unsigned long lpj;
1265
1266	lpj = preset_lpj ? preset_lpj : 1000000; /* some guess */
1267	loops_per_msec = (unsigned long long)lpj / 1000 * HZ;
1268
1269	get_option(str: &str, pint: &boot_delay);
1270	if (boot_delay > 10 * 1000)
1271	boot_delay = 0;
1272
1273	pr_debug("boot_delay: %u, preset_lpj: %ld, lpj: %lu, "
1274	"HZ: %d, loops_per_msec: %llu\n",
1275	boot_delay, preset_lpj, lpj, HZ, loops_per_msec);
1276	return 0;
1277	}
1278	early_param("boot_delay", boot_delay_setup);
1279
1280	static void boot_delay_msec(int level)
1281	{
1282	unsigned long long k;
1283	unsigned long timeout;
1284
1285	if ((boot_delay == 0 || system_state >= SYSTEM_RUNNING)
1286	|| suppress_message_printing(level)) {
1287	return;
1288	}
1289
1290	k = (unsigned long long)loops_per_msec * boot_delay;
1291
1292	timeout = jiffies + msecs_to_jiffies(m: boot_delay);
1293	while (k) {
1294	k--;
1295	cpu_relax();
1296	/*
1297	* use (volatile) jiffies to prevent
1298	* compiler reduction; loop termination via jiffies
1299	* is secondary and may or may not happen.
1300	*/
1301	if (time_after(jiffies, timeout))
1302	break;
1303	touch_nmi_watchdog();
1304	}
1305	}
1306	#else
1307	static inline void boot_delay_msec(int level)
1308	{
1309	}
1310	#endif
1311
1312	static bool printk_time = IS_ENABLED(CONFIG_PRINTK_TIME);
1313	module_param_named(time, printk_time, bool, S_IRUGO | S_IWUSR);
1314
1315	static size_t print_syslog(unsigned int level, char *buf)
1316	{
1317	return sprintf(buf, fmt: "<%u>", level);
1318	}
1319
1320	static size_t print_time(u64 ts, char *buf)
1321	{
1322	unsigned long rem_nsec = do_div(ts, 1000000000);
1323
1324	return sprintf(buf, fmt: "[%5lu.%06lu]",
1325	(unsigned long)ts, rem_nsec / 1000);
1326	}
1327
1328	#ifdef CONFIG_PRINTK_CALLER
1329	static size_t print_caller(u32 id, char *buf)
1330	{
1331	char caller[12];
1332
1333	snprintf(buf: caller, size: sizeof(caller), fmt: "%c%u",
1334	id & 0x80000000 ? 'C' : 'T', id & ~0x80000000);
1335	return sprintf(buf, fmt: "[%6s]", caller);
1336	}
1337	#else
1338	#define print_caller(id, buf) 0
1339	#endif
1340
1341	static size_t info_print_prefix(const struct printk_info *info, bool syslog,
1342	bool time, char *buf)
1343	{
1344	size_t len = 0;
1345
1346	if (syslog)
1347	len = print_syslog(level: (info->facility << 3) | info->level, buf);
1348
1349	if (time)
1350	len += print_time(ts: info->ts_nsec, buf: buf + len);
1351
1352	len += print_caller(id: info->caller_id, buf: buf + len);
1353
1354	if (IS_ENABLED(CONFIG_PRINTK_CALLER) || time) {
1355	buf[len++] = ' ';
1356	buf[len] = '\0';
1357	}
1358
1359	return len;
1360	}
1361
1362	/*
1363	* Prepare the record for printing. The text is shifted within the given
1364	* buffer to avoid a need for another one. The following operations are
1365	* done:
1366	*
1367	* - Add prefix for each line.
1368	* - Drop truncated lines that no longer fit into the buffer.
1369	* - Add the trailing newline that has been removed in vprintk_store().
1370	* - Add a string terminator.
1371	*
1372	* Since the produced string is always terminated, the maximum possible
1373	* return value is @r->text_buf_size - 1;
1374	*
1375	* Return: The length of the updated/prepared text, including the added
1376	* prefixes and the newline. The terminator is not counted. The dropped
1377	* line(s) are not counted.
1378	*/
1379	static size_t record_print_text(struct printk_record *r, bool syslog,
1380	bool time)
1381	{
1382	size_t text_len = r->info->text_len;
1383	size_t buf_size = r->text_buf_size;
1384	char *text = r->text_buf;
1385	char prefix[PRINTK_PREFIX_MAX];
1386	bool truncated = false;
1387	size_t prefix_len;
1388	size_t line_len;
1389	size_t len = 0;
1390	char *next;
1391
1392	/*
1393	* If the message was truncated because the buffer was not large
1394	* enough, treat the available text as if it were the full text.
1395	*/
1396	if (text_len > buf_size)
1397	text_len = buf_size;
1398
1399	prefix_len = info_print_prefix(info: r->info, syslog, time, buf: prefix);
1400
1401	/*
1402	* @text_len: bytes of unprocessed text
1403	* @line_len: bytes of current line _without_ newline
1404	* @text: pointer to beginning of current line
1405	* @len: number of bytes prepared in r->text_buf
1406	*/
1407	for (;;) {
1408	next = memchr(p: text, c: '\n', size: text_len);
1409	if (next) {
1410	line_len = next - text;
1411	} else {
1412	/* Drop truncated line(s). */
1413	if (truncated)
1414	break;
1415	line_len = text_len;
1416	}
1417
1418	/*
1419	* Truncate the text if there is not enough space to add the
1420	* prefix and a trailing newline and a terminator.
1421	*/
1422	if (len + prefix_len + text_len + 1 + 1 > buf_size) {
1423	/* Drop even the current line if no space. */
1424	if (len + prefix_len + line_len + 1 + 1 > buf_size)
1425	break;
1426
1427	text_len = buf_size - len - prefix_len - 1 - 1;
1428	truncated = true;
1429	}
1430
1431	memmove(text + prefix_len, text, text_len);
1432	memcpy(text, prefix, prefix_len);
1433
1434	/*
1435	* Increment the prepared length to include the text and
1436	* prefix that were just moved+copied. Also increment for the
1437	* newline at the end of this line. If this is the last line,
1438	* there is no newline, but it will be added immediately below.
1439	*/
1440	len += prefix_len + line_len + 1;
1441	if (text_len == line_len) {
1442	/*
1443	* This is the last line. Add the trailing newline
1444	* removed in vprintk_store().
1445	*/
1446	text[prefix_len + line_len] = '\n';
1447	break;
1448	}
1449
1450	/*
1451	* Advance beyond the added prefix and the related line with
1452	* its newline.
1453	*/
1454	text += prefix_len + line_len + 1;
1455
1456	/*
1457	* The remaining text has only decreased by the line with its
1458	* newline.
1459	*
1460	* Note that @text_len can become zero. It happens when @text
1461	* ended with a newline (either due to truncation or the
1462	* original string ending with "\n\n"). The loop is correctly
1463	* repeated and (if not truncated) an empty line with a prefix
1464	* will be prepared.
1465	*/
1466	text_len -= line_len + 1;
1467	}
1468
1469	/*
1470	* If a buffer was provided, it will be terminated. Space for the
1471	* string terminator is guaranteed to be available. The terminator is
1472	* not counted in the return value.
1473	*/
1474	if (buf_size > 0)
1475	r->text_buf[len] = 0;
1476
1477	return len;
1478	}
1479
1480	static size_t get_record_print_text_size(struct printk_info *info,
1481	unsigned int line_count,
1482	bool syslog, bool time)
1483	{
1484	char prefix[PRINTK_PREFIX_MAX];
1485	size_t prefix_len;
1486
1487	prefix_len = info_print_prefix(info, syslog, time, buf: prefix);
1488
1489	/*
1490	* Each line will be preceded with a prefix. The intermediate
1491	* newlines are already within the text, but a final trailing
1492	* newline will be added.
1493	*/
1494	return ((prefix_len * line_count) + info->text_len + 1);
1495	}
1496
1497	/*
1498	* Beginning with @start_seq, find the first record where it and all following
1499	* records up to (but not including) @max_seq fit into @size.
1500	*
1501	* @max_seq is simply an upper bound and does not need to exist. If the caller
1502	* does not require an upper bound, -1 can be used for @max_seq.
1503	*/
1504	static u64 find_first_fitting_seq(u64 start_seq, u64 max_seq, size_t size,
1505	bool syslog, bool time)
1506	{
1507	struct printk_info info;
1508	unsigned int line_count;
1509	size_t len = 0;
1510	u64 seq;
1511
1512	/* Determine the size of the records up to @max_seq. */
1513	prb_for_each_info(start_seq, prb, seq, &info, &line_count) {
1514	if (info.seq >= max_seq)
1515	break;
1516	len += get_record_print_text_size(info: &info, line_count, syslog, time);
1517	}
1518
1519	/*
1520	* Adjust the upper bound for the next loop to avoid subtracting
1521	* lengths that were never added.
1522	*/
1523	if (seq < max_seq)
1524	max_seq = seq;
1525
1526	/*
1527	* Move first record forward until length fits into the buffer. Ignore
1528	* newest messages that were not counted in the above cycle. Messages
1529	* might appear and get lost in the meantime. This is a best effort
1530	* that prevents an infinite loop that could occur with a retry.
1531	*/
1532	prb_for_each_info(start_seq, prb, seq, &info, &line_count) {
1533	if (len <= size || info.seq >= max_seq)
1534	break;
1535	len -= get_record_print_text_size(info: &info, line_count, syslog, time);
1536	}
1537
1538	return seq;
1539	}
1540
1541	/* The caller is responsible for making sure @size is greater than 0. */
1542	static int syslog_print(char __user *buf, int size)
1543	{
1544	struct printk_info info;
1545	struct printk_record r;
1546	char *text;
1547	int len = 0;
1548	u64 seq;
1549
1550	text = kmalloc(PRINTK_MESSAGE_MAX, GFP_KERNEL);
1551	if (!text)
1552	return -ENOMEM;
1553
1554	prb_rec_init_rd(r: &r, info: &info, text_buf: text, PRINTK_MESSAGE_MAX);
1555
1556	mutex_lock(&syslog_lock);
1557
1558	/*
1559	* Wait for the @syslog_seq record to be available. @syslog_seq may
1560	* change while waiting.
1561	*/
1562	do {
1563	seq = syslog_seq;
1564
1565	mutex_unlock(lock: &syslog_lock);
1566	/*
1567	* Guarantee this task is visible on the waitqueue before
1568	* checking the wake condition.
1569	*
1570	* The full memory barrier within set_current_state() of
1571	* prepare_to_wait_event() pairs with the full memory barrier
1572	* within wq_has_sleeper().
1573	*
1574	* This pairs with __wake_up_klogd:A.
1575	*/
1576	len = wait_event_interruptible(log_wait,
1577	prb_read_valid(prb, seq, NULL)); /* LMM(syslog_print:A) */
1578	mutex_lock(&syslog_lock);
1579
1580	if (len)
1581	goto out;
1582	} while (syslog_seq != seq);
1583
1584	/*
1585	* Copy records that fit into the buffer. The above cycle makes sure
1586	* that the first record is always available.
1587	*/
1588	do {
1589	size_t n;
1590	size_t skip;
1591	int err;
1592
1593	if (!prb_read_valid(rb: prb, seq: syslog_seq, r: &r))
1594	break;
1595
1596	if (r.info->seq != syslog_seq) {
1597	/* message is gone, move to next valid one */
1598	syslog_seq = r.info->seq;
1599	syslog_partial = 0;
1600	}
1601
1602	/*
1603	* To keep reading/counting partial line consistent,
1604	* use printk_time value as of the beginning of a line.
1605	*/
1606	if (!syslog_partial)
1607	syslog_time = printk_time;
1608
1609	skip = syslog_partial;
1610	n = record_print_text(r: &r, syslog: true, time: syslog_time);
1611	if (n - syslog_partial <= size) {
1612	/* message fits into buffer, move forward */
1613	syslog_seq = r.info->seq + 1;
1614	n -= syslog_partial;
1615	syslog_partial = 0;
1616	} else if (!len){
1617	/* partial read(), remember position */
1618	n = size;
1619	syslog_partial += n;
1620	} else
1621	n = 0;
1622
1623	if (!n)
1624	break;
1625
1626	mutex_unlock(lock: &syslog_lock);
1627	err = copy_to_user(to: buf, from: text + skip, n);
1628	mutex_lock(&syslog_lock);
1629
1630	if (err) {
1631	if (!len)
1632	len = -EFAULT;
1633	break;
1634	}
1635
1636	len += n;
1637	size -= n;
1638	buf += n;
1639	} while (size);
1640	out:
1641	mutex_unlock(lock: &syslog_lock);
1642	kfree(objp: text);
1643	return len;
1644	}
1645
1646	static int syslog_print_all(char __user *buf, int size, bool clear)
1647	{
1648	struct printk_info info;
1649	struct printk_record r;
1650	char *text;
1651	int len = 0;
1652	u64 seq;
1653	bool time;
1654
1655	text = kmalloc(PRINTK_MESSAGE_MAX, GFP_KERNEL);
1656	if (!text)
1657	return -ENOMEM;
1658
1659	time = printk_time;
1660	/*
1661	* Find first record that fits, including all following records,
1662	* into the user-provided buffer for this dump.
1663	*/
1664	seq = find_first_fitting_seq(start_seq: latched_seq_read_nolock(ls: &clear_seq), max_seq: -1,
1665	size, syslog: true, time);
1666
1667	prb_rec_init_rd(r: &r, info: &info, text_buf: text, PRINTK_MESSAGE_MAX);
1668
1669	prb_for_each_record(seq, prb, seq, &r) {
1670	int textlen;
1671
1672	textlen = record_print_text(r: &r, syslog: true, time);
1673
1674	if (len + textlen > size) {
1675	seq--;
1676	break;
1677	}
1678
1679	if (copy_to_user(to: buf + len, from: text, n: textlen))
1680	len = -EFAULT;
1681	else
1682	len += textlen;
1683
1684	if (len < 0)
1685	break;
1686	}
1687
1688	if (clear) {
1689	mutex_lock(&syslog_lock);
1690	latched_seq_write(ls: &clear_seq, val: seq);
1691	mutex_unlock(lock: &syslog_lock);
1692	}
1693
1694	kfree(objp: text);
1695	return len;
1696	}
1697
1698	static void syslog_clear(void)
1699	{
1700	mutex_lock(&syslog_lock);
1701	latched_seq_write(ls: &clear_seq, val: prb_next_seq(rb: prb));
1702	mutex_unlock(lock: &syslog_lock);
1703	}
1704
1705	int do_syslog(int type, char __user *buf, int len, int source)
1706	{
1707	struct printk_info info;
1708	bool clear = false;
1709	static int saved_console_loglevel = LOGLEVEL_DEFAULT;
1710	int error;
1711
1712	error = check_syslog_permissions(type, source);
1713	if (error)
1714	return error;
1715
1716	switch (type) {
1717	case SYSLOG_ACTION_CLOSE: /* Close log */
1718	break;
1719	case SYSLOG_ACTION_OPEN: /* Open log */
1720	break;
1721	case SYSLOG_ACTION_READ: /* Read from log */
1722	if (!buf || len < 0)
1723	return -EINVAL;
1724	if (!len)
1725	return 0;
1726	if (!access_ok(buf, len))
1727	return -EFAULT;
1728	error = syslog_print(buf, size: len);
1729	break;
1730	/* Read/clear last kernel messages */
1731	case SYSLOG_ACTION_READ_CLEAR:
1732	clear = true;
1733	fallthrough;
1734	/* Read last kernel messages */
1735	case SYSLOG_ACTION_READ_ALL:
1736	if (!buf || len < 0)
1737	return -EINVAL;
1738	if (!len)
1739	return 0;
1740	if (!access_ok(buf, len))
1741	return -EFAULT;
1742	error = syslog_print_all(buf, size: len, clear);
1743	break;
1744	/* Clear ring buffer */
1745	case SYSLOG_ACTION_CLEAR:
1746	syslog_clear();
1747	break;
1748	/* Disable logging to console */
1749	case SYSLOG_ACTION_CONSOLE_OFF:
1750	if (saved_console_loglevel == LOGLEVEL_DEFAULT)
1751	saved_console_loglevel = console_loglevel;
1752	console_loglevel = minimum_console_loglevel;
1753	break;
1754	/* Enable logging to console */
1755	case SYSLOG_ACTION_CONSOLE_ON:
1756	if (saved_console_loglevel != LOGLEVEL_DEFAULT) {
1757	console_loglevel = saved_console_loglevel;
1758	saved_console_loglevel = LOGLEVEL_DEFAULT;
1759	}
1760	break;
1761	/* Set level of messages printed to console */
1762	case SYSLOG_ACTION_CONSOLE_LEVEL:
1763	if (len < 1 || len > 8)
1764	return -EINVAL;
1765	if (len < minimum_console_loglevel)
1766	len = minimum_console_loglevel;
1767	console_loglevel = len;
1768	/* Implicitly re-enable logging to console */
1769	saved_console_loglevel = LOGLEVEL_DEFAULT;
1770	break;
1771	/* Number of chars in the log buffer */
1772	case SYSLOG_ACTION_SIZE_UNREAD:
1773	mutex_lock(&syslog_lock);
1774	if (!prb_read_valid_info(rb: prb, seq: syslog_seq, info: &info, NULL)) {
1775	/* No unread messages. */
1776	mutex_unlock(lock: &syslog_lock);
1777	return 0;
1778	}
1779	if (info.seq != syslog_seq) {
1780	/* messages are gone, move to first one */
1781	syslog_seq = info.seq;
1782	syslog_partial = 0;
1783	}
1784	if (source == SYSLOG_FROM_PROC) {
1785	/*
1786	* Short-cut for poll(/"proc/kmsg") which simply checks
1787	* for pending data, not the size; return the count of
1788	* records, not the length.
1789	*/
1790	error = prb_next_seq(rb: prb) - syslog_seq;
1791	} else {
1792	bool time = syslog_partial ? syslog_time : printk_time;
1793	unsigned int line_count;
1794	u64 seq;
1795
1796	prb_for_each_info(syslog_seq, prb, seq, &info,
1797	&line_count) {
1798	error += get_record_print_text_size(info: &info, line_count,
1799	syslog: true, time);
1800	time = printk_time;
1801	}
1802	error -= syslog_partial;
1803	}
1804	mutex_unlock(lock: &syslog_lock);
1805	break;
1806	/* Size of the log buffer */
1807	case SYSLOG_ACTION_SIZE_BUFFER:
1808	error = log_buf_len;
1809	break;
1810	default:
1811	error = -EINVAL;
1812	break;
1813	}
1814
1815	return error;
1816	}
1817
1818	SYSCALL_DEFINE3(syslog, int, type, char __user *, buf, int, len)
1819	{
1820	return do_syslog(type, buf, len, SYSLOG_FROM_READER);
1821	}
1822
1823	/*
1824	* Special console_lock variants that help to reduce the risk of soft-lockups.
1825	* They allow to pass console_lock to another printk() call using a busy wait.
1826	*/
1827
1828	#ifdef CONFIG_LOCKDEP
1829	static struct lockdep_map console_owner_dep_map = {
1830	.name = "console_owner"
1831	};
1832	#endif
1833
1834	static DEFINE_RAW_SPINLOCK(console_owner_lock);
1835	static struct task_struct *console_owner;
1836	static bool console_waiter;
1837
1838	/**
1839	* console_lock_spinning_enable - mark beginning of code where another
1840	* thread might safely busy wait
1841	*
1842	* This basically converts console_lock into a spinlock. This marks
1843	* the section where the console_lock owner can not sleep, because
1844	* there may be a waiter spinning (like a spinlock). Also it must be
1845	* ready to hand over the lock at the end of the section.
1846	*/
1847	static void console_lock_spinning_enable(void)
1848	{
1849	raw_spin_lock(&console_owner_lock);
1850	console_owner = current;
1851	raw_spin_unlock(&console_owner_lock);
1852
1853	/* The waiter may spin on us after setting console_owner */
1854	spin_acquire(&console_owner_dep_map, 0, 0, _THIS_IP_);
1855	}
1856
1857	/**
1858	* console_lock_spinning_disable_and_check - mark end of code where another
1859	* thread was able to busy wait and check if there is a waiter
1860	* @cookie: cookie returned from console_srcu_read_lock()
1861	*
1862	* This is called at the end of the section where spinning is allowed.
1863	* It has two functions. First, it is a signal that it is no longer
1864	* safe to start busy waiting for the lock. Second, it checks if
1865	* there is a busy waiter and passes the lock rights to her.
1866	*
1867	* Important: Callers lose both the console_lock and the SRCU read lock if
1868	* there was a busy waiter. They must not touch items synchronized by
1869	* console_lock or SRCU read lock in this case.
1870	*
1871	* Return: 1 if the lock rights were passed, 0 otherwise.
1872	*/
1873	static int console_lock_spinning_disable_and_check(int cookie)
1874	{
1875	int waiter;
1876
1877	raw_spin_lock(&console_owner_lock);
1878	waiter = READ_ONCE(console_waiter);
1879	console_owner = NULL;
1880	raw_spin_unlock(&console_owner_lock);
1881
1882	if (!waiter) {
1883	spin_release(&console_owner_dep_map, _THIS_IP_);
1884	return 0;
1885	}
1886
1887	/* The waiter is now free to continue */
1888	WRITE_ONCE(console_waiter, false);
1889
1890	spin_release(&console_owner_dep_map, _THIS_IP_);
1891
1892	/*
1893	* Preserve lockdep lock ordering. Release the SRCU read lock before
1894	* releasing the console_lock.
1895	*/
1896	console_srcu_read_unlock(cookie);
1897
1898	/*
1899	* Hand off console_lock to waiter. The waiter will perform
1900	* the up(). After this, the waiter is the console_lock owner.
1901	*/
1902	mutex_release(&console_lock_dep_map, _THIS_IP_);
1903	return 1;
1904	}
1905
1906	/**
1907	* console_trylock_spinning - try to get console_lock by busy waiting
1908	*
1909	* This allows to busy wait for the console_lock when the current
1910	* owner is running in specially marked sections. It means that
1911	* the current owner is running and cannot reschedule until it
1912	* is ready to lose the lock.
1913	*
1914	* Return: 1 if we got the lock, 0 othrewise
1915	*/
1916	static int console_trylock_spinning(void)
1917	{
1918	struct task_struct *owner = NULL;
1919	bool waiter;
1920	bool spin = false;
1921	unsigned long flags;
1922
1923	if (console_trylock())
1924	return 1;
1925
1926	/*
1927	* It's unsafe to spin once a panic has begun. If we are the
1928	* panic CPU, we may have already halted the owner of the
1929	* console_sem. If we are not the panic CPU, then we should
1930	* avoid taking console_sem, so the panic CPU has a better
1931	* chance of cleanly acquiring it later.
1932	*/
1933	if (panic_in_progress())
1934	return 0;
1935
1936	printk_safe_enter_irqsave(flags);
1937
1938	raw_spin_lock(&console_owner_lock);
1939	owner = READ_ONCE(console_owner);
1940	waiter = READ_ONCE(console_waiter);
1941	if (!waiter && owner && owner != current) {
1942	WRITE_ONCE(console_waiter, true);
1943	spin = true;
1944	}
1945	raw_spin_unlock(&console_owner_lock);
1946
1947	/*
1948	* If there is an active printk() writing to the
1949	* consoles, instead of having it write our data too,
1950	* see if we can offload that load from the active
1951	* printer, and do some printing ourselves.
1952	* Go into a spin only if there isn't already a waiter
1953	* spinning, and there is an active printer, and
1954	* that active printer isn't us (recursive printk?).
1955	*/
1956	if (!spin) {
1957	printk_safe_exit_irqrestore(flags);
1958	return 0;
1959	}
1960
1961	/* We spin waiting for the owner to release us */
1962	spin_acquire(&console_owner_dep_map, 0, 0, _THIS_IP_);
1963	/* Owner will clear console_waiter on hand off */
1964	while (READ_ONCE(console_waiter))
1965	cpu_relax();
1966	spin_release(&console_owner_dep_map, _THIS_IP_);
1967
1968	printk_safe_exit_irqrestore(flags);
1969	/*
1970	* The owner passed the console lock to us.
1971	* Since we did not spin on console lock, annotate
1972	* this as a trylock. Otherwise lockdep will
1973	* complain.
1974	*/
1975	mutex_acquire(&console_lock_dep_map, 0, 1, _THIS_IP_);
1976
1977	return 1;
1978	}
1979
1980	/*
1981	* Recursion is tracked separately on each CPU. If NMIs are supported, an
1982	* additional NMI context per CPU is also separately tracked. Until per-CPU
1983	* is available, a separate "early tracking" is performed.
1984	*/
1985	static DEFINE_PER_CPU(u8, printk_count);
1986	static u8 printk_count_early;
1987	#ifdef CONFIG_HAVE_NMI
1988	static DEFINE_PER_CPU(u8, printk_count_nmi);
1989	static u8 printk_count_nmi_early;
1990	#endif
1991
1992	/*
1993	* Recursion is limited to keep the output sane. printk() should not require
1994	* more than 1 level of recursion (allowing, for example, printk() to trigger
1995	* a WARN), but a higher value is used in case some printk-internal errors
1996	* exist, such as the ringbuffer validation checks failing.
1997	*/
1998	#define PRINTK_MAX_RECURSION 3
1999
2000	/*
2001	* Return a pointer to the dedicated counter for the CPU+context of the
2002	* caller.
2003	*/
2004	static u8 *__printk_recursion_counter(void)
2005	{
2006	#ifdef CONFIG_HAVE_NMI
2007	if (in_nmi()) {
2008	if (printk_percpu_data_ready())
2009	return this_cpu_ptr(&printk_count_nmi);
2010	return &printk_count_nmi_early;
2011	}
2012	#endif
2013	if (printk_percpu_data_ready())
2014	return this_cpu_ptr(&printk_count);
2015	return &printk_count_early;
2016	}
2017
2018	/*
2019	* Enter recursion tracking. Interrupts are disabled to simplify tracking.
2020	* The caller must check the boolean return value to see if the recursion is
2021	* allowed. On failure, interrupts are not disabled.
2022	*
2023	* @recursion_ptr must be a variable of type (u8 *) and is the same variable
2024	* that is passed to printk_exit_irqrestore().
2025	*/
2026	#define printk_enter_irqsave(recursion_ptr, flags) \
2027	({ \
2028	bool success = true; \
2029	\
2030	typecheck(u8 *, recursion_ptr); \
2031	local_irq_save(flags); \
2032	(recursion_ptr) = __printk_recursion_counter(); \
2033	if (*(recursion_ptr) > PRINTK_MAX_RECURSION) { \
2034	local_irq_restore(flags); \
2035	success = false; \
2036	} else { \
2037	(*(recursion_ptr))++; \
2038	} \
2039	success; \
2040	})
2041
2042	/* Exit recursion tracking, restoring interrupts. */
2043	#define printk_exit_irqrestore(recursion_ptr, flags) \
2044	do { \
2045	typecheck(u8 *, recursion_ptr); \
2046	(*(recursion_ptr))--; \
2047	local_irq_restore(flags); \
2048	} while (0)
2049
2050	int printk_delay_msec __read_mostly;
2051
2052	static inline void printk_delay(int level)
2053	{
2054	boot_delay_msec(level);
2055
2056	if (unlikely(printk_delay_msec)) {
2057	int m = printk_delay_msec;
2058
2059	while (m--) {
2060	mdelay(1);
2061	touch_nmi_watchdog();
2062	}
2063	}
2064	}
2065
2066	static inline u32 printk_caller_id(void)
2067	{
2068	return in_task() ? task_pid_nr(current) :
2069	0x80000000 + smp_processor_id();
2070	}
2071
2072	/**
2073	* printk_parse_prefix - Parse level and control flags.
2074	*
2075	* @text: The terminated text message.
2076	* @level: A pointer to the current level value, will be updated.
2077	* @flags: A pointer to the current printk_info flags, will be updated.
2078	*
2079	* @level may be NULL if the caller is not interested in the parsed value.
2080	* Otherwise the variable pointed to by @level must be set to
2081	* LOGLEVEL_DEFAULT in order to be updated with the parsed value.
2082	*
2083	* @flags may be NULL if the caller is not interested in the parsed value.
2084	* Otherwise the variable pointed to by @flags will be OR'd with the parsed
2085	* value.
2086	*
2087	* Return: The length of the parsed level and control flags.
2088	*/
2089	u16 printk_parse_prefix(const char *text, int *level,
2090	enum printk_info_flags *flags)
2091	{
2092	u16 prefix_len = 0;
2093	int kern_level;
2094
2095	while (*text) {
2096	kern_level = printk_get_level(buffer: text);
2097	if (!kern_level)
2098	break;
2099
2100	switch (kern_level) {
2101	case '0' ... '7':
2102	if (level && *level == LOGLEVEL_DEFAULT)
2103	*level = kern_level - '0';
2104	break;
2105	case 'c': /* KERN_CONT */
2106	if (flags)
2107	*flags |= LOG_CONT;
2108	}
2109
2110	prefix_len += 2;
2111	text += 2;
2112	}
2113
2114	return prefix_len;
2115	}
2116
2117	__printf(5, 0)
2118	static u16 printk_sprint(char *text, u16 size, int facility,
2119	enum printk_info_flags *flags, const char *fmt,
2120	va_list args)
2121	{
2122	u16 text_len;
2123
2124	text_len = vscnprintf(buf: text, size, fmt, args);
2125
2126	/* Mark and strip a trailing newline. */
2127	if (text_len && text[text_len - 1] == '\n') {
2128	text_len--;
2129	*flags |= LOG_NEWLINE;
2130	}
2131
2132	/* Strip log level and control flags. */
2133	if (facility == 0) {
2134	u16 prefix_len;
2135
2136	prefix_len = printk_parse_prefix(text, NULL, NULL);
2137	if (prefix_len) {
2138	text_len -= prefix_len;
2139	memmove(text, text + prefix_len, text_len);
2140	}
2141	}
2142
2143	trace_console(text, len: text_len);
2144
2145	return text_len;
2146	}
2147
2148	__printf(4, 0)
2149	int vprintk_store(int facility, int level,
2150	const struct dev_printk_info *dev_info,
2151	const char *fmt, va_list args)
2152	{
2153	struct prb_reserved_entry e;
2154	enum printk_info_flags flags = 0;
2155	struct printk_record r;
2156	unsigned long irqflags;
2157	u16 trunc_msg_len = 0;
2158	char prefix_buf[8];
2159	u8 *recursion_ptr;
2160	u16 reserve_size;
2161	va_list args2;
2162	u32 caller_id;
2163	u16 text_len;
2164	int ret = 0;
2165	u64 ts_nsec;
2166
2167	if (!printk_enter_irqsave(recursion_ptr, irqflags))
2168	return 0;
2169
2170	/*
2171	* Since the duration of printk() can vary depending on the message
2172	* and state of the ringbuffer, grab the timestamp now so that it is
2173	* close to the call of printk(). This provides a more deterministic
2174	* timestamp with respect to the caller.
2175	*/
2176	ts_nsec = local_clock();
2177
2178	caller_id = printk_caller_id();
2179
2180	/*
2181	* The sprintf needs to come first since the syslog prefix might be
2182	* passed in as a parameter. An extra byte must be reserved so that
2183	* later the vscnprintf() into the reserved buffer has room for the
2184	* terminating '\0', which is not counted by vsnprintf().
2185	*/
2186	va_copy(args2, args);
2187	reserve_size = vsnprintf(buf: &prefix_buf[0], size: sizeof(prefix_buf), fmt, args: args2) + 1;
2188	va_end(args2);
2189
2190	if (reserve_size > PRINTKRB_RECORD_MAX)
2191	reserve_size = PRINTKRB_RECORD_MAX;
2192
2193	/* Extract log level or control flags. */
2194	if (facility == 0)
2195	printk_parse_prefix(text: &prefix_buf[0], level: &level, flags: &flags);
2196
2197	if (level == LOGLEVEL_DEFAULT)
2198	level = default_message_loglevel;
2199
2200	if (dev_info)
2201	flags |= LOG_NEWLINE;
2202
2203	if (flags & LOG_CONT) {
2204	prb_rec_init_wr(r: &r, text_buf_size: reserve_size);
2205	if (prb_reserve_in_last(e: &e, rb: prb, r: &r, caller_id, PRINTKRB_RECORD_MAX)) {
2206	text_len = printk_sprint(text: &r.text_buf[r.info->text_len], size: reserve_size,
2207	facility, flags: &flags, fmt, args);
2208	r.info->text_len += text_len;
2209
2210	if (flags & LOG_NEWLINE) {
2211	r.info->flags |= LOG_NEWLINE;
2212	prb_final_commit(e: &e);
2213	} else {
2214	prb_commit(e: &e);
2215	}
2216
2217	ret = text_len;
2218	goto out;
2219	}
2220	}
2221
2222	/*
2223	* Explicitly initialize the record before every prb_reserve() call.
2224	* prb_reserve_in_last() and prb_reserve() purposely invalidate the
2225	* structure when they fail.
2226	*/
2227	prb_rec_init_wr(r: &r, text_buf_size: reserve_size);
2228	if (!prb_reserve(e: &e, rb: prb, r: &r)) {
2229	/* truncate the message if it is too long for empty buffer */
2230	truncate_msg(text_len: &reserve_size, trunc_msg_len: &trunc_msg_len);
2231
2232	prb_rec_init_wr(r: &r, text_buf_size: reserve_size + trunc_msg_len);
2233	if (!prb_reserve(e: &e, rb: prb, r: &r))
2234	goto out;
2235	}
2236
2237	/* fill message */
2238	text_len = printk_sprint(text: &r.text_buf[0], size: reserve_size, facility, flags: &flags, fmt, args);
2239	if (trunc_msg_len)
2240	memcpy(&r.text_buf[text_len], trunc_msg, trunc_msg_len);
2241	r.info->text_len = text_len + trunc_msg_len;
2242	r.info->facility = facility;
2243	r.info->level = level & 7;
2244	r.info->flags = flags & 0x1f;
2245	r.info->ts_nsec = ts_nsec;
2246	r.info->caller_id = caller_id;
2247	if (dev_info)
2248	memcpy(&r.info->dev_info, dev_info, sizeof(r.info->dev_info));
2249
2250	/* A message without a trailing newline can be continued. */
2251	if (!(flags & LOG_NEWLINE))
2252	prb_commit(e: &e);
2253	else
2254	prb_final_commit(e: &e);
2255
2256	ret = text_len + trunc_msg_len;
2257	out:
2258	printk_exit_irqrestore(recursion_ptr, irqflags);
2259	return ret;
2260	}
2261
2262	asmlinkage int vprintk_emit(int facility, int level,
2263	const struct dev_printk_info *dev_info,
2264	const char *fmt, va_list args)
2265	{
2266	int printed_len;
2267	bool in_sched = false;
2268
2269	/* Suppress unimportant messages after panic happens */
2270	if (unlikely(suppress_printk))
2271	return 0;
2272
2273	if (unlikely(suppress_panic_printk) &&
2274	atomic_read(v: &panic_cpu) != raw_smp_processor_id())
2275	return 0;
2276
2277	if (level == LOGLEVEL_SCHED) {
2278	level = LOGLEVEL_DEFAULT;
2279	in_sched = true;
2280	}
2281
2282	printk_delay(level);
2283
2284	printed_len = vprintk_store(facility, level, dev_info, fmt, args);
2285
2286	/* If called from the scheduler, we can not call up(). */
2287	if (!in_sched) {
2288	/*
2289	* The caller may be holding system-critical or
2290	* timing-sensitive locks. Disable preemption during
2291	* printing of all remaining records to all consoles so that
2292	* this context can return as soon as possible. Hopefully
2293	* another printk() caller will take over the printing.
2294	*/
2295	preempt_disable();
2296	/*
2297	* Try to acquire and then immediately release the console
2298	* semaphore. The release will print out buffers. With the
2299	* spinning variant, this context tries to take over the
2300	* printing from another printing context.
2301	*/
2302	if (console_trylock_spinning())
2303	console_unlock();
2304	preempt_enable();
2305	}
2306
2307	if (in_sched)
2308	defer_console_output();
2309	else
2310	wake_up_klogd();
2311
2312	return printed_len;
2313	}
2314	EXPORT_SYMBOL(vprintk_emit);
2315
2316	int vprintk_default(const char *fmt, va_list args)
2317	{
2318	return vprintk_emit(0, LOGLEVEL_DEFAULT, NULL, fmt, args);
2319	}
2320	EXPORT_SYMBOL_GPL(vprintk_default);
2321
2322	asmlinkage __visible int _printk(const char *fmt, ...)
2323	{
2324	va_list args;
2325	int r;
2326
2327	va_start(args, fmt);
2328	r = vprintk(fmt, args);
2329	va_end(args);
2330
2331	return r;
2332	}
2333	EXPORT_SYMBOL(_printk);
2334
2335	static bool pr_flush(int timeout_ms, bool reset_on_progress);
2336	static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress);
2337
2338	#else /* CONFIG_PRINTK */
2339
2340	#define printk_time false
2341
2342	#define prb_read_valid(rb, seq, r) false
2343	#define prb_first_valid_seq(rb) 0
2344	#define prb_next_seq(rb) 0
2345
2346	static u64 syslog_seq;
2347
2348	static bool pr_flush(int timeout_ms, bool reset_on_progress) { return true; }
2349	static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress) { return true; }
2350
2351	#endif /* CONFIG_PRINTK */
2352
2353	#ifdef CONFIG_EARLY_PRINTK
2354	struct console *early_console;
2355
2356	asmlinkage __visible void early_printk(const char *fmt, ...)
2357	{
2358	va_list ap;
2359	char buf[512];
2360	int n;
2361
2362	if (!early_console)
2363	return;
2364
2365	va_start(ap, fmt);
2366	n = vscnprintf(buf, size: sizeof(buf), fmt, args: ap);
2367	va_end(ap);
2368
2369	early_console->write(early_console, buf, n);
2370	}
2371	#endif
2372
2373	static void set_user_specified(struct console_cmdline *c, bool user_specified)
2374	{
2375	if (!user_specified)
2376	return;
2377
2378	/*
2379	* @c console was defined by the user on the command line.
2380	* Do not clear when added twice also by SPCR or the device tree.
2381	*/
2382	c->user_specified = true;
2383	/* At least one console defined by the user on the command line. */
2384	console_set_on_cmdline = 1;
2385	}
2386
2387	static int __add_preferred_console(const char *name, const short idx, char *options,
2388	char *brl_options, bool user_specified)
2389	{
2390	struct console_cmdline *c;
2391	int i;
2392
2393	/*
2394	* We use a signed short index for struct console for device drivers to
2395	* indicate a not yet assigned index or port. However, a negative index
2396	* value is not valid for preferred console.
2397	*/
2398	if (idx < 0)
2399	return -EINVAL;
2400
2401	/*
2402	* See if this tty is not yet registered, and
2403	* if we have a slot free.
2404	*/
2405	for (i = 0, c = console_cmdline;
2406	i < MAX_CMDLINECONSOLES && c->name[0];
2407	i++, c++) {
2408	if (strcmp(c->name, name) == 0 && c->index == idx) {
2409	if (!brl_options)
2410	preferred_console = i;
2411	set_user_specified(c, user_specified);
2412	return 0;
2413	}
2414	}
2415	if (i == MAX_CMDLINECONSOLES)
2416	return -E2BIG;
2417	if (!brl_options)
2418	preferred_console = i;
2419	strscpy(p: c->name, q: name, size: sizeof(c->name));
2420	c->options = options;
2421	set_user_specified(c, user_specified);
2422	braille_set_options(c, brl_options);
2423
2424	c->index = idx;
2425	return 0;
2426	}
2427
2428	static int __init console_msg_format_setup(char *str)
2429	{
2430	if (!strcmp(str, "syslog"))
2431	console_msg_format = MSG_FORMAT_SYSLOG;
2432	if (!strcmp(str, "default"))
2433	console_msg_format = MSG_FORMAT_DEFAULT;
2434	return 1;
2435	}
2436	__setup("console_msg_format=", console_msg_format_setup);
2437
2438	/*
2439	* Set up a console. Called via do_early_param() in init/main.c
2440	* for each "console=" parameter in the boot command line.
2441	*/
2442	static int __init console_setup(char *str)
2443	{
2444	char buf[sizeof(console_cmdline[0].name) + 4]; /* 4 for "ttyS" */
2445	char *s, *options, *brl_options = NULL;
2446	int idx;
2447
2448	/*
2449	* console="" or console=null have been suggested as a way to
2450	* disable console output. Use ttynull that has been created
2451	* for exactly this purpose.
2452	*/
2453	if (str[0] == 0 || strcmp(str, "null") == 0) {
2454	__add_preferred_console(name: "ttynull", idx: 0, NULL, NULL, user_specified: true);
2455	return 1;
2456	}
2457
2458	if (_braille_console_setup(str: &str, brl_options: &brl_options))
2459	return 1;
2460
2461	/*
2462	* Decode str into name, index, options.
2463	*/
2464	if (str[0] >= '0' && str[0] <= '9') {
2465	strcpy(p: buf, q: "ttyS");
2466	strncpy(p: buf + 4, q: str, size: sizeof(buf) - 5);
2467	} else {
2468	strncpy(p: buf, q: str, size: sizeof(buf) - 1);
2469	}
2470	buf[sizeof(buf) - 1] = 0;
2471	options = strchr(str, ',');
2472	if (options)
2473	*(options++) = 0;
2474	#ifdef __sparc__
2475	if (!strcmp(str, "ttya"))
2476	strcpy(buf, "ttyS0");
2477	if (!strcmp(str, "ttyb"))
2478	strcpy(buf, "ttyS1");
2479	#endif
2480	for (s = buf; *s; s++)
2481	if (isdigit(c: *s) || *s == ',')
2482	break;
2483	idx = simple_strtoul(s, NULL, 10);
2484	*s = 0;
2485
2486	__add_preferred_console(name: buf, idx, options, brl_options, user_specified: true);
2487	return 1;
2488	}
2489	__setup("console=", console_setup);
2490
2491	/**
2492	* add_preferred_console - add a device to the list of preferred consoles.
2493	* @name: device name
2494	* @idx: device index
2495	* @options: options for this console
2496	*
2497	* The last preferred console added will be used for kernel messages
2498	* and stdin/out/err for init. Normally this is used by console_setup
2499	* above to handle user-supplied console arguments; however it can also
2500	* be used by arch-specific code either to override the user or more
2501	* commonly to provide a default console (ie from PROM variables) when
2502	* the user has not supplied one.
2503	*/
2504	int add_preferred_console(const char *name, const short idx, char *options)
2505	{
2506	return __add_preferred_console(name, idx, options, NULL, user_specified: false);
2507	}
2508
2509	bool console_suspend_enabled = true;
2510	EXPORT_SYMBOL(console_suspend_enabled);
2511
2512	static int __init console_suspend_disable(char *str)
2513	{
2514	console_suspend_enabled = false;
2515	return 1;
2516	}
2517	__setup("no_console_suspend", console_suspend_disable);
2518	module_param_named(console_suspend, console_suspend_enabled,
2519	bool, S_IRUGO | S_IWUSR);
2520	MODULE_PARM_DESC(console_suspend, "suspend console during suspend"
2521	" and hibernate operations");
2522
2523	static bool printk_console_no_auto_verbose;
2524
2525	void console_verbose(void)
2526	{
2527	if (console_loglevel && !printk_console_no_auto_verbose)
2528	console_loglevel = CONSOLE_LOGLEVEL_MOTORMOUTH;
2529	}
2530	EXPORT_SYMBOL_GPL(console_verbose);
2531
2532	module_param_named(console_no_auto_verbose, printk_console_no_auto_verbose, bool, 0644);
2533	MODULE_PARM_DESC(console_no_auto_verbose, "Disable console loglevel raise to highest on oops/panic/etc");
2534
2535	/**
2536	* suspend_console - suspend the console subsystem
2537	*
2538	* This disables printk() while we go into suspend states
2539	*/
2540	void suspend_console(void)
2541	{
2542	struct console *con;
2543
2544	if (!console_suspend_enabled)
2545	return;
2546	pr_info("Suspending console(s) (use no_console_suspend to debug)\n");
2547	pr_flush(timeout_ms: 1000, reset_on_progress: true);
2548
2549	console_list_lock();
2550	for_each_console(con)
2551	console_srcu_write_flags(con, flags: con->flags | CON_SUSPENDED);
2552	console_list_unlock();
2553
2554	/*
2555	* Ensure that all SRCU list walks have completed. All printing
2556	* contexts must be able to see that they are suspended so that it
2557	* is guaranteed that all printing has stopped when this function
2558	* completes.
2559	*/
2560	synchronize_srcu(ssp: &console_srcu);
2561	}
2562
2563	void resume_console(void)
2564	{
2565	struct console *con;
2566
2567	if (!console_suspend_enabled)
2568	return;
2569
2570	console_list_lock();
2571	for_each_console(con)
2572	console_srcu_write_flags(con, flags: con->flags & ~CON_SUSPENDED);
2573	console_list_unlock();
2574
2575	/*
2576	* Ensure that all SRCU list walks have completed. All printing
2577	* contexts must be able to see they are no longer suspended so
2578	* that they are guaranteed to wake up and resume printing.
2579	*/
2580	synchronize_srcu(ssp: &console_srcu);
2581
2582	pr_flush(timeout_ms: 1000, reset_on_progress: true);
2583	}
2584
2585	/**
2586	* console_cpu_notify - print deferred console messages after CPU hotplug
2587	* @cpu: unused
2588	*
2589	* If printk() is called from a CPU that is not online yet, the messages
2590	* will be printed on the console only if there are CON_ANYTIME consoles.
2591	* This function is called when a new CPU comes online (or fails to come
2592	* up) or goes offline.
2593	*/
2594	static int console_cpu_notify(unsigned int cpu)
2595	{
2596	if (!cpuhp_tasks_frozen) {
2597	/* If trylock fails, someone else is doing the printing */
2598	if (console_trylock())
2599	console_unlock();
2600	}
2601	return 0;
2602	}
2603
2604	/*
2605	* Return true if a panic is in progress on a remote CPU.
2606	*
2607	* On true, the local CPU should immediately release any printing resources
2608	* that may be needed by the panic CPU.
2609	*/
2610	bool other_cpu_in_panic(void)
2611	{
2612	if (!panic_in_progress())
2613	return false;
2614
2615	/*
2616	* We can use raw_smp_processor_id() here because it is impossible for
2617	* the task to be migrated to the panic_cpu, or away from it. If
2618	* panic_cpu has already been set, and we're not currently executing on
2619	* that CPU, then we never will be.
2620	*/
2621	return atomic_read(v: &panic_cpu) != raw_smp_processor_id();
2622	}
2623
2624	/**
2625	* console_lock - block the console subsystem from printing
2626	*
2627	* Acquires a lock which guarantees that no consoles will
2628	* be in or enter their write() callback.
2629	*
2630	* Can sleep, returns nothing.
2631	*/
2632	void console_lock(void)
2633	{
2634	might_sleep();
2635
2636	/* On panic, the console_lock must be left to the panic cpu. */
2637	while (other_cpu_in_panic())
2638	msleep(msecs: 1000);
2639
2640	down_console_sem();
2641	console_locked = 1;
2642	console_may_schedule = 1;
2643	}
2644	EXPORT_SYMBOL(console_lock);
2645
2646	/**
2647	* console_trylock - try to block the console subsystem from printing
2648	*
2649	* Try to acquire a lock which guarantees that no consoles will
2650	* be in or enter their write() callback.
2651	*
2652	* returns 1 on success, and 0 on failure to acquire the lock.
2653	*/
2654	int console_trylock(void)
2655	{
2656	/* On panic, the console_lock must be left to the panic cpu. */
2657	if (other_cpu_in_panic())
2658	return 0;
2659	if (down_trylock_console_sem())
2660	return 0;
2661	console_locked = 1;
2662	console_may_schedule = 0;
2663	return 1;
2664	}
2665	EXPORT_SYMBOL(console_trylock);
2666
2667	int is_console_locked(void)
2668	{
2669	return console_locked;
2670	}
2671	EXPORT_SYMBOL(is_console_locked);
2672
2673	/*
2674	* Check if the given console is currently capable and allowed to print
2675	* records.
2676	*
2677	* Requires the console_srcu_read_lock.
2678	*/
2679	static inline bool console_is_usable(struct console *con)
2680	{
2681	short flags = console_srcu_read_flags(con);
2682
2683	if (!(flags & CON_ENABLED))
2684	return false;
2685
2686	if ((flags & CON_SUSPENDED))
2687	return false;
2688
2689	if (!con->write)
2690	return false;
2691
2692	/*
2693	* Console drivers may assume that per-cpu resources have been
2694	* allocated. So unless they're explicitly marked as being able to
2695	* cope (CON_ANYTIME) don't call them until this CPU is officially up.
2696	*/
2697	if (!cpu_online(raw_smp_processor_id()) && !(flags & CON_ANYTIME))
2698	return false;
2699
2700	return true;
2701	}
2702
2703	static void __console_unlock(void)
2704	{
2705	console_locked = 0;
2706	up_console_sem();
2707	}
2708
2709	#ifdef CONFIG_PRINTK
2710
2711	/*
2712	* Prepend the message in @pmsg->pbufs->outbuf with a "dropped message". This
2713	* is achieved by shifting the existing message over and inserting the dropped
2714	* message.
2715	*
2716	* @pmsg is the printk message to prepend.
2717	*
2718	* @dropped is the dropped count to report in the dropped message.
2719	*
2720	* If the message text in @pmsg->pbufs->outbuf does not have enough space for
2721	* the dropped message, the message text will be sufficiently truncated.
2722	*
2723	* If @pmsg->pbufs->outbuf is modified, @pmsg->outbuf_len is updated.
2724	*/
2725	void console_prepend_dropped(struct printk_message *pmsg, unsigned long dropped)
2726	{
2727	struct printk_buffers *pbufs = pmsg->pbufs;
2728	const size_t scratchbuf_sz = sizeof(pbufs->scratchbuf);
2729	const size_t outbuf_sz = sizeof(pbufs->outbuf);
2730	char *scratchbuf = &pbufs->scratchbuf[0];
2731	char *outbuf = &pbufs->outbuf[0];
2732	size_t len;
2733
2734	len = scnprintf(buf: scratchbuf, size: scratchbuf_sz,
2735	fmt: "** %lu printk messages dropped **\n", dropped);
2736
2737	/*
2738	* Make sure outbuf is sufficiently large before prepending.
2739	* Keep at least the prefix when the message must be truncated.
2740	* It is a rather theoretical problem when someone tries to
2741	* use a minimalist buffer.
2742	*/
2743	if (WARN_ON_ONCE(len + PRINTK_PREFIX_MAX >= outbuf_sz))
2744	return;
2745
2746	if (pmsg->outbuf_len + len >= outbuf_sz) {
2747	/* Truncate the message, but keep it terminated. */
2748	pmsg->outbuf_len = outbuf_sz - (len + 1);
2749	outbuf[pmsg->outbuf_len] = 0;
2750	}
2751
2752	memmove(outbuf + len, outbuf, pmsg->outbuf_len + 1);
2753	memcpy(outbuf, scratchbuf, len);
2754	pmsg->outbuf_len += len;
2755	}
2756
2757	/*
2758	* Read and format the specified record (or a later record if the specified
2759	* record is not available).
2760	*
2761	* @pmsg will contain the formatted result. @pmsg->pbufs must point to a
2762	* struct printk_buffers.
2763	*
2764	* @seq is the record to read and format. If it is not available, the next
2765	* valid record is read.
2766	*
2767	* @is_extended specifies if the message should be formatted for extended
2768	* console output.
2769	*
2770	* @may_supress specifies if records may be skipped based on loglevel.
2771	*
2772	* Returns false if no record is available. Otherwise true and all fields
2773	* of @pmsg are valid. (See the documentation of struct printk_message
2774	* for information about the @pmsg fields.)
2775	*/
2776	bool printk_get_next_message(struct printk_message *pmsg, u64 seq,
2777	bool is_extended, bool may_suppress)
2778	{
2779	static int panic_console_dropped;
2780
2781	struct printk_buffers *pbufs = pmsg->pbufs;
2782	const size_t scratchbuf_sz = sizeof(pbufs->scratchbuf);
2783	const size_t outbuf_sz = sizeof(pbufs->outbuf);
2784	char *scratchbuf = &pbufs->scratchbuf[0];
2785	char *outbuf = &pbufs->outbuf[0];
2786	struct printk_info info;
2787	struct printk_record r;
2788	size_t len = 0;
2789
2790	/*
2791	* Formatting extended messages requires a separate buffer, so use the
2792	* scratch buffer to read in the ringbuffer text.
2793	*
2794	* Formatting normal messages is done in-place, so read the ringbuffer
2795	* text directly into the output buffer.
2796	*/
2797	if (is_extended)
2798	prb_rec_init_rd(r: &r, info: &info, text_buf: scratchbuf, text_buf_size: scratchbuf_sz);
2799	else
2800	prb_rec_init_rd(r: &r, info: &info, text_buf: outbuf, text_buf_size: outbuf_sz);
2801
2802	if (!prb_read_valid(rb: prb, seq, r: &r))
2803	return false;
2804
2805	pmsg->seq = r.info->seq;
2806	pmsg->dropped = r.info->seq - seq;
2807
2808	/*
2809	* Check for dropped messages in panic here so that printk
2810	* suppression can occur as early as possible if necessary.
2811	*/
2812	if (pmsg->dropped &&
2813	panic_in_progress() &&
2814	panic_console_dropped++ > 10) {
2815	suppress_panic_printk = 1;
2816	pr_warn_once("Too many dropped messages. Suppress messages on non-panic CPUs to prevent livelock.\n");
2817	}
2818
2819	/* Skip record that has level above the console loglevel. */
2820	if (may_suppress && suppress_message_printing(level: r.info->level))
2821	goto out;
2822
2823	if (is_extended) {
2824	len = info_print_ext_header(buf: outbuf, size: outbuf_sz, info: r.info);
2825	len += msg_print_ext_body(buf: outbuf + len, size: outbuf_sz - len,
2826	text: &r.text_buf[0], text_len: r.info->text_len, dev_info: &r.info->dev_info);
2827	} else {
2828	len = record_print_text(r: &r, syslog: console_msg_format & MSG_FORMAT_SYSLOG, time: printk_time);
2829	}
2830	out:
2831	pmsg->outbuf_len = len;
2832	return true;
2833	}
2834
2835	/*
2836	* Used as the printk buffers for non-panic, serialized console printing.
2837	* This is for legacy (!CON_NBCON) as well as all boot (CON_BOOT) consoles.
2838	* Its usage requires the console_lock held.
2839	*/
2840	struct printk_buffers printk_shared_pbufs;
2841
2842	/*
2843	* Print one record for the given console. The record printed is whatever
2844	* record is the next available record for the given console.
2845	*
2846	* @handover will be set to true if a printk waiter has taken over the
2847	* console_lock, in which case the caller is no longer holding both the
2848	* console_lock and the SRCU read lock. Otherwise it is set to false.
2849	*
2850	* @cookie is the cookie from the SRCU read lock.
2851	*
2852	* Returns false if the given console has no next record to print, otherwise
2853	* true.
2854	*
2855	* Requires the console_lock and the SRCU read lock.
2856	*/
2857	static bool console_emit_next_record(struct console *con, bool *handover, int cookie)
2858	{
2859	bool is_extended = console_srcu_read_flags(con) & CON_EXTENDED;
2860	char *outbuf = &printk_shared_pbufs.outbuf[0];
2861	struct printk_message pmsg = {
2862	.pbufs = &printk_shared_pbufs,
2863	};
2864	unsigned long flags;
2865
2866	*handover = false;
2867
2868	if (!printk_get_next_message(pmsg: &pmsg, seq: con->seq, is_extended, may_suppress: true))
2869	return false;
2870
2871	con->dropped += pmsg.dropped;
2872
2873	/* Skip messages of formatted length 0. */
2874	if (pmsg.outbuf_len == 0) {
2875	con->seq = pmsg.seq + 1;
2876	goto skip;
2877	}
2878
2879	if (con->dropped && !is_extended) {
2880	console_prepend_dropped(pmsg: &pmsg, dropped: con->dropped);
2881	con->dropped = 0;
2882	}
2883
2884	/*
2885	* While actively printing out messages, if another printk()
2886	* were to occur on another CPU, it may wait for this one to
2887	* finish. This task can not be preempted if there is a
2888	* waiter waiting to take over.
2889	*
2890	* Interrupts are disabled because the hand over to a waiter
2891	* must not be interrupted until the hand over is completed
2892	* (@console_waiter is cleared).
2893	*/
2894	printk_safe_enter_irqsave(flags);
2895	console_lock_spinning_enable();
2896
2897	/* Do not trace print latency. */
2898	stop_critical_timings();
2899
2900	/* Write everything out to the hardware. */
2901	con->write(con, outbuf, pmsg.outbuf_len);
2902
2903	start_critical_timings();
2904
2905	con->seq = pmsg.seq + 1;
2906
2907	*handover = console_lock_spinning_disable_and_check(cookie);
2908	printk_safe_exit_irqrestore(flags);
2909	skip:
2910	return true;
2911	}
2912
2913	#else
2914
2915	static bool console_emit_next_record(struct console *con, bool *handover, int cookie)
2916	{
2917	*handover = false;
2918	return false;
2919	}
2920
2921	#endif /* CONFIG_PRINTK */
2922
2923	/*
2924	* Print out all remaining records to all consoles.
2925	*
2926	* @do_cond_resched is set by the caller. It can be true only in schedulable
2927	* context.
2928	*
2929	* @next_seq is set to the sequence number after the last available record.
2930	* The value is valid only when this function returns true. It means that all
2931	* usable consoles are completely flushed.
2932	*
2933	* @handover will be set to true if a printk waiter has taken over the
2934	* console_lock, in which case the caller is no longer holding the
2935	* console_lock. Otherwise it is set to false.
2936	*
2937	* Returns true when there was at least one usable console and all messages
2938	* were flushed to all usable consoles. A returned false informs the caller
2939	* that everything was not flushed (either there were no usable consoles or
2940	* another context has taken over printing or it is a panic situation and this
2941	* is not the panic CPU). Regardless the reason, the caller should assume it
2942	* is not useful to immediately try again.
2943	*
2944	* Requires the console_lock.
2945	*/
2946	static bool console_flush_all(bool do_cond_resched, u64 *next_seq, bool *handover)
2947	{
2948	bool any_usable = false;
2949	struct console *con;
2950	bool any_progress;
2951	int cookie;
2952
2953	*next_seq = 0;
2954	*handover = false;
2955
2956	do {
2957	any_progress = false;
2958
2959	cookie = console_srcu_read_lock();
2960	for_each_console_srcu(con) {
2961	bool progress;
2962
2963	if (!console_is_usable(con))
2964	continue;
2965	any_usable = true;
2966
2967	progress = console_emit_next_record(con, handover, cookie);
2968
2969	/*
2970	* If a handover has occurred, the SRCU read lock
2971	* is already released.
2972	*/
2973	if (*handover)
2974	return false;
2975
2976	/* Track the next of the highest seq flushed. */
2977	if (con->seq > *next_seq)
2978	*next_seq = con->seq;
2979
2980	if (!progress)
2981	continue;
2982	any_progress = true;
2983
2984	/* Allow panic_cpu to take over the consoles safely. */
2985	if (other_cpu_in_panic())
2986	goto abandon;
2987
2988	if (do_cond_resched)
2989	cond_resched();
2990	}
2991	console_srcu_read_unlock(cookie);
2992	} while (any_progress);
2993
2994	return any_usable;
2995
2996	abandon:
2997	console_srcu_read_unlock(cookie);
2998	return false;
2999	}
3000
3001	/**
3002	* console_unlock - unblock the console subsystem from printing
3003	*
3004	* Releases the console_lock which the caller holds to block printing of
3005	* the console subsystem.
3006	*
3007	* While the console_lock was held, console output may have been buffered
3008	* by printk(). If this is the case, console_unlock(); emits
3009	* the output prior to releasing the lock.
3010	*
3011	* console_unlock(); may be called from any context.
3012	*/
3013	void console_unlock(void)
3014	{
3015	bool do_cond_resched;
3016	bool handover;
3017	bool flushed;
3018	u64 next_seq;
3019
3020	/*
3021	* Console drivers are called with interrupts disabled, so
3022	* @console_may_schedule should be cleared before; however, we may
3023	* end up dumping a lot of lines, for example, if called from
3024	* console registration path, and should invoke cond_resched()
3025	* between lines if allowable. Not doing so can cause a very long
3026	* scheduling stall on a slow console leading to RCU stall and
3027	* softlockup warnings which exacerbate the issue with more
3028	* messages practically incapacitating the system. Therefore, create
3029	* a local to use for the printing loop.
3030	*/
3031	do_cond_resched = console_may_schedule;
3032
3033	do {
3034	console_may_schedule = 0;
3035
3036	flushed = console_flush_all(do_cond_resched, next_seq: &next_seq, handover: &handover);
3037	if (!handover)
3038	__console_unlock();
3039
3040	/*
3041	* Abort if there was a failure to flush all messages to all
3042	* usable consoles. Either it is not possible to flush (in
3043	* which case it would be an infinite loop of retrying) or
3044	* another context has taken over printing.
3045	*/
3046	if (!flushed)
3047	break;
3048
3049	/*
3050	* Some context may have added new records after
3051	* console_flush_all() but before unlocking the console.
3052	* Re-check if there is a new record to flush. If the trylock
3053	* fails, another context is already handling the printing.
3054	*/
3055	} while (prb_read_valid(rb: prb, seq: next_seq, NULL) && console_trylock());
3056	}
3057	EXPORT_SYMBOL(console_unlock);
3058
3059	/**
3060	* console_conditional_schedule - yield the CPU if required
3061	*
3062	* If the console code is currently allowed to sleep, and
3063	* if this CPU should yield the CPU to another task, do
3064	* so here.
3065	*
3066	* Must be called within console_lock();.
3067	*/
3068	void __sched console_conditional_schedule(void)
3069	{
3070	if (console_may_schedule)
3071	cond_resched();
3072	}
3073	EXPORT_SYMBOL(console_conditional_schedule);
3074
3075	void console_unblank(void)
3076	{
3077	bool found_unblank = false;
3078	struct console *c;
3079	int cookie;
3080
3081	/*
3082	* First check if there are any consoles implementing the unblank()
3083	* callback. If not, there is no reason to continue and take the
3084	* console lock, which in particular can be dangerous if
3085	* @oops_in_progress is set.
3086	*/
3087	cookie = console_srcu_read_lock();
3088	for_each_console_srcu(c) {
3089	if ((console_srcu_read_flags(con: c) & CON_ENABLED) && c->unblank) {
3090	found_unblank = true;
3091	break;
3092	}
3093	}
3094	console_srcu_read_unlock(cookie);
3095	if (!found_unblank)
3096	return;
3097
3098	/*
3099	* Stop console printing because the unblank() callback may
3100	* assume the console is not within its write() callback.
3101	*
3102	* If @oops_in_progress is set, this may be an atomic context.
3103	* In that case, attempt a trylock as best-effort.
3104	*/
3105	if (oops_in_progress) {
3106	/* Semaphores are not NMI-safe. */
3107	if (in_nmi())
3108	return;
3109
3110	/*
3111	* Attempting to trylock the console lock can deadlock
3112	* if another CPU was stopped while modifying the
3113	* semaphore. "Hope and pray" that this is not the
3114	* current situation.
3115	*/
3116	if (down_trylock_console_sem() != 0)
3117	return;
3118	} else
3119	console_lock();
3120
3121	console_locked = 1;
3122	console_may_schedule = 0;
3123
3124	cookie = console_srcu_read_lock();
3125	for_each_console_srcu(c) {
3126	if ((console_srcu_read_flags(con: c) & CON_ENABLED) && c->unblank)
3127	c->unblank();
3128	}
3129	console_srcu_read_unlock(cookie);
3130
3131	console_unlock();
3132
3133	if (!oops_in_progress)
3134	pr_flush(timeout_ms: 1000, reset_on_progress: true);
3135	}
3136
3137	/**
3138	* console_flush_on_panic - flush console content on panic
3139	* @mode: flush all messages in buffer or just the pending ones
3140	*
3141	* Immediately output all pending messages no matter what.
3142	*/
3143	void console_flush_on_panic(enum con_flush_mode mode)
3144	{
3145	bool handover;
3146	u64 next_seq;
3147
3148	/*
3149	* Ignore the console lock and flush out the messages. Attempting a
3150	* trylock would not be useful because:
3151	*
3152	* - if it is contended, it must be ignored anyway
3153	* - console_lock() and console_trylock() block and fail
3154	* respectively in panic for non-panic CPUs
3155	* - semaphores are not NMI-safe
3156	*/
3157
3158	/*
3159	* If another context is holding the console lock,
3160	* @console_may_schedule might be set. Clear it so that
3161	* this context does not call cond_resched() while flushing.
3162	*/
3163	console_may_schedule = 0;
3164
3165	if (mode == CONSOLE_REPLAY_ALL) {
3166	struct console *c;
3167	short flags;
3168	int cookie;
3169	u64 seq;
3170
3171	seq = prb_first_valid_seq(rb: prb);
3172
3173	cookie = console_srcu_read_lock();
3174	for_each_console_srcu(c) {
3175	flags = console_srcu_read_flags(con: c);
3176
3177	if (flags & CON_NBCON) {
3178	nbcon_seq_force(con: c, seq);
3179	} else {
3180	/*
3181	* This is an unsynchronized assignment. On
3182	* panic legacy consoles are only best effort.
3183	*/
3184	c->seq = seq;
3185	}
3186	}
3187	console_srcu_read_unlock(cookie);
3188	}
3189
3190	console_flush_all(do_cond_resched: false, next_seq: &next_seq, handover: &handover);
3191	}
3192
3193	/*
3194	* Return the console tty driver structure and its associated index
3195	*/
3196	struct tty_driver *console_device(int *index)
3197	{
3198	struct console *c;
3199	struct tty_driver *driver = NULL;
3200	int cookie;
3201
3202	/*
3203	* Take console_lock to serialize device() callback with
3204	* other console operations. For example, fg_console is
3205	* modified under console_lock when switching vt.
3206	*/
3207	console_lock();
3208
3209	cookie = console_srcu_read_lock();
3210	for_each_console_srcu(c) {
3211	if (!c->device)
3212	continue;
3213	driver = c->device(c, index);
3214	if (driver)
3215	break;
3216	}
3217	console_srcu_read_unlock(cookie);
3218
3219	console_unlock();
3220	return driver;
3221	}
3222
3223	/*
3224	* Prevent further output on the passed console device so that (for example)
3225	* serial drivers can disable console output before suspending a port, and can
3226	* re-enable output afterwards.
3227	*/
3228	void console_stop(struct console *console)
3229	{
3230	__pr_flush(con: console, timeout_ms: 1000, reset_on_progress: true);
3231	console_list_lock();
3232	console_srcu_write_flags(con: console, flags: console->flags & ~CON_ENABLED);
3233	console_list_unlock();
3234
3235	/*
3236	* Ensure that all SRCU list walks have completed. All contexts must
3237	* be able to see that this console is disabled so that (for example)
3238	* the caller can suspend the port without risk of another context
3239	* using the port.
3240	*/
3241	synchronize_srcu(ssp: &console_srcu);
3242	}
3243	EXPORT_SYMBOL(console_stop);
3244
3245	void console_start(struct console *console)
3246	{
3247	console_list_lock();
3248	console_srcu_write_flags(con: console, flags: console->flags | CON_ENABLED);
3249	console_list_unlock();
3250	__pr_flush(con: console, timeout_ms: 1000, reset_on_progress: true);
3251	}
3252	EXPORT_SYMBOL(console_start);
3253
3254	static int __read_mostly keep_bootcon;
3255
3256	static int __init keep_bootcon_setup(char *str)
3257	{
3258	keep_bootcon = 1;
3259	pr_info("debug: skip boot console de-registration.\n");
3260
3261	return 0;
3262	}
3263
3264	early_param("keep_bootcon", keep_bootcon_setup);
3265
3266	/*
3267	* This is called by register_console() to try to match
3268	* the newly registered console with any of the ones selected
3269	* by either the command line or add_preferred_console() and
3270	* setup/enable it.
3271	*
3272	* Care need to be taken with consoles that are statically
3273	* enabled such as netconsole
3274	*/
3275	static int try_enable_preferred_console(struct console *newcon,
3276	bool user_specified)
3277	{
3278	struct console_cmdline *c;
3279	int i, err;
3280
3281	for (i = 0, c = console_cmdline;
3282	i < MAX_CMDLINECONSOLES && c->name[0];
3283	i++, c++) {
3284	if (c->user_specified != user_specified)
3285	continue;
3286	if (!newcon->match ||
3287	newcon->match(newcon, c->name, c->index, c->options) != 0) {
3288	/* default matching */
3289	BUILD_BUG_ON(sizeof(c->name) != sizeof(newcon->name));
3290	if (strcmp(c->name, newcon->name) != 0)
3291	continue;
3292	if (newcon->index >= 0 &&
3293	newcon->index != c->index)
3294	continue;
3295	if (newcon->index < 0)
3296	newcon->index = c->index;
3297
3298	if (_braille_register_console(console: newcon, c))
3299	return 0;
3300
3301	if (newcon->setup &&
3302	(err = newcon->setup(newcon, c->options)) != 0)
3303	return err;
3304	}
3305	newcon->flags |= CON_ENABLED;
3306	if (i == preferred_console)
3307	newcon->flags |= CON_CONSDEV;
3308	return 0;
3309	}
3310
3311	/*
3312	* Some consoles, such as pstore and netconsole, can be enabled even
3313	* without matching. Accept the pre-enabled consoles only when match()
3314	* and setup() had a chance to be called.
3315	*/
3316	if (newcon->flags & CON_ENABLED && c->user_specified == user_specified)
3317	return 0;
3318
3319	return -ENOENT;
3320	}
3321
3322	/* Try to enable the console unconditionally */
3323	static void try_enable_default_console(struct console *newcon)
3324	{
3325	if (newcon->index < 0)
3326	newcon->index = 0;
3327
3328	if (newcon->setup && newcon->setup(newcon, NULL) != 0)
3329	return;
3330
3331	newcon->flags |= CON_ENABLED;
3332
3333	if (newcon->device)
3334	newcon->flags |= CON_CONSDEV;
3335	}
3336
3337	static void console_init_seq(struct console *newcon, bool bootcon_registered)
3338	{
3339	struct console *con;
3340	bool handover;
3341
3342	if (newcon->flags & (CON_PRINTBUFFER | CON_BOOT)) {
3343	/* Get a consistent copy of @syslog_seq. */
3344	mutex_lock(&syslog_lock);
3345	newcon->seq = syslog_seq;
3346	mutex_unlock(lock: &syslog_lock);
3347	} else {
3348	/* Begin with next message added to ringbuffer. */
3349	newcon->seq = prb_next_seq(rb: prb);
3350
3351	/*
3352	* If any enabled boot consoles are due to be unregistered
3353	* shortly, some may not be caught up and may be the same
3354	* device as @newcon. Since it is not known which boot console
3355	* is the same device, flush all consoles and, if necessary,
3356	* start with the message of the enabled boot console that is
3357	* the furthest behind.
3358	*/
3359	if (bootcon_registered && !keep_bootcon) {
3360	/*
3361	* Hold the console_lock to stop console printing and
3362	* guarantee safe access to console->seq.
3363	*/
3364	console_lock();
3365
3366	/*
3367	* Flush all consoles and set the console to start at
3368	* the next unprinted sequence number.
3369	*/
3370	if (!console_flush_all(do_cond_resched: true, next_seq: &newcon->seq, handover: &handover)) {
3371	/*
3372	* Flushing failed. Just choose the lowest
3373	* sequence of the enabled boot consoles.
3374	*/
3375
3376	/*
3377	* If there was a handover, this context no
3378	* longer holds the console_lock.
3379	*/
3380	if (handover)
3381	console_lock();
3382
3383	newcon->seq = prb_next_seq(rb: prb);
3384	for_each_console(con) {
3385	if ((con->flags & CON_BOOT) &&
3386	(con->flags & CON_ENABLED) &&
3387	con->seq < newcon->seq) {
3388	newcon->seq = con->seq;
3389	}
3390	}
3391	}
3392
3393	console_unlock();
3394	}
3395	}
3396	}
3397
3398	#define console_first() \
3399	hlist_entry(console_list.first, struct console, node)
3400
3401	static int unregister_console_locked(struct console *console);
3402
3403	/*
3404	* The console driver calls this routine during kernel initialization
3405	* to register the console printing procedure with printk() and to
3406	* print any messages that were printed by the kernel before the
3407	* console driver was initialized.
3408	*
3409	* This can happen pretty early during the boot process (because of
3410	* early_printk) - sometimes before setup_arch() completes - be careful
3411	* of what kernel features are used - they may not be initialised yet.
3412	*
3413	* There are two types of consoles - bootconsoles (early_printk) and
3414	* "real" consoles (everything which is not a bootconsole) which are
3415	* handled differently.
3416	* - Any number of bootconsoles can be registered at any time.
3417	* - As soon as a "real" console is registered, all bootconsoles
3418	* will be unregistered automatically.
3419	* - Once a "real" console is registered, any attempt to register a
3420	* bootconsoles will be rejected
3421	*/
3422	void register_console(struct console *newcon)
3423	{
3424	struct console *con;
3425	bool bootcon_registered = false;
3426	bool realcon_registered = false;
3427	int err;
3428
3429	console_list_lock();
3430
3431	for_each_console(con) {
3432	if (WARN(con == newcon, "console '%s%d' already registered\n",
3433	con->name, con->index)) {
3434	goto unlock;
3435	}
3436
3437	if (con->flags & CON_BOOT)
3438	bootcon_registered = true;
3439	else
3440	realcon_registered = true;
3441	}
3442
3443	/* Do not register boot consoles when there already is a real one. */
3444	if ((newcon->flags & CON_BOOT) && realcon_registered) {
3445	pr_info("Too late to register bootconsole %s%d\n",
3446	newcon->name, newcon->index);
3447	goto unlock;
3448	}
3449
3450	if (newcon->flags & CON_NBCON) {
3451	/*
3452	* Ensure the nbcon console buffers can be allocated
3453	* before modifying any global data.
3454	*/
3455	if (!nbcon_alloc(con: newcon))
3456	goto unlock;
3457	}
3458
3459	/*
3460	* See if we want to enable this console driver by default.
3461	*
3462	* Nope when a console is preferred by the command line, device
3463	* tree, or SPCR.
3464	*
3465	* The first real console with tty binding (driver) wins. More
3466	* consoles might get enabled before the right one is found.
3467	*
3468	* Note that a console with tty binding will have CON_CONSDEV
3469	* flag set and will be first in the list.
3470	*/
3471	if (preferred_console < 0) {
3472	if (hlist_empty(h: &console_list) || !console_first()->device ||
3473	console_first()->flags & CON_BOOT) {
3474	try_enable_default_console(newcon);
3475	}
3476	}
3477
3478	/* See if this console matches one we selected on the command line */
3479	err = try_enable_preferred_console(newcon, user_specified: true);
3480
3481	/* If not, try to match against the platform default(s) */
3482	if (err == -ENOENT)
3483	err = try_enable_preferred_console(newcon, user_specified: false);
3484
3485	/* printk() messages are not printed to the Braille console. */
3486	if (err || newcon->flags & CON_BRL) {
3487	if (newcon->flags & CON_NBCON)
3488	nbcon_free(con: newcon);
3489	goto unlock;
3490	}
3491
3492	/*
3493	* If we have a bootconsole, and are switching to a real console,
3494	* don't print everything out again, since when the boot console, and
3495	* the real console are the same physical device, it's annoying to
3496	* see the beginning boot messages twice
3497	*/
3498	if (bootcon_registered &&
3499	((newcon->flags & (CON_CONSDEV | CON_BOOT)) == CON_CONSDEV)) {
3500	newcon->flags &= ~CON_PRINTBUFFER;
3501	}
3502
3503	newcon->dropped = 0;
3504	console_init_seq(newcon, bootcon_registered);
3505
3506	if (newcon->flags & CON_NBCON)
3507	nbcon_init(con: newcon);
3508
3509	/*
3510	* Put this console in the list - keep the
3511	* preferred driver at the head of the list.
3512	*/
3513	if (hlist_empty(h: &console_list)) {
3514	/* Ensure CON_CONSDEV is always set for the head. */
3515	newcon->flags |= CON_CONSDEV;
3516	hlist_add_head_rcu(n: &newcon->node, h: &console_list);
3517
3518	} else if (newcon->flags & CON_CONSDEV) {
3519	/* Only the new head can have CON_CONSDEV set. */
3520	console_srcu_write_flags(console_first(), console_first()->flags & ~CON_CONSDEV);
3521	hlist_add_head_rcu(n: &newcon->node, h: &console_list);
3522
3523	} else {
3524	hlist_add_behind_rcu(n: &newcon->node, prev: console_list.first);
3525	}
3526
3527	/*
3528	* No need to synchronize SRCU here! The caller does not rely
3529	* on all contexts being able to see the new console before
3530	* register_console() completes.
3531	*/
3532
3533	console_sysfs_notify();
3534
3535	/*
3536	* By unregistering the bootconsoles after we enable the real console
3537	* we get the "console xxx enabled" message on all the consoles -
3538	* boot consoles, real consoles, etc - this is to ensure that end
3539	* users know there might be something in the kernel's log buffer that
3540	* went to the bootconsole (that they do not see on the real console)
3541	*/
3542	con_printk(KERN_INFO, newcon, "enabled\n");
3543	if (bootcon_registered &&
3544	((newcon->flags & (CON_CONSDEV | CON_BOOT)) == CON_CONSDEV) &&
3545	!keep_bootcon) {
3546	struct hlist_node *tmp;
3547
3548	hlist_for_each_entry_safe(con, tmp, &console_list, node) {
3549	if (con->flags & CON_BOOT)
3550	unregister_console_locked(console: con);
3551	}
3552	}
3553	unlock:
3554	console_list_unlock();
3555	}
3556	EXPORT_SYMBOL(register_console);
3557
3558	/* Must be called under console_list_lock(). */
3559	static int unregister_console_locked(struct console *console)
3560	{
3561	int res;
3562
3563	lockdep_assert_console_list_lock_held();
3564
3565	con_printk(KERN_INFO, console, "disabled\n");
3566
3567	res = _braille_unregister_console(console);
3568	if (res < 0)
3569	return res;
3570	if (res > 0)
3571	return 0;
3572
3573	/* Disable it unconditionally */
3574	console_srcu_write_flags(con: console, flags: console->flags & ~CON_ENABLED);
3575
3576	if (!console_is_registered_locked(con: console))
3577	return -ENODEV;
3578
3579	hlist_del_init_rcu(n: &console->node);
3580
3581	/*
3582	* <HISTORICAL>
3583	* If this isn't the last console and it has CON_CONSDEV set, we
3584	* need to set it on the next preferred console.
3585	* </HISTORICAL>
3586	*
3587	* The above makes no sense as there is no guarantee that the next
3588	* console has any device attached. Oh well....
3589	*/
3590	if (!hlist_empty(h: &console_list) && console->flags & CON_CONSDEV)
3591	console_srcu_write_flags(console_first(), console_first()->flags | CON_CONSDEV);
3592
3593	/*
3594	* Ensure that all SRCU list walks have completed. All contexts
3595	* must not be able to see this console in the list so that any
3596	* exit/cleanup routines can be performed safely.
3597	*/
3598	synchronize_srcu(ssp: &console_srcu);
3599
3600	if (console->flags & CON_NBCON)
3601	nbcon_free(con: console);
3602
3603	console_sysfs_notify();
3604
3605	if (console->exit)
3606	res = console->exit(console);
3607
3608	return res;
3609	}
3610
3611	int unregister_console(struct console *console)
3612	{
3613	int res;
3614
3615	console_list_lock();
3616	res = unregister_console_locked(console);
3617	console_list_unlock();
3618	return res;
3619	}
3620	EXPORT_SYMBOL(unregister_console);
3621
3622	/**
3623	* console_force_preferred_locked - force a registered console preferred
3624	* @con: The registered console to force preferred.
3625	*
3626	* Must be called under console_list_lock().
3627	*/
3628	void console_force_preferred_locked(struct console *con)
3629	{
3630	struct console *cur_pref_con;
3631
3632	if (!console_is_registered_locked(con))
3633	return;
3634
3635	cur_pref_con = console_first();
3636
3637	/* Already preferred? */
3638	if (cur_pref_con == con)
3639	return;
3640
3641	/*
3642	* Delete, but do not re-initialize the entry. This allows the console
3643	* to continue to appear registered (via any hlist_unhashed_lockless()
3644	* checks), even though it was briefly removed from the console list.
3645	*/
3646	hlist_del_rcu(n: &con->node);
3647
3648	/*
3649	* Ensure that all SRCU list walks have completed so that the console
3650	* can be added to the beginning of the console list and its forward
3651	* list pointer can be re-initialized.
3652	*/
3653	synchronize_srcu(ssp: &console_srcu);
3654
3655	con->flags |= CON_CONSDEV;
3656	WARN_ON(!con->device);
3657
3658	/* Only the new head can have CON_CONSDEV set. */
3659	console_srcu_write_flags(con: cur_pref_con, flags: cur_pref_con->flags & ~CON_CONSDEV);
3660	hlist_add_head_rcu(n: &con->node, h: &console_list);
3661	}
3662	EXPORT_SYMBOL(console_force_preferred_locked);
3663
3664	/*
3665	* Initialize the console device. This is called *early*, so
3666	* we can't necessarily depend on lots of kernel help here.
3667	* Just do some early initializations, and do the complex setup
3668	* later.
3669	*/
3670	void __init console_init(void)
3671	{
3672	int ret;
3673	initcall_t call;
3674	initcall_entry_t *ce;
3675
3676	/* Setup the default TTY line discipline. */
3677	n_tty_init();
3678
3679	/*
3680	* set up the console device so that later boot sequences can
3681	* inform about problems etc..
3682	*/
3683	ce = __con_initcall_start;
3684	trace_initcall_level(level: "console");
3685	while (ce < __con_initcall_end) {
3686	call = initcall_from_entry(entry: ce);
3687	trace_initcall_start(func: call);
3688	ret = call();
3689	trace_initcall_finish(func: call, ret);
3690	ce++;
3691	}
3692	}
3693
3694	/*
3695	* Some boot consoles access data that is in the init section and which will
3696	* be discarded after the initcalls have been run. To make sure that no code
3697	* will access this data, unregister the boot consoles in a late initcall.
3698	*
3699	* If for some reason, such as deferred probe or the driver being a loadable
3700	* module, the real console hasn't registered yet at this point, there will
3701	* be a brief interval in which no messages are logged to the console, which
3702	* makes it difficult to diagnose problems that occur during this time.
3703	*
3704	* To mitigate this problem somewhat, only unregister consoles whose memory
3705	* intersects with the init section. Note that all other boot consoles will
3706	* get unregistered when the real preferred console is registered.
3707	*/
3708	static int __init printk_late_init(void)
3709	{
3710	struct hlist_node *tmp;
3711	struct console *con;
3712	int ret;
3713
3714	console_list_lock();
3715	hlist_for_each_entry_safe(con, tmp, &console_list, node) {
3716	if (!(con->flags & CON_BOOT))
3717	continue;
3718
3719	/* Check addresses that might be used for enabled consoles. */
3720	if (init_section_intersects(virt: con, size: sizeof(*con)) ||
3721	init_section_contains(virt: con->write, size: 0) ||
3722	init_section_contains(virt: con->read, size: 0) ||
3723	init_section_contains(virt: con->device, size: 0) ||
3724	init_section_contains(virt: con->unblank, size: 0) ||
3725	init_section_contains(virt: con->data, size: 0)) {
3726	/*
3727	* Please, consider moving the reported consoles out
3728	* of the init section.
3729	*/
3730	pr_warn("bootconsole [%s%d] uses init memory and must be disabled even before the real one is ready\n",
3731	con->name, con->index);
3732	unregister_console_locked(console: con);
3733	}
3734	}
3735	console_list_unlock();
3736
3737	ret = cpuhp_setup_state_nocalls(state: CPUHP_PRINTK_DEAD, name: "printk:dead", NULL,
3738	teardown: console_cpu_notify);
3739	WARN_ON(ret < 0);
3740	ret = cpuhp_setup_state_nocalls(state: CPUHP_AP_ONLINE_DYN, name: "printk:online",
3741	startup: console_cpu_notify, NULL);
3742	WARN_ON(ret < 0);
3743	printk_sysctl_init();
3744	return 0;
3745	}
3746	late_initcall(printk_late_init);
3747
3748	#if defined CONFIG_PRINTK
3749	/* If @con is specified, only wait for that console. Otherwise wait for all. */
3750	static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress)
3751	{
3752	unsigned long timeout_jiffies = msecs_to_jiffies(m: timeout_ms);
3753	unsigned long remaining_jiffies = timeout_jiffies;
3754	struct console *c;
3755	u64 last_diff = 0;
3756	u64 printk_seq;
3757	short flags;
3758	int cookie;
3759	u64 diff;
3760	u64 seq;
3761
3762	might_sleep();
3763
3764	seq = prb_next_seq(rb: prb);
3765
3766	/* Flush the consoles so that records up to @seq are printed. */
3767	console_lock();
3768	console_unlock();
3769
3770	for (;;) {
3771	unsigned long begin_jiffies;
3772	unsigned long slept_jiffies;
3773
3774	diff = 0;
3775
3776	/*
3777	* Hold the console_lock to guarantee safe access to
3778	* console->seq. Releasing console_lock flushes more
3779	* records in case @seq is still not printed on all
3780	* usable consoles.
3781	*/
3782	console_lock();
3783
3784	cookie = console_srcu_read_lock();
3785	for_each_console_srcu(c) {
3786	if (con && con != c)
3787	continue;
3788
3789	flags = console_srcu_read_flags(con: c);
3790
3791	/*
3792	* If consoles are not usable, it cannot be expected
3793	* that they make forward progress, so only increment
3794	* @diff for usable consoles.
3795	*/
3796	if (!console_is_usable(con: c))
3797	continue;
3798
3799	if (flags & CON_NBCON) {
3800	printk_seq = nbcon_seq_read(con: c);
3801	} else {
3802	printk_seq = c->seq;
3803	}
3804
3805	if (printk_seq < seq)
3806	diff += seq - printk_seq;
3807	}
3808	console_srcu_read_unlock(cookie);
3809
3810	if (diff != last_diff && reset_on_progress)
3811	remaining_jiffies = timeout_jiffies;
3812
3813	console_unlock();
3814
3815	/* Note: @diff is 0 if there are no usable consoles. */
3816	if (diff == 0 || remaining_jiffies == 0)
3817	break;
3818
3819	/* msleep(1) might sleep much longer. Check time by jiffies. */
3820	begin_jiffies = jiffies;
3821	msleep(msecs: 1);
3822	slept_jiffies = jiffies - begin_jiffies;
3823
3824	remaining_jiffies -= min(slept_jiffies, remaining_jiffies);
3825
3826	last_diff = diff;
3827	}
3828
3829	return (diff == 0);
3830	}
3831
3832	/**
3833	* pr_flush() - Wait for printing threads to catch up.
3834	*
3835	* @timeout_ms: The maximum time (in ms) to wait.
3836	* @reset_on_progress: Reset the timeout if forward progress is seen.
3837	*
3838	* A value of 0 for @timeout_ms means no waiting will occur. A value of -1
3839	* represents infinite waiting.
3840	*
3841	* If @reset_on_progress is true, the timeout will be reset whenever any
3842	* printer has been seen to make some forward progress.
3843	*
3844	* Context: Process context. May sleep while acquiring console lock.
3845	* Return: true if all usable printers are caught up.
3846	*/
3847	static bool pr_flush(int timeout_ms, bool reset_on_progress)
3848	{
3849	return __pr_flush(NULL, timeout_ms, reset_on_progress);
3850	}
3851
3852	/*
3853	* Delayed printk version, for scheduler-internal messages:
3854	*/
3855	#define PRINTK_PENDING_WAKEUP 0x01
3856	#define PRINTK_PENDING_OUTPUT 0x02
3857
3858	static DEFINE_PER_CPU(int, printk_pending);
3859
3860	static void wake_up_klogd_work_func(struct irq_work *irq_work)
3861	{
3862	int pending = this_cpu_xchg(printk_pending, 0);
3863
3864	if (pending & PRINTK_PENDING_OUTPUT) {
3865	/* If trylock fails, someone else is doing the printing */
3866	if (console_trylock())
3867	console_unlock();
3868	}
3869
3870	if (pending & PRINTK_PENDING_WAKEUP)
3871	wake_up_interruptible(&log_wait);
3872	}
3873
3874	static DEFINE_PER_CPU(struct irq_work, wake_up_klogd_work) =
3875	IRQ_WORK_INIT_LAZY(wake_up_klogd_work_func);
3876
3877	static void __wake_up_klogd(int val)
3878	{
3879	if (!printk_percpu_data_ready())
3880	return;
3881
3882	preempt_disable();
3883	/*
3884	* Guarantee any new records can be seen by tasks preparing to wait
3885	* before this context checks if the wait queue is empty.
3886	*
3887	* The full memory barrier within wq_has_sleeper() pairs with the full
3888	* memory barrier within set_current_state() of
3889	* prepare_to_wait_event(), which is called after ___wait_event() adds
3890	* the waiter but before it has checked the wait condition.
3891	*
3892	* This pairs with devkmsg_read:A and syslog_print:A.
3893	*/
3894	if (wq_has_sleeper(wq_head: &log_wait) || /* LMM(__wake_up_klogd:A) */
3895	(val & PRINTK_PENDING_OUTPUT)) {
3896	this_cpu_or(printk_pending, val);
3897	irq_work_queue(this_cpu_ptr(&wake_up_klogd_work));
3898	}
3899	preempt_enable();
3900	}
3901
3902	/**
3903	* wake_up_klogd - Wake kernel logging daemon
3904	*
3905	* Use this function when new records have been added to the ringbuffer
3906	* and the console printing of those records has already occurred or is
3907	* known to be handled by some other context. This function will only
3908	* wake the logging daemon.
3909	*
3910	* Context: Any context.
3911	*/
3912	void wake_up_klogd(void)
3913	{
3914	__wake_up_klogd(PRINTK_PENDING_WAKEUP);
3915	}
3916
3917	/**
3918	* defer_console_output - Wake kernel logging daemon and trigger
3919	* console printing in a deferred context
3920	*
3921	* Use this function when new records have been added to the ringbuffer,
3922	* this context is responsible for console printing those records, but
3923	* the current context is not allowed to perform the console printing.
3924	* Trigger an irq_work context to perform the console printing. This
3925	* function also wakes the logging daemon.
3926	*
3927	* Context: Any context.
3928	*/
3929	void defer_console_output(void)
3930	{
3931	/*
3932	* New messages may have been added directly to the ringbuffer
3933	* using vprintk_store(), so wake any waiters as well.
3934	*/
3935	__wake_up_klogd(PRINTK_PENDING_WAKEUP | PRINTK_PENDING_OUTPUT);
3936	}
3937
3938	void printk_trigger_flush(void)
3939	{
3940	defer_console_output();
3941	}
3942
3943	int vprintk_deferred(const char *fmt, va_list args)
3944	{
3945	return vprintk_emit(0, LOGLEVEL_SCHED, NULL, fmt, args);
3946	}
3947
3948	int _printk_deferred(const char *fmt, ...)
3949	{
3950	va_list args;
3951	int r;
3952
3953	va_start(args, fmt);
3954	r = vprintk_deferred(fmt, args);
3955	va_end(args);
3956
3957	return r;
3958	}
3959
3960	/*
3961	* printk rate limiting, lifted from the networking subsystem.
3962	*
3963	* This enforces a rate limit: not more than 10 kernel messages
3964	* every 5s to make a denial-of-service attack impossible.
3965	*/
3966	DEFINE_RATELIMIT_STATE(printk_ratelimit_state, 5 * HZ, 10);
3967
3968	int __printk_ratelimit(const char *func)
3969	{
3970	return ___ratelimit(rs: &printk_ratelimit_state, func);
3971	}
3972	EXPORT_SYMBOL(__printk_ratelimit);
3973
3974	/**
3975	* printk_timed_ratelimit - caller-controlled printk ratelimiting
3976	* @caller_jiffies: pointer to caller's state
3977	* @interval_msecs: minimum interval between prints
3978	*
3979	* printk_timed_ratelimit() returns true if more than @interval_msecs
3980	* milliseconds have elapsed since the last time printk_timed_ratelimit()
3981	* returned true.
3982	*/
3983	bool printk_timed_ratelimit(unsigned long *caller_jiffies,
3984	unsigned int interval_msecs)
3985	{
3986	unsigned long elapsed = jiffies - *caller_jiffies;
3987
3988	if (*caller_jiffies && elapsed <= msecs_to_jiffies(m: interval_msecs))
3989	return false;
3990
3991	*caller_jiffies = jiffies;
3992	return true;
3993	}
3994	EXPORT_SYMBOL(printk_timed_ratelimit);
3995
3996	static DEFINE_SPINLOCK(dump_list_lock);
3997	static LIST_HEAD(dump_list);
3998
3999	/**
4000	* kmsg_dump_register - register a kernel log dumper.
4001	* @dumper: pointer to the kmsg_dumper structure
4002	*
4003	* Adds a kernel log dumper to the system. The dump callback in the
4004	* structure will be called when the kernel oopses or panics and must be
4005	* set. Returns zero on success and %-EINVAL or %-EBUSY otherwise.
4006	*/
4007	int kmsg_dump_register(struct kmsg_dumper *dumper)
4008	{
4009	unsigned long flags;
4010	int err = -EBUSY;
4011
4012	/* The dump callback needs to be set */
4013	if (!dumper->dump)
4014	return -EINVAL;
4015
4016	spin_lock_irqsave(&dump_list_lock, flags);
4017	/* Don't allow registering multiple times */
4018	if (!dumper->registered) {
4019	dumper->registered = 1;
4020	list_add_tail_rcu(new: &dumper->list, head: &dump_list);
4021	err = 0;
4022	}
4023	spin_unlock_irqrestore(lock: &dump_list_lock, flags);
4024
4025	return err;
4026	}
4027	EXPORT_SYMBOL_GPL(kmsg_dump_register);
4028
4029	/**
4030	* kmsg_dump_unregister - unregister a kmsg dumper.
4031	* @dumper: pointer to the kmsg_dumper structure
4032	*
4033	* Removes a dump device from the system. Returns zero on success and
4034	* %-EINVAL otherwise.
4035	*/
4036	int kmsg_dump_unregister(struct kmsg_dumper *dumper)
4037	{
4038	unsigned long flags;
4039	int err = -EINVAL;
4040
4041	spin_lock_irqsave(&dump_list_lock, flags);
4042	if (dumper->registered) {
4043	dumper->registered = 0;
4044	list_del_rcu(entry: &dumper->list);
4045	err = 0;
4046	}
4047	spin_unlock_irqrestore(lock: &dump_list_lock, flags);
4048	synchronize_rcu();
4049
4050	return err;
4051	}
4052	EXPORT_SYMBOL_GPL(kmsg_dump_unregister);
4053
4054	static bool always_kmsg_dump;
4055	module_param_named(always_kmsg_dump, always_kmsg_dump, bool, S_IRUGO | S_IWUSR);
4056
4057	const char *kmsg_dump_reason_str(enum kmsg_dump_reason reason)
4058	{
4059	switch (reason) {
4060	case KMSG_DUMP_PANIC:
4061	return "Panic";
4062	case KMSG_DUMP_OOPS:
4063	return "Oops";
4064	case KMSG_DUMP_EMERG:
4065	return "Emergency";
4066	case KMSG_DUMP_SHUTDOWN:
4067	return "Shutdown";
4068	default:
4069	return "Unknown";
4070	}
4071	}
4072	EXPORT_SYMBOL_GPL(kmsg_dump_reason_str);
4073
4074	/**
4075	* kmsg_dump - dump kernel log to kernel message dumpers.
4076	* @reason: the reason (oops, panic etc) for dumping
4077	*
4078	* Call each of the registered dumper's dump() callback, which can
4079	* retrieve the kmsg records with kmsg_dump_get_line() or
4080	* kmsg_dump_get_buffer().
4081	*/
4082	void kmsg_dump(enum kmsg_dump_reason reason)
4083	{
4084	struct kmsg_dumper *dumper;
4085
4086	rcu_read_lock();
4087	list_for_each_entry_rcu(dumper, &dump_list, list) {
4088	enum kmsg_dump_reason max_reason = dumper->max_reason;
4089
4090	/*
4091	* If client has not provided a specific max_reason, default
4092	* to KMSG_DUMP_OOPS, unless always_kmsg_dump was set.
4093	*/
4094	if (max_reason == KMSG_DUMP_UNDEF) {
4095	max_reason = always_kmsg_dump ? KMSG_DUMP_MAX :
4096	KMSG_DUMP_OOPS;
4097	}
4098	if (reason > max_reason)
4099	continue;
4100
4101	/* invoke dumper which will iterate over records */
4102	dumper->dump(dumper, reason);
4103	}
4104	rcu_read_unlock();
4105	}
4106
4107	/**
4108	* kmsg_dump_get_line - retrieve one kmsg log line
4109	* @iter: kmsg dump iterator
4110	* @syslog: include the "<4>" prefixes
4111	* @line: buffer to copy the line to
4112	* @size: maximum size of the buffer
4113	* @len: length of line placed into buffer
4114	*
4115	* Start at the beginning of the kmsg buffer, with the oldest kmsg
4116	* record, and copy one record into the provided buffer.
4117	*
4118	* Consecutive calls will return the next available record moving
4119	* towards the end of the buffer with the youngest messages.
4120	*
4121	* A return value of FALSE indicates that there are no more records to
4122	* read.
4123	*/
4124	bool kmsg_dump_get_line(struct kmsg_dump_iter *iter, bool syslog,
4125	char *line, size_t size, size_t *len)
4126	{
4127	u64 min_seq = latched_seq_read_nolock(ls: &clear_seq);
4128	struct printk_info info;
4129	unsigned int line_count;
4130	struct printk_record r;
4131	size_t l = 0;
4132	bool ret = false;
4133
4134	if (iter->cur_seq < min_seq)
4135	iter->cur_seq = min_seq;
4136
4137	prb_rec_init_rd(r: &r, info: &info, text_buf: line, text_buf_size: size);
4138
4139	/* Read text or count text lines? */
4140	if (line) {
4141	if (!prb_read_valid(rb: prb, seq: iter->cur_seq, r: &r))
4142	goto out;
4143	l = record_print_text(r: &r, syslog, time: printk_time);
4144	} else {
4145	if (!prb_read_valid_info(rb: prb, seq: iter->cur_seq,
4146	info: &info, line_count: &line_count)) {
4147	goto out;
4148	}
4149	l = get_record_print_text_size(info: &info, line_count, syslog,
4150	time: printk_time);
4151
4152	}
4153
4154	iter->cur_seq = r.info->seq + 1;
4155	ret = true;
4156	out:
4157	if (len)
4158	*len = l;
4159	return ret;
4160	}
4161	EXPORT_SYMBOL_GPL(kmsg_dump_get_line);
4162
4163	/**
4164	* kmsg_dump_get_buffer - copy kmsg log lines
4165	* @iter: kmsg dump iterator
4166	* @syslog: include the "<4>" prefixes
4167	* @buf: buffer to copy the line to
4168	* @size: maximum size of the buffer
4169	* @len_out: length of line placed into buffer
4170	*
4171	* Start at the end of the kmsg buffer and fill the provided buffer
4172	* with as many of the *youngest* kmsg records that fit into it.
4173	* If the buffer is large enough, all available kmsg records will be
4174	* copied with a single call.
4175	*
4176	* Consecutive calls will fill the buffer with the next block of
4177	* available older records, not including the earlier retrieved ones.
4178	*
4179	* A return value of FALSE indicates that there are no more records to
4180	* read.
4181	*/
4182	bool kmsg_dump_get_buffer(struct kmsg_dump_iter *iter, bool syslog,
4183	char *buf, size_t size, size_t *len_out)
4184	{
4185	u64 min_seq = latched_seq_read_nolock(ls: &clear_seq);
4186	struct printk_info info;
4187	struct printk_record r;
4188	u64 seq;
4189	u64 next_seq;
4190	size_t len = 0;
4191	bool ret = false;
4192	bool time = printk_time;
4193
4194	if (!buf || !size)
4195	goto out;
4196
4197	if (iter->cur_seq < min_seq)
4198	iter->cur_seq = min_seq;
4199
4200	if (prb_read_valid_info(rb: prb, seq: iter->cur_seq, info: &info, NULL)) {
4201	if (info.seq != iter->cur_seq) {
4202	/* messages are gone, move to first available one */
4203	iter->cur_seq = info.seq;
4204	}
4205	}
4206
4207	/* last entry */
4208	if (iter->cur_seq >= iter->next_seq)
4209	goto out;
4210
4211	/*
4212	* Find first record that fits, including all following records,
4213	* into the user-provided buffer for this dump. Pass in size-1
4214	* because this function (by way of record_print_text()) will
4215	* not write more than size-1 bytes of text into @buf.
4216	*/
4217	seq = find_first_fitting_seq(start_seq: iter->cur_seq, max_seq: iter->next_seq,
4218	size: size - 1, syslog, time);
4219
4220	/*
4221	* Next kmsg_dump_get_buffer() invocation will dump block of
4222	* older records stored right before this one.
4223	*/
4224	next_seq = seq;
4225
4226	prb_rec_init_rd(r: &r, info: &info, text_buf: buf, text_buf_size: size);
4227
4228	prb_for_each_record(seq, prb, seq, &r) {
4229	if (r.info->seq >= iter->next_seq)
4230	break;
4231
4232	len += record_print_text(r: &r, syslog, time);
4233
4234	/* Adjust record to store to remaining buffer space. */
4235	prb_rec_init_rd(r: &r, info: &info, text_buf: buf + len, text_buf_size: size - len);
4236	}
4237
4238	iter->next_seq = next_seq;
4239	ret = true;
4240	out:
4241	if (len_out)
4242	*len_out = len;
4243	return ret;
4244	}
4245	EXPORT_SYMBOL_GPL(kmsg_dump_get_buffer);
4246
4247	/**
4248	* kmsg_dump_rewind - reset the iterator
4249	* @iter: kmsg dump iterator
4250	*
4251	* Reset the dumper's iterator so that kmsg_dump_get_line() and
4252	* kmsg_dump_get_buffer() can be called again and used multiple
4253	* times within the same dumper.dump() callback.
4254	*/
4255	void kmsg_dump_rewind(struct kmsg_dump_iter *iter)
4256	{
4257	iter->cur_seq = latched_seq_read_nolock(ls: &clear_seq);
4258	iter->next_seq = prb_next_seq(rb: prb);
4259	}
4260	EXPORT_SYMBOL_GPL(kmsg_dump_rewind);
4261
4262	#endif
4263
4264	#ifdef CONFIG_SMP
4265	static atomic_t printk_cpu_sync_owner = ATOMIC_INIT(-1);
4266	static atomic_t printk_cpu_sync_nested = ATOMIC_INIT(0);
4267
4268	/**
4269	* __printk_cpu_sync_wait() - Busy wait until the printk cpu-reentrant
4270	* spinning lock is not owned by any CPU.
4271	*
4272	* Context: Any context.
4273	*/
4274	void __printk_cpu_sync_wait(void)
4275	{
4276	do {
4277	cpu_relax();
4278	} while (atomic_read(v: &printk_cpu_sync_owner) != -1);
4279	}
4280	EXPORT_SYMBOL(__printk_cpu_sync_wait);
4281
4282	/**
4283	* __printk_cpu_sync_try_get() - Try to acquire the printk cpu-reentrant
4284	* spinning lock.
4285	*
4286	* If no processor has the lock, the calling processor takes the lock and
4287	* becomes the owner. If the calling processor is already the owner of the
4288	* lock, this function succeeds immediately.
4289	*
4290	* Context: Any context. Expects interrupts to be disabled.
4291	* Return: 1 on success, otherwise 0.
4292	*/
4293	int __printk_cpu_sync_try_get(void)
4294	{
4295	int cpu;
4296	int old;
4297
4298	cpu = smp_processor_id();
4299
4300	/*
4301	* Guarantee loads and stores from this CPU when it is the lock owner
4302	* are _not_ visible to the previous lock owner. This pairs with
4303	* __printk_cpu_sync_put:B.
4304	*
4305	* Memory barrier involvement:
4306	*
4307	* If __printk_cpu_sync_try_get:A reads from __printk_cpu_sync_put:B,
4308	* then __printk_cpu_sync_put:A can never read from
4309	* __printk_cpu_sync_try_get:B.
4310	*
4311	* Relies on:
4312	*
4313	* RELEASE from __printk_cpu_sync_put:A to __printk_cpu_sync_put:B
4314	* of the previous CPU
4315	* matching
4316	* ACQUIRE from __printk_cpu_sync_try_get:A to
4317	* __printk_cpu_sync_try_get:B of this CPU
4318	*/
4319	old = atomic_cmpxchg_acquire(v: &printk_cpu_sync_owner, old: -1,
4320	new: cpu); /* LMM(__printk_cpu_sync_try_get:A) */
4321	if (old == -1) {
4322	/*
4323	* This CPU is now the owner and begins loading/storing
4324	* data: LMM(__printk_cpu_sync_try_get:B)
4325	*/
4326	return 1;
4327
4328	} else if (old == cpu) {
4329	/* This CPU is already the owner. */
4330	atomic_inc(v: &printk_cpu_sync_nested);
4331	return 1;
4332	}
4333
4334	return 0;
4335	}
4336	EXPORT_SYMBOL(__printk_cpu_sync_try_get);
4337
4338	/**
4339	* __printk_cpu_sync_put() - Release the printk cpu-reentrant spinning lock.
4340	*
4341	* The calling processor must be the owner of the lock.
4342	*
4343	* Context: Any context. Expects interrupts to be disabled.
4344	*/
4345	void __printk_cpu_sync_put(void)
4346	{
4347	if (atomic_read(v: &printk_cpu_sync_nested)) {
4348	atomic_dec(v: &printk_cpu_sync_nested);
4349	return;
4350	}
4351
4352	/*
4353	* This CPU is finished loading/storing data:
4354	* LMM(__printk_cpu_sync_put:A)
4355	*/
4356
4357	/*
4358	* Guarantee loads and stores from this CPU when it was the
4359	* lock owner are visible to the next lock owner. This pairs
4360	* with __printk_cpu_sync_try_get:A.
4361	*
4362	* Memory barrier involvement:
4363	*
4364	* If __printk_cpu_sync_try_get:A reads from __printk_cpu_sync_put:B,
4365	* then __printk_cpu_sync_try_get:B reads from __printk_cpu_sync_put:A.
4366	*
4367	* Relies on:
4368	*
4369	* RELEASE from __printk_cpu_sync_put:A to __printk_cpu_sync_put:B
4370	* of this CPU
4371	* matching
4372	* ACQUIRE from __printk_cpu_sync_try_get:A to
4373	* __printk_cpu_sync_try_get:B of the next CPU
4374	*/
4375	atomic_set_release(v: &printk_cpu_sync_owner,
4376	i: -1); /* LMM(__printk_cpu_sync_put:B) */
4377	}
4378	EXPORT_SYMBOL(__printk_cpu_sync_put);
4379	#endif /* CONFIG_SMP */
4380