1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
4	* Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
5	* Copyright(C) 2006-2007 Timesys Corp., Thomas Gleixner
6	*
7	* High-resolution kernel timers
8	*
9	* In contrast to the low-resolution timeout API, aka timer wheel,
10	* hrtimers provide finer resolution and accuracy depending on system
11	* configuration and capabilities.
12	*
13	* Started by: Thomas Gleixner and Ingo Molnar
14	*
15	* Credits:
16	* Based on the original timer wheel code
17	*
18	* Help, testing, suggestions, bugfixes, improvements were
19	* provided by:
20	*
21	* George Anzinger, Andrew Morton, Steven Rostedt, Roman Zippel
22	* et. al.
23	*/
24
25	#include <linux/cpu.h>
26	#include <linux/export.h>
27	#include <linux/percpu.h>
28	#include <linux/hrtimer.h>
29	#include <linux/notifier.h>
30	#include <linux/syscalls.h>
31	#include <linux/interrupt.h>
32	#include <linux/tick.h>
33	#include <linux/err.h>
34	#include <linux/debugobjects.h>
35	#include <linux/sched/signal.h>
36	#include <linux/sched/sysctl.h>
37	#include <linux/sched/rt.h>
38	#include <linux/sched/deadline.h>
39	#include <linux/sched/nohz.h>
40	#include <linux/sched/debug.h>
41	#include <linux/sched/isolation.h>
42	#include <linux/timer.h>
43	#include <linux/freezer.h>
44	#include <linux/compat.h>
45
46	#include <linux/uaccess.h>
47
48	#include <trace/events/timer.h>
49
50	#include "tick-internal.h"
51
52	/*
53	* Masks for selecting the soft and hard context timers from
54	* cpu_base->active
55	*/
56	#define MASK_SHIFT (HRTIMER_BASE_MONOTONIC_SOFT)
57	#define HRTIMER_ACTIVE_HARD ((1U << MASK_SHIFT) - 1)
58	#define HRTIMER_ACTIVE_SOFT (HRTIMER_ACTIVE_HARD << MASK_SHIFT)
59	#define HRTIMER_ACTIVE_ALL (HRTIMER_ACTIVE_SOFT | HRTIMER_ACTIVE_HARD)
60
61	/*
62	* The timer bases:
63	*
64	* There are more clockids than hrtimer bases. Thus, we index
65	* into the timer bases by the hrtimer_base_type enum. When trying
66	* to reach a base using a clockid, hrtimer_clockid_to_base()
67	* is used to convert from clockid to the proper hrtimer_base_type.
68	*/
69	DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) =
70	{
71	.lock = __RAW_SPIN_LOCK_UNLOCKED(hrtimer_bases.lock),
72	.clock_base =
73	{
74	{
75	.index = HRTIMER_BASE_MONOTONIC,
76	.clockid = CLOCK_MONOTONIC,
77	.get_time = &ktime_get,
78	},
79	{
80	.index = HRTIMER_BASE_REALTIME,
81	.clockid = CLOCK_REALTIME,
82	.get_time = &ktime_get_real,
83	},
84	{
85	.index = HRTIMER_BASE_BOOTTIME,
86	.clockid = CLOCK_BOOTTIME,
87	.get_time = &ktime_get_boottime,
88	},
89	{
90	.index = HRTIMER_BASE_TAI,
91	.clockid = CLOCK_TAI,
92	.get_time = &ktime_get_clocktai,
93	},
94	{
95	.index = HRTIMER_BASE_MONOTONIC_SOFT,
96	.clockid = CLOCK_MONOTONIC,
97	.get_time = &ktime_get,
98	},
99	{
100	.index = HRTIMER_BASE_REALTIME_SOFT,
101	.clockid = CLOCK_REALTIME,
102	.get_time = &ktime_get_real,
103	},
104	{
105	.index = HRTIMER_BASE_BOOTTIME_SOFT,
106	.clockid = CLOCK_BOOTTIME,
107	.get_time = &ktime_get_boottime,
108	},
109	{
110	.index = HRTIMER_BASE_TAI_SOFT,
111	.clockid = CLOCK_TAI,
112	.get_time = &ktime_get_clocktai,
113	},
114	}
115	};
116
117	static const int hrtimer_clock_to_base_table[MAX_CLOCKS] = {
118	/* Make sure we catch unsupported clockids */
119	[0 ... MAX_CLOCKS - 1] = HRTIMER_MAX_CLOCK_BASES,
120
121	[CLOCK_REALTIME] = HRTIMER_BASE_REALTIME,
122	[CLOCK_MONOTONIC] = HRTIMER_BASE_MONOTONIC,
123	[CLOCK_BOOTTIME] = HRTIMER_BASE_BOOTTIME,
124	[CLOCK_TAI] = HRTIMER_BASE_TAI,
125	};
126
127	/*
128	* Functions and macros which are different for UP/SMP systems are kept in a
129	* single place
130	*/
131	#ifdef CONFIG_SMP
132
133	/*
134	* We require the migration_base for lock_hrtimer_base()/switch_hrtimer_base()
135	* such that hrtimer_callback_running() can unconditionally dereference
136	* timer->base->cpu_base
137	*/
138	static struct hrtimer_cpu_base migration_cpu_base = {
139	.clock_base = { {
140	.cpu_base = &migration_cpu_base,
141	.seq = SEQCNT_RAW_SPINLOCK_ZERO(migration_cpu_base.seq,
142	&migration_cpu_base.lock),
143	}, },
144	};
145
146	#define migration_base migration_cpu_base.clock_base[0]
147
148	static inline bool is_migration_base(struct hrtimer_clock_base *base)
149	{
150	return base == &migration_base;
151	}
152
153	/*
154	* We are using hashed locking: holding per_cpu(hrtimer_bases)[n].lock
155	* means that all timers which are tied to this base via timer->base are
156	* locked, and the base itself is locked too.
157	*
158	* So __run_timers/migrate_timers can safely modify all timers which could
159	* be found on the lists/queues.
160	*
161	* When the timer's base is locked, and the timer removed from list, it is
162	* possible to set timer->base = &migration_base and drop the lock: the timer
163	* remains locked.
164	*/
165	static
166	struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer,
167	unsigned long *flags)
168	__acquires(&timer->base->lock)
169	{
170	struct hrtimer_clock_base *base;
171
172	for (;;) {
173	base = READ_ONCE(timer->base);
174	if (likely(base != &migration_base)) {
175	raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);
176	if (likely(base == timer->base))
177	return base;
178	/* The timer has migrated to another CPU: */
179	raw_spin_unlock_irqrestore(&base->cpu_base->lock, *flags);
180	}
181	cpu_relax();
182	}
183	}
184
185	/*
186	* We do not migrate the timer when it is expiring before the next
187	* event on the target cpu. When high resolution is enabled, we cannot
188	* reprogram the target cpu hardware and we would cause it to fire
189	* late. To keep it simple, we handle the high resolution enabled and
190	* disabled case similar.
191	*
192	* Called with cpu_base->lock of target cpu held.
193	*/
194	static int
195	hrtimer_check_target(struct hrtimer *timer, struct hrtimer_clock_base *new_base)
196	{
197	ktime_t expires;
198
199	expires = ktime_sub(hrtimer_get_expires(timer), new_base->offset);
200	return expires < new_base->cpu_base->expires_next;
201	}
202
203	static inline
204	struct hrtimer_cpu_base *get_target_base(struct hrtimer_cpu_base *base,
205	int pinned)
206	{
207	#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
208	if (static_branch_likely(&timers_migration_enabled) && !pinned)
209	return &per_cpu(hrtimer_bases, get_nohz_timer_target());
210	#endif
211	return base;
212	}
213
214	/*
215	* We switch the timer base to a power-optimized selected CPU target,
216	* if:
217	* - NO_HZ_COMMON is enabled
218	* - timer migration is enabled
219	* - the timer callback is not running
220	* - the timer is not the first expiring timer on the new target
221	*
222	* If one of the above requirements is not fulfilled we move the timer
223	* to the current CPU or leave it on the previously assigned CPU if
224	* the timer callback is currently running.
225	*/
226	static inline struct hrtimer_clock_base *
227	switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base,
228	int pinned)
229	{
230	struct hrtimer_cpu_base *new_cpu_base, *this_cpu_base;
231	struct hrtimer_clock_base *new_base;
232	int basenum = base->index;
233
234	this_cpu_base = this_cpu_ptr(&hrtimer_bases);
235	new_cpu_base = get_target_base(base: this_cpu_base, pinned);
236	again:
237	new_base = &new_cpu_base->clock_base[basenum];
238
239	if (base != new_base) {
240	/*
241	* We are trying to move timer to new_base.
242	* However we can't change timer's base while it is running,
243	* so we keep it on the same CPU. No hassle vs. reprogramming
244	* the event source in the high resolution case. The softirq
245	* code will take care of this when the timer function has
246	* completed. There is no conflict as we hold the lock until
247	* the timer is enqueued.
248	*/
249	if (unlikely(hrtimer_callback_running(timer)))
250	return base;
251
252	/* See the comment in lock_hrtimer_base() */
253	WRITE_ONCE(timer->base, &migration_base);
254	raw_spin_unlock(&base->cpu_base->lock);
255	raw_spin_lock(&new_base->cpu_base->lock);
256
257	if (new_cpu_base != this_cpu_base &&
258	hrtimer_check_target(timer, new_base)) {
259	raw_spin_unlock(&new_base->cpu_base->lock);
260	raw_spin_lock(&base->cpu_base->lock);
261	new_cpu_base = this_cpu_base;
262	WRITE_ONCE(timer->base, base);
263	goto again;
264	}
265	WRITE_ONCE(timer->base, new_base);
266	} else {
267	if (new_cpu_base != this_cpu_base &&
268	hrtimer_check_target(timer, new_base)) {
269	new_cpu_base = this_cpu_base;
270	goto again;
271	}
272	}
273	return new_base;
274	}
275
276	#else /* CONFIG_SMP */
277
278	static inline bool is_migration_base(struct hrtimer_clock_base *base)
279	{
280	return false;
281	}
282
283	static inline struct hrtimer_clock_base *
284	lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
285	__acquires(&timer->base->cpu_base->lock)
286	{
287	struct hrtimer_clock_base *base = timer->base;
288
289	raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);
290
291	return base;
292	}
293
294	# define switch_hrtimer_base(t, b, p) (b)
295
296	#endif /* !CONFIG_SMP */
297
298	/*
299	* Functions for the union type storage format of ktime_t which are
300	* too large for inlining:
301	*/
302	#if BITS_PER_LONG < 64
303	/*
304	* Divide a ktime value by a nanosecond value
305	*/
306	s64 __ktime_divns(const ktime_t kt, s64 div)
307	{
308	int sft = 0;
309	s64 dclc;
310	u64 tmp;
311
312	dclc = ktime_to_ns(kt);
313	tmp = dclc < 0 ? -dclc : dclc;
314
315	/* Make sure the divisor is less than 2^32: */
316	while (div >> 32) {
317	sft++;
318	div >>= 1;
319	}
320	tmp >>= sft;
321	do_div(tmp, (u32) div);
322	return dclc < 0 ? -tmp : tmp;
323	}
324	EXPORT_SYMBOL_GPL(__ktime_divns);
325	#endif /* BITS_PER_LONG >= 64 */
326
327	/*
328	* Add two ktime values and do a safety check for overflow:
329	*/
330	ktime_t ktime_add_safe(const ktime_t lhs, const ktime_t rhs)
331	{
332	ktime_t res = ktime_add_unsafe(lhs, rhs);
333
334	/*
335	* We use KTIME_SEC_MAX here, the maximum timeout which we can
336	* return to user space in a timespec:
337	*/
338	if (res < 0 || res < lhs || res < rhs)
339	res = ktime_set(KTIME_SEC_MAX, nsecs: 0);
340
341	return res;
342	}
343
344	EXPORT_SYMBOL_GPL(ktime_add_safe);
345
346	#ifdef CONFIG_DEBUG_OBJECTS_TIMERS
347
348	static const struct debug_obj_descr hrtimer_debug_descr;
349
350	static void *hrtimer_debug_hint(void *addr)
351	{
352	return ((struct hrtimer *) addr)->function;
353	}
354
355	/*
356	* fixup_init is called when:
357	* - an active object is initialized
358	*/
359	static bool hrtimer_fixup_init(void *addr, enum debug_obj_state state)
360	{
361	struct hrtimer *timer = addr;
362
363	switch (state) {
364	case ODEBUG_STATE_ACTIVE:
365	hrtimer_cancel(timer);
366	debug_object_init(addr: timer, descr: &hrtimer_debug_descr);
367	return true;
368	default:
369	return false;
370	}
371	}
372
373	/*
374	* fixup_activate is called when:
375	* - an active object is activated
376	* - an unknown non-static object is activated
377	*/
378	static bool hrtimer_fixup_activate(void *addr, enum debug_obj_state state)
379	{
380	switch (state) {
381	case ODEBUG_STATE_ACTIVE:
382	WARN_ON(1);
383	fallthrough;
384	default:
385	return false;
386	}
387	}
388
389	/*
390	* fixup_free is called when:
391	* - an active object is freed
392	*/
393	static bool hrtimer_fixup_free(void *addr, enum debug_obj_state state)
394	{
395	struct hrtimer *timer = addr;
396
397	switch (state) {
398	case ODEBUG_STATE_ACTIVE:
399	hrtimer_cancel(timer);
400	debug_object_free(addr: timer, descr: &hrtimer_debug_descr);
401	return true;
402	default:
403	return false;
404	}
405	}
406
407	static const struct debug_obj_descr hrtimer_debug_descr = {
408	.name = "hrtimer",
409	.debug_hint = hrtimer_debug_hint,
410	.fixup_init = hrtimer_fixup_init,
411	.fixup_activate = hrtimer_fixup_activate,
412	.fixup_free = hrtimer_fixup_free,
413	};
414
415	static inline void debug_hrtimer_init(struct hrtimer *timer)
416	{
417	debug_object_init(addr: timer, descr: &hrtimer_debug_descr);
418	}
419
420	static inline void debug_hrtimer_activate(struct hrtimer *timer,
421	enum hrtimer_mode mode)
422	{
423	debug_object_activate(addr: timer, descr: &hrtimer_debug_descr);
424	}
425
426	static inline void debug_hrtimer_deactivate(struct hrtimer *timer)
427	{
428	debug_object_deactivate(addr: timer, descr: &hrtimer_debug_descr);
429	}
430
431	static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
432	enum hrtimer_mode mode);
433
434	void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t clock_id,
435	enum hrtimer_mode mode)
436	{
437	debug_object_init_on_stack(addr: timer, descr: &hrtimer_debug_descr);
438	__hrtimer_init(timer, clock_id, mode);
439	}
440	EXPORT_SYMBOL_GPL(hrtimer_init_on_stack);
441
442	static void __hrtimer_init_sleeper(struct hrtimer_sleeper *sl,
443	clockid_t clock_id, enum hrtimer_mode mode);
444
445	void hrtimer_init_sleeper_on_stack(struct hrtimer_sleeper *sl,
446	clockid_t clock_id, enum hrtimer_mode mode)
447	{
448	debug_object_init_on_stack(addr: &sl->timer, descr: &hrtimer_debug_descr);
449	__hrtimer_init_sleeper(sl, clock_id, mode);
450	}
451	EXPORT_SYMBOL_GPL(hrtimer_init_sleeper_on_stack);
452
453	void destroy_hrtimer_on_stack(struct hrtimer *timer)
454	{
455	debug_object_free(addr: timer, descr: &hrtimer_debug_descr);
456	}
457	EXPORT_SYMBOL_GPL(destroy_hrtimer_on_stack);
458
459	#else
460
461	static inline void debug_hrtimer_init(struct hrtimer *timer) { }
462	static inline void debug_hrtimer_activate(struct hrtimer *timer,
463	enum hrtimer_mode mode) { }
464	static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { }
465	#endif
466
467	static inline void
468	debug_init(struct hrtimer *timer, clockid_t clockid,
469	enum hrtimer_mode mode)
470	{
471	debug_hrtimer_init(timer);
472	trace_hrtimer_init(hrtimer: timer, clockid, mode);
473	}
474
475	static inline void debug_activate(struct hrtimer *timer,
476	enum hrtimer_mode mode)
477	{
478	debug_hrtimer_activate(timer, mode);
479	trace_hrtimer_start(hrtimer: timer, mode);
480	}
481
482	static inline void debug_deactivate(struct hrtimer *timer)
483	{
484	debug_hrtimer_deactivate(timer);
485	trace_hrtimer_cancel(hrtimer: timer);
486	}
487
488	static struct hrtimer_clock_base *
489	__next_base(struct hrtimer_cpu_base *cpu_base, unsigned int *active)
490	{
491	unsigned int idx;
492
493	if (!*active)
494	return NULL;
495
496	idx = __ffs(*active);
497	*active &= ~(1U << idx);
498
499	return &cpu_base->clock_base[idx];
500	}
501
502	#define for_each_active_base(base, cpu_base, active) \
503	while ((base = __next_base((cpu_base), &(active))))
504
505	static ktime_t __hrtimer_next_event_base(struct hrtimer_cpu_base *cpu_base,
506	const struct hrtimer *exclude,
507	unsigned int active,
508	ktime_t expires_next)
509	{
510	struct hrtimer_clock_base *base;
511	ktime_t expires;
512
513	for_each_active_base(base, cpu_base, active) {
514	struct timerqueue_node *next;
515	struct hrtimer *timer;
516
517	next = timerqueue_getnext(head: &base->active);
518	timer = container_of(next, struct hrtimer, node);
519	if (timer == exclude) {
520	/* Get to the next timer in the queue. */
521	next = timerqueue_iterate_next(node: next);
522	if (!next)
523	continue;
524
525	timer = container_of(next, struct hrtimer, node);
526	}
527	expires = ktime_sub(hrtimer_get_expires(timer), base->offset);
528	if (expires < expires_next) {
529	expires_next = expires;
530
531	/* Skip cpu_base update if a timer is being excluded. */
532	if (exclude)
533	continue;
534
535	if (timer->is_soft)
536	cpu_base->softirq_next_timer = timer;
537	else
538	cpu_base->next_timer = timer;
539	}
540	}
541	/*
542	* clock_was_set() might have changed base->offset of any of
543	* the clock bases so the result might be negative. Fix it up
544	* to prevent a false positive in clockevents_program_event().
545	*/
546	if (expires_next < 0)
547	expires_next = 0;
548	return expires_next;
549	}
550
551	/*
552	* Recomputes cpu_base::*next_timer and returns the earliest expires_next
553	* but does not set cpu_base::*expires_next, that is done by
554	* hrtimer[_force]_reprogram and hrtimer_interrupt only. When updating
555	* cpu_base::*expires_next right away, reprogramming logic would no longer
556	* work.
557	*
558	* When a softirq is pending, we can ignore the HRTIMER_ACTIVE_SOFT bases,
559	* those timers will get run whenever the softirq gets handled, at the end of
560	* hrtimer_run_softirq(), hrtimer_update_softirq_timer() will re-add these bases.
561	*
562	* Therefore softirq values are those from the HRTIMER_ACTIVE_SOFT clock bases.
563	* The !softirq values are the minima across HRTIMER_ACTIVE_ALL, unless an actual
564	* softirq is pending, in which case they're the minima of HRTIMER_ACTIVE_HARD.
565	*
566	* @active_mask must be one of:
567	* - HRTIMER_ACTIVE_ALL,
568	* - HRTIMER_ACTIVE_SOFT, or
569	* - HRTIMER_ACTIVE_HARD.
570	*/
571	static ktime_t
572	__hrtimer_get_next_event(struct hrtimer_cpu_base *cpu_base, unsigned int active_mask)
573	{
574	unsigned int active;
575	struct hrtimer *next_timer = NULL;
576	ktime_t expires_next = KTIME_MAX;
577
578	if (!cpu_base->softirq_activated && (active_mask & HRTIMER_ACTIVE_SOFT)) {
579	active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;
580	cpu_base->softirq_next_timer = NULL;
581	expires_next = __hrtimer_next_event_base(cpu_base, NULL,
582	active, KTIME_MAX);
583
584	next_timer = cpu_base->softirq_next_timer;
585	}
586
587	if (active_mask & HRTIMER_ACTIVE_HARD) {
588	active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;
589	cpu_base->next_timer = next_timer;
590	expires_next = __hrtimer_next_event_base(cpu_base, NULL, active,
591	expires_next);
592	}
593
594	return expires_next;
595	}
596
597	static ktime_t hrtimer_update_next_event(struct hrtimer_cpu_base *cpu_base)
598	{
599	ktime_t expires_next, soft = KTIME_MAX;
600
601	/*
602	* If the soft interrupt has already been activated, ignore the
603	* soft bases. They will be handled in the already raised soft
604	* interrupt.
605	*/
606	if (!cpu_base->softirq_activated) {
607	soft = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);
608	/*
609	* Update the soft expiry time. clock_settime() might have
610	* affected it.
611	*/
612	cpu_base->softirq_expires_next = soft;
613	}
614
615	expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_HARD);
616	/*
617	* If a softirq timer is expiring first, update cpu_base->next_timer
618	* and program the hardware with the soft expiry time.
619	*/
620	if (expires_next > soft) {
621	cpu_base->next_timer = cpu_base->softirq_next_timer;
622	expires_next = soft;
623	}
624
625	return expires_next;
626	}
627
628	static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base)
629	{
630	ktime_t *offs_real = &base->clock_base[HRTIMER_BASE_REALTIME].offset;
631	ktime_t *offs_boot = &base->clock_base[HRTIMER_BASE_BOOTTIME].offset;
632	ktime_t *offs_tai = &base->clock_base[HRTIMER_BASE_TAI].offset;
633
634	ktime_t now = ktime_get_update_offsets_now(cwsseq: &base->clock_was_set_seq,
635	offs_real, offs_boot, offs_tai);
636
637	base->clock_base[HRTIMER_BASE_REALTIME_SOFT].offset = *offs_real;
638	base->clock_base[HRTIMER_BASE_BOOTTIME_SOFT].offset = *offs_boot;
639	base->clock_base[HRTIMER_BASE_TAI_SOFT].offset = *offs_tai;
640
641	return now;
642	}
643
644	/*
645	* Is the high resolution mode active ?
646	*/
647	static inline int __hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base)
648	{
649	return IS_ENABLED(CONFIG_HIGH_RES_TIMERS) ?
650	cpu_base->hres_active : 0;
651	}
652
653	static inline int hrtimer_hres_active(void)
654	{
655	return __hrtimer_hres_active(this_cpu_ptr(&hrtimer_bases));
656	}
657
658	static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base,
659	struct hrtimer *next_timer,
660	ktime_t expires_next)
661	{
662	cpu_base->expires_next = expires_next;
663
664	/*
665	* If hres is not active, hardware does not have to be
666	* reprogrammed yet.
667	*
668	* If a hang was detected in the last timer interrupt then we
669	* leave the hang delay active in the hardware. We want the
670	* system to make progress. That also prevents the following
671	* scenario:
672	* T1 expires 50ms from now
673	* T2 expires 5s from now
674	*
675	* T1 is removed, so this code is called and would reprogram
676	* the hardware to 5s from now. Any hrtimer_start after that
677	* will not reprogram the hardware due to hang_detected being
678	* set. So we'd effectively block all timers until the T2 event
679	* fires.
680	*/
681	if (!__hrtimer_hres_active(cpu_base) || cpu_base->hang_detected)
682	return;
683
684	tick_program_event(expires: expires_next, force: 1);
685	}
686
687	/*
688	* Reprogram the event source with checking both queues for the
689	* next event
690	* Called with interrupts disabled and base->lock held
691	*/
692	static void
693	hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, int skip_equal)
694	{
695	ktime_t expires_next;
696
697	expires_next = hrtimer_update_next_event(cpu_base);
698
699	if (skip_equal && expires_next == cpu_base->expires_next)
700	return;
701
702	__hrtimer_reprogram(cpu_base, next_timer: cpu_base->next_timer, expires_next);
703	}
704
705	/* High resolution timer related functions */
706	#ifdef CONFIG_HIGH_RES_TIMERS
707
708	/*
709	* High resolution timer enabled ?
710	*/
711	static bool hrtimer_hres_enabled __read_mostly = true;
712	unsigned int hrtimer_resolution __read_mostly = LOW_RES_NSEC;
713	EXPORT_SYMBOL_GPL(hrtimer_resolution);
714
715	/*
716	* Enable / Disable high resolution mode
717	*/
718	static int __init setup_hrtimer_hres(char *str)
719	{
720	return (kstrtobool(s: str, res: &hrtimer_hres_enabled) == 0);
721	}
722
723	__setup("highres=", setup_hrtimer_hres);
724
725	/*
726	* hrtimer_high_res_enabled - query, if the highres mode is enabled
727	*/
728	static inline int hrtimer_is_hres_enabled(void)
729	{
730	return hrtimer_hres_enabled;
731	}
732
733	static void retrigger_next_event(void *arg);
734
735	/*
736	* Switch to high resolution mode
737	*/
738	static void hrtimer_switch_to_hres(void)
739	{
740	struct hrtimer_cpu_base *base = this_cpu_ptr(&hrtimer_bases);
741
742	if (tick_init_highres()) {
743	pr_warn("Could not switch to high resolution mode on CPU %u\n",
744	base->cpu);
745	return;
746	}
747	base->hres_active = 1;
748	hrtimer_resolution = HIGH_RES_NSEC;
749
750	tick_setup_sched_timer(hrtimer: true);
751	/* "Retrigger" the interrupt to get things going */
752	retrigger_next_event(NULL);
753	}
754
755	#else
756
757	static inline int hrtimer_is_hres_enabled(void) { return 0; }
758	static inline void hrtimer_switch_to_hres(void) { }
759
760	#endif /* CONFIG_HIGH_RES_TIMERS */
761	/*
762	* Retrigger next event is called after clock was set with interrupts
763	* disabled through an SMP function call or directly from low level
764	* resume code.
765	*
766	* This is only invoked when:
767	* - CONFIG_HIGH_RES_TIMERS is enabled.
768	* - CONFIG_NOHZ_COMMON is enabled
769	*
770	* For the other cases this function is empty and because the call sites
771	* are optimized out it vanishes as well, i.e. no need for lots of
772	* #ifdeffery.
773	*/
774	static void retrigger_next_event(void *arg)
775	{
776	struct hrtimer_cpu_base *base = this_cpu_ptr(&hrtimer_bases);
777
778	/*
779	* When high resolution mode or nohz is active, then the offsets of
780	* CLOCK_REALTIME/TAI/BOOTTIME have to be updated. Otherwise the
781	* next tick will take care of that.
782	*
783	* If high resolution mode is active then the next expiring timer
784	* must be reevaluated and the clock event device reprogrammed if
785	* necessary.
786	*
787	* In the NOHZ case the update of the offset and the reevaluation
788	* of the next expiring timer is enough. The return from the SMP
789	* function call will take care of the reprogramming in case the
790	* CPU was in a NOHZ idle sleep.
791	*/
792	if (!__hrtimer_hres_active(cpu_base: base) && !tick_nohz_active)
793	return;
794
795	raw_spin_lock(&base->lock);
796	hrtimer_update_base(base);
797	if (__hrtimer_hres_active(cpu_base: base))
798	hrtimer_force_reprogram(cpu_base: base, skip_equal: 0);
799	else
800	hrtimer_update_next_event(cpu_base: base);
801	raw_spin_unlock(&base->lock);
802	}
803
804	/*
805	* When a timer is enqueued and expires earlier than the already enqueued
806	* timers, we have to check, whether it expires earlier than the timer for
807	* which the clock event device was armed.
808	*
809	* Called with interrupts disabled and base->cpu_base.lock held
810	*/
811	static void hrtimer_reprogram(struct hrtimer *timer, bool reprogram)
812	{
813	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
814	struct hrtimer_clock_base *base = timer->base;
815	ktime_t expires = ktime_sub(hrtimer_get_expires(timer), base->offset);
816
817	WARN_ON_ONCE(hrtimer_get_expires_tv64(timer) < 0);
818
819	/*
820	* CLOCK_REALTIME timer might be requested with an absolute
821	* expiry time which is less than base->offset. Set it to 0.
822	*/
823	if (expires < 0)
824	expires = 0;
825
826	if (timer->is_soft) {
827	/*
828	* soft hrtimer could be started on a remote CPU. In this
829	* case softirq_expires_next needs to be updated on the
830	* remote CPU. The soft hrtimer will not expire before the
831	* first hard hrtimer on the remote CPU -
832	* hrtimer_check_target() prevents this case.
833	*/
834	struct hrtimer_cpu_base *timer_cpu_base = base->cpu_base;
835
836	if (timer_cpu_base->softirq_activated)
837	return;
838
839	if (!ktime_before(cmp1: expires, cmp2: timer_cpu_base->softirq_expires_next))
840	return;
841
842	timer_cpu_base->softirq_next_timer = timer;
843	timer_cpu_base->softirq_expires_next = expires;
844
845	if (!ktime_before(cmp1: expires, cmp2: timer_cpu_base->expires_next) ||
846	!reprogram)
847	return;
848	}
849
850	/*
851	* If the timer is not on the current cpu, we cannot reprogram
852	* the other cpus clock event device.
853	*/
854	if (base->cpu_base != cpu_base)
855	return;
856
857	if (expires >= cpu_base->expires_next)
858	return;
859
860	/*
861	* If the hrtimer interrupt is running, then it will reevaluate the
862	* clock bases and reprogram the clock event device.
863	*/
864	if (cpu_base->in_hrtirq)
865	return;
866
867	cpu_base->next_timer = timer;
868
869	__hrtimer_reprogram(cpu_base, next_timer: timer, expires_next: expires);
870	}
871
872	static bool update_needs_ipi(struct hrtimer_cpu_base *cpu_base,
873	unsigned int active)
874	{
875	struct hrtimer_clock_base *base;
876	unsigned int seq;
877	ktime_t expires;
878
879	/*
880	* Update the base offsets unconditionally so the following
881	* checks whether the SMP function call is required works.
882	*
883	* The update is safe even when the remote CPU is in the hrtimer
884	* interrupt or the hrtimer soft interrupt and expiring affected
885	* bases. Either it will see the update before handling a base or
886	* it will see it when it finishes the processing and reevaluates
887	* the next expiring timer.
888	*/
889	seq = cpu_base->clock_was_set_seq;
890	hrtimer_update_base(base: cpu_base);
891
892	/*
893	* If the sequence did not change over the update then the
894	* remote CPU already handled it.
895	*/
896	if (seq == cpu_base->clock_was_set_seq)
897	return false;
898
899	/*
900	* If the remote CPU is currently handling an hrtimer interrupt, it
901	* will reevaluate the first expiring timer of all clock bases
902	* before reprogramming. Nothing to do here.
903	*/
904	if (cpu_base->in_hrtirq)
905	return false;
906
907	/*
908	* Walk the affected clock bases and check whether the first expiring
909	* timer in a clock base is moving ahead of the first expiring timer of
910	* @cpu_base. If so, the IPI must be invoked because per CPU clock
911	* event devices cannot be remotely reprogrammed.
912	*/
913	active &= cpu_base->active_bases;
914
915	for_each_active_base(base, cpu_base, active) {
916	struct timerqueue_node *next;
917
918	next = timerqueue_getnext(head: &base->active);
919	expires = ktime_sub(next->expires, base->offset);
920	if (expires < cpu_base->expires_next)
921	return true;
922
923	/* Extra check for softirq clock bases */
924	if (base->clockid < HRTIMER_BASE_MONOTONIC_SOFT)
925	continue;
926	if (cpu_base->softirq_activated)
927	continue;
928	if (expires < cpu_base->softirq_expires_next)
929	return true;
930	}
931	return false;
932	}
933
934	/*
935	* Clock was set. This might affect CLOCK_REALTIME, CLOCK_TAI and
936	* CLOCK_BOOTTIME (for late sleep time injection).
937	*
938	* This requires to update the offsets for these clocks
939	* vs. CLOCK_MONOTONIC. When high resolution timers are enabled, then this
940	* also requires to eventually reprogram the per CPU clock event devices
941	* when the change moves an affected timer ahead of the first expiring
942	* timer on that CPU. Obviously remote per CPU clock event devices cannot
943	* be reprogrammed. The other reason why an IPI has to be sent is when the
944	* system is in !HIGH_RES and NOHZ mode. The NOHZ mode updates the offsets
945	* in the tick, which obviously might be stopped, so this has to bring out
946	* the remote CPU which might sleep in idle to get this sorted.
947	*/
948	void clock_was_set(unsigned int bases)
949	{
950	struct hrtimer_cpu_base *cpu_base = raw_cpu_ptr(&hrtimer_bases);
951	cpumask_var_t mask;
952	int cpu;
953
954	if (!__hrtimer_hres_active(cpu_base) && !tick_nohz_active)
955	goto out_timerfd;
956
957	if (!zalloc_cpumask_var(mask: &mask, GFP_KERNEL)) {
958	on_each_cpu(func: retrigger_next_event, NULL, wait: 1);
959	goto out_timerfd;
960	}
961
962	/* Avoid interrupting CPUs if possible */
963	cpus_read_lock();
964	for_each_online_cpu(cpu) {
965	unsigned long flags;
966
967	cpu_base = &per_cpu(hrtimer_bases, cpu);
968	raw_spin_lock_irqsave(&cpu_base->lock, flags);
969
970	if (update_needs_ipi(cpu_base, active: bases))
971	cpumask_set_cpu(cpu, dstp: mask);
972
973	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
974	}
975
976	preempt_disable();
977	smp_call_function_many(mask, func: retrigger_next_event, NULL, wait: 1);
978	preempt_enable();
979	cpus_read_unlock();
980	free_cpumask_var(mask);
981
982	out_timerfd:
983	timerfd_clock_was_set();
984	}
985
986	static void clock_was_set_work(struct work_struct *work)
987	{
988	clock_was_set(CLOCK_SET_WALL);
989	}
990
991	static DECLARE_WORK(hrtimer_work, clock_was_set_work);
992
993	/*
994	* Called from timekeeping code to reprogram the hrtimer interrupt device
995	* on all cpus and to notify timerfd.
996	*/
997	void clock_was_set_delayed(void)
998	{
999	schedule_work(work: &hrtimer_work);
1000	}
1001
1002	/*
1003	* Called during resume either directly from via timekeeping_resume()
1004	* or in the case of s2idle from tick_unfreeze() to ensure that the
1005	* hrtimers are up to date.
1006	*/
1007	void hrtimers_resume_local(void)
1008	{
1009	lockdep_assert_irqs_disabled();
1010	/* Retrigger on the local CPU */
1011	retrigger_next_event(NULL);
1012	}
1013
1014	/*
1015	* Counterpart to lock_hrtimer_base above:
1016	*/
1017	static inline
1018	void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
1019	__releases(&timer->base->cpu_base->lock)
1020	{
1021	raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags);
1022	}
1023
1024	/**
1025	* hrtimer_forward() - forward the timer expiry
1026	* @timer: hrtimer to forward
1027	* @now: forward past this time
1028	* @interval: the interval to forward
1029	*
1030	* Forward the timer expiry so it will expire in the future.
1031	*
1032	* .. note::
1033	* This only updates the timer expiry value and does not requeue the timer.
1034	*
1035	* There is also a variant of the function hrtimer_forward_now().
1036	*
1037	* Context: Can be safely called from the callback function of @timer. If called
1038	* from other contexts @timer must neither be enqueued nor running the
1039	* callback and the caller needs to take care of serialization.
1040	*
1041	* Return: The number of overruns are returned.
1042	*/
1043	u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval)
1044	{
1045	u64 orun = 1;
1046	ktime_t delta;
1047
1048	delta = ktime_sub(now, hrtimer_get_expires(timer));
1049
1050	if (delta < 0)
1051	return 0;
1052
1053	if (WARN_ON(timer->state & HRTIMER_STATE_ENQUEUED))
1054	return 0;
1055
1056	if (interval < hrtimer_resolution)
1057	interval = hrtimer_resolution;
1058
1059	if (unlikely(delta >= interval)) {
1060	s64 incr = ktime_to_ns(kt: interval);
1061
1062	orun = ktime_divns(kt: delta, div: incr);
1063	hrtimer_add_expires_ns(timer, ns: incr * orun);
1064	if (hrtimer_get_expires_tv64(timer) > now)
1065	return orun;
1066	/*
1067	* This (and the ktime_add() below) is the
1068	* correction for exact:
1069	*/
1070	orun++;
1071	}
1072	hrtimer_add_expires(timer, time: interval);
1073
1074	return orun;
1075	}
1076	EXPORT_SYMBOL_GPL(hrtimer_forward);
1077
1078	/*
1079	* enqueue_hrtimer - internal function to (re)start a timer
1080	*
1081	* The timer is inserted in expiry order. Insertion into the
1082	* red black tree is O(log(n)). Must hold the base lock.
1083	*
1084	* Returns 1 when the new timer is the leftmost timer in the tree.
1085	*/
1086	static int enqueue_hrtimer(struct hrtimer *timer,
1087	struct hrtimer_clock_base *base,
1088	enum hrtimer_mode mode)
1089	{
1090	debug_activate(timer, mode);
1091	WARN_ON_ONCE(!base->cpu_base->online);
1092
1093	base->cpu_base->active_bases |= 1 << base->index;
1094
1095	/* Pairs with the lockless read in hrtimer_is_queued() */
1096	WRITE_ONCE(timer->state, HRTIMER_STATE_ENQUEUED);
1097
1098	return timerqueue_add(head: &base->active, node: &timer->node);
1099	}
1100
1101	/*
1102	* __remove_hrtimer - internal function to remove a timer
1103	*
1104	* Caller must hold the base lock.
1105	*
1106	* High resolution timer mode reprograms the clock event device when the
1107	* timer is the one which expires next. The caller can disable this by setting
1108	* reprogram to zero. This is useful, when the context does a reprogramming
1109	* anyway (e.g. timer interrupt)
1110	*/
1111	static void __remove_hrtimer(struct hrtimer *timer,
1112	struct hrtimer_clock_base *base,
1113	u8 newstate, int reprogram)
1114	{
1115	struct hrtimer_cpu_base *cpu_base = base->cpu_base;
1116	u8 state = timer->state;
1117
1118	/* Pairs with the lockless read in hrtimer_is_queued() */
1119	WRITE_ONCE(timer->state, newstate);
1120	if (!(state & HRTIMER_STATE_ENQUEUED))
1121	return;
1122
1123	if (!timerqueue_del(head: &base->active, node: &timer->node))
1124	cpu_base->active_bases &= ~(1 << base->index);
1125
1126	/*
1127	* Note: If reprogram is false we do not update
1128	* cpu_base->next_timer. This happens when we remove the first
1129	* timer on a remote cpu. No harm as we never dereference
1130	* cpu_base->next_timer. So the worst thing what can happen is
1131	* an superfluous call to hrtimer_force_reprogram() on the
1132	* remote cpu later on if the same timer gets enqueued again.
1133	*/
1134	if (reprogram && timer == cpu_base->next_timer)
1135	hrtimer_force_reprogram(cpu_base, skip_equal: 1);
1136	}
1137
1138	/*
1139	* remove hrtimer, called with base lock held
1140	*/
1141	static inline int
1142	remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base,
1143	bool restart, bool keep_local)
1144	{
1145	u8 state = timer->state;
1146
1147	if (state & HRTIMER_STATE_ENQUEUED) {
1148	bool reprogram;
1149
1150	/*
1151	* Remove the timer and force reprogramming when high
1152	* resolution mode is active and the timer is on the current
1153	* CPU. If we remove a timer on another CPU, reprogramming is
1154	* skipped. The interrupt event on this CPU is fired and
1155	* reprogramming happens in the interrupt handler. This is a
1156	* rare case and less expensive than a smp call.
1157	*/
1158	debug_deactivate(timer);
1159	reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases);
1160
1161	/*
1162	* If the timer is not restarted then reprogramming is
1163	* required if the timer is local. If it is local and about
1164	* to be restarted, avoid programming it twice (on removal
1165	* and a moment later when it's requeued).
1166	*/
1167	if (!restart)
1168	state = HRTIMER_STATE_INACTIVE;
1169	else
1170	reprogram &= !keep_local;
1171
1172	__remove_hrtimer(timer, base, newstate: state, reprogram);
1173	return 1;
1174	}
1175	return 0;
1176	}
1177
1178	static inline ktime_t hrtimer_update_lowres(struct hrtimer *timer, ktime_t tim,
1179	const enum hrtimer_mode mode)
1180	{
1181	#ifdef CONFIG_TIME_LOW_RES
1182	/*
1183	* CONFIG_TIME_LOW_RES indicates that the system has no way to return
1184	* granular time values. For relative timers we add hrtimer_resolution
1185	* (i.e. one jiffie) to prevent short timeouts.
1186	*/
1187	timer->is_rel = mode & HRTIMER_MODE_REL;
1188	if (timer->is_rel)
1189	tim = ktime_add_safe(tim, hrtimer_resolution);
1190	#endif
1191	return tim;
1192	}
1193
1194	static void
1195	hrtimer_update_softirq_timer(struct hrtimer_cpu_base *cpu_base, bool reprogram)
1196	{
1197	ktime_t expires;
1198
1199	/*
1200	* Find the next SOFT expiration.
1201	*/
1202	expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);
1203
1204	/*
1205	* reprogramming needs to be triggered, even if the next soft
1206	* hrtimer expires at the same time than the next hard
1207	* hrtimer. cpu_base->softirq_expires_next needs to be updated!
1208	*/
1209	if (expires == KTIME_MAX)
1210	return;
1211
1212	/*
1213	* cpu_base->*next_timer is recomputed by __hrtimer_get_next_event()
1214	* cpu_base->*expires_next is only set by hrtimer_reprogram()
1215	*/
1216	hrtimer_reprogram(timer: cpu_base->softirq_next_timer, reprogram);
1217	}
1218
1219	static int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
1220	u64 delta_ns, const enum hrtimer_mode mode,
1221	struct hrtimer_clock_base *base)
1222	{
1223	struct hrtimer_clock_base *new_base;
1224	bool force_local, first;
1225
1226	/*
1227	* If the timer is on the local cpu base and is the first expiring
1228	* timer then this might end up reprogramming the hardware twice
1229	* (on removal and on enqueue). To avoid that by prevent the
1230	* reprogram on removal, keep the timer local to the current CPU
1231	* and enforce reprogramming after it is queued no matter whether
1232	* it is the new first expiring timer again or not.
1233	*/
1234	force_local = base->cpu_base == this_cpu_ptr(&hrtimer_bases);
1235	force_local &= base->cpu_base->next_timer == timer;
1236
1237	/*
1238	* Remove an active timer from the queue. In case it is not queued
1239	* on the current CPU, make sure that remove_hrtimer() updates the
1240	* remote data correctly.
1241	*
1242	* If it's on the current CPU and the first expiring timer, then
1243	* skip reprogramming, keep the timer local and enforce
1244	* reprogramming later if it was the first expiring timer. This
1245	* avoids programming the underlying clock event twice (once at
1246	* removal and once after enqueue).
1247	*/
1248	remove_hrtimer(timer, base, restart: true, keep_local: force_local);
1249
1250	if (mode & HRTIMER_MODE_REL)
1251	tim = ktime_add_safe(tim, base->get_time());
1252
1253	tim = hrtimer_update_lowres(timer, tim, mode);
1254
1255	hrtimer_set_expires_range_ns(timer, time: tim, delta: delta_ns);
1256
1257	/* Switch the timer base, if necessary: */
1258	if (!force_local) {
1259	new_base = switch_hrtimer_base(timer, base,
1260	pinned: mode & HRTIMER_MODE_PINNED);
1261	} else {
1262	new_base = base;
1263	}
1264
1265	first = enqueue_hrtimer(timer, base: new_base, mode);
1266	if (!force_local)
1267	return first;
1268
1269	/*
1270	* Timer was forced to stay on the current CPU to avoid
1271	* reprogramming on removal and enqueue. Force reprogram the
1272	* hardware by evaluating the new first expiring timer.
1273	*/
1274	hrtimer_force_reprogram(cpu_base: new_base->cpu_base, skip_equal: 1);
1275	return 0;
1276	}
1277
1278	/**
1279	* hrtimer_start_range_ns - (re)start an hrtimer
1280	* @timer: the timer to be added
1281	* @tim: expiry time
1282	* @delta_ns: "slack" range for the timer
1283	* @mode: timer mode: absolute (HRTIMER_MODE_ABS) or
1284	* relative (HRTIMER_MODE_REL), and pinned (HRTIMER_MODE_PINNED);
1285	* softirq based mode is considered for debug purpose only!
1286	*/
1287	void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
1288	u64 delta_ns, const enum hrtimer_mode mode)
1289	{
1290	struct hrtimer_clock_base *base;
1291	unsigned long flags;
1292
1293	/*
1294	* Check whether the HRTIMER_MODE_SOFT bit and hrtimer.is_soft
1295	* match on CONFIG_PREEMPT_RT = n. With PREEMPT_RT check the hard
1296	* expiry mode because unmarked timers are moved to softirq expiry.
1297	*/
1298	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
1299	WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft);
1300	else
1301	WARN_ON_ONCE(!(mode & HRTIMER_MODE_HARD) ^ !timer->is_hard);
1302
1303	base = lock_hrtimer_base(timer, flags: &flags);
1304
1305	if (__hrtimer_start_range_ns(timer, tim, delta_ns, mode, base))
1306	hrtimer_reprogram(timer, reprogram: true);
1307
1308	unlock_hrtimer_base(timer, flags: &flags);
1309	}
1310	EXPORT_SYMBOL_GPL(hrtimer_start_range_ns);
1311
1312	/**
1313	* hrtimer_try_to_cancel - try to deactivate a timer
1314	* @timer: hrtimer to stop
1315	*
1316	* Returns:
1317	*
1318	* * 0 when the timer was not active
1319	* * 1 when the timer was active
1320	* * -1 when the timer is currently executing the callback function and
1321	* cannot be stopped
1322	*/
1323	int hrtimer_try_to_cancel(struct hrtimer *timer)
1324	{
1325	struct hrtimer_clock_base *base;
1326	unsigned long flags;
1327	int ret = -1;
1328
1329	/*
1330	* Check lockless first. If the timer is not active (neither
1331	* enqueued nor running the callback, nothing to do here. The
1332	* base lock does not serialize against a concurrent enqueue,
1333	* so we can avoid taking it.
1334	*/
1335	if (!hrtimer_active(timer))
1336	return 0;
1337
1338	base = lock_hrtimer_base(timer, flags: &flags);
1339
1340	if (!hrtimer_callback_running(timer))
1341	ret = remove_hrtimer(timer, base, restart: false, keep_local: false);
1342
1343	unlock_hrtimer_base(timer, flags: &flags);
1344
1345	return ret;
1346
1347	}
1348	EXPORT_SYMBOL_GPL(hrtimer_try_to_cancel);
1349
1350	#ifdef CONFIG_PREEMPT_RT
1351	static void hrtimer_cpu_base_init_expiry_lock(struct hrtimer_cpu_base *base)
1352	{
1353	spin_lock_init(&base->softirq_expiry_lock);
1354	}
1355
1356	static void hrtimer_cpu_base_lock_expiry(struct hrtimer_cpu_base *base)
1357	{
1358	spin_lock(&base->softirq_expiry_lock);
1359	}
1360
1361	static void hrtimer_cpu_base_unlock_expiry(struct hrtimer_cpu_base *base)
1362	{
1363	spin_unlock(&base->softirq_expiry_lock);
1364	}
1365
1366	/*
1367	* The counterpart to hrtimer_cancel_wait_running().
1368	*
1369	* If there is a waiter for cpu_base->expiry_lock, then it was waiting for
1370	* the timer callback to finish. Drop expiry_lock and reacquire it. That
1371	* allows the waiter to acquire the lock and make progress.
1372	*/
1373	static void hrtimer_sync_wait_running(struct hrtimer_cpu_base *cpu_base,
1374	unsigned long flags)
1375	{
1376	if (atomic_read(&cpu_base->timer_waiters)) {
1377	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1378	spin_unlock(&cpu_base->softirq_expiry_lock);
1379	spin_lock(&cpu_base->softirq_expiry_lock);
1380	raw_spin_lock_irq(&cpu_base->lock);
1381	}
1382	}
1383
1384	/*
1385	* This function is called on PREEMPT_RT kernels when the fast path
1386	* deletion of a timer failed because the timer callback function was
1387	* running.
1388	*
1389	* This prevents priority inversion: if the soft irq thread is preempted
1390	* in the middle of a timer callback, then calling del_timer_sync() can
1391	* lead to two issues:
1392	*
1393	* - If the caller is on a remote CPU then it has to spin wait for the timer
1394	* handler to complete. This can result in unbound priority inversion.
1395	*
1396	* - If the caller originates from the task which preempted the timer
1397	* handler on the same CPU, then spin waiting for the timer handler to
1398	* complete is never going to end.
1399	*/
1400	void hrtimer_cancel_wait_running(const struct hrtimer *timer)
1401	{
1402	/* Lockless read. Prevent the compiler from reloading it below */
1403	struct hrtimer_clock_base *base = READ_ONCE(timer->base);
1404
1405	/*
1406	* Just relax if the timer expires in hard interrupt context or if
1407	* it is currently on the migration base.
1408	*/
1409	if (!timer->is_soft || is_migration_base(base)) {
1410	cpu_relax();
1411	return;
1412	}
1413
1414	/*
1415	* Mark the base as contended and grab the expiry lock, which is
1416	* held by the softirq across the timer callback. Drop the lock
1417	* immediately so the softirq can expire the next timer. In theory
1418	* the timer could already be running again, but that's more than
1419	* unlikely and just causes another wait loop.
1420	*/
1421	atomic_inc(&base->cpu_base->timer_waiters);
1422	spin_lock_bh(&base->cpu_base->softirq_expiry_lock);
1423	atomic_dec(&base->cpu_base->timer_waiters);
1424	spin_unlock_bh(&base->cpu_base->softirq_expiry_lock);
1425	}
1426	#else
1427	static inline void
1428	hrtimer_cpu_base_init_expiry_lock(struct hrtimer_cpu_base *base) { }
1429	static inline void
1430	hrtimer_cpu_base_lock_expiry(struct hrtimer_cpu_base *base) { }
1431	static inline void
1432	hrtimer_cpu_base_unlock_expiry(struct hrtimer_cpu_base *base) { }
1433	static inline void hrtimer_sync_wait_running(struct hrtimer_cpu_base *base,
1434	unsigned long flags) { }
1435	#endif
1436
1437	/**
1438	* hrtimer_cancel - cancel a timer and wait for the handler to finish.
1439	* @timer: the timer to be cancelled
1440	*
1441	* Returns:
1442	* 0 when the timer was not active
1443	* 1 when the timer was active
1444	*/
1445	int hrtimer_cancel(struct hrtimer *timer)
1446	{
1447	int ret;
1448
1449	do {
1450	ret = hrtimer_try_to_cancel(timer);
1451
1452	if (ret < 0)
1453	hrtimer_cancel_wait_running(timer);
1454	} while (ret < 0);
1455	return ret;
1456	}
1457	EXPORT_SYMBOL_GPL(hrtimer_cancel);
1458
1459	/**
1460	* __hrtimer_get_remaining - get remaining time for the timer
1461	* @timer: the timer to read
1462	* @adjust: adjust relative timers when CONFIG_TIME_LOW_RES=y
1463	*/
1464	ktime_t __hrtimer_get_remaining(const struct hrtimer *timer, bool adjust)
1465	{
1466	unsigned long flags;
1467	ktime_t rem;
1468
1469	lock_hrtimer_base(timer, flags: &flags);
1470	if (IS_ENABLED(CONFIG_TIME_LOW_RES) && adjust)
1471	rem = hrtimer_expires_remaining_adjusted(timer);
1472	else
1473	rem = hrtimer_expires_remaining(timer);
1474	unlock_hrtimer_base(timer, flags: &flags);
1475
1476	return rem;
1477	}
1478	EXPORT_SYMBOL_GPL(__hrtimer_get_remaining);
1479
1480	#ifdef CONFIG_NO_HZ_COMMON
1481	/**
1482	* hrtimer_get_next_event - get the time until next expiry event
1483	*
1484	* Returns the next expiry time or KTIME_MAX if no timer is pending.
1485	*/
1486	u64 hrtimer_get_next_event(void)
1487	{
1488	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
1489	u64 expires = KTIME_MAX;
1490	unsigned long flags;
1491
1492	raw_spin_lock_irqsave(&cpu_base->lock, flags);
1493
1494	if (!__hrtimer_hres_active(cpu_base))
1495	expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);
1496
1497	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1498
1499	return expires;
1500	}
1501
1502	/**
1503	* hrtimer_next_event_without - time until next expiry event w/o one timer
1504	* @exclude: timer to exclude
1505	*
1506	* Returns the next expiry time over all timers except for the @exclude one or
1507	* KTIME_MAX if none of them is pending.
1508	*/
1509	u64 hrtimer_next_event_without(const struct hrtimer *exclude)
1510	{
1511	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
1512	u64 expires = KTIME_MAX;
1513	unsigned long flags;
1514
1515	raw_spin_lock_irqsave(&cpu_base->lock, flags);
1516
1517	if (__hrtimer_hres_active(cpu_base)) {
1518	unsigned int active;
1519
1520	if (!cpu_base->softirq_activated) {
1521	active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;
1522	expires = __hrtimer_next_event_base(cpu_base, exclude,
1523	active, KTIME_MAX);
1524	}
1525	active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;
1526	expires = __hrtimer_next_event_base(cpu_base, exclude, active,
1527	expires_next: expires);
1528	}
1529
1530	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1531
1532	return expires;
1533	}
1534	#endif
1535
1536	static inline int hrtimer_clockid_to_base(clockid_t clock_id)
1537	{
1538	if (likely(clock_id < MAX_CLOCKS)) {
1539	int base = hrtimer_clock_to_base_table[clock_id];
1540
1541	if (likely(base != HRTIMER_MAX_CLOCK_BASES))
1542	return base;
1543	}
1544	WARN(1, "Invalid clockid %d. Using MONOTONIC\n", clock_id);
1545	return HRTIMER_BASE_MONOTONIC;
1546	}
1547
1548	static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
1549	enum hrtimer_mode mode)
1550	{
1551	bool softtimer = !!(mode & HRTIMER_MODE_SOFT);
1552	struct hrtimer_cpu_base *cpu_base;
1553	int base;
1554
1555	/*
1556	* On PREEMPT_RT enabled kernels hrtimers which are not explicitly
1557	* marked for hard interrupt expiry mode are moved into soft
1558	* interrupt context for latency reasons and because the callbacks
1559	* can invoke functions which might sleep on RT, e.g. spin_lock().
1560	*/
1561	if (IS_ENABLED(CONFIG_PREEMPT_RT) && !(mode & HRTIMER_MODE_HARD))
1562	softtimer = true;
1563
1564	memset(timer, 0, sizeof(struct hrtimer));
1565
1566	cpu_base = raw_cpu_ptr(&hrtimer_bases);
1567
1568	/*
1569	* POSIX magic: Relative CLOCK_REALTIME timers are not affected by
1570	* clock modifications, so they needs to become CLOCK_MONOTONIC to
1571	* ensure POSIX compliance.
1572	*/
1573	if (clock_id == CLOCK_REALTIME && mode & HRTIMER_MODE_REL)
1574	clock_id = CLOCK_MONOTONIC;
1575
1576	base = softtimer ? HRTIMER_MAX_CLOCK_BASES / 2 : 0;
1577	base += hrtimer_clockid_to_base(clock_id);
1578	timer->is_soft = softtimer;
1579	timer->is_hard = !!(mode & HRTIMER_MODE_HARD);
1580	timer->base = &cpu_base->clock_base[base];
1581	timerqueue_init(node: &timer->node);
1582	}
1583
1584	/**
1585	* hrtimer_init - initialize a timer to the given clock
1586	* @timer: the timer to be initialized
1587	* @clock_id: the clock to be used
1588	* @mode: The modes which are relevant for initialization:
1589	* HRTIMER_MODE_ABS, HRTIMER_MODE_REL, HRTIMER_MODE_ABS_SOFT,
1590	* HRTIMER_MODE_REL_SOFT
1591	*
1592	* The PINNED variants of the above can be handed in,
1593	* but the PINNED bit is ignored as pinning happens
1594	* when the hrtimer is started
1595	*/
1596	void hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
1597	enum hrtimer_mode mode)
1598	{
1599	debug_init(timer, clockid: clock_id, mode);
1600	__hrtimer_init(timer, clock_id, mode);
1601	}
1602	EXPORT_SYMBOL_GPL(hrtimer_init);
1603
1604	/*
1605	* A timer is active, when it is enqueued into the rbtree or the
1606	* callback function is running or it's in the state of being migrated
1607	* to another cpu.
1608	*
1609	* It is important for this function to not return a false negative.
1610	*/
1611	bool hrtimer_active(const struct hrtimer *timer)
1612	{
1613	struct hrtimer_clock_base *base;
1614	unsigned int seq;
1615
1616	do {
1617	base = READ_ONCE(timer->base);
1618	seq = raw_read_seqcount_begin(&base->seq);
1619
1620	if (timer->state != HRTIMER_STATE_INACTIVE ||
1621	base->running == timer)
1622	return true;
1623
1624	} while (read_seqcount_retry(&base->seq, seq) ||
1625	base != READ_ONCE(timer->base));
1626
1627	return false;
1628	}
1629	EXPORT_SYMBOL_GPL(hrtimer_active);
1630
1631	/*
1632	* The write_seqcount_barrier()s in __run_hrtimer() split the thing into 3
1633	* distinct sections:
1634	*
1635	* - queued: the timer is queued
1636	* - callback: the timer is being ran
1637	* - post: the timer is inactive or (re)queued
1638	*
1639	* On the read side we ensure we observe timer->state and cpu_base->running
1640	* from the same section, if anything changed while we looked at it, we retry.
1641	* This includes timer->base changing because sequence numbers alone are
1642	* insufficient for that.
1643	*
1644	* The sequence numbers are required because otherwise we could still observe
1645	* a false negative if the read side got smeared over multiple consecutive
1646	* __run_hrtimer() invocations.
1647	*/
1648
1649	static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base,
1650	struct hrtimer_clock_base *base,
1651	struct hrtimer *timer, ktime_t *now,
1652	unsigned long flags) __must_hold(&cpu_base->lock)
1653	{
1654	enum hrtimer_restart (*fn)(struct hrtimer *);
1655	bool expires_in_hardirq;
1656	int restart;
1657
1658	lockdep_assert_held(&cpu_base->lock);
1659
1660	debug_deactivate(timer);
1661	base->running = timer;
1662
1663	/*
1664	* Separate the ->running assignment from the ->state assignment.
1665	*
1666	* As with a regular write barrier, this ensures the read side in
1667	* hrtimer_active() cannot observe base->running == NULL &&
1668	* timer->state == INACTIVE.
1669	*/
1670	raw_write_seqcount_barrier(&base->seq);
1671
1672	__remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, reprogram: 0);
1673	fn = timer->function;
1674
1675	/*
1676	* Clear the 'is relative' flag for the TIME_LOW_RES case. If the
1677	* timer is restarted with a period then it becomes an absolute
1678	* timer. If its not restarted it does not matter.
1679	*/
1680	if (IS_ENABLED(CONFIG_TIME_LOW_RES))
1681	timer->is_rel = false;
1682
1683	/*
1684	* The timer is marked as running in the CPU base, so it is
1685	* protected against migration to a different CPU even if the lock
1686	* is dropped.
1687	*/
1688	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1689	trace_hrtimer_expire_entry(hrtimer: timer, now);
1690	expires_in_hardirq = lockdep_hrtimer_enter(timer);
1691
1692	restart = fn(timer);
1693
1694	lockdep_hrtimer_exit(expires_in_hardirq);
1695	trace_hrtimer_expire_exit(hrtimer: timer);
1696	raw_spin_lock_irq(&cpu_base->lock);
1697
1698	/*
1699	* Note: We clear the running state after enqueue_hrtimer and
1700	* we do not reprogram the event hardware. Happens either in
1701	* hrtimer_start_range_ns() or in hrtimer_interrupt()
1702	*
1703	* Note: Because we dropped the cpu_base->lock above,
1704	* hrtimer_start_range_ns() can have popped in and enqueued the timer
1705	* for us already.
1706	*/
1707	if (restart != HRTIMER_NORESTART &&
1708	!(timer->state & HRTIMER_STATE_ENQUEUED))
1709	enqueue_hrtimer(timer, base, mode: HRTIMER_MODE_ABS);
1710
1711	/*
1712	* Separate the ->running assignment from the ->state assignment.
1713	*
1714	* As with a regular write barrier, this ensures the read side in
1715	* hrtimer_active() cannot observe base->running.timer == NULL &&
1716	* timer->state == INACTIVE.
1717	*/
1718	raw_write_seqcount_barrier(&base->seq);
1719
1720	WARN_ON_ONCE(base->running != timer);
1721	base->running = NULL;
1722	}
1723
1724	static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now,
1725	unsigned long flags, unsigned int active_mask)
1726	{
1727	struct hrtimer_clock_base *base;
1728	unsigned int active = cpu_base->active_bases & active_mask;
1729
1730	for_each_active_base(base, cpu_base, active) {
1731	struct timerqueue_node *node;
1732	ktime_t basenow;
1733
1734	basenow = ktime_add(now, base->offset);
1735
1736	while ((node = timerqueue_getnext(head: &base->active))) {
1737	struct hrtimer *timer;
1738
1739	timer = container_of(node, struct hrtimer, node);
1740
1741	/*
1742	* The immediate goal for using the softexpires is
1743	* minimizing wakeups, not running timers at the
1744	* earliest interrupt after their soft expiration.
1745	* This allows us to avoid using a Priority Search
1746	* Tree, which can answer a stabbing query for
1747	* overlapping intervals and instead use the simple
1748	* BST we already have.
1749	* We don't add extra wakeups by delaying timers that
1750	* are right-of a not yet expired timer, because that
1751	* timer will have to trigger a wakeup anyway.
1752	*/
1753	if (basenow < hrtimer_get_softexpires_tv64(timer))
1754	break;
1755
1756	__run_hrtimer(cpu_base, base, timer, now: &basenow, flags);
1757	if (active_mask == HRTIMER_ACTIVE_SOFT)
1758	hrtimer_sync_wait_running(base: cpu_base, flags);
1759	}
1760	}
1761	}
1762
1763	static __latent_entropy void hrtimer_run_softirq(struct softirq_action *h)
1764	{
1765	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
1766	unsigned long flags;
1767	ktime_t now;
1768
1769	hrtimer_cpu_base_lock_expiry(base: cpu_base);
1770	raw_spin_lock_irqsave(&cpu_base->lock, flags);
1771
1772	now = hrtimer_update_base(base: cpu_base);
1773	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_SOFT);
1774
1775	cpu_base->softirq_activated = 0;
1776	hrtimer_update_softirq_timer(cpu_base, reprogram: true);
1777
1778	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1779	hrtimer_cpu_base_unlock_expiry(base: cpu_base);
1780	}
1781
1782	#ifdef CONFIG_HIGH_RES_TIMERS
1783
1784	/*
1785	* High resolution timer interrupt
1786	* Called with interrupts disabled
1787	*/
1788	void hrtimer_interrupt(struct clock_event_device *dev)
1789	{
1790	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
1791	ktime_t expires_next, now, entry_time, delta;
1792	unsigned long flags;
1793	int retries = 0;
1794
1795	BUG_ON(!cpu_base->hres_active);
1796	cpu_base->nr_events++;
1797	dev->next_event = KTIME_MAX;
1798
1799	raw_spin_lock_irqsave(&cpu_base->lock, flags);
1800	entry_time = now = hrtimer_update_base(base: cpu_base);
1801	retry:
1802	cpu_base->in_hrtirq = 1;
1803	/*
1804	* We set expires_next to KTIME_MAX here with cpu_base->lock
1805	* held to prevent that a timer is enqueued in our queue via
1806	* the migration code. This does not affect enqueueing of
1807	* timers which run their callback and need to be requeued on
1808	* this CPU.
1809	*/
1810	cpu_base->expires_next = KTIME_MAX;
1811
1812	if (!ktime_before(cmp1: now, cmp2: cpu_base->softirq_expires_next)) {
1813	cpu_base->softirq_expires_next = KTIME_MAX;
1814	cpu_base->softirq_activated = 1;
1815	raise_softirq_irqoff(nr: HRTIMER_SOFTIRQ);
1816	}
1817
1818	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);
1819
1820	/* Reevaluate the clock bases for the [soft] next expiry */
1821	expires_next = hrtimer_update_next_event(cpu_base);
1822	/*
1823	* Store the new expiry value so the migration code can verify
1824	* against it.
1825	*/
1826	cpu_base->expires_next = expires_next;
1827	cpu_base->in_hrtirq = 0;
1828	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1829
1830	/* Reprogramming necessary ? */
1831	if (!tick_program_event(expires: expires_next, force: 0)) {
1832	cpu_base->hang_detected = 0;
1833	return;
1834	}
1835
1836	/*
1837	* The next timer was already expired due to:
1838	* - tracing
1839	* - long lasting callbacks
1840	* - being scheduled away when running in a VM
1841	*
1842	* We need to prevent that we loop forever in the hrtimer
1843	* interrupt routine. We give it 3 attempts to avoid
1844	* overreacting on some spurious event.
1845	*
1846	* Acquire base lock for updating the offsets and retrieving
1847	* the current time.
1848	*/
1849	raw_spin_lock_irqsave(&cpu_base->lock, flags);
1850	now = hrtimer_update_base(base: cpu_base);
1851	cpu_base->nr_retries++;
1852	if (++retries < 3)
1853	goto retry;
1854	/*
1855	* Give the system a chance to do something else than looping
1856	* here. We stored the entry time, so we know exactly how long
1857	* we spent here. We schedule the next event this amount of
1858	* time away.
1859	*/
1860	cpu_base->nr_hangs++;
1861	cpu_base->hang_detected = 1;
1862	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1863
1864	delta = ktime_sub(now, entry_time);
1865	if ((unsigned int)delta > cpu_base->max_hang_time)
1866	cpu_base->max_hang_time = (unsigned int) delta;
1867	/*
1868	* Limit it to a sensible value as we enforce a longer
1869	* delay. Give the CPU at least 100ms to catch up.
1870	*/
1871	if (delta > 100 * NSEC_PER_MSEC)
1872	expires_next = ktime_add_ns(now, 100 * NSEC_PER_MSEC);
1873	else
1874	expires_next = ktime_add(now, delta);
1875	tick_program_event(expires: expires_next, force: 1);
1876	pr_warn_once("hrtimer: interrupt took %llu ns\n", ktime_to_ns(delta));
1877	}
1878
1879	/* called with interrupts disabled */
1880	static inline void __hrtimer_peek_ahead_timers(void)
1881	{
1882	struct tick_device *td;
1883
1884	if (!hrtimer_hres_active())
1885	return;
1886
1887	td = this_cpu_ptr(&tick_cpu_device);
1888	if (td && td->evtdev)
1889	hrtimer_interrupt(dev: td->evtdev);
1890	}
1891
1892	#else /* CONFIG_HIGH_RES_TIMERS */
1893
1894	static inline void __hrtimer_peek_ahead_timers(void) { }
1895
1896	#endif /* !CONFIG_HIGH_RES_TIMERS */
1897
1898	/*
1899	* Called from run_local_timers in hardirq context every jiffy
1900	*/
1901	void hrtimer_run_queues(void)
1902	{
1903	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
1904	unsigned long flags;
1905	ktime_t now;
1906
1907	if (__hrtimer_hres_active(cpu_base))
1908	return;
1909
1910	/*
1911	* This _is_ ugly: We have to check periodically, whether we
1912	* can switch to highres and / or nohz mode. The clocksource
1913	* switch happens with xtime_lock held. Notification from
1914	* there only sets the check bit in the tick_oneshot code,
1915	* otherwise we might deadlock vs. xtime_lock.
1916	*/
1917	if (tick_check_oneshot_change(allow_nohz: !hrtimer_is_hres_enabled())) {
1918	hrtimer_switch_to_hres();
1919	return;
1920	}
1921
1922	raw_spin_lock_irqsave(&cpu_base->lock, flags);
1923	now = hrtimer_update_base(base: cpu_base);
1924
1925	if (!ktime_before(cmp1: now, cmp2: cpu_base->softirq_expires_next)) {
1926	cpu_base->softirq_expires_next = KTIME_MAX;
1927	cpu_base->softirq_activated = 1;
1928	raise_softirq_irqoff(nr: HRTIMER_SOFTIRQ);
1929	}
1930
1931	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);
1932	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1933	}
1934
1935	/*
1936	* Sleep related functions:
1937	*/
1938	static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer)
1939	{
1940	struct hrtimer_sleeper *t =
1941	container_of(timer, struct hrtimer_sleeper, timer);
1942	struct task_struct *task = t->task;
1943
1944	t->task = NULL;
1945	if (task)
1946	wake_up_process(tsk: task);
1947
1948	return HRTIMER_NORESTART;
1949	}
1950
1951	/**
1952	* hrtimer_sleeper_start_expires - Start a hrtimer sleeper timer
1953	* @sl: sleeper to be started
1954	* @mode: timer mode abs/rel
1955	*
1956	* Wrapper around hrtimer_start_expires() for hrtimer_sleeper based timers
1957	* to allow PREEMPT_RT to tweak the delivery mode (soft/hardirq context)
1958	*/
1959	void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl,
1960	enum hrtimer_mode mode)
1961	{
1962	/*
1963	* Make the enqueue delivery mode check work on RT. If the sleeper
1964	* was initialized for hard interrupt delivery, force the mode bit.
1965	* This is a special case for hrtimer_sleepers because
1966	* hrtimer_init_sleeper() determines the delivery mode on RT so the
1967	* fiddling with this decision is avoided at the call sites.
1968	*/
1969	if (IS_ENABLED(CONFIG_PREEMPT_RT) && sl->timer.is_hard)
1970	mode |= HRTIMER_MODE_HARD;
1971
1972	hrtimer_start_expires(timer: &sl->timer, mode);
1973	}
1974	EXPORT_SYMBOL_GPL(hrtimer_sleeper_start_expires);
1975
1976	static void __hrtimer_init_sleeper(struct hrtimer_sleeper *sl,
1977	clockid_t clock_id, enum hrtimer_mode mode)
1978	{
1979	/*
1980	* On PREEMPT_RT enabled kernels hrtimers which are not explicitly
1981	* marked for hard interrupt expiry mode are moved into soft
1982	* interrupt context either for latency reasons or because the
1983	* hrtimer callback takes regular spinlocks or invokes other
1984	* functions which are not suitable for hard interrupt context on
1985	* PREEMPT_RT.
1986	*
1987	* The hrtimer_sleeper callback is RT compatible in hard interrupt
1988	* context, but there is a latency concern: Untrusted userspace can
1989	* spawn many threads which arm timers for the same expiry time on
1990	* the same CPU. That causes a latency spike due to the wakeup of
1991	* a gazillion threads.
1992	*
1993	* OTOH, privileged real-time user space applications rely on the
1994	* low latency of hard interrupt wakeups. If the current task is in
1995	* a real-time scheduling class, mark the mode for hard interrupt
1996	* expiry.
1997	*/
1998	if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
1999	if (task_is_realtime(current) && !(mode & HRTIMER_MODE_SOFT))
2000	mode |= HRTIMER_MODE_HARD;
2001	}
2002
2003	__hrtimer_init(timer: &sl->timer, clock_id, mode);
2004	sl->timer.function = hrtimer_wakeup;
2005	sl->task = current;
2006	}
2007
2008	/**
2009	* hrtimer_init_sleeper - initialize sleeper to the given clock
2010	* @sl: sleeper to be initialized
2011	* @clock_id: the clock to be used
2012	* @mode: timer mode abs/rel
2013	*/
2014	void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, clockid_t clock_id,
2015	enum hrtimer_mode mode)
2016	{
2017	debug_init(timer: &sl->timer, clockid: clock_id, mode);
2018	__hrtimer_init_sleeper(sl, clock_id, mode);
2019
2020	}
2021	EXPORT_SYMBOL_GPL(hrtimer_init_sleeper);
2022
2023	int nanosleep_copyout(struct restart_block *restart, struct timespec64 *ts)
2024	{
2025	switch(restart->nanosleep.type) {
2026	#ifdef CONFIG_COMPAT_32BIT_TIME
2027	case TT_COMPAT:
2028	if (put_old_timespec32(ts, restart->nanosleep.compat_rmtp))
2029	return -EFAULT;
2030	break;
2031	#endif
2032	case TT_NATIVE:
2033	if (put_timespec64(ts, uts: restart->nanosleep.rmtp))
2034	return -EFAULT;
2035	break;
2036	default:
2037	BUG();
2038	}
2039	return -ERESTART_RESTARTBLOCK;
2040	}
2041
2042	static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode)
2043	{
2044	struct restart_block *restart;
2045
2046	do {
2047	set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE);
2048	hrtimer_sleeper_start_expires(t, mode);
2049
2050	if (likely(t->task))
2051	schedule();
2052
2053	hrtimer_cancel(&t->timer);
2054	mode = HRTIMER_MODE_ABS;
2055
2056	} while (t->task && !signal_pending(current));
2057
2058	__set_current_state(TASK_RUNNING);
2059
2060	if (!t->task)
2061	return 0;
2062
2063	restart = &current->restart_block;
2064	if (restart->nanosleep.type != TT_NONE) {
2065	ktime_t rem = hrtimer_expires_remaining(timer: &t->timer);
2066	struct timespec64 rmt;
2067
2068	if (rem <= 0)
2069	return 0;
2070	rmt = ktime_to_timespec64(rem);
2071
2072	return nanosleep_copyout(restart, ts: &rmt);
2073	}
2074	return -ERESTART_RESTARTBLOCK;
2075	}
2076
2077	static long __sched hrtimer_nanosleep_restart(struct restart_block *restart)
2078	{
2079	struct hrtimer_sleeper t;
2080	int ret;
2081
2082	hrtimer_init_sleeper_on_stack(&t, restart->nanosleep.clockid,
2083	HRTIMER_MODE_ABS);
2084	hrtimer_set_expires_tv64(timer: &t.timer, tv64: restart->nanosleep.expires);
2085	ret = do_nanosleep(t: &t, mode: HRTIMER_MODE_ABS);
2086	destroy_hrtimer_on_stack(&t.timer);
2087	return ret;
2088	}
2089
2090	long hrtimer_nanosleep(ktime_t rqtp, const enum hrtimer_mode mode,
2091	const clockid_t clockid)
2092	{
2093	struct restart_block *restart;
2094	struct hrtimer_sleeper t;
2095	int ret = 0;
2096	u64 slack;
2097
2098	slack = current->timer_slack_ns;
2099	if (rt_task(current))
2100	slack = 0;
2101
2102	hrtimer_init_sleeper_on_stack(&t, clockid, mode);
2103	hrtimer_set_expires_range_ns(timer: &t.timer, time: rqtp, delta: slack);
2104	ret = do_nanosleep(t: &t, mode);
2105	if (ret != -ERESTART_RESTARTBLOCK)
2106	goto out;
2107
2108	/* Absolute timers do not update the rmtp value and restart: */
2109	if (mode == HRTIMER_MODE_ABS) {
2110	ret = -ERESTARTNOHAND;
2111	goto out;
2112	}
2113
2114	restart = &current->restart_block;
2115	restart->nanosleep.clockid = t.timer.base->clockid;
2116	restart->nanosleep.expires = hrtimer_get_expires_tv64(timer: &t.timer);
2117	set_restart_fn(restart, fn: hrtimer_nanosleep_restart);
2118	out:
2119	destroy_hrtimer_on_stack(&t.timer);
2120	return ret;
2121	}
2122
2123	#ifdef CONFIG_64BIT
2124
2125	SYSCALL_DEFINE2(nanosleep, struct __kernel_timespec __user *, rqtp,
2126	struct __kernel_timespec __user *, rmtp)
2127	{
2128	struct timespec64 tu;
2129
2130	if (get_timespec64(ts: &tu, uts: rqtp))
2131	return -EFAULT;
2132
2133	if (!timespec64_valid(ts: &tu))
2134	return -EINVAL;
2135
2136	current->restart_block.fn = do_no_restart_syscall;
2137	current->restart_block.nanosleep.type = rmtp ? TT_NATIVE : TT_NONE;
2138	current->restart_block.nanosleep.rmtp = rmtp;
2139	return hrtimer_nanosleep(rqtp: timespec64_to_ktime(ts: tu), mode: HRTIMER_MODE_REL,
2140	CLOCK_MONOTONIC);
2141	}
2142
2143	#endif
2144
2145	#ifdef CONFIG_COMPAT_32BIT_TIME
2146
2147	SYSCALL_DEFINE2(nanosleep_time32, struct old_timespec32 __user *, rqtp,
2148	struct old_timespec32 __user *, rmtp)
2149	{
2150	struct timespec64 tu;
2151
2152	if (get_old_timespec32(&tu, rqtp))
2153	return -EFAULT;
2154
2155	if (!timespec64_valid(ts: &tu))
2156	return -EINVAL;
2157
2158	current->restart_block.fn = do_no_restart_syscall;
2159	current->restart_block.nanosleep.type = rmtp ? TT_COMPAT : TT_NONE;
2160	current->restart_block.nanosleep.compat_rmtp = rmtp;
2161	return hrtimer_nanosleep(rqtp: timespec64_to_ktime(ts: tu), mode: HRTIMER_MODE_REL,
2162	CLOCK_MONOTONIC);
2163	}
2164	#endif
2165
2166	/*
2167	* Functions related to boot-time initialization:
2168	*/
2169	int hrtimers_prepare_cpu(unsigned int cpu)
2170	{
2171	struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu);
2172	int i;
2173
2174	for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
2175	struct hrtimer_clock_base *clock_b = &cpu_base->clock_base[i];
2176
2177	clock_b->cpu_base = cpu_base;
2178	seqcount_raw_spinlock_init(&clock_b->seq, &cpu_base->lock);
2179	timerqueue_init_head(head: &clock_b->active);
2180	}
2181
2182	cpu_base->cpu = cpu;
2183	cpu_base->active_bases = 0;
2184	cpu_base->hres_active = 0;
2185	cpu_base->hang_detected = 0;
2186	cpu_base->next_timer = NULL;
2187	cpu_base->softirq_next_timer = NULL;
2188	cpu_base->expires_next = KTIME_MAX;
2189	cpu_base->softirq_expires_next = KTIME_MAX;
2190	cpu_base->online = 1;
2191	hrtimer_cpu_base_init_expiry_lock(base: cpu_base);
2192	return 0;
2193	}
2194
2195	#ifdef CONFIG_HOTPLUG_CPU
2196
2197	static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base,
2198	struct hrtimer_clock_base *new_base)
2199	{
2200	struct hrtimer *timer;
2201	struct timerqueue_node *node;
2202
2203	while ((node = timerqueue_getnext(head: &old_base->active))) {
2204	timer = container_of(node, struct hrtimer, node);
2205	BUG_ON(hrtimer_callback_running(timer));
2206	debug_deactivate(timer);
2207
2208	/*
2209	* Mark it as ENQUEUED not INACTIVE otherwise the
2210	* timer could be seen as !active and just vanish away
2211	* under us on another CPU
2212	*/
2213	__remove_hrtimer(timer, base: old_base, HRTIMER_STATE_ENQUEUED, reprogram: 0);
2214	timer->base = new_base;
2215	/*
2216	* Enqueue the timers on the new cpu. This does not
2217	* reprogram the event device in case the timer
2218	* expires before the earliest on this CPU, but we run
2219	* hrtimer_interrupt after we migrated everything to
2220	* sort out already expired timers and reprogram the
2221	* event device.
2222	*/
2223	enqueue_hrtimer(timer, base: new_base, mode: HRTIMER_MODE_ABS);
2224	}
2225	}
2226
2227	int hrtimers_cpu_dying(unsigned int dying_cpu)
2228	{
2229	int i, ncpu = cpumask_any_and(cpu_active_mask, housekeeping_cpumask(HK_TYPE_TIMER));
2230	struct hrtimer_cpu_base *old_base, *new_base;
2231
2232	old_base = this_cpu_ptr(&hrtimer_bases);
2233	new_base = &per_cpu(hrtimer_bases, ncpu);
2234
2235	/*
2236	* The caller is globally serialized and nobody else
2237	* takes two locks at once, deadlock is not possible.
2238	*/
2239	raw_spin_lock(&old_base->lock);
2240	raw_spin_lock_nested(&new_base->lock, SINGLE_DEPTH_NESTING);
2241
2242	for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
2243	migrate_hrtimer_list(old_base: &old_base->clock_base[i],
2244	new_base: &new_base->clock_base[i]);
2245	}
2246
2247	/*
2248	* The migration might have changed the first expiring softirq
2249	* timer on this CPU. Update it.
2250	*/
2251	__hrtimer_get_next_event(cpu_base: new_base, HRTIMER_ACTIVE_SOFT);
2252	/* Tell the other CPU to retrigger the next event */
2253	smp_call_function_single(cpuid: ncpu, func: retrigger_next_event, NULL, wait: 0);
2254
2255	raw_spin_unlock(&new_base->lock);
2256	old_base->online = 0;
2257	raw_spin_unlock(&old_base->lock);
2258
2259	return 0;
2260	}
2261
2262	#endif /* CONFIG_HOTPLUG_CPU */
2263
2264	void __init hrtimers_init(void)
2265	{
2266	hrtimers_prepare_cpu(smp_processor_id());
2267	open_softirq(nr: HRTIMER_SOFTIRQ, action: hrtimer_run_softirq);
2268	}
2269
2270	/**
2271	* schedule_hrtimeout_range_clock - sleep until timeout
2272	* @expires: timeout value (ktime_t)
2273	* @delta: slack in expires timeout (ktime_t) for SCHED_OTHER tasks
2274	* @mode: timer mode
2275	* @clock_id: timer clock to be used
2276	*/
2277	int __sched
2278	schedule_hrtimeout_range_clock(ktime_t *expires, u64 delta,
2279	const enum hrtimer_mode mode, clockid_t clock_id)
2280	{
2281	struct hrtimer_sleeper t;
2282
2283	/*
2284	* Optimize when a zero timeout value is given. It does not
2285	* matter whether this is an absolute or a relative time.
2286	*/
2287	if (expires && *expires == 0) {
2288	__set_current_state(TASK_RUNNING);
2289	return 0;
2290	}
2291
2292	/*
2293	* A NULL parameter means "infinite"
2294	*/
2295	if (!expires) {
2296	schedule();
2297	return -EINTR;
2298	}
2299
2300	/*
2301	* Override any slack passed by the user if under
2302	* rt contraints.
2303	*/
2304	if (rt_task(current))
2305	delta = 0;
2306
2307	hrtimer_init_sleeper_on_stack(&t, clock_id, mode);
2308	hrtimer_set_expires_range_ns(timer: &t.timer, time: *expires, delta);
2309	hrtimer_sleeper_start_expires(&t, mode);
2310
2311	if (likely(t.task))
2312	schedule();
2313
2314	hrtimer_cancel(&t.timer);
2315	destroy_hrtimer_on_stack(&t.timer);
2316
2317	__set_current_state(TASK_RUNNING);
2318
2319	return !t.task ? 0 : -EINTR;
2320	}
2321	EXPORT_SYMBOL_GPL(schedule_hrtimeout_range_clock);
2322
2323	/**
2324	* schedule_hrtimeout_range - sleep until timeout
2325	* @expires: timeout value (ktime_t)
2326	* @delta: slack in expires timeout (ktime_t) for SCHED_OTHER tasks
2327	* @mode: timer mode
2328	*
2329	* Make the current task sleep until the given expiry time has
2330	* elapsed. The routine will return immediately unless
2331	* the current task state has been set (see set_current_state()).
2332	*
2333	* The @delta argument gives the kernel the freedom to schedule the
2334	* actual wakeup to a time that is both power and performance friendly
2335	* for regular (non RT/DL) tasks.
2336	* The kernel give the normal best effort behavior for "@expires+@delta",
2337	* but may decide to fire the timer earlier, but no earlier than @expires.
2338	*
2339	* You can set the task state as follows -
2340	*
2341	* %TASK_UNINTERRUPTIBLE - at least @timeout time is guaranteed to
2342	* pass before the routine returns unless the current task is explicitly
2343	* woken up, (e.g. by wake_up_process()).
2344	*
2345	* %TASK_INTERRUPTIBLE - the routine may return early if a signal is
2346	* delivered to the current task or the current task is explicitly woken
2347	* up.
2348	*
2349	* The current task state is guaranteed to be TASK_RUNNING when this
2350	* routine returns.
2351	*
2352	* Returns 0 when the timer has expired. If the task was woken before the
2353	* timer expired by a signal (only possible in state TASK_INTERRUPTIBLE) or
2354	* by an explicit wakeup, it returns -EINTR.
2355	*/
2356	int __sched schedule_hrtimeout_range(ktime_t *expires, u64 delta,
2357	const enum hrtimer_mode mode)
2358	{
2359	return schedule_hrtimeout_range_clock(expires, delta, mode,
2360	CLOCK_MONOTONIC);
2361	}
2362	EXPORT_SYMBOL_GPL(schedule_hrtimeout_range);
2363
2364	/**
2365	* schedule_hrtimeout - sleep until timeout
2366	* @expires: timeout value (ktime_t)
2367	* @mode: timer mode
2368	*
2369	* Make the current task sleep until the given expiry time has
2370	* elapsed. The routine will return immediately unless
2371	* the current task state has been set (see set_current_state()).
2372	*
2373	* You can set the task state as follows -
2374	*
2375	* %TASK_UNINTERRUPTIBLE - at least @timeout time is guaranteed to
2376	* pass before the routine returns unless the current task is explicitly
2377	* woken up, (e.g. by wake_up_process()).
2378	*
2379	* %TASK_INTERRUPTIBLE - the routine may return early if a signal is
2380	* delivered to the current task or the current task is explicitly woken
2381	* up.
2382	*
2383	* The current task state is guaranteed to be TASK_RUNNING when this
2384	* routine returns.
2385	*
2386	* Returns 0 when the timer has expired. If the task was woken before the
2387	* timer expired by a signal (only possible in state TASK_INTERRUPTIBLE) or
2388	* by an explicit wakeup, it returns -EINTR.
2389	*/
2390	int __sched schedule_hrtimeout(ktime_t *expires,
2391	const enum hrtimer_mode mode)
2392	{
2393	return schedule_hrtimeout_range(expires, 0, mode);
2394	}
2395	EXPORT_SYMBOL_GPL(schedule_hrtimeout);
2396