1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
2 | #ifndef _KERNEL_TIME_MIGRATION_H |
3 | #define _KERNEL_TIME_MIGRATION_H |
4 | |
5 | /* Per group capacity. Must be a power of 2! */ |
6 | #define TMIGR_CHILDREN_PER_GROUP 8 |
7 | |
8 | /** |
9 | * struct tmigr_event - a timer event associated to a CPU |
10 | * @nextevt: The node to enqueue an event in the parent group queue |
11 | * @cpu: The CPU to which this event belongs |
12 | * @ignore: Hint whether the event could be ignored; it is set when |
13 | * CPU or group is active; |
14 | */ |
15 | struct tmigr_event { |
16 | struct timerqueue_node nextevt; |
17 | unsigned int cpu; |
18 | bool ignore; |
19 | }; |
20 | |
21 | /** |
22 | * struct tmigr_group - timer migration hierarchy group |
23 | * @lock: Lock protecting the event information and group hierarchy |
24 | * information during setup |
25 | * @parent: Pointer to the parent group |
26 | * @groupevt: Next event of the group which is only used when the |
27 | * group is !active. The group event is then queued into |
28 | * the parent timer queue. |
29 | * Ignore bit of @groupevt is set when the group is active. |
30 | * @next_expiry: Base monotonic expiry time of the next event of the |
31 | * group; It is used for the racy lockless check whether a |
32 | * remote expiry is required; it is always reliable |
33 | * @events: Timer queue for child events queued in the group |
34 | * @migr_state: State of the group (see union tmigr_state) |
35 | * @level: Hierarchy level of the group; Required during setup |
36 | * @numa_node: Required for setup only to make sure CPU and low level |
37 | * group information is NUMA local. It is set to NUMA node |
38 | * as long as the group level is per NUMA node (level < |
39 | * tmigr_crossnode_level); otherwise it is set to |
40 | * NUMA_NO_NODE |
41 | * @num_children: Counter of group children to make sure the group is only |
42 | * filled with TMIGR_CHILDREN_PER_GROUP; Required for setup |
43 | * only |
44 | * @childmask: childmask of the group in the parent group; is set |
45 | * during setup and will never change; can be read |
46 | * lockless |
47 | * @list: List head that is added to the per level |
48 | * tmigr_level_list; is required during setup when a |
49 | * new group needs to be connected to the existing |
50 | * hierarchy groups |
51 | */ |
52 | struct tmigr_group { |
53 | raw_spinlock_t lock; |
54 | struct tmigr_group *parent; |
55 | struct tmigr_event groupevt; |
56 | u64 next_expiry; |
57 | struct timerqueue_head events; |
58 | atomic_t migr_state; |
59 | unsigned int level; |
60 | int numa_node; |
61 | unsigned int num_children; |
62 | u8 childmask; |
63 | struct list_head list; |
64 | }; |
65 | |
66 | /** |
67 | * struct tmigr_cpu - timer migration per CPU group |
68 | * @lock: Lock protecting the tmigr_cpu group information |
69 | * @online: Indicates whether the CPU is online; In deactivate path |
70 | * it is required to know whether the migrator in the top |
71 | * level group is to be set offline, while a timer is |
72 | * pending. Then another online CPU needs to be notified to |
73 | * take over the migrator role. Furthermore the information |
74 | * is required in CPU hotplug path as the CPU is able to go |
75 | * idle before the timer migration hierarchy hotplug AP is |
76 | * reached. During this phase, the CPU has to handle the |
77 | * global timers on its own and must not act as a migrator. |
78 | * @idle: Indicates whether the CPU is idle in the timer migration |
79 | * hierarchy |
80 | * @remote: Is set when timers of the CPU are expired remotely |
81 | * @tmgroup: Pointer to the parent group |
82 | * @childmask: childmask of tmigr_cpu in the parent group |
83 | * @wakeup: Stores the first timer when the timer migration |
84 | * hierarchy is completely idle and remote expiry was done; |
85 | * is returned to timer code in the idle path and is only |
86 | * used in idle path. |
87 | * @cpuevt: CPU event which could be enqueued into the parent group |
88 | */ |
89 | struct tmigr_cpu { |
90 | raw_spinlock_t lock; |
91 | bool online; |
92 | bool idle; |
93 | bool remote; |
94 | struct tmigr_group *tmgroup; |
95 | u8 childmask; |
96 | u64 wakeup; |
97 | struct tmigr_event cpuevt; |
98 | }; |
99 | |
100 | /** |
101 | * union tmigr_state - state of tmigr_group |
102 | * @state: Combined version of the state - only used for atomic |
103 | * read/cmpxchg function |
104 | * @struct: Split version of the state - only use the struct members to |
105 | * update information to stay independent of endianness |
106 | */ |
107 | union tmigr_state { |
108 | u32 state; |
109 | /** |
110 | * struct - split state of tmigr_group |
111 | * @active: Contains each childmask bit of the active children |
112 | * @migrator: Contains childmask of the child which is migrator |
113 | * @seq: Sequence counter needs to be increased when an update |
114 | * to the tmigr_state is done. It prevents a race when |
115 | * updates in the child groups are propagated in changed |
116 | * order. Detailed information about the scenario is |
117 | * given in the documentation at the begin of |
118 | * timer_migration.c. |
119 | */ |
120 | struct { |
121 | u8 active; |
122 | u8 migrator; |
123 | u16 seq; |
124 | } __packed; |
125 | }; |
126 | |
127 | #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON) |
128 | extern void tmigr_handle_remote(void); |
129 | extern bool tmigr_requires_handle_remote(void); |
130 | extern void tmigr_cpu_activate(void); |
131 | extern u64 tmigr_cpu_deactivate(u64 nextevt); |
132 | extern u64 tmigr_cpu_new_timer(u64 nextevt); |
133 | extern u64 tmigr_quick_check(u64 nextevt); |
134 | #else |
135 | static inline void tmigr_handle_remote(void) { } |
136 | static inline bool tmigr_requires_handle_remote(void) { return false; } |
137 | static inline void tmigr_cpu_activate(void) { } |
138 | #endif |
139 | |
140 | #endif |
141 | |