1 | #ifdef CONFIG_SMP |
2 | #include "sched-pelt.h" |
3 | |
4 | int __update_load_avg_blocked_se(u64 now, struct sched_entity *se); |
5 | int __update_load_avg_se(u64 now, struct cfs_rq *cfs_rq, struct sched_entity *se); |
6 | int __update_load_avg_cfs_rq(u64 now, struct cfs_rq *cfs_rq); |
7 | int update_rt_rq_load_avg(u64 now, struct rq *rq, int running); |
8 | int update_dl_rq_load_avg(u64 now, struct rq *rq, int running); |
9 | |
10 | #ifdef CONFIG_SCHED_THERMAL_PRESSURE |
11 | int update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity); |
12 | |
13 | static inline u64 thermal_load_avg(struct rq *rq) |
14 | { |
15 | return READ_ONCE(rq->avg_thermal.load_avg); |
16 | } |
17 | #else |
18 | static inline int |
19 | update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity) |
20 | { |
21 | return 0; |
22 | } |
23 | |
24 | static inline u64 thermal_load_avg(struct rq *rq) |
25 | { |
26 | return 0; |
27 | } |
28 | #endif |
29 | |
30 | #ifdef CONFIG_HAVE_SCHED_AVG_IRQ |
31 | int update_irq_load_avg(struct rq *rq, u64 running); |
32 | #else |
33 | static inline int |
34 | update_irq_load_avg(struct rq *rq, u64 running) |
35 | { |
36 | return 0; |
37 | } |
38 | #endif |
39 | |
40 | #define PELT_MIN_DIVIDER (LOAD_AVG_MAX - 1024) |
41 | |
42 | static inline u32 get_pelt_divider(struct sched_avg *avg) |
43 | { |
44 | return PELT_MIN_DIVIDER + avg->period_contrib; |
45 | } |
46 | |
47 | static inline void cfs_se_util_change(struct sched_avg *avg) |
48 | { |
49 | unsigned int enqueued; |
50 | |
51 | if (!sched_feat(UTIL_EST)) |
52 | return; |
53 | |
54 | /* Avoid store if the flag has been already reset */ |
55 | enqueued = avg->util_est.enqueued; |
56 | if (!(enqueued & UTIL_AVG_UNCHANGED)) |
57 | return; |
58 | |
59 | /* Reset flag to report util_avg has been updated */ |
60 | enqueued &= ~UTIL_AVG_UNCHANGED; |
61 | WRITE_ONCE(avg->util_est.enqueued, enqueued); |
62 | } |
63 | |
64 | static inline u64 rq_clock_pelt(struct rq *rq) |
65 | { |
66 | lockdep_assert_rq_held(rq); |
67 | assert_clock_updated(rq); |
68 | |
69 | return rq->clock_pelt - rq->lost_idle_time; |
70 | } |
71 | |
72 | /* The rq is idle, we can sync to clock_task */ |
73 | static inline void _update_idle_rq_clock_pelt(struct rq *rq) |
74 | { |
75 | rq->clock_pelt = rq_clock_task(rq); |
76 | |
77 | u64_u32_store(rq->clock_idle, rq_clock(rq)); |
78 | /* Paired with smp_rmb in migrate_se_pelt_lag() */ |
79 | smp_wmb(); |
80 | u64_u32_store(rq->clock_pelt_idle, rq_clock_pelt(rq)); |
81 | } |
82 | |
83 | /* |
84 | * The clock_pelt scales the time to reflect the effective amount of |
85 | * computation done during the running delta time but then sync back to |
86 | * clock_task when rq is idle. |
87 | * |
88 | * |
89 | * absolute time | 1| 2| 3| 4| 5| 6| 7| 8| 9|10|11|12|13|14|15|16 |
90 | * @ max capacity ------******---------------******--------------- |
91 | * @ half capacity ------************---------************--------- |
92 | * clock pelt | 1| 2| 3| 4| 7| 8| 9| 10| 11|14|15|16 |
93 | * |
94 | */ |
95 | static inline void update_rq_clock_pelt(struct rq *rq, s64 delta) |
96 | { |
97 | if (unlikely(is_idle_task(rq->curr))) { |
98 | _update_idle_rq_clock_pelt(rq); |
99 | return; |
100 | } |
101 | |
102 | /* |
103 | * When a rq runs at a lower compute capacity, it will need |
104 | * more time to do the same amount of work than at max |
105 | * capacity. In order to be invariant, we scale the delta to |
106 | * reflect how much work has been really done. |
107 | * Running longer results in stealing idle time that will |
108 | * disturb the load signal compared to max capacity. This |
109 | * stolen idle time will be automatically reflected when the |
110 | * rq will be idle and the clock will be synced with |
111 | * rq_clock_task. |
112 | */ |
113 | |
114 | /* |
115 | * Scale the elapsed time to reflect the real amount of |
116 | * computation |
117 | */ |
118 | delta = cap_scale(delta, arch_scale_cpu_capacity(cpu_of(rq))); |
119 | delta = cap_scale(delta, arch_scale_freq_capacity(cpu_of(rq))); |
120 | |
121 | rq->clock_pelt += delta; |
122 | } |
123 | |
124 | /* |
125 | * When rq becomes idle, we have to check if it has lost idle time |
126 | * because it was fully busy. A rq is fully used when the /Sum util_sum |
127 | * is greater or equal to: |
128 | * (LOAD_AVG_MAX - 1024 + rq->cfs.avg.period_contrib) << SCHED_CAPACITY_SHIFT; |
129 | * For optimization and computing rounding purpose, we don't take into account |
130 | * the position in the current window (period_contrib) and we use the higher |
131 | * bound of util_sum to decide. |
132 | */ |
133 | static inline void update_idle_rq_clock_pelt(struct rq *rq) |
134 | { |
135 | u32 divider = ((LOAD_AVG_MAX - 1024) << SCHED_CAPACITY_SHIFT) - LOAD_AVG_MAX; |
136 | u32 util_sum = rq->cfs.avg.util_sum; |
137 | util_sum += rq->avg_rt.util_sum; |
138 | util_sum += rq->avg_dl.util_sum; |
139 | |
140 | /* |
141 | * Reflecting stolen time makes sense only if the idle |
142 | * phase would be present at max capacity. As soon as the |
143 | * utilization of a rq has reached the maximum value, it is |
144 | * considered as an always running rq without idle time to |
145 | * steal. This potential idle time is considered as lost in |
146 | * this case. We keep track of this lost idle time compare to |
147 | * rq's clock_task. |
148 | */ |
149 | if (util_sum >= divider) |
150 | rq->lost_idle_time += rq_clock_task(rq) - rq->clock_pelt; |
151 | |
152 | _update_idle_rq_clock_pelt(rq); |
153 | } |
154 | |
155 | #ifdef CONFIG_CFS_BANDWIDTH |
156 | static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) |
157 | { |
158 | u64 throttled; |
159 | |
160 | if (unlikely(cfs_rq->throttle_count)) |
161 | throttled = U64_MAX; |
162 | else |
163 | throttled = cfs_rq->throttled_clock_pelt_time; |
164 | |
165 | u64_u32_store(cfs_rq->throttled_pelt_idle, throttled); |
166 | } |
167 | |
168 | /* rq->task_clock normalized against any time this cfs_rq has spent throttled */ |
169 | static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) |
170 | { |
171 | if (unlikely(cfs_rq->throttle_count)) |
172 | return cfs_rq->throttled_clock_pelt - cfs_rq->throttled_clock_pelt_time; |
173 | |
174 | return rq_clock_pelt(rq: rq_of(cfs_rq)) - cfs_rq->throttled_clock_pelt_time; |
175 | } |
176 | #else |
177 | static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) { } |
178 | static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) |
179 | { |
180 | return rq_clock_pelt(rq_of(cfs_rq)); |
181 | } |
182 | #endif |
183 | |
184 | #else |
185 | |
186 | static inline int |
187 | update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq) |
188 | { |
189 | return 0; |
190 | } |
191 | |
192 | static inline int |
193 | update_rt_rq_load_avg(u64 now, struct rq *rq, int running) |
194 | { |
195 | return 0; |
196 | } |
197 | |
198 | static inline int |
199 | update_dl_rq_load_avg(u64 now, struct rq *rq, int running) |
200 | { |
201 | return 0; |
202 | } |
203 | |
204 | static inline int |
205 | update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity) |
206 | { |
207 | return 0; |
208 | } |
209 | |
210 | static inline u64 thermal_load_avg(struct rq *rq) |
211 | { |
212 | return 0; |
213 | } |
214 | |
215 | static inline int |
216 | update_irq_load_avg(struct rq *rq, u64 running) |
217 | { |
218 | return 0; |
219 | } |
220 | |
221 | static inline u64 rq_clock_pelt(struct rq *rq) |
222 | { |
223 | return rq_clock_task(rq); |
224 | } |
225 | |
226 | static inline void |
227 | update_rq_clock_pelt(struct rq *rq, s64 delta) { } |
228 | |
229 | static inline void |
230 | update_idle_rq_clock_pelt(struct rq *rq) { } |
231 | |
232 | static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) { } |
233 | #endif |
234 | |
235 | |
236 | |