1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (C) 2013 Red Hat |
4 | * Author: Rob Clark <robdclark@gmail.com> |
5 | */ |
6 | |
7 | #include "msm_gpu.h" |
8 | #include "msm_gpu_trace.h" |
9 | |
10 | #include <linux/devfreq.h> |
11 | #include <linux/devfreq_cooling.h> |
12 | #include <linux/math64.h> |
13 | #include <linux/units.h> |
14 | |
15 | /* |
16 | * Power Management: |
17 | */ |
18 | |
19 | static int msm_devfreq_target(struct device *dev, unsigned long *freq, |
20 | u32 flags) |
21 | { |
22 | struct msm_gpu *gpu = dev_to_gpu(dev); |
23 | struct msm_gpu_devfreq *df = &gpu->devfreq; |
24 | struct dev_pm_opp *opp; |
25 | |
26 | /* |
27 | * Note that devfreq_recommended_opp() can modify the freq |
28 | * to something that actually is in the opp table: |
29 | */ |
30 | opp = devfreq_recommended_opp(dev, freq, flags); |
31 | if (IS_ERR(ptr: opp)) |
32 | return PTR_ERR(ptr: opp); |
33 | |
34 | trace_msm_gpu_freq_change(freq: dev_pm_opp_get_freq(opp)); |
35 | |
36 | /* |
37 | * If the GPU is idle, devfreq is not aware, so just stash |
38 | * the new target freq (to use when we return to active) |
39 | */ |
40 | if (df->idle_freq) { |
41 | df->idle_freq = *freq; |
42 | dev_pm_opp_put(opp); |
43 | return 0; |
44 | } |
45 | |
46 | if (gpu->funcs->gpu_set_freq) { |
47 | mutex_lock(&df->lock); |
48 | gpu->funcs->gpu_set_freq(gpu, opp, df->suspended); |
49 | mutex_unlock(lock: &df->lock); |
50 | } else { |
51 | dev_pm_opp_set_rate(dev, target_freq: *freq); |
52 | } |
53 | |
54 | dev_pm_opp_put(opp); |
55 | |
56 | return 0; |
57 | } |
58 | |
59 | static unsigned long get_freq(struct msm_gpu *gpu) |
60 | { |
61 | struct msm_gpu_devfreq *df = &gpu->devfreq; |
62 | |
63 | /* |
64 | * If the GPU is idle, use the shadow/saved freq to avoid |
65 | * confusing devfreq (which is unaware that we are switching |
66 | * to lowest freq until the device is active again) |
67 | */ |
68 | if (df->idle_freq) |
69 | return df->idle_freq; |
70 | |
71 | if (gpu->funcs->gpu_get_freq) |
72 | return gpu->funcs->gpu_get_freq(gpu); |
73 | |
74 | return clk_get_rate(clk: gpu->core_clk); |
75 | } |
76 | |
77 | static int msm_devfreq_get_dev_status(struct device *dev, |
78 | struct devfreq_dev_status *status) |
79 | { |
80 | struct msm_gpu *gpu = dev_to_gpu(dev); |
81 | struct msm_gpu_devfreq *df = &gpu->devfreq; |
82 | u64 busy_cycles, busy_time; |
83 | unsigned long sample_rate; |
84 | ktime_t time; |
85 | |
86 | mutex_lock(&df->lock); |
87 | |
88 | status->current_frequency = get_freq(gpu); |
89 | time = ktime_get(); |
90 | status->total_time = ktime_us_delta(later: time, earlier: df->time); |
91 | df->time = time; |
92 | |
93 | if (df->suspended) { |
94 | mutex_unlock(lock: &df->lock); |
95 | status->busy_time = 0; |
96 | return 0; |
97 | } |
98 | |
99 | busy_cycles = gpu->funcs->gpu_busy(gpu, &sample_rate); |
100 | busy_time = busy_cycles - df->busy_cycles; |
101 | df->busy_cycles = busy_cycles; |
102 | |
103 | mutex_unlock(lock: &df->lock); |
104 | |
105 | busy_time *= USEC_PER_SEC; |
106 | busy_time = div64_ul(busy_time, sample_rate); |
107 | if (WARN_ON(busy_time > ~0LU)) |
108 | busy_time = ~0LU; |
109 | |
110 | status->busy_time = busy_time; |
111 | |
112 | return 0; |
113 | } |
114 | |
115 | static int msm_devfreq_get_cur_freq(struct device *dev, unsigned long *freq) |
116 | { |
117 | *freq = get_freq(gpu: dev_to_gpu(dev)); |
118 | |
119 | return 0; |
120 | } |
121 | |
122 | static struct devfreq_dev_profile msm_devfreq_profile = { |
123 | .timer = DEVFREQ_TIMER_DELAYED, |
124 | .polling_ms = 50, |
125 | .target = msm_devfreq_target, |
126 | .get_dev_status = msm_devfreq_get_dev_status, |
127 | .get_cur_freq = msm_devfreq_get_cur_freq, |
128 | }; |
129 | |
130 | static void msm_devfreq_boost_work(struct kthread_work *work); |
131 | static void msm_devfreq_idle_work(struct kthread_work *work); |
132 | |
133 | static bool has_devfreq(struct msm_gpu *gpu) |
134 | { |
135 | struct msm_gpu_devfreq *df = &gpu->devfreq; |
136 | return !!df->devfreq; |
137 | } |
138 | |
139 | void msm_devfreq_init(struct msm_gpu *gpu) |
140 | { |
141 | struct msm_gpu_devfreq *df = &gpu->devfreq; |
142 | struct msm_drm_private *priv = gpu->dev->dev_private; |
143 | |
144 | /* We need target support to do devfreq */ |
145 | if (!gpu->funcs->gpu_busy) |
146 | return; |
147 | |
148 | /* |
149 | * Setup default values for simple_ondemand governor tuning. We |
150 | * want to throttle up at 50% load for the double-buffer case, |
151 | * where due to stalling waiting for vblank we could get stuck |
152 | * at (for ex) 30fps at 50% utilization. |
153 | */ |
154 | priv->gpu_devfreq_config.upthreshold = 50; |
155 | priv->gpu_devfreq_config.downdifferential = 10; |
156 | |
157 | mutex_init(&df->lock); |
158 | |
159 | dev_pm_qos_add_request(dev: &gpu->pdev->dev, req: &df->boost_freq, |
160 | type: DEV_PM_QOS_MIN_FREQUENCY, value: 0); |
161 | |
162 | msm_devfreq_profile.initial_freq = gpu->fast_rate; |
163 | |
164 | /* |
165 | * Don't set the freq_table or max_state and let devfreq build the table |
166 | * from OPP |
167 | * After a deferred probe, these may have be left to non-zero values, |
168 | * so set them back to zero before creating the devfreq device |
169 | */ |
170 | msm_devfreq_profile.freq_table = NULL; |
171 | msm_devfreq_profile.max_state = 0; |
172 | |
173 | df->devfreq = devm_devfreq_add_device(dev: &gpu->pdev->dev, |
174 | profile: &msm_devfreq_profile, DEVFREQ_GOV_SIMPLE_ONDEMAND, |
175 | data: &priv->gpu_devfreq_config); |
176 | |
177 | if (IS_ERR(ptr: df->devfreq)) { |
178 | DRM_DEV_ERROR(&gpu->pdev->dev, "Couldn't initialize GPU devfreq\n" ); |
179 | dev_pm_qos_remove_request(req: &df->boost_freq); |
180 | df->devfreq = NULL; |
181 | return; |
182 | } |
183 | |
184 | devfreq_suspend_device(devfreq: df->devfreq); |
185 | |
186 | gpu->cooling = of_devfreq_cooling_register(np: gpu->pdev->dev.of_node, df: df->devfreq); |
187 | if (IS_ERR(ptr: gpu->cooling)) { |
188 | DRM_DEV_ERROR(&gpu->pdev->dev, |
189 | "Couldn't register GPU cooling device\n" ); |
190 | gpu->cooling = NULL; |
191 | } |
192 | |
193 | msm_hrtimer_work_init(work: &df->boost_work, worker: gpu->worker, fn: msm_devfreq_boost_work, |
194 | CLOCK_MONOTONIC, mode: HRTIMER_MODE_REL); |
195 | msm_hrtimer_work_init(work: &df->idle_work, worker: gpu->worker, fn: msm_devfreq_idle_work, |
196 | CLOCK_MONOTONIC, mode: HRTIMER_MODE_REL); |
197 | } |
198 | |
199 | static void cancel_idle_work(struct msm_gpu_devfreq *df) |
200 | { |
201 | hrtimer_cancel(timer: &df->idle_work.timer); |
202 | kthread_cancel_work_sync(work: &df->idle_work.work); |
203 | } |
204 | |
205 | static void cancel_boost_work(struct msm_gpu_devfreq *df) |
206 | { |
207 | hrtimer_cancel(timer: &df->boost_work.timer); |
208 | kthread_cancel_work_sync(work: &df->boost_work.work); |
209 | } |
210 | |
211 | void msm_devfreq_cleanup(struct msm_gpu *gpu) |
212 | { |
213 | struct msm_gpu_devfreq *df = &gpu->devfreq; |
214 | |
215 | if (!has_devfreq(gpu)) |
216 | return; |
217 | |
218 | devfreq_cooling_unregister(dfc: gpu->cooling); |
219 | dev_pm_qos_remove_request(req: &df->boost_freq); |
220 | } |
221 | |
222 | void msm_devfreq_resume(struct msm_gpu *gpu) |
223 | { |
224 | struct msm_gpu_devfreq *df = &gpu->devfreq; |
225 | unsigned long sample_rate; |
226 | |
227 | if (!has_devfreq(gpu)) |
228 | return; |
229 | |
230 | mutex_lock(&df->lock); |
231 | df->busy_cycles = gpu->funcs->gpu_busy(gpu, &sample_rate); |
232 | df->time = ktime_get(); |
233 | df->suspended = false; |
234 | mutex_unlock(lock: &df->lock); |
235 | |
236 | devfreq_resume_device(devfreq: df->devfreq); |
237 | } |
238 | |
239 | void msm_devfreq_suspend(struct msm_gpu *gpu) |
240 | { |
241 | struct msm_gpu_devfreq *df = &gpu->devfreq; |
242 | |
243 | if (!has_devfreq(gpu)) |
244 | return; |
245 | |
246 | mutex_lock(&df->lock); |
247 | df->suspended = true; |
248 | mutex_unlock(lock: &df->lock); |
249 | |
250 | devfreq_suspend_device(devfreq: df->devfreq); |
251 | |
252 | cancel_idle_work(df); |
253 | cancel_boost_work(df); |
254 | } |
255 | |
256 | static void msm_devfreq_boost_work(struct kthread_work *work) |
257 | { |
258 | struct msm_gpu_devfreq *df = container_of(work, |
259 | struct msm_gpu_devfreq, boost_work.work); |
260 | |
261 | dev_pm_qos_update_request(req: &df->boost_freq, new_value: 0); |
262 | } |
263 | |
264 | void msm_devfreq_boost(struct msm_gpu *gpu, unsigned factor) |
265 | { |
266 | struct msm_gpu_devfreq *df = &gpu->devfreq; |
267 | uint64_t freq; |
268 | |
269 | if (!has_devfreq(gpu)) |
270 | return; |
271 | |
272 | freq = get_freq(gpu); |
273 | freq *= factor; |
274 | |
275 | /* |
276 | * A nice little trap is that PM QoS operates in terms of KHz, |
277 | * while devfreq operates in terms of Hz: |
278 | */ |
279 | do_div(freq, HZ_PER_KHZ); |
280 | |
281 | dev_pm_qos_update_request(req: &df->boost_freq, new_value: freq); |
282 | |
283 | msm_hrtimer_queue_work(work: &df->boost_work, |
284 | wakeup_time: ms_to_ktime(ms: msm_devfreq_profile.polling_ms), |
285 | mode: HRTIMER_MODE_REL); |
286 | } |
287 | |
288 | void msm_devfreq_active(struct msm_gpu *gpu) |
289 | { |
290 | struct msm_gpu_devfreq *df = &gpu->devfreq; |
291 | unsigned int idle_time; |
292 | unsigned long target_freq; |
293 | |
294 | if (!has_devfreq(gpu)) |
295 | return; |
296 | |
297 | /* |
298 | * Cancel any pending transition to idle frequency: |
299 | */ |
300 | cancel_idle_work(df); |
301 | |
302 | /* |
303 | * Hold devfreq lock to synchronize with get_dev_status()/ |
304 | * target() callbacks |
305 | */ |
306 | mutex_lock(&df->devfreq->lock); |
307 | |
308 | target_freq = df->idle_freq; |
309 | |
310 | idle_time = ktime_to_ms(ktime_sub(ktime_get(), df->idle_time)); |
311 | |
312 | df->idle_freq = 0; |
313 | |
314 | /* |
315 | * We could have become active again before the idle work had a |
316 | * chance to run, in which case the df->idle_freq would have |
317 | * still been zero. In this case, no need to change freq. |
318 | */ |
319 | if (target_freq) |
320 | msm_devfreq_target(dev: &gpu->pdev->dev, freq: &target_freq, flags: 0); |
321 | |
322 | mutex_unlock(lock: &df->devfreq->lock); |
323 | |
324 | /* |
325 | * If we've been idle for a significant fraction of a polling |
326 | * interval, then we won't meet the threshold of busyness for |
327 | * the governor to ramp up the freq.. so give some boost |
328 | */ |
329 | if (idle_time > msm_devfreq_profile.polling_ms) { |
330 | msm_devfreq_boost(gpu, factor: 2); |
331 | } |
332 | } |
333 | |
334 | |
335 | static void msm_devfreq_idle_work(struct kthread_work *work) |
336 | { |
337 | struct msm_gpu_devfreq *df = container_of(work, |
338 | struct msm_gpu_devfreq, idle_work.work); |
339 | struct msm_gpu *gpu = container_of(df, struct msm_gpu, devfreq); |
340 | struct msm_drm_private *priv = gpu->dev->dev_private; |
341 | unsigned long idle_freq, target_freq = 0; |
342 | |
343 | /* |
344 | * Hold devfreq lock to synchronize with get_dev_status()/ |
345 | * target() callbacks |
346 | */ |
347 | mutex_lock(&df->devfreq->lock); |
348 | |
349 | idle_freq = get_freq(gpu); |
350 | |
351 | if (priv->gpu_clamp_to_idle) |
352 | msm_devfreq_target(dev: &gpu->pdev->dev, freq: &target_freq, flags: 0); |
353 | |
354 | df->idle_time = ktime_get(); |
355 | df->idle_freq = idle_freq; |
356 | |
357 | mutex_unlock(lock: &df->devfreq->lock); |
358 | } |
359 | |
360 | void msm_devfreq_idle(struct msm_gpu *gpu) |
361 | { |
362 | struct msm_gpu_devfreq *df = &gpu->devfreq; |
363 | |
364 | if (!has_devfreq(gpu)) |
365 | return; |
366 | |
367 | msm_hrtimer_queue_work(work: &df->idle_work, wakeup_time: ms_to_ktime(ms: 1), |
368 | mode: HRTIMER_MODE_REL); |
369 | } |
370 | |