1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2013 Red Hat
4 * Author: Rob Clark <robdclark@gmail.com>
5 */
6
7#include "msm_gpu.h"
8#include "msm_gpu_trace.h"
9
10#include <linux/devfreq.h>
11#include <linux/devfreq_cooling.h>
12#include <linux/math64.h>
13#include <linux/units.h>
14
15/*
16 * Power Management:
17 */
18
19static int msm_devfreq_target(struct device *dev, unsigned long *freq,
20 u32 flags)
21{
22 struct msm_gpu *gpu = dev_to_gpu(dev);
23 struct msm_gpu_devfreq *df = &gpu->devfreq;
24 struct dev_pm_opp *opp;
25
26 /*
27 * Note that devfreq_recommended_opp() can modify the freq
28 * to something that actually is in the opp table:
29 */
30 opp = devfreq_recommended_opp(dev, freq, flags);
31 if (IS_ERR(ptr: opp))
32 return PTR_ERR(ptr: opp);
33
34 trace_msm_gpu_freq_change(freq: dev_pm_opp_get_freq(opp));
35
36 /*
37 * If the GPU is idle, devfreq is not aware, so just stash
38 * the new target freq (to use when we return to active)
39 */
40 if (df->idle_freq) {
41 df->idle_freq = *freq;
42 dev_pm_opp_put(opp);
43 return 0;
44 }
45
46 if (gpu->funcs->gpu_set_freq) {
47 mutex_lock(&df->lock);
48 gpu->funcs->gpu_set_freq(gpu, opp, df->suspended);
49 mutex_unlock(lock: &df->lock);
50 } else {
51 dev_pm_opp_set_rate(dev, target_freq: *freq);
52 }
53
54 dev_pm_opp_put(opp);
55
56 return 0;
57}
58
59static unsigned long get_freq(struct msm_gpu *gpu)
60{
61 struct msm_gpu_devfreq *df = &gpu->devfreq;
62
63 /*
64 * If the GPU is idle, use the shadow/saved freq to avoid
65 * confusing devfreq (which is unaware that we are switching
66 * to lowest freq until the device is active again)
67 */
68 if (df->idle_freq)
69 return df->idle_freq;
70
71 if (gpu->funcs->gpu_get_freq)
72 return gpu->funcs->gpu_get_freq(gpu);
73
74 return clk_get_rate(clk: gpu->core_clk);
75}
76
77static int msm_devfreq_get_dev_status(struct device *dev,
78 struct devfreq_dev_status *status)
79{
80 struct msm_gpu *gpu = dev_to_gpu(dev);
81 struct msm_gpu_devfreq *df = &gpu->devfreq;
82 u64 busy_cycles, busy_time;
83 unsigned long sample_rate;
84 ktime_t time;
85
86 mutex_lock(&df->lock);
87
88 status->current_frequency = get_freq(gpu);
89 time = ktime_get();
90 status->total_time = ktime_us_delta(later: time, earlier: df->time);
91 df->time = time;
92
93 if (df->suspended) {
94 mutex_unlock(lock: &df->lock);
95 status->busy_time = 0;
96 return 0;
97 }
98
99 busy_cycles = gpu->funcs->gpu_busy(gpu, &sample_rate);
100 busy_time = busy_cycles - df->busy_cycles;
101 df->busy_cycles = busy_cycles;
102
103 mutex_unlock(lock: &df->lock);
104
105 busy_time *= USEC_PER_SEC;
106 busy_time = div64_ul(busy_time, sample_rate);
107 if (WARN_ON(busy_time > ~0LU))
108 busy_time = ~0LU;
109
110 status->busy_time = busy_time;
111
112 return 0;
113}
114
115static int msm_devfreq_get_cur_freq(struct device *dev, unsigned long *freq)
116{
117 *freq = get_freq(gpu: dev_to_gpu(dev));
118
119 return 0;
120}
121
122static struct devfreq_dev_profile msm_devfreq_profile = {
123 .timer = DEVFREQ_TIMER_DELAYED,
124 .polling_ms = 50,
125 .target = msm_devfreq_target,
126 .get_dev_status = msm_devfreq_get_dev_status,
127 .get_cur_freq = msm_devfreq_get_cur_freq,
128};
129
130static void msm_devfreq_boost_work(struct kthread_work *work);
131static void msm_devfreq_idle_work(struct kthread_work *work);
132
133static bool has_devfreq(struct msm_gpu *gpu)
134{
135 struct msm_gpu_devfreq *df = &gpu->devfreq;
136 return !!df->devfreq;
137}
138
139void msm_devfreq_init(struct msm_gpu *gpu)
140{
141 struct msm_gpu_devfreq *df = &gpu->devfreq;
142 struct msm_drm_private *priv = gpu->dev->dev_private;
143
144 /* We need target support to do devfreq */
145 if (!gpu->funcs->gpu_busy)
146 return;
147
148 /*
149 * Setup default values for simple_ondemand governor tuning. We
150 * want to throttle up at 50% load for the double-buffer case,
151 * where due to stalling waiting for vblank we could get stuck
152 * at (for ex) 30fps at 50% utilization.
153 */
154 priv->gpu_devfreq_config.upthreshold = 50;
155 priv->gpu_devfreq_config.downdifferential = 10;
156
157 mutex_init(&df->lock);
158
159 dev_pm_qos_add_request(dev: &gpu->pdev->dev, req: &df->boost_freq,
160 type: DEV_PM_QOS_MIN_FREQUENCY, value: 0);
161
162 msm_devfreq_profile.initial_freq = gpu->fast_rate;
163
164 /*
165 * Don't set the freq_table or max_state and let devfreq build the table
166 * from OPP
167 * After a deferred probe, these may have be left to non-zero values,
168 * so set them back to zero before creating the devfreq device
169 */
170 msm_devfreq_profile.freq_table = NULL;
171 msm_devfreq_profile.max_state = 0;
172
173 df->devfreq = devm_devfreq_add_device(dev: &gpu->pdev->dev,
174 profile: &msm_devfreq_profile, DEVFREQ_GOV_SIMPLE_ONDEMAND,
175 data: &priv->gpu_devfreq_config);
176
177 if (IS_ERR(ptr: df->devfreq)) {
178 DRM_DEV_ERROR(&gpu->pdev->dev, "Couldn't initialize GPU devfreq\n");
179 dev_pm_qos_remove_request(req: &df->boost_freq);
180 df->devfreq = NULL;
181 return;
182 }
183
184 devfreq_suspend_device(devfreq: df->devfreq);
185
186 gpu->cooling = of_devfreq_cooling_register(np: gpu->pdev->dev.of_node, df: df->devfreq);
187 if (IS_ERR(ptr: gpu->cooling)) {
188 DRM_DEV_ERROR(&gpu->pdev->dev,
189 "Couldn't register GPU cooling device\n");
190 gpu->cooling = NULL;
191 }
192
193 msm_hrtimer_work_init(work: &df->boost_work, worker: gpu->worker, fn: msm_devfreq_boost_work,
194 CLOCK_MONOTONIC, mode: HRTIMER_MODE_REL);
195 msm_hrtimer_work_init(work: &df->idle_work, worker: gpu->worker, fn: msm_devfreq_idle_work,
196 CLOCK_MONOTONIC, mode: HRTIMER_MODE_REL);
197}
198
199static void cancel_idle_work(struct msm_gpu_devfreq *df)
200{
201 hrtimer_cancel(timer: &df->idle_work.timer);
202 kthread_cancel_work_sync(work: &df->idle_work.work);
203}
204
205static void cancel_boost_work(struct msm_gpu_devfreq *df)
206{
207 hrtimer_cancel(timer: &df->boost_work.timer);
208 kthread_cancel_work_sync(work: &df->boost_work.work);
209}
210
211void msm_devfreq_cleanup(struct msm_gpu *gpu)
212{
213 struct msm_gpu_devfreq *df = &gpu->devfreq;
214
215 if (!has_devfreq(gpu))
216 return;
217
218 devfreq_cooling_unregister(dfc: gpu->cooling);
219 dev_pm_qos_remove_request(req: &df->boost_freq);
220}
221
222void msm_devfreq_resume(struct msm_gpu *gpu)
223{
224 struct msm_gpu_devfreq *df = &gpu->devfreq;
225 unsigned long sample_rate;
226
227 if (!has_devfreq(gpu))
228 return;
229
230 mutex_lock(&df->lock);
231 df->busy_cycles = gpu->funcs->gpu_busy(gpu, &sample_rate);
232 df->time = ktime_get();
233 df->suspended = false;
234 mutex_unlock(lock: &df->lock);
235
236 devfreq_resume_device(devfreq: df->devfreq);
237}
238
239void msm_devfreq_suspend(struct msm_gpu *gpu)
240{
241 struct msm_gpu_devfreq *df = &gpu->devfreq;
242
243 if (!has_devfreq(gpu))
244 return;
245
246 mutex_lock(&df->lock);
247 df->suspended = true;
248 mutex_unlock(lock: &df->lock);
249
250 devfreq_suspend_device(devfreq: df->devfreq);
251
252 cancel_idle_work(df);
253 cancel_boost_work(df);
254}
255
256static void msm_devfreq_boost_work(struct kthread_work *work)
257{
258 struct msm_gpu_devfreq *df = container_of(work,
259 struct msm_gpu_devfreq, boost_work.work);
260
261 dev_pm_qos_update_request(req: &df->boost_freq, new_value: 0);
262}
263
264void msm_devfreq_boost(struct msm_gpu *gpu, unsigned factor)
265{
266 struct msm_gpu_devfreq *df = &gpu->devfreq;
267 uint64_t freq;
268
269 if (!has_devfreq(gpu))
270 return;
271
272 freq = get_freq(gpu);
273 freq *= factor;
274
275 /*
276 * A nice little trap is that PM QoS operates in terms of KHz,
277 * while devfreq operates in terms of Hz:
278 */
279 do_div(freq, HZ_PER_KHZ);
280
281 dev_pm_qos_update_request(req: &df->boost_freq, new_value: freq);
282
283 msm_hrtimer_queue_work(work: &df->boost_work,
284 wakeup_time: ms_to_ktime(ms: msm_devfreq_profile.polling_ms),
285 mode: HRTIMER_MODE_REL);
286}
287
288void msm_devfreq_active(struct msm_gpu *gpu)
289{
290 struct msm_gpu_devfreq *df = &gpu->devfreq;
291 unsigned int idle_time;
292 unsigned long target_freq;
293
294 if (!has_devfreq(gpu))
295 return;
296
297 /*
298 * Cancel any pending transition to idle frequency:
299 */
300 cancel_idle_work(df);
301
302 /*
303 * Hold devfreq lock to synchronize with get_dev_status()/
304 * target() callbacks
305 */
306 mutex_lock(&df->devfreq->lock);
307
308 target_freq = df->idle_freq;
309
310 idle_time = ktime_to_ms(ktime_sub(ktime_get(), df->idle_time));
311
312 df->idle_freq = 0;
313
314 /*
315 * We could have become active again before the idle work had a
316 * chance to run, in which case the df->idle_freq would have
317 * still been zero. In this case, no need to change freq.
318 */
319 if (target_freq)
320 msm_devfreq_target(dev: &gpu->pdev->dev, freq: &target_freq, flags: 0);
321
322 mutex_unlock(lock: &df->devfreq->lock);
323
324 /*
325 * If we've been idle for a significant fraction of a polling
326 * interval, then we won't meet the threshold of busyness for
327 * the governor to ramp up the freq.. so give some boost
328 */
329 if (idle_time > msm_devfreq_profile.polling_ms) {
330 msm_devfreq_boost(gpu, factor: 2);
331 }
332}
333
334
335static void msm_devfreq_idle_work(struct kthread_work *work)
336{
337 struct msm_gpu_devfreq *df = container_of(work,
338 struct msm_gpu_devfreq, idle_work.work);
339 struct msm_gpu *gpu = container_of(df, struct msm_gpu, devfreq);
340 struct msm_drm_private *priv = gpu->dev->dev_private;
341 unsigned long idle_freq, target_freq = 0;
342
343 /*
344 * Hold devfreq lock to synchronize with get_dev_status()/
345 * target() callbacks
346 */
347 mutex_lock(&df->devfreq->lock);
348
349 idle_freq = get_freq(gpu);
350
351 if (priv->gpu_clamp_to_idle)
352 msm_devfreq_target(dev: &gpu->pdev->dev, freq: &target_freq, flags: 0);
353
354 df->idle_time = ktime_get();
355 df->idle_freq = idle_freq;
356
357 mutex_unlock(lock: &df->devfreq->lock);
358}
359
360void msm_devfreq_idle(struct msm_gpu *gpu)
361{
362 struct msm_gpu_devfreq *df = &gpu->devfreq;
363
364 if (!has_devfreq(gpu))
365 return;
366
367 msm_hrtimer_queue_work(work: &df->idle_work, wakeup_time: ms_to_ktime(ms: 1),
368 mode: HRTIMER_MODE_REL);
369}
370

source code of linux/drivers/gpu/drm/msm/msm_gpu_devfreq.c