1 | /* |
2 | * Copyright © 2012-2014 Intel Corporation |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
5 | * copy of this software and associated documentation files (the "Software"), |
6 | * to deal in the Software without restriction, including without limitation |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
8 | * and/or sell copies of the Software, and to permit persons to whom the |
9 | * Software is furnished to do so, subject to the following conditions: |
10 | * |
11 | * The above copyright notice and this permission notice (including the next |
12 | * paragraph) shall be included in all copies or substantial portions of the |
13 | * Software. |
14 | * |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
21 | * IN THE SOFTWARE. |
22 | * |
23 | * Authors: |
24 | * Eugeni Dodonov <eugeni.dodonov@intel.com> |
25 | * Daniel Vetter <daniel.vetter@ffwll.ch> |
26 | * |
27 | */ |
28 | |
29 | #include <linux/pm_runtime.h> |
30 | |
31 | #include <drm/drm_print.h> |
32 | |
33 | #include "i915_drv.h" |
34 | #include "i915_trace.h" |
35 | |
36 | /** |
37 | * DOC: runtime pm |
38 | * |
39 | * The i915 driver supports dynamic enabling and disabling of entire hardware |
40 | * blocks at runtime. This is especially important on the display side where |
41 | * software is supposed to control many power gates manually on recent hardware, |
42 | * since on the GT side a lot of the power management is done by the hardware. |
43 | * But even there some manual control at the device level is required. |
44 | * |
45 | * Since i915 supports a diverse set of platforms with a unified codebase and |
46 | * hardware engineers just love to shuffle functionality around between power |
47 | * domains there's a sizeable amount of indirection required. This file provides |
48 | * generic functions to the driver for grabbing and releasing references for |
49 | * abstract power domains. It then maps those to the actual power wells |
50 | * present for a given platform. |
51 | */ |
52 | |
53 | static struct drm_i915_private *rpm_to_i915(struct intel_runtime_pm *rpm) |
54 | { |
55 | return container_of(rpm, struct drm_i915_private, runtime_pm); |
56 | } |
57 | |
58 | #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM) |
59 | |
60 | static void init_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm) |
61 | { |
62 | ref_tracker_dir_init(dir: &rpm->debug, INTEL_REFTRACK_DEAD_COUNT, name: dev_name(dev: rpm->kdev)); |
63 | } |
64 | |
65 | static intel_wakeref_t |
66 | track_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm) |
67 | { |
68 | if (!rpm->available || rpm->no_wakeref_tracking) |
69 | return -1; |
70 | |
71 | return intel_ref_tracker_alloc(dir: &rpm->debug); |
72 | } |
73 | |
74 | static void untrack_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm, |
75 | intel_wakeref_t wakeref) |
76 | { |
77 | if (!rpm->available || rpm->no_wakeref_tracking) |
78 | return; |
79 | |
80 | intel_ref_tracker_free(dir: &rpm->debug, handle: wakeref); |
81 | } |
82 | |
83 | static void untrack_all_intel_runtime_pm_wakerefs(struct intel_runtime_pm *rpm) |
84 | { |
85 | ref_tracker_dir_exit(dir: &rpm->debug); |
86 | } |
87 | |
88 | static noinline void |
89 | __intel_wakeref_dec_and_check_tracking(struct intel_runtime_pm *rpm) |
90 | { |
91 | unsigned long flags; |
92 | |
93 | if (!atomic_dec_and_lock_irqsave(&rpm->wakeref_count, |
94 | &rpm->debug.lock, |
95 | flags)) |
96 | return; |
97 | |
98 | ref_tracker_dir_print_locked(dir: &rpm->debug, INTEL_REFTRACK_PRINT_LIMIT); |
99 | spin_unlock_irqrestore(lock: &rpm->debug.lock, flags); |
100 | } |
101 | |
102 | void print_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm, |
103 | struct drm_printer *p) |
104 | { |
105 | intel_ref_tracker_show(dir: &rpm->debug, p); |
106 | } |
107 | |
108 | #else |
109 | |
110 | static void init_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm) |
111 | { |
112 | } |
113 | |
114 | static intel_wakeref_t |
115 | track_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm) |
116 | { |
117 | return -1; |
118 | } |
119 | |
120 | static void untrack_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm, |
121 | intel_wakeref_t wakeref) |
122 | { |
123 | } |
124 | |
125 | static void |
126 | __intel_wakeref_dec_and_check_tracking(struct intel_runtime_pm *rpm) |
127 | { |
128 | atomic_dec(&rpm->wakeref_count); |
129 | } |
130 | |
131 | static void |
132 | untrack_all_intel_runtime_pm_wakerefs(struct intel_runtime_pm *rpm) |
133 | { |
134 | } |
135 | |
136 | #endif |
137 | |
138 | static void |
139 | intel_runtime_pm_acquire(struct intel_runtime_pm *rpm, bool wakelock) |
140 | { |
141 | if (wakelock) { |
142 | atomic_add(i: 1 + INTEL_RPM_WAKELOCK_BIAS, v: &rpm->wakeref_count); |
143 | assert_rpm_wakelock_held(rpm); |
144 | } else { |
145 | atomic_inc(v: &rpm->wakeref_count); |
146 | assert_rpm_raw_wakeref_held(rpm); |
147 | } |
148 | } |
149 | |
150 | static void |
151 | intel_runtime_pm_release(struct intel_runtime_pm *rpm, int wakelock) |
152 | { |
153 | if (wakelock) { |
154 | assert_rpm_wakelock_held(rpm); |
155 | atomic_sub(INTEL_RPM_WAKELOCK_BIAS, v: &rpm->wakeref_count); |
156 | } else { |
157 | assert_rpm_raw_wakeref_held(rpm); |
158 | } |
159 | |
160 | __intel_wakeref_dec_and_check_tracking(rpm); |
161 | } |
162 | |
163 | static intel_wakeref_t __intel_runtime_pm_get(struct intel_runtime_pm *rpm, |
164 | bool wakelock) |
165 | { |
166 | struct drm_i915_private *i915 = rpm_to_i915(rpm); |
167 | int ret; |
168 | |
169 | ret = pm_runtime_get_sync(dev: rpm->kdev); |
170 | drm_WARN_ONCE(&i915->drm, ret < 0, |
171 | "pm_runtime_get_sync() failed: %d\n" , ret); |
172 | |
173 | intel_runtime_pm_acquire(rpm, wakelock); |
174 | |
175 | return track_intel_runtime_pm_wakeref(rpm); |
176 | } |
177 | |
178 | /** |
179 | * intel_runtime_pm_get_raw - grab a raw runtime pm reference |
180 | * @rpm: the intel_runtime_pm structure |
181 | * |
182 | * This is the unlocked version of intel_display_power_is_enabled() and should |
183 | * only be used from error capture and recovery code where deadlocks are |
184 | * possible. |
185 | * This function grabs a device-level runtime pm reference (mostly used for |
186 | * asynchronous PM management from display code) and ensures that it is powered |
187 | * up. Raw references are not considered during wakelock assert checks. |
188 | * |
189 | * Any runtime pm reference obtained by this function must have a symmetric |
190 | * call to intel_runtime_pm_put_raw() to release the reference again. |
191 | * |
192 | * Returns: the wakeref cookie to pass to intel_runtime_pm_put_raw(), evaluates |
193 | * as True if the wakeref was acquired, or False otherwise. |
194 | */ |
195 | intel_wakeref_t intel_runtime_pm_get_raw(struct intel_runtime_pm *rpm) |
196 | { |
197 | return __intel_runtime_pm_get(rpm, wakelock: false); |
198 | } |
199 | |
200 | /** |
201 | * intel_runtime_pm_get - grab a runtime pm reference |
202 | * @rpm: the intel_runtime_pm structure |
203 | * |
204 | * This function grabs a device-level runtime pm reference (mostly used for GEM |
205 | * code to ensure the GTT or GT is on) and ensures that it is powered up. |
206 | * |
207 | * Any runtime pm reference obtained by this function must have a symmetric |
208 | * call to intel_runtime_pm_put() to release the reference again. |
209 | * |
210 | * Returns: the wakeref cookie to pass to intel_runtime_pm_put() |
211 | */ |
212 | intel_wakeref_t intel_runtime_pm_get(struct intel_runtime_pm *rpm) |
213 | { |
214 | return __intel_runtime_pm_get(rpm, wakelock: true); |
215 | } |
216 | |
217 | /** |
218 | * __intel_runtime_pm_get_if_active - grab a runtime pm reference if device is active |
219 | * @rpm: the intel_runtime_pm structure |
220 | * @ignore_usecount: get a ref even if dev->power.usage_count is 0 |
221 | * |
222 | * This function grabs a device-level runtime pm reference if the device is |
223 | * already active and ensures that it is powered up. It is illegal to try |
224 | * and access the HW should intel_runtime_pm_get_if_active() report failure. |
225 | * |
226 | * If @ignore_usecount is true, a reference will be acquired even if there is no |
227 | * user requiring the device to be powered up (dev->power.usage_count == 0). |
228 | * If the function returns false in this case then it's guaranteed that the |
229 | * device's runtime suspend hook has been called already or that it will be |
230 | * called (and hence it's also guaranteed that the device's runtime resume |
231 | * hook will be called eventually). |
232 | * |
233 | * Any runtime pm reference obtained by this function must have a symmetric |
234 | * call to intel_runtime_pm_put() to release the reference again. |
235 | * |
236 | * Returns: the wakeref cookie to pass to intel_runtime_pm_put(), evaluates |
237 | * as True if the wakeref was acquired, or False otherwise. |
238 | */ |
239 | static intel_wakeref_t __intel_runtime_pm_get_if_active(struct intel_runtime_pm *rpm, |
240 | bool ignore_usecount) |
241 | { |
242 | if (IS_ENABLED(CONFIG_PM)) { |
243 | /* |
244 | * In cases runtime PM is disabled by the RPM core and we get |
245 | * an -EINVAL return value we are not supposed to call this |
246 | * function, since the power state is undefined. This applies |
247 | * atm to the late/early system suspend/resume handlers. |
248 | */ |
249 | if ((ignore_usecount && |
250 | pm_runtime_get_if_active(dev: rpm->kdev) <= 0) || |
251 | (!ignore_usecount && |
252 | pm_runtime_get_if_in_use(dev: rpm->kdev) <= 0)) |
253 | return 0; |
254 | } |
255 | |
256 | intel_runtime_pm_acquire(rpm, wakelock: true); |
257 | |
258 | return track_intel_runtime_pm_wakeref(rpm); |
259 | } |
260 | |
261 | intel_wakeref_t intel_runtime_pm_get_if_in_use(struct intel_runtime_pm *rpm) |
262 | { |
263 | return __intel_runtime_pm_get_if_active(rpm, ignore_usecount: false); |
264 | } |
265 | |
266 | intel_wakeref_t intel_runtime_pm_get_if_active(struct intel_runtime_pm *rpm) |
267 | { |
268 | return __intel_runtime_pm_get_if_active(rpm, ignore_usecount: true); |
269 | } |
270 | |
271 | /** |
272 | * intel_runtime_pm_get_noresume - grab a runtime pm reference |
273 | * @rpm: the intel_runtime_pm structure |
274 | * |
275 | * This function grabs a device-level runtime pm reference (mostly used for GEM |
276 | * code to ensure the GTT or GT is on). |
277 | * |
278 | * It will _not_ power up the device but instead only check that it's powered |
279 | * on. Therefore it is only valid to call this functions from contexts where |
280 | * the device is known to be powered up and where trying to power it up would |
281 | * result in hilarity and deadlocks. That pretty much means only the system |
282 | * suspend/resume code where this is used to grab runtime pm references for |
283 | * delayed setup down in work items. |
284 | * |
285 | * Any runtime pm reference obtained by this function must have a symmetric |
286 | * call to intel_runtime_pm_put() to release the reference again. |
287 | * |
288 | * Returns: the wakeref cookie to pass to intel_runtime_pm_put() |
289 | */ |
290 | intel_wakeref_t intel_runtime_pm_get_noresume(struct intel_runtime_pm *rpm) |
291 | { |
292 | assert_rpm_wakelock_held(rpm); |
293 | pm_runtime_get_noresume(dev: rpm->kdev); |
294 | |
295 | intel_runtime_pm_acquire(rpm, wakelock: true); |
296 | |
297 | return track_intel_runtime_pm_wakeref(rpm); |
298 | } |
299 | |
300 | static void __intel_runtime_pm_put(struct intel_runtime_pm *rpm, |
301 | intel_wakeref_t wref, |
302 | bool wakelock) |
303 | { |
304 | struct device *kdev = rpm->kdev; |
305 | |
306 | untrack_intel_runtime_pm_wakeref(rpm, wakeref: wref); |
307 | |
308 | intel_runtime_pm_release(rpm, wakelock); |
309 | |
310 | pm_runtime_mark_last_busy(dev: kdev); |
311 | pm_runtime_put_autosuspend(dev: kdev); |
312 | } |
313 | |
314 | /** |
315 | * intel_runtime_pm_put_raw - release a raw runtime pm reference |
316 | * @rpm: the intel_runtime_pm structure |
317 | * @wref: wakeref acquired for the reference that is being released |
318 | * |
319 | * This function drops the device-level runtime pm reference obtained by |
320 | * intel_runtime_pm_get_raw() and might power down the corresponding |
321 | * hardware block right away if this is the last reference. |
322 | */ |
323 | void |
324 | intel_runtime_pm_put_raw(struct intel_runtime_pm *rpm, intel_wakeref_t wref) |
325 | { |
326 | __intel_runtime_pm_put(rpm, wref, wakelock: false); |
327 | } |
328 | |
329 | /** |
330 | * intel_runtime_pm_put_unchecked - release an unchecked runtime pm reference |
331 | * @rpm: the intel_runtime_pm structure |
332 | * |
333 | * This function drops the device-level runtime pm reference obtained by |
334 | * intel_runtime_pm_get() and might power down the corresponding |
335 | * hardware block right away if this is the last reference. |
336 | * |
337 | * This function exists only for historical reasons and should be avoided in |
338 | * new code, as the correctness of its use cannot be checked. Always use |
339 | * intel_runtime_pm_put() instead. |
340 | */ |
341 | void intel_runtime_pm_put_unchecked(struct intel_runtime_pm *rpm) |
342 | { |
343 | __intel_runtime_pm_put(rpm, wref: -1, wakelock: true); |
344 | } |
345 | |
346 | #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM) |
347 | /** |
348 | * intel_runtime_pm_put - release a runtime pm reference |
349 | * @rpm: the intel_runtime_pm structure |
350 | * @wref: wakeref acquired for the reference that is being released |
351 | * |
352 | * This function drops the device-level runtime pm reference obtained by |
353 | * intel_runtime_pm_get() and might power down the corresponding |
354 | * hardware block right away if this is the last reference. |
355 | */ |
356 | void intel_runtime_pm_put(struct intel_runtime_pm *rpm, intel_wakeref_t wref) |
357 | { |
358 | __intel_runtime_pm_put(rpm, wref, wakelock: true); |
359 | } |
360 | #endif |
361 | |
362 | /** |
363 | * intel_runtime_pm_enable - enable runtime pm |
364 | * @rpm: the intel_runtime_pm structure |
365 | * |
366 | * This function enables runtime pm at the end of the driver load sequence. |
367 | * |
368 | * Note that this function does currently not enable runtime pm for the |
369 | * subordinate display power domains. That is done by |
370 | * intel_power_domains_enable(). |
371 | */ |
372 | void intel_runtime_pm_enable(struct intel_runtime_pm *rpm) |
373 | { |
374 | struct drm_i915_private *i915 = rpm_to_i915(rpm); |
375 | struct device *kdev = rpm->kdev; |
376 | |
377 | /* |
378 | * Disable the system suspend direct complete optimization, which can |
379 | * leave the device suspended skipping the driver's suspend handlers |
380 | * if the device was already runtime suspended. This is needed due to |
381 | * the difference in our runtime and system suspend sequence and |
382 | * becaue the HDA driver may require us to enable the audio power |
383 | * domain during system suspend. |
384 | */ |
385 | dev_pm_set_driver_flags(dev: kdev, DPM_FLAG_NO_DIRECT_COMPLETE); |
386 | |
387 | pm_runtime_set_autosuspend_delay(dev: kdev, delay: 10000); /* 10s */ |
388 | pm_runtime_mark_last_busy(dev: kdev); |
389 | |
390 | /* |
391 | * Take a permanent reference to disable the RPM functionality and drop |
392 | * it only when unloading the driver. Use the low level get/put helpers, |
393 | * so the driver's own RPM reference tracking asserts also work on |
394 | * platforms without RPM support. |
395 | */ |
396 | if (!rpm->available) { |
397 | int ret; |
398 | |
399 | pm_runtime_dont_use_autosuspend(dev: kdev); |
400 | ret = pm_runtime_get_sync(dev: kdev); |
401 | drm_WARN(&i915->drm, ret < 0, |
402 | "pm_runtime_get_sync() failed: %d\n" , ret); |
403 | } else { |
404 | pm_runtime_use_autosuspend(dev: kdev); |
405 | } |
406 | |
407 | /* |
408 | * FIXME: Temp hammer to keep autosupend disable on lmem supported platforms. |
409 | * As per PCIe specs 5.3.1.4.1, all iomem read write request over a PCIe |
410 | * function will be unsupported in case PCIe endpoint function is in D3. |
411 | * Let's keep i915 autosuspend control 'on' till we fix all known issue |
412 | * with lmem access in D3. |
413 | */ |
414 | if (!IS_DGFX(i915)) |
415 | pm_runtime_allow(dev: kdev); |
416 | |
417 | /* |
418 | * The core calls the driver load handler with an RPM reference held. |
419 | * We drop that here and will reacquire it during unloading in |
420 | * intel_power_domains_fini(). |
421 | */ |
422 | pm_runtime_put_autosuspend(dev: kdev); |
423 | } |
424 | |
425 | void intel_runtime_pm_disable(struct intel_runtime_pm *rpm) |
426 | { |
427 | struct drm_i915_private *i915 = rpm_to_i915(rpm); |
428 | struct device *kdev = rpm->kdev; |
429 | |
430 | /* Transfer rpm ownership back to core */ |
431 | drm_WARN(&i915->drm, pm_runtime_get_sync(kdev) < 0, |
432 | "Failed to pass rpm ownership back to core\n" ); |
433 | |
434 | pm_runtime_dont_use_autosuspend(dev: kdev); |
435 | |
436 | if (!rpm->available) |
437 | pm_runtime_put(dev: kdev); |
438 | } |
439 | |
440 | void intel_runtime_pm_driver_release(struct intel_runtime_pm *rpm) |
441 | { |
442 | struct drm_i915_private *i915 = rpm_to_i915(rpm); |
443 | int count = atomic_read(v: &rpm->wakeref_count); |
444 | |
445 | intel_wakeref_auto_fini(wf: &rpm->userfault_wakeref); |
446 | |
447 | drm_WARN(&i915->drm, count, |
448 | "i915 raw-wakerefs=%d wakelocks=%d on cleanup\n" , |
449 | intel_rpm_raw_wakeref_count(count), |
450 | intel_rpm_wakelock_count(count)); |
451 | } |
452 | |
453 | void intel_runtime_pm_driver_last_release(struct intel_runtime_pm *rpm) |
454 | { |
455 | intel_runtime_pm_driver_release(rpm); |
456 | untrack_all_intel_runtime_pm_wakerefs(rpm); |
457 | } |
458 | |
459 | void intel_runtime_pm_init_early(struct intel_runtime_pm *rpm) |
460 | { |
461 | struct drm_i915_private *i915 = rpm_to_i915(rpm); |
462 | struct pci_dev *pdev = to_pci_dev(i915->drm.dev); |
463 | struct device *kdev = &pdev->dev; |
464 | |
465 | rpm->kdev = kdev; |
466 | rpm->available = HAS_RUNTIME_PM(i915); |
467 | atomic_set(v: &rpm->wakeref_count, i: 0); |
468 | |
469 | init_intel_runtime_pm_wakeref(rpm); |
470 | INIT_LIST_HEAD(list: &rpm->lmem_userfault_list); |
471 | spin_lock_init(&rpm->lmem_userfault_lock); |
472 | intel_wakeref_auto_init(wf: &rpm->userfault_wakeref, i915); |
473 | } |
474 | |