1 | /* |
2 | * SPDX-License-Identifier: MIT |
3 | * |
4 | * Copyright © 2019 Intel Corporation |
5 | */ |
6 | |
7 | #include <linux/wait_bit.h> |
8 | |
9 | #include "intel_runtime_pm.h" |
10 | #include "intel_wakeref.h" |
11 | #include "i915_drv.h" |
12 | |
13 | int __intel_wakeref_get_first(struct intel_wakeref *wf) |
14 | { |
15 | intel_wakeref_t wakeref; |
16 | int ret = 0; |
17 | |
18 | wakeref = intel_runtime_pm_get(rpm: &wf->i915->runtime_pm); |
19 | /* |
20 | * Treat get/put as different subclasses, as we may need to run |
21 | * the put callback from under the shrinker and do not want to |
22 | * cross-contanimate that callback with any extra work performed |
23 | * upon acquiring the wakeref. |
24 | */ |
25 | mutex_lock_nested(lock: &wf->mutex, SINGLE_DEPTH_NESTING); |
26 | |
27 | if (!atomic_read(v: &wf->count)) { |
28 | INTEL_WAKEREF_BUG_ON(wf->wakeref); |
29 | wf->wakeref = wakeref; |
30 | wakeref = 0; |
31 | |
32 | ret = wf->ops->get(wf); |
33 | if (ret) { |
34 | wakeref = xchg(&wf->wakeref, 0); |
35 | wake_up_var(var: &wf->wakeref); |
36 | goto unlock; |
37 | } |
38 | |
39 | smp_mb__before_atomic(); /* release wf->count */ |
40 | } |
41 | |
42 | atomic_inc(v: &wf->count); |
43 | INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0); |
44 | |
45 | unlock: |
46 | mutex_unlock(lock: &wf->mutex); |
47 | if (unlikely(wakeref)) |
48 | intel_runtime_pm_put(rpm: &wf->i915->runtime_pm, wref: wakeref); |
49 | |
50 | return ret; |
51 | } |
52 | |
53 | static void ____intel_wakeref_put_last(struct intel_wakeref *wf) |
54 | { |
55 | intel_wakeref_t wakeref = 0; |
56 | |
57 | INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0); |
58 | if (unlikely(!atomic_dec_and_test(&wf->count))) |
59 | goto unlock; |
60 | |
61 | /* ops->put() must reschedule its own release on error/deferral */ |
62 | if (likely(!wf->ops->put(wf))) { |
63 | INTEL_WAKEREF_BUG_ON(!wf->wakeref); |
64 | wakeref = xchg(&wf->wakeref, 0); |
65 | wake_up_var(var: &wf->wakeref); |
66 | } |
67 | |
68 | unlock: |
69 | mutex_unlock(lock: &wf->mutex); |
70 | if (wakeref) |
71 | intel_runtime_pm_put(rpm: &wf->i915->runtime_pm, wref: wakeref); |
72 | } |
73 | |
74 | void __intel_wakeref_put_last(struct intel_wakeref *wf, unsigned long flags) |
75 | { |
76 | INTEL_WAKEREF_BUG_ON(delayed_work_pending(&wf->work)); |
77 | |
78 | /* Assume we are not in process context and so cannot sleep. */ |
79 | if (flags & INTEL_WAKEREF_PUT_ASYNC || !mutex_trylock(lock: &wf->mutex)) { |
80 | mod_delayed_work(wq: wf->i915->unordered_wq, dwork: &wf->work, |
81 | FIELD_GET(INTEL_WAKEREF_PUT_DELAY, flags)); |
82 | return; |
83 | } |
84 | |
85 | ____intel_wakeref_put_last(wf); |
86 | } |
87 | |
88 | static void __intel_wakeref_put_work(struct work_struct *wrk) |
89 | { |
90 | struct intel_wakeref *wf = container_of(wrk, typeof(*wf), work.work); |
91 | |
92 | if (atomic_add_unless(v: &wf->count, a: -1, u: 1)) |
93 | return; |
94 | |
95 | mutex_lock(&wf->mutex); |
96 | ____intel_wakeref_put_last(wf); |
97 | } |
98 | |
99 | void __intel_wakeref_init(struct intel_wakeref *wf, |
100 | struct drm_i915_private *i915, |
101 | const struct intel_wakeref_ops *ops, |
102 | struct intel_wakeref_lockclass *key, |
103 | const char *name) |
104 | { |
105 | wf->i915 = i915; |
106 | wf->ops = ops; |
107 | |
108 | __mutex_init(lock: &wf->mutex, name: "wakeref.mutex" , key: &key->mutex); |
109 | atomic_set(v: &wf->count, i: 0); |
110 | wf->wakeref = 0; |
111 | |
112 | INIT_DELAYED_WORK(&wf->work, __intel_wakeref_put_work); |
113 | lockdep_init_map(lock: &wf->work.work.lockdep_map, |
114 | name: "wakeref.work" , key: &key->work, subclass: 0); |
115 | |
116 | #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_WAKEREF) |
117 | ref_tracker_dir_init(dir: &wf->debug, INTEL_REFTRACK_DEAD_COUNT, name); |
118 | #endif |
119 | } |
120 | |
121 | int intel_wakeref_wait_for_idle(struct intel_wakeref *wf) |
122 | { |
123 | int err; |
124 | |
125 | might_sleep(); |
126 | |
127 | err = wait_var_event_killable(&wf->wakeref, |
128 | !intel_wakeref_is_active(wf)); |
129 | if (err) |
130 | return err; |
131 | |
132 | intel_wakeref_unlock_wait(wf); |
133 | return 0; |
134 | } |
135 | |
136 | static void wakeref_auto_timeout(struct timer_list *t) |
137 | { |
138 | struct intel_wakeref_auto *wf = from_timer(wf, t, timer); |
139 | intel_wakeref_t wakeref; |
140 | unsigned long flags; |
141 | |
142 | if (!refcount_dec_and_lock_irqsave(r: &wf->count, lock: &wf->lock, flags: &flags)) |
143 | return; |
144 | |
145 | wakeref = fetch_and_zero(&wf->wakeref); |
146 | spin_unlock_irqrestore(lock: &wf->lock, flags); |
147 | |
148 | intel_runtime_pm_put(rpm: &wf->i915->runtime_pm, wref: wakeref); |
149 | } |
150 | |
151 | void intel_wakeref_auto_init(struct intel_wakeref_auto *wf, |
152 | struct drm_i915_private *i915) |
153 | { |
154 | spin_lock_init(&wf->lock); |
155 | timer_setup(&wf->timer, wakeref_auto_timeout, 0); |
156 | refcount_set(r: &wf->count, n: 0); |
157 | wf->wakeref = 0; |
158 | wf->i915 = i915; |
159 | } |
160 | |
161 | void intel_wakeref_auto(struct intel_wakeref_auto *wf, unsigned long timeout) |
162 | { |
163 | unsigned long flags; |
164 | |
165 | if (!timeout) { |
166 | if (del_timer_sync(timer: &wf->timer)) |
167 | wakeref_auto_timeout(t: &wf->timer); |
168 | return; |
169 | } |
170 | |
171 | /* Our mission is that we only extend an already active wakeref */ |
172 | assert_rpm_wakelock_held(rpm: &wf->i915->runtime_pm); |
173 | |
174 | if (!refcount_inc_not_zero(r: &wf->count)) { |
175 | spin_lock_irqsave(&wf->lock, flags); |
176 | if (!refcount_inc_not_zero(r: &wf->count)) { |
177 | INTEL_WAKEREF_BUG_ON(wf->wakeref); |
178 | wf->wakeref = |
179 | intel_runtime_pm_get_if_in_use(rpm: &wf->i915->runtime_pm); |
180 | refcount_set(r: &wf->count, n: 1); |
181 | } |
182 | spin_unlock_irqrestore(lock: &wf->lock, flags); |
183 | } |
184 | |
185 | /* |
186 | * If we extend a pending timer, we will only get a single timer |
187 | * callback and so need to cancel the local inc by running the |
188 | * elided callback to keep the wf->count balanced. |
189 | */ |
190 | if (mod_timer(timer: &wf->timer, expires: jiffies + timeout)) |
191 | wakeref_auto_timeout(t: &wf->timer); |
192 | } |
193 | |
194 | void intel_wakeref_auto_fini(struct intel_wakeref_auto *wf) |
195 | { |
196 | intel_wakeref_auto(wf, timeout: 0); |
197 | INTEL_WAKEREF_BUG_ON(wf->wakeref); |
198 | } |
199 | |
200 | void intel_ref_tracker_show(struct ref_tracker_dir *dir, |
201 | struct drm_printer *p) |
202 | { |
203 | const size_t buf_size = PAGE_SIZE; |
204 | char *buf, *sb, *se; |
205 | size_t count; |
206 | |
207 | buf = kmalloc(size: buf_size, GFP_NOWAIT); |
208 | if (!buf) |
209 | return; |
210 | |
211 | count = ref_tracker_dir_snprint(dir, buf, size: buf_size); |
212 | if (!count) |
213 | goto free; |
214 | /* printk does not like big buffers, so we split it */ |
215 | for (sb = buf; *sb; sb = se + 1) { |
216 | se = strchrnul(sb, '\n'); |
217 | drm_printf(p, f: "%.*s" , (int)(se - sb + 1), sb); |
218 | if (!*se) |
219 | break; |
220 | } |
221 | if (count >= buf_size) |
222 | drm_printf(p, f: "\n...dropped %zd extra bytes of leak report.\n" , |
223 | count + 1 - buf_size); |
224 | free: |
225 | kfree(objp: buf); |
226 | } |
227 | |