1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Tegra host1x Command DMA |
4 | * |
5 | * Copyright (c) 2010-2013, NVIDIA Corporation. |
6 | */ |
7 | |
8 | #include <linux/slab.h> |
9 | #include <linux/scatterlist.h> |
10 | #include <linux/dma-mapping.h> |
11 | |
12 | #include "../cdma.h" |
13 | #include "../channel.h" |
14 | #include "../dev.h" |
15 | #include "../debug.h" |
16 | |
17 | /* |
18 | * Put the restart at the end of pushbuffer memory |
19 | */ |
20 | static void push_buffer_init(struct push_buffer *pb) |
21 | { |
22 | *(u32 *)(pb->mapped + pb->size) = host1x_opcode_restart(0); |
23 | } |
24 | |
25 | /* |
26 | * Increment timedout buffer's syncpt via CPU. |
27 | */ |
28 | static void cdma_timeout_cpu_incr(struct host1x_cdma *cdma, u32 getptr, |
29 | u32 syncpt_incrs, u32 syncval, u32 nr_slots) |
30 | { |
31 | unsigned int i; |
32 | |
33 | for (i = 0; i < syncpt_incrs; i++) |
34 | host1x_syncpt_incr(sp: cdma->timeout.syncpt); |
35 | |
36 | /* after CPU incr, ensure shadow is up to date */ |
37 | host1x_syncpt_load(sp: cdma->timeout.syncpt); |
38 | } |
39 | |
40 | /* |
41 | * Start channel DMA |
42 | */ |
43 | static void cdma_start(struct host1x_cdma *cdma) |
44 | { |
45 | struct host1x_channel *ch = cdma_to_channel(cdma); |
46 | u64 start, end; |
47 | |
48 | if (cdma->running) |
49 | return; |
50 | |
51 | cdma->last_pos = cdma->push_buffer.pos; |
52 | start = cdma->push_buffer.dma; |
53 | end = cdma->push_buffer.size + 4; |
54 | |
55 | host1x_ch_writel(ch, r: HOST1X_CHANNEL_DMACTRL_DMASTOP, |
56 | v: HOST1X_CHANNEL_DMACTRL); |
57 | |
58 | /* set base, put and end pointer */ |
59 | host1x_ch_writel(ch, lower_32_bits(start), v: HOST1X_CHANNEL_DMASTART); |
60 | #if HOST1X_HW >= 6 |
61 | host1x_ch_writel(ch, upper_32_bits(start), HOST1X_CHANNEL_DMASTART_HI); |
62 | #endif |
63 | host1x_ch_writel(ch, r: cdma->push_buffer.pos, v: HOST1X_CHANNEL_DMAPUT); |
64 | #if HOST1X_HW >= 6 |
65 | host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMAPUT_HI); |
66 | #endif |
67 | host1x_ch_writel(ch, lower_32_bits(end), v: HOST1X_CHANNEL_DMAEND); |
68 | #if HOST1X_HW >= 6 |
69 | host1x_ch_writel(ch, upper_32_bits(end), HOST1X_CHANNEL_DMAEND_HI); |
70 | #endif |
71 | |
72 | /* reset GET */ |
73 | host1x_ch_writel(ch, r: HOST1X_CHANNEL_DMACTRL_DMASTOP | |
74 | HOST1X_CHANNEL_DMACTRL_DMAGETRST | |
75 | HOST1X_CHANNEL_DMACTRL_DMAINITGET, |
76 | v: HOST1X_CHANNEL_DMACTRL); |
77 | |
78 | /* start the command DMA */ |
79 | host1x_ch_writel(ch, r: 0, v: HOST1X_CHANNEL_DMACTRL); |
80 | |
81 | cdma->running = true; |
82 | } |
83 | |
84 | /* |
85 | * Similar to cdma_start(), but rather than starting from an idle |
86 | * state (where DMA GET is set to DMA PUT), on a timeout we restore |
87 | * DMA GET from an explicit value (so DMA may again be pending). |
88 | */ |
89 | static void cdma_timeout_restart(struct host1x_cdma *cdma, u32 getptr) |
90 | { |
91 | struct host1x *host1x = cdma_to_host1x(cdma); |
92 | struct host1x_channel *ch = cdma_to_channel(cdma); |
93 | u64 start, end; |
94 | |
95 | if (cdma->running) |
96 | return; |
97 | |
98 | cdma->last_pos = cdma->push_buffer.pos; |
99 | |
100 | host1x_ch_writel(ch, r: HOST1X_CHANNEL_DMACTRL_DMASTOP, |
101 | v: HOST1X_CHANNEL_DMACTRL); |
102 | |
103 | start = cdma->push_buffer.dma; |
104 | end = cdma->push_buffer.size + 4; |
105 | |
106 | /* set base, end pointer (all of memory) */ |
107 | host1x_ch_writel(ch, lower_32_bits(start), v: HOST1X_CHANNEL_DMASTART); |
108 | #if HOST1X_HW >= 6 |
109 | host1x_ch_writel(ch, upper_32_bits(start), HOST1X_CHANNEL_DMASTART_HI); |
110 | #endif |
111 | host1x_ch_writel(ch, lower_32_bits(end), v: HOST1X_CHANNEL_DMAEND); |
112 | #if HOST1X_HW >= 6 |
113 | host1x_ch_writel(ch, upper_32_bits(end), HOST1X_CHANNEL_DMAEND_HI); |
114 | #endif |
115 | |
116 | /* set GET, by loading the value in PUT (then reset GET) */ |
117 | host1x_ch_writel(ch, r: getptr, v: HOST1X_CHANNEL_DMAPUT); |
118 | host1x_ch_writel(ch, r: HOST1X_CHANNEL_DMACTRL_DMASTOP | |
119 | HOST1X_CHANNEL_DMACTRL_DMAGETRST | |
120 | HOST1X_CHANNEL_DMACTRL_DMAINITGET, |
121 | v: HOST1X_CHANNEL_DMACTRL); |
122 | |
123 | dev_dbg(host1x->dev, |
124 | "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n" , __func__, |
125 | host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET), |
126 | host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT), |
127 | cdma->last_pos); |
128 | |
129 | /* deassert GET reset and set PUT */ |
130 | host1x_ch_writel(ch, r: HOST1X_CHANNEL_DMACTRL_DMASTOP, |
131 | v: HOST1X_CHANNEL_DMACTRL); |
132 | host1x_ch_writel(ch, r: cdma->push_buffer.pos, v: HOST1X_CHANNEL_DMAPUT); |
133 | |
134 | /* start the command DMA */ |
135 | host1x_ch_writel(ch, r: 0, v: HOST1X_CHANNEL_DMACTRL); |
136 | |
137 | cdma->running = true; |
138 | } |
139 | |
140 | /* |
141 | * Kick channel DMA into action by writing its PUT offset (if it has changed) |
142 | */ |
143 | static void cdma_flush(struct host1x_cdma *cdma) |
144 | { |
145 | struct host1x_channel *ch = cdma_to_channel(cdma); |
146 | |
147 | if (cdma->push_buffer.pos != cdma->last_pos) { |
148 | host1x_ch_writel(ch, r: cdma->push_buffer.pos, |
149 | v: HOST1X_CHANNEL_DMAPUT); |
150 | cdma->last_pos = cdma->push_buffer.pos; |
151 | } |
152 | } |
153 | |
154 | static void cdma_stop(struct host1x_cdma *cdma) |
155 | { |
156 | struct host1x_channel *ch = cdma_to_channel(cdma); |
157 | |
158 | mutex_lock(&cdma->lock); |
159 | |
160 | if (cdma->running) { |
161 | host1x_cdma_wait_locked(cdma, event: CDMA_EVENT_SYNC_QUEUE_EMPTY); |
162 | host1x_ch_writel(ch, r: HOST1X_CHANNEL_DMACTRL_DMASTOP, |
163 | v: HOST1X_CHANNEL_DMACTRL); |
164 | cdma->running = false; |
165 | } |
166 | |
167 | mutex_unlock(lock: &cdma->lock); |
168 | } |
169 | |
170 | static void cdma_hw_cmdproc_stop(struct host1x *host, struct host1x_channel *ch, |
171 | bool stop) |
172 | { |
173 | #if HOST1X_HW >= 6 |
174 | host1x_ch_writel(ch, stop ? 0x1 : 0x0, HOST1X_CHANNEL_CMDPROC_STOP); |
175 | #else |
176 | u32 cmdproc_stop = host1x_sync_readl(host1x: host, r: HOST1X_SYNC_CMDPROC_STOP); |
177 | if (stop) |
178 | cmdproc_stop |= BIT(ch->id); |
179 | else |
180 | cmdproc_stop &= ~BIT(ch->id); |
181 | host1x_sync_writel(host1x: host, r: cmdproc_stop, v: HOST1X_SYNC_CMDPROC_STOP); |
182 | #endif |
183 | } |
184 | |
185 | static void cdma_hw_teardown(struct host1x *host, struct host1x_channel *ch) |
186 | { |
187 | #if HOST1X_HW >= 6 |
188 | host1x_ch_writel(ch, 0x1, HOST1X_CHANNEL_TEARDOWN); |
189 | #else |
190 | host1x_sync_writel(host1x: host, BIT(ch->id), v: HOST1X_SYNC_CH_TEARDOWN); |
191 | #endif |
192 | } |
193 | |
194 | /* |
195 | * Stops both channel's command processor and CDMA immediately. |
196 | * Also, tears down the channel and resets corresponding module. |
197 | */ |
198 | static void cdma_freeze(struct host1x_cdma *cdma) |
199 | { |
200 | struct host1x *host = cdma_to_host1x(cdma); |
201 | struct host1x_channel *ch = cdma_to_channel(cdma); |
202 | |
203 | if (cdma->torndown && !cdma->running) { |
204 | dev_warn(host->dev, "Already torn down\n" ); |
205 | return; |
206 | } |
207 | |
208 | dev_dbg(host->dev, "freezing channel (id %d)\n" , ch->id); |
209 | |
210 | cdma_hw_cmdproc_stop(host, ch, stop: true); |
211 | |
212 | dev_dbg(host->dev, "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n" , |
213 | __func__, host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET), |
214 | host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT), |
215 | cdma->last_pos); |
216 | |
217 | host1x_ch_writel(ch, r: HOST1X_CHANNEL_DMACTRL_DMASTOP, |
218 | v: HOST1X_CHANNEL_DMACTRL); |
219 | |
220 | cdma_hw_teardown(host, ch); |
221 | |
222 | cdma->running = false; |
223 | cdma->torndown = true; |
224 | } |
225 | |
226 | static void cdma_resume(struct host1x_cdma *cdma, u32 getptr) |
227 | { |
228 | struct host1x *host1x = cdma_to_host1x(cdma); |
229 | struct host1x_channel *ch = cdma_to_channel(cdma); |
230 | |
231 | dev_dbg(host1x->dev, |
232 | "resuming channel (id %u, DMAGET restart = 0x%x)\n" , |
233 | ch->id, getptr); |
234 | |
235 | cdma_hw_cmdproc_stop(host: host1x, ch, stop: false); |
236 | |
237 | cdma->torndown = false; |
238 | cdma_timeout_restart(cdma, getptr); |
239 | } |
240 | |
241 | static void timeout_release_mlock(struct host1x_cdma *cdma) |
242 | { |
243 | #if HOST1X_HW >= 8 |
244 | /* Tegra186 and Tegra194 require a more complicated MLOCK release |
245 | * sequence. Furthermore, those chips by default don't enforce MLOCKs, |
246 | * so it turns out that if we don't /actually/ need MLOCKs, we can just |
247 | * ignore them. |
248 | * |
249 | * As such, for now just implement this on Tegra234 where things are |
250 | * stricter but also easy to implement. |
251 | */ |
252 | struct host1x_channel *ch = cdma_to_channel(cdma); |
253 | struct host1x *host1x = cdma_to_host1x(cdma); |
254 | u32 offset; |
255 | |
256 | switch (ch->client->class) { |
257 | case HOST1X_CLASS_VIC: |
258 | offset = HOST1X_COMMON_VIC_MLOCK; |
259 | break; |
260 | case HOST1X_CLASS_NVDEC: |
261 | offset = HOST1X_COMMON_NVDEC_MLOCK; |
262 | break; |
263 | default: |
264 | WARN(1, "%s was not updated for class %u" , __func__, ch->client->class); |
265 | return; |
266 | } |
267 | |
268 | host1x_common_writel(host1x, 0x0, offset); |
269 | #endif |
270 | } |
271 | |
272 | /* |
273 | * If this timeout fires, it indicates the current sync_queue entry has |
274 | * exceeded its TTL and the userctx should be timed out and remaining |
275 | * submits already issued cleaned up (future submits return an error). |
276 | */ |
277 | static void cdma_timeout_handler(struct work_struct *work) |
278 | { |
279 | u32 syncpt_val; |
280 | struct host1x_cdma *cdma; |
281 | struct host1x *host1x; |
282 | struct host1x_channel *ch; |
283 | |
284 | cdma = container_of(to_delayed_work(work), struct host1x_cdma, |
285 | timeout.wq); |
286 | host1x = cdma_to_host1x(cdma); |
287 | ch = cdma_to_channel(cdma); |
288 | |
289 | host1x_debug_dump(cdma_to_host1x(cdma)); |
290 | |
291 | mutex_lock(&cdma->lock); |
292 | |
293 | if (!cdma->timeout.client) { |
294 | dev_dbg(host1x->dev, |
295 | "cdma_timeout: expired, but has no clientid\n" ); |
296 | mutex_unlock(lock: &cdma->lock); |
297 | return; |
298 | } |
299 | |
300 | /* stop processing to get a clean snapshot */ |
301 | cdma_hw_cmdproc_stop(host: host1x, ch, stop: true); |
302 | |
303 | syncpt_val = host1x_syncpt_load(sp: cdma->timeout.syncpt); |
304 | |
305 | /* has buffer actually completed? */ |
306 | if ((s32)(syncpt_val - cdma->timeout.syncpt_val) >= 0) { |
307 | dev_dbg(host1x->dev, |
308 | "cdma_timeout: expired, but buffer had completed\n" ); |
309 | /* restore */ |
310 | cdma_hw_cmdproc_stop(host: host1x, ch, stop: false); |
311 | mutex_unlock(lock: &cdma->lock); |
312 | return; |
313 | } |
314 | |
315 | dev_warn(host1x->dev, "%s: timeout: %u (%s), HW thresh %d, done %d\n" , |
316 | __func__, cdma->timeout.syncpt->id, cdma->timeout.syncpt->name, |
317 | syncpt_val, cdma->timeout.syncpt_val); |
318 | |
319 | /* stop HW, resetting channel/module */ |
320 | host1x_hw_cdma_freeze(host: host1x, cdma); |
321 | |
322 | /* release any held MLOCK */ |
323 | timeout_release_mlock(cdma); |
324 | |
325 | host1x_cdma_update_sync_queue(cdma, dev: ch->dev); |
326 | mutex_unlock(lock: &cdma->lock); |
327 | } |
328 | |
329 | /* |
330 | * Init timeout resources |
331 | */ |
332 | static int cdma_timeout_init(struct host1x_cdma *cdma) |
333 | { |
334 | INIT_DELAYED_WORK(&cdma->timeout.wq, cdma_timeout_handler); |
335 | cdma->timeout.initialized = true; |
336 | |
337 | return 0; |
338 | } |
339 | |
340 | /* |
341 | * Clean up timeout resources |
342 | */ |
343 | static void cdma_timeout_destroy(struct host1x_cdma *cdma) |
344 | { |
345 | if (cdma->timeout.initialized) |
346 | cancel_delayed_work(dwork: &cdma->timeout.wq); |
347 | |
348 | cdma->timeout.initialized = false; |
349 | } |
350 | |
351 | static const struct host1x_cdma_ops host1x_cdma_ops = { |
352 | .start = cdma_start, |
353 | .stop = cdma_stop, |
354 | .flush = cdma_flush, |
355 | |
356 | .timeout_init = cdma_timeout_init, |
357 | .timeout_destroy = cdma_timeout_destroy, |
358 | .freeze = cdma_freeze, |
359 | .resume = cdma_resume, |
360 | .timeout_cpu_incr = cdma_timeout_cpu_incr, |
361 | }; |
362 | |
363 | static const struct host1x_pushbuffer_ops host1x_pushbuffer_ops = { |
364 | .init = push_buffer_init, |
365 | }; |
366 | |