1 | // SPDX-License-Identifier: GPL-2.0 OR MIT |
2 | /* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */ |
3 | |
4 | #include <linux/interrupt.h> |
5 | #include <linux/io.h> |
6 | #include <linux/device.h> |
7 | #include <linux/slab.h> |
8 | |
9 | #include <drm/lima_drm.h> |
10 | |
11 | #include "lima_device.h" |
12 | #include "lima_pp.h" |
13 | #include "lima_dlbu.h" |
14 | #include "lima_bcast.h" |
15 | #include "lima_vm.h" |
16 | #include "lima_regs.h" |
17 | |
18 | #define pp_write(reg, data) writel(data, ip->iomem + reg) |
19 | #define pp_read(reg) readl(ip->iomem + reg) |
20 | |
21 | static void lima_pp_handle_irq(struct lima_ip *ip, u32 state) |
22 | { |
23 | struct lima_device *dev = ip->dev; |
24 | struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp; |
25 | |
26 | if (state & LIMA_PP_IRQ_MASK_ERROR) { |
27 | u32 status = pp_read(LIMA_PP_STATUS); |
28 | |
29 | dev_err(dev->dev, "%s error irq state=%x status=%x\n" , |
30 | lima_ip_name(ip), state, status); |
31 | |
32 | pipe->error = true; |
33 | |
34 | /* mask all interrupts before hard reset */ |
35 | pp_write(LIMA_PP_INT_MASK, 0); |
36 | } |
37 | |
38 | pp_write(LIMA_PP_INT_CLEAR, state); |
39 | } |
40 | |
41 | static irqreturn_t lima_pp_irq_handler(int irq, void *data) |
42 | { |
43 | struct lima_ip *ip = data; |
44 | struct lima_device *dev = ip->dev; |
45 | struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp; |
46 | u32 state = pp_read(LIMA_PP_INT_STATUS); |
47 | |
48 | /* for shared irq case */ |
49 | if (!state) |
50 | return IRQ_NONE; |
51 | |
52 | lima_pp_handle_irq(ip, state); |
53 | |
54 | if (atomic_dec_and_test(v: &pipe->task)) |
55 | lima_sched_pipe_task_done(pipe); |
56 | |
57 | return IRQ_HANDLED; |
58 | } |
59 | |
60 | static irqreturn_t lima_pp_bcast_irq_handler(int irq, void *data) |
61 | { |
62 | int i; |
63 | irqreturn_t ret = IRQ_NONE; |
64 | struct lima_ip *pp_bcast = data; |
65 | struct lima_device *dev = pp_bcast->dev; |
66 | struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp; |
67 | struct drm_lima_m450_pp_frame *frame; |
68 | |
69 | /* for shared irq case */ |
70 | if (!pipe->current_task) |
71 | return IRQ_NONE; |
72 | |
73 | frame = pipe->current_task->frame; |
74 | |
75 | for (i = 0; i < frame->num_pp; i++) { |
76 | struct lima_ip *ip = pipe->processor[i]; |
77 | u32 status, state; |
78 | |
79 | if (pipe->done & (1 << i)) |
80 | continue; |
81 | |
82 | /* status read first in case int state change in the middle |
83 | * which may miss the interrupt handling |
84 | */ |
85 | status = pp_read(LIMA_PP_STATUS); |
86 | state = pp_read(LIMA_PP_INT_STATUS); |
87 | |
88 | if (state) { |
89 | lima_pp_handle_irq(ip, state); |
90 | ret = IRQ_HANDLED; |
91 | } else { |
92 | if (status & LIMA_PP_STATUS_RENDERING_ACTIVE) |
93 | continue; |
94 | } |
95 | |
96 | pipe->done |= (1 << i); |
97 | if (atomic_dec_and_test(v: &pipe->task)) |
98 | lima_sched_pipe_task_done(pipe); |
99 | } |
100 | |
101 | return ret; |
102 | } |
103 | |
104 | static void lima_pp_soft_reset_async(struct lima_ip *ip) |
105 | { |
106 | if (ip->data.async_reset) |
107 | return; |
108 | |
109 | pp_write(LIMA_PP_INT_MASK, 0); |
110 | pp_write(LIMA_PP_INT_RAWSTAT, LIMA_PP_IRQ_MASK_ALL); |
111 | pp_write(LIMA_PP_CTRL, LIMA_PP_CTRL_SOFT_RESET); |
112 | ip->data.async_reset = true; |
113 | } |
114 | |
115 | static int lima_pp_soft_reset_poll(struct lima_ip *ip) |
116 | { |
117 | return !(pp_read(LIMA_PP_STATUS) & LIMA_PP_STATUS_RENDERING_ACTIVE) && |
118 | pp_read(LIMA_PP_INT_RAWSTAT) == LIMA_PP_IRQ_RESET_COMPLETED; |
119 | } |
120 | |
121 | static int lima_pp_soft_reset_async_wait_one(struct lima_ip *ip) |
122 | { |
123 | struct lima_device *dev = ip->dev; |
124 | int ret; |
125 | |
126 | ret = lima_poll_timeout(ip, func: lima_pp_soft_reset_poll, sleep_us: 0, timeout_us: 100); |
127 | if (ret) { |
128 | dev_err(dev->dev, "%s reset time out\n" , lima_ip_name(ip)); |
129 | return ret; |
130 | } |
131 | |
132 | pp_write(LIMA_PP_INT_CLEAR, LIMA_PP_IRQ_MASK_ALL); |
133 | pp_write(LIMA_PP_INT_MASK, LIMA_PP_IRQ_MASK_USED); |
134 | return 0; |
135 | } |
136 | |
137 | static int lima_pp_soft_reset_async_wait(struct lima_ip *ip) |
138 | { |
139 | int i, err = 0; |
140 | |
141 | if (!ip->data.async_reset) |
142 | return 0; |
143 | |
144 | if (ip->id == lima_ip_pp_bcast) { |
145 | struct lima_device *dev = ip->dev; |
146 | struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp; |
147 | struct drm_lima_m450_pp_frame *frame = pipe->current_task->frame; |
148 | |
149 | for (i = 0; i < frame->num_pp; i++) |
150 | err |= lima_pp_soft_reset_async_wait_one(ip: pipe->processor[i]); |
151 | } else |
152 | err = lima_pp_soft_reset_async_wait_one(ip); |
153 | |
154 | ip->data.async_reset = false; |
155 | return err; |
156 | } |
157 | |
158 | static void lima_pp_write_frame(struct lima_ip *ip, u32 *frame, u32 *wb) |
159 | { |
160 | int i, j, n = 0; |
161 | |
162 | for (i = 0; i < LIMA_PP_FRAME_REG_NUM; i++) |
163 | writel(val: frame[i], addr: ip->iomem + LIMA_PP_FRAME + i * 4); |
164 | |
165 | for (i = 0; i < 3; i++) { |
166 | for (j = 0; j < LIMA_PP_WB_REG_NUM; j++) |
167 | writel(val: wb[n++], addr: ip->iomem + LIMA_PP_WB(i) + j * 4); |
168 | } |
169 | } |
170 | |
171 | static int lima_pp_bus_stop_poll(struct lima_ip *ip) |
172 | { |
173 | return !!(pp_read(LIMA_PP_STATUS) & LIMA_PP_STATUS_BUS_STOPPED); |
174 | } |
175 | |
176 | static int lima_pp_hard_reset_poll(struct lima_ip *ip) |
177 | { |
178 | pp_write(LIMA_PP_PERF_CNT_0_LIMIT, 0xC01A0000); |
179 | return pp_read(LIMA_PP_PERF_CNT_0_LIMIT) == 0xC01A0000; |
180 | } |
181 | |
182 | static int lima_pp_hard_reset(struct lima_ip *ip) |
183 | { |
184 | struct lima_device *dev = ip->dev; |
185 | int ret; |
186 | |
187 | pp_write(LIMA_PP_PERF_CNT_0_LIMIT, 0xC0FFE000); |
188 | pp_write(LIMA_PP_INT_MASK, 0); |
189 | |
190 | pp_write(LIMA_PP_CTRL, LIMA_PP_CTRL_STOP_BUS); |
191 | ret = lima_poll_timeout(ip, func: lima_pp_bus_stop_poll, sleep_us: 10, timeout_us: 100); |
192 | if (ret) { |
193 | dev_err(dev->dev, "%s bus stop timeout\n" , lima_ip_name(ip)); |
194 | return ret; |
195 | } |
196 | |
197 | pp_write(LIMA_PP_CTRL, LIMA_PP_CTRL_FORCE_RESET); |
198 | ret = lima_poll_timeout(ip, func: lima_pp_hard_reset_poll, sleep_us: 10, timeout_us: 100); |
199 | if (ret) { |
200 | dev_err(dev->dev, "%s hard reset timeout\n" , lima_ip_name(ip)); |
201 | return ret; |
202 | } |
203 | |
204 | pp_write(LIMA_PP_PERF_CNT_0_LIMIT, 0); |
205 | pp_write(LIMA_PP_INT_CLEAR, LIMA_PP_IRQ_MASK_ALL); |
206 | pp_write(LIMA_PP_INT_MASK, LIMA_PP_IRQ_MASK_USED); |
207 | |
208 | /* |
209 | * if there was an async soft reset queued, |
210 | * don't wait for it in the next job |
211 | */ |
212 | ip->data.async_reset = false; |
213 | |
214 | return 0; |
215 | } |
216 | |
217 | static void lima_pp_print_version(struct lima_ip *ip) |
218 | { |
219 | u32 version, major, minor; |
220 | char *name; |
221 | |
222 | version = pp_read(LIMA_PP_VERSION); |
223 | major = (version >> 8) & 0xFF; |
224 | minor = version & 0xFF; |
225 | switch (version >> 16) { |
226 | case 0xC807: |
227 | name = "mali200" ; |
228 | break; |
229 | case 0xCE07: |
230 | name = "mali300" ; |
231 | break; |
232 | case 0xCD07: |
233 | name = "mali400" ; |
234 | break; |
235 | case 0xCF07: |
236 | name = "mali450" ; |
237 | break; |
238 | default: |
239 | name = "unknown" ; |
240 | break; |
241 | } |
242 | dev_info(ip->dev->dev, "%s - %s version major %d minor %d\n" , |
243 | lima_ip_name(ip), name, major, minor); |
244 | } |
245 | |
246 | static int lima_pp_hw_init(struct lima_ip *ip) |
247 | { |
248 | ip->data.async_reset = false; |
249 | lima_pp_soft_reset_async(ip); |
250 | return lima_pp_soft_reset_async_wait(ip); |
251 | } |
252 | |
253 | int lima_pp_resume(struct lima_ip *ip) |
254 | { |
255 | return lima_pp_hw_init(ip); |
256 | } |
257 | |
258 | void lima_pp_suspend(struct lima_ip *ip) |
259 | { |
260 | |
261 | } |
262 | |
263 | int lima_pp_init(struct lima_ip *ip) |
264 | { |
265 | struct lima_device *dev = ip->dev; |
266 | int err; |
267 | |
268 | lima_pp_print_version(ip); |
269 | |
270 | err = lima_pp_hw_init(ip); |
271 | if (err) |
272 | return err; |
273 | |
274 | err = devm_request_irq(dev: dev->dev, irq: ip->irq, handler: lima_pp_irq_handler, |
275 | IRQF_SHARED, devname: lima_ip_name(ip), dev_id: ip); |
276 | if (err) { |
277 | dev_err(dev->dev, "%s fail to request irq\n" , |
278 | lima_ip_name(ip)); |
279 | return err; |
280 | } |
281 | |
282 | dev->pp_version = pp_read(LIMA_PP_VERSION); |
283 | |
284 | return 0; |
285 | } |
286 | |
287 | void lima_pp_fini(struct lima_ip *ip) |
288 | { |
289 | |
290 | } |
291 | |
292 | int lima_pp_bcast_resume(struct lima_ip *ip) |
293 | { |
294 | /* PP has been reset by individual PP resume */ |
295 | ip->data.async_reset = false; |
296 | return 0; |
297 | } |
298 | |
299 | void lima_pp_bcast_suspend(struct lima_ip *ip) |
300 | { |
301 | |
302 | } |
303 | |
304 | int lima_pp_bcast_init(struct lima_ip *ip) |
305 | { |
306 | struct lima_device *dev = ip->dev; |
307 | int err; |
308 | |
309 | err = devm_request_irq(dev: dev->dev, irq: ip->irq, handler: lima_pp_bcast_irq_handler, |
310 | IRQF_SHARED, devname: lima_ip_name(ip), dev_id: ip); |
311 | if (err) { |
312 | dev_err(dev->dev, "%s fail to request irq\n" , |
313 | lima_ip_name(ip)); |
314 | return err; |
315 | } |
316 | |
317 | return 0; |
318 | } |
319 | |
320 | void lima_pp_bcast_fini(struct lima_ip *ip) |
321 | { |
322 | |
323 | } |
324 | |
325 | static int lima_pp_task_validate(struct lima_sched_pipe *pipe, |
326 | struct lima_sched_task *task) |
327 | { |
328 | u32 num_pp; |
329 | |
330 | if (pipe->bcast_processor) { |
331 | struct drm_lima_m450_pp_frame *f = task->frame; |
332 | |
333 | num_pp = f->num_pp; |
334 | |
335 | if (f->_pad) |
336 | return -EINVAL; |
337 | } else { |
338 | struct drm_lima_m400_pp_frame *f = task->frame; |
339 | |
340 | num_pp = f->num_pp; |
341 | } |
342 | |
343 | if (num_pp == 0 || num_pp > pipe->num_processor) |
344 | return -EINVAL; |
345 | |
346 | return 0; |
347 | } |
348 | |
349 | static void lima_pp_task_run(struct lima_sched_pipe *pipe, |
350 | struct lima_sched_task *task) |
351 | { |
352 | if (pipe->bcast_processor) { |
353 | struct drm_lima_m450_pp_frame *frame = task->frame; |
354 | struct lima_device *dev = pipe->bcast_processor->dev; |
355 | struct lima_ip *ip = pipe->bcast_processor; |
356 | int i; |
357 | |
358 | pipe->done = 0; |
359 | atomic_set(v: &pipe->task, i: frame->num_pp); |
360 | |
361 | if (frame->use_dlbu) { |
362 | lima_dlbu_enable(dev, num_pp: frame->num_pp); |
363 | |
364 | frame->frame[LIMA_PP_FRAME >> 2] = LIMA_VA_RESERVE_DLBU; |
365 | lima_dlbu_set_reg(ip: dev->ip + lima_ip_dlbu, reg: frame->dlbu_regs); |
366 | } else |
367 | lima_dlbu_disable(dev); |
368 | |
369 | lima_bcast_enable(dev, num_pp: frame->num_pp); |
370 | |
371 | lima_pp_soft_reset_async_wait(ip); |
372 | |
373 | lima_pp_write_frame(ip, frame: frame->frame, wb: frame->wb); |
374 | |
375 | for (i = 0; i < frame->num_pp; i++) { |
376 | struct lima_ip *ip = pipe->processor[i]; |
377 | |
378 | pp_write(LIMA_PP_STACK, frame->fragment_stack_address[i]); |
379 | if (!frame->use_dlbu) |
380 | pp_write(LIMA_PP_FRAME, frame->plbu_array_address[i]); |
381 | } |
382 | |
383 | pp_write(LIMA_PP_CTRL, LIMA_PP_CTRL_START_RENDERING); |
384 | } else { |
385 | struct drm_lima_m400_pp_frame *frame = task->frame; |
386 | int i; |
387 | |
388 | atomic_set(v: &pipe->task, i: frame->num_pp); |
389 | |
390 | for (i = 0; i < frame->num_pp; i++) { |
391 | struct lima_ip *ip = pipe->processor[i]; |
392 | |
393 | frame->frame[LIMA_PP_FRAME >> 2] = |
394 | frame->plbu_array_address[i]; |
395 | frame->frame[LIMA_PP_STACK >> 2] = |
396 | frame->fragment_stack_address[i]; |
397 | |
398 | lima_pp_soft_reset_async_wait(ip); |
399 | |
400 | lima_pp_write_frame(ip, frame: frame->frame, wb: frame->wb); |
401 | |
402 | pp_write(LIMA_PP_CTRL, LIMA_PP_CTRL_START_RENDERING); |
403 | } |
404 | } |
405 | } |
406 | |
407 | static void lima_pp_task_fini(struct lima_sched_pipe *pipe) |
408 | { |
409 | if (pipe->bcast_processor) |
410 | lima_pp_soft_reset_async(ip: pipe->bcast_processor); |
411 | else { |
412 | int i; |
413 | |
414 | for (i = 0; i < pipe->num_processor; i++) |
415 | lima_pp_soft_reset_async(ip: pipe->processor[i]); |
416 | } |
417 | } |
418 | |
419 | static void lima_pp_task_error(struct lima_sched_pipe *pipe) |
420 | { |
421 | int i; |
422 | |
423 | for (i = 0; i < pipe->num_processor; i++) { |
424 | struct lima_ip *ip = pipe->processor[i]; |
425 | |
426 | dev_err(ip->dev->dev, "%s task error %d int_state=%x status=%x\n" , |
427 | lima_ip_name(ip), i, pp_read(LIMA_PP_INT_STATUS), |
428 | pp_read(LIMA_PP_STATUS)); |
429 | |
430 | lima_pp_hard_reset(ip); |
431 | } |
432 | } |
433 | |
434 | static void lima_pp_task_mmu_error(struct lima_sched_pipe *pipe) |
435 | { |
436 | if (atomic_dec_and_test(v: &pipe->task)) |
437 | lima_sched_pipe_task_done(pipe); |
438 | } |
439 | |
440 | static struct kmem_cache *lima_pp_task_slab; |
441 | static int lima_pp_task_slab_refcnt; |
442 | |
443 | int lima_pp_pipe_init(struct lima_device *dev) |
444 | { |
445 | int frame_size; |
446 | struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp; |
447 | |
448 | if (dev->id == lima_gpu_mali400) |
449 | frame_size = sizeof(struct drm_lima_m400_pp_frame); |
450 | else |
451 | frame_size = sizeof(struct drm_lima_m450_pp_frame); |
452 | |
453 | if (!lima_pp_task_slab) { |
454 | lima_pp_task_slab = kmem_cache_create_usercopy( |
455 | name: "lima_pp_task" , size: sizeof(struct lima_sched_task) + frame_size, |
456 | align: 0, SLAB_HWCACHE_ALIGN, useroffset: sizeof(struct lima_sched_task), |
457 | usersize: frame_size, NULL); |
458 | if (!lima_pp_task_slab) |
459 | return -ENOMEM; |
460 | } |
461 | lima_pp_task_slab_refcnt++; |
462 | |
463 | pipe->frame_size = frame_size; |
464 | pipe->task_slab = lima_pp_task_slab; |
465 | |
466 | pipe->task_validate = lima_pp_task_validate; |
467 | pipe->task_run = lima_pp_task_run; |
468 | pipe->task_fini = lima_pp_task_fini; |
469 | pipe->task_error = lima_pp_task_error; |
470 | pipe->task_mmu_error = lima_pp_task_mmu_error; |
471 | |
472 | return 0; |
473 | } |
474 | |
475 | void lima_pp_pipe_fini(struct lima_device *dev) |
476 | { |
477 | if (!--lima_pp_task_slab_refcnt) { |
478 | kmem_cache_destroy(s: lima_pp_task_slab); |
479 | lima_pp_task_slab = NULL; |
480 | } |
481 | } |
482 | |