1// SPDX-License-Identifier: GPL-2.0 OR MIT
2/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
3
4#include <linux/interrupt.h>
5#include <linux/io.h>
6#include <linux/device.h>
7#include <linux/slab.h>
8
9#include <drm/lima_drm.h>
10
11#include "lima_device.h"
12#include "lima_pp.h"
13#include "lima_dlbu.h"
14#include "lima_bcast.h"
15#include "lima_vm.h"
16#include "lima_regs.h"
17
18#define pp_write(reg, data) writel(data, ip->iomem + reg)
19#define pp_read(reg) readl(ip->iomem + reg)
20
21static void lima_pp_handle_irq(struct lima_ip *ip, u32 state)
22{
23 struct lima_device *dev = ip->dev;
24 struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp;
25
26 if (state & LIMA_PP_IRQ_MASK_ERROR) {
27 u32 status = pp_read(LIMA_PP_STATUS);
28
29 dev_err(dev->dev, "%s error irq state=%x status=%x\n",
30 lima_ip_name(ip), state, status);
31
32 pipe->error = true;
33
34 /* mask all interrupts before hard reset */
35 pp_write(LIMA_PP_INT_MASK, 0);
36 }
37
38 pp_write(LIMA_PP_INT_CLEAR, state);
39}
40
41static irqreturn_t lima_pp_irq_handler(int irq, void *data)
42{
43 struct lima_ip *ip = data;
44 struct lima_device *dev = ip->dev;
45 struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp;
46 u32 state = pp_read(LIMA_PP_INT_STATUS);
47
48 /* for shared irq case */
49 if (!state)
50 return IRQ_NONE;
51
52 lima_pp_handle_irq(ip, state);
53
54 if (atomic_dec_and_test(v: &pipe->task))
55 lima_sched_pipe_task_done(pipe);
56
57 return IRQ_HANDLED;
58}
59
60static irqreturn_t lima_pp_bcast_irq_handler(int irq, void *data)
61{
62 int i;
63 irqreturn_t ret = IRQ_NONE;
64 struct lima_ip *pp_bcast = data;
65 struct lima_device *dev = pp_bcast->dev;
66 struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp;
67 struct drm_lima_m450_pp_frame *frame;
68
69 /* for shared irq case */
70 if (!pipe->current_task)
71 return IRQ_NONE;
72
73 frame = pipe->current_task->frame;
74
75 for (i = 0; i < frame->num_pp; i++) {
76 struct lima_ip *ip = pipe->processor[i];
77 u32 status, state;
78
79 if (pipe->done & (1 << i))
80 continue;
81
82 /* status read first in case int state change in the middle
83 * which may miss the interrupt handling
84 */
85 status = pp_read(LIMA_PP_STATUS);
86 state = pp_read(LIMA_PP_INT_STATUS);
87
88 if (state) {
89 lima_pp_handle_irq(ip, state);
90 ret = IRQ_HANDLED;
91 } else {
92 if (status & LIMA_PP_STATUS_RENDERING_ACTIVE)
93 continue;
94 }
95
96 pipe->done |= (1 << i);
97 if (atomic_dec_and_test(v: &pipe->task))
98 lima_sched_pipe_task_done(pipe);
99 }
100
101 return ret;
102}
103
104static void lima_pp_soft_reset_async(struct lima_ip *ip)
105{
106 if (ip->data.async_reset)
107 return;
108
109 pp_write(LIMA_PP_INT_MASK, 0);
110 pp_write(LIMA_PP_INT_RAWSTAT, LIMA_PP_IRQ_MASK_ALL);
111 pp_write(LIMA_PP_CTRL, LIMA_PP_CTRL_SOFT_RESET);
112 ip->data.async_reset = true;
113}
114
115static int lima_pp_soft_reset_poll(struct lima_ip *ip)
116{
117 return !(pp_read(LIMA_PP_STATUS) & LIMA_PP_STATUS_RENDERING_ACTIVE) &&
118 pp_read(LIMA_PP_INT_RAWSTAT) == LIMA_PP_IRQ_RESET_COMPLETED;
119}
120
121static int lima_pp_soft_reset_async_wait_one(struct lima_ip *ip)
122{
123 struct lima_device *dev = ip->dev;
124 int ret;
125
126 ret = lima_poll_timeout(ip, func: lima_pp_soft_reset_poll, sleep_us: 0, timeout_us: 100);
127 if (ret) {
128 dev_err(dev->dev, "%s reset time out\n", lima_ip_name(ip));
129 return ret;
130 }
131
132 pp_write(LIMA_PP_INT_CLEAR, LIMA_PP_IRQ_MASK_ALL);
133 pp_write(LIMA_PP_INT_MASK, LIMA_PP_IRQ_MASK_USED);
134 return 0;
135}
136
137static int lima_pp_soft_reset_async_wait(struct lima_ip *ip)
138{
139 int i, err = 0;
140
141 if (!ip->data.async_reset)
142 return 0;
143
144 if (ip->id == lima_ip_pp_bcast) {
145 struct lima_device *dev = ip->dev;
146 struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp;
147 struct drm_lima_m450_pp_frame *frame = pipe->current_task->frame;
148
149 for (i = 0; i < frame->num_pp; i++)
150 err |= lima_pp_soft_reset_async_wait_one(ip: pipe->processor[i]);
151 } else
152 err = lima_pp_soft_reset_async_wait_one(ip);
153
154 ip->data.async_reset = false;
155 return err;
156}
157
158static void lima_pp_write_frame(struct lima_ip *ip, u32 *frame, u32 *wb)
159{
160 int i, j, n = 0;
161
162 for (i = 0; i < LIMA_PP_FRAME_REG_NUM; i++)
163 writel(val: frame[i], addr: ip->iomem + LIMA_PP_FRAME + i * 4);
164
165 for (i = 0; i < 3; i++) {
166 for (j = 0; j < LIMA_PP_WB_REG_NUM; j++)
167 writel(val: wb[n++], addr: ip->iomem + LIMA_PP_WB(i) + j * 4);
168 }
169}
170
171static int lima_pp_bus_stop_poll(struct lima_ip *ip)
172{
173 return !!(pp_read(LIMA_PP_STATUS) & LIMA_PP_STATUS_BUS_STOPPED);
174}
175
176static int lima_pp_hard_reset_poll(struct lima_ip *ip)
177{
178 pp_write(LIMA_PP_PERF_CNT_0_LIMIT, 0xC01A0000);
179 return pp_read(LIMA_PP_PERF_CNT_0_LIMIT) == 0xC01A0000;
180}
181
182static int lima_pp_hard_reset(struct lima_ip *ip)
183{
184 struct lima_device *dev = ip->dev;
185 int ret;
186
187 pp_write(LIMA_PP_PERF_CNT_0_LIMIT, 0xC0FFE000);
188 pp_write(LIMA_PP_INT_MASK, 0);
189
190 pp_write(LIMA_PP_CTRL, LIMA_PP_CTRL_STOP_BUS);
191 ret = lima_poll_timeout(ip, func: lima_pp_bus_stop_poll, sleep_us: 10, timeout_us: 100);
192 if (ret) {
193 dev_err(dev->dev, "%s bus stop timeout\n", lima_ip_name(ip));
194 return ret;
195 }
196
197 pp_write(LIMA_PP_CTRL, LIMA_PP_CTRL_FORCE_RESET);
198 ret = lima_poll_timeout(ip, func: lima_pp_hard_reset_poll, sleep_us: 10, timeout_us: 100);
199 if (ret) {
200 dev_err(dev->dev, "%s hard reset timeout\n", lima_ip_name(ip));
201 return ret;
202 }
203
204 pp_write(LIMA_PP_PERF_CNT_0_LIMIT, 0);
205 pp_write(LIMA_PP_INT_CLEAR, LIMA_PP_IRQ_MASK_ALL);
206 pp_write(LIMA_PP_INT_MASK, LIMA_PP_IRQ_MASK_USED);
207
208 /*
209 * if there was an async soft reset queued,
210 * don't wait for it in the next job
211 */
212 ip->data.async_reset = false;
213
214 return 0;
215}
216
217static void lima_pp_print_version(struct lima_ip *ip)
218{
219 u32 version, major, minor;
220 char *name;
221
222 version = pp_read(LIMA_PP_VERSION);
223 major = (version >> 8) & 0xFF;
224 minor = version & 0xFF;
225 switch (version >> 16) {
226 case 0xC807:
227 name = "mali200";
228 break;
229 case 0xCE07:
230 name = "mali300";
231 break;
232 case 0xCD07:
233 name = "mali400";
234 break;
235 case 0xCF07:
236 name = "mali450";
237 break;
238 default:
239 name = "unknown";
240 break;
241 }
242 dev_info(ip->dev->dev, "%s - %s version major %d minor %d\n",
243 lima_ip_name(ip), name, major, minor);
244}
245
246static int lima_pp_hw_init(struct lima_ip *ip)
247{
248 ip->data.async_reset = false;
249 lima_pp_soft_reset_async(ip);
250 return lima_pp_soft_reset_async_wait(ip);
251}
252
253int lima_pp_resume(struct lima_ip *ip)
254{
255 return lima_pp_hw_init(ip);
256}
257
258void lima_pp_suspend(struct lima_ip *ip)
259{
260
261}
262
263int lima_pp_init(struct lima_ip *ip)
264{
265 struct lima_device *dev = ip->dev;
266 int err;
267
268 lima_pp_print_version(ip);
269
270 err = lima_pp_hw_init(ip);
271 if (err)
272 return err;
273
274 err = devm_request_irq(dev: dev->dev, irq: ip->irq, handler: lima_pp_irq_handler,
275 IRQF_SHARED, devname: lima_ip_name(ip), dev_id: ip);
276 if (err) {
277 dev_err(dev->dev, "%s fail to request irq\n",
278 lima_ip_name(ip));
279 return err;
280 }
281
282 dev->pp_version = pp_read(LIMA_PP_VERSION);
283
284 return 0;
285}
286
287void lima_pp_fini(struct lima_ip *ip)
288{
289
290}
291
292int lima_pp_bcast_resume(struct lima_ip *ip)
293{
294 /* PP has been reset by individual PP resume */
295 ip->data.async_reset = false;
296 return 0;
297}
298
299void lima_pp_bcast_suspend(struct lima_ip *ip)
300{
301
302}
303
304int lima_pp_bcast_init(struct lima_ip *ip)
305{
306 struct lima_device *dev = ip->dev;
307 int err;
308
309 err = devm_request_irq(dev: dev->dev, irq: ip->irq, handler: lima_pp_bcast_irq_handler,
310 IRQF_SHARED, devname: lima_ip_name(ip), dev_id: ip);
311 if (err) {
312 dev_err(dev->dev, "%s fail to request irq\n",
313 lima_ip_name(ip));
314 return err;
315 }
316
317 return 0;
318}
319
320void lima_pp_bcast_fini(struct lima_ip *ip)
321{
322
323}
324
325static int lima_pp_task_validate(struct lima_sched_pipe *pipe,
326 struct lima_sched_task *task)
327{
328 u32 num_pp;
329
330 if (pipe->bcast_processor) {
331 struct drm_lima_m450_pp_frame *f = task->frame;
332
333 num_pp = f->num_pp;
334
335 if (f->_pad)
336 return -EINVAL;
337 } else {
338 struct drm_lima_m400_pp_frame *f = task->frame;
339
340 num_pp = f->num_pp;
341 }
342
343 if (num_pp == 0 || num_pp > pipe->num_processor)
344 return -EINVAL;
345
346 return 0;
347}
348
349static void lima_pp_task_run(struct lima_sched_pipe *pipe,
350 struct lima_sched_task *task)
351{
352 if (pipe->bcast_processor) {
353 struct drm_lima_m450_pp_frame *frame = task->frame;
354 struct lima_device *dev = pipe->bcast_processor->dev;
355 struct lima_ip *ip = pipe->bcast_processor;
356 int i;
357
358 pipe->done = 0;
359 atomic_set(v: &pipe->task, i: frame->num_pp);
360
361 if (frame->use_dlbu) {
362 lima_dlbu_enable(dev, num_pp: frame->num_pp);
363
364 frame->frame[LIMA_PP_FRAME >> 2] = LIMA_VA_RESERVE_DLBU;
365 lima_dlbu_set_reg(ip: dev->ip + lima_ip_dlbu, reg: frame->dlbu_regs);
366 } else
367 lima_dlbu_disable(dev);
368
369 lima_bcast_enable(dev, num_pp: frame->num_pp);
370
371 lima_pp_soft_reset_async_wait(ip);
372
373 lima_pp_write_frame(ip, frame: frame->frame, wb: frame->wb);
374
375 for (i = 0; i < frame->num_pp; i++) {
376 struct lima_ip *ip = pipe->processor[i];
377
378 pp_write(LIMA_PP_STACK, frame->fragment_stack_address[i]);
379 if (!frame->use_dlbu)
380 pp_write(LIMA_PP_FRAME, frame->plbu_array_address[i]);
381 }
382
383 pp_write(LIMA_PP_CTRL, LIMA_PP_CTRL_START_RENDERING);
384 } else {
385 struct drm_lima_m400_pp_frame *frame = task->frame;
386 int i;
387
388 atomic_set(v: &pipe->task, i: frame->num_pp);
389
390 for (i = 0; i < frame->num_pp; i++) {
391 struct lima_ip *ip = pipe->processor[i];
392
393 frame->frame[LIMA_PP_FRAME >> 2] =
394 frame->plbu_array_address[i];
395 frame->frame[LIMA_PP_STACK >> 2] =
396 frame->fragment_stack_address[i];
397
398 lima_pp_soft_reset_async_wait(ip);
399
400 lima_pp_write_frame(ip, frame: frame->frame, wb: frame->wb);
401
402 pp_write(LIMA_PP_CTRL, LIMA_PP_CTRL_START_RENDERING);
403 }
404 }
405}
406
407static void lima_pp_task_fini(struct lima_sched_pipe *pipe)
408{
409 if (pipe->bcast_processor)
410 lima_pp_soft_reset_async(ip: pipe->bcast_processor);
411 else {
412 int i;
413
414 for (i = 0; i < pipe->num_processor; i++)
415 lima_pp_soft_reset_async(ip: pipe->processor[i]);
416 }
417}
418
419static void lima_pp_task_error(struct lima_sched_pipe *pipe)
420{
421 int i;
422
423 for (i = 0; i < pipe->num_processor; i++) {
424 struct lima_ip *ip = pipe->processor[i];
425
426 dev_err(ip->dev->dev, "%s task error %d int_state=%x status=%x\n",
427 lima_ip_name(ip), i, pp_read(LIMA_PP_INT_STATUS),
428 pp_read(LIMA_PP_STATUS));
429
430 lima_pp_hard_reset(ip);
431 }
432}
433
434static void lima_pp_task_mmu_error(struct lima_sched_pipe *pipe)
435{
436 if (atomic_dec_and_test(v: &pipe->task))
437 lima_sched_pipe_task_done(pipe);
438}
439
440static struct kmem_cache *lima_pp_task_slab;
441static int lima_pp_task_slab_refcnt;
442
443int lima_pp_pipe_init(struct lima_device *dev)
444{
445 int frame_size;
446 struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp;
447
448 if (dev->id == lima_gpu_mali400)
449 frame_size = sizeof(struct drm_lima_m400_pp_frame);
450 else
451 frame_size = sizeof(struct drm_lima_m450_pp_frame);
452
453 if (!lima_pp_task_slab) {
454 lima_pp_task_slab = kmem_cache_create_usercopy(
455 name: "lima_pp_task", size: sizeof(struct lima_sched_task) + frame_size,
456 align: 0, SLAB_HWCACHE_ALIGN, useroffset: sizeof(struct lima_sched_task),
457 usersize: frame_size, NULL);
458 if (!lima_pp_task_slab)
459 return -ENOMEM;
460 }
461 lima_pp_task_slab_refcnt++;
462
463 pipe->frame_size = frame_size;
464 pipe->task_slab = lima_pp_task_slab;
465
466 pipe->task_validate = lima_pp_task_validate;
467 pipe->task_run = lima_pp_task_run;
468 pipe->task_fini = lima_pp_task_fini;
469 pipe->task_error = lima_pp_task_error;
470 pipe->task_mmu_error = lima_pp_task_mmu_error;
471
472 return 0;
473}
474
475void lima_pp_pipe_fini(struct lima_device *dev)
476{
477 if (!--lima_pp_task_slab_refcnt) {
478 kmem_cache_destroy(s: lima_pp_task_slab);
479 lima_pp_task_slab = NULL;
480 }
481}
482

source code of linux/drivers/gpu/drm/lima/lima_pp.c