1 | // SPDX-License-Identifier: GPL-2.0 OR MIT |
2 | /* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */ |
3 | |
4 | #include <linux/interrupt.h> |
5 | #include <linux/iopoll.h> |
6 | #include <linux/device.h> |
7 | #include <linux/slab.h> |
8 | |
9 | #include <drm/lima_drm.h> |
10 | |
11 | #include "lima_device.h" |
12 | #include "lima_gp.h" |
13 | #include "lima_regs.h" |
14 | #include "lima_gem.h" |
15 | #include "lima_vm.h" |
16 | |
17 | #define gp_write(reg, data) writel(data, ip->iomem + reg) |
18 | #define gp_read(reg) readl(ip->iomem + reg) |
19 | |
20 | static irqreturn_t lima_gp_irq_handler(int irq, void *data) |
21 | { |
22 | struct lima_ip *ip = data; |
23 | struct lima_device *dev = ip->dev; |
24 | struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_gp; |
25 | struct lima_sched_task *task = pipe->current_task; |
26 | u32 state = gp_read(LIMA_GP_INT_STAT); |
27 | u32 status = gp_read(LIMA_GP_STATUS); |
28 | bool done = false; |
29 | |
30 | /* for shared irq case */ |
31 | if (!state) |
32 | return IRQ_NONE; |
33 | |
34 | if (state & LIMA_GP_IRQ_MASK_ERROR) { |
35 | if ((state & LIMA_GP_IRQ_MASK_ERROR) == |
36 | LIMA_GP_IRQ_PLBU_OUT_OF_MEM) { |
37 | dev_dbg(dev->dev, "%s out of heap irq status=%x\n" , |
38 | lima_ip_name(ip), status); |
39 | } else { |
40 | dev_err(dev->dev, "%s error irq state=%x status=%x\n" , |
41 | lima_ip_name(ip), state, status); |
42 | if (task) |
43 | task->recoverable = false; |
44 | } |
45 | |
46 | /* mask all interrupts before hard reset */ |
47 | gp_write(LIMA_GP_INT_MASK, 0); |
48 | |
49 | pipe->error = true; |
50 | done = true; |
51 | } else { |
52 | bool valid = state & (LIMA_GP_IRQ_VS_END_CMD_LST | |
53 | LIMA_GP_IRQ_PLBU_END_CMD_LST); |
54 | bool active = status & (LIMA_GP_STATUS_VS_ACTIVE | |
55 | LIMA_GP_STATUS_PLBU_ACTIVE); |
56 | done = valid && !active; |
57 | pipe->error = false; |
58 | } |
59 | |
60 | gp_write(LIMA_GP_INT_CLEAR, state); |
61 | |
62 | if (done) |
63 | lima_sched_pipe_task_done(pipe); |
64 | |
65 | return IRQ_HANDLED; |
66 | } |
67 | |
68 | static void lima_gp_soft_reset_async(struct lima_ip *ip) |
69 | { |
70 | if (ip->data.async_reset) |
71 | return; |
72 | |
73 | gp_write(LIMA_GP_INT_MASK, 0); |
74 | gp_write(LIMA_GP_INT_CLEAR, LIMA_GP_IRQ_RESET_COMPLETED); |
75 | gp_write(LIMA_GP_CMD, LIMA_GP_CMD_SOFT_RESET); |
76 | ip->data.async_reset = true; |
77 | } |
78 | |
79 | static int lima_gp_soft_reset_async_wait(struct lima_ip *ip) |
80 | { |
81 | struct lima_device *dev = ip->dev; |
82 | int err; |
83 | u32 v; |
84 | |
85 | if (!ip->data.async_reset) |
86 | return 0; |
87 | |
88 | err = readl_poll_timeout(ip->iomem + LIMA_GP_INT_RAWSTAT, v, |
89 | v & LIMA_GP_IRQ_RESET_COMPLETED, |
90 | 0, 100); |
91 | if (err) { |
92 | dev_err(dev->dev, "%s soft reset time out\n" , |
93 | lima_ip_name(ip)); |
94 | return err; |
95 | } |
96 | |
97 | gp_write(LIMA_GP_INT_CLEAR, LIMA_GP_IRQ_MASK_ALL); |
98 | gp_write(LIMA_GP_INT_MASK, LIMA_GP_IRQ_MASK_USED); |
99 | |
100 | ip->data.async_reset = false; |
101 | return 0; |
102 | } |
103 | |
104 | static int lima_gp_task_validate(struct lima_sched_pipe *pipe, |
105 | struct lima_sched_task *task) |
106 | { |
107 | struct drm_lima_gp_frame *frame = task->frame; |
108 | u32 *f = frame->frame; |
109 | (void)pipe; |
110 | |
111 | if (f[LIMA_GP_VSCL_START_ADDR >> 2] > |
112 | f[LIMA_GP_VSCL_END_ADDR >> 2] || |
113 | f[LIMA_GP_PLBUCL_START_ADDR >> 2] > |
114 | f[LIMA_GP_PLBUCL_END_ADDR >> 2] || |
115 | f[LIMA_GP_PLBU_ALLOC_START_ADDR >> 2] > |
116 | f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2]) |
117 | return -EINVAL; |
118 | |
119 | if (f[LIMA_GP_VSCL_START_ADDR >> 2] == |
120 | f[LIMA_GP_VSCL_END_ADDR >> 2] && |
121 | f[LIMA_GP_PLBUCL_START_ADDR >> 2] == |
122 | f[LIMA_GP_PLBUCL_END_ADDR >> 2]) |
123 | return -EINVAL; |
124 | |
125 | return 0; |
126 | } |
127 | |
128 | static void lima_gp_task_run(struct lima_sched_pipe *pipe, |
129 | struct lima_sched_task *task) |
130 | { |
131 | struct lima_ip *ip = pipe->processor[0]; |
132 | struct drm_lima_gp_frame *frame = task->frame; |
133 | u32 *f = frame->frame; |
134 | u32 cmd = 0; |
135 | int i; |
136 | |
137 | /* update real heap buffer size for GP */ |
138 | for (i = 0; i < task->num_bos; i++) { |
139 | struct lima_bo *bo = task->bos[i]; |
140 | |
141 | if (bo->heap_size && |
142 | lima_vm_get_va(vm: task->vm, bo) == |
143 | f[LIMA_GP_PLBU_ALLOC_START_ADDR >> 2]) { |
144 | f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2] = |
145 | f[LIMA_GP_PLBU_ALLOC_START_ADDR >> 2] + |
146 | bo->heap_size; |
147 | task->recoverable = true; |
148 | task->heap = bo; |
149 | break; |
150 | } |
151 | } |
152 | |
153 | if (f[LIMA_GP_VSCL_START_ADDR >> 2] != |
154 | f[LIMA_GP_VSCL_END_ADDR >> 2]) |
155 | cmd |= LIMA_GP_CMD_START_VS; |
156 | if (f[LIMA_GP_PLBUCL_START_ADDR >> 2] != |
157 | f[LIMA_GP_PLBUCL_END_ADDR >> 2]) |
158 | cmd |= LIMA_GP_CMD_START_PLBU; |
159 | |
160 | /* before any hw ops, wait last success task async soft reset */ |
161 | lima_gp_soft_reset_async_wait(ip); |
162 | |
163 | for (i = 0; i < LIMA_GP_FRAME_REG_NUM; i++) |
164 | writel(val: f[i], addr: ip->iomem + LIMA_GP_VSCL_START_ADDR + i * 4); |
165 | |
166 | gp_write(LIMA_GP_CMD, LIMA_GP_CMD_UPDATE_PLBU_ALLOC); |
167 | gp_write(LIMA_GP_CMD, cmd); |
168 | } |
169 | |
170 | static int lima_gp_bus_stop_poll(struct lima_ip *ip) |
171 | { |
172 | return !!(gp_read(LIMA_GP_STATUS) & LIMA_GP_STATUS_BUS_STOPPED); |
173 | } |
174 | |
175 | static int lima_gp_hard_reset_poll(struct lima_ip *ip) |
176 | { |
177 | gp_write(LIMA_GP_PERF_CNT_0_LIMIT, 0xC01A0000); |
178 | return gp_read(LIMA_GP_PERF_CNT_0_LIMIT) == 0xC01A0000; |
179 | } |
180 | |
181 | static int lima_gp_hard_reset(struct lima_ip *ip) |
182 | { |
183 | struct lima_device *dev = ip->dev; |
184 | int ret; |
185 | |
186 | gp_write(LIMA_GP_PERF_CNT_0_LIMIT, 0xC0FFE000); |
187 | gp_write(LIMA_GP_INT_MASK, 0); |
188 | |
189 | gp_write(LIMA_GP_CMD, LIMA_GP_CMD_STOP_BUS); |
190 | ret = lima_poll_timeout(ip, func: lima_gp_bus_stop_poll, sleep_us: 10, timeout_us: 100); |
191 | if (ret) { |
192 | dev_err(dev->dev, "%s bus stop timeout\n" , lima_ip_name(ip)); |
193 | return ret; |
194 | } |
195 | gp_write(LIMA_GP_CMD, LIMA_GP_CMD_RESET); |
196 | ret = lima_poll_timeout(ip, func: lima_gp_hard_reset_poll, sleep_us: 10, timeout_us: 100); |
197 | if (ret) { |
198 | dev_err(dev->dev, "%s hard reset timeout\n" , lima_ip_name(ip)); |
199 | return ret; |
200 | } |
201 | |
202 | gp_write(LIMA_GP_PERF_CNT_0_LIMIT, 0); |
203 | gp_write(LIMA_GP_INT_CLEAR, LIMA_GP_IRQ_MASK_ALL); |
204 | gp_write(LIMA_GP_INT_MASK, LIMA_GP_IRQ_MASK_USED); |
205 | |
206 | /* |
207 | * if there was an async soft reset queued, |
208 | * don't wait for it in the next job |
209 | */ |
210 | ip->data.async_reset = false; |
211 | |
212 | return 0; |
213 | } |
214 | |
215 | static void lima_gp_task_fini(struct lima_sched_pipe *pipe) |
216 | { |
217 | lima_gp_soft_reset_async(ip: pipe->processor[0]); |
218 | } |
219 | |
220 | static void lima_gp_task_error(struct lima_sched_pipe *pipe) |
221 | { |
222 | struct lima_ip *ip = pipe->processor[0]; |
223 | |
224 | dev_err(ip->dev->dev, "%s task error int_state=%x status=%x\n" , |
225 | lima_ip_name(ip), gp_read(LIMA_GP_INT_STAT), |
226 | gp_read(LIMA_GP_STATUS)); |
227 | |
228 | lima_gp_hard_reset(ip); |
229 | } |
230 | |
231 | static void lima_gp_task_mmu_error(struct lima_sched_pipe *pipe) |
232 | { |
233 | lima_sched_pipe_task_done(pipe); |
234 | } |
235 | |
236 | static int lima_gp_task_recover(struct lima_sched_pipe *pipe) |
237 | { |
238 | struct lima_ip *ip = pipe->processor[0]; |
239 | struct lima_sched_task *task = pipe->current_task; |
240 | struct drm_lima_gp_frame *frame = task->frame; |
241 | u32 *f = frame->frame; |
242 | size_t fail_size = |
243 | f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2] - |
244 | f[LIMA_GP_PLBU_ALLOC_START_ADDR >> 2]; |
245 | |
246 | if (fail_size == task->heap->heap_size) { |
247 | int ret; |
248 | |
249 | ret = lima_heap_alloc(bo: task->heap, vm: task->vm); |
250 | if (ret < 0) |
251 | return ret; |
252 | } |
253 | |
254 | gp_write(LIMA_GP_INT_MASK, LIMA_GP_IRQ_MASK_USED); |
255 | /* Resume from where we stopped, i.e. new start is old end */ |
256 | gp_write(LIMA_GP_PLBU_ALLOC_START_ADDR, |
257 | f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2]); |
258 | f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2] = |
259 | f[LIMA_GP_PLBU_ALLOC_START_ADDR >> 2] + task->heap->heap_size; |
260 | gp_write(LIMA_GP_PLBU_ALLOC_END_ADDR, |
261 | f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2]); |
262 | gp_write(LIMA_GP_CMD, LIMA_GP_CMD_UPDATE_PLBU_ALLOC); |
263 | return 0; |
264 | } |
265 | |
266 | static void lima_gp_print_version(struct lima_ip *ip) |
267 | { |
268 | u32 version, major, minor; |
269 | char *name; |
270 | |
271 | version = gp_read(LIMA_GP_VERSION); |
272 | major = (version >> 8) & 0xFF; |
273 | minor = version & 0xFF; |
274 | switch (version >> 16) { |
275 | case 0xA07: |
276 | name = "mali200" ; |
277 | break; |
278 | case 0xC07: |
279 | name = "mali300" ; |
280 | break; |
281 | case 0xB07: |
282 | name = "mali400" ; |
283 | break; |
284 | case 0xD07: |
285 | name = "mali450" ; |
286 | break; |
287 | default: |
288 | name = "unknown" ; |
289 | break; |
290 | } |
291 | dev_info(ip->dev->dev, "%s - %s version major %d minor %d\n" , |
292 | lima_ip_name(ip), name, major, minor); |
293 | } |
294 | |
295 | static struct kmem_cache *lima_gp_task_slab; |
296 | static int lima_gp_task_slab_refcnt; |
297 | |
298 | static int lima_gp_hw_init(struct lima_ip *ip) |
299 | { |
300 | ip->data.async_reset = false; |
301 | lima_gp_soft_reset_async(ip); |
302 | return lima_gp_soft_reset_async_wait(ip); |
303 | } |
304 | |
305 | int lima_gp_resume(struct lima_ip *ip) |
306 | { |
307 | return lima_gp_hw_init(ip); |
308 | } |
309 | |
310 | void lima_gp_suspend(struct lima_ip *ip) |
311 | { |
312 | |
313 | } |
314 | |
315 | int lima_gp_init(struct lima_ip *ip) |
316 | { |
317 | struct lima_device *dev = ip->dev; |
318 | int err; |
319 | |
320 | lima_gp_print_version(ip); |
321 | |
322 | err = lima_gp_hw_init(ip); |
323 | if (err) |
324 | return err; |
325 | |
326 | err = devm_request_irq(dev: dev->dev, irq: ip->irq, handler: lima_gp_irq_handler, |
327 | IRQF_SHARED, devname: lima_ip_name(ip), dev_id: ip); |
328 | if (err) { |
329 | dev_err(dev->dev, "%s fail to request irq\n" , |
330 | lima_ip_name(ip)); |
331 | return err; |
332 | } |
333 | |
334 | dev->gp_version = gp_read(LIMA_GP_VERSION); |
335 | |
336 | return 0; |
337 | } |
338 | |
339 | void lima_gp_fini(struct lima_ip *ip) |
340 | { |
341 | |
342 | } |
343 | |
344 | int lima_gp_pipe_init(struct lima_device *dev) |
345 | { |
346 | int frame_size = sizeof(struct drm_lima_gp_frame); |
347 | struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_gp; |
348 | |
349 | if (!lima_gp_task_slab) { |
350 | lima_gp_task_slab = kmem_cache_create_usercopy( |
351 | name: "lima_gp_task" , size: sizeof(struct lima_sched_task) + frame_size, |
352 | align: 0, SLAB_HWCACHE_ALIGN, useroffset: sizeof(struct lima_sched_task), |
353 | usersize: frame_size, NULL); |
354 | if (!lima_gp_task_slab) |
355 | return -ENOMEM; |
356 | } |
357 | lima_gp_task_slab_refcnt++; |
358 | |
359 | pipe->frame_size = frame_size; |
360 | pipe->task_slab = lima_gp_task_slab; |
361 | |
362 | pipe->task_validate = lima_gp_task_validate; |
363 | pipe->task_run = lima_gp_task_run; |
364 | pipe->task_fini = lima_gp_task_fini; |
365 | pipe->task_error = lima_gp_task_error; |
366 | pipe->task_mmu_error = lima_gp_task_mmu_error; |
367 | pipe->task_recover = lima_gp_task_recover; |
368 | |
369 | return 0; |
370 | } |
371 | |
372 | void lima_gp_pipe_fini(struct lima_device *dev) |
373 | { |
374 | if (!--lima_gp_task_slab_refcnt) { |
375 | kmem_cache_destroy(s: lima_gp_task_slab); |
376 | lima_gp_task_slab = NULL; |
377 | } |
378 | } |
379 | |