1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright (C) 2017 Etnaviv Project |
4 | */ |
5 | |
6 | #include <linux/moduleparam.h> |
7 | |
8 | #include "etnaviv_drv.h" |
9 | #include "etnaviv_dump.h" |
10 | #include "etnaviv_gem.h" |
11 | #include "etnaviv_gpu.h" |
12 | #include "etnaviv_sched.h" |
13 | #include "state.xml.h" |
14 | |
15 | static int etnaviv_job_hang_limit = 0; |
16 | module_param_named(job_hang_limit, etnaviv_job_hang_limit, int , 0444); |
17 | static int etnaviv_hw_jobs_limit = 4; |
18 | module_param_named(hw_job_limit, etnaviv_hw_jobs_limit, int , 0444); |
19 | |
20 | static struct dma_fence *etnaviv_sched_run_job(struct drm_sched_job *sched_job) |
21 | { |
22 | struct etnaviv_gem_submit *submit = to_etnaviv_submit(sched_job); |
23 | struct dma_fence *fence = NULL; |
24 | |
25 | if (likely(!sched_job->s_fence->finished.error)) |
26 | fence = etnaviv_gpu_submit(submit); |
27 | else |
28 | dev_dbg(submit->gpu->dev, "skipping bad job\n" ); |
29 | |
30 | return fence; |
31 | } |
32 | |
33 | static enum drm_gpu_sched_stat etnaviv_sched_timedout_job(struct drm_sched_job |
34 | *sched_job) |
35 | { |
36 | struct etnaviv_gem_submit *submit = to_etnaviv_submit(sched_job); |
37 | struct etnaviv_gpu *gpu = submit->gpu; |
38 | u32 dma_addr; |
39 | int change; |
40 | |
41 | /* block scheduler */ |
42 | drm_sched_stop(sched: &gpu->sched, bad: sched_job); |
43 | |
44 | /* |
45 | * If the GPU managed to complete this jobs fence, the timout is |
46 | * spurious. Bail out. |
47 | */ |
48 | if (dma_fence_is_signaled(fence: submit->out_fence)) |
49 | goto out_no_timeout; |
50 | |
51 | /* |
52 | * If the GPU is still making forward progress on the front-end (which |
53 | * should never loop) we shift out the timeout to give it a chance to |
54 | * finish the job. |
55 | */ |
56 | dma_addr = gpu_read(gpu, VIVS_FE_DMA_ADDRESS); |
57 | change = dma_addr - gpu->hangcheck_dma_addr; |
58 | if (gpu->state == ETNA_GPU_STATE_RUNNING && |
59 | (gpu->completed_fence != gpu->hangcheck_fence || |
60 | change < 0 || change > 16)) { |
61 | gpu->hangcheck_dma_addr = dma_addr; |
62 | gpu->hangcheck_fence = gpu->completed_fence; |
63 | goto out_no_timeout; |
64 | } |
65 | |
66 | if(sched_job) |
67 | drm_sched_increase_karma(bad: sched_job); |
68 | |
69 | /* get the GPU back into the init state */ |
70 | etnaviv_core_dump(submit); |
71 | etnaviv_gpu_recover_hang(submit); |
72 | |
73 | drm_sched_resubmit_jobs(sched: &gpu->sched); |
74 | |
75 | drm_sched_start(sched: &gpu->sched, full_recovery: true); |
76 | return DRM_GPU_SCHED_STAT_NOMINAL; |
77 | |
78 | out_no_timeout: |
79 | /* restart scheduler after GPU is usable again */ |
80 | drm_sched_start(sched: &gpu->sched, full_recovery: true); |
81 | return DRM_GPU_SCHED_STAT_NOMINAL; |
82 | } |
83 | |
84 | static void etnaviv_sched_free_job(struct drm_sched_job *sched_job) |
85 | { |
86 | struct etnaviv_gem_submit *submit = to_etnaviv_submit(sched_job); |
87 | |
88 | drm_sched_job_cleanup(job: sched_job); |
89 | |
90 | etnaviv_submit_put(submit); |
91 | } |
92 | |
93 | static const struct drm_sched_backend_ops etnaviv_sched_ops = { |
94 | .run_job = etnaviv_sched_run_job, |
95 | .timedout_job = etnaviv_sched_timedout_job, |
96 | .free_job = etnaviv_sched_free_job, |
97 | }; |
98 | |
99 | int etnaviv_sched_push_job(struct etnaviv_gem_submit *submit) |
100 | { |
101 | struct etnaviv_gpu *gpu = submit->gpu; |
102 | int ret; |
103 | |
104 | /* |
105 | * Hold the sched lock across the whole operation to avoid jobs being |
106 | * pushed out of order with regard to their sched fence seqnos as |
107 | * allocated in drm_sched_job_arm. |
108 | */ |
109 | mutex_lock(&gpu->sched_lock); |
110 | |
111 | drm_sched_job_arm(job: &submit->sched_job); |
112 | |
113 | submit->out_fence = dma_fence_get(fence: &submit->sched_job.s_fence->finished); |
114 | ret = xa_alloc_cyclic(xa: &gpu->user_fences, id: &submit->out_fence_id, |
115 | entry: submit->out_fence, xa_limit_32b, |
116 | next: &gpu->next_user_fence, GFP_KERNEL); |
117 | if (ret < 0) { |
118 | drm_sched_job_cleanup(job: &submit->sched_job); |
119 | goto out_unlock; |
120 | } |
121 | |
122 | /* the scheduler holds on to the job now */ |
123 | kref_get(kref: &submit->refcount); |
124 | |
125 | drm_sched_entity_push_job(sched_job: &submit->sched_job); |
126 | |
127 | out_unlock: |
128 | mutex_unlock(lock: &gpu->sched_lock); |
129 | |
130 | return ret; |
131 | } |
132 | |
133 | int etnaviv_sched_init(struct etnaviv_gpu *gpu) |
134 | { |
135 | int ret; |
136 | |
137 | ret = drm_sched_init(sched: &gpu->sched, ops: &etnaviv_sched_ops, NULL, |
138 | num_rqs: DRM_SCHED_PRIORITY_COUNT, |
139 | credit_limit: etnaviv_hw_jobs_limit, hang_limit: etnaviv_job_hang_limit, |
140 | timeout: msecs_to_jiffies(m: 500), NULL, NULL, |
141 | name: dev_name(dev: gpu->dev), dev: gpu->dev); |
142 | if (ret) |
143 | return ret; |
144 | |
145 | return 0; |
146 | } |
147 | |
148 | void etnaviv_sched_fini(struct etnaviv_gpu *gpu) |
149 | { |
150 | drm_sched_fini(sched: &gpu->sched); |
151 | } |
152 | |