1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * fence-chain: chain fences together in a timeline |
4 | * |
5 | * Copyright (C) 2018 Advanced Micro Devices, Inc. |
6 | * Authors: |
7 | * Christian König <christian.koenig@amd.com> |
8 | */ |
9 | |
10 | #include <linux/dma-fence-chain.h> |
11 | |
12 | static bool dma_fence_chain_enable_signaling(struct dma_fence *fence); |
13 | |
14 | /** |
15 | * dma_fence_chain_get_prev - use RCU to get a reference to the previous fence |
16 | * @chain: chain node to get the previous node from |
17 | * |
18 | * Use dma_fence_get_rcu_safe to get a reference to the previous fence of the |
19 | * chain node. |
20 | */ |
21 | static struct dma_fence *dma_fence_chain_get_prev(struct dma_fence_chain *chain) |
22 | { |
23 | struct dma_fence *prev; |
24 | |
25 | rcu_read_lock(); |
26 | prev = dma_fence_get_rcu_safe(fencep: &chain->prev); |
27 | rcu_read_unlock(); |
28 | return prev; |
29 | } |
30 | |
31 | /** |
32 | * dma_fence_chain_walk - chain walking function |
33 | * @fence: current chain node |
34 | * |
35 | * Walk the chain to the next node. Returns the next fence or NULL if we are at |
36 | * the end of the chain. Garbage collects chain nodes which are already |
37 | * signaled. |
38 | */ |
39 | struct dma_fence *dma_fence_chain_walk(struct dma_fence *fence) |
40 | { |
41 | struct dma_fence_chain *chain, *prev_chain; |
42 | struct dma_fence *prev, *replacement, *tmp; |
43 | |
44 | chain = to_dma_fence_chain(fence); |
45 | if (!chain) { |
46 | dma_fence_put(fence); |
47 | return NULL; |
48 | } |
49 | |
50 | while ((prev = dma_fence_chain_get_prev(chain))) { |
51 | |
52 | prev_chain = to_dma_fence_chain(fence: prev); |
53 | if (prev_chain) { |
54 | if (!dma_fence_is_signaled(fence: prev_chain->fence)) |
55 | break; |
56 | |
57 | replacement = dma_fence_chain_get_prev(chain: prev_chain); |
58 | } else { |
59 | if (!dma_fence_is_signaled(fence: prev)) |
60 | break; |
61 | |
62 | replacement = NULL; |
63 | } |
64 | |
65 | tmp = unrcu_pointer(cmpxchg(&chain->prev, RCU_INITIALIZER(prev), |
66 | RCU_INITIALIZER(replacement))); |
67 | if (tmp == prev) |
68 | dma_fence_put(fence: tmp); |
69 | else |
70 | dma_fence_put(fence: replacement); |
71 | dma_fence_put(fence: prev); |
72 | } |
73 | |
74 | dma_fence_put(fence); |
75 | return prev; |
76 | } |
77 | EXPORT_SYMBOL(dma_fence_chain_walk); |
78 | |
79 | /** |
80 | * dma_fence_chain_find_seqno - find fence chain node by seqno |
81 | * @pfence: pointer to the chain node where to start |
82 | * @seqno: the sequence number to search for |
83 | * |
84 | * Advance the fence pointer to the chain node which will signal this sequence |
85 | * number. If no sequence number is provided then this is a no-op. |
86 | * |
87 | * Returns EINVAL if the fence is not a chain node or the sequence number has |
88 | * not yet advanced far enough. |
89 | */ |
90 | int dma_fence_chain_find_seqno(struct dma_fence **pfence, uint64_t seqno) |
91 | { |
92 | struct dma_fence_chain *chain; |
93 | |
94 | if (!seqno) |
95 | return 0; |
96 | |
97 | chain = to_dma_fence_chain(fence: *pfence); |
98 | if (!chain || chain->base.seqno < seqno) |
99 | return -EINVAL; |
100 | |
101 | dma_fence_chain_for_each(*pfence, &chain->base) { |
102 | if ((*pfence)->context != chain->base.context || |
103 | to_dma_fence_chain(fence: *pfence)->prev_seqno < seqno) |
104 | break; |
105 | } |
106 | dma_fence_put(fence: &chain->base); |
107 | |
108 | return 0; |
109 | } |
110 | EXPORT_SYMBOL(dma_fence_chain_find_seqno); |
111 | |
112 | static const char *dma_fence_chain_get_driver_name(struct dma_fence *fence) |
113 | { |
114 | return "dma_fence_chain" ; |
115 | } |
116 | |
117 | static const char *dma_fence_chain_get_timeline_name(struct dma_fence *fence) |
118 | { |
119 | return "unbound" ; |
120 | } |
121 | |
122 | static void dma_fence_chain_irq_work(struct irq_work *work) |
123 | { |
124 | struct dma_fence_chain *chain; |
125 | |
126 | chain = container_of(work, typeof(*chain), work); |
127 | |
128 | /* Try to rearm the callback */ |
129 | if (!dma_fence_chain_enable_signaling(fence: &chain->base)) |
130 | /* Ok, we are done. No more unsignaled fences left */ |
131 | dma_fence_signal(fence: &chain->base); |
132 | dma_fence_put(fence: &chain->base); |
133 | } |
134 | |
135 | static void dma_fence_chain_cb(struct dma_fence *f, struct dma_fence_cb *cb) |
136 | { |
137 | struct dma_fence_chain *chain; |
138 | |
139 | chain = container_of(cb, typeof(*chain), cb); |
140 | init_irq_work(work: &chain->work, func: dma_fence_chain_irq_work); |
141 | irq_work_queue(work: &chain->work); |
142 | dma_fence_put(fence: f); |
143 | } |
144 | |
145 | static bool dma_fence_chain_enable_signaling(struct dma_fence *fence) |
146 | { |
147 | struct dma_fence_chain *head = to_dma_fence_chain(fence); |
148 | |
149 | dma_fence_get(fence: &head->base); |
150 | dma_fence_chain_for_each(fence, &head->base) { |
151 | struct dma_fence *f = dma_fence_chain_contained(fence); |
152 | |
153 | dma_fence_get(fence: f); |
154 | if (!dma_fence_add_callback(fence: f, cb: &head->cb, func: dma_fence_chain_cb)) { |
155 | dma_fence_put(fence); |
156 | return true; |
157 | } |
158 | dma_fence_put(fence: f); |
159 | } |
160 | dma_fence_put(fence: &head->base); |
161 | return false; |
162 | } |
163 | |
164 | static bool dma_fence_chain_signaled(struct dma_fence *fence) |
165 | { |
166 | dma_fence_chain_for_each(fence, fence) { |
167 | struct dma_fence *f = dma_fence_chain_contained(fence); |
168 | |
169 | if (!dma_fence_is_signaled(fence: f)) { |
170 | dma_fence_put(fence); |
171 | return false; |
172 | } |
173 | } |
174 | |
175 | return true; |
176 | } |
177 | |
178 | static void dma_fence_chain_release(struct dma_fence *fence) |
179 | { |
180 | struct dma_fence_chain *chain = to_dma_fence_chain(fence); |
181 | struct dma_fence *prev; |
182 | |
183 | /* Manually unlink the chain as much as possible to avoid recursion |
184 | * and potential stack overflow. |
185 | */ |
186 | while ((prev = rcu_dereference_protected(chain->prev, true))) { |
187 | struct dma_fence_chain *prev_chain; |
188 | |
189 | if (kref_read(kref: &prev->refcount) > 1) |
190 | break; |
191 | |
192 | prev_chain = to_dma_fence_chain(fence: prev); |
193 | if (!prev_chain) |
194 | break; |
195 | |
196 | /* No need for atomic operations since we hold the last |
197 | * reference to prev_chain. |
198 | */ |
199 | chain->prev = prev_chain->prev; |
200 | RCU_INIT_POINTER(prev_chain->prev, NULL); |
201 | dma_fence_put(fence: prev); |
202 | } |
203 | dma_fence_put(fence: prev); |
204 | |
205 | dma_fence_put(fence: chain->fence); |
206 | dma_fence_free(fence); |
207 | } |
208 | |
209 | |
210 | static void dma_fence_chain_set_deadline(struct dma_fence *fence, |
211 | ktime_t deadline) |
212 | { |
213 | dma_fence_chain_for_each(fence, fence) { |
214 | struct dma_fence *f = dma_fence_chain_contained(fence); |
215 | |
216 | dma_fence_set_deadline(fence: f, deadline); |
217 | } |
218 | } |
219 | |
220 | const struct dma_fence_ops dma_fence_chain_ops = { |
221 | .use_64bit_seqno = true, |
222 | .get_driver_name = dma_fence_chain_get_driver_name, |
223 | .get_timeline_name = dma_fence_chain_get_timeline_name, |
224 | .enable_signaling = dma_fence_chain_enable_signaling, |
225 | .signaled = dma_fence_chain_signaled, |
226 | .release = dma_fence_chain_release, |
227 | .set_deadline = dma_fence_chain_set_deadline, |
228 | }; |
229 | EXPORT_SYMBOL(dma_fence_chain_ops); |
230 | |
231 | /** |
232 | * dma_fence_chain_init - initialize a fence chain |
233 | * @chain: the chain node to initialize |
234 | * @prev: the previous fence |
235 | * @fence: the current fence |
236 | * @seqno: the sequence number to use for the fence chain |
237 | * |
238 | * Initialize a new chain node and either start a new chain or add the node to |
239 | * the existing chain of the previous fence. |
240 | */ |
241 | void dma_fence_chain_init(struct dma_fence_chain *chain, |
242 | struct dma_fence *prev, |
243 | struct dma_fence *fence, |
244 | uint64_t seqno) |
245 | { |
246 | struct dma_fence_chain *prev_chain = to_dma_fence_chain(fence: prev); |
247 | uint64_t context; |
248 | |
249 | spin_lock_init(&chain->lock); |
250 | rcu_assign_pointer(chain->prev, prev); |
251 | chain->fence = fence; |
252 | chain->prev_seqno = 0; |
253 | |
254 | /* Try to reuse the context of the previous chain node. */ |
255 | if (prev_chain && __dma_fence_is_later(f1: seqno, f2: prev->seqno, ops: prev->ops)) { |
256 | context = prev->context; |
257 | chain->prev_seqno = prev->seqno; |
258 | } else { |
259 | context = dma_fence_context_alloc(num: 1); |
260 | /* Make sure that we always have a valid sequence number. */ |
261 | if (prev_chain) |
262 | seqno = max(prev->seqno, seqno); |
263 | } |
264 | |
265 | dma_fence_init(fence: &chain->base, ops: &dma_fence_chain_ops, |
266 | lock: &chain->lock, context, seqno); |
267 | |
268 | /* |
269 | * Chaining dma_fence_chain container together is only allowed through |
270 | * the prev fence and not through the contained fence. |
271 | * |
272 | * The correct way of handling this is to flatten out the fence |
273 | * structure into a dma_fence_array by the caller instead. |
274 | */ |
275 | WARN_ON(dma_fence_is_chain(fence)); |
276 | } |
277 | EXPORT_SYMBOL(dma_fence_chain_init); |
278 | |