1 | /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ |
2 | /* |
3 | * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. |
4 | * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. |
5 | */ |
6 | |
7 | #ifndef RXE_QUEUE_H |
8 | #define RXE_QUEUE_H |
9 | |
10 | /* Implements a simple circular buffer that is shared between user |
11 | * and the driver and can be resized. The requested element size is |
12 | * rounded up to a power of 2 and the number of elements in the buffer |
13 | * is also rounded up to a power of 2. Since the queue is empty when |
14 | * the producer and consumer indices match the maximum capacity of the |
15 | * queue is one less than the number of element slots. |
16 | * |
17 | * Notes: |
18 | * - The driver indices are always masked off to q->index_mask |
19 | * before storing so do not need to be checked on reads. |
20 | * - The user whether user space or kernel is generally |
21 | * not trusted so its parameters are masked to make sure |
22 | * they do not access the queue out of bounds on reads. |
23 | * - The driver indices for queues must not be written |
24 | * by user so a local copy is used and a shared copy is |
25 | * stored when the local copy is changed. |
26 | * - By passing the type in the parameter list separate from q |
27 | * the compiler can eliminate the switch statement when the |
28 | * actual queue type is known when the function is called at |
29 | * compile time. |
30 | * - These queues are lock free. The user and driver must protect |
31 | * changes to their end of the queues with locks if more than one |
32 | * CPU can be accessing it at the same time. |
33 | */ |
34 | |
35 | /** |
36 | * enum queue_type - type of queue |
37 | * @QUEUE_TYPE_TO_CLIENT: Queue is written by rxe driver and |
38 | * read by client which may be a user space |
39 | * application or a kernel ulp. |
40 | * Used by rxe internals only. |
41 | * @QUEUE_TYPE_FROM_CLIENT: Queue is written by client and |
42 | * read by rxe driver. |
43 | * Used by rxe internals only. |
44 | * @QUEUE_TYPE_FROM_ULP: Queue is written by kernel ulp and |
45 | * read by rxe driver. |
46 | * Used by kernel verbs APIs only on |
47 | * behalf of ulps. |
48 | * @QUEUE_TYPE_TO_ULP: Queue is written by rxe driver and |
49 | * read by kernel ulp. |
50 | * Used by kernel verbs APIs only on |
51 | * behalf of ulps. |
52 | */ |
53 | enum queue_type { |
54 | QUEUE_TYPE_TO_CLIENT, |
55 | QUEUE_TYPE_FROM_CLIENT, |
56 | QUEUE_TYPE_FROM_ULP, |
57 | QUEUE_TYPE_TO_ULP, |
58 | }; |
59 | |
60 | struct rxe_queue_buf; |
61 | |
62 | struct rxe_queue { |
63 | struct rxe_dev *rxe; |
64 | struct rxe_queue_buf *buf; |
65 | struct rxe_mmap_info *ip; |
66 | size_t buf_size; |
67 | size_t elem_size; |
68 | unsigned int log2_elem_size; |
69 | u32 index_mask; |
70 | enum queue_type type; |
71 | /* private copy of index for shared queues between |
72 | * driver and clients. Driver reads and writes |
73 | * this copy and then replicates to rxe_queue_buf |
74 | * for read access by clients. |
75 | */ |
76 | u32 index; |
77 | }; |
78 | |
79 | int do_mmap_info(struct rxe_dev *rxe, struct mminfo __user *outbuf, |
80 | struct ib_udata *udata, struct rxe_queue_buf *buf, |
81 | size_t buf_size, struct rxe_mmap_info **ip_p); |
82 | |
83 | void rxe_queue_reset(struct rxe_queue *q); |
84 | |
85 | struct rxe_queue *rxe_queue_init(struct rxe_dev *rxe, int *num_elem, |
86 | unsigned int elem_size, enum queue_type type); |
87 | |
88 | int rxe_queue_resize(struct rxe_queue *q, unsigned int *num_elem_p, |
89 | unsigned int elem_size, struct ib_udata *udata, |
90 | struct mminfo __user *outbuf, |
91 | spinlock_t *producer_lock, spinlock_t *consumer_lock); |
92 | |
93 | void rxe_queue_cleanup(struct rxe_queue *queue); |
94 | |
95 | static inline u32 queue_next_index(struct rxe_queue *q, int index) |
96 | { |
97 | return (index + 1) & q->index_mask; |
98 | } |
99 | |
100 | static inline u32 queue_get_producer(const struct rxe_queue *q, |
101 | enum queue_type type) |
102 | { |
103 | u32 prod; |
104 | |
105 | switch (type) { |
106 | case QUEUE_TYPE_FROM_CLIENT: |
107 | /* used by rxe, client owns the index */ |
108 | prod = smp_load_acquire(&q->buf->producer_index); |
109 | break; |
110 | case QUEUE_TYPE_TO_CLIENT: |
111 | /* used by rxe which owns the index */ |
112 | prod = q->index; |
113 | break; |
114 | case QUEUE_TYPE_FROM_ULP: |
115 | /* used by ulp which owns the index */ |
116 | prod = q->buf->producer_index; |
117 | break; |
118 | case QUEUE_TYPE_TO_ULP: |
119 | /* used by ulp, rxe owns the index */ |
120 | prod = smp_load_acquire(&q->buf->producer_index); |
121 | break; |
122 | } |
123 | |
124 | return prod; |
125 | } |
126 | |
127 | static inline u32 queue_get_consumer(const struct rxe_queue *q, |
128 | enum queue_type type) |
129 | { |
130 | u32 cons; |
131 | |
132 | switch (type) { |
133 | case QUEUE_TYPE_FROM_CLIENT: |
134 | /* used by rxe which owns the index */ |
135 | cons = q->index; |
136 | break; |
137 | case QUEUE_TYPE_TO_CLIENT: |
138 | /* used by rxe, client owns the index */ |
139 | cons = smp_load_acquire(&q->buf->consumer_index); |
140 | break; |
141 | case QUEUE_TYPE_FROM_ULP: |
142 | /* used by ulp, rxe owns the index */ |
143 | cons = smp_load_acquire(&q->buf->consumer_index); |
144 | break; |
145 | case QUEUE_TYPE_TO_ULP: |
146 | /* used by ulp which owns the index */ |
147 | cons = q->buf->consumer_index; |
148 | break; |
149 | } |
150 | |
151 | return cons; |
152 | } |
153 | |
154 | static inline int queue_empty(struct rxe_queue *q, enum queue_type type) |
155 | { |
156 | u32 prod = queue_get_producer(q, type); |
157 | u32 cons = queue_get_consumer(q, type); |
158 | |
159 | return ((prod - cons) & q->index_mask) == 0; |
160 | } |
161 | |
162 | static inline int queue_full(struct rxe_queue *q, enum queue_type type) |
163 | { |
164 | u32 prod = queue_get_producer(q, type); |
165 | u32 cons = queue_get_consumer(q, type); |
166 | |
167 | return ((prod + 1 - cons) & q->index_mask) == 0; |
168 | } |
169 | |
170 | static inline u32 queue_count(const struct rxe_queue *q, |
171 | enum queue_type type) |
172 | { |
173 | u32 prod = queue_get_producer(q, type); |
174 | u32 cons = queue_get_consumer(q, type); |
175 | |
176 | return (prod - cons) & q->index_mask; |
177 | } |
178 | |
179 | static inline void queue_advance_producer(struct rxe_queue *q, |
180 | enum queue_type type) |
181 | { |
182 | u32 prod; |
183 | |
184 | switch (type) { |
185 | case QUEUE_TYPE_FROM_CLIENT: |
186 | /* used by rxe, client owns the index */ |
187 | if (WARN_ON(1)) |
188 | pr_warn("%s: attempt to advance client index\n" , |
189 | __func__); |
190 | break; |
191 | case QUEUE_TYPE_TO_CLIENT: |
192 | /* used by rxe which owns the index */ |
193 | prod = q->index; |
194 | prod = (prod + 1) & q->index_mask; |
195 | q->index = prod; |
196 | /* release so client can read it safely */ |
197 | smp_store_release(&q->buf->producer_index, prod); |
198 | break; |
199 | case QUEUE_TYPE_FROM_ULP: |
200 | /* used by ulp which owns the index */ |
201 | prod = q->buf->producer_index; |
202 | prod = (prod + 1) & q->index_mask; |
203 | /* release so rxe can read it safely */ |
204 | smp_store_release(&q->buf->producer_index, prod); |
205 | break; |
206 | case QUEUE_TYPE_TO_ULP: |
207 | /* used by ulp, rxe owns the index */ |
208 | if (WARN_ON(1)) |
209 | pr_warn("%s: attempt to advance driver index\n" , |
210 | __func__); |
211 | break; |
212 | } |
213 | } |
214 | |
215 | static inline void queue_advance_consumer(struct rxe_queue *q, |
216 | enum queue_type type) |
217 | { |
218 | u32 cons; |
219 | |
220 | switch (type) { |
221 | case QUEUE_TYPE_FROM_CLIENT: |
222 | /* used by rxe which owns the index */ |
223 | cons = (q->index + 1) & q->index_mask; |
224 | q->index = cons; |
225 | /* release so client can read it safely */ |
226 | smp_store_release(&q->buf->consumer_index, cons); |
227 | break; |
228 | case QUEUE_TYPE_TO_CLIENT: |
229 | /* used by rxe, client owns the index */ |
230 | if (WARN_ON(1)) |
231 | pr_warn("%s: attempt to advance client index\n" , |
232 | __func__); |
233 | break; |
234 | case QUEUE_TYPE_FROM_ULP: |
235 | /* used by ulp, rxe owns the index */ |
236 | if (WARN_ON(1)) |
237 | pr_warn("%s: attempt to advance driver index\n" , |
238 | __func__); |
239 | break; |
240 | case QUEUE_TYPE_TO_ULP: |
241 | /* used by ulp which owns the index */ |
242 | cons = q->buf->consumer_index; |
243 | cons = (cons + 1) & q->index_mask; |
244 | /* release so rxe can read it safely */ |
245 | smp_store_release(&q->buf->consumer_index, cons); |
246 | break; |
247 | } |
248 | } |
249 | |
250 | static inline void *queue_producer_addr(struct rxe_queue *q, |
251 | enum queue_type type) |
252 | { |
253 | u32 prod = queue_get_producer(q, type); |
254 | |
255 | return q->buf->data + (prod << q->log2_elem_size); |
256 | } |
257 | |
258 | static inline void *queue_consumer_addr(struct rxe_queue *q, |
259 | enum queue_type type) |
260 | { |
261 | u32 cons = queue_get_consumer(q, type); |
262 | |
263 | return q->buf->data + (cons << q->log2_elem_size); |
264 | } |
265 | |
266 | static inline void *queue_addr_from_index(struct rxe_queue *q, u32 index) |
267 | { |
268 | return q->buf->data + ((index & q->index_mask) |
269 | << q->log2_elem_size); |
270 | } |
271 | |
272 | static inline u32 queue_index_from_addr(const struct rxe_queue *q, |
273 | const void *addr) |
274 | { |
275 | return (((u8 *)addr - q->buf->data) >> q->log2_elem_size) |
276 | & q->index_mask; |
277 | } |
278 | |
279 | static inline void *queue_head(struct rxe_queue *q, enum queue_type type) |
280 | { |
281 | return queue_empty(q, type) ? NULL : queue_consumer_addr(q, type); |
282 | } |
283 | |
284 | #endif /* RXE_QUEUE_H */ |
285 | |