1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | /* |
3 | * Shared Memory Communications over RDMA (SMC-R) and RoCE |
4 | * |
5 | * Connection Data Control (CDC) |
6 | * |
7 | * Copyright IBM Corp. 2016 |
8 | * |
9 | * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com> |
10 | */ |
11 | |
12 | #ifndef SMC_CDC_H |
13 | #define SMC_CDC_H |
14 | |
15 | #include <linux/kernel.h> /* max_t */ |
16 | #include <linux/atomic.h> |
17 | #include <linux/in.h> |
18 | #include <linux/compiler.h> |
19 | |
20 | #include "smc.h" |
21 | #include "smc_core.h" |
22 | #include "smc_wr.h" |
23 | |
24 | #define SMC_CDC_MSG_TYPE 0xFE |
25 | |
26 | /* in network byte order */ |
27 | union smc_cdc_cursor { /* SMC cursor */ |
28 | struct { |
29 | __be16 reserved; |
30 | __be16 wrap; |
31 | __be32 count; |
32 | }; |
33 | #ifdef KERNEL_HAS_ATOMIC64 |
34 | atomic64_t acurs; /* for atomic processing */ |
35 | #else |
36 | u64 acurs; /* for atomic processing */ |
37 | #endif |
38 | } __aligned(8); |
39 | |
40 | /* in network byte order */ |
41 | struct smc_cdc_msg { |
42 | struct smc_wr_rx_hdr common; /* .type = 0xFE */ |
43 | u8 len; /* 44 */ |
44 | __be16 seqno; |
45 | __be32 token; |
46 | union smc_cdc_cursor prod; |
47 | union smc_cdc_cursor cons; /* piggy backed "ack" */ |
48 | struct smc_cdc_producer_flags prod_flags; |
49 | struct smc_cdc_conn_state_flags conn_state_flags; |
50 | u8 reserved[18]; |
51 | }; |
52 | |
53 | /* SMC-D cursor format */ |
54 | union smcd_cdc_cursor { |
55 | struct { |
56 | u16 wrap; |
57 | u32 count; |
58 | struct smc_cdc_producer_flags prod_flags; |
59 | struct smc_cdc_conn_state_flags conn_state_flags; |
60 | } __packed; |
61 | #ifdef KERNEL_HAS_ATOMIC64 |
62 | atomic64_t acurs; /* for atomic processing */ |
63 | #else |
64 | u64 acurs; /* for atomic processing */ |
65 | #endif |
66 | } __aligned(8); |
67 | |
68 | /* CDC message for SMC-D */ |
69 | struct smcd_cdc_msg { |
70 | struct smc_wr_rx_hdr common; /* Type = 0xFE */ |
71 | u8 res1[7]; |
72 | union smcd_cdc_cursor prod; |
73 | union smcd_cdc_cursor cons; |
74 | u8 res3[8]; |
75 | } __aligned(8); |
76 | |
77 | static inline bool smc_cdc_rxed_any_close(struct smc_connection *conn) |
78 | { |
79 | return conn->local_rx_ctrl.conn_state_flags.peer_conn_abort || |
80 | conn->local_rx_ctrl.conn_state_flags.peer_conn_closed; |
81 | } |
82 | |
83 | static inline bool smc_cdc_rxed_any_close_or_senddone( |
84 | struct smc_connection *conn) |
85 | { |
86 | return smc_cdc_rxed_any_close(conn) || |
87 | conn->local_rx_ctrl.conn_state_flags.peer_done_writing; |
88 | } |
89 | |
90 | static inline void smc_curs_add(int size, union smc_host_cursor *curs, |
91 | int value) |
92 | { |
93 | curs->count += value; |
94 | if (curs->count >= size) { |
95 | curs->wrap++; |
96 | curs->count -= size; |
97 | } |
98 | } |
99 | |
100 | /* Copy cursor src into tgt */ |
101 | static inline void smc_curs_copy(union smc_host_cursor *tgt, |
102 | union smc_host_cursor *src, |
103 | struct smc_connection *conn) |
104 | { |
105 | #ifndef KERNEL_HAS_ATOMIC64 |
106 | unsigned long flags; |
107 | |
108 | spin_lock_irqsave(&conn->acurs_lock, flags); |
109 | tgt->acurs = src->acurs; |
110 | spin_unlock_irqrestore(&conn->acurs_lock, flags); |
111 | #else |
112 | atomic64_set(v: &tgt->acurs, i: atomic64_read(v: &src->acurs)); |
113 | #endif |
114 | } |
115 | |
116 | static inline void smc_curs_copy_net(union smc_cdc_cursor *tgt, |
117 | union smc_cdc_cursor *src, |
118 | struct smc_connection *conn) |
119 | { |
120 | #ifndef KERNEL_HAS_ATOMIC64 |
121 | unsigned long flags; |
122 | |
123 | spin_lock_irqsave(&conn->acurs_lock, flags); |
124 | tgt->acurs = src->acurs; |
125 | spin_unlock_irqrestore(&conn->acurs_lock, flags); |
126 | #else |
127 | atomic64_set(v: &tgt->acurs, i: atomic64_read(v: &src->acurs)); |
128 | #endif |
129 | } |
130 | |
131 | static inline void smcd_curs_copy(union smcd_cdc_cursor *tgt, |
132 | union smcd_cdc_cursor *src, |
133 | struct smc_connection *conn) |
134 | { |
135 | #ifndef KERNEL_HAS_ATOMIC64 |
136 | unsigned long flags; |
137 | |
138 | spin_lock_irqsave(&conn->acurs_lock, flags); |
139 | tgt->acurs = src->acurs; |
140 | spin_unlock_irqrestore(&conn->acurs_lock, flags); |
141 | #else |
142 | atomic64_set(v: &tgt->acurs, i: atomic64_read(v: &src->acurs)); |
143 | #endif |
144 | } |
145 | |
146 | /* calculate cursor difference between old and new, where old <= new and |
147 | * difference cannot exceed size |
148 | */ |
149 | static inline int smc_curs_diff(unsigned int size, |
150 | union smc_host_cursor *old, |
151 | union smc_host_cursor *new) |
152 | { |
153 | if (old->wrap != new->wrap) |
154 | return max_t(int, 0, |
155 | ((size - old->count) + new->count)); |
156 | |
157 | return max_t(int, 0, (new->count - old->count)); |
158 | } |
159 | |
160 | /* calculate cursor difference between old and new - returns negative |
161 | * value in case old > new |
162 | */ |
163 | static inline int smc_curs_comp(unsigned int size, |
164 | union smc_host_cursor *old, |
165 | union smc_host_cursor *new) |
166 | { |
167 | if (old->wrap > new->wrap || |
168 | (old->wrap == new->wrap && old->count > new->count)) |
169 | return -smc_curs_diff(size, old: new, new: old); |
170 | return smc_curs_diff(size, old, new); |
171 | } |
172 | |
173 | /* calculate cursor difference between old and new, where old <= new and |
174 | * difference may exceed size |
175 | */ |
176 | static inline int smc_curs_diff_large(unsigned int size, |
177 | union smc_host_cursor *old, |
178 | union smc_host_cursor *new) |
179 | { |
180 | if (old->wrap < new->wrap) |
181 | return min_t(int, |
182 | (size - old->count) + new->count + |
183 | (new->wrap - old->wrap - 1) * size, |
184 | size); |
185 | |
186 | if (old->wrap > new->wrap) /* wrap has switched from 0xffff to 0x0000 */ |
187 | return min_t(int, |
188 | (size - old->count) + new->count + |
189 | (new->wrap + 0xffff - old->wrap) * size, |
190 | size); |
191 | |
192 | return max_t(int, 0, (new->count - old->count)); |
193 | } |
194 | |
195 | static inline void smc_host_cursor_to_cdc(union smc_cdc_cursor *peer, |
196 | union smc_host_cursor *local, |
197 | union smc_host_cursor *save, |
198 | struct smc_connection *conn) |
199 | { |
200 | smc_curs_copy(tgt: save, src: local, conn); |
201 | peer->count = htonl(save->count); |
202 | peer->wrap = htons(save->wrap); |
203 | /* peer->reserved = htons(0); must be ensured by caller */ |
204 | } |
205 | |
206 | static inline void smc_host_msg_to_cdc(struct smc_cdc_msg *peer, |
207 | struct smc_connection *conn, |
208 | union smc_host_cursor *save) |
209 | { |
210 | struct smc_host_cdc_msg *local = &conn->local_tx_ctrl; |
211 | |
212 | peer->common.type = local->common.type; |
213 | peer->len = local->len; |
214 | peer->seqno = htons(local->seqno); |
215 | peer->token = htonl(local->token); |
216 | smc_host_cursor_to_cdc(peer: &peer->prod, local: &local->prod, save, conn); |
217 | smc_host_cursor_to_cdc(peer: &peer->cons, local: &local->cons, save, conn); |
218 | peer->prod_flags = local->prod_flags; |
219 | peer->conn_state_flags = local->conn_state_flags; |
220 | } |
221 | |
222 | static inline void smc_cdc_cursor_to_host(union smc_host_cursor *local, |
223 | union smc_cdc_cursor *peer, |
224 | struct smc_connection *conn) |
225 | { |
226 | union smc_host_cursor temp, old; |
227 | union smc_cdc_cursor net; |
228 | |
229 | smc_curs_copy(tgt: &old, src: local, conn); |
230 | smc_curs_copy_net(tgt: &net, src: peer, conn); |
231 | temp.count = ntohl(net.count); |
232 | temp.wrap = ntohs(net.wrap); |
233 | if ((old.wrap > temp.wrap) && temp.wrap) |
234 | return; |
235 | if ((old.wrap == temp.wrap) && |
236 | (old.count > temp.count)) |
237 | return; |
238 | smc_curs_copy(tgt: local, src: &temp, conn); |
239 | } |
240 | |
241 | static inline void smcr_cdc_msg_to_host(struct smc_host_cdc_msg *local, |
242 | struct smc_cdc_msg *peer, |
243 | struct smc_connection *conn) |
244 | { |
245 | local->common.type = peer->common.type; |
246 | local->len = peer->len; |
247 | local->seqno = ntohs(peer->seqno); |
248 | local->token = ntohl(peer->token); |
249 | smc_cdc_cursor_to_host(local: &local->prod, peer: &peer->prod, conn); |
250 | smc_cdc_cursor_to_host(local: &local->cons, peer: &peer->cons, conn); |
251 | local->prod_flags = peer->prod_flags; |
252 | local->conn_state_flags = peer->conn_state_flags; |
253 | } |
254 | |
255 | static inline void smcd_cdc_msg_to_host(struct smc_host_cdc_msg *local, |
256 | struct smcd_cdc_msg *peer, |
257 | struct smc_connection *conn) |
258 | { |
259 | union smc_host_cursor temp; |
260 | |
261 | temp.wrap = peer->prod.wrap; |
262 | temp.count = peer->prod.count; |
263 | smc_curs_copy(tgt: &local->prod, src: &temp, conn); |
264 | |
265 | temp.wrap = peer->cons.wrap; |
266 | temp.count = peer->cons.count; |
267 | smc_curs_copy(tgt: &local->cons, src: &temp, conn); |
268 | local->prod_flags = peer->cons.prod_flags; |
269 | local->conn_state_flags = peer->cons.conn_state_flags; |
270 | } |
271 | |
272 | static inline void smc_cdc_msg_to_host(struct smc_host_cdc_msg *local, |
273 | struct smc_cdc_msg *peer, |
274 | struct smc_connection *conn) |
275 | { |
276 | if (conn->lgr->is_smcd) |
277 | smcd_cdc_msg_to_host(local, peer: (struct smcd_cdc_msg *)peer, conn); |
278 | else |
279 | smcr_cdc_msg_to_host(local, peer, conn); |
280 | } |
281 | |
282 | struct smc_cdc_tx_pend { |
283 | struct smc_connection *conn; /* socket connection */ |
284 | union smc_host_cursor cursor; /* tx sndbuf cursor sent */ |
285 | union smc_host_cursor p_cursor; /* rx RMBE cursor produced */ |
286 | u16 ctrl_seq; /* conn. tx sequence # */ |
287 | }; |
288 | |
289 | int smc_cdc_get_free_slot(struct smc_connection *conn, |
290 | struct smc_link *link, |
291 | struct smc_wr_buf **wr_buf, |
292 | struct smc_rdma_wr **wr_rdma_buf, |
293 | struct smc_cdc_tx_pend **pend); |
294 | void smc_cdc_wait_pend_tx_wr(struct smc_connection *conn); |
295 | int smc_cdc_msg_send(struct smc_connection *conn, struct smc_wr_buf *wr_buf, |
296 | struct smc_cdc_tx_pend *pend); |
297 | int smc_cdc_get_slot_and_msg_send(struct smc_connection *conn); |
298 | int smcd_cdc_msg_send(struct smc_connection *conn); |
299 | int smcr_cdc_msg_send_validation(struct smc_connection *conn, |
300 | struct smc_cdc_tx_pend *pend, |
301 | struct smc_wr_buf *wr_buf); |
302 | int smc_cdc_init(void) __init; |
303 | void smcd_cdc_rx_init(struct smc_connection *conn); |
304 | |
305 | #endif /* SMC_CDC_H */ |
306 | |