1 | /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ |
2 | /* |
3 | * Copyright(c) 2015 - 2018 Intel Corporation. |
4 | */ |
5 | |
6 | #ifndef HFI1_VERBS_H |
7 | #define HFI1_VERBS_H |
8 | |
9 | #include <linux/types.h> |
10 | #include <linux/seqlock.h> |
11 | #include <linux/kernel.h> |
12 | #include <linux/interrupt.h> |
13 | #include <linux/kref.h> |
14 | #include <linux/workqueue.h> |
15 | #include <linux/kthread.h> |
16 | #include <linux/completion.h> |
17 | #include <linux/slab.h> |
18 | #include <rdma/ib_pack.h> |
19 | #include <rdma/ib_user_verbs.h> |
20 | #include <rdma/ib_mad.h> |
21 | #include <rdma/ib_hdrs.h> |
22 | #include <rdma/rdma_vt.h> |
23 | #include <rdma/rdmavt_qp.h> |
24 | #include <rdma/rdmavt_cq.h> |
25 | |
26 | struct hfi1_ctxtdata; |
27 | struct hfi1_pportdata; |
28 | struct hfi1_devdata; |
29 | struct hfi1_packet; |
30 | |
31 | #include "iowait.h" |
32 | #include "tid_rdma.h" |
33 | #include "opfn.h" |
34 | |
35 | #define HFI1_MAX_RDMA_ATOMIC 16 |
36 | |
37 | /* |
38 | * Increment this value if any changes that break userspace ABI |
39 | * compatibility are made. |
40 | */ |
41 | #define HFI1_UVERBS_ABI_VERSION 2 |
42 | |
43 | /* IB Performance Manager status values */ |
44 | #define IB_PMA_SAMPLE_STATUS_DONE 0x00 |
45 | #define IB_PMA_SAMPLE_STATUS_STARTED 0x01 |
46 | #define IB_PMA_SAMPLE_STATUS_RUNNING 0x02 |
47 | |
48 | /* Mandatory IB performance counter select values. */ |
49 | #define IB_PMA_PORT_XMIT_DATA cpu_to_be16(0x0001) |
50 | #define IB_PMA_PORT_RCV_DATA cpu_to_be16(0x0002) |
51 | #define IB_PMA_PORT_XMIT_PKTS cpu_to_be16(0x0003) |
52 | #define IB_PMA_PORT_RCV_PKTS cpu_to_be16(0x0004) |
53 | #define IB_PMA_PORT_XMIT_WAIT cpu_to_be16(0x0005) |
54 | |
55 | #define HFI1_VENDOR_IPG cpu_to_be16(0xFFA0) |
56 | |
57 | #define IB_DEFAULT_GID_PREFIX cpu_to_be64(0xfe80000000000000ULL) |
58 | #define OPA_BTH_MIG_REQ BIT(31) |
59 | |
60 | #define RC_OP(x) IB_OPCODE_RC_##x |
61 | #define UC_OP(x) IB_OPCODE_UC_##x |
62 | |
63 | /* flags passed by hfi1_ib_rcv() */ |
64 | enum { |
65 | HFI1_HAS_GRH = (1 << 0), |
66 | }; |
67 | |
68 | #define LRH_16B_BYTES (sizeof_field(struct hfi1_16b_header, lrh)) |
69 | #define LRH_16B_DWORDS (LRH_16B_BYTES / sizeof(u32)) |
70 | #define LRH_9B_BYTES (sizeof_field(struct ib_header, lrh)) |
71 | #define LRH_9B_DWORDS (LRH_9B_BYTES / sizeof(u32)) |
72 | |
73 | /* 24Bits for qpn, upper 8Bits reserved */ |
74 | struct opa_16b_mgmt { |
75 | __be32 dest_qpn; |
76 | __be32 src_qpn; |
77 | }; |
78 | |
79 | struct { |
80 | u32 [4]; |
81 | union { |
82 | struct { |
83 | struct ib_grh ; |
84 | struct ib_other_headers ; |
85 | } ; |
86 | struct ib_other_headers ; |
87 | struct opa_16b_mgmt ; |
88 | } ; |
89 | } __packed; |
90 | |
91 | struct { |
92 | union { |
93 | struct ib_header ; /* 9B header */ |
94 | struct hfi1_16b_header ; /* 16B header */ |
95 | }; |
96 | u8 ; /* 9B or 16B */ |
97 | } __packed; |
98 | |
99 | struct hfi1_ahg_info { |
100 | u32 ahgdesc[2]; |
101 | u16 tx_flags; |
102 | u8 ahgcount; |
103 | u8 ahgidx; |
104 | }; |
105 | |
106 | struct { |
107 | __le64 ; |
108 | struct hfi1_opa_header ; |
109 | } __packed; |
110 | |
111 | /* |
112 | * hfi1 specific data structures that will be hidden from rvt after the queue |
113 | * pair is made common |
114 | */ |
115 | struct hfi1_qp_priv { |
116 | struct hfi1_ahg_info *s_ahg; /* ahg info for next header */ |
117 | struct sdma_engine *s_sde; /* current sde */ |
118 | struct send_context *s_sendcontext; /* current sendcontext */ |
119 | struct hfi1_ctxtdata *rcd; /* QP's receive context */ |
120 | struct page **pages; /* for TID page scan */ |
121 | u32 tid_enqueue; /* saved when tid waited */ |
122 | u8 s_sc; /* SC[0..4] for next packet */ |
123 | struct iowait s_iowait; |
124 | struct timer_list s_tid_timer; /* for timing tid wait */ |
125 | struct timer_list s_tid_retry_timer; /* for timing tid ack */ |
126 | struct list_head tid_wait; /* for queueing tid space */ |
127 | struct hfi1_opfn_data opfn; |
128 | struct tid_flow_state flow_state; |
129 | struct tid_rdma_qp_params tid_rdma; |
130 | struct rvt_qp *owner; |
131 | u16 s_running_pkt_size; |
132 | u8 hdr_type; /* 9B or 16B */ |
133 | struct rvt_sge_state tid_ss; /* SGE state pointer for 2nd leg */ |
134 | atomic_t n_requests; /* # of TID RDMA requests in the */ |
135 | /* queue */ |
136 | atomic_t n_tid_requests; /* # of sent TID RDMA requests */ |
137 | unsigned long tid_timer_timeout_jiffies; |
138 | unsigned long tid_retry_timeout_jiffies; |
139 | |
140 | /* variables for the TID RDMA SE state machine */ |
141 | u8 s_state; |
142 | u8 s_retry; |
143 | u8 rnr_nak_state; /* RNR NAK state */ |
144 | u8 s_nak_state; |
145 | u32 s_nak_psn; |
146 | u32 s_flags; |
147 | u32 s_tid_cur; |
148 | u32 s_tid_head; |
149 | u32 s_tid_tail; |
150 | u32 r_tid_head; /* Most recently added TID RDMA request */ |
151 | u32 r_tid_tail; /* the last completed TID RDMA request */ |
152 | u32 r_tid_ack; /* the TID RDMA request to be ACK'ed */ |
153 | u32 r_tid_alloc; /* Request for which we are allocating resources */ |
154 | u32 pending_tid_w_segs; /* Num of pending tid write segments */ |
155 | u32 pending_tid_w_resp; /* Num of pending tid write responses */ |
156 | u32 alloc_w_segs; /* Number of segments for which write */ |
157 | /* resources have been allocated for this QP */ |
158 | |
159 | /* For TID RDMA READ */ |
160 | u32 tid_r_reqs; /* Num of tid reads requested */ |
161 | u32 tid_r_comp; /* Num of tid reads completed */ |
162 | u32 pending_tid_r_segs; /* Num of pending tid read segments */ |
163 | u16 pkts_ps; /* packets per segment */ |
164 | u8 timeout_shift; /* account for number of packets per segment */ |
165 | |
166 | u32 r_next_psn_kdeth; |
167 | u32 r_next_psn_kdeth_save; |
168 | u32 s_resync_psn; |
169 | u8 sync_pt; /* Set when QP reaches sync point */ |
170 | u8 resync; |
171 | }; |
172 | |
173 | #define HFI1_QP_WQE_INVALID ((u32)-1) |
174 | |
175 | struct hfi1_swqe_priv { |
176 | struct tid_rdma_request tid_req; |
177 | struct rvt_sge_state ss; /* Used for TID RDMA READ Request */ |
178 | }; |
179 | |
180 | struct hfi1_ack_priv { |
181 | struct rvt_sge_state ss; /* used for TID WRITE RESP */ |
182 | struct tid_rdma_request tid_req; |
183 | }; |
184 | |
185 | /* |
186 | * This structure is used to hold commonly lookedup and computed values during |
187 | * the send engine progress. |
188 | */ |
189 | struct iowait_work; |
190 | struct hfi1_pkt_state { |
191 | struct hfi1_ibdev *dev; |
192 | struct hfi1_ibport *ibp; |
193 | struct hfi1_pportdata *ppd; |
194 | struct verbs_txreq *s_txreq; |
195 | struct iowait_work *wait; |
196 | unsigned long flags; |
197 | unsigned long timeout; |
198 | unsigned long timeout_int; |
199 | int cpu; |
200 | u8 opcode; |
201 | bool in_thread; |
202 | bool pkts_sent; |
203 | }; |
204 | |
205 | #define HFI1_PSN_CREDIT 16 |
206 | |
207 | struct hfi1_opcode_stats { |
208 | u64 n_packets; /* number of packets */ |
209 | u64 n_bytes; /* total number of bytes */ |
210 | }; |
211 | |
212 | struct hfi1_opcode_stats_perctx { |
213 | struct hfi1_opcode_stats stats[256]; |
214 | }; |
215 | |
216 | static inline void inc_opstats( |
217 | u32 tlen, |
218 | struct hfi1_opcode_stats *stats) |
219 | { |
220 | #ifdef CONFIG_DEBUG_FS |
221 | stats->n_bytes += tlen; |
222 | stats->n_packets++; |
223 | #endif |
224 | } |
225 | |
226 | struct hfi1_ibport { |
227 | struct rvt_qp __rcu *qp[2]; |
228 | struct rvt_ibport rvp; |
229 | |
230 | /* the first 16 entries are sl_to_vl for !OPA */ |
231 | u8 sl_to_sc[32]; |
232 | u8 sc_to_sl[32]; |
233 | }; |
234 | |
235 | struct hfi1_ibdev { |
236 | struct rvt_dev_info rdi; /* Must be first */ |
237 | |
238 | /* QP numbers are shared by all IB ports */ |
239 | /* protect txwait list */ |
240 | seqlock_t txwait_lock ____cacheline_aligned_in_smp; |
241 | struct list_head txwait; /* list for wait verbs_txreq */ |
242 | struct list_head memwait; /* list for wait kernel memory */ |
243 | struct kmem_cache *verbs_txreq_cache; |
244 | u64 n_txwait; |
245 | u64 n_kmem_wait; |
246 | u64 n_tidwait; |
247 | |
248 | /* protect iowait lists */ |
249 | seqlock_t iowait_lock ____cacheline_aligned_in_smp; |
250 | u64 n_piowait; |
251 | u64 n_piodrain; |
252 | struct timer_list mem_timer; |
253 | |
254 | #ifdef CONFIG_DEBUG_FS |
255 | /* per HFI debugfs */ |
256 | struct dentry *hfi1_ibdev_dbg; |
257 | /* per HFI symlinks to above */ |
258 | struct dentry *hfi1_ibdev_link; |
259 | #ifdef CONFIG_FAULT_INJECTION |
260 | struct fault *fault; |
261 | #endif |
262 | #endif |
263 | }; |
264 | |
265 | static inline struct hfi1_ibdev *to_idev(struct ib_device *ibdev) |
266 | { |
267 | struct rvt_dev_info *rdi; |
268 | |
269 | rdi = container_of(ibdev, struct rvt_dev_info, ibdev); |
270 | return container_of(rdi, struct hfi1_ibdev, rdi); |
271 | } |
272 | |
273 | static inline struct rvt_qp *iowait_to_qp(struct iowait *s_iowait) |
274 | { |
275 | struct hfi1_qp_priv *priv; |
276 | |
277 | priv = container_of(s_iowait, struct hfi1_qp_priv, s_iowait); |
278 | return priv->owner; |
279 | } |
280 | |
281 | /* |
282 | * This must be called with s_lock held. |
283 | */ |
284 | void hfi1_bad_pkey(struct hfi1_ibport *ibp, u32 key, u32 sl, |
285 | u32 qp1, u32 qp2, u32 lid1, u32 lid2); |
286 | void hfi1_cap_mask_chg(struct rvt_dev_info *rdi, u32 port_num); |
287 | void hfi1_sys_guid_chg(struct hfi1_ibport *ibp); |
288 | void hfi1_node_desc_chg(struct hfi1_ibport *ibp); |
289 | int hfi1_process_mad(struct ib_device *ibdev, int mad_flags, u32 port, |
290 | const struct ib_wc *in_wc, const struct ib_grh *in_grh, |
291 | const struct ib_mad *in_mad, struct ib_mad *out_mad, |
292 | size_t *out_mad_size, u16 *out_mad_pkey_index); |
293 | |
294 | /* |
295 | * The PSN_MASK and PSN_SHIFT allow for |
296 | * 1) comparing two PSNs |
297 | * 2) returning the PSN with any upper bits masked |
298 | * 3) returning the difference between to PSNs |
299 | * |
300 | * The number of significant bits in the PSN must |
301 | * necessarily be at least one bit less than |
302 | * the container holding the PSN. |
303 | */ |
304 | #define PSN_MASK 0x7FFFFFFF |
305 | #define PSN_SHIFT 1 |
306 | #define PSN_MODIFY_MASK 0xFFFFFF |
307 | |
308 | /* |
309 | * Compare two PSNs |
310 | * Returns an integer <, ==, or > than zero. |
311 | */ |
312 | static inline int cmp_psn(u32 a, u32 b) |
313 | { |
314 | return (((int)a) - ((int)b)) << PSN_SHIFT; |
315 | } |
316 | |
317 | /* |
318 | * Return masked PSN |
319 | */ |
320 | static inline u32 mask_psn(u32 a) |
321 | { |
322 | return a & PSN_MASK; |
323 | } |
324 | |
325 | /* |
326 | * Return delta between two PSNs |
327 | */ |
328 | static inline u32 delta_psn(u32 a, u32 b) |
329 | { |
330 | return (((int)a - (int)b) << PSN_SHIFT) >> PSN_SHIFT; |
331 | } |
332 | |
333 | static inline struct tid_rdma_request *wqe_to_tid_req(struct rvt_swqe *wqe) |
334 | { |
335 | return &((struct hfi1_swqe_priv *)wqe->priv)->tid_req; |
336 | } |
337 | |
338 | static inline struct tid_rdma_request *ack_to_tid_req(struct rvt_ack_entry *e) |
339 | { |
340 | return &((struct hfi1_ack_priv *)e->priv)->tid_req; |
341 | } |
342 | |
343 | /* |
344 | * Look through all the active flows for a TID RDMA request and find |
345 | * the one (if it exists) that contains the specified PSN. |
346 | */ |
347 | static inline u32 __full_flow_psn(struct flow_state *state, u32 psn) |
348 | { |
349 | return mask_psn(a: (state->generation << HFI1_KDETH_BTH_SEQ_SHIFT) | |
350 | (psn & HFI1_KDETH_BTH_SEQ_MASK)); |
351 | } |
352 | |
353 | static inline u32 full_flow_psn(struct tid_rdma_flow *flow, u32 psn) |
354 | { |
355 | return __full_flow_psn(state: &flow->flow_state, psn); |
356 | } |
357 | |
358 | struct verbs_txreq; |
359 | void hfi1_put_txreq(struct verbs_txreq *tx); |
360 | |
361 | int hfi1_verbs_send(struct rvt_qp *qp, struct hfi1_pkt_state *ps); |
362 | |
363 | void hfi1_cnp_rcv(struct hfi1_packet *packet); |
364 | |
365 | void hfi1_uc_rcv(struct hfi1_packet *packet); |
366 | |
367 | void hfi1_rc_rcv(struct hfi1_packet *packet); |
368 | |
369 | void hfi1_rc_hdrerr( |
370 | struct hfi1_ctxtdata *rcd, |
371 | struct hfi1_packet *packet, |
372 | struct rvt_qp *qp); |
373 | |
374 | u8 ah_to_sc(struct ib_device *ibdev, struct rdma_ah_attr *ah_attr); |
375 | |
376 | void hfi1_rc_verbs_aborted(struct rvt_qp *qp, struct hfi1_opa_header *opah); |
377 | void hfi1_rc_send_complete(struct rvt_qp *qp, struct hfi1_opa_header *opah); |
378 | |
379 | void hfi1_ud_rcv(struct hfi1_packet *packet); |
380 | |
381 | int hfi1_lookup_pkey_idx(struct hfi1_ibport *ibp, u16 pkey); |
382 | |
383 | void hfi1_migrate_qp(struct rvt_qp *qp); |
384 | |
385 | int hfi1_check_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr, |
386 | int attr_mask, struct ib_udata *udata); |
387 | |
388 | void hfi1_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr, |
389 | int attr_mask, struct ib_udata *udata); |
390 | void hfi1_restart_rc(struct rvt_qp *qp, u32 psn, int wait); |
391 | int hfi1_setup_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe, |
392 | bool *call_send); |
393 | |
394 | int hfi1_ruc_check_hdr(struct hfi1_ibport *ibp, struct hfi1_packet *packet); |
395 | |
396 | u32 hfi1_make_grh(struct hfi1_ibport *ibp, struct ib_grh *hdr, |
397 | const struct ib_global_route *grh, u32 hwords, u32 nwords); |
398 | |
399 | void (struct rvt_qp *qp, struct ib_other_headers *ohdr, |
400 | u32 bth0, u32 bth1, u32 bth2, int middle, |
401 | struct hfi1_pkt_state *ps); |
402 | |
403 | bool hfi1_schedule_send_yield(struct rvt_qp *qp, struct hfi1_pkt_state *ps, |
404 | bool tid); |
405 | |
406 | void _hfi1_do_send(struct work_struct *work); |
407 | |
408 | void hfi1_do_send_from_rvt(struct rvt_qp *qp); |
409 | |
410 | void hfi1_do_send(struct rvt_qp *qp, bool in_thread); |
411 | |
412 | void hfi1_send_rc_ack(struct hfi1_packet *packet, bool is_fecn); |
413 | |
414 | int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps); |
415 | |
416 | int hfi1_make_uc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps); |
417 | |
418 | int hfi1_make_ud_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps); |
419 | |
420 | int hfi1_register_ib_device(struct hfi1_devdata *); |
421 | |
422 | void hfi1_unregister_ib_device(struct hfi1_devdata *); |
423 | |
424 | void hfi1_kdeth_eager_rcv(struct hfi1_packet *packet); |
425 | |
426 | void hfi1_kdeth_expected_rcv(struct hfi1_packet *packet); |
427 | |
428 | void hfi1_ib_rcv(struct hfi1_packet *packet); |
429 | |
430 | void hfi1_16B_rcv(struct hfi1_packet *packet); |
431 | |
432 | unsigned hfi1_get_npkeys(struct hfi1_devdata *); |
433 | |
434 | int hfi1_verbs_send_dma(struct rvt_qp *qp, struct hfi1_pkt_state *ps, |
435 | u64 pbc); |
436 | |
437 | int hfi1_verbs_send_pio(struct rvt_qp *qp, struct hfi1_pkt_state *ps, |
438 | u64 pbc); |
439 | |
440 | static inline bool opa_bth_is_migration(struct ib_other_headers *ohdr) |
441 | { |
442 | return ohdr->bth[1] & cpu_to_be32(OPA_BTH_MIG_REQ); |
443 | } |
444 | |
445 | void hfi1_wait_kmem(struct rvt_qp *qp); |
446 | |
447 | static inline void hfi1_trdma_send_complete(struct rvt_qp *qp, |
448 | struct rvt_swqe *wqe, |
449 | enum ib_wc_status status) |
450 | { |
451 | trdma_clean_swqe(qp, wqe); |
452 | rvt_send_complete(qp, wqe, status); |
453 | } |
454 | |
455 | extern const enum ib_wc_opcode ib_hfi1_wc_opcode[]; |
456 | |
457 | extern const u8 hdr_len_by_opcode[]; |
458 | |
459 | extern const int ib_rvt_state_ops[]; |
460 | |
461 | extern __be64 ib_hfi1_sys_image_guid; /* in network order */ |
462 | |
463 | extern unsigned int hfi1_max_cqes; |
464 | |
465 | extern unsigned int hfi1_max_cqs; |
466 | |
467 | extern unsigned int hfi1_max_qp_wrs; |
468 | |
469 | extern unsigned int hfi1_max_qps; |
470 | |
471 | extern unsigned int hfi1_max_sges; |
472 | |
473 | extern unsigned int hfi1_max_mcast_grps; |
474 | |
475 | extern unsigned int hfi1_max_mcast_qp_attached; |
476 | |
477 | extern unsigned int hfi1_max_srqs; |
478 | |
479 | extern unsigned int hfi1_max_srq_sges; |
480 | |
481 | extern unsigned int hfi1_max_srq_wrs; |
482 | |
483 | extern unsigned short piothreshold; |
484 | |
485 | extern const u32 ib_hfi1_rnr_table[]; |
486 | |
487 | #endif /* HFI1_VERBS_H */ |
488 | |