1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | /* |
3 | * Shared Memory Communications over RDMA (SMC-R) and RoCE |
4 | * |
5 | * Work Requests exploiting Infiniband API |
6 | * |
7 | * Copyright IBM Corp. 2016 |
8 | * |
9 | * Author(s): Steffen Maier <maier@linux.vnet.ibm.com> |
10 | */ |
11 | |
12 | #ifndef SMC_WR_H |
13 | #define SMC_WR_H |
14 | |
15 | #include <linux/atomic.h> |
16 | #include <rdma/ib_verbs.h> |
17 | #include <asm/div64.h> |
18 | |
19 | #include "smc.h" |
20 | #include "smc_core.h" |
21 | |
22 | #define SMC_WR_BUF_CNT 16 /* # of ctrl buffers per link */ |
23 | |
24 | #define SMC_WR_TX_WAIT_FREE_SLOT_TIME (10 * HZ) |
25 | |
26 | #define SMC_WR_TX_SIZE 44 /* actual size of wr_send data (<=SMC_WR_BUF_SIZE) */ |
27 | |
28 | #define SMC_WR_TX_PEND_PRIV_SIZE 32 |
29 | |
30 | struct smc_wr_tx_pend_priv { |
31 | u8 priv[SMC_WR_TX_PEND_PRIV_SIZE]; |
32 | }; |
33 | |
34 | typedef void (*smc_wr_tx_handler)(struct smc_wr_tx_pend_priv *, |
35 | struct smc_link *, |
36 | enum ib_wc_status); |
37 | |
38 | typedef bool (*smc_wr_tx_filter)(struct smc_wr_tx_pend_priv *, |
39 | unsigned long); |
40 | |
41 | typedef void (*smc_wr_tx_dismisser)(struct smc_wr_tx_pend_priv *); |
42 | |
43 | struct smc_wr_rx_handler { |
44 | struct hlist_node list; /* hash table collision resolution */ |
45 | void (*handler)(struct ib_wc *, void *); |
46 | u8 type; |
47 | }; |
48 | |
49 | /* Only used by RDMA write WRs. |
50 | * All other WRs (CDC/LLC) use smc_wr_tx_send handling WR_ID implicitly |
51 | */ |
52 | static inline long smc_wr_tx_get_next_wr_id(struct smc_link *link) |
53 | { |
54 | return atomic_long_inc_return(v: &link->wr_tx_id); |
55 | } |
56 | |
57 | static inline void smc_wr_tx_set_wr_id(atomic_long_t *wr_tx_id, long val) |
58 | { |
59 | atomic_long_set(v: wr_tx_id, i: val); |
60 | } |
61 | |
62 | static inline bool smc_wr_tx_link_hold(struct smc_link *link) |
63 | { |
64 | if (!smc_link_sendable(lnk: link)) |
65 | return false; |
66 | percpu_ref_get(ref: &link->wr_tx_refs); |
67 | return true; |
68 | } |
69 | |
70 | static inline void smc_wr_tx_link_put(struct smc_link *link) |
71 | { |
72 | percpu_ref_put(ref: &link->wr_tx_refs); |
73 | } |
74 | |
75 | static inline void smc_wr_drain_cq(struct smc_link *lnk) |
76 | { |
77 | wait_event(lnk->wr_rx_empty_wait, lnk->wr_rx_id_compl == lnk->wr_rx_id); |
78 | } |
79 | |
80 | static inline void smc_wr_wakeup_tx_wait(struct smc_link *lnk) |
81 | { |
82 | wake_up_all(&lnk->wr_tx_wait); |
83 | } |
84 | |
85 | static inline void smc_wr_wakeup_reg_wait(struct smc_link *lnk) |
86 | { |
87 | wake_up(&lnk->wr_reg_wait); |
88 | } |
89 | |
90 | /* post a new receive work request to fill a completed old work request entry */ |
91 | static inline int smc_wr_rx_post(struct smc_link *link) |
92 | { |
93 | int rc; |
94 | u64 wr_id, temp_wr_id; |
95 | u32 index; |
96 | |
97 | wr_id = ++link->wr_rx_id; /* tasklet context, thus not atomic */ |
98 | temp_wr_id = wr_id; |
99 | index = do_div(temp_wr_id, link->wr_rx_cnt); |
100 | link->wr_rx_ibs[index].wr_id = wr_id; |
101 | rc = ib_post_recv(qp: link->roce_qp, recv_wr: &link->wr_rx_ibs[index], NULL); |
102 | return rc; |
103 | } |
104 | |
105 | int smc_wr_create_link(struct smc_link *lnk); |
106 | int smc_wr_alloc_link_mem(struct smc_link *lnk); |
107 | int smc_wr_alloc_lgr_mem(struct smc_link_group *lgr); |
108 | void smc_wr_free_link(struct smc_link *lnk); |
109 | void smc_wr_free_link_mem(struct smc_link *lnk); |
110 | void smc_wr_free_lgr_mem(struct smc_link_group *lgr); |
111 | void smc_wr_remember_qp_attr(struct smc_link *lnk); |
112 | void smc_wr_remove_dev(struct smc_ib_device *smcibdev); |
113 | void smc_wr_add_dev(struct smc_ib_device *smcibdev); |
114 | |
115 | int smc_wr_tx_get_free_slot(struct smc_link *link, smc_wr_tx_handler handler, |
116 | struct smc_wr_buf **wr_buf, |
117 | struct smc_rdma_wr **wrs, |
118 | struct smc_wr_tx_pend_priv **wr_pend_priv); |
119 | int smc_wr_tx_get_v2_slot(struct smc_link *link, |
120 | smc_wr_tx_handler handler, |
121 | struct smc_wr_v2_buf **wr_buf, |
122 | struct smc_wr_tx_pend_priv **wr_pend_priv); |
123 | int smc_wr_tx_put_slot(struct smc_link *link, |
124 | struct smc_wr_tx_pend_priv *wr_pend_priv); |
125 | int smc_wr_tx_send(struct smc_link *link, |
126 | struct smc_wr_tx_pend_priv *wr_pend_priv); |
127 | int smc_wr_tx_v2_send(struct smc_link *link, |
128 | struct smc_wr_tx_pend_priv *priv, int len); |
129 | int smc_wr_tx_send_wait(struct smc_link *link, struct smc_wr_tx_pend_priv *priv, |
130 | unsigned long timeout); |
131 | void smc_wr_tx_cq_handler(struct ib_cq *ib_cq, void *cq_context); |
132 | void smc_wr_tx_wait_no_pending_sends(struct smc_link *link); |
133 | |
134 | int smc_wr_rx_register_handler(struct smc_wr_rx_handler *handler); |
135 | int smc_wr_rx_post_init(struct smc_link *link); |
136 | void smc_wr_rx_cq_handler(struct ib_cq *ib_cq, void *cq_context); |
137 | int smc_wr_reg_send(struct smc_link *link, struct ib_mr *mr); |
138 | |
139 | #endif /* SMC_WR_H */ |
140 | |