1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | #include <linux/socket.h> |
3 | #include <linux/in.h> |
4 | #include <linux/in6.h> |
5 | #include <rdma/ib_verbs.h> |
6 | #include <rdma/rdma_cm.h> |
7 | #include <rdma/rw.h> |
8 | #include <scsi/iser.h> |
9 | |
10 | |
11 | #define DRV_NAME "isert" |
12 | #define PFX DRV_NAME ": " |
13 | |
14 | #define isert_dbg(fmt, arg...) \ |
15 | do { \ |
16 | if (unlikely(isert_debug_level > 2)) \ |
17 | printk(KERN_DEBUG PFX "%s: " fmt,\ |
18 | __func__ , ## arg); \ |
19 | } while (0) |
20 | |
21 | #define isert_warn(fmt, arg...) \ |
22 | do { \ |
23 | if (unlikely(isert_debug_level > 0)) \ |
24 | pr_warn(PFX "%s: " fmt, \ |
25 | __func__ , ## arg); \ |
26 | } while (0) |
27 | |
28 | #define isert_info(fmt, arg...) \ |
29 | do { \ |
30 | if (unlikely(isert_debug_level > 1)) \ |
31 | pr_info(PFX "%s: " fmt, \ |
32 | __func__ , ## arg); \ |
33 | } while (0) |
34 | |
35 | #define isert_err(fmt, arg...) \ |
36 | pr_err(PFX "%s: " fmt, __func__ , ## arg) |
37 | |
38 | /* Constant PDU lengths calculations */ |
39 | #define (sizeof(struct iser_ctrl) + \ |
40 | sizeof(struct iscsi_hdr)) |
41 | #define ISER_RX_PAYLOAD_SIZE (ISER_HEADERS_LEN + ISCSI_DEF_MAX_RECV_SEG_LEN) |
42 | |
43 | /* QP settings */ |
44 | /* Maximal bounds on received asynchronous PDUs */ |
45 | #define ISERT_MAX_TX_MISC_PDUS 4 /* NOOP_IN(2) , ASYNC_EVENT(2) */ |
46 | |
47 | #define ISERT_MAX_RX_MISC_PDUS 6 /* |
48 | * NOOP_OUT(2), TEXT(1), |
49 | * SCSI_TMFUNC(2), LOGOUT(1) |
50 | */ |
51 | |
52 | #define ISCSI_DEF_XMIT_CMDS_MAX 128 /* from libiscsi.h, must be power of 2 */ |
53 | |
54 | #define ISERT_QP_MAX_RECV_DTOS (ISCSI_DEF_XMIT_CMDS_MAX) |
55 | |
56 | #define ISERT_MIN_POSTED_RX (ISCSI_DEF_XMIT_CMDS_MAX >> 2) |
57 | |
58 | #define ISERT_QP_MAX_REQ_DTOS (ISCSI_DEF_XMIT_CMDS_MAX + \ |
59 | ISERT_MAX_TX_MISC_PDUS + \ |
60 | ISERT_MAX_RX_MISC_PDUS) |
61 | |
62 | /* |
63 | * RX size is default of 8k plus headers, but data needs to align to |
64 | * 512 boundary, so use 1024 to have the extra space for alignment. |
65 | */ |
66 | #define ISER_RX_SIZE (ISCSI_DEF_MAX_RECV_SEG_LEN + 1024) |
67 | |
68 | /* Minimum I/O size is 512KB */ |
69 | #define ISCSI_ISER_MIN_SG_TABLESIZE 128 |
70 | |
71 | /* Maximum support is 16MB I/O size */ |
72 | #define ISCSI_ISER_MAX_SG_TABLESIZE 4096 |
73 | |
74 | enum isert_desc_type { |
75 | ISCSI_TX_CONTROL, |
76 | ISCSI_TX_DATAIN |
77 | }; |
78 | |
79 | enum iser_conn_state { |
80 | ISER_CONN_INIT, |
81 | ISER_CONN_UP, |
82 | ISER_CONN_BOUND, |
83 | ISER_CONN_FULL_FEATURE, |
84 | ISER_CONN_TERMINATING, |
85 | ISER_CONN_DOWN, |
86 | }; |
87 | |
88 | struct iser_rx_desc { |
89 | char buf[ISER_RX_SIZE]; |
90 | u64 dma_addr; |
91 | struct ib_sge rx_sg; |
92 | struct ib_cqe rx_cqe; |
93 | bool in_use; |
94 | }; |
95 | |
96 | static inline struct iser_rx_desc *cqe_to_rx_desc(struct ib_cqe *cqe) |
97 | { |
98 | return container_of(cqe, struct iser_rx_desc, rx_cqe); |
99 | } |
100 | |
101 | static void *isert_get_iser_hdr(struct iser_rx_desc *desc) |
102 | { |
103 | return PTR_ALIGN(desc->buf + ISER_HEADERS_LEN, 512) - ISER_HEADERS_LEN; |
104 | } |
105 | |
106 | static size_t isert_get_hdr_offset(struct iser_rx_desc *desc) |
107 | { |
108 | return isert_get_iser_hdr(desc) - (void *)desc->buf; |
109 | } |
110 | |
111 | static void *isert_get_iscsi_hdr(struct iser_rx_desc *desc) |
112 | { |
113 | return isert_get_iser_hdr(desc) + sizeof(struct iser_ctrl); |
114 | } |
115 | |
116 | static void *isert_get_data(struct iser_rx_desc *desc) |
117 | { |
118 | void *data = isert_get_iser_hdr(desc) + ISER_HEADERS_LEN; |
119 | |
120 | WARN_ON((uintptr_t)data & 511); |
121 | return data; |
122 | } |
123 | |
124 | struct iser_tx_desc { |
125 | struct iser_ctrl ; |
126 | struct iscsi_hdr ; |
127 | enum isert_desc_type type; |
128 | u64 dma_addr; |
129 | struct ib_sge tx_sg[2]; |
130 | struct ib_cqe tx_cqe; |
131 | int num_sge; |
132 | struct ib_send_wr send_wr; |
133 | } __packed; |
134 | |
135 | static inline struct iser_tx_desc *cqe_to_tx_desc(struct ib_cqe *cqe) |
136 | { |
137 | return container_of(cqe, struct iser_tx_desc, tx_cqe); |
138 | } |
139 | |
140 | struct isert_cmd { |
141 | uint32_t read_stag; |
142 | uint32_t write_stag; |
143 | uint64_t read_va; |
144 | uint64_t write_va; |
145 | uint32_t inv_rkey; |
146 | u64 pdu_buf_dma; |
147 | u32 pdu_buf_len; |
148 | struct isert_conn *conn; |
149 | struct iscsit_cmd *iscsit_cmd; |
150 | struct iser_tx_desc tx_desc; |
151 | struct iser_rx_desc *rx_desc; |
152 | struct rdma_rw_ctx rw; |
153 | struct work_struct comp_work; |
154 | struct scatterlist sg; |
155 | bool ctx_init_done; |
156 | }; |
157 | |
158 | static inline struct isert_cmd *tx_desc_to_cmd(struct iser_tx_desc *desc) |
159 | { |
160 | return container_of(desc, struct isert_cmd, tx_desc); |
161 | } |
162 | |
163 | struct isert_device; |
164 | |
165 | struct isert_conn { |
166 | enum iser_conn_state state; |
167 | u32 responder_resources; |
168 | u32 initiator_depth; |
169 | bool pi_support; |
170 | struct iser_rx_desc *login_desc; |
171 | char *login_rsp_buf; |
172 | int login_req_len; |
173 | u64 login_rsp_dma; |
174 | struct iser_rx_desc *rx_descs; |
175 | struct ib_recv_wr rx_wr[ISERT_QP_MAX_RECV_DTOS]; |
176 | struct iscsit_conn *conn; |
177 | struct list_head node; |
178 | struct completion login_comp; |
179 | struct completion login_req_comp; |
180 | struct iser_tx_desc login_tx_desc; |
181 | struct rdma_cm_id *cm_id; |
182 | struct ib_qp *qp; |
183 | struct ib_cq *cq; |
184 | u32 cq_size; |
185 | struct isert_device *device; |
186 | struct mutex mutex; |
187 | struct kref kref; |
188 | struct work_struct release_work; |
189 | bool logout_posted; |
190 | bool snd_w_inv; |
191 | wait_queue_head_t rem_wait; |
192 | bool dev_removed; |
193 | }; |
194 | |
195 | struct isert_device { |
196 | bool pi_capable; |
197 | int refcount; |
198 | struct ib_device *ib_device; |
199 | struct ib_pd *pd; |
200 | struct isert_comp *comps; |
201 | int comps_used; |
202 | struct list_head dev_node; |
203 | }; |
204 | |
205 | struct isert_np { |
206 | struct iscsi_np *np; |
207 | struct semaphore sem; |
208 | struct rdma_cm_id *cm_id; |
209 | struct mutex mutex; |
210 | struct list_head accepted; |
211 | struct list_head pending; |
212 | }; |
213 | |