1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (c) 2016 Chelsio Communications, Inc. |
4 | */ |
5 | |
6 | #include "cxgbit.h" |
7 | |
8 | static void |
9 | cxgbit_set_one_ppod(struct cxgbi_pagepod *ppod, |
10 | struct cxgbi_task_tag_info *ttinfo, |
11 | struct scatterlist **sg_pp, unsigned int *sg_off) |
12 | { |
13 | struct scatterlist *sg = sg_pp ? *sg_pp : NULL; |
14 | unsigned int offset = sg_off ? *sg_off : 0; |
15 | dma_addr_t addr = 0UL; |
16 | unsigned int len = 0; |
17 | int i; |
18 | |
19 | memcpy(ppod, &ttinfo->hdr, sizeof(struct cxgbi_pagepod_hdr)); |
20 | |
21 | if (sg) { |
22 | addr = sg_dma_address(sg); |
23 | len = sg_dma_len(sg); |
24 | } |
25 | |
26 | for (i = 0; i < PPOD_PAGES_MAX; i++) { |
27 | if (sg) { |
28 | ppod->addr[i] = cpu_to_be64(addr + offset); |
29 | offset += PAGE_SIZE; |
30 | if (offset == (len + sg->offset)) { |
31 | offset = 0; |
32 | sg = sg_next(sg); |
33 | if (sg) { |
34 | addr = sg_dma_address(sg); |
35 | len = sg_dma_len(sg); |
36 | } |
37 | } |
38 | } else { |
39 | ppod->addr[i] = 0ULL; |
40 | } |
41 | } |
42 | |
43 | /* |
44 | * the fifth address needs to be repeated in the next ppod, so do |
45 | * not move sg |
46 | */ |
47 | if (sg_pp) { |
48 | *sg_pp = sg; |
49 | *sg_off = offset; |
50 | } |
51 | |
52 | if (offset == len) { |
53 | offset = 0; |
54 | if (sg) { |
55 | sg = sg_next(sg); |
56 | if (sg) |
57 | addr = sg_dma_address(sg); |
58 | } |
59 | } |
60 | ppod->addr[i] = sg ? cpu_to_be64(addr + offset) : 0ULL; |
61 | } |
62 | |
63 | static struct sk_buff * |
64 | cxgbit_ppod_init_idata(struct cxgbit_device *cdev, struct cxgbi_ppm *ppm, |
65 | unsigned int idx, unsigned int npods, unsigned int tid) |
66 | { |
67 | struct ulp_mem_io *req; |
68 | struct ulptx_idata *idata; |
69 | unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ppm->llimit; |
70 | unsigned int dlen = npods << PPOD_SIZE_SHIFT; |
71 | unsigned int wr_len = roundup(sizeof(struct ulp_mem_io) + |
72 | sizeof(struct ulptx_idata) + dlen, 16); |
73 | struct sk_buff *skb; |
74 | |
75 | skb = alloc_skb(size: wr_len, GFP_KERNEL); |
76 | if (!skb) |
77 | return NULL; |
78 | |
79 | req = __skb_put(skb, len: wr_len); |
80 | INIT_ULPTX_WR(req, wr_len, 0, tid); |
81 | req->wr.wr_hi = htonl(FW_WR_OP_V(FW_ULPTX_WR) | |
82 | FW_WR_ATOMIC_V(0)); |
83 | req->cmd = htonl(ULPTX_CMD_V(ULP_TX_MEM_WRITE) | |
84 | ULP_MEMIO_ORDER_V(0) | |
85 | T5_ULP_MEMIO_IMM_V(1)); |
86 | req->dlen = htonl(ULP_MEMIO_DATA_LEN_V(dlen >> 5)); |
87 | req->lock_addr = htonl(ULP_MEMIO_ADDR_V(pm_addr >> 5)); |
88 | req->len16 = htonl(DIV_ROUND_UP(wr_len - sizeof(req->wr), 16)); |
89 | |
90 | idata = (struct ulptx_idata *)(req + 1); |
91 | idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM)); |
92 | idata->len = htonl(dlen); |
93 | |
94 | return skb; |
95 | } |
96 | |
97 | static int |
98 | cxgbit_ppod_write_idata(struct cxgbi_ppm *ppm, struct cxgbit_sock *csk, |
99 | struct cxgbi_task_tag_info *ttinfo, unsigned int idx, |
100 | unsigned int npods, struct scatterlist **sg_pp, |
101 | unsigned int *sg_off) |
102 | { |
103 | struct cxgbit_device *cdev = csk->com.cdev; |
104 | struct sk_buff *skb; |
105 | struct ulp_mem_io *req; |
106 | struct ulptx_idata *idata; |
107 | struct cxgbi_pagepod *ppod; |
108 | unsigned int i; |
109 | |
110 | skb = cxgbit_ppod_init_idata(cdev, ppm, idx, npods, tid: csk->tid); |
111 | if (!skb) |
112 | return -ENOMEM; |
113 | |
114 | req = (struct ulp_mem_io *)skb->data; |
115 | idata = (struct ulptx_idata *)(req + 1); |
116 | ppod = (struct cxgbi_pagepod *)(idata + 1); |
117 | |
118 | for (i = 0; i < npods; i++, ppod++) |
119 | cxgbit_set_one_ppod(ppod, ttinfo, sg_pp, sg_off); |
120 | |
121 | __skb_queue_tail(list: &csk->ppodq, newsk: skb); |
122 | |
123 | return 0; |
124 | } |
125 | |
126 | static int |
127 | cxgbit_ddp_set_map(struct cxgbi_ppm *ppm, struct cxgbit_sock *csk, |
128 | struct cxgbi_task_tag_info *ttinfo) |
129 | { |
130 | unsigned int pidx = ttinfo->idx; |
131 | unsigned int npods = ttinfo->npods; |
132 | unsigned int i, cnt; |
133 | struct scatterlist *sg = ttinfo->sgl; |
134 | unsigned int offset = 0; |
135 | int ret = 0; |
136 | |
137 | for (i = 0; i < npods; i += cnt, pidx += cnt) { |
138 | cnt = npods - i; |
139 | |
140 | if (cnt > ULPMEM_IDATA_MAX_NPPODS) |
141 | cnt = ULPMEM_IDATA_MAX_NPPODS; |
142 | |
143 | ret = cxgbit_ppod_write_idata(ppm, csk, ttinfo, idx: pidx, npods: cnt, |
144 | sg_pp: &sg, sg_off: &offset); |
145 | if (ret < 0) |
146 | break; |
147 | } |
148 | |
149 | return ret; |
150 | } |
151 | |
152 | static int cxgbit_ddp_sgl_check(struct scatterlist *sg, |
153 | unsigned int nents) |
154 | { |
155 | unsigned int last_sgidx = nents - 1; |
156 | unsigned int i; |
157 | |
158 | for (i = 0; i < nents; i++, sg = sg_next(sg)) { |
159 | unsigned int len = sg->length + sg->offset; |
160 | |
161 | if ((sg->offset & 0x3) || (i && sg->offset) || |
162 | ((i != last_sgidx) && (len != PAGE_SIZE))) { |
163 | return -EINVAL; |
164 | } |
165 | } |
166 | |
167 | return 0; |
168 | } |
169 | |
170 | static int |
171 | cxgbit_ddp_reserve(struct cxgbit_sock *csk, struct cxgbi_task_tag_info *ttinfo, |
172 | unsigned int xferlen) |
173 | { |
174 | struct cxgbit_device *cdev = csk->com.cdev; |
175 | struct cxgbi_ppm *ppm = cdev2ppm(cdev); |
176 | struct scatterlist *sgl = ttinfo->sgl; |
177 | unsigned int sgcnt = ttinfo->nents; |
178 | unsigned int sg_offset = sgl->offset; |
179 | int ret; |
180 | |
181 | if ((xferlen < DDP_THRESHOLD) || (!sgcnt)) { |
182 | pr_debug("ppm 0x%p, pgidx %u, xfer %u, sgcnt %u, NO ddp.\n" , |
183 | ppm, ppm->tformat.pgsz_idx_dflt, |
184 | xferlen, ttinfo->nents); |
185 | return -EINVAL; |
186 | } |
187 | |
188 | if (cxgbit_ddp_sgl_check(sg: sgl, nents: sgcnt) < 0) |
189 | return -EINVAL; |
190 | |
191 | ttinfo->nr_pages = (xferlen + sgl->offset + |
192 | (1 << PAGE_SHIFT) - 1) >> PAGE_SHIFT; |
193 | |
194 | /* |
195 | * the ddp tag will be used for the ttt in the outgoing r2t pdu |
196 | */ |
197 | ret = cxgbi_ppm_ppods_reserve(ppm, ttinfo->nr_pages, 0, &ttinfo->idx, |
198 | &ttinfo->tag, 0); |
199 | if (ret < 0) |
200 | return ret; |
201 | ttinfo->npods = ret; |
202 | |
203 | sgl->offset = 0; |
204 | ret = dma_map_sg(&ppm->pdev->dev, sgl, sgcnt, DMA_FROM_DEVICE); |
205 | sgl->offset = sg_offset; |
206 | if (!ret) { |
207 | pr_debug("%s: 0x%x, xfer %u, sgl %u dma mapping err.\n" , |
208 | __func__, 0, xferlen, sgcnt); |
209 | goto rel_ppods; |
210 | } |
211 | |
212 | cxgbi_ppm_make_ppod_hdr(ppm, ttinfo->tag, csk->tid, sgl->offset, |
213 | xferlen, &ttinfo->hdr); |
214 | |
215 | ret = cxgbit_ddp_set_map(ppm, csk, ttinfo); |
216 | if (ret < 0) { |
217 | __skb_queue_purge(list: &csk->ppodq); |
218 | dma_unmap_sg(&ppm->pdev->dev, sgl, sgcnt, DMA_FROM_DEVICE); |
219 | goto rel_ppods; |
220 | } |
221 | |
222 | return 0; |
223 | |
224 | rel_ppods: |
225 | cxgbi_ppm_ppod_release(ppm, ttinfo->idx); |
226 | return -EINVAL; |
227 | } |
228 | |
229 | void |
230 | cxgbit_get_r2t_ttt(struct iscsit_conn *conn, struct iscsit_cmd *cmd, |
231 | struct iscsi_r2t *r2t) |
232 | { |
233 | struct cxgbit_sock *csk = conn->context; |
234 | struct cxgbit_device *cdev = csk->com.cdev; |
235 | struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd); |
236 | struct cxgbi_task_tag_info *ttinfo = &ccmd->ttinfo; |
237 | int ret; |
238 | |
239 | if ((!ccmd->setup_ddp) || |
240 | (!test_bit(CSK_DDP_ENABLE, &csk->com.flags))) |
241 | goto out; |
242 | |
243 | ccmd->setup_ddp = false; |
244 | |
245 | ttinfo->sgl = cmd->se_cmd.t_data_sg; |
246 | ttinfo->nents = cmd->se_cmd.t_data_nents; |
247 | |
248 | ret = cxgbit_ddp_reserve(csk, ttinfo, xferlen: cmd->se_cmd.data_length); |
249 | if (ret < 0) { |
250 | pr_debug("csk 0x%p, cmd 0x%p, xfer len %u, sgcnt %u no ddp.\n" , |
251 | csk, cmd, cmd->se_cmd.data_length, ttinfo->nents); |
252 | |
253 | ttinfo->sgl = NULL; |
254 | ttinfo->nents = 0; |
255 | } else { |
256 | ccmd->release = true; |
257 | } |
258 | out: |
259 | pr_debug("cdev 0x%p, cmd 0x%p, tag 0x%x\n" , cdev, cmd, ttinfo->tag); |
260 | r2t->targ_xfer_tag = ttinfo->tag; |
261 | } |
262 | |
263 | void cxgbit_unmap_cmd(struct iscsit_conn *conn, struct iscsit_cmd *cmd) |
264 | { |
265 | struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd); |
266 | |
267 | if (ccmd->release) { |
268 | if (cmd->se_cmd.se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) { |
269 | put_page(page: sg_page(sg: &ccmd->sg)); |
270 | } else { |
271 | struct cxgbit_sock *csk = conn->context; |
272 | struct cxgbit_device *cdev = csk->com.cdev; |
273 | struct cxgbi_ppm *ppm = cdev2ppm(cdev); |
274 | struct cxgbi_task_tag_info *ttinfo = &ccmd->ttinfo; |
275 | |
276 | /* Abort the TCP conn if DDP is not complete to |
277 | * avoid any possibility of DDP after freeing |
278 | * the cmd. |
279 | */ |
280 | if (unlikely(cmd->write_data_done != |
281 | cmd->se_cmd.data_length)) |
282 | cxgbit_abort_conn(csk); |
283 | |
284 | if (unlikely(ttinfo->sgl)) { |
285 | dma_unmap_sg(&ppm->pdev->dev, ttinfo->sgl, |
286 | ttinfo->nents, DMA_FROM_DEVICE); |
287 | ttinfo->nents = 0; |
288 | ttinfo->sgl = NULL; |
289 | } |
290 | cxgbi_ppm_ppod_release(ppm, ttinfo->idx); |
291 | } |
292 | ccmd->release = false; |
293 | } |
294 | } |
295 | |
296 | int cxgbit_ddp_init(struct cxgbit_device *cdev) |
297 | { |
298 | struct cxgb4_lld_info *lldi = &cdev->lldi; |
299 | struct net_device *ndev = cdev->lldi.ports[0]; |
300 | struct cxgbi_tag_format tformat; |
301 | int ret, i; |
302 | |
303 | if (!lldi->vr->iscsi.size) { |
304 | pr_warn("%s, iscsi NOT enabled, check config!\n" , ndev->name); |
305 | return -EACCES; |
306 | } |
307 | |
308 | memset(&tformat, 0, sizeof(struct cxgbi_tag_format)); |
309 | for (i = 0; i < 4; i++) |
310 | tformat.pgsz_order[i] = (lldi->iscsi_pgsz_order >> (i << 3)) |
311 | & 0xF; |
312 | cxgbi_tagmask_check(lldi->iscsi_tagmask, &tformat); |
313 | |
314 | ret = cxgbi_ppm_init(lldi->iscsi_ppm, cdev->lldi.ports[0], |
315 | cdev->lldi.pdev, &cdev->lldi, &tformat, |
316 | lldi->vr->iscsi.size, lldi->iscsi_llimit, |
317 | lldi->vr->iscsi.start, 2, |
318 | lldi->vr->ppod_edram.start, |
319 | lldi->vr->ppod_edram.size); |
320 | if (ret >= 0) { |
321 | struct cxgbi_ppm *ppm = (struct cxgbi_ppm *)(*lldi->iscsi_ppm); |
322 | |
323 | if ((ppm->tformat.pgsz_idx_dflt < DDP_PGIDX_MAX) && |
324 | (ppm->ppmax >= 1024)) |
325 | set_bit(nr: CDEV_DDP_ENABLE, addr: &cdev->flags); |
326 | ret = 0; |
327 | } |
328 | |
329 | return ret; |
330 | } |
331 | |