1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* Copyright (C) 2020 Marvell. */ |
3 | |
4 | #include "otx2_cptvf.h" |
5 | #include "otx2_cpt_common.h" |
6 | |
7 | /* Default timeout when waiting for free pending entry in us */ |
8 | #define CPT_PENTRY_TIMEOUT 1000 |
9 | #define CPT_PENTRY_STEP 50 |
10 | |
11 | /* Default threshold for stopping and resuming sender requests */ |
12 | #define CPT_IQ_STOP_MARGIN 128 |
13 | #define CPT_IQ_RESUME_MARGIN 512 |
14 | |
15 | /* Default command timeout in seconds */ |
16 | #define CPT_COMMAND_TIMEOUT 4 |
17 | #define CPT_TIME_IN_RESET_COUNT 5 |
18 | |
19 | static void otx2_cpt_dump_sg_list(struct pci_dev *pdev, |
20 | struct otx2_cpt_req_info *req) |
21 | { |
22 | int i; |
23 | |
24 | pr_debug("Gather list size %d\n" , req->in_cnt); |
25 | for (i = 0; i < req->in_cnt; i++) { |
26 | pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%llx\n" , i, |
27 | req->in[i].size, req->in[i].vptr, |
28 | req->in[i].dma_addr); |
29 | pr_debug("Buffer hexdump (%d bytes)\n" , |
30 | req->in[i].size); |
31 | print_hex_dump_debug("" , DUMP_PREFIX_NONE, 16, 1, |
32 | req->in[i].vptr, req->in[i].size, false); |
33 | } |
34 | pr_debug("Scatter list size %d\n" , req->out_cnt); |
35 | for (i = 0; i < req->out_cnt; i++) { |
36 | pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%llx\n" , i, |
37 | req->out[i].size, req->out[i].vptr, |
38 | req->out[i].dma_addr); |
39 | pr_debug("Buffer hexdump (%d bytes)\n" , req->out[i].size); |
40 | print_hex_dump_debug("" , DUMP_PREFIX_NONE, 16, 1, |
41 | req->out[i].vptr, req->out[i].size, false); |
42 | } |
43 | } |
44 | |
45 | static inline struct otx2_cpt_pending_entry *get_free_pending_entry( |
46 | struct otx2_cpt_pending_queue *q, |
47 | int qlen) |
48 | { |
49 | struct otx2_cpt_pending_entry *ent = NULL; |
50 | |
51 | ent = &q->head[q->rear]; |
52 | if (unlikely(ent->busy)) |
53 | return NULL; |
54 | |
55 | q->rear++; |
56 | if (unlikely(q->rear == qlen)) |
57 | q->rear = 0; |
58 | |
59 | return ent; |
60 | } |
61 | |
62 | static inline u32 modulo_inc(u32 index, u32 length, u32 inc) |
63 | { |
64 | if (WARN_ON(inc > length)) |
65 | inc = length; |
66 | |
67 | index += inc; |
68 | if (unlikely(index >= length)) |
69 | index -= length; |
70 | |
71 | return index; |
72 | } |
73 | |
74 | static inline void free_pentry(struct otx2_cpt_pending_entry *pentry) |
75 | { |
76 | pentry->completion_addr = NULL; |
77 | pentry->info = NULL; |
78 | pentry->callback = NULL; |
79 | pentry->areq = NULL; |
80 | pentry->resume_sender = false; |
81 | pentry->busy = false; |
82 | } |
83 | |
84 | static int process_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req, |
85 | struct otx2_cpt_pending_queue *pqueue, |
86 | struct otx2_cptlf_info *lf) |
87 | { |
88 | struct otx2_cptvf_request *cpt_req = &req->req; |
89 | struct otx2_cpt_pending_entry *pentry = NULL; |
90 | union otx2_cpt_ctrl_info *ctrl = &req->ctrl; |
91 | struct otx2_cpt_inst_info *info = NULL; |
92 | union otx2_cpt_res_s *result = NULL; |
93 | struct otx2_cpt_iq_command iq_cmd; |
94 | union otx2_cpt_inst_s cptinst; |
95 | int retry, ret = 0; |
96 | u8 resume_sender; |
97 | gfp_t gfp; |
98 | |
99 | gfp = (req->areq->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : |
100 | GFP_ATOMIC; |
101 | if (unlikely(!otx2_cptlf_started(lf->lfs))) |
102 | return -ENODEV; |
103 | |
104 | info = lf->lfs->ops->cpt_sg_info_create(pdev, req, gfp); |
105 | if (unlikely(!info)) { |
106 | dev_err(&pdev->dev, "Setting up cpt inst info failed" ); |
107 | return -ENOMEM; |
108 | } |
109 | cpt_req->dlen = info->dlen; |
110 | |
111 | result = info->completion_addr; |
112 | result->s.compcode = OTX2_CPT_COMPLETION_CODE_INIT; |
113 | |
114 | spin_lock_bh(lock: &pqueue->lock); |
115 | pentry = get_free_pending_entry(q: pqueue, qlen: pqueue->qlen); |
116 | retry = CPT_PENTRY_TIMEOUT / CPT_PENTRY_STEP; |
117 | while (unlikely(!pentry) && retry--) { |
118 | spin_unlock_bh(lock: &pqueue->lock); |
119 | udelay(CPT_PENTRY_STEP); |
120 | spin_lock_bh(lock: &pqueue->lock); |
121 | pentry = get_free_pending_entry(q: pqueue, qlen: pqueue->qlen); |
122 | } |
123 | |
124 | if (unlikely(!pentry)) { |
125 | ret = -ENOSPC; |
126 | goto destroy_info; |
127 | } |
128 | |
129 | /* |
130 | * Check if we are close to filling in entire pending queue, |
131 | * if so then tell the sender to stop/sleep by returning -EBUSY |
132 | * We do it only for context which can sleep (GFP_KERNEL) |
133 | */ |
134 | if (gfp == GFP_KERNEL && |
135 | pqueue->pending_count > (pqueue->qlen - CPT_IQ_STOP_MARGIN)) { |
136 | pentry->resume_sender = true; |
137 | } else |
138 | pentry->resume_sender = false; |
139 | resume_sender = pentry->resume_sender; |
140 | pqueue->pending_count++; |
141 | |
142 | pentry->completion_addr = info->completion_addr; |
143 | pentry->info = info; |
144 | pentry->callback = req->callback; |
145 | pentry->areq = req->areq; |
146 | pentry->busy = true; |
147 | info->pentry = pentry; |
148 | info->time_in = jiffies; |
149 | info->req = req; |
150 | |
151 | /* Fill in the command */ |
152 | iq_cmd.cmd.u = 0; |
153 | iq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags); |
154 | iq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1); |
155 | iq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2); |
156 | iq_cmd.cmd.s.dlen = cpu_to_be16(cpt_req->dlen); |
157 | |
158 | /* 64-bit swap for microcode data reads, not needed for addresses*/ |
159 | cpu_to_be64s(&iq_cmd.cmd.u); |
160 | iq_cmd.dptr = info->dptr_baddr | info->gthr_sz << 60; |
161 | iq_cmd.rptr = info->rptr_baddr | info->sctr_sz << 60; |
162 | iq_cmd.cptr.s.cptr = cpt_req->cptr_dma; |
163 | iq_cmd.cptr.s.grp = ctrl->s.grp; |
164 | |
165 | /* Fill in the CPT_INST_S type command for HW interpretation */ |
166 | otx2_cpt_fill_inst(cptinst: &cptinst, iq_cmd: &iq_cmd, comp_baddr: info->comp_baddr); |
167 | |
168 | /* Print debug info if enabled */ |
169 | otx2_cpt_dump_sg_list(pdev, req); |
170 | pr_debug("Cpt_inst_s hexdump (%d bytes)\n" , OTX2_CPT_INST_SIZE); |
171 | print_hex_dump_debug("" , 0, 16, 1, &cptinst, OTX2_CPT_INST_SIZE, false); |
172 | pr_debug("Dptr hexdump (%d bytes)\n" , cpt_req->dlen); |
173 | print_hex_dump_debug("" , 0, 16, 1, info->in_buffer, |
174 | cpt_req->dlen, false); |
175 | |
176 | /* Send CPT command */ |
177 | lf->lfs->ops->send_cmd(&cptinst, 1, lf); |
178 | |
179 | /* |
180 | * We allocate and prepare pending queue entry in critical section |
181 | * together with submitting CPT instruction to CPT instruction queue |
182 | * to make sure that order of CPT requests is the same in both |
183 | * pending and instruction queues |
184 | */ |
185 | spin_unlock_bh(lock: &pqueue->lock); |
186 | |
187 | ret = resume_sender ? -EBUSY : -EINPROGRESS; |
188 | return ret; |
189 | |
190 | destroy_info: |
191 | spin_unlock_bh(lock: &pqueue->lock); |
192 | otx2_cpt_info_destroy(pdev, info); |
193 | return ret; |
194 | } |
195 | |
196 | int otx2_cpt_do_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req, |
197 | int cpu_num) |
198 | { |
199 | struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev); |
200 | struct otx2_cptlfs_info *lfs = &cptvf->lfs; |
201 | |
202 | return process_request(pdev: lfs->pdev, req, pqueue: &lfs->lf[cpu_num].pqueue, |
203 | lf: &lfs->lf[cpu_num]); |
204 | } |
205 | |
206 | static int cpt_process_ccode(struct otx2_cptlfs_info *lfs, |
207 | union otx2_cpt_res_s *cpt_status, |
208 | struct otx2_cpt_inst_info *info, |
209 | u32 *res_code) |
210 | { |
211 | u8 uc_ccode = lfs->ops->cpt_get_uc_compcode(cpt_status); |
212 | u8 ccode = lfs->ops->cpt_get_compcode(cpt_status); |
213 | struct pci_dev *pdev = lfs->pdev; |
214 | |
215 | switch (ccode) { |
216 | case OTX2_CPT_COMP_E_FAULT: |
217 | dev_err(&pdev->dev, |
218 | "Request failed with DMA fault\n" ); |
219 | otx2_cpt_dump_sg_list(pdev, req: info->req); |
220 | break; |
221 | |
222 | case OTX2_CPT_COMP_E_HWERR: |
223 | dev_err(&pdev->dev, |
224 | "Request failed with hardware error\n" ); |
225 | otx2_cpt_dump_sg_list(pdev, req: info->req); |
226 | break; |
227 | |
228 | case OTX2_CPT_COMP_E_INSTERR: |
229 | dev_err(&pdev->dev, |
230 | "Request failed with instruction error\n" ); |
231 | otx2_cpt_dump_sg_list(pdev, req: info->req); |
232 | break; |
233 | |
234 | case OTX2_CPT_COMP_E_NOTDONE: |
235 | /* check for timeout */ |
236 | if (time_after_eq(jiffies, info->time_in + |
237 | CPT_COMMAND_TIMEOUT * HZ)) |
238 | dev_warn(&pdev->dev, |
239 | "Request timed out 0x%p" , info->req); |
240 | else if (info->extra_time < CPT_TIME_IN_RESET_COUNT) { |
241 | info->time_in = jiffies; |
242 | info->extra_time++; |
243 | } |
244 | return 1; |
245 | |
246 | case OTX2_CPT_COMP_E_GOOD: |
247 | case OTX2_CPT_COMP_E_WARN: |
248 | /* |
249 | * Check microcode completion code, it is only valid |
250 | * when completion code is CPT_COMP_E::GOOD |
251 | */ |
252 | if (uc_ccode != OTX2_CPT_UCC_SUCCESS) { |
253 | /* |
254 | * If requested hmac is truncated and ucode returns |
255 | * s/g write length error then we report success |
256 | * because ucode writes as many bytes of calculated |
257 | * hmac as available in gather buffer and reports |
258 | * s/g write length error if number of bytes in gather |
259 | * buffer is less than full hmac size. |
260 | */ |
261 | if (info->req->is_trunc_hmac && |
262 | uc_ccode == OTX2_CPT_UCC_SG_WRITE_LENGTH) { |
263 | *res_code = 0; |
264 | break; |
265 | } |
266 | |
267 | dev_err(&pdev->dev, |
268 | "Request failed with software error code 0x%x\n" , |
269 | cpt_status->s.uc_compcode); |
270 | otx2_cpt_dump_sg_list(pdev, req: info->req); |
271 | break; |
272 | } |
273 | /* Request has been processed with success */ |
274 | *res_code = 0; |
275 | break; |
276 | |
277 | default: |
278 | dev_err(&pdev->dev, |
279 | "Request returned invalid status %d\n" , ccode); |
280 | break; |
281 | } |
282 | return 0; |
283 | } |
284 | |
285 | static inline void process_pending_queue(struct otx2_cptlfs_info *lfs, |
286 | struct otx2_cpt_pending_queue *pqueue) |
287 | { |
288 | struct otx2_cpt_pending_entry *resume_pentry = NULL; |
289 | void (*callback)(int status, void *arg, void *req); |
290 | struct otx2_cpt_pending_entry *pentry = NULL; |
291 | union otx2_cpt_res_s *cpt_status = NULL; |
292 | struct otx2_cpt_inst_info *info = NULL; |
293 | struct otx2_cpt_req_info *req = NULL; |
294 | struct crypto_async_request *areq; |
295 | struct pci_dev *pdev = lfs->pdev; |
296 | u32 res_code, resume_index; |
297 | |
298 | while (1) { |
299 | spin_lock_bh(lock: &pqueue->lock); |
300 | pentry = &pqueue->head[pqueue->front]; |
301 | |
302 | if (WARN_ON(!pentry)) { |
303 | spin_unlock_bh(lock: &pqueue->lock); |
304 | break; |
305 | } |
306 | |
307 | res_code = -EINVAL; |
308 | if (unlikely(!pentry->busy)) { |
309 | spin_unlock_bh(lock: &pqueue->lock); |
310 | break; |
311 | } |
312 | |
313 | if (unlikely(!pentry->callback)) { |
314 | dev_err(&pdev->dev, "Callback NULL\n" ); |
315 | goto process_pentry; |
316 | } |
317 | |
318 | info = pentry->info; |
319 | if (unlikely(!info)) { |
320 | dev_err(&pdev->dev, "Pending entry post arg NULL\n" ); |
321 | goto process_pentry; |
322 | } |
323 | |
324 | req = info->req; |
325 | if (unlikely(!req)) { |
326 | dev_err(&pdev->dev, "Request NULL\n" ); |
327 | goto process_pentry; |
328 | } |
329 | |
330 | cpt_status = pentry->completion_addr; |
331 | if (unlikely(!cpt_status)) { |
332 | dev_err(&pdev->dev, "Completion address NULL\n" ); |
333 | goto process_pentry; |
334 | } |
335 | |
336 | if (cpt_process_ccode(lfs, cpt_status, info, res_code: &res_code)) { |
337 | spin_unlock_bh(lock: &pqueue->lock); |
338 | return; |
339 | } |
340 | info->pdev = pdev; |
341 | |
342 | process_pentry: |
343 | /* |
344 | * Check if we should inform sending side to resume |
345 | * We do it CPT_IQ_RESUME_MARGIN elements in advance before |
346 | * pending queue becomes empty |
347 | */ |
348 | resume_index = modulo_inc(index: pqueue->front, length: pqueue->qlen, |
349 | CPT_IQ_RESUME_MARGIN); |
350 | resume_pentry = &pqueue->head[resume_index]; |
351 | if (resume_pentry && |
352 | resume_pentry->resume_sender) { |
353 | resume_pentry->resume_sender = false; |
354 | callback = resume_pentry->callback; |
355 | areq = resume_pentry->areq; |
356 | |
357 | if (callback) { |
358 | spin_unlock_bh(lock: &pqueue->lock); |
359 | |
360 | /* |
361 | * EINPROGRESS is an indication for sending |
362 | * side that it can resume sending requests |
363 | */ |
364 | callback(-EINPROGRESS, areq, info); |
365 | spin_lock_bh(lock: &pqueue->lock); |
366 | } |
367 | } |
368 | |
369 | callback = pentry->callback; |
370 | areq = pentry->areq; |
371 | free_pentry(pentry); |
372 | |
373 | pqueue->pending_count--; |
374 | pqueue->front = modulo_inc(index: pqueue->front, length: pqueue->qlen, inc: 1); |
375 | spin_unlock_bh(lock: &pqueue->lock); |
376 | |
377 | /* |
378 | * Call callback after current pending entry has been |
379 | * processed, we don't do it if the callback pointer is |
380 | * invalid. |
381 | */ |
382 | if (callback) |
383 | callback(res_code, areq, info); |
384 | } |
385 | } |
386 | |
387 | void otx2_cpt_post_process(struct otx2_cptlf_wqe *wqe) |
388 | { |
389 | process_pending_queue(lfs: wqe->lfs, |
390 | pqueue: &wqe->lfs->lf[wqe->lf_num].pqueue); |
391 | } |
392 | |
393 | int otx2_cpt_get_kcrypto_eng_grp_num(struct pci_dev *pdev) |
394 | { |
395 | struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev); |
396 | |
397 | return cptvf->lfs.kcrypto_eng_grp_num; |
398 | } |
399 | |