1// SPDX-License-Identifier: GPL-2.0
2#include <linux/cpumask.h>
3#include <linux/dma-mapping.h>
4#include <linux/dmapool.h>
5#include <linux/delay.h>
6#include <linux/gfp.h>
7#include <linux/kernel.h>
8#include <linux/module.h>
9#include <linux/pci_regs.h>
10#include <linux/vmalloc.h>
11#include <linux/pci.h>
12
13#include "nitrox_dev.h"
14#include "nitrox_common.h"
15#include "nitrox_req.h"
16#include "nitrox_csr.h"
17
18#define CRYPTO_CTX_SIZE 256
19
20/* packet inuput ring alignments */
21#define PKTIN_Q_ALIGN_BYTES 16
22/* AQM Queue input alignments */
23#define AQM_Q_ALIGN_BYTES 32
24
25static int nitrox_cmdq_init(struct nitrox_cmdq *cmdq, int align_bytes)
26{
27 struct nitrox_device *ndev = cmdq->ndev;
28
29 cmdq->qsize = (ndev->qlen * cmdq->instr_size) + align_bytes;
30 cmdq->unalign_base = dma_alloc_coherent(DEV(ndev), size: cmdq->qsize,
31 dma_handle: &cmdq->unalign_dma,
32 GFP_KERNEL);
33 if (!cmdq->unalign_base)
34 return -ENOMEM;
35
36 cmdq->dma = PTR_ALIGN(cmdq->unalign_dma, align_bytes);
37 cmdq->base = cmdq->unalign_base + (cmdq->dma - cmdq->unalign_dma);
38 cmdq->write_idx = 0;
39
40 spin_lock_init(&cmdq->cmd_qlock);
41 spin_lock_init(&cmdq->resp_qlock);
42 spin_lock_init(&cmdq->backlog_qlock);
43
44 INIT_LIST_HEAD(list: &cmdq->response_head);
45 INIT_LIST_HEAD(list: &cmdq->backlog_head);
46 INIT_WORK(&cmdq->backlog_qflush, backlog_qflush_work);
47
48 atomic_set(v: &cmdq->pending_count, i: 0);
49 atomic_set(v: &cmdq->backlog_count, i: 0);
50 return 0;
51}
52
53static void nitrox_cmdq_reset(struct nitrox_cmdq *cmdq)
54{
55 cmdq->write_idx = 0;
56 atomic_set(v: &cmdq->pending_count, i: 0);
57 atomic_set(v: &cmdq->backlog_count, i: 0);
58}
59
60static void nitrox_cmdq_cleanup(struct nitrox_cmdq *cmdq)
61{
62 struct nitrox_device *ndev;
63
64 if (!cmdq)
65 return;
66
67 if (!cmdq->unalign_base)
68 return;
69
70 ndev = cmdq->ndev;
71 cancel_work_sync(work: &cmdq->backlog_qflush);
72
73 dma_free_coherent(DEV(ndev), size: cmdq->qsize,
74 cpu_addr: cmdq->unalign_base, dma_handle: cmdq->unalign_dma);
75 nitrox_cmdq_reset(cmdq);
76
77 cmdq->dbell_csr_addr = NULL;
78 cmdq->compl_cnt_csr_addr = NULL;
79 cmdq->unalign_base = NULL;
80 cmdq->base = NULL;
81 cmdq->unalign_dma = 0;
82 cmdq->dma = 0;
83 cmdq->qsize = 0;
84 cmdq->instr_size = 0;
85}
86
87static void nitrox_free_aqm_queues(struct nitrox_device *ndev)
88{
89 int i;
90
91 for (i = 0; i < ndev->nr_queues; i++) {
92 nitrox_cmdq_cleanup(cmdq: ndev->aqmq[i]);
93 kfree_sensitive(objp: ndev->aqmq[i]);
94 ndev->aqmq[i] = NULL;
95 }
96}
97
98static int nitrox_alloc_aqm_queues(struct nitrox_device *ndev)
99{
100 int i, err;
101
102 for (i = 0; i < ndev->nr_queues; i++) {
103 struct nitrox_cmdq *cmdq;
104 u64 offset;
105
106 cmdq = kzalloc_node(size: sizeof(*cmdq), GFP_KERNEL, node: ndev->node);
107 if (!cmdq) {
108 err = -ENOMEM;
109 goto aqmq_fail;
110 }
111
112 cmdq->ndev = ndev;
113 cmdq->qno = i;
114 cmdq->instr_size = sizeof(struct aqmq_command_s);
115
116 /* AQM Queue Doorbell Counter Register Address */
117 offset = AQMQ_DRBLX(i);
118 cmdq->dbell_csr_addr = NITROX_CSR_ADDR(ndev, offset);
119 /* AQM Queue Commands Completed Count Register Address */
120 offset = AQMQ_CMD_CNTX(i);
121 cmdq->compl_cnt_csr_addr = NITROX_CSR_ADDR(ndev, offset);
122
123 err = nitrox_cmdq_init(cmdq, AQM_Q_ALIGN_BYTES);
124 if (err) {
125 kfree_sensitive(objp: cmdq);
126 goto aqmq_fail;
127 }
128 ndev->aqmq[i] = cmdq;
129 }
130
131 return 0;
132
133aqmq_fail:
134 nitrox_free_aqm_queues(ndev);
135 return err;
136}
137
138static void nitrox_free_pktin_queues(struct nitrox_device *ndev)
139{
140 int i;
141
142 for (i = 0; i < ndev->nr_queues; i++) {
143 struct nitrox_cmdq *cmdq = &ndev->pkt_inq[i];
144
145 nitrox_cmdq_cleanup(cmdq);
146 }
147 kfree(objp: ndev->pkt_inq);
148 ndev->pkt_inq = NULL;
149}
150
151static int nitrox_alloc_pktin_queues(struct nitrox_device *ndev)
152{
153 int i, err;
154
155 ndev->pkt_inq = kcalloc_node(n: ndev->nr_queues,
156 size: sizeof(struct nitrox_cmdq),
157 GFP_KERNEL, node: ndev->node);
158 if (!ndev->pkt_inq)
159 return -ENOMEM;
160
161 for (i = 0; i < ndev->nr_queues; i++) {
162 struct nitrox_cmdq *cmdq;
163 u64 offset;
164
165 cmdq = &ndev->pkt_inq[i];
166 cmdq->ndev = ndev;
167 cmdq->qno = i;
168 cmdq->instr_size = sizeof(struct nps_pkt_instr);
169
170 /* packet input ring doorbell address */
171 offset = NPS_PKT_IN_INSTR_BAOFF_DBELLX(i);
172 cmdq->dbell_csr_addr = NITROX_CSR_ADDR(ndev, offset);
173 /* packet solicit port completion count address */
174 offset = NPS_PKT_SLC_CNTSX(i);
175 cmdq->compl_cnt_csr_addr = NITROX_CSR_ADDR(ndev, offset);
176
177 err = nitrox_cmdq_init(cmdq, PKTIN_Q_ALIGN_BYTES);
178 if (err)
179 goto pktq_fail;
180 }
181 return 0;
182
183pktq_fail:
184 nitrox_free_pktin_queues(ndev);
185 return err;
186}
187
188static int create_crypto_dma_pool(struct nitrox_device *ndev)
189{
190 size_t size;
191
192 /* Crypto context pool, 16 byte aligned */
193 size = CRYPTO_CTX_SIZE + sizeof(struct ctx_hdr);
194 ndev->ctx_pool = dma_pool_create(name: "nitrox-context",
195 DEV(ndev), size, align: 16, allocation: 0);
196 if (!ndev->ctx_pool)
197 return -ENOMEM;
198
199 return 0;
200}
201
202static void destroy_crypto_dma_pool(struct nitrox_device *ndev)
203{
204 if (!ndev->ctx_pool)
205 return;
206
207 dma_pool_destroy(pool: ndev->ctx_pool);
208 ndev->ctx_pool = NULL;
209}
210
211/*
212 * crypto_alloc_context - Allocate crypto context from pool
213 * @ndev: NITROX Device
214 */
215void *crypto_alloc_context(struct nitrox_device *ndev)
216{
217 struct ctx_hdr *ctx;
218 struct crypto_ctx_hdr *chdr;
219 void *vaddr;
220 dma_addr_t dma;
221
222 chdr = kmalloc(size: sizeof(*chdr), GFP_KERNEL);
223 if (!chdr)
224 return NULL;
225
226 vaddr = dma_pool_zalloc(pool: ndev->ctx_pool, GFP_KERNEL, handle: &dma);
227 if (!vaddr) {
228 kfree(objp: chdr);
229 return NULL;
230 }
231
232 /* fill meta data */
233 ctx = vaddr;
234 ctx->pool = ndev->ctx_pool;
235 ctx->dma = dma;
236 ctx->ctx_dma = dma + sizeof(struct ctx_hdr);
237
238 chdr->pool = ndev->ctx_pool;
239 chdr->dma = dma;
240 chdr->vaddr = vaddr;
241
242 return chdr;
243}
244
245/**
246 * crypto_free_context - Free crypto context to pool
247 * @ctx: context to free
248 */
249void crypto_free_context(void *ctx)
250{
251 struct crypto_ctx_hdr *ctxp;
252
253 if (!ctx)
254 return;
255
256 ctxp = ctx;
257 dma_pool_free(pool: ctxp->pool, vaddr: ctxp->vaddr, addr: ctxp->dma);
258 kfree(objp: ctxp);
259}
260
261/**
262 * nitrox_common_sw_init - allocate software resources.
263 * @ndev: NITROX device
264 *
265 * Allocates crypto context pools and command queues etc.
266 *
267 * Return: 0 on success, or a negative error code on error.
268 */
269int nitrox_common_sw_init(struct nitrox_device *ndev)
270{
271 int err = 0;
272
273 /* per device crypto context pool */
274 err = create_crypto_dma_pool(ndev);
275 if (err)
276 return err;
277
278 err = nitrox_alloc_pktin_queues(ndev);
279 if (err)
280 destroy_crypto_dma_pool(ndev);
281
282 err = nitrox_alloc_aqm_queues(ndev);
283 if (err) {
284 nitrox_free_pktin_queues(ndev);
285 destroy_crypto_dma_pool(ndev);
286 }
287
288 return err;
289}
290
291/**
292 * nitrox_common_sw_cleanup - free software resources.
293 * @ndev: NITROX device
294 */
295void nitrox_common_sw_cleanup(struct nitrox_device *ndev)
296{
297 nitrox_free_aqm_queues(ndev);
298 nitrox_free_pktin_queues(ndev);
299 destroy_crypto_dma_pool(ndev);
300}
301

source code of linux/drivers/crypto/cavium/nitrox/nitrox_lib.c