1 | // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) |
2 | /* Copyright (c) 2020 Marvell International Ltd. */ |
3 | |
4 | #include <linux/dma-mapping.h> |
5 | #include <linux/qed/qed_chain.h> |
6 | #include <linux/vmalloc.h> |
7 | |
8 | #include "qed_dev_api.h" |
9 | |
10 | static void qed_chain_init(struct qed_chain *chain, |
11 | const struct qed_chain_init_params *params, |
12 | u32 page_cnt) |
13 | { |
14 | memset(chain, 0, sizeof(*chain)); |
15 | |
16 | chain->elem_size = params->elem_size; |
17 | chain->intended_use = params->intended_use; |
18 | chain->mode = params->mode; |
19 | chain->cnt_type = params->cnt_type; |
20 | |
21 | chain->elem_per_page = ELEMS_PER_PAGE(params->elem_size, |
22 | params->page_size); |
23 | chain->usable_per_page = USABLE_ELEMS_PER_PAGE(params->elem_size, |
24 | params->page_size, |
25 | params->mode); |
26 | chain->elem_unusable = UNUSABLE_ELEMS_PER_PAGE(params->elem_size, |
27 | params->mode); |
28 | |
29 | chain->elem_per_page_mask = chain->elem_per_page - 1; |
30 | chain->next_page_mask = chain->usable_per_page & |
31 | chain->elem_per_page_mask; |
32 | |
33 | chain->page_size = params->page_size; |
34 | chain->page_cnt = page_cnt; |
35 | chain->capacity = chain->usable_per_page * page_cnt; |
36 | chain->size = chain->elem_per_page * page_cnt; |
37 | |
38 | if (params->ext_pbl_virt) { |
39 | chain->pbl_sp.table_virt = params->ext_pbl_virt; |
40 | chain->pbl_sp.table_phys = params->ext_pbl_phys; |
41 | |
42 | chain->b_external_pbl = true; |
43 | } |
44 | } |
45 | |
46 | static void qed_chain_init_next_ptr_elem(const struct qed_chain *chain, |
47 | void *virt_curr, void *virt_next, |
48 | dma_addr_t phys_next) |
49 | { |
50 | struct qed_chain_next *next; |
51 | u32 size; |
52 | |
53 | size = chain->elem_size * chain->usable_per_page; |
54 | next = virt_curr + size; |
55 | |
56 | DMA_REGPAIR_LE(next->next_phys, phys_next); |
57 | next->next_virt = virt_next; |
58 | } |
59 | |
60 | static void qed_chain_init_mem(struct qed_chain *chain, void *virt_addr, |
61 | dma_addr_t phys_addr) |
62 | { |
63 | chain->p_virt_addr = virt_addr; |
64 | chain->p_phys_addr = phys_addr; |
65 | } |
66 | |
67 | static void qed_chain_free_next_ptr(struct qed_dev *cdev, |
68 | struct qed_chain *chain) |
69 | { |
70 | struct device *dev = &cdev->pdev->dev; |
71 | struct qed_chain_next *next; |
72 | dma_addr_t phys, phys_next; |
73 | void *virt, *virt_next; |
74 | u32 size, i; |
75 | |
76 | size = chain->elem_size * chain->usable_per_page; |
77 | virt = chain->p_virt_addr; |
78 | phys = chain->p_phys_addr; |
79 | |
80 | for (i = 0; i < chain->page_cnt; i++) { |
81 | if (!virt) |
82 | break; |
83 | |
84 | next = virt + size; |
85 | virt_next = next->next_virt; |
86 | phys_next = HILO_DMA_REGPAIR(next->next_phys); |
87 | |
88 | dma_free_coherent(dev, size: chain->page_size, cpu_addr: virt, dma_handle: phys); |
89 | |
90 | virt = virt_next; |
91 | phys = phys_next; |
92 | } |
93 | } |
94 | |
95 | static void qed_chain_free_single(struct qed_dev *cdev, |
96 | struct qed_chain *chain) |
97 | { |
98 | if (!chain->p_virt_addr) |
99 | return; |
100 | |
101 | dma_free_coherent(dev: &cdev->pdev->dev, size: chain->page_size, |
102 | cpu_addr: chain->p_virt_addr, dma_handle: chain->p_phys_addr); |
103 | } |
104 | |
105 | static void qed_chain_free_pbl(struct qed_dev *cdev, struct qed_chain *chain) |
106 | { |
107 | struct device *dev = &cdev->pdev->dev; |
108 | struct addr_tbl_entry *entry; |
109 | u32 i; |
110 | |
111 | if (!chain->pbl.pp_addr_tbl) |
112 | return; |
113 | |
114 | for (i = 0; i < chain->page_cnt; i++) { |
115 | entry = chain->pbl.pp_addr_tbl + i; |
116 | if (!entry->virt_addr) |
117 | break; |
118 | |
119 | dma_free_coherent(dev, size: chain->page_size, cpu_addr: entry->virt_addr, |
120 | dma_handle: entry->dma_map); |
121 | } |
122 | |
123 | if (!chain->b_external_pbl) |
124 | dma_free_coherent(dev, size: chain->pbl_sp.table_size, |
125 | cpu_addr: chain->pbl_sp.table_virt, |
126 | dma_handle: chain->pbl_sp.table_phys); |
127 | |
128 | vfree(addr: chain->pbl.pp_addr_tbl); |
129 | chain->pbl.pp_addr_tbl = NULL; |
130 | } |
131 | |
132 | /** |
133 | * qed_chain_free() - Free chain DMA memory. |
134 | * |
135 | * @cdev: Main device structure. |
136 | * @chain: Chain to free. |
137 | */ |
138 | void qed_chain_free(struct qed_dev *cdev, struct qed_chain *chain) |
139 | { |
140 | switch (chain->mode) { |
141 | case QED_CHAIN_MODE_NEXT_PTR: |
142 | qed_chain_free_next_ptr(cdev, chain); |
143 | break; |
144 | case QED_CHAIN_MODE_SINGLE: |
145 | qed_chain_free_single(cdev, chain); |
146 | break; |
147 | case QED_CHAIN_MODE_PBL: |
148 | qed_chain_free_pbl(cdev, chain); |
149 | break; |
150 | default: |
151 | return; |
152 | } |
153 | |
154 | qed_chain_init_mem(chain, NULL, phys_addr: 0); |
155 | } |
156 | |
157 | static int |
158 | qed_chain_alloc_sanity_check(struct qed_dev *cdev, |
159 | const struct qed_chain_init_params *params, |
160 | u32 page_cnt) |
161 | { |
162 | u64 chain_size; |
163 | |
164 | chain_size = ELEMS_PER_PAGE(params->elem_size, params->page_size); |
165 | chain_size *= page_cnt; |
166 | |
167 | if (!chain_size) |
168 | return -EINVAL; |
169 | |
170 | /* The actual chain size can be larger than the maximal possible value |
171 | * after rounding up the requested elements number to pages, and after |
172 | * taking into account the unusuable elements (next-ptr elements). |
173 | * The size of a "u16" chain can be (U16_MAX + 1) since the chain |
174 | * size/capacity fields are of u32 type. |
175 | */ |
176 | switch (params->cnt_type) { |
177 | case QED_CHAIN_CNT_TYPE_U16: |
178 | if (chain_size > U16_MAX + 1) |
179 | break; |
180 | |
181 | return 0; |
182 | case QED_CHAIN_CNT_TYPE_U32: |
183 | if (chain_size > U32_MAX) |
184 | break; |
185 | |
186 | return 0; |
187 | default: |
188 | return -EINVAL; |
189 | } |
190 | |
191 | DP_NOTICE(cdev, |
192 | "The actual chain size (0x%llx) is larger than the maximal possible value\n" , |
193 | chain_size); |
194 | |
195 | return -EINVAL; |
196 | } |
197 | |
198 | static int qed_chain_alloc_next_ptr(struct qed_dev *cdev, |
199 | struct qed_chain *chain) |
200 | { |
201 | struct device *dev = &cdev->pdev->dev; |
202 | void *virt, *virt_prev = NULL; |
203 | dma_addr_t phys; |
204 | u32 i; |
205 | |
206 | for (i = 0; i < chain->page_cnt; i++) { |
207 | virt = dma_alloc_coherent(dev, size: chain->page_size, dma_handle: &phys, |
208 | GFP_KERNEL); |
209 | if (!virt) |
210 | return -ENOMEM; |
211 | |
212 | if (i == 0) { |
213 | qed_chain_init_mem(chain, virt_addr: virt, phys_addr: phys); |
214 | qed_chain_reset(p_chain: chain); |
215 | } else { |
216 | qed_chain_init_next_ptr_elem(chain, virt_curr: virt_prev, virt_next: virt, |
217 | phys_next: phys); |
218 | } |
219 | |
220 | virt_prev = virt; |
221 | } |
222 | |
223 | /* Last page's next element should point to the beginning of the |
224 | * chain. |
225 | */ |
226 | qed_chain_init_next_ptr_elem(chain, virt_curr: virt_prev, virt_next: chain->p_virt_addr, |
227 | phys_next: chain->p_phys_addr); |
228 | |
229 | return 0; |
230 | } |
231 | |
232 | static int qed_chain_alloc_single(struct qed_dev *cdev, |
233 | struct qed_chain *chain) |
234 | { |
235 | dma_addr_t phys; |
236 | void *virt; |
237 | |
238 | virt = dma_alloc_coherent(dev: &cdev->pdev->dev, size: chain->page_size, |
239 | dma_handle: &phys, GFP_KERNEL); |
240 | if (!virt) |
241 | return -ENOMEM; |
242 | |
243 | qed_chain_init_mem(chain, virt_addr: virt, phys_addr: phys); |
244 | qed_chain_reset(p_chain: chain); |
245 | |
246 | return 0; |
247 | } |
248 | |
249 | static int qed_chain_alloc_pbl(struct qed_dev *cdev, struct qed_chain *chain) |
250 | { |
251 | struct device *dev = &cdev->pdev->dev; |
252 | struct addr_tbl_entry *addr_tbl; |
253 | dma_addr_t phys, pbl_phys; |
254 | __le64 *pbl_virt; |
255 | u32 page_cnt, i; |
256 | size_t size; |
257 | void *virt; |
258 | |
259 | page_cnt = chain->page_cnt; |
260 | |
261 | size = array_size(page_cnt, sizeof(*addr_tbl)); |
262 | if (unlikely(size == SIZE_MAX)) |
263 | return -EOVERFLOW; |
264 | |
265 | addr_tbl = vzalloc(size); |
266 | if (!addr_tbl) |
267 | return -ENOMEM; |
268 | |
269 | chain->pbl.pp_addr_tbl = addr_tbl; |
270 | |
271 | if (chain->b_external_pbl) { |
272 | pbl_virt = chain->pbl_sp.table_virt; |
273 | goto alloc_pages; |
274 | } |
275 | |
276 | size = array_size(page_cnt, sizeof(*pbl_virt)); |
277 | if (unlikely(size == SIZE_MAX)) |
278 | return -EOVERFLOW; |
279 | |
280 | pbl_virt = dma_alloc_coherent(dev, size, dma_handle: &pbl_phys, GFP_KERNEL); |
281 | if (!pbl_virt) |
282 | return -ENOMEM; |
283 | |
284 | chain->pbl_sp.table_virt = pbl_virt; |
285 | chain->pbl_sp.table_phys = pbl_phys; |
286 | chain->pbl_sp.table_size = size; |
287 | |
288 | alloc_pages: |
289 | for (i = 0; i < page_cnt; i++) { |
290 | virt = dma_alloc_coherent(dev, size: chain->page_size, dma_handle: &phys, |
291 | GFP_KERNEL); |
292 | if (!virt) |
293 | return -ENOMEM; |
294 | |
295 | if (i == 0) { |
296 | qed_chain_init_mem(chain, virt_addr: virt, phys_addr: phys); |
297 | qed_chain_reset(p_chain: chain); |
298 | } |
299 | |
300 | /* Fill the PBL table with the physical address of the page */ |
301 | pbl_virt[i] = cpu_to_le64(phys); |
302 | |
303 | /* Keep the virtual address of the page */ |
304 | addr_tbl[i].virt_addr = virt; |
305 | addr_tbl[i].dma_map = phys; |
306 | } |
307 | |
308 | return 0; |
309 | } |
310 | |
311 | /** |
312 | * qed_chain_alloc() - Allocate and initialize a chain. |
313 | * |
314 | * @cdev: Main device structure. |
315 | * @chain: Chain to be processed. |
316 | * @params: Chain initialization parameters. |
317 | * |
318 | * Return: 0 on success, negative errno otherwise. |
319 | */ |
320 | int qed_chain_alloc(struct qed_dev *cdev, struct qed_chain *chain, |
321 | struct qed_chain_init_params *params) |
322 | { |
323 | u32 page_cnt; |
324 | int rc; |
325 | |
326 | if (!params->page_size) |
327 | params->page_size = QED_CHAIN_PAGE_SIZE; |
328 | |
329 | if (params->mode == QED_CHAIN_MODE_SINGLE) |
330 | page_cnt = 1; |
331 | else |
332 | page_cnt = QED_CHAIN_PAGE_CNT(params->num_elems, |
333 | params->elem_size, |
334 | params->page_size, |
335 | params->mode); |
336 | |
337 | rc = qed_chain_alloc_sanity_check(cdev, params, page_cnt); |
338 | if (rc) { |
339 | DP_NOTICE(cdev, |
340 | "Cannot allocate a chain with the given arguments:\n" ); |
341 | DP_NOTICE(cdev, |
342 | "[use_mode %d, mode %d, cnt_type %d, num_elems %d, elem_size %zu, page_size %u]\n" , |
343 | params->intended_use, params->mode, params->cnt_type, |
344 | params->num_elems, params->elem_size, |
345 | params->page_size); |
346 | return rc; |
347 | } |
348 | |
349 | qed_chain_init(chain, params, page_cnt); |
350 | |
351 | switch (params->mode) { |
352 | case QED_CHAIN_MODE_NEXT_PTR: |
353 | rc = qed_chain_alloc_next_ptr(cdev, chain); |
354 | break; |
355 | case QED_CHAIN_MODE_SINGLE: |
356 | rc = qed_chain_alloc_single(cdev, chain); |
357 | break; |
358 | case QED_CHAIN_MODE_PBL: |
359 | rc = qed_chain_alloc_pbl(cdev, chain); |
360 | break; |
361 | default: |
362 | return -EINVAL; |
363 | } |
364 | |
365 | if (!rc) |
366 | return 0; |
367 | |
368 | qed_chain_free(cdev, chain); |
369 | |
370 | return rc; |
371 | } |
372 | |