1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | /* Copyright (c) 2016-2017 HiSilicon Limited. */ |
3 | |
4 | #ifndef _SEC_DRV_H_ |
5 | #define _SEC_DRV_H_ |
6 | |
7 | #include <crypto/algapi.h> |
8 | #include <linux/kfifo.h> |
9 | |
10 | #define SEC_MAX_SGE_NUM 64 |
11 | #define SEC_HW_RING_NUM 3 |
12 | |
13 | #define SEC_CMD_RING 0 |
14 | #define SEC_OUTORDER_RING 1 |
15 | #define SEC_DBG_RING 2 |
16 | |
17 | /* A reasonable length to balance memory use against flexibility */ |
18 | #define SEC_QUEUE_LEN 512 |
19 | |
20 | #define SEC_MAX_SGE_NUM 64 |
21 | |
22 | struct sec_bd_info { |
23 | #define SEC_BD_W0_T_LEN_M GENMASK(4, 0) |
24 | #define SEC_BD_W0_T_LEN_S 0 |
25 | |
26 | #define SEC_BD_W0_C_WIDTH_M GENMASK(6, 5) |
27 | #define SEC_BD_W0_C_WIDTH_S 5 |
28 | #define SEC_C_WIDTH_AES_128BIT 0 |
29 | #define SEC_C_WIDTH_AES_8BIT 1 |
30 | #define SEC_C_WIDTH_AES_1BIT 2 |
31 | #define SEC_C_WIDTH_DES_64BIT 0 |
32 | #define SEC_C_WIDTH_DES_8BIT 1 |
33 | #define SEC_C_WIDTH_DES_1BIT 2 |
34 | |
35 | #define SEC_BD_W0_C_MODE_M GENMASK(9, 7) |
36 | #define SEC_BD_W0_C_MODE_S 7 |
37 | #define SEC_C_MODE_ECB 0 |
38 | #define SEC_C_MODE_CBC 1 |
39 | #define SEC_C_MODE_CTR 4 |
40 | #define SEC_C_MODE_CCM 5 |
41 | #define SEC_C_MODE_GCM 6 |
42 | #define SEC_C_MODE_XTS 7 |
43 | |
44 | #define SEC_BD_W0_SEQ BIT(10) |
45 | #define SEC_BD_W0_DE BIT(11) |
46 | #define SEC_BD_W0_DAT_SKIP_M GENMASK(13, 12) |
47 | #define SEC_BD_W0_DAT_SKIP_S 12 |
48 | #define SEC_BD_W0_C_GRAN_SIZE_19_16_M GENMASK(17, 14) |
49 | #define SEC_BD_W0_C_GRAN_SIZE_19_16_S 14 |
50 | |
51 | #define SEC_BD_W0_CIPHER_M GENMASK(19, 18) |
52 | #define SEC_BD_W0_CIPHER_S 18 |
53 | #define SEC_CIPHER_NULL 0 |
54 | #define SEC_CIPHER_ENCRYPT 1 |
55 | #define SEC_CIPHER_DECRYPT 2 |
56 | |
57 | #define SEC_BD_W0_AUTH_M GENMASK(21, 20) |
58 | #define SEC_BD_W0_AUTH_S 20 |
59 | #define SEC_AUTH_NULL 0 |
60 | #define SEC_AUTH_MAC 1 |
61 | #define SEC_AUTH_VERIF 2 |
62 | |
63 | #define SEC_BD_W0_AI_GEN BIT(22) |
64 | #define SEC_BD_W0_CI_GEN BIT(23) |
65 | #define SEC_BD_W0_NO_HPAD BIT(24) |
66 | #define SEC_BD_W0_HM_M GENMASK(26, 25) |
67 | #define SEC_BD_W0_HM_S 25 |
68 | #define SEC_BD_W0_ICV_OR_SKEY_EN_M GENMASK(28, 27) |
69 | #define SEC_BD_W0_ICV_OR_SKEY_EN_S 27 |
70 | |
71 | /* Multi purpose field - gran size bits for send, flag for recv */ |
72 | #define SEC_BD_W0_FLAG_M GENMASK(30, 29) |
73 | #define SEC_BD_W0_C_GRAN_SIZE_21_20_M GENMASK(30, 29) |
74 | #define SEC_BD_W0_FLAG_S 29 |
75 | #define SEC_BD_W0_C_GRAN_SIZE_21_20_S 29 |
76 | |
77 | #define SEC_BD_W0_DONE BIT(31) |
78 | u32 w0; |
79 | |
80 | #define SEC_BD_W1_AUTH_GRAN_SIZE_M GENMASK(21, 0) |
81 | #define SEC_BD_W1_AUTH_GRAN_SIZE_S 0 |
82 | #define SEC_BD_W1_M_KEY_EN BIT(22) |
83 | #define SEC_BD_W1_BD_INVALID BIT(23) |
84 | #define SEC_BD_W1_ADDR_TYPE BIT(24) |
85 | |
86 | #define SEC_BD_W1_A_ALG_M GENMASK(28, 25) |
87 | #define SEC_BD_W1_A_ALG_S 25 |
88 | #define SEC_A_ALG_SHA1 0 |
89 | #define SEC_A_ALG_SHA256 1 |
90 | #define SEC_A_ALG_MD5 2 |
91 | #define SEC_A_ALG_SHA224 3 |
92 | #define SEC_A_ALG_HMAC_SHA1 8 |
93 | #define SEC_A_ALG_HMAC_SHA224 10 |
94 | #define SEC_A_ALG_HMAC_SHA256 11 |
95 | #define SEC_A_ALG_HMAC_MD5 12 |
96 | #define SEC_A_ALG_AES_XCBC 13 |
97 | #define SEC_A_ALG_AES_CMAC 14 |
98 | |
99 | #define SEC_BD_W1_C_ALG_M GENMASK(31, 29) |
100 | #define SEC_BD_W1_C_ALG_S 29 |
101 | #define SEC_C_ALG_DES 0 |
102 | #define SEC_C_ALG_3DES 1 |
103 | #define SEC_C_ALG_AES 2 |
104 | |
105 | u32 w1; |
106 | |
107 | #define SEC_BD_W2_C_GRAN_SIZE_15_0_M GENMASK(15, 0) |
108 | #define SEC_BD_W2_C_GRAN_SIZE_15_0_S 0 |
109 | #define SEC_BD_W2_GRAN_NUM_M GENMASK(31, 16) |
110 | #define SEC_BD_W2_GRAN_NUM_S 16 |
111 | u32 w2; |
112 | |
113 | #define SEC_BD_W3_AUTH_LEN_OFFSET_M GENMASK(9, 0) |
114 | #define SEC_BD_W3_AUTH_LEN_OFFSET_S 0 |
115 | #define SEC_BD_W3_CIPHER_LEN_OFFSET_M GENMASK(19, 10) |
116 | #define SEC_BD_W3_CIPHER_LEN_OFFSET_S 10 |
117 | #define SEC_BD_W3_MAC_LEN_M GENMASK(24, 20) |
118 | #define SEC_BD_W3_MAC_LEN_S 20 |
119 | #define SEC_BD_W3_A_KEY_LEN_M GENMASK(29, 25) |
120 | #define SEC_BD_W3_A_KEY_LEN_S 25 |
121 | #define SEC_BD_W3_C_KEY_LEN_M GENMASK(31, 30) |
122 | #define SEC_BD_W3_C_KEY_LEN_S 30 |
123 | #define SEC_KEY_LEN_AES_128 0 |
124 | #define SEC_KEY_LEN_AES_192 1 |
125 | #define SEC_KEY_LEN_AES_256 2 |
126 | #define SEC_KEY_LEN_DES 1 |
127 | #define SEC_KEY_LEN_3DES_3_KEY 1 |
128 | #define SEC_KEY_LEN_3DES_2_KEY 3 |
129 | u32 w3; |
130 | |
131 | /* W4,5 */ |
132 | union { |
133 | u32 authkey_addr_lo; |
134 | u32 authiv_addr_lo; |
135 | }; |
136 | union { |
137 | u32 authkey_addr_hi; |
138 | u32 authiv_addr_hi; |
139 | }; |
140 | |
141 | /* W6,7 */ |
142 | u32 cipher_key_addr_lo; |
143 | u32 cipher_key_addr_hi; |
144 | |
145 | /* W8,9 */ |
146 | u32 cipher_iv_addr_lo; |
147 | u32 cipher_iv_addr_hi; |
148 | |
149 | /* W10,11 */ |
150 | u32 data_addr_lo; |
151 | u32 data_addr_hi; |
152 | |
153 | /* W12,13 */ |
154 | u32 mac_addr_lo; |
155 | u32 mac_addr_hi; |
156 | |
157 | /* W14,15 */ |
158 | u32 cipher_destin_addr_lo; |
159 | u32 cipher_destin_addr_hi; |
160 | }; |
161 | |
162 | enum sec_mem_region { |
163 | SEC_COMMON = 0, |
164 | SEC_SAA, |
165 | SEC_NUM_ADDR_REGIONS |
166 | }; |
167 | |
168 | #define SEC_NAME_SIZE 64 |
169 | #define SEC_Q_NUM 16 |
170 | |
171 | |
172 | /** |
173 | * struct sec_queue_ring_cmd - store information about a SEC HW cmd ring |
174 | * @used: Local counter used to cheaply establish if the ring is empty. |
175 | * @lock: Protect against simultaneous adjusting of the read and write pointers. |
176 | * @vaddr: Virtual address for the ram pages used for the ring. |
177 | * @paddr: Physical address of the dma mapped region of ram used for the ring. |
178 | * @callback: Callback function called on a ring element completing. |
179 | */ |
180 | struct sec_queue_ring_cmd { |
181 | atomic_t used; |
182 | struct mutex lock; |
183 | struct sec_bd_info *vaddr; |
184 | dma_addr_t paddr; |
185 | void (*callback)(struct sec_bd_info *resp, void *ctx); |
186 | }; |
187 | |
188 | struct sec_debug_bd_info; |
189 | struct sec_queue_ring_db { |
190 | struct sec_debug_bd_info *vaddr; |
191 | dma_addr_t paddr; |
192 | }; |
193 | |
194 | struct sec_out_bd_info; |
195 | struct sec_queue_ring_cq { |
196 | struct sec_out_bd_info *vaddr; |
197 | dma_addr_t paddr; |
198 | }; |
199 | |
200 | struct sec_dev_info; |
201 | |
202 | enum sec_cipher_alg { |
203 | SEC_C_DES_ECB_64, |
204 | SEC_C_DES_CBC_64, |
205 | |
206 | SEC_C_3DES_ECB_192_3KEY, |
207 | SEC_C_3DES_ECB_192_2KEY, |
208 | |
209 | SEC_C_3DES_CBC_192_3KEY, |
210 | SEC_C_3DES_CBC_192_2KEY, |
211 | |
212 | SEC_C_AES_ECB_128, |
213 | SEC_C_AES_ECB_192, |
214 | SEC_C_AES_ECB_256, |
215 | |
216 | SEC_C_AES_CBC_128, |
217 | SEC_C_AES_CBC_192, |
218 | SEC_C_AES_CBC_256, |
219 | |
220 | SEC_C_AES_CTR_128, |
221 | SEC_C_AES_CTR_192, |
222 | SEC_C_AES_CTR_256, |
223 | |
224 | SEC_C_AES_XTS_128, |
225 | SEC_C_AES_XTS_256, |
226 | |
227 | SEC_C_NULL, |
228 | }; |
229 | |
230 | /** |
231 | * struct sec_alg_tfm_ctx - hardware specific tranformation context |
232 | * @cipher_alg: Cipher algorithm enabled include encryption mode. |
233 | * @key: Key storage if required. |
234 | * @pkey: DMA address for the key storage. |
235 | * @req_template: Request template to save time on setup. |
236 | * @queue: The hardware queue associated with this tfm context. |
237 | * @lock: Protect key and pkey to ensure they are consistent |
238 | * @auth_buf: Current context buffer for auth operations. |
239 | * @backlog: The backlog queue used for cases where our buffers aren't |
240 | * large enough. |
241 | */ |
242 | struct sec_alg_tfm_ctx { |
243 | enum sec_cipher_alg cipher_alg; |
244 | u8 *key; |
245 | dma_addr_t pkey; |
246 | struct sec_bd_info req_template; |
247 | struct sec_queue *queue; |
248 | struct mutex lock; |
249 | u8 *auth_buf; |
250 | struct list_head backlog; |
251 | }; |
252 | |
253 | /** |
254 | * struct sec_request - data associate with a single crypto request |
255 | * @elements: List of subparts of this request (hardware size restriction) |
256 | * @num_elements: The number of subparts (used as an optimization) |
257 | * @lock: Protect elements of this structure against concurrent change. |
258 | * @tfm_ctx: hardware specific context. |
259 | * @len_in: length of in sgl from upper layers |
260 | * @len_out: length of out sgl from upper layers |
261 | * @dma_iv: initialization vector - phsyical address |
262 | * @err: store used to track errors across subelements of this request. |
263 | * @req_base: pointer to base element of associate crypto context. |
264 | * This is needed to allow shared handling skcipher, ahash etc. |
265 | * @cb: completion callback. |
266 | * @backlog_head: list head to allow backlog maintenance. |
267 | * |
268 | * The hardware is limited in the maximum size of data that it can |
269 | * process from a single BD. Typically this is fairly large (32MB) |
270 | * but still requires the complexity of splitting the incoming |
271 | * skreq up into a number of elements complete with appropriate |
272 | * iv chaining. |
273 | */ |
274 | struct sec_request { |
275 | struct list_head elements; |
276 | int num_elements; |
277 | struct mutex lock; |
278 | struct sec_alg_tfm_ctx *tfm_ctx; |
279 | int len_in; |
280 | int len_out; |
281 | dma_addr_t dma_iv; |
282 | int err; |
283 | struct crypto_async_request *req_base; |
284 | void (*cb)(struct sec_bd_info *resp, struct crypto_async_request *req); |
285 | struct list_head backlog_head; |
286 | }; |
287 | |
288 | /** |
289 | * struct sec_request_el - A subpart of a request. |
290 | * @head: allow us to attach this to the list in the sec_request |
291 | * @req: hardware block descriptor corresponding to this request subpart |
292 | * @in: hardware sgl for input - virtual address |
293 | * @dma_in: hardware sgl for input - physical address |
294 | * @sgl_in: scatterlist for this request subpart |
295 | * @out: hardware sgl for output - virtual address |
296 | * @dma_out: hardware sgl for output - physical address |
297 | * @sgl_out: scatterlist for this request subpart |
298 | * @sec_req: The request which this subpart forms a part of |
299 | * @el_length: Number of bytes in this subpart. Needed to locate |
300 | * last ivsize chunk for iv chaining. |
301 | */ |
302 | struct sec_request_el { |
303 | struct list_head head; |
304 | struct sec_bd_info req; |
305 | struct sec_hw_sgl *in; |
306 | dma_addr_t dma_in; |
307 | struct scatterlist *sgl_in; |
308 | struct sec_hw_sgl *out; |
309 | dma_addr_t dma_out; |
310 | struct scatterlist *sgl_out; |
311 | struct sec_request *sec_req; |
312 | size_t el_length; |
313 | }; |
314 | |
315 | /** |
316 | * struct sec_queue - All the information about a HW queue |
317 | * @dev_info: The parent SEC device to which this queue belongs. |
318 | * @task_irq: Completion interrupt for the queue. |
319 | * @name: Human readable queue description also used as irq name. |
320 | * @ring: The several HW rings associated with one queue. |
321 | * @regs: The iomapped device registers |
322 | * @queue_id: Index of the queue used for naming and resource selection. |
323 | * @in_use: Flag to say if the queue is in use. |
324 | * @expected: The next expected element to finish assuming we were in order. |
325 | * @uprocessed: A bitmap to track which OoO elements are done but not handled. |
326 | * @softqueue: A software queue used when chaining requirements prevent direct |
327 | * use of the hardware queues. |
328 | * @havesoftqueue: A flag to say we have a queues - as we may need one for the |
329 | * current mode. |
330 | * @queuelock: Protect the soft queue from concurrent changes to avoid some |
331 | * potential loss of data races. |
332 | * @shadow: Pointers back to the shadow copy of the hardware ring element |
333 | * need because we can't store any context reference in the bd element. |
334 | */ |
335 | struct sec_queue { |
336 | struct sec_dev_info *dev_info; |
337 | int task_irq; |
338 | char name[SEC_NAME_SIZE]; |
339 | struct sec_queue_ring_cmd ring_cmd; |
340 | struct sec_queue_ring_cq ring_cq; |
341 | struct sec_queue_ring_db ring_db; |
342 | void __iomem *regs; |
343 | u32 queue_id; |
344 | bool in_use; |
345 | int expected; |
346 | |
347 | DECLARE_BITMAP(unprocessed, SEC_QUEUE_LEN); |
348 | DECLARE_KFIFO_PTR(softqueue, typeof(struct sec_request_el *)); |
349 | bool havesoftqueue; |
350 | spinlock_t queuelock; |
351 | void *shadow[SEC_QUEUE_LEN]; |
352 | }; |
353 | |
354 | /** |
355 | * struct sec_hw_sge: Track each of the 64 element SEC HW SGL entries |
356 | * @buf: The IOV dma address for this entry. |
357 | * @len: Length of this IOV. |
358 | * @pad: Reserved space. |
359 | */ |
360 | struct sec_hw_sge { |
361 | dma_addr_t buf; |
362 | unsigned int len; |
363 | unsigned int pad; |
364 | }; |
365 | |
366 | /** |
367 | * struct sec_hw_sgl: One hardware SGL entry. |
368 | * @next_sgl: The next entry if we need to chain dma address. Null if last. |
369 | * @entry_sum_in_chain: The full count of SGEs - only matters for first SGL. |
370 | * @entry_sum_in_sgl: The number of SGEs in this SGL element. |
371 | * @flag: Unused in skciphers. |
372 | * @serial_num: Unsued in skciphers. |
373 | * @cpuid: Currently unused. |
374 | * @data_bytes_in_sgl: Count of bytes from all SGEs in this SGL. |
375 | * @next: Virtual address used to stash the next sgl - useful in completion. |
376 | * @reserved: A reserved field not currently used. |
377 | * @sge_entries: The (up to) 64 Scatter Gather Entries, representing IOVs. |
378 | * @node: Currently unused. |
379 | */ |
380 | struct sec_hw_sgl { |
381 | dma_addr_t next_sgl; |
382 | u16 entry_sum_in_chain; |
383 | u16 entry_sum_in_sgl; |
384 | u32 flag; |
385 | u64 serial_num; |
386 | u32 cpuid; |
387 | u32 data_bytes_in_sgl; |
388 | struct sec_hw_sgl *next; |
389 | u64 reserved; |
390 | struct sec_hw_sge sge_entries[SEC_MAX_SGE_NUM]; |
391 | u8 node[16]; |
392 | }; |
393 | |
394 | struct dma_pool; |
395 | |
396 | /** |
397 | * struct sec_dev_info: The full SEC unit comprising queues and processors. |
398 | * @sec_id: Index used to track which SEC this is when more than one is present. |
399 | * @num_saas: The number of backed processors enabled. |
400 | * @regs: iomapped register regions shared by whole SEC unit. |
401 | * @dev_lock: Protects concurrent queue allocation / freeing for the SEC. |
402 | * @queues: The 16 queues that this SEC instance provides. |
403 | * @dev: Device pointer. |
404 | * @hw_sgl_pool: DMA pool used to mimise mapping for the scatter gather lists. |
405 | */ |
406 | struct sec_dev_info { |
407 | int sec_id; |
408 | int num_saas; |
409 | void __iomem *regs[SEC_NUM_ADDR_REGIONS]; |
410 | struct mutex dev_lock; |
411 | int queues_in_use; |
412 | struct sec_queue queues[SEC_Q_NUM]; |
413 | struct device *dev; |
414 | struct dma_pool *hw_sgl_pool; |
415 | }; |
416 | |
417 | int sec_queue_send(struct sec_queue *queue, struct sec_bd_info *msg, void *ctx); |
418 | bool sec_queue_can_enqueue(struct sec_queue *queue, int num); |
419 | int sec_queue_stop_release(struct sec_queue *queue); |
420 | struct sec_queue *sec_queue_alloc_start_safe(void); |
421 | bool sec_queue_empty(struct sec_queue *queue); |
422 | |
423 | /* Algorithm specific elements from sec_algs.c */ |
424 | void sec_alg_callback(struct sec_bd_info *resp, void *ctx); |
425 | int sec_algs_register(void); |
426 | void sec_algs_unregister(void); |
427 | |
428 | #endif /* _SEC_DRV_H_ */ |
429 | |