1 | |
2 | /* SPDX-License-Identifier: GPL-2.0-only */ |
3 | /* |
4 | * Copyright 2016 Broadcom |
5 | */ |
6 | |
7 | #ifndef _CIPHER_H |
8 | #define _CIPHER_H |
9 | |
10 | #include <linux/atomic.h> |
11 | #include <linux/mailbox/brcm-message.h> |
12 | #include <linux/mailbox_client.h> |
13 | #include <crypto/aes.h> |
14 | #include <crypto/internal/hash.h> |
15 | #include <crypto/internal/skcipher.h> |
16 | #include <crypto/aead.h> |
17 | #include <crypto/arc4.h> |
18 | #include <crypto/gcm.h> |
19 | #include <crypto/sha1.h> |
20 | #include <crypto/sha2.h> |
21 | #include <crypto/sha3.h> |
22 | |
23 | #include "spu.h" |
24 | #include "spum.h" |
25 | #include "spu2.h" |
26 | |
27 | /* Driver supports up to MAX_SPUS SPU blocks */ |
28 | #define MAX_SPUS 16 |
29 | |
30 | #define ARC4_STATE_SIZE 4 |
31 | |
32 | #define CCM_AES_IV_SIZE 16 |
33 | #define CCM_ESP_IV_SIZE 8 |
34 | #define RFC4543_ICV_SIZE 16 |
35 | |
36 | #define MAX_KEY_SIZE ARC4_MAX_KEY_SIZE |
37 | #define MAX_IV_SIZE AES_BLOCK_SIZE |
38 | #define MAX_DIGEST_SIZE SHA3_512_DIGEST_SIZE |
39 | #define MAX_ASSOC_SIZE 512 |
40 | |
41 | /* size of salt value for AES-GCM-ESP and AES-CCM-ESP */ |
42 | #define GCM_ESP_SALT_SIZE 4 |
43 | #define CCM_ESP_SALT_SIZE 3 |
44 | #define MAX_SALT_SIZE GCM_ESP_SALT_SIZE |
45 | #define GCM_ESP_SALT_OFFSET 0 |
46 | #define CCM_ESP_SALT_OFFSET 1 |
47 | |
48 | #define GCM_ESP_DIGESTSIZE 16 |
49 | |
50 | #define MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE |
51 | |
52 | /* |
53 | * Maximum number of bytes from a non-final hash request that can be deferred |
54 | * until more data is available. With new crypto API framework, this |
55 | * can be no more than one block of data. |
56 | */ |
57 | #define HASH_CARRY_MAX MAX_HASH_BLOCK_SIZE |
58 | |
59 | /* Force at least 4-byte alignment of all SPU message fields */ |
60 | #define SPU_MSG_ALIGN 4 |
61 | |
62 | /* Number of times to resend mailbox message if mb queue is full */ |
63 | #define SPU_MB_RETRY_MAX 1000 |
64 | |
65 | /* op_counts[] indexes */ |
66 | enum op_type { |
67 | SPU_OP_CIPHER, |
68 | SPU_OP_HASH, |
69 | SPU_OP_HMAC, |
70 | SPU_OP_AEAD, |
71 | SPU_OP_NUM |
72 | }; |
73 | |
74 | enum spu_spu_type { |
75 | SPU_TYPE_SPUM, |
76 | SPU_TYPE_SPU2, |
77 | }; |
78 | |
79 | /* |
80 | * SPUM_NS2 and SPUM_NSP are the SPU-M block on Northstar 2 and Northstar Plus, |
81 | * respectively. |
82 | */ |
83 | enum spu_spu_subtype { |
84 | SPU_SUBTYPE_SPUM_NS2, |
85 | SPU_SUBTYPE_SPUM_NSP, |
86 | SPU_SUBTYPE_SPU2_V1, |
87 | SPU_SUBTYPE_SPU2_V2 |
88 | }; |
89 | |
90 | struct spu_type_subtype { |
91 | enum spu_spu_type type; |
92 | enum spu_spu_subtype subtype; |
93 | }; |
94 | |
95 | struct cipher_op { |
96 | enum spu_cipher_alg alg; |
97 | enum spu_cipher_mode mode; |
98 | }; |
99 | |
100 | struct auth_op { |
101 | enum hash_alg alg; |
102 | enum hash_mode mode; |
103 | }; |
104 | |
105 | struct iproc_alg_s { |
106 | u32 type; |
107 | union { |
108 | struct skcipher_alg skcipher; |
109 | struct ahash_alg hash; |
110 | struct aead_alg aead; |
111 | } alg; |
112 | struct cipher_op cipher_info; |
113 | struct auth_op auth_info; |
114 | bool auth_first; |
115 | bool registered; |
116 | }; |
117 | |
118 | /* |
119 | * Buffers for a SPU request/reply message pair. All part of one structure to |
120 | * allow a single alloc per request. |
121 | */ |
122 | struct spu_msg_buf { |
123 | /* Request message fragments */ |
124 | |
125 | /* |
126 | * SPU request message header. For SPU-M, holds MH, EMH, SCTX, BDESC, |
127 | * and BD header. For SPU2, holds FMD, OMD. |
128 | */ |
129 | u8 bcm_spu_req_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)]; |
130 | |
131 | /* IV or counter. Size to include salt. Also used for XTS tweek. */ |
132 | u8 iv_ctr[ALIGN(2 * AES_BLOCK_SIZE, SPU_MSG_ALIGN)]; |
133 | |
134 | /* Hash digest. request and response. */ |
135 | u8 digest[ALIGN(MAX_DIGEST_SIZE, SPU_MSG_ALIGN)]; |
136 | |
137 | /* SPU request message padding */ |
138 | u8 spu_req_pad[ALIGN(SPU_PAD_LEN_MAX, SPU_MSG_ALIGN)]; |
139 | |
140 | /* SPU-M request message STATUS field */ |
141 | u8 tx_stat[ALIGN(SPU_TX_STATUS_LEN, SPU_MSG_ALIGN)]; |
142 | |
143 | /* Response message fragments */ |
144 | |
145 | /* SPU response message header */ |
146 | u8 spu_resp_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)]; |
147 | |
148 | /* SPU response message STATUS field padding */ |
149 | u8 rx_stat_pad[ALIGN(SPU_STAT_PAD_MAX, SPU_MSG_ALIGN)]; |
150 | |
151 | /* SPU response message STATUS field */ |
152 | u8 rx_stat[ALIGN(SPU_RX_STATUS_LEN, SPU_MSG_ALIGN)]; |
153 | |
154 | union { |
155 | /* Buffers only used for skcipher */ |
156 | struct { |
157 | /* |
158 | * Field used for either SUPDT when RC4 is used |
159 | * -OR- tweak value when XTS/AES is used |
160 | */ |
161 | u8 supdt_tweak[ALIGN(SPU_SUPDT_LEN, SPU_MSG_ALIGN)]; |
162 | } c; |
163 | |
164 | /* Buffers only used for aead */ |
165 | struct { |
166 | /* SPU response pad for GCM data */ |
167 | u8 gcmpad[ALIGN(AES_BLOCK_SIZE, SPU_MSG_ALIGN)]; |
168 | |
169 | /* SPU request msg padding for GCM AAD */ |
170 | u8 req_aad_pad[ALIGN(SPU_PAD_LEN_MAX, SPU_MSG_ALIGN)]; |
171 | |
172 | /* SPU response data to be discarded */ |
173 | u8 resp_aad[ALIGN(MAX_ASSOC_SIZE + MAX_IV_SIZE, |
174 | SPU_MSG_ALIGN)]; |
175 | } a; |
176 | }; |
177 | }; |
178 | |
179 | struct iproc_ctx_s { |
180 | u8 enckey[MAX_KEY_SIZE + ARC4_STATE_SIZE]; |
181 | unsigned int enckeylen; |
182 | |
183 | u8 authkey[MAX_KEY_SIZE + ARC4_STATE_SIZE]; |
184 | unsigned int authkeylen; |
185 | |
186 | u8 salt[MAX_SALT_SIZE]; |
187 | unsigned int salt_len; |
188 | unsigned int salt_offset; |
189 | u8 iv[MAX_IV_SIZE]; |
190 | |
191 | unsigned int digestsize; |
192 | |
193 | struct iproc_alg_s *alg; |
194 | bool is_esp; |
195 | |
196 | struct cipher_op cipher; |
197 | enum spu_cipher_type cipher_type; |
198 | |
199 | struct auth_op auth; |
200 | bool auth_first; |
201 | |
202 | /* |
203 | * The maximum length in bytes of the payload in a SPU message for this |
204 | * context. For SPU-M, the payload is the combination of AAD and data. |
205 | * For SPU2, the payload is just data. A value of SPU_MAX_PAYLOAD_INF |
206 | * indicates that there is no limit to the length of the SPU message |
207 | * payload. |
208 | */ |
209 | unsigned int max_payload; |
210 | |
211 | struct crypto_aead *fallback_cipher; |
212 | |
213 | /* auth_type is determined during processing of request */ |
214 | |
215 | u8 ipad[MAX_HASH_BLOCK_SIZE]; |
216 | u8 opad[MAX_HASH_BLOCK_SIZE]; |
217 | |
218 | /* |
219 | * Buffer to hold SPU message header template. Template is created at |
220 | * setkey time for skcipher requests, since most of the fields in the |
221 | * header are known at that time. At request time, just fill in a few |
222 | * missing pieces related to length of data in the request and IVs, etc. |
223 | */ |
224 | u8 bcm_spu_req_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)]; |
225 | |
226 | /* Length of SPU request header */ |
227 | u16 spu_req_hdr_len; |
228 | |
229 | /* Expected length of SPU response header */ |
230 | u16 spu_resp_hdr_len; |
231 | |
232 | /* |
233 | * shash descriptor - needed to perform incremental hashing in |
234 | * software, when hw doesn't support it. |
235 | */ |
236 | struct shash_desc *shash; |
237 | |
238 | bool is_rfc4543; /* RFC 4543 style of GMAC */ |
239 | }; |
240 | |
241 | /* state from iproc_reqctx_s necessary for hash state export/import */ |
242 | struct spu_hash_export_s { |
243 | unsigned int total_todo; |
244 | unsigned int total_sent; |
245 | u8 hash_carry[HASH_CARRY_MAX]; |
246 | unsigned int hash_carry_len; |
247 | u8 incr_hash[MAX_DIGEST_SIZE]; |
248 | bool is_sw_hmac; |
249 | }; |
250 | |
251 | struct iproc_reqctx_s { |
252 | /* general context */ |
253 | struct crypto_async_request *parent; |
254 | |
255 | /* only valid after enqueue() */ |
256 | struct iproc_ctx_s *ctx; |
257 | |
258 | u8 chan_idx; /* Mailbox channel to be used to submit this request */ |
259 | |
260 | /* total todo, rx'd, and sent for this request */ |
261 | unsigned int total_todo; |
262 | unsigned int total_received; /* only valid for skcipher */ |
263 | unsigned int total_sent; |
264 | |
265 | /* |
266 | * num bytes sent to hw from the src sg in this request. This can differ |
267 | * from total_sent for incremental hashing. total_sent includes previous |
268 | * init() and update() data. src_sent does not. |
269 | */ |
270 | unsigned int src_sent; |
271 | |
272 | /* |
273 | * For AEAD requests, start of associated data. This will typically |
274 | * point to the beginning of the src scatterlist from the request, |
275 | * since assoc data is at the beginning of the src scatterlist rather |
276 | * than in its own sg. |
277 | */ |
278 | struct scatterlist *assoc; |
279 | |
280 | /* |
281 | * scatterlist entry and offset to start of data for next chunk. Crypto |
282 | * API src scatterlist for AEAD starts with AAD, if present. For first |
283 | * chunk, src_sg is sg entry at beginning of input data (after AAD). |
284 | * src_skip begins at the offset in that sg entry where data begins. |
285 | */ |
286 | struct scatterlist *src_sg; |
287 | int src_nents; /* Number of src entries with data */ |
288 | u32 src_skip; /* bytes of current sg entry already used */ |
289 | |
290 | /* |
291 | * Same for destination. For AEAD, if there is AAD, output data must |
292 | * be written at offset following AAD. |
293 | */ |
294 | struct scatterlist *dst_sg; |
295 | int dst_nents; /* Number of dst entries with data */ |
296 | u32 dst_skip; /* bytes of current sg entry already written */ |
297 | |
298 | /* Mailbox message used to send this request to PDC driver */ |
299 | struct brcm_message mb_mssg; |
300 | |
301 | bool bd_suppress; /* suppress BD field in SPU response? */ |
302 | |
303 | /* cipher context */ |
304 | bool is_encrypt; |
305 | |
306 | /* |
307 | * CBC mode: IV. CTR mode: counter. Else empty. Used as a DMA |
308 | * buffer for AEAD requests. So allocate as DMAable memory. If IV |
309 | * concatenated with salt, includes the salt. |
310 | */ |
311 | u8 *iv_ctr; |
312 | /* Length of IV or counter, in bytes */ |
313 | unsigned int iv_ctr_len; |
314 | |
315 | /* |
316 | * Hash requests can be of any size, whether initial, update, or final. |
317 | * A non-final request must be submitted to the SPU as an integral |
318 | * number of blocks. This may leave data at the end of the request |
319 | * that is not a full block. Since the request is non-final, it cannot |
320 | * be padded. So, we write the remainder to this hash_carry buffer and |
321 | * hold it until the next request arrives. The carry data is then |
322 | * submitted at the beginning of the data in the next SPU msg. |
323 | * hash_carry_len is the number of bytes currently in hash_carry. These |
324 | * fields are only used for ahash requests. |
325 | */ |
326 | u8 hash_carry[HASH_CARRY_MAX]; |
327 | unsigned int hash_carry_len; |
328 | unsigned int is_final; /* is this the final for the hash op? */ |
329 | |
330 | /* |
331 | * Digest from incremental hash is saved here to include in next hash |
332 | * operation. Cannot be stored in req->result for truncated hashes, |
333 | * since result may be sized for final digest. Cannot be saved in |
334 | * msg_buf because that gets deleted between incremental hash ops |
335 | * and is not saved as part of export(). |
336 | */ |
337 | u8 incr_hash[MAX_DIGEST_SIZE]; |
338 | |
339 | /* hmac context */ |
340 | bool is_sw_hmac; |
341 | |
342 | gfp_t gfp; |
343 | |
344 | /* Buffers used to build SPU request and response messages */ |
345 | struct spu_msg_buf msg_buf; |
346 | |
347 | struct aead_request req; |
348 | }; |
349 | |
350 | /* |
351 | * Structure encapsulates a set of function pointers specific to the type of |
352 | * SPU hardware running. These functions handling creation and parsing of |
353 | * SPU request messages and SPU response messages. Includes hardware-specific |
354 | * values read from device tree. |
355 | */ |
356 | struct spu_hw { |
357 | void (*spu_dump_msg_hdr)(u8 *buf, unsigned int buf_len); |
358 | u32 (*spu_ctx_max_payload)(enum spu_cipher_alg cipher_alg, |
359 | enum spu_cipher_mode cipher_mode, |
360 | unsigned int blocksize); |
361 | u32 (*spu_payload_length)(u8 *spu_hdr); |
362 | u16 (*spu_response_hdr_len)(u16 auth_key_len, u16 enc_key_len, |
363 | bool is_hash); |
364 | u16 (*spu_hash_pad_len)(enum hash_alg hash_alg, |
365 | enum hash_mode hash_mode, u32 chunksize, |
366 | u16 hash_block_size); |
367 | u32 (*spu_gcm_ccm_pad_len)(enum spu_cipher_mode cipher_mode, |
368 | unsigned int data_size); |
369 | u32 (*spu_assoc_resp_len)(enum spu_cipher_mode cipher_mode, |
370 | unsigned int assoc_len, |
371 | unsigned int iv_len, bool is_encrypt); |
372 | u8 (*spu_aead_ivlen)(enum spu_cipher_mode cipher_mode, |
373 | u16 iv_len); |
374 | enum hash_type (*spu_hash_type)(u32 src_sent); |
375 | u32 (*spu_digest_size)(u32 digest_size, enum hash_alg alg, |
376 | enum hash_type); |
377 | u32 (*spu_create_request)(u8 *spu_hdr, |
378 | struct spu_request_opts *req_opts, |
379 | struct spu_cipher_parms *cipher_parms, |
380 | struct spu_hash_parms *hash_parms, |
381 | struct spu_aead_parms *aead_parms, |
382 | unsigned int data_size); |
383 | u16 (*spu_cipher_req_init)(u8 *spu_hdr, |
384 | struct spu_cipher_parms *cipher_parms); |
385 | void (*spu_cipher_req_finish)(u8 *spu_hdr, |
386 | u16 spu_req_hdr_len, |
387 | unsigned int is_inbound, |
388 | struct spu_cipher_parms *cipher_parms, |
389 | unsigned int data_size); |
390 | void (*spu_request_pad)(u8 *pad_start, u32 gcm_padding, |
391 | u32 hash_pad_len, enum hash_alg auth_alg, |
392 | enum hash_mode auth_mode, |
393 | unsigned int total_sent, u32 status_padding); |
394 | u8 (*spu_xts_tweak_in_payload)(void); |
395 | u8 (*spu_tx_status_len)(void); |
396 | u8 (*spu_rx_status_len)(void); |
397 | int (*spu_status_process)(u8 *statp); |
398 | void (*spu_ccm_update_iv)(unsigned int digestsize, |
399 | struct spu_cipher_parms *cipher_parms, |
400 | unsigned int assoclen, unsigned int chunksize, |
401 | bool is_encrypt, bool is_esp); |
402 | u32 (*spu_wordalign_padlen)(u32 data_size); |
403 | |
404 | /* The base virtual address of the SPU hw registers */ |
405 | void __iomem *reg_vbase[MAX_SPUS]; |
406 | |
407 | /* Version of the SPU hardware */ |
408 | enum spu_spu_type spu_type; |
409 | |
410 | /* Sub-version of the SPU hardware */ |
411 | enum spu_spu_subtype spu_subtype; |
412 | |
413 | /* The number of SPUs on this platform */ |
414 | u32 num_spu; |
415 | |
416 | /* The number of SPU channels on this platform */ |
417 | u32 num_chan; |
418 | }; |
419 | |
420 | struct bcm_device_private { |
421 | struct platform_device *pdev; |
422 | |
423 | struct spu_hw spu; |
424 | |
425 | atomic_t session_count; /* number of streams active */ |
426 | atomic_t stream_count; /* monotonic counter for streamID's */ |
427 | |
428 | /* Length of BCM header. Set to 0 when hw does not expect BCM HEADER. */ |
429 | u8 bcm_hdr_len; |
430 | |
431 | /* The index of the channel to use for the next crypto request */ |
432 | atomic_t next_chan; |
433 | |
434 | struct dentry *debugfs_dir; |
435 | struct dentry *debugfs_stats; |
436 | |
437 | /* Number of request bytes processed and result bytes returned */ |
438 | atomic64_t bytes_in; |
439 | atomic64_t bytes_out; |
440 | |
441 | /* Number of operations of each type */ |
442 | atomic_t op_counts[SPU_OP_NUM]; |
443 | |
444 | atomic_t cipher_cnt[CIPHER_ALG_LAST][CIPHER_MODE_LAST]; |
445 | atomic_t hash_cnt[HASH_ALG_LAST]; |
446 | atomic_t hmac_cnt[HASH_ALG_LAST]; |
447 | atomic_t aead_cnt[AEAD_TYPE_LAST]; |
448 | |
449 | /* Number of calls to setkey() for each operation type */ |
450 | atomic_t setkey_cnt[SPU_OP_NUM]; |
451 | |
452 | /* Number of times request was resubmitted because mb was full */ |
453 | atomic_t mb_no_spc; |
454 | |
455 | /* Number of mailbox send failures */ |
456 | atomic_t mb_send_fail; |
457 | |
458 | /* Number of ICV check failures for AEAD messages */ |
459 | atomic_t bad_icv; |
460 | |
461 | struct mbox_client mcl; |
462 | |
463 | /* Array of mailbox channel pointers, one for each channel */ |
464 | struct mbox_chan **mbox; |
465 | }; |
466 | |
467 | extern struct bcm_device_private iproc_priv; |
468 | |
469 | #endif |
470 | |