1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2/* Copyright (C) 2018 Netronome Systems, Inc */
3/* Copyright (C) 2021 Corigine, Inc */
4
5#include <linux/module.h>
6#include <linux/kernel.h>
7#include <linux/init.h>
8#include <linux/netdevice.h>
9#include <asm/unaligned.h>
10#include <linux/ktime.h>
11#include <net/xfrm.h>
12
13#include "../nfpcore/nfp_dev.h"
14#include "../nfp_net_ctrl.h"
15#include "../nfp_net.h"
16#include "crypto.h"
17
18#define NFP_NET_IPSEC_MAX_SA_CNT (16 * 1024) /* Firmware support a maximum of 16K SA offload */
19
20/* IPsec config message cmd codes */
21enum nfp_ipsec_cfg_mssg_cmd_codes {
22 NFP_IPSEC_CFG_MSSG_ADD_SA, /* Add a new SA */
23 NFP_IPSEC_CFG_MSSG_INV_SA /* Invalidate an existing SA */
24};
25
26/* IPsec config message response codes */
27enum nfp_ipsec_cfg_mssg_rsp_codes {
28 NFP_IPSEC_CFG_MSSG_OK,
29 NFP_IPSEC_CFG_MSSG_FAILED,
30 NFP_IPSEC_CFG_MSSG_SA_VALID,
31 NFP_IPSEC_CFG_MSSG_SA_HASH_ADD_FAILED,
32 NFP_IPSEC_CFG_MSSG_SA_HASH_DEL_FAILED,
33 NFP_IPSEC_CFG_MSSG_SA_INVALID_CMD
34};
35
36/* Protocol */
37enum nfp_ipsec_sa_prot {
38 NFP_IPSEC_PROTOCOL_AH = 0,
39 NFP_IPSEC_PROTOCOL_ESP = 1
40};
41
42/* Mode */
43enum nfp_ipsec_sa_mode {
44 NFP_IPSEC_PROTMODE_TRANSPORT = 0,
45 NFP_IPSEC_PROTMODE_TUNNEL = 1
46};
47
48/* Cipher types */
49enum nfp_ipsec_sa_cipher {
50 NFP_IPSEC_CIPHER_NULL,
51 NFP_IPSEC_CIPHER_3DES,
52 NFP_IPSEC_CIPHER_AES128,
53 NFP_IPSEC_CIPHER_AES192,
54 NFP_IPSEC_CIPHER_AES256,
55 NFP_IPSEC_CIPHER_AES128_NULL,
56 NFP_IPSEC_CIPHER_AES192_NULL,
57 NFP_IPSEC_CIPHER_AES256_NULL,
58 NFP_IPSEC_CIPHER_CHACHA20
59};
60
61/* Cipher modes */
62enum nfp_ipsec_sa_cipher_mode {
63 NFP_IPSEC_CIMODE_ECB,
64 NFP_IPSEC_CIMODE_CBC,
65 NFP_IPSEC_CIMODE_CFB,
66 NFP_IPSEC_CIMODE_OFB,
67 NFP_IPSEC_CIMODE_CTR
68};
69
70/* Hash types */
71enum nfp_ipsec_sa_hash_type {
72 NFP_IPSEC_HASH_NONE,
73 NFP_IPSEC_HASH_MD5_96,
74 NFP_IPSEC_HASH_SHA1_96,
75 NFP_IPSEC_HASH_SHA256_96,
76 NFP_IPSEC_HASH_SHA384_96,
77 NFP_IPSEC_HASH_SHA512_96,
78 NFP_IPSEC_HASH_MD5_128,
79 NFP_IPSEC_HASH_SHA1_80,
80 NFP_IPSEC_HASH_SHA256_128,
81 NFP_IPSEC_HASH_SHA384_192,
82 NFP_IPSEC_HASH_SHA512_256,
83 NFP_IPSEC_HASH_GF128_128,
84 NFP_IPSEC_HASH_POLY1305_128
85};
86
87/* IPSEC_CFG_MSSG_ADD_SA */
88struct nfp_ipsec_cfg_add_sa {
89 u32 ciph_key[8]; /* Cipher Key */
90 union {
91 u32 auth_key[16]; /* Authentication Key */
92 struct nfp_ipsec_aesgcm { /* AES-GCM-ESP fields */
93 u32 salt; /* Initialized with SA */
94 u32 resv[15];
95 } aesgcm_fields;
96 };
97 struct sa_ctrl_word {
98 uint32_t hash :4; /* From nfp_ipsec_sa_hash_type */
99 uint32_t cimode :4; /* From nfp_ipsec_sa_cipher_mode */
100 uint32_t cipher :4; /* From nfp_ipsec_sa_cipher */
101 uint32_t mode :2; /* From nfp_ipsec_sa_mode */
102 uint32_t proto :2; /* From nfp_ipsec_sa_prot */
103 uint32_t dir :1; /* SA direction */
104 uint32_t resv0 :12;
105 uint32_t encap_dsbl:1; /* Encap/Decap disable */
106 uint32_t resv1 :2; /* Must be set to 0 */
107 } ctrl_word;
108 u32 spi; /* SPI Value */
109 uint32_t pmtu_limit :16; /* PMTU Limit */
110 uint32_t resv0 :5;
111 uint32_t ipv6 :1; /* Outbound IPv6 addr format */
112 uint32_t resv1 :10;
113 u32 resv2[2];
114 u32 src_ip[4]; /* Src IP addr */
115 u32 dst_ip[4]; /* Dst IP addr */
116 u32 resv3[6];
117};
118
119/* IPSEC_CFG_MSSG */
120struct nfp_ipsec_cfg_mssg {
121 union {
122 struct{
123 uint32_t cmd:16; /* One of nfp_ipsec_cfg_mssg_cmd_codes */
124 uint32_t rsp:16; /* One of nfp_ipsec_cfg_mssg_rsp_codes */
125 uint32_t sa_idx:16; /* SA table index */
126 uint32_t spare0:16;
127 struct nfp_ipsec_cfg_add_sa cfg_add_sa;
128 };
129 u32 raw[64];
130 };
131};
132
133static int nfp_net_ipsec_cfg(struct nfp_net *nn, struct nfp_mbox_amsg_entry *entry)
134{
135 unsigned int offset = nn->tlv_caps.mbox_off + NFP_NET_CFG_MBOX_SIMPLE_VAL;
136 struct nfp_ipsec_cfg_mssg *msg = (struct nfp_ipsec_cfg_mssg *)entry->msg;
137 int i, msg_size, ret;
138
139 ret = nfp_net_mbox_lock(nn, data_size: sizeof(*msg));
140 if (ret)
141 return ret;
142
143 msg_size = ARRAY_SIZE(msg->raw);
144 for (i = 0; i < msg_size; i++)
145 nn_writel(nn, off: offset + 4 * i, val: msg->raw[i]);
146
147 ret = nfp_net_mbox_reconfig(nn, mbox_cmd: entry->cmd);
148 if (ret < 0) {
149 nn_ctrl_bar_unlock(nn);
150 return ret;
151 }
152
153 /* For now we always read the whole message response back */
154 for (i = 0; i < msg_size; i++)
155 msg->raw[i] = nn_readl(nn, off: offset + 4 * i);
156
157 nn_ctrl_bar_unlock(nn);
158
159 switch (msg->rsp) {
160 case NFP_IPSEC_CFG_MSSG_OK:
161 return 0;
162 case NFP_IPSEC_CFG_MSSG_SA_INVALID_CMD:
163 return -EINVAL;
164 case NFP_IPSEC_CFG_MSSG_SA_VALID:
165 return -EEXIST;
166 case NFP_IPSEC_CFG_MSSG_FAILED:
167 case NFP_IPSEC_CFG_MSSG_SA_HASH_ADD_FAILED:
168 case NFP_IPSEC_CFG_MSSG_SA_HASH_DEL_FAILED:
169 return -EIO;
170 default:
171 return -EINVAL;
172 }
173}
174
175static int set_aes_keylen(struct nfp_ipsec_cfg_add_sa *cfg, int alg, int keylen)
176{
177 bool aes_gmac = (alg == SADB_X_EALG_NULL_AES_GMAC);
178
179 switch (keylen) {
180 case 128:
181 cfg->ctrl_word.cipher = aes_gmac ? NFP_IPSEC_CIPHER_AES128_NULL :
182 NFP_IPSEC_CIPHER_AES128;
183 break;
184 case 192:
185 cfg->ctrl_word.cipher = aes_gmac ? NFP_IPSEC_CIPHER_AES192_NULL :
186 NFP_IPSEC_CIPHER_AES192;
187 break;
188 case 256:
189 cfg->ctrl_word.cipher = aes_gmac ? NFP_IPSEC_CIPHER_AES256_NULL :
190 NFP_IPSEC_CIPHER_AES256;
191 break;
192 default:
193 return -EINVAL;
194 }
195
196 return 0;
197}
198
199static void set_md5hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len)
200{
201 switch (*trunc_len) {
202 case 96:
203 cfg->ctrl_word.hash = NFP_IPSEC_HASH_MD5_96;
204 break;
205 case 128:
206 cfg->ctrl_word.hash = NFP_IPSEC_HASH_MD5_128;
207 break;
208 default:
209 *trunc_len = 0;
210 }
211}
212
213static void set_sha1hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len)
214{
215 switch (*trunc_len) {
216 case 96:
217 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA1_96;
218 break;
219 case 80:
220 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA1_80;
221 break;
222 default:
223 *trunc_len = 0;
224 }
225}
226
227static void set_sha2_256hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len)
228{
229 switch (*trunc_len) {
230 case 96:
231 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA256_96;
232 break;
233 case 128:
234 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA256_128;
235 break;
236 default:
237 *trunc_len = 0;
238 }
239}
240
241static void set_sha2_384hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len)
242{
243 switch (*trunc_len) {
244 case 96:
245 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA384_96;
246 break;
247 case 192:
248 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA384_192;
249 break;
250 default:
251 *trunc_len = 0;
252 }
253}
254
255static void set_sha2_512hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len)
256{
257 switch (*trunc_len) {
258 case 96:
259 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA512_96;
260 break;
261 case 256:
262 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA512_256;
263 break;
264 default:
265 *trunc_len = 0;
266 }
267}
268
269static int nfp_net_xfrm_add_state(struct xfrm_state *x,
270 struct netlink_ext_ack *extack)
271{
272 struct net_device *netdev = x->xso.real_dev;
273 struct nfp_ipsec_cfg_mssg msg = {};
274 int i, key_len, trunc_len, err = 0;
275 struct nfp_ipsec_cfg_add_sa *cfg;
276 struct nfp_net *nn;
277 unsigned int saidx;
278
279 nn = netdev_priv(dev: netdev);
280 cfg = &msg.cfg_add_sa;
281
282 /* General */
283 switch (x->props.mode) {
284 case XFRM_MODE_TUNNEL:
285 cfg->ctrl_word.mode = NFP_IPSEC_PROTMODE_TUNNEL;
286 break;
287 case XFRM_MODE_TRANSPORT:
288 cfg->ctrl_word.mode = NFP_IPSEC_PROTMODE_TRANSPORT;
289 break;
290 default:
291 NL_SET_ERR_MSG_MOD(extack, "Unsupported mode for xfrm offload");
292 return -EINVAL;
293 }
294
295 switch (x->id.proto) {
296 case IPPROTO_ESP:
297 cfg->ctrl_word.proto = NFP_IPSEC_PROTOCOL_ESP;
298 break;
299 case IPPROTO_AH:
300 cfg->ctrl_word.proto = NFP_IPSEC_PROTOCOL_AH;
301 break;
302 default:
303 NL_SET_ERR_MSG_MOD(extack, "Unsupported protocol for xfrm offload");
304 return -EINVAL;
305 }
306
307 if (x->props.flags & XFRM_STATE_ESN) {
308 NL_SET_ERR_MSG_MOD(extack, "Unsupported XFRM_REPLAY_MODE_ESN for xfrm offload");
309 return -EINVAL;
310 }
311
312 if (x->xso.type != XFRM_DEV_OFFLOAD_CRYPTO) {
313 NL_SET_ERR_MSG_MOD(extack, "Unsupported xfrm offload type");
314 return -EINVAL;
315 }
316
317 cfg->spi = ntohl(x->id.spi);
318
319 /* Hash/Authentication */
320 if (x->aalg)
321 trunc_len = x->aalg->alg_trunc_len;
322 else
323 trunc_len = 0;
324
325 switch (x->props.aalgo) {
326 case SADB_AALG_NONE:
327 if (x->aead) {
328 trunc_len = -1;
329 } else {
330 NL_SET_ERR_MSG_MOD(extack, "Unsupported authentication algorithm");
331 return -EINVAL;
332 }
333 break;
334 case SADB_X_AALG_NULL:
335 cfg->ctrl_word.hash = NFP_IPSEC_HASH_NONE;
336 trunc_len = -1;
337 break;
338 case SADB_AALG_MD5HMAC:
339 if (nn->pdev->device == PCI_DEVICE_ID_NFP3800) {
340 NL_SET_ERR_MSG_MOD(extack, "Unsupported authentication algorithm");
341 return -EINVAL;
342 }
343 set_md5hmac(cfg, trunc_len: &trunc_len);
344 break;
345 case SADB_AALG_SHA1HMAC:
346 set_sha1hmac(cfg, trunc_len: &trunc_len);
347 break;
348 case SADB_X_AALG_SHA2_256HMAC:
349 set_sha2_256hmac(cfg, trunc_len: &trunc_len);
350 break;
351 case SADB_X_AALG_SHA2_384HMAC:
352 set_sha2_384hmac(cfg, trunc_len: &trunc_len);
353 break;
354 case SADB_X_AALG_SHA2_512HMAC:
355 set_sha2_512hmac(cfg, trunc_len: &trunc_len);
356 break;
357 default:
358 NL_SET_ERR_MSG_MOD(extack, "Unsupported authentication algorithm");
359 return -EINVAL;
360 }
361
362 if (!trunc_len) {
363 NL_SET_ERR_MSG_MOD(extack, "Unsupported authentication algorithm trunc length");
364 return -EINVAL;
365 }
366
367 if (x->aalg) {
368 key_len = DIV_ROUND_UP(x->aalg->alg_key_len, BITS_PER_BYTE);
369 if (key_len > sizeof(cfg->auth_key)) {
370 NL_SET_ERR_MSG_MOD(extack, "Insufficient space for offloaded auth key");
371 return -EINVAL;
372 }
373 for (i = 0; i < key_len / sizeof(cfg->auth_key[0]) ; i++)
374 cfg->auth_key[i] = get_unaligned_be32(p: x->aalg->alg_key +
375 sizeof(cfg->auth_key[0]) * i);
376 }
377
378 /* Encryption */
379 switch (x->props.ealgo) {
380 case SADB_EALG_NONE:
381 /* The xfrm descriptor for CHACAH20_POLY1305 does not set the algorithm id, which
382 * is the default value SADB_EALG_NONE. In the branch of SADB_EALG_NONE, driver
383 * uses algorithm name to identify CHACAH20_POLY1305's algorithm.
384 */
385 if (x->aead && !strcmp(x->aead->alg_name, "rfc7539esp(chacha20,poly1305)")) {
386 if (nn->pdev->device != PCI_DEVICE_ID_NFP3800) {
387 NL_SET_ERR_MSG_MOD(extack,
388 "Unsupported encryption algorithm for offload");
389 return -EINVAL;
390 }
391 if (x->aead->alg_icv_len != 128) {
392 NL_SET_ERR_MSG_MOD(extack,
393 "ICV must be 128bit with CHACHA20_POLY1305");
394 return -EINVAL;
395 }
396
397 /* Aead->alg_key_len includes 32-bit salt */
398 if (x->aead->alg_key_len - 32 != 256) {
399 NL_SET_ERR_MSG_MOD(extack, "Unsupported CHACHA20 key length");
400 return -EINVAL;
401 }
402
403 /* The CHACHA20's mode is not configured */
404 cfg->ctrl_word.hash = NFP_IPSEC_HASH_POLY1305_128;
405 cfg->ctrl_word.cipher = NFP_IPSEC_CIPHER_CHACHA20;
406 break;
407 }
408 fallthrough;
409 case SADB_EALG_NULL:
410 cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CBC;
411 cfg->ctrl_word.cipher = NFP_IPSEC_CIPHER_NULL;
412 break;
413 case SADB_EALG_3DESCBC:
414 if (nn->pdev->device == PCI_DEVICE_ID_NFP3800) {
415 NL_SET_ERR_MSG_MOD(extack, "Unsupported encryption algorithm for offload");
416 return -EINVAL;
417 }
418 cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CBC;
419 cfg->ctrl_word.cipher = NFP_IPSEC_CIPHER_3DES;
420 break;
421 case SADB_X_EALG_AES_GCM_ICV16:
422 case SADB_X_EALG_NULL_AES_GMAC:
423 if (!x->aead) {
424 NL_SET_ERR_MSG_MOD(extack, "Invalid AES key data");
425 return -EINVAL;
426 }
427
428 if (x->aead->alg_icv_len != 128) {
429 NL_SET_ERR_MSG_MOD(extack, "ICV must be 128bit with SADB_X_EALG_AES_GCM_ICV16");
430 return -EINVAL;
431 }
432 cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CTR;
433 cfg->ctrl_word.hash = NFP_IPSEC_HASH_GF128_128;
434
435 /* Aead->alg_key_len includes 32-bit salt */
436 if (set_aes_keylen(cfg, alg: x->props.ealgo, keylen: x->aead->alg_key_len - 32)) {
437 NL_SET_ERR_MSG_MOD(extack, "Unsupported AES key length");
438 return -EINVAL;
439 }
440 break;
441 case SADB_X_EALG_AESCBC:
442 cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CBC;
443 if (!x->ealg) {
444 NL_SET_ERR_MSG_MOD(extack, "Invalid AES key data");
445 return -EINVAL;
446 }
447 if (set_aes_keylen(cfg, alg: x->props.ealgo, keylen: x->ealg->alg_key_len) < 0) {
448 NL_SET_ERR_MSG_MOD(extack, "Unsupported AES key length");
449 return -EINVAL;
450 }
451 break;
452 default:
453 NL_SET_ERR_MSG_MOD(extack, "Unsupported encryption algorithm for offload");
454 return -EINVAL;
455 }
456
457 if (x->aead) {
458 int key_offset = 0;
459 int salt_len = 4;
460
461 key_len = DIV_ROUND_UP(x->aead->alg_key_len, BITS_PER_BYTE);
462 key_len -= salt_len;
463
464 if (key_len > sizeof(cfg->ciph_key)) {
465 NL_SET_ERR_MSG_MOD(extack, "aead: Insufficient space for offloaded key");
466 return -EINVAL;
467 }
468
469 /* The CHACHA20's key order needs to be adjusted based on hardware design.
470 * Other's key order: {K0, K1, K2, K3, K4, K5, K6, K7}
471 * CHACHA20's key order: {K4, K5, K6, K7, K0, K1, K2, K3}
472 */
473 if (!strcmp(x->aead->alg_name, "rfc7539esp(chacha20,poly1305)"))
474 key_offset = key_len / sizeof(cfg->ciph_key[0]) >> 1;
475
476 for (i = 0; i < key_len / sizeof(cfg->ciph_key[0]); i++) {
477 int index = (i + key_offset) % (key_len / sizeof(cfg->ciph_key[0]));
478
479 cfg->ciph_key[index] = get_unaligned_be32(p: x->aead->alg_key +
480 sizeof(cfg->ciph_key[0]) * i);
481 }
482
483 /* Load up the salt */
484 cfg->aesgcm_fields.salt = get_unaligned_be32(p: x->aead->alg_key + key_len);
485 }
486
487 if (x->ealg) {
488 key_len = DIV_ROUND_UP(x->ealg->alg_key_len, BITS_PER_BYTE);
489
490 if (key_len > sizeof(cfg->ciph_key)) {
491 NL_SET_ERR_MSG_MOD(extack, "ealg: Insufficient space for offloaded key");
492 return -EINVAL;
493 }
494 for (i = 0; i < key_len / sizeof(cfg->ciph_key[0]) ; i++)
495 cfg->ciph_key[i] = get_unaligned_be32(p: x->ealg->alg_key +
496 sizeof(cfg->ciph_key[0]) * i);
497 }
498
499 /* IP related info */
500 switch (x->props.family) {
501 case AF_INET:
502 cfg->ipv6 = 0;
503 cfg->src_ip[0] = ntohl(x->props.saddr.a4);
504 cfg->dst_ip[0] = ntohl(x->id.daddr.a4);
505 break;
506 case AF_INET6:
507 cfg->ipv6 = 1;
508 for (i = 0; i < 4; i++) {
509 cfg->src_ip[i] = ntohl(x->props.saddr.a6[i]);
510 cfg->dst_ip[i] = ntohl(x->id.daddr.a6[i]);
511 }
512 break;
513 default:
514 NL_SET_ERR_MSG_MOD(extack, "Unsupported address family");
515 return -EINVAL;
516 }
517
518 /* Maximum nic IPsec code could handle. Other limits may apply. */
519 cfg->pmtu_limit = 0xffff;
520 cfg->ctrl_word.encap_dsbl = 1;
521
522 /* SA direction */
523 cfg->ctrl_word.dir = x->xso.dir;
524
525 /* Find unused SA data*/
526 err = xa_alloc(xa: &nn->xa_ipsec, id: &saidx, entry: x,
527 XA_LIMIT(0, NFP_NET_IPSEC_MAX_SA_CNT - 1), GFP_KERNEL);
528 if (err < 0) {
529 NL_SET_ERR_MSG_MOD(extack, "Unable to get sa_data number for IPsec");
530 return err;
531 }
532
533 /* Allocate saidx and commit the SA */
534 msg.cmd = NFP_IPSEC_CFG_MSSG_ADD_SA;
535 msg.sa_idx = saidx;
536 err = nfp_net_sched_mbox_amsg_work(nn, NFP_NET_CFG_MBOX_CMD_IPSEC, data: &msg,
537 len: sizeof(msg), cb: nfp_net_ipsec_cfg);
538 if (err) {
539 xa_erase(&nn->xa_ipsec, index: saidx);
540 NL_SET_ERR_MSG_MOD(extack, "Failed to issue IPsec command");
541 return err;
542 }
543
544 /* 0 is invalid offload_handle for kernel */
545 x->xso.offload_handle = saidx + 1;
546 return 0;
547}
548
549static void nfp_net_xfrm_del_state(struct xfrm_state *x)
550{
551 struct nfp_ipsec_cfg_mssg msg = {
552 .cmd = NFP_IPSEC_CFG_MSSG_INV_SA,
553 .sa_idx = x->xso.offload_handle - 1,
554 };
555 struct net_device *netdev = x->xso.real_dev;
556 struct nfp_net *nn;
557 int err;
558
559 nn = netdev_priv(dev: netdev);
560 err = nfp_net_sched_mbox_amsg_work(nn, NFP_NET_CFG_MBOX_CMD_IPSEC, data: &msg,
561 len: sizeof(msg), cb: nfp_net_ipsec_cfg);
562 if (err)
563 nn_warn(nn, "Failed to invalidate SA in hardware\n");
564
565 xa_erase(&nn->xa_ipsec, index: x->xso.offload_handle - 1);
566}
567
568static bool nfp_net_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x)
569{
570 if (x->props.family == AF_INET)
571 /* Offload with IPv4 options is not supported yet */
572 return ip_hdr(skb)->ihl == 5;
573
574 /* Offload with IPv6 extension headers is not support yet */
575 return !(ipv6_ext_hdr(nexthdr: ipv6_hdr(skb)->nexthdr));
576}
577
578static const struct xfrmdev_ops nfp_net_ipsec_xfrmdev_ops = {
579 .xdo_dev_state_add = nfp_net_xfrm_add_state,
580 .xdo_dev_state_delete = nfp_net_xfrm_del_state,
581 .xdo_dev_offload_ok = nfp_net_ipsec_offload_ok,
582};
583
584void nfp_net_ipsec_init(struct nfp_net *nn)
585{
586 if (!(nn->cap_w1 & NFP_NET_CFG_CTRL_IPSEC))
587 return;
588
589 xa_init_flags(xa: &nn->xa_ipsec, XA_FLAGS_ALLOC);
590 nn->dp.netdev->xfrmdev_ops = &nfp_net_ipsec_xfrmdev_ops;
591}
592
593void nfp_net_ipsec_clean(struct nfp_net *nn)
594{
595 if (!(nn->cap_w1 & NFP_NET_CFG_CTRL_IPSEC))
596 return;
597
598 WARN_ON(!xa_empty(&nn->xa_ipsec));
599 xa_destroy(&nn->xa_ipsec);
600}
601
602bool nfp_net_ipsec_tx_prep(struct nfp_net_dp *dp, struct sk_buff *skb,
603 struct nfp_ipsec_offload *offload_info)
604{
605 struct xfrm_offload *xo = xfrm_offload(skb);
606 struct xfrm_state *x;
607
608 x = xfrm_input_state(skb);
609 if (!x)
610 return false;
611
612 offload_info->seq_hi = xo->seq.hi;
613 offload_info->seq_low = xo->seq.low;
614 offload_info->handle = x->xso.offload_handle;
615
616 return true;
617}
618
619int nfp_net_ipsec_rx(struct nfp_meta_parsed *meta, struct sk_buff *skb)
620{
621 struct net_device *netdev = skb->dev;
622 struct xfrm_offload *xo;
623 struct xfrm_state *x;
624 struct sec_path *sp;
625 struct nfp_net *nn;
626 u32 saidx;
627
628 nn = netdev_priv(dev: netdev);
629
630 saidx = meta->ipsec_saidx - 1;
631 if (saidx >= NFP_NET_IPSEC_MAX_SA_CNT)
632 return -EINVAL;
633
634 sp = secpath_set(skb);
635 if (unlikely(!sp))
636 return -ENOMEM;
637
638 xa_lock(&nn->xa_ipsec);
639 x = xa_load(&nn->xa_ipsec, index: saidx);
640 xa_unlock(&nn->xa_ipsec);
641 if (!x)
642 return -EINVAL;
643
644 xfrm_state_hold(x);
645 sp->xvec[sp->len++] = x;
646 sp->olen++;
647 xo = xfrm_offload(skb);
648 xo->flags = CRYPTO_DONE;
649 xo->status = CRYPTO_SUCCESS;
650
651 return 0;
652}
653

source code of linux/drivers/net/ethernet/netronome/nfp/crypto/ipsec.c