1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * xfrm_policy.c |
4 | * |
5 | * Changes: |
6 | * Mitsuru KANDA @USAGI |
7 | * Kazunori MIYAZAWA @USAGI |
8 | * Kunihiro Ishiguro <kunihiro@ipinfusion.com> |
9 | * IPv6 support |
10 | * Kazunori MIYAZAWA @USAGI |
11 | * YOSHIFUJI Hideaki |
12 | * Split up af-specific portion |
13 | * Derek Atkins <derek@ihtfp.com> Add the post_input processor |
14 | * |
15 | */ |
16 | |
17 | #include <linux/err.h> |
18 | #include <linux/slab.h> |
19 | #include <linux/kmod.h> |
20 | #include <linux/list.h> |
21 | #include <linux/spinlock.h> |
22 | #include <linux/workqueue.h> |
23 | #include <linux/notifier.h> |
24 | #include <linux/netdevice.h> |
25 | #include <linux/netfilter.h> |
26 | #include <linux/module.h> |
27 | #include <linux/cache.h> |
28 | #include <linux/cpu.h> |
29 | #include <linux/audit.h> |
30 | #include <linux/rhashtable.h> |
31 | #include <linux/if_tunnel.h> |
32 | #include <net/dst.h> |
33 | #include <net/flow.h> |
34 | #include <net/inet_ecn.h> |
35 | #include <net/xfrm.h> |
36 | #include <net/ip.h> |
37 | #include <net/gre.h> |
38 | #if IS_ENABLED(CONFIG_IPV6_MIP6) |
39 | #include <net/mip6.h> |
40 | #endif |
41 | #ifdef CONFIG_XFRM_STATISTICS |
42 | #include <net/snmp.h> |
43 | #endif |
44 | #ifdef CONFIG_XFRM_ESPINTCP |
45 | #include <net/espintcp.h> |
46 | #endif |
47 | |
48 | #include "xfrm_hash.h" |
49 | |
50 | #define XFRM_QUEUE_TMO_MIN ((unsigned)(HZ/10)) |
51 | #define XFRM_QUEUE_TMO_MAX ((unsigned)(60*HZ)) |
52 | #define XFRM_MAX_QUEUE_LEN 100 |
53 | |
54 | struct xfrm_flo { |
55 | struct dst_entry *dst_orig; |
56 | u8 flags; |
57 | }; |
58 | |
59 | /* prefixes smaller than this are stored in lists, not trees. */ |
60 | #define INEXACT_PREFIXLEN_IPV4 16 |
61 | #define INEXACT_PREFIXLEN_IPV6 48 |
62 | |
63 | struct xfrm_pol_inexact_node { |
64 | struct rb_node node; |
65 | union { |
66 | xfrm_address_t addr; |
67 | struct rcu_head rcu; |
68 | }; |
69 | u8 prefixlen; |
70 | |
71 | struct rb_root root; |
72 | |
73 | /* the policies matching this node, can be empty list */ |
74 | struct hlist_head hhead; |
75 | }; |
76 | |
77 | /* xfrm inexact policy search tree: |
78 | * xfrm_pol_inexact_bin = hash(dir,type,family,if_id); |
79 | * | |
80 | * +---- root_d: sorted by daddr:prefix |
81 | * | | |
82 | * | xfrm_pol_inexact_node |
83 | * | | |
84 | * | +- root: sorted by saddr/prefix |
85 | * | | | |
86 | * | | xfrm_pol_inexact_node |
87 | * | | | |
88 | * | | + root: unused |
89 | * | | | |
90 | * | | + hhead: saddr:daddr policies |
91 | * | | |
92 | * | +- coarse policies and all any:daddr policies |
93 | * | |
94 | * +---- root_s: sorted by saddr:prefix |
95 | * | | |
96 | * | xfrm_pol_inexact_node |
97 | * | | |
98 | * | + root: unused |
99 | * | | |
100 | * | + hhead: saddr:any policies |
101 | * | |
102 | * +---- coarse policies and all any:any policies |
103 | * |
104 | * Lookups return four candidate lists: |
105 | * 1. any:any list from top-level xfrm_pol_inexact_bin |
106 | * 2. any:daddr list from daddr tree |
107 | * 3. saddr:daddr list from 2nd level daddr tree |
108 | * 4. saddr:any list from saddr tree |
109 | * |
110 | * This result set then needs to be searched for the policy with |
111 | * the lowest priority. If two results have same prio, youngest one wins. |
112 | */ |
113 | |
114 | struct xfrm_pol_inexact_key { |
115 | possible_net_t net; |
116 | u32 if_id; |
117 | u16 family; |
118 | u8 dir, type; |
119 | }; |
120 | |
121 | struct xfrm_pol_inexact_bin { |
122 | struct xfrm_pol_inexact_key k; |
123 | struct rhash_head head; |
124 | /* list containing '*:*' policies */ |
125 | struct hlist_head hhead; |
126 | |
127 | seqcount_spinlock_t count; |
128 | /* tree sorted by daddr/prefix */ |
129 | struct rb_root root_d; |
130 | |
131 | /* tree sorted by saddr/prefix */ |
132 | struct rb_root root_s; |
133 | |
134 | /* slow path below */ |
135 | struct list_head inexact_bins; |
136 | struct rcu_head rcu; |
137 | }; |
138 | |
139 | enum xfrm_pol_inexact_candidate_type { |
140 | XFRM_POL_CAND_BOTH, |
141 | XFRM_POL_CAND_SADDR, |
142 | XFRM_POL_CAND_DADDR, |
143 | XFRM_POL_CAND_ANY, |
144 | |
145 | XFRM_POL_CAND_MAX, |
146 | }; |
147 | |
148 | struct xfrm_pol_inexact_candidates { |
149 | struct hlist_head *res[XFRM_POL_CAND_MAX]; |
150 | }; |
151 | |
152 | struct xfrm_flow_keys { |
153 | struct flow_dissector_key_basic basic; |
154 | struct flow_dissector_key_control control; |
155 | union { |
156 | struct flow_dissector_key_ipv4_addrs ipv4; |
157 | struct flow_dissector_key_ipv6_addrs ipv6; |
158 | } addrs; |
159 | struct flow_dissector_key_ip ip; |
160 | struct flow_dissector_key_icmp icmp; |
161 | struct flow_dissector_key_ports ports; |
162 | struct flow_dissector_key_keyid gre; |
163 | }; |
164 | |
165 | static struct flow_dissector xfrm_session_dissector __ro_after_init; |
166 | |
167 | static DEFINE_SPINLOCK(xfrm_if_cb_lock); |
168 | static struct xfrm_if_cb const __rcu *xfrm_if_cb __read_mostly; |
169 | |
170 | static DEFINE_SPINLOCK(xfrm_policy_afinfo_lock); |
171 | static struct xfrm_policy_afinfo const __rcu *xfrm_policy_afinfo[AF_INET6 + 1] |
172 | __read_mostly; |
173 | |
174 | static struct kmem_cache *xfrm_dst_cache __ro_after_init; |
175 | |
176 | static struct rhashtable xfrm_policy_inexact_table; |
177 | static const struct rhashtable_params xfrm_pol_inexact_params; |
178 | |
179 | static void xfrm_init_pmtu(struct xfrm_dst **bundle, int nr); |
180 | static int stale_bundle(struct dst_entry *dst); |
181 | static int xfrm_bundle_ok(struct xfrm_dst *xdst); |
182 | static void xfrm_policy_queue_process(struct timer_list *t); |
183 | |
184 | static void __xfrm_policy_link(struct xfrm_policy *pol, int dir); |
185 | static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol, |
186 | int dir); |
187 | |
188 | static struct xfrm_pol_inexact_bin * |
189 | xfrm_policy_inexact_lookup(struct net *net, u8 type, u16 family, u8 dir, |
190 | u32 if_id); |
191 | |
192 | static struct xfrm_pol_inexact_bin * |
193 | xfrm_policy_inexact_lookup_rcu(struct net *net, |
194 | u8 type, u16 family, u8 dir, u32 if_id); |
195 | static struct xfrm_policy * |
196 | xfrm_policy_insert_list(struct hlist_head *chain, struct xfrm_policy *policy, |
197 | bool excl); |
198 | static void xfrm_policy_insert_inexact_list(struct hlist_head *chain, |
199 | struct xfrm_policy *policy); |
200 | |
201 | static bool |
202 | xfrm_policy_find_inexact_candidates(struct xfrm_pol_inexact_candidates *cand, |
203 | struct xfrm_pol_inexact_bin *b, |
204 | const xfrm_address_t *saddr, |
205 | const xfrm_address_t *daddr); |
206 | |
207 | static inline bool xfrm_pol_hold_rcu(struct xfrm_policy *policy) |
208 | { |
209 | return refcount_inc_not_zero(r: &policy->refcnt); |
210 | } |
211 | |
212 | static inline bool |
213 | __xfrm4_selector_match(const struct xfrm_selector *sel, const struct flowi *fl) |
214 | { |
215 | const struct flowi4 *fl4 = &fl->u.ip4; |
216 | |
217 | return addr4_match(a1: fl4->daddr, a2: sel->daddr.a4, prefixlen: sel->prefixlen_d) && |
218 | addr4_match(a1: fl4->saddr, a2: sel->saddr.a4, prefixlen: sel->prefixlen_s) && |
219 | !((xfrm_flowi_dport(fl, uli: &fl4->uli) ^ sel->dport) & sel->dport_mask) && |
220 | !((xfrm_flowi_sport(fl, uli: &fl4->uli) ^ sel->sport) & sel->sport_mask) && |
221 | (fl4->flowi4_proto == sel->proto || !sel->proto) && |
222 | (fl4->flowi4_oif == sel->ifindex || !sel->ifindex); |
223 | } |
224 | |
225 | static inline bool |
226 | __xfrm6_selector_match(const struct xfrm_selector *sel, const struct flowi *fl) |
227 | { |
228 | const struct flowi6 *fl6 = &fl->u.ip6; |
229 | |
230 | return addr_match(token1: &fl6->daddr, token2: &sel->daddr, prefixlen: sel->prefixlen_d) && |
231 | addr_match(token1: &fl6->saddr, token2: &sel->saddr, prefixlen: sel->prefixlen_s) && |
232 | !((xfrm_flowi_dport(fl, uli: &fl6->uli) ^ sel->dport) & sel->dport_mask) && |
233 | !((xfrm_flowi_sport(fl, uli: &fl6->uli) ^ sel->sport) & sel->sport_mask) && |
234 | (fl6->flowi6_proto == sel->proto || !sel->proto) && |
235 | (fl6->flowi6_oif == sel->ifindex || !sel->ifindex); |
236 | } |
237 | |
238 | bool xfrm_selector_match(const struct xfrm_selector *sel, const struct flowi *fl, |
239 | unsigned short family) |
240 | { |
241 | switch (family) { |
242 | case AF_INET: |
243 | return __xfrm4_selector_match(sel, fl); |
244 | case AF_INET6: |
245 | return __xfrm6_selector_match(sel, fl); |
246 | } |
247 | return false; |
248 | } |
249 | |
250 | static const struct xfrm_policy_afinfo *xfrm_policy_get_afinfo(unsigned short family) |
251 | { |
252 | const struct xfrm_policy_afinfo *afinfo; |
253 | |
254 | if (unlikely(family >= ARRAY_SIZE(xfrm_policy_afinfo))) |
255 | return NULL; |
256 | rcu_read_lock(); |
257 | afinfo = rcu_dereference(xfrm_policy_afinfo[family]); |
258 | if (unlikely(!afinfo)) |
259 | rcu_read_unlock(); |
260 | return afinfo; |
261 | } |
262 | |
263 | /* Called with rcu_read_lock(). */ |
264 | static const struct xfrm_if_cb *xfrm_if_get_cb(void) |
265 | { |
266 | return rcu_dereference(xfrm_if_cb); |
267 | } |
268 | |
269 | struct dst_entry *__xfrm_dst_lookup(struct net *net, int tos, int oif, |
270 | const xfrm_address_t *saddr, |
271 | const xfrm_address_t *daddr, |
272 | int family, u32 mark) |
273 | { |
274 | const struct xfrm_policy_afinfo *afinfo; |
275 | struct dst_entry *dst; |
276 | |
277 | afinfo = xfrm_policy_get_afinfo(family); |
278 | if (unlikely(afinfo == NULL)) |
279 | return ERR_PTR(error: -EAFNOSUPPORT); |
280 | |
281 | dst = afinfo->dst_lookup(net, tos, oif, saddr, daddr, mark); |
282 | |
283 | rcu_read_unlock(); |
284 | |
285 | return dst; |
286 | } |
287 | EXPORT_SYMBOL(__xfrm_dst_lookup); |
288 | |
289 | static inline struct dst_entry *xfrm_dst_lookup(struct xfrm_state *x, |
290 | int tos, int oif, |
291 | xfrm_address_t *prev_saddr, |
292 | xfrm_address_t *prev_daddr, |
293 | int family, u32 mark) |
294 | { |
295 | struct net *net = xs_net(x); |
296 | xfrm_address_t *saddr = &x->props.saddr; |
297 | xfrm_address_t *daddr = &x->id.daddr; |
298 | struct dst_entry *dst; |
299 | |
300 | if (x->type->flags & XFRM_TYPE_LOCAL_COADDR) { |
301 | saddr = x->coaddr; |
302 | daddr = prev_daddr; |
303 | } |
304 | if (x->type->flags & XFRM_TYPE_REMOTE_COADDR) { |
305 | saddr = prev_saddr; |
306 | daddr = x->coaddr; |
307 | } |
308 | |
309 | dst = __xfrm_dst_lookup(net, tos, oif, saddr, daddr, family, mark); |
310 | |
311 | if (!IS_ERR(ptr: dst)) { |
312 | if (prev_saddr != saddr) |
313 | memcpy(prev_saddr, saddr, sizeof(*prev_saddr)); |
314 | if (prev_daddr != daddr) |
315 | memcpy(prev_daddr, daddr, sizeof(*prev_daddr)); |
316 | } |
317 | |
318 | return dst; |
319 | } |
320 | |
321 | static inline unsigned long make_jiffies(long secs) |
322 | { |
323 | if (secs >= (MAX_SCHEDULE_TIMEOUT-1)/HZ) |
324 | return MAX_SCHEDULE_TIMEOUT-1; |
325 | else |
326 | return secs*HZ; |
327 | } |
328 | |
329 | static void xfrm_policy_timer(struct timer_list *t) |
330 | { |
331 | struct xfrm_policy *xp = from_timer(xp, t, timer); |
332 | time64_t now = ktime_get_real_seconds(); |
333 | time64_t next = TIME64_MAX; |
334 | int warn = 0; |
335 | int dir; |
336 | |
337 | read_lock(&xp->lock); |
338 | |
339 | if (unlikely(xp->walk.dead)) |
340 | goto out; |
341 | |
342 | dir = xfrm_policy_id2dir(index: xp->index); |
343 | |
344 | if (xp->lft.hard_add_expires_seconds) { |
345 | time64_t tmo = xp->lft.hard_add_expires_seconds + |
346 | xp->curlft.add_time - now; |
347 | if (tmo <= 0) |
348 | goto expired; |
349 | if (tmo < next) |
350 | next = tmo; |
351 | } |
352 | if (xp->lft.hard_use_expires_seconds) { |
353 | time64_t tmo = xp->lft.hard_use_expires_seconds + |
354 | (READ_ONCE(xp->curlft.use_time) ? : xp->curlft.add_time) - now; |
355 | if (tmo <= 0) |
356 | goto expired; |
357 | if (tmo < next) |
358 | next = tmo; |
359 | } |
360 | if (xp->lft.soft_add_expires_seconds) { |
361 | time64_t tmo = xp->lft.soft_add_expires_seconds + |
362 | xp->curlft.add_time - now; |
363 | if (tmo <= 0) { |
364 | warn = 1; |
365 | tmo = XFRM_KM_TIMEOUT; |
366 | } |
367 | if (tmo < next) |
368 | next = tmo; |
369 | } |
370 | if (xp->lft.soft_use_expires_seconds) { |
371 | time64_t tmo = xp->lft.soft_use_expires_seconds + |
372 | (READ_ONCE(xp->curlft.use_time) ? : xp->curlft.add_time) - now; |
373 | if (tmo <= 0) { |
374 | warn = 1; |
375 | tmo = XFRM_KM_TIMEOUT; |
376 | } |
377 | if (tmo < next) |
378 | next = tmo; |
379 | } |
380 | |
381 | if (warn) |
382 | km_policy_expired(pol: xp, dir, hard: 0, portid: 0); |
383 | if (next != TIME64_MAX && |
384 | !mod_timer(timer: &xp->timer, expires: jiffies + make_jiffies(secs: next))) |
385 | xfrm_pol_hold(policy: xp); |
386 | |
387 | out: |
388 | read_unlock(&xp->lock); |
389 | xfrm_pol_put(policy: xp); |
390 | return; |
391 | |
392 | expired: |
393 | read_unlock(&xp->lock); |
394 | if (!xfrm_policy_delete(pol: xp, dir)) |
395 | km_policy_expired(pol: xp, dir, hard: 1, portid: 0); |
396 | xfrm_pol_put(policy: xp); |
397 | } |
398 | |
399 | /* Allocate xfrm_policy. Not used here, it is supposed to be used by pfkeyv2 |
400 | * SPD calls. |
401 | */ |
402 | |
403 | struct xfrm_policy *xfrm_policy_alloc(struct net *net, gfp_t gfp) |
404 | { |
405 | struct xfrm_policy *policy; |
406 | |
407 | policy = kzalloc(size: sizeof(struct xfrm_policy), flags: gfp); |
408 | |
409 | if (policy) { |
410 | write_pnet(pnet: &policy->xp_net, net); |
411 | INIT_LIST_HEAD(list: &policy->walk.all); |
412 | INIT_HLIST_NODE(h: &policy->bydst_inexact_list); |
413 | INIT_HLIST_NODE(h: &policy->bydst); |
414 | INIT_HLIST_NODE(h: &policy->byidx); |
415 | rwlock_init(&policy->lock); |
416 | refcount_set(r: &policy->refcnt, n: 1); |
417 | skb_queue_head_init(list: &policy->polq.hold_queue); |
418 | timer_setup(&policy->timer, xfrm_policy_timer, 0); |
419 | timer_setup(&policy->polq.hold_timer, |
420 | xfrm_policy_queue_process, 0); |
421 | } |
422 | return policy; |
423 | } |
424 | EXPORT_SYMBOL(xfrm_policy_alloc); |
425 | |
426 | static void xfrm_policy_destroy_rcu(struct rcu_head *head) |
427 | { |
428 | struct xfrm_policy *policy = container_of(head, struct xfrm_policy, rcu); |
429 | |
430 | security_xfrm_policy_free(ctx: policy->security); |
431 | kfree(objp: policy); |
432 | } |
433 | |
434 | /* Destroy xfrm_policy: descendant resources must be released to this moment. */ |
435 | |
436 | void xfrm_policy_destroy(struct xfrm_policy *policy) |
437 | { |
438 | BUG_ON(!policy->walk.dead); |
439 | |
440 | if (del_timer(timer: &policy->timer) || del_timer(timer: &policy->polq.hold_timer)) |
441 | BUG(); |
442 | |
443 | xfrm_dev_policy_free(x: policy); |
444 | call_rcu(head: &policy->rcu, func: xfrm_policy_destroy_rcu); |
445 | } |
446 | EXPORT_SYMBOL(xfrm_policy_destroy); |
447 | |
448 | /* Rule must be locked. Release descendant resources, announce |
449 | * entry dead. The rule must be unlinked from lists to the moment. |
450 | */ |
451 | |
452 | static void xfrm_policy_kill(struct xfrm_policy *policy) |
453 | { |
454 | write_lock_bh(&policy->lock); |
455 | policy->walk.dead = 1; |
456 | write_unlock_bh(&policy->lock); |
457 | |
458 | atomic_inc(v: &policy->genid); |
459 | |
460 | if (del_timer(timer: &policy->polq.hold_timer)) |
461 | xfrm_pol_put(policy); |
462 | skb_queue_purge(list: &policy->polq.hold_queue); |
463 | |
464 | if (del_timer(timer: &policy->timer)) |
465 | xfrm_pol_put(policy); |
466 | |
467 | xfrm_pol_put(policy); |
468 | } |
469 | |
470 | static unsigned int xfrm_policy_hashmax __read_mostly = 1 * 1024 * 1024; |
471 | |
472 | static inline unsigned int idx_hash(struct net *net, u32 index) |
473 | { |
474 | return __idx_hash(index, hmask: net->xfrm.policy_idx_hmask); |
475 | } |
476 | |
477 | /* calculate policy hash thresholds */ |
478 | static void __get_hash_thresh(struct net *net, |
479 | unsigned short family, int dir, |
480 | u8 *dbits, u8 *sbits) |
481 | { |
482 | switch (family) { |
483 | case AF_INET: |
484 | *dbits = net->xfrm.policy_bydst[dir].dbits4; |
485 | *sbits = net->xfrm.policy_bydst[dir].sbits4; |
486 | break; |
487 | |
488 | case AF_INET6: |
489 | *dbits = net->xfrm.policy_bydst[dir].dbits6; |
490 | *sbits = net->xfrm.policy_bydst[dir].sbits6; |
491 | break; |
492 | |
493 | default: |
494 | *dbits = 0; |
495 | *sbits = 0; |
496 | } |
497 | } |
498 | |
499 | static struct hlist_head *policy_hash_bysel(struct net *net, |
500 | const struct xfrm_selector *sel, |
501 | unsigned short family, int dir) |
502 | { |
503 | unsigned int hmask = net->xfrm.policy_bydst[dir].hmask; |
504 | unsigned int hash; |
505 | u8 dbits; |
506 | u8 sbits; |
507 | |
508 | __get_hash_thresh(net, family, dir, dbits: &dbits, sbits: &sbits); |
509 | hash = __sel_hash(sel, family, hmask, dbits, sbits); |
510 | |
511 | if (hash == hmask + 1) |
512 | return NULL; |
513 | |
514 | return rcu_dereference_check(net->xfrm.policy_bydst[dir].table, |
515 | lockdep_is_held(&net->xfrm.xfrm_policy_lock)) + hash; |
516 | } |
517 | |
518 | static struct hlist_head *policy_hash_direct(struct net *net, |
519 | const xfrm_address_t *daddr, |
520 | const xfrm_address_t *saddr, |
521 | unsigned short family, int dir) |
522 | { |
523 | unsigned int hmask = net->xfrm.policy_bydst[dir].hmask; |
524 | unsigned int hash; |
525 | u8 dbits; |
526 | u8 sbits; |
527 | |
528 | __get_hash_thresh(net, family, dir, dbits: &dbits, sbits: &sbits); |
529 | hash = __addr_hash(daddr, saddr, family, hmask, dbits, sbits); |
530 | |
531 | return rcu_dereference_check(net->xfrm.policy_bydst[dir].table, |
532 | lockdep_is_held(&net->xfrm.xfrm_policy_lock)) + hash; |
533 | } |
534 | |
535 | static void xfrm_dst_hash_transfer(struct net *net, |
536 | struct hlist_head *list, |
537 | struct hlist_head *ndsttable, |
538 | unsigned int nhashmask, |
539 | int dir) |
540 | { |
541 | struct hlist_node *tmp, *entry0 = NULL; |
542 | struct xfrm_policy *pol; |
543 | unsigned int h0 = 0; |
544 | u8 dbits; |
545 | u8 sbits; |
546 | |
547 | redo: |
548 | hlist_for_each_entry_safe(pol, tmp, list, bydst) { |
549 | unsigned int h; |
550 | |
551 | __get_hash_thresh(net, family: pol->family, dir, dbits: &dbits, sbits: &sbits); |
552 | h = __addr_hash(daddr: &pol->selector.daddr, saddr: &pol->selector.saddr, |
553 | family: pol->family, hmask: nhashmask, dbits, sbits); |
554 | if (!entry0 || pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) { |
555 | hlist_del_rcu(n: &pol->bydst); |
556 | hlist_add_head_rcu(n: &pol->bydst, h: ndsttable + h); |
557 | h0 = h; |
558 | } else { |
559 | if (h != h0) |
560 | continue; |
561 | hlist_del_rcu(n: &pol->bydst); |
562 | hlist_add_behind_rcu(n: &pol->bydst, prev: entry0); |
563 | } |
564 | entry0 = &pol->bydst; |
565 | } |
566 | if (!hlist_empty(h: list)) { |
567 | entry0 = NULL; |
568 | goto redo; |
569 | } |
570 | } |
571 | |
572 | static void xfrm_idx_hash_transfer(struct hlist_head *list, |
573 | struct hlist_head *nidxtable, |
574 | unsigned int nhashmask) |
575 | { |
576 | struct hlist_node *tmp; |
577 | struct xfrm_policy *pol; |
578 | |
579 | hlist_for_each_entry_safe(pol, tmp, list, byidx) { |
580 | unsigned int h; |
581 | |
582 | h = __idx_hash(index: pol->index, hmask: nhashmask); |
583 | hlist_add_head(n: &pol->byidx, h: nidxtable+h); |
584 | } |
585 | } |
586 | |
587 | static unsigned long xfrm_new_hash_mask(unsigned int old_hmask) |
588 | { |
589 | return ((old_hmask + 1) << 1) - 1; |
590 | } |
591 | |
592 | static void xfrm_bydst_resize(struct net *net, int dir) |
593 | { |
594 | unsigned int hmask = net->xfrm.policy_bydst[dir].hmask; |
595 | unsigned int nhashmask = xfrm_new_hash_mask(old_hmask: hmask); |
596 | unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head); |
597 | struct hlist_head *ndst = xfrm_hash_alloc(sz: nsize); |
598 | struct hlist_head *odst; |
599 | int i; |
600 | |
601 | if (!ndst) |
602 | return; |
603 | |
604 | spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock); |
605 | write_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation); |
606 | |
607 | odst = rcu_dereference_protected(net->xfrm.policy_bydst[dir].table, |
608 | lockdep_is_held(&net->xfrm.xfrm_policy_lock)); |
609 | |
610 | for (i = hmask; i >= 0; i--) |
611 | xfrm_dst_hash_transfer(net, list: odst + i, ndsttable: ndst, nhashmask, dir); |
612 | |
613 | rcu_assign_pointer(net->xfrm.policy_bydst[dir].table, ndst); |
614 | net->xfrm.policy_bydst[dir].hmask = nhashmask; |
615 | |
616 | write_seqcount_end(&net->xfrm.xfrm_policy_hash_generation); |
617 | spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock); |
618 | |
619 | synchronize_rcu(); |
620 | |
621 | xfrm_hash_free(n: odst, sz: (hmask + 1) * sizeof(struct hlist_head)); |
622 | } |
623 | |
624 | static void xfrm_byidx_resize(struct net *net) |
625 | { |
626 | unsigned int hmask = net->xfrm.policy_idx_hmask; |
627 | unsigned int nhashmask = xfrm_new_hash_mask(old_hmask: hmask); |
628 | unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head); |
629 | struct hlist_head *oidx = net->xfrm.policy_byidx; |
630 | struct hlist_head *nidx = xfrm_hash_alloc(sz: nsize); |
631 | int i; |
632 | |
633 | if (!nidx) |
634 | return; |
635 | |
636 | spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock); |
637 | |
638 | for (i = hmask; i >= 0; i--) |
639 | xfrm_idx_hash_transfer(list: oidx + i, nidxtable: nidx, nhashmask); |
640 | |
641 | net->xfrm.policy_byidx = nidx; |
642 | net->xfrm.policy_idx_hmask = nhashmask; |
643 | |
644 | spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock); |
645 | |
646 | xfrm_hash_free(n: oidx, sz: (hmask + 1) * sizeof(struct hlist_head)); |
647 | } |
648 | |
649 | static inline int xfrm_bydst_should_resize(struct net *net, int dir, int *total) |
650 | { |
651 | unsigned int cnt = net->xfrm.policy_count[dir]; |
652 | unsigned int hmask = net->xfrm.policy_bydst[dir].hmask; |
653 | |
654 | if (total) |
655 | *total += cnt; |
656 | |
657 | if ((hmask + 1) < xfrm_policy_hashmax && |
658 | cnt > hmask) |
659 | return 1; |
660 | |
661 | return 0; |
662 | } |
663 | |
664 | static inline int xfrm_byidx_should_resize(struct net *net, int total) |
665 | { |
666 | unsigned int hmask = net->xfrm.policy_idx_hmask; |
667 | |
668 | if ((hmask + 1) < xfrm_policy_hashmax && |
669 | total > hmask) |
670 | return 1; |
671 | |
672 | return 0; |
673 | } |
674 | |
675 | void xfrm_spd_getinfo(struct net *net, struct xfrmk_spdinfo *si) |
676 | { |
677 | si->incnt = net->xfrm.policy_count[XFRM_POLICY_IN]; |
678 | si->outcnt = net->xfrm.policy_count[XFRM_POLICY_OUT]; |
679 | si->fwdcnt = net->xfrm.policy_count[XFRM_POLICY_FWD]; |
680 | si->inscnt = net->xfrm.policy_count[XFRM_POLICY_IN+XFRM_POLICY_MAX]; |
681 | si->outscnt = net->xfrm.policy_count[XFRM_POLICY_OUT+XFRM_POLICY_MAX]; |
682 | si->fwdscnt = net->xfrm.policy_count[XFRM_POLICY_FWD+XFRM_POLICY_MAX]; |
683 | si->spdhcnt = net->xfrm.policy_idx_hmask; |
684 | si->spdhmcnt = xfrm_policy_hashmax; |
685 | } |
686 | EXPORT_SYMBOL(xfrm_spd_getinfo); |
687 | |
688 | static DEFINE_MUTEX(hash_resize_mutex); |
689 | static void xfrm_hash_resize(struct work_struct *work) |
690 | { |
691 | struct net *net = container_of(work, struct net, xfrm.policy_hash_work); |
692 | int dir, total; |
693 | |
694 | mutex_lock(&hash_resize_mutex); |
695 | |
696 | total = 0; |
697 | for (dir = 0; dir < XFRM_POLICY_MAX; dir++) { |
698 | if (xfrm_bydst_should_resize(net, dir, total: &total)) |
699 | xfrm_bydst_resize(net, dir); |
700 | } |
701 | if (xfrm_byidx_should_resize(net, total)) |
702 | xfrm_byidx_resize(net); |
703 | |
704 | mutex_unlock(lock: &hash_resize_mutex); |
705 | } |
706 | |
707 | /* Make sure *pol can be inserted into fastbin. |
708 | * Useful to check that later insert requests will be successful |
709 | * (provided xfrm_policy_lock is held throughout). |
710 | */ |
711 | static struct xfrm_pol_inexact_bin * |
712 | xfrm_policy_inexact_alloc_bin(const struct xfrm_policy *pol, u8 dir) |
713 | { |
714 | struct xfrm_pol_inexact_bin *bin, *prev; |
715 | struct xfrm_pol_inexact_key k = { |
716 | .family = pol->family, |
717 | .type = pol->type, |
718 | .dir = dir, |
719 | .if_id = pol->if_id, |
720 | }; |
721 | struct net *net = xp_net(xp: pol); |
722 | |
723 | lockdep_assert_held(&net->xfrm.xfrm_policy_lock); |
724 | |
725 | write_pnet(pnet: &k.net, net); |
726 | bin = rhashtable_lookup_fast(ht: &xfrm_policy_inexact_table, key: &k, |
727 | params: xfrm_pol_inexact_params); |
728 | if (bin) |
729 | return bin; |
730 | |
731 | bin = kzalloc(size: sizeof(*bin), GFP_ATOMIC); |
732 | if (!bin) |
733 | return NULL; |
734 | |
735 | bin->k = k; |
736 | INIT_HLIST_HEAD(&bin->hhead); |
737 | bin->root_d = RB_ROOT; |
738 | bin->root_s = RB_ROOT; |
739 | seqcount_spinlock_init(&bin->count, &net->xfrm.xfrm_policy_lock); |
740 | |
741 | prev = rhashtable_lookup_get_insert_key(ht: &xfrm_policy_inexact_table, |
742 | key: &bin->k, obj: &bin->head, |
743 | params: xfrm_pol_inexact_params); |
744 | if (!prev) { |
745 | list_add(new: &bin->inexact_bins, head: &net->xfrm.inexact_bins); |
746 | return bin; |
747 | } |
748 | |
749 | kfree(objp: bin); |
750 | |
751 | return IS_ERR(ptr: prev) ? NULL : prev; |
752 | } |
753 | |
754 | static bool xfrm_pol_inexact_addr_use_any_list(const xfrm_address_t *addr, |
755 | int family, u8 prefixlen) |
756 | { |
757 | if (xfrm_addr_any(addr, family)) |
758 | return true; |
759 | |
760 | if (family == AF_INET6 && prefixlen < INEXACT_PREFIXLEN_IPV6) |
761 | return true; |
762 | |
763 | if (family == AF_INET && prefixlen < INEXACT_PREFIXLEN_IPV4) |
764 | return true; |
765 | |
766 | return false; |
767 | } |
768 | |
769 | static bool |
770 | xfrm_policy_inexact_insert_use_any_list(const struct xfrm_policy *policy) |
771 | { |
772 | const xfrm_address_t *addr; |
773 | bool saddr_any, daddr_any; |
774 | u8 prefixlen; |
775 | |
776 | addr = &policy->selector.saddr; |
777 | prefixlen = policy->selector.prefixlen_s; |
778 | |
779 | saddr_any = xfrm_pol_inexact_addr_use_any_list(addr, |
780 | family: policy->family, |
781 | prefixlen); |
782 | addr = &policy->selector.daddr; |
783 | prefixlen = policy->selector.prefixlen_d; |
784 | daddr_any = xfrm_pol_inexact_addr_use_any_list(addr, |
785 | family: policy->family, |
786 | prefixlen); |
787 | return saddr_any && daddr_any; |
788 | } |
789 | |
790 | static void xfrm_pol_inexact_node_init(struct xfrm_pol_inexact_node *node, |
791 | const xfrm_address_t *addr, u8 prefixlen) |
792 | { |
793 | node->addr = *addr; |
794 | node->prefixlen = prefixlen; |
795 | } |
796 | |
797 | static struct xfrm_pol_inexact_node * |
798 | xfrm_pol_inexact_node_alloc(const xfrm_address_t *addr, u8 prefixlen) |
799 | { |
800 | struct xfrm_pol_inexact_node *node; |
801 | |
802 | node = kzalloc(size: sizeof(*node), GFP_ATOMIC); |
803 | if (node) |
804 | xfrm_pol_inexact_node_init(node, addr, prefixlen); |
805 | |
806 | return node; |
807 | } |
808 | |
809 | static int xfrm_policy_addr_delta(const xfrm_address_t *a, |
810 | const xfrm_address_t *b, |
811 | u8 prefixlen, u16 family) |
812 | { |
813 | u32 ma, mb, mask; |
814 | unsigned int pdw, pbi; |
815 | int delta = 0; |
816 | |
817 | switch (family) { |
818 | case AF_INET: |
819 | if (prefixlen == 0) |
820 | return 0; |
821 | mask = ~0U << (32 - prefixlen); |
822 | ma = ntohl(a->a4) & mask; |
823 | mb = ntohl(b->a4) & mask; |
824 | if (ma < mb) |
825 | delta = -1; |
826 | else if (ma > mb) |
827 | delta = 1; |
828 | break; |
829 | case AF_INET6: |
830 | pdw = prefixlen >> 5; |
831 | pbi = prefixlen & 0x1f; |
832 | |
833 | if (pdw) { |
834 | delta = memcmp(p: a->a6, q: b->a6, size: pdw << 2); |
835 | if (delta) |
836 | return delta; |
837 | } |
838 | if (pbi) { |
839 | mask = ~0U << (32 - pbi); |
840 | ma = ntohl(a->a6[pdw]) & mask; |
841 | mb = ntohl(b->a6[pdw]) & mask; |
842 | if (ma < mb) |
843 | delta = -1; |
844 | else if (ma > mb) |
845 | delta = 1; |
846 | } |
847 | break; |
848 | default: |
849 | break; |
850 | } |
851 | |
852 | return delta; |
853 | } |
854 | |
855 | static void xfrm_policy_inexact_list_reinsert(struct net *net, |
856 | struct xfrm_pol_inexact_node *n, |
857 | u16 family) |
858 | { |
859 | unsigned int matched_s, matched_d; |
860 | struct xfrm_policy *policy, *p; |
861 | |
862 | matched_s = 0; |
863 | matched_d = 0; |
864 | |
865 | list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) { |
866 | struct hlist_node *newpos = NULL; |
867 | bool matches_s, matches_d; |
868 | |
869 | if (policy->walk.dead || !policy->bydst_reinsert) |
870 | continue; |
871 | |
872 | WARN_ON_ONCE(policy->family != family); |
873 | |
874 | policy->bydst_reinsert = false; |
875 | hlist_for_each_entry(p, &n->hhead, bydst) { |
876 | if (policy->priority > p->priority) |
877 | newpos = &p->bydst; |
878 | else if (policy->priority == p->priority && |
879 | policy->pos > p->pos) |
880 | newpos = &p->bydst; |
881 | else |
882 | break; |
883 | } |
884 | |
885 | if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET) |
886 | hlist_add_behind_rcu(n: &policy->bydst, prev: newpos); |
887 | else |
888 | hlist_add_head_rcu(n: &policy->bydst, h: &n->hhead); |
889 | |
890 | /* paranoia checks follow. |
891 | * Check that the reinserted policy matches at least |
892 | * saddr or daddr for current node prefix. |
893 | * |
894 | * Matching both is fine, matching saddr in one policy |
895 | * (but not daddr) and then matching only daddr in another |
896 | * is a bug. |
897 | */ |
898 | matches_s = xfrm_policy_addr_delta(a: &policy->selector.saddr, |
899 | b: &n->addr, |
900 | prefixlen: n->prefixlen, |
901 | family) == 0; |
902 | matches_d = xfrm_policy_addr_delta(a: &policy->selector.daddr, |
903 | b: &n->addr, |
904 | prefixlen: n->prefixlen, |
905 | family) == 0; |
906 | if (matches_s && matches_d) |
907 | continue; |
908 | |
909 | WARN_ON_ONCE(!matches_s && !matches_d); |
910 | if (matches_s) |
911 | matched_s++; |
912 | if (matches_d) |
913 | matched_d++; |
914 | WARN_ON_ONCE(matched_s && matched_d); |
915 | } |
916 | } |
917 | |
918 | static void xfrm_policy_inexact_node_reinsert(struct net *net, |
919 | struct xfrm_pol_inexact_node *n, |
920 | struct rb_root *new, |
921 | u16 family) |
922 | { |
923 | struct xfrm_pol_inexact_node *node; |
924 | struct rb_node **p, *parent; |
925 | |
926 | /* we should not have another subtree here */ |
927 | WARN_ON_ONCE(!RB_EMPTY_ROOT(&n->root)); |
928 | restart: |
929 | parent = NULL; |
930 | p = &new->rb_node; |
931 | while (*p) { |
932 | u8 prefixlen; |
933 | int delta; |
934 | |
935 | parent = *p; |
936 | node = rb_entry(*p, struct xfrm_pol_inexact_node, node); |
937 | |
938 | prefixlen = min(node->prefixlen, n->prefixlen); |
939 | |
940 | delta = xfrm_policy_addr_delta(a: &n->addr, b: &node->addr, |
941 | prefixlen, family); |
942 | if (delta < 0) { |
943 | p = &parent->rb_left; |
944 | } else if (delta > 0) { |
945 | p = &parent->rb_right; |
946 | } else { |
947 | bool same_prefixlen = node->prefixlen == n->prefixlen; |
948 | struct xfrm_policy *tmp; |
949 | |
950 | hlist_for_each_entry(tmp, &n->hhead, bydst) { |
951 | tmp->bydst_reinsert = true; |
952 | hlist_del_rcu(n: &tmp->bydst); |
953 | } |
954 | |
955 | node->prefixlen = prefixlen; |
956 | |
957 | xfrm_policy_inexact_list_reinsert(net, n: node, family); |
958 | |
959 | if (same_prefixlen) { |
960 | kfree_rcu(n, rcu); |
961 | return; |
962 | } |
963 | |
964 | rb_erase(*p, new); |
965 | kfree_rcu(n, rcu); |
966 | n = node; |
967 | goto restart; |
968 | } |
969 | } |
970 | |
971 | rb_link_node_rcu(node: &n->node, parent, rb_link: p); |
972 | rb_insert_color(&n->node, new); |
973 | } |
974 | |
975 | /* merge nodes v and n */ |
976 | static void xfrm_policy_inexact_node_merge(struct net *net, |
977 | struct xfrm_pol_inexact_node *v, |
978 | struct xfrm_pol_inexact_node *n, |
979 | u16 family) |
980 | { |
981 | struct xfrm_pol_inexact_node *node; |
982 | struct xfrm_policy *tmp; |
983 | struct rb_node *rnode; |
984 | |
985 | /* To-be-merged node v has a subtree. |
986 | * |
987 | * Dismantle it and insert its nodes to n->root. |
988 | */ |
989 | while ((rnode = rb_first(&v->root)) != NULL) { |
990 | node = rb_entry(rnode, struct xfrm_pol_inexact_node, node); |
991 | rb_erase(&node->node, &v->root); |
992 | xfrm_policy_inexact_node_reinsert(net, n: node, new: &n->root, |
993 | family); |
994 | } |
995 | |
996 | hlist_for_each_entry(tmp, &v->hhead, bydst) { |
997 | tmp->bydst_reinsert = true; |
998 | hlist_del_rcu(n: &tmp->bydst); |
999 | } |
1000 | |
1001 | xfrm_policy_inexact_list_reinsert(net, n, family); |
1002 | } |
1003 | |
1004 | static struct xfrm_pol_inexact_node * |
1005 | xfrm_policy_inexact_insert_node(struct net *net, |
1006 | struct rb_root *root, |
1007 | xfrm_address_t *addr, |
1008 | u16 family, u8 prefixlen, u8 dir) |
1009 | { |
1010 | struct xfrm_pol_inexact_node *cached = NULL; |
1011 | struct rb_node **p, *parent = NULL; |
1012 | struct xfrm_pol_inexact_node *node; |
1013 | |
1014 | p = &root->rb_node; |
1015 | while (*p) { |
1016 | int delta; |
1017 | |
1018 | parent = *p; |
1019 | node = rb_entry(*p, struct xfrm_pol_inexact_node, node); |
1020 | |
1021 | delta = xfrm_policy_addr_delta(a: addr, b: &node->addr, |
1022 | prefixlen: node->prefixlen, |
1023 | family); |
1024 | if (delta == 0 && prefixlen >= node->prefixlen) { |
1025 | WARN_ON_ONCE(cached); /* ipsec policies got lost */ |
1026 | return node; |
1027 | } |
1028 | |
1029 | if (delta < 0) |
1030 | p = &parent->rb_left; |
1031 | else |
1032 | p = &parent->rb_right; |
1033 | |
1034 | if (prefixlen < node->prefixlen) { |
1035 | delta = xfrm_policy_addr_delta(a: addr, b: &node->addr, |
1036 | prefixlen, |
1037 | family); |
1038 | if (delta) |
1039 | continue; |
1040 | |
1041 | /* This node is a subnet of the new prefix. It needs |
1042 | * to be removed and re-inserted with the smaller |
1043 | * prefix and all nodes that are now also covered |
1044 | * by the reduced prefixlen. |
1045 | */ |
1046 | rb_erase(&node->node, root); |
1047 | |
1048 | if (!cached) { |
1049 | xfrm_pol_inexact_node_init(node, addr, |
1050 | prefixlen); |
1051 | cached = node; |
1052 | } else { |
1053 | /* This node also falls within the new |
1054 | * prefixlen. Merge the to-be-reinserted |
1055 | * node and this one. |
1056 | */ |
1057 | xfrm_policy_inexact_node_merge(net, v: node, |
1058 | n: cached, family); |
1059 | kfree_rcu(node, rcu); |
1060 | } |
1061 | |
1062 | /* restart */ |
1063 | p = &root->rb_node; |
1064 | parent = NULL; |
1065 | } |
1066 | } |
1067 | |
1068 | node = cached; |
1069 | if (!node) { |
1070 | node = xfrm_pol_inexact_node_alloc(addr, prefixlen); |
1071 | if (!node) |
1072 | return NULL; |
1073 | } |
1074 | |
1075 | rb_link_node_rcu(node: &node->node, parent, rb_link: p); |
1076 | rb_insert_color(&node->node, root); |
1077 | |
1078 | return node; |
1079 | } |
1080 | |
1081 | static void xfrm_policy_inexact_gc_tree(struct rb_root *r, bool rm) |
1082 | { |
1083 | struct xfrm_pol_inexact_node *node; |
1084 | struct rb_node *rn = rb_first(r); |
1085 | |
1086 | while (rn) { |
1087 | node = rb_entry(rn, struct xfrm_pol_inexact_node, node); |
1088 | |
1089 | xfrm_policy_inexact_gc_tree(r: &node->root, rm); |
1090 | rn = rb_next(rn); |
1091 | |
1092 | if (!hlist_empty(h: &node->hhead) || !RB_EMPTY_ROOT(&node->root)) { |
1093 | WARN_ON_ONCE(rm); |
1094 | continue; |
1095 | } |
1096 | |
1097 | rb_erase(&node->node, r); |
1098 | kfree_rcu(node, rcu); |
1099 | } |
1100 | } |
1101 | |
1102 | static void __xfrm_policy_inexact_prune_bin(struct xfrm_pol_inexact_bin *b, bool net_exit) |
1103 | { |
1104 | write_seqcount_begin(&b->count); |
1105 | xfrm_policy_inexact_gc_tree(r: &b->root_d, rm: net_exit); |
1106 | xfrm_policy_inexact_gc_tree(r: &b->root_s, rm: net_exit); |
1107 | write_seqcount_end(&b->count); |
1108 | |
1109 | if (!RB_EMPTY_ROOT(&b->root_d) || !RB_EMPTY_ROOT(&b->root_s) || |
1110 | !hlist_empty(h: &b->hhead)) { |
1111 | WARN_ON_ONCE(net_exit); |
1112 | return; |
1113 | } |
1114 | |
1115 | if (rhashtable_remove_fast(ht: &xfrm_policy_inexact_table, obj: &b->head, |
1116 | params: xfrm_pol_inexact_params) == 0) { |
1117 | list_del(entry: &b->inexact_bins); |
1118 | kfree_rcu(b, rcu); |
1119 | } |
1120 | } |
1121 | |
1122 | static void xfrm_policy_inexact_prune_bin(struct xfrm_pol_inexact_bin *b) |
1123 | { |
1124 | struct net *net = read_pnet(pnet: &b->k.net); |
1125 | |
1126 | spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock); |
1127 | __xfrm_policy_inexact_prune_bin(b, net_exit: false); |
1128 | spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock); |
1129 | } |
1130 | |
1131 | static void __xfrm_policy_inexact_flush(struct net *net) |
1132 | { |
1133 | struct xfrm_pol_inexact_bin *bin, *t; |
1134 | |
1135 | lockdep_assert_held(&net->xfrm.xfrm_policy_lock); |
1136 | |
1137 | list_for_each_entry_safe(bin, t, &net->xfrm.inexact_bins, inexact_bins) |
1138 | __xfrm_policy_inexact_prune_bin(b: bin, net_exit: false); |
1139 | } |
1140 | |
1141 | static struct hlist_head * |
1142 | xfrm_policy_inexact_alloc_chain(struct xfrm_pol_inexact_bin *bin, |
1143 | struct xfrm_policy *policy, u8 dir) |
1144 | { |
1145 | struct xfrm_pol_inexact_node *n; |
1146 | struct net *net; |
1147 | |
1148 | net = xp_net(xp: policy); |
1149 | lockdep_assert_held(&net->xfrm.xfrm_policy_lock); |
1150 | |
1151 | if (xfrm_policy_inexact_insert_use_any_list(policy)) |
1152 | return &bin->hhead; |
1153 | |
1154 | if (xfrm_pol_inexact_addr_use_any_list(addr: &policy->selector.daddr, |
1155 | family: policy->family, |
1156 | prefixlen: policy->selector.prefixlen_d)) { |
1157 | write_seqcount_begin(&bin->count); |
1158 | n = xfrm_policy_inexact_insert_node(net, |
1159 | root: &bin->root_s, |
1160 | addr: &policy->selector.saddr, |
1161 | family: policy->family, |
1162 | prefixlen: policy->selector.prefixlen_s, |
1163 | dir); |
1164 | write_seqcount_end(&bin->count); |
1165 | if (!n) |
1166 | return NULL; |
1167 | |
1168 | return &n->hhead; |
1169 | } |
1170 | |
1171 | /* daddr is fixed */ |
1172 | write_seqcount_begin(&bin->count); |
1173 | n = xfrm_policy_inexact_insert_node(net, |
1174 | root: &bin->root_d, |
1175 | addr: &policy->selector.daddr, |
1176 | family: policy->family, |
1177 | prefixlen: policy->selector.prefixlen_d, dir); |
1178 | write_seqcount_end(&bin->count); |
1179 | if (!n) |
1180 | return NULL; |
1181 | |
1182 | /* saddr is wildcard */ |
1183 | if (xfrm_pol_inexact_addr_use_any_list(addr: &policy->selector.saddr, |
1184 | family: policy->family, |
1185 | prefixlen: policy->selector.prefixlen_s)) |
1186 | return &n->hhead; |
1187 | |
1188 | write_seqcount_begin(&bin->count); |
1189 | n = xfrm_policy_inexact_insert_node(net, |
1190 | root: &n->root, |
1191 | addr: &policy->selector.saddr, |
1192 | family: policy->family, |
1193 | prefixlen: policy->selector.prefixlen_s, dir); |
1194 | write_seqcount_end(&bin->count); |
1195 | if (!n) |
1196 | return NULL; |
1197 | |
1198 | return &n->hhead; |
1199 | } |
1200 | |
1201 | static struct xfrm_policy * |
1202 | xfrm_policy_inexact_insert(struct xfrm_policy *policy, u8 dir, int excl) |
1203 | { |
1204 | struct xfrm_pol_inexact_bin *bin; |
1205 | struct xfrm_policy *delpol; |
1206 | struct hlist_head *chain; |
1207 | struct net *net; |
1208 | |
1209 | bin = xfrm_policy_inexact_alloc_bin(pol: policy, dir); |
1210 | if (!bin) |
1211 | return ERR_PTR(error: -ENOMEM); |
1212 | |
1213 | net = xp_net(xp: policy); |
1214 | lockdep_assert_held(&net->xfrm.xfrm_policy_lock); |
1215 | |
1216 | chain = xfrm_policy_inexact_alloc_chain(bin, policy, dir); |
1217 | if (!chain) { |
1218 | __xfrm_policy_inexact_prune_bin(b: bin, net_exit: false); |
1219 | return ERR_PTR(error: -ENOMEM); |
1220 | } |
1221 | |
1222 | delpol = xfrm_policy_insert_list(chain, policy, excl); |
1223 | if (delpol && excl) { |
1224 | __xfrm_policy_inexact_prune_bin(b: bin, net_exit: false); |
1225 | return ERR_PTR(error: -EEXIST); |
1226 | } |
1227 | |
1228 | chain = &net->xfrm.policy_inexact[dir]; |
1229 | xfrm_policy_insert_inexact_list(chain, policy); |
1230 | |
1231 | if (delpol) |
1232 | __xfrm_policy_inexact_prune_bin(b: bin, net_exit: false); |
1233 | |
1234 | return delpol; |
1235 | } |
1236 | |
1237 | static void xfrm_hash_rebuild(struct work_struct *work) |
1238 | { |
1239 | struct net *net = container_of(work, struct net, |
1240 | xfrm.policy_hthresh.work); |
1241 | unsigned int hmask; |
1242 | struct xfrm_policy *pol; |
1243 | struct xfrm_policy *policy; |
1244 | struct hlist_head *chain; |
1245 | struct hlist_head *odst; |
1246 | struct hlist_node *newpos; |
1247 | int i; |
1248 | int dir; |
1249 | unsigned seq; |
1250 | u8 lbits4, rbits4, lbits6, rbits6; |
1251 | |
1252 | mutex_lock(&hash_resize_mutex); |
1253 | |
1254 | /* read selector prefixlen thresholds */ |
1255 | do { |
1256 | seq = read_seqbegin(sl: &net->xfrm.policy_hthresh.lock); |
1257 | |
1258 | lbits4 = net->xfrm.policy_hthresh.lbits4; |
1259 | rbits4 = net->xfrm.policy_hthresh.rbits4; |
1260 | lbits6 = net->xfrm.policy_hthresh.lbits6; |
1261 | rbits6 = net->xfrm.policy_hthresh.rbits6; |
1262 | } while (read_seqretry(sl: &net->xfrm.policy_hthresh.lock, start: seq)); |
1263 | |
1264 | spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock); |
1265 | write_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation); |
1266 | |
1267 | /* make sure that we can insert the indirect policies again before |
1268 | * we start with destructive action. |
1269 | */ |
1270 | list_for_each_entry(policy, &net->xfrm.policy_all, walk.all) { |
1271 | struct xfrm_pol_inexact_bin *bin; |
1272 | u8 dbits, sbits; |
1273 | |
1274 | if (policy->walk.dead) |
1275 | continue; |
1276 | |
1277 | dir = xfrm_policy_id2dir(index: policy->index); |
1278 | if (dir >= XFRM_POLICY_MAX) |
1279 | continue; |
1280 | |
1281 | if ((dir & XFRM_POLICY_MASK) == XFRM_POLICY_OUT) { |
1282 | if (policy->family == AF_INET) { |
1283 | dbits = rbits4; |
1284 | sbits = lbits4; |
1285 | } else { |
1286 | dbits = rbits6; |
1287 | sbits = lbits6; |
1288 | } |
1289 | } else { |
1290 | if (policy->family == AF_INET) { |
1291 | dbits = lbits4; |
1292 | sbits = rbits4; |
1293 | } else { |
1294 | dbits = lbits6; |
1295 | sbits = rbits6; |
1296 | } |
1297 | } |
1298 | |
1299 | if (policy->selector.prefixlen_d < dbits || |
1300 | policy->selector.prefixlen_s < sbits) |
1301 | continue; |
1302 | |
1303 | bin = xfrm_policy_inexact_alloc_bin(pol: policy, dir); |
1304 | if (!bin) |
1305 | goto out_unlock; |
1306 | |
1307 | if (!xfrm_policy_inexact_alloc_chain(bin, policy, dir)) |
1308 | goto out_unlock; |
1309 | } |
1310 | |
1311 | /* reset the bydst and inexact table in all directions */ |
1312 | for (dir = 0; dir < XFRM_POLICY_MAX; dir++) { |
1313 | struct hlist_node *n; |
1314 | |
1315 | hlist_for_each_entry_safe(policy, n, |
1316 | &net->xfrm.policy_inexact[dir], |
1317 | bydst_inexact_list) { |
1318 | hlist_del_rcu(n: &policy->bydst); |
1319 | hlist_del_init(n: &policy->bydst_inexact_list); |
1320 | } |
1321 | |
1322 | hmask = net->xfrm.policy_bydst[dir].hmask; |
1323 | odst = net->xfrm.policy_bydst[dir].table; |
1324 | for (i = hmask; i >= 0; i--) { |
1325 | hlist_for_each_entry_safe(policy, n, odst + i, bydst) |
1326 | hlist_del_rcu(n: &policy->bydst); |
1327 | } |
1328 | if ((dir & XFRM_POLICY_MASK) == XFRM_POLICY_OUT) { |
1329 | /* dir out => dst = remote, src = local */ |
1330 | net->xfrm.policy_bydst[dir].dbits4 = rbits4; |
1331 | net->xfrm.policy_bydst[dir].sbits4 = lbits4; |
1332 | net->xfrm.policy_bydst[dir].dbits6 = rbits6; |
1333 | net->xfrm.policy_bydst[dir].sbits6 = lbits6; |
1334 | } else { |
1335 | /* dir in/fwd => dst = local, src = remote */ |
1336 | net->xfrm.policy_bydst[dir].dbits4 = lbits4; |
1337 | net->xfrm.policy_bydst[dir].sbits4 = rbits4; |
1338 | net->xfrm.policy_bydst[dir].dbits6 = lbits6; |
1339 | net->xfrm.policy_bydst[dir].sbits6 = rbits6; |
1340 | } |
1341 | } |
1342 | |
1343 | /* re-insert all policies by order of creation */ |
1344 | list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) { |
1345 | if (policy->walk.dead) |
1346 | continue; |
1347 | dir = xfrm_policy_id2dir(index: policy->index); |
1348 | if (dir >= XFRM_POLICY_MAX) { |
1349 | /* skip socket policies */ |
1350 | continue; |
1351 | } |
1352 | newpos = NULL; |
1353 | chain = policy_hash_bysel(net, sel: &policy->selector, |
1354 | family: policy->family, dir); |
1355 | |
1356 | if (!chain) { |
1357 | void *p = xfrm_policy_inexact_insert(policy, dir, excl: 0); |
1358 | |
1359 | WARN_ONCE(IS_ERR(p), "reinsert: %ld\n" , PTR_ERR(p)); |
1360 | continue; |
1361 | } |
1362 | |
1363 | hlist_for_each_entry(pol, chain, bydst) { |
1364 | if (policy->priority >= pol->priority) |
1365 | newpos = &pol->bydst; |
1366 | else |
1367 | break; |
1368 | } |
1369 | if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET) |
1370 | hlist_add_behind_rcu(n: &policy->bydst, prev: newpos); |
1371 | else |
1372 | hlist_add_head_rcu(n: &policy->bydst, h: chain); |
1373 | } |
1374 | |
1375 | out_unlock: |
1376 | __xfrm_policy_inexact_flush(net); |
1377 | write_seqcount_end(&net->xfrm.xfrm_policy_hash_generation); |
1378 | spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock); |
1379 | |
1380 | mutex_unlock(lock: &hash_resize_mutex); |
1381 | } |
1382 | |
1383 | void xfrm_policy_hash_rebuild(struct net *net) |
1384 | { |
1385 | schedule_work(work: &net->xfrm.policy_hthresh.work); |
1386 | } |
1387 | EXPORT_SYMBOL(xfrm_policy_hash_rebuild); |
1388 | |
1389 | /* Generate new index... KAME seems to generate them ordered by cost |
1390 | * of an absolute inpredictability of ordering of rules. This will not pass. */ |
1391 | static u32 xfrm_gen_index(struct net *net, int dir, u32 index) |
1392 | { |
1393 | for (;;) { |
1394 | struct hlist_head *list; |
1395 | struct xfrm_policy *p; |
1396 | u32 idx; |
1397 | int found; |
1398 | |
1399 | if (!index) { |
1400 | idx = (net->xfrm.idx_generator | dir); |
1401 | net->xfrm.idx_generator += 8; |
1402 | } else { |
1403 | idx = index; |
1404 | index = 0; |
1405 | } |
1406 | |
1407 | if (idx == 0) |
1408 | idx = 8; |
1409 | list = net->xfrm.policy_byidx + idx_hash(net, index: idx); |
1410 | found = 0; |
1411 | hlist_for_each_entry(p, list, byidx) { |
1412 | if (p->index == idx) { |
1413 | found = 1; |
1414 | break; |
1415 | } |
1416 | } |
1417 | if (!found) |
1418 | return idx; |
1419 | } |
1420 | } |
1421 | |
1422 | static inline int selector_cmp(struct xfrm_selector *s1, struct xfrm_selector *s2) |
1423 | { |
1424 | u32 *p1 = (u32 *) s1; |
1425 | u32 *p2 = (u32 *) s2; |
1426 | int len = sizeof(struct xfrm_selector) / sizeof(u32); |
1427 | int i; |
1428 | |
1429 | for (i = 0; i < len; i++) { |
1430 | if (p1[i] != p2[i]) |
1431 | return 1; |
1432 | } |
1433 | |
1434 | return 0; |
1435 | } |
1436 | |
1437 | static void xfrm_policy_requeue(struct xfrm_policy *old, |
1438 | struct xfrm_policy *new) |
1439 | { |
1440 | struct xfrm_policy_queue *pq = &old->polq; |
1441 | struct sk_buff_head list; |
1442 | |
1443 | if (skb_queue_empty(list: &pq->hold_queue)) |
1444 | return; |
1445 | |
1446 | __skb_queue_head_init(list: &list); |
1447 | |
1448 | spin_lock_bh(lock: &pq->hold_queue.lock); |
1449 | skb_queue_splice_init(list: &pq->hold_queue, head: &list); |
1450 | if (del_timer(timer: &pq->hold_timer)) |
1451 | xfrm_pol_put(policy: old); |
1452 | spin_unlock_bh(lock: &pq->hold_queue.lock); |
1453 | |
1454 | pq = &new->polq; |
1455 | |
1456 | spin_lock_bh(lock: &pq->hold_queue.lock); |
1457 | skb_queue_splice(list: &list, head: &pq->hold_queue); |
1458 | pq->timeout = XFRM_QUEUE_TMO_MIN; |
1459 | if (!mod_timer(timer: &pq->hold_timer, expires: jiffies)) |
1460 | xfrm_pol_hold(policy: new); |
1461 | spin_unlock_bh(lock: &pq->hold_queue.lock); |
1462 | } |
1463 | |
1464 | static inline bool xfrm_policy_mark_match(const struct xfrm_mark *mark, |
1465 | struct xfrm_policy *pol) |
1466 | { |
1467 | return mark->v == pol->mark.v && mark->m == pol->mark.m; |
1468 | } |
1469 | |
1470 | static u32 xfrm_pol_bin_key(const void *data, u32 len, u32 seed) |
1471 | { |
1472 | const struct xfrm_pol_inexact_key *k = data; |
1473 | u32 a = k->type << 24 | k->dir << 16 | k->family; |
1474 | |
1475 | return jhash_3words(a, b: k->if_id, c: net_hash_mix(net: read_pnet(pnet: &k->net)), |
1476 | initval: seed); |
1477 | } |
1478 | |
1479 | static u32 xfrm_pol_bin_obj(const void *data, u32 len, u32 seed) |
1480 | { |
1481 | const struct xfrm_pol_inexact_bin *b = data; |
1482 | |
1483 | return xfrm_pol_bin_key(data: &b->k, len: 0, seed); |
1484 | } |
1485 | |
1486 | static int xfrm_pol_bin_cmp(struct rhashtable_compare_arg *arg, |
1487 | const void *ptr) |
1488 | { |
1489 | const struct xfrm_pol_inexact_key *key = arg->key; |
1490 | const struct xfrm_pol_inexact_bin *b = ptr; |
1491 | int ret; |
1492 | |
1493 | if (!net_eq(net1: read_pnet(pnet: &b->k.net), net2: read_pnet(pnet: &key->net))) |
1494 | return -1; |
1495 | |
1496 | ret = b->k.dir ^ key->dir; |
1497 | if (ret) |
1498 | return ret; |
1499 | |
1500 | ret = b->k.type ^ key->type; |
1501 | if (ret) |
1502 | return ret; |
1503 | |
1504 | ret = b->k.family ^ key->family; |
1505 | if (ret) |
1506 | return ret; |
1507 | |
1508 | return b->k.if_id ^ key->if_id; |
1509 | } |
1510 | |
1511 | static const struct rhashtable_params xfrm_pol_inexact_params = { |
1512 | .head_offset = offsetof(struct xfrm_pol_inexact_bin, head), |
1513 | .hashfn = xfrm_pol_bin_key, |
1514 | .obj_hashfn = xfrm_pol_bin_obj, |
1515 | .obj_cmpfn = xfrm_pol_bin_cmp, |
1516 | .automatic_shrinking = true, |
1517 | }; |
1518 | |
1519 | static void xfrm_policy_insert_inexact_list(struct hlist_head *chain, |
1520 | struct xfrm_policy *policy) |
1521 | { |
1522 | struct xfrm_policy *pol, *delpol = NULL; |
1523 | struct hlist_node *newpos = NULL; |
1524 | int i = 0; |
1525 | |
1526 | hlist_for_each_entry(pol, chain, bydst_inexact_list) { |
1527 | if (pol->type == policy->type && |
1528 | pol->if_id == policy->if_id && |
1529 | !selector_cmp(s1: &pol->selector, s2: &policy->selector) && |
1530 | xfrm_policy_mark_match(mark: &policy->mark, pol) && |
1531 | xfrm_sec_ctx_match(s1: pol->security, s2: policy->security) && |
1532 | !WARN_ON(delpol)) { |
1533 | delpol = pol; |
1534 | if (policy->priority > pol->priority) |
1535 | continue; |
1536 | } else if (policy->priority >= pol->priority) { |
1537 | newpos = &pol->bydst_inexact_list; |
1538 | continue; |
1539 | } |
1540 | if (delpol) |
1541 | break; |
1542 | } |
1543 | |
1544 | if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET) |
1545 | hlist_add_behind_rcu(n: &policy->bydst_inexact_list, prev: newpos); |
1546 | else |
1547 | hlist_add_head_rcu(n: &policy->bydst_inexact_list, h: chain); |
1548 | |
1549 | hlist_for_each_entry(pol, chain, bydst_inexact_list) { |
1550 | pol->pos = i; |
1551 | i++; |
1552 | } |
1553 | } |
1554 | |
1555 | static struct xfrm_policy *xfrm_policy_insert_list(struct hlist_head *chain, |
1556 | struct xfrm_policy *policy, |
1557 | bool excl) |
1558 | { |
1559 | struct xfrm_policy *pol, *newpos = NULL, *delpol = NULL; |
1560 | |
1561 | hlist_for_each_entry(pol, chain, bydst) { |
1562 | if (pol->type == policy->type && |
1563 | pol->if_id == policy->if_id && |
1564 | !selector_cmp(s1: &pol->selector, s2: &policy->selector) && |
1565 | xfrm_policy_mark_match(mark: &policy->mark, pol) && |
1566 | xfrm_sec_ctx_match(s1: pol->security, s2: policy->security) && |
1567 | !WARN_ON(delpol)) { |
1568 | if (excl) |
1569 | return ERR_PTR(error: -EEXIST); |
1570 | delpol = pol; |
1571 | if (policy->priority > pol->priority) |
1572 | continue; |
1573 | } else if (policy->priority >= pol->priority) { |
1574 | newpos = pol; |
1575 | continue; |
1576 | } |
1577 | if (delpol) |
1578 | break; |
1579 | } |
1580 | |
1581 | if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET) |
1582 | hlist_add_behind_rcu(n: &policy->bydst, prev: &newpos->bydst); |
1583 | else |
1584 | /* Packet offload policies enter to the head |
1585 | * to speed-up lookups. |
1586 | */ |
1587 | hlist_add_head_rcu(n: &policy->bydst, h: chain); |
1588 | |
1589 | return delpol; |
1590 | } |
1591 | |
1592 | int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl) |
1593 | { |
1594 | struct net *net = xp_net(xp: policy); |
1595 | struct xfrm_policy *delpol; |
1596 | struct hlist_head *chain; |
1597 | |
1598 | spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock); |
1599 | chain = policy_hash_bysel(net, sel: &policy->selector, family: policy->family, dir); |
1600 | if (chain) |
1601 | delpol = xfrm_policy_insert_list(chain, policy, excl); |
1602 | else |
1603 | delpol = xfrm_policy_inexact_insert(policy, dir, excl); |
1604 | |
1605 | if (IS_ERR(ptr: delpol)) { |
1606 | spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock); |
1607 | return PTR_ERR(ptr: delpol); |
1608 | } |
1609 | |
1610 | __xfrm_policy_link(pol: policy, dir); |
1611 | |
1612 | /* After previous checking, family can either be AF_INET or AF_INET6 */ |
1613 | if (policy->family == AF_INET) |
1614 | rt_genid_bump_ipv4(net); |
1615 | else |
1616 | rt_genid_bump_ipv6(net); |
1617 | |
1618 | if (delpol) { |
1619 | xfrm_policy_requeue(old: delpol, new: policy); |
1620 | __xfrm_policy_unlink(pol: delpol, dir); |
1621 | } |
1622 | policy->index = delpol ? delpol->index : xfrm_gen_index(net, dir, index: policy->index); |
1623 | hlist_add_head(n: &policy->byidx, h: net->xfrm.policy_byidx+idx_hash(net, index: policy->index)); |
1624 | policy->curlft.add_time = ktime_get_real_seconds(); |
1625 | policy->curlft.use_time = 0; |
1626 | if (!mod_timer(timer: &policy->timer, expires: jiffies + HZ)) |
1627 | xfrm_pol_hold(policy); |
1628 | spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock); |
1629 | |
1630 | if (delpol) |
1631 | xfrm_policy_kill(policy: delpol); |
1632 | else if (xfrm_bydst_should_resize(net, dir, NULL)) |
1633 | schedule_work(work: &net->xfrm.policy_hash_work); |
1634 | |
1635 | return 0; |
1636 | } |
1637 | EXPORT_SYMBOL(xfrm_policy_insert); |
1638 | |
1639 | static struct xfrm_policy * |
1640 | __xfrm_policy_bysel_ctx(struct hlist_head *chain, const struct xfrm_mark *mark, |
1641 | u32 if_id, u8 type, int dir, struct xfrm_selector *sel, |
1642 | struct xfrm_sec_ctx *ctx) |
1643 | { |
1644 | struct xfrm_policy *pol; |
1645 | |
1646 | if (!chain) |
1647 | return NULL; |
1648 | |
1649 | hlist_for_each_entry(pol, chain, bydst) { |
1650 | if (pol->type == type && |
1651 | pol->if_id == if_id && |
1652 | xfrm_policy_mark_match(mark, pol) && |
1653 | !selector_cmp(s1: sel, s2: &pol->selector) && |
1654 | xfrm_sec_ctx_match(s1: ctx, s2: pol->security)) |
1655 | return pol; |
1656 | } |
1657 | |
1658 | return NULL; |
1659 | } |
1660 | |
1661 | struct xfrm_policy * |
1662 | xfrm_policy_bysel_ctx(struct net *net, const struct xfrm_mark *mark, u32 if_id, |
1663 | u8 type, int dir, struct xfrm_selector *sel, |
1664 | struct xfrm_sec_ctx *ctx, int delete, int *err) |
1665 | { |
1666 | struct xfrm_pol_inexact_bin *bin = NULL; |
1667 | struct xfrm_policy *pol, *ret = NULL; |
1668 | struct hlist_head *chain; |
1669 | |
1670 | *err = 0; |
1671 | spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock); |
1672 | chain = policy_hash_bysel(net, sel, family: sel->family, dir); |
1673 | if (!chain) { |
1674 | struct xfrm_pol_inexact_candidates cand; |
1675 | int i; |
1676 | |
1677 | bin = xfrm_policy_inexact_lookup(net, type, |
1678 | family: sel->family, dir, if_id); |
1679 | if (!bin) { |
1680 | spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock); |
1681 | return NULL; |
1682 | } |
1683 | |
1684 | if (!xfrm_policy_find_inexact_candidates(cand: &cand, b: bin, |
1685 | saddr: &sel->saddr, |
1686 | daddr: &sel->daddr)) { |
1687 | spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock); |
1688 | return NULL; |
1689 | } |
1690 | |
1691 | pol = NULL; |
1692 | for (i = 0; i < ARRAY_SIZE(cand.res); i++) { |
1693 | struct xfrm_policy *tmp; |
1694 | |
1695 | tmp = __xfrm_policy_bysel_ctx(chain: cand.res[i], mark, |
1696 | if_id, type, dir, |
1697 | sel, ctx); |
1698 | if (!tmp) |
1699 | continue; |
1700 | |
1701 | if (!pol || tmp->pos < pol->pos) |
1702 | pol = tmp; |
1703 | } |
1704 | } else { |
1705 | pol = __xfrm_policy_bysel_ctx(chain, mark, if_id, type, dir, |
1706 | sel, ctx); |
1707 | } |
1708 | |
1709 | if (pol) { |
1710 | xfrm_pol_hold(policy: pol); |
1711 | if (delete) { |
1712 | *err = security_xfrm_policy_delete(ctx: pol->security); |
1713 | if (*err) { |
1714 | spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock); |
1715 | return pol; |
1716 | } |
1717 | __xfrm_policy_unlink(pol, dir); |
1718 | } |
1719 | ret = pol; |
1720 | } |
1721 | spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock); |
1722 | |
1723 | if (ret && delete) |
1724 | xfrm_policy_kill(policy: ret); |
1725 | if (bin && delete) |
1726 | xfrm_policy_inexact_prune_bin(b: bin); |
1727 | return ret; |
1728 | } |
1729 | EXPORT_SYMBOL(xfrm_policy_bysel_ctx); |
1730 | |
1731 | struct xfrm_policy * |
1732 | xfrm_policy_byid(struct net *net, const struct xfrm_mark *mark, u32 if_id, |
1733 | u8 type, int dir, u32 id, int delete, int *err) |
1734 | { |
1735 | struct xfrm_policy *pol, *ret; |
1736 | struct hlist_head *chain; |
1737 | |
1738 | *err = -ENOENT; |
1739 | if (xfrm_policy_id2dir(index: id) != dir) |
1740 | return NULL; |
1741 | |
1742 | *err = 0; |
1743 | spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock); |
1744 | chain = net->xfrm.policy_byidx + idx_hash(net, index: id); |
1745 | ret = NULL; |
1746 | hlist_for_each_entry(pol, chain, byidx) { |
1747 | if (pol->type == type && pol->index == id && |
1748 | pol->if_id == if_id && xfrm_policy_mark_match(mark, pol)) { |
1749 | xfrm_pol_hold(policy: pol); |
1750 | if (delete) { |
1751 | *err = security_xfrm_policy_delete( |
1752 | ctx: pol->security); |
1753 | if (*err) { |
1754 | spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock); |
1755 | return pol; |
1756 | } |
1757 | __xfrm_policy_unlink(pol, dir); |
1758 | } |
1759 | ret = pol; |
1760 | break; |
1761 | } |
1762 | } |
1763 | spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock); |
1764 | |
1765 | if (ret && delete) |
1766 | xfrm_policy_kill(policy: ret); |
1767 | return ret; |
1768 | } |
1769 | EXPORT_SYMBOL(xfrm_policy_byid); |
1770 | |
1771 | #ifdef CONFIG_SECURITY_NETWORK_XFRM |
1772 | static inline int |
1773 | xfrm_policy_flush_secctx_check(struct net *net, u8 type, bool task_valid) |
1774 | { |
1775 | struct xfrm_policy *pol; |
1776 | int err = 0; |
1777 | |
1778 | list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) { |
1779 | if (pol->walk.dead || |
1780 | xfrm_policy_id2dir(index: pol->index) >= XFRM_POLICY_MAX || |
1781 | pol->type != type) |
1782 | continue; |
1783 | |
1784 | err = security_xfrm_policy_delete(ctx: pol->security); |
1785 | if (err) { |
1786 | xfrm_audit_policy_delete(xp: pol, result: 0, task_valid); |
1787 | return err; |
1788 | } |
1789 | } |
1790 | return err; |
1791 | } |
1792 | |
1793 | static inline int xfrm_dev_policy_flush_secctx_check(struct net *net, |
1794 | struct net_device *dev, |
1795 | bool task_valid) |
1796 | { |
1797 | struct xfrm_policy *pol; |
1798 | int err = 0; |
1799 | |
1800 | list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) { |
1801 | if (pol->walk.dead || |
1802 | xfrm_policy_id2dir(index: pol->index) >= XFRM_POLICY_MAX || |
1803 | pol->xdo.dev != dev) |
1804 | continue; |
1805 | |
1806 | err = security_xfrm_policy_delete(ctx: pol->security); |
1807 | if (err) { |
1808 | xfrm_audit_policy_delete(xp: pol, result: 0, task_valid); |
1809 | return err; |
1810 | } |
1811 | } |
1812 | return err; |
1813 | } |
1814 | #else |
1815 | static inline int |
1816 | xfrm_policy_flush_secctx_check(struct net *net, u8 type, bool task_valid) |
1817 | { |
1818 | return 0; |
1819 | } |
1820 | |
1821 | static inline int xfrm_dev_policy_flush_secctx_check(struct net *net, |
1822 | struct net_device *dev, |
1823 | bool task_valid) |
1824 | { |
1825 | return 0; |
1826 | } |
1827 | #endif |
1828 | |
1829 | int xfrm_policy_flush(struct net *net, u8 type, bool task_valid) |
1830 | { |
1831 | int dir, err = 0, cnt = 0; |
1832 | struct xfrm_policy *pol; |
1833 | |
1834 | spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock); |
1835 | |
1836 | err = xfrm_policy_flush_secctx_check(net, type, task_valid); |
1837 | if (err) |
1838 | goto out; |
1839 | |
1840 | again: |
1841 | list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) { |
1842 | if (pol->walk.dead) |
1843 | continue; |
1844 | |
1845 | dir = xfrm_policy_id2dir(index: pol->index); |
1846 | if (dir >= XFRM_POLICY_MAX || |
1847 | pol->type != type) |
1848 | continue; |
1849 | |
1850 | __xfrm_policy_unlink(pol, dir); |
1851 | spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock); |
1852 | xfrm_dev_policy_delete(x: pol); |
1853 | cnt++; |
1854 | xfrm_audit_policy_delete(xp: pol, result: 1, task_valid); |
1855 | xfrm_policy_kill(policy: pol); |
1856 | spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock); |
1857 | goto again; |
1858 | } |
1859 | if (cnt) |
1860 | __xfrm_policy_inexact_flush(net); |
1861 | else |
1862 | err = -ESRCH; |
1863 | out: |
1864 | spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock); |
1865 | return err; |
1866 | } |
1867 | EXPORT_SYMBOL(xfrm_policy_flush); |
1868 | |
1869 | int xfrm_dev_policy_flush(struct net *net, struct net_device *dev, |
1870 | bool task_valid) |
1871 | { |
1872 | int dir, err = 0, cnt = 0; |
1873 | struct xfrm_policy *pol; |
1874 | |
1875 | spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock); |
1876 | |
1877 | err = xfrm_dev_policy_flush_secctx_check(net, dev, task_valid); |
1878 | if (err) |
1879 | goto out; |
1880 | |
1881 | again: |
1882 | list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) { |
1883 | if (pol->walk.dead) |
1884 | continue; |
1885 | |
1886 | dir = xfrm_policy_id2dir(index: pol->index); |
1887 | if (dir >= XFRM_POLICY_MAX || |
1888 | pol->xdo.dev != dev) |
1889 | continue; |
1890 | |
1891 | __xfrm_policy_unlink(pol, dir); |
1892 | spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock); |
1893 | xfrm_dev_policy_delete(x: pol); |
1894 | cnt++; |
1895 | xfrm_audit_policy_delete(xp: pol, result: 1, task_valid); |
1896 | xfrm_policy_kill(policy: pol); |
1897 | spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock); |
1898 | goto again; |
1899 | } |
1900 | if (cnt) |
1901 | __xfrm_policy_inexact_flush(net); |
1902 | else |
1903 | err = -ESRCH; |
1904 | out: |
1905 | spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock); |
1906 | return err; |
1907 | } |
1908 | EXPORT_SYMBOL(xfrm_dev_policy_flush); |
1909 | |
1910 | int xfrm_policy_walk(struct net *net, struct xfrm_policy_walk *walk, |
1911 | int (*func)(struct xfrm_policy *, int, int, void*), |
1912 | void *data) |
1913 | { |
1914 | struct xfrm_policy *pol; |
1915 | struct xfrm_policy_walk_entry *x; |
1916 | int error = 0; |
1917 | |
1918 | if (walk->type >= XFRM_POLICY_TYPE_MAX && |
1919 | walk->type != XFRM_POLICY_TYPE_ANY) |
1920 | return -EINVAL; |
1921 | |
1922 | if (list_empty(head: &walk->walk.all) && walk->seq != 0) |
1923 | return 0; |
1924 | |
1925 | spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock); |
1926 | if (list_empty(head: &walk->walk.all)) |
1927 | x = list_first_entry(&net->xfrm.policy_all, struct xfrm_policy_walk_entry, all); |
1928 | else |
1929 | x = list_first_entry(&walk->walk.all, |
1930 | struct xfrm_policy_walk_entry, all); |
1931 | |
1932 | list_for_each_entry_from(x, &net->xfrm.policy_all, all) { |
1933 | if (x->dead) |
1934 | continue; |
1935 | pol = container_of(x, struct xfrm_policy, walk); |
1936 | if (walk->type != XFRM_POLICY_TYPE_ANY && |
1937 | walk->type != pol->type) |
1938 | continue; |
1939 | error = func(pol, xfrm_policy_id2dir(index: pol->index), |
1940 | walk->seq, data); |
1941 | if (error) { |
1942 | list_move_tail(list: &walk->walk.all, head: &x->all); |
1943 | goto out; |
1944 | } |
1945 | walk->seq++; |
1946 | } |
1947 | if (walk->seq == 0) { |
1948 | error = -ENOENT; |
1949 | goto out; |
1950 | } |
1951 | list_del_init(entry: &walk->walk.all); |
1952 | out: |
1953 | spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock); |
1954 | return error; |
1955 | } |
1956 | EXPORT_SYMBOL(xfrm_policy_walk); |
1957 | |
1958 | void xfrm_policy_walk_init(struct xfrm_policy_walk *walk, u8 type) |
1959 | { |
1960 | INIT_LIST_HEAD(list: &walk->walk.all); |
1961 | walk->walk.dead = 1; |
1962 | walk->type = type; |
1963 | walk->seq = 0; |
1964 | } |
1965 | EXPORT_SYMBOL(xfrm_policy_walk_init); |
1966 | |
1967 | void xfrm_policy_walk_done(struct xfrm_policy_walk *walk, struct net *net) |
1968 | { |
1969 | if (list_empty(head: &walk->walk.all)) |
1970 | return; |
1971 | |
1972 | spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock); /*FIXME where is net? */ |
1973 | list_del(entry: &walk->walk.all); |
1974 | spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock); |
1975 | } |
1976 | EXPORT_SYMBOL(xfrm_policy_walk_done); |
1977 | |
1978 | /* |
1979 | * Find policy to apply to this flow. |
1980 | * |
1981 | * Returns 0 if policy found, else an -errno. |
1982 | */ |
1983 | static int xfrm_policy_match(const struct xfrm_policy *pol, |
1984 | const struct flowi *fl, |
1985 | u8 type, u16 family, u32 if_id) |
1986 | { |
1987 | const struct xfrm_selector *sel = &pol->selector; |
1988 | int ret = -ESRCH; |
1989 | bool match; |
1990 | |
1991 | if (pol->family != family || |
1992 | pol->if_id != if_id || |
1993 | (fl->flowi_mark & pol->mark.m) != pol->mark.v || |
1994 | pol->type != type) |
1995 | return ret; |
1996 | |
1997 | match = xfrm_selector_match(sel, fl, family); |
1998 | if (match) |
1999 | ret = security_xfrm_policy_lookup(ctx: pol->security, fl_secid: fl->flowi_secid); |
2000 | return ret; |
2001 | } |
2002 | |
2003 | static struct xfrm_pol_inexact_node * |
2004 | xfrm_policy_lookup_inexact_addr(const struct rb_root *r, |
2005 | seqcount_spinlock_t *count, |
2006 | const xfrm_address_t *addr, u16 family) |
2007 | { |
2008 | const struct rb_node *parent; |
2009 | int seq; |
2010 | |
2011 | again: |
2012 | seq = read_seqcount_begin(count); |
2013 | |
2014 | parent = rcu_dereference_raw(r->rb_node); |
2015 | while (parent) { |
2016 | struct xfrm_pol_inexact_node *node; |
2017 | int delta; |
2018 | |
2019 | node = rb_entry(parent, struct xfrm_pol_inexact_node, node); |
2020 | |
2021 | delta = xfrm_policy_addr_delta(a: addr, b: &node->addr, |
2022 | prefixlen: node->prefixlen, family); |
2023 | if (delta < 0) { |
2024 | parent = rcu_dereference_raw(parent->rb_left); |
2025 | continue; |
2026 | } else if (delta > 0) { |
2027 | parent = rcu_dereference_raw(parent->rb_right); |
2028 | continue; |
2029 | } |
2030 | |
2031 | return node; |
2032 | } |
2033 | |
2034 | if (read_seqcount_retry(count, seq)) |
2035 | goto again; |
2036 | |
2037 | return NULL; |
2038 | } |
2039 | |
2040 | static bool |
2041 | xfrm_policy_find_inexact_candidates(struct xfrm_pol_inexact_candidates *cand, |
2042 | struct xfrm_pol_inexact_bin *b, |
2043 | const xfrm_address_t *saddr, |
2044 | const xfrm_address_t *daddr) |
2045 | { |
2046 | struct xfrm_pol_inexact_node *n; |
2047 | u16 family; |
2048 | |
2049 | if (!b) |
2050 | return false; |
2051 | |
2052 | family = b->k.family; |
2053 | memset(cand, 0, sizeof(*cand)); |
2054 | cand->res[XFRM_POL_CAND_ANY] = &b->hhead; |
2055 | |
2056 | n = xfrm_policy_lookup_inexact_addr(r: &b->root_d, count: &b->count, addr: daddr, |
2057 | family); |
2058 | if (n) { |
2059 | cand->res[XFRM_POL_CAND_DADDR] = &n->hhead; |
2060 | n = xfrm_policy_lookup_inexact_addr(r: &n->root, count: &b->count, addr: saddr, |
2061 | family); |
2062 | if (n) |
2063 | cand->res[XFRM_POL_CAND_BOTH] = &n->hhead; |
2064 | } |
2065 | |
2066 | n = xfrm_policy_lookup_inexact_addr(r: &b->root_s, count: &b->count, addr: saddr, |
2067 | family); |
2068 | if (n) |
2069 | cand->res[XFRM_POL_CAND_SADDR] = &n->hhead; |
2070 | |
2071 | return true; |
2072 | } |
2073 | |
2074 | static struct xfrm_pol_inexact_bin * |
2075 | xfrm_policy_inexact_lookup_rcu(struct net *net, u8 type, u16 family, |
2076 | u8 dir, u32 if_id) |
2077 | { |
2078 | struct xfrm_pol_inexact_key k = { |
2079 | .family = family, |
2080 | .type = type, |
2081 | .dir = dir, |
2082 | .if_id = if_id, |
2083 | }; |
2084 | |
2085 | write_pnet(pnet: &k.net, net); |
2086 | |
2087 | return rhashtable_lookup(ht: &xfrm_policy_inexact_table, key: &k, |
2088 | params: xfrm_pol_inexact_params); |
2089 | } |
2090 | |
2091 | static struct xfrm_pol_inexact_bin * |
2092 | xfrm_policy_inexact_lookup(struct net *net, u8 type, u16 family, |
2093 | u8 dir, u32 if_id) |
2094 | { |
2095 | struct xfrm_pol_inexact_bin *bin; |
2096 | |
2097 | lockdep_assert_held(&net->xfrm.xfrm_policy_lock); |
2098 | |
2099 | rcu_read_lock(); |
2100 | bin = xfrm_policy_inexact_lookup_rcu(net, type, family, dir, if_id); |
2101 | rcu_read_unlock(); |
2102 | |
2103 | return bin; |
2104 | } |
2105 | |
2106 | static struct xfrm_policy * |
2107 | __xfrm_policy_eval_candidates(struct hlist_head *chain, |
2108 | struct xfrm_policy *prefer, |
2109 | const struct flowi *fl, |
2110 | u8 type, u16 family, u32 if_id) |
2111 | { |
2112 | u32 priority = prefer ? prefer->priority : ~0u; |
2113 | struct xfrm_policy *pol; |
2114 | |
2115 | if (!chain) |
2116 | return NULL; |
2117 | |
2118 | hlist_for_each_entry_rcu(pol, chain, bydst) { |
2119 | int err; |
2120 | |
2121 | if (pol->priority > priority) |
2122 | break; |
2123 | |
2124 | err = xfrm_policy_match(pol, fl, type, family, if_id); |
2125 | if (err) { |
2126 | if (err != -ESRCH) |
2127 | return ERR_PTR(error: err); |
2128 | |
2129 | continue; |
2130 | } |
2131 | |
2132 | if (prefer) { |
2133 | /* matches. Is it older than *prefer? */ |
2134 | if (pol->priority == priority && |
2135 | prefer->pos < pol->pos) |
2136 | return prefer; |
2137 | } |
2138 | |
2139 | return pol; |
2140 | } |
2141 | |
2142 | return NULL; |
2143 | } |
2144 | |
2145 | static struct xfrm_policy * |
2146 | xfrm_policy_eval_candidates(struct xfrm_pol_inexact_candidates *cand, |
2147 | struct xfrm_policy *prefer, |
2148 | const struct flowi *fl, |
2149 | u8 type, u16 family, u32 if_id) |
2150 | { |
2151 | struct xfrm_policy *tmp; |
2152 | int i; |
2153 | |
2154 | for (i = 0; i < ARRAY_SIZE(cand->res); i++) { |
2155 | tmp = __xfrm_policy_eval_candidates(chain: cand->res[i], |
2156 | prefer, |
2157 | fl, type, family, if_id); |
2158 | if (!tmp) |
2159 | continue; |
2160 | |
2161 | if (IS_ERR(ptr: tmp)) |
2162 | return tmp; |
2163 | prefer = tmp; |
2164 | } |
2165 | |
2166 | return prefer; |
2167 | } |
2168 | |
2169 | static struct xfrm_policy *xfrm_policy_lookup_bytype(struct net *net, u8 type, |
2170 | const struct flowi *fl, |
2171 | u16 family, u8 dir, |
2172 | u32 if_id) |
2173 | { |
2174 | struct xfrm_pol_inexact_candidates cand; |
2175 | const xfrm_address_t *daddr, *saddr; |
2176 | struct xfrm_pol_inexact_bin *bin; |
2177 | struct xfrm_policy *pol, *ret; |
2178 | struct hlist_head *chain; |
2179 | unsigned int sequence; |
2180 | int err; |
2181 | |
2182 | daddr = xfrm_flowi_daddr(fl, family); |
2183 | saddr = xfrm_flowi_saddr(fl, family); |
2184 | if (unlikely(!daddr || !saddr)) |
2185 | return NULL; |
2186 | |
2187 | rcu_read_lock(); |
2188 | retry: |
2189 | do { |
2190 | sequence = read_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation); |
2191 | chain = policy_hash_direct(net, daddr, saddr, family, dir); |
2192 | } while (read_seqcount_retry(&net->xfrm.xfrm_policy_hash_generation, sequence)); |
2193 | |
2194 | ret = NULL; |
2195 | hlist_for_each_entry_rcu(pol, chain, bydst) { |
2196 | err = xfrm_policy_match(pol, fl, type, family, if_id); |
2197 | if (err) { |
2198 | if (err == -ESRCH) |
2199 | continue; |
2200 | else { |
2201 | ret = ERR_PTR(error: err); |
2202 | goto fail; |
2203 | } |
2204 | } else { |
2205 | ret = pol; |
2206 | break; |
2207 | } |
2208 | } |
2209 | if (ret && ret->xdo.type == XFRM_DEV_OFFLOAD_PACKET) |
2210 | goto skip_inexact; |
2211 | |
2212 | bin = xfrm_policy_inexact_lookup_rcu(net, type, family, dir, if_id); |
2213 | if (!bin || !xfrm_policy_find_inexact_candidates(cand: &cand, b: bin, saddr, |
2214 | daddr)) |
2215 | goto skip_inexact; |
2216 | |
2217 | pol = xfrm_policy_eval_candidates(cand: &cand, prefer: ret, fl, type, |
2218 | family, if_id); |
2219 | if (pol) { |
2220 | ret = pol; |
2221 | if (IS_ERR(ptr: pol)) |
2222 | goto fail; |
2223 | } |
2224 | |
2225 | skip_inexact: |
2226 | if (read_seqcount_retry(&net->xfrm.xfrm_policy_hash_generation, sequence)) |
2227 | goto retry; |
2228 | |
2229 | if (ret && !xfrm_pol_hold_rcu(policy: ret)) |
2230 | goto retry; |
2231 | fail: |
2232 | rcu_read_unlock(); |
2233 | |
2234 | return ret; |
2235 | } |
2236 | |
2237 | static struct xfrm_policy *xfrm_policy_lookup(struct net *net, |
2238 | const struct flowi *fl, |
2239 | u16 family, u8 dir, u32 if_id) |
2240 | { |
2241 | #ifdef CONFIG_XFRM_SUB_POLICY |
2242 | struct xfrm_policy *pol; |
2243 | |
2244 | pol = xfrm_policy_lookup_bytype(net, type: XFRM_POLICY_TYPE_SUB, fl, family, |
2245 | dir, if_id); |
2246 | if (pol != NULL) |
2247 | return pol; |
2248 | #endif |
2249 | return xfrm_policy_lookup_bytype(net, type: XFRM_POLICY_TYPE_MAIN, fl, family, |
2250 | dir, if_id); |
2251 | } |
2252 | |
2253 | static struct xfrm_policy *xfrm_sk_policy_lookup(const struct sock *sk, int dir, |
2254 | const struct flowi *fl, |
2255 | u16 family, u32 if_id) |
2256 | { |
2257 | struct xfrm_policy *pol; |
2258 | |
2259 | rcu_read_lock(); |
2260 | again: |
2261 | pol = rcu_dereference(sk->sk_policy[dir]); |
2262 | if (pol != NULL) { |
2263 | bool match; |
2264 | int err = 0; |
2265 | |
2266 | if (pol->family != family) { |
2267 | pol = NULL; |
2268 | goto out; |
2269 | } |
2270 | |
2271 | match = xfrm_selector_match(sel: &pol->selector, fl, family); |
2272 | if (match) { |
2273 | if ((READ_ONCE(sk->sk_mark) & pol->mark.m) != pol->mark.v || |
2274 | pol->if_id != if_id) { |
2275 | pol = NULL; |
2276 | goto out; |
2277 | } |
2278 | err = security_xfrm_policy_lookup(ctx: pol->security, |
2279 | fl_secid: fl->flowi_secid); |
2280 | if (!err) { |
2281 | if (!xfrm_pol_hold_rcu(policy: pol)) |
2282 | goto again; |
2283 | } else if (err == -ESRCH) { |
2284 | pol = NULL; |
2285 | } else { |
2286 | pol = ERR_PTR(error: err); |
2287 | } |
2288 | } else |
2289 | pol = NULL; |
2290 | } |
2291 | out: |
2292 | rcu_read_unlock(); |
2293 | return pol; |
2294 | } |
2295 | |
2296 | static void __xfrm_policy_link(struct xfrm_policy *pol, int dir) |
2297 | { |
2298 | struct net *net = xp_net(xp: pol); |
2299 | |
2300 | list_add(new: &pol->walk.all, head: &net->xfrm.policy_all); |
2301 | net->xfrm.policy_count[dir]++; |
2302 | xfrm_pol_hold(policy: pol); |
2303 | } |
2304 | |
2305 | static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol, |
2306 | int dir) |
2307 | { |
2308 | struct net *net = xp_net(xp: pol); |
2309 | |
2310 | if (list_empty(head: &pol->walk.all)) |
2311 | return NULL; |
2312 | |
2313 | /* Socket policies are not hashed. */ |
2314 | if (!hlist_unhashed(h: &pol->bydst)) { |
2315 | hlist_del_rcu(n: &pol->bydst); |
2316 | hlist_del_init(n: &pol->bydst_inexact_list); |
2317 | hlist_del(n: &pol->byidx); |
2318 | } |
2319 | |
2320 | list_del_init(entry: &pol->walk.all); |
2321 | net->xfrm.policy_count[dir]--; |
2322 | |
2323 | return pol; |
2324 | } |
2325 | |
2326 | static void xfrm_sk_policy_link(struct xfrm_policy *pol, int dir) |
2327 | { |
2328 | __xfrm_policy_link(pol, dir: XFRM_POLICY_MAX + dir); |
2329 | } |
2330 | |
2331 | static void xfrm_sk_policy_unlink(struct xfrm_policy *pol, int dir) |
2332 | { |
2333 | __xfrm_policy_unlink(pol, dir: XFRM_POLICY_MAX + dir); |
2334 | } |
2335 | |
2336 | int xfrm_policy_delete(struct xfrm_policy *pol, int dir) |
2337 | { |
2338 | struct net *net = xp_net(xp: pol); |
2339 | |
2340 | spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock); |
2341 | pol = __xfrm_policy_unlink(pol, dir); |
2342 | spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock); |
2343 | if (pol) { |
2344 | xfrm_dev_policy_delete(x: pol); |
2345 | xfrm_policy_kill(policy: pol); |
2346 | return 0; |
2347 | } |
2348 | return -ENOENT; |
2349 | } |
2350 | EXPORT_SYMBOL(xfrm_policy_delete); |
2351 | |
2352 | int xfrm_sk_policy_insert(struct sock *sk, int dir, struct xfrm_policy *pol) |
2353 | { |
2354 | struct net *net = sock_net(sk); |
2355 | struct xfrm_policy *old_pol; |
2356 | |
2357 | #ifdef CONFIG_XFRM_SUB_POLICY |
2358 | if (pol && pol->type != XFRM_POLICY_TYPE_MAIN) |
2359 | return -EINVAL; |
2360 | #endif |
2361 | |
2362 | spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock); |
2363 | old_pol = rcu_dereference_protected(sk->sk_policy[dir], |
2364 | lockdep_is_held(&net->xfrm.xfrm_policy_lock)); |
2365 | if (pol) { |
2366 | pol->curlft.add_time = ktime_get_real_seconds(); |
2367 | pol->index = xfrm_gen_index(net, dir: XFRM_POLICY_MAX+dir, index: 0); |
2368 | xfrm_sk_policy_link(pol, dir); |
2369 | } |
2370 | rcu_assign_pointer(sk->sk_policy[dir], pol); |
2371 | if (old_pol) { |
2372 | if (pol) |
2373 | xfrm_policy_requeue(old: old_pol, new: pol); |
2374 | |
2375 | /* Unlinking succeeds always. This is the only function |
2376 | * allowed to delete or replace socket policy. |
2377 | */ |
2378 | xfrm_sk_policy_unlink(pol: old_pol, dir); |
2379 | } |
2380 | spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock); |
2381 | |
2382 | if (old_pol) { |
2383 | xfrm_policy_kill(policy: old_pol); |
2384 | } |
2385 | return 0; |
2386 | } |
2387 | |
2388 | static struct xfrm_policy *clone_policy(const struct xfrm_policy *old, int dir) |
2389 | { |
2390 | struct xfrm_policy *newp = xfrm_policy_alloc(xp_net(xp: old), GFP_ATOMIC); |
2391 | struct net *net = xp_net(xp: old); |
2392 | |
2393 | if (newp) { |
2394 | newp->selector = old->selector; |
2395 | if (security_xfrm_policy_clone(old_ctx: old->security, |
2396 | new_ctxp: &newp->security)) { |
2397 | kfree(objp: newp); |
2398 | return NULL; /* ENOMEM */ |
2399 | } |
2400 | newp->lft = old->lft; |
2401 | newp->curlft = old->curlft; |
2402 | newp->mark = old->mark; |
2403 | newp->if_id = old->if_id; |
2404 | newp->action = old->action; |
2405 | newp->flags = old->flags; |
2406 | newp->xfrm_nr = old->xfrm_nr; |
2407 | newp->index = old->index; |
2408 | newp->type = old->type; |
2409 | newp->family = old->family; |
2410 | memcpy(newp->xfrm_vec, old->xfrm_vec, |
2411 | newp->xfrm_nr*sizeof(struct xfrm_tmpl)); |
2412 | spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock); |
2413 | xfrm_sk_policy_link(pol: newp, dir); |
2414 | spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock); |
2415 | xfrm_pol_put(policy: newp); |
2416 | } |
2417 | return newp; |
2418 | } |
2419 | |
2420 | int __xfrm_sk_clone_policy(struct sock *sk, const struct sock *osk) |
2421 | { |
2422 | const struct xfrm_policy *p; |
2423 | struct xfrm_policy *np; |
2424 | int i, ret = 0; |
2425 | |
2426 | rcu_read_lock(); |
2427 | for (i = 0; i < 2; i++) { |
2428 | p = rcu_dereference(osk->sk_policy[i]); |
2429 | if (p) { |
2430 | np = clone_policy(old: p, dir: i); |
2431 | if (unlikely(!np)) { |
2432 | ret = -ENOMEM; |
2433 | break; |
2434 | } |
2435 | rcu_assign_pointer(sk->sk_policy[i], np); |
2436 | } |
2437 | } |
2438 | rcu_read_unlock(); |
2439 | return ret; |
2440 | } |
2441 | |
2442 | static int |
2443 | xfrm_get_saddr(struct net *net, int oif, xfrm_address_t *local, |
2444 | xfrm_address_t *remote, unsigned short family, u32 mark) |
2445 | { |
2446 | int err; |
2447 | const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family); |
2448 | |
2449 | if (unlikely(afinfo == NULL)) |
2450 | return -EINVAL; |
2451 | err = afinfo->get_saddr(net, oif, local, remote, mark); |
2452 | rcu_read_unlock(); |
2453 | return err; |
2454 | } |
2455 | |
2456 | /* Resolve list of templates for the flow, given policy. */ |
2457 | |
2458 | static int |
2459 | xfrm_tmpl_resolve_one(struct xfrm_policy *policy, const struct flowi *fl, |
2460 | struct xfrm_state **xfrm, unsigned short family) |
2461 | { |
2462 | struct net *net = xp_net(xp: policy); |
2463 | int nx; |
2464 | int i, error; |
2465 | xfrm_address_t *daddr = xfrm_flowi_daddr(fl, family); |
2466 | xfrm_address_t *saddr = xfrm_flowi_saddr(fl, family); |
2467 | xfrm_address_t tmp; |
2468 | |
2469 | for (nx = 0, i = 0; i < policy->xfrm_nr; i++) { |
2470 | struct xfrm_state *x; |
2471 | xfrm_address_t *remote = daddr; |
2472 | xfrm_address_t *local = saddr; |
2473 | struct xfrm_tmpl *tmpl = &policy->xfrm_vec[i]; |
2474 | |
2475 | if (tmpl->mode == XFRM_MODE_TUNNEL || |
2476 | tmpl->mode == XFRM_MODE_BEET) { |
2477 | remote = &tmpl->id.daddr; |
2478 | local = &tmpl->saddr; |
2479 | if (xfrm_addr_any(addr: local, family: tmpl->encap_family)) { |
2480 | error = xfrm_get_saddr(net, oif: fl->flowi_oif, |
2481 | local: &tmp, remote, |
2482 | family: tmpl->encap_family, mark: 0); |
2483 | if (error) |
2484 | goto fail; |
2485 | local = &tmp; |
2486 | } |
2487 | } |
2488 | |
2489 | x = xfrm_state_find(daddr: remote, saddr: local, fl, tmpl, pol: policy, err: &error, |
2490 | family, if_id: policy->if_id); |
2491 | |
2492 | if (x && x->km.state == XFRM_STATE_VALID) { |
2493 | xfrm[nx++] = x; |
2494 | daddr = remote; |
2495 | saddr = local; |
2496 | continue; |
2497 | } |
2498 | if (x) { |
2499 | error = (x->km.state == XFRM_STATE_ERROR ? |
2500 | -EINVAL : -EAGAIN); |
2501 | xfrm_state_put(x); |
2502 | } else if (error == -ESRCH) { |
2503 | error = -EAGAIN; |
2504 | } |
2505 | |
2506 | if (!tmpl->optional) |
2507 | goto fail; |
2508 | } |
2509 | return nx; |
2510 | |
2511 | fail: |
2512 | for (nx--; nx >= 0; nx--) |
2513 | xfrm_state_put(x: xfrm[nx]); |
2514 | return error; |
2515 | } |
2516 | |
2517 | static int |
2518 | xfrm_tmpl_resolve(struct xfrm_policy **pols, int npols, const struct flowi *fl, |
2519 | struct xfrm_state **xfrm, unsigned short family) |
2520 | { |
2521 | struct xfrm_state *tp[XFRM_MAX_DEPTH]; |
2522 | struct xfrm_state **tpp = (npols > 1) ? tp : xfrm; |
2523 | int cnx = 0; |
2524 | int error; |
2525 | int ret; |
2526 | int i; |
2527 | |
2528 | for (i = 0; i < npols; i++) { |
2529 | if (cnx + pols[i]->xfrm_nr >= XFRM_MAX_DEPTH) { |
2530 | error = -ENOBUFS; |
2531 | goto fail; |
2532 | } |
2533 | |
2534 | ret = xfrm_tmpl_resolve_one(policy: pols[i], fl, xfrm: &tpp[cnx], family); |
2535 | if (ret < 0) { |
2536 | error = ret; |
2537 | goto fail; |
2538 | } else |
2539 | cnx += ret; |
2540 | } |
2541 | |
2542 | /* found states are sorted for outbound processing */ |
2543 | if (npols > 1) |
2544 | xfrm_state_sort(dst: xfrm, src: tpp, n: cnx, family); |
2545 | |
2546 | return cnx; |
2547 | |
2548 | fail: |
2549 | for (cnx--; cnx >= 0; cnx--) |
2550 | xfrm_state_put(x: tpp[cnx]); |
2551 | return error; |
2552 | |
2553 | } |
2554 | |
2555 | static int xfrm_get_tos(const struct flowi *fl, int family) |
2556 | { |
2557 | if (family == AF_INET) |
2558 | return IPTOS_RT_MASK & fl->u.ip4.flowi4_tos; |
2559 | |
2560 | return 0; |
2561 | } |
2562 | |
2563 | static inline struct xfrm_dst *xfrm_alloc_dst(struct net *net, int family) |
2564 | { |
2565 | const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family); |
2566 | struct dst_ops *dst_ops; |
2567 | struct xfrm_dst *xdst; |
2568 | |
2569 | if (!afinfo) |
2570 | return ERR_PTR(error: -EINVAL); |
2571 | |
2572 | switch (family) { |
2573 | case AF_INET: |
2574 | dst_ops = &net->xfrm.xfrm4_dst_ops; |
2575 | break; |
2576 | #if IS_ENABLED(CONFIG_IPV6) |
2577 | case AF_INET6: |
2578 | dst_ops = &net->xfrm.xfrm6_dst_ops; |
2579 | break; |
2580 | #endif |
2581 | default: |
2582 | BUG(); |
2583 | } |
2584 | xdst = dst_alloc(ops: dst_ops, NULL, DST_OBSOLETE_NONE, flags: 0); |
2585 | |
2586 | if (likely(xdst)) { |
2587 | memset_after(xdst, 0, u.dst); |
2588 | } else |
2589 | xdst = ERR_PTR(error: -ENOBUFS); |
2590 | |
2591 | rcu_read_unlock(); |
2592 | |
2593 | return xdst; |
2594 | } |
2595 | |
2596 | static void xfrm_init_path(struct xfrm_dst *path, struct dst_entry *dst, |
2597 | int ) |
2598 | { |
2599 | if (dst->ops->family == AF_INET6) { |
2600 | struct rt6_info *rt = (struct rt6_info *)dst; |
2601 | path->path_cookie = rt6_get_cookie(rt); |
2602 | path->u.rt6.rt6i_nfheader_len = nfheader_len; |
2603 | } |
2604 | } |
2605 | |
2606 | static inline int xfrm_fill_dst(struct xfrm_dst *xdst, struct net_device *dev, |
2607 | const struct flowi *fl) |
2608 | { |
2609 | const struct xfrm_policy_afinfo *afinfo = |
2610 | xfrm_policy_get_afinfo(family: xdst->u.dst.ops->family); |
2611 | int err; |
2612 | |
2613 | if (!afinfo) |
2614 | return -EINVAL; |
2615 | |
2616 | err = afinfo->fill_dst(xdst, dev, fl); |
2617 | |
2618 | rcu_read_unlock(); |
2619 | |
2620 | return err; |
2621 | } |
2622 | |
2623 | |
2624 | /* Allocate chain of dst_entry's, attach known xfrm's, calculate |
2625 | * all the metrics... Shortly, bundle a bundle. |
2626 | */ |
2627 | |
2628 | static struct dst_entry *xfrm_bundle_create(struct xfrm_policy *policy, |
2629 | struct xfrm_state **xfrm, |
2630 | struct xfrm_dst **bundle, |
2631 | int nx, |
2632 | const struct flowi *fl, |
2633 | struct dst_entry *dst) |
2634 | { |
2635 | const struct xfrm_state_afinfo *afinfo; |
2636 | const struct xfrm_mode *inner_mode; |
2637 | struct net *net = xp_net(xp: policy); |
2638 | unsigned long now = jiffies; |
2639 | struct net_device *dev; |
2640 | struct xfrm_dst *xdst_prev = NULL; |
2641 | struct xfrm_dst *xdst0 = NULL; |
2642 | int i = 0; |
2643 | int err; |
2644 | int = 0; |
2645 | int = 0; |
2646 | int trailer_len = 0; |
2647 | int tos; |
2648 | int family = policy->selector.family; |
2649 | xfrm_address_t saddr, daddr; |
2650 | |
2651 | xfrm_flowi_addr_get(fl, saddr: &saddr, daddr: &daddr, family); |
2652 | |
2653 | tos = xfrm_get_tos(fl, family); |
2654 | |
2655 | dst_hold(dst); |
2656 | |
2657 | for (; i < nx; i++) { |
2658 | struct xfrm_dst *xdst = xfrm_alloc_dst(net, family); |
2659 | struct dst_entry *dst1 = &xdst->u.dst; |
2660 | |
2661 | err = PTR_ERR(ptr: xdst); |
2662 | if (IS_ERR(ptr: xdst)) { |
2663 | dst_release(dst); |
2664 | goto put_states; |
2665 | } |
2666 | |
2667 | bundle[i] = xdst; |
2668 | if (!xdst_prev) |
2669 | xdst0 = xdst; |
2670 | else |
2671 | /* Ref count is taken during xfrm_alloc_dst() |
2672 | * No need to do dst_clone() on dst1 |
2673 | */ |
2674 | xfrm_dst_set_child(xdst: xdst_prev, child: &xdst->u.dst); |
2675 | |
2676 | if (xfrm[i]->sel.family == AF_UNSPEC) { |
2677 | inner_mode = xfrm_ip2inner_mode(x: xfrm[i], |
2678 | ipproto: xfrm_af2proto(family)); |
2679 | if (!inner_mode) { |
2680 | err = -EAFNOSUPPORT; |
2681 | dst_release(dst); |
2682 | goto put_states; |
2683 | } |
2684 | } else |
2685 | inner_mode = &xfrm[i]->inner_mode; |
2686 | |
2687 | xdst->route = dst; |
2688 | dst_copy_metrics(dest: dst1, src: dst); |
2689 | |
2690 | if (xfrm[i]->props.mode != XFRM_MODE_TRANSPORT) { |
2691 | __u32 mark = 0; |
2692 | int oif; |
2693 | |
2694 | if (xfrm[i]->props.smark.v || xfrm[i]->props.smark.m) |
2695 | mark = xfrm_smark_get(mark: fl->flowi_mark, x: xfrm[i]); |
2696 | |
2697 | family = xfrm[i]->props.family; |
2698 | oif = fl->flowi_oif ? : fl->flowi_l3mdev; |
2699 | dst = xfrm_dst_lookup(x: xfrm[i], tos, oif, |
2700 | prev_saddr: &saddr, prev_daddr: &daddr, family, mark); |
2701 | err = PTR_ERR(ptr: dst); |
2702 | if (IS_ERR(ptr: dst)) |
2703 | goto put_states; |
2704 | } else |
2705 | dst_hold(dst); |
2706 | |
2707 | dst1->xfrm = xfrm[i]; |
2708 | xdst->xfrm_genid = xfrm[i]->genid; |
2709 | |
2710 | dst1->obsolete = DST_OBSOLETE_FORCE_CHK; |
2711 | dst1->lastuse = now; |
2712 | |
2713 | dst1->input = dst_discard; |
2714 | |
2715 | rcu_read_lock(); |
2716 | afinfo = xfrm_state_afinfo_get_rcu(family: inner_mode->family); |
2717 | if (likely(afinfo)) |
2718 | dst1->output = afinfo->output; |
2719 | else |
2720 | dst1->output = dst_discard_out; |
2721 | rcu_read_unlock(); |
2722 | |
2723 | xdst_prev = xdst; |
2724 | |
2725 | header_len += xfrm[i]->props.header_len; |
2726 | if (xfrm[i]->type->flags & XFRM_TYPE_NON_FRAGMENT) |
2727 | nfheader_len += xfrm[i]->props.header_len; |
2728 | trailer_len += xfrm[i]->props.trailer_len; |
2729 | } |
2730 | |
2731 | xfrm_dst_set_child(xdst: xdst_prev, child: dst); |
2732 | xdst0->path = dst; |
2733 | |
2734 | err = -ENODEV; |
2735 | dev = dst->dev; |
2736 | if (!dev) |
2737 | goto free_dst; |
2738 | |
2739 | xfrm_init_path(path: xdst0, dst, nfheader_len); |
2740 | xfrm_init_pmtu(bundle, nr: nx); |
2741 | |
2742 | for (xdst_prev = xdst0; xdst_prev != (struct xfrm_dst *)dst; |
2743 | xdst_prev = (struct xfrm_dst *) xfrm_dst_child(dst: &xdst_prev->u.dst)) { |
2744 | err = xfrm_fill_dst(xdst: xdst_prev, dev, fl); |
2745 | if (err) |
2746 | goto free_dst; |
2747 | |
2748 | xdst_prev->u.dst.header_len = header_len; |
2749 | xdst_prev->u.dst.trailer_len = trailer_len; |
2750 | header_len -= xdst_prev->u.dst.xfrm->props.header_len; |
2751 | trailer_len -= xdst_prev->u.dst.xfrm->props.trailer_len; |
2752 | } |
2753 | |
2754 | return &xdst0->u.dst; |
2755 | |
2756 | put_states: |
2757 | for (; i < nx; i++) |
2758 | xfrm_state_put(x: xfrm[i]); |
2759 | free_dst: |
2760 | if (xdst0) |
2761 | dst_release_immediate(dst: &xdst0->u.dst); |
2762 | |
2763 | return ERR_PTR(error: err); |
2764 | } |
2765 | |
2766 | static int xfrm_expand_policies(const struct flowi *fl, u16 family, |
2767 | struct xfrm_policy **pols, |
2768 | int *num_pols, int *num_xfrms) |
2769 | { |
2770 | int i; |
2771 | |
2772 | if (*num_pols == 0 || !pols[0]) { |
2773 | *num_pols = 0; |
2774 | *num_xfrms = 0; |
2775 | return 0; |
2776 | } |
2777 | if (IS_ERR(ptr: pols[0])) { |
2778 | *num_pols = 0; |
2779 | return PTR_ERR(ptr: pols[0]); |
2780 | } |
2781 | |
2782 | *num_xfrms = pols[0]->xfrm_nr; |
2783 | |
2784 | #ifdef CONFIG_XFRM_SUB_POLICY |
2785 | if (pols[0]->action == XFRM_POLICY_ALLOW && |
2786 | pols[0]->type != XFRM_POLICY_TYPE_MAIN) { |
2787 | pols[1] = xfrm_policy_lookup_bytype(net: xp_net(xp: pols[0]), |
2788 | type: XFRM_POLICY_TYPE_MAIN, |
2789 | fl, family, |
2790 | dir: XFRM_POLICY_OUT, |
2791 | if_id: pols[0]->if_id); |
2792 | if (pols[1]) { |
2793 | if (IS_ERR(ptr: pols[1])) { |
2794 | xfrm_pols_put(pols, npols: *num_pols); |
2795 | *num_pols = 0; |
2796 | return PTR_ERR(ptr: pols[1]); |
2797 | } |
2798 | (*num_pols)++; |
2799 | (*num_xfrms) += pols[1]->xfrm_nr; |
2800 | } |
2801 | } |
2802 | #endif |
2803 | for (i = 0; i < *num_pols; i++) { |
2804 | if (pols[i]->action != XFRM_POLICY_ALLOW) { |
2805 | *num_xfrms = -1; |
2806 | break; |
2807 | } |
2808 | } |
2809 | |
2810 | return 0; |
2811 | |
2812 | } |
2813 | |
2814 | static struct xfrm_dst * |
2815 | xfrm_resolve_and_create_bundle(struct xfrm_policy **pols, int num_pols, |
2816 | const struct flowi *fl, u16 family, |
2817 | struct dst_entry *dst_orig) |
2818 | { |
2819 | struct net *net = xp_net(xp: pols[0]); |
2820 | struct xfrm_state *xfrm[XFRM_MAX_DEPTH]; |
2821 | struct xfrm_dst *bundle[XFRM_MAX_DEPTH]; |
2822 | struct xfrm_dst *xdst; |
2823 | struct dst_entry *dst; |
2824 | int err; |
2825 | |
2826 | /* Try to instantiate a bundle */ |
2827 | err = xfrm_tmpl_resolve(pols, npols: num_pols, fl, xfrm, family); |
2828 | if (err <= 0) { |
2829 | if (err == 0) |
2830 | return NULL; |
2831 | |
2832 | if (err != -EAGAIN) |
2833 | XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLERROR); |
2834 | return ERR_PTR(error: err); |
2835 | } |
2836 | |
2837 | dst = xfrm_bundle_create(policy: pols[0], xfrm, bundle, nx: err, fl, dst: dst_orig); |
2838 | if (IS_ERR(ptr: dst)) { |
2839 | XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTBUNDLEGENERROR); |
2840 | return ERR_CAST(ptr: dst); |
2841 | } |
2842 | |
2843 | xdst = (struct xfrm_dst *)dst; |
2844 | xdst->num_xfrms = err; |
2845 | xdst->num_pols = num_pols; |
2846 | memcpy(xdst->pols, pols, sizeof(struct xfrm_policy *) * num_pols); |
2847 | xdst->policy_genid = atomic_read(v: &pols[0]->genid); |
2848 | |
2849 | return xdst; |
2850 | } |
2851 | |
2852 | static void xfrm_policy_queue_process(struct timer_list *t) |
2853 | { |
2854 | struct sk_buff *skb; |
2855 | struct sock *sk; |
2856 | struct dst_entry *dst; |
2857 | struct xfrm_policy *pol = from_timer(pol, t, polq.hold_timer); |
2858 | struct net *net = xp_net(xp: pol); |
2859 | struct xfrm_policy_queue *pq = &pol->polq; |
2860 | struct flowi fl; |
2861 | struct sk_buff_head list; |
2862 | __u32 skb_mark; |
2863 | |
2864 | spin_lock(lock: &pq->hold_queue.lock); |
2865 | skb = skb_peek(list_: &pq->hold_queue); |
2866 | if (!skb) { |
2867 | spin_unlock(lock: &pq->hold_queue.lock); |
2868 | goto out; |
2869 | } |
2870 | dst = skb_dst(skb); |
2871 | sk = skb->sk; |
2872 | |
2873 | /* Fixup the mark to support VTI. */ |
2874 | skb_mark = skb->mark; |
2875 | skb->mark = pol->mark.v; |
2876 | xfrm_decode_session(net, skb, fl: &fl, family: dst->ops->family); |
2877 | skb->mark = skb_mark; |
2878 | spin_unlock(lock: &pq->hold_queue.lock); |
2879 | |
2880 | dst_hold(dst: xfrm_dst_path(dst)); |
2881 | dst = xfrm_lookup(net, dst_orig: xfrm_dst_path(dst), fl: &fl, sk, flags: XFRM_LOOKUP_QUEUE); |
2882 | if (IS_ERR(ptr: dst)) |
2883 | goto purge_queue; |
2884 | |
2885 | if (dst->flags & DST_XFRM_QUEUE) { |
2886 | dst_release(dst); |
2887 | |
2888 | if (pq->timeout >= XFRM_QUEUE_TMO_MAX) |
2889 | goto purge_queue; |
2890 | |
2891 | pq->timeout = pq->timeout << 1; |
2892 | if (!mod_timer(timer: &pq->hold_timer, expires: jiffies + pq->timeout)) |
2893 | xfrm_pol_hold(policy: pol); |
2894 | goto out; |
2895 | } |
2896 | |
2897 | dst_release(dst); |
2898 | |
2899 | __skb_queue_head_init(list: &list); |
2900 | |
2901 | spin_lock(lock: &pq->hold_queue.lock); |
2902 | pq->timeout = 0; |
2903 | skb_queue_splice_init(list: &pq->hold_queue, head: &list); |
2904 | spin_unlock(lock: &pq->hold_queue.lock); |
2905 | |
2906 | while (!skb_queue_empty(list: &list)) { |
2907 | skb = __skb_dequeue(list: &list); |
2908 | |
2909 | /* Fixup the mark to support VTI. */ |
2910 | skb_mark = skb->mark; |
2911 | skb->mark = pol->mark.v; |
2912 | xfrm_decode_session(net, skb, fl: &fl, family: skb_dst(skb)->ops->family); |
2913 | skb->mark = skb_mark; |
2914 | |
2915 | dst_hold(dst: xfrm_dst_path(dst: skb_dst(skb))); |
2916 | dst = xfrm_lookup(net, dst_orig: xfrm_dst_path(dst: skb_dst(skb)), fl: &fl, sk: skb->sk, flags: 0); |
2917 | if (IS_ERR(ptr: dst)) { |
2918 | kfree_skb(skb); |
2919 | continue; |
2920 | } |
2921 | |
2922 | nf_reset_ct(skb); |
2923 | skb_dst_drop(skb); |
2924 | skb_dst_set(skb, dst); |
2925 | |
2926 | dst_output(net, sk: skb->sk, skb); |
2927 | } |
2928 | |
2929 | out: |
2930 | xfrm_pol_put(policy: pol); |
2931 | return; |
2932 | |
2933 | purge_queue: |
2934 | pq->timeout = 0; |
2935 | skb_queue_purge(list: &pq->hold_queue); |
2936 | xfrm_pol_put(policy: pol); |
2937 | } |
2938 | |
2939 | static int xdst_queue_output(struct net *net, struct sock *sk, struct sk_buff *skb) |
2940 | { |
2941 | unsigned long sched_next; |
2942 | struct dst_entry *dst = skb_dst(skb); |
2943 | struct xfrm_dst *xdst = (struct xfrm_dst *) dst; |
2944 | struct xfrm_policy *pol = xdst->pols[0]; |
2945 | struct xfrm_policy_queue *pq = &pol->polq; |
2946 | |
2947 | if (unlikely(skb_fclone_busy(sk, skb))) { |
2948 | kfree_skb(skb); |
2949 | return 0; |
2950 | } |
2951 | |
2952 | if (pq->hold_queue.qlen > XFRM_MAX_QUEUE_LEN) { |
2953 | kfree_skb(skb); |
2954 | return -EAGAIN; |
2955 | } |
2956 | |
2957 | skb_dst_force(skb); |
2958 | |
2959 | spin_lock_bh(lock: &pq->hold_queue.lock); |
2960 | |
2961 | if (!pq->timeout) |
2962 | pq->timeout = XFRM_QUEUE_TMO_MIN; |
2963 | |
2964 | sched_next = jiffies + pq->timeout; |
2965 | |
2966 | if (del_timer(timer: &pq->hold_timer)) { |
2967 | if (time_before(pq->hold_timer.expires, sched_next)) |
2968 | sched_next = pq->hold_timer.expires; |
2969 | xfrm_pol_put(policy: pol); |
2970 | } |
2971 | |
2972 | __skb_queue_tail(list: &pq->hold_queue, newsk: skb); |
2973 | if (!mod_timer(timer: &pq->hold_timer, expires: sched_next)) |
2974 | xfrm_pol_hold(policy: pol); |
2975 | |
2976 | spin_unlock_bh(lock: &pq->hold_queue.lock); |
2977 | |
2978 | return 0; |
2979 | } |
2980 | |
2981 | static struct xfrm_dst *xfrm_create_dummy_bundle(struct net *net, |
2982 | struct xfrm_flo *xflo, |
2983 | const struct flowi *fl, |
2984 | int num_xfrms, |
2985 | u16 family) |
2986 | { |
2987 | int err; |
2988 | struct net_device *dev; |
2989 | struct dst_entry *dst; |
2990 | struct dst_entry *dst1; |
2991 | struct xfrm_dst *xdst; |
2992 | |
2993 | xdst = xfrm_alloc_dst(net, family); |
2994 | if (IS_ERR(ptr: xdst)) |
2995 | return xdst; |
2996 | |
2997 | if (!(xflo->flags & XFRM_LOOKUP_QUEUE) || |
2998 | net->xfrm.sysctl_larval_drop || |
2999 | num_xfrms <= 0) |
3000 | return xdst; |
3001 | |
3002 | dst = xflo->dst_orig; |
3003 | dst1 = &xdst->u.dst; |
3004 | dst_hold(dst); |
3005 | xdst->route = dst; |
3006 | |
3007 | dst_copy_metrics(dest: dst1, src: dst); |
3008 | |
3009 | dst1->obsolete = DST_OBSOLETE_FORCE_CHK; |
3010 | dst1->flags |= DST_XFRM_QUEUE; |
3011 | dst1->lastuse = jiffies; |
3012 | |
3013 | dst1->input = dst_discard; |
3014 | dst1->output = xdst_queue_output; |
3015 | |
3016 | dst_hold(dst); |
3017 | xfrm_dst_set_child(xdst, child: dst); |
3018 | xdst->path = dst; |
3019 | |
3020 | xfrm_init_path(path: (struct xfrm_dst *)dst1, dst, nfheader_len: 0); |
3021 | |
3022 | err = -ENODEV; |
3023 | dev = dst->dev; |
3024 | if (!dev) |
3025 | goto free_dst; |
3026 | |
3027 | err = xfrm_fill_dst(xdst, dev, fl); |
3028 | if (err) |
3029 | goto free_dst; |
3030 | |
3031 | out: |
3032 | return xdst; |
3033 | |
3034 | free_dst: |
3035 | dst_release(dst: dst1); |
3036 | xdst = ERR_PTR(error: err); |
3037 | goto out; |
3038 | } |
3039 | |
3040 | static struct xfrm_dst *xfrm_bundle_lookup(struct net *net, |
3041 | const struct flowi *fl, |
3042 | u16 family, u8 dir, |
3043 | struct xfrm_flo *xflo, u32 if_id) |
3044 | { |
3045 | struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX]; |
3046 | int num_pols = 0, num_xfrms = 0, err; |
3047 | struct xfrm_dst *xdst; |
3048 | |
3049 | /* Resolve policies to use if we couldn't get them from |
3050 | * previous cache entry */ |
3051 | num_pols = 1; |
3052 | pols[0] = xfrm_policy_lookup(net, fl, family, dir, if_id); |
3053 | err = xfrm_expand_policies(fl, family, pols, |
3054 | num_pols: &num_pols, num_xfrms: &num_xfrms); |
3055 | if (err < 0) |
3056 | goto inc_error; |
3057 | if (num_pols == 0) |
3058 | return NULL; |
3059 | if (num_xfrms <= 0) |
3060 | goto make_dummy_bundle; |
3061 | |
3062 | xdst = xfrm_resolve_and_create_bundle(pols, num_pols, fl, family, |
3063 | dst_orig: xflo->dst_orig); |
3064 | if (IS_ERR(ptr: xdst)) { |
3065 | err = PTR_ERR(ptr: xdst); |
3066 | if (err == -EREMOTE) { |
3067 | xfrm_pols_put(pols, npols: num_pols); |
3068 | return NULL; |
3069 | } |
3070 | |
3071 | if (err != -EAGAIN) |
3072 | goto error; |
3073 | goto make_dummy_bundle; |
3074 | } else if (xdst == NULL) { |
3075 | num_xfrms = 0; |
3076 | goto make_dummy_bundle; |
3077 | } |
3078 | |
3079 | return xdst; |
3080 | |
3081 | make_dummy_bundle: |
3082 | /* We found policies, but there's no bundles to instantiate: |
3083 | * either because the policy blocks, has no transformations or |
3084 | * we could not build template (no xfrm_states).*/ |
3085 | xdst = xfrm_create_dummy_bundle(net, xflo, fl, num_xfrms, family); |
3086 | if (IS_ERR(ptr: xdst)) { |
3087 | xfrm_pols_put(pols, npols: num_pols); |
3088 | return ERR_CAST(ptr: xdst); |
3089 | } |
3090 | xdst->num_pols = num_pols; |
3091 | xdst->num_xfrms = num_xfrms; |
3092 | memcpy(xdst->pols, pols, sizeof(struct xfrm_policy *) * num_pols); |
3093 | |
3094 | return xdst; |
3095 | |
3096 | inc_error: |
3097 | XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLERROR); |
3098 | error: |
3099 | xfrm_pols_put(pols, npols: num_pols); |
3100 | return ERR_PTR(error: err); |
3101 | } |
3102 | |
3103 | static struct dst_entry *make_blackhole(struct net *net, u16 family, |
3104 | struct dst_entry *dst_orig) |
3105 | { |
3106 | const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family); |
3107 | struct dst_entry *ret; |
3108 | |
3109 | if (!afinfo) { |
3110 | dst_release(dst: dst_orig); |
3111 | return ERR_PTR(error: -EINVAL); |
3112 | } else { |
3113 | ret = afinfo->blackhole_route(net, dst_orig); |
3114 | } |
3115 | rcu_read_unlock(); |
3116 | |
3117 | return ret; |
3118 | } |
3119 | |
3120 | /* Finds/creates a bundle for given flow and if_id |
3121 | * |
3122 | * At the moment we eat a raw IP route. Mostly to speed up lookups |
3123 | * on interfaces with disabled IPsec. |
3124 | * |
3125 | * xfrm_lookup uses an if_id of 0 by default, and is provided for |
3126 | * compatibility |
3127 | */ |
3128 | struct dst_entry *xfrm_lookup_with_ifid(struct net *net, |
3129 | struct dst_entry *dst_orig, |
3130 | const struct flowi *fl, |
3131 | const struct sock *sk, |
3132 | int flags, u32 if_id) |
3133 | { |
3134 | struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX]; |
3135 | struct xfrm_dst *xdst; |
3136 | struct dst_entry *dst, *route; |
3137 | u16 family = dst_orig->ops->family; |
3138 | u8 dir = XFRM_POLICY_OUT; |
3139 | int i, err, num_pols, num_xfrms = 0, drop_pols = 0; |
3140 | |
3141 | dst = NULL; |
3142 | xdst = NULL; |
3143 | route = NULL; |
3144 | |
3145 | sk = sk_const_to_full_sk(sk); |
3146 | if (sk && sk->sk_policy[XFRM_POLICY_OUT]) { |
3147 | num_pols = 1; |
3148 | pols[0] = xfrm_sk_policy_lookup(sk, dir: XFRM_POLICY_OUT, fl, family, |
3149 | if_id); |
3150 | err = xfrm_expand_policies(fl, family, pols, |
3151 | num_pols: &num_pols, num_xfrms: &num_xfrms); |
3152 | if (err < 0) |
3153 | goto dropdst; |
3154 | |
3155 | if (num_pols) { |
3156 | if (num_xfrms <= 0) { |
3157 | drop_pols = num_pols; |
3158 | goto no_transform; |
3159 | } |
3160 | |
3161 | xdst = xfrm_resolve_and_create_bundle( |
3162 | pols, num_pols, fl, |
3163 | family, dst_orig); |
3164 | |
3165 | if (IS_ERR(ptr: xdst)) { |
3166 | xfrm_pols_put(pols, npols: num_pols); |
3167 | err = PTR_ERR(ptr: xdst); |
3168 | if (err == -EREMOTE) |
3169 | goto nopol; |
3170 | |
3171 | goto dropdst; |
3172 | } else if (xdst == NULL) { |
3173 | num_xfrms = 0; |
3174 | drop_pols = num_pols; |
3175 | goto no_transform; |
3176 | } |
3177 | |
3178 | route = xdst->route; |
3179 | } |
3180 | } |
3181 | |
3182 | if (xdst == NULL) { |
3183 | struct xfrm_flo xflo; |
3184 | |
3185 | xflo.dst_orig = dst_orig; |
3186 | xflo.flags = flags; |
3187 | |
3188 | /* To accelerate a bit... */ |
3189 | if (!if_id && ((dst_orig->flags & DST_NOXFRM) || |
3190 | !net->xfrm.policy_count[XFRM_POLICY_OUT])) |
3191 | goto nopol; |
3192 | |
3193 | xdst = xfrm_bundle_lookup(net, fl, family, dir, xflo: &xflo, if_id); |
3194 | if (xdst == NULL) |
3195 | goto nopol; |
3196 | if (IS_ERR(ptr: xdst)) { |
3197 | err = PTR_ERR(ptr: xdst); |
3198 | goto dropdst; |
3199 | } |
3200 | |
3201 | num_pols = xdst->num_pols; |
3202 | num_xfrms = xdst->num_xfrms; |
3203 | memcpy(pols, xdst->pols, sizeof(struct xfrm_policy *) * num_pols); |
3204 | route = xdst->route; |
3205 | } |
3206 | |
3207 | dst = &xdst->u.dst; |
3208 | if (route == NULL && num_xfrms > 0) { |
3209 | /* The only case when xfrm_bundle_lookup() returns a |
3210 | * bundle with null route, is when the template could |
3211 | * not be resolved. It means policies are there, but |
3212 | * bundle could not be created, since we don't yet |
3213 | * have the xfrm_state's. We need to wait for KM to |
3214 | * negotiate new SA's or bail out with error.*/ |
3215 | if (net->xfrm.sysctl_larval_drop) { |
3216 | XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTNOSTATES); |
3217 | err = -EREMOTE; |
3218 | goto error; |
3219 | } |
3220 | |
3221 | err = -EAGAIN; |
3222 | |
3223 | XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTNOSTATES); |
3224 | goto error; |
3225 | } |
3226 | |
3227 | no_transform: |
3228 | if (num_pols == 0) |
3229 | goto nopol; |
3230 | |
3231 | if ((flags & XFRM_LOOKUP_ICMP) && |
3232 | !(pols[0]->flags & XFRM_POLICY_ICMP)) { |
3233 | err = -ENOENT; |
3234 | goto error; |
3235 | } |
3236 | |
3237 | for (i = 0; i < num_pols; i++) |
3238 | WRITE_ONCE(pols[i]->curlft.use_time, ktime_get_real_seconds()); |
3239 | |
3240 | if (num_xfrms < 0) { |
3241 | /* Prohibit the flow */ |
3242 | XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLBLOCK); |
3243 | err = -EPERM; |
3244 | goto error; |
3245 | } else if (num_xfrms > 0) { |
3246 | /* Flow transformed */ |
3247 | dst_release(dst: dst_orig); |
3248 | } else { |
3249 | /* Flow passes untransformed */ |
3250 | dst_release(dst); |
3251 | dst = dst_orig; |
3252 | } |
3253 | ok: |
3254 | xfrm_pols_put(pols, npols: drop_pols); |
3255 | if (dst && dst->xfrm && |
3256 | dst->xfrm->props.mode == XFRM_MODE_TUNNEL) |
3257 | dst->flags |= DST_XFRM_TUNNEL; |
3258 | return dst; |
3259 | |
3260 | nopol: |
3261 | if ((!dst_orig->dev || !(dst_orig->dev->flags & IFF_LOOPBACK)) && |
3262 | net->xfrm.policy_default[dir] == XFRM_USERPOLICY_BLOCK) { |
3263 | err = -EPERM; |
3264 | goto error; |
3265 | } |
3266 | if (!(flags & XFRM_LOOKUP_ICMP)) { |
3267 | dst = dst_orig; |
3268 | goto ok; |
3269 | } |
3270 | err = -ENOENT; |
3271 | error: |
3272 | dst_release(dst); |
3273 | dropdst: |
3274 | if (!(flags & XFRM_LOOKUP_KEEP_DST_REF)) |
3275 | dst_release(dst: dst_orig); |
3276 | xfrm_pols_put(pols, npols: drop_pols); |
3277 | return ERR_PTR(error: err); |
3278 | } |
3279 | EXPORT_SYMBOL(xfrm_lookup_with_ifid); |
3280 | |
3281 | /* Main function: finds/creates a bundle for given flow. |
3282 | * |
3283 | * At the moment we eat a raw IP route. Mostly to speed up lookups |
3284 | * on interfaces with disabled IPsec. |
3285 | */ |
3286 | struct dst_entry *xfrm_lookup(struct net *net, struct dst_entry *dst_orig, |
3287 | const struct flowi *fl, const struct sock *sk, |
3288 | int flags) |
3289 | { |
3290 | return xfrm_lookup_with_ifid(net, dst_orig, fl, sk, flags, 0); |
3291 | } |
3292 | EXPORT_SYMBOL(xfrm_lookup); |
3293 | |
3294 | /* Callers of xfrm_lookup_route() must ensure a call to dst_output(). |
3295 | * Otherwise we may send out blackholed packets. |
3296 | */ |
3297 | struct dst_entry *xfrm_lookup_route(struct net *net, struct dst_entry *dst_orig, |
3298 | const struct flowi *fl, |
3299 | const struct sock *sk, int flags) |
3300 | { |
3301 | struct dst_entry *dst = xfrm_lookup(net, dst_orig, fl, sk, |
3302 | flags | XFRM_LOOKUP_QUEUE | |
3303 | XFRM_LOOKUP_KEEP_DST_REF); |
3304 | |
3305 | if (PTR_ERR(ptr: dst) == -EREMOTE) |
3306 | return make_blackhole(net, family: dst_orig->ops->family, dst_orig); |
3307 | |
3308 | if (IS_ERR(ptr: dst)) |
3309 | dst_release(dst: dst_orig); |
3310 | |
3311 | return dst; |
3312 | } |
3313 | EXPORT_SYMBOL(xfrm_lookup_route); |
3314 | |
3315 | static inline int |
3316 | xfrm_secpath_reject(int idx, struct sk_buff *skb, const struct flowi *fl) |
3317 | { |
3318 | struct sec_path *sp = skb_sec_path(skb); |
3319 | struct xfrm_state *x; |
3320 | |
3321 | if (!sp || idx < 0 || idx >= sp->len) |
3322 | return 0; |
3323 | x = sp->xvec[idx]; |
3324 | if (!x->type->reject) |
3325 | return 0; |
3326 | return x->type->reject(x, skb, fl); |
3327 | } |
3328 | |
3329 | /* When skb is transformed back to its "native" form, we have to |
3330 | * check policy restrictions. At the moment we make this in maximally |
3331 | * stupid way. Shame on me. :-) Of course, connected sockets must |
3332 | * have policy cached at them. |
3333 | */ |
3334 | |
3335 | static inline int |
3336 | xfrm_state_ok(const struct xfrm_tmpl *tmpl, const struct xfrm_state *x, |
3337 | unsigned short family, u32 if_id) |
3338 | { |
3339 | if (xfrm_state_kern(x)) |
3340 | return tmpl->optional && !xfrm_state_addr_cmp(tmpl, x, family: tmpl->encap_family); |
3341 | return x->id.proto == tmpl->id.proto && |
3342 | (x->id.spi == tmpl->id.spi || !tmpl->id.spi) && |
3343 | (x->props.reqid == tmpl->reqid || !tmpl->reqid) && |
3344 | x->props.mode == tmpl->mode && |
3345 | (tmpl->allalgs || (tmpl->aalgos & (1<<x->props.aalgo)) || |
3346 | !(xfrm_id_proto_match(proto: tmpl->id.proto, IPSEC_PROTO_ANY))) && |
3347 | !(x->props.mode != XFRM_MODE_TRANSPORT && |
3348 | xfrm_state_addr_cmp(tmpl, x, family)) && |
3349 | (if_id == 0 || if_id == x->if_id); |
3350 | } |
3351 | |
3352 | /* |
3353 | * 0 or more than 0 is returned when validation is succeeded (either bypass |
3354 | * because of optional transport mode, or next index of the matched secpath |
3355 | * state with the template. |
3356 | * -1 is returned when no matching template is found. |
3357 | * Otherwise "-2 - errored_index" is returned. |
3358 | */ |
3359 | static inline int |
3360 | xfrm_policy_ok(const struct xfrm_tmpl *tmpl, const struct sec_path *sp, int start, |
3361 | unsigned short family, u32 if_id) |
3362 | { |
3363 | int idx = start; |
3364 | |
3365 | if (tmpl->optional) { |
3366 | if (tmpl->mode == XFRM_MODE_TRANSPORT) |
3367 | return start; |
3368 | } else |
3369 | start = -1; |
3370 | for (; idx < sp->len; idx++) { |
3371 | if (xfrm_state_ok(tmpl, x: sp->xvec[idx], family, if_id)) |
3372 | return ++idx; |
3373 | if (sp->xvec[idx]->props.mode != XFRM_MODE_TRANSPORT) { |
3374 | if (idx < sp->verified_cnt) { |
3375 | /* Secpath entry previously verified, consider optional and |
3376 | * continue searching |
3377 | */ |
3378 | continue; |
3379 | } |
3380 | |
3381 | if (start == -1) |
3382 | start = -2-idx; |
3383 | break; |
3384 | } |
3385 | } |
3386 | return start; |
3387 | } |
3388 | |
3389 | static void |
3390 | decode_session4(const struct xfrm_flow_keys *flkeys, struct flowi *fl, bool reverse) |
3391 | { |
3392 | struct flowi4 *fl4 = &fl->u.ip4; |
3393 | |
3394 | memset(fl4, 0, sizeof(struct flowi4)); |
3395 | |
3396 | if (reverse) { |
3397 | fl4->saddr = flkeys->addrs.ipv4.dst; |
3398 | fl4->daddr = flkeys->addrs.ipv4.src; |
3399 | fl4->fl4_sport = flkeys->ports.dst; |
3400 | fl4->fl4_dport = flkeys->ports.src; |
3401 | } else { |
3402 | fl4->saddr = flkeys->addrs.ipv4.src; |
3403 | fl4->daddr = flkeys->addrs.ipv4.dst; |
3404 | fl4->fl4_sport = flkeys->ports.src; |
3405 | fl4->fl4_dport = flkeys->ports.dst; |
3406 | } |
3407 | |
3408 | switch (flkeys->basic.ip_proto) { |
3409 | case IPPROTO_GRE: |
3410 | fl4->fl4_gre_key = flkeys->gre.keyid; |
3411 | break; |
3412 | case IPPROTO_ICMP: |
3413 | fl4->fl4_icmp_type = flkeys->icmp.type; |
3414 | fl4->fl4_icmp_code = flkeys->icmp.code; |
3415 | break; |
3416 | } |
3417 | |
3418 | fl4->flowi4_proto = flkeys->basic.ip_proto; |
3419 | fl4->flowi4_tos = flkeys->ip.tos; |
3420 | } |
3421 | |
3422 | #if IS_ENABLED(CONFIG_IPV6) |
3423 | static void |
3424 | decode_session6(const struct xfrm_flow_keys *flkeys, struct flowi *fl, bool reverse) |
3425 | { |
3426 | struct flowi6 *fl6 = &fl->u.ip6; |
3427 | |
3428 | memset(fl6, 0, sizeof(struct flowi6)); |
3429 | |
3430 | if (reverse) { |
3431 | fl6->saddr = flkeys->addrs.ipv6.dst; |
3432 | fl6->daddr = flkeys->addrs.ipv6.src; |
3433 | fl6->fl6_sport = flkeys->ports.dst; |
3434 | fl6->fl6_dport = flkeys->ports.src; |
3435 | } else { |
3436 | fl6->saddr = flkeys->addrs.ipv6.src; |
3437 | fl6->daddr = flkeys->addrs.ipv6.dst; |
3438 | fl6->fl6_sport = flkeys->ports.src; |
3439 | fl6->fl6_dport = flkeys->ports.dst; |
3440 | } |
3441 | |
3442 | switch (flkeys->basic.ip_proto) { |
3443 | case IPPROTO_GRE: |
3444 | fl6->fl6_gre_key = flkeys->gre.keyid; |
3445 | break; |
3446 | case IPPROTO_ICMPV6: |
3447 | fl6->fl6_icmp_type = flkeys->icmp.type; |
3448 | fl6->fl6_icmp_code = flkeys->icmp.code; |
3449 | break; |
3450 | } |
3451 | |
3452 | fl6->flowi6_proto = flkeys->basic.ip_proto; |
3453 | } |
3454 | #endif |
3455 | |
3456 | int __xfrm_decode_session(struct net *net, struct sk_buff *skb, struct flowi *fl, |
3457 | unsigned int family, int reverse) |
3458 | { |
3459 | struct xfrm_flow_keys flkeys; |
3460 | |
3461 | memset(&flkeys, 0, sizeof(flkeys)); |
3462 | __skb_flow_dissect(net, skb, flow_dissector: &xfrm_session_dissector, target_container: &flkeys, |
3463 | NULL, proto: 0, nhoff: 0, hlen: 0, FLOW_DISSECTOR_F_STOP_AT_ENCAP); |
3464 | |
3465 | switch (family) { |
3466 | case AF_INET: |
3467 | decode_session4(flkeys: &flkeys, fl, reverse); |
3468 | break; |
3469 | #if IS_ENABLED(CONFIG_IPV6) |
3470 | case AF_INET6: |
3471 | decode_session6(flkeys: &flkeys, fl, reverse); |
3472 | break; |
3473 | #endif |
3474 | default: |
3475 | return -EAFNOSUPPORT; |
3476 | } |
3477 | |
3478 | fl->flowi_mark = skb->mark; |
3479 | if (reverse) { |
3480 | fl->flowi_oif = skb->skb_iif; |
3481 | } else { |
3482 | int oif = 0; |
3483 | |
3484 | if (skb_dst(skb) && skb_dst(skb)->dev) |
3485 | oif = skb_dst(skb)->dev->ifindex; |
3486 | |
3487 | fl->flowi_oif = oif; |
3488 | } |
3489 | |
3490 | return security_xfrm_decode_session(skb, secid: &fl->flowi_secid); |
3491 | } |
3492 | EXPORT_SYMBOL(__xfrm_decode_session); |
3493 | |
3494 | static inline int secpath_has_nontransport(const struct sec_path *sp, int k, int *idxp) |
3495 | { |
3496 | for (; k < sp->len; k++) { |
3497 | if (sp->xvec[k]->props.mode != XFRM_MODE_TRANSPORT) { |
3498 | *idxp = k; |
3499 | return 1; |
3500 | } |
3501 | } |
3502 | |
3503 | return 0; |
3504 | } |
3505 | |
3506 | int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb, |
3507 | unsigned short family) |
3508 | { |
3509 | struct net *net = dev_net(dev: skb->dev); |
3510 | struct xfrm_policy *pol; |
3511 | struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX]; |
3512 | int npols = 0; |
3513 | int xfrm_nr; |
3514 | int pi; |
3515 | int reverse; |
3516 | struct flowi fl; |
3517 | int xerr_idx = -1; |
3518 | const struct xfrm_if_cb *ifcb; |
3519 | struct sec_path *sp; |
3520 | u32 if_id = 0; |
3521 | |
3522 | rcu_read_lock(); |
3523 | ifcb = xfrm_if_get_cb(); |
3524 | |
3525 | if (ifcb) { |
3526 | struct xfrm_if_decode_session_result r; |
3527 | |
3528 | if (ifcb->decode_session(skb, family, &r)) { |
3529 | if_id = r.if_id; |
3530 | net = r.net; |
3531 | } |
3532 | } |
3533 | rcu_read_unlock(); |
3534 | |
3535 | reverse = dir & ~XFRM_POLICY_MASK; |
3536 | dir &= XFRM_POLICY_MASK; |
3537 | |
3538 | if (__xfrm_decode_session(net, skb, &fl, family, reverse) < 0) { |
3539 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR); |
3540 | return 0; |
3541 | } |
3542 | |
3543 | nf_nat_decode_session(skb, fl: &fl, family); |
3544 | |
3545 | /* First, check used SA against their selectors. */ |
3546 | sp = skb_sec_path(skb); |
3547 | if (sp) { |
3548 | int i; |
3549 | |
3550 | for (i = sp->len - 1; i >= 0; i--) { |
3551 | struct xfrm_state *x = sp->xvec[i]; |
3552 | if (!xfrm_selector_match(sel: &x->sel, fl: &fl, family)) { |
3553 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMISMATCH); |
3554 | return 0; |
3555 | } |
3556 | } |
3557 | } |
3558 | |
3559 | pol = NULL; |
3560 | sk = sk_to_full_sk(sk); |
3561 | if (sk && sk->sk_policy[dir]) { |
3562 | pol = xfrm_sk_policy_lookup(sk, dir, fl: &fl, family, if_id); |
3563 | if (IS_ERR(ptr: pol)) { |
3564 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR); |
3565 | return 0; |
3566 | } |
3567 | } |
3568 | |
3569 | if (!pol) |
3570 | pol = xfrm_policy_lookup(net, fl: &fl, family, dir, if_id); |
3571 | |
3572 | if (IS_ERR(ptr: pol)) { |
3573 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR); |
3574 | return 0; |
3575 | } |
3576 | |
3577 | if (!pol) { |
3578 | if (net->xfrm.policy_default[dir] == XFRM_USERPOLICY_BLOCK) { |
3579 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOPOLS); |
3580 | return 0; |
3581 | } |
3582 | |
3583 | if (sp && secpath_has_nontransport(sp, k: 0, idxp: &xerr_idx)) { |
3584 | xfrm_secpath_reject(idx: xerr_idx, skb, fl: &fl); |
3585 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOPOLS); |
3586 | return 0; |
3587 | } |
3588 | return 1; |
3589 | } |
3590 | |
3591 | /* This lockless write can happen from different cpus. */ |
3592 | WRITE_ONCE(pol->curlft.use_time, ktime_get_real_seconds()); |
3593 | |
3594 | pols[0] = pol; |
3595 | npols++; |
3596 | #ifdef CONFIG_XFRM_SUB_POLICY |
3597 | if (pols[0]->type != XFRM_POLICY_TYPE_MAIN) { |
3598 | pols[1] = xfrm_policy_lookup_bytype(net, type: XFRM_POLICY_TYPE_MAIN, |
3599 | fl: &fl, family, |
3600 | dir: XFRM_POLICY_IN, if_id); |
3601 | if (pols[1]) { |
3602 | if (IS_ERR(ptr: pols[1])) { |
3603 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR); |
3604 | xfrm_pol_put(policy: pols[0]); |
3605 | return 0; |
3606 | } |
3607 | /* This write can happen from different cpus. */ |
3608 | WRITE_ONCE(pols[1]->curlft.use_time, |
3609 | ktime_get_real_seconds()); |
3610 | npols++; |
3611 | } |
3612 | } |
3613 | #endif |
3614 | |
3615 | if (pol->action == XFRM_POLICY_ALLOW) { |
3616 | static struct sec_path dummy; |
3617 | struct xfrm_tmpl *tp[XFRM_MAX_DEPTH]; |
3618 | struct xfrm_tmpl *stp[XFRM_MAX_DEPTH]; |
3619 | struct xfrm_tmpl **tpp = tp; |
3620 | int ti = 0; |
3621 | int i, k; |
3622 | |
3623 | sp = skb_sec_path(skb); |
3624 | if (!sp) |
3625 | sp = &dummy; |
3626 | |
3627 | for (pi = 0; pi < npols; pi++) { |
3628 | if (pols[pi] != pol && |
3629 | pols[pi]->action != XFRM_POLICY_ALLOW) { |
3630 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLBLOCK); |
3631 | goto reject; |
3632 | } |
3633 | if (ti + pols[pi]->xfrm_nr >= XFRM_MAX_DEPTH) { |
3634 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR); |
3635 | goto reject_error; |
3636 | } |
3637 | for (i = 0; i < pols[pi]->xfrm_nr; i++) |
3638 | tpp[ti++] = &pols[pi]->xfrm_vec[i]; |
3639 | } |
3640 | xfrm_nr = ti; |
3641 | |
3642 | if (npols > 1) { |
3643 | xfrm_tmpl_sort(dst: stp, src: tpp, n: xfrm_nr, family); |
3644 | tpp = stp; |
3645 | } |
3646 | |
3647 | /* For each tunnel xfrm, find the first matching tmpl. |
3648 | * For each tmpl before that, find corresponding xfrm. |
3649 | * Order is _important_. Later we will implement |
3650 | * some barriers, but at the moment barriers |
3651 | * are implied between each two transformations. |
3652 | * Upon success, marks secpath entries as having been |
3653 | * verified to allow them to be skipped in future policy |
3654 | * checks (e.g. nested tunnels). |
3655 | */ |
3656 | for (i = xfrm_nr-1, k = 0; i >= 0; i--) { |
3657 | k = xfrm_policy_ok(tmpl: tpp[i], sp, start: k, family, if_id); |
3658 | if (k < 0) { |
3659 | if (k < -1) |
3660 | /* "-2 - errored_index" returned */ |
3661 | xerr_idx = -(2+k); |
3662 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINTMPLMISMATCH); |
3663 | goto reject; |
3664 | } |
3665 | } |
3666 | |
3667 | if (secpath_has_nontransport(sp, k, idxp: &xerr_idx)) { |
3668 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINTMPLMISMATCH); |
3669 | goto reject; |
3670 | } |
3671 | |
3672 | xfrm_pols_put(pols, npols); |
3673 | sp->verified_cnt = k; |
3674 | |
3675 | return 1; |
3676 | } |
3677 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLBLOCK); |
3678 | |
3679 | reject: |
3680 | xfrm_secpath_reject(idx: xerr_idx, skb, fl: &fl); |
3681 | reject_error: |
3682 | xfrm_pols_put(pols, npols); |
3683 | return 0; |
3684 | } |
3685 | EXPORT_SYMBOL(__xfrm_policy_check); |
3686 | |
3687 | int __xfrm_route_forward(struct sk_buff *skb, unsigned short family) |
3688 | { |
3689 | struct net *net = dev_net(dev: skb->dev); |
3690 | struct flowi fl; |
3691 | struct dst_entry *dst; |
3692 | int res = 1; |
3693 | |
3694 | if (xfrm_decode_session(net, skb, fl: &fl, family) < 0) { |
3695 | XFRM_INC_STATS(net, LINUX_MIB_XFRMFWDHDRERROR); |
3696 | return 0; |
3697 | } |
3698 | |
3699 | skb_dst_force(skb); |
3700 | if (!skb_dst(skb)) { |
3701 | XFRM_INC_STATS(net, LINUX_MIB_XFRMFWDHDRERROR); |
3702 | return 0; |
3703 | } |
3704 | |
3705 | dst = xfrm_lookup(net, skb_dst(skb), &fl, NULL, XFRM_LOOKUP_QUEUE); |
3706 | if (IS_ERR(ptr: dst)) { |
3707 | res = 0; |
3708 | dst = NULL; |
3709 | } |
3710 | skb_dst_set(skb, dst); |
3711 | return res; |
3712 | } |
3713 | EXPORT_SYMBOL(__xfrm_route_forward); |
3714 | |
3715 | /* Optimize later using cookies and generation ids. */ |
3716 | |
3717 | static struct dst_entry *xfrm_dst_check(struct dst_entry *dst, u32 cookie) |
3718 | { |
3719 | /* Code (such as __xfrm4_bundle_create()) sets dst->obsolete |
3720 | * to DST_OBSOLETE_FORCE_CHK to force all XFRM destinations to |
3721 | * get validated by dst_ops->check on every use. We do this |
3722 | * because when a normal route referenced by an XFRM dst is |
3723 | * obsoleted we do not go looking around for all parent |
3724 | * referencing XFRM dsts so that we can invalidate them. It |
3725 | * is just too much work. Instead we make the checks here on |
3726 | * every use. For example: |
3727 | * |
3728 | * XFRM dst A --> IPv4 dst X |
3729 | * |
3730 | * X is the "xdst->route" of A (X is also the "dst->path" of A |
3731 | * in this example). If X is marked obsolete, "A" will not |
3732 | * notice. That's what we are validating here via the |
3733 | * stale_bundle() check. |
3734 | * |
3735 | * When a dst is removed from the fib tree, DST_OBSOLETE_DEAD will |
3736 | * be marked on it. |
3737 | * This will force stale_bundle() to fail on any xdst bundle with |
3738 | * this dst linked in it. |
3739 | */ |
3740 | if (dst->obsolete < 0 && !stale_bundle(dst)) |
3741 | return dst; |
3742 | |
3743 | return NULL; |
3744 | } |
3745 | |
3746 | static int stale_bundle(struct dst_entry *dst) |
3747 | { |
3748 | return !xfrm_bundle_ok(xdst: (struct xfrm_dst *)dst); |
3749 | } |
3750 | |
3751 | void xfrm_dst_ifdown(struct dst_entry *dst, struct net_device *dev) |
3752 | { |
3753 | while ((dst = xfrm_dst_child(dst)) && dst->xfrm && dst->dev == dev) { |
3754 | dst->dev = blackhole_netdev; |
3755 | dev_hold(dev: dst->dev); |
3756 | dev_put(dev); |
3757 | } |
3758 | } |
3759 | EXPORT_SYMBOL(xfrm_dst_ifdown); |
3760 | |
3761 | static void xfrm_link_failure(struct sk_buff *skb) |
3762 | { |
3763 | /* Impossible. Such dst must be popped before reaches point of failure. */ |
3764 | } |
3765 | |
3766 | static struct dst_entry *xfrm_negative_advice(struct dst_entry *dst) |
3767 | { |
3768 | if (dst) { |
3769 | if (dst->obsolete) { |
3770 | dst_release(dst); |
3771 | dst = NULL; |
3772 | } |
3773 | } |
3774 | return dst; |
3775 | } |
3776 | |
3777 | static void xfrm_init_pmtu(struct xfrm_dst **bundle, int nr) |
3778 | { |
3779 | while (nr--) { |
3780 | struct xfrm_dst *xdst = bundle[nr]; |
3781 | u32 pmtu, route_mtu_cached; |
3782 | struct dst_entry *dst; |
3783 | |
3784 | dst = &xdst->u.dst; |
3785 | pmtu = dst_mtu(dst: xfrm_dst_child(dst)); |
3786 | xdst->child_mtu_cached = pmtu; |
3787 | |
3788 | pmtu = xfrm_state_mtu(x: dst->xfrm, mtu: pmtu); |
3789 | |
3790 | route_mtu_cached = dst_mtu(dst: xdst->route); |
3791 | xdst->route_mtu_cached = route_mtu_cached; |
3792 | |
3793 | if (pmtu > route_mtu_cached) |
3794 | pmtu = route_mtu_cached; |
3795 | |
3796 | dst_metric_set(dst, RTAX_MTU, val: pmtu); |
3797 | } |
3798 | } |
3799 | |
3800 | /* Check that the bundle accepts the flow and its components are |
3801 | * still valid. |
3802 | */ |
3803 | |
3804 | static int xfrm_bundle_ok(struct xfrm_dst *first) |
3805 | { |
3806 | struct xfrm_dst *bundle[XFRM_MAX_DEPTH]; |
3807 | struct dst_entry *dst = &first->u.dst; |
3808 | struct xfrm_dst *xdst; |
3809 | int start_from, nr; |
3810 | u32 mtu; |
3811 | |
3812 | if (!dst_check(dst: xfrm_dst_path(dst), cookie: ((struct xfrm_dst *)dst)->path_cookie) || |
3813 | (dst->dev && !netif_running(dev: dst->dev))) |
3814 | return 0; |
3815 | |
3816 | if (dst->flags & DST_XFRM_QUEUE) |
3817 | return 1; |
3818 | |
3819 | start_from = nr = 0; |
3820 | do { |
3821 | struct xfrm_dst *xdst = (struct xfrm_dst *)dst; |
3822 | |
3823 | if (dst->xfrm->km.state != XFRM_STATE_VALID) |
3824 | return 0; |
3825 | if (xdst->xfrm_genid != dst->xfrm->genid) |
3826 | return 0; |
3827 | if (xdst->num_pols > 0 && |
3828 | xdst->policy_genid != atomic_read(v: &xdst->pols[0]->genid)) |
3829 | return 0; |
3830 | |
3831 | bundle[nr++] = xdst; |
3832 | |
3833 | mtu = dst_mtu(dst: xfrm_dst_child(dst)); |
3834 | if (xdst->child_mtu_cached != mtu) { |
3835 | start_from = nr; |
3836 | xdst->child_mtu_cached = mtu; |
3837 | } |
3838 | |
3839 | if (!dst_check(dst: xdst->route, cookie: xdst->route_cookie)) |
3840 | return 0; |
3841 | mtu = dst_mtu(dst: xdst->route); |
3842 | if (xdst->route_mtu_cached != mtu) { |
3843 | start_from = nr; |
3844 | xdst->route_mtu_cached = mtu; |
3845 | } |
3846 | |
3847 | dst = xfrm_dst_child(dst); |
3848 | } while (dst->xfrm); |
3849 | |
3850 | if (likely(!start_from)) |
3851 | return 1; |
3852 | |
3853 | xdst = bundle[start_from - 1]; |
3854 | mtu = xdst->child_mtu_cached; |
3855 | while (start_from--) { |
3856 | dst = &xdst->u.dst; |
3857 | |
3858 | mtu = xfrm_state_mtu(x: dst->xfrm, mtu); |
3859 | if (mtu > xdst->route_mtu_cached) |
3860 | mtu = xdst->route_mtu_cached; |
3861 | dst_metric_set(dst, RTAX_MTU, val: mtu); |
3862 | if (!start_from) |
3863 | break; |
3864 | |
3865 | xdst = bundle[start_from - 1]; |
3866 | xdst->child_mtu_cached = mtu; |
3867 | } |
3868 | |
3869 | return 1; |
3870 | } |
3871 | |
3872 | static unsigned int xfrm_default_advmss(const struct dst_entry *dst) |
3873 | { |
3874 | return dst_metric_advmss(dst: xfrm_dst_path(dst)); |
3875 | } |
3876 | |
3877 | static unsigned int xfrm_mtu(const struct dst_entry *dst) |
3878 | { |
3879 | unsigned int mtu = dst_metric_raw(dst, RTAX_MTU); |
3880 | |
3881 | return mtu ? : dst_mtu(dst: xfrm_dst_path(dst)); |
3882 | } |
3883 | |
3884 | static const void *xfrm_get_dst_nexthop(const struct dst_entry *dst, |
3885 | const void *daddr) |
3886 | { |
3887 | while (dst->xfrm) { |
3888 | const struct xfrm_state *xfrm = dst->xfrm; |
3889 | |
3890 | dst = xfrm_dst_child(dst); |
3891 | |
3892 | if (xfrm->props.mode == XFRM_MODE_TRANSPORT) |
3893 | continue; |
3894 | if (xfrm->type->flags & XFRM_TYPE_REMOTE_COADDR) |
3895 | daddr = xfrm->coaddr; |
3896 | else if (!(xfrm->type->flags & XFRM_TYPE_LOCAL_COADDR)) |
3897 | daddr = &xfrm->id.daddr; |
3898 | } |
3899 | return daddr; |
3900 | } |
3901 | |
3902 | static struct neighbour *xfrm_neigh_lookup(const struct dst_entry *dst, |
3903 | struct sk_buff *skb, |
3904 | const void *daddr) |
3905 | { |
3906 | const struct dst_entry *path = xfrm_dst_path(dst); |
3907 | |
3908 | if (!skb) |
3909 | daddr = xfrm_get_dst_nexthop(dst, daddr); |
3910 | return path->ops->neigh_lookup(path, skb, daddr); |
3911 | } |
3912 | |
3913 | static void xfrm_confirm_neigh(const struct dst_entry *dst, const void *daddr) |
3914 | { |
3915 | const struct dst_entry *path = xfrm_dst_path(dst); |
3916 | |
3917 | daddr = xfrm_get_dst_nexthop(dst, daddr); |
3918 | path->ops->confirm_neigh(path, daddr); |
3919 | } |
3920 | |
3921 | int xfrm_policy_register_afinfo(const struct xfrm_policy_afinfo *afinfo, int family) |
3922 | { |
3923 | int err = 0; |
3924 | |
3925 | if (WARN_ON(family >= ARRAY_SIZE(xfrm_policy_afinfo))) |
3926 | return -EAFNOSUPPORT; |
3927 | |
3928 | spin_lock(lock: &xfrm_policy_afinfo_lock); |
3929 | if (unlikely(xfrm_policy_afinfo[family] != NULL)) |
3930 | err = -EEXIST; |
3931 | else { |
3932 | struct dst_ops *dst_ops = afinfo->dst_ops; |
3933 | if (likely(dst_ops->kmem_cachep == NULL)) |
3934 | dst_ops->kmem_cachep = xfrm_dst_cache; |
3935 | if (likely(dst_ops->check == NULL)) |
3936 | dst_ops->check = xfrm_dst_check; |
3937 | if (likely(dst_ops->default_advmss == NULL)) |
3938 | dst_ops->default_advmss = xfrm_default_advmss; |
3939 | if (likely(dst_ops->mtu == NULL)) |
3940 | dst_ops->mtu = xfrm_mtu; |
3941 | if (likely(dst_ops->negative_advice == NULL)) |
3942 | dst_ops->negative_advice = xfrm_negative_advice; |
3943 | if (likely(dst_ops->link_failure == NULL)) |
3944 | dst_ops->link_failure = xfrm_link_failure; |
3945 | if (likely(dst_ops->neigh_lookup == NULL)) |
3946 | dst_ops->neigh_lookup = xfrm_neigh_lookup; |
3947 | if (likely(!dst_ops->confirm_neigh)) |
3948 | dst_ops->confirm_neigh = xfrm_confirm_neigh; |
3949 | rcu_assign_pointer(xfrm_policy_afinfo[family], afinfo); |
3950 | } |
3951 | spin_unlock(lock: &xfrm_policy_afinfo_lock); |
3952 | |
3953 | return err; |
3954 | } |
3955 | EXPORT_SYMBOL(xfrm_policy_register_afinfo); |
3956 | |
3957 | void xfrm_policy_unregister_afinfo(const struct xfrm_policy_afinfo *afinfo) |
3958 | { |
3959 | struct dst_ops *dst_ops = afinfo->dst_ops; |
3960 | int i; |
3961 | |
3962 | for (i = 0; i < ARRAY_SIZE(xfrm_policy_afinfo); i++) { |
3963 | if (xfrm_policy_afinfo[i] != afinfo) |
3964 | continue; |
3965 | RCU_INIT_POINTER(xfrm_policy_afinfo[i], NULL); |
3966 | break; |
3967 | } |
3968 | |
3969 | synchronize_rcu(); |
3970 | |
3971 | dst_ops->kmem_cachep = NULL; |
3972 | dst_ops->check = NULL; |
3973 | dst_ops->negative_advice = NULL; |
3974 | dst_ops->link_failure = NULL; |
3975 | } |
3976 | EXPORT_SYMBOL(xfrm_policy_unregister_afinfo); |
3977 | |
3978 | void xfrm_if_register_cb(const struct xfrm_if_cb *ifcb) |
3979 | { |
3980 | spin_lock(lock: &xfrm_if_cb_lock); |
3981 | rcu_assign_pointer(xfrm_if_cb, ifcb); |
3982 | spin_unlock(lock: &xfrm_if_cb_lock); |
3983 | } |
3984 | EXPORT_SYMBOL(xfrm_if_register_cb); |
3985 | |
3986 | void xfrm_if_unregister_cb(void) |
3987 | { |
3988 | RCU_INIT_POINTER(xfrm_if_cb, NULL); |
3989 | synchronize_rcu(); |
3990 | } |
3991 | EXPORT_SYMBOL(xfrm_if_unregister_cb); |
3992 | |
3993 | #ifdef CONFIG_XFRM_STATISTICS |
3994 | static int __net_init xfrm_statistics_init(struct net *net) |
3995 | { |
3996 | int rv; |
3997 | net->mib.xfrm_statistics = alloc_percpu(struct linux_xfrm_mib); |
3998 | if (!net->mib.xfrm_statistics) |
3999 | return -ENOMEM; |
4000 | rv = xfrm_proc_init(net); |
4001 | if (rv < 0) |
4002 | free_percpu(pdata: net->mib.xfrm_statistics); |
4003 | return rv; |
4004 | } |
4005 | |
4006 | static void xfrm_statistics_fini(struct net *net) |
4007 | { |
4008 | xfrm_proc_fini(net); |
4009 | free_percpu(pdata: net->mib.xfrm_statistics); |
4010 | } |
4011 | #else |
4012 | static int __net_init xfrm_statistics_init(struct net *net) |
4013 | { |
4014 | return 0; |
4015 | } |
4016 | |
4017 | static void xfrm_statistics_fini(struct net *net) |
4018 | { |
4019 | } |
4020 | #endif |
4021 | |
4022 | static int __net_init xfrm_policy_init(struct net *net) |
4023 | { |
4024 | unsigned int hmask, sz; |
4025 | int dir, err; |
4026 | |
4027 | if (net_eq(net1: net, net2: &init_net)) { |
4028 | xfrm_dst_cache = kmem_cache_create(name: "xfrm_dst_cache" , |
4029 | size: sizeof(struct xfrm_dst), |
4030 | align: 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, |
4031 | NULL); |
4032 | err = rhashtable_init(ht: &xfrm_policy_inexact_table, |
4033 | params: &xfrm_pol_inexact_params); |
4034 | BUG_ON(err); |
4035 | } |
4036 | |
4037 | hmask = 8 - 1; |
4038 | sz = (hmask+1) * sizeof(struct hlist_head); |
4039 | |
4040 | net->xfrm.policy_byidx = xfrm_hash_alloc(sz); |
4041 | if (!net->xfrm.policy_byidx) |
4042 | goto out_byidx; |
4043 | net->xfrm.policy_idx_hmask = hmask; |
4044 | |
4045 | for (dir = 0; dir < XFRM_POLICY_MAX; dir++) { |
4046 | struct xfrm_policy_hash *htab; |
4047 | |
4048 | net->xfrm.policy_count[dir] = 0; |
4049 | net->xfrm.policy_count[XFRM_POLICY_MAX + dir] = 0; |
4050 | INIT_HLIST_HEAD(&net->xfrm.policy_inexact[dir]); |
4051 | |
4052 | htab = &net->xfrm.policy_bydst[dir]; |
4053 | htab->table = xfrm_hash_alloc(sz); |
4054 | if (!htab->table) |
4055 | goto out_bydst; |
4056 | htab->hmask = hmask; |
4057 | htab->dbits4 = 32; |
4058 | htab->sbits4 = 32; |
4059 | htab->dbits6 = 128; |
4060 | htab->sbits6 = 128; |
4061 | } |
4062 | net->xfrm.policy_hthresh.lbits4 = 32; |
4063 | net->xfrm.policy_hthresh.rbits4 = 32; |
4064 | net->xfrm.policy_hthresh.lbits6 = 128; |
4065 | net->xfrm.policy_hthresh.rbits6 = 128; |
4066 | |
4067 | seqlock_init(&net->xfrm.policy_hthresh.lock); |
4068 | |
4069 | INIT_LIST_HEAD(list: &net->xfrm.policy_all); |
4070 | INIT_LIST_HEAD(list: &net->xfrm.inexact_bins); |
4071 | INIT_WORK(&net->xfrm.policy_hash_work, xfrm_hash_resize); |
4072 | INIT_WORK(&net->xfrm.policy_hthresh.work, xfrm_hash_rebuild); |
4073 | return 0; |
4074 | |
4075 | out_bydst: |
4076 | for (dir--; dir >= 0; dir--) { |
4077 | struct xfrm_policy_hash *htab; |
4078 | |
4079 | htab = &net->xfrm.policy_bydst[dir]; |
4080 | xfrm_hash_free(n: htab->table, sz); |
4081 | } |
4082 | xfrm_hash_free(n: net->xfrm.policy_byidx, sz); |
4083 | out_byidx: |
4084 | return -ENOMEM; |
4085 | } |
4086 | |
4087 | static void xfrm_policy_fini(struct net *net) |
4088 | { |
4089 | struct xfrm_pol_inexact_bin *b, *t; |
4090 | unsigned int sz; |
4091 | int dir; |
4092 | |
4093 | flush_work(work: &net->xfrm.policy_hash_work); |
4094 | #ifdef CONFIG_XFRM_SUB_POLICY |
4095 | xfrm_policy_flush(net, XFRM_POLICY_TYPE_SUB, false); |
4096 | #endif |
4097 | xfrm_policy_flush(net, XFRM_POLICY_TYPE_MAIN, false); |
4098 | |
4099 | WARN_ON(!list_empty(&net->xfrm.policy_all)); |
4100 | |
4101 | for (dir = 0; dir < XFRM_POLICY_MAX; dir++) { |
4102 | struct xfrm_policy_hash *htab; |
4103 | |
4104 | WARN_ON(!hlist_empty(&net->xfrm.policy_inexact[dir])); |
4105 | |
4106 | htab = &net->xfrm.policy_bydst[dir]; |
4107 | sz = (htab->hmask + 1) * sizeof(struct hlist_head); |
4108 | WARN_ON(!hlist_empty(htab->table)); |
4109 | xfrm_hash_free(n: htab->table, sz); |
4110 | } |
4111 | |
4112 | sz = (net->xfrm.policy_idx_hmask + 1) * sizeof(struct hlist_head); |
4113 | WARN_ON(!hlist_empty(net->xfrm.policy_byidx)); |
4114 | xfrm_hash_free(n: net->xfrm.policy_byidx, sz); |
4115 | |
4116 | spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock); |
4117 | list_for_each_entry_safe(b, t, &net->xfrm.inexact_bins, inexact_bins) |
4118 | __xfrm_policy_inexact_prune_bin(b, net_exit: true); |
4119 | spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock); |
4120 | } |
4121 | |
4122 | static int __net_init xfrm_net_init(struct net *net) |
4123 | { |
4124 | int rv; |
4125 | |
4126 | /* Initialize the per-net locks here */ |
4127 | spin_lock_init(&net->xfrm.xfrm_state_lock); |
4128 | spin_lock_init(&net->xfrm.xfrm_policy_lock); |
4129 | seqcount_spinlock_init(&net->xfrm.xfrm_policy_hash_generation, &net->xfrm.xfrm_policy_lock); |
4130 | mutex_init(&net->xfrm.xfrm_cfg_mutex); |
4131 | net->xfrm.policy_default[XFRM_POLICY_IN] = XFRM_USERPOLICY_ACCEPT; |
4132 | net->xfrm.policy_default[XFRM_POLICY_FWD] = XFRM_USERPOLICY_ACCEPT; |
4133 | net->xfrm.policy_default[XFRM_POLICY_OUT] = XFRM_USERPOLICY_ACCEPT; |
4134 | |
4135 | rv = xfrm_statistics_init(net); |
4136 | if (rv < 0) |
4137 | goto out_statistics; |
4138 | rv = xfrm_state_init(net); |
4139 | if (rv < 0) |
4140 | goto out_state; |
4141 | rv = xfrm_policy_init(net); |
4142 | if (rv < 0) |
4143 | goto out_policy; |
4144 | rv = xfrm_sysctl_init(net); |
4145 | if (rv < 0) |
4146 | goto out_sysctl; |
4147 | |
4148 | return 0; |
4149 | |
4150 | out_sysctl: |
4151 | xfrm_policy_fini(net); |
4152 | out_policy: |
4153 | xfrm_state_fini(net); |
4154 | out_state: |
4155 | xfrm_statistics_fini(net); |
4156 | out_statistics: |
4157 | return rv; |
4158 | } |
4159 | |
4160 | static void __net_exit xfrm_net_exit(struct net *net) |
4161 | { |
4162 | xfrm_sysctl_fini(net); |
4163 | xfrm_policy_fini(net); |
4164 | xfrm_state_fini(net); |
4165 | xfrm_statistics_fini(net); |
4166 | } |
4167 | |
4168 | static struct pernet_operations __net_initdata xfrm_net_ops = { |
4169 | .init = xfrm_net_init, |
4170 | .exit = xfrm_net_exit, |
4171 | }; |
4172 | |
4173 | static const struct flow_dissector_key xfrm_flow_dissector_keys[] = { |
4174 | { |
4175 | .key_id = FLOW_DISSECTOR_KEY_CONTROL, |
4176 | .offset = offsetof(struct xfrm_flow_keys, control), |
4177 | }, |
4178 | { |
4179 | .key_id = FLOW_DISSECTOR_KEY_BASIC, |
4180 | .offset = offsetof(struct xfrm_flow_keys, basic), |
4181 | }, |
4182 | { |
4183 | .key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS, |
4184 | .offset = offsetof(struct xfrm_flow_keys, addrs.ipv4), |
4185 | }, |
4186 | { |
4187 | .key_id = FLOW_DISSECTOR_KEY_IPV6_ADDRS, |
4188 | .offset = offsetof(struct xfrm_flow_keys, addrs.ipv6), |
4189 | }, |
4190 | { |
4191 | .key_id = FLOW_DISSECTOR_KEY_PORTS, |
4192 | .offset = offsetof(struct xfrm_flow_keys, ports), |
4193 | }, |
4194 | { |
4195 | .key_id = FLOW_DISSECTOR_KEY_GRE_KEYID, |
4196 | .offset = offsetof(struct xfrm_flow_keys, gre), |
4197 | }, |
4198 | { |
4199 | .key_id = FLOW_DISSECTOR_KEY_IP, |
4200 | .offset = offsetof(struct xfrm_flow_keys, ip), |
4201 | }, |
4202 | { |
4203 | .key_id = FLOW_DISSECTOR_KEY_ICMP, |
4204 | .offset = offsetof(struct xfrm_flow_keys, icmp), |
4205 | }, |
4206 | }; |
4207 | |
4208 | void __init xfrm_init(void) |
4209 | { |
4210 | skb_flow_dissector_init(flow_dissector: &xfrm_session_dissector, |
4211 | key: xfrm_flow_dissector_keys, |
4212 | ARRAY_SIZE(xfrm_flow_dissector_keys)); |
4213 | |
4214 | register_pernet_subsys(&xfrm_net_ops); |
4215 | xfrm_dev_init(); |
4216 | xfrm_input_init(); |
4217 | |
4218 | #ifdef CONFIG_XFRM_ESPINTCP |
4219 | espintcp_init(); |
4220 | #endif |
4221 | } |
4222 | |
4223 | #ifdef CONFIG_AUDITSYSCALL |
4224 | static void xfrm_audit_common_policyinfo(struct xfrm_policy *xp, |
4225 | struct audit_buffer *audit_buf) |
4226 | { |
4227 | struct xfrm_sec_ctx *ctx = xp->security; |
4228 | struct xfrm_selector *sel = &xp->selector; |
4229 | |
4230 | if (ctx) |
4231 | audit_log_format(ab: audit_buf, fmt: " sec_alg=%u sec_doi=%u sec_obj=%s" , |
4232 | ctx->ctx_alg, ctx->ctx_doi, ctx->ctx_str); |
4233 | |
4234 | switch (sel->family) { |
4235 | case AF_INET: |
4236 | audit_log_format(ab: audit_buf, fmt: " src=%pI4" , &sel->saddr.a4); |
4237 | if (sel->prefixlen_s != 32) |
4238 | audit_log_format(ab: audit_buf, fmt: " src_prefixlen=%d" , |
4239 | sel->prefixlen_s); |
4240 | audit_log_format(ab: audit_buf, fmt: " dst=%pI4" , &sel->daddr.a4); |
4241 | if (sel->prefixlen_d != 32) |
4242 | audit_log_format(ab: audit_buf, fmt: " dst_prefixlen=%d" , |
4243 | sel->prefixlen_d); |
4244 | break; |
4245 | case AF_INET6: |
4246 | audit_log_format(ab: audit_buf, fmt: " src=%pI6" , sel->saddr.a6); |
4247 | if (sel->prefixlen_s != 128) |
4248 | audit_log_format(ab: audit_buf, fmt: " src_prefixlen=%d" , |
4249 | sel->prefixlen_s); |
4250 | audit_log_format(ab: audit_buf, fmt: " dst=%pI6" , sel->daddr.a6); |
4251 | if (sel->prefixlen_d != 128) |
4252 | audit_log_format(ab: audit_buf, fmt: " dst_prefixlen=%d" , |
4253 | sel->prefixlen_d); |
4254 | break; |
4255 | } |
4256 | } |
4257 | |
4258 | void xfrm_audit_policy_add(struct xfrm_policy *xp, int result, bool task_valid) |
4259 | { |
4260 | struct audit_buffer *audit_buf; |
4261 | |
4262 | audit_buf = xfrm_audit_start(op: "SPD-add" ); |
4263 | if (audit_buf == NULL) |
4264 | return; |
4265 | xfrm_audit_helper_usrinfo(task_valid, audit_buf); |
4266 | audit_log_format(ab: audit_buf, fmt: " res=%u" , result); |
4267 | xfrm_audit_common_policyinfo(xp, audit_buf); |
4268 | audit_log_end(ab: audit_buf); |
4269 | } |
4270 | EXPORT_SYMBOL_GPL(xfrm_audit_policy_add); |
4271 | |
4272 | void xfrm_audit_policy_delete(struct xfrm_policy *xp, int result, |
4273 | bool task_valid) |
4274 | { |
4275 | struct audit_buffer *audit_buf; |
4276 | |
4277 | audit_buf = xfrm_audit_start(op: "SPD-delete" ); |
4278 | if (audit_buf == NULL) |
4279 | return; |
4280 | xfrm_audit_helper_usrinfo(task_valid, audit_buf); |
4281 | audit_log_format(ab: audit_buf, fmt: " res=%u" , result); |
4282 | xfrm_audit_common_policyinfo(xp, audit_buf); |
4283 | audit_log_end(ab: audit_buf); |
4284 | } |
4285 | EXPORT_SYMBOL_GPL(xfrm_audit_policy_delete); |
4286 | #endif |
4287 | |
4288 | #ifdef CONFIG_XFRM_MIGRATE |
4289 | static bool xfrm_migrate_selector_match(const struct xfrm_selector *sel_cmp, |
4290 | const struct xfrm_selector *sel_tgt) |
4291 | { |
4292 | if (sel_cmp->proto == IPSEC_ULPROTO_ANY) { |
4293 | if (sel_tgt->family == sel_cmp->family && |
4294 | xfrm_addr_equal(a: &sel_tgt->daddr, b: &sel_cmp->daddr, |
4295 | family: sel_cmp->family) && |
4296 | xfrm_addr_equal(a: &sel_tgt->saddr, b: &sel_cmp->saddr, |
4297 | family: sel_cmp->family) && |
4298 | sel_tgt->prefixlen_d == sel_cmp->prefixlen_d && |
4299 | sel_tgt->prefixlen_s == sel_cmp->prefixlen_s) { |
4300 | return true; |
4301 | } |
4302 | } else { |
4303 | if (memcmp(p: sel_tgt, q: sel_cmp, size: sizeof(*sel_tgt)) == 0) { |
4304 | return true; |
4305 | } |
4306 | } |
4307 | return false; |
4308 | } |
4309 | |
4310 | static struct xfrm_policy *xfrm_migrate_policy_find(const struct xfrm_selector *sel, |
4311 | u8 dir, u8 type, struct net *net, u32 if_id) |
4312 | { |
4313 | struct xfrm_policy *pol, *ret = NULL; |
4314 | struct hlist_head *chain; |
4315 | u32 priority = ~0U; |
4316 | |
4317 | spin_lock_bh(lock: &net->xfrm.xfrm_policy_lock); |
4318 | chain = policy_hash_direct(net, daddr: &sel->daddr, saddr: &sel->saddr, family: sel->family, dir); |
4319 | hlist_for_each_entry(pol, chain, bydst) { |
4320 | if ((if_id == 0 || pol->if_id == if_id) && |
4321 | xfrm_migrate_selector_match(sel_cmp: sel, sel_tgt: &pol->selector) && |
4322 | pol->type == type) { |
4323 | ret = pol; |
4324 | priority = ret->priority; |
4325 | break; |
4326 | } |
4327 | } |
4328 | chain = &net->xfrm.policy_inexact[dir]; |
4329 | hlist_for_each_entry(pol, chain, bydst_inexact_list) { |
4330 | if ((pol->priority >= priority) && ret) |
4331 | break; |
4332 | |
4333 | if ((if_id == 0 || pol->if_id == if_id) && |
4334 | xfrm_migrate_selector_match(sel_cmp: sel, sel_tgt: &pol->selector) && |
4335 | pol->type == type) { |
4336 | ret = pol; |
4337 | break; |
4338 | } |
4339 | } |
4340 | |
4341 | xfrm_pol_hold(policy: ret); |
4342 | |
4343 | spin_unlock_bh(lock: &net->xfrm.xfrm_policy_lock); |
4344 | |
4345 | return ret; |
4346 | } |
4347 | |
4348 | static int migrate_tmpl_match(const struct xfrm_migrate *m, const struct xfrm_tmpl *t) |
4349 | { |
4350 | int match = 0; |
4351 | |
4352 | if (t->mode == m->mode && t->id.proto == m->proto && |
4353 | (m->reqid == 0 || t->reqid == m->reqid)) { |
4354 | switch (t->mode) { |
4355 | case XFRM_MODE_TUNNEL: |
4356 | case XFRM_MODE_BEET: |
4357 | if (xfrm_addr_equal(a: &t->id.daddr, b: &m->old_daddr, |
4358 | family: m->old_family) && |
4359 | xfrm_addr_equal(a: &t->saddr, b: &m->old_saddr, |
4360 | family: m->old_family)) { |
4361 | match = 1; |
4362 | } |
4363 | break; |
4364 | case XFRM_MODE_TRANSPORT: |
4365 | /* in case of transport mode, template does not store |
4366 | any IP addresses, hence we just compare mode and |
4367 | protocol */ |
4368 | match = 1; |
4369 | break; |
4370 | default: |
4371 | break; |
4372 | } |
4373 | } |
4374 | return match; |
4375 | } |
4376 | |
4377 | /* update endpoint address(es) of template(s) */ |
4378 | static int xfrm_policy_migrate(struct xfrm_policy *pol, |
4379 | struct xfrm_migrate *m, int num_migrate, |
4380 | struct netlink_ext_ack *extack) |
4381 | { |
4382 | struct xfrm_migrate *mp; |
4383 | int i, j, n = 0; |
4384 | |
4385 | write_lock_bh(&pol->lock); |
4386 | if (unlikely(pol->walk.dead)) { |
4387 | /* target policy has been deleted */ |
4388 | NL_SET_ERR_MSG(extack, "Target policy not found" ); |
4389 | write_unlock_bh(&pol->lock); |
4390 | return -ENOENT; |
4391 | } |
4392 | |
4393 | for (i = 0; i < pol->xfrm_nr; i++) { |
4394 | for (j = 0, mp = m; j < num_migrate; j++, mp++) { |
4395 | if (!migrate_tmpl_match(m: mp, t: &pol->xfrm_vec[i])) |
4396 | continue; |
4397 | n++; |
4398 | if (pol->xfrm_vec[i].mode != XFRM_MODE_TUNNEL && |
4399 | pol->xfrm_vec[i].mode != XFRM_MODE_BEET) |
4400 | continue; |
4401 | /* update endpoints */ |
4402 | memcpy(&pol->xfrm_vec[i].id.daddr, &mp->new_daddr, |
4403 | sizeof(pol->xfrm_vec[i].id.daddr)); |
4404 | memcpy(&pol->xfrm_vec[i].saddr, &mp->new_saddr, |
4405 | sizeof(pol->xfrm_vec[i].saddr)); |
4406 | pol->xfrm_vec[i].encap_family = mp->new_family; |
4407 | /* flush bundles */ |
4408 | atomic_inc(v: &pol->genid); |
4409 | } |
4410 | } |
4411 | |
4412 | write_unlock_bh(&pol->lock); |
4413 | |
4414 | if (!n) |
4415 | return -ENODATA; |
4416 | |
4417 | return 0; |
4418 | } |
4419 | |
4420 | static int xfrm_migrate_check(const struct xfrm_migrate *m, int num_migrate, |
4421 | struct netlink_ext_ack *extack) |
4422 | { |
4423 | int i, j; |
4424 | |
4425 | if (num_migrate < 1 || num_migrate > XFRM_MAX_DEPTH) { |
4426 | NL_SET_ERR_MSG(extack, "Invalid number of SAs to migrate, must be 0 < num <= XFRM_MAX_DEPTH (6)" ); |
4427 | return -EINVAL; |
4428 | } |
4429 | |
4430 | for (i = 0; i < num_migrate; i++) { |
4431 | if (xfrm_addr_any(addr: &m[i].new_daddr, family: m[i].new_family) || |
4432 | xfrm_addr_any(addr: &m[i].new_saddr, family: m[i].new_family)) { |
4433 | NL_SET_ERR_MSG(extack, "Addresses in the MIGRATE attribute's list cannot be null" ); |
4434 | return -EINVAL; |
4435 | } |
4436 | |
4437 | /* check if there is any duplicated entry */ |
4438 | for (j = i + 1; j < num_migrate; j++) { |
4439 | if (!memcmp(p: &m[i].old_daddr, q: &m[j].old_daddr, |
4440 | size: sizeof(m[i].old_daddr)) && |
4441 | !memcmp(p: &m[i].old_saddr, q: &m[j].old_saddr, |
4442 | size: sizeof(m[i].old_saddr)) && |
4443 | m[i].proto == m[j].proto && |
4444 | m[i].mode == m[j].mode && |
4445 | m[i].reqid == m[j].reqid && |
4446 | m[i].old_family == m[j].old_family) { |
4447 | NL_SET_ERR_MSG(extack, "Entries in the MIGRATE attribute's list must be unique" ); |
4448 | return -EINVAL; |
4449 | } |
4450 | } |
4451 | } |
4452 | |
4453 | return 0; |
4454 | } |
4455 | |
4456 | int xfrm_migrate(const struct xfrm_selector *sel, u8 dir, u8 type, |
4457 | struct xfrm_migrate *m, int num_migrate, |
4458 | struct xfrm_kmaddress *k, struct net *net, |
4459 | struct xfrm_encap_tmpl *encap, u32 if_id, |
4460 | struct netlink_ext_ack *extack) |
4461 | { |
4462 | int i, err, nx_cur = 0, nx_new = 0; |
4463 | struct xfrm_policy *pol = NULL; |
4464 | struct xfrm_state *x, *xc; |
4465 | struct xfrm_state *x_cur[XFRM_MAX_DEPTH]; |
4466 | struct xfrm_state *x_new[XFRM_MAX_DEPTH]; |
4467 | struct xfrm_migrate *mp; |
4468 | |
4469 | /* Stage 0 - sanity checks */ |
4470 | err = xfrm_migrate_check(m, num_migrate, extack); |
4471 | if (err < 0) |
4472 | goto out; |
4473 | |
4474 | if (dir >= XFRM_POLICY_MAX) { |
4475 | NL_SET_ERR_MSG(extack, "Invalid policy direction" ); |
4476 | err = -EINVAL; |
4477 | goto out; |
4478 | } |
4479 | |
4480 | /* Stage 1 - find policy */ |
4481 | pol = xfrm_migrate_policy_find(sel, dir, type, net, if_id); |
4482 | if (!pol) { |
4483 | NL_SET_ERR_MSG(extack, "Target policy not found" ); |
4484 | err = -ENOENT; |
4485 | goto out; |
4486 | } |
4487 | |
4488 | /* Stage 2 - find and update state(s) */ |
4489 | for (i = 0, mp = m; i < num_migrate; i++, mp++) { |
4490 | if ((x = xfrm_migrate_state_find(m: mp, net, if_id))) { |
4491 | x_cur[nx_cur] = x; |
4492 | nx_cur++; |
4493 | xc = xfrm_state_migrate(x, m: mp, encap); |
4494 | if (xc) { |
4495 | x_new[nx_new] = xc; |
4496 | nx_new++; |
4497 | } else { |
4498 | err = -ENODATA; |
4499 | goto restore_state; |
4500 | } |
4501 | } |
4502 | } |
4503 | |
4504 | /* Stage 3 - update policy */ |
4505 | err = xfrm_policy_migrate(pol, m, num_migrate, extack); |
4506 | if (err < 0) |
4507 | goto restore_state; |
4508 | |
4509 | /* Stage 4 - delete old state(s) */ |
4510 | if (nx_cur) { |
4511 | xfrm_states_put(states: x_cur, n: nx_cur); |
4512 | xfrm_states_delete(states: x_cur, n: nx_cur); |
4513 | } |
4514 | |
4515 | /* Stage 5 - announce */ |
4516 | km_migrate(sel, dir, type, m, num_bundles: num_migrate, k, encap); |
4517 | |
4518 | xfrm_pol_put(policy: pol); |
4519 | |
4520 | return 0; |
4521 | out: |
4522 | return err; |
4523 | |
4524 | restore_state: |
4525 | if (pol) |
4526 | xfrm_pol_put(policy: pol); |
4527 | if (nx_cur) |
4528 | xfrm_states_put(states: x_cur, n: nx_cur); |
4529 | if (nx_new) |
4530 | xfrm_states_delete(states: x_new, n: nx_new); |
4531 | |
4532 | return err; |
4533 | } |
4534 | EXPORT_SYMBOL(xfrm_migrate); |
4535 | #endif |
4536 | |