1	// SPDX-License-Identifier: GPL-2.0-only
2	#include <linux/types.h>
3	#include <linux/skbuff.h>
4	#include <linux/socket.h>
5	#include <linux/sysctl.h>
6	#include <linux/net.h>
7	#include <linux/module.h>
8	#include <linux/if_arp.h>
9	#include <linux/ipv6.h>
10	#include <linux/mpls.h>
11	#include <linux/netconf.h>
12	#include <linux/nospec.h>
13	#include <linux/vmalloc.h>
14	#include <linux/percpu.h>
15	#include <net/gso.h>
16	#include <net/ip.h>
17	#include <net/dst.h>
18	#include <net/sock.h>
19	#include <net/arp.h>
20	#include <net/ip_fib.h>
21	#include <net/netevent.h>
22	#include <net/ip_tunnels.h>
23	#include <net/netns/generic.h>
24	#if IS_ENABLED(CONFIG_IPV6)
25	#include <net/ipv6.h>
26	#endif
27	#include <net/ipv6_stubs.h>
28	#include <net/rtnh.h>
29	#include "internal.h"
30
31	/* max memory we will use for mpls_route */
32	#define MAX_MPLS_ROUTE_MEM 4096
33
34	/* Maximum number of labels to look ahead at when selecting a path of
35	* a multipath route
36	*/
37	#define MAX_MP_SELECT_LABELS 4
38
39	#define MPLS_NEIGH_TABLE_UNSPEC (NEIGH_LINK_TABLE + 1)
40
41	static int label_limit = (1 << 20) - 1;
42	static int ttl_max = 255;
43
44	#if IS_ENABLED(CONFIG_NET_IP_TUNNEL)
45	static size_t ipgre_mpls_encap_hlen(struct ip_tunnel_encap *e)
46	{
47	return sizeof(struct mpls_shim_hdr);
48	}
49
50	static const struct ip_tunnel_encap_ops mpls_iptun_ops = {
51	.encap_hlen = ipgre_mpls_encap_hlen,
52	};
53
54	static int ipgre_tunnel_encap_add_mpls_ops(void)
55	{
56	return ip_tunnel_encap_add_ops(op: &mpls_iptun_ops, num: TUNNEL_ENCAP_MPLS);
57	}
58
59	static void ipgre_tunnel_encap_del_mpls_ops(void)
60	{
61	ip_tunnel_encap_del_ops(op: &mpls_iptun_ops, num: TUNNEL_ENCAP_MPLS);
62	}
63	#else
64	static int ipgre_tunnel_encap_add_mpls_ops(void)
65	{
66	return 0;
67	}
68
69	static void ipgre_tunnel_encap_del_mpls_ops(void)
70	{
71	}
72	#endif
73
74	static void rtmsg_lfib(int event, u32 label, struct mpls_route *rt,
75	struct nlmsghdr *nlh, struct net *net, u32 portid,
76	unsigned int nlm_flags);
77
78	static struct mpls_route *mpls_route_input_rcu(struct net *net, unsigned index)
79	{
80	struct mpls_route *rt = NULL;
81
82	if (index < net->mpls.platform_labels) {
83	struct mpls_route __rcu **platform_label =
84	rcu_dereference(net->mpls.platform_label);
85	rt = rcu_dereference(platform_label[index]);
86	}
87	return rt;
88	}
89
90	bool mpls_output_possible(const struct net_device *dev)
91	{
92	return dev && (dev->flags & IFF_UP) && netif_carrier_ok(dev);
93	}
94	EXPORT_SYMBOL_GPL(mpls_output_possible);
95
96	static u8 *__mpls_nh_via(struct mpls_route *rt, struct mpls_nh *nh)
97	{
98	return (u8 *)nh + rt->rt_via_offset;
99	}
100
101	static const u8 *mpls_nh_via(const struct mpls_route *rt,
102	const struct mpls_nh *nh)
103	{
104	return __mpls_nh_via(rt: (struct mpls_route *)rt, nh: (struct mpls_nh *)nh);
105	}
106
107	static unsigned int mpls_nh_header_size(const struct mpls_nh *nh)
108	{
109	/* The size of the layer 2.5 labels to be added for this route */
110	return nh->nh_labels * sizeof(struct mpls_shim_hdr);
111	}
112
113	unsigned int mpls_dev_mtu(const struct net_device *dev)
114	{
115	/* The amount of data the layer 2 frame can hold */
116	return dev->mtu;
117	}
118	EXPORT_SYMBOL_GPL(mpls_dev_mtu);
119
120	bool mpls_pkt_too_big(const struct sk_buff *skb, unsigned int mtu)
121	{
122	if (skb->len <= mtu)
123	return false;
124
125	if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu))
126	return false;
127
128	return true;
129	}
130	EXPORT_SYMBOL_GPL(mpls_pkt_too_big);
131
132	void mpls_stats_inc_outucastpkts(struct net_device *dev,
133	const struct sk_buff *skb)
134	{
135	struct mpls_dev *mdev;
136
137	if (skb->protocol == htons(ETH_P_MPLS_UC)) {
138	mdev = mpls_dev_get(dev);
139	if (mdev)
140	MPLS_INC_STATS_LEN(mdev, skb->len,
141	tx_packets,
142	tx_bytes);
143	} else if (skb->protocol == htons(ETH_P_IP)) {
144	IP_UPD_PO_STATS(dev_net(dev), IPSTATS_MIB_OUT, skb->len);
145	#if IS_ENABLED(CONFIG_IPV6)
146	} else if (skb->protocol == htons(ETH_P_IPV6)) {
147	struct inet6_dev *in6dev = __in6_dev_get(dev);
148
149	if (in6dev)
150	IP6_UPD_PO_STATS(dev_net(dev), in6dev,
151	IPSTATS_MIB_OUT, skb->len);
152	#endif
153	}
154	}
155	EXPORT_SYMBOL_GPL(mpls_stats_inc_outucastpkts);
156
157	static u32 mpls_multipath_hash(struct mpls_route *rt, struct sk_buff *skb)
158	{
159	struct mpls_entry_decoded dec;
160	unsigned int mpls_hdr_len = 0;
161	struct mpls_shim_hdr *hdr;
162	bool eli_seen = false;
163	int label_index;
164	u32 hash = 0;
165
166	for (label_index = 0; label_index < MAX_MP_SELECT_LABELS;
167	label_index++) {
168	mpls_hdr_len += sizeof(*hdr);
169	if (!pskb_may_pull(skb, len: mpls_hdr_len))
170	break;
171
172	/* Read and decode the current label */
173	hdr = mpls_hdr(skb) + label_index;
174	dec = mpls_entry_decode(hdr);
175
176	/* RFC6790 - reserved labels MUST NOT be used as keys
177	* for the load-balancing function
178	*/
179	if (likely(dec.label >= MPLS_LABEL_FIRST_UNRESERVED)) {
180	hash = jhash_1word(a: dec.label, initval: hash);
181
182	/* The entropy label follows the entropy label
183	* indicator, so this means that the entropy
184	* label was just added to the hash - no need to
185	* go any deeper either in the label stack or in the
186	* payload
187	*/
188	if (eli_seen)
189	break;
190	} else if (dec.label == MPLS_LABEL_ENTROPY) {
191	eli_seen = true;
192	}
193
194	if (!dec.bos)
195	continue;
196
197	/* found bottom label; does skb have room for a header? */
198	if (pskb_may_pull(skb, len: mpls_hdr_len + sizeof(struct iphdr))) {
199	const struct iphdr *v4hdr;
200
201	v4hdr = (const struct iphdr *)(hdr + 1);
202	if (v4hdr->version == 4) {
203	hash = jhash_3words(ntohl(v4hdr->saddr),
204	ntohl(v4hdr->daddr),
205	c: v4hdr->protocol, initval: hash);
206	} else if (v4hdr->version == 6 &&
207	pskb_may_pull(skb, len: mpls_hdr_len +
208	sizeof(struct ipv6hdr))) {
209	const struct ipv6hdr *v6hdr;
210
211	v6hdr = (const struct ipv6hdr *)(hdr + 1);
212	hash = __ipv6_addr_jhash(a: &v6hdr->saddr, initval: hash);
213	hash = __ipv6_addr_jhash(a: &v6hdr->daddr, initval: hash);
214	hash = jhash_1word(a: v6hdr->nexthdr, initval: hash);
215	}
216	}
217
218	break;
219	}
220
221	return hash;
222	}
223
224	static struct mpls_nh *mpls_get_nexthop(struct mpls_route *rt, u8 index)
225	{
226	return (struct mpls_nh *)((u8 *)rt->rt_nh + index * rt->rt_nh_size);
227	}
228
229	/* number of alive nexthops (rt->rt_nhn_alive) and the flags for
230	* a next hop (nh->nh_flags) are modified by netdev event handlers.
231	* Since those fields can change at any moment, use READ_ONCE to
232	* access both.
233	*/
234	static const struct mpls_nh *mpls_select_multipath(struct mpls_route *rt,
235	struct sk_buff *skb)
236	{
237	u32 hash = 0;
238	int nh_index = 0;
239	int n = 0;
240	u8 alive;
241
242	/* No need to look further into packet if there's only
243	* one path
244	*/
245	if (rt->rt_nhn == 1)
246	return rt->rt_nh;
247
248	alive = READ_ONCE(rt->rt_nhn_alive);
249	if (alive == 0)
250	return NULL;
251
252	hash = mpls_multipath_hash(rt, skb);
253	nh_index = hash % alive;
254	if (alive == rt->rt_nhn)
255	goto out;
256	for_nexthops(rt) {
257	unsigned int nh_flags = READ_ONCE(nh->nh_flags);
258
259	if (nh_flags & (RTNH_F_DEAD | RTNH_F_LINKDOWN))
260	continue;
261	if (n == nh_index)
262	return nh;
263	n++;
264	} endfor_nexthops(rt);
265
266	out:
267	return mpls_get_nexthop(rt, index: nh_index);
268	}
269
270	static bool mpls_egress(struct net *net, struct mpls_route *rt,
271	struct sk_buff *skb, struct mpls_entry_decoded dec)
272	{
273	enum mpls_payload_type payload_type;
274	bool success = false;
275
276	/* The IPv4 code below accesses through the IPv4 header
277	* checksum, which is 12 bytes into the packet.
278	* The IPv6 code below accesses through the IPv6 hop limit
279	* which is 8 bytes into the packet.
280	*
281	* For all supported cases there should always be at least 12
282	* bytes of packet data present. The IPv4 header is 20 bytes
283	* without options and the IPv6 header is always 40 bytes
284	* long.
285	*/
286	if (!pskb_may_pull(skb, len: 12))
287	return false;
288
289	payload_type = rt->rt_payload_type;
290	if (payload_type == MPT_UNSPEC)
291	payload_type = ip_hdr(skb)->version;
292
293	switch (payload_type) {
294	case MPT_IPV4: {
295	struct iphdr *hdr4 = ip_hdr(skb);
296	u8 new_ttl;
297	skb->protocol = htons(ETH_P_IP);
298
299	/* If propagating TTL, take the decremented TTL from
300	* the incoming MPLS header, otherwise decrement the
301	* TTL, but only if not 0 to avoid underflow.
302	*/
303	if (rt->rt_ttl_propagate == MPLS_TTL_PROP_ENABLED ||
304	(rt->rt_ttl_propagate == MPLS_TTL_PROP_DEFAULT &&
305	net->mpls.ip_ttl_propagate))
306	new_ttl = dec.ttl;
307	else
308	new_ttl = hdr4->ttl ? hdr4->ttl - 1 : 0;
309
310	csum_replace2(sum: &hdr4->check,
311	htons(hdr4->ttl << 8),
312	htons(new_ttl << 8));
313	hdr4->ttl = new_ttl;
314	success = true;
315	break;
316	}
317	case MPT_IPV6: {
318	struct ipv6hdr *hdr6 = ipv6_hdr(skb);
319	skb->protocol = htons(ETH_P_IPV6);
320
321	/* If propagating TTL, take the decremented TTL from
322	* the incoming MPLS header, otherwise decrement the
323	* hop limit, but only if not 0 to avoid underflow.
324	*/
325	if (rt->rt_ttl_propagate == MPLS_TTL_PROP_ENABLED ||
326	(rt->rt_ttl_propagate == MPLS_TTL_PROP_DEFAULT &&
327	net->mpls.ip_ttl_propagate))
328	hdr6->hop_limit = dec.ttl;
329	else if (hdr6->hop_limit)
330	hdr6->hop_limit = hdr6->hop_limit - 1;
331	success = true;
332	break;
333	}
334	case MPT_UNSPEC:
335	/* Should have decided which protocol it is by now */
336	break;
337	}
338
339	return success;
340	}
341
342	static int mpls_forward(struct sk_buff *skb, struct net_device *dev,
343	struct packet_type *pt, struct net_device *orig_dev)
344	{
345	struct net *net = dev_net(dev);
346	struct mpls_shim_hdr *hdr;
347	const struct mpls_nh *nh;
348	struct mpls_route *rt;
349	struct mpls_entry_decoded dec;
350	struct net_device *out_dev;
351	struct mpls_dev *out_mdev;
352	struct mpls_dev *mdev;
353	unsigned int hh_len;
354	unsigned int new_header_size;
355	unsigned int mtu;
356	int err;
357
358	/* Careful this entire function runs inside of an rcu critical section */
359
360	mdev = mpls_dev_get(dev);
361	if (!mdev)
362	goto drop;
363
364	MPLS_INC_STATS_LEN(mdev, skb->len, rx_packets,
365	rx_bytes);
366
367	if (!mdev->input_enabled) {
368	MPLS_INC_STATS(mdev, rx_dropped);
369	goto drop;
370	}
371
372	if (skb->pkt_type != PACKET_HOST)
373	goto err;
374
375	if ((skb = skb_share_check(skb, GFP_ATOMIC)) == NULL)
376	goto err;
377
378	if (!pskb_may_pull(skb, len: sizeof(*hdr)))
379	goto err;
380
381	skb_dst_drop(skb);
382
383	/* Read and decode the label */
384	hdr = mpls_hdr(skb);
385	dec = mpls_entry_decode(hdr);
386
387	rt = mpls_route_input_rcu(net, index: dec.label);
388	if (!rt) {
389	MPLS_INC_STATS(mdev, rx_noroute);
390	goto drop;
391	}
392
393	nh = mpls_select_multipath(rt, skb);
394	if (!nh)
395	goto err;
396
397	/* Pop the label */
398	skb_pull(skb, len: sizeof(*hdr));
399	skb_reset_network_header(skb);
400
401	skb_orphan(skb);
402
403	if (skb_warn_if_lro(skb))
404	goto err;
405
406	skb_forward_csum(skb);
407
408	/* Verify ttl is valid */
409	if (dec.ttl <= 1)
410	goto err;
411
412	/* Find the output device */
413	out_dev = nh->nh_dev;
414	if (!mpls_output_possible(out_dev))
415	goto tx_err;
416
417	/* Verify the destination can hold the packet */
418	new_header_size = mpls_nh_header_size(nh);
419	mtu = mpls_dev_mtu(out_dev);
420	if (mpls_pkt_too_big(skb, mtu - new_header_size))
421	goto tx_err;
422
423	hh_len = LL_RESERVED_SPACE(out_dev);
424	if (!out_dev->header_ops)
425	hh_len = 0;
426
427	/* Ensure there is enough space for the headers in the skb */
428	if (skb_cow(skb, headroom: hh_len + new_header_size))
429	goto tx_err;
430
431	skb->dev = out_dev;
432	skb->protocol = htons(ETH_P_MPLS_UC);
433
434	dec.ttl -= 1;
435	if (unlikely(!new_header_size && dec.bos)) {
436	/* Penultimate hop popping */
437	if (!mpls_egress(net: dev_net(dev: out_dev), rt, skb, dec))
438	goto err;
439	} else {
440	bool bos;
441	int i;
442	skb_push(skb, len: new_header_size);
443	skb_reset_network_header(skb);
444	/* Push the new labels */
445	hdr = mpls_hdr(skb);
446	bos = dec.bos;
447	for (i = nh->nh_labels - 1; i >= 0; i--) {
448	hdr[i] = mpls_entry_encode(label: nh->nh_label[i],
449	ttl: dec.ttl, tc: 0, bos);
450	bos = false;
451	}
452	}
453
454	mpls_stats_inc_outucastpkts(out_dev, skb);
455
456	/* If via wasn't specified then send out using device address */
457	if (nh->nh_via_table == MPLS_NEIGH_TABLE_UNSPEC)
458	err = neigh_xmit(fam: NEIGH_LINK_TABLE, out_dev,
459	out_dev->dev_addr, skb);
460	else
461	err = neigh_xmit(fam: nh->nh_via_table, out_dev,
462	mpls_nh_via(rt, nh), skb);
463	if (err)
464	net_dbg_ratelimited("%s: packet transmission failed: %d\n",
465	__func__, err);
466	return 0;
467
468	tx_err:
469	out_mdev = out_dev ? mpls_dev_get(dev: out_dev) : NULL;
470	if (out_mdev)
471	MPLS_INC_STATS(out_mdev, tx_errors);
472	goto drop;
473	err:
474	MPLS_INC_STATS(mdev, rx_errors);
475	drop:
476	kfree_skb(skb);
477	return NET_RX_DROP;
478	}
479
480	static struct packet_type mpls_packet_type __read_mostly = {
481	.type = cpu_to_be16(ETH_P_MPLS_UC),
482	.func = mpls_forward,
483	};
484
485	static const struct nla_policy rtm_mpls_policy[RTA_MAX+1] = {
486	[RTA_DST] = { .type = NLA_U32 },
487	[RTA_OIF] = { .type = NLA_U32 },
488	[RTA_TTL_PROPAGATE] = { .type = NLA_U8 },
489	};
490
491	struct mpls_route_config {
492	u32 rc_protocol;
493	u32 rc_ifindex;
494	u8 rc_via_table;
495	u8 rc_via_alen;
496	u8 rc_via[MAX_VIA_ALEN];
497	u32 rc_label;
498	u8 rc_ttl_propagate;
499	u8 rc_output_labels;
500	u32 rc_output_label[MAX_NEW_LABELS];
501	u32 rc_nlflags;
502	enum mpls_payload_type rc_payload_type;
503	struct nl_info rc_nlinfo;
504	struct rtnexthop *rc_mp;
505	int rc_mp_len;
506	};
507
508	/* all nexthops within a route have the same size based on max
509	* number of labels and max via length for a hop
510	*/
511	static struct mpls_route *mpls_rt_alloc(u8 num_nh, u8 max_alen, u8 max_labels)
512	{
513	u8 nh_size = MPLS_NH_SIZE(max_labels, max_alen);
514	struct mpls_route *rt;
515	size_t size;
516
517	size = sizeof(*rt) + num_nh * nh_size;
518	if (size > MAX_MPLS_ROUTE_MEM)
519	return ERR_PTR(error: -EINVAL);
520
521	rt = kzalloc(size, GFP_KERNEL);
522	if (!rt)
523	return ERR_PTR(error: -ENOMEM);
524
525	rt->rt_nhn = num_nh;
526	rt->rt_nhn_alive = num_nh;
527	rt->rt_nh_size = nh_size;
528	rt->rt_via_offset = MPLS_NH_VIA_OFF(max_labels);
529
530	return rt;
531	}
532
533	static void mpls_rt_free(struct mpls_route *rt)
534	{
535	if (rt)
536	kfree_rcu(rt, rt_rcu);
537	}
538
539	static void mpls_notify_route(struct net *net, unsigned index,
540	struct mpls_route *old, struct mpls_route *new,
541	const struct nl_info *info)
542	{
543	struct nlmsghdr *nlh = info ? info->nlh : NULL;
544	unsigned portid = info ? info->portid : 0;
545	int event = new ? RTM_NEWROUTE : RTM_DELROUTE;
546	struct mpls_route *rt = new ? new : old;
547	unsigned nlm_flags = (old && new) ? NLM_F_REPLACE : 0;
548	/* Ignore reserved labels for now */
549	if (rt && (index >= MPLS_LABEL_FIRST_UNRESERVED))
550	rtmsg_lfib(event, label: index, rt, nlh, net, portid, nlm_flags);
551	}
552
553	static void mpls_route_update(struct net *net, unsigned index,
554	struct mpls_route *new,
555	const struct nl_info *info)
556	{
557	struct mpls_route __rcu **platform_label;
558	struct mpls_route *rt;
559
560	ASSERT_RTNL();
561
562	platform_label = rtnl_dereference(net->mpls.platform_label);
563	rt = rtnl_dereference(platform_label[index]);
564	rcu_assign_pointer(platform_label[index], new);
565
566	mpls_notify_route(net, index, old: rt, new, info);
567
568	/* If we removed a route free it now */
569	mpls_rt_free(rt);
570	}
571
572	static unsigned find_free_label(struct net *net)
573	{
574	struct mpls_route __rcu **platform_label;
575	size_t platform_labels;
576	unsigned index;
577
578	platform_label = rtnl_dereference(net->mpls.platform_label);
579	platform_labels = net->mpls.platform_labels;
580	for (index = MPLS_LABEL_FIRST_UNRESERVED; index < platform_labels;
581	index++) {
582	if (!rtnl_dereference(platform_label[index]))
583	return index;
584	}
585	return LABEL_NOT_SPECIFIED;
586	}
587
588	#if IS_ENABLED(CONFIG_INET)
589	static struct net_device *inet_fib_lookup_dev(struct net *net,
590	const void *addr)
591	{
592	struct net_device *dev;
593	struct rtable *rt;
594	struct in_addr daddr;
595
596	memcpy(&daddr, addr, sizeof(struct in_addr));
597	rt = ip_route_output(net, daddr: daddr.s_addr, saddr: 0, tos: 0, oif: 0);
598	if (IS_ERR(ptr: rt))
599	return ERR_CAST(ptr: rt);
600
601	dev = rt->dst.dev;
602	dev_hold(dev);
603
604	ip_rt_put(rt);
605
606	return dev;
607	}
608	#else
609	static struct net_device *inet_fib_lookup_dev(struct net *net,
610	const void *addr)
611	{
612	return ERR_PTR(-EAFNOSUPPORT);
613	}
614	#endif
615
616	#if IS_ENABLED(CONFIG_IPV6)
617	static struct net_device *inet6_fib_lookup_dev(struct net *net,
618	const void *addr)
619	{
620	struct net_device *dev;
621	struct dst_entry *dst;
622	struct flowi6 fl6;
623
624	if (!ipv6_stub)
625	return ERR_PTR(error: -EAFNOSUPPORT);
626
627	memset(&fl6, 0, sizeof(fl6));
628	memcpy(&fl6.daddr, addr, sizeof(struct in6_addr));
629	dst = ipv6_stub->ipv6_dst_lookup_flow(net, NULL, &fl6, NULL);
630	if (IS_ERR(ptr: dst))
631	return ERR_CAST(ptr: dst);
632
633	dev = dst->dev;
634	dev_hold(dev);
635	dst_release(dst);
636
637	return dev;
638	}
639	#else
640	static struct net_device *inet6_fib_lookup_dev(struct net *net,
641	const void *addr)
642	{
643	return ERR_PTR(-EAFNOSUPPORT);
644	}
645	#endif
646
647	static struct net_device *find_outdev(struct net *net,
648	struct mpls_route *rt,
649	struct mpls_nh *nh, int oif)
650	{
651	struct net_device *dev = NULL;
652
653	if (!oif) {
654	switch (nh->nh_via_table) {
655	case NEIGH_ARP_TABLE:
656	dev = inet_fib_lookup_dev(net, addr: mpls_nh_via(rt, nh));
657	break;
658	case NEIGH_ND_TABLE:
659	dev = inet6_fib_lookup_dev(net, addr: mpls_nh_via(rt, nh));
660	break;
661	case NEIGH_LINK_TABLE:
662	break;
663	}
664	} else {
665	dev = dev_get_by_index(net, ifindex: oif);
666	}
667
668	if (!dev)
669	return ERR_PTR(error: -ENODEV);
670
671	if (IS_ERR(ptr: dev))
672	return dev;
673
674	/* The caller is holding rtnl anyways, so release the dev reference */
675	dev_put(dev);
676
677	return dev;
678	}
679
680	static int mpls_nh_assign_dev(struct net *net, struct mpls_route *rt,
681	struct mpls_nh *nh, int oif)
682	{
683	struct net_device *dev = NULL;
684	int err = -ENODEV;
685
686	dev = find_outdev(net, rt, nh, oif);
687	if (IS_ERR(ptr: dev)) {
688	err = PTR_ERR(ptr: dev);
689	dev = NULL;
690	goto errout;
691	}
692
693	/* Ensure this is a supported device */
694	err = -EINVAL;
695	if (!mpls_dev_get(dev))
696	goto errout;
697
698	if ((nh->nh_via_table == NEIGH_LINK_TABLE) &&
699	(dev->addr_len != nh->nh_via_alen))
700	goto errout;
701
702	nh->nh_dev = dev;
703
704	if (!(dev->flags & IFF_UP)) {
705	nh->nh_flags |= RTNH_F_DEAD;
706	} else {
707	unsigned int flags;
708
709	flags = dev_get_flags(dev);
710	if (!(flags & (IFF_RUNNING | IFF_LOWER_UP)))
711	nh->nh_flags |= RTNH_F_LINKDOWN;
712	}
713
714	return 0;
715
716	errout:
717	return err;
718	}
719
720	static int nla_get_via(const struct nlattr *nla, u8 *via_alen, u8 *via_table,
721	u8 via_addr[], struct netlink_ext_ack *extack)
722	{
723	struct rtvia *via = nla_data(nla);
724	int err = -EINVAL;
725	int alen;
726
727	if (nla_len(nla) < offsetof(struct rtvia, rtvia_addr)) {
728	NL_SET_ERR_MSG_ATTR(extack, nla,
729	"Invalid attribute length for RTA_VIA");
730	goto errout;
731	}
732	alen = nla_len(nla) -
733	offsetof(struct rtvia, rtvia_addr);
734	if (alen > MAX_VIA_ALEN) {
735	NL_SET_ERR_MSG_ATTR(extack, nla,
736	"Invalid address length for RTA_VIA");
737	goto errout;
738	}
739
740	/* Validate the address family */
741	switch (via->rtvia_family) {
742	case AF_PACKET:
743	*via_table = NEIGH_LINK_TABLE;
744	break;
745	case AF_INET:
746	*via_table = NEIGH_ARP_TABLE;
747	if (alen != 4)
748	goto errout;
749	break;
750	case AF_INET6:
751	*via_table = NEIGH_ND_TABLE;
752	if (alen != 16)
753	goto errout;
754	break;
755	default:
756	/* Unsupported address family */
757	goto errout;
758	}
759
760	memcpy(via_addr, via->rtvia_addr, alen);
761	*via_alen = alen;
762	err = 0;
763
764	errout:
765	return err;
766	}
767
768	static int mpls_nh_build_from_cfg(struct mpls_route_config *cfg,
769	struct mpls_route *rt)
770	{
771	struct net *net = cfg->rc_nlinfo.nl_net;
772	struct mpls_nh *nh = rt->rt_nh;
773	int err;
774	int i;
775
776	if (!nh)
777	return -ENOMEM;
778
779	nh->nh_labels = cfg->rc_output_labels;
780	for (i = 0; i < nh->nh_labels; i++)
781	nh->nh_label[i] = cfg->rc_output_label[i];
782
783	nh->nh_via_table = cfg->rc_via_table;
784	memcpy(__mpls_nh_via(rt, nh), cfg->rc_via, cfg->rc_via_alen);
785	nh->nh_via_alen = cfg->rc_via_alen;
786
787	err = mpls_nh_assign_dev(net, rt, nh, oif: cfg->rc_ifindex);
788	if (err)
789	goto errout;
790
791	if (nh->nh_flags & (RTNH_F_DEAD | RTNH_F_LINKDOWN))
792	rt->rt_nhn_alive--;
793
794	return 0;
795
796	errout:
797	return err;
798	}
799
800	static int mpls_nh_build(struct net *net, struct mpls_route *rt,
801	struct mpls_nh *nh, int oif, struct nlattr *via,
802	struct nlattr *newdst, u8 max_labels,
803	struct netlink_ext_ack *extack)
804	{
805	int err = -ENOMEM;
806
807	if (!nh)
808	goto errout;
809
810	if (newdst) {
811	err = nla_get_labels(nla: newdst, max_labels, labels: &nh->nh_labels,
812	label: nh->nh_label, extack);
813	if (err)
814	goto errout;
815	}
816
817	if (via) {
818	err = nla_get_via(nla: via, via_alen: &nh->nh_via_alen, via_table: &nh->nh_via_table,
819	via_addr: __mpls_nh_via(rt, nh), extack);
820	if (err)
821	goto errout;
822	} else {
823	nh->nh_via_table = MPLS_NEIGH_TABLE_UNSPEC;
824	}
825
826	err = mpls_nh_assign_dev(net, rt, nh, oif);
827	if (err)
828	goto errout;
829
830	return 0;
831
832	errout:
833	return err;
834	}
835
836	static u8 mpls_count_nexthops(struct rtnexthop *rtnh, int len,
837	u8 cfg_via_alen, u8 *max_via_alen,
838	u8 *max_labels)
839	{
840	int remaining = len;
841	u8 nhs = 0;
842
843	*max_via_alen = 0;
844	*max_labels = 0;
845
846	while (rtnh_ok(rtnh, remaining)) {
847	struct nlattr *nla, *attrs = rtnh_attrs(rtnh);
848	int attrlen;
849	u8 n_labels = 0;
850
851	attrlen = rtnh_attrlen(rtnh);
852	nla = nla_find(head: attrs, len: attrlen, attrtype: RTA_VIA);
853	if (nla && nla_len(nla) >=
854	offsetof(struct rtvia, rtvia_addr)) {
855	int via_alen = nla_len(nla) -
856	offsetof(struct rtvia, rtvia_addr);
857
858	if (via_alen <= MAX_VIA_ALEN)
859	*max_via_alen = max_t(u16, *max_via_alen,
860	via_alen);
861	}
862
863	nla = nla_find(head: attrs, len: attrlen, attrtype: RTA_NEWDST);
864	if (nla &&
865	nla_get_labels(nla, MAX_NEW_LABELS, labels: &n_labels,
866	NULL, NULL) != 0)
867	return 0;
868
869	*max_labels = max_t(u8, *max_labels, n_labels);
870
871	/* number of nexthops is tracked by a u8.
872	* Check for overflow.
873	*/
874	if (nhs == 255)
875	return 0;
876	nhs++;
877
878	rtnh = rtnh_next(rtnh, remaining: &remaining);
879	}
880
881	/* leftover implies invalid nexthop configuration, discard it */
882	return remaining > 0 ? 0 : nhs;
883	}
884
885	static int mpls_nh_build_multi(struct mpls_route_config *cfg,
886	struct mpls_route *rt, u8 max_labels,
887	struct netlink_ext_ack *extack)
888	{
889	struct rtnexthop *rtnh = cfg->rc_mp;
890	struct nlattr *nla_via, *nla_newdst;
891	int remaining = cfg->rc_mp_len;
892	int err = 0;
893	u8 nhs = 0;
894
895	change_nexthops(rt) {
896	int attrlen;
897
898	nla_via = NULL;
899	nla_newdst = NULL;
900
901	err = -EINVAL;
902	if (!rtnh_ok(rtnh, remaining))
903	goto errout;
904
905	/* neither weighted multipath nor any flags
906	* are supported
907	*/
908	if (rtnh->rtnh_hops || rtnh->rtnh_flags)
909	goto errout;
910
911	attrlen = rtnh_attrlen(rtnh);
912	if (attrlen > 0) {
913	struct nlattr *attrs = rtnh_attrs(rtnh);
914
915	nla_via = nla_find(head: attrs, len: attrlen, attrtype: RTA_VIA);
916	nla_newdst = nla_find(head: attrs, len: attrlen, attrtype: RTA_NEWDST);
917	}
918
919	err = mpls_nh_build(net: cfg->rc_nlinfo.nl_net, rt, nh,
920	oif: rtnh->rtnh_ifindex, via: nla_via, newdst: nla_newdst,
921	max_labels, extack);
922	if (err)
923	goto errout;
924
925	if (nh->nh_flags & (RTNH_F_DEAD | RTNH_F_LINKDOWN))
926	rt->rt_nhn_alive--;
927
928	rtnh = rtnh_next(rtnh, remaining: &remaining);
929	nhs++;
930	} endfor_nexthops(rt);
931
932	rt->rt_nhn = nhs;
933
934	return 0;
935
936	errout:
937	return err;
938	}
939
940	static bool mpls_label_ok(struct net *net, unsigned int *index,
941	struct netlink_ext_ack *extack)
942	{
943	bool is_ok = true;
944
945	/* Reserved labels may not be set */
946	if (*index < MPLS_LABEL_FIRST_UNRESERVED) {
947	NL_SET_ERR_MSG(extack,
948	"Invalid label - must be MPLS_LABEL_FIRST_UNRESERVED or higher");
949	is_ok = false;
950	}
951
952	/* The full 20 bit range may not be supported. */
953	if (is_ok && *index >= net->mpls.platform_labels) {
954	NL_SET_ERR_MSG(extack,
955	"Label >= configured maximum in platform_labels");
956	is_ok = false;
957	}
958
959	*index = array_index_nospec(*index, net->mpls.platform_labels);
960	return is_ok;
961	}
962
963	static int mpls_route_add(struct mpls_route_config *cfg,
964	struct netlink_ext_ack *extack)
965	{
966	struct mpls_route __rcu **platform_label;
967	struct net *net = cfg->rc_nlinfo.nl_net;
968	struct mpls_route *rt, *old;
969	int err = -EINVAL;
970	u8 max_via_alen;
971	unsigned index;
972	u8 max_labels;
973	u8 nhs;
974
975	index = cfg->rc_label;
976
977	/* If a label was not specified during insert pick one */
978	if ((index == LABEL_NOT_SPECIFIED) &&
979	(cfg->rc_nlflags & NLM_F_CREATE)) {
980	index = find_free_label(net);
981	}
982
983	if (!mpls_label_ok(net, index: &index, extack))
984	goto errout;
985
986	/* Append makes no sense with mpls */
987	err = -EOPNOTSUPP;
988	if (cfg->rc_nlflags & NLM_F_APPEND) {
989	NL_SET_ERR_MSG(extack, "MPLS does not support route append");
990	goto errout;
991	}
992
993	err = -EEXIST;
994	platform_label = rtnl_dereference(net->mpls.platform_label);
995	old = rtnl_dereference(platform_label[index]);
996	if ((cfg->rc_nlflags & NLM_F_EXCL) && old)
997	goto errout;
998
999	err = -EEXIST;
1000	if (!(cfg->rc_nlflags & NLM_F_REPLACE) && old)
1001	goto errout;
1002
1003	err = -ENOENT;
1004	if (!(cfg->rc_nlflags & NLM_F_CREATE) && !old)
1005	goto errout;
1006
1007	err = -EINVAL;
1008	if (cfg->rc_mp) {
1009	nhs = mpls_count_nexthops(rtnh: cfg->rc_mp, len: cfg->rc_mp_len,
1010	cfg_via_alen: cfg->rc_via_alen, max_via_alen: &max_via_alen,
1011	max_labels: &max_labels);
1012	} else {
1013	max_via_alen = cfg->rc_via_alen;
1014	max_labels = cfg->rc_output_labels;
1015	nhs = 1;
1016	}
1017
1018	if (nhs == 0) {
1019	NL_SET_ERR_MSG(extack, "Route does not contain a nexthop");
1020	goto errout;
1021	}
1022
1023	rt = mpls_rt_alloc(num_nh: nhs, max_alen: max_via_alen, max_labels);
1024	if (IS_ERR(ptr: rt)) {
1025	err = PTR_ERR(ptr: rt);
1026	goto errout;
1027	}
1028
1029	rt->rt_protocol = cfg->rc_protocol;
1030	rt->rt_payload_type = cfg->rc_payload_type;
1031	rt->rt_ttl_propagate = cfg->rc_ttl_propagate;
1032
1033	if (cfg->rc_mp)
1034	err = mpls_nh_build_multi(cfg, rt, max_labels, extack);
1035	else
1036	err = mpls_nh_build_from_cfg(cfg, rt);
1037	if (err)
1038	goto freert;
1039
1040	mpls_route_update(net, index, new: rt, info: &cfg->rc_nlinfo);
1041
1042	return 0;
1043
1044	freert:
1045	mpls_rt_free(rt);
1046	errout:
1047	return err;
1048	}
1049
1050	static int mpls_route_del(struct mpls_route_config *cfg,
1051	struct netlink_ext_ack *extack)
1052	{
1053	struct net *net = cfg->rc_nlinfo.nl_net;
1054	unsigned index;
1055	int err = -EINVAL;
1056
1057	index = cfg->rc_label;
1058
1059	if (!mpls_label_ok(net, index: &index, extack))
1060	goto errout;
1061
1062	mpls_route_update(net, index, NULL, info: &cfg->rc_nlinfo);
1063
1064	err = 0;
1065	errout:
1066	return err;
1067	}
1068
1069	static void mpls_get_stats(struct mpls_dev *mdev,
1070	struct mpls_link_stats *stats)
1071	{
1072	struct mpls_pcpu_stats *p;
1073	int i;
1074
1075	memset(stats, 0, sizeof(*stats));
1076
1077	for_each_possible_cpu(i) {
1078	struct mpls_link_stats local;
1079	unsigned int start;
1080
1081	p = per_cpu_ptr(mdev->stats, i);
1082	do {
1083	start = u64_stats_fetch_begin(syncp: &p->syncp);
1084	local = p->stats;
1085	} while (u64_stats_fetch_retry(syncp: &p->syncp, start));
1086
1087	stats->rx_packets += local.rx_packets;
1088	stats->rx_bytes += local.rx_bytes;
1089	stats->tx_packets += local.tx_packets;
1090	stats->tx_bytes += local.tx_bytes;
1091	stats->rx_errors += local.rx_errors;
1092	stats->tx_errors += local.tx_errors;
1093	stats->rx_dropped += local.rx_dropped;
1094	stats->tx_dropped += local.tx_dropped;
1095	stats->rx_noroute += local.rx_noroute;
1096	}
1097	}
1098
1099	static int mpls_fill_stats_af(struct sk_buff *skb,
1100	const struct net_device *dev)
1101	{
1102	struct mpls_link_stats *stats;
1103	struct mpls_dev *mdev;
1104	struct nlattr *nla;
1105
1106	mdev = mpls_dev_get(dev);
1107	if (!mdev)
1108	return -ENODATA;
1109
1110	nla = nla_reserve_64bit(skb, attrtype: MPLS_STATS_LINK,
1111	attrlen: sizeof(struct mpls_link_stats),
1112	padattr: MPLS_STATS_UNSPEC);
1113	if (!nla)
1114	return -EMSGSIZE;
1115
1116	stats = nla_data(nla);
1117	mpls_get_stats(mdev, stats);
1118
1119	return 0;
1120	}
1121
1122	static size_t mpls_get_stats_af_size(const struct net_device *dev)
1123	{
1124	struct mpls_dev *mdev;
1125
1126	mdev = mpls_dev_get(dev);
1127	if (!mdev)
1128	return 0;
1129
1130	return nla_total_size_64bit(payload: sizeof(struct mpls_link_stats));
1131	}
1132
1133	static int mpls_netconf_fill_devconf(struct sk_buff *skb, struct mpls_dev *mdev,
1134	u32 portid, u32 seq, int event,
1135	unsigned int flags, int type)
1136	{
1137	struct nlmsghdr *nlh;
1138	struct netconfmsg *ncm;
1139	bool all = false;
1140
1141	nlh = nlmsg_put(skb, portid, seq, type: event, payload: sizeof(struct netconfmsg),
1142	flags);
1143	if (!nlh)
1144	return -EMSGSIZE;
1145
1146	if (type == NETCONFA_ALL)
1147	all = true;
1148
1149	ncm = nlmsg_data(nlh);
1150	ncm->ncm_family = AF_MPLS;
1151
1152	if (nla_put_s32(skb, attrtype: NETCONFA_IFINDEX, value: mdev->dev->ifindex) < 0)
1153	goto nla_put_failure;
1154
1155	if ((all || type == NETCONFA_INPUT) &&
1156	nla_put_s32(skb, attrtype: NETCONFA_INPUT,
1157	value: mdev->input_enabled) < 0)
1158	goto nla_put_failure;
1159
1160	nlmsg_end(skb, nlh);
1161	return 0;
1162
1163	nla_put_failure:
1164	nlmsg_cancel(skb, nlh);
1165	return -EMSGSIZE;
1166	}
1167
1168	static int mpls_netconf_msgsize_devconf(int type)
1169	{
1170	int size = NLMSG_ALIGN(sizeof(struct netconfmsg))
1171	+ nla_total_size(payload: 4); /* NETCONFA_IFINDEX */
1172	bool all = false;
1173
1174	if (type == NETCONFA_ALL)
1175	all = true;
1176
1177	if (all || type == NETCONFA_INPUT)
1178	size += nla_total_size(payload: 4);
1179
1180	return size;
1181	}
1182
1183	static void mpls_netconf_notify_devconf(struct net *net, int event,
1184	int type, struct mpls_dev *mdev)
1185	{
1186	struct sk_buff *skb;
1187	int err = -ENOBUFS;
1188
1189	skb = nlmsg_new(payload: mpls_netconf_msgsize_devconf(type), GFP_KERNEL);
1190	if (!skb)
1191	goto errout;
1192
1193	err = mpls_netconf_fill_devconf(skb, mdev, portid: 0, seq: 0, event, flags: 0, type);
1194	if (err < 0) {
1195	/* -EMSGSIZE implies BUG in mpls_netconf_msgsize_devconf() */
1196	WARN_ON(err == -EMSGSIZE);
1197	kfree_skb(skb);
1198	goto errout;
1199	}
1200
1201	rtnl_notify(skb, net, pid: 0, RTNLGRP_MPLS_NETCONF, NULL, GFP_KERNEL);
1202	return;
1203	errout:
1204	if (err < 0)
1205	rtnl_set_sk_err(net, RTNLGRP_MPLS_NETCONF, error: err);
1206	}
1207
1208	static const struct nla_policy devconf_mpls_policy[NETCONFA_MAX + 1] = {
1209	[NETCONFA_IFINDEX] = { .len = sizeof(int) },
1210	};
1211
1212	static int mpls_netconf_valid_get_req(struct sk_buff *skb,
1213	const struct nlmsghdr *nlh,
1214	struct nlattr **tb,
1215	struct netlink_ext_ack *extack)
1216	{
1217	int i, err;
1218
1219	if (nlh->nlmsg_len < nlmsg_msg_size(payload: sizeof(struct netconfmsg))) {
1220	NL_SET_ERR_MSG_MOD(extack,
1221	"Invalid header for netconf get request");
1222	return -EINVAL;
1223	}
1224
1225	if (!netlink_strict_get_check(skb))
1226	return nlmsg_parse_deprecated(nlh, hdrlen: sizeof(struct netconfmsg),
1227	tb, NETCONFA_MAX,
1228	policy: devconf_mpls_policy, extack);
1229
1230	err = nlmsg_parse_deprecated_strict(nlh, hdrlen: sizeof(struct netconfmsg),
1231	tb, NETCONFA_MAX,
1232	policy: devconf_mpls_policy, extack);
1233	if (err)
1234	return err;
1235
1236	for (i = 0; i <= NETCONFA_MAX; i++) {
1237	if (!tb[i])
1238	continue;
1239
1240	switch (i) {
1241	case NETCONFA_IFINDEX:
1242	break;
1243	default:
1244	NL_SET_ERR_MSG_MOD(extack, "Unsupported attribute in netconf get request");
1245	return -EINVAL;
1246	}
1247	}
1248
1249	return 0;
1250	}
1251
1252	static int mpls_netconf_get_devconf(struct sk_buff *in_skb,
1253	struct nlmsghdr *nlh,
1254	struct netlink_ext_ack *extack)
1255	{
1256	struct net *net = sock_net(sk: in_skb->sk);
1257	struct nlattr *tb[NETCONFA_MAX + 1];
1258	struct net_device *dev;
1259	struct mpls_dev *mdev;
1260	struct sk_buff *skb;
1261	int ifindex;
1262	int err;
1263
1264	err = mpls_netconf_valid_get_req(skb: in_skb, nlh, tb, extack);
1265	if (err < 0)
1266	goto errout;
1267
1268	err = -EINVAL;
1269	if (!tb[NETCONFA_IFINDEX])
1270	goto errout;
1271
1272	ifindex = nla_get_s32(nla: tb[NETCONFA_IFINDEX]);
1273	dev = __dev_get_by_index(net, ifindex);
1274	if (!dev)
1275	goto errout;
1276
1277	mdev = mpls_dev_get(dev);
1278	if (!mdev)
1279	goto errout;
1280
1281	err = -ENOBUFS;
1282	skb = nlmsg_new(payload: mpls_netconf_msgsize_devconf(NETCONFA_ALL), GFP_KERNEL);
1283	if (!skb)
1284	goto errout;
1285
1286	err = mpls_netconf_fill_devconf(skb, mdev,
1287	NETLINK_CB(in_skb).portid,
1288	seq: nlh->nlmsg_seq, RTM_NEWNETCONF, flags: 0,
1289	NETCONFA_ALL);
1290	if (err < 0) {
1291	/* -EMSGSIZE implies BUG in mpls_netconf_msgsize_devconf() */
1292	WARN_ON(err == -EMSGSIZE);
1293	kfree_skb(skb);
1294	goto errout;
1295	}
1296	err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
1297	errout:
1298	return err;
1299	}
1300
1301	static int mpls_netconf_dump_devconf(struct sk_buff *skb,
1302	struct netlink_callback *cb)
1303	{
1304	const struct nlmsghdr *nlh = cb->nlh;
1305	struct net *net = sock_net(sk: skb->sk);
1306	struct hlist_head *head;
1307	struct net_device *dev;
1308	struct mpls_dev *mdev;
1309	int idx, s_idx;
1310	int h, s_h;
1311
1312	if (cb->strict_check) {
1313	struct netlink_ext_ack *extack = cb->extack;
1314	struct netconfmsg *ncm;
1315
1316	if (nlh->nlmsg_len < nlmsg_msg_size(payload: sizeof(*ncm))) {
1317	NL_SET_ERR_MSG_MOD(extack, "Invalid header for netconf dump request");
1318	return -EINVAL;
1319	}
1320
1321	if (nlmsg_attrlen(nlh, hdrlen: sizeof(*ncm))) {
1322	NL_SET_ERR_MSG_MOD(extack, "Invalid data after header in netconf dump request");
1323	return -EINVAL;
1324	}
1325	}
1326
1327	s_h = cb->args[0];
1328	s_idx = idx = cb->args[1];
1329
1330	for (h = s_h; h < NETDEV_HASHENTRIES; h++, s_idx = 0) {
1331	idx = 0;
1332	head = &net->dev_index_head[h];
1333	rcu_read_lock();
1334	cb->seq = net->dev_base_seq;
1335	hlist_for_each_entry_rcu(dev, head, index_hlist) {
1336	if (idx < s_idx)
1337	goto cont;
1338	mdev = mpls_dev_get(dev);
1339	if (!mdev)
1340	goto cont;
1341	if (mpls_netconf_fill_devconf(skb, mdev,
1342	NETLINK_CB(cb->skb).portid,
1343	seq: nlh->nlmsg_seq,
1344	RTM_NEWNETCONF,
1345	NLM_F_MULTI,
1346	NETCONFA_ALL) < 0) {
1347	rcu_read_unlock();
1348	goto done;
1349	}
1350	nl_dump_check_consistent(cb, nlh: nlmsg_hdr(skb));
1351	cont:
1352	idx++;
1353	}
1354	rcu_read_unlock();
1355	}
1356	done:
1357	cb->args[0] = h;
1358	cb->args[1] = idx;
1359
1360	return skb->len;
1361	}
1362
1363	#define MPLS_PERDEV_SYSCTL_OFFSET(field) \
1364	(&((struct mpls_dev *)0)->field)
1365
1366	static int mpls_conf_proc(struct ctl_table *ctl, int write,
1367	void *buffer, size_t *lenp, loff_t *ppos)
1368	{
1369	int oval = *(int *)ctl->data;
1370	int ret = proc_dointvec(ctl, write, buffer, lenp, ppos);
1371
1372	if (write) {
1373	struct mpls_dev *mdev = ctl->extra1;
1374	int i = (int *)ctl->data - (int *)mdev;
1375	struct net *net = ctl->extra2;
1376	int val = *(int *)ctl->data;
1377
1378	if (i == offsetof(struct mpls_dev, input_enabled) &&
1379	val != oval) {
1380	mpls_netconf_notify_devconf(net, RTM_NEWNETCONF,
1381	type: NETCONFA_INPUT, mdev);
1382	}
1383	}
1384
1385	return ret;
1386	}
1387
1388	static const struct ctl_table mpls_dev_table[] = {
1389	{
1390	.procname = "input",
1391	.maxlen = sizeof(int),
1392	.mode = 0644,
1393	.proc_handler = mpls_conf_proc,
1394	.data = MPLS_PERDEV_SYSCTL_OFFSET(input_enabled),
1395	},
1396	{ }
1397	};
1398
1399	static int mpls_dev_sysctl_register(struct net_device *dev,
1400	struct mpls_dev *mdev)
1401	{
1402	char path[sizeof("net/mpls/conf/") + IFNAMSIZ];
1403	struct net *net = dev_net(dev);
1404	struct ctl_table *table;
1405	int i;
1406
1407	table = kmemdup(p: &mpls_dev_table, size: sizeof(mpls_dev_table), GFP_KERNEL);
1408	if (!table)
1409	goto out;
1410
1411	/* Table data contains only offsets relative to the base of
1412	* the mdev at this point, so make them absolute.
1413	*/
1414	for (i = 0; i < ARRAY_SIZE(mpls_dev_table); i++) {
1415	table[i].data = (char *)mdev + (uintptr_t)table[i].data;
1416	table[i].extra1 = mdev;
1417	table[i].extra2 = net;
1418	}
1419
1420	snprintf(buf: path, size: sizeof(path), fmt: "net/mpls/conf/%s", dev->name);
1421
1422	mdev->sysctl = register_net_sysctl_sz(net, path, table,
1423	ARRAY_SIZE(mpls_dev_table));
1424	if (!mdev->sysctl)
1425	goto free;
1426
1427	mpls_netconf_notify_devconf(net, RTM_NEWNETCONF, NETCONFA_ALL, mdev);
1428	return 0;
1429
1430	free:
1431	kfree(objp: table);
1432	out:
1433	mdev->sysctl = NULL;
1434	return -ENOBUFS;
1435	}
1436
1437	static void mpls_dev_sysctl_unregister(struct net_device *dev,
1438	struct mpls_dev *mdev)
1439	{
1440	struct net *net = dev_net(dev);
1441	struct ctl_table *table;
1442
1443	if (!mdev->sysctl)
1444	return;
1445
1446	table = mdev->sysctl->ctl_table_arg;
1447	unregister_net_sysctl_table(header: mdev->sysctl);
1448	kfree(objp: table);
1449
1450	mpls_netconf_notify_devconf(net, RTM_DELNETCONF, type: 0, mdev);
1451	}
1452
1453	static struct mpls_dev *mpls_add_dev(struct net_device *dev)
1454	{
1455	struct mpls_dev *mdev;
1456	int err = -ENOMEM;
1457	int i;
1458
1459	ASSERT_RTNL();
1460
1461	mdev = kzalloc(size: sizeof(*mdev), GFP_KERNEL);
1462	if (!mdev)
1463	return ERR_PTR(error: err);
1464
1465	mdev->stats = alloc_percpu(struct mpls_pcpu_stats);
1466	if (!mdev->stats)
1467	goto free;
1468
1469	for_each_possible_cpu(i) {
1470	struct mpls_pcpu_stats *mpls_stats;
1471
1472	mpls_stats = per_cpu_ptr(mdev->stats, i);
1473	u64_stats_init(syncp: &mpls_stats->syncp);
1474	}
1475
1476	mdev->dev = dev;
1477
1478	err = mpls_dev_sysctl_register(dev, mdev);
1479	if (err)
1480	goto free;
1481
1482	rcu_assign_pointer(dev->mpls_ptr, mdev);
1483
1484	return mdev;
1485
1486	free:
1487	free_percpu(pdata: mdev->stats);
1488	kfree(objp: mdev);
1489	return ERR_PTR(error: err);
1490	}
1491
1492	static void mpls_dev_destroy_rcu(struct rcu_head *head)
1493	{
1494	struct mpls_dev *mdev = container_of(head, struct mpls_dev, rcu);
1495
1496	free_percpu(pdata: mdev->stats);
1497	kfree(objp: mdev);
1498	}
1499
1500	static int mpls_ifdown(struct net_device *dev, int event)
1501	{
1502	struct mpls_route __rcu **platform_label;
1503	struct net *net = dev_net(dev);
1504	unsigned index;
1505
1506	platform_label = rtnl_dereference(net->mpls.platform_label);
1507	for (index = 0; index < net->mpls.platform_labels; index++) {
1508	struct mpls_route *rt = rtnl_dereference(platform_label[index]);
1509	bool nh_del = false;
1510	u8 alive = 0;
1511
1512	if (!rt)
1513	continue;
1514
1515	if (event == NETDEV_UNREGISTER) {
1516	u8 deleted = 0;
1517
1518	for_nexthops(rt) {
1519	if (!nh->nh_dev || nh->nh_dev == dev)
1520	deleted++;
1521	if (nh->nh_dev == dev)
1522	nh_del = true;
1523	} endfor_nexthops(rt);
1524
1525	/* if there are no more nexthops, delete the route */
1526	if (deleted == rt->rt_nhn) {
1527	mpls_route_update(net, index, NULL, NULL);
1528	continue;
1529	}
1530
1531	if (nh_del) {
1532	size_t size = sizeof(*rt) + rt->rt_nhn *
1533	rt->rt_nh_size;
1534	struct mpls_route *orig = rt;
1535
1536	rt = kmemdup(p: orig, size, GFP_KERNEL);
1537	if (!rt)
1538	return -ENOMEM;
1539	}
1540	}
1541
1542	change_nexthops(rt) {
1543	unsigned int nh_flags = nh->nh_flags;
1544
1545	if (nh->nh_dev != dev)
1546	goto next;
1547
1548	switch (event) {
1549	case NETDEV_DOWN:
1550	case NETDEV_UNREGISTER:
1551	nh_flags |= RTNH_F_DEAD;
1552	fallthrough;
1553	case NETDEV_CHANGE:
1554	nh_flags |= RTNH_F_LINKDOWN;
1555	break;
1556	}
1557	if (event == NETDEV_UNREGISTER)
1558	nh->nh_dev = NULL;
1559
1560	if (nh->nh_flags != nh_flags)
1561	WRITE_ONCE(nh->nh_flags, nh_flags);
1562	next:
1563	if (!(nh_flags & (RTNH_F_DEAD | RTNH_F_LINKDOWN)))
1564	alive++;
1565	} endfor_nexthops(rt);
1566
1567	WRITE_ONCE(rt->rt_nhn_alive, alive);
1568
1569	if (nh_del)
1570	mpls_route_update(net, index, new: rt, NULL);
1571	}
1572
1573	return 0;
1574	}
1575
1576	static void mpls_ifup(struct net_device *dev, unsigned int flags)
1577	{
1578	struct mpls_route __rcu **platform_label;
1579	struct net *net = dev_net(dev);
1580	unsigned index;
1581	u8 alive;
1582
1583	platform_label = rtnl_dereference(net->mpls.platform_label);
1584	for (index = 0; index < net->mpls.platform_labels; index++) {
1585	struct mpls_route *rt = rtnl_dereference(platform_label[index]);
1586
1587	if (!rt)
1588	continue;
1589
1590	alive = 0;
1591	change_nexthops(rt) {
1592	unsigned int nh_flags = nh->nh_flags;
1593
1594	if (!(nh_flags & flags)) {
1595	alive++;
1596	continue;
1597	}
1598	if (nh->nh_dev != dev)
1599	continue;
1600	alive++;
1601	nh_flags &= ~flags;
1602	WRITE_ONCE(nh->nh_flags, nh_flags);
1603	} endfor_nexthops(rt);
1604
1605	WRITE_ONCE(rt->rt_nhn_alive, alive);
1606	}
1607	}
1608
1609	static int mpls_dev_notify(struct notifier_block *this, unsigned long event,
1610	void *ptr)
1611	{
1612	struct net_device *dev = netdev_notifier_info_to_dev(info: ptr);
1613	struct mpls_dev *mdev;
1614	unsigned int flags;
1615	int err;
1616
1617	if (event == NETDEV_REGISTER) {
1618	mdev = mpls_add_dev(dev);
1619	if (IS_ERR(ptr: mdev))
1620	return notifier_from_errno(err: PTR_ERR(ptr: mdev));
1621
1622	return NOTIFY_OK;
1623	}
1624
1625	mdev = mpls_dev_get(dev);
1626	if (!mdev)
1627	return NOTIFY_OK;
1628
1629	switch (event) {
1630
1631	case NETDEV_DOWN:
1632	err = mpls_ifdown(dev, event);
1633	if (err)
1634	return notifier_from_errno(err);
1635	break;
1636	case NETDEV_UP:
1637	flags = dev_get_flags(dev);
1638	if (flags & (IFF_RUNNING | IFF_LOWER_UP))
1639	mpls_ifup(dev, RTNH_F_DEAD | RTNH_F_LINKDOWN);
1640	else
1641	mpls_ifup(dev, RTNH_F_DEAD);
1642	break;
1643	case NETDEV_CHANGE:
1644	flags = dev_get_flags(dev);
1645	if (flags & (IFF_RUNNING | IFF_LOWER_UP)) {
1646	mpls_ifup(dev, RTNH_F_DEAD | RTNH_F_LINKDOWN);
1647	} else {
1648	err = mpls_ifdown(dev, event);
1649	if (err)
1650	return notifier_from_errno(err);
1651	}
1652	break;
1653	case NETDEV_UNREGISTER:
1654	err = mpls_ifdown(dev, event);
1655	if (err)
1656	return notifier_from_errno(err);
1657	mdev = mpls_dev_get(dev);
1658	if (mdev) {
1659	mpls_dev_sysctl_unregister(dev, mdev);
1660	RCU_INIT_POINTER(dev->mpls_ptr, NULL);
1661	call_rcu(head: &mdev->rcu, func: mpls_dev_destroy_rcu);
1662	}
1663	break;
1664	case NETDEV_CHANGENAME:
1665	mdev = mpls_dev_get(dev);
1666	if (mdev) {
1667	mpls_dev_sysctl_unregister(dev, mdev);
1668	err = mpls_dev_sysctl_register(dev, mdev);
1669	if (err)
1670	return notifier_from_errno(err);
1671	}
1672	break;
1673	}
1674	return NOTIFY_OK;
1675	}
1676
1677	static struct notifier_block mpls_dev_notifier = {
1678	.notifier_call = mpls_dev_notify,
1679	};
1680
1681	static int nla_put_via(struct sk_buff *skb,
1682	u8 table, const void *addr, int alen)
1683	{
1684	static const int table_to_family[NEIGH_NR_TABLES + 1] = {
1685	AF_INET, AF_INET6, AF_DECnet, AF_PACKET,
1686	};
1687	struct nlattr *nla;
1688	struct rtvia *via;
1689	int family = AF_UNSPEC;
1690
1691	nla = nla_reserve(skb, attrtype: RTA_VIA, attrlen: alen + 2);
1692	if (!nla)
1693	return -EMSGSIZE;
1694
1695	if (table <= NEIGH_NR_TABLES)
1696	family = table_to_family[table];
1697
1698	via = nla_data(nla);
1699	via->rtvia_family = family;
1700	memcpy(via->rtvia_addr, addr, alen);
1701	return 0;
1702	}
1703
1704	int nla_put_labels(struct sk_buff *skb, int attrtype,
1705	u8 labels, const u32 label[])
1706	{
1707	struct nlattr *nla;
1708	struct mpls_shim_hdr *nla_label;
1709	bool bos;
1710	int i;
1711	nla = nla_reserve(skb, attrtype, attrlen: labels*4);
1712	if (!nla)
1713	return -EMSGSIZE;
1714
1715	nla_label = nla_data(nla);
1716	bos = true;
1717	for (i = labels - 1; i >= 0; i--) {
1718	nla_label[i] = mpls_entry_encode(label: label[i], ttl: 0, tc: 0, bos);
1719	bos = false;
1720	}
1721
1722	return 0;
1723	}
1724	EXPORT_SYMBOL_GPL(nla_put_labels);
1725
1726	int nla_get_labels(const struct nlattr *nla, u8 max_labels, u8 *labels,
1727	u32 label[], struct netlink_ext_ack *extack)
1728	{
1729	unsigned len = nla_len(nla);
1730	struct mpls_shim_hdr *nla_label;
1731	u8 nla_labels;
1732	bool bos;
1733	int i;
1734
1735	/* len needs to be an even multiple of 4 (the label size). Number
1736	* of labels is a u8 so check for overflow.
1737	*/
1738	if (len & 3 || len / 4 > 255) {
1739	NL_SET_ERR_MSG_ATTR(extack, nla,
1740	"Invalid length for labels attribute");
1741	return -EINVAL;
1742	}
1743
1744	/* Limit the number of new labels allowed */
1745	nla_labels = len/4;
1746	if (nla_labels > max_labels) {
1747	NL_SET_ERR_MSG(extack, "Too many labels");
1748	return -EINVAL;
1749	}
1750
1751	/* when label == NULL, caller wants number of labels */
1752	if (!label)
1753	goto out;
1754
1755	nla_label = nla_data(nla);
1756	bos = true;
1757	for (i = nla_labels - 1; i >= 0; i--, bos = false) {
1758	struct mpls_entry_decoded dec;
1759	dec = mpls_entry_decode(hdr: nla_label + i);
1760
1761	/* Ensure the bottom of stack flag is properly set
1762	* and ttl and tc are both clear.
1763	*/
1764	if (dec.ttl) {
1765	NL_SET_ERR_MSG_ATTR(extack, nla,
1766	"TTL in label must be 0");
1767	return -EINVAL;
1768	}
1769
1770	if (dec.tc) {
1771	NL_SET_ERR_MSG_ATTR(extack, nla,
1772	"Traffic class in label must be 0");
1773	return -EINVAL;
1774	}
1775
1776	if (dec.bos != bos) {
1777	NL_SET_BAD_ATTR(extack, nla);
1778	if (bos) {
1779	NL_SET_ERR_MSG(extack,
1780	"BOS bit must be set in first label");
1781	} else {
1782	NL_SET_ERR_MSG(extack,
1783	"BOS bit can only be set in first label");
1784	}
1785	return -EINVAL;
1786	}
1787
1788	switch (dec.label) {
1789	case MPLS_LABEL_IMPLNULL:
1790	/* RFC3032: This is a label that an LSR may
1791	* assign and distribute, but which never
1792	* actually appears in the encapsulation.
1793	*/
1794	NL_SET_ERR_MSG_ATTR(extack, nla,
1795	"Implicit NULL Label (3) can not be used in encapsulation");
1796	return -EINVAL;
1797	}
1798
1799	label[i] = dec.label;
1800	}
1801	out:
1802	*labels = nla_labels;
1803	return 0;
1804	}
1805	EXPORT_SYMBOL_GPL(nla_get_labels);
1806
1807	static int rtm_to_route_config(struct sk_buff *skb,
1808	struct nlmsghdr *nlh,
1809	struct mpls_route_config *cfg,
1810	struct netlink_ext_ack *extack)
1811	{
1812	struct rtmsg *rtm;
1813	struct nlattr *tb[RTA_MAX+1];
1814	int index;
1815	int err;
1816
1817	err = nlmsg_parse_deprecated(nlh, hdrlen: sizeof(*rtm), tb, RTA_MAX,
1818	policy: rtm_mpls_policy, extack);
1819	if (err < 0)
1820	goto errout;
1821
1822	err = -EINVAL;
1823	rtm = nlmsg_data(nlh);
1824
1825	if (rtm->rtm_family != AF_MPLS) {
1826	NL_SET_ERR_MSG(extack, "Invalid address family in rtmsg");
1827	goto errout;
1828	}
1829	if (rtm->rtm_dst_len != 20) {
1830	NL_SET_ERR_MSG(extack, "rtm_dst_len must be 20 for MPLS");
1831	goto errout;
1832	}
1833	if (rtm->rtm_src_len != 0) {
1834	NL_SET_ERR_MSG(extack, "rtm_src_len must be 0 for MPLS");
1835	goto errout;
1836	}
1837	if (rtm->rtm_tos != 0) {
1838	NL_SET_ERR_MSG(extack, "rtm_tos must be 0 for MPLS");
1839	goto errout;
1840	}
1841	if (rtm->rtm_table != RT_TABLE_MAIN) {
1842	NL_SET_ERR_MSG(extack,
1843	"MPLS only supports the main route table");
1844	goto errout;
1845	}
1846	/* Any value is acceptable for rtm_protocol */
1847
1848	/* As mpls uses destination specific addresses
1849	* (or source specific address in the case of multicast)
1850	* all addresses have universal scope.
1851	*/
1852	if (rtm->rtm_scope != RT_SCOPE_UNIVERSE) {
1853	NL_SET_ERR_MSG(extack,
1854	"Invalid route scope - MPLS only supports UNIVERSE");
1855	goto errout;
1856	}
1857	if (rtm->rtm_type != RTN_UNICAST) {
1858	NL_SET_ERR_MSG(extack,
1859	"Invalid route type - MPLS only supports UNICAST");
1860	goto errout;
1861	}
1862	if (rtm->rtm_flags != 0) {
1863	NL_SET_ERR_MSG(extack, "rtm_flags must be 0 for MPLS");
1864	goto errout;
1865	}
1866
1867	cfg->rc_label = LABEL_NOT_SPECIFIED;
1868	cfg->rc_protocol = rtm->rtm_protocol;
1869	cfg->rc_via_table = MPLS_NEIGH_TABLE_UNSPEC;
1870	cfg->rc_ttl_propagate = MPLS_TTL_PROP_DEFAULT;
1871	cfg->rc_nlflags = nlh->nlmsg_flags;
1872	cfg->rc_nlinfo.portid = NETLINK_CB(skb).portid;
1873	cfg->rc_nlinfo.nlh = nlh;
1874	cfg->rc_nlinfo.nl_net = sock_net(sk: skb->sk);
1875
1876	for (index = 0; index <= RTA_MAX; index++) {
1877	struct nlattr *nla = tb[index];
1878	if (!nla)
1879	continue;
1880
1881	switch (index) {
1882	case RTA_OIF:
1883	cfg->rc_ifindex = nla_get_u32(nla);
1884	break;
1885	case RTA_NEWDST:
1886	if (nla_get_labels(nla, MAX_NEW_LABELS,
1887	&cfg->rc_output_labels,
1888	cfg->rc_output_label, extack))
1889	goto errout;
1890	break;
1891	case RTA_DST:
1892	{
1893	u8 label_count;
1894	if (nla_get_labels(nla, 1, &label_count,
1895	&cfg->rc_label, extack))
1896	goto errout;
1897
1898	if (!mpls_label_ok(net: cfg->rc_nlinfo.nl_net,
1899	index: &cfg->rc_label, extack))
1900	goto errout;
1901	break;
1902	}
1903	case RTA_GATEWAY:
1904	NL_SET_ERR_MSG(extack, "MPLS does not support RTA_GATEWAY attribute");
1905	goto errout;
1906	case RTA_VIA:
1907	{
1908	if (nla_get_via(nla, via_alen: &cfg->rc_via_alen,
1909	via_table: &cfg->rc_via_table, via_addr: cfg->rc_via,
1910	extack))
1911	goto errout;
1912	break;
1913	}
1914	case RTA_MULTIPATH:
1915	{
1916	cfg->rc_mp = nla_data(nla);
1917	cfg->rc_mp_len = nla_len(nla);
1918	break;
1919	}
1920	case RTA_TTL_PROPAGATE:
1921	{
1922	u8 ttl_propagate = nla_get_u8(nla);
1923
1924	if (ttl_propagate > 1) {
1925	NL_SET_ERR_MSG_ATTR(extack, nla,
1926	"RTA_TTL_PROPAGATE can only be 0 or 1");
1927	goto errout;
1928	}
1929	cfg->rc_ttl_propagate = ttl_propagate ?
1930	MPLS_TTL_PROP_ENABLED :
1931	MPLS_TTL_PROP_DISABLED;
1932	break;
1933	}
1934	default:
1935	NL_SET_ERR_MSG_ATTR(extack, nla, "Unknown attribute");
1936	/* Unsupported attribute */
1937	goto errout;
1938	}
1939	}
1940
1941	err = 0;
1942	errout:
1943	return err;
1944	}
1945
1946	static int mpls_rtm_delroute(struct sk_buff *skb, struct nlmsghdr *nlh,
1947	struct netlink_ext_ack *extack)
1948	{
1949	struct mpls_route_config *cfg;
1950	int err;
1951
1952	cfg = kzalloc(size: sizeof(*cfg), GFP_KERNEL);
1953	if (!cfg)
1954	return -ENOMEM;
1955
1956	err = rtm_to_route_config(skb, nlh, cfg, extack);
1957	if (err < 0)
1958	goto out;
1959
1960	err = mpls_route_del(cfg, extack);
1961	out:
1962	kfree(objp: cfg);
1963
1964	return err;
1965	}
1966
1967
1968	static int mpls_rtm_newroute(struct sk_buff *skb, struct nlmsghdr *nlh,
1969	struct netlink_ext_ack *extack)
1970	{
1971	struct mpls_route_config *cfg;
1972	int err;
1973
1974	cfg = kzalloc(size: sizeof(*cfg), GFP_KERNEL);
1975	if (!cfg)
1976	return -ENOMEM;
1977
1978	err = rtm_to_route_config(skb, nlh, cfg, extack);
1979	if (err < 0)
1980	goto out;
1981
1982	err = mpls_route_add(cfg, extack);
1983	out:
1984	kfree(objp: cfg);
1985
1986	return err;
1987	}
1988
1989	static int mpls_dump_route(struct sk_buff *skb, u32 portid, u32 seq, int event,
1990	u32 label, struct mpls_route *rt, int flags)
1991	{
1992	struct net_device *dev;
1993	struct nlmsghdr *nlh;
1994	struct rtmsg *rtm;
1995
1996	nlh = nlmsg_put(skb, portid, seq, type: event, payload: sizeof(*rtm), flags);
1997	if (nlh == NULL)
1998	return -EMSGSIZE;
1999
2000	rtm = nlmsg_data(nlh);
2001	rtm->rtm_family = AF_MPLS;
2002	rtm->rtm_dst_len = 20;
2003	rtm->rtm_src_len = 0;
2004	rtm->rtm_tos = 0;
2005	rtm->rtm_table = RT_TABLE_MAIN;
2006	rtm->rtm_protocol = rt->rt_protocol;
2007	rtm->rtm_scope = RT_SCOPE_UNIVERSE;
2008	rtm->rtm_type = RTN_UNICAST;
2009	rtm->rtm_flags = 0;
2010
2011	if (nla_put_labels(skb, RTA_DST, 1, &label))
2012	goto nla_put_failure;
2013
2014	if (rt->rt_ttl_propagate != MPLS_TTL_PROP_DEFAULT) {
2015	bool ttl_propagate =
2016	rt->rt_ttl_propagate == MPLS_TTL_PROP_ENABLED;
2017
2018	if (nla_put_u8(skb, attrtype: RTA_TTL_PROPAGATE,
2019	value: ttl_propagate))
2020	goto nla_put_failure;
2021	}
2022	if (rt->rt_nhn == 1) {
2023	const struct mpls_nh *nh = rt->rt_nh;
2024
2025	if (nh->nh_labels &&
2026	nla_put_labels(skb, RTA_NEWDST, nh->nh_labels,
2027	nh->nh_label))
2028	goto nla_put_failure;
2029	if (nh->nh_via_table != MPLS_NEIGH_TABLE_UNSPEC &&
2030	nla_put_via(skb, table: nh->nh_via_table, addr: mpls_nh_via(rt, nh),
2031	alen: nh->nh_via_alen))
2032	goto nla_put_failure;
2033	dev = nh->nh_dev;
2034	if (dev && nla_put_u32(skb, attrtype: RTA_OIF, value: dev->ifindex))
2035	goto nla_put_failure;
2036	if (nh->nh_flags & RTNH_F_LINKDOWN)
2037	rtm->rtm_flags |= RTNH_F_LINKDOWN;
2038	if (nh->nh_flags & RTNH_F_DEAD)
2039	rtm->rtm_flags |= RTNH_F_DEAD;
2040	} else {
2041	struct rtnexthop *rtnh;
2042	struct nlattr *mp;
2043	u8 linkdown = 0;
2044	u8 dead = 0;
2045
2046	mp = nla_nest_start_noflag(skb, attrtype: RTA_MULTIPATH);
2047	if (!mp)
2048	goto nla_put_failure;
2049
2050	for_nexthops(rt) {
2051	dev = nh->nh_dev;
2052	if (!dev)
2053	continue;
2054
2055	rtnh = nla_reserve_nohdr(skb, attrlen: sizeof(*rtnh));
2056	if (!rtnh)
2057	goto nla_put_failure;
2058
2059	rtnh->rtnh_ifindex = dev->ifindex;
2060	if (nh->nh_flags & RTNH_F_LINKDOWN) {
2061	rtnh->rtnh_flags |= RTNH_F_LINKDOWN;
2062	linkdown++;
2063	}
2064	if (nh->nh_flags & RTNH_F_DEAD) {
2065	rtnh->rtnh_flags |= RTNH_F_DEAD;
2066	dead++;
2067	}
2068
2069	if (nh->nh_labels && nla_put_labels(skb, RTA_NEWDST,
2070	nh->nh_labels,
2071	nh->nh_label))
2072	goto nla_put_failure;
2073	if (nh->nh_via_table != MPLS_NEIGH_TABLE_UNSPEC &&
2074	nla_put_via(skb, table: nh->nh_via_table,
2075	addr: mpls_nh_via(rt, nh),
2076	alen: nh->nh_via_alen))
2077	goto nla_put_failure;
2078
2079	/* length of rtnetlink header + attributes */
2080	rtnh->rtnh_len = nlmsg_get_pos(skb) - (void *)rtnh;
2081	} endfor_nexthops(rt);
2082
2083	if (linkdown == rt->rt_nhn)
2084	rtm->rtm_flags |= RTNH_F_LINKDOWN;
2085	if (dead == rt->rt_nhn)
2086	rtm->rtm_flags |= RTNH_F_DEAD;
2087
2088	nla_nest_end(skb, start: mp);
2089	}
2090
2091	nlmsg_end(skb, nlh);
2092	return 0;
2093
2094	nla_put_failure:
2095	nlmsg_cancel(skb, nlh);
2096	return -EMSGSIZE;
2097	}
2098
2099	#if IS_ENABLED(CONFIG_INET)
2100	static int mpls_valid_fib_dump_req(struct net *net, const struct nlmsghdr *nlh,
2101	struct fib_dump_filter *filter,
2102	struct netlink_callback *cb)
2103	{
2104	return ip_valid_fib_dump_req(net, nlh, filter, cb);
2105	}
2106	#else
2107	static int mpls_valid_fib_dump_req(struct net *net, const struct nlmsghdr *nlh,
2108	struct fib_dump_filter *filter,
2109	struct netlink_callback *cb)
2110	{
2111	struct netlink_ext_ack *extack = cb->extack;
2112	struct nlattr *tb[RTA_MAX + 1];
2113	struct rtmsg *rtm;
2114	int err, i;
2115
2116	if (nlh->nlmsg_len < nlmsg_msg_size(sizeof(*rtm))) {
2117	NL_SET_ERR_MSG_MOD(extack, "Invalid header for FIB dump request");
2118	return -EINVAL;
2119	}
2120
2121	rtm = nlmsg_data(nlh);
2122	if (rtm->rtm_dst_len || rtm->rtm_src_len || rtm->rtm_tos ||
2123	rtm->rtm_table || rtm->rtm_scope || rtm->rtm_type ||
2124	rtm->rtm_flags) {
2125	NL_SET_ERR_MSG_MOD(extack, "Invalid values in header for FIB dump request");
2126	return -EINVAL;
2127	}
2128
2129	if (rtm->rtm_protocol) {
2130	filter->protocol = rtm->rtm_protocol;
2131	filter->filter_set = 1;
2132	cb->answer_flags = NLM_F_DUMP_FILTERED;
2133	}
2134
2135	err = nlmsg_parse_deprecated_strict(nlh, sizeof(*rtm), tb, RTA_MAX,
2136	rtm_mpls_policy, extack);
2137	if (err < 0)
2138	return err;
2139
2140	for (i = 0; i <= RTA_MAX; ++i) {
2141	int ifindex;
2142
2143	if (i == RTA_OIF) {
2144	ifindex = nla_get_u32(tb[i]);
2145	filter->dev = __dev_get_by_index(net, ifindex);
2146	if (!filter->dev)
2147	return -ENODEV;
2148	filter->filter_set = 1;
2149	} else if (tb[i]) {
2150	NL_SET_ERR_MSG_MOD(extack, "Unsupported attribute in dump request");
2151	return -EINVAL;
2152	}
2153	}
2154
2155	return 0;
2156	}
2157	#endif
2158
2159	static bool mpls_rt_uses_dev(struct mpls_route *rt,
2160	const struct net_device *dev)
2161	{
2162	if (rt->rt_nhn == 1) {
2163	struct mpls_nh *nh = rt->rt_nh;
2164
2165	if (nh->nh_dev == dev)
2166	return true;
2167	} else {
2168	for_nexthops(rt) {
2169	if (nh->nh_dev == dev)
2170	return true;
2171	} endfor_nexthops(rt);
2172	}
2173
2174	return false;
2175	}
2176
2177	static int mpls_dump_routes(struct sk_buff *skb, struct netlink_callback *cb)
2178	{
2179	const struct nlmsghdr *nlh = cb->nlh;
2180	struct net *net = sock_net(sk: skb->sk);
2181	struct mpls_route __rcu **platform_label;
2182	struct fib_dump_filter filter = {};
2183	unsigned int flags = NLM_F_MULTI;
2184	size_t platform_labels;
2185	unsigned int index;
2186
2187	ASSERT_RTNL();
2188
2189	if (cb->strict_check) {
2190	int err;
2191
2192	err = mpls_valid_fib_dump_req(net, nlh, filter: &filter, cb);
2193	if (err < 0)
2194	return err;
2195
2196	/* for MPLS, there is only 1 table with fixed type and flags.
2197	* If either are set in the filter then return nothing.
2198	*/
2199	if ((filter.table_id && filter.table_id != RT_TABLE_MAIN) ||
2200	(filter.rt_type && filter.rt_type != RTN_UNICAST) ||
2201	filter.flags)
2202	return skb->len;
2203	}
2204
2205	index = cb->args[0];
2206	if (index < MPLS_LABEL_FIRST_UNRESERVED)
2207	index = MPLS_LABEL_FIRST_UNRESERVED;
2208
2209	platform_label = rtnl_dereference(net->mpls.platform_label);
2210	platform_labels = net->mpls.platform_labels;
2211
2212	if (filter.filter_set)
2213	flags |= NLM_F_DUMP_FILTERED;
2214
2215	for (; index < platform_labels; index++) {
2216	struct mpls_route *rt;
2217
2218	rt = rtnl_dereference(platform_label[index]);
2219	if (!rt)
2220	continue;
2221
2222	if ((filter.dev && !mpls_rt_uses_dev(rt, dev: filter.dev)) ||
2223	(filter.protocol && rt->rt_protocol != filter.protocol))
2224	continue;
2225
2226	if (mpls_dump_route(skb, NETLINK_CB(cb->skb).portid,
2227	seq: cb->nlh->nlmsg_seq, RTM_NEWROUTE,
2228	label: index, rt, flags) < 0)
2229	break;
2230	}
2231	cb->args[0] = index;
2232
2233	return skb->len;
2234	}
2235
2236	static inline size_t lfib_nlmsg_size(struct mpls_route *rt)
2237	{
2238	size_t payload =
2239	NLMSG_ALIGN(sizeof(struct rtmsg))
2240	+ nla_total_size(payload: 4) /* RTA_DST */
2241	+ nla_total_size(payload: 1); /* RTA_TTL_PROPAGATE */
2242
2243	if (rt->rt_nhn == 1) {
2244	struct mpls_nh *nh = rt->rt_nh;
2245
2246	if (nh->nh_dev)
2247	payload += nla_total_size(payload: 4); /* RTA_OIF */
2248	if (nh->nh_via_table != MPLS_NEIGH_TABLE_UNSPEC) /* RTA_VIA */
2249	payload += nla_total_size(payload: 2 + nh->nh_via_alen);
2250	if (nh->nh_labels) /* RTA_NEWDST */
2251	payload += nla_total_size(payload: nh->nh_labels * 4);
2252	} else {
2253	/* each nexthop is packed in an attribute */
2254	size_t nhsize = 0;
2255
2256	for_nexthops(rt) {
2257	if (!nh->nh_dev)
2258	continue;
2259	nhsize += nla_total_size(payload: sizeof(struct rtnexthop));
2260	/* RTA_VIA */
2261	if (nh->nh_via_table != MPLS_NEIGH_TABLE_UNSPEC)
2262	nhsize += nla_total_size(payload: 2 + nh->nh_via_alen);
2263	if (nh->nh_labels)
2264	nhsize += nla_total_size(payload: nh->nh_labels * 4);
2265	} endfor_nexthops(rt);
2266	/* nested attribute */
2267	payload += nla_total_size(payload: nhsize);
2268	}
2269
2270	return payload;
2271	}
2272
2273	static void rtmsg_lfib(int event, u32 label, struct mpls_route *rt,
2274	struct nlmsghdr *nlh, struct net *net, u32 portid,
2275	unsigned int nlm_flags)
2276	{
2277	struct sk_buff *skb;
2278	u32 seq = nlh ? nlh->nlmsg_seq : 0;
2279	int err = -ENOBUFS;
2280
2281	skb = nlmsg_new(payload: lfib_nlmsg_size(rt), GFP_KERNEL);
2282	if (skb == NULL)
2283	goto errout;
2284
2285	err = mpls_dump_route(skb, portid, seq, event, label, rt, flags: nlm_flags);
2286	if (err < 0) {
2287	/* -EMSGSIZE implies BUG in lfib_nlmsg_size */
2288	WARN_ON(err == -EMSGSIZE);
2289	kfree_skb(skb);
2290	goto errout;
2291	}
2292	rtnl_notify(skb, net, pid: portid, RTNLGRP_MPLS_ROUTE, nlh, GFP_KERNEL);
2293
2294	return;
2295	errout:
2296	if (err < 0)
2297	rtnl_set_sk_err(net, RTNLGRP_MPLS_ROUTE, error: err);
2298	}
2299
2300	static int mpls_valid_getroute_req(struct sk_buff *skb,
2301	const struct nlmsghdr *nlh,
2302	struct nlattr **tb,
2303	struct netlink_ext_ack *extack)
2304	{
2305	struct rtmsg *rtm;
2306	int i, err;
2307
2308	if (nlh->nlmsg_len < nlmsg_msg_size(payload: sizeof(*rtm))) {
2309	NL_SET_ERR_MSG_MOD(extack,
2310	"Invalid header for get route request");
2311	return -EINVAL;
2312	}
2313
2314	if (!netlink_strict_get_check(skb))
2315	return nlmsg_parse_deprecated(nlh, hdrlen: sizeof(*rtm), tb, RTA_MAX,
2316	policy: rtm_mpls_policy, extack);
2317
2318	rtm = nlmsg_data(nlh);
2319	if ((rtm->rtm_dst_len && rtm->rtm_dst_len != 20) ||
2320	rtm->rtm_src_len || rtm->rtm_tos || rtm->rtm_table ||
2321	rtm->rtm_protocol || rtm->rtm_scope || rtm->rtm_type) {
2322	NL_SET_ERR_MSG_MOD(extack, "Invalid values in header for get route request");
2323	return -EINVAL;
2324	}
2325	if (rtm->rtm_flags & ~RTM_F_FIB_MATCH) {
2326	NL_SET_ERR_MSG_MOD(extack,
2327	"Invalid flags for get route request");
2328	return -EINVAL;
2329	}
2330
2331	err = nlmsg_parse_deprecated_strict(nlh, hdrlen: sizeof(*rtm), tb, RTA_MAX,
2332	policy: rtm_mpls_policy, extack);
2333	if (err)
2334	return err;
2335
2336	if ((tb[RTA_DST] || tb[RTA_NEWDST]) && !rtm->rtm_dst_len) {
2337	NL_SET_ERR_MSG_MOD(extack, "rtm_dst_len must be 20 for MPLS");
2338	return -EINVAL;
2339	}
2340
2341	for (i = 0; i <= RTA_MAX; i++) {
2342	if (!tb[i])
2343	continue;
2344
2345	switch (i) {
2346	case RTA_DST:
2347	case RTA_NEWDST:
2348	break;
2349	default:
2350	NL_SET_ERR_MSG_MOD(extack, "Unsupported attribute in get route request");
2351	return -EINVAL;
2352	}
2353	}
2354
2355	return 0;
2356	}
2357
2358	static int mpls_getroute(struct sk_buff *in_skb, struct nlmsghdr *in_nlh,
2359	struct netlink_ext_ack *extack)
2360	{
2361	struct net *net = sock_net(sk: in_skb->sk);
2362	u32 portid = NETLINK_CB(in_skb).portid;
2363	u32 in_label = LABEL_NOT_SPECIFIED;
2364	struct nlattr *tb[RTA_MAX + 1];
2365	u32 labels[MAX_NEW_LABELS];
2366	struct mpls_shim_hdr *hdr;
2367	unsigned int hdr_size = 0;
2368	const struct mpls_nh *nh;
2369	struct net_device *dev;
2370	struct mpls_route *rt;
2371	struct rtmsg *rtm, *r;
2372	struct nlmsghdr *nlh;
2373	struct sk_buff *skb;
2374	u8 n_labels;
2375	int err;
2376
2377	err = mpls_valid_getroute_req(skb: in_skb, nlh: in_nlh, tb, extack);
2378	if (err < 0)
2379	goto errout;
2380
2381	rtm = nlmsg_data(nlh: in_nlh);
2382
2383	if (tb[RTA_DST]) {
2384	u8 label_count;
2385
2386	if (nla_get_labels(tb[RTA_DST], 1, &label_count,
2387	&in_label, extack)) {
2388	err = -EINVAL;
2389	goto errout;
2390	}
2391
2392	if (!mpls_label_ok(net, index: &in_label, extack)) {
2393	err = -EINVAL;
2394	goto errout;
2395	}
2396	}
2397
2398	rt = mpls_route_input_rcu(net, index: in_label);
2399	if (!rt) {
2400	err = -ENETUNREACH;
2401	goto errout;
2402	}
2403
2404	if (rtm->rtm_flags & RTM_F_FIB_MATCH) {
2405	skb = nlmsg_new(payload: lfib_nlmsg_size(rt), GFP_KERNEL);
2406	if (!skb) {
2407	err = -ENOBUFS;
2408	goto errout;
2409	}
2410
2411	err = mpls_dump_route(skb, portid, seq: in_nlh->nlmsg_seq,
2412	RTM_NEWROUTE, label: in_label, rt, flags: 0);
2413	if (err < 0) {
2414	/* -EMSGSIZE implies BUG in lfib_nlmsg_size */
2415	WARN_ON(err == -EMSGSIZE);
2416	goto errout_free;
2417	}
2418
2419	return rtnl_unicast(skb, net, pid: portid);
2420	}
2421
2422	if (tb[RTA_NEWDST]) {
2423	if (nla_get_labels(tb[RTA_NEWDST], MAX_NEW_LABELS, &n_labels,
2424	labels, extack) != 0) {
2425	err = -EINVAL;
2426	goto errout;
2427	}
2428
2429	hdr_size = n_labels * sizeof(struct mpls_shim_hdr);
2430	}
2431
2432	skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2433	if (!skb) {
2434	err = -ENOBUFS;
2435	goto errout;
2436	}
2437
2438	skb->protocol = htons(ETH_P_MPLS_UC);
2439
2440	if (hdr_size) {
2441	bool bos;
2442	int i;
2443
2444	if (skb_cow(skb, headroom: hdr_size)) {
2445	err = -ENOBUFS;
2446	goto errout_free;
2447	}
2448
2449	skb_reserve(skb, len: hdr_size);
2450	skb_push(skb, len: hdr_size);
2451	skb_reset_network_header(skb);
2452
2453	/* Push new labels */
2454	hdr = mpls_hdr(skb);
2455	bos = true;
2456	for (i = n_labels - 1; i >= 0; i--) {
2457	hdr[i] = mpls_entry_encode(label: labels[i],
2458	ttl: 1, tc: 0, bos);
2459	bos = false;
2460	}
2461	}
2462
2463	nh = mpls_select_multipath(rt, skb);
2464	if (!nh) {
2465	err = -ENETUNREACH;
2466	goto errout_free;
2467	}
2468
2469	if (hdr_size) {
2470	skb_pull(skb, len: hdr_size);
2471	skb_reset_network_header(skb);
2472	}
2473
2474	nlh = nlmsg_put(skb, portid, seq: in_nlh->nlmsg_seq,
2475	RTM_NEWROUTE, payload: sizeof(*r), flags: 0);
2476	if (!nlh) {
2477	err = -EMSGSIZE;
2478	goto errout_free;
2479	}
2480
2481	r = nlmsg_data(nlh);
2482	r->rtm_family = AF_MPLS;
2483	r->rtm_dst_len = 20;
2484	r->rtm_src_len = 0;
2485	r->rtm_table = RT_TABLE_MAIN;
2486	r->rtm_type = RTN_UNICAST;
2487	r->rtm_scope = RT_SCOPE_UNIVERSE;
2488	r->rtm_protocol = rt->rt_protocol;
2489	r->rtm_flags = 0;
2490
2491	if (nla_put_labels(skb, RTA_DST, 1, &in_label))
2492	goto nla_put_failure;
2493
2494	if (nh->nh_labels &&
2495	nla_put_labels(skb, RTA_NEWDST, nh->nh_labels,
2496	nh->nh_label))
2497	goto nla_put_failure;
2498
2499	if (nh->nh_via_table != MPLS_NEIGH_TABLE_UNSPEC &&
2500	nla_put_via(skb, table: nh->nh_via_table, addr: mpls_nh_via(rt, nh),
2501	alen: nh->nh_via_alen))
2502	goto nla_put_failure;
2503	dev = nh->nh_dev;
2504	if (dev && nla_put_u32(skb, attrtype: RTA_OIF, value: dev->ifindex))
2505	goto nla_put_failure;
2506
2507	nlmsg_end(skb, nlh);
2508
2509	err = rtnl_unicast(skb, net, pid: portid);
2510	errout:
2511	return err;
2512
2513	nla_put_failure:
2514	nlmsg_cancel(skb, nlh);
2515	err = -EMSGSIZE;
2516	errout_free:
2517	kfree_skb(skb);
2518	return err;
2519	}
2520
2521	static int resize_platform_label_table(struct net *net, size_t limit)
2522	{
2523	size_t size = sizeof(struct mpls_route *) * limit;
2524	size_t old_limit;
2525	size_t cp_size;
2526	struct mpls_route __rcu **labels = NULL, **old;
2527	struct mpls_route *rt0 = NULL, *rt2 = NULL;
2528	unsigned index;
2529
2530	if (size) {
2531	labels = kvzalloc(size, GFP_KERNEL);
2532	if (!labels)
2533	goto nolabels;
2534	}
2535
2536	/* In case the predefined labels need to be populated */
2537	if (limit > MPLS_LABEL_IPV4NULL) {
2538	struct net_device *lo = net->loopback_dev;
2539	rt0 = mpls_rt_alloc(num_nh: 1, max_alen: lo->addr_len, max_labels: 0);
2540	if (IS_ERR(ptr: rt0))
2541	goto nort0;
2542	rt0->rt_nh->nh_dev = lo;
2543	rt0->rt_protocol = RTPROT_KERNEL;
2544	rt0->rt_payload_type = MPT_IPV4;
2545	rt0->rt_ttl_propagate = MPLS_TTL_PROP_DEFAULT;
2546	rt0->rt_nh->nh_via_table = NEIGH_LINK_TABLE;
2547	rt0->rt_nh->nh_via_alen = lo->addr_len;
2548	memcpy(__mpls_nh_via(rt0, rt0->rt_nh), lo->dev_addr,
2549	lo->addr_len);
2550	}
2551	if (limit > MPLS_LABEL_IPV6NULL) {
2552	struct net_device *lo = net->loopback_dev;
2553	rt2 = mpls_rt_alloc(num_nh: 1, max_alen: lo->addr_len, max_labels: 0);
2554	if (IS_ERR(ptr: rt2))
2555	goto nort2;
2556	rt2->rt_nh->nh_dev = lo;
2557	rt2->rt_protocol = RTPROT_KERNEL;
2558	rt2->rt_payload_type = MPT_IPV6;
2559	rt2->rt_ttl_propagate = MPLS_TTL_PROP_DEFAULT;
2560	rt2->rt_nh->nh_via_table = NEIGH_LINK_TABLE;
2561	rt2->rt_nh->nh_via_alen = lo->addr_len;
2562	memcpy(__mpls_nh_via(rt2, rt2->rt_nh), lo->dev_addr,
2563	lo->addr_len);
2564	}
2565
2566	rtnl_lock();
2567	/* Remember the original table */
2568	old = rtnl_dereference(net->mpls.platform_label);
2569	old_limit = net->mpls.platform_labels;
2570
2571	/* Free any labels beyond the new table */
2572	for (index = limit; index < old_limit; index++)
2573	mpls_route_update(net, index, NULL, NULL);
2574
2575	/* Copy over the old labels */
2576	cp_size = size;
2577	if (old_limit < limit)
2578	cp_size = old_limit * sizeof(struct mpls_route *);
2579
2580	memcpy(labels, old, cp_size);
2581
2582	/* If needed set the predefined labels */
2583	if ((old_limit <= MPLS_LABEL_IPV6NULL) &&
2584	(limit > MPLS_LABEL_IPV6NULL)) {
2585	RCU_INIT_POINTER(labels[MPLS_LABEL_IPV6NULL], rt2);
2586	rt2 = NULL;
2587	}
2588
2589	if ((old_limit <= MPLS_LABEL_IPV4NULL) &&
2590	(limit > MPLS_LABEL_IPV4NULL)) {
2591	RCU_INIT_POINTER(labels[MPLS_LABEL_IPV4NULL], rt0);
2592	rt0 = NULL;
2593	}
2594
2595	/* Update the global pointers */
2596	net->mpls.platform_labels = limit;
2597	rcu_assign_pointer(net->mpls.platform_label, labels);
2598
2599	rtnl_unlock();
2600
2601	mpls_rt_free(rt: rt2);
2602	mpls_rt_free(rt: rt0);
2603
2604	if (old) {
2605	synchronize_rcu();
2606	kvfree(addr: old);
2607	}
2608	return 0;
2609
2610	nort2:
2611	mpls_rt_free(rt: rt0);
2612	nort0:
2613	kvfree(addr: labels);
2614	nolabels:
2615	return -ENOMEM;
2616	}
2617
2618	static int mpls_platform_labels(struct ctl_table *table, int write,
2619	void *buffer, size_t *lenp, loff_t *ppos)
2620	{
2621	struct net *net = table->data;
2622	int platform_labels = net->mpls.platform_labels;
2623	int ret;
2624	struct ctl_table tmp = {
2625	.procname = table->procname,
2626	.data = &platform_labels,
2627	.maxlen = sizeof(int),
2628	.mode = table->mode,
2629	.extra1 = SYSCTL_ZERO,
2630	.extra2 = &label_limit,
2631	};
2632
2633	ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos);
2634
2635	if (write && ret == 0)
2636	ret = resize_platform_label_table(net, limit: platform_labels);
2637
2638	return ret;
2639	}
2640
2641	#define MPLS_NS_SYSCTL_OFFSET(field) \
2642	(&((struct net *)0)->field)
2643
2644	static const struct ctl_table mpls_table[] = {
2645	{
2646	.procname = "platform_labels",
2647	.data = NULL,
2648	.maxlen = sizeof(int),
2649	.mode = 0644,
2650	.proc_handler = mpls_platform_labels,
2651	},
2652	{
2653	.procname = "ip_ttl_propagate",
2654	.data = MPLS_NS_SYSCTL_OFFSET(mpls.ip_ttl_propagate),
2655	.maxlen = sizeof(int),
2656	.mode = 0644,
2657	.proc_handler = proc_dointvec_minmax,
2658	.extra1 = SYSCTL_ZERO,
2659	.extra2 = SYSCTL_ONE,
2660	},
2661	{
2662	.procname = "default_ttl",
2663	.data = MPLS_NS_SYSCTL_OFFSET(mpls.default_ttl),
2664	.maxlen = sizeof(int),
2665	.mode = 0644,
2666	.proc_handler = proc_dointvec_minmax,
2667	.extra1 = SYSCTL_ONE,
2668	.extra2 = &ttl_max,
2669	},
2670	{ }
2671	};
2672
2673	static int mpls_net_init(struct net *net)
2674	{
2675	struct ctl_table *table;
2676	int i;
2677
2678	net->mpls.platform_labels = 0;
2679	net->mpls.platform_label = NULL;
2680	net->mpls.ip_ttl_propagate = 1;
2681	net->mpls.default_ttl = 255;
2682
2683	table = kmemdup(p: mpls_table, size: sizeof(mpls_table), GFP_KERNEL);
2684	if (table == NULL)
2685	return -ENOMEM;
2686
2687	/* Table data contains only offsets relative to the base of
2688	* the mdev at this point, so make them absolute.
2689	*/
2690	for (i = 0; i < ARRAY_SIZE(mpls_table) - 1; i++)
2691	table[i].data = (char *)net + (uintptr_t)table[i].data;
2692
2693	net->mpls.ctl = register_net_sysctl_sz(net, path: "net/mpls", table,
2694	ARRAY_SIZE(mpls_table));
2695	if (net->mpls.ctl == NULL) {
2696	kfree(objp: table);
2697	return -ENOMEM;
2698	}
2699
2700	return 0;
2701	}
2702
2703	static void mpls_net_exit(struct net *net)
2704	{
2705	struct mpls_route __rcu **platform_label;
2706	size_t platform_labels;
2707	struct ctl_table *table;
2708	unsigned int index;
2709
2710	table = net->mpls.ctl->ctl_table_arg;
2711	unregister_net_sysctl_table(header: net->mpls.ctl);
2712	kfree(objp: table);
2713
2714	/* An rcu grace period has passed since there was a device in
2715	* the network namespace (and thus the last in flight packet)
2716	* left this network namespace. This is because
2717	* unregister_netdevice_many and netdev_run_todo has completed
2718	* for each network device that was in this network namespace.
2719	*
2720	* As such no additional rcu synchronization is necessary when
2721	* freeing the platform_label table.
2722	*/
2723	rtnl_lock();
2724	platform_label = rtnl_dereference(net->mpls.platform_label);
2725	platform_labels = net->mpls.platform_labels;
2726	for (index = 0; index < platform_labels; index++) {
2727	struct mpls_route *rt = rtnl_dereference(platform_label[index]);
2728	RCU_INIT_POINTER(platform_label[index], NULL);
2729	mpls_notify_route(net, index, old: rt, NULL, NULL);
2730	mpls_rt_free(rt);
2731	}
2732	rtnl_unlock();
2733
2734	kvfree(addr: platform_label);
2735	}
2736
2737	static struct pernet_operations mpls_net_ops = {
2738	.init = mpls_net_init,
2739	.exit = mpls_net_exit,
2740	};
2741
2742	static struct rtnl_af_ops mpls_af_ops __read_mostly = {
2743	.family = AF_MPLS,
2744	.fill_stats_af = mpls_fill_stats_af,
2745	.get_stats_af_size = mpls_get_stats_af_size,
2746	};
2747
2748	static int __init mpls_init(void)
2749	{
2750	int err;
2751
2752	BUILD_BUG_ON(sizeof(struct mpls_shim_hdr) != 4);
2753
2754	err = register_pernet_subsys(&mpls_net_ops);
2755	if (err)
2756	goto out;
2757
2758	err = register_netdevice_notifier(nb: &mpls_dev_notifier);
2759	if (err)
2760	goto out_unregister_pernet;
2761
2762	dev_add_pack(pt: &mpls_packet_type);
2763
2764	rtnl_af_register(ops: &mpls_af_ops);
2765
2766	rtnl_register_module(THIS_MODULE, PF_MPLS, RTM_NEWROUTE,
2767	mpls_rtm_newroute, NULL, flags: 0);
2768	rtnl_register_module(THIS_MODULE, PF_MPLS, RTM_DELROUTE,
2769	mpls_rtm_delroute, NULL, flags: 0);
2770	rtnl_register_module(THIS_MODULE, PF_MPLS, RTM_GETROUTE,
2771	mpls_getroute, mpls_dump_routes, flags: 0);
2772	rtnl_register_module(THIS_MODULE, PF_MPLS, RTM_GETNETCONF,
2773	mpls_netconf_get_devconf,
2774	mpls_netconf_dump_devconf, flags: 0);
2775	err = ipgre_tunnel_encap_add_mpls_ops();
2776	if (err)
2777	pr_err("Can't add mpls over gre tunnel ops\n");
2778
2779	err = 0;
2780	out:
2781	return err;
2782
2783	out_unregister_pernet:
2784	unregister_pernet_subsys(&mpls_net_ops);
2785	goto out;
2786	}
2787	module_init(mpls_init);
2788
2789	static void __exit mpls_exit(void)
2790	{
2791	rtnl_unregister_all(PF_MPLS);
2792	rtnl_af_unregister(ops: &mpls_af_ops);
2793	dev_remove_pack(pt: &mpls_packet_type);
2794	unregister_netdevice_notifier(nb: &mpls_dev_notifier);
2795	unregister_pernet_subsys(&mpls_net_ops);
2796	ipgre_tunnel_encap_del_mpls_ops();
2797	}
2798	module_exit(mpls_exit);
2799
2800	MODULE_DESCRIPTION("MultiProtocol Label Switching");
2801	MODULE_LICENSE("GPL v2");
2802	MODULE_ALIAS_NETPROTO(PF_MPLS);
2803