1/* include/net/xdp.h
2 *
3 * Copyright (c) 2017 Jesper Dangaard Brouer, Red Hat Inc.
4 * Released under terms in GPL version 2. See COPYING.
5 */
6#ifndef __LINUX_NET_XDP_H__
7#define __LINUX_NET_XDP_H__
8
9/**
10 * DOC: XDP RX-queue information
11 *
12 * The XDP RX-queue info (xdp_rxq_info) is associated with the driver
13 * level RX-ring queues. It is information that is specific to how
14 * the driver have configured a given RX-ring queue.
15 *
16 * Each xdp_buff frame received in the driver carry a (pointer)
17 * reference to this xdp_rxq_info structure. This provides the XDP
18 * data-path read-access to RX-info for both kernel and bpf-side
19 * (limited subset).
20 *
21 * For now, direct access is only safe while running in NAPI/softirq
22 * context. Contents is read-mostly and must not be updated during
23 * driver NAPI/softirq poll.
24 *
25 * The driver usage API is a register and unregister API.
26 *
27 * The struct is not directly tied to the XDP prog. A new XDP prog
28 * can be attached as long as it doesn't change the underlying
29 * RX-ring. If the RX-ring does change significantly, the NIC driver
30 * naturally need to stop the RX-ring before purging and reallocating
31 * memory. In that process the driver MUST call unregistor (which
32 * also apply for driver shutdown and unload). The register API is
33 * also mandatory during RX-ring setup.
34 */
35
36enum xdp_mem_type {
37 MEM_TYPE_PAGE_SHARED = 0, /* Split-page refcnt based model */
38 MEM_TYPE_PAGE_ORDER0, /* Orig XDP full page model */
39 MEM_TYPE_PAGE_POOL,
40 MEM_TYPE_ZERO_COPY,
41 MEM_TYPE_MAX,
42};
43
44/* XDP flags for ndo_xdp_xmit */
45#define XDP_XMIT_FLUSH (1U << 0) /* doorbell signal consumer */
46#define XDP_XMIT_FLAGS_MASK XDP_XMIT_FLUSH
47
48struct xdp_mem_info {
49 u32 type; /* enum xdp_mem_type, but known size type */
50 u32 id;
51};
52
53struct page_pool;
54
55struct zero_copy_allocator {
56 void (*free)(struct zero_copy_allocator *zca, unsigned long handle);
57};
58
59struct xdp_rxq_info {
60 struct net_device *dev;
61 u32 queue_index;
62 u32 reg_state;
63 struct xdp_mem_info mem;
64} ____cacheline_aligned; /* perf critical, avoid false-sharing */
65
66struct xdp_buff {
67 void *data;
68 void *data_end;
69 void *data_meta;
70 void *data_hard_start;
71 unsigned long handle;
72 struct xdp_rxq_info *rxq;
73};
74
75struct xdp_frame {
76 void *data;
77 u16 len;
78 u16 headroom;
79 u16 metasize;
80 /* Lifetime of xdp_rxq_info is limited to NAPI/enqueue time,
81 * while mem info is valid on remote CPU.
82 */
83 struct xdp_mem_info mem;
84 struct net_device *dev_rx; /* used by cpumap */
85};
86
87/* Clear kernel pointers in xdp_frame */
88static inline void xdp_scrub_frame(struct xdp_frame *frame)
89{
90 frame->data = NULL;
91 frame->dev_rx = NULL;
92}
93
94struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp);
95
96/* Convert xdp_buff to xdp_frame */
97static inline
98struct xdp_frame *convert_to_xdp_frame(struct xdp_buff *xdp)
99{
100 struct xdp_frame *xdp_frame;
101 int metasize;
102 int headroom;
103
104 if (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY)
105 return xdp_convert_zc_to_xdp_frame(xdp);
106
107 /* Assure headroom is available for storing info */
108 headroom = xdp->data - xdp->data_hard_start;
109 metasize = xdp->data - xdp->data_meta;
110 metasize = metasize > 0 ? metasize : 0;
111 if (unlikely((headroom - metasize) < sizeof(*xdp_frame)))
112 return NULL;
113
114 /* Store info in top of packet */
115 xdp_frame = xdp->data_hard_start;
116
117 xdp_frame->data = xdp->data;
118 xdp_frame->len = xdp->data_end - xdp->data;
119 xdp_frame->headroom = headroom - sizeof(*xdp_frame);
120 xdp_frame->metasize = metasize;
121
122 /* rxq only valid until napi_schedule ends, convert to xdp_mem_info */
123 xdp_frame->mem = xdp->rxq->mem;
124
125 return xdp_frame;
126}
127
128void xdp_return_frame(struct xdp_frame *xdpf);
129void xdp_return_frame_rx_napi(struct xdp_frame *xdpf);
130void xdp_return_buff(struct xdp_buff *xdp);
131
132int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq,
133 struct net_device *dev, u32 queue_index);
134void xdp_rxq_info_unreg(struct xdp_rxq_info *xdp_rxq);
135void xdp_rxq_info_unused(struct xdp_rxq_info *xdp_rxq);
136bool xdp_rxq_info_is_reg(struct xdp_rxq_info *xdp_rxq);
137int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq,
138 enum xdp_mem_type type, void *allocator);
139void xdp_rxq_info_unreg_mem_model(struct xdp_rxq_info *xdp_rxq);
140
141/* Drivers not supporting XDP metadata can use this helper, which
142 * rejects any room expansion for metadata as a result.
143 */
144static __always_inline void
145xdp_set_data_meta_invalid(struct xdp_buff *xdp)
146{
147 xdp->data_meta = xdp->data + 1;
148}
149
150static __always_inline bool
151xdp_data_meta_unsupported(const struct xdp_buff *xdp)
152{
153 return unlikely(xdp->data_meta > xdp->data);
154}
155
156struct xdp_attachment_info {
157 struct bpf_prog *prog;
158 u32 flags;
159};
160
161struct netdev_bpf;
162int xdp_attachment_query(struct xdp_attachment_info *info,
163 struct netdev_bpf *bpf);
164bool xdp_attachment_flags_ok(struct xdp_attachment_info *info,
165 struct netdev_bpf *bpf);
166void xdp_attachment_setup(struct xdp_attachment_info *info,
167 struct netdev_bpf *bpf);
168
169#endif /* __LINUX_NET_XDP_H__ */
170