1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | /* AF_XDP internal functions |
3 | * Copyright(c) 2018 Intel Corporation. |
4 | */ |
5 | |
6 | #ifndef _LINUX_XDP_SOCK_H |
7 | #define _LINUX_XDP_SOCK_H |
8 | |
9 | #include <linux/bpf.h> |
10 | #include <linux/workqueue.h> |
11 | #include <linux/if_xdp.h> |
12 | #include <linux/mutex.h> |
13 | #include <linux/spinlock.h> |
14 | #include <linux/mm.h> |
15 | #include <net/sock.h> |
16 | |
17 | #define XDP_UMEM_SG_FLAG (1 << 1) |
18 | |
19 | struct net_device; |
20 | struct xsk_queue; |
21 | struct xdp_buff; |
22 | |
23 | struct xdp_umem { |
24 | void *addrs; |
25 | u64 size; |
26 | u32 headroom; |
27 | u32 chunk_size; |
28 | u32 chunks; |
29 | u32 npgs; |
30 | struct user_struct *user; |
31 | refcount_t users; |
32 | u8 flags; |
33 | u8 tx_metadata_len; |
34 | bool zc; |
35 | struct page **pgs; |
36 | int id; |
37 | struct list_head xsk_dma_list; |
38 | struct work_struct work; |
39 | }; |
40 | |
41 | struct xsk_map { |
42 | struct bpf_map map; |
43 | spinlock_t lock; /* Synchronize map updates */ |
44 | atomic_t count; |
45 | struct xdp_sock __rcu *xsk_map[]; |
46 | }; |
47 | |
48 | struct xdp_sock { |
49 | /* struct sock must be the first member of struct xdp_sock */ |
50 | struct sock sk; |
51 | struct xsk_queue *rx ____cacheline_aligned_in_smp; |
52 | struct net_device *dev; |
53 | struct xdp_umem *umem; |
54 | struct list_head flush_node; |
55 | struct xsk_buff_pool *pool; |
56 | u16 queue_id; |
57 | bool zc; |
58 | bool sg; |
59 | enum { |
60 | XSK_READY = 0, |
61 | XSK_BOUND, |
62 | XSK_UNBOUND, |
63 | } state; |
64 | |
65 | struct xsk_queue *tx ____cacheline_aligned_in_smp; |
66 | struct list_head tx_list; |
67 | /* record the number of tx descriptors sent by this xsk and |
68 | * when it exceeds MAX_PER_SOCKET_BUDGET, an opportunity needs |
69 | * to be given to other xsks for sending tx descriptors, thereby |
70 | * preventing other XSKs from being starved. |
71 | */ |
72 | u32 tx_budget_spent; |
73 | |
74 | /* Protects generic receive. */ |
75 | spinlock_t rx_lock; |
76 | |
77 | /* Statistics */ |
78 | u64 rx_dropped; |
79 | u64 rx_queue_full; |
80 | |
81 | /* When __xsk_generic_xmit() must return before it sees the EOP descriptor for the current |
82 | * packet, the partially built skb is saved here so that packet building can resume in next |
83 | * call of __xsk_generic_xmit(). |
84 | */ |
85 | struct sk_buff *skb; |
86 | |
87 | struct list_head map_list; |
88 | /* Protects map_list */ |
89 | spinlock_t map_list_lock; |
90 | /* Protects multiple processes in the control path */ |
91 | struct mutex mutex; |
92 | struct xsk_queue *fq_tmp; /* Only as tmp storage before bind */ |
93 | struct xsk_queue *cq_tmp; /* Only as tmp storage before bind */ |
94 | }; |
95 | |
96 | /* |
97 | * AF_XDP TX metadata hooks for network devices. |
98 | * The following hooks can be defined; unless noted otherwise, they are |
99 | * optional and can be filled with a null pointer. |
100 | * |
101 | * void (*tmo_request_timestamp)(void *priv) |
102 | * Called when AF_XDP frame requested egress timestamp. |
103 | * |
104 | * u64 (*tmo_fill_timestamp)(void *priv) |
105 | * Called when AF_XDP frame, that had requested egress timestamp, |
106 | * received a completion. The hook needs to return the actual HW timestamp. |
107 | * |
108 | * void (*tmo_request_checksum)(u16 csum_start, u16 csum_offset, void *priv) |
109 | * Called when AF_XDP frame requested HW checksum offload. csum_start |
110 | * indicates position where checksumming should start. |
111 | * csum_offset indicates position where checksum should be stored. |
112 | * |
113 | */ |
114 | struct xsk_tx_metadata_ops { |
115 | void (*tmo_request_timestamp)(void *priv); |
116 | u64 (*tmo_fill_timestamp)(void *priv); |
117 | void (*tmo_request_checksum)(u16 csum_start, u16 csum_offset, void *priv); |
118 | }; |
119 | |
120 | #ifdef CONFIG_XDP_SOCKETS |
121 | |
122 | int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp); |
123 | int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp); |
124 | void __xsk_map_flush(void); |
125 | |
126 | /** |
127 | * xsk_tx_metadata_to_compl - Save enough relevant metadata information |
128 | * to perform tx completion in the future. |
129 | * @meta: pointer to AF_XDP metadata area |
130 | * @compl: pointer to output struct xsk_tx_metadata_to_compl |
131 | * |
132 | * This function should be called by the networking device when |
133 | * it prepares AF_XDP egress packet. The value of @compl should be stored |
134 | * and passed to xsk_tx_metadata_complete upon TX completion. |
135 | */ |
136 | static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta, |
137 | struct xsk_tx_metadata_compl *compl) |
138 | { |
139 | if (!meta) |
140 | return; |
141 | |
142 | if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP) |
143 | compl->tx_timestamp = &meta->completion.tx_timestamp; |
144 | else |
145 | compl->tx_timestamp = NULL; |
146 | } |
147 | |
148 | /** |
149 | * xsk_tx_metadata_request - Evaluate AF_XDP TX metadata at submission |
150 | * and call appropriate xsk_tx_metadata_ops operation. |
151 | * @meta: pointer to AF_XDP metadata area |
152 | * @ops: pointer to struct xsk_tx_metadata_ops |
153 | * @priv: pointer to driver-private aread |
154 | * |
155 | * This function should be called by the networking device when |
156 | * it prepares AF_XDP egress packet. |
157 | */ |
158 | static inline void xsk_tx_metadata_request(const struct xsk_tx_metadata *meta, |
159 | const struct xsk_tx_metadata_ops *ops, |
160 | void *priv) |
161 | { |
162 | if (!meta) |
163 | return; |
164 | |
165 | if (ops->tmo_request_timestamp) |
166 | if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP) |
167 | ops->tmo_request_timestamp(priv); |
168 | |
169 | if (ops->tmo_request_checksum) |
170 | if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM) |
171 | ops->tmo_request_checksum(meta->request.csum_start, |
172 | meta->request.csum_offset, priv); |
173 | } |
174 | |
175 | /** |
176 | * xsk_tx_metadata_complete - Evaluate AF_XDP TX metadata at completion |
177 | * and call appropriate xsk_tx_metadata_ops operation. |
178 | * @compl: pointer to completion metadata produced from xsk_tx_metadata_to_compl |
179 | * @ops: pointer to struct xsk_tx_metadata_ops |
180 | * @priv: pointer to driver-private aread |
181 | * |
182 | * This function should be called by the networking device upon |
183 | * AF_XDP egress completion. |
184 | */ |
185 | static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl, |
186 | const struct xsk_tx_metadata_ops *ops, |
187 | void *priv) |
188 | { |
189 | if (!compl) |
190 | return; |
191 | if (!compl->tx_timestamp) |
192 | return; |
193 | |
194 | *compl->tx_timestamp = ops->tmo_fill_timestamp(priv); |
195 | } |
196 | |
197 | #else |
198 | |
199 | static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) |
200 | { |
201 | return -ENOTSUPP; |
202 | } |
203 | |
204 | static inline int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp) |
205 | { |
206 | return -EOPNOTSUPP; |
207 | } |
208 | |
209 | static inline void __xsk_map_flush(void) |
210 | { |
211 | } |
212 | |
213 | static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta, |
214 | struct xsk_tx_metadata_compl *compl) |
215 | { |
216 | } |
217 | |
218 | static inline void xsk_tx_metadata_request(struct xsk_tx_metadata *meta, |
219 | const struct xsk_tx_metadata_ops *ops, |
220 | void *priv) |
221 | { |
222 | } |
223 | |
224 | static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl, |
225 | const struct xsk_tx_metadata_ops *ops, |
226 | void *priv) |
227 | { |
228 | } |
229 | |
230 | #endif /* CONFIG_XDP_SOCKETS */ |
231 | |
232 | #if defined(CONFIG_XDP_SOCKETS) && defined(CONFIG_DEBUG_NET) |
233 | bool xsk_map_check_flush(void); |
234 | #else |
235 | static inline bool xsk_map_check_flush(void) |
236 | { |
237 | return false; |
238 | } |
239 | #endif |
240 | |
241 | #endif /* _LINUX_XDP_SOCK_H */ |
242 | |