1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | #ifndef __NET_ACT_API_H |
3 | #define __NET_ACT_API_H |
4 | |
5 | /* |
6 | * Public action API for classifiers/qdiscs |
7 | */ |
8 | |
9 | #include <linux/refcount.h> |
10 | #include <net/flow_offload.h> |
11 | #include <net/sch_generic.h> |
12 | #include <net/pkt_sched.h> |
13 | #include <net/net_namespace.h> |
14 | #include <net/netns/generic.h> |
15 | |
16 | struct tcf_idrinfo { |
17 | struct mutex lock; |
18 | struct idr action_idr; |
19 | struct net *net; |
20 | }; |
21 | |
22 | struct tc_action_ops; |
23 | |
24 | struct tc_action { |
25 | const struct tc_action_ops *ops; |
26 | __u32 type; /* for backward compat(TCA_OLD_COMPAT) */ |
27 | struct tcf_idrinfo *idrinfo; |
28 | |
29 | u32 tcfa_index; |
30 | refcount_t tcfa_refcnt; |
31 | atomic_t tcfa_bindcnt; |
32 | int tcfa_action; |
33 | struct tcf_t tcfa_tm; |
34 | struct gnet_stats_basic_sync tcfa_bstats; |
35 | struct gnet_stats_basic_sync tcfa_bstats_hw; |
36 | struct gnet_stats_queue tcfa_qstats; |
37 | struct net_rate_estimator __rcu *tcfa_rate_est; |
38 | spinlock_t tcfa_lock; |
39 | struct gnet_stats_basic_sync __percpu *cpu_bstats; |
40 | struct gnet_stats_basic_sync __percpu *cpu_bstats_hw; |
41 | struct gnet_stats_queue __percpu *cpu_qstats; |
42 | struct tc_cookie __rcu *user_cookie; |
43 | struct tcf_chain __rcu *goto_chain; |
44 | u32 tcfa_flags; |
45 | u8 hw_stats; |
46 | u8 used_hw_stats; |
47 | bool used_hw_stats_valid; |
48 | u32 in_hw_count; |
49 | }; |
50 | #define tcf_index common.tcfa_index |
51 | #define tcf_refcnt common.tcfa_refcnt |
52 | #define tcf_bindcnt common.tcfa_bindcnt |
53 | #define tcf_action common.tcfa_action |
54 | #define tcf_tm common.tcfa_tm |
55 | #define tcf_bstats common.tcfa_bstats |
56 | #define tcf_qstats common.tcfa_qstats |
57 | #define tcf_rate_est common.tcfa_rate_est |
58 | #define tcf_lock common.tcfa_lock |
59 | |
60 | #define TCA_ACT_HW_STATS_ANY (TCA_ACT_HW_STATS_IMMEDIATE | \ |
61 | TCA_ACT_HW_STATS_DELAYED) |
62 | |
63 | /* Reserve 16 bits for user-space. See TCA_ACT_FLAGS_NO_PERCPU_STATS. */ |
64 | #define TCA_ACT_FLAGS_USER_BITS 16 |
65 | #define TCA_ACT_FLAGS_USER_MASK 0xffff |
66 | #define TCA_ACT_FLAGS_POLICE (1U << TCA_ACT_FLAGS_USER_BITS) |
67 | #define TCA_ACT_FLAGS_BIND (1U << (TCA_ACT_FLAGS_USER_BITS + 1)) |
68 | #define TCA_ACT_FLAGS_REPLACE (1U << (TCA_ACT_FLAGS_USER_BITS + 2)) |
69 | #define TCA_ACT_FLAGS_NO_RTNL (1U << (TCA_ACT_FLAGS_USER_BITS + 3)) |
70 | #define TCA_ACT_FLAGS_AT_INGRESS (1U << (TCA_ACT_FLAGS_USER_BITS + 4)) |
71 | |
72 | /* Update lastuse only if needed, to avoid dirtying a cache line. |
73 | * We use a temp variable to avoid fetching jiffies twice. |
74 | */ |
75 | static inline void tcf_lastuse_update(struct tcf_t *tm) |
76 | { |
77 | unsigned long now = jiffies; |
78 | |
79 | if (tm->lastuse != now) |
80 | tm->lastuse = now; |
81 | if (unlikely(!tm->firstuse)) |
82 | tm->firstuse = now; |
83 | } |
84 | |
85 | static inline void tcf_tm_dump(struct tcf_t *dtm, const struct tcf_t *stm) |
86 | { |
87 | dtm->install = jiffies_to_clock_t(x: jiffies - stm->install); |
88 | dtm->lastuse = jiffies_to_clock_t(x: jiffies - stm->lastuse); |
89 | dtm->firstuse = stm->firstuse ? |
90 | jiffies_to_clock_t(x: jiffies - stm->firstuse) : 0; |
91 | dtm->expires = jiffies_to_clock_t(x: stm->expires); |
92 | } |
93 | |
94 | static inline enum flow_action_hw_stats tc_act_hw_stats(u8 hw_stats) |
95 | { |
96 | if (WARN_ON_ONCE(hw_stats > TCA_ACT_HW_STATS_ANY)) |
97 | return FLOW_ACTION_HW_STATS_DONT_CARE; |
98 | else if (!hw_stats) |
99 | return FLOW_ACTION_HW_STATS_DISABLED; |
100 | |
101 | return hw_stats; |
102 | } |
103 | |
104 | typedef void (*tc_action_priv_destructor)(void *priv); |
105 | |
106 | struct tc_action_ops { |
107 | struct list_head head; |
108 | char kind[IFNAMSIZ]; |
109 | enum tca_id id; /* identifier should match kind */ |
110 | unsigned int net_id; |
111 | size_t size; |
112 | struct module *owner; |
113 | int (*act)(struct sk_buff *, const struct tc_action *, |
114 | struct tcf_result *); /* called under RCU BH lock*/ |
115 | int (*dump)(struct sk_buff *, struct tc_action *, int, int); |
116 | void (*cleanup)(struct tc_action *); |
117 | int (*lookup)(struct net *net, struct tc_action **a, u32 index); |
118 | int (*init)(struct net *net, struct nlattr *nla, |
119 | struct nlattr *est, struct tc_action **act, |
120 | struct tcf_proto *tp, |
121 | u32 flags, struct netlink_ext_ack *extack); |
122 | int (*walk)(struct net *, struct sk_buff *, |
123 | struct netlink_callback *, int, |
124 | const struct tc_action_ops *, |
125 | struct netlink_ext_ack *); |
126 | void (*stats_update)(struct tc_action *, u64, u64, u64, u64, bool); |
127 | size_t (*get_fill_size)(const struct tc_action *act); |
128 | struct net_device *(*get_dev)(const struct tc_action *a, |
129 | tc_action_priv_destructor *destructor); |
130 | struct psample_group * |
131 | (*get_psample_group)(const struct tc_action *a, |
132 | tc_action_priv_destructor *destructor); |
133 | int (*offload_act_setup)(struct tc_action *act, void *entry_data, |
134 | u32 *index_inc, bool bind, |
135 | struct netlink_ext_ack *extack); |
136 | }; |
137 | |
138 | #ifdef CONFIG_NET_CLS_ACT |
139 | |
140 | #define ACT_P_CREATED 1 |
141 | #define ACT_P_DELETED 1 |
142 | |
143 | struct tc_action_net { |
144 | struct tcf_idrinfo *idrinfo; |
145 | const struct tc_action_ops *ops; |
146 | }; |
147 | |
148 | static inline |
149 | int tc_action_net_init(struct net *net, struct tc_action_net *tn, |
150 | const struct tc_action_ops *ops) |
151 | { |
152 | int err = 0; |
153 | |
154 | tn->idrinfo = kmalloc(size: sizeof(*tn->idrinfo), GFP_KERNEL); |
155 | if (!tn->idrinfo) |
156 | return -ENOMEM; |
157 | tn->ops = ops; |
158 | tn->idrinfo->net = net; |
159 | mutex_init(&tn->idrinfo->lock); |
160 | idr_init(idr: &tn->idrinfo->action_idr); |
161 | return err; |
162 | } |
163 | |
164 | void tcf_idrinfo_destroy(const struct tc_action_ops *ops, |
165 | struct tcf_idrinfo *idrinfo); |
166 | |
167 | static inline void tc_action_net_exit(struct list_head *net_list, |
168 | unsigned int id) |
169 | { |
170 | struct net *net; |
171 | |
172 | rtnl_lock(); |
173 | list_for_each_entry(net, net_list, exit_list) { |
174 | struct tc_action_net *tn = net_generic(net, id); |
175 | |
176 | tcf_idrinfo_destroy(ops: tn->ops, idrinfo: tn->idrinfo); |
177 | kfree(objp: tn->idrinfo); |
178 | } |
179 | rtnl_unlock(); |
180 | } |
181 | |
182 | int tcf_generic_walker(struct tc_action_net *tn, struct sk_buff *skb, |
183 | struct netlink_callback *cb, int type, |
184 | const struct tc_action_ops *ops, |
185 | struct netlink_ext_ack *extack); |
186 | int tcf_idr_search(struct tc_action_net *tn, struct tc_action **a, u32 index); |
187 | int tcf_idr_create(struct tc_action_net *tn, u32 index, struct nlattr *est, |
188 | struct tc_action **a, const struct tc_action_ops *ops, |
189 | int bind, bool cpustats, u32 flags); |
190 | int tcf_idr_create_from_flags(struct tc_action_net *tn, u32 index, |
191 | struct nlattr *est, struct tc_action **a, |
192 | const struct tc_action_ops *ops, int bind, |
193 | u32 flags); |
194 | void tcf_idr_insert_many(struct tc_action *actions[]); |
195 | void tcf_idr_cleanup(struct tc_action_net *tn, u32 index); |
196 | int tcf_idr_check_alloc(struct tc_action_net *tn, u32 *index, |
197 | struct tc_action **a, int bind); |
198 | int tcf_idr_release(struct tc_action *a, bool bind); |
199 | |
200 | int tcf_register_action(struct tc_action_ops *a, struct pernet_operations *ops); |
201 | int tcf_unregister_action(struct tc_action_ops *a, |
202 | struct pernet_operations *ops); |
203 | int tcf_action_destroy(struct tc_action *actions[], int bind); |
204 | int tcf_action_exec(struct sk_buff *skb, struct tc_action **actions, |
205 | int nr_actions, struct tcf_result *res); |
206 | int tcf_action_init(struct net *net, struct tcf_proto *tp, struct nlattr *nla, |
207 | struct nlattr *est, |
208 | struct tc_action *actions[], int init_res[], size_t *attr_size, |
209 | u32 flags, u32 fl_flags, struct netlink_ext_ack *extack); |
210 | struct tc_action_ops *tc_action_load_ops(struct nlattr *nla, bool police, |
211 | bool rtnl_held, |
212 | struct netlink_ext_ack *extack); |
213 | struct tc_action *tcf_action_init_1(struct net *net, struct tcf_proto *tp, |
214 | struct nlattr *nla, struct nlattr *est, |
215 | struct tc_action_ops *a_o, int *init_res, |
216 | u32 flags, struct netlink_ext_ack *extack); |
217 | int tcf_action_dump(struct sk_buff *skb, struct tc_action *actions[], int bind, |
218 | int ref, bool terse); |
219 | int tcf_action_dump_old(struct sk_buff *skb, struct tc_action *a, int, int); |
220 | int tcf_action_dump_1(struct sk_buff *skb, struct tc_action *a, int, int); |
221 | |
222 | static inline void tcf_action_update_bstats(struct tc_action *a, |
223 | struct sk_buff *skb) |
224 | { |
225 | if (likely(a->cpu_bstats)) { |
226 | bstats_update(this_cpu_ptr(a->cpu_bstats), skb); |
227 | return; |
228 | } |
229 | spin_lock(lock: &a->tcfa_lock); |
230 | bstats_update(bstats: &a->tcfa_bstats, skb); |
231 | spin_unlock(lock: &a->tcfa_lock); |
232 | } |
233 | |
234 | static inline void tcf_action_inc_drop_qstats(struct tc_action *a) |
235 | { |
236 | if (likely(a->cpu_qstats)) { |
237 | qstats_drop_inc(this_cpu_ptr(a->cpu_qstats)); |
238 | return; |
239 | } |
240 | spin_lock(lock: &a->tcfa_lock); |
241 | qstats_drop_inc(qstats: &a->tcfa_qstats); |
242 | spin_unlock(lock: &a->tcfa_lock); |
243 | } |
244 | |
245 | static inline void tcf_action_inc_overlimit_qstats(struct tc_action *a) |
246 | { |
247 | if (likely(a->cpu_qstats)) { |
248 | qstats_overlimit_inc(this_cpu_ptr(a->cpu_qstats)); |
249 | return; |
250 | } |
251 | spin_lock(lock: &a->tcfa_lock); |
252 | qstats_overlimit_inc(qstats: &a->tcfa_qstats); |
253 | spin_unlock(lock: &a->tcfa_lock); |
254 | } |
255 | |
256 | void tcf_action_update_stats(struct tc_action *a, u64 bytes, u64 packets, |
257 | u64 drops, bool hw); |
258 | int tcf_action_copy_stats(struct sk_buff *, struct tc_action *, int); |
259 | |
260 | int tcf_action_update_hw_stats(struct tc_action *action); |
261 | int tcf_action_reoffload_cb(flow_indr_block_bind_cb_t *cb, |
262 | void *cb_priv, bool add); |
263 | int tcf_action_check_ctrlact(int action, struct tcf_proto *tp, |
264 | struct tcf_chain **handle, |
265 | struct netlink_ext_ack *newchain); |
266 | struct tcf_chain *tcf_action_set_ctrlact(struct tc_action *a, int action, |
267 | struct tcf_chain *newchain); |
268 | |
269 | #ifdef CONFIG_INET |
270 | DECLARE_STATIC_KEY_FALSE(tcf_frag_xmit_count); |
271 | #endif |
272 | |
273 | int tcf_dev_queue_xmit(struct sk_buff *skb, int (*xmit)(struct sk_buff *skb)); |
274 | |
275 | #else /* !CONFIG_NET_CLS_ACT */ |
276 | |
277 | static inline int tcf_action_reoffload_cb(flow_indr_block_bind_cb_t *cb, |
278 | void *cb_priv, bool add) { |
279 | return 0; |
280 | } |
281 | |
282 | #endif /* CONFIG_NET_CLS_ACT */ |
283 | |
284 | static inline void tcf_action_stats_update(struct tc_action *a, u64 bytes, |
285 | u64 packets, u64 drops, |
286 | u64 lastuse, bool hw) |
287 | { |
288 | #ifdef CONFIG_NET_CLS_ACT |
289 | if (!a->ops->stats_update) |
290 | return; |
291 | |
292 | a->ops->stats_update(a, bytes, packets, drops, lastuse, hw); |
293 | #endif |
294 | } |
295 | |
296 | |
297 | #endif |
298 | |