1 | // SPDX-License-Identifier: GPL-2.0 |
2 | |
3 | #include "vmlinux.h" |
4 | |
5 | #include <bpf/bpf_helpers.h> |
6 | #include <bpf/bpf_tracing.h> |
7 | |
8 | char _license[] SEC("license" ) = "GPL" ; |
9 | |
10 | #define USEC_PER_SEC 1000000UL |
11 | |
12 | #define min(a, b) ((a) < (b) ? (a) : (b)) |
13 | |
14 | static inline struct tcp_sock *tcp_sk(const struct sock *sk) |
15 | { |
16 | return (struct tcp_sock *)sk; |
17 | } |
18 | |
19 | static inline unsigned int tcp_left_out(const struct tcp_sock *tp) |
20 | { |
21 | return tp->sacked_out + tp->lost_out; |
22 | } |
23 | |
24 | static inline unsigned int tcp_packets_in_flight(const struct tcp_sock *tp) |
25 | { |
26 | return tp->packets_out - tcp_left_out(tp) + tp->retrans_out; |
27 | } |
28 | |
29 | SEC("struct_ops/write_sk_pacing_init" ) |
30 | void BPF_PROG(write_sk_pacing_init, struct sock *sk) |
31 | { |
32 | #ifdef ENABLE_ATOMICS_TESTS |
33 | __sync_bool_compare_and_swap(&sk->sk_pacing_status, SK_PACING_NONE, |
34 | SK_PACING_NEEDED); |
35 | #else |
36 | sk->sk_pacing_status = SK_PACING_NEEDED; |
37 | #endif |
38 | } |
39 | |
40 | SEC("struct_ops/write_sk_pacing_cong_control" ) |
41 | void BPF_PROG(write_sk_pacing_cong_control, struct sock *sk, |
42 | const struct rate_sample *rs) |
43 | { |
44 | struct tcp_sock *tp = tcp_sk(sk: sk); |
45 | unsigned long rate = |
46 | ((tp->snd_cwnd * tp->mss_cache * USEC_PER_SEC) << 3) / |
47 | (tp->srtt_us ?: 1U << 3); |
48 | sk->sk_pacing_rate = min(rate, sk->sk_max_pacing_rate); |
49 | tp->app_limited = (tp->delivered + tcp_packets_in_flight(tp)) ?: 1; |
50 | } |
51 | |
52 | SEC("struct_ops/write_sk_pacing_ssthresh" ) |
53 | __u32 BPF_PROG(write_sk_pacing_ssthresh, struct sock *sk) |
54 | { |
55 | return tcp_sk(sk)->snd_ssthresh; |
56 | } |
57 | |
58 | SEC("struct_ops/write_sk_pacing_undo_cwnd" ) |
59 | __u32 BPF_PROG(write_sk_pacing_undo_cwnd, struct sock *sk) |
60 | { |
61 | return tcp_sk(sk)->snd_cwnd; |
62 | } |
63 | |
64 | SEC(".struct_ops" ) |
65 | struct tcp_congestion_ops write_sk_pacing = { |
66 | .init = (void *)write_sk_pacing_init, |
67 | .cong_control = (void *)write_sk_pacing_cong_control, |
68 | .ssthresh = (void *)write_sk_pacing_ssthresh, |
69 | .undo_cwnd = (void *)write_sk_pacing_undo_cwnd, |
70 | .name = "bpf_w_sk_pacing" , |
71 | }; |
72 | |