1 | /* This test is a demo of using get_socket_uid and get_socket_cookie |
2 | * helper function to do per socket based network traffic monitoring. |
3 | * It requires iptables version higher then 1.6.1. to load pinned eBPF |
4 | * program into the xt_bpf match. |
5 | * |
6 | * TEST: |
7 | * ./run_cookie_uid_helper_example.sh -option |
8 | * option: |
9 | * -t: do traffic monitoring test, the program will continuously |
10 | * print out network traffic happens after program started A sample |
11 | * output is shown below: |
12 | * |
13 | * cookie: 877, uid: 0x3e8, Pakcet Count: 20, Bytes Count: 11058 |
14 | * cookie: 132, uid: 0x0, Pakcet Count: 2, Bytes Count: 286 |
15 | * cookie: 812, uid: 0x3e8, Pakcet Count: 3, Bytes Count: 1726 |
16 | * cookie: 802, uid: 0x3e8, Pakcet Count: 2, Bytes Count: 104 |
17 | * cookie: 877, uid: 0x3e8, Pakcet Count: 20, Bytes Count: 11058 |
18 | * cookie: 831, uid: 0x3e8, Pakcet Count: 2, Bytes Count: 104 |
19 | * cookie: 0, uid: 0x0, Pakcet Count: 6, Bytes Count: 712 |
20 | * cookie: 880, uid: 0xfffe, Pakcet Count: 1, Bytes Count: 70 |
21 | * |
22 | * -s: do getsockopt SO_COOKIE test, the program will set up a pair of |
23 | * UDP sockets and send packets between them. And read out the traffic data |
24 | * directly from the ebpf map based on the socket cookie. |
25 | * |
26 | * Clean up: if using shell script, the script file will delete the iptables |
27 | * rule and unmount the bpf program when exit. Else the iptables rule need |
28 | * to be deleted by hand, see run_cookie_uid_helper_example.sh for detail. |
29 | */ |
30 | |
31 | #define _GNU_SOURCE |
32 | |
33 | #define offsetof(type, member) __builtin_offsetof(type, member) |
34 | #define ARRAY_SIZE(x) (sizeof(x) / sizeof(*(x))) |
35 | |
36 | #include <arpa/inet.h> |
37 | #include <errno.h> |
38 | #include <error.h> |
39 | #include <limits.h> |
40 | #include <linux/bpf.h> |
41 | #include <linux/if_ether.h> |
42 | #include <net/if.h> |
43 | #include <signal.h> |
44 | #include <stdbool.h> |
45 | #include <stdint.h> |
46 | #include <stdio.h> |
47 | #include <stdlib.h> |
48 | #include <string.h> |
49 | #include <sys/socket.h> |
50 | #include <sys/stat.h> |
51 | #include <sys/types.h> |
52 | #include <unistd.h> |
53 | #include <bpf/bpf.h> |
54 | #include "bpf_insn.h" |
55 | |
56 | #define PORT 8888 |
57 | |
58 | struct stats { |
59 | uint32_t uid; |
60 | uint64_t packets; |
61 | uint64_t bytes; |
62 | }; |
63 | |
64 | static int map_fd, prog_fd; |
65 | |
66 | static bool test_finish; |
67 | |
68 | static void maps_create(void) |
69 | { |
70 | map_fd = bpf_map_create(BPF_MAP_TYPE_HASH, NULL, sizeof(uint32_t), |
71 | sizeof(struct stats), 100, NULL); |
72 | if (map_fd < 0) |
73 | error(status: 1, errno, format: "map create failed!\n" ); |
74 | } |
75 | |
76 | static void prog_load(void) |
77 | { |
78 | static char log_buf[1 << 16]; |
79 | |
80 | struct bpf_insn prog[] = { |
81 | /* |
82 | * Save sk_buff for future usage. value stored in R6 to R10 will |
83 | * not be reset after a bpf helper function call. |
84 | */ |
85 | BPF_MOV64_REG(BPF_REG_6, BPF_REG_1), |
86 | /* |
87 | * pc1: BPF_FUNC_get_socket_cookie takes one parameter, |
88 | * R1: sk_buff |
89 | */ |
90 | BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, |
91 | BPF_FUNC_get_socket_cookie), |
92 | /* pc2-4: save &socketCookie to r7 for future usage*/ |
93 | BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, -8), |
94 | BPF_MOV64_REG(BPF_REG_7, BPF_REG_10), |
95 | BPF_ALU64_IMM(BPF_ADD, BPF_REG_7, -8), |
96 | /* |
97 | * pc5-8: set up the registers for BPF_FUNC_map_lookup_elem, |
98 | * it takes two parameters (R1: map_fd, R2: &socket_cookie) |
99 | */ |
100 | BPF_LD_MAP_FD(BPF_REG_1, map_fd), |
101 | BPF_MOV64_REG(BPF_REG_2, BPF_REG_7), |
102 | BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, |
103 | BPF_FUNC_map_lookup_elem), |
104 | /* |
105 | * pc9. if r0 != 0x0, go to pc+14, since we have the cookie |
106 | * stored already |
107 | * Otherwise do pc10-22 to setup a new data entry. |
108 | */ |
109 | BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 14), |
110 | BPF_MOV64_REG(BPF_REG_1, BPF_REG_6), |
111 | BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, |
112 | BPF_FUNC_get_socket_uid), |
113 | /* |
114 | * Place a struct stats in the R10 stack and sequentially |
115 | * place the member value into the memory. Packets value |
116 | * is set by directly place a IMM value 1 into the stack. |
117 | */ |
118 | BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, |
119 | -32 + (__s16)offsetof(struct stats, uid)), |
120 | BPF_ST_MEM(BPF_DW, BPF_REG_10, |
121 | -32 + (__s16)offsetof(struct stats, packets), 1), |
122 | /* |
123 | * __sk_buff is a special struct used for eBPF program to |
124 | * directly access some sk_buff field. |
125 | */ |
126 | BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_6, |
127 | offsetof(struct __sk_buff, len)), |
128 | BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_1, |
129 | -32 + (__s16)offsetof(struct stats, bytes)), |
130 | /* |
131 | * add new map entry using BPF_FUNC_map_update_elem, it takes |
132 | * 4 parameters (R1: map_fd, R2: &socket_cookie, R3: &stats, |
133 | * R4: flags) |
134 | */ |
135 | BPF_LD_MAP_FD(BPF_REG_1, map_fd), |
136 | BPF_MOV64_REG(BPF_REG_2, BPF_REG_7), |
137 | BPF_MOV64_REG(BPF_REG_3, BPF_REG_10), |
138 | BPF_ALU64_IMM(BPF_ADD, BPF_REG_3, -32), |
139 | BPF_MOV64_IMM(BPF_REG_4, 0), |
140 | BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, |
141 | BPF_FUNC_map_update_elem), |
142 | BPF_JMP_IMM(BPF_JA, 0, 0, 5), |
143 | /* |
144 | * pc24-30 update the packet info to a exist data entry, it can |
145 | * be done by directly write to pointers instead of using |
146 | * BPF_FUNC_map_update_elem helper function |
147 | */ |
148 | BPF_MOV64_REG(BPF_REG_9, BPF_REG_0), |
149 | BPF_MOV64_IMM(BPF_REG_1, 1), |
150 | BPF_ATOMIC_OP(BPF_DW, BPF_ADD, BPF_REG_9, BPF_REG_1, |
151 | offsetof(struct stats, packets)), |
152 | BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_6, |
153 | offsetof(struct __sk_buff, len)), |
154 | BPF_ATOMIC_OP(BPF_DW, BPF_ADD, BPF_REG_9, BPF_REG_1, |
155 | offsetof(struct stats, bytes)), |
156 | BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_6, |
157 | offsetof(struct __sk_buff, len)), |
158 | BPF_EXIT_INSN(), |
159 | }; |
160 | LIBBPF_OPTS(bpf_prog_load_opts, opts, |
161 | .log_buf = log_buf, |
162 | .log_size = sizeof(log_buf), |
163 | ); |
164 | |
165 | prog_fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL" , |
166 | prog, ARRAY_SIZE(prog), &opts); |
167 | if (prog_fd < 0) |
168 | error(status: 1, errno, format: "failed to load prog\n%s\n" , log_buf); |
169 | } |
170 | |
171 | static void prog_attach_iptables(char *file) |
172 | { |
173 | int ret; |
174 | char rules[256]; |
175 | |
176 | if (bpf_obj_pin(prog_fd, file)) |
177 | error(status: 1, errno, format: "bpf_obj_pin" ); |
178 | if (strlen(s: file) > 50) { |
179 | printf(format: "file path too long: %s\n" , file); |
180 | exit(status: 1); |
181 | } |
182 | ret = snprintf(s: rules, maxlen: sizeof(rules), |
183 | format: "iptables -A OUTPUT -m bpf --object-pinned %s -j ACCEPT" , |
184 | file); |
185 | if (ret < 0 || ret >= sizeof(rules)) { |
186 | printf(format: "error constructing iptables command\n" ); |
187 | exit(status: 1); |
188 | } |
189 | ret = system(command: rules); |
190 | if (ret < 0) { |
191 | printf(format: "iptables rule update failed: %d/n" , WEXITSTATUS(ret)); |
192 | exit(status: 1); |
193 | } |
194 | } |
195 | |
196 | static void print_table(void) |
197 | { |
198 | struct stats curEntry; |
199 | uint32_t curN = UINT32_MAX; |
200 | uint32_t nextN; |
201 | int res; |
202 | |
203 | while (bpf_map_get_next_key(map_fd, &curN, &nextN) > -1) { |
204 | curN = nextN; |
205 | res = bpf_map_lookup_elem(map_fd, &curN, &curEntry); |
206 | if (res < 0) { |
207 | error(status: 1, errno, format: "fail to get entry value of Key: %u\n" , |
208 | curN); |
209 | } else { |
210 | printf(format: "cookie: %u, uid: 0x%x, Packet Count: %lu," |
211 | " Bytes Count: %lu\n" , curN, curEntry.uid, |
212 | curEntry.packets, curEntry.bytes); |
213 | } |
214 | } |
215 | } |
216 | |
217 | static void udp_client(void) |
218 | { |
219 | struct sockaddr_in si_other = {0}; |
220 | struct sockaddr_in si_me = {0}; |
221 | struct stats dataEntry; |
222 | int s_rcv, s_send, i, recv_len; |
223 | char message = 'a'; |
224 | char buf; |
225 | uint64_t cookie; |
226 | int res; |
227 | socklen_t cookie_len = sizeof(cookie); |
228 | socklen_t slen = sizeof(si_other); |
229 | |
230 | s_rcv = socket(PF_INET, SOCK_DGRAM, protocol: 0); |
231 | if (s_rcv < 0) |
232 | error(status: 1, errno, format: "rcv socket creat failed!\n" ); |
233 | si_other.sin_family = AF_INET; |
234 | si_other.sin_port = htons(PORT); |
235 | if (inet_aton(cp: "127.0.0.1" , inp: &si_other.sin_addr) == 0) |
236 | error(status: 1, errno, format: "inet_aton\n" ); |
237 | if (bind(fd: s_rcv, addr: (struct sockaddr *)&si_other, len: sizeof(si_other)) == -1) |
238 | error(status: 1, errno, format: "bind\n" ); |
239 | s_send = socket(PF_INET, SOCK_DGRAM, protocol: 0); |
240 | if (s_send < 0) |
241 | error(status: 1, errno, format: "send socket creat failed!\n" ); |
242 | res = getsockopt(fd: s_send, SOL_SOCKET, SO_COOKIE, optval: &cookie, optlen: &cookie_len); |
243 | if (res < 0) |
244 | printf(format: "get cookie failed: %s\n" , strerror(errno)); |
245 | res = bpf_map_lookup_elem(map_fd, &cookie, &dataEntry); |
246 | if (res != -1) |
247 | error(status: 1, errno, format: "socket stat found while flow not active\n" ); |
248 | for (i = 0; i < 10; i++) { |
249 | res = sendto(fd: s_send, buf: &message, n: sizeof(message), flags: 0, |
250 | addr: (struct sockaddr *)&si_other, addr_len: slen); |
251 | if (res == -1) |
252 | error(status: 1, errno, format: "send\n" ); |
253 | if (res != sizeof(message)) |
254 | error(status: 1, errnum: 0, format: "%uB != %luB\n" , res, sizeof(message)); |
255 | recv_len = recvfrom(fd: s_rcv, buf: &buf, n: sizeof(buf), flags: 0, |
256 | addr: (struct sockaddr *)&si_me, addr_len: &slen); |
257 | if (recv_len < 0) |
258 | error(status: 1, errno, format: "receive\n" ); |
259 | res = memcmp(s1: &(si_other.sin_addr), s2: &(si_me.sin_addr), |
260 | n: sizeof(si_me.sin_addr)); |
261 | if (res != 0) |
262 | error(status: 1, EFAULT, format: "sender addr error: %d\n" , res); |
263 | printf(format: "Message received: %c\n" , buf); |
264 | res = bpf_map_lookup_elem(map_fd, &cookie, &dataEntry); |
265 | if (res < 0) |
266 | error(status: 1, errno, format: "lookup sk stat failed, cookie: %lu\n" , |
267 | cookie); |
268 | printf(format: "cookie: %lu, uid: 0x%x, Packet Count: %lu," |
269 | " Bytes Count: %lu\n\n" , cookie, dataEntry.uid, |
270 | dataEntry.packets, dataEntry.bytes); |
271 | } |
272 | close(fd: s_send); |
273 | close(fd: s_rcv); |
274 | } |
275 | |
276 | static int usage(void) |
277 | { |
278 | printf(format: "Usage: ./run_cookie_uid_helper_example.sh" |
279 | " bpfObjName -option\n" |
280 | " -t traffic monitor test\n" |
281 | " -s getsockopt cookie test\n" ); |
282 | return 1; |
283 | } |
284 | |
285 | static void finish(int ret) |
286 | { |
287 | test_finish = true; |
288 | } |
289 | |
290 | int main(int argc, char *argv[]) |
291 | { |
292 | int opt; |
293 | bool cfg_test_traffic = false; |
294 | bool cfg_test_cookie = false; |
295 | |
296 | if (argc != 3) |
297 | return usage(); |
298 | while ((opt = getopt(argc: argc, argv: argv, shortopts: "ts" )) != -1) { |
299 | switch (opt) { |
300 | case 't': |
301 | cfg_test_traffic = true; |
302 | break; |
303 | case 's': |
304 | cfg_test_cookie = true; |
305 | break; |
306 | |
307 | default: |
308 | printf(format: "unknown option %c\n" , opt); |
309 | usage(); |
310 | return -1; |
311 | } |
312 | } |
313 | maps_create(); |
314 | prog_load(); |
315 | prog_attach_iptables(file: argv[2]); |
316 | if (cfg_test_traffic) { |
317 | if (signal(SIGINT, handler: finish) == SIG_ERR) |
318 | error(status: 1, errno, format: "register SIGINT handler failed" ); |
319 | if (signal(SIGTERM, handler: finish) == SIG_ERR) |
320 | error(status: 1, errno, format: "register SIGTERM handler failed" ); |
321 | while (!test_finish) { |
322 | print_table(); |
323 | printf(format: "\n" ); |
324 | sleep(seconds: 1); |
325 | } |
326 | } else if (cfg_test_cookie) { |
327 | udp_client(); |
328 | } |
329 | close(fd: prog_fd); |
330 | close(fd: map_fd); |
331 | return 0; |
332 | } |
333 | |