1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com |
3 | * Copyright (c) 2016 Facebook |
4 | * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io |
5 | */ |
6 | #include <uapi/linux/btf.h> |
7 | #include <linux/kernel.h> |
8 | #include <linux/types.h> |
9 | #include <linux/bpf.h> |
10 | #include <linux/bpf_verifier.h> |
11 | #include <linux/math64.h> |
12 | #include <linux/string.h> |
13 | |
14 | #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args) |
15 | |
16 | static bool bpf_verifier_log_attr_valid(const struct bpf_verifier_log *log) |
17 | { |
18 | /* ubuf and len_total should both be specified (or not) together */ |
19 | if (!!log->ubuf != !!log->len_total) |
20 | return false; |
21 | /* log buf without log_level is meaningless */ |
22 | if (log->ubuf && log->level == 0) |
23 | return false; |
24 | if (log->level & ~BPF_LOG_MASK) |
25 | return false; |
26 | if (log->len_total > UINT_MAX >> 2) |
27 | return false; |
28 | return true; |
29 | } |
30 | |
31 | int bpf_vlog_init(struct bpf_verifier_log *log, u32 log_level, |
32 | char __user *log_buf, u32 log_size) |
33 | { |
34 | log->level = log_level; |
35 | log->ubuf = log_buf; |
36 | log->len_total = log_size; |
37 | |
38 | /* log attributes have to be sane */ |
39 | if (!bpf_verifier_log_attr_valid(log)) |
40 | return -EINVAL; |
41 | |
42 | return 0; |
43 | } |
44 | |
45 | static void bpf_vlog_update_len_max(struct bpf_verifier_log *log, u32 add_len) |
46 | { |
47 | /* add_len includes terminal \0, so no need for +1. */ |
48 | u64 len = log->end_pos + add_len; |
49 | |
50 | /* log->len_max could be larger than our current len due to |
51 | * bpf_vlog_reset() calls, so we maintain the max of any length at any |
52 | * previous point |
53 | */ |
54 | if (len > UINT_MAX) |
55 | log->len_max = UINT_MAX; |
56 | else if (len > log->len_max) |
57 | log->len_max = len; |
58 | } |
59 | |
60 | void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt, |
61 | va_list args) |
62 | { |
63 | u64 cur_pos; |
64 | u32 new_n, n; |
65 | |
66 | n = vscnprintf(buf: log->kbuf, BPF_VERIFIER_TMP_LOG_SIZE, fmt, args); |
67 | |
68 | if (log->level == BPF_LOG_KERNEL) { |
69 | bool newline = n > 0 && log->kbuf[n - 1] == '\n'; |
70 | |
71 | pr_err("BPF: %s%s" , log->kbuf, newline ? "" : "\n" ); |
72 | return; |
73 | } |
74 | |
75 | n += 1; /* include terminating zero */ |
76 | bpf_vlog_update_len_max(log, add_len: n); |
77 | |
78 | if (log->level & BPF_LOG_FIXED) { |
79 | /* check if we have at least something to put into user buf */ |
80 | new_n = 0; |
81 | if (log->end_pos < log->len_total) { |
82 | new_n = min_t(u32, log->len_total - log->end_pos, n); |
83 | log->kbuf[new_n - 1] = '\0'; |
84 | } |
85 | |
86 | cur_pos = log->end_pos; |
87 | log->end_pos += n - 1; /* don't count terminating '\0' */ |
88 | |
89 | if (log->ubuf && new_n && |
90 | copy_to_user(to: log->ubuf + cur_pos, from: log->kbuf, n: new_n)) |
91 | goto fail; |
92 | } else { |
93 | u64 new_end, new_start; |
94 | u32 buf_start, buf_end, new_n; |
95 | |
96 | new_end = log->end_pos + n; |
97 | if (new_end - log->start_pos >= log->len_total) |
98 | new_start = new_end - log->len_total; |
99 | else |
100 | new_start = log->start_pos; |
101 | |
102 | log->start_pos = new_start; |
103 | log->end_pos = new_end - 1; /* don't count terminating '\0' */ |
104 | |
105 | if (!log->ubuf) |
106 | return; |
107 | |
108 | new_n = min(n, log->len_total); |
109 | cur_pos = new_end - new_n; |
110 | div_u64_rem(dividend: cur_pos, divisor: log->len_total, remainder: &buf_start); |
111 | div_u64_rem(dividend: new_end, divisor: log->len_total, remainder: &buf_end); |
112 | /* new_end and buf_end are exclusive indices, so if buf_end is |
113 | * exactly zero, then it actually points right to the end of |
114 | * ubuf and there is no wrap around |
115 | */ |
116 | if (buf_end == 0) |
117 | buf_end = log->len_total; |
118 | |
119 | /* if buf_start > buf_end, we wrapped around; |
120 | * if buf_start == buf_end, then we fill ubuf completely; we |
121 | * can't have buf_start == buf_end to mean that there is |
122 | * nothing to write, because we always write at least |
123 | * something, even if terminal '\0' |
124 | */ |
125 | if (buf_start < buf_end) { |
126 | /* message fits within contiguous chunk of ubuf */ |
127 | if (copy_to_user(to: log->ubuf + buf_start, |
128 | from: log->kbuf + n - new_n, |
129 | n: buf_end - buf_start)) |
130 | goto fail; |
131 | } else { |
132 | /* message wraps around the end of ubuf, copy in two chunks */ |
133 | if (copy_to_user(to: log->ubuf + buf_start, |
134 | from: log->kbuf + n - new_n, |
135 | n: log->len_total - buf_start)) |
136 | goto fail; |
137 | if (copy_to_user(to: log->ubuf, |
138 | from: log->kbuf + n - buf_end, |
139 | n: buf_end)) |
140 | goto fail; |
141 | } |
142 | } |
143 | |
144 | return; |
145 | fail: |
146 | log->ubuf = NULL; |
147 | } |
148 | |
149 | void bpf_vlog_reset(struct bpf_verifier_log *log, u64 new_pos) |
150 | { |
151 | char zero = 0; |
152 | u32 pos; |
153 | |
154 | if (WARN_ON_ONCE(new_pos > log->end_pos)) |
155 | return; |
156 | |
157 | if (!bpf_verifier_log_needed(log) || log->level == BPF_LOG_KERNEL) |
158 | return; |
159 | |
160 | /* if position to which we reset is beyond current log window, |
161 | * then we didn't preserve any useful content and should adjust |
162 | * start_pos to end up with an empty log (start_pos == end_pos) |
163 | */ |
164 | log->end_pos = new_pos; |
165 | if (log->end_pos < log->start_pos) |
166 | log->start_pos = log->end_pos; |
167 | |
168 | if (!log->ubuf) |
169 | return; |
170 | |
171 | if (log->level & BPF_LOG_FIXED) |
172 | pos = log->end_pos + 1; |
173 | else |
174 | div_u64_rem(dividend: new_pos, divisor: log->len_total, remainder: &pos); |
175 | |
176 | if (pos < log->len_total && put_user(zero, log->ubuf + pos)) |
177 | log->ubuf = NULL; |
178 | } |
179 | |
180 | static void bpf_vlog_reverse_kbuf(char *buf, int len) |
181 | { |
182 | int i, j; |
183 | |
184 | for (i = 0, j = len - 1; i < j; i++, j--) |
185 | swap(buf[i], buf[j]); |
186 | } |
187 | |
188 | static int bpf_vlog_reverse_ubuf(struct bpf_verifier_log *log, int start, int end) |
189 | { |
190 | /* we split log->kbuf into two equal parts for both ends of array */ |
191 | int n = sizeof(log->kbuf) / 2, nn; |
192 | char *lbuf = log->kbuf, *rbuf = log->kbuf + n; |
193 | |
194 | /* Read ubuf's section [start, end) two chunks at a time, from left |
195 | * and right side; within each chunk, swap all the bytes; after that |
196 | * reverse the order of lbuf and rbuf and write result back to ubuf. |
197 | * This way we'll end up with swapped contents of specified |
198 | * [start, end) ubuf segment. |
199 | */ |
200 | while (end - start > 1) { |
201 | nn = min(n, (end - start ) / 2); |
202 | |
203 | if (copy_from_user(to: lbuf, from: log->ubuf + start, n: nn)) |
204 | return -EFAULT; |
205 | if (copy_from_user(to: rbuf, from: log->ubuf + end - nn, n: nn)) |
206 | return -EFAULT; |
207 | |
208 | bpf_vlog_reverse_kbuf(buf: lbuf, len: nn); |
209 | bpf_vlog_reverse_kbuf(buf: rbuf, len: nn); |
210 | |
211 | /* we write lbuf to the right end of ubuf, while rbuf to the |
212 | * left one to end up with properly reversed overall ubuf |
213 | */ |
214 | if (copy_to_user(to: log->ubuf + start, from: rbuf, n: nn)) |
215 | return -EFAULT; |
216 | if (copy_to_user(to: log->ubuf + end - nn, from: lbuf, n: nn)) |
217 | return -EFAULT; |
218 | |
219 | start += nn; |
220 | end -= nn; |
221 | } |
222 | |
223 | return 0; |
224 | } |
225 | |
226 | int bpf_vlog_finalize(struct bpf_verifier_log *log, u32 *log_size_actual) |
227 | { |
228 | u32 sublen; |
229 | int err; |
230 | |
231 | *log_size_actual = 0; |
232 | if (!log || log->level == 0 || log->level == BPF_LOG_KERNEL) |
233 | return 0; |
234 | |
235 | if (!log->ubuf) |
236 | goto skip_log_rotate; |
237 | /* If we never truncated log, there is nothing to move around. */ |
238 | if (log->start_pos == 0) |
239 | goto skip_log_rotate; |
240 | |
241 | /* Otherwise we need to rotate log contents to make it start from the |
242 | * buffer beginning and be a continuous zero-terminated string. Note |
243 | * that if log->start_pos != 0 then we definitely filled up entire log |
244 | * buffer with no gaps, and we just need to shift buffer contents to |
245 | * the left by (log->start_pos % log->len_total) bytes. |
246 | * |
247 | * Unfortunately, user buffer could be huge and we don't want to |
248 | * allocate temporary kernel memory of the same size just to shift |
249 | * contents in a straightforward fashion. Instead, we'll be clever and |
250 | * do in-place array rotation. This is a leetcode-style problem, which |
251 | * could be solved by three rotations. |
252 | * |
253 | * Let's say we have log buffer that has to be shifted left by 7 bytes |
254 | * (spaces and vertical bar is just for demonstrative purposes): |
255 | * E F G H I J K | A B C D |
256 | * |
257 | * First, we reverse entire array: |
258 | * D C B A | K J I H G F E |
259 | * |
260 | * Then we rotate first 4 bytes (DCBA) and separately last 7 bytes |
261 | * (KJIHGFE), resulting in a properly rotated array: |
262 | * A B C D | E F G H I J K |
263 | * |
264 | * We'll utilize log->kbuf to read user memory chunk by chunk, swap |
265 | * bytes, and write them back. Doing it byte-by-byte would be |
266 | * unnecessarily inefficient. Altogether we are going to read and |
267 | * write each byte twice, for total 4 memory copies between kernel and |
268 | * user space. |
269 | */ |
270 | |
271 | /* length of the chopped off part that will be the beginning; |
272 | * len(ABCD) in the example above |
273 | */ |
274 | div_u64_rem(dividend: log->start_pos, divisor: log->len_total, remainder: &sublen); |
275 | sublen = log->len_total - sublen; |
276 | |
277 | err = bpf_vlog_reverse_ubuf(log, start: 0, end: log->len_total); |
278 | err = err ?: bpf_vlog_reverse_ubuf(log, start: 0, end: sublen); |
279 | err = err ?: bpf_vlog_reverse_ubuf(log, start: sublen, end: log->len_total); |
280 | if (err) |
281 | log->ubuf = NULL; |
282 | |
283 | skip_log_rotate: |
284 | *log_size_actual = log->len_max; |
285 | |
286 | /* properly initialized log has either both ubuf!=NULL and len_total>0 |
287 | * or ubuf==NULL and len_total==0, so if this condition doesn't hold, |
288 | * we got a fault somewhere along the way, so report it back |
289 | */ |
290 | if (!!log->ubuf != !!log->len_total) |
291 | return -EFAULT; |
292 | |
293 | /* did truncation actually happen? */ |
294 | if (log->ubuf && log->len_max > log->len_total) |
295 | return -ENOSPC; |
296 | |
297 | return 0; |
298 | } |
299 | |
300 | /* log_level controls verbosity level of eBPF verifier. |
301 | * bpf_verifier_log_write() is used to dump the verification trace to the log, |
302 | * so the user can figure out what's wrong with the program |
303 | */ |
304 | __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env, |
305 | const char *fmt, ...) |
306 | { |
307 | va_list args; |
308 | |
309 | if (!bpf_verifier_log_needed(log: &env->log)) |
310 | return; |
311 | |
312 | va_start(args, fmt); |
313 | bpf_verifier_vlog(log: &env->log, fmt, args); |
314 | va_end(args); |
315 | } |
316 | EXPORT_SYMBOL_GPL(bpf_verifier_log_write); |
317 | |
318 | __printf(2, 3) void bpf_log(struct bpf_verifier_log *log, |
319 | const char *fmt, ...) |
320 | { |
321 | va_list args; |
322 | |
323 | if (!bpf_verifier_log_needed(log)) |
324 | return; |
325 | |
326 | va_start(args, fmt); |
327 | bpf_verifier_vlog(log, fmt, args); |
328 | va_end(args); |
329 | } |
330 | EXPORT_SYMBOL_GPL(bpf_log); |
331 | |
332 | static const struct bpf_line_info * |
333 | find_linfo(const struct bpf_verifier_env *env, u32 insn_off) |
334 | { |
335 | const struct bpf_line_info *linfo; |
336 | const struct bpf_prog *prog; |
337 | u32 nr_linfo; |
338 | int l, r, m; |
339 | |
340 | prog = env->prog; |
341 | nr_linfo = prog->aux->nr_linfo; |
342 | |
343 | if (!nr_linfo || insn_off >= prog->len) |
344 | return NULL; |
345 | |
346 | linfo = prog->aux->linfo; |
347 | /* Loop invariant: linfo[l].insn_off <= insns_off. |
348 | * linfo[0].insn_off == 0 which always satisfies above condition. |
349 | * Binary search is searching for rightmost linfo entry that satisfies |
350 | * the above invariant, giving us the desired record that covers given |
351 | * instruction offset. |
352 | */ |
353 | l = 0; |
354 | r = nr_linfo - 1; |
355 | while (l < r) { |
356 | /* (r - l + 1) / 2 means we break a tie to the right, so if: |
357 | * l=1, r=2, linfo[l].insn_off <= insn_off, linfo[r].insn_off > insn_off, |
358 | * then m=2, we see that linfo[m].insn_off > insn_off, and so |
359 | * r becomes 1 and we exit the loop with correct l==1. |
360 | * If the tie was broken to the left, m=1 would end us up in |
361 | * an endless loop where l and m stay at 1 and r stays at 2. |
362 | */ |
363 | m = l + (r - l + 1) / 2; |
364 | if (linfo[m].insn_off <= insn_off) |
365 | l = m; |
366 | else |
367 | r = m - 1; |
368 | } |
369 | |
370 | return &linfo[l]; |
371 | } |
372 | |
373 | static const char *ltrim(const char *s) |
374 | { |
375 | while (isspace(*s)) |
376 | s++; |
377 | |
378 | return s; |
379 | } |
380 | |
381 | __printf(3, 4) void verbose_linfo(struct bpf_verifier_env *env, |
382 | u32 insn_off, |
383 | const char *prefix_fmt, ...) |
384 | { |
385 | const struct bpf_line_info *linfo, *prev_linfo; |
386 | const struct btf *btf; |
387 | const char *s, *fname; |
388 | |
389 | if (!bpf_verifier_log_needed(log: &env->log)) |
390 | return; |
391 | |
392 | prev_linfo = env->prev_linfo; |
393 | linfo = find_linfo(env, insn_off); |
394 | if (!linfo || linfo == prev_linfo) |
395 | return; |
396 | |
397 | /* It often happens that two separate linfo records point to the same |
398 | * source code line, but have differing column numbers. Given verifier |
399 | * log doesn't emit column information, from user perspective we just |
400 | * end up emitting the same source code line twice unnecessarily. |
401 | * So instead check that previous and current linfo record point to |
402 | * the same file (file_name_offs match) and the same line number, and |
403 | * avoid emitting duplicated source code line in such case. |
404 | */ |
405 | if (prev_linfo && linfo->file_name_off == prev_linfo->file_name_off && |
406 | BPF_LINE_INFO_LINE_NUM(linfo->line_col) == BPF_LINE_INFO_LINE_NUM(prev_linfo->line_col)) |
407 | return; |
408 | |
409 | if (prefix_fmt) { |
410 | va_list args; |
411 | |
412 | va_start(args, prefix_fmt); |
413 | bpf_verifier_vlog(log: &env->log, fmt: prefix_fmt, args); |
414 | va_end(args); |
415 | } |
416 | |
417 | btf = env->prog->aux->btf; |
418 | s = ltrim(s: btf_name_by_offset(btf, offset: linfo->line_off)); |
419 | verbose(env, "%s" , s); /* source code line */ |
420 | |
421 | s = btf_name_by_offset(btf, offset: linfo->file_name_off); |
422 | /* leave only file name */ |
423 | fname = strrchr(s, '/'); |
424 | fname = fname ? fname + 1 : s; |
425 | verbose(env, " @ %s:%u\n" , fname, BPF_LINE_INFO_LINE_NUM(linfo->line_col)); |
426 | |
427 | env->prev_linfo = linfo; |
428 | } |
429 | |
430 | static const char *btf_type_name(const struct btf *btf, u32 id) |
431 | { |
432 | return btf_name_by_offset(btf, offset: btf_type_by_id(btf, type_id: id)->name_off); |
433 | } |
434 | |
435 | /* string representation of 'enum bpf_reg_type' |
436 | * |
437 | * Note that reg_type_str() can not appear more than once in a single verbose() |
438 | * statement. |
439 | */ |
440 | const char *reg_type_str(struct bpf_verifier_env *env, enum bpf_reg_type type) |
441 | { |
442 | char postfix[16] = {0}, prefix[64] = {0}; |
443 | static const char * const str[] = { |
444 | [NOT_INIT] = "?" , |
445 | [SCALAR_VALUE] = "scalar" , |
446 | [PTR_TO_CTX] = "ctx" , |
447 | [CONST_PTR_TO_MAP] = "map_ptr" , |
448 | [PTR_TO_MAP_VALUE] = "map_value" , |
449 | [PTR_TO_STACK] = "fp" , |
450 | [PTR_TO_PACKET] = "pkt" , |
451 | [PTR_TO_PACKET_META] = "pkt_meta" , |
452 | [PTR_TO_PACKET_END] = "pkt_end" , |
453 | [PTR_TO_FLOW_KEYS] = "flow_keys" , |
454 | [PTR_TO_SOCKET] = "sock" , |
455 | [PTR_TO_SOCK_COMMON] = "sock_common" , |
456 | [PTR_TO_TCP_SOCK] = "tcp_sock" , |
457 | [PTR_TO_TP_BUFFER] = "tp_buffer" , |
458 | [PTR_TO_XDP_SOCK] = "xdp_sock" , |
459 | [PTR_TO_BTF_ID] = "ptr_" , |
460 | [PTR_TO_MEM] = "mem" , |
461 | [PTR_TO_ARENA] = "arena" , |
462 | [PTR_TO_BUF] = "buf" , |
463 | [PTR_TO_FUNC] = "func" , |
464 | [PTR_TO_MAP_KEY] = "map_key" , |
465 | [CONST_PTR_TO_DYNPTR] = "dynptr_ptr" , |
466 | }; |
467 | |
468 | if (type & PTR_MAYBE_NULL) { |
469 | if (base_type(type) == PTR_TO_BTF_ID) |
470 | strncpy(p: postfix, q: "or_null_" , size: 16); |
471 | else |
472 | strncpy(p: postfix, q: "_or_null" , size: 16); |
473 | } |
474 | |
475 | snprintf(buf: prefix, size: sizeof(prefix), fmt: "%s%s%s%s%s%s%s" , |
476 | type & MEM_RDONLY ? "rdonly_" : "" , |
477 | type & MEM_RINGBUF ? "ringbuf_" : "" , |
478 | type & MEM_USER ? "user_" : "" , |
479 | type & MEM_PERCPU ? "percpu_" : "" , |
480 | type & MEM_RCU ? "rcu_" : "" , |
481 | type & PTR_UNTRUSTED ? "untrusted_" : "" , |
482 | type & PTR_TRUSTED ? "trusted_" : "" |
483 | ); |
484 | |
485 | snprintf(buf: env->tmp_str_buf, TMP_STR_BUF_LEN, fmt: "%s%s%s" , |
486 | prefix, str[base_type(type)], postfix); |
487 | return env->tmp_str_buf; |
488 | } |
489 | |
490 | const char *dynptr_type_str(enum bpf_dynptr_type type) |
491 | { |
492 | switch (type) { |
493 | case BPF_DYNPTR_TYPE_LOCAL: |
494 | return "local" ; |
495 | case BPF_DYNPTR_TYPE_RINGBUF: |
496 | return "ringbuf" ; |
497 | case BPF_DYNPTR_TYPE_SKB: |
498 | return "skb" ; |
499 | case BPF_DYNPTR_TYPE_XDP: |
500 | return "xdp" ; |
501 | case BPF_DYNPTR_TYPE_INVALID: |
502 | return "<invalid>" ; |
503 | default: |
504 | WARN_ONCE(1, "unknown dynptr type %d\n" , type); |
505 | return "<unknown>" ; |
506 | } |
507 | } |
508 | |
509 | const char *iter_type_str(const struct btf *btf, u32 btf_id) |
510 | { |
511 | if (!btf || btf_id == 0) |
512 | return "<invalid>" ; |
513 | |
514 | /* we already validated that type is valid and has conforming name */ |
515 | return btf_type_name(btf, id: btf_id) + sizeof(ITER_PREFIX) - 1; |
516 | } |
517 | |
518 | const char *iter_state_str(enum bpf_iter_state state) |
519 | { |
520 | switch (state) { |
521 | case BPF_ITER_STATE_ACTIVE: |
522 | return "active" ; |
523 | case BPF_ITER_STATE_DRAINED: |
524 | return "drained" ; |
525 | case BPF_ITER_STATE_INVALID: |
526 | return "<invalid>" ; |
527 | default: |
528 | WARN_ONCE(1, "unknown iter state %d\n" , state); |
529 | return "<unknown>" ; |
530 | } |
531 | } |
532 | |
533 | static char slot_type_char[] = { |
534 | [STACK_INVALID] = '?', |
535 | [STACK_SPILL] = 'r', |
536 | [STACK_MISC] = 'm', |
537 | [STACK_ZERO] = '0', |
538 | [STACK_DYNPTR] = 'd', |
539 | [STACK_ITER] = 'i', |
540 | }; |
541 | |
542 | static void print_liveness(struct bpf_verifier_env *env, |
543 | enum bpf_reg_liveness live) |
544 | { |
545 | if (live & (REG_LIVE_READ | REG_LIVE_WRITTEN | REG_LIVE_DONE)) |
546 | verbose(env, "_" ); |
547 | if (live & REG_LIVE_READ) |
548 | verbose(env, "r" ); |
549 | if (live & REG_LIVE_WRITTEN) |
550 | verbose(env, "w" ); |
551 | if (live & REG_LIVE_DONE) |
552 | verbose(env, "D" ); |
553 | } |
554 | |
555 | #define UNUM_MAX_DECIMAL U16_MAX |
556 | #define SNUM_MAX_DECIMAL S16_MAX |
557 | #define SNUM_MIN_DECIMAL S16_MIN |
558 | |
559 | static bool is_unum_decimal(u64 num) |
560 | { |
561 | return num <= UNUM_MAX_DECIMAL; |
562 | } |
563 | |
564 | static bool is_snum_decimal(s64 num) |
565 | { |
566 | return num >= SNUM_MIN_DECIMAL && num <= SNUM_MAX_DECIMAL; |
567 | } |
568 | |
569 | static void verbose_unum(struct bpf_verifier_env *env, u64 num) |
570 | { |
571 | if (is_unum_decimal(num)) |
572 | verbose(env, "%llu" , num); |
573 | else |
574 | verbose(env, "%#llx" , num); |
575 | } |
576 | |
577 | static void verbose_snum(struct bpf_verifier_env *env, s64 num) |
578 | { |
579 | if (is_snum_decimal(num)) |
580 | verbose(env, "%lld" , num); |
581 | else |
582 | verbose(env, "%#llx" , num); |
583 | } |
584 | |
585 | int tnum_strn(char *str, size_t size, struct tnum a) |
586 | { |
587 | /* print as a constant, if tnum is fully known */ |
588 | if (a.mask == 0) { |
589 | if (is_unum_decimal(num: a.value)) |
590 | return snprintf(buf: str, size, fmt: "%llu" , a.value); |
591 | else |
592 | return snprintf(buf: str, size, fmt: "%#llx" , a.value); |
593 | } |
594 | return snprintf(buf: str, size, fmt: "(%#llx; %#llx)" , a.value, a.mask); |
595 | } |
596 | EXPORT_SYMBOL_GPL(tnum_strn); |
597 | |
598 | static void print_scalar_ranges(struct bpf_verifier_env *env, |
599 | const struct bpf_reg_state *reg, |
600 | const char **sep) |
601 | { |
602 | /* For signed ranges, we want to unify 64-bit and 32-bit values in the |
603 | * output as much as possible, but there is a bit of a complication. |
604 | * If we choose to print values as decimals, this is natural to do, |
605 | * because negative 64-bit and 32-bit values >= -S32_MIN have the same |
606 | * representation due to sign extension. But if we choose to print |
607 | * them in hex format (see is_snum_decimal()), then sign extension is |
608 | * misleading. |
609 | * E.g., smin=-2 and smin32=-2 are exactly the same in decimal, but in |
610 | * hex they will be smin=0xfffffffffffffffe and smin32=0xfffffffe, two |
611 | * very different numbers. |
612 | * So we avoid sign extension if we choose to print values in hex. |
613 | */ |
614 | struct { |
615 | const char *name; |
616 | u64 val; |
617 | bool omit; |
618 | } minmaxs[] = { |
619 | {"smin" , reg->smin_value, reg->smin_value == S64_MIN}, |
620 | {"smax" , reg->smax_value, reg->smax_value == S64_MAX}, |
621 | {"umin" , reg->umin_value, reg->umin_value == 0}, |
622 | {"umax" , reg->umax_value, reg->umax_value == U64_MAX}, |
623 | {"smin32" , |
624 | is_snum_decimal(num: (s64)reg->s32_min_value) |
625 | ? (s64)reg->s32_min_value |
626 | : (u32)reg->s32_min_value, reg->s32_min_value == S32_MIN}, |
627 | {"smax32" , |
628 | is_snum_decimal(num: (s64)reg->s32_max_value) |
629 | ? (s64)reg->s32_max_value |
630 | : (u32)reg->s32_max_value, reg->s32_max_value == S32_MAX}, |
631 | {"umin32" , reg->u32_min_value, reg->u32_min_value == 0}, |
632 | {"umax32" , reg->u32_max_value, reg->u32_max_value == U32_MAX}, |
633 | }, *m1, *m2, *mend = &minmaxs[ARRAY_SIZE(minmaxs)]; |
634 | bool neg1, neg2; |
635 | |
636 | for (m1 = &minmaxs[0]; m1 < mend; m1++) { |
637 | if (m1->omit) |
638 | continue; |
639 | |
640 | neg1 = m1->name[0] == 's' && (s64)m1->val < 0; |
641 | |
642 | verbose(env, "%s%s=" , *sep, m1->name); |
643 | *sep = "," ; |
644 | |
645 | for (m2 = m1 + 2; m2 < mend; m2 += 2) { |
646 | if (m2->omit || m2->val != m1->val) |
647 | continue; |
648 | /* don't mix negatives with positives */ |
649 | neg2 = m2->name[0] == 's' && (s64)m2->val < 0; |
650 | if (neg2 != neg1) |
651 | continue; |
652 | m2->omit = true; |
653 | verbose(env, "%s=" , m2->name); |
654 | } |
655 | |
656 | if (m1->name[0] == 's') |
657 | verbose_snum(env, num: m1->val); |
658 | else |
659 | verbose_unum(env, num: m1->val); |
660 | } |
661 | } |
662 | |
663 | static bool type_is_map_ptr(enum bpf_reg_type t) { |
664 | switch (base_type(type: t)) { |
665 | case CONST_PTR_TO_MAP: |
666 | case PTR_TO_MAP_KEY: |
667 | case PTR_TO_MAP_VALUE: |
668 | return true; |
669 | default: |
670 | return false; |
671 | } |
672 | } |
673 | |
674 | /* |
675 | * _a stands for append, was shortened to avoid multiline statements below. |
676 | * This macro is used to output a comma separated list of attributes. |
677 | */ |
678 | #define verbose_a(fmt, ...) ({ verbose(env, "%s" fmt, sep, ##__VA_ARGS__); sep = ","; }) |
679 | |
680 | static void print_reg_state(struct bpf_verifier_env *env, |
681 | const struct bpf_func_state *state, |
682 | const struct bpf_reg_state *reg) |
683 | { |
684 | enum bpf_reg_type t; |
685 | const char *sep = "" ; |
686 | |
687 | t = reg->type; |
688 | if (t == SCALAR_VALUE && reg->precise) |
689 | verbose(env, "P" ); |
690 | if (t == SCALAR_VALUE && tnum_is_const(a: reg->var_off)) { |
691 | /* reg->off should be 0 for SCALAR_VALUE */ |
692 | verbose_snum(env, num: reg->var_off.value + reg->off); |
693 | return; |
694 | } |
695 | |
696 | verbose(env, "%s" , reg_type_str(env, t)); |
697 | if (t == PTR_TO_ARENA) |
698 | return; |
699 | if (t == PTR_TO_STACK) { |
700 | if (state->frameno != reg->frameno) |
701 | verbose(env, "[%d]" , reg->frameno); |
702 | if (tnum_is_const(a: reg->var_off)) { |
703 | verbose_snum(env, num: reg->var_off.value + reg->off); |
704 | return; |
705 | } |
706 | } |
707 | if (base_type(type: t) == PTR_TO_BTF_ID) |
708 | verbose(env, "%s" , btf_type_name(reg->btf, reg->btf_id)); |
709 | verbose(env, "(" ); |
710 | if (reg->id) |
711 | verbose_a("id=%d" , reg->id); |
712 | if (reg->ref_obj_id) |
713 | verbose_a("ref_obj_id=%d" , reg->ref_obj_id); |
714 | if (type_is_non_owning_ref(type: reg->type)) |
715 | verbose_a("%s" , "non_own_ref" ); |
716 | if (type_is_map_ptr(t)) { |
717 | if (reg->map_ptr->name[0]) |
718 | verbose_a("map=%s" , reg->map_ptr->name); |
719 | verbose_a("ks=%d,vs=%d" , |
720 | reg->map_ptr->key_size, |
721 | reg->map_ptr->value_size); |
722 | } |
723 | if (t != SCALAR_VALUE && reg->off) { |
724 | verbose_a("off=" ); |
725 | verbose_snum(env, num: reg->off); |
726 | } |
727 | if (type_is_pkt_pointer(type: t)) { |
728 | verbose_a("r=" ); |
729 | verbose_unum(env, num: reg->range); |
730 | } |
731 | if (base_type(type: t) == PTR_TO_MEM) { |
732 | verbose_a("sz=" ); |
733 | verbose_unum(env, num: reg->mem_size); |
734 | } |
735 | if (t == CONST_PTR_TO_DYNPTR) |
736 | verbose_a("type=%s" , dynptr_type_str(reg->dynptr.type)); |
737 | if (tnum_is_const(a: reg->var_off)) { |
738 | /* a pointer register with fixed offset */ |
739 | if (reg->var_off.value) { |
740 | verbose_a("imm=" ); |
741 | verbose_snum(env, num: reg->var_off.value); |
742 | } |
743 | } else { |
744 | print_scalar_ranges(env, reg, sep: &sep); |
745 | if (!tnum_is_unknown(a: reg->var_off)) { |
746 | char tn_buf[48]; |
747 | |
748 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
749 | verbose_a("var_off=%s" , tn_buf); |
750 | } |
751 | } |
752 | verbose(env, ")" ); |
753 | } |
754 | |
755 | void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_func_state *state, |
756 | bool print_all) |
757 | { |
758 | const struct bpf_reg_state *reg; |
759 | int i; |
760 | |
761 | if (state->frameno) |
762 | verbose(env, " frame%d:" , state->frameno); |
763 | for (i = 0; i < MAX_BPF_REG; i++) { |
764 | reg = &state->regs[i]; |
765 | if (reg->type == NOT_INIT) |
766 | continue; |
767 | if (!print_all && !reg_scratched(env, regno: i)) |
768 | continue; |
769 | verbose(env, " R%d" , i); |
770 | print_liveness(env, live: reg->live); |
771 | verbose(env, "=" ); |
772 | print_reg_state(env, state, reg); |
773 | } |
774 | for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { |
775 | char types_buf[BPF_REG_SIZE + 1]; |
776 | const char *sep = "" ; |
777 | bool valid = false; |
778 | u8 slot_type; |
779 | int j; |
780 | |
781 | if (!print_all && !stack_slot_scratched(env, regno: i)) |
782 | continue; |
783 | |
784 | for (j = 0; j < BPF_REG_SIZE; j++) { |
785 | slot_type = state->stack[i].slot_type[j]; |
786 | if (slot_type != STACK_INVALID) |
787 | valid = true; |
788 | types_buf[j] = slot_type_char[slot_type]; |
789 | } |
790 | types_buf[BPF_REG_SIZE] = 0; |
791 | if (!valid) |
792 | continue; |
793 | |
794 | reg = &state->stack[i].spilled_ptr; |
795 | switch (state->stack[i].slot_type[BPF_REG_SIZE - 1]) { |
796 | case STACK_SPILL: |
797 | /* print MISC/ZERO/INVALID slots above subreg spill */ |
798 | for (j = 0; j < BPF_REG_SIZE; j++) |
799 | if (state->stack[i].slot_type[j] == STACK_SPILL) |
800 | break; |
801 | types_buf[j] = '\0'; |
802 | |
803 | verbose(env, " fp%d" , (-i - 1) * BPF_REG_SIZE); |
804 | print_liveness(env, live: reg->live); |
805 | verbose(env, "=%s" , types_buf); |
806 | print_reg_state(env, state, reg); |
807 | break; |
808 | case STACK_DYNPTR: |
809 | /* skip to main dynptr slot */ |
810 | i += BPF_DYNPTR_NR_SLOTS - 1; |
811 | reg = &state->stack[i].spilled_ptr; |
812 | |
813 | verbose(env, " fp%d" , (-i - 1) * BPF_REG_SIZE); |
814 | print_liveness(env, live: reg->live); |
815 | verbose(env, "=dynptr_%s(" , dynptr_type_str(reg->dynptr.type)); |
816 | if (reg->id) |
817 | verbose_a("id=%d" , reg->id); |
818 | if (reg->ref_obj_id) |
819 | verbose_a("ref_id=%d" , reg->ref_obj_id); |
820 | if (reg->dynptr_id) |
821 | verbose_a("dynptr_id=%d" , reg->dynptr_id); |
822 | verbose(env, ")" ); |
823 | break; |
824 | case STACK_ITER: |
825 | /* only main slot has ref_obj_id set; skip others */ |
826 | if (!reg->ref_obj_id) |
827 | continue; |
828 | |
829 | verbose(env, " fp%d" , (-i - 1) * BPF_REG_SIZE); |
830 | print_liveness(env, live: reg->live); |
831 | verbose(env, "=iter_%s(ref_id=%d,state=%s,depth=%u)" , |
832 | iter_type_str(reg->iter.btf, reg->iter.btf_id), |
833 | reg->ref_obj_id, iter_state_str(reg->iter.state), |
834 | reg->iter.depth); |
835 | break; |
836 | case STACK_MISC: |
837 | case STACK_ZERO: |
838 | default: |
839 | verbose(env, " fp%d" , (-i - 1) * BPF_REG_SIZE); |
840 | print_liveness(env, live: reg->live); |
841 | verbose(env, "=%s" , types_buf); |
842 | break; |
843 | } |
844 | } |
845 | if (state->acquired_refs && state->refs[0].id) { |
846 | verbose(env, " refs=%d" , state->refs[0].id); |
847 | for (i = 1; i < state->acquired_refs; i++) |
848 | if (state->refs[i].id) |
849 | verbose(env, ",%d" , state->refs[i].id); |
850 | } |
851 | if (state->in_callback_fn) |
852 | verbose(env, " cb" ); |
853 | if (state->in_async_callback_fn) |
854 | verbose(env, " async_cb" ); |
855 | verbose(env, "\n" ); |
856 | if (!print_all) |
857 | mark_verifier_state_clean(env); |
858 | } |
859 | |
860 | static inline u32 vlog_alignment(u32 pos) |
861 | { |
862 | return round_up(max(pos + BPF_LOG_MIN_ALIGNMENT / 2, BPF_LOG_ALIGNMENT), |
863 | BPF_LOG_MIN_ALIGNMENT) - pos - 1; |
864 | } |
865 | |
866 | void print_insn_state(struct bpf_verifier_env *env, const struct bpf_func_state *state) |
867 | { |
868 | if (env->prev_log_pos && env->prev_log_pos == env->log.end_pos) { |
869 | /* remove new line character */ |
870 | bpf_vlog_reset(log: &env->log, new_pos: env->prev_log_pos - 1); |
871 | verbose(env, "%*c;" , vlog_alignment(env->prev_insn_print_pos), ' '); |
872 | } else { |
873 | verbose(env, "%d:" , env->insn_idx); |
874 | } |
875 | print_verifier_state(env, state, print_all: false); |
876 | } |
877 | |