1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * KCSAN test with various race scenarious to test runtime behaviour. Since the |
4 | * interface with which KCSAN's reports are obtained is via the console, this is |
5 | * the output we should verify. For each test case checks the presence (or |
6 | * absence) of generated reports. Relies on 'console' tracepoint to capture |
7 | * reports as they appear in the kernel log. |
8 | * |
9 | * Makes use of KUnit for test organization, and the Torture framework for test |
10 | * thread control. |
11 | * |
12 | * Copyright (C) 2020, Google LLC. |
13 | * Author: Marco Elver <elver@google.com> |
14 | */ |
15 | |
16 | #define pr_fmt(fmt) "kcsan_test: " fmt |
17 | |
18 | #include <kunit/test.h> |
19 | #include <linux/atomic.h> |
20 | #include <linux/bitops.h> |
21 | #include <linux/jiffies.h> |
22 | #include <linux/kcsan-checks.h> |
23 | #include <linux/kernel.h> |
24 | #include <linux/mutex.h> |
25 | #include <linux/sched.h> |
26 | #include <linux/seqlock.h> |
27 | #include <linux/spinlock.h> |
28 | #include <linux/string.h> |
29 | #include <linux/timer.h> |
30 | #include <linux/torture.h> |
31 | #include <linux/tracepoint.h> |
32 | #include <linux/types.h> |
33 | #include <trace/events/printk.h> |
34 | |
35 | #define KCSAN_TEST_REQUIRES(test, cond) do { \ |
36 | if (!(cond)) \ |
37 | kunit_skip((test), "Test requires: " #cond); \ |
38 | } while (0) |
39 | |
40 | #ifdef CONFIG_CC_HAS_TSAN_COMPOUND_READ_BEFORE_WRITE |
41 | #define __KCSAN_ACCESS_RW(alt) (KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE) |
42 | #else |
43 | #define __KCSAN_ACCESS_RW(alt) (alt) |
44 | #endif |
45 | |
46 | /* Points to current test-case memory access "kernels". */ |
47 | static void (*access_kernels[2])(void); |
48 | |
49 | static struct task_struct **threads; /* Lists of threads. */ |
50 | static unsigned long end_time; /* End time of test. */ |
51 | |
52 | /* Report as observed from console. */ |
53 | static struct { |
54 | spinlock_t lock; |
55 | int nlines; |
56 | char lines[3][512]; |
57 | } observed = { |
58 | .lock = __SPIN_LOCK_UNLOCKED(observed.lock), |
59 | }; |
60 | |
61 | /* Setup test checking loop. */ |
62 | static __no_kcsan inline void |
63 | begin_test_checks(void (*func1)(void), void (*func2)(void)) |
64 | { |
65 | kcsan_disable_current(); |
66 | |
67 | /* |
68 | * Require at least as long as KCSAN_REPORT_ONCE_IN_MS, to ensure at |
69 | * least one race is reported. |
70 | */ |
71 | end_time = jiffies + msecs_to_jiffies(CONFIG_KCSAN_REPORT_ONCE_IN_MS + 500); |
72 | |
73 | /* Signal start; release potential initialization of shared data. */ |
74 | smp_store_release(&access_kernels[0], func1); |
75 | smp_store_release(&access_kernels[1], func2); |
76 | } |
77 | |
78 | /* End test checking loop. */ |
79 | static __no_kcsan inline bool |
80 | end_test_checks(bool stop) |
81 | { |
82 | if (!stop && time_before(jiffies, end_time)) { |
83 | /* Continue checking */ |
84 | might_sleep(); |
85 | return false; |
86 | } |
87 | |
88 | kcsan_enable_current(); |
89 | return true; |
90 | } |
91 | |
92 | /* |
93 | * Probe for console output: checks if a race was reported, and obtains observed |
94 | * lines of interest. |
95 | */ |
96 | __no_kcsan |
97 | static void probe_console(void *ignore, const char *buf, size_t len) |
98 | { |
99 | unsigned long flags; |
100 | int nlines; |
101 | |
102 | /* |
103 | * Note that KCSAN reports under a global lock, so we do not risk the |
104 | * possibility of having multiple reports interleaved. If that were the |
105 | * case, we'd expect tests to fail. |
106 | */ |
107 | |
108 | spin_lock_irqsave(&observed.lock, flags); |
109 | nlines = observed.nlines; |
110 | |
111 | if (strnstr(buf, "BUG: KCSAN: " , len) && strnstr(buf, "test_" , len)) { |
112 | /* |
113 | * KCSAN report and related to the test. |
114 | * |
115 | * The provided @buf is not NUL-terminated; copy no more than |
116 | * @len bytes and let strscpy() add the missing NUL-terminator. |
117 | */ |
118 | strscpy(observed.lines[0], buf, min(len + 1, sizeof(observed.lines[0]))); |
119 | nlines = 1; |
120 | } else if ((nlines == 1 || nlines == 2) && strnstr(buf, "bytes by" , len)) { |
121 | strscpy(observed.lines[nlines++], buf, min(len + 1, sizeof(observed.lines[0]))); |
122 | |
123 | if (strnstr(buf, "race at unknown origin" , len)) { |
124 | if (WARN_ON(nlines != 2)) |
125 | goto out; |
126 | |
127 | /* No second line of interest. */ |
128 | strcpy(p: observed.lines[nlines++], q: "<none>" ); |
129 | } |
130 | } |
131 | |
132 | out: |
133 | WRITE_ONCE(observed.nlines, nlines); /* Publish new nlines. */ |
134 | spin_unlock_irqrestore(lock: &observed.lock, flags); |
135 | } |
136 | |
137 | /* Check if a report related to the test exists. */ |
138 | __no_kcsan |
139 | static bool report_available(void) |
140 | { |
141 | return READ_ONCE(observed.nlines) == ARRAY_SIZE(observed.lines); |
142 | } |
143 | |
144 | /* Report information we expect in a report. */ |
145 | struct expect_report { |
146 | /* Access information of both accesses. */ |
147 | struct { |
148 | void *fn; /* Function pointer to expected function of top frame. */ |
149 | void *addr; /* Address of access; unchecked if NULL. */ |
150 | size_t size; /* Size of access; unchecked if @addr is NULL. */ |
151 | int type; /* Access type, see KCSAN_ACCESS definitions. */ |
152 | } access[2]; |
153 | }; |
154 | |
155 | /* Check observed report matches information in @r. */ |
156 | __no_kcsan |
157 | static bool __report_matches(const struct expect_report *r) |
158 | { |
159 | const bool is_assert = (r->access[0].type | r->access[1].type) & KCSAN_ACCESS_ASSERT; |
160 | bool ret = false; |
161 | unsigned long flags; |
162 | typeof(*observed.lines) *expect; |
163 | const char *end; |
164 | char *cur; |
165 | int i; |
166 | |
167 | /* Doubled-checked locking. */ |
168 | if (!report_available()) |
169 | return false; |
170 | |
171 | expect = kmalloc(size: sizeof(observed.lines), GFP_KERNEL); |
172 | if (WARN_ON(!expect)) |
173 | return false; |
174 | |
175 | /* Generate expected report contents. */ |
176 | |
177 | /* Title */ |
178 | cur = expect[0]; |
179 | end = &expect[0][sizeof(expect[0]) - 1]; |
180 | cur += scnprintf(buf: cur, size: end - cur, fmt: "BUG: KCSAN: %s in " , |
181 | is_assert ? "assert: race" : "data-race" ); |
182 | if (r->access[1].fn) { |
183 | char tmp[2][64]; |
184 | int cmp; |
185 | |
186 | /* Expect lexographically sorted function names in title. */ |
187 | scnprintf(buf: tmp[0], size: sizeof(tmp[0]), fmt: "%pS" , r->access[0].fn); |
188 | scnprintf(buf: tmp[1], size: sizeof(tmp[1]), fmt: "%pS" , r->access[1].fn); |
189 | cmp = strcmp(tmp[0], tmp[1]); |
190 | cur += scnprintf(buf: cur, size: end - cur, fmt: "%ps / %ps" , |
191 | cmp < 0 ? r->access[0].fn : r->access[1].fn, |
192 | cmp < 0 ? r->access[1].fn : r->access[0].fn); |
193 | } else { |
194 | scnprintf(buf: cur, size: end - cur, fmt: "%pS" , r->access[0].fn); |
195 | /* The exact offset won't match, remove it. */ |
196 | cur = strchr(expect[0], '+'); |
197 | if (cur) |
198 | *cur = '\0'; |
199 | } |
200 | |
201 | /* Access 1 */ |
202 | cur = expect[1]; |
203 | end = &expect[1][sizeof(expect[1]) - 1]; |
204 | if (!r->access[1].fn) |
205 | cur += scnprintf(buf: cur, size: end - cur, fmt: "race at unknown origin, with " ); |
206 | |
207 | /* Access 1 & 2 */ |
208 | for (i = 0; i < 2; ++i) { |
209 | const int ty = r->access[i].type; |
210 | const char *const access_type = |
211 | (ty & KCSAN_ACCESS_ASSERT) ? |
212 | ((ty & KCSAN_ACCESS_WRITE) ? |
213 | "assert no accesses" : |
214 | "assert no writes" ) : |
215 | ((ty & KCSAN_ACCESS_WRITE) ? |
216 | ((ty & KCSAN_ACCESS_COMPOUND) ? |
217 | "read-write" : |
218 | "write" ) : |
219 | "read" ); |
220 | const bool is_atomic = (ty & KCSAN_ACCESS_ATOMIC); |
221 | const bool is_scoped = (ty & KCSAN_ACCESS_SCOPED); |
222 | const char *const access_type_aux = |
223 | (is_atomic && is_scoped) ? " (marked, reordered)" |
224 | : (is_atomic ? " (marked)" |
225 | : (is_scoped ? " (reordered)" : "" )); |
226 | |
227 | if (i == 1) { |
228 | /* Access 2 */ |
229 | cur = expect[2]; |
230 | end = &expect[2][sizeof(expect[2]) - 1]; |
231 | |
232 | if (!r->access[1].fn) { |
233 | /* Dummy string if no second access is available. */ |
234 | strcpy(p: cur, q: "<none>" ); |
235 | break; |
236 | } |
237 | } |
238 | |
239 | cur += scnprintf(buf: cur, size: end - cur, fmt: "%s%s to " , access_type, |
240 | access_type_aux); |
241 | |
242 | if (r->access[i].addr) /* Address is optional. */ |
243 | cur += scnprintf(buf: cur, size: end - cur, fmt: "0x%px of %zu bytes" , |
244 | r->access[i].addr, r->access[i].size); |
245 | } |
246 | |
247 | spin_lock_irqsave(&observed.lock, flags); |
248 | if (!report_available()) |
249 | goto out; /* A new report is being captured. */ |
250 | |
251 | /* Finally match expected output to what we actually observed. */ |
252 | ret = strstr(observed.lines[0], expect[0]) && |
253 | /* Access info may appear in any order. */ |
254 | ((strstr(observed.lines[1], expect[1]) && |
255 | strstr(observed.lines[2], expect[2])) || |
256 | (strstr(observed.lines[1], expect[2]) && |
257 | strstr(observed.lines[2], expect[1]))); |
258 | out: |
259 | spin_unlock_irqrestore(lock: &observed.lock, flags); |
260 | kfree(objp: expect); |
261 | return ret; |
262 | } |
263 | |
264 | static __always_inline const struct expect_report * |
265 | __report_set_scoped(struct expect_report *r, int accesses) |
266 | { |
267 | BUILD_BUG_ON(accesses > 3); |
268 | |
269 | if (accesses & 1) |
270 | r->access[0].type |= KCSAN_ACCESS_SCOPED; |
271 | else |
272 | r->access[0].type &= ~KCSAN_ACCESS_SCOPED; |
273 | |
274 | if (accesses & 2) |
275 | r->access[1].type |= KCSAN_ACCESS_SCOPED; |
276 | else |
277 | r->access[1].type &= ~KCSAN_ACCESS_SCOPED; |
278 | |
279 | return r; |
280 | } |
281 | |
282 | __no_kcsan |
283 | static bool report_matches_any_reordered(struct expect_report *r) |
284 | { |
285 | return __report_matches(r: __report_set_scoped(r, accesses: 0)) || |
286 | __report_matches(r: __report_set_scoped(r, accesses: 1)) || |
287 | __report_matches(r: __report_set_scoped(r, accesses: 2)) || |
288 | __report_matches(r: __report_set_scoped(r, accesses: 3)); |
289 | } |
290 | |
291 | #ifdef CONFIG_KCSAN_WEAK_MEMORY |
292 | /* Due to reordering accesses, any access may appear as "(reordered)". */ |
293 | #define report_matches report_matches_any_reordered |
294 | #else |
295 | #define report_matches __report_matches |
296 | #endif |
297 | |
298 | /* ===== Test kernels ===== */ |
299 | |
300 | static long test_sink; |
301 | static long test_var; |
302 | /* @test_array should be large enough to fall into multiple watchpoint slots. */ |
303 | static long test_array[3 * PAGE_SIZE / sizeof(long)]; |
304 | static struct { |
305 | long val[8]; |
306 | } test_struct; |
307 | static DEFINE_SEQLOCK(test_seqlock); |
308 | static DEFINE_SPINLOCK(test_spinlock); |
309 | static DEFINE_MUTEX(test_mutex); |
310 | |
311 | /* |
312 | * Helper to avoid compiler optimizing out reads, and to generate source values |
313 | * for writes. |
314 | */ |
315 | __no_kcsan |
316 | static noinline void sink_value(long v) { WRITE_ONCE(test_sink, v); } |
317 | |
318 | /* |
319 | * Generates a delay and some accesses that enter the runtime but do not produce |
320 | * data races. |
321 | */ |
322 | static noinline void test_delay(int iter) |
323 | { |
324 | while (iter--) |
325 | sink_value(READ_ONCE(test_sink)); |
326 | } |
327 | |
328 | static noinline void test_kernel_read(void) { sink_value(v: test_var); } |
329 | |
330 | static noinline void test_kernel_write(void) |
331 | { |
332 | test_var = READ_ONCE_NOCHECK(test_sink) + 1; |
333 | } |
334 | |
335 | static noinline void test_kernel_write_nochange(void) { test_var = 42; } |
336 | |
337 | /* Suffixed by value-change exception filter. */ |
338 | static noinline void test_kernel_write_nochange_rcu(void) { test_var = 42; } |
339 | |
340 | static noinline void test_kernel_read_atomic(void) |
341 | { |
342 | sink_value(READ_ONCE(test_var)); |
343 | } |
344 | |
345 | static noinline void test_kernel_write_atomic(void) |
346 | { |
347 | WRITE_ONCE(test_var, READ_ONCE_NOCHECK(test_sink) + 1); |
348 | } |
349 | |
350 | static noinline void test_kernel_atomic_rmw(void) |
351 | { |
352 | /* Use builtin, so we can set up the "bad" atomic/non-atomic scenario. */ |
353 | __atomic_fetch_add(&test_var, 1, __ATOMIC_RELAXED); |
354 | } |
355 | |
356 | __no_kcsan |
357 | static noinline void test_kernel_write_uninstrumented(void) { test_var++; } |
358 | |
359 | static noinline void test_kernel_data_race(void) { data_race(test_var++); } |
360 | |
361 | static noinline void test_kernel_assert_writer(void) |
362 | { |
363 | ASSERT_EXCLUSIVE_WRITER(test_var); |
364 | } |
365 | |
366 | static noinline void test_kernel_assert_access(void) |
367 | { |
368 | ASSERT_EXCLUSIVE_ACCESS(test_var); |
369 | } |
370 | |
371 | #define TEST_CHANGE_BITS 0xff00ff00 |
372 | |
373 | static noinline void test_kernel_change_bits(void) |
374 | { |
375 | if (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) { |
376 | /* |
377 | * Avoid race of unknown origin for this test, just pretend they |
378 | * are atomic. |
379 | */ |
380 | kcsan_nestable_atomic_begin(); |
381 | test_var ^= TEST_CHANGE_BITS; |
382 | kcsan_nestable_atomic_end(); |
383 | } else |
384 | WRITE_ONCE(test_var, READ_ONCE(test_var) ^ TEST_CHANGE_BITS); |
385 | } |
386 | |
387 | static noinline void test_kernel_assert_bits_change(void) |
388 | { |
389 | ASSERT_EXCLUSIVE_BITS(test_var, TEST_CHANGE_BITS); |
390 | } |
391 | |
392 | static noinline void test_kernel_assert_bits_nochange(void) |
393 | { |
394 | ASSERT_EXCLUSIVE_BITS(test_var, ~TEST_CHANGE_BITS); |
395 | } |
396 | |
397 | /* |
398 | * Scoped assertions do trigger anywhere in scope. However, the report should |
399 | * still only point at the start of the scope. |
400 | */ |
401 | static noinline void test_enter_scope(void) |
402 | { |
403 | int x = 0; |
404 | |
405 | /* Unrelated accesses to scoped assert. */ |
406 | READ_ONCE(test_sink); |
407 | kcsan_check_read(&x, sizeof(x)); |
408 | } |
409 | |
410 | static noinline void test_kernel_assert_writer_scoped(void) |
411 | { |
412 | ASSERT_EXCLUSIVE_WRITER_SCOPED(test_var); |
413 | test_enter_scope(); |
414 | } |
415 | |
416 | static noinline void test_kernel_assert_access_scoped(void) |
417 | { |
418 | ASSERT_EXCLUSIVE_ACCESS_SCOPED(test_var); |
419 | test_enter_scope(); |
420 | } |
421 | |
422 | static noinline void test_kernel_rmw_array(void) |
423 | { |
424 | int i; |
425 | |
426 | for (i = 0; i < ARRAY_SIZE(test_array); ++i) |
427 | test_array[i]++; |
428 | } |
429 | |
430 | static noinline void test_kernel_write_struct(void) |
431 | { |
432 | kcsan_check_write(&test_struct, sizeof(test_struct)); |
433 | kcsan_disable_current(); |
434 | test_struct.val[3]++; /* induce value change */ |
435 | kcsan_enable_current(); |
436 | } |
437 | |
438 | static noinline void test_kernel_write_struct_part(void) |
439 | { |
440 | test_struct.val[3] = 42; |
441 | } |
442 | |
443 | static noinline void test_kernel_read_struct_zero_size(void) |
444 | { |
445 | kcsan_check_read(&test_struct.val[3], 0); |
446 | } |
447 | |
448 | static noinline void test_kernel_jiffies_reader(void) |
449 | { |
450 | sink_value(v: (long)jiffies); |
451 | } |
452 | |
453 | static noinline void test_kernel_seqlock_reader(void) |
454 | { |
455 | unsigned int seq; |
456 | |
457 | do { |
458 | seq = read_seqbegin(sl: &test_seqlock); |
459 | sink_value(v: test_var); |
460 | } while (read_seqretry(sl: &test_seqlock, start: seq)); |
461 | } |
462 | |
463 | static noinline void test_kernel_seqlock_writer(void) |
464 | { |
465 | unsigned long flags; |
466 | |
467 | write_seqlock_irqsave(&test_seqlock, flags); |
468 | test_var++; |
469 | write_sequnlock_irqrestore(sl: &test_seqlock, flags); |
470 | } |
471 | |
472 | static noinline void test_kernel_atomic_builtins(void) |
473 | { |
474 | /* |
475 | * Generate concurrent accesses, expecting no reports, ensuring KCSAN |
476 | * treats builtin atomics as actually atomic. |
477 | */ |
478 | __atomic_load_n(&test_var, __ATOMIC_RELAXED); |
479 | } |
480 | |
481 | static noinline void test_kernel_xor_1bit(void) |
482 | { |
483 | /* Do not report data races between the read-writes. */ |
484 | kcsan_nestable_atomic_begin(); |
485 | test_var ^= 0x10000; |
486 | kcsan_nestable_atomic_end(); |
487 | } |
488 | |
489 | #define TEST_KERNEL_LOCKED(name, acquire, release) \ |
490 | static noinline void test_kernel_##name(void) \ |
491 | { \ |
492 | long *flag = &test_struct.val[0]; \ |
493 | long v = 0; \ |
494 | if (!(acquire)) \ |
495 | return; \ |
496 | while (v++ < 100) { \ |
497 | test_var++; \ |
498 | barrier(); \ |
499 | } \ |
500 | release; \ |
501 | test_delay(10); \ |
502 | } |
503 | |
504 | TEST_KERNEL_LOCKED(with_memorder, |
505 | cmpxchg_acquire(flag, 0, 1) == 0, |
506 | smp_store_release(flag, 0)); |
507 | TEST_KERNEL_LOCKED(wrong_memorder, |
508 | cmpxchg_relaxed(flag, 0, 1) == 0, |
509 | WRITE_ONCE(*flag, 0)); |
510 | TEST_KERNEL_LOCKED(atomic_builtin_with_memorder, |
511 | __atomic_compare_exchange_n(flag, &v, 1, 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED), |
512 | __atomic_store_n(flag, 0, __ATOMIC_RELEASE)); |
513 | TEST_KERNEL_LOCKED(atomic_builtin_wrong_memorder, |
514 | __atomic_compare_exchange_n(flag, &v, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED), |
515 | __atomic_store_n(flag, 0, __ATOMIC_RELAXED)); |
516 | |
517 | /* ===== Test cases ===== */ |
518 | |
519 | /* |
520 | * Tests that various barriers have the expected effect on internal state. Not |
521 | * exhaustive on atomic_t operations. Unlike the selftest, also checks for |
522 | * too-strict barrier instrumentation; these can be tolerated, because it does |
523 | * not cause false positives, but at least we should be aware of such cases. |
524 | */ |
525 | static void test_barrier_nothreads(struct kunit *test) |
526 | { |
527 | #ifdef CONFIG_KCSAN_WEAK_MEMORY |
528 | struct kcsan_scoped_access *reorder_access = ¤t->kcsan_ctx.reorder_access; |
529 | #else |
530 | struct kcsan_scoped_access *reorder_access = NULL; |
531 | #endif |
532 | arch_spinlock_t arch_spinlock = __ARCH_SPIN_LOCK_UNLOCKED; |
533 | atomic_t dummy; |
534 | |
535 | KCSAN_TEST_REQUIRES(test, reorder_access != NULL); |
536 | KCSAN_TEST_REQUIRES(test, IS_ENABLED(CONFIG_SMP)); |
537 | |
538 | #define __KCSAN_EXPECT_BARRIER(access_type, barrier, order_before, name) \ |
539 | do { \ |
540 | reorder_access->type = (access_type) | KCSAN_ACCESS_SCOPED; \ |
541 | reorder_access->size = sizeof(test_var); \ |
542 | barrier; \ |
543 | KUNIT_EXPECT_EQ_MSG(test, reorder_access->size, \ |
544 | order_before ? 0 : sizeof(test_var), \ |
545 | "improperly instrumented type=(" #access_type "): " name); \ |
546 | } while (0) |
547 | #define KCSAN_EXPECT_READ_BARRIER(b, o) __KCSAN_EXPECT_BARRIER(0, b, o, #b) |
548 | #define KCSAN_EXPECT_WRITE_BARRIER(b, o) __KCSAN_EXPECT_BARRIER(KCSAN_ACCESS_WRITE, b, o, #b) |
549 | #define KCSAN_EXPECT_RW_BARRIER(b, o) __KCSAN_EXPECT_BARRIER(KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE, b, o, #b) |
550 | |
551 | /* |
552 | * Lockdep initialization can strengthen certain locking operations due |
553 | * to calling into instrumented files; "warm up" our locks. |
554 | */ |
555 | spin_lock(lock: &test_spinlock); |
556 | spin_unlock(lock: &test_spinlock); |
557 | mutex_lock(&test_mutex); |
558 | mutex_unlock(lock: &test_mutex); |
559 | |
560 | /* Force creating a valid entry in reorder_access first. */ |
561 | test_var = 0; |
562 | while (test_var++ < 1000000 && reorder_access->size != sizeof(test_var)) |
563 | __kcsan_check_read(&test_var, sizeof(test_var)); |
564 | KUNIT_ASSERT_EQ(test, reorder_access->size, sizeof(test_var)); |
565 | |
566 | kcsan_nestable_atomic_begin(); /* No watchpoints in called functions. */ |
567 | |
568 | KCSAN_EXPECT_READ_BARRIER(mb(), true); |
569 | KCSAN_EXPECT_READ_BARRIER(wmb(), false); |
570 | KCSAN_EXPECT_READ_BARRIER(rmb(), true); |
571 | KCSAN_EXPECT_READ_BARRIER(smp_mb(), true); |
572 | KCSAN_EXPECT_READ_BARRIER(smp_wmb(), false); |
573 | KCSAN_EXPECT_READ_BARRIER(smp_rmb(), true); |
574 | KCSAN_EXPECT_READ_BARRIER(dma_wmb(), false); |
575 | KCSAN_EXPECT_READ_BARRIER(dma_rmb(), true); |
576 | KCSAN_EXPECT_READ_BARRIER(smp_mb__before_atomic(), true); |
577 | KCSAN_EXPECT_READ_BARRIER(smp_mb__after_atomic(), true); |
578 | KCSAN_EXPECT_READ_BARRIER(smp_mb__after_spinlock(), true); |
579 | KCSAN_EXPECT_READ_BARRIER(smp_store_mb(test_var, 0), true); |
580 | KCSAN_EXPECT_READ_BARRIER(smp_load_acquire(&test_var), false); |
581 | KCSAN_EXPECT_READ_BARRIER(smp_store_release(&test_var, 0), true); |
582 | KCSAN_EXPECT_READ_BARRIER(xchg(&test_var, 0), true); |
583 | KCSAN_EXPECT_READ_BARRIER(xchg_release(&test_var, 0), true); |
584 | KCSAN_EXPECT_READ_BARRIER(xchg_relaxed(&test_var, 0), false); |
585 | KCSAN_EXPECT_READ_BARRIER(cmpxchg(&test_var, 0, 0), true); |
586 | KCSAN_EXPECT_READ_BARRIER(cmpxchg_release(&test_var, 0, 0), true); |
587 | KCSAN_EXPECT_READ_BARRIER(cmpxchg_relaxed(&test_var, 0, 0), false); |
588 | KCSAN_EXPECT_READ_BARRIER(atomic_read(&dummy), false); |
589 | KCSAN_EXPECT_READ_BARRIER(atomic_read_acquire(&dummy), false); |
590 | KCSAN_EXPECT_READ_BARRIER(atomic_set(&dummy, 0), false); |
591 | KCSAN_EXPECT_READ_BARRIER(atomic_set_release(&dummy, 0), true); |
592 | KCSAN_EXPECT_READ_BARRIER(atomic_add(1, &dummy), false); |
593 | KCSAN_EXPECT_READ_BARRIER(atomic_add_return(1, &dummy), true); |
594 | KCSAN_EXPECT_READ_BARRIER(atomic_add_return_acquire(1, &dummy), false); |
595 | KCSAN_EXPECT_READ_BARRIER(atomic_add_return_release(1, &dummy), true); |
596 | KCSAN_EXPECT_READ_BARRIER(atomic_add_return_relaxed(1, &dummy), false); |
597 | KCSAN_EXPECT_READ_BARRIER(atomic_fetch_add(1, &dummy), true); |
598 | KCSAN_EXPECT_READ_BARRIER(atomic_fetch_add_acquire(1, &dummy), false); |
599 | KCSAN_EXPECT_READ_BARRIER(atomic_fetch_add_release(1, &dummy), true); |
600 | KCSAN_EXPECT_READ_BARRIER(atomic_fetch_add_relaxed(1, &dummy), false); |
601 | KCSAN_EXPECT_READ_BARRIER(test_and_set_bit(0, &test_var), true); |
602 | KCSAN_EXPECT_READ_BARRIER(test_and_clear_bit(0, &test_var), true); |
603 | KCSAN_EXPECT_READ_BARRIER(test_and_change_bit(0, &test_var), true); |
604 | KCSAN_EXPECT_READ_BARRIER(clear_bit_unlock(0, &test_var), true); |
605 | KCSAN_EXPECT_READ_BARRIER(__clear_bit_unlock(0, &test_var), true); |
606 | KCSAN_EXPECT_READ_BARRIER(arch_spin_lock(&arch_spinlock), false); |
607 | KCSAN_EXPECT_READ_BARRIER(arch_spin_unlock(&arch_spinlock), true); |
608 | KCSAN_EXPECT_READ_BARRIER(spin_lock(&test_spinlock), false); |
609 | KCSAN_EXPECT_READ_BARRIER(spin_unlock(&test_spinlock), true); |
610 | KCSAN_EXPECT_READ_BARRIER(mutex_lock(&test_mutex), false); |
611 | KCSAN_EXPECT_READ_BARRIER(mutex_unlock(&test_mutex), true); |
612 | |
613 | KCSAN_EXPECT_WRITE_BARRIER(mb(), true); |
614 | KCSAN_EXPECT_WRITE_BARRIER(wmb(), true); |
615 | KCSAN_EXPECT_WRITE_BARRIER(rmb(), false); |
616 | KCSAN_EXPECT_WRITE_BARRIER(smp_mb(), true); |
617 | KCSAN_EXPECT_WRITE_BARRIER(smp_wmb(), true); |
618 | KCSAN_EXPECT_WRITE_BARRIER(smp_rmb(), false); |
619 | KCSAN_EXPECT_WRITE_BARRIER(dma_wmb(), true); |
620 | KCSAN_EXPECT_WRITE_BARRIER(dma_rmb(), false); |
621 | KCSAN_EXPECT_WRITE_BARRIER(smp_mb__before_atomic(), true); |
622 | KCSAN_EXPECT_WRITE_BARRIER(smp_mb__after_atomic(), true); |
623 | KCSAN_EXPECT_WRITE_BARRIER(smp_mb__after_spinlock(), true); |
624 | KCSAN_EXPECT_WRITE_BARRIER(smp_store_mb(test_var, 0), true); |
625 | KCSAN_EXPECT_WRITE_BARRIER(smp_load_acquire(&test_var), false); |
626 | KCSAN_EXPECT_WRITE_BARRIER(smp_store_release(&test_var, 0), true); |
627 | KCSAN_EXPECT_WRITE_BARRIER(xchg(&test_var, 0), true); |
628 | KCSAN_EXPECT_WRITE_BARRIER(xchg_release(&test_var, 0), true); |
629 | KCSAN_EXPECT_WRITE_BARRIER(xchg_relaxed(&test_var, 0), false); |
630 | KCSAN_EXPECT_WRITE_BARRIER(cmpxchg(&test_var, 0, 0), true); |
631 | KCSAN_EXPECT_WRITE_BARRIER(cmpxchg_release(&test_var, 0, 0), true); |
632 | KCSAN_EXPECT_WRITE_BARRIER(cmpxchg_relaxed(&test_var, 0, 0), false); |
633 | KCSAN_EXPECT_WRITE_BARRIER(atomic_read(&dummy), false); |
634 | KCSAN_EXPECT_WRITE_BARRIER(atomic_read_acquire(&dummy), false); |
635 | KCSAN_EXPECT_WRITE_BARRIER(atomic_set(&dummy, 0), false); |
636 | KCSAN_EXPECT_WRITE_BARRIER(atomic_set_release(&dummy, 0), true); |
637 | KCSAN_EXPECT_WRITE_BARRIER(atomic_add(1, &dummy), false); |
638 | KCSAN_EXPECT_WRITE_BARRIER(atomic_add_return(1, &dummy), true); |
639 | KCSAN_EXPECT_WRITE_BARRIER(atomic_add_return_acquire(1, &dummy), false); |
640 | KCSAN_EXPECT_WRITE_BARRIER(atomic_add_return_release(1, &dummy), true); |
641 | KCSAN_EXPECT_WRITE_BARRIER(atomic_add_return_relaxed(1, &dummy), false); |
642 | KCSAN_EXPECT_WRITE_BARRIER(atomic_fetch_add(1, &dummy), true); |
643 | KCSAN_EXPECT_WRITE_BARRIER(atomic_fetch_add_acquire(1, &dummy), false); |
644 | KCSAN_EXPECT_WRITE_BARRIER(atomic_fetch_add_release(1, &dummy), true); |
645 | KCSAN_EXPECT_WRITE_BARRIER(atomic_fetch_add_relaxed(1, &dummy), false); |
646 | KCSAN_EXPECT_WRITE_BARRIER(test_and_set_bit(0, &test_var), true); |
647 | KCSAN_EXPECT_WRITE_BARRIER(test_and_clear_bit(0, &test_var), true); |
648 | KCSAN_EXPECT_WRITE_BARRIER(test_and_change_bit(0, &test_var), true); |
649 | KCSAN_EXPECT_WRITE_BARRIER(clear_bit_unlock(0, &test_var), true); |
650 | KCSAN_EXPECT_WRITE_BARRIER(__clear_bit_unlock(0, &test_var), true); |
651 | KCSAN_EXPECT_WRITE_BARRIER(arch_spin_lock(&arch_spinlock), false); |
652 | KCSAN_EXPECT_WRITE_BARRIER(arch_spin_unlock(&arch_spinlock), true); |
653 | KCSAN_EXPECT_WRITE_BARRIER(spin_lock(&test_spinlock), false); |
654 | KCSAN_EXPECT_WRITE_BARRIER(spin_unlock(&test_spinlock), true); |
655 | KCSAN_EXPECT_WRITE_BARRIER(mutex_lock(&test_mutex), false); |
656 | KCSAN_EXPECT_WRITE_BARRIER(mutex_unlock(&test_mutex), true); |
657 | |
658 | KCSAN_EXPECT_RW_BARRIER(mb(), true); |
659 | KCSAN_EXPECT_RW_BARRIER(wmb(), true); |
660 | KCSAN_EXPECT_RW_BARRIER(rmb(), true); |
661 | KCSAN_EXPECT_RW_BARRIER(smp_mb(), true); |
662 | KCSAN_EXPECT_RW_BARRIER(smp_wmb(), true); |
663 | KCSAN_EXPECT_RW_BARRIER(smp_rmb(), true); |
664 | KCSAN_EXPECT_RW_BARRIER(dma_wmb(), true); |
665 | KCSAN_EXPECT_RW_BARRIER(dma_rmb(), true); |
666 | KCSAN_EXPECT_RW_BARRIER(smp_mb__before_atomic(), true); |
667 | KCSAN_EXPECT_RW_BARRIER(smp_mb__after_atomic(), true); |
668 | KCSAN_EXPECT_RW_BARRIER(smp_mb__after_spinlock(), true); |
669 | KCSAN_EXPECT_RW_BARRIER(smp_store_mb(test_var, 0), true); |
670 | KCSAN_EXPECT_RW_BARRIER(smp_load_acquire(&test_var), false); |
671 | KCSAN_EXPECT_RW_BARRIER(smp_store_release(&test_var, 0), true); |
672 | KCSAN_EXPECT_RW_BARRIER(xchg(&test_var, 0), true); |
673 | KCSAN_EXPECT_RW_BARRIER(xchg_release(&test_var, 0), true); |
674 | KCSAN_EXPECT_RW_BARRIER(xchg_relaxed(&test_var, 0), false); |
675 | KCSAN_EXPECT_RW_BARRIER(cmpxchg(&test_var, 0, 0), true); |
676 | KCSAN_EXPECT_RW_BARRIER(cmpxchg_release(&test_var, 0, 0), true); |
677 | KCSAN_EXPECT_RW_BARRIER(cmpxchg_relaxed(&test_var, 0, 0), false); |
678 | KCSAN_EXPECT_RW_BARRIER(atomic_read(&dummy), false); |
679 | KCSAN_EXPECT_RW_BARRIER(atomic_read_acquire(&dummy), false); |
680 | KCSAN_EXPECT_RW_BARRIER(atomic_set(&dummy, 0), false); |
681 | KCSAN_EXPECT_RW_BARRIER(atomic_set_release(&dummy, 0), true); |
682 | KCSAN_EXPECT_RW_BARRIER(atomic_add(1, &dummy), false); |
683 | KCSAN_EXPECT_RW_BARRIER(atomic_add_return(1, &dummy), true); |
684 | KCSAN_EXPECT_RW_BARRIER(atomic_add_return_acquire(1, &dummy), false); |
685 | KCSAN_EXPECT_RW_BARRIER(atomic_add_return_release(1, &dummy), true); |
686 | KCSAN_EXPECT_RW_BARRIER(atomic_add_return_relaxed(1, &dummy), false); |
687 | KCSAN_EXPECT_RW_BARRIER(atomic_fetch_add(1, &dummy), true); |
688 | KCSAN_EXPECT_RW_BARRIER(atomic_fetch_add_acquire(1, &dummy), false); |
689 | KCSAN_EXPECT_RW_BARRIER(atomic_fetch_add_release(1, &dummy), true); |
690 | KCSAN_EXPECT_RW_BARRIER(atomic_fetch_add_relaxed(1, &dummy), false); |
691 | KCSAN_EXPECT_RW_BARRIER(test_and_set_bit(0, &test_var), true); |
692 | KCSAN_EXPECT_RW_BARRIER(test_and_clear_bit(0, &test_var), true); |
693 | KCSAN_EXPECT_RW_BARRIER(test_and_change_bit(0, &test_var), true); |
694 | KCSAN_EXPECT_RW_BARRIER(clear_bit_unlock(0, &test_var), true); |
695 | KCSAN_EXPECT_RW_BARRIER(__clear_bit_unlock(0, &test_var), true); |
696 | KCSAN_EXPECT_RW_BARRIER(arch_spin_lock(&arch_spinlock), false); |
697 | KCSAN_EXPECT_RW_BARRIER(arch_spin_unlock(&arch_spinlock), true); |
698 | KCSAN_EXPECT_RW_BARRIER(spin_lock(&test_spinlock), false); |
699 | KCSAN_EXPECT_RW_BARRIER(spin_unlock(&test_spinlock), true); |
700 | KCSAN_EXPECT_RW_BARRIER(mutex_lock(&test_mutex), false); |
701 | KCSAN_EXPECT_RW_BARRIER(mutex_unlock(&test_mutex), true); |
702 | KCSAN_EXPECT_READ_BARRIER(xor_unlock_is_negative_byte(1, &test_var), true); |
703 | KCSAN_EXPECT_WRITE_BARRIER(xor_unlock_is_negative_byte(1, &test_var), true); |
704 | KCSAN_EXPECT_RW_BARRIER(xor_unlock_is_negative_byte(1, &test_var), true); |
705 | kcsan_nestable_atomic_end(); |
706 | } |
707 | |
708 | /* Simple test with normal data race. */ |
709 | __no_kcsan |
710 | static void test_basic(struct kunit *test) |
711 | { |
712 | struct expect_report expect = { |
713 | .access = { |
714 | { test_kernel_write, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, |
715 | { test_kernel_read, &test_var, sizeof(test_var), 0 }, |
716 | }, |
717 | }; |
718 | struct expect_report never = { |
719 | .access = { |
720 | { test_kernel_read, &test_var, sizeof(test_var), 0 }, |
721 | { test_kernel_read, &test_var, sizeof(test_var), 0 }, |
722 | }, |
723 | }; |
724 | bool match_expect = false; |
725 | bool match_never = false; |
726 | |
727 | begin_test_checks(func1: test_kernel_write, func2: test_kernel_read); |
728 | do { |
729 | match_expect |= report_matches(r: &expect); |
730 | match_never = report_matches(r: &never); |
731 | } while (!end_test_checks(stop: match_never)); |
732 | KUNIT_EXPECT_TRUE(test, match_expect); |
733 | KUNIT_EXPECT_FALSE(test, match_never); |
734 | } |
735 | |
736 | /* |
737 | * Stress KCSAN with lots of concurrent races on different addresses until |
738 | * timeout. |
739 | */ |
740 | __no_kcsan |
741 | static void test_concurrent_races(struct kunit *test) |
742 | { |
743 | struct expect_report expect = { |
744 | .access = { |
745 | /* NULL will match any address. */ |
746 | { test_kernel_rmw_array, NULL, 0, __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) }, |
747 | { test_kernel_rmw_array, NULL, 0, __KCSAN_ACCESS_RW(0) }, |
748 | }, |
749 | }; |
750 | struct expect_report never = { |
751 | .access = { |
752 | { test_kernel_rmw_array, NULL, 0, 0 }, |
753 | { test_kernel_rmw_array, NULL, 0, 0 }, |
754 | }, |
755 | }; |
756 | bool match_expect = false; |
757 | bool match_never = false; |
758 | |
759 | begin_test_checks(func1: test_kernel_rmw_array, func2: test_kernel_rmw_array); |
760 | do { |
761 | match_expect |= report_matches(r: &expect); |
762 | match_never |= report_matches(r: &never); |
763 | } while (!end_test_checks(stop: false)); |
764 | KUNIT_EXPECT_TRUE(test, match_expect); /* Sanity check matches exist. */ |
765 | KUNIT_EXPECT_FALSE(test, match_never); |
766 | } |
767 | |
768 | /* Test the KCSAN_REPORT_VALUE_CHANGE_ONLY option. */ |
769 | __no_kcsan |
770 | static void test_novalue_change(struct kunit *test) |
771 | { |
772 | struct expect_report expect_rw = { |
773 | .access = { |
774 | { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, |
775 | { test_kernel_read, &test_var, sizeof(test_var), 0 }, |
776 | }, |
777 | }; |
778 | struct expect_report expect_ww = { |
779 | .access = { |
780 | { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, |
781 | { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, |
782 | }, |
783 | }; |
784 | bool match_expect = false; |
785 | |
786 | test_kernel_write_nochange(); /* Reset value. */ |
787 | begin_test_checks(func1: test_kernel_write_nochange, func2: test_kernel_read); |
788 | do { |
789 | match_expect = report_matches(r: &expect_rw) || report_matches(r: &expect_ww); |
790 | } while (!end_test_checks(stop: match_expect)); |
791 | if (IS_ENABLED(CONFIG_KCSAN_REPORT_VALUE_CHANGE_ONLY)) |
792 | KUNIT_EXPECT_FALSE(test, match_expect); |
793 | else |
794 | KUNIT_EXPECT_TRUE(test, match_expect); |
795 | } |
796 | |
797 | /* |
798 | * Test that the rules where the KCSAN_REPORT_VALUE_CHANGE_ONLY option should |
799 | * never apply work. |
800 | */ |
801 | __no_kcsan |
802 | static void test_novalue_change_exception(struct kunit *test) |
803 | { |
804 | struct expect_report expect_rw = { |
805 | .access = { |
806 | { test_kernel_write_nochange_rcu, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, |
807 | { test_kernel_read, &test_var, sizeof(test_var), 0 }, |
808 | }, |
809 | }; |
810 | struct expect_report expect_ww = { |
811 | .access = { |
812 | { test_kernel_write_nochange_rcu, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, |
813 | { test_kernel_write_nochange_rcu, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, |
814 | }, |
815 | }; |
816 | bool match_expect = false; |
817 | |
818 | test_kernel_write_nochange_rcu(); /* Reset value. */ |
819 | begin_test_checks(func1: test_kernel_write_nochange_rcu, func2: test_kernel_read); |
820 | do { |
821 | match_expect = report_matches(r: &expect_rw) || report_matches(r: &expect_ww); |
822 | } while (!end_test_checks(stop: match_expect)); |
823 | KUNIT_EXPECT_TRUE(test, match_expect); |
824 | } |
825 | |
826 | /* Test that data races of unknown origin are reported. */ |
827 | __no_kcsan |
828 | static void test_unknown_origin(struct kunit *test) |
829 | { |
830 | struct expect_report expect = { |
831 | .access = { |
832 | { test_kernel_read, &test_var, sizeof(test_var), 0 }, |
833 | { NULL }, |
834 | }, |
835 | }; |
836 | bool match_expect = false; |
837 | |
838 | begin_test_checks(func1: test_kernel_write_uninstrumented, func2: test_kernel_read); |
839 | do { |
840 | match_expect = report_matches(r: &expect); |
841 | } while (!end_test_checks(stop: match_expect)); |
842 | if (IS_ENABLED(CONFIG_KCSAN_REPORT_RACE_UNKNOWN_ORIGIN)) |
843 | KUNIT_EXPECT_TRUE(test, match_expect); |
844 | else |
845 | KUNIT_EXPECT_FALSE(test, match_expect); |
846 | } |
847 | |
848 | /* Test KCSAN_ASSUME_PLAIN_WRITES_ATOMIC if it is selected. */ |
849 | __no_kcsan |
850 | static void test_write_write_assume_atomic(struct kunit *test) |
851 | { |
852 | struct expect_report expect = { |
853 | .access = { |
854 | { test_kernel_write, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, |
855 | { test_kernel_write, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, |
856 | }, |
857 | }; |
858 | bool match_expect = false; |
859 | |
860 | begin_test_checks(func1: test_kernel_write, func2: test_kernel_write); |
861 | do { |
862 | sink_value(READ_ONCE(test_var)); /* induce value-change */ |
863 | match_expect = report_matches(r: &expect); |
864 | } while (!end_test_checks(stop: match_expect)); |
865 | if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC)) |
866 | KUNIT_EXPECT_FALSE(test, match_expect); |
867 | else |
868 | KUNIT_EXPECT_TRUE(test, match_expect); |
869 | } |
870 | |
871 | /* |
872 | * Test that data races with writes larger than word-size are always reported, |
873 | * even if KCSAN_ASSUME_PLAIN_WRITES_ATOMIC is selected. |
874 | */ |
875 | __no_kcsan |
876 | static void test_write_write_struct(struct kunit *test) |
877 | { |
878 | struct expect_report expect = { |
879 | .access = { |
880 | { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE }, |
881 | { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE }, |
882 | }, |
883 | }; |
884 | bool match_expect = false; |
885 | |
886 | begin_test_checks(func1: test_kernel_write_struct, func2: test_kernel_write_struct); |
887 | do { |
888 | match_expect = report_matches(r: &expect); |
889 | } while (!end_test_checks(stop: match_expect)); |
890 | KUNIT_EXPECT_TRUE(test, match_expect); |
891 | } |
892 | |
893 | /* |
894 | * Test that data races where only one write is larger than word-size are always |
895 | * reported, even if KCSAN_ASSUME_PLAIN_WRITES_ATOMIC is selected. |
896 | */ |
897 | __no_kcsan |
898 | static void test_write_write_struct_part(struct kunit *test) |
899 | { |
900 | struct expect_report expect = { |
901 | .access = { |
902 | { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE }, |
903 | { test_kernel_write_struct_part, &test_struct.val[3], sizeof(test_struct.val[3]), KCSAN_ACCESS_WRITE }, |
904 | }, |
905 | }; |
906 | bool match_expect = false; |
907 | |
908 | begin_test_checks(func1: test_kernel_write_struct, func2: test_kernel_write_struct_part); |
909 | do { |
910 | match_expect = report_matches(r: &expect); |
911 | } while (!end_test_checks(stop: match_expect)); |
912 | KUNIT_EXPECT_TRUE(test, match_expect); |
913 | } |
914 | |
915 | /* Test that races with atomic accesses never result in reports. */ |
916 | __no_kcsan |
917 | static void test_read_atomic_write_atomic(struct kunit *test) |
918 | { |
919 | bool match_never = false; |
920 | |
921 | begin_test_checks(func1: test_kernel_read_atomic, func2: test_kernel_write_atomic); |
922 | do { |
923 | match_never = report_available(); |
924 | } while (!end_test_checks(stop: match_never)); |
925 | KUNIT_EXPECT_FALSE(test, match_never); |
926 | } |
927 | |
928 | /* Test that a race with an atomic and plain access result in reports. */ |
929 | __no_kcsan |
930 | static void test_read_plain_atomic_write(struct kunit *test) |
931 | { |
932 | struct expect_report expect = { |
933 | .access = { |
934 | { test_kernel_read, &test_var, sizeof(test_var), 0 }, |
935 | { test_kernel_write_atomic, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC }, |
936 | }, |
937 | }; |
938 | bool match_expect = false; |
939 | |
940 | KCSAN_TEST_REQUIRES(test, !IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)); |
941 | |
942 | begin_test_checks(func1: test_kernel_read, func2: test_kernel_write_atomic); |
943 | do { |
944 | match_expect = report_matches(r: &expect); |
945 | } while (!end_test_checks(stop: match_expect)); |
946 | KUNIT_EXPECT_TRUE(test, match_expect); |
947 | } |
948 | |
949 | /* Test that atomic RMWs generate correct report. */ |
950 | __no_kcsan |
951 | static void test_read_plain_atomic_rmw(struct kunit *test) |
952 | { |
953 | struct expect_report expect = { |
954 | .access = { |
955 | { test_kernel_read, &test_var, sizeof(test_var), 0 }, |
956 | { test_kernel_atomic_rmw, &test_var, sizeof(test_var), |
957 | KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC }, |
958 | }, |
959 | }; |
960 | bool match_expect = false; |
961 | |
962 | KCSAN_TEST_REQUIRES(test, !IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)); |
963 | |
964 | begin_test_checks(func1: test_kernel_read, func2: test_kernel_atomic_rmw); |
965 | do { |
966 | match_expect = report_matches(r: &expect); |
967 | } while (!end_test_checks(stop: match_expect)); |
968 | KUNIT_EXPECT_TRUE(test, match_expect); |
969 | } |
970 | |
971 | /* Zero-sized accesses should never cause data race reports. */ |
972 | __no_kcsan |
973 | static void test_zero_size_access(struct kunit *test) |
974 | { |
975 | struct expect_report expect = { |
976 | .access = { |
977 | { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE }, |
978 | { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE }, |
979 | }, |
980 | }; |
981 | struct expect_report never = { |
982 | .access = { |
983 | { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE }, |
984 | { test_kernel_read_struct_zero_size, &test_struct.val[3], 0, 0 }, |
985 | }, |
986 | }; |
987 | bool match_expect = false; |
988 | bool match_never = false; |
989 | |
990 | begin_test_checks(func1: test_kernel_write_struct, func2: test_kernel_read_struct_zero_size); |
991 | do { |
992 | match_expect |= report_matches(r: &expect); |
993 | match_never = report_matches(r: &never); |
994 | } while (!end_test_checks(stop: match_never)); |
995 | KUNIT_EXPECT_TRUE(test, match_expect); /* Sanity check. */ |
996 | KUNIT_EXPECT_FALSE(test, match_never); |
997 | } |
998 | |
999 | /* Test the data_race() macro. */ |
1000 | __no_kcsan |
1001 | static void test_data_race(struct kunit *test) |
1002 | { |
1003 | bool match_never = false; |
1004 | |
1005 | begin_test_checks(func1: test_kernel_data_race, func2: test_kernel_data_race); |
1006 | do { |
1007 | match_never = report_available(); |
1008 | } while (!end_test_checks(stop: match_never)); |
1009 | KUNIT_EXPECT_FALSE(test, match_never); |
1010 | } |
1011 | |
1012 | __no_kcsan |
1013 | static void test_assert_exclusive_writer(struct kunit *test) |
1014 | { |
1015 | struct expect_report expect = { |
1016 | .access = { |
1017 | { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT }, |
1018 | { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, |
1019 | }, |
1020 | }; |
1021 | bool match_expect = false; |
1022 | |
1023 | begin_test_checks(func1: test_kernel_assert_writer, func2: test_kernel_write_nochange); |
1024 | do { |
1025 | match_expect = report_matches(r: &expect); |
1026 | } while (!end_test_checks(stop: match_expect)); |
1027 | KUNIT_EXPECT_TRUE(test, match_expect); |
1028 | } |
1029 | |
1030 | __no_kcsan |
1031 | static void test_assert_exclusive_access(struct kunit *test) |
1032 | { |
1033 | struct expect_report expect = { |
1034 | .access = { |
1035 | { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE }, |
1036 | { test_kernel_read, &test_var, sizeof(test_var), 0 }, |
1037 | }, |
1038 | }; |
1039 | bool match_expect = false; |
1040 | |
1041 | begin_test_checks(func1: test_kernel_assert_access, func2: test_kernel_read); |
1042 | do { |
1043 | match_expect = report_matches(r: &expect); |
1044 | } while (!end_test_checks(stop: match_expect)); |
1045 | KUNIT_EXPECT_TRUE(test, match_expect); |
1046 | } |
1047 | |
1048 | __no_kcsan |
1049 | static void test_assert_exclusive_access_writer(struct kunit *test) |
1050 | { |
1051 | struct expect_report expect_access_writer = { |
1052 | .access = { |
1053 | { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE }, |
1054 | { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT }, |
1055 | }, |
1056 | }; |
1057 | struct expect_report expect_access_access = { |
1058 | .access = { |
1059 | { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE }, |
1060 | { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE }, |
1061 | }, |
1062 | }; |
1063 | struct expect_report never = { |
1064 | .access = { |
1065 | { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT }, |
1066 | { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT }, |
1067 | }, |
1068 | }; |
1069 | bool match_expect_access_writer = false; |
1070 | bool match_expect_access_access = false; |
1071 | bool match_never = false; |
1072 | |
1073 | begin_test_checks(func1: test_kernel_assert_access, func2: test_kernel_assert_writer); |
1074 | do { |
1075 | match_expect_access_writer |= report_matches(r: &expect_access_writer); |
1076 | match_expect_access_access |= report_matches(r: &expect_access_access); |
1077 | match_never |= report_matches(r: &never); |
1078 | } while (!end_test_checks(stop: match_never)); |
1079 | KUNIT_EXPECT_TRUE(test, match_expect_access_writer); |
1080 | KUNIT_EXPECT_TRUE(test, match_expect_access_access); |
1081 | KUNIT_EXPECT_FALSE(test, match_never); |
1082 | } |
1083 | |
1084 | __no_kcsan |
1085 | static void test_assert_exclusive_bits_change(struct kunit *test) |
1086 | { |
1087 | struct expect_report expect = { |
1088 | .access = { |
1089 | { test_kernel_assert_bits_change, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT }, |
1090 | { test_kernel_change_bits, &test_var, sizeof(test_var), |
1091 | KCSAN_ACCESS_WRITE | (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS) ? 0 : KCSAN_ACCESS_ATOMIC) }, |
1092 | }, |
1093 | }; |
1094 | bool match_expect = false; |
1095 | |
1096 | begin_test_checks(func1: test_kernel_assert_bits_change, func2: test_kernel_change_bits); |
1097 | do { |
1098 | match_expect = report_matches(r: &expect); |
1099 | } while (!end_test_checks(stop: match_expect)); |
1100 | KUNIT_EXPECT_TRUE(test, match_expect); |
1101 | } |
1102 | |
1103 | __no_kcsan |
1104 | static void test_assert_exclusive_bits_nochange(struct kunit *test) |
1105 | { |
1106 | bool match_never = false; |
1107 | |
1108 | begin_test_checks(func1: test_kernel_assert_bits_nochange, func2: test_kernel_change_bits); |
1109 | do { |
1110 | match_never = report_available(); |
1111 | } while (!end_test_checks(stop: match_never)); |
1112 | KUNIT_EXPECT_FALSE(test, match_never); |
1113 | } |
1114 | |
1115 | __no_kcsan |
1116 | static void test_assert_exclusive_writer_scoped(struct kunit *test) |
1117 | { |
1118 | struct expect_report expect_start = { |
1119 | .access = { |
1120 | { test_kernel_assert_writer_scoped, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_SCOPED }, |
1121 | { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, |
1122 | }, |
1123 | }; |
1124 | struct expect_report expect_inscope = { |
1125 | .access = { |
1126 | { test_enter_scope, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_SCOPED }, |
1127 | { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, |
1128 | }, |
1129 | }; |
1130 | bool match_expect_start = false; |
1131 | bool match_expect_inscope = false; |
1132 | |
1133 | begin_test_checks(func1: test_kernel_assert_writer_scoped, func2: test_kernel_write_nochange); |
1134 | do { |
1135 | match_expect_start |= report_matches(r: &expect_start); |
1136 | match_expect_inscope |= report_matches(r: &expect_inscope); |
1137 | } while (!end_test_checks(stop: match_expect_inscope)); |
1138 | KUNIT_EXPECT_TRUE(test, match_expect_start); |
1139 | KUNIT_EXPECT_FALSE(test, match_expect_inscope); |
1140 | } |
1141 | |
1142 | __no_kcsan |
1143 | static void test_assert_exclusive_access_scoped(struct kunit *test) |
1144 | { |
1145 | struct expect_report expect_start1 = { |
1146 | .access = { |
1147 | { test_kernel_assert_access_scoped, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_SCOPED }, |
1148 | { test_kernel_read, &test_var, sizeof(test_var), 0 }, |
1149 | }, |
1150 | }; |
1151 | struct expect_report expect_start2 = { |
1152 | .access = { expect_start1.access[0], expect_start1.access[0] }, |
1153 | }; |
1154 | struct expect_report expect_inscope = { |
1155 | .access = { |
1156 | { .fn: test_enter_scope, .addr: &test_var, .size: sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_SCOPED }, |
1157 | { test_kernel_read, &test_var, sizeof(test_var), 0 }, |
1158 | }, |
1159 | }; |
1160 | bool match_expect_start = false; |
1161 | bool match_expect_inscope = false; |
1162 | |
1163 | begin_test_checks(func1: test_kernel_assert_access_scoped, func2: test_kernel_read); |
1164 | end_time += msecs_to_jiffies(m: 1000); /* This test requires a bit more time. */ |
1165 | do { |
1166 | match_expect_start |= report_matches(r: &expect_start1) || report_matches(r: &expect_start2); |
1167 | match_expect_inscope |= report_matches(r: &expect_inscope); |
1168 | } while (!end_test_checks(stop: match_expect_inscope)); |
1169 | KUNIT_EXPECT_TRUE(test, match_expect_start); |
1170 | KUNIT_EXPECT_FALSE(test, match_expect_inscope); |
1171 | } |
1172 | |
1173 | /* |
1174 | * jiffies is special (declared to be volatile) and its accesses are typically |
1175 | * not marked; this test ensures that the compiler nor KCSAN gets confused about |
1176 | * jiffies's declaration on different architectures. |
1177 | */ |
1178 | __no_kcsan |
1179 | static void test_jiffies_noreport(struct kunit *test) |
1180 | { |
1181 | bool match_never = false; |
1182 | |
1183 | begin_test_checks(func1: test_kernel_jiffies_reader, func2: test_kernel_jiffies_reader); |
1184 | do { |
1185 | match_never = report_available(); |
1186 | } while (!end_test_checks(stop: match_never)); |
1187 | KUNIT_EXPECT_FALSE(test, match_never); |
1188 | } |
1189 | |
1190 | /* Test that racing accesses in seqlock critical sections are not reported. */ |
1191 | __no_kcsan |
1192 | static void test_seqlock_noreport(struct kunit *test) |
1193 | { |
1194 | bool match_never = false; |
1195 | |
1196 | begin_test_checks(func1: test_kernel_seqlock_reader, func2: test_kernel_seqlock_writer); |
1197 | do { |
1198 | match_never = report_available(); |
1199 | } while (!end_test_checks(stop: match_never)); |
1200 | KUNIT_EXPECT_FALSE(test, match_never); |
1201 | } |
1202 | |
1203 | /* |
1204 | * Test atomic builtins work and required instrumentation functions exist. We |
1205 | * also test that KCSAN understands they're atomic by racing with them via |
1206 | * test_kernel_atomic_builtins(), and expect no reports. |
1207 | * |
1208 | * The atomic builtins _SHOULD NOT_ be used in normal kernel code! |
1209 | */ |
1210 | static void test_atomic_builtins(struct kunit *test) |
1211 | { |
1212 | bool match_never = false; |
1213 | |
1214 | begin_test_checks(func1: test_kernel_atomic_builtins, func2: test_kernel_atomic_builtins); |
1215 | do { |
1216 | long tmp; |
1217 | |
1218 | kcsan_enable_current(); |
1219 | |
1220 | __atomic_store_n(&test_var, 42L, __ATOMIC_RELAXED); |
1221 | KUNIT_EXPECT_EQ(test, 42L, __atomic_load_n(&test_var, __ATOMIC_RELAXED)); |
1222 | |
1223 | KUNIT_EXPECT_EQ(test, 42L, __atomic_exchange_n(&test_var, 20, __ATOMIC_RELAXED)); |
1224 | KUNIT_EXPECT_EQ(test, 20L, test_var); |
1225 | |
1226 | tmp = 20L; |
1227 | KUNIT_EXPECT_TRUE(test, __atomic_compare_exchange_n(&test_var, &tmp, 30L, |
1228 | 0, __ATOMIC_RELAXED, |
1229 | __ATOMIC_RELAXED)); |
1230 | KUNIT_EXPECT_EQ(test, tmp, 20L); |
1231 | KUNIT_EXPECT_EQ(test, test_var, 30L); |
1232 | KUNIT_EXPECT_FALSE(test, __atomic_compare_exchange_n(&test_var, &tmp, 40L, |
1233 | 1, __ATOMIC_RELAXED, |
1234 | __ATOMIC_RELAXED)); |
1235 | KUNIT_EXPECT_EQ(test, tmp, 30L); |
1236 | KUNIT_EXPECT_EQ(test, test_var, 30L); |
1237 | |
1238 | KUNIT_EXPECT_EQ(test, 30L, __atomic_fetch_add(&test_var, 1, __ATOMIC_RELAXED)); |
1239 | KUNIT_EXPECT_EQ(test, 31L, __atomic_fetch_sub(&test_var, 1, __ATOMIC_RELAXED)); |
1240 | KUNIT_EXPECT_EQ(test, 30L, __atomic_fetch_and(&test_var, 0xf, __ATOMIC_RELAXED)); |
1241 | KUNIT_EXPECT_EQ(test, 14L, __atomic_fetch_xor(&test_var, 0xf, __ATOMIC_RELAXED)); |
1242 | KUNIT_EXPECT_EQ(test, 1L, __atomic_fetch_or(&test_var, 0xf0, __ATOMIC_RELAXED)); |
1243 | KUNIT_EXPECT_EQ(test, 241L, __atomic_fetch_nand(&test_var, 0xf, __ATOMIC_RELAXED)); |
1244 | KUNIT_EXPECT_EQ(test, -2L, test_var); |
1245 | |
1246 | __atomic_thread_fence(__ATOMIC_SEQ_CST); |
1247 | __atomic_signal_fence(__ATOMIC_SEQ_CST); |
1248 | |
1249 | kcsan_disable_current(); |
1250 | |
1251 | match_never = report_available(); |
1252 | } while (!end_test_checks(stop: match_never)); |
1253 | KUNIT_EXPECT_FALSE(test, match_never); |
1254 | } |
1255 | |
1256 | __no_kcsan |
1257 | static void test_1bit_value_change(struct kunit *test) |
1258 | { |
1259 | struct expect_report expect = { |
1260 | .access = { |
1261 | { test_kernel_read, &test_var, sizeof(test_var), 0 }, |
1262 | { test_kernel_xor_1bit, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) }, |
1263 | }, |
1264 | }; |
1265 | bool match = false; |
1266 | |
1267 | begin_test_checks(func1: test_kernel_read, func2: test_kernel_xor_1bit); |
1268 | do { |
1269 | match = IS_ENABLED(CONFIG_KCSAN_PERMISSIVE) |
1270 | ? report_available() |
1271 | : report_matches(r: &expect); |
1272 | } while (!end_test_checks(stop: match)); |
1273 | if (IS_ENABLED(CONFIG_KCSAN_PERMISSIVE)) |
1274 | KUNIT_EXPECT_FALSE(test, match); |
1275 | else |
1276 | KUNIT_EXPECT_TRUE(test, match); |
1277 | } |
1278 | |
1279 | __no_kcsan |
1280 | static void test_correct_barrier(struct kunit *test) |
1281 | { |
1282 | struct expect_report expect = { |
1283 | .access = { |
1284 | { test_kernel_with_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) }, |
1285 | { test_kernel_with_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(0) }, |
1286 | }, |
1287 | }; |
1288 | bool match_expect = false; |
1289 | |
1290 | test_struct.val[0] = 0; /* init unlocked */ |
1291 | begin_test_checks(func1: test_kernel_with_memorder, func2: test_kernel_with_memorder); |
1292 | do { |
1293 | match_expect = report_matches_any_reordered(r: &expect); |
1294 | } while (!end_test_checks(stop: match_expect)); |
1295 | KUNIT_EXPECT_FALSE(test, match_expect); |
1296 | } |
1297 | |
1298 | __no_kcsan |
1299 | static void test_missing_barrier(struct kunit *test) |
1300 | { |
1301 | struct expect_report expect = { |
1302 | .access = { |
1303 | { test_kernel_wrong_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) }, |
1304 | { test_kernel_wrong_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(0) }, |
1305 | }, |
1306 | }; |
1307 | bool match_expect = false; |
1308 | |
1309 | test_struct.val[0] = 0; /* init unlocked */ |
1310 | begin_test_checks(func1: test_kernel_wrong_memorder, func2: test_kernel_wrong_memorder); |
1311 | do { |
1312 | match_expect = report_matches_any_reordered(r: &expect); |
1313 | } while (!end_test_checks(stop: match_expect)); |
1314 | if (IS_ENABLED(CONFIG_KCSAN_WEAK_MEMORY)) |
1315 | KUNIT_EXPECT_TRUE(test, match_expect); |
1316 | else |
1317 | KUNIT_EXPECT_FALSE(test, match_expect); |
1318 | } |
1319 | |
1320 | __no_kcsan |
1321 | static void test_atomic_builtins_correct_barrier(struct kunit *test) |
1322 | { |
1323 | struct expect_report expect = { |
1324 | .access = { |
1325 | { test_kernel_atomic_builtin_with_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) }, |
1326 | { test_kernel_atomic_builtin_with_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(0) }, |
1327 | }, |
1328 | }; |
1329 | bool match_expect = false; |
1330 | |
1331 | test_struct.val[0] = 0; /* init unlocked */ |
1332 | begin_test_checks(func1: test_kernel_atomic_builtin_with_memorder, |
1333 | func2: test_kernel_atomic_builtin_with_memorder); |
1334 | do { |
1335 | match_expect = report_matches_any_reordered(r: &expect); |
1336 | } while (!end_test_checks(stop: match_expect)); |
1337 | KUNIT_EXPECT_FALSE(test, match_expect); |
1338 | } |
1339 | |
1340 | __no_kcsan |
1341 | static void test_atomic_builtins_missing_barrier(struct kunit *test) |
1342 | { |
1343 | struct expect_report expect = { |
1344 | .access = { |
1345 | { test_kernel_atomic_builtin_wrong_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) }, |
1346 | { test_kernel_atomic_builtin_wrong_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(0) }, |
1347 | }, |
1348 | }; |
1349 | bool match_expect = false; |
1350 | |
1351 | test_struct.val[0] = 0; /* init unlocked */ |
1352 | begin_test_checks(func1: test_kernel_atomic_builtin_wrong_memorder, |
1353 | func2: test_kernel_atomic_builtin_wrong_memorder); |
1354 | do { |
1355 | match_expect = report_matches_any_reordered(r: &expect); |
1356 | } while (!end_test_checks(stop: match_expect)); |
1357 | if (IS_ENABLED(CONFIG_KCSAN_WEAK_MEMORY)) |
1358 | KUNIT_EXPECT_TRUE(test, match_expect); |
1359 | else |
1360 | KUNIT_EXPECT_FALSE(test, match_expect); |
1361 | } |
1362 | |
1363 | /* |
1364 | * Generate thread counts for all test cases. Values generated are in interval |
1365 | * [2, 5] followed by exponentially increasing thread counts from 8 to 32. |
1366 | * |
1367 | * The thread counts are chosen to cover potentially interesting boundaries and |
1368 | * corner cases (2 to 5), and then stress the system with larger counts. |
1369 | */ |
1370 | static const void *nthreads_gen_params(const void *prev, char *desc) |
1371 | { |
1372 | long nthreads = (long)prev; |
1373 | |
1374 | if (nthreads < 0 || nthreads >= 32) |
1375 | nthreads = 0; /* stop */ |
1376 | else if (!nthreads) |
1377 | nthreads = 2; /* initial value */ |
1378 | else if (nthreads < 5) |
1379 | nthreads++; |
1380 | else if (nthreads == 5) |
1381 | nthreads = 8; |
1382 | else |
1383 | nthreads *= 2; |
1384 | |
1385 | if (!preempt_model_preemptible() || |
1386 | !IS_ENABLED(CONFIG_KCSAN_INTERRUPT_WATCHER)) { |
1387 | /* |
1388 | * Without any preemption, keep 2 CPUs free for other tasks, one |
1389 | * of which is the main test case function checking for |
1390 | * completion or failure. |
1391 | */ |
1392 | const long min_unused_cpus = preempt_model_none() ? 2 : 0; |
1393 | const long min_required_cpus = 2 + min_unused_cpus; |
1394 | |
1395 | if (num_online_cpus() < min_required_cpus) { |
1396 | pr_err_once("Too few online CPUs (%u < %ld) for test\n" , |
1397 | num_online_cpus(), min_required_cpus); |
1398 | nthreads = 0; |
1399 | } else if (nthreads >= num_online_cpus() - min_unused_cpus) { |
1400 | /* Use negative value to indicate last param. */ |
1401 | nthreads = -(num_online_cpus() - min_unused_cpus); |
1402 | pr_warn_once("Limiting number of threads to %ld (only %d online CPUs)\n" , |
1403 | -nthreads, num_online_cpus()); |
1404 | } |
1405 | } |
1406 | |
1407 | snprintf(buf: desc, KUNIT_PARAM_DESC_SIZE, fmt: "threads=%ld" , abs(nthreads)); |
1408 | return (void *)nthreads; |
1409 | } |
1410 | |
1411 | #define KCSAN_KUNIT_CASE(test_name) KUNIT_CASE_PARAM(test_name, nthreads_gen_params) |
1412 | static struct kunit_case kcsan_test_cases[] = { |
1413 | KUNIT_CASE(test_barrier_nothreads), |
1414 | KCSAN_KUNIT_CASE(test_basic), |
1415 | KCSAN_KUNIT_CASE(test_concurrent_races), |
1416 | KCSAN_KUNIT_CASE(test_novalue_change), |
1417 | KCSAN_KUNIT_CASE(test_novalue_change_exception), |
1418 | KCSAN_KUNIT_CASE(test_unknown_origin), |
1419 | KCSAN_KUNIT_CASE(test_write_write_assume_atomic), |
1420 | KCSAN_KUNIT_CASE(test_write_write_struct), |
1421 | KCSAN_KUNIT_CASE(test_write_write_struct_part), |
1422 | KCSAN_KUNIT_CASE(test_read_atomic_write_atomic), |
1423 | KCSAN_KUNIT_CASE(test_read_plain_atomic_write), |
1424 | KCSAN_KUNIT_CASE(test_read_plain_atomic_rmw), |
1425 | KCSAN_KUNIT_CASE(test_zero_size_access), |
1426 | KCSAN_KUNIT_CASE(test_data_race), |
1427 | KCSAN_KUNIT_CASE(test_assert_exclusive_writer), |
1428 | KCSAN_KUNIT_CASE(test_assert_exclusive_access), |
1429 | KCSAN_KUNIT_CASE(test_assert_exclusive_access_writer), |
1430 | KCSAN_KUNIT_CASE(test_assert_exclusive_bits_change), |
1431 | KCSAN_KUNIT_CASE(test_assert_exclusive_bits_nochange), |
1432 | KCSAN_KUNIT_CASE(test_assert_exclusive_writer_scoped), |
1433 | KCSAN_KUNIT_CASE(test_assert_exclusive_access_scoped), |
1434 | KCSAN_KUNIT_CASE(test_jiffies_noreport), |
1435 | KCSAN_KUNIT_CASE(test_seqlock_noreport), |
1436 | KCSAN_KUNIT_CASE(test_atomic_builtins), |
1437 | KCSAN_KUNIT_CASE(test_1bit_value_change), |
1438 | KCSAN_KUNIT_CASE(test_correct_barrier), |
1439 | KCSAN_KUNIT_CASE(test_missing_barrier), |
1440 | KCSAN_KUNIT_CASE(test_atomic_builtins_correct_barrier), |
1441 | KCSAN_KUNIT_CASE(test_atomic_builtins_missing_barrier), |
1442 | {}, |
1443 | }; |
1444 | |
1445 | /* ===== End test cases ===== */ |
1446 | |
1447 | /* Concurrent accesses from interrupts. */ |
1448 | __no_kcsan |
1449 | static void access_thread_timer(struct timer_list *timer) |
1450 | { |
1451 | static atomic_t cnt = ATOMIC_INIT(0); |
1452 | unsigned int idx; |
1453 | void (*func)(void); |
1454 | |
1455 | idx = (unsigned int)atomic_inc_return(v: &cnt) % ARRAY_SIZE(access_kernels); |
1456 | /* Acquire potential initialization. */ |
1457 | func = smp_load_acquire(&access_kernels[idx]); |
1458 | if (func) |
1459 | func(); |
1460 | } |
1461 | |
1462 | /* The main loop for each thread. */ |
1463 | __no_kcsan |
1464 | static int access_thread(void *arg) |
1465 | { |
1466 | struct timer_list timer; |
1467 | unsigned int cnt = 0; |
1468 | unsigned int idx; |
1469 | void (*func)(void); |
1470 | |
1471 | timer_setup_on_stack(&timer, access_thread_timer, 0); |
1472 | do { |
1473 | might_sleep(); |
1474 | |
1475 | if (!timer_pending(timer: &timer)) |
1476 | mod_timer(timer: &timer, expires: jiffies + 1); |
1477 | else { |
1478 | /* Iterate through all kernels. */ |
1479 | idx = cnt++ % ARRAY_SIZE(access_kernels); |
1480 | /* Acquire potential initialization. */ |
1481 | func = smp_load_acquire(&access_kernels[idx]); |
1482 | if (func) |
1483 | func(); |
1484 | } |
1485 | } while (!torture_must_stop()); |
1486 | del_timer_sync(timer: &timer); |
1487 | destroy_timer_on_stack(timer: &timer); |
1488 | |
1489 | torture_kthread_stopping(title: "access_thread" ); |
1490 | return 0; |
1491 | } |
1492 | |
1493 | __no_kcsan |
1494 | static int test_init(struct kunit *test) |
1495 | { |
1496 | unsigned long flags; |
1497 | int nthreads; |
1498 | int i; |
1499 | |
1500 | spin_lock_irqsave(&observed.lock, flags); |
1501 | for (i = 0; i < ARRAY_SIZE(observed.lines); ++i) |
1502 | observed.lines[i][0] = '\0'; |
1503 | observed.nlines = 0; |
1504 | spin_unlock_irqrestore(lock: &observed.lock, flags); |
1505 | |
1506 | if (strstr(test->name, "nothreads" )) |
1507 | return 0; |
1508 | |
1509 | if (!torture_init_begin(ttype: (char *)test->name, v: 1)) |
1510 | return -EBUSY; |
1511 | |
1512 | if (WARN_ON(threads)) |
1513 | goto err; |
1514 | |
1515 | for (i = 0; i < ARRAY_SIZE(access_kernels); ++i) { |
1516 | if (WARN_ON(access_kernels[i])) |
1517 | goto err; |
1518 | } |
1519 | |
1520 | nthreads = abs((long)test->param_value); |
1521 | if (WARN_ON(!nthreads)) |
1522 | goto err; |
1523 | |
1524 | threads = kcalloc(n: nthreads + 1, size: sizeof(struct task_struct *), GFP_KERNEL); |
1525 | if (WARN_ON(!threads)) |
1526 | goto err; |
1527 | |
1528 | threads[nthreads] = NULL; |
1529 | for (i = 0; i < nthreads; ++i) { |
1530 | if (torture_create_kthread(access_thread, NULL, threads[i])) |
1531 | goto err; |
1532 | } |
1533 | |
1534 | torture_init_end(); |
1535 | |
1536 | return 0; |
1537 | |
1538 | err: |
1539 | kfree(objp: threads); |
1540 | threads = NULL; |
1541 | torture_init_end(); |
1542 | return -EINVAL; |
1543 | } |
1544 | |
1545 | __no_kcsan |
1546 | static void test_exit(struct kunit *test) |
1547 | { |
1548 | struct task_struct **stop_thread; |
1549 | int i; |
1550 | |
1551 | if (strstr(test->name, "nothreads" )) |
1552 | return; |
1553 | |
1554 | if (torture_cleanup_begin()) |
1555 | return; |
1556 | |
1557 | for (i = 0; i < ARRAY_SIZE(access_kernels); ++i) |
1558 | WRITE_ONCE(access_kernels[i], NULL); |
1559 | |
1560 | if (threads) { |
1561 | for (stop_thread = threads; *stop_thread; stop_thread++) |
1562 | torture_stop_kthread(reader_thread, *stop_thread); |
1563 | |
1564 | kfree(objp: threads); |
1565 | threads = NULL; |
1566 | } |
1567 | |
1568 | torture_cleanup_end(); |
1569 | } |
1570 | |
1571 | __no_kcsan |
1572 | static void register_tracepoints(void) |
1573 | { |
1574 | register_trace_console(probe: probe_console, NULL); |
1575 | } |
1576 | |
1577 | __no_kcsan |
1578 | static void unregister_tracepoints(void) |
1579 | { |
1580 | unregister_trace_console(probe: probe_console, NULL); |
1581 | } |
1582 | |
1583 | static int kcsan_suite_init(struct kunit_suite *suite) |
1584 | { |
1585 | register_tracepoints(); |
1586 | return 0; |
1587 | } |
1588 | |
1589 | static void kcsan_suite_exit(struct kunit_suite *suite) |
1590 | { |
1591 | unregister_tracepoints(); |
1592 | tracepoint_synchronize_unregister(); |
1593 | } |
1594 | |
1595 | static struct kunit_suite kcsan_test_suite = { |
1596 | .name = "kcsan" , |
1597 | .test_cases = kcsan_test_cases, |
1598 | .init = test_init, |
1599 | .exit = test_exit, |
1600 | .suite_init = kcsan_suite_init, |
1601 | .suite_exit = kcsan_suite_exit, |
1602 | }; |
1603 | |
1604 | kunit_test_suites(&kcsan_test_suite); |
1605 | |
1606 | MODULE_LICENSE("GPL v2" ); |
1607 | MODULE_AUTHOR("Marco Elver <elver@google.com>" ); |
1608 | |