Warning: This file is not a C or C++ file. It does not have highlighting.
| 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
|---|---|
| 2 | #ifndef __LINUX_COOKIE_H |
| 3 | #define __LINUX_COOKIE_H |
| 4 | |
| 5 | #include <linux/atomic.h> |
| 6 | #include <linux/percpu.h> |
| 7 | #include <asm/local.h> |
| 8 | |
| 9 | struct pcpu_gen_cookie { |
| 10 | local_t nesting; |
| 11 | u64 last; |
| 12 | } __aligned(16); |
| 13 | |
| 14 | struct gen_cookie { |
| 15 | struct pcpu_gen_cookie __percpu *local; |
| 16 | atomic64_t forward_last ____cacheline_aligned_in_smp; |
| 17 | atomic64_t reverse_last; |
| 18 | }; |
| 19 | |
| 20 | #define COOKIE_LOCAL_BATCH 4096 |
| 21 | |
| 22 | #define DEFINE_COOKIE(name) \ |
| 23 | static DEFINE_PER_CPU(struct pcpu_gen_cookie, __##name); \ |
| 24 | static struct gen_cookie name = { \ |
| 25 | .local = &__##name, \ |
| 26 | .forward_last = ATOMIC64_INIT(0), \ |
| 27 | .reverse_last = ATOMIC64_INIT(0), \ |
| 28 | } |
| 29 | |
| 30 | static __always_inline u64 gen_cookie_next(struct gen_cookie *gc) |
| 31 | { |
| 32 | struct pcpu_gen_cookie *local = this_cpu_ptr(gc->local); |
| 33 | u64 val; |
| 34 | |
| 35 | if (likely(local_inc_return(&local->nesting) == 1)) { |
| 36 | val = local->last; |
| 37 | if (__is_defined(CONFIG_SMP) && |
| 38 | unlikely((val & (COOKIE_LOCAL_BATCH - 1)) == 0)) { |
| 39 | s64 next = atomic64_add_return(COOKIE_LOCAL_BATCH, |
| 40 | &gc->forward_last); |
| 41 | val = next - COOKIE_LOCAL_BATCH; |
| 42 | } |
| 43 | local->last = ++val; |
| 44 | } else { |
| 45 | val = atomic64_dec_return(&gc->reverse_last); |
| 46 | } |
| 47 | local_dec(&local->nesting); |
| 48 | return val; |
| 49 | } |
| 50 | |
| 51 | #endif /* __LINUX_COOKIE_H */ |
| 52 |
Warning: This file is not a C or C++ file. It does not have highlighting.