Warning: That file was not part of the compilation database. It may have many parsing errors.
1 | #ifndef __ASM_X86_REFCOUNT_H |
---|---|
2 | #define __ASM_X86_REFCOUNT_H |
3 | /* |
4 | * x86-specific implementation of refcount_t. Based on PAX_REFCOUNT from |
5 | * PaX/grsecurity. |
6 | */ |
7 | #include <linux/refcount.h> |
8 | #include <asm/bug.h> |
9 | |
10 | /* |
11 | * This is the first portion of the refcount error handling, which lives in |
12 | * .text.unlikely, and is jumped to from the CPU flag check (in the |
13 | * following macros). This saves the refcount value location into CX for |
14 | * the exception handler to use (in mm/extable.c), and then triggers the |
15 | * central refcount exception. The fixup address for the exception points |
16 | * back to the regular execution flow in .text. |
17 | */ |
18 | #define _REFCOUNT_EXCEPTION \ |
19 | ".pushsection .text..refcount\n" \ |
20 | "111:\tlea %[var], %%" _ASM_CX "\n" \ |
21 | "112:\t" ASM_UD2 "\n" \ |
22 | ASM_UNREACHABLE \ |
23 | ".popsection\n" \ |
24 | "113:\n" \ |
25 | _ASM_EXTABLE_REFCOUNT(112b, 113b) |
26 | |
27 | /* Trigger refcount exception if refcount result is negative. */ |
28 | #define REFCOUNT_CHECK_LT_ZERO \ |
29 | "js 111f\n\t" \ |
30 | _REFCOUNT_EXCEPTION |
31 | |
32 | /* Trigger refcount exception if refcount result is zero or negative. */ |
33 | #define REFCOUNT_CHECK_LE_ZERO \ |
34 | "jz 111f\n\t" \ |
35 | REFCOUNT_CHECK_LT_ZERO |
36 | |
37 | /* Trigger refcount exception unconditionally. */ |
38 | #define REFCOUNT_ERROR \ |
39 | "jmp 111f\n\t" \ |
40 | _REFCOUNT_EXCEPTION |
41 | |
42 | static __always_inline void refcount_add(unsigned int i, refcount_t *r) |
43 | { |
44 | asm volatile(LOCK_PREFIX "addl %1,%0\n\t" |
45 | REFCOUNT_CHECK_LT_ZERO |
46 | : [var] "+m"(r->refs.counter) |
47 | : "ir"(i) |
48 | : "cc", "cx"); |
49 | } |
50 | |
51 | static __always_inline void refcount_inc(refcount_t *r) |
52 | { |
53 | asm volatile(LOCK_PREFIX "incl %0\n\t" |
54 | REFCOUNT_CHECK_LT_ZERO |
55 | : [var] "+m"(r->refs.counter) |
56 | : : "cc", "cx"); |
57 | } |
58 | |
59 | static __always_inline void refcount_dec(refcount_t *r) |
60 | { |
61 | asm volatile(LOCK_PREFIX "decl %0\n\t" |
62 | REFCOUNT_CHECK_LE_ZERO |
63 | : [var] "+m"(r->refs.counter) |
64 | : : "cc", "cx"); |
65 | } |
66 | |
67 | static __always_inline __must_check |
68 | bool refcount_sub_and_test(unsigned int i, refcount_t *r) |
69 | { |
70 | bool ret = GEN_BINARY_SUFFIXED_RMWcc(LOCK_PREFIX "subl", |
71 | REFCOUNT_CHECK_LT_ZERO, |
72 | r->refs.counter, e, "er", i, "cx"); |
73 | |
74 | if (ret) { |
75 | smp_acquire__after_ctrl_dep(); |
76 | return true; |
77 | } |
78 | |
79 | return false; |
80 | } |
81 | |
82 | static __always_inline __must_check bool refcount_dec_and_test(refcount_t *r) |
83 | { |
84 | bool ret = GEN_UNARY_SUFFIXED_RMWcc(LOCK_PREFIX "decl", |
85 | REFCOUNT_CHECK_LT_ZERO, |
86 | r->refs.counter, e, "cx"); |
87 | |
88 | if (ret) { |
89 | smp_acquire__after_ctrl_dep(); |
90 | return true; |
91 | } |
92 | |
93 | return false; |
94 | } |
95 | |
96 | static __always_inline __must_check |
97 | bool refcount_add_not_zero(unsigned int i, refcount_t *r) |
98 | { |
99 | int c, result; |
100 | |
101 | c = atomic_read(&(r->refs)); |
102 | do { |
103 | if (unlikely(c == 0)) |
104 | return false; |
105 | |
106 | result = c + i; |
107 | |
108 | /* Did we try to increment from/to an undesirable state? */ |
109 | if (unlikely(c < 0 || c == INT_MAX || result < c)) { |
110 | asm volatile(REFCOUNT_ERROR |
111 | : : [var] "m"(r->refs.counter) |
112 | : "cc", "cx"); |
113 | break; |
114 | } |
115 | |
116 | } while (!atomic_try_cmpxchg(&(r->refs), &c, result)); |
117 | |
118 | return c != 0; |
119 | } |
120 | |
121 | static __always_inline __must_check bool refcount_inc_not_zero(refcount_t *r) |
122 | { |
123 | return refcount_add_not_zero(1, r); |
124 | } |
125 | |
126 | #endif |
127 |
Warning: That file was not part of the compilation database. It may have many parsing errors.