1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _LINUX_PAGE_REF_H
3#define _LINUX_PAGE_REF_H
4
5#include <linux/atomic.h>
6#include <linux/mm_types.h>
7#include <linux/page-flags.h>
8#include <linux/tracepoint-defs.h>
9
10extern struct tracepoint __tracepoint_page_ref_set;
11extern struct tracepoint __tracepoint_page_ref_mod;
12extern struct tracepoint __tracepoint_page_ref_mod_and_test;
13extern struct tracepoint __tracepoint_page_ref_mod_and_return;
14extern struct tracepoint __tracepoint_page_ref_mod_unless;
15extern struct tracepoint __tracepoint_page_ref_freeze;
16extern struct tracepoint __tracepoint_page_ref_unfreeze;
17
18#ifdef CONFIG_DEBUG_PAGE_REF
19
20/*
21 * Ideally we would want to use the trace_<tracepoint>_enabled() helper
22 * functions. But due to include header file issues, that is not
23 * feasible. Instead we have to open code the static key functions.
24 *
25 * See trace_##name##_enabled(void) in include/linux/tracepoint.h
26 */
27#define page_ref_tracepoint_active(t) static_key_false(&(t).key)
28
29extern void __page_ref_set(struct page *page, int v);
30extern void __page_ref_mod(struct page *page, int v);
31extern void __page_ref_mod_and_test(struct page *page, int v, int ret);
32extern void __page_ref_mod_and_return(struct page *page, int v, int ret);
33extern void __page_ref_mod_unless(struct page *page, int v, int u);
34extern void __page_ref_freeze(struct page *page, int v, int ret);
35extern void __page_ref_unfreeze(struct page *page, int v);
36
37#else
38
39#define page_ref_tracepoint_active(t) false
40
41static inline void __page_ref_set(struct page *page, int v)
42{
43}
44static inline void __page_ref_mod(struct page *page, int v)
45{
46}
47static inline void __page_ref_mod_and_test(struct page *page, int v, int ret)
48{
49}
50static inline void __page_ref_mod_and_return(struct page *page, int v, int ret)
51{
52}
53static inline void __page_ref_mod_unless(struct page *page, int v, int u)
54{
55}
56static inline void __page_ref_freeze(struct page *page, int v, int ret)
57{
58}
59static inline void __page_ref_unfreeze(struct page *page, int v)
60{
61}
62
63#endif
64
65static inline int page_ref_count(struct page *page)
66{
67 return atomic_read(&page->_refcount);
68}
69
70static inline int page_count(struct page *page)
71{
72 return atomic_read(&compound_head(page)->_refcount);
73}
74
75static inline void set_page_count(struct page *page, int v)
76{
77 atomic_set(&page->_refcount, v);
78 if (page_ref_tracepoint_active(__tracepoint_page_ref_set))
79 __page_ref_set(page, v);
80}
81
82/*
83 * Setup the page count before being freed into the page allocator for
84 * the first time (boot or memory hotplug)
85 */
86static inline void init_page_count(struct page *page)
87{
88 set_page_count(page, 1);
89}
90
91static inline void page_ref_add(struct page *page, int nr)
92{
93 atomic_add(nr, &page->_refcount);
94 if (page_ref_tracepoint_active(__tracepoint_page_ref_mod))
95 __page_ref_mod(page, nr);
96}
97
98static inline void page_ref_sub(struct page *page, int nr)
99{
100 atomic_sub(nr, &page->_refcount);
101 if (page_ref_tracepoint_active(__tracepoint_page_ref_mod))
102 __page_ref_mod(page, -nr);
103}
104
105static inline void page_ref_inc(struct page *page)
106{
107 atomic_inc(&page->_refcount);
108 if (page_ref_tracepoint_active(__tracepoint_page_ref_mod))
109 __page_ref_mod(page, 1);
110}
111
112static inline void page_ref_dec(struct page *page)
113{
114 atomic_dec(&page->_refcount);
115 if (page_ref_tracepoint_active(__tracepoint_page_ref_mod))
116 __page_ref_mod(page, -1);
117}
118
119static inline int page_ref_sub_and_test(struct page *page, int nr)
120{
121 int ret = atomic_sub_and_test(nr, &page->_refcount);
122
123 if (page_ref_tracepoint_active(__tracepoint_page_ref_mod_and_test))
124 __page_ref_mod_and_test(page, -nr, ret);
125 return ret;
126}
127
128static inline int page_ref_inc_return(struct page *page)
129{
130 int ret = atomic_inc_return(&page->_refcount);
131
132 if (page_ref_tracepoint_active(__tracepoint_page_ref_mod_and_return))
133 __page_ref_mod_and_return(page, 1, ret);
134 return ret;
135}
136
137static inline int page_ref_dec_and_test(struct page *page)
138{
139 int ret = atomic_dec_and_test(&page->_refcount);
140
141 if (page_ref_tracepoint_active(__tracepoint_page_ref_mod_and_test))
142 __page_ref_mod_and_test(page, -1, ret);
143 return ret;
144}
145
146static inline int page_ref_dec_return(struct page *page)
147{
148 int ret = atomic_dec_return(&page->_refcount);
149
150 if (page_ref_tracepoint_active(__tracepoint_page_ref_mod_and_return))
151 __page_ref_mod_and_return(page, -1, ret);
152 return ret;
153}
154
155static inline int page_ref_add_unless(struct page *page, int nr, int u)
156{
157 int ret = atomic_add_unless(&page->_refcount, nr, u);
158
159 if (page_ref_tracepoint_active(__tracepoint_page_ref_mod_unless))
160 __page_ref_mod_unless(page, nr, ret);
161 return ret;
162}
163
164static inline int page_ref_freeze(struct page *page, int count)
165{
166 int ret = likely(atomic_cmpxchg(&page->_refcount, count, 0) == count);
167
168 if (page_ref_tracepoint_active(__tracepoint_page_ref_freeze))
169 __page_ref_freeze(page, count, ret);
170 return ret;
171}
172
173static inline void page_ref_unfreeze(struct page *page, int count)
174{
175 VM_BUG_ON_PAGE(page_count(page) != 0, page);
176 VM_BUG_ON(count == 0);
177
178 atomic_set_release(&page->_refcount, count);
179 if (page_ref_tracepoint_active(__tracepoint_page_ref_unfreeze))
180 __page_ref_unfreeze(page, count);
181}
182
183#endif
184