1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | #ifndef _MM_PERCPU_INTERNAL_H |
3 | #define _MM_PERCPU_INTERNAL_H |
4 | |
5 | #include <linux/types.h> |
6 | #include <linux/percpu.h> |
7 | #include <linux/memcontrol.h> |
8 | |
9 | /* |
10 | * pcpu_block_md is the metadata block struct. |
11 | * Each chunk's bitmap is split into a number of full blocks. |
12 | * All units are in terms of bits. |
13 | * |
14 | * The scan hint is the largest known contiguous area before the contig hint. |
15 | * It is not necessarily the actual largest contig hint though. There is an |
16 | * invariant that the scan_hint_start > contig_hint_start iff |
17 | * scan_hint == contig_hint. This is necessary because when scanning forward, |
18 | * we don't know if a new contig hint would be better than the current one. |
19 | */ |
20 | struct pcpu_block_md { |
21 | int scan_hint; /* scan hint for block */ |
22 | int scan_hint_start; /* block relative starting |
23 | position of the scan hint */ |
24 | int contig_hint; /* contig hint for block */ |
25 | int contig_hint_start; /* block relative starting |
26 | position of the contig hint */ |
27 | int left_free; /* size of free space along |
28 | the left side of the block */ |
29 | int right_free; /* size of free space along |
30 | the right side of the block */ |
31 | int first_free; /* block position of first free */ |
32 | int nr_bits; /* total bits responsible for */ |
33 | }; |
34 | |
35 | struct pcpu_chunk { |
36 | #ifdef CONFIG_PERCPU_STATS |
37 | int nr_alloc; /* # of allocations */ |
38 | size_t max_alloc_size; /* largest allocation size */ |
39 | #endif |
40 | |
41 | struct list_head list; /* linked to pcpu_slot lists */ |
42 | int free_bytes; /* free bytes in the chunk */ |
43 | struct pcpu_block_md chunk_md; |
44 | unsigned long *bound_map; /* boundary map */ |
45 | |
46 | /* |
47 | * base_addr is the base address of this chunk. |
48 | * To reduce false sharing, current layout is optimized to make sure |
49 | * base_addr locate in the different cacheline with free_bytes and |
50 | * chunk_md. |
51 | */ |
52 | void *base_addr ____cacheline_aligned_in_smp; |
53 | |
54 | unsigned long *alloc_map; /* allocation map */ |
55 | struct pcpu_block_md *md_blocks; /* metadata blocks */ |
56 | |
57 | void *data; /* chunk data */ |
58 | bool immutable; /* no [de]population allowed */ |
59 | bool isolated; /* isolated from active chunk |
60 | slots */ |
61 | int start_offset; /* the overlap with the previous |
62 | region to have a page aligned |
63 | base_addr */ |
64 | int end_offset; /* additional area required to |
65 | have the region end page |
66 | aligned */ |
67 | #ifdef CONFIG_MEMCG_KMEM |
68 | struct obj_cgroup **obj_cgroups; /* vector of object cgroups */ |
69 | #endif |
70 | |
71 | int nr_pages; /* # of pages served by this chunk */ |
72 | int nr_populated; /* # of populated pages */ |
73 | int nr_empty_pop_pages; /* # of empty populated pages */ |
74 | unsigned long populated[]; /* populated bitmap */ |
75 | }; |
76 | |
77 | extern spinlock_t pcpu_lock; |
78 | |
79 | extern struct list_head *pcpu_chunk_lists; |
80 | extern int pcpu_nr_slots; |
81 | extern int pcpu_sidelined_slot; |
82 | extern int pcpu_to_depopulate_slot; |
83 | extern int pcpu_nr_empty_pop_pages; |
84 | |
85 | extern struct pcpu_chunk *pcpu_first_chunk; |
86 | extern struct pcpu_chunk *pcpu_reserved_chunk; |
87 | |
88 | /** |
89 | * pcpu_chunk_nr_blocks - converts nr_pages to # of md_blocks |
90 | * @chunk: chunk of interest |
91 | * |
92 | * This conversion is from the number of physical pages that the chunk |
93 | * serves to the number of bitmap blocks used. |
94 | */ |
95 | static inline int pcpu_chunk_nr_blocks(struct pcpu_chunk *chunk) |
96 | { |
97 | return chunk->nr_pages * PAGE_SIZE / PCPU_BITMAP_BLOCK_SIZE; |
98 | } |
99 | |
100 | /** |
101 | * pcpu_nr_pages_to_map_bits - converts the pages to size of bitmap |
102 | * @pages: number of physical pages |
103 | * |
104 | * This conversion is from physical pages to the number of bits |
105 | * required in the bitmap. |
106 | */ |
107 | static inline int pcpu_nr_pages_to_map_bits(int pages) |
108 | { |
109 | return pages * PAGE_SIZE / PCPU_MIN_ALLOC_SIZE; |
110 | } |
111 | |
112 | /** |
113 | * pcpu_chunk_map_bits - helper to convert nr_pages to size of bitmap |
114 | * @chunk: chunk of interest |
115 | * |
116 | * This conversion is from the number of physical pages that the chunk |
117 | * serves to the number of bits in the bitmap. |
118 | */ |
119 | static inline int pcpu_chunk_map_bits(struct pcpu_chunk *chunk) |
120 | { |
121 | return pcpu_nr_pages_to_map_bits(pages: chunk->nr_pages); |
122 | } |
123 | |
124 | /** |
125 | * pcpu_obj_full_size - helper to calculate size of each accounted object |
126 | * @size: size of area to allocate in bytes |
127 | * |
128 | * For each accounted object there is an extra space which is used to store |
129 | * obj_cgroup membership if kmemcg is not disabled. Charge it too. |
130 | */ |
131 | static inline size_t pcpu_obj_full_size(size_t size) |
132 | { |
133 | size_t = 0; |
134 | |
135 | #ifdef CONFIG_MEMCG_KMEM |
136 | if (!mem_cgroup_kmem_disabled()) |
137 | extra_size += size / PCPU_MIN_ALLOC_SIZE * sizeof(struct obj_cgroup *); |
138 | #endif |
139 | |
140 | return size * num_possible_cpus() + extra_size; |
141 | } |
142 | |
143 | #ifdef CONFIG_PERCPU_STATS |
144 | |
145 | #include <linux/spinlock.h> |
146 | |
147 | struct percpu_stats { |
148 | u64 nr_alloc; /* lifetime # of allocations */ |
149 | u64 nr_dealloc; /* lifetime # of deallocations */ |
150 | u64 nr_cur_alloc; /* current # of allocations */ |
151 | u64 nr_max_alloc; /* max # of live allocations */ |
152 | u32 nr_chunks; /* current # of live chunks */ |
153 | u32 nr_max_chunks; /* max # of live chunks */ |
154 | size_t min_alloc_size; /* min allocation size */ |
155 | size_t max_alloc_size; /* max allocation size */ |
156 | }; |
157 | |
158 | extern struct percpu_stats pcpu_stats; |
159 | extern struct pcpu_alloc_info pcpu_stats_ai; |
160 | |
161 | /* |
162 | * For debug purposes. We don't care about the flexible array. |
163 | */ |
164 | static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai) |
165 | { |
166 | memcpy(&pcpu_stats_ai, ai, sizeof(struct pcpu_alloc_info)); |
167 | |
168 | /* initialize min_alloc_size to unit_size */ |
169 | pcpu_stats.min_alloc_size = pcpu_stats_ai.unit_size; |
170 | } |
171 | |
172 | /* |
173 | * pcpu_stats_area_alloc - increment area allocation stats |
174 | * @chunk: the location of the area being allocated |
175 | * @size: size of area to allocate in bytes |
176 | * |
177 | * CONTEXT: |
178 | * pcpu_lock. |
179 | */ |
180 | static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size) |
181 | { |
182 | lockdep_assert_held(&pcpu_lock); |
183 | |
184 | pcpu_stats.nr_alloc++; |
185 | pcpu_stats.nr_cur_alloc++; |
186 | pcpu_stats.nr_max_alloc = |
187 | max(pcpu_stats.nr_max_alloc, pcpu_stats.nr_cur_alloc); |
188 | pcpu_stats.min_alloc_size = |
189 | min(pcpu_stats.min_alloc_size, size); |
190 | pcpu_stats.max_alloc_size = |
191 | max(pcpu_stats.max_alloc_size, size); |
192 | |
193 | chunk->nr_alloc++; |
194 | chunk->max_alloc_size = max(chunk->max_alloc_size, size); |
195 | } |
196 | |
197 | /* |
198 | * pcpu_stats_area_dealloc - decrement allocation stats |
199 | * @chunk: the location of the area being deallocated |
200 | * |
201 | * CONTEXT: |
202 | * pcpu_lock. |
203 | */ |
204 | static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk) |
205 | { |
206 | lockdep_assert_held(&pcpu_lock); |
207 | |
208 | pcpu_stats.nr_dealloc++; |
209 | pcpu_stats.nr_cur_alloc--; |
210 | |
211 | chunk->nr_alloc--; |
212 | } |
213 | |
214 | /* |
215 | * pcpu_stats_chunk_alloc - increment chunk stats |
216 | */ |
217 | static inline void pcpu_stats_chunk_alloc(void) |
218 | { |
219 | unsigned long flags; |
220 | spin_lock_irqsave(&pcpu_lock, flags); |
221 | |
222 | pcpu_stats.nr_chunks++; |
223 | pcpu_stats.nr_max_chunks = |
224 | max(pcpu_stats.nr_max_chunks, pcpu_stats.nr_chunks); |
225 | |
226 | spin_unlock_irqrestore(lock: &pcpu_lock, flags); |
227 | } |
228 | |
229 | /* |
230 | * pcpu_stats_chunk_dealloc - decrement chunk stats |
231 | */ |
232 | static inline void pcpu_stats_chunk_dealloc(void) |
233 | { |
234 | unsigned long flags; |
235 | spin_lock_irqsave(&pcpu_lock, flags); |
236 | |
237 | pcpu_stats.nr_chunks--; |
238 | |
239 | spin_unlock_irqrestore(lock: &pcpu_lock, flags); |
240 | } |
241 | |
242 | #else |
243 | |
244 | static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai) |
245 | { |
246 | } |
247 | |
248 | static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size) |
249 | { |
250 | } |
251 | |
252 | static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk) |
253 | { |
254 | } |
255 | |
256 | static inline void pcpu_stats_chunk_alloc(void) |
257 | { |
258 | } |
259 | |
260 | static inline void pcpu_stats_chunk_dealloc(void) |
261 | { |
262 | } |
263 | |
264 | #endif /* !CONFIG_PERCPU_STATS */ |
265 | |
266 | #endif |
267 | |