1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | #ifndef _MM_PERCPU_INTERNAL_H |
3 | #define _MM_PERCPU_INTERNAL_H |
4 | |
5 | #include <linux/types.h> |
6 | #include <linux/percpu.h> |
7 | |
8 | /* |
9 | * pcpu_block_md is the metadata block struct. |
10 | * Each chunk's bitmap is split into a number of full blocks. |
11 | * All units are in terms of bits. |
12 | */ |
13 | struct pcpu_block_md { |
14 | int contig_hint; /* contig hint for block */ |
15 | int contig_hint_start; /* block relative starting |
16 | position of the contig hint */ |
17 | int left_free; /* size of free space along |
18 | the left side of the block */ |
19 | int right_free; /* size of free space along |
20 | the right side of the block */ |
21 | int first_free; /* block position of first free */ |
22 | }; |
23 | |
24 | struct pcpu_chunk { |
25 | #ifdef CONFIG_PERCPU_STATS |
26 | int nr_alloc; /* # of allocations */ |
27 | size_t max_alloc_size; /* largest allocation size */ |
28 | #endif |
29 | |
30 | struct list_head list; /* linked to pcpu_slot lists */ |
31 | int free_bytes; /* free bytes in the chunk */ |
32 | int contig_bits; /* max contiguous size hint */ |
33 | int contig_bits_start; /* contig_bits starting |
34 | offset */ |
35 | void *base_addr; /* base address of this chunk */ |
36 | |
37 | unsigned long *alloc_map; /* allocation map */ |
38 | unsigned long *bound_map; /* boundary map */ |
39 | struct pcpu_block_md *md_blocks; /* metadata blocks */ |
40 | |
41 | void *data; /* chunk data */ |
42 | int first_bit; /* no free below this */ |
43 | bool immutable; /* no [de]population allowed */ |
44 | int start_offset; /* the overlap with the previous |
45 | region to have a page aligned |
46 | base_addr */ |
47 | int end_offset; /* additional area required to |
48 | have the region end page |
49 | aligned */ |
50 | |
51 | int nr_pages; /* # of pages served by this chunk */ |
52 | int nr_populated; /* # of populated pages */ |
53 | int nr_empty_pop_pages; /* # of empty populated pages */ |
54 | unsigned long populated[]; /* populated bitmap */ |
55 | }; |
56 | |
57 | extern spinlock_t pcpu_lock; |
58 | |
59 | extern struct list_head *pcpu_slot; |
60 | extern int pcpu_nr_slots; |
61 | extern int pcpu_nr_empty_pop_pages; |
62 | |
63 | extern struct pcpu_chunk *pcpu_first_chunk; |
64 | extern struct pcpu_chunk *pcpu_reserved_chunk; |
65 | |
66 | /** |
67 | * pcpu_chunk_nr_blocks - converts nr_pages to # of md_blocks |
68 | * @chunk: chunk of interest |
69 | * |
70 | * This conversion is from the number of physical pages that the chunk |
71 | * serves to the number of bitmap blocks used. |
72 | */ |
73 | static inline int pcpu_chunk_nr_blocks(struct pcpu_chunk *chunk) |
74 | { |
75 | return chunk->nr_pages * PAGE_SIZE / PCPU_BITMAP_BLOCK_SIZE; |
76 | } |
77 | |
78 | /** |
79 | * pcpu_nr_pages_to_map_bits - converts the pages to size of bitmap |
80 | * @pages: number of physical pages |
81 | * |
82 | * This conversion is from physical pages to the number of bits |
83 | * required in the bitmap. |
84 | */ |
85 | static inline int pcpu_nr_pages_to_map_bits(int pages) |
86 | { |
87 | return pages * PAGE_SIZE / PCPU_MIN_ALLOC_SIZE; |
88 | } |
89 | |
90 | /** |
91 | * pcpu_chunk_map_bits - helper to convert nr_pages to size of bitmap |
92 | * @chunk: chunk of interest |
93 | * |
94 | * This conversion is from the number of physical pages that the chunk |
95 | * serves to the number of bits in the bitmap. |
96 | */ |
97 | static inline int pcpu_chunk_map_bits(struct pcpu_chunk *chunk) |
98 | { |
99 | return pcpu_nr_pages_to_map_bits(chunk->nr_pages); |
100 | } |
101 | |
102 | #ifdef CONFIG_PERCPU_STATS |
103 | |
104 | #include <linux/spinlock.h> |
105 | |
106 | struct percpu_stats { |
107 | u64 nr_alloc; /* lifetime # of allocations */ |
108 | u64 nr_dealloc; /* lifetime # of deallocations */ |
109 | u64 nr_cur_alloc; /* current # of allocations */ |
110 | u64 nr_max_alloc; /* max # of live allocations */ |
111 | u32 nr_chunks; /* current # of live chunks */ |
112 | u32 nr_max_chunks; /* max # of live chunks */ |
113 | size_t min_alloc_size; /* min allocaiton size */ |
114 | size_t max_alloc_size; /* max allocation size */ |
115 | }; |
116 | |
117 | extern struct percpu_stats pcpu_stats; |
118 | extern struct pcpu_alloc_info pcpu_stats_ai; |
119 | |
120 | /* |
121 | * For debug purposes. We don't care about the flexible array. |
122 | */ |
123 | static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai) |
124 | { |
125 | memcpy(&pcpu_stats_ai, ai, sizeof(struct pcpu_alloc_info)); |
126 | |
127 | /* initialize min_alloc_size to unit_size */ |
128 | pcpu_stats.min_alloc_size = pcpu_stats_ai.unit_size; |
129 | } |
130 | |
131 | /* |
132 | * pcpu_stats_area_alloc - increment area allocation stats |
133 | * @chunk: the location of the area being allocated |
134 | * @size: size of area to allocate in bytes |
135 | * |
136 | * CONTEXT: |
137 | * pcpu_lock. |
138 | */ |
139 | static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size) |
140 | { |
141 | lockdep_assert_held(&pcpu_lock); |
142 | |
143 | pcpu_stats.nr_alloc++; |
144 | pcpu_stats.nr_cur_alloc++; |
145 | pcpu_stats.nr_max_alloc = |
146 | max(pcpu_stats.nr_max_alloc, pcpu_stats.nr_cur_alloc); |
147 | pcpu_stats.min_alloc_size = |
148 | min(pcpu_stats.min_alloc_size, size); |
149 | pcpu_stats.max_alloc_size = |
150 | max(pcpu_stats.max_alloc_size, size); |
151 | |
152 | chunk->nr_alloc++; |
153 | chunk->max_alloc_size = max(chunk->max_alloc_size, size); |
154 | } |
155 | |
156 | /* |
157 | * pcpu_stats_area_dealloc - decrement allocation stats |
158 | * @chunk: the location of the area being deallocated |
159 | * |
160 | * CONTEXT: |
161 | * pcpu_lock. |
162 | */ |
163 | static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk) |
164 | { |
165 | lockdep_assert_held(&pcpu_lock); |
166 | |
167 | pcpu_stats.nr_dealloc++; |
168 | pcpu_stats.nr_cur_alloc--; |
169 | |
170 | chunk->nr_alloc--; |
171 | } |
172 | |
173 | /* |
174 | * pcpu_stats_chunk_alloc - increment chunk stats |
175 | */ |
176 | static inline void pcpu_stats_chunk_alloc(void) |
177 | { |
178 | unsigned long flags; |
179 | spin_lock_irqsave(&pcpu_lock, flags); |
180 | |
181 | pcpu_stats.nr_chunks++; |
182 | pcpu_stats.nr_max_chunks = |
183 | max(pcpu_stats.nr_max_chunks, pcpu_stats.nr_chunks); |
184 | |
185 | spin_unlock_irqrestore(&pcpu_lock, flags); |
186 | } |
187 | |
188 | /* |
189 | * pcpu_stats_chunk_dealloc - decrement chunk stats |
190 | */ |
191 | static inline void pcpu_stats_chunk_dealloc(void) |
192 | { |
193 | unsigned long flags; |
194 | spin_lock_irqsave(&pcpu_lock, flags); |
195 | |
196 | pcpu_stats.nr_chunks--; |
197 | |
198 | spin_unlock_irqrestore(&pcpu_lock, flags); |
199 | } |
200 | |
201 | #else |
202 | |
203 | static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai) |
204 | { |
205 | } |
206 | |
207 | static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size) |
208 | { |
209 | } |
210 | |
211 | static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk) |
212 | { |
213 | } |
214 | |
215 | static inline void pcpu_stats_chunk_alloc(void) |
216 | { |
217 | } |
218 | |
219 | static inline void pcpu_stats_chunk_dealloc(void) |
220 | { |
221 | } |
222 | |
223 | #endif /* !CONFIG_PERCPU_STATS */ |
224 | |
225 | #endif |
226 | |