1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * f2fs iostat support |
4 | * |
5 | * Copyright 2021 Google LLC |
6 | * Author: Daeho Jeong <daehojeong@google.com> |
7 | */ |
8 | |
9 | #include <linux/fs.h> |
10 | #include <linux/f2fs_fs.h> |
11 | #include <linux/seq_file.h> |
12 | |
13 | #include "f2fs.h" |
14 | #include "iostat.h" |
15 | #include <trace/events/f2fs.h> |
16 | |
17 | static struct kmem_cache *bio_iostat_ctx_cache; |
18 | static mempool_t *bio_iostat_ctx_pool; |
19 | |
20 | static inline unsigned long long iostat_get_avg_bytes(struct f2fs_sb_info *sbi, |
21 | enum iostat_type type) |
22 | { |
23 | return sbi->iostat_count[type] ? div64_u64(dividend: sbi->iostat_bytes[type], |
24 | divisor: sbi->iostat_count[type]) : 0; |
25 | } |
26 | |
27 | #define IOSTAT_INFO_SHOW(name, type) \ |
28 | seq_printf(seq, "%-23s %-16llu %-16llu %-16llu\n", \ |
29 | name":", sbi->iostat_bytes[type], \ |
30 | sbi->iostat_count[type], \ |
31 | iostat_get_avg_bytes(sbi, type)) |
32 | |
33 | int __maybe_unused iostat_info_seq_show(struct seq_file *seq, void *offset) |
34 | { |
35 | struct super_block *sb = seq->private; |
36 | struct f2fs_sb_info *sbi = F2FS_SB(sb); |
37 | |
38 | if (!sbi->iostat_enable) |
39 | return 0; |
40 | |
41 | seq_printf(m: seq, fmt: "time: %-16llu\n" , ktime_get_real_seconds()); |
42 | seq_printf(m: seq, fmt: "\t\t\t%-16s %-16s %-16s\n" , |
43 | "io_bytes" , "count" , "avg_bytes" ); |
44 | |
45 | /* print app write IOs */ |
46 | seq_puts(m: seq, s: "[WRITE]\n" ); |
47 | IOSTAT_INFO_SHOW("app buffered data" , APP_BUFFERED_IO); |
48 | IOSTAT_INFO_SHOW("app direct data" , APP_DIRECT_IO); |
49 | IOSTAT_INFO_SHOW("app mapped data" , APP_MAPPED_IO); |
50 | IOSTAT_INFO_SHOW("app buffered cdata" , APP_BUFFERED_CDATA_IO); |
51 | IOSTAT_INFO_SHOW("app mapped cdata" , APP_MAPPED_CDATA_IO); |
52 | |
53 | /* print fs write IOs */ |
54 | IOSTAT_INFO_SHOW("fs data" , FS_DATA_IO); |
55 | IOSTAT_INFO_SHOW("fs cdata" , FS_CDATA_IO); |
56 | IOSTAT_INFO_SHOW("fs node" , FS_NODE_IO); |
57 | IOSTAT_INFO_SHOW("fs meta" , FS_META_IO); |
58 | IOSTAT_INFO_SHOW("fs gc data" , FS_GC_DATA_IO); |
59 | IOSTAT_INFO_SHOW("fs gc node" , FS_GC_NODE_IO); |
60 | IOSTAT_INFO_SHOW("fs cp data" , FS_CP_DATA_IO); |
61 | IOSTAT_INFO_SHOW("fs cp node" , FS_CP_NODE_IO); |
62 | IOSTAT_INFO_SHOW("fs cp meta" , FS_CP_META_IO); |
63 | |
64 | /* print app read IOs */ |
65 | seq_puts(m: seq, s: "[READ]\n" ); |
66 | IOSTAT_INFO_SHOW("app buffered data" , APP_BUFFERED_READ_IO); |
67 | IOSTAT_INFO_SHOW("app direct data" , APP_DIRECT_READ_IO); |
68 | IOSTAT_INFO_SHOW("app mapped data" , APP_MAPPED_READ_IO); |
69 | IOSTAT_INFO_SHOW("app buffered cdata" , APP_BUFFERED_CDATA_READ_IO); |
70 | IOSTAT_INFO_SHOW("app mapped cdata" , APP_MAPPED_CDATA_READ_IO); |
71 | |
72 | /* print fs read IOs */ |
73 | IOSTAT_INFO_SHOW("fs data" , FS_DATA_READ_IO); |
74 | IOSTAT_INFO_SHOW("fs gc data" , FS_GDATA_READ_IO); |
75 | IOSTAT_INFO_SHOW("fs cdata" , FS_CDATA_READ_IO); |
76 | IOSTAT_INFO_SHOW("fs node" , FS_NODE_READ_IO); |
77 | IOSTAT_INFO_SHOW("fs meta" , FS_META_READ_IO); |
78 | |
79 | /* print other IOs */ |
80 | seq_puts(m: seq, s: "[OTHER]\n" ); |
81 | IOSTAT_INFO_SHOW("fs discard" , FS_DISCARD_IO); |
82 | IOSTAT_INFO_SHOW("fs flush" , FS_FLUSH_IO); |
83 | IOSTAT_INFO_SHOW("fs zone reset" , FS_ZONE_RESET_IO); |
84 | |
85 | return 0; |
86 | } |
87 | |
88 | static inline void __record_iostat_latency(struct f2fs_sb_info *sbi) |
89 | { |
90 | int io, idx; |
91 | struct f2fs_iostat_latency iostat_lat[MAX_IO_TYPE][NR_PAGE_TYPE]; |
92 | struct iostat_lat_info *io_lat = sbi->iostat_io_lat; |
93 | unsigned long flags; |
94 | |
95 | spin_lock_irqsave(&sbi->iostat_lat_lock, flags); |
96 | for (idx = 0; idx < MAX_IO_TYPE; idx++) { |
97 | for (io = 0; io < NR_PAGE_TYPE; io++) { |
98 | iostat_lat[idx][io].peak_lat = |
99 | jiffies_to_msecs(j: io_lat->peak_lat[idx][io]); |
100 | iostat_lat[idx][io].cnt = io_lat->bio_cnt[idx][io]; |
101 | iostat_lat[idx][io].avg_lat = iostat_lat[idx][io].cnt ? |
102 | jiffies_to_msecs(j: io_lat->sum_lat[idx][io]) / iostat_lat[idx][io].cnt : 0; |
103 | io_lat->sum_lat[idx][io] = 0; |
104 | io_lat->peak_lat[idx][io] = 0; |
105 | io_lat->bio_cnt[idx][io] = 0; |
106 | } |
107 | } |
108 | spin_unlock_irqrestore(lock: &sbi->iostat_lat_lock, flags); |
109 | |
110 | trace_f2fs_iostat_latency(sbi, iostat_lat); |
111 | } |
112 | |
113 | static inline void f2fs_record_iostat(struct f2fs_sb_info *sbi) |
114 | { |
115 | unsigned long long iostat_diff[NR_IO_TYPE]; |
116 | int i; |
117 | unsigned long flags; |
118 | |
119 | if (time_is_after_jiffies(sbi->iostat_next_period)) |
120 | return; |
121 | |
122 | /* Need double check under the lock */ |
123 | spin_lock_irqsave(&sbi->iostat_lock, flags); |
124 | if (time_is_after_jiffies(sbi->iostat_next_period)) { |
125 | spin_unlock_irqrestore(lock: &sbi->iostat_lock, flags); |
126 | return; |
127 | } |
128 | sbi->iostat_next_period = jiffies + |
129 | msecs_to_jiffies(m: sbi->iostat_period_ms); |
130 | |
131 | for (i = 0; i < NR_IO_TYPE; i++) { |
132 | iostat_diff[i] = sbi->iostat_bytes[i] - |
133 | sbi->prev_iostat_bytes[i]; |
134 | sbi->prev_iostat_bytes[i] = sbi->iostat_bytes[i]; |
135 | } |
136 | spin_unlock_irqrestore(lock: &sbi->iostat_lock, flags); |
137 | |
138 | trace_f2fs_iostat(sbi, iostat: iostat_diff); |
139 | |
140 | __record_iostat_latency(sbi); |
141 | } |
142 | |
143 | void f2fs_reset_iostat(struct f2fs_sb_info *sbi) |
144 | { |
145 | struct iostat_lat_info *io_lat = sbi->iostat_io_lat; |
146 | int i; |
147 | |
148 | spin_lock_irq(lock: &sbi->iostat_lock); |
149 | for (i = 0; i < NR_IO_TYPE; i++) { |
150 | sbi->iostat_count[i] = 0; |
151 | sbi->iostat_bytes[i] = 0; |
152 | sbi->prev_iostat_bytes[i] = 0; |
153 | } |
154 | spin_unlock_irq(lock: &sbi->iostat_lock); |
155 | |
156 | spin_lock_irq(lock: &sbi->iostat_lat_lock); |
157 | memset(io_lat, 0, sizeof(struct iostat_lat_info)); |
158 | spin_unlock_irq(lock: &sbi->iostat_lat_lock); |
159 | } |
160 | |
161 | static inline void __f2fs_update_iostat(struct f2fs_sb_info *sbi, |
162 | enum iostat_type type, unsigned long long io_bytes) |
163 | { |
164 | sbi->iostat_bytes[type] += io_bytes; |
165 | sbi->iostat_count[type]++; |
166 | } |
167 | |
168 | void f2fs_update_iostat(struct f2fs_sb_info *sbi, struct inode *inode, |
169 | enum iostat_type type, unsigned long long io_bytes) |
170 | { |
171 | unsigned long flags; |
172 | |
173 | if (!sbi->iostat_enable) |
174 | return; |
175 | |
176 | spin_lock_irqsave(&sbi->iostat_lock, flags); |
177 | __f2fs_update_iostat(sbi, type, io_bytes); |
178 | |
179 | if (type == APP_BUFFERED_IO || type == APP_DIRECT_IO) |
180 | __f2fs_update_iostat(sbi, type: APP_WRITE_IO, io_bytes); |
181 | |
182 | if (type == APP_BUFFERED_READ_IO || type == APP_DIRECT_READ_IO) |
183 | __f2fs_update_iostat(sbi, type: APP_READ_IO, io_bytes); |
184 | |
185 | #ifdef CONFIG_F2FS_FS_COMPRESSION |
186 | if (inode && f2fs_compressed_file(inode)) { |
187 | if (type == APP_BUFFERED_IO) |
188 | __f2fs_update_iostat(sbi, type: APP_BUFFERED_CDATA_IO, io_bytes); |
189 | |
190 | if (type == APP_BUFFERED_READ_IO) |
191 | __f2fs_update_iostat(sbi, type: APP_BUFFERED_CDATA_READ_IO, io_bytes); |
192 | |
193 | if (type == APP_MAPPED_READ_IO) |
194 | __f2fs_update_iostat(sbi, type: APP_MAPPED_CDATA_READ_IO, io_bytes); |
195 | |
196 | if (type == APP_MAPPED_IO) |
197 | __f2fs_update_iostat(sbi, type: APP_MAPPED_CDATA_IO, io_bytes); |
198 | |
199 | if (type == FS_DATA_READ_IO) |
200 | __f2fs_update_iostat(sbi, type: FS_CDATA_READ_IO, io_bytes); |
201 | |
202 | if (type == FS_DATA_IO) |
203 | __f2fs_update_iostat(sbi, type: FS_CDATA_IO, io_bytes); |
204 | } |
205 | #endif |
206 | |
207 | spin_unlock_irqrestore(lock: &sbi->iostat_lock, flags); |
208 | |
209 | f2fs_record_iostat(sbi); |
210 | } |
211 | |
212 | static inline void __update_iostat_latency(struct bio_iostat_ctx *iostat_ctx, |
213 | enum iostat_lat_type lat_type) |
214 | { |
215 | unsigned long ts_diff; |
216 | unsigned int page_type = iostat_ctx->type; |
217 | struct f2fs_sb_info *sbi = iostat_ctx->sbi; |
218 | struct iostat_lat_info *io_lat = sbi->iostat_io_lat; |
219 | unsigned long flags; |
220 | |
221 | if (!sbi->iostat_enable) |
222 | return; |
223 | |
224 | ts_diff = jiffies - iostat_ctx->submit_ts; |
225 | if (page_type == META_FLUSH) { |
226 | page_type = META; |
227 | } else if (page_type >= NR_PAGE_TYPE) { |
228 | f2fs_warn(sbi, "%s: %d over NR_PAGE_TYPE" , __func__, page_type); |
229 | return; |
230 | } |
231 | |
232 | spin_lock_irqsave(&sbi->iostat_lat_lock, flags); |
233 | io_lat->sum_lat[lat_type][page_type] += ts_diff; |
234 | io_lat->bio_cnt[lat_type][page_type]++; |
235 | if (ts_diff > io_lat->peak_lat[lat_type][page_type]) |
236 | io_lat->peak_lat[lat_type][page_type] = ts_diff; |
237 | spin_unlock_irqrestore(lock: &sbi->iostat_lat_lock, flags); |
238 | } |
239 | |
240 | void iostat_update_and_unbind_ctx(struct bio *bio) |
241 | { |
242 | struct bio_iostat_ctx *iostat_ctx = bio->bi_private; |
243 | enum iostat_lat_type lat_type; |
244 | |
245 | if (op_is_write(op: bio_op(bio))) { |
246 | lat_type = bio->bi_opf & REQ_SYNC ? |
247 | WRITE_SYNC_IO : WRITE_ASYNC_IO; |
248 | bio->bi_private = iostat_ctx->sbi; |
249 | } else { |
250 | lat_type = READ_IO; |
251 | bio->bi_private = iostat_ctx->post_read_ctx; |
252 | } |
253 | |
254 | __update_iostat_latency(iostat_ctx, lat_type); |
255 | mempool_free(element: iostat_ctx, pool: bio_iostat_ctx_pool); |
256 | } |
257 | |
258 | void iostat_alloc_and_bind_ctx(struct f2fs_sb_info *sbi, |
259 | struct bio *bio, struct bio_post_read_ctx *ctx) |
260 | { |
261 | struct bio_iostat_ctx *iostat_ctx; |
262 | /* Due to the mempool, this never fails. */ |
263 | iostat_ctx = mempool_alloc(pool: bio_iostat_ctx_pool, GFP_NOFS); |
264 | iostat_ctx->sbi = sbi; |
265 | iostat_ctx->submit_ts = 0; |
266 | iostat_ctx->type = 0; |
267 | iostat_ctx->post_read_ctx = ctx; |
268 | bio->bi_private = iostat_ctx; |
269 | } |
270 | |
271 | int __init f2fs_init_iostat_processing(void) |
272 | { |
273 | bio_iostat_ctx_cache = |
274 | kmem_cache_create(name: "f2fs_bio_iostat_ctx" , |
275 | size: sizeof(struct bio_iostat_ctx), align: 0, flags: 0, NULL); |
276 | if (!bio_iostat_ctx_cache) |
277 | goto fail; |
278 | bio_iostat_ctx_pool = |
279 | mempool_create_slab_pool(NUM_PREALLOC_IOSTAT_CTXS, |
280 | kc: bio_iostat_ctx_cache); |
281 | if (!bio_iostat_ctx_pool) |
282 | goto fail_free_cache; |
283 | return 0; |
284 | |
285 | fail_free_cache: |
286 | kmem_cache_destroy(s: bio_iostat_ctx_cache); |
287 | fail: |
288 | return -ENOMEM; |
289 | } |
290 | |
291 | void f2fs_destroy_iostat_processing(void) |
292 | { |
293 | mempool_destroy(pool: bio_iostat_ctx_pool); |
294 | kmem_cache_destroy(s: bio_iostat_ctx_cache); |
295 | } |
296 | |
297 | int f2fs_init_iostat(struct f2fs_sb_info *sbi) |
298 | { |
299 | /* init iostat info */ |
300 | spin_lock_init(&sbi->iostat_lock); |
301 | spin_lock_init(&sbi->iostat_lat_lock); |
302 | sbi->iostat_enable = false; |
303 | sbi->iostat_period_ms = DEFAULT_IOSTAT_PERIOD_MS; |
304 | sbi->iostat_io_lat = f2fs_kzalloc(sbi, size: sizeof(struct iostat_lat_info), |
305 | GFP_KERNEL); |
306 | if (!sbi->iostat_io_lat) |
307 | return -ENOMEM; |
308 | |
309 | return 0; |
310 | } |
311 | |
312 | void f2fs_destroy_iostat(struct f2fs_sb_info *sbi) |
313 | { |
314 | kfree(objp: sbi->iostat_io_lat); |
315 | } |
316 | |